From 90cc9576f540e43de4b7ac5d7bfee02a87ea5f58 Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Thu, 11 Jun 2020 17:46:29 -0400 Subject: [PATCH 1/9] Increase Cloudflare page size Increase Cloudflare page size to reduce request count `GET zones` has a MAX of 50 and a default of 20 https://api.cloudflare.com/#zone-list-zones `GET zones/:zone_identifier/dns_records` has a MAX of 100 and a default of 20 https://api.cloudflare.com/#dns-records-for-a-zone-list-dns-records --- octodns/provider/cloudflare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 698fbee..96febf4 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -142,7 +142,7 @@ class CloudflareProvider(BaseProvider): zones = [] while page: resp = self._try_request('GET', '/zones', - params={'page': page}) + params={'page': page, 'per_page': 50}) zones += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: @@ -251,7 +251,7 @@ class CloudflareProvider(BaseProvider): path = '/zones/{}/dns_records'.format(zone_id) page = 1 while page: - resp = self._try_request('GET', path, params={'page': page}) + resp = self._try_request('GET', path, params={'page': page, 'per_page': 100}) records += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: From b80d1575e6d5268e3ee69c14adcc8e2ad9dcfa88 Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Thu, 11 Jun 2020 17:56:31 -0400 Subject: [PATCH 2/9] Update tests with new per_page params --- octodns/provider/cloudflare.py | 3 ++- tests/test_octodns_provider_cloudflare.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 96febf4..96c9f5e 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -251,7 +251,8 @@ class CloudflareProvider(BaseProvider): path = '/zones/{}/dns_records'.format(zone_id) page = 1 while page: - resp = self._try_request('GET', path, params={'page': page, 'per_page': 100}) + resp = self._try_request('GET', path, params={'page': page, + 'per_page': 100}) records += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 08608ea..735d95c 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -426,7 +426,7 @@ class TestCloudflareProvider(TestCase): # get the list of zones, create a zone, add some records, update # something, and delete something provider._request.assert_has_calls([ - call('GET', '/zones', params={'page': 1}), + call('GET', '/zones', params={'page': 1, 'per_page': 50}), call('POST', '/zones', data={ 'jump_start': False, 'name': 'unit.tests' @@ -531,7 +531,7 @@ class TestCloudflareProvider(TestCase): # Get zones, create zone, create a record, delete a record provider._request.assert_has_calls([ - call('GET', '/zones', params={'page': 1}), + call('GET', '/zones', params={'page': 1, 'per_page': 50}), call('POST', '/zones', data={ 'jump_start': False, 'name': 'unit.tests' @@ -1302,7 +1302,8 @@ class TestCloudflareProvider(TestCase): provider._request.side_effect = [result] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # One retry required provider._zones = None @@ -1313,7 +1314,8 @@ class TestCloudflareProvider(TestCase): ] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # Two retries required provider._zones = None @@ -1325,7 +1327,8 @@ class TestCloudflareProvider(TestCase): ] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # # Exhaust our retries provider._zones = None From bac16622426d21768ebf6b4c895936965a3ffd66 Mon Sep 17 00:00:00 2001 From: DavHau Date: Mon, 15 Jun 2020 16:51:28 +0000 Subject: [PATCH 3/9] fix: dependency 'ipaddress' unnecessary for py >= 3.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c56aa82..142b209 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ setup( 'PyYaml>=4.2b1', 'dnspython>=1.15.0', 'futures>=3.2.0; python_version<"3.2"', - 'ipaddress>=1.0.22', + 'ipaddress>=1.0.22; python_version<"3.2"', 'natsort>=5.5.0', 'pycountry>=19.8.18', 'pycountry-convert>=0.7.2', From 84048dbde9df4de639141d32ca4722c538b2b92b Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Mon, 22 Jun 2020 17:27:41 -0400 Subject: [PATCH 4/9] Cloudflare: Make page size configurable --- octodns/provider/cloudflare.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 96c9f5e..db937e5 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -58,6 +58,10 @@ class CloudflareProvider(BaseProvider): retry_count: 4 # Optional. Default: 300. Number of seconds to wait before retrying. retry_period: 300 + # Optional. Default: 50. Number of zones per page. + zones_per_page: 50 + # Optional. Default: 100. Number of dns records per page. + records_per_page: 100 Note: The "proxied" flag of "A", "AAAA" and "CNAME" records can be managed via the YAML provider like so: @@ -78,7 +82,8 @@ class CloudflareProvider(BaseProvider): TIMEOUT = 15 def __init__(self, id, email=None, token=None, cdn=False, retry_count=4, - retry_period=300, *args, **kwargs): + retry_period=300, zones_per_page=50, records_per_page=100, + *args, **kwargs): self.log = getLogger('CloudflareProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, email=%s, token=***, cdn=%s', id, email, cdn) @@ -99,6 +104,8 @@ class CloudflareProvider(BaseProvider): self.cdn = cdn self.retry_count = retry_count self.retry_period = retry_period + self.zones_per_page = zones_per_page + self.records_per_page = records_per_page self._sess = sess self._zones = None @@ -142,7 +149,10 @@ class CloudflareProvider(BaseProvider): zones = [] while page: resp = self._try_request('GET', '/zones', - params={'page': page, 'per_page': 50}) + params={ + 'page': page, + 'per_page': self.zones_per_page + }) zones += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: @@ -252,7 +262,7 @@ class CloudflareProvider(BaseProvider): page = 1 while page: resp = self._try_request('GET', path, params={'page': page, - 'per_page': 100}) + 'per_page': self.records_per_page}) records += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: From 4d006e94a212f2babce6aeeebaf3eaba3961e4d0 Mon Sep 17 00:00:00 2001 From: Phelps Williams Date: Wed, 15 Jul 2020 18:17:33 -0700 Subject: [PATCH 5/9] Adding environment variable record injection Per the discussion on https://github.com/github/octodns/issues/583 here is a work in progress of environment variable injection for discussion. --- octodns/source/envvar.py | 105 ++++++++++++++++++++++++++++ tests/test_octodns_source_envvar.py | 34 +++++++++ 2 files changed, 139 insertions(+) create mode 100644 octodns/source/envvar.py create mode 100644 tests/test_octodns_source_envvar.py diff --git a/octodns/source/envvar.py b/octodns/source/envvar.py new file mode 100644 index 0000000..b755632 --- /dev/null +++ b/octodns/source/envvar.py @@ -0,0 +1,105 @@ + +import logging +import os + +from ..record import Record +from .base import BaseSource + + +class EnvVarSourceException(Exception): + pass + + +class EnvironmentVariableNotFoundException(EnvVarSourceException): + def __init__(self, data): + super(EnvironmentVariableNotFoundException, self).__init__( + 'Unknown environment variable {}'.format(data)) + + +class EnvVarSource(BaseSource): + ''' + This source allows for environment variables to be embedded at octodns + execution time into zones. Intended to capture artifacts of deployment to + facilitate operational objectives. + + The TXT record generated will only have a single value. + + The record name cannot conflict with any other co-existing sources. If + this occurs, an exception will be thrown. + + Possible use cases include: + - Embedding a version number into a TXT record to monitor update + propagation across authoritative providers. + - Capturing identifying information about the deployment process to + record where and when the zone was updated. + + version: + class: octodns.source.envvar.EnvVarSource + # The environment variable in question, in this example the username + # currently executing octodns + variable: USER + # The TXT record name to embed the value found at the above + # environment variable + record: deployuser + # The TTL of the TXT record (optional, default 60) + ttl: 3600 + + This source is then combined with other sources in the octodns config + file: + + zones: + netflix.com.: + sources: + - yaml + - version + targets: + - ultra + - ns1 + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('TXT')) + + DEFAULT_TTL = 60 + + def __init__(self, id, variable, record, ttl=DEFAULT_TTL): + self.log = logging.getLogger('{}[{}]'.format( + self.__class__.__name__, id)) + self.log.debug('__init__: id=%s, variable=%s, record=%s, ' + 'ttl=%d', id, variable, record, ttl) + super(EnvVarSource, self).__init__(id) + self.envvar = variable + self.record = record + self.ttl = ttl + self.value = None + + def _read_variable(self): + self.value = os.environ.get(self.envvar) + if self.value is None: + raise EnvironmentVariableNotFoundException(self.envvar) + + self.log.debug('_read_variable: successfully loaded var=%s val=%s', + self.envvar, self.value) + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + # if target: + # TODO: Environment Variable Source cannot act as a target, + # throw exception? + # return + + before = len(zone.records) + + self._read_variable() + + # We don't need to worry about conflicting records here because the + # manager will deconflict sources on our behalf. + payload = {'ttl': self.ttl, 'type': 'TXT', 'values': [self.value]} + record = Record.new(zone, self.record, payload, source=self, + lenient=lenient) + zone.add_record(record, lenient=lenient) + + self.log.info('populate: found %s records, exists=False', + len(zone.records) - before) diff --git a/tests/test_octodns_source_envvar.py b/tests/test_octodns_source_envvar.py new file mode 100644 index 0000000..e562aa0 --- /dev/null +++ b/tests/test_octodns_source_envvar.py @@ -0,0 +1,34 @@ +from six import text_type +from unittest import TestCase +from unittest.mock import patch + +from octodns.source.envvar import EnvVarSource +from octodns.source.envvar import EnvironmentVariableNotFoundException +from octodns.zone import Zone + + +class TestEnvVarSource(TestCase): + + def test_read_variable(self): + envvar = 'OCTODNS_TEST_ENVIRONMENT_VARIABLE' + source = EnvVarSource('testid', envvar, 'recordname', ttl=120) + with self.assertRaises(EnvironmentVariableNotFoundException) as ctx: + source._read_variable() + msg = 'Unknown environment variable {}'.format(envvar) + self.assertEquals(msg, text_type(ctx.exception)) + + with patch.dict('os.environ', {envvar: 'testvalue'}): + source._read_variable() + self.assertEquals(source.value, 'testvalue') + + def test_populate(self): + envvar = 'TEST_VAR' + value = 'somevalue' + record = 'testrecord' + source = EnvVarSource('testid', envvar, record) + zone = Zone('unit.tests.', []) + + with patch.dict('os.environ', {envvar: value}): + source.populate(zone) + + # TODO: Validate zone and record From 0a342aa6c2586a87b8a0a7df74ce85e579a4aa96 Mon Sep 17 00:00:00 2001 From: Phelps Williams Date: Fri, 17 Jul 2020 12:09:20 -0700 Subject: [PATCH 6/9] EnvVar: Integrating review feedback and finishing tests --- octodns/source/envvar.py | 29 ++++++++++++----------------- tests/test_octodns_source_envvar.py | 19 +++++++++++++------ 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/octodns/source/envvar.py b/octodns/source/envvar.py index b755632..adf267a 100644 --- a/octodns/source/envvar.py +++ b/octodns/source/envvar.py @@ -40,7 +40,7 @@ class EnvVarSource(BaseSource): variable: USER # The TXT record name to embed the value found at the above # environment variable - record: deployuser + name: deployuser # The TTL of the TXT record (optional, default 60) ttl: 3600 @@ -62,42 +62,37 @@ class EnvVarSource(BaseSource): DEFAULT_TTL = 60 - def __init__(self, id, variable, record, ttl=DEFAULT_TTL): + def __init__(self, id, variable, name, ttl=DEFAULT_TTL): self.log = logging.getLogger('{}[{}]'.format( self.__class__.__name__, id)) - self.log.debug('__init__: id=%s, variable=%s, record=%s, ' - 'ttl=%d', id, variable, record, ttl) + self.log.debug('__init__: id=%s, variable=%s, name=%s, ' + 'ttl=%d', id, variable, name, ttl) super(EnvVarSource, self).__init__(id) self.envvar = variable - self.record = record + self.name = name self.ttl = ttl - self.value = None def _read_variable(self): - self.value = os.environ.get(self.envvar) - if self.value is None: + value = os.environ.get(self.envvar) + if value is None: raise EnvironmentVariableNotFoundException(self.envvar) self.log.debug('_read_variable: successfully loaded var=%s val=%s', - self.envvar, self.value) + self.envvar, value) + return value def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) - # if target: - # TODO: Environment Variable Source cannot act as a target, - # throw exception? - # return - before = len(zone.records) - self._read_variable() + value = self._read_variable() # We don't need to worry about conflicting records here because the # manager will deconflict sources on our behalf. - payload = {'ttl': self.ttl, 'type': 'TXT', 'values': [self.value]} - record = Record.new(zone, self.record, payload, source=self, + payload = {'ttl': self.ttl, 'type': 'TXT', 'values': [value]} + record = Record.new(zone, self.name, payload, source=self, lenient=lenient) zone.add_record(record, lenient=lenient) diff --git a/tests/test_octodns_source_envvar.py b/tests/test_octodns_source_envvar.py index e562aa0..0714883 100644 --- a/tests/test_octodns_source_envvar.py +++ b/tests/test_octodns_source_envvar.py @@ -18,17 +18,24 @@ class TestEnvVarSource(TestCase): self.assertEquals(msg, text_type(ctx.exception)) with patch.dict('os.environ', {envvar: 'testvalue'}): - source._read_variable() - self.assertEquals(source.value, 'testvalue') + value = source._read_variable() + self.assertEquals(value, 'testvalue') def test_populate(self): envvar = 'TEST_VAR' value = 'somevalue' - record = 'testrecord' - source = EnvVarSource('testid', envvar, record) - zone = Zone('unit.tests.', []) + name = 'testrecord' + zone_name = 'unit.tests.' + source = EnvVarSource('testid', envvar, name) + zone = Zone(zone_name, []) with patch.dict('os.environ', {envvar: value}): source.populate(zone) - # TODO: Validate zone and record + self.assertEquals(1, len(zone.records)) + record = list(zone.records)[0] + self.assertEquals(name, record.name) + self.assertEquals('{}.{}'.format(name, zone_name), record.fqdn) + self.assertEquals('TXT', record._type) + self.assertEquals(1, len(record.values)) + self.assertEquals(value, record.values[0]) From c75df0d8ed5a7fee2f7ff08d4987cfc390a870d6 Mon Sep 17 00:00:00 2001 From: Phelps Williams Date: Fri, 17 Jul 2020 12:29:17 -0700 Subject: [PATCH 7/9] Adding entry in readme for environment variable support --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d4e7171..995776a 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DnsimpleProvider](/octodns/provider/dnsimple.py) | | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | dyn | All | Both | | | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | +| [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | No CNAME support, missing `NA` geo target | From 427b8a1a061acb164edf0ecd1bafe497cce85f18 Mon Sep 17 00:00:00 2001 From: Justin B Newman Date: Mon, 20 Jul 2020 12:48:47 -0500 Subject: [PATCH 8/9] Add support for wildcard SRV records, as shown in RFC 2782 --- octodns/record/__init__.py | 2 +- tests/test_octodns_record.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 04eb2da..849e035 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -1209,7 +1209,7 @@ class SrvValue(EqualityTupleMixin): class SrvRecord(_ValuesMixin, Record): _type = 'SRV' _value_type = SrvValue - _name_re = re.compile(r'^_[^\.]+\.[^\.]+') + _name_re = re.compile(r'^(\*|_[^\.]+)\.[^\.]+') @classmethod def validate(cls, name, fqdn, data): diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index e2917b3..08a3e7a 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -2155,6 +2155,18 @@ class TestRecordValidation(TestCase): } }) + # permit wildcard entries + Record.new(self.zone, '*._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + 'target': 'food.bar.baz.' + } + }) + # invalid name with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, 'neup', { From 5c248b476db1fd045c1fffdbfaa09086b2c0ddb4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 20 Jul 2020 13:23:40 -0700 Subject: [PATCH 9/9] According to docs ipaddress was 3.3, requires for ipaddress too Also corrects futures to 3.2 in requires --- requirements.txt | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0c6cc97..dd1643f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,10 +7,10 @@ dnspython==1.16.0 docutils==0.16 dyn==1.8.1 edgegrid-python==1.1.1 -futures==3.2.0; python_version < '3.0' +futures==3.2.0; python_version < '3.2' google-cloud-core==1.3.0 google-cloud-dns==0.32.0 -ipaddress==1.0.23 +ipaddress==1.0.23; python_version < '3.3' jmespath==0.10.0 msrestazure==0.6.4 natsort==6.2.1 diff --git a/setup.py b/setup.py index 142b209..9394e7f 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ setup( 'PyYaml>=4.2b1', 'dnspython>=1.15.0', 'futures>=3.2.0; python_version<"3.2"', - 'ipaddress>=1.0.22; python_version<"3.2"', + 'ipaddress>=1.0.22; python_version<"3.3"', 'natsort>=5.5.0', 'pycountry>=19.8.18', 'pycountry-convert>=0.7.2',