From 90cc9576f540e43de4b7ac5d7bfee02a87ea5f58 Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Thu, 11 Jun 2020 17:46:29 -0400 Subject: [PATCH 1/6] Increase Cloudflare page size Increase Cloudflare page size to reduce request count `GET zones` has a MAX of 50 and a default of 20 https://api.cloudflare.com/#zone-list-zones `GET zones/:zone_identifier/dns_records` has a MAX of 100 and a default of 20 https://api.cloudflare.com/#dns-records-for-a-zone-list-dns-records --- octodns/provider/cloudflare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 698fbee..96febf4 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -142,7 +142,7 @@ class CloudflareProvider(BaseProvider): zones = [] while page: resp = self._try_request('GET', '/zones', - params={'page': page}) + params={'page': page, 'per_page': 50}) zones += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: @@ -251,7 +251,7 @@ class CloudflareProvider(BaseProvider): path = '/zones/{}/dns_records'.format(zone_id) page = 1 while page: - resp = self._try_request('GET', path, params={'page': page}) + resp = self._try_request('GET', path, params={'page': page, 'per_page': 100}) records += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: From b80d1575e6d5268e3ee69c14adcc8e2ad9dcfa88 Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Thu, 11 Jun 2020 17:56:31 -0400 Subject: [PATCH 2/6] Update tests with new per_page params --- octodns/provider/cloudflare.py | 3 ++- tests/test_octodns_provider_cloudflare.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 96febf4..96c9f5e 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -251,7 +251,8 @@ class CloudflareProvider(BaseProvider): path = '/zones/{}/dns_records'.format(zone_id) page = 1 while page: - resp = self._try_request('GET', path, params={'page': page, 'per_page': 100}) + resp = self._try_request('GET', path, params={'page': page, + 'per_page': 100}) records += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 08608ea..735d95c 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -426,7 +426,7 @@ class TestCloudflareProvider(TestCase): # get the list of zones, create a zone, add some records, update # something, and delete something provider._request.assert_has_calls([ - call('GET', '/zones', params={'page': 1}), + call('GET', '/zones', params={'page': 1, 'per_page': 50}), call('POST', '/zones', data={ 'jump_start': False, 'name': 'unit.tests' @@ -531,7 +531,7 @@ class TestCloudflareProvider(TestCase): # Get zones, create zone, create a record, delete a record provider._request.assert_has_calls([ - call('GET', '/zones', params={'page': 1}), + call('GET', '/zones', params={'page': 1, 'per_page': 50}), call('POST', '/zones', data={ 'jump_start': False, 'name': 'unit.tests' @@ -1302,7 +1302,8 @@ class TestCloudflareProvider(TestCase): provider._request.side_effect = [result] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # One retry required provider._zones = None @@ -1313,7 +1314,8 @@ class TestCloudflareProvider(TestCase): ] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # Two retries required provider._zones = None @@ -1325,7 +1327,8 @@ class TestCloudflareProvider(TestCase): ] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # # Exhaust our retries provider._zones = None From bac16622426d21768ebf6b4c895936965a3ffd66 Mon Sep 17 00:00:00 2001 From: DavHau Date: Mon, 15 Jun 2020 16:51:28 +0000 Subject: [PATCH 3/6] fix: dependency 'ipaddress' unnecessary for py >= 3.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c56aa82..142b209 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ setup( 'PyYaml>=4.2b1', 'dnspython>=1.15.0', 'futures>=3.2.0; python_version<"3.2"', - 'ipaddress>=1.0.22', + 'ipaddress>=1.0.22; python_version<"3.2"', 'natsort>=5.5.0', 'pycountry>=19.8.18', 'pycountry-convert>=0.7.2', From 84048dbde9df4de639141d32ca4722c538b2b92b Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Mon, 22 Jun 2020 17:27:41 -0400 Subject: [PATCH 4/6] Cloudflare: Make page size configurable --- octodns/provider/cloudflare.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 96c9f5e..db937e5 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -58,6 +58,10 @@ class CloudflareProvider(BaseProvider): retry_count: 4 # Optional. Default: 300. Number of seconds to wait before retrying. retry_period: 300 + # Optional. Default: 50. Number of zones per page. + zones_per_page: 50 + # Optional. Default: 100. Number of dns records per page. + records_per_page: 100 Note: The "proxied" flag of "A", "AAAA" and "CNAME" records can be managed via the YAML provider like so: @@ -78,7 +82,8 @@ class CloudflareProvider(BaseProvider): TIMEOUT = 15 def __init__(self, id, email=None, token=None, cdn=False, retry_count=4, - retry_period=300, *args, **kwargs): + retry_period=300, zones_per_page=50, records_per_page=100, + *args, **kwargs): self.log = getLogger('CloudflareProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, email=%s, token=***, cdn=%s', id, email, cdn) @@ -99,6 +104,8 @@ class CloudflareProvider(BaseProvider): self.cdn = cdn self.retry_count = retry_count self.retry_period = retry_period + self.zones_per_page = zones_per_page + self.records_per_page = records_per_page self._sess = sess self._zones = None @@ -142,7 +149,10 @@ class CloudflareProvider(BaseProvider): zones = [] while page: resp = self._try_request('GET', '/zones', - params={'page': page, 'per_page': 50}) + params={ + 'page': page, + 'per_page': self.zones_per_page + }) zones += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: @@ -252,7 +262,7 @@ class CloudflareProvider(BaseProvider): page = 1 while page: resp = self._try_request('GET', path, params={'page': page, - 'per_page': 100}) + 'per_page': self.records_per_page}) records += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: From 427b8a1a061acb164edf0ecd1bafe497cce85f18 Mon Sep 17 00:00:00 2001 From: Justin B Newman Date: Mon, 20 Jul 2020 12:48:47 -0500 Subject: [PATCH 5/6] Add support for wildcard SRV records, as shown in RFC 2782 --- octodns/record/__init__.py | 2 +- tests/test_octodns_record.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 04eb2da..849e035 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -1209,7 +1209,7 @@ class SrvValue(EqualityTupleMixin): class SrvRecord(_ValuesMixin, Record): _type = 'SRV' _value_type = SrvValue - _name_re = re.compile(r'^_[^\.]+\.[^\.]+') + _name_re = re.compile(r'^(\*|_[^\.]+)\.[^\.]+') @classmethod def validate(cls, name, fqdn, data): diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index e2917b3..08a3e7a 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -2155,6 +2155,18 @@ class TestRecordValidation(TestCase): } }) + # permit wildcard entries + Record.new(self.zone, '*._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + 'target': 'food.bar.baz.' + } + }) + # invalid name with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, 'neup', { From 5c248b476db1fd045c1fffdbfaa09086b2c0ddb4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 20 Jul 2020 13:23:40 -0700 Subject: [PATCH 6/6] According to docs ipaddress was 3.3, requires for ipaddress too Also corrects futures to 3.2 in requires --- requirements.txt | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0c6cc97..dd1643f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,10 +7,10 @@ dnspython==1.16.0 docutils==0.16 dyn==1.8.1 edgegrid-python==1.1.1 -futures==3.2.0; python_version < '3.0' +futures==3.2.0; python_version < '3.2' google-cloud-core==1.3.0 google-cloud-dns==0.32.0 -ipaddress==1.0.23 +ipaddress==1.0.23; python_version < '3.3' jmespath==0.10.0 msrestazure==0.6.4 natsort==6.2.1 diff --git a/setup.py b/setup.py index 142b209..9394e7f 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ setup( 'PyYaml>=4.2b1', 'dnspython>=1.15.0', 'futures>=3.2.0; python_version<"3.2"', - 'ipaddress>=1.0.22; python_version<"3.2"', + 'ipaddress>=1.0.22; python_version<"3.3"', 'natsort>=5.5.0', 'pycountry>=19.8.18', 'pycountry-convert>=0.7.2',