From 7eaf03dec40a809a36b5554e5391f64656827bef Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 30 May 2019 17:33:42 -0700 Subject: [PATCH 01/86] started fast dns --- octodns/provider/fastdns.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 octodns/provider/fastdns.py diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py new file mode 100644 index 0000000..59318cc --- /dev/null +++ b/octodns/provider/fastdns.py @@ -0,0 +1 @@ +## Provider file for Akamai Fast DNS \ No newline at end of file From 95681972b7722e8d880574ba56f7ccbbae45dcf3 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 30 May 2019 18:25:28 -0700 Subject: [PATCH 02/86] starting authentication for akamai fastdns --- octodns/provider/fastdns.py | 55 ++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 59318cc..3504e02 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -1 +1,54 @@ -## Provider file for Akamai Fast DNS \ No newline at end of file +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +## octodns specfic imports: + +import + +import logging +from functools import reduce +from ..record import Record +from .base import BaseProvider + + +class _AkamaiRecord(object): + pass + + + + +class AkamaiProvider(BaseProvider): + pass + + def __init__(self, id, client_secret, host, access_token, client_token, + *args, **kwargs): + + self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, ') + super(AkamaiProvider, self).__init__(id, *args, **kwargs) + + ## generate edgegrid + #### create credential file, and write credentials to it + credFile = open('tempCred.txt', 'w') + + credFile.write('client_secret = ') + credFile.write(str(client_secret)) + credFile.write('\n\n') + + credFile.write('host = %s', str(host)) + credFile.write('\n\n') + + credFile.write('access_token = %s', str(access_token)) + credFile.write('\n\n') + + credFile.write('client_token = %s', str(client_token)) + + credFile.close() + + #### generate edgegrid using tool + + #### delete temp txt file \ No newline at end of file From f7239e30a5766f19b14cffc5dbe79159a6989d37 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 30 May 2019 18:33:03 -0700 Subject: [PATCH 03/86] fixed import bug --- octodns/provider/fastdns.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 3504e02..2a24edc 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -7,8 +7,6 @@ from __future__ import absolute_import, division, print_function, \ ## octodns specfic imports: -import - import logging from functools import reduce from ..record import Record From 99875a128999c5090c9359cfac1d30850ce48a02 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 30 May 2019 18:36:31 -0700 Subject: [PATCH 04/86] fixed base class errors --- octodns/provider/fastdns.py | 7 ++++++- src/octodns | 1 + src/pip-delete-this-directory.txt | 5 +++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 160000 src/octodns create mode 100644 src/pip-delete-this-directory.txt diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 2a24edc..a9591d4 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -20,8 +20,13 @@ class _AkamaiRecord(object): class AkamaiProvider(BaseProvider): - pass + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', + 'TXT')) + + def __init__(self, id, client_secret, host, access_token, client_token, *args, **kwargs): diff --git a/src/octodns b/src/octodns new file mode 160000 index 0000000..b57cbbb --- /dev/null +++ b/src/octodns @@ -0,0 +1 @@ +Subproject commit b57cbbb09230684552f1fa4db5fb53a921486b96 diff --git a/src/pip-delete-this-directory.txt b/src/pip-delete-this-directory.txt new file mode 100644 index 0000000..c8883ea --- /dev/null +++ b/src/pip-delete-this-directory.txt @@ -0,0 +1,5 @@ +This file is placed here by pip to indicate the source was put +here by pip. + +Once this package is successfully installed this source code will be +deleted (unless you remove this file). From 383452809010f423cac9e48f28af5685bc6f2ee4 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 30 May 2019 18:49:41 -0700 Subject: [PATCH 05/86] fixed file writing error --- octodns/provider/fastdns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index a9591d4..be4df69 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -42,13 +42,13 @@ class AkamaiProvider(BaseProvider): credFile.write(str(client_secret)) credFile.write('\n\n') - credFile.write('host = %s', str(host)) + credFile.write('host = ' + str(host)) credFile.write('\n\n') - credFile.write('access_token = %s', str(access_token)) + credFile.write('access_token = ' + str(access_token)) credFile.write('\n\n') - credFile.write('client_token = %s', str(client_token)) + credFile.write('client_token = ' + str(client_token)) credFile.close() From e7d82ac75d74f1daf2cb9ad76e67f1cab690a4de Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Tue, 4 Jun 2019 12:16:14 -0700 Subject: [PATCH 06/86] sync --- octodns/provider/fastdns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index be4df69..439f924 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -51,6 +51,7 @@ class AkamaiProvider(BaseProvider): credFile.write('client_token = ' + str(client_token)) credFile.close() + #### generate edgegrid using tool From 064b30c4d1b9f29eb3f392b33838664d1665a4fc Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 09:29:45 -0700 Subject: [PATCH 07/86] began authentication script --- octodns/provider/fastdns.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 439f924..c955cdf 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -49,10 +49,16 @@ class AkamaiProvider(BaseProvider): credFile.write('\n\n') credFile.write('client_token = ' + str(client_token)) + credFile.write('\n') credFile.close() - + #### generate edgegrid using tool - #### delete temp txt file \ No newline at end of file + #### delete temp txt file + + def populate(self, zone, target=False, lenient=False): + pass + + From 34b618fbee77de96bf1c8bc6afb4fd8762296c6b Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 09:45:27 -0700 Subject: [PATCH 08/86] began authentication script --- octodns/provider/fastdns.py | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index c955cdf..a01f71e 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -6,6 +6,9 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals ## octodns specfic imports: +from os.path import expanduser +import ConfigParser + import logging from functools import reduce @@ -34,8 +37,49 @@ class AkamaiProvider(BaseProvider): self.log.debug('__init__: id=%s, ') super(AkamaiProvider, self).__init__(id, *args, **kwargs) + self._authenticate(self, client_secret, host, access_token, client_token) + + + def _authenticate(self, client_secret, host, access_token, client_token) + ## generate edgegrid + section_name = "dns" + home = expanduser("~") + index = 0 + fields = {} + + ## process original .edgerc file + origConfig = ConfigParser.ConfigParser() + filename = "%s/.edgerc" % home + + # If this is a new file, create it + if not os.path.isfile(filename): + open(filename, 'a+').close() + + origConfig.read(filename) + + + if section_name in origConfig.sections(): + print (">>> Replacing section: %s" % section_name) + + + + sys.stdout.write ("*** OK TO REPLACE section %s? *** [Y|n]:" % section_name) + real_raw_input = vars(__builtins__).get('raw_input',input) + choice = real_raw_input().lower() + if choice == "n": + print ("Not replacing section.") + exit(0) + + replace_section = True + else: + print ("+++ Creating section: %s" % section_name) + replace_section = False + + #### create credential file, and write credentials to it + + credFile = open('tempCred.txt', 'w') credFile.write('client_secret = ') From e1a139d18753ecb7f68acf7c9a051c686fe5332d Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 14:31:26 -0400 Subject: [PATCH 09/86] revamed authentication --- octodns/provider/fastdns.py | 94 +++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 50 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index a01f71e..597c68a 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -9,6 +9,8 @@ from __future__ import absolute_import, division, print_function, \ from os.path import expanduser import ConfigParser +# import os ## command line workaround + import logging from functools import reduce @@ -42,65 +44,57 @@ class AkamaiProvider(BaseProvider): def _authenticate(self, client_secret, host, access_token, client_token) - ## generate edgegrid - section_name = "dns" - home = expanduser("~") - index = 0 - fields = {} + # ## generate edgegrid + # section_name = "dns" + # home = expanduser("~") + # index = 0 + # fields = {} - ## process original .edgerc file - origConfig = ConfigParser.ConfigParser() - filename = "%s/.edgerc" % home + # ## process original .edgerc file + # origConfig = ConfigParser.ConfigParser() + # filename = "%s/.edgerc" % home - # If this is a new file, create it - if not os.path.isfile(filename): - open(filename, 'a+').close() - - origConfig.read(filename) - - - if section_name in origConfig.sections(): - print (">>> Replacing section: %s" % section_name) + # # If this is a new file, create it + # if not os.path.isfile(filename): + # open(filename, 'a+').close() - - - sys.stdout.write ("*** OK TO REPLACE section %s? *** [Y|n]:" % section_name) - real_raw_input = vars(__builtins__).get('raw_input',input) - choice = real_raw_input().lower() - if choice == "n": - print ("Not replacing section.") - exit(0) - - replace_section = True - else: - print ("+++ Creating section: %s" % section_name) - replace_section = False + # origConfig.read(filename) + + # ## credential type already exists, prompt to overwrite + # if section_name in origConfig.sections(): + # print (">>> Replacing section: %s" % section_name) + # sys.stdout.write ("*** OK TO REPLACE section %s? *** [Y|n]:" % section_name) + # real_raw_input = vars(__builtins__).get('raw_input',input) + # choice = real_raw_input().lower() + # if choice == "n": + # print ("Not replacing section.") + # return + # replace_section = True + # else: + # print ("+++ Creating section: %s" % section_name) + # replace_section = False + + # ## open the ~/.edgerc file for writing + # Config = ConfigParser.ConfigParser() + # Config.read(filename) + # confile = open(filename, 'w') + + + # ## add the new section + #### create credential file, and write credentials to it + with open('~/.edgerc', 'a') as credFile: - credFile = open('tempCred.txt', 'w') - - credFile.write('client_secret = ') - credFile.write(str(client_secret)) - credFile.write('\n\n') - - credFile.write('host = ' + str(host)) - credFile.write('\n\n') - - credFile.write('access_token = ' + str(access_token)) - credFile.write('\n\n') - - credFile.write('client_token = ' + str(client_token)) - credFile.write('\n') - - credFile.close() - - - #### generate edgegrid using tool + credFile.write('[dns]\n') - #### delete temp txt file + credFile.write('client_secret = ' + str(client_secret) + '\n') + credFile.write('host = ' + str(host) + '\n') + credFile.write('access_token = ' + str(access_token) + '\n') + credFile.write('client_token = ' + str(client_token) + '\n') + credFile.close() def populate(self, zone, target=False, lenient=False): pass From d4867c4268c90408e3db49c1f92deaa68a8869d1 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 14:40:41 -0400 Subject: [PATCH 10/86] fixed typo in authernticate --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 597c68a..94c049e 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -42,7 +42,7 @@ class AkamaiProvider(BaseProvider): self._authenticate(self, client_secret, host, access_token, client_token) - def _authenticate(self, client_secret, host, access_token, client_token) + def _authenticate(self, client_secret, host, access_token, client_token): # ## generate edgegrid # section_name = "dns" From f21915308850a7402d75b5e28a6d8647ec4f8321 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 14:49:58 -0400 Subject: [PATCH 11/86] fixed call to auth fxn --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 94c049e..e8fb35c 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -39,7 +39,7 @@ class AkamaiProvider(BaseProvider): self.log.debug('__init__: id=%s, ') super(AkamaiProvider, self).__init__(id, *args, **kwargs) - self._authenticate(self, client_secret, host, access_token, client_token) + self._authenticate(client_secret, host, access_token, client_token) def _authenticate(self, client_secret, host, access_token, client_token): From 3d61baea5060ed1ff75ec14d6b8a2153b8c99780 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 15:00:04 -0400 Subject: [PATCH 12/86] fixed issue with accessing home dir --- octodns/provider/fastdns.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index e8fb35c..1ce91d0 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -86,7 +86,10 @@ class AkamaiProvider(BaseProvider): #### create credential file, and write credentials to it - with open('~/.edgerc', 'a') as credFile: + + home = expanduser("~") + filename = "%s/.edgerc" % home + with open(filename, 'a') as credFile: credFile.write('[dns]\n') From 2ea26273b807b1e4a955133a391512a25cc82d10 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 17:25:16 -0400 Subject: [PATCH 13/86] started populate fxn --- octodns/provider/fastdns.py | 57 +++++++------------------------------ 1 file changed, 10 insertions(+), 47 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 1ce91d0..56e94f8 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -7,9 +7,9 @@ from __future__ import absolute_import, division, print_function, \ ## octodns specfic imports: from os.path import expanduser -import ConfigParser - -# import os ## command line workaround +import requests +from akamai.edgegrid import EdgeGridAuth +from urlparse import urljoin import logging @@ -40,53 +40,12 @@ class AkamaiProvider(BaseProvider): super(AkamaiProvider, self).__init__(id, *args, **kwargs) self._authenticate(client_secret, host, access_token, client_token) + self._zone_records = {} def _authenticate(self, client_secret, host, access_token, client_token): - # ## generate edgegrid - # section_name = "dns" - # home = expanduser("~") - # index = 0 - # fields = {} - - # ## process original .edgerc file - # origConfig = ConfigParser.ConfigParser() - # filename = "%s/.edgerc" % home - - # # If this is a new file, create it - # if not os.path.isfile(filename): - # open(filename, 'a+').close() - - # origConfig.read(filename) - - # ## credential type already exists, prompt to overwrite - # if section_name in origConfig.sections(): - # print (">>> Replacing section: %s" % section_name) - # sys.stdout.write ("*** OK TO REPLACE section %s? *** [Y|n]:" % section_name) - # real_raw_input = vars(__builtins__).get('raw_input',input) - # choice = real_raw_input().lower() - # if choice == "n": - # print ("Not replacing section.") - # return - # replace_section = True - # else: - # print ("+++ Creating section: %s" % section_name) - # replace_section = False - - # ## open the ~/.edgerc file for writing - # Config = ConfigParser.ConfigParser() - # Config.read(filename) - # confile = open(filename, 'w') - - - # ## add the new section - - - - #### create credential file, and write credentials to it - - + ## generate edgegrid home = expanduser("~") filename = "%s/.edgerc" % home with open(filename, 'a') as credFile: @@ -100,6 +59,10 @@ class AkamaiProvider(BaseProvider): credFile.close() def populate(self, zone, target=False, lenient=False): - pass + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + print ("populate(%s)", zone.name) + From 66136f475f2b41a3e3c3cac1f195dc71fba1e595 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 17:27:42 -0400 Subject: [PATCH 14/86] fixed typo --- octodns/provider/fastdns.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 56e94f8..7cd0593 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -59,10 +59,11 @@ class AkamaiProvider(BaseProvider): credFile.close() def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, - target, lenient) + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) print ("populate(%s)", zone.name) + return + From 4a68f7df259f40e06679512c8b8fcc7c059fc593 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:32:46 -0400 Subject: [PATCH 15/86] added dns client class, changed auth method --- octodns/provider/fastdns.py | 39 ++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 7cd0593..fe5602c 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -22,6 +22,29 @@ class _AkamaiRecord(object): pass +class AkamaiClient(object): + + def __init__(self, _client_secret, _host, _access_token, _client_token): + + self.base = "https://" + _host + "/config-dns/v1/zones/" + + sess = requests.Session() + sess.auth = EdgeGridAuth( + client_token=_client_token, + client_secret=_client_secret, + access_token=_access_token + ) + self._sess = sess + + + def getZone(self, name): + path = urljoin(self.base, name) + result = self.sess.get(path) + return result.json() + + + # def _get(self, method, path, params=None, data=None): + # url = '{}{}{}'.format(self.) class AkamaiProvider(BaseProvider): @@ -39,7 +62,12 @@ class AkamaiProvider(BaseProvider): self.log.debug('__init__: id=%s, ') super(AkamaiProvider, self).__init__(id, *args, **kwargs) - self._authenticate(client_secret, host, access_token, client_token) + + self._dns_client = AkamaiClient(client_secret, host, access_token, client_token) + + + + #self._authenticate(client_secret, host, access_token, client_token) self._zone_records = {} @@ -61,9 +89,14 @@ class AkamaiProvider(BaseProvider): def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) - print ("populate(%s)", zone.name) + print ("populate()", zone.name) - return + result = self._dns_client.getZone() + + print(result) + + return + From 6afab8c90a1033c51809a5d94f39edacaf7286fb Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:35:04 -0400 Subject: [PATCH 16/86] fixed typo --- octodns/provider/fastdns.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index fe5602c..afbbfc2 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -90,13 +90,14 @@ class AkamaiProvider(BaseProvider): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) print ("populate()", zone.name) + - result = self._dns_client.getZone() + result = self._dns_client.getZone(zone.name) print(result) return - + From b385b1ddb5d2a5bc7b4c3eb3e3907c4ef80ef7b4 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:36:16 -0400 Subject: [PATCH 17/86] fixed typo --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index afbbfc2..da9d3ee 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -39,7 +39,7 @@ class AkamaiClient(object): def getZone(self, name): path = urljoin(self.base, name) - result = self.sess.get(path) + result = self._sess.get(path) return result.json() From 32f01da506961ad7ab0daf7ff3677aeadcb463fe Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:39:24 -0400 Subject: [PATCH 18/86] fixed zone name trailing . --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index da9d3ee..4d0d2a9 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -91,7 +91,7 @@ class AkamaiProvider(BaseProvider): print ("populate()", zone.name) - + zone_name = zone.name[:len(zone_name)-1] result = self._dns_client.getZone(zone.name) print(result) From d2ec707255a1323de8fcaa26e792ea2390f75fa3 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:40:30 -0400 Subject: [PATCH 19/86] fixed typo --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 4d0d2a9..b0b3c8e 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -91,7 +91,7 @@ class AkamaiProvider(BaseProvider): print ("populate()", zone.name) - zone_name = zone.name[:len(zone_name)-1] + zone_name = zone.name[:len(zone.name)-1] result = self._dns_client.getZone(zone.name) print(result) From a9fe4a9cf6bba95fbeb951ee2384323a59c263e8 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:42:26 -0400 Subject: [PATCH 20/86] fixed typo --- octodns/provider/fastdns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index b0b3c8e..a58fbe5 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -92,7 +92,8 @@ class AkamaiProvider(BaseProvider): print ("populate()", zone.name) zone_name = zone.name[:len(zone.name)-1] - result = self._dns_client.getZone(zone.name) + + result = self._dns_client.getZone(zone_name) print(result) From 9c43c92339eaa0996fba256a9c92a538d49fac3c Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:52:55 -0400 Subject: [PATCH 21/86] json parse --- octodns/provider/fastdns.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index a58fbe5..7f50cb1 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -6,10 +6,10 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals ## octodns specfic imports: -from os.path import expanduser import requests from akamai.edgegrid import EdgeGridAuth from urlparse import urljoin +import json import logging @@ -65,12 +65,13 @@ class AkamaiProvider(BaseProvider): self._dns_client = AkamaiClient(client_secret, host, access_token, client_token) - - #self._authenticate(client_secret, host, access_token, client_token) self._zone_records = {} + + + def _authenticate(self, client_secret, host, access_token, client_token): ## generate edgegrid @@ -95,7 +96,11 @@ class AkamaiProvider(BaseProvider): result = self._dns_client.getZone(zone_name) - print(result) + + print "\n\n" + print json.dumps(result, indent=4, separators=(',', ': ')) + print "\n\n" + return From ec5e7a173ecc90b9a719bcc31a62fad2c6891a0c Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:54:19 -0400 Subject: [PATCH 22/86] fixed typo --- octodns/provider/fastdns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 7f50cb1..caf17ae 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -97,9 +97,9 @@ class AkamaiProvider(BaseProvider): result = self._dns_client.getZone(zone_name) - print "\n\n" - print json.dumps(result, indent=4, separators=(',', ': ')) - print "\n\n" + print ("\n\n") + print (json.dumps(result, indent=4, separators=(',', ': '))) + print ("\n\n") return From c7b39053fe4d4876854d35faeb13ee69960dc39b Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 19:16:37 -0400 Subject: [PATCH 23/86] sync --- octodns/provider/fastdns.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index caf17ae..fc37834 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -43,8 +43,6 @@ class AkamaiClient(object): return result.json() - # def _get(self, method, path, params=None, data=None): - # url = '{}{}{}'.format(self.) class AkamaiProvider(BaseProvider): @@ -90,10 +88,8 @@ class AkamaiProvider(BaseProvider): def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) - print ("populate()", zone.name) - + zone_name = zone.name[:len(zone.name)-1] - result = self._dns_client.getZone(zone_name) @@ -101,7 +97,6 @@ class AkamaiProvider(BaseProvider): print (json.dumps(result, indent=4, separators=(',', ': '))) print ("\n\n") - return From 4943b72b4cdfcf25d5eee2287ec4ea4752e616f4 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Mon, 10 Jun 2019 07:54:57 -0700 Subject: [PATCH 24/86] sync --- octodns/provider/.vscode/settings.json | 2 ++ octodns/provider/fastdns.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 octodns/provider/.vscode/settings.json diff --git a/octodns/provider/.vscode/settings.json b/octodns/provider/.vscode/settings.json new file mode 100644 index 0000000..7a73a41 --- /dev/null +++ b/octodns/provider/.vscode/settings.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index fc37834..1405d17 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -40,8 +40,9 @@ class AkamaiClient(object): def getZone(self, name): path = urljoin(self.base, name) result = self._sess.get(path) - return result.json() + return result.json() + # return result @@ -91,9 +92,14 @@ class AkamaiProvider(BaseProvider): zone_name = zone.name[:len(zone.name)-1] result = self._dns_client.getZone(zone_name) + + print() + print() + + print(type(result)) + #print(result.text) - print ("\n\n") print (json.dumps(result, indent=4, separators=(',', ': '))) print ("\n\n") From 1aa9424e213dc0e70e6a03164778aae8a723db05 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Mon, 10 Jun 2019 14:23:26 -0700 Subject: [PATCH 25/86] working sync --- src/octodns | 1 - src/pip-delete-this-directory.txt | 5 ----- 2 files changed, 6 deletions(-) delete mode 160000 src/octodns delete mode 100644 src/pip-delete-this-directory.txt diff --git a/src/octodns b/src/octodns deleted file mode 160000 index b57cbbb..0000000 --- a/src/octodns +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b57cbbb09230684552f1fa4db5fb53a921486b96 diff --git a/src/pip-delete-this-directory.txt b/src/pip-delete-this-directory.txt deleted file mode 100644 index c8883ea..0000000 --- a/src/pip-delete-this-directory.txt +++ /dev/null @@ -1,5 +0,0 @@ -This file is placed here by pip to indicate the source was put -here by pip. - -Once this package is successfully installed this source code will be -deleted (unless you remove this file). From b58210328ff2ca7942fd6457ca45ca4c861aeb6f Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Wed, 12 Jun 2019 20:31:11 -0400 Subject: [PATCH 26/86] implemented API v2 - client fxns to request records functionality --- octodns/provider/fastdns.py | 127 +++++++++++++++++++++++++++--------- 1 file changed, 95 insertions(+), 32 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 1405d17..a6d5cae 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -18,6 +18,30 @@ from ..record import Record from .base import BaseProvider +class AkamaiClientException(Exception): + pass + +class AkamaiClientBadRequest(AkamaiClientException): #400 + def __init__(self): + super(AkamaiClientBadRequest, self).__init__('Bad request') + +class AkamaiClientNotAuthorized(AkamaiClientException): #403 + def __init__(self): + super(AkamaiClientNotAuthorized, self).__init__('Forbidden') + +class AkamaiClientNotFound(AkamaiClientException): #404 + def __init__(self): + super(AkamaiClientNotFound, self).__init__('Not found') + +class AkamaiClientNotAcceptable(AkamaiClientException): #406 + def __init__(self): + super(AkamaiClientNotAcceptable, self).__init__('Not acceptable') + +class AkamaiClientGenericExcp(AkamaiClientException): + def __init__(self, num): + super(AkamaiClientGenericExcp, self).__init__('HTTP Error: ' + str(num)) + + class _AkamaiRecord(object): pass @@ -26,7 +50,8 @@ class AkamaiClient(object): def __init__(self, _client_secret, _host, _access_token, _client_token): - self.base = "https://" + _host + "/config-dns/v1/zones/" + self.base = "https://" + _host + "/config-dns/v2/zones/" + self.basev1 = "https://" + _host + "/config-dns/v1/zones/" sess = requests.Session() sess.auth = EdgeGridAuth( @@ -36,22 +61,71 @@ class AkamaiClient(object): ) self._sess = sess + def _request(self, method, path, params=None, data=None): + # url = '{}{}'.format(self.base, path) + url = urljoin(self.base, path) + print(url) + resp = self._sess.request(method, url, params=params, json=data) + + if resp.status_code == 400: + raise AkamaiClientBadRequest() + if resp.status_code == 403: + raise AkamaiClientNotAuthorized() + if resp.status_code == 404: + raise AkamaiClientNotFound() + if resp.status_code == 406: + raise AkamaiClientNotAcceptable() + resp.raise_for_status() - def getZone(self, name): - path = urljoin(self.base, name) - result = self._sess.get(path) + return resp + + # def getZone(self, name): + # path = name + "/recordsets/" + # result = self._request('GET', path, "sortBy=type&types=A") + + # return result.json() + + # def getNames(self, name): + # path = name + "/names/" + # result = self._request('GET', path) + + # return result.json() + + + + def record_get(self, zone, name, record_type): + + path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + result = self._request('GET', path) + return result.json() - # return result + def record_create(self, zone, name, record_type, params): + path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + result = self._request('POST', path, data=params) + + return result + + + def record_delete(self, zone, name, record_type): + path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + result = self._request('DELETE', path) + + return result + def record_replace(self, zone, name, record_type, params): + path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + result = self._request('PUT', path, data=params) + + return result class AkamaiProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', - 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', + 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, client_secret, host, access_token, client_token, @@ -64,44 +138,33 @@ class AkamaiProvider(BaseProvider): self._dns_client = AkamaiClient(client_secret, host, access_token, client_token) - #self._authenticate(client_secret, host, access_token, client_token) - self._zone_records = {} - - + self._zone_records = {} - def _authenticate(self, client_secret, host, access_token, client_token): - - ## generate edgegrid - home = expanduser("~") - filename = "%s/.edgerc" % home - with open(filename, 'a') as credFile: - - credFile.write('[dns]\n') - - credFile.write('client_secret = ' + str(client_secret) + '\n') - credFile.write('host = ' + str(host) + '\n') - credFile.write('access_token = ' + str(access_token) + '\n') - credFile.write('client_token = ' + str(client_token) + '\n') - credFile.close() def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) zone_name = zone.name[:len(zone.name)-1] - result = self._dns_client.getZone(zone_name) + zone = self._dns_client.getZone(zone_name) + names = self._dns_client.getNames(zone_name) - - print() - print() - print(type(result)) + # print(type(result)) #print(result.text) - print (json.dumps(result, indent=4, separators=(',', ': '))) - print ("\n\n") + zoneFile = open('zoneFile.json', 'w') + namesFile = open('namesFile.json', 'w') + zoneFile.write(json.dumps(zone, indent=4, separators=(',', ': '))) + namesFile.write(json.dumps(names, indent=4, separators=(',', ': '))) + zoneFile.close() + namesFile.close() + + + + #print (json.dumps(result, indent=4, separators=(',', ': '))) return From e85fce6fe0751e426222d21a2837e466b0910078 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 13 Jun 2019 13:16:25 -0400 Subject: [PATCH 27/86] added test class to test API calls -- to be removed later --- octodns/provider/fastdns.py | 78 ++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index a6d5cae..cb5123d 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -64,7 +64,23 @@ class AkamaiClient(object): def _request(self, method, path, params=None, data=None): # url = '{}{}'.format(self.base, path) url = urljoin(self.base, path) - print(url) + print(method, url) + + req = requests.Request(method, url, params=params, json=data) + prepped = req.prepare() + + filename = str(method) + ".json" + f = open(filename, 'w') + # f.write(json.dumps(prepped, indent=4, separators=(',', ': '))) + f.write(method + " " + url + "\n") + f.write("headers: \n") + f.write(str(prepped.headers)) + f.write("\nbody: \n") + f.write(str(prepped.body)) + f.close() + + + ''' resp = self._sess.request(method, url, params=params, json=data) if resp.status_code == 400: @@ -78,20 +94,22 @@ class AkamaiClient(object): resp.raise_for_status() return resp + ''' + return prepped + ''' + def getZone(self, name): + path = name + "/recordsets/" + result = self._request('GET', path, "sortBy=type&types=A") - # def getZone(self, name): - # path = name + "/recordsets/" - # result = self._request('GET', path, "sortBy=type&types=A") - - # return result.json() - - # def getNames(self, name): - # path = name + "/names/" - # result = self._request('GET', path) + return result.json() - # return result.json() + def getNames(self, name): + path = name + "/names/" + result = self._request('GET', path) + return result.json() + ''' def record_get(self, zone, name, record_type): @@ -99,6 +117,7 @@ class AkamaiClient(object): path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) result = self._request('GET', path) + return return result.json() def record_create(self, zone, name, record_type, params): @@ -145,29 +164,36 @@ class AkamaiProvider(BaseProvider): def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) + self._test(zone) - zone_name = zone.name[:len(zone.name)-1] - zone = self._dns_client.getZone(zone_name) - names = self._dns_client.getNames(zone_name) - - - # print(type(result)) - #print(result.text) - zoneFile = open('zoneFile.json', 'w') - namesFile = open('namesFile.json', 'w') - zoneFile.write(json.dumps(zone, indent=4, separators=(',', ': '))) - namesFile.write(json.dumps(names, indent=4, separators=(',', ': '))) - zoneFile.close() - namesFile.close() + def _test(self, zone) : + zone_name = zone.name[:len(zone.name)-1] - #print (json.dumps(result, indent=4, separators=(',', ': '))) + record_name = "www.exampleRecordName.com" + record_type = "AAAA" + params = { + "name": "www.example.com", + "type": "AAAA", + "ttl": 300, + "rdata": [ + "10.0.0.2", + "10.0.0.3" + ] + } + + self._dns_client.record_get(zone_name, record_name, record_type) + self._dns_client.record_create(zone_name, record_name, record_type, params) + self._dns_client.record_delete(zone_name, record_name, record_type) + self._dns_client.record_replace(zone_name, record_name, record_type, params) - return + # domainRequest = self._dns_client.getZone(zone_name) + # names = self._dns_client.getNames(zone_name) + return From ea9c90c3bb7ed47ef83a1fdeaf0d51a12177019a Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 13 Jun 2019 18:26:56 -0400 Subject: [PATCH 28/86] major commit: refactored exceptions, implemented testing suite, implm zones get, zone recordset get, record api calls --- octodns/provider/fastdns.py | 313 +++++++++++++++++++++++++++++------- 1 file changed, 252 insertions(+), 61 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index cb5123d..56aa883 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -17,10 +17,28 @@ from functools import reduce from ..record import Record from .base import BaseProvider +TESTING = False class AkamaiClientException(Exception): - pass - + + _errorMessages = { + 400: "400: Bad request", + 403: "403: Access is forbidden", + 404: "404: Resource not found", + 405: "405: Method not supported", + 406: "406: Not Acceptable", + 409: "409: Request not allowed due to conflict with current state of resource", + 415: "415: Unsupported media type", + 422: "422: Request body contains an error preventing processing", + 500: "500: Internal server error" + } + + def __init__(self, code): + message = self._errorMessages.get(code) + super(AkamaiClientException, self).__init__(message) + + +''' class AkamaiClientBadRequest(AkamaiClientException): #400 def __init__(self): super(AkamaiClientBadRequest, self).__init__('Bad request') @@ -39,8 +57,8 @@ class AkamaiClientNotAcceptable(AkamaiClientException): #406 class AkamaiClientGenericExcp(AkamaiClientException): def __init__(self, num): - super(AkamaiClientGenericExcp, self).__init__('HTTP Error: ' + str(num)) - + super(AkamaiClientGenericExcp, self).__init__('HTTP Error: ' + str(num)) +''' class _AkamaiRecord(object): pass @@ -50,8 +68,8 @@ class AkamaiClient(object): def __init__(self, _client_secret, _host, _access_token, _client_token): - self.base = "https://" + _host + "/config-dns/v2/zones/" - self.basev1 = "https://" + _host + "/config-dns/v1/zones/" + self.base = "https://" + _host + "/config-dns/v2/" + self.basev1 = "https://" + _host + "/config-dns/v1/" sess = requests.Session() sess.auth = EdgeGridAuth( @@ -60,85 +78,126 @@ class AkamaiClient(object): access_token=_access_token ) self._sess = sess - - def _request(self, method, path, params=None, data=None): - # url = '{}{}'.format(self.base, path) - url = urljoin(self.base, path) - print(method, url) - req = requests.Request(method, url, params=params, json=data) - prepped = req.prepare() - filename = str(method) + ".json" + def _writePrepped(self, prepped): + filename = str(prepped.method) + ".json" f = open(filename, 'w') - # f.write(json.dumps(prepped, indent=4, separators=(',', ': '))) - f.write(method + " " + url + "\n") + f.write("headers: \n") f.write(str(prepped.headers)) f.write("\nbody: \n") f.write(str(prepped.body)) + f.write("\nhooks:\n") + f.write(str(prepped.hooks)) + f.write("\nmethod: " + str(prepped.method)) + f.write("\nurl: " + str(prepped.url) + "\n") + f.close() + def _request(self, method, path, params=None, data=None): + + url = urljoin(self.base, path) + + if (TESTING): + print("testing mode") + print(method, url) + req = requests.Request(method, url, params=params, json=data) + prepped = req.prepare() + self._writePrepped(prepped) - ''' resp = self._sess.request(method, url, params=params, json=data) - if resp.status_code == 400: - raise AkamaiClientBadRequest() - if resp.status_code == 403: - raise AkamaiClientNotAuthorized() - if resp.status_code == 404: - raise AkamaiClientNotFound() - if resp.status_code == 406: - raise AkamaiClientNotAcceptable() + + if resp.status_code > 299: + raise AkamaiClientException(resp.status_code) + + # if resp.status_code == 400: + # raise AkamaiClientBadRequest() + # if resp.status_code == 403: + # raise AkamaiClientNotAuthorized() + # if resp.status_code == 404: + # raise AkamaiClientNotFound() + # if resp.status_code == 406: + # raise AkamaiClientNotAcceptable() resp.raise_for_status() - return resp - ''' - return prepped - - ''' - def getZone(self, name): - path = name + "/recordsets/" - result = self._request('GET', path, "sortBy=type&types=A") - - return result.json() - - def getNames(self, name): - path = name + "/names/" - result = self._request('GET', path) - - return result.json() - ''' + return resp + def record_get(self, zone, name, record_type): - path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) result = self._request('GET', path) - return - return result.json() + return result def record_create(self, zone, name, record_type, params): - path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) result = self._request('POST', path, data=params) return result - def record_delete(self, zone, name, record_type): - path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) result = self._request('DELETE', path) + if result.status_code == 204: + print ("successfully deleted ", path) + return result def record_replace(self, zone, name, record_type, params): - path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) result = self._request('PUT', path, data=params) return result + + def zones_get(self, contractIds=None, page=None, pageSize=None, search=None, + showAll="true", sortBy="zone", types=None): + path = 'zones' + + params = { + "contracIds": contractIds, + "page": page, + "pageSize": pageSize, + "search": search, + "showAll": showAll, + "sortBy": sortBy, + "types": types + } + + result = self._request('GET', path, params=params) + + return result + + + def zone_create(self): + pass + + + def zone_recordset_get(self, zone, params=None): + path = 'zones/{}/recordsets'.format(zone) + result = self._request('GET', path, params=params) + + return result + + def zone_recordset_create(self, zone): + pass + + def zone_recordset_replace(self, zone): + pass + + + def master_zone_file_get(self, zone): + path = 'zones/{}/zone-file'.format(zone) + result = self._request('GET', path) + + return result + + class AkamaiProvider(BaseProvider): SUPPORTS_GEO = False @@ -172,28 +231,160 @@ class AkamaiProvider(BaseProvider): zone_name = zone.name[:len(zone.name)-1] - - record_name = "www.exampleRecordName.com" - record_type = "AAAA" + record_name = "octo.basir-test.com" + record_type = "A" params = { - "name": "www.example.com", - "type": "AAAA", + "name": "octo.basir-test.com", + "type": "A", "ttl": 300, "rdata": [ "10.0.0.2", "10.0.0.3" ] } - - self._dns_client.record_get(zone_name, record_name, record_type) - self._dns_client.record_create(zone_name, record_name, record_type, params) - self._dns_client.record_delete(zone_name, record_name, record_type) - self._dns_client.record_replace(zone_name, record_name, record_type, params) + repl_params = { + "name": "octo.basir-test.com", + "type": "A", + "ttl": 300, + "rdata": [ + "99.99.99.99", + "10.0.0.3", + "1.2.3.4" + ] + } - # domainRequest = self._dns_client.getZone(zone_name) - # names = self._dns_client.getNames(zone_name) + print("\n\nRunning test: record get..........\n") + self._test_record_get(zone_name, "test.basir-test.com", record_type) + print("\n\nRunning test: record create..........\n") + self._test_record_create(zone_name, record_name, record_type, params) + print("\n\nRunning test: record replace..........\n") + self._test_record_replace(zone_name, record_name, record_type, repl_params) + print("\n\nRunning test: record delete..........\n") + self._test_record_delete(zone_name, record_name, record_type) + + # print("\n\nRunning test: zones get..........\n") + # self._test_zones_get() + # print("\n\nRunning test: zone recordset get..........\n") + # self._test_zones_recordset_get(zone_name) return + def _test_record_get(self, zone_name, record_name, record_type): + try: + get = self._dns_client.record_get(zone_name, record_name, record_type) + except AkamaiClientException as e: + print ("record get test failed") + print (e.message) + + else: + print("record get test result: ") + print(json.dumps(get.json(), indent=4, separators=(',', ': '))) + + return + + def _test_record_delete(self, zone_name, record_name, record_type): + + try: + delete = self._dns_client.record_delete(zone_name, record_name, record_type) + except AkamaiClientException as e: + print("delete failed") + print(e.message) + return + + + try: + self._dns_client.record_get(zone_name, record_name, record_type) + except AkamaiClientException as e: + print("get on record failed as expected, since record was succesfully deleted") + print ("(Probably Ignore):", e.message) + print ("delete status:", delete.status_code) + else: + print("unexpected condition in test delete") + + return + + def _test_record_create(self, zone_name, record_name, record_type, params): + + try: + create = self._dns_client.record_create(zone_name, record_name, record_type, params) + except AkamaiClientException as e: + print ("create unsuccessful, presumably because it already exists") + print ("(Probably Ignore)", e.message) + else: + print("initial create of", create.json().get("name"), "succesful: ", create.status_code) + + return + + def _test_record_replace(self, zone_name, record_name, record_type, params): + + ## create record to be replaced, if it doesn't already exist + try: + old_params = { + "name": record_name, + "type": record_type, + "ttl": 300, + "rdata": [ + "10.0.0.2", + "10.0.0.3" + ] + } + create = self._dns_client.record_create(zone_name, record_name, record_type, old_params) + except AkamaiClientException as e: + print ("initial create unsuccessful, presumably because it already exists") + print ("(Probably Ignore)", e.message) + else: + print("initial create of record to be replaced", create.json().get("name"), "succesful: ", create.status_code) + + + ## test replace + try: + replace = self._dns_client.record_replace(zone_name, record_name, record_type, params) + except AkamaiClientException as e: + print("replace failed") + print(e.message) + return + else: + try: + record = self._dns_client.record_get(zone_name, record_name, record_type) + except AkamaiClientException as e: + print("retrieval in replacement failed") + print(e.message) + else: + new_data = record.json() + + if (new_data != params): + print("replace failed, records don't match") + print("current data:") + print(new_data) + print("expected data:") + print(params) + + else: + print("replace succesful") + print("replace status:", replace.status_code) + + def _test_zones_get(self): + try: + zonesList = self._dns_client.zones_get() + except AkamaiClientException as e: + print ("zones get test failed") + print (e.message) + + else: + print("zones list: ") + print(json.dumps(zonesList.json(), indent=4, separators=(',', ': '))) + + return + + def _test_zones_recordset_get(self, zone_name): + try: + zoneRecordset = self._dns_client.zone_recordset_get(zone_name) + except AkamaiClientException as e: + print("zone recordset retrieval test failed") + print (e.message) + else: + print("zone recordset: ") + print(json.dumps(zoneRecordset.json(), indent=4, separators=(',', ': '))) + return From 9844f4293a31834c455fd7e00cf999d693e00ee8 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 13 Jun 2019 19:51:56 -0400 Subject: [PATCH 29/86] implemented records(), and zone_records(), fixed sone_recordset_get(), started populate() --- octodns/provider/fastdns.py | 152 +++++++++++++++++++++--------------- 1 file changed, 87 insertions(+), 65 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 56aa883..b94ff61 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -10,14 +10,14 @@ import requests from akamai.edgegrid import EdgeGridAuth from urlparse import urljoin import json - +from collections import defaultdict import logging from functools import reduce from ..record import Record from .base import BaseProvider -TESTING = False +TESTING = True class AkamaiClientException(Exception): @@ -38,28 +38,6 @@ class AkamaiClientException(Exception): super(AkamaiClientException, self).__init__(message) -''' -class AkamaiClientBadRequest(AkamaiClientException): #400 - def __init__(self): - super(AkamaiClientBadRequest, self).__init__('Bad request') - -class AkamaiClientNotAuthorized(AkamaiClientException): #403 - def __init__(self): - super(AkamaiClientNotAuthorized, self).__init__('Forbidden') - -class AkamaiClientNotFound(AkamaiClientException): #404 - def __init__(self): - super(AkamaiClientNotFound, self).__init__('Not found') - -class AkamaiClientNotAcceptable(AkamaiClientException): #406 - def __init__(self): - super(AkamaiClientNotAcceptable, self).__init__('Not acceptable') - -class AkamaiClientGenericExcp(AkamaiClientException): - def __init__(self, num): - super(AkamaiClientGenericExcp, self).__init__('HTTP Error: ' + str(num)) -''' - class _AkamaiRecord(object): pass @@ -95,10 +73,12 @@ class AkamaiClient(object): f.close() - def _request(self, method, path, params=None, data=None): + def _request(self, method, path, params=None, data=None, v1=False): url = urljoin(self.base, path) - + if v1: + url = urljoin(self.basev1, path) + if (TESTING): print("testing mode") print(method, url) @@ -111,15 +91,7 @@ class AkamaiClient(object): if resp.status_code > 299: raise AkamaiClientException(resp.status_code) - - # if resp.status_code == 400: - # raise AkamaiClientBadRequest() - # if resp.status_code == 403: - # raise AkamaiClientNotAuthorized() - # if resp.status_code == 404: - # raise AkamaiClientNotFound() - # if resp.status_code == 406: - # raise AkamaiClientNotAcceptable() + resp.raise_for_status() @@ -154,47 +126,64 @@ class AkamaiClient(object): return result - def zones_get(self, contractIds=None, page=None, pageSize=None, search=None, showAll="true", sortBy="zone", types=None): path = 'zones' params = { - "contracIds": contractIds, - "page": page, - "pageSize": pageSize, - "search": search, - "showAll": showAll, - "sortBy": sortBy, - "types": types + 'contractIds': contractIds, + 'page': page, + 'pageSize': pageSize, + 'search': search, + 'showAll': showAll, + 'sortBy': sortBy, + 'types': types } result = self._request('GET', path, params=params) return result - - def zone_create(self): - pass + def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, + showAll="true", sortBy="name", types=None): - def zone_recordset_get(self, zone, params=None): + params = { + 'page': page, + 'pageSize': pageSize, + 'search': search, + 'showAll': showAll, + 'sortBy': sortBy, + 'types': types + } + path = 'zones/{}/recordsets'.format(zone) result = self._request('GET', path, params=params) return result - def zone_recordset_create(self, zone): - pass - - def zone_recordset_replace(self, zone): - pass + def records(self, zone_name): + + recordset = self.zone_recordset_get(zone_name, showAll="true").json().get("recordsets") + # print (type(recordset)) + print(recordset) + return recordset def master_zone_file_get(self, zone): + path = 'zones/{}/zone-file'.format(zone) - result = self._request('GET', path) - + + try: + result = self._request('GET', path) + + except AkamaiClientException as e: + # not working with API v2, API v1 fallback + path = 'zones/{}'.format(zone) + result = self._request('GET', path, v1=True) + print("Using API v1 fallback") + print("(Probably Ignore)", e.message) + return result @@ -220,10 +209,26 @@ class AkamaiProvider(BaseProvider): self._zone_records = {} + def zone_records(self, zone): + if zone.name not in self._zone_records: + try: + self._zone_records[zone.name] = self._dns_client.records(zone.name[:-1]) + + except AkamaiClientException: + return [] + + return self._zone_records[zone.name] + def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) - self._test(zone) + + # self._test(zone) + + values = defaultdict(lambda: defaultdict(list)) + + for record in self.zone_records(zone): + pass @@ -254,20 +259,23 @@ class AkamaiProvider(BaseProvider): } - print("\n\nRunning test: record get..........\n") - self._test_record_get(zone_name, "test.basir-test.com", record_type) - print("\n\nRunning test: record create..........\n") - self._test_record_create(zone_name, record_name, record_type, params) - print("\n\nRunning test: record replace..........\n") - self._test_record_replace(zone_name, record_name, record_type, repl_params) - print("\n\nRunning test: record delete..........\n") - self._test_record_delete(zone_name, record_name, record_type) + # print("\n\nRunning test: record get..........\n") + # self._test_record_get(zone_name, "test.basir-test.com", record_type) + # print("\n\nRunning test: record create..........\n") + # self._test_record_create(zone_name, record_name, record_type, params) + # print("\n\nRunning test: record replace..........\n") + # self._test_record_replace(zone_name, record_name, record_type, repl_params) + # print("\n\nRunning test: record delete..........\n") + # self._test_record_delete(zone_name, record_name, record_type) # print("\n\nRunning test: zones get..........\n") # self._test_zones_get() - # print("\n\nRunning test: zone recordset get..........\n") - # self._test_zones_recordset_get(zone_name) + print("\n\nRunning test: zone recordset get..........\n") + self._test_zones_recordset_get(zone_name) + + # print("\n\nRunning test: Master Zone File get..........\n") + # self._test_master_zone_file_get(zone_name) return @@ -388,3 +396,17 @@ class AkamaiProvider(BaseProvider): print("zone recordset: ") print(json.dumps(zoneRecordset.json(), indent=4, separators=(',', ': '))) return + + def _test_master_zone_file_get(self, zone_name): + try: + mzf = self._dns_client.master_zone_file_get(zone_name) + + except AkamaiClientException as e: + print("MZF retrieval test failed") + print (e.message) + + else: + print("Master Zone File:") + print(json.dumps(mzf.json(), indent=4, separators=(',', ': '))) + + return From b8c94386a87be103c6b058c8e6f354639d5afe47 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 13 Jun 2019 20:05:33 -0400 Subject: [PATCH 30/86] thurs sync --- octodns/provider/fastdns.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index b94ff61..de8e7d6 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -166,8 +166,6 @@ class AkamaiClient(object): recordset = self.zone_recordset_get(zone_name, showAll="true").json().get("recordsets") - # print (type(recordset)) - print(recordset) return recordset def master_zone_file_get(self, zone): @@ -228,7 +226,14 @@ class AkamaiProvider(BaseProvider): values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): - pass + _type=record['type'] + if _type not in self.SUPPORTS: + continue + elif _type == 'TXT' and record['content'].startswith('ALIAS for'): + # ALIAS has a "ride along" TXT record with 'ALIAS for XXXX', + # we're ignoring it + continue + values[record['name']][record['type']].append(record) From 98f7935f9044532ecb5653aa09e872b7c6128d15 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Fri, 14 Jun 2019 19:06:19 -0700 Subject: [PATCH 31/86] major update: progressed on populate(), implemented _data_for_TYPE fxns, but not all are verfied to work as expected yet --- octodns/.vscode/settings.json | 3 + octodns/provider/.vscode/settings.json | 1 + octodns/provider/fastdns.py | 999 ++++++++++++++----------- 3 files changed, 586 insertions(+), 417 deletions(-) create mode 100644 octodns/.vscode/settings.json diff --git a/octodns/.vscode/settings.json b/octodns/.vscode/settings.json new file mode 100644 index 0000000..38e6c1e --- /dev/null +++ b/octodns/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "C:\\Python27\\python.exe" +} \ No newline at end of file diff --git a/octodns/provider/.vscode/settings.json b/octodns/provider/.vscode/settings.json index 7a73a41..88322cc 100644 --- a/octodns/provider/.vscode/settings.json +++ b/octodns/provider/.vscode/settings.json @@ -1,2 +1,3 @@ { + "python.pythonPath": "C:\\Python27\\python.exe" } \ No newline at end of file diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index de8e7d6..bcb1f32 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -1,417 +1,582 @@ -# -# -# - -from __future__ import absolute_import, division, print_function, \ - unicode_literals - -## octodns specfic imports: -import requests -from akamai.edgegrid import EdgeGridAuth -from urlparse import urljoin -import json -from collections import defaultdict - -import logging -from functools import reduce -from ..record import Record -from .base import BaseProvider - -TESTING = True - -class AkamaiClientException(Exception): - - _errorMessages = { - 400: "400: Bad request", - 403: "403: Access is forbidden", - 404: "404: Resource not found", - 405: "405: Method not supported", - 406: "406: Not Acceptable", - 409: "409: Request not allowed due to conflict with current state of resource", - 415: "415: Unsupported media type", - 422: "422: Request body contains an error preventing processing", - 500: "500: Internal server error" - } - - def __init__(self, code): - message = self._errorMessages.get(code) - super(AkamaiClientException, self).__init__(message) - - -class _AkamaiRecord(object): - pass - - -class AkamaiClient(object): - - def __init__(self, _client_secret, _host, _access_token, _client_token): - - self.base = "https://" + _host + "/config-dns/v2/" - self.basev1 = "https://" + _host + "/config-dns/v1/" - - sess = requests.Session() - sess.auth = EdgeGridAuth( - client_token=_client_token, - client_secret=_client_secret, - access_token=_access_token - ) - self._sess = sess - - - def _writePrepped(self, prepped): - filename = str(prepped.method) + ".json" - f = open(filename, 'w') - - f.write("headers: \n") - f.write(str(prepped.headers)) - f.write("\nbody: \n") - f.write(str(prepped.body)) - f.write("\nhooks:\n") - f.write(str(prepped.hooks)) - f.write("\nmethod: " + str(prepped.method)) - f.write("\nurl: " + str(prepped.url) + "\n") - - f.close() - - def _request(self, method, path, params=None, data=None, v1=False): - - url = urljoin(self.base, path) - if v1: - url = urljoin(self.basev1, path) - - if (TESTING): - print("testing mode") - print(method, url) - req = requests.Request(method, url, params=params, json=data) - prepped = req.prepare() - self._writePrepped(prepped) - - resp = self._sess.request(method, url, params=params, json=data) - - - if resp.status_code > 299: - raise AkamaiClientException(resp.status_code) - - resp.raise_for_status() - - - return resp - - - def record_get(self, zone, name, record_type): - - path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('GET', path) - - return result - - def record_create(self, zone, name, record_type, params): - path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('POST', path, data=params) - - return result - - def record_delete(self, zone, name, record_type): - path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('DELETE', path) - - if result.status_code == 204: - print ("successfully deleted ", path) - - return result - - def record_replace(self, zone, name, record_type, params): - path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('PUT', path, data=params) - - return result - - def zones_get(self, contractIds=None, page=None, pageSize=None, search=None, - showAll="true", sortBy="zone", types=None): - path = 'zones' - - params = { - 'contractIds': contractIds, - 'page': page, - 'pageSize': pageSize, - 'search': search, - 'showAll': showAll, - 'sortBy': sortBy, - 'types': types - } - - result = self._request('GET', path, params=params) - - return result - - def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, - showAll="true", sortBy="name", types=None): - - - params = { - 'page': page, - 'pageSize': pageSize, - 'search': search, - 'showAll': showAll, - 'sortBy': sortBy, - 'types': types - } - - path = 'zones/{}/recordsets'.format(zone) - result = self._request('GET', path, params=params) - - return result - - def records(self, zone_name): - - recordset = self.zone_recordset_get(zone_name, showAll="true").json().get("recordsets") - - return recordset - - def master_zone_file_get(self, zone): - - path = 'zones/{}/zone-file'.format(zone) - - try: - result = self._request('GET', path) - - except AkamaiClientException as e: - # not working with API v2, API v1 fallback - path = 'zones/{}'.format(zone) - result = self._request('GET', path, v1=True) - print("Using API v1 fallback") - print("(Probably Ignore)", e.message) - - return result - - -class AkamaiProvider(BaseProvider): - - SUPPORTS_GEO = False - SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', - 'SRV', 'SSHFP', 'TXT')) - - - def __init__(self, id, client_secret, host, access_token, client_token, - *args, **kwargs): - - self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) - self.log.debug('__init__: id=%s, ') - super(AkamaiProvider, self).__init__(id, *args, **kwargs) - - - self._dns_client = AkamaiClient(client_secret, host, access_token, client_token) - - - self._zone_records = {} - - - def zone_records(self, zone): - if zone.name not in self._zone_records: - try: - self._zone_records[zone.name] = self._dns_client.records(zone.name[:-1]) - - except AkamaiClientException: - return [] - - return self._zone_records[zone.name] - - - def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) - - # self._test(zone) - - values = defaultdict(lambda: defaultdict(list)) - - for record in self.zone_records(zone): - _type=record['type'] - if _type not in self.SUPPORTS: - continue - elif _type == 'TXT' and record['content'].startswith('ALIAS for'): - # ALIAS has a "ride along" TXT record with 'ALIAS for XXXX', - # we're ignoring it - continue - values[record['name']][record['type']].append(record) - - - - def _test(self, zone) : - - zone_name = zone.name[:len(zone.name)-1] - - record_name = "octo.basir-test.com" - record_type = "A" - params = { - "name": "octo.basir-test.com", - "type": "A", - "ttl": 300, - "rdata": [ - "10.0.0.2", - "10.0.0.3" - ] - } - repl_params = { - "name": "octo.basir-test.com", - "type": "A", - "ttl": 300, - "rdata": [ - "99.99.99.99", - "10.0.0.3", - "1.2.3.4" - ] - } - - - # print("\n\nRunning test: record get..........\n") - # self._test_record_get(zone_name, "test.basir-test.com", record_type) - # print("\n\nRunning test: record create..........\n") - # self._test_record_create(zone_name, record_name, record_type, params) - # print("\n\nRunning test: record replace..........\n") - # self._test_record_replace(zone_name, record_name, record_type, repl_params) - # print("\n\nRunning test: record delete..........\n") - # self._test_record_delete(zone_name, record_name, record_type) - - # print("\n\nRunning test: zones get..........\n") - # self._test_zones_get() - - print("\n\nRunning test: zone recordset get..........\n") - self._test_zones_recordset_get(zone_name) - - # print("\n\nRunning test: Master Zone File get..........\n") - # self._test_master_zone_file_get(zone_name) - - return - - def _test_record_get(self, zone_name, record_name, record_type): - try: - get = self._dns_client.record_get(zone_name, record_name, record_type) - except AkamaiClientException as e: - print ("record get test failed") - print (e.message) - - else: - print("record get test result: ") - print(json.dumps(get.json(), indent=4, separators=(',', ': '))) - - return - - def _test_record_delete(self, zone_name, record_name, record_type): - - try: - delete = self._dns_client.record_delete(zone_name, record_name, record_type) - except AkamaiClientException as e: - print("delete failed") - print(e.message) - return - - - try: - self._dns_client.record_get(zone_name, record_name, record_type) - except AkamaiClientException as e: - print("get on record failed as expected, since record was succesfully deleted") - print ("(Probably Ignore):", e.message) - print ("delete status:", delete.status_code) - else: - print("unexpected condition in test delete") - - return - - def _test_record_create(self, zone_name, record_name, record_type, params): - - try: - create = self._dns_client.record_create(zone_name, record_name, record_type, params) - except AkamaiClientException as e: - print ("create unsuccessful, presumably because it already exists") - print ("(Probably Ignore)", e.message) - else: - print("initial create of", create.json().get("name"), "succesful: ", create.status_code) - - return - - def _test_record_replace(self, zone_name, record_name, record_type, params): - - ## create record to be replaced, if it doesn't already exist - try: - old_params = { - "name": record_name, - "type": record_type, - "ttl": 300, - "rdata": [ - "10.0.0.2", - "10.0.0.3" - ] - } - create = self._dns_client.record_create(zone_name, record_name, record_type, old_params) - except AkamaiClientException as e: - print ("initial create unsuccessful, presumably because it already exists") - print ("(Probably Ignore)", e.message) - else: - print("initial create of record to be replaced", create.json().get("name"), "succesful: ", create.status_code) - - - ## test replace - try: - replace = self._dns_client.record_replace(zone_name, record_name, record_type, params) - except AkamaiClientException as e: - print("replace failed") - print(e.message) - return - else: - try: - record = self._dns_client.record_get(zone_name, record_name, record_type) - except AkamaiClientException as e: - print("retrieval in replacement failed") - print(e.message) - else: - new_data = record.json() - - if (new_data != params): - print("replace failed, records don't match") - print("current data:") - print(new_data) - print("expected data:") - print(params) - - else: - print("replace succesful") - print("replace status:", replace.status_code) - - def _test_zones_get(self): - try: - zonesList = self._dns_client.zones_get() - except AkamaiClientException as e: - print ("zones get test failed") - print (e.message) - - else: - print("zones list: ") - print(json.dumps(zonesList.json(), indent=4, separators=(',', ': '))) - - return - - def _test_zones_recordset_get(self, zone_name): - try: - zoneRecordset = self._dns_client.zone_recordset_get(zone_name) - except AkamaiClientException as e: - print("zone recordset retrieval test failed") - print (e.message) - else: - print("zone recordset: ") - print(json.dumps(zoneRecordset.json(), indent=4, separators=(',', ': '))) - return - - def _test_master_zone_file_get(self, zone_name): - try: - mzf = self._dns_client.master_zone_file_get(zone_name) - - except AkamaiClientException as e: - print("MZF retrieval test failed") - print (e.message) - - else: - print("Master Zone File:") - print(json.dumps(mzf.json(), indent=4, separators=(',', ': '))) - - return +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +## octodns specfic imports: +import requests +from akamai.edgegrid import EdgeGridAuth +from urlparse import urljoin +import json +from collections import defaultdict + +import logging +from ..record import Record +from .base import BaseProvider + +TESTING = True + +class AkamaiClientException(Exception): + + _errorMessages = { + 400: "400: Bad request", + 403: "403: Access is forbidden", + 404: "404: Resource not found", + 405: "405: Method not supported", + 406: "406: Not Acceptable", + 409: "409: Request not allowed due to conflict with current state of resource", + 415: "415: Unsupported media type", + 422: "422: Request body contains an error preventing processing", + 500: "500: Internal server error" + } + + def __init__(self, code): + message = self._errorMessages.get(code) + super(AkamaiClientException, self).__init__(message) + + +class _AkamaiRecord(object): + pass + + +class AkamaiClient(object): + + def __init__(self, _client_secret, _host, _access_token, _client_token): + + self.base = "https://" + _host + "/config-dns/v2/" + self.basev1 = "https://" + _host + "/config-dns/v1/" + + sess = requests.Session() + sess.auth = EdgeGridAuth( + client_token=_client_token, + client_secret=_client_secret, + access_token=_access_token + ) + self._sess = sess + + + def _writePrepped(self, prepped): + filename = str(prepped.method) + ".json" + f = open(filename, 'w') + + f.write("headers: \n") + f.write(str(prepped.headers)) + f.write("\nbody: \n") + f.write(str(prepped.body)) + f.write("\nhooks:\n") + f.write(str(prepped.hooks)) + f.write("\nmethod: " + str(prepped.method)) + f.write("\nurl: " + str(prepped.url) + "\n") + + f.close() + + def _request(self, method, path, params=None, data=None, v1=False): + + url = urljoin(self.base, path) + if v1: + url = urljoin(self.basev1, path) + + if (TESTING): + print("testing mode") + print(method, url) + req = requests.Request(method, url, params=params, json=data) + prepped = req.prepare() + self._writePrepped(prepped) + + resp = self._sess.request(method, url, params=params, json=data) + + + if resp.status_code > 299: + raise AkamaiClientException(resp.status_code) + + resp.raise_for_status() + + + return resp + + + def record_get(self, zone, name, record_type): + + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) + result = self._request('GET', path) + + return result + + def record_create(self, zone, name, record_type, params): + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) + result = self._request('POST', path, data=params) + + return result + + def record_delete(self, zone, name, record_type): + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) + result = self._request('DELETE', path) + + if result.status_code == 204: + print ("successfully deleted ", path) + + return result + + def record_replace(self, zone, name, record_type, params): + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) + result = self._request('PUT', path, data=params) + + return result + + def zones_get(self, contractIds=None, page=None, pageSize=None, search=None, + showAll="true", sortBy="zone", types=None): + path = 'zones' + + params = { + 'contractIds': contractIds, + 'page': page, + 'pageSize': pageSize, + 'search': search, + 'showAll': showAll, + 'sortBy': sortBy, + 'types': types + } + + result = self._request('GET', path, params=params) + + return result + + def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, + showAll="true", sortBy="name", types=None): + + + params = { + 'page': page, + 'pageSize': pageSize, + 'search': search, + 'showAll': showAll, + 'sortBy': sortBy, + 'types': types + } + + path = 'zones/{}/recordsets'.format(zone) + result = self._request('GET', path, params=params) + + + return result + + def records(self, zone_name): + + resp = self.zone_recordset_get(zone_name, showAll="true") + recordset = resp.json().get("recordsets") + + return recordset + + def master_zone_file_get(self, zone): + + path = 'zones/{}/zone-file'.format(zone) + + try: + result = self._request('GET', path) + + except AkamaiClientException as e: + # not working with API v2, API v1 fallback + path = 'zones/{}'.format(zone) + result = self._request('GET', path, v1=True) + print("Using API v1 fallback") + print("(Probably Ignore)", e.message) + + return result + + +class AkamaiProvider(BaseProvider): + + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', + 'SRV', 'SSHFP', 'TXT')) + # SUPPORTS = set(('NAPTR', 'MX')) + + def __init__(self, id, client_secret, host, access_token, client_token, + *args, **kwargs): + + self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, ') + super(AkamaiProvider, self).__init__(id, *args, **kwargs) + + self._dns_client = AkamaiClient(client_secret, host, access_token, + client_token) + + self._zone_records = {} + + + def zone_records(self, zone): + """ returns records for a zone, finds it if not present, or + returns empty if doesnt exist + """ + if zone.name not in self._zone_records: + try: + name = zone.name[:-1] + self._zone_records[zone.name] = self._dns_client.records(name) + + except AkamaiClientException: + return [] + + + # fileName = "zone_records_for_" + zone.name +".json" + # path = "/mnt/c/Users/bajamil/Desktop/" + fileName + # f = open(path, 'w') + # f.write(json.dumps(self._zone_records[zone.name], indent=4, separators=(',', ': '))) + # f.close() + + return self._zone_records[zone.name] + + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s', zone.name) + # self._test(zone) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + + _type =record.get('type') + _name = record.get('name') + + if _type not in self.SUPPORTS: + continue + + values[_name][_type].append(record) + + + + fileName = "values_for_" + zone.name +".json" + path = "/mnt/c/Users/bajamil/Desktop/" + fileName + f = open(path, 'w') + f.write(json.dumps(values, indent=4, separators=(',', ': '))) + f.close() + + return ### + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + + data = data_for(_type, records) + + print() + print ("data processed for", name, ":") + print (json.dumps(data, indent=4, separators=(',', ': '))) + print() + + record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) + + + exists = zone.name in self._zone_records + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + + return exists + + + def _data_for_multiple(self, _type, records): + + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': [r['rdata'][0] for r in records] + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_SPF = _data_for_multiple + + def _data_for_CNAME(self, _type, records): + record = records[0] + value = record['rdata'][0] + if (value[-1] != "."): + value = '{}.'.format(value) + + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'value': value + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + print ("MX record: ", record) + values.append({ + 'preference': record['priority'], + 'exchange': '{}.'.format(record['rdata'][0]) + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_NAPTR(self, _type, records): + print ("NAPTR Record") + print(json.dumps(records, indent=4, separators=(',', ': '))) + values = [] + for record in records: + order, preference, flags, service, regexp, replacement = \ + record['rdata'][0].split(' ', 5) + + values.append({ + 'flags': flags[1:-1], + 'order': order, + 'preference': preference, + 'regexp': regexp[1:-1], + 'replacement': replacement, + 'service': service[1:-1] + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + def _data_for_NS(self, _type, records): + values = [] + for record in records: + rdata = record['rdata'] + for content in rdata: + if content[-1] != '.': + content = '{}.'.format(content) + values.append(content) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'value': values + } + + def _data_for_PTR(self, _type, records): + return self._data_for_multiple(_type, records[0]) + + # rdata[0]? + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + weight, port, target = record['rdata'].split(', ', 2) + values.append({ + 'port': port, + 'priority': record['priority'], + 'target': '{}.'.format(target), + 'weight': weight + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + # rdata[0]? + def _data_for_SSHFP(self, _type, records): + values = [] + algorithm, fp_type, fingerprint = record['rdata'].split(' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint': fingerprint, + 'fingerprint_type': fp_type + }) + + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + def _data_for_TXT(self, _type, records): + values = [] + for r in records: + for content in r['rdata']: + values.append(content.replace(';', '\\;')) + + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + # 'values': [r['rdata'][0].replace(';', '\\;') for r in records] + } + + + + + def _test(self, zone) : + + zone_name = zone.name[:len(zone.name)-1] + + record_name = "octo.basir-test.com" + record_type = "A" + params = { + "name": "octo.basir-test.com", + "type": "A", + "ttl": 300, + "rdata": [ + "10.0.0.2", + "10.0.0.3" + ] + } + repl_params = { + "name": "octo.basir-test.com", + "type": "A", + "ttl": 300, + "rdata": [ + "99.99.99.99", + "10.0.0.3", + "1.2.3.4" + ] + } + + + # print("\n\nRunning test: record get..........\n") + # self._test_record_get(zone_name, "test.basir-test.com", record_type) + # print("\n\nRunning test: record create..........\n") + # self._test_record_create(zone_name, record_name, record_type, params) + # print("\n\nRunning test: record replace..........\n") + # self._test_record_replace(zone_name, record_name, record_type, repl_params) + # print("\n\nRunning test: record delete..........\n") + # self._test_record_delete(zone_name, record_name, record_type) + + # print("\n\nRunning test: zones get..........\n") + # self._test_zones_get() + + print("\n\nRunning test: zone recordset get..........\n") + self._test_zones_recordset_get(zone_name) + + # print("\n\nRunning test: Master Zone File get..........\n") + # self._test_master_zone_file_get(zone_name) + + return + + def _test_record_get(self, zone_name, record_name, record_type): + try: + get = self._dns_client.record_get(zone_name, record_name, record_type) + except AkamaiClientException as e: + print ("record get test failed") + print (e.message) + + else: + print("record get test result: ") + print(json.dumps(get.json(), indent=4, separators=(',', ': '))) + + return + + def _test_record_delete(self, zone_name, record_name, record_type): + + try: + delete = self._dns_client.record_delete(zone_name, record_name, record_type) + except AkamaiClientException as e: + print("delete failed") + print(e.message) + return + + + try: + self._dns_client.record_get(zone_name, record_name, record_type) + except AkamaiClientException as e: + print("get on record failed as expected, since record was succesfully deleted") + print ("(Probably Ignore):", e.message) + print ("delete status:", delete.status_code) + else: + print("unexpected condition in test delete") + + return + + def _test_record_create(self, zone_name, record_name, record_type, params): + + try: + create = self._dns_client.record_create(zone_name, record_name, record_type, params) + except AkamaiClientException as e: + print ("create unsuccessful, presumably because it already exists") + print ("(Probably Ignore)", e.message) + else: + print("initial create of", create.json().get("name"), "succesful: ", create.status_code) + + return + + def _test_record_replace(self, zone_name, record_name, record_type, params): + + ## create record to be replaced, if it doesn't already exist + try: + old_params = { + "name": record_name, + "type": record_type, + "ttl": 300, + "rdata": [ + "10.0.0.2", + "10.0.0.3" + ] + } + create = self._dns_client.record_create(zone_name, record_name, record_type, old_params) + except AkamaiClientException as e: + print ("initial create unsuccessful, presumably because it already exists") + print ("(Probably Ignore)", e.message) + else: + print("initial create of record to be replaced", create.json().get("name"), "succesful: ", create.status_code) + + + ## test replace + try: + replace = self._dns_client.record_replace(zone_name, record_name, record_type, params) + except AkamaiClientException as e: + print("replace failed") + print(e.message) + return + else: + try: + record = self._dns_client.record_get(zone_name, record_name, record_type) + except AkamaiClientException as e: + print("retrieval in replacement failed") + print(e.message) + else: + new_data = record.json() + + if (new_data != params): + print("replace failed, records don't match") + print("current data:") + print(new_data) + print("expected data:") + print(params) + + else: + print("replace succesful") + print("replace status:", replace.status_code) + + def _test_zones_get(self): + try: + zonesList = self._dns_client.zones_get() + except AkamaiClientException as e: + print ("zones get test failed") + print (e.message) + + else: + print("zones list: ") + print(json.dumps(zonesList.json(), indent=4, separators=(',', ': '))) + + return + + def _test_zones_recordset_get(self, zone_name): + try: + zoneRecordset = self._dns_client.zone_recordset_get(zone_name) + except AkamaiClientException as e: + print("zone recordset retrieval test failed") + print (e.message) + else: + print("zone recordset: ") + print(json.dumps(zoneRecordset.json(), indent=4, separators=(',', ': '))) + return + + def _test_master_zone_file_get(self, zone_name): + try: + mzf = self._dns_client.master_zone_file_get(zone_name) + + except AkamaiClientException as e: + print("MZF retrieval test failed") + print (e.message) + + else: + print("Master Zone File:") + print(json.dumps(mzf.json(), indent=4, separators=(',', ': '))) + + return From 6db6052d28fc10afa861c3b83f9f82160d393b8b Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Mon, 17 Jun 2019 10:24:34 -0700 Subject: [PATCH 32/86] friday sync --- octodns/provider/fastdns.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index bcb1f32..686c81c 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -247,26 +247,24 @@ class AkamaiProvider(BaseProvider): - fileName = "values_for_" + zone.name +".json" - path = "/mnt/c/Users/bajamil/Desktop/" + fileName - f = open(path, 'w') - f.write(json.dumps(values, indent=4, separators=(',', ': '))) - f.close() - return ### before = len(zone.records) for name, types in values.items(): - for _type, records in types.items(): - data_for = getattr(self, '_data_for_{}'.format(_type)) - - data = data_for(_type, records) + # for _type, records in types.items(): + for _type in types.items(): + for rdata in _type['rdata']: + + + data_for = getattr(self, '_data_for_{}'.format(_type)) + + data = data_for(_type, records, rdata) - print() - print ("data processed for", name, ":") - print (json.dumps(data, indent=4, separators=(',', ': '))) - print() + print() + print ("data processed for", name, ":") + print (json.dumps(data, indent=4, separators=(',', ': '))) + print() - record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) + record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) exists = zone.name in self._zone_records From 67f7af58e632345570f8bd2b854dd82c4797905f Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Mon, 17 Jun 2019 10:24:58 -0700 Subject: [PATCH 33/86] friday sync --- octodns/provider/fastdns.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 686c81c..0874709 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -235,7 +235,6 @@ class AkamaiProvider(BaseProvider): # self._test(zone) values = defaultdict(lambda: defaultdict(list)) - for record in self.zone_records(zone): _type =record.get('type') _name = record.get('name') From dfd6e8cfdaf051b9e1122f4146c449e967d85a88 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Tue, 18 Jun 2019 14:43:38 -0400 Subject: [PATCH 34/86] Completed populate + helper functions --- .vscode/settings.json | 3 + octodns/provider/fastdns.py | 227 +++++++++++++----------------------- 2 files changed, 81 insertions(+), 149 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..38e6c1e --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "C:\\Python27\\python.exe" +} \ No newline at end of file diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 0874709..bdd3b82 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -16,7 +16,6 @@ import logging from ..record import Record from .base import BaseProvider -TESTING = True class AkamaiClientException(Exception): @@ -37,10 +36,6 @@ class AkamaiClientException(Exception): super(AkamaiClientException, self).__init__(message) -class _AkamaiRecord(object): - pass - - class AkamaiClient(object): def __init__(self, _client_secret, _host, _access_token, _client_token): @@ -57,45 +52,20 @@ class AkamaiClient(object): self._sess = sess - def _writePrepped(self, prepped): - filename = str(prepped.method) + ".json" - f = open(filename, 'w') - - f.write("headers: \n") - f.write(str(prepped.headers)) - f.write("\nbody: \n") - f.write(str(prepped.body)) - f.write("\nhooks:\n") - f.write(str(prepped.hooks)) - f.write("\nmethod: " + str(prepped.method)) - f.write("\nurl: " + str(prepped.url) + "\n") - - f.close() - def _request(self, method, path, params=None, data=None, v1=False): url = urljoin(self.base, path) if v1: url = urljoin(self.basev1, path) - if (TESTING): - print("testing mode") - print(method, url) - req = requests.Request(method, url, params=params, json=data) - prepped = req.prepare() - self._writePrepped(prepped) - resp = self._sess.request(method, url, params=params, json=data) - if resp.status_code > 299: raise AkamaiClientException(resp.status_code) - resp.raise_for_status() - return resp - + return resp def record_get(self, zone, name, record_type): @@ -146,7 +116,6 @@ class AkamaiClient(object): def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, showAll="true", sortBy="name", types=None): - params = { 'page': page, 'pageSize': pageSize, @@ -193,8 +162,7 @@ class AkamaiProvider(BaseProvider): SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'SSHFP', 'TXT')) - # SUPPORTS = set(('NAPTR', 'MX')) - + def __init__(self, id, client_secret, host, access_token, client_token, *args, **kwargs): @@ -210,7 +178,7 @@ class AkamaiProvider(BaseProvider): def zone_records(self, zone): """ returns records for a zone, finds it if not present, or - returns empty if doesnt exist + returns empty if can't find a match """ if zone.name not in self._zone_records: try: @@ -220,184 +188,145 @@ class AkamaiProvider(BaseProvider): except AkamaiClientException: return [] - - # fileName = "zone_records_for_" + zone.name +".json" - # path = "/mnt/c/Users/bajamil/Desktop/" + fileName - # f = open(path, 'w') - # f.write(json.dumps(self._zone_records[zone.name], indent=4, separators=(',', ': '))) - # f.close() - return self._zone_records[zone.name] - def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s', zone.name) - # self._test(zone) values = defaultdict(lambda: defaultdict(list)) - + for record in self.zone_records(zone): + _type =record.get('type') - _name = record.get('name') + ## Akamai sends down prefix.zonename., while OctoDNS only expects prefix + _name = record.get('name').split("." + zone.name[:-1], 1)[0] + if _name == zone.name[:-1] : + _name = '' ## root / @ if _type not in self.SUPPORTS: continue - values[_name][_type].append(record) - - - before = len(zone.records) for name, types in values.items(): - # for _type, records in types.items(): - for _type in types.items(): - for rdata in _type['rdata']: - - - data_for = getattr(self, '_data_for_{}'.format(_type)) - - data = data_for(_type, records, rdata) - - print() - print ("data processed for", name, ":") - print (json.dumps(data, indent=4, separators=(',', ': '))) - print() - - record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) - - - exists = zone.name in self._zone_records - self.log.info('populate: found %s records, exists=%s', - len(zone.records) - before, exists) + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records[0]), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) - return exists + exists = zone.name in self._zone_records + found = len(zone.records) - before + self.log.info('populate: found %s records, exists=%s', found, exists) - def _data_for_multiple(self, _type, records): + def _data_for_multiple(self, _type, _records): return { - 'ttl': records[0]['ttl'], - 'type': _type, - 'values': [r['rdata'][0] for r in records] - } + 'ttl': _records['ttl'], + 'type': _type, + 'values': [r for r in _records['rdata']] + } _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple + _data_for_NS = _data_for_multiple _data_for_SPF = _data_for_multiple - def _data_for_CNAME(self, _type, records): - record = records[0] - value = record['rdata'][0] - if (value[-1] != "."): + def _data_for_CNAME(self, _type, _records): + value = _records['rdata'][0] + if (value[-1] != '.') : value = '{}.'.format(value) - + return { - 'ttl': records[0]['ttl'], + 'ttl': _records['ttl'], 'type': _type, 'value': value } - def _data_for_MX(self, _type, records): + def _data_for_MX(self, _type, _records): values = [] - for record in records: - print ("MX record: ", record) + for r in _records['rdata']: + preference, exchange = r.split(" ", 1) values.append({ - 'preference': record['priority'], - 'exchange': '{}.'.format(record['rdata'][0]) - }) + 'preference': preference, + 'exchange' : exchange + }) return { - 'ttl': records[0]['ttl'], + 'ttl': _records['ttl'], 'type': _type, 'values': values } - def _data_for_NAPTR(self, _type, records): - print ("NAPTR Record") - print(json.dumps(records, indent=4, separators=(',', ': '))) + def _data_for_NAPTR(self, _type, _records): values = [] - for record in records: - order, preference, flags, service, regexp, replacement = \ - record['rdata'][0].split(' ', 5) - + for r in _records['rdata']: + order, preference, flags, service, regexp, repl = r.split(' ', 5) + values.append({ 'flags': flags[1:-1], 'order': order, 'preference': preference, 'regexp': regexp[1:-1], - 'replacement': replacement, + 'replacement': repl, 'service': service[1:-1] }) return { 'type': _type, - 'ttl': records[0]['ttl'], + 'ttl': _records['ttl'], 'values': values } + + def _data_for_PTR(self, _type, _records): - def _data_for_NS(self, _type, records): - values = [] - for record in records: - rdata = record['rdata'] - for content in rdata: - if content[-1] != '.': - content = '{}.'.format(content) - values.append(content) return { - 'ttl': records[0]['ttl'], + 'ttl': _records['ttl'], 'type': _type, - 'value': values - } - - def _data_for_PTR(self, _type, records): - return self._data_for_multiple(_type, records[0]) + 'value' : _records['rdata'][0] + } - # rdata[0]? - def _data_for_SRV(self, _type, records): + def _data_for_SRV(self, _type, _records): values = [] - for record in records: - weight, port, target = record['rdata'].split(', ', 2) + for r in _records['rdata']: + priority, weight, port, target = r.split(' ', 3) values.append({ 'port': port, - 'priority': record['priority'], - 'target': '{}.'.format(target), + 'priority': priority, + 'target': target, 'weight': weight }) return { 'type': _type, - 'ttl': records[0]['ttl'], + 'ttl': _records['ttl'], 'values': values } - # rdata[0]? - def _data_for_SSHFP(self, _type, records): + def _data_for_SSHFP(self, _type, _records): values = [] - algorithm, fp_type, fingerprint = record['rdata'].split(' ', 2) - values.append({ - 'algorithm': algorithm, - 'fingerprint': fingerprint, - 'fingerprint_type': fp_type - }) - + for r in _records['rdata']: + algorithm, fp_type, fingerprint = r.split(' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint': fingerprint, + 'fingerprint_type': fp_type + }) return { 'type': _type, - 'ttl': records[0]['ttl'], + 'ttl': _records['ttl'], 'values': values } - def _data_for_TXT(self, _type, records): + def _data_for_TXT(self, _type, _records): values = [] - for r in records: - for content in r['rdata']: - values.append(content.replace(';', '\\;')) + for r in _records['rdata']: + r = r[1:-1] + values.append(r.replace(';', '\\;')) return { - 'ttl': records[0]['ttl'], + 'ttl': _records['ttl'], 'type': _type, 'values': values - # 'values': [r['rdata'][0].replace(';', '\\;') for r in records] } - - def _test(self, zone) : @@ -426,23 +355,23 @@ class AkamaiProvider(BaseProvider): } - # print("\n\nRunning test: record get..........\n") - # self._test_record_get(zone_name, "test.basir-test.com", record_type) - # print("\n\nRunning test: record create..........\n") - # self._test_record_create(zone_name, record_name, record_type, params) - # print("\n\nRunning test: record replace..........\n") - # self._test_record_replace(zone_name, record_name, record_type, repl_params) - # print("\n\nRunning test: record delete..........\n") - # self._test_record_delete(zone_name, record_name, record_type) + print("\n\nRunning test: record get..........\n") + self._test_record_get(zone_name, "test.basir-test.com", record_type) + print("\n\nRunning test: record create..........\n") + self._test_record_create(zone_name, record_name, record_type, params) + print("\n\nRunning test: record replace..........\n") + self._test_record_replace(zone_name, record_name, record_type, repl_params) + print("\n\nRunning test: record delete..........\n") + self._test_record_delete(zone_name, record_name, record_type) - # print("\n\nRunning test: zones get..........\n") - # self._test_zones_get() + print("\n\nRunning test: zones get..........\n") + self._test_zones_get() print("\n\nRunning test: zone recordset get..........\n") self._test_zones_recordset_get(zone_name) - # print("\n\nRunning test: Master Zone File get..........\n") - # self._test_master_zone_file_get(zone_name) + print("\n\nRunning test: Master Zone File get..........\n") + self._test_master_zone_file_get(zone_name) return From c87967a239f748ed72f04232cd0f23ffecc01fe9 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Tue, 18 Jun 2019 18:16:15 -0400 Subject: [PATCH 35/86] tuesday sync --- octodns/provider/fastdns.py | 44 +++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index bdd3b82..99a0b93 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -95,6 +95,23 @@ class AkamaiClient(object): return result + def zone_get(self, zone): + path = 'zones/{}'.format(zone) + result = self._request('GET', path) + + return result + + def zone_create(self, contractId, params, gid=None): + path = 'zones?contractId={}'.format(contractId) + + if gid is not None: + path += '&gid={}'.format(gid) + + result = self._request('POST', path, data=params) + + return result + + def zones_get(self, contractIds=None, page=None, pageSize=None, search=None, showAll="true", sortBy="zone", types=None): path = 'zones' @@ -164,7 +181,7 @@ class AkamaiProvider(BaseProvider): 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, client_secret, host, access_token, client_token, - *args, **kwargs): + contractId=None, gid=None, *args, **kwargs): self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, ') @@ -174,7 +191,8 @@ class AkamaiProvider(BaseProvider): client_token) self._zone_records = {} - + self._contractId = contractId + self._gid = gid def zone_records(self, zone): """ returns records for a zone, finds it if not present, or @@ -218,6 +236,7 @@ class AkamaiProvider(BaseProvider): found = len(zone.records) - before self.log.info('populate: found %s records, exists=%s', found, exists) + return exists def _data_for_multiple(self, _type, _records): @@ -328,6 +347,27 @@ class AkamaiProvider(BaseProvider): } + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + domain_name = desired.name[:-1] + + + try: + self._dns_client.zone_get(domain_name) + except AkamaiClientException: + self.log.debug('_apply: no matching zone, creating domain') + + params = self._build_zone_config(domain_name) + + self._dns_client.zone_create(self._contractId, params, self._gid) + + + def _build_zone_config(self, zone, type="primary", comment=None, + def _test(self, zone) : zone_name = zone.name[:len(zone.name)-1] From c1528e3221e25c4e0d053d93d8bffb71a48d5190 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Wed, 19 Jun 2019 19:21:28 -0400 Subject: [PATCH 36/86] implemented _apply_delete, other misc. fixes --- octodns/provider/fastdns.py | 176 +++++++++++++++++++++++++++++------- 1 file changed, 141 insertions(+), 35 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 99a0b93..6306145 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -21,6 +21,7 @@ class AkamaiClientException(Exception): _errorMessages = { 400: "400: Bad request", + 401: "401: Unauthorized", 403: "403: Access is forbidden", 404: "404: Resource not found", 405: "405: Method not supported", @@ -42,6 +43,7 @@ class AkamaiClient(object): self.base = "https://" + _host + "/config-dns/v2/" self.basev1 = "https://" + _host + "/config-dns/v1/" + self.basehost = "https://" + _host sess = requests.Session() sess.auth = EdgeGridAuth( @@ -61,12 +63,14 @@ class AkamaiClient(object): resp = self._sess.request(method, url, params=params, json=data) if resp.status_code > 299: + # print(resp.status_code) raise AkamaiClientException(resp.status_code) resp.raise_for_status() return resp + def record_get(self, zone, name, record_type): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) @@ -82,6 +86,7 @@ class AkamaiClient(object): def record_delete(self, zone, name, record_type): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) + print(path) result = self._request('DELETE', path) if result.status_code == 204: @@ -95,6 +100,7 @@ class AkamaiClient(object): return result + def zone_get(self, zone): path = 'zones/{}'.format(zone) result = self._request('GET', path) @@ -148,7 +154,19 @@ class AkamaiClient(object): return result - def records(self, zone_name): + + def contracts_get(self, gid=None): + path = 'data/contracts' + if gid is not None: + path += '?gid={}'.format(gid) + + # result = self._sess.request('GET', path) + + result = self._request('GET', path) + + return result + + def recordsets_get(self, zone_name): resp = self.zone_recordset_get(zone_name, showAll="true") recordset = resp.json().get("recordsets") @@ -181,7 +199,7 @@ class AkamaiProvider(BaseProvider): 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, client_secret, host, access_token, client_token, - contractId=None, gid=None, *args, **kwargs): + contract_id=None, gid=None, *args, **kwargs): self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, ') @@ -191,7 +209,7 @@ class AkamaiProvider(BaseProvider): client_token) self._zone_records = {} - self._contractId = contractId + self._contractId = contract_id self._gid = gid def zone_records(self, zone): @@ -201,7 +219,7 @@ class AkamaiProvider(BaseProvider): if zone.name not in self._zone_records: try: name = zone.name[:-1] - self._zone_records[zone.name] = self._dns_client.records(name) + self._zone_records[zone.name] = self._dns_client.recordsets_get(name) except AkamaiClientException: return [] @@ -238,12 +256,12 @@ class AkamaiProvider(BaseProvider): return exists - def _data_for_multiple(self, _type, _records): + def _data_for_multiple(self, _type, records): return { - 'ttl': _records['ttl'], + 'ttl':records['ttl'], 'type': _type, - 'values': [r for r in _records['rdata']] + 'values': [r for r in records['rdata']] } _data_for_A = _data_for_multiple @@ -251,34 +269,34 @@ class AkamaiProvider(BaseProvider): _data_for_NS = _data_for_multiple _data_for_SPF = _data_for_multiple - def _data_for_CNAME(self, _type, _records): - value = _records['rdata'][0] + def _data_for_CNAME(self, _type, records): + value =records['rdata'][0] if (value[-1] != '.') : value = '{}.'.format(value) return { - 'ttl': _records['ttl'], + 'ttl':records['ttl'], 'type': _type, 'value': value } - def _data_for_MX(self, _type, _records): + def _data_for_MX(self, _type, records): values = [] - for r in _records['rdata']: + for r in records['rdata']: preference, exchange = r.split(" ", 1) values.append({ 'preference': preference, 'exchange' : exchange }) return { - 'ttl': _records['ttl'], + 'ttl':records['ttl'], 'type': _type, 'values': values } - def _data_for_NAPTR(self, _type, _records): + def _data_for_NAPTR(self, _type, records): values = [] - for r in _records['rdata']: + for r in records['rdata']: order, preference, flags, service, regexp, repl = r.split(' ', 5) values.append({ @@ -291,21 +309,21 @@ class AkamaiProvider(BaseProvider): }) return { 'type': _type, - 'ttl': _records['ttl'], + 'ttl':records['ttl'], 'values': values } - def _data_for_PTR(self, _type, _records): + def _data_for_PTR(self, _type, records): return { - 'ttl': _records['ttl'], + 'ttl':records['ttl'], 'type': _type, - 'value' : _records['rdata'][0] + 'value' :records['rdata'][0] } - def _data_for_SRV(self, _type, _records): + def _data_for_SRV(self, _type, records): values = [] - for r in _records['rdata']: + for r in records['rdata']: priority, weight, port, target = r.split(' ', 3) values.append({ 'port': port, @@ -315,13 +333,13 @@ class AkamaiProvider(BaseProvider): }) return { 'type': _type, - 'ttl': _records['ttl'], + 'ttl':records['ttl'], 'values': values } - def _data_for_SSHFP(self, _type, _records): + def _data_for_SSHFP(self, _type, records): values = [] - for r in _records['rdata']: + for r in records['rdata']: algorithm, fp_type, fingerprint = r.split(' ', 2) values.append({ 'algorithm': algorithm, @@ -330,18 +348,18 @@ class AkamaiProvider(BaseProvider): }) return { 'type': _type, - 'ttl': _records['ttl'], + 'ttl': records['ttl'], 'values': values } - def _data_for_TXT(self, _type, _records): + def _data_for_TXT(self, _type, records): values = [] - for r in _records['rdata']: + for r in records['rdata']: r = r[1:-1] values.append(r.replace(';', '\\;')) return { - 'ttl': _records['ttl'], + 'ttl': records['ttl'], 'type': _type, 'values': values } @@ -353,20 +371,108 @@ class AkamaiProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - domain_name = desired.name[:-1] + zone_name = desired.name[:-1] + print(zone_name) try: - self._dns_client.zone_get(domain_name) - except AkamaiClientException: - self.log.debug('_apply: no matching zone, creating domain') - - params = self._build_zone_config(domain_name) + self._dns_client.zone_get(zone_name) + except AkamaiClientException as e: + + print("zone not found, creating zone") + self.log.debug('_apply: no matching zone, creating zone') + params = self._build_zone_config(zone_name) self._dns_client.zone_create(self._contractId, params, self._gid) + for change in changes: + class_name = change.__class__.__name__ + print() + print(class_name) + if (class_name == "Delete" ): + print(change.existing.name) + print(change.existing._type) + print(change.existing.data) + elif class_name == "Update": + print(change.existing.name) + print(change.existing._type) + print(change.existing.data) + print("----------------->") + print(change.new.name) + print(change.new._type) + print(change.new.data) + else: + print(change.new.name) + print(change.new._type) + print(change.new.data) + print() + print(change) + print() + getattr(self, '_apply_{}'.format(class_name))(change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) + + def _apply_Create(self, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + pass + # self._dns_client.record_create(new.zone.name[:-1], params) + + def _apply_Delete(self, change): + existing = change.existing + zone = existing.zone.name[:-1] + name = existing.name + '.' + zone + record_type = existing._type + + result = self._dns_client.record_delete(zone, name, record_type) + + return result + + def _build_zone_config(self, zone, _type=None, comment=None, masters=[]): + + if _type is None: + _type="primary" + + if self._contractId is None: + self._set_default_contractId() + + return { + "zone": zone, + "type": _type, + "comment": comment, + "masters": masters + } + + def _set_default_contractId(self): + ''' if no contractId is set, but one is required to create a new zone, + this function will try to retrieve any contracts available to the + user, and use the first one it finds + ''' + + try: + + request = self._dns_client.zones_get(self._gid) + response = request.json() + zones = response['zones'] + + contractId = zones[0]['contractId'] + + + self._contractId = contractId + self.log.info("contractId not specified, using contractId=%s", contractId) - def _build_zone_config(self, zone, type="primary", comment=None, + except KeyError: + self.log.debug("_get_default_contractId: key error") + raise + + except: + self.log.debug("_get_default_contractId: unable to find a contractId") + raise + + return + def _test(self, zone) : From a7fb63623e1eaa0f057bb08d730089f81dea1ab1 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 20 Jun 2019 19:43:40 -0400 Subject: [PATCH 37/86] fixed apply_delete, create, and update. need to impl param_for_ fxns --- octodns/provider/fastdns.py | 159 ++++++++++++++++++++++++++---------- 1 file changed, 115 insertions(+), 44 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 6306145..42bf23b 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -78,9 +78,9 @@ class AkamaiClient(object): return result - def record_create(self, zone, name, record_type, params): + def record_create(self, zone, name, record_type, content): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('POST', path, data=params) + result = self._request('POST', path, data=content) return result @@ -94,9 +94,9 @@ class AkamaiClient(object): return result - def record_replace(self, zone, name, record_type, params): + def record_replace(self, zone, name, record_type, content): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('PUT', path, data=params) + result = self._request('PUT', path, data=content) return result @@ -368,67 +368,122 @@ class AkamaiProvider(BaseProvider): def _apply(self, plan): desired = plan.desired changes = plan.changes - self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, - len(changes)) + self.log.debug('_apply: zone=%s, changes=%d', desired.name, len(changes)) zone_name = desired.name[:-1] - print(zone_name) - - try: self._dns_client.zone_get(zone_name) - except AkamaiClientException as e: - - print("zone not found, creating zone") - self.log.debug('_apply: no matching zone, creating zone') + except AkamaiClientException: + self.log.info("zone not found, creating zone") params = self._build_zone_config(zone_name) self._dns_client.zone_create(self._contractId, params, self._gid) for change in changes: class_name = change.__class__.__name__ - print() - print(class_name) - if (class_name == "Delete" ): - print(change.existing.name) - print(change.existing._type) - print(change.existing.data) - elif class_name == "Update": - print(change.existing.name) - print(change.existing._type) - print(change.existing.data) - print("----------------->") - print(change.new.name) - print(change.new._type) - print(change.new.data) - else: - print(change.new.name) - print(change.new._type) - print(change.new.data) - print() - print(change) - print() getattr(self, '_apply_{}'.format(class_name))(change) # Clear out the cache if any self._zone_records.pop(desired.name, None) def _apply_Create(self, change): + new = change.new - params_for = getattr(self, '_params_for_{}'.format(new._type)) - for params in params_for(new): - pass - # self._dns_client.record_create(new.zone.name[:-1], params) + record_type = new._type + + params_for = getattr(self, '_params_for_{}'.format(record_type)) + values = self._get_values(new.data) + rdata = params_for(values) + + zone = new.zone.name[:-1] + + name = self._set_full_name(new.name, zone) + + content = { + "name": name, + "type": record_type, + "ttl" : new.ttl, + "rdata" : rdata + } + + print() + print ("mock create") + print ("zone=", zone, "name=", name , "record_type=", record_type) + print("content:") + print(json.dumps(content, indent=4, separators=(',', ': '))) + print() + + return + self._dns_client.record_create(zone, name, record_type, content) + + return def _apply_Delete(self, change): - existing = change.existing - zone = existing.zone.name[:-1] - name = existing.name + '.' + zone - record_type = existing._type - result = self._dns_client.record_delete(zone, name, record_type) + zone = change.existing.zone.name[:-1] + name = self._set_full_name(change.existing.name, zone) + record_type = change.existing._type + + self._dns_client.record_delete(zone, name, record_type) + + return + + def _apply_Update(self, change): + + new = change.new + record_type = new._type + + params_for = getattr(self, '_params_for_{}'.format(record_type)) + rdata = params_for(new.data) + + zone = new.zone.name[:-1] + name = self._set_full_name(new.name, zone) + + content = { + "name": name, + "type": record_type, + "ttl" : new.ttl, + "rdata" : rdata + } + + self._dns_client.record_create(zone, name, record_type, content) + + return + + + def _params_for_multiple(self, values): + + rdata = [r for r in values] + + return rdata + + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + _params_for_NS = _params_for_multiple + _params_for_SPF = _params_for_multiple + + def _params_for_CNAME(self, values): + pass + + def _params_for_MX(self, values): + pass + + def _params_for_NAPTR(self, values): + pass + + def _params_for_PTR(self, values): + pass + + def _params_for_SRV(self, values): + pass + + def _params_for_SSHFP(self, values): + pass + + def _params_for_TXT(self, values): + pass - return result def _build_zone_config(self, zone, _type=None, comment=None, masters=[]): @@ -473,6 +528,22 @@ class AkamaiProvider(BaseProvider): return + def _get_values(self, data): + + try: + vals = data['values'] + except KeyError: + vals = data['value'] + + return vals + + def _set_full_name(self, name, zone): + name = name + '.' + zone + if (name[0] == '.'): ## octodns's name for root is '' + name = name[1:] + + return name + def _test(self, zone) : From 0eb1f97ae39af8acce6145624a8e01cf801af921 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 20 Jun 2019 20:21:35 -0400 Subject: [PATCH 38/86] started implementing _params_for fxns --- octodns/provider/fastdns.py | 56 +++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 42bf23b..cf19c79 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -396,7 +396,6 @@ class AkamaiProvider(BaseProvider): rdata = params_for(values) zone = new.zone.name[:-1] - name = self._set_full_name(new.name, zone) content = { @@ -434,7 +433,8 @@ class AkamaiProvider(BaseProvider): record_type = new._type params_for = getattr(self, '_params_for_{}'.format(record_type)) - rdata = params_for(new.data) + values = self._get_values(new.data) + rdata = params_for(values) zone = new.zone.name[:-1] name = self._set_full_name(new.name, zone) @@ -446,34 +446,58 @@ class AkamaiProvider(BaseProvider): "rdata" : rdata } - self._dns_client.record_create(zone, name, record_type, content) + self._dns_client.record_replace(zone, name, record_type, content) return def _params_for_multiple(self, values): - - rdata = [r for r in values] - - return rdata + return [r for r in values] + def _params_for_single(self, values): + return values[0] _params_for_A = _params_for_multiple _params_for_AAAA = _params_for_multiple _params_for_NS = _params_for_multiple _params_for_SPF = _params_for_multiple - def _params_for_CNAME(self, values): - pass - def _params_for_MX(self, values): - pass + _params_for_CNAME = _params_for_single + _params_for_PTR = _params_for_single + + def _params_for_MX(self, values): + rdata = [] + + for r in values: + preference = r.preference + exchange = r.exchange + + record = '{} {}'.format(preference, exchange) + + rdata.append(record) + + return rdata + def _params_for_NAPTR(self, values): - pass + rdata = [] + + for r in values: + order = r.order + preference = r.preference + flags = "\"" + r.flags + "\"" + service = "\"" + r.service + "\"" + regexp = "\"" + r.regexp + "\"" + repl = r.replacement + + record = '{} {} {} {} {}'.format(order, preference, flags, service, + regexp, repl) + # record = ' '.join([order, preference, flags, service, regexp, repl]) + rdata.append(record) + + return rdata - def _params_for_PTR(self, values): - pass def _params_for_SRV(self, values): pass @@ -485,6 +509,7 @@ class AkamaiProvider(BaseProvider): pass + def _build_zone_config(self, zone, _type=None, comment=None, masters=[]): if _type is None: @@ -545,6 +570,9 @@ class AkamaiProvider(BaseProvider): return name + + + def _test(self, zone) : zone_name = zone.name[:len(zone.name)-1] From 3edf6969f8e44711c3c6cc72da91bc31c5049a79 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Fri, 21 Jun 2019 13:48:17 -0400 Subject: [PATCH 39/86] finished implementing applys, MVP reached --- octodns/provider/fastdns.py | 72 +++++++++++++++++++++---------------- ta | 0 targets=[azuredns, | 0 3 files changed, 41 insertions(+), 31 deletions(-) create mode 100644 ta create mode 100644 targets=[azuredns, diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index cf19c79..745fc34 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -26,7 +26,7 @@ class AkamaiClientException(Exception): 404: "404: Resource not found", 405: "405: Method not supported", 406: "406: Not Acceptable", - 409: "409: Request not allowed due to conflict with current state of resource", + 409: "409: Request not allowed due to conflict with current state", 415: "415: Unsupported media type", 422: "422: Request body contains an error preventing processing", 500: "500: Internal server error" @@ -43,7 +43,6 @@ class AkamaiClient(object): self.base = "https://" + _host + "/config-dns/v2/" self.basev1 = "https://" + _host + "/config-dns/v1/" - self.basehost = "https://" + _host sess = requests.Session() sess.auth = EdgeGridAuth( @@ -63,11 +62,9 @@ class AkamaiClient(object): resp = self._sess.request(method, url, params=params, json=data) if resp.status_code > 299: - # print(resp.status_code) raise AkamaiClientException(resp.status_code) resp.raise_for_status() - return resp @@ -160,8 +157,6 @@ class AkamaiClient(object): if gid is not None: path += '?gid={}'.format(gid) - # result = self._sess.request('GET', path) - result = self._request('GET', path) return result @@ -405,14 +400,6 @@ class AkamaiProvider(BaseProvider): "rdata" : rdata } - print() - print ("mock create") - print ("zone=", zone, "name=", name , "record_type=", record_type) - print("content:") - print(json.dumps(content, indent=4, separators=(',', ': '))) - print() - - return self._dns_client.record_create(zone, name, record_type, content) return @@ -455,24 +442,23 @@ class AkamaiProvider(BaseProvider): return [r for r in values] def _params_for_single(self, values): - return values[0] + return values _params_for_A = _params_for_multiple _params_for_AAAA = _params_for_multiple _params_for_NS = _params_for_multiple _params_for_SPF = _params_for_multiple - _params_for_CNAME = _params_for_single _params_for_PTR = _params_for_single - def _params_for_MX(self, values): rdata = [] for r in values: - preference = r.preference - exchange = r.exchange + print(json.dumps(r, indent=4, separators=(',', ': '))) + preference = r['preference'] + exchange = r['exchange'] record = '{} {}'.format(preference, exchange) @@ -484,29 +470,53 @@ class AkamaiProvider(BaseProvider): rdata = [] for r in values: - order = r.order - preference = r.preference - flags = "\"" + r.flags + "\"" - service = "\"" + r.service + "\"" - regexp = "\"" + r.regexp + "\"" - repl = r.replacement + order = r['order'] + preference = r['preference'] + flags = "\"" + r['flags'] + "\"" + service = "\"" + r['service'] + "\"" + regexp = "\"" + r['regexp'] + "\"" + repl = r['replacement'] - record = '{} {} {} {} {}'.format(order, preference, flags, service, + record = '{} {} {} {} {} {}'.format(order, preference, flags, service, regexp, repl) # record = ' '.join([order, preference, flags, service, regexp, repl]) rdata.append(record) return rdata - def _params_for_SRV(self, values): - pass + rdata = [] + for r in values: + priority = r['priority'] + weight = r['weight'] + port = r['port'] + target = r['target'] + record = '{} {} {} {}'.format(priority, weight, port, target) + rdata.append(record) + + return rdata + def _params_for_SSHFP(self, values): - pass + rdata = [] + for r in values: + algorithm = r['algorithm'] + fp_type = r['fingerprint_type'] + fp = r['fingerprint'] + record = '{} {} {}'.format(algorithm, fp_type, fp) + rdata.append(record) + + return rdata + def _params_for_TXT(self, values): - pass + rdata = [] + + for r in values: + txt = "\"" + r.replace('\\;', ';') + "\"" + rdata.append(txt) + + return rdata @@ -558,7 +568,7 @@ class AkamaiProvider(BaseProvider): try: vals = data['values'] except KeyError: - vals = data['value'] + vals = [data['value']] return vals diff --git a/ta b/ta new file mode 100644 index 0000000..e69de29 diff --git a/targets=[azuredns, b/targets=[azuredns, new file mode 100644 index 0000000..e69de29 From 8417a4475a8fa7d80a202a0e6a469654bb612731 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Fri, 21 Jun 2019 15:04:27 -0400 Subject: [PATCH 40/86] cleaned up code --- octodns/provider/fastdns.py | 531 +++++++++++------------------------- 1 file changed, 159 insertions(+), 372 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 745fc34..4d6f1cd 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -5,11 +5,9 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -## octodns specfic imports: import requests from akamai.edgegrid import EdgeGridAuth from urlparse import urljoin -import json from collections import defaultdict import logging @@ -32,17 +30,30 @@ class AkamaiClientException(Exception): 500: "500: Internal server error" } - def __init__(self, code): - message = self._errorMessages.get(code) - super(AkamaiClientException, self).__init__(message) + def __init__(self, resp): + try: + message = self._errorMessages.get(resp.status_code) + super(AkamaiClientException, self).__init__(message) + + except: + resp.raise_for_status() class AkamaiClient(object): + ''' + Client for making calls to Akamai Fast DNS API using Python Requests + + Fast DNS Zone Management API V2, found here: + developer.akamai.com/api/web_performance/fast_dns_zone_management/v2.html + + Info on Python Requests library: + https://2.python-requests.org/en/master/ + + ''' def __init__(self, _client_secret, _host, _access_token, _client_token): self.base = "https://" + _host + "/config-dns/v2/" - self.basev1 = "https://" + _host + "/config-dns/v1/" sess = requests.Session() sess.auth = EdgeGridAuth( @@ -56,25 +67,14 @@ class AkamaiClient(object): def _request(self, method, path, params=None, data=None, v1=False): url = urljoin(self.base, path) - if v1: - url = urljoin(self.basev1, path) - resp = self._sess.request(method, url, params=params, json=data) if resp.status_code > 299: - raise AkamaiClientException(resp.status_code) - resp.raise_for_status() + raise AkamaiClientException(resp) return resp - def record_get(self, zone, name, record_type): - - path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('GET', path) - - return result - def record_create(self, zone, name, record_type, content): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) result = self._request('POST', path, data=content) @@ -83,12 +83,8 @@ class AkamaiClient(object): def record_delete(self, zone, name, record_type): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - print(path) result = self._request('DELETE', path) - if result.status_code == 204: - print ("successfully deleted ", path) - return result def record_replace(self, zone, name, record_type, content): @@ -115,24 +111,6 @@ class AkamaiClient(object): return result - def zones_get(self, contractIds=None, page=None, pageSize=None, search=None, - showAll="true", sortBy="zone", types=None): - path = 'zones' - - params = { - 'contractIds': contractIds, - 'page': page, - 'pageSize': pageSize, - 'search': search, - 'showAll': showAll, - 'sortBy': sortBy, - 'types': types - } - - result = self._request('GET', path, params=params) - - return result - def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, showAll="true", sortBy="name", types=None): @@ -151,16 +129,6 @@ class AkamaiClient(object): return result - - def contracts_get(self, gid=None): - path = 'data/contracts' - if gid is not None: - path += '?gid={}'.format(gid) - - result = self._request('GET', path) - - return result - def recordsets_get(self, zone_name): resp = self.zone_recordset_get(zone_name, showAll="true") @@ -168,24 +136,55 @@ class AkamaiClient(object): return recordset - def master_zone_file_get(self, zone): - - path = 'zones/{}/zone-file'.format(zone) - try: - result = self._request('GET', path) +class AkamaiProvider(BaseProvider): - except AkamaiClientException as e: - # not working with API v2, API v1 fallback - path = 'zones/{}'.format(zone) - result = self._request('GET', path, v1=True) - print("Using API v1 fallback") - print("(Probably Ignore)", e.message) + ''' + Akamai Fast DNS Provider + + fastdns.py: + + Example config file with variables: + " + --- + providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config (example path to directory of zone files) + fastdns: + class: octodns.provider.fastdns.AkamaiProvider + client_secret: env/AKAMAI_CLIENT_SECRET + host: env/AKAMAI_HOST + access_token: env/AKAMAI_ACCESS_TOKEN + client_token: env/AKAMAI_CLIENT_TOKEN + contract_id: env/AKAMAI_CONTRACT_ID (optional) + + zones: + example.com.: + sources: + - config + targets: + - fastdns + " + + The first four variables above can be hidden in environment variables + and octoDNS will automatically search for them in the shell. It is + possible to also hard-code into the config file: eg, contract_id. + + The first four values can be found by generating credentials: + https://control.akamai.com/ + Configure > Organization > Manage APIs > New API Client for me + Select appropriate group, and fill relevant fields. + For "API Service Name", select "DNS—Zone Record Management" + and then set appropriate Access level (Read-Write to make changes). + Then select the "New Credential" button to generate values for above - return result + The contract_id paramater is optional, and only required for creating + a new zone. If the zone being managed already exists in Akamai for the + user in question, then this paramater is not needed. + ''' -class AkamaiProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False @@ -208,8 +207,8 @@ class AkamaiProvider(BaseProvider): self._gid = gid def zone_records(self, zone): - """ returns records for a zone, finds it if not present, or - returns empty if can't find a match + """ returns records for a zone, looks for it if not present, or + returns empty [] if can't find a match """ if zone.name not in self._zone_records: try: @@ -228,7 +227,7 @@ class AkamaiProvider(BaseProvider): for record in self.zone_records(zone): _type =record.get('type') - ## Akamai sends down prefix.zonename., while OctoDNS only expects prefix + ## Akamai sends down prefix.zonename., while octodns expects prefix _name = record.get('name').split("." + zone.name[:-1], 1)[0] if _name == zone.name[:-1] : _name = '' ## root / @ @@ -251,6 +250,85 @@ class AkamaiProvider(BaseProvider): return exists + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('apply: zone=%s, changes=%d', desired.name, len(changes)) + + zone_name = desired.name[:-1] + try: + self._dns_client.zone_get(zone_name) + + except AkamaiClientException: + self.log.info("zone not found, creating zone") + params = self._build_zone_config(zone_name) + self._dns_client.zone_create(self._contractId, params, self._gid) + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) + + def _apply_Create(self, change): + + new = change.new + record_type = new._type + + params_for = getattr(self, '_params_for_{}'.format(record_type)) + values = self._get_values(new.data) + rdata = params_for(values) + + zone = new.zone.name[:-1] + name = self._set_full_name(new.name, zone) + + content = { + "name": name, + "type": record_type, + "ttl" : new.ttl, + "rdata" : rdata + } + + self._dns_client.record_create(zone, name, record_type, content) + + return + + def _apply_Delete(self, change): + + zone = change.existing.zone.name[:-1] + name = self._set_full_name(change.existing.name, zone) + record_type = change.existing._type + + self._dns_client.record_delete(zone, name, record_type) + + return + + def _apply_Update(self, change): + + new = change.new + record_type = new._type + + params_for = getattr(self, '_params_for_{}'.format(record_type)) + values = self._get_values(new.data) + rdata = params_for(values) + + zone = new.zone.name[:-1] + name = self._set_full_name(new.name, zone) + + content = { + "name": name, + "type": record_type, + "ttl" : new.ttl, + "rdata" : rdata + } + + self._dns_client.record_replace(zone, name, record_type, content) + + return + + def _data_for_multiple(self, _type, records): return { @@ -326,6 +404,7 @@ class AkamaiProvider(BaseProvider): 'target': target, 'weight': weight }) + return { 'type': _type, 'ttl':records['ttl'], @@ -341,6 +420,7 @@ class AkamaiProvider(BaseProvider): 'fingerprint': fingerprint, 'fingerprint_type': fp_type }) + return { 'type': _type, 'ttl': records['ttl'], @@ -358,86 +438,8 @@ class AkamaiProvider(BaseProvider): 'type': _type, 'values': values } - - - def _apply(self, plan): - desired = plan.desired - changes = plan.changes - self.log.debug('_apply: zone=%s, changes=%d', desired.name, len(changes)) - - zone_name = desired.name[:-1] - try: - self._dns_client.zone_get(zone_name) - - except AkamaiClientException: - self.log.info("zone not found, creating zone") - params = self._build_zone_config(zone_name) - self._dns_client.zone_create(self._contractId, params, self._gid) - - for change in changes: - class_name = change.__class__.__name__ - getattr(self, '_apply_{}'.format(class_name))(change) - - # Clear out the cache if any - self._zone_records.pop(desired.name, None) - - def _apply_Create(self, change): - - new = change.new - record_type = new._type - - params_for = getattr(self, '_params_for_{}'.format(record_type)) - values = self._get_values(new.data) - rdata = params_for(values) - - zone = new.zone.name[:-1] - name = self._set_full_name(new.name, zone) - - content = { - "name": name, - "type": record_type, - "ttl" : new.ttl, - "rdata" : rdata - } - - self._dns_client.record_create(zone, name, record_type, content) - - return - - def _apply_Delete(self, change): - - zone = change.existing.zone.name[:-1] - name = self._set_full_name(change.existing.name, zone) - record_type = change.existing._type - - self._dns_client.record_delete(zone, name, record_type) - - return - - def _apply_Update(self, change): - - new = change.new - record_type = new._type - - params_for = getattr(self, '_params_for_{}'.format(record_type)) - values = self._get_values(new.data) - rdata = params_for(values) - - zone = new.zone.name[:-1] - name = self._set_full_name(new.name, zone) - - content = { - "name": name, - "type": record_type, - "ttl" : new.ttl, - "rdata" : rdata - } - - self._dns_client.record_replace(zone, name, record_type, content) - - return - - + + def _params_for_multiple(self, values): return [r for r in values] @@ -456,12 +458,10 @@ class AkamaiProvider(BaseProvider): rdata = [] for r in values: - print(json.dumps(r, indent=4, separators=(',', ': '))) preference = r['preference'] exchange = r['exchange'] record = '{} {}'.format(preference, exchange) - rdata.append(record) return rdata @@ -470,16 +470,14 @@ class AkamaiProvider(BaseProvider): rdata = [] for r in values: - order = r['order'] - preference = r['preference'] - flags = "\"" + r['flags'] + "\"" - service = "\"" + r['service'] + "\"" - regexp = "\"" + r['regexp'] + "\"" - repl = r['replacement'] + ordr = r['order'] + pref = r['preference'] + flg = "\"" + r['flags'] + "\"" + srvc = "\"" + r['service'] + "\"" + rgx = "\"" + r['regexp'] + "\"" + rpl = r['replacement'] - record = '{} {} {} {} {} {}'.format(order, preference, flags, service, - regexp, repl) - # record = ' '.join([order, preference, flags, service, regexp, repl]) + record = '{} {} {} {} {} {}'.format(ordr, pref, flg, srvc, rgx, rpl) rdata.append(record) return rdata @@ -518,15 +516,13 @@ class AkamaiProvider(BaseProvider): return rdata - - def _build_zone_config(self, zone, _type=None, comment=None, masters=[]): if _type is None: _type="primary" if self._contractId is None: - self._set_default_contractId() + raise NameError("contractId not specified to create zone") return { "zone": zone, @@ -535,34 +531,6 @@ class AkamaiProvider(BaseProvider): "masters": masters } - def _set_default_contractId(self): - ''' if no contractId is set, but one is required to create a new zone, - this function will try to retrieve any contracts available to the - user, and use the first one it finds - ''' - - try: - - request = self._dns_client.zones_get(self._gid) - response = request.json() - zones = response['zones'] - - contractId = zones[0]['contractId'] - - - self._contractId = contractId - self.log.info("contractId not specified, using contractId=%s", contractId) - - except KeyError: - self.log.debug("_get_default_contractId: key error") - raise - - except: - self.log.debug("_get_default_contractId: unable to find a contractId") - raise - - return - def _get_values(self, data): try: @@ -574,190 +542,9 @@ class AkamaiProvider(BaseProvider): def _set_full_name(self, name, zone): name = name + '.' + zone - if (name[0] == '.'): ## octodns's name for root is '' + + ## octodns's name for root is '' + if (name[0] == '.'): name = name[1:] return name - - - - - - def _test(self, zone) : - - zone_name = zone.name[:len(zone.name)-1] - - record_name = "octo.basir-test.com" - record_type = "A" - params = { - "name": "octo.basir-test.com", - "type": "A", - "ttl": 300, - "rdata": [ - "10.0.0.2", - "10.0.0.3" - ] - } - repl_params = { - "name": "octo.basir-test.com", - "type": "A", - "ttl": 300, - "rdata": [ - "99.99.99.99", - "10.0.0.3", - "1.2.3.4" - ] - } - - - print("\n\nRunning test: record get..........\n") - self._test_record_get(zone_name, "test.basir-test.com", record_type) - print("\n\nRunning test: record create..........\n") - self._test_record_create(zone_name, record_name, record_type, params) - print("\n\nRunning test: record replace..........\n") - self._test_record_replace(zone_name, record_name, record_type, repl_params) - print("\n\nRunning test: record delete..........\n") - self._test_record_delete(zone_name, record_name, record_type) - - print("\n\nRunning test: zones get..........\n") - self._test_zones_get() - - print("\n\nRunning test: zone recordset get..........\n") - self._test_zones_recordset_get(zone_name) - - print("\n\nRunning test: Master Zone File get..........\n") - self._test_master_zone_file_get(zone_name) - - return - - def _test_record_get(self, zone_name, record_name, record_type): - try: - get = self._dns_client.record_get(zone_name, record_name, record_type) - except AkamaiClientException as e: - print ("record get test failed") - print (e.message) - - else: - print("record get test result: ") - print(json.dumps(get.json(), indent=4, separators=(',', ': '))) - - return - - def _test_record_delete(self, zone_name, record_name, record_type): - - try: - delete = self._dns_client.record_delete(zone_name, record_name, record_type) - except AkamaiClientException as e: - print("delete failed") - print(e.message) - return - - - try: - self._dns_client.record_get(zone_name, record_name, record_type) - except AkamaiClientException as e: - print("get on record failed as expected, since record was succesfully deleted") - print ("(Probably Ignore):", e.message) - print ("delete status:", delete.status_code) - else: - print("unexpected condition in test delete") - - return - - def _test_record_create(self, zone_name, record_name, record_type, params): - - try: - create = self._dns_client.record_create(zone_name, record_name, record_type, params) - except AkamaiClientException as e: - print ("create unsuccessful, presumably because it already exists") - print ("(Probably Ignore)", e.message) - else: - print("initial create of", create.json().get("name"), "succesful: ", create.status_code) - - return - - def _test_record_replace(self, zone_name, record_name, record_type, params): - - ## create record to be replaced, if it doesn't already exist - try: - old_params = { - "name": record_name, - "type": record_type, - "ttl": 300, - "rdata": [ - "10.0.0.2", - "10.0.0.3" - ] - } - create = self._dns_client.record_create(zone_name, record_name, record_type, old_params) - except AkamaiClientException as e: - print ("initial create unsuccessful, presumably because it already exists") - print ("(Probably Ignore)", e.message) - else: - print("initial create of record to be replaced", create.json().get("name"), "succesful: ", create.status_code) - - - ## test replace - try: - replace = self._dns_client.record_replace(zone_name, record_name, record_type, params) - except AkamaiClientException as e: - print("replace failed") - print(e.message) - return - else: - try: - record = self._dns_client.record_get(zone_name, record_name, record_type) - except AkamaiClientException as e: - print("retrieval in replacement failed") - print(e.message) - else: - new_data = record.json() - - if (new_data != params): - print("replace failed, records don't match") - print("current data:") - print(new_data) - print("expected data:") - print(params) - - else: - print("replace succesful") - print("replace status:", replace.status_code) - - def _test_zones_get(self): - try: - zonesList = self._dns_client.zones_get() - except AkamaiClientException as e: - print ("zones get test failed") - print (e.message) - - else: - print("zones list: ") - print(json.dumps(zonesList.json(), indent=4, separators=(',', ': '))) - - return - - def _test_zones_recordset_get(self, zone_name): - try: - zoneRecordset = self._dns_client.zone_recordset_get(zone_name) - except AkamaiClientException as e: - print("zone recordset retrieval test failed") - print (e.message) - else: - print("zone recordset: ") - print(json.dumps(zoneRecordset.json(), indent=4, separators=(',', ': '))) - return - - def _test_master_zone_file_get(self, zone_name): - try: - mzf = self._dns_client.master_zone_file_get(zone_name) - - except AkamaiClientException as e: - print("MZF retrieval test failed") - print (e.message) - - else: - print("Master Zone File:") - print(json.dumps(mzf.json(), indent=4, separators=(',', ': '))) - - return From 173d41aa9930346e8d92e1521e89ab3cf94cf795 Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 21 Jun 2019 15:06:14 -0400 Subject: [PATCH 41/86] Delete targets=[azuredns, --- targets=[azuredns, | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 targets=[azuredns, diff --git a/targets=[azuredns, b/targets=[azuredns, deleted file mode 100644 index e69de29..0000000 From c7251444c82d1562ac1d280f3e300cb337b3717f Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 21 Jun 2019 15:07:14 -0400 Subject: [PATCH 42/86] Delete ta --- ta | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 ta diff --git a/ta b/ta deleted file mode 100644 index e69de29..0000000 From 2397030cf6d2e7c8d1698b1a66e888a3f34eef4d Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 21 Jun 2019 15:07:24 -0400 Subject: [PATCH 43/86] Delete settings.json --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 38e6c1e..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.pythonPath": "C:\\Python27\\python.exe" -} \ No newline at end of file From 6f4e60ac141fd53eb29f112f2b82238a5610d8de Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 21 Jun 2019 15:08:11 -0400 Subject: [PATCH 44/86] Delete settings.json --- octodns/.vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 octodns/.vscode/settings.json diff --git a/octodns/.vscode/settings.json b/octodns/.vscode/settings.json deleted file mode 100644 index 38e6c1e..0000000 --- a/octodns/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.pythonPath": "C:\\Python27\\python.exe" -} \ No newline at end of file From 70a1627757cb6eafd32596a3a35acd7626df68c5 Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 21 Jun 2019 15:08:43 -0400 Subject: [PATCH 45/86] Delete settings.json --- octodns/provider/.vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 octodns/provider/.vscode/settings.json diff --git a/octodns/provider/.vscode/settings.json b/octodns/provider/.vscode/settings.json deleted file mode 100644 index 88322cc..0000000 --- a/octodns/provider/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.pythonPath": "C:\\Python27\\python.exe" -} \ No newline at end of file From 5b143e422c2edbbb73d220baf428415b432698de Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Fri, 21 Jun 2019 17:26:54 -0400 Subject: [PATCH 46/86] cleanup - fixed lint issues --- octodns/provider/fastdns.py | 178 +++++++++++++++++------------------- 1 file changed, 85 insertions(+), 93 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 4d6f1cd..26c4351 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -12,13 +12,13 @@ from collections import defaultdict import logging from ..record import Record -from .base import BaseProvider +from .base import BaseProvider class AkamaiClientException(Exception): - + _errorMessages = { - 400: "400: Bad request", + 400: "400: Bad request", 401: "401: Unauthorized", 403: "403: Access is forbidden", 404: "404: Resource not found", @@ -35,15 +35,15 @@ class AkamaiClientException(Exception): message = self._errorMessages.get(resp.status_code) super(AkamaiClientException, self).__init__(message) - except: + except: resp.raise_for_status() class AkamaiClient(object): - ''' + ''' Client for making calls to Akamai Fast DNS API using Python Requests - Fast DNS Zone Management API V2, found here: + Fast DNS Zone Management API V2, found here: developer.akamai.com/api/web_performance/fast_dns_zone_management/v2.html Info on Python Requests library: @@ -63,9 +63,8 @@ class AkamaiClient(object): ) self._sess = sess - def _request(self, method, path, params=None, data=None, v1=False): - + url = urljoin(self.base, path) resp = self._sess.request(method, url, params=params, json=data) @@ -73,7 +72,6 @@ class AkamaiClient(object): raise AkamaiClientException(resp) return resp - def record_create(self, zone, name, record_type, content): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) @@ -84,7 +82,7 @@ class AkamaiClient(object): def record_delete(self, zone, name, record_type): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) result = self._request('DELETE', path) - + return result def record_replace(self, zone, name, record_type, content): @@ -93,7 +91,6 @@ class AkamaiClient(object): return result - def zone_get(self, zone): path = 'zones/{}'.format(zone) result = self._request('GET', path) @@ -105,17 +102,16 @@ class AkamaiClient(object): if gid is not None: path += '&gid={}'.format(gid) - + result = self._request('POST', path, data=params) - - return result + return result def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, - showAll="true", sortBy="name", types=None): + showAll="true", sortBy="name", types=None): params = { - 'page': page, + 'page': page, 'pageSize': pageSize, 'search': search, 'showAll': showAll, @@ -126,11 +122,10 @@ class AkamaiClient(object): path = 'zones/{}/recordsets'.format(zone) result = self._request('GET', path, params=params) - return result def recordsets_get(self, zone_name): - + resp = self.zone_recordset_get(zone_name, showAll="true") recordset = resp.json().get("recordsets") @@ -170,8 +165,8 @@ class AkamaiProvider(BaseProvider): The first four variables above can be hidden in environment variables and octoDNS will automatically search for them in the shell. It is possible to also hard-code into the config file: eg, contract_id. - - The first four values can be found by generating credentials: + + The first four values can be found by generating credentials: https://control.akamai.com/ Configure > Organization > Manage APIs > New API Client for me Select appropriate group, and fill relevant fields. @@ -179,41 +174,41 @@ class AkamaiProvider(BaseProvider): and then set appropriate Access level (Read-Write to make changes). Then select the "New Credential" button to generate values for above - The contract_id paramater is optional, and only required for creating - a new zone. If the zone being managed already exists in Akamai for the - user in question, then this paramater is not needed. + The contract_id paramater is optional, and only required for creating + a new zone. If the zone being managed already exists in Akamai for the + user in question, then this paramater is not needed. ''' - SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'SSHFP', 'TXT')) - def __init__(self, id, client_secret, host, access_token, client_token, - contract_id=None, gid=None, *args, **kwargs): - + def __init__(self, id, client_secret, host, access_token, client_token, + contract_id=None, gid=None, *args, **kwargs): + self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, ') super(AkamaiProvider, self).__init__(id, *args, **kwargs) - self._dns_client = AkamaiClient(client_secret, host, access_token, - client_token) - + self._dns_client = AkamaiClient(client_secret, host, access_token, + client_token) + self._zone_records = {} self._contractId = contract_id self._gid = gid def zone_records(self, zone): - """ returns records for a zone, looks for it if not present, or + """ returns records for a zone, looks for it if not present, or returns empty [] if can't find a match """ if zone.name not in self._zone_records: try: name = zone.name[:-1] - self._zone_records[zone.name] = self._dns_client.recordsets_get(name) + self._zone_records[zone.name] = \ + self._dns_client.recordsets_get(name) except AkamaiClientException: return [] @@ -225,12 +220,12 @@ class AkamaiProvider(BaseProvider): values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): - - _type =record.get('type') - ## Akamai sends down prefix.zonename., while octodns expects prefix - _name = record.get('name').split("." + zone.name[:-1], 1)[0] - if _name == zone.name[:-1] : - _name = '' ## root / @ + + _type = record.get('type') + # Akamai sends down prefix.zonename., while octodns expects prefix + _name = record.get('name').split("." + zone.name[:-1], 1)[0] + if _name == zone.name[:-1]: + _name = '' # root / @ if _type not in self.SUPPORTS: continue @@ -240,21 +235,20 @@ class AkamaiProvider(BaseProvider): for name, types in values.items(): for _type, records in types.items(): data_for = getattr(self, '_data_for_{}'.format(_type)) - record = Record.new(zone, name, data_for(_type, records[0]), - source=self, lenient=lenient) + record = Record.new(zone, name, data_for(_type, records[0]), + source=self, lenient=lenient) zone.add_record(record, lenient=lenient) - + exists = zone.name in self._zone_records found = len(zone.records) - before - self.log.info('populate: found %s records, exists=%s', found, exists) + self.log.info('populate: found %s records, exists=%s', found, exists) return exists - def _apply(self, plan): desired = plan.desired changes = plan.changes - self.log.debug('apply: zone=%s, changes=%d', desired.name, len(changes)) + self.log.debug('apply: zone=%s, chnges=%d', desired.name, len(changes)) zone_name = desired.name[:-1] try: @@ -287,13 +281,13 @@ class AkamaiProvider(BaseProvider): content = { "name": name, "type": record_type, - "ttl" : new.ttl, - "rdata" : rdata + "ttl": new.ttl, + "rdata": rdata } self._dns_client.record_create(zone, name, record_type, content) - return + return def _apply_Delete(self, change): @@ -303,10 +297,10 @@ class AkamaiProvider(BaseProvider): self._dns_client.record_delete(zone, name, record_type) - return + return def _apply_Update(self, change): - + new = change.new record_type = new._type @@ -320,35 +314,34 @@ class AkamaiProvider(BaseProvider): content = { "name": name, "type": record_type, - "ttl" : new.ttl, - "rdata" : rdata + "ttl": new.ttl, + "rdata": rdata } self._dns_client.record_replace(zone, name, record_type, content) - return - + return def _data_for_multiple(self, _type, records): - + return { - 'ttl':records['ttl'], - 'type': _type, + 'ttl': records['ttl'], + 'type': _type, 'values': [r for r in records['rdata']] - } + } _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple _data_for_NS = _data_for_multiple _data_for_SPF = _data_for_multiple - def _data_for_CNAME(self, _type, records): - value =records['rdata'][0] - if (value[-1] != '.') : + def _data_for_CNAME(self, _type, records): + value = records['rdata'][0] + if (value[-1] != '.'): value = '{}.'.format(value) return { - 'ttl':records['ttl'], + 'ttl': records['ttl'], 'type': _type, 'value': value } @@ -359,10 +352,10 @@ class AkamaiProvider(BaseProvider): preference, exchange = r.split(" ", 1) values.append({ 'preference': preference, - 'exchange' : exchange - }) + 'exchange': exchange + }) return { - 'ttl':records['ttl'], + 'ttl': records['ttl'], 'type': _type, 'values': values } @@ -371,29 +364,29 @@ class AkamaiProvider(BaseProvider): values = [] for r in records['rdata']: order, preference, flags, service, regexp, repl = r.split(' ', 5) - + values.append({ 'flags': flags[1:-1], 'order': order, 'preference': preference, 'regexp': regexp[1:-1], - 'replacement': repl, + 'replacement': repl, 'service': service[1:-1] }) return { 'type': _type, - 'ttl':records['ttl'], + 'ttl': records['ttl'], 'values': values } - + def _data_for_PTR(self, _type, records): return { - 'ttl':records['ttl'], + 'ttl': records['ttl'], 'type': _type, - 'value' :records['rdata'][0] - } - + 'value': records['rdata'][0] + } + def _data_for_SRV(self, _type, records): values = [] for r in records['rdata']: @@ -407,7 +400,7 @@ class AkamaiProvider(BaseProvider): return { 'type': _type, - 'ttl':records['ttl'], + 'ttl': records['ttl'], 'values': values } @@ -420,11 +413,11 @@ class AkamaiProvider(BaseProvider): 'fingerprint': fingerprint, 'fingerprint_type': fp_type }) - + return { 'type': _type, 'ttl': records['ttl'], - 'values': values + 'values': values } def _data_for_TXT(self, _type, records): @@ -438,8 +431,7 @@ class AkamaiProvider(BaseProvider): 'type': _type, 'values': values } - - + def _params_for_multiple(self, values): return [r for r in values] @@ -453,7 +445,7 @@ class AkamaiProvider(BaseProvider): _params_for_CNAME = _params_for_single _params_for_PTR = _params_for_single - + def _params_for_MX(self, values): rdata = [] @@ -471,15 +463,15 @@ class AkamaiProvider(BaseProvider): for r in values: ordr = r['order'] - pref = r['preference'] + prf = r['preference'] flg = "\"" + r['flags'] + "\"" srvc = "\"" + r['service'] + "\"" rgx = "\"" + r['regexp'] + "\"" rpl = r['replacement'] - - record = '{} {} {} {} {} {}'.format(ordr, pref, flg, srvc, rgx, rpl) + + record = '{} {} {} {} {} {}'.format(ordr, prf, flg, srvc, rgx, rpl) rdata.append(record) - + return rdata def _params_for_SRV(self, values): @@ -494,7 +486,7 @@ class AkamaiProvider(BaseProvider): rdata.append(record) return rdata - + def _params_for_SSHFP(self, values): rdata = [] for r in values: @@ -506,10 +498,10 @@ class AkamaiProvider(BaseProvider): rdata.append(record) return rdata - + def _params_for_TXT(self, values): rdata = [] - + for r in values: txt = "\"" + r.replace('\\;', ';') + "\"" rdata.append(txt) @@ -518,11 +510,11 @@ class AkamaiProvider(BaseProvider): def _build_zone_config(self, zone, _type=None, comment=None, masters=[]): - if _type is None: - _type="primary" - + if _type is None: + _type = "primary" + if self._contractId is None: - raise NameError("contractId not specified to create zone") + raise NameError("contractId not specified to create zone") return { "zone": zone, @@ -537,14 +529,14 @@ class AkamaiProvider(BaseProvider): vals = data['values'] except KeyError: vals = [data['value']] - + return vals def _set_full_name(self, name, zone): name = name + '.' + zone - ## octodns's name for root is '' - if (name[0] == '.'): + # octodns's name for root is '' + if (name[0] == '.'): name = name[1:] return name From 550f5b14a391d741e223ba0ef64e74cb18b6f6a4 Mon Sep 17 00:00:00 2001 From: Basir Date: Wed, 26 Jun 2019 16:19:52 -0400 Subject: [PATCH 47/86] started fastdns testing --- tests/test_octodns_provider_fastdns.py | 70 ++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tests/test_octodns_provider_fastdns.py diff --git a/tests/test_octodns_provider_fastdns.py b/tests/test_octodns_provider_fastdns.py new file mode 100644 index 0000000..baf28ea --- /dev/null +++ b/tests/test_octodns_provider_fastdns.py @@ -0,0 +1,70 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from mock import Mock, call +from os.path import dirname, join +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.record import Record +from octodns.provider.fastdns import AkamaiProvider, AkamaiClientException +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestFastdnsProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + + def test_populate(self): + provider = AkamaiProvider("test", "client_secret", "host", "access_token", + "client_token") + + # Bad Auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, + text='{"message": "Authentication failed"}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals("401: Unauthorized", ctx.exception.message) + + # general error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existant zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='{"message": "Domain `foo.bar` not found"}') + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # # No diffs == no changes + # with requests_mock() as mock: + # base = 'https://api.dnsimple.com/v2/42/zones/unit.tests/' \ + # 'records?page=' + # with open('tests/fixtures/dnsimple-page-1.json') as fh: + # mock.get('{}{}'.format(base, 1), text=fh.read()) + # with open('tests/fixtures/dnsimple-page-2.json') as fh: + # mock.get('{}{}'.format(base, 2), text=fh.read()) + + # zone = Zone('unit.tests.', []) + # provider.populate(zone) + # self.assertEquals(16, len(zone.records)) + # changes = self.expected.changes(zone, provider) + # self.assertEquals(0, len(changes)) \ No newline at end of file From ea3b84091ea4f4760d3a165bec24ac293cdea092 Mon Sep 17 00:00:00 2001 From: Basir Date: Wed, 26 Jun 2019 16:25:57 -0400 Subject: [PATCH 48/86] encoding issue --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 26c4351..7ad5988 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -170,7 +170,7 @@ class AkamaiProvider(BaseProvider): https://control.akamai.com/ Configure > Organization > Manage APIs > New API Client for me Select appropriate group, and fill relevant fields. - For "API Service Name", select "DNS—Zone Record Management" + For API Service Name, select DNS—Zone Record Management and then set appropriate Access level (Read-Write to make changes). Then select the "New Credential" button to generate values for above From 5729358e3703fc8231b6ac819184967578d7a193 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Tue, 16 Jul 2019 19:27:24 +0000 Subject: [PATCH 49/86] removed redundant recordsets_get fxn from AkamaiClient --- octodns/provider/fastdns.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 7ad5988..90323b2 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -124,14 +124,6 @@ class AkamaiClient(object): return result - def recordsets_get(self, zone_name): - - resp = self.zone_recordset_get(zone_name, showAll="true") - recordset = resp.json().get("recordsets") - - return recordset - - class AkamaiProvider(BaseProvider): ''' @@ -207,8 +199,9 @@ class AkamaiProvider(BaseProvider): if zone.name not in self._zone_records: try: name = zone.name[:-1] - self._zone_records[zone.name] = \ - self._dns_client.recordsets_get(name) + resp = self._dns_client.zone_recordset_get(name, showAll="true") + resp = resp.json().get("recordsets") + self._zone_records[zone.name] = resp except AkamaiClientException: return [] From 3e3bfe5cf5d691bed7023f100a366acbba7eaf42 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Wed, 17 Jul 2019 23:34:58 +0000 Subject: [PATCH 50/86] fastdns provider fixes: - fixed exception handling in AkamaiClientException - fixed zone_records and zone_recordset fxns in AkamaiProvider and AkamaiClient - refactored _build_zone_config in AkamaiProvider --- octodns/provider/fastdns.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 90323b2..3dd52e7 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -19,7 +19,6 @@ class AkamaiClientException(Exception): _errorMessages = { 400: "400: Bad request", - 401: "401: Unauthorized", 403: "403: Access is forbidden", 404: "404: Resource not found", 405: "405: Method not supported", @@ -32,7 +31,7 @@ class AkamaiClientException(Exception): def __init__(self, resp): try: - message = self._errorMessages.get(resp.status_code) + message = self._errorMessages[resp.status_code] super(AkamaiClientException, self).__init__(message) except: @@ -107,7 +106,7 @@ class AkamaiClient(object): return result - def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, + def zone_recordset_get(self, zone, page=None, pageSize=None, search=None, showAll="true", sortBy="name", types=None): params = { @@ -124,6 +123,7 @@ class AkamaiClient(object): return result + class AkamaiProvider(BaseProvider): ''' @@ -162,7 +162,7 @@ class AkamaiProvider(BaseProvider): https://control.akamai.com/ Configure > Organization > Manage APIs > New API Client for me Select appropriate group, and fill relevant fields. - For API Service Name, select DNS—Zone Record Management + For API Service Name, select DNS-Zone Record Management and then set appropriate Access level (Read-Write to make changes). Then select the "New Credential" button to generate values for above @@ -199,9 +199,11 @@ class AkamaiProvider(BaseProvider): if zone.name not in self._zone_records: try: name = zone.name[:-1] - resp = self._dns_client.zone_recordset_get(name, showAll="true") - resp = resp.json().get("recordsets") - self._zone_records[zone.name] = resp + response = self._dns_client.zone_recordset_get(name) + self._zone_records[zone.name] = response.json()["recordsets"] + + except KeyError: + return [] except AkamaiClientException: return [] @@ -501,10 +503,8 @@ class AkamaiProvider(BaseProvider): return rdata - def _build_zone_config(self, zone, _type=None, comment=None, masters=[]): - - if _type is None: - _type = "primary" + def _build_zone_config(self, zone, _type="primary", comment=None, + masters=[]): if self._contractId is None: raise NameError("contractId not specified to create zone") From 41c59861440400524307aa78e5ec3e684b26d34c Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Wed, 17 Jul 2019 23:43:56 +0000 Subject: [PATCH 51/86] Fastdns provider: - added test cases for testin populate() - added mock request responses in tests/fixtures/ - updated requirements.txt to require edgegrid-python --- requirements.txt | 1 + tests/fixtures/fastdns-invalid-content.json | 35 +++++ tests/fixtures/fastdns-records.json | 157 ++++++++++++++++++++ tests/test_octodns_provider_fastdns.py | 60 +++++--- 4 files changed, 232 insertions(+), 21 deletions(-) create mode 100644 tests/fixtures/fastdns-invalid-content.json create mode 100644 tests/fixtures/fastdns-records.json diff --git a/requirements.txt b/requirements.txt index d100c96..75dc1df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ botocore==1.10.5 dnspython==1.15.0 docutils==0.14 dyn==1.8.1 +edgegrid-python==1.1.1 futures==3.2.0 google-cloud-core==0.28.1 google-cloud-dns==0.29.0 diff --git a/tests/fixtures/fastdns-invalid-content.json b/tests/fixtures/fastdns-invalid-content.json new file mode 100644 index 0000000..8932f66 --- /dev/null +++ b/tests/fixtures/fastdns-invalid-content.json @@ -0,0 +1,35 @@ +{ + "recordsets": [ + { + "rdata": [ + "", + "12 20 foo-2.unit.tests." + ], + "type": "SRV", + "name": "_srv._tcp.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "", + "1 1" + ], + "type": "SSHFP", + "name": "unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "", + "100 \"U\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" ." + ], + "type": "NAPTR", + "name": "naptr.unit.tests", + "ttl": 600 + } + ], + "metadata": { + "totalElements": 3, + "showAll": true + } +} \ No newline at end of file diff --git a/tests/fixtures/fastdns-records.json b/tests/fixtures/fastdns-records.json new file mode 100644 index 0000000..4693eb1 --- /dev/null +++ b/tests/fixtures/fastdns-records.json @@ -0,0 +1,157 @@ +{ + "recordsets": [ + { + "rdata": [ + "10 20 30 foo-1.unit.tests.", + "12 20 30 foo-2.unit.tests." + ], + "type": "SRV", + "name": "_srv._tcp.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ], + "type": "AAAA", + "name": "aaaa.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "ns1.akam.net.", + "ns2.akam.net.", + "ns3.akam.net.", + "ns4.akam.net." + ], + "type": "NS", + "name": "unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "1.2.3.4", + "1.2.3.5" + ], + "type": "A", + "name": "unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "ns1.akam.net hostmaster.akamai.com 1489074932 86400 7200 604800 300" + ], + "type": "SOA", + "name": "unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49", + "1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73" + ], + "type": "SSHFP", + "name": "unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "unit.tests." + ], + "type": "CNAME", + "name": "cname.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "unit.tests." + ], + "type": "CNAME", + "name": "excluded.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "unit.tests." + ], + "type": "CNAME", + "name": "included.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "10 smtp-4.unit.tests.", + "20 smtp-2.unit.tests.", + "30 smtp-3.unit.tests.", + "40 smtp-1.unit.tests." + ], + "type": "MX", + "name": "mx.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "10 100 \"S\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" .", + "100 100 \"U\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" ." + ], + "type": "NAPTR", + "name": "naptr.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "foo.bar.com." + ], + "type": "PTR", + "name": "ptr.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "\"v=spf1 ip4:192.168.0.1/16-all\"" + ], + "type": "SPF", + "name": "spf.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "ns1.unit.tests.", + "ns2.unit.tests." + ], + "type": "NS", + "name": "under.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "\"Bah bah black sheep\"", + "\"have you any wool.\"", + "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"" + ], + "type": "TXT", + "name": "txt.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "2.2.3.6" + ], + "type": "A", + "name": "www.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "2.2.3.6" + ], + "type": "A", + "name": "www.sub.unit.tests", + "ttl": 300 + } + ], + "metadata": { + "totalElements": 16, + "showAll": true + } +} \ No newline at end of file diff --git a/tests/test_octodns_provider_fastdns.py b/tests/test_octodns_provider_fastdns.py index baf28ea..8c841fe 100644 --- a/tests/test_octodns_provider_fastdns.py +++ b/tests/test_octodns_provider_fastdns.py @@ -22,19 +22,32 @@ class TestFastdnsProvider(TestCase): source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) + # Our test suite differs a bit, add our NS and remove the simple one + expected.add_record(Record.new(expected, 'under', { + 'ttl': 3600, + 'type': 'NS', + 'values': [ + 'ns1.unit.tests.', + 'ns2.unit.tests.', + ] + })) + for record in list(expected.records): + if record.name == 'sub' and record._type == 'NS': + expected._remove_record(record) + break + def test_populate(self): - provider = AkamaiProvider("test", "client_secret", "host", "access_token", - "client_token") - + provider = AkamaiProvider("test", "secret", "akam.com", "atok", "ctok") + # Bad Auth with requests_mock() as mock: - mock.get(ANY, status_code=401, - text='{"message": "Authentication failed"}') + mock.get(ANY, status_code=401, text='{"message": "Unauthorized"}') with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals("401: Unauthorized", ctx.exception.message) + + self.assertEquals(401, ctx.exception.response.status_code) # general error with requests_mock() as mock: @@ -45,7 +58,7 @@ class TestFastdnsProvider(TestCase): provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) - # Non-existant zone doesn't populate anything + # Non-existant zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=404, text='{"message": "Domain `foo.bar` not found"}') @@ -54,17 +67,22 @@ class TestFastdnsProvider(TestCase): provider.populate(zone) self.assertEquals(set(), zone.records) - # # No diffs == no changes - # with requests_mock() as mock: - # base = 'https://api.dnsimple.com/v2/42/zones/unit.tests/' \ - # 'records?page=' - # with open('tests/fixtures/dnsimple-page-1.json') as fh: - # mock.get('{}{}'.format(base, 1), text=fh.read()) - # with open('tests/fixtures/dnsimple-page-2.json') as fh: - # mock.get('{}{}'.format(base, 2), text=fh.read()) - - # zone = Zone('unit.tests.', []) - # provider.populate(zone) - # self.assertEquals(16, len(zone.records)) - # changes = self.expected.changes(zone, provider) - # self.assertEquals(0, len(changes)) \ No newline at end of file + # No diffs == no changes + with requests_mock() as mock: + + with open('tests/fixtures/fastdns-records.json') as fh: + mock.get(ANY, text=fh.read()) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(16, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 2nd populate makes no network calls/all from cache + again = Zone('unit.tests.', []) + provider.populate(again) + self.assertEquals(16, len(again.records)) + + # bust the cache + del provider._zone_records[zone.name] \ No newline at end of file From 852d607715078c682c1c1bb74a365a430279acc7 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 18 Jul 2019 01:07:42 +0000 Subject: [PATCH 52/86] fastdns: - added test cases for test_apply, testing for basic create, update, delete records - added previous state json file in test/fixtures to do the above --- tests/fixtures/fastdns-records-prev.json | 166 +++++++++++++++++++++++ tests/test_octodns_provider_fastdns.py | 22 ++- 2 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/fastdns-records-prev.json diff --git a/tests/fixtures/fastdns-records-prev.json b/tests/fixtures/fastdns-records-prev.json new file mode 100644 index 0000000..04ce274 --- /dev/null +++ b/tests/fixtures/fastdns-records-prev.json @@ -0,0 +1,166 @@ +{ + "recordsets": [ + { + "rdata": [ + "10 20 30 foo-1.unit.tests.", + "12 20 30 foo-2.unit.tests." + ], + "type": "SRV", + "name": "_srv._tcp.old.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "10 20 30 foo-1.unit.tests.", + "12 20 30 foo-2.unit.tests." + ], + "type": "SRV", + "name": "_srv._tcp.old.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "2601:644:500:e210:62f8:1dff:feb8:9471" + ], + "type": "AAAA", + "name": "aaaa.old.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "ns1.akam.net.", + "ns2.akam.net.", + "ns3.akam.net.", + "ns4.akam.net." + ], + "type": "NS", + "name": "old.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "1.2.3.4", + "1.2.3.5" + ], + "type": "A", + "name": "old.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "ns1.akam.net hostmaster.akamai.com 1489074932 86400 7200 604800 300" + ], + "type": "SOA", + "name": "unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49", + "1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73" + ], + "type": "SSHFP", + "name": "old.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "unit.tests." + ], + "type": "CNAME", + "name": "old.cname.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "unit.tests." + ], + "type": "CNAME", + "name": "excluded.old.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "unit.tests." + ], + "type": "CNAME", + "name": "included.old.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "10 smtp-4.unit.tests.", + "20 smtp-2.unit.tests.", + "30 smtp-3.unit.tests.", + "40 smtp-1.unit.tests." + ], + "type": "MX", + "name": "mx.old.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "10 100 \"S\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" .", + "100 100 \"U\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" ." + ], + "type": "NAPTR", + "name": "naptr.old.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "foo.bar.com." + ], + "type": "PTR", + "name": "ptr.old.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "\"v=spf1 ip4:192.168.0.1/16-all\"" + ], + "type": "SPF", + "name": "spf.old.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "ns1.unit.tests.", + "ns2.unit.tests." + ], + "type": "NS", + "name": "under.old.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "\"Bah bah black sheep\"", + "\"have you any wool.\"", + "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"" + ], + "type": "TXT", + "name": "txt.old.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "2.2.3.7" + ], + "type": "A", + "name": "www.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "2.2.3.6" + ], + "type": "A", + "name": "www.sub.old.unit.tests", + "ttl": 300 + } + ], + "metadata": { + "totalElements": 16, + "showAll": true + } +} \ No newline at end of file diff --git a/tests/test_octodns_provider_fastdns.py b/tests/test_octodns_provider_fastdns.py index 8c841fe..4df36c0 100644 --- a/tests/test_octodns_provider_fastdns.py +++ b/tests/test_octodns_provider_fastdns.py @@ -5,14 +5,14 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from mock import Mock, call +# from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock from unittest import TestCase from octodns.record import Record -from octodns.provider.fastdns import AkamaiProvider, AkamaiClientException +from octodns.provider.fastdns import AkamaiProvider from octodns.provider.yaml import YamlProvider from octodns.zone import Zone @@ -85,4 +85,20 @@ class TestFastdnsProvider(TestCase): self.assertEquals(16, len(again.records)) # bust the cache - del provider._zone_records[zone.name] \ No newline at end of file + del provider._zone_records[zone.name] + + def test_apply(self): + provider = AkamaiProvider("test", "secret", "akam.com", "atok", "ctok") + + with requests_mock() as mock: + + with open('tests/fixtures/fastdns-records-prev.json') as fh: + mock.get(ANY, text=fh.read()) + + plan = provider.plan(self.expected) + mock.post(ANY, status_code=201) + mock.put(ANY, status_code=200) + mock.delete(ANY, status_code=204) + + changes = provider.apply(plan) + self.assertEquals(29, changes) From 1596add89f54c32af3e8c005f53507b3d0f39489 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 18 Jul 2019 20:46:05 +0000 Subject: [PATCH 53/86] fastdns provider: - added unit tests for apply functions - added another json config file for this - 100% code coverage reached --- .../fixtures/fastdns-records-prev-other.json | 166 ++++++++++++++++++ tests/test_octodns_provider_fastdns.py | 48 ++++- 2 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/fastdns-records-prev-other.json diff --git a/tests/fixtures/fastdns-records-prev-other.json b/tests/fixtures/fastdns-records-prev-other.json new file mode 100644 index 0000000..acae3ec --- /dev/null +++ b/tests/fixtures/fastdns-records-prev-other.json @@ -0,0 +1,166 @@ +{ + "recordsets": [ + { + "rdata": [ + "10 20 30 foo-1.other.tests.", + "12 20 30 foo-2.other.tests." + ], + "type": "SRV", + "name": "_srv._tcp.old.other.tests", + "ttl": 600 + }, + { + "rdata": [ + "10 20 30 foo-1.other.tests.", + "12 20 30 foo-2.other.tests." + ], + "type": "SRV", + "name": "_srv._tcp.old.other.tests", + "ttl": 600 + }, + { + "rdata": [ + "2601:644:500:e210:62f8:1dff:feb8:9471" + ], + "type": "AAAA", + "name": "aaaa.old.other.tests", + "ttl": 600 + }, + { + "rdata": [ + "ns1.akam.net.", + "ns2.akam.net.", + "ns3.akam.net.", + "ns4.akam.net." + ], + "type": "NS", + "name": "old.other.tests", + "ttl": 3600 + }, + { + "rdata": [ + "1.2.3.4", + "1.2.3.5" + ], + "type": "A", + "name": "old.other.tests", + "ttl": 300 + }, + { + "rdata": [ + "ns1.akam.net hostmaster.akamai.com 1489074932 86400 7200 604800 300" + ], + "type": "SOA", + "name": "other.tests", + "ttl": 3600 + }, + { + "rdata": [ + "1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49", + "1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73" + ], + "type": "SSHFP", + "name": "old.other.tests", + "ttl": 3600 + }, + { + "rdata": [ + "other.tests." + ], + "type": "CNAME", + "name": "old.cname.other.tests", + "ttl": 300 + }, + { + "rdata": [ + "other.tests." + ], + "type": "CNAME", + "name": "excluded.old.other.tests", + "ttl": 3600 + }, + { + "rdata": [ + "other.tests." + ], + "type": "CNAME", + "name": "included.old.other.tests", + "ttl": 3600 + }, + { + "rdata": [ + "10 smtp-4.other.tests.", + "20 smtp-2.other.tests.", + "30 smtp-3.other.tests.", + "40 smtp-1.other.tests." + ], + "type": "MX", + "name": "mx.old.other.tests", + "ttl": 300 + }, + { + "rdata": [ + "10 100 \"S\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" .", + "100 100 \"U\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" ." + ], + "type": "NAPTR", + "name": "naptr.old.other.tests", + "ttl": 600 + }, + { + "rdata": [ + "foo.bar.com." + ], + "type": "PTR", + "name": "ptr.old.other.tests", + "ttl": 300 + }, + { + "rdata": [ + "\"v=spf1 ip4:192.168.0.1/16-all\"" + ], + "type": "SPF", + "name": "spf.old.other.tests", + "ttl": 600 + }, + { + "rdata": [ + "ns1.other.tests.", + "ns2.other.tests." + ], + "type": "NS", + "name": "under.old.other.tests", + "ttl": 3600 + }, + { + "rdata": [ + "\"Bah bah black sheep\"", + "\"have you any wool.\"", + "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"" + ], + "type": "TXT", + "name": "txt.old.other.tests", + "ttl": 600 + }, + { + "rdata": [ + "2.2.3.7" + ], + "type": "A", + "name": "www.other.tests", + "ttl": 300 + }, + { + "rdata": [ + "2.2.3.6" + ], + "type": "A", + "name": "www.sub.old.other.tests", + "ttl": 300 + } + ], + "metadata": { + "totalElements": 16, + "showAll": true + } +} \ No newline at end of file diff --git a/tests/test_octodns_provider_fastdns.py b/tests/test_octodns_provider_fastdns.py index 4df36c0..5f503c7 100644 --- a/tests/test_octodns_provider_fastdns.py +++ b/tests/test_octodns_provider_fastdns.py @@ -88,8 +88,10 @@ class TestFastdnsProvider(TestCase): del provider._zone_records[zone.name] def test_apply(self): - provider = AkamaiProvider("test", "secret", "akam.com", "atok", "ctok") + provider = AkamaiProvider("test", "s", "akam.com", "atok", "ctok", + "cid", "gid") + # tests create update delete through previous state config json with requests_mock() as mock: with open('tests/fixtures/fastdns-records-prev.json') as fh: @@ -102,3 +104,47 @@ class TestFastdnsProvider(TestCase): changes = provider.apply(plan) self.assertEquals(29, changes) + + # Test against a zone that doesn't exist yet + with requests_mock() as mock: + with open('tests/fixtures/fastdns-records-prev-other.json') as fh: + mock.get(ANY, status_code=404) + + plan = provider.plan(self.expected) + mock.post(ANY, status_code=201) + mock.put(ANY, status_code=200) + mock.delete(ANY, status_code=204) + + changes = provider.apply(plan) + self.assertEquals(14, changes) + + # Test against a zone that doesn't exist yet, but gid not provided + with requests_mock() as mock: + with open('tests/fixtures/fastdns-records-prev-other.json') as fh: + mock.get(ANY, status_code=404) + provider = AkamaiProvider("test", "s", "akam.com", "atok", "ctok", + "cid") + plan = provider.plan(self.expected) + mock.post(ANY, status_code=201) + mock.put(ANY, status_code=200) + mock.delete(ANY, status_code=204) + + changes = provider.apply(plan) + self.assertEquals(14, changes) + + # Test against a zone that doesn't exist, but cid not provided + + with requests_mock() as mock: + mock.get(ANY, status_code=404) + + provider = AkamaiProvider("test", "s", "akam.com", "atok", "ctok") + plan = provider.plan(self.expected) + mock.post(ANY, status_code=201) + mock.put(ANY, status_code=200) + mock.delete(ANY, status_code=204) + + try: + changes = provider.apply(plan) + except NameError as e: + expected = "contractId not specified to create zone" + self.assertEquals(e.message, expected) From e3f6c4c711b314bc5ea0616327f4d3c4629bf758 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 18 Jul 2019 21:07:14 +0000 Subject: [PATCH 54/86] fastdns: changed sshfp to convert text to lowercase, since akamai changes to caps --- octodns/provider/fastdns.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 3dd52e7..e4a3c66 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -202,10 +202,7 @@ class AkamaiProvider(BaseProvider): response = self._dns_client.zone_recordset_get(name) self._zone_records[zone.name] = response.json()["recordsets"] - except KeyError: - return [] - - except AkamaiClientException: + except (AkamaiClientException, KeyError): return [] return self._zone_records[zone.name] @@ -405,7 +402,7 @@ class AkamaiProvider(BaseProvider): algorithm, fp_type, fingerprint = r.split(' ', 2) values.append({ 'algorithm': algorithm, - 'fingerprint': fingerprint, + 'fingerprint': fingerprint.lower(), 'fingerprint_type': fp_type }) From 5a741eb6007fed26d79d14288f7b95294d850a1b Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 18 Jul 2019 21:16:34 +0000 Subject: [PATCH 55/86] fastdns: fixed typo in test fixture --- tests/fixtures/fastdns-records-prev.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures/fastdns-records-prev.json b/tests/fixtures/fastdns-records-prev.json index 04ce274..b07c63f 100644 --- a/tests/fixtures/fastdns-records-prev.json +++ b/tests/fixtures/fastdns-records-prev.json @@ -65,7 +65,7 @@ }, { "rdata": [ - "unit.tests." + "unit.tests" ], "type": "CNAME", "name": "old.cname.unit.tests", From 79dd35f7a74d50c854e460f8dd11a3222f7fb384 Mon Sep 17 00:00:00 2001 From: Basir Date: Thu, 18 Jul 2019 17:56:18 -0400 Subject: [PATCH 56/86] fastdns: fixed an issue with handling spaces in SPF records --- octodns/provider/fastdns.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index e4a3c66..f6d3bd3 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -433,7 +433,6 @@ class AkamaiProvider(BaseProvider): _params_for_A = _params_for_multiple _params_for_AAAA = _params_for_multiple _params_for_NS = _params_for_multiple - _params_for_SPF = _params_for_multiple _params_for_CNAME = _params_for_single _params_for_PTR = _params_for_single @@ -466,6 +465,15 @@ class AkamaiProvider(BaseProvider): return rdata + def _params_for_SPF(self, values): + rdata = [] + + for r in values: + txt = "\"" + r.replace('\\;', ';') + "\"" + rdata.append(txt) + + return rdata + def _params_for_SRV(self, values): rdata = [] for r in values: From 76e799873e51fa78138a85efd194ec35594ce3ef Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 19 Jul 2019 14:32:51 -0400 Subject: [PATCH 57/86] Update octodns/provider/fastdns.py Co-Authored-By: Ross McFarland --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index f6d3bd3..0bc0aa2 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -import requests +from requests import Session from akamai.edgegrid import EdgeGridAuth from urlparse import urljoin from collections import defaultdict From d53eb7ad8e2400fe0266802617cf82ee76391305 Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 19 Jul 2019 14:32:58 -0400 Subject: [PATCH 58/86] Update octodns/provider/fastdns.py Co-Authored-By: Ross McFarland --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 0bc0aa2..45d6aa1 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -10,7 +10,7 @@ from akamai.edgegrid import EdgeGridAuth from urlparse import urljoin from collections import defaultdict -import logging +from logging import getLogger from ..record import Record from .base import BaseProvider From 91e7ac6d36c22f5570896296f1fac9632c78fa1d Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 19 Jul 2019 14:33:23 -0400 Subject: [PATCH 59/86] Update octodns/provider/fastdns.py Co-Authored-By: Ross McFarland --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 45d6aa1..d0d2009 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -181,7 +181,7 @@ class AkamaiProvider(BaseProvider): def __init__(self, id, client_secret, host, access_token, client_token, contract_id=None, gid=None, *args, **kwargs): - self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) + self.log = getLogger('AkamaiProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, ') super(AkamaiProvider, self).__init__(id, *args, **kwargs) From 41a5d78728ffc90d90f89d1ad966c4a84c3fbebe Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Fri, 19 Jul 2019 20:53:34 +0000 Subject: [PATCH 60/86] fastdns: refactored how exceptions are handled --- octodns/provider/fastdns.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index d0d2009..fb199d3 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -18,25 +18,18 @@ from .base import BaseProvider class AkamaiClientException(Exception): _errorMessages = { - 400: "400: Bad request", - 403: "403: Access is forbidden", - 404: "404: Resource not found", - 405: "405: Method not supported", - 406: "406: Not Acceptable", - 409: "409: Request not allowed due to conflict with current state", - 415: "415: Unsupported media type", - 422: "422: Request body contains an error preventing processing", - 500: "500: Internal server error" + 404: "404: Resource not found" } def __init__(self, resp): try: message = self._errorMessages[resp.status_code] - super(AkamaiClientException, self).__init__(message) - except: + except KeyError: resp.raise_for_status() + super(AkamaiClientException, self).__init__(message) + class AkamaiClient(object): ''' @@ -54,7 +47,7 @@ class AkamaiClient(object): self.base = "https://" + _host + "/config-dns/v2/" - sess = requests.Session() + sess = Session() sess.auth = EdgeGridAuth( client_token=_client_token, client_secret=_client_secret, From daa6fecc9b15eb123a3563de89ce5f4660be87f9 Mon Sep 17 00:00:00 2001 From: Kitten King Date: Sat, 20 Jul 2019 06:31:05 +0000 Subject: [PATCH 61/86] Fix Typos --- CHANGELOG.md | 2 +- octodns/provider/route53.py | 14 +++++++------- tests/test_octodns_provider_cloudflare.py | 6 +++--- tests/test_octodns_provider_digitalocean.py | 4 ++-- tests/test_octodns_provider_dnsimple.py | 4 ++-- tests/test_octodns_provider_dnsmadeeasy.py | 4 ++-- tests/test_octodns_provider_ns1.py | 4 ++-- tests/test_octodns_provider_powerdns.py | 8 ++++---- tests/test_octodns_record.py | 6 +++--- 9 files changed, 26 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e2b243..4f283ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,7 +112,7 @@ Adds an OVH provider. ## v0.8.6 - 2017-09-06 - CAA record type, -Misc fixes and improvments. +Misc fixes and improvements. * Azure TXT record fix * PowerDNS api support for https diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 4cf7c99..7b21573 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -528,13 +528,13 @@ def _mod_keyer(mod): # before all changes, followed by a "CREATE", internally in the AWS API. # Because of this, we order changes as follows: # - Delete any records that we wish to delete that are GEOS - # (because they are never targetted by anything) + # (because they are never targeted by anything) # - Delete any records that we wish to delete that are SECONDARY - # (because they are no longer targetted by GEOS) + # (because they are no longer targeted by GEOS) # - Delete any records that we wish to delete that are PRIMARY - # (because they are no longer targetted by SECONDARY) + # (because they are no longer targeted by SECONDARY) # - Delete any records that we wish to delete that are VALUES - # (because they are no longer targetted by PRIMARY) + # (because they are no longer targeted by PRIMARY) # - CREATE/UPSERT any records that are VALUES # (because they don't depend on other records) # - CREATE/UPSERT any records that are PRIMARY @@ -1028,7 +1028,7 @@ class Route53Provider(BaseProvider): .get('healthcheck', {}) \ .get('measure_latency', True) - def _health_check_equivilent(self, host, path, protocol, port, + def _health_check_equivalent(self, host, path, protocol, port, measure_latency, health_check, value=None): config = health_check['HealthCheckConfig'] @@ -1080,7 +1080,7 @@ class Route53Provider(BaseProvider): if not health_check['CallerReference'].startswith(expected_ref): # not match, ignore continue - if self._health_check_equivilent(healthcheck_host, + if self._health_check_equivalent(healthcheck_host, healthcheck_path, healthcheck_protocol, healthcheck_port, @@ -1237,7 +1237,7 @@ class Route53Provider(BaseProvider): health_check = self.health_checks[health_check_id] caller_ref = health_check['CallerReference'] if caller_ref.startswith(self.HEALTH_CHECK_VERSION): - if self._health_check_equivilent(healthcheck_host, + if self._health_check_equivalent(healthcheck_host, healthcheck_path, healthcheck_protocol, healthcheck_port, diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 25f2b58..5c6d503 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -102,7 +102,7 @@ class TestCloudflareProvider(TestCase): provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) - # Non-existant zone doesn't populate anything + # Non-existent zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=200, json=self.empty) @@ -110,7 +110,7 @@ class TestCloudflareProvider(TestCase): provider.populate(zone) self.assertEquals(set(), zone.records) - # re-populating the same non-existant zone uses cache and makes no + # re-populating the same non-existent zone uses cache and makes no # calls again = Zone('unit.tests.', []) provider.populate(again) @@ -173,7 +173,7 @@ class TestCloudflareProvider(TestCase): }, # zone create ] + [None] * 20 # individual record creates - # non-existant zone, create everything + # non-existent zone, create everything plan = provider.plan(self.expected) self.assertEquals(12, len(plan.changes)) self.assertEquals(12, provider.apply(plan)) diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index ddc6bc2..fce62b1 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -61,7 +61,7 @@ class TestDigitalOceanProvider(TestCase): provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) - # Non-existant zone doesn't populate anything + # Non-existent zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=404, text='{"id":"not_found","message":"The resource you ' @@ -153,7 +153,7 @@ class TestDigitalOceanProvider(TestCase): } } - # non-existant domain, create everything + # non-existent domain, create everything resp.json.side_effect = [ DigitalOceanClientNotFound, # no zone in populate DigitalOceanClientNotFound, # no domain during apply diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 896425e..b458ad7 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -58,7 +58,7 @@ class TestDnsimpleProvider(TestCase): provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) - # Non-existant zone doesn't populate anything + # Non-existent zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=404, text='{"message": "Domain `foo.bar` not found"}') @@ -122,7 +122,7 @@ class TestDnsimpleProvider(TestCase): resp.json = Mock() provider._client._request = Mock(return_value=resp) - # non-existant domain, create everything + # non-existent domain, create everything resp.json.side_effect = [ DnsimpleClientNotFound, # no zone in populate DnsimpleClientNotFound, # no domain during apply diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index 04cf0ee..bb663fb 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -87,7 +87,7 @@ class TestDnsMadeEasyProvider(TestCase): provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) - # Non-existant zone doesn't populate anything + # Non-existent zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=404, text='') @@ -130,7 +130,7 @@ class TestDnsMadeEasyProvider(TestCase): with open('tests/fixtures/dnsmadeeasy-domains.json') as fh: domains = json.load(fh) - # non-existant domain, create everything + # non-existent domain, create everything resp.json.side_effect = [ DnsMadeEasyClientNotFound, # no zone in populate DnsMadeEasyClientNotFound, # no domain during apply diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 8530b62..178ce53 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -191,7 +191,7 @@ class TestNs1Provider(TestCase): self.assertEquals(load_mock.side_effect, ctx.exception) self.assertEquals(('unit.tests',), load_mock.call_args[0]) - # Non-existant zone doesn't populate anything + # Non-existent zone doesn't populate anything load_mock.reset_mock() load_mock.side_effect = \ ResourceException('server error: zone not found') @@ -323,7 +323,7 @@ class TestNs1Provider(TestCase): provider.apply(plan) self.assertEquals(create_mock.side_effect, ctx.exception) - # non-existant zone, create + # non-existent zone, create load_mock.reset_mock() create_mock.reset_mock() load_mock.side_effect = \ diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 067dc74..2ce8519 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -41,7 +41,7 @@ with open('./tests/fixtures/powerdns-full-data.json') as fh: class TestPowerDnsProvider(TestCase): def test_provider(self): - provider = PowerDnsProvider('test', 'non.existant', 'api-key', + provider = PowerDnsProvider('test', 'non.existent', 'api-key', nameserver_values=['8.8.8.8.', '9.9.9.9.']) @@ -63,7 +63,7 @@ class TestPowerDnsProvider(TestCase): provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) - # Non-existant zone doesn't populate anything + # Non-existent zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=422, json={'error': "Could not find domain 'unit.tests.'"}) @@ -163,7 +163,7 @@ class TestPowerDnsProvider(TestCase): provider.apply(plan) def test_small_change(self): - provider = PowerDnsProvider('test', 'non.existant', 'api-key') + provider = PowerDnsProvider('test', 'non.existent', 'api-key') expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) @@ -203,7 +203,7 @@ class TestPowerDnsProvider(TestCase): def test_existing_nameservers(self): ns_values = ['8.8.8.8.', '9.9.9.9.'] - provider = PowerDnsProvider('test', 'non.existant', 'api-key', + provider = PowerDnsProvider('test', 'non.existent', 'api-key', nameserver_values=ns_values) expected = Zone('unit.tests.', []) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 2b11364..d6ed2d9 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -2845,7 +2845,7 @@ class TestDynamicRecords(TestCase): self.assertEquals(['rule 1 invalid pool "[]"'], ctx.exception.reasons) - # rule references non-existant pool + # rule references non-existent pool a_data = { 'dynamic': { 'pools': { @@ -2864,7 +2864,7 @@ class TestDynamicRecords(TestCase): }, 'rules': [{ 'geos': ['NA-US-CA'], - 'pool': 'non-existant', + 'pool': 'non-existent', }, { 'pool': 'one', }], @@ -2878,7 +2878,7 @@ class TestDynamicRecords(TestCase): } with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, 'bad', a_data) - self.assertEquals(["rule 1 undefined pool \"non-existant\""], + self.assertEquals(["rule 1 undefined pool \"non-existent\""], ctx.exception.reasons) # rule with invalid geos From 57c96cf40f7b2996b2d779ffec534d175189ebfc Mon Sep 17 00:00:00 2001 From: Shnayder Anton Date: Mon, 22 Jul 2019 13:18:45 +0300 Subject: [PATCH 62/86] Add Selectel as OctoDNS provider --- README.md | 1 + octodns/provider/selectel.py | 303 ++++++++++++++++++ tests/test_octodns_provider_selectel.py | 401 ++++++++++++++++++++++++ 3 files changed, 705 insertions(+) create mode 100644 octodns/provider/selectel.py create mode 100644 tests/test_octodns_provider_selectel.py diff --git a/README.md b/README.md index aa9950e..737ba9b 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,7 @@ The above command pulled the existing data out of Route53 and placed the results | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | | | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Both | CNAME health checks don't support a Host header | +| [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | | [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only | diff --git a/octodns/provider/selectel.py b/octodns/provider/selectel.py new file mode 100644 index 0000000..0c576a4 --- /dev/null +++ b/octodns/provider/selectel.py @@ -0,0 +1,303 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from collections import defaultdict + +from logging import getLogger + +from requests import Session + +from ..record import Record, Update +from .base import BaseProvider + + +class SelectelAuthenticationRequired(Exception): + def __init__(self, msg): + message = 'Authorization failed. Invalid or empty token.' + super(SelectelAuthenticationRequired, self).__init__(message) + + +class SelectelProvider(BaseProvider): + SUPPORTS_GEO = False + + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'TXT', 'SPF', 'SRV')) + + MIN_TTL = 60 + + LIMIT = 50 + + API_URL = 'https://api.selectel.ru/domains/v1' + + def __init__(self, id, token, *args, **kwargs): + self.log = getLogger('SelectelProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s', id) + super(SelectelProvider, self).__init__(id, *args, **kwargs) + + self._sess = Session() + self._sess.headers.update({ + 'X-Token': token, + 'Content-Type': 'application/json', + }) + self._zone_records = {} + self._domain_list = self.domain_list() + self._zones = None + + def _request(self, method, path, params=None, data=None): + self.log.debug('_request: method=%s, path=%s', method, path) + + url = '{}{}'.format(self.API_URL, path) + resp = self._sess.request(method, url, params=params, json=data) + + self.log.debug('_request: status=%s', resp.status_code) + if resp.status_code == 401: + raise SelectelAuthenticationRequired(resp.text) + elif resp.status_code == 404: + return {} + resp.raise_for_status() + if method == 'DELETE': + return {} + return resp.json() + + def _get_total_count(self, path): + url = '{}{}'.format(self.API_URL, path) + resp = self._sess.request('HEAD', url) + return int(resp.headers['X-Total-Count']) + + def _request_with_pagination(self, path, total_count): + result = [] + for offset in range(0, total_count, self.LIMIT): + result += self._request('GET', path, + params={'limit': self.LIMIT, + 'offset': offset}) + return result + + def _include_change(self, change): + if isinstance(change, Update): + existing = change.existing.data + new = change.new.data + new['ttl'] = max(self.MIN_TTL, new['ttl']) + if new == existing: + return False + return True + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + zone_name = desired.name[:-1] + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name).lower())(zone_name, + change) + + def _apply_create(self, zone_name, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self.create_record(zone_name, params) + + def _apply_update(self, zone_name, change): + self._apply_delete(zone_name, change) + self._apply_create(zone_name, change) + + def _apply_delete(self, zone_name, change): + existing = change.existing + self.delete_record(zone_name, existing._type, existing.name) + + def _params_for_multiple(self, record): + for value in record.values: + yield { + 'content': value, + 'name': record.fqdn, + 'ttl': max(self.MIN_TTL, record.ttl), + 'type': record._type, + } + + def _params_for_single(self, record): + yield { + 'content': record.value, + 'name': record.fqdn, + 'ttl': max(self.MIN_TTL, record.ttl), + 'type': record._type + } + + def _params_for_MX(self, record): + for value in record.values: + yield { + 'content': value.exchange, + 'name': record.fqdn, + 'ttl': max(self.MIN_TTL, record.ttl), + 'type': record._type, + 'priority': value.preference + } + + def _params_for_SRV(self, record): + for value in record.values: + yield { + 'name': record.fqdn, + 'target': value.target, + 'ttl': max(self.MIN_TTL, record.ttl), + 'type': record._type, + 'port': value.port, + 'weight': value.weight, + 'priority': value.priority + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + _params_for_NS = _params_for_multiple + _params_for_TXT = _params_for_multiple + _params_for_SPF = _params_for_multiple + + _params_for_CNAME = _params_for_single + + def _data_for_A(self, _type, records): + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': [r['content'] for r in records], + } + + _data_for_AAAA = _data_for_A + + def _data_for_NS(self, _type, records): + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': ['{}.'.format(r['content']) for r in records], + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + values.append({ + 'preference': record['priority'], + 'exchange': '{}.'.format(record['content']), + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + + def _data_for_CNAME(self, _type, records): + only = records[0] + return { + 'ttl': only['ttl'], + 'type': _type, + 'value': '{}.'.format(only['content']) + } + + def _data_for_TXT(self, _type, records): + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': [r['content'] for r in records], + } + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + values.append({ + 'priority': record['priority'], + 'weight': record['weight'], + 'port': record['port'], + 'target': '{}.'.format(record['target']), + }) + + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values, + } + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', + zone.name, target, lenient) + before = len(zone.records) + records = self.zone_records(zone) + if records: + values = defaultdict(lambda: defaultdict(list)) + for record in records: + name = zone.hostname_from_fqdn(record['name']) + _type = record['type'] + if _type in self.SUPPORTS: + values[name][record['type']].append(record) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + data = data_for(_type, records) + record = Record.new(zone, name, data, source=self, + lenient=lenient) + zone.add_record(record) + self.log.info('populate: found %s records', + len(zone.records) - before) + + def domain_list(self): + path = '/' + domains = {} + domains_list = [] + + total_count = self._get_total_count(path) + domains_list = self._request_with_pagination(path, total_count) + + for domain in domains_list: + domains[domain['name']] = domain + return domains + + def zone_records(self, zone): + path = '/{}/records/'.format(zone.name[:-1]) + zone_records = [] + + total_count = self._get_total_count(path) + zone_records = self._request_with_pagination(path, total_count) + + self._zone_records[zone.name] = zone_records + return self._zone_records[zone.name] + + def create_domain(self, name, zone=""): + path = '/' + + data = { + 'name': name, + 'bind_zone': zone, + } + + resp = self._request('POST', path, data=data) + self._domain_list[name] = resp + return resp + + def create_record(self, zone_name, data): + self.log.debug('Create record. Zone: %s, data %s', zone_name, data) + if zone_name in self._domain_list.keys(): + domain_id = self._domain_list[zone_name]['id'] + else: + domain_id = self.create_domain(zone_name)['id'] + + path = '/{}/records/'.format(domain_id) + return self._request('POST', path, data=data) + + def delete_record(self, domain, _type, zone): + self.log.debug('Delete record. Domain: %s, Type: %s', domain, _type) + + domain_id = self._domain_list[domain]['id'] + records = self._zone_records.get('{}.'.format(domain), False) + if not records: + path = '/{}/records/'.format(domain_id) + records = self._request('GET', path) + + for record in records: + full_domain = domain + if zone: + full_domain = '{}{}'.format(zone, domain) + if record['type'] == _type and record['name'] == full_domain: + path = '/{}/records/{}'.format(domain_id, record['id']) + return self._request('DELETE', path) + + self.log.debug('Delete record failed (Record not found)') diff --git a/tests/test_octodns_provider_selectel.py b/tests/test_octodns_provider_selectel.py new file mode 100644 index 0000000..a2ba39e --- /dev/null +++ b/tests/test_octodns_provider_selectel.py @@ -0,0 +1,401 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from unittest import TestCase + +import requests_mock + +from octodns.provider.selectel import SelectelProvider +from octodns.record import Record, Update +from octodns.zone import Zone + + +class TestSelectelProvider(TestCase): + API_URL = 'https://api.selectel.ru/domains/v1' + + api_record = [] + + zone = Zone('unit.tests.', []) + expected = set() + + domain = [{"name": "unit.tests", "id": 100000}] + + # A, subdomain='' + api_record.append({ + 'type': 'A', + 'ttl': 100, + 'content': '1.2.3.4', + 'name': 'unit.tests', + 'id': 1 + }) + expected.add(Record.new(zone, '', { + 'ttl': 100, + 'type': 'A', + 'value': '1.2.3.4', + })) + + # A, subdomain='sub' + api_record.append({ + 'type': 'A', + 'ttl': 200, + 'content': '1.2.3.4', + 'name': 'sub.unit.tests', + 'id': 2 + }) + expected.add(Record.new(zone, 'sub', { + 'ttl': 200, + 'type': 'A', + 'value': '1.2.3.4', + })) + + # CNAME + api_record.append({ + 'type': 'CNAME', + 'ttl': 300, + 'content': 'unit.tests', + 'name': 'www2.unit.tests', + 'id': 3 + }) + expected.add(Record.new(zone, 'www2', { + 'ttl': 300, + 'type': 'CNAME', + 'value': 'unit.tests.', + })) + + # MX + api_record.append({ + 'type': 'MX', + 'ttl': 400, + 'content': 'mx1.unit.tests', + 'priority': 10, + 'name': 'unit.tests', + 'id': 4 + }) + expected.add(Record.new(zone, '', { + 'ttl': 400, + 'type': 'MX', + 'values': [{ + 'preference': 10, + 'exchange': 'mx1.unit.tests.', + }] + })) + + # NS + api_record.append({ + 'type': 'NS', + 'ttl': 600, + 'content': 'ns1.unit.tests', + 'name': 'unit.tests.', + 'id': 6 + }) + api_record.append({ + 'type': 'NS', + 'ttl': 600, + 'content': 'ns2.unit.tests', + 'name': 'unit.tests', + 'id': 7 + }) + expected.add(Record.new(zone, '', { + 'ttl': 600, + 'type': 'NS', + 'values': ['ns1.unit.tests.', 'ns2.unit.tests.'], + })) + + # NS with sub + api_record.append({ + 'type': 'NS', + 'ttl': 700, + 'content': 'ns3.unit.tests', + 'name': 'www3.unit.tests', + 'id': 8 + }) + api_record.append({ + 'type': 'NS', + 'ttl': 700, + 'content': 'ns4.unit.tests', + 'name': 'www3.unit.tests', + 'id': 9 + }) + expected.add(Record.new(zone, 'www3', { + 'ttl': 700, + 'type': 'NS', + 'values': ['ns3.unit.tests.', 'ns4.unit.tests.'], + })) + + # SRV + api_record.append({ + 'type': 'SRV', + 'ttl': 800, + 'target': 'foo-1.unit.tests', + 'weight': 20, + 'priority': 10, + 'port': 30, + 'id': 10, + 'name': '_srv._tcp.unit.tests' + }) + api_record.append({ + 'type': 'SRV', + 'ttl': 800, + 'target': 'foo-2.unit.tests', + 'name': '_srv._tcp.unit.tests', + 'weight': 50, + 'priority': 40, + 'port': 60, + 'id': 11 + }) + expected.add(Record.new(zone, '_srv._tcp', { + 'ttl': 800, + 'type': 'SRV', + 'values': [{ + 'priority': 10, + 'weight': 20, + 'port': 30, + 'target': 'foo-1.unit.tests.', + }, { + 'priority': 40, + 'weight': 50, + 'port': 60, + 'target': 'foo-2.unit.tests.', + }] + })) + + # AAAA + aaaa_record = { + 'type': 'AAAA', + 'ttl': 200, + 'content': '1:1ec:1::1', + 'name': 'unit.tests', + 'id': 15 + } + api_record.append(aaaa_record) + expected.add(Record.new(zone, '', { + 'ttl': 200, + 'type': 'AAAA', + 'value': '1:1ec:1::1', + })) + + # TXT + api_record.append({ + 'type': 'TXT', + 'ttl': 300, + 'content': 'little text', + 'name': 'text.unit.tests', + 'id': 16 + }) + expected.add(Record.new(zone, 'text', { + 'ttl': 200, + 'type': 'TXT', + 'value': 'little text', + })) + + @requests_mock.Mocker() + def test_populate(self, fake_http): + zone = Zone('unit.tests.', []) + fake_http.get('{}/unit.tests/records/'.format(self.API_URL), + json=self.api_record) + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.head('{}/unit.tests/records/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.api_record))}) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + + provider = SelectelProvider(123, 'secret_token') + provider.populate(zone) + + self.assertEquals(self.expected, zone.records) + + @requests_mock.Mocker() + def test_populate_invalid_record(self, fake_http): + more_record = self.api_record + more_record.append({"name": "unit.tests", + "id": 100001, + "content": "support.unit.tests.", + "ttl": 300, "ns": "ns1.unit.tests", + "type": "SOA", + "email": "support@unit.tests"}) + + zone = Zone('unit.tests.', []) + fake_http.get('{}/unit.tests/records/'.format(self.API_URL), + json=more_record) + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.head('{}/unit.tests/records/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.api_record))}) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + + zone.add_record(Record.new(self.zone, 'unsup', { + 'ttl': 200, + 'type': 'NAPTR', + 'value': { + 'order': 40, + 'preference': 70, + 'flags': 'U', + 'service': 'SIP+D2U', + 'regexp': '!^.*$!sip:info@bar.example.com!', + 'replacement': '.', + } + })) + + provider = SelectelProvider(123, 'secret_token') + provider.populate(zone) + + self.assertNotEqual(self.expected, zone.records) + + @requests_mock.Mocker() + def test_apply(self, fake_http): + + fake_http.get('{}/unit.tests/records/'.format(self.API_URL), + json=list()) + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.head('{}/unit.tests/records/'.format(self.API_URL), + headers={'X-Total-Count': '0'}) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + fake_http.post('{}/100000/records/'.format(self.API_URL), json=list()) + + provider = SelectelProvider(123, 'test_token') + + zone = Zone('unit.tests.', []) + + for record in self.expected: + zone.add_record(record) + + plan = provider.plan(zone) + self.assertEquals(8, len(plan.changes)) + self.assertEquals(8, provider.apply(plan)) + + @requests_mock.Mocker() + def test_domain_list(self, fake_http): + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + + expected = {'unit.tests': self.domain[0]} + provider = SelectelProvider(123, 'test_token') + + result = provider.domain_list() + self.assertEquals(result, expected) + + @requests_mock.Mocker() + def test_authentication_fail(self, fake_http): + fake_http.get('{}/'.format(self.API_URL), status_code=401) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + + with self.assertRaises(Exception) as ctx: + SelectelProvider(123, 'fail_token') + self.assertEquals(ctx.exception.message, + 'Authorization failed. Invalid or empty token.') + + @requests_mock.Mocker() + def test_not_exist_domain(self, fake_http): + fake_http.get('{}/'.format(self.API_URL), status_code=404, json='') + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + + fake_http.post('{}/'.format(self.API_URL), + json={"name": "unit.tests", + "create_date": 1507154178, + "id": 100000}) + fake_http.get('{}/unit.tests/records/'.format(self.API_URL), + json=list()) + fake_http.head('{}/unit.tests/records/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.api_record))}) + fake_http.post('{}/100000/records/'.format(self.API_URL), + json=list()) + + provider = SelectelProvider(123, 'test_token') + + zone = Zone('unit.tests.', []) + + for record in self.expected: + zone.add_record(record) + + plan = provider.plan(zone) + self.assertEquals(8, len(plan.changes)) + self.assertEquals(8, provider.apply(plan)) + + @requests_mock.Mocker() + def test_delete_no_exist_record(self, fake_http): + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.get('{}/100000/records/'.format(self.API_URL), json=list()) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + fake_http.head('{}/unit.tests/records/'.format(self.API_URL), + headers={'X-Total-Count': '0'}) + + provider = SelectelProvider(123, 'test_token') + + zone = Zone('unit.tests.', []) + + provider.delete_record('unit.tests', 'NS', zone) + + @requests_mock.Mocker() + def test_change_record(self, fake_http): + exist_record = [self.aaaa_record, + {"content": "6.6.5.7", + "ttl": 100, + "type": "A", + "id": 100001, + "name": "delete.unit.tests"}, + {"content": "9.8.2.1", + "ttl": 100, + "type": "A", + "id": 100002, + "name": "unit.tests"}] # exist + fake_http.get('{}/unit.tests/records/'.format(self.API_URL), + json=exist_record) + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.get('{}/100000/records/'.format(self.API_URL), + json=exist_record) + fake_http.head('{}/unit.tests/records/'.format(self.API_URL), + headers={'X-Total-Count': str(len(exist_record))}) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + fake_http.head('{}/100000/records/'.format(self.API_URL), + headers={'X-Total-Count': str(len(exist_record))}) + fake_http.post('{}/100000/records/'.format(self.API_URL), + json=list()) + fake_http.delete('{}/100000/records/100001'.format(self.API_URL), + text="") + fake_http.delete('{}/100000/records/100002'.format(self.API_URL), + text="") + + provider = SelectelProvider(123, 'test_token') + + zone = Zone('unit.tests.', []) + + for record in self.expected: + zone.add_record(record) + + plan = provider.plan(zone) + self.assertEquals(8, len(plan.changes)) + self.assertEquals(8, provider.apply(plan)) + + @requests_mock.Mocker() + def test_include_change_returns_false(self, fake_http): + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + provider = SelectelProvider(123, 'test_token') + zone = Zone('unit.tests.', []) + + exist_record = Record.new(zone, '', { + 'ttl': 60, + 'type': 'A', + 'values': ['1.1.1.1', '2.2.2.2'] + }) + new = Record.new(zone, '', { + 'ttl': 10, + 'type': 'A', + 'values': ['1.1.1.1', '2.2.2.2'] + }) + change = Update(exist_record, new) + + include_change = provider._include_change(change) + + self.assertFalse(include_change) From a85d4e25eda59d59ebc320b1a6597ac9d2848f1a Mon Sep 17 00:00:00 2001 From: Basir Date: Mon, 29 Jul 2019 13:03:23 -0400 Subject: [PATCH 63/86] fastdns: refactored AkamaiClientNotFound exception class --- octodns/provider/fastdns.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index fb199d3..bf19953 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -15,19 +15,10 @@ from ..record import Record from .base import BaseProvider -class AkamaiClientException(Exception): - - _errorMessages = { - 404: "404: Resource not found" - } +class AkamaiClientNotFound(Exception): def __init__(self, resp): - try: - message = self._errorMessages[resp.status_code] - - except KeyError: - resp.raise_for_status() - + message = "404: Resource not found" super(AkamaiClientException, self).__init__(message) From 93e1e0939d5cb3d8218bb11cc3fd62cfda9f83ca Mon Sep 17 00:00:00 2001 From: Basir Date: Mon, 29 Jul 2019 13:07:04 -0400 Subject: [PATCH 64/86] fastdns: fixed how exceptions are handled in _request --- octodns/provider/fastdns.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index bf19953..6e9a00d 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -51,8 +51,9 @@ class AkamaiClient(object): url = urljoin(self.base, path) resp = self._sess.request(method, url, params=params, json=data) - if resp.status_code > 299: - raise AkamaiClientException(resp) + if resp.status_code == 404: + raise AkamaiClientNotFound(resp) + resp.raise_for_status() return resp From 55210ebb05cd4e9077fa324dd2619de83e6cb046 Mon Sep 17 00:00:00 2001 From: Basir Date: Mon, 29 Jul 2019 13:09:18 -0400 Subject: [PATCH 65/86] fastdns: fixed naming in AkamaiClient init --- octodns/provider/fastdns.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 6e9a00d..ab80882 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -34,15 +34,15 @@ class AkamaiClient(object): ''' - def __init__(self, _client_secret, _host, _access_token, _client_token): + def __init__(self, client_secret, host, access_token, client_token): - self.base = "https://" + _host + "/config-dns/v2/" + self.base = "https://" + host + "/config-dns/v2/" sess = Session() sess.auth = EdgeGridAuth( - client_token=_client_token, - client_secret=_client_secret, - access_token=_access_token + client_token=client_token, + client_secret=client_secret, + access_token=access_token ) self._sess = sess From b2bd91a998aa5a3e7c046d9a32eea3b505071109 Mon Sep 17 00:00:00 2001 From: Basir Date: Mon, 29 Jul 2019 13:25:26 -0400 Subject: [PATCH 66/86] fastdns: resolved naming mismatch for exception class --- octodns/provider/fastdns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index ab80882..f851303 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -19,7 +19,7 @@ class AkamaiClientNotFound(Exception): def __init__(self, resp): message = "404: Resource not found" - super(AkamaiClientException, self).__init__(message) + super(AkamaiClientNotFound, self).__init__(message) class AkamaiClient(object): @@ -187,7 +187,7 @@ class AkamaiProvider(BaseProvider): response = self._dns_client.zone_recordset_get(name) self._zone_records[zone.name] = response.json()["recordsets"] - except (AkamaiClientException, KeyError): + except (AkamaiClientNotFound, KeyError): return [] return self._zone_records[zone.name] @@ -231,7 +231,7 @@ class AkamaiProvider(BaseProvider): try: self._dns_client.zone_get(zone_name) - except AkamaiClientException: + except AkamaiClientNotFound: self.log.info("zone not found, creating zone") params = self._build_zone_config(zone_name) self._dns_client.zone_create(self._contractId, params, self._gid) From 76a9d358e82bb65a3a1cad83cabd5f79325c3b2f Mon Sep 17 00:00:00 2001 From: Shnayder Anton <41984158+TruePack@users.noreply.github.com> Date: Thu, 1 Aug 2019 12:53:25 +0300 Subject: [PATCH 67/86] Add more explicit constant name for pagination Co-Authored-By: Ross McFarland --- octodns/provider/selectel.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/selectel.py b/octodns/provider/selectel.py index 0c576a4..bfbbb8f 100644 --- a/octodns/provider/selectel.py +++ b/octodns/provider/selectel.py @@ -28,7 +28,7 @@ class SelectelProvider(BaseProvider): MIN_TTL = 60 - LIMIT = 50 + PAGINATION_LIMIT = 50 API_URL = 'https://api.selectel.ru/domains/v1' @@ -69,9 +69,9 @@ class SelectelProvider(BaseProvider): def _request_with_pagination(self, path, total_count): result = [] - for offset in range(0, total_count, self.LIMIT): + for offset in range(0, total_count, self.PAGINATION_LIMIT): result += self._request('GET', path, - params={'limit': self.LIMIT, + params={'limit': self.PAGINATION_LIMIT, 'offset': offset}) return result From 74a87d4933e021f056a38382488881731504c2c2 Mon Sep 17 00:00:00 2001 From: Anton Shnayder Date: Thu, 1 Aug 2019 13:33:53 +0300 Subject: [PATCH 68/86] Add debug for _include_changes --- octodns/provider/selectel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/octodns/provider/selectel.py b/octodns/provider/selectel.py index bfbbb8f..072b8cf 100644 --- a/octodns/provider/selectel.py +++ b/octodns/provider/selectel.py @@ -81,6 +81,8 @@ class SelectelProvider(BaseProvider): new = change.new.data new['ttl'] = max(self.MIN_TTL, new['ttl']) if new == existing: + self.log.debug('_include_changes: new=%s, found existing=%s', + new, existing) return False return True From c0315d25baca80e967a0e14af2bbd267adb98e09 Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Mon, 29 Jul 2019 23:12:27 -0700 Subject: [PATCH 69/86] add Constellix Provider --- README.md | 1 + octodns/provider/constellix.py | 447 ++++++++++++++++ tests/fixtures/constellix-domains.json | 28 + tests/fixtures/constellix-records.json | 598 ++++++++++++++++++++++ tests/test_octodns_provider_constellix.py | 218 ++++++++ 5 files changed, 1292 insertions(+) create mode 100644 octodns/provider/constellix.py create mode 100644 tests/fixtures/constellix-domains.json create mode 100644 tests/fixtures/constellix-records.json create mode 100644 tests/test_octodns_provider_constellix.py diff --git a/README.md b/README.md index 737ba9b..75a006c 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,7 @@ The above command pulled the existing data out of Route53 and placed the results |--|--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, MX, NS, SPF, SRV, TXT | No | CAA tags restricted | +| [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | | [DnsMadeEasyProvider](/octodns/provider/dnsmadeeasy.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | | All | No | CAA tags restricted | diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py new file mode 100644 index 0000000..939284d --- /dev/null +++ b/octodns/provider/constellix.py @@ -0,0 +1,447 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from collections import defaultdict +from requests import Session +from base64 import b64encode +from ipaddress import ip_address +import hashlib +import hmac +import logging +import time + +from ..record import Record +from .base import BaseProvider + + +class ConstellixClientException(Exception): + pass + + +class ConstellixClientBadRequest(ConstellixClientException): + + def __init__(self, resp): + errors = resp.json()['errors'] + super(ConstellixClientBadRequest, self).__init__( + '\n - {}'.format('\n - '.join(errors))) + + +class ConstellixClientUnauthorized(ConstellixClientException): + + def __init__(self): + super(ConstellixClientUnauthorized, self).__init__('Unauthorized') + + +class ConstellixClientNotFound(ConstellixClientException): + + def __init__(self): + super(ConstellixClientNotFound, self).__init__('Not Found') + + +class ConstellixClient(object): + BASE = 'https://api.dns.constellix.com/v1/domains' + + def __init__(self, api_key, secret_key, ratelimit_delay=0.0): + self.api_key = api_key + self.secret_key = secret_key + self.ratelimit_delay = ratelimit_delay + self._sess = Session() + self._sess.headers.update({'x-cnsdns-apiKey': self.api_key}) + self._domains = None + + def _current_time(self): + return str(int(time.time() * 1000)) + + def _hmac_hash(self, now): + return hmac.new(self.secret_key.encode('utf-8'), now.encode('utf-8'), + digestmod=hashlib.sha1).digest() + + def _request(self, method, path, params=None, data=None): + now = self._current_time() + hmac_hash = self._hmac_hash(now) + + headers = { + 'x-cnsdns-hmac': b64encode(hmac_hash), + 'x-cnsdns-requestDate': now + } + + url = '{}{}'.format(self.BASE, path) + resp = self._sess.request(method, url, headers=headers, + params=params, json=data) + if resp.status_code == 400: + raise ConstellixClientBadRequest(resp) + if resp.status_code == 401: + raise ConstellixClientUnauthorized() + if resp.status_code == 404: + raise ConstellixClientNotFound() + resp.raise_for_status() + time.sleep(self.ratelimit_delay) + return resp + + @property + def domains(self): + if self._domains is None: + zones = [] + + resp = self._request('GET', '/').json() + zones += resp + + self._domains = {'{}.'.format(z['name']): z['id'] for z in zones} + + return self._domains + + def domain(self, name): + path = '/{}'.format(self.domains.get(name)) + return self._request('GET', path).json() + + def domain_create(self, name): + self._request('POST', '/', data={'names': [name]}) + + def _absolutize_value(self, value, zone_name): + if value == '': + value = zone_name + elif not value.endswith('.'): + value = '{}.{}'.format(value, zone_name) + + return value + + def records(self, zone_name): + zone_id = self.domains.get(zone_name, False) + path = '/{}/records'.format(zone_id) + + resp = self._request('GET', path).json() + for record in resp: + # change ANAME records to ALIAS + if record['type'] == 'ANAME': + record['type'] = 'ALIAS' + + # change relative values to absolute + value = record['value'] + if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'SRV']: + if isinstance(value, unicode): + record['value'] = self._absolutize_value(value, + zone_name) + if isinstance(value, list): + for v in value: + v['value'] = self._absolutize_value(v['value'], + zone_name) + + # compress IPv6 addresses + if record['type'] == 'AAAA': + for i, v in enumerate(value): + value[i] = str(ip_address(v)) + + return resp + + def record_create(self, zone_name, record_type, params): + # change ALIAS records to ANAME + if record_type == 'ALIAS': + record_type = 'ANAME' + + zone_id = self.domains.get(zone_name, False) + path = '/{}/records/{}'.format(zone_id, record_type) + + self._request('POST', path, data=params) + + def record_delete(self, zone_name, record_type, record_id): + zone_id = self.domains.get(zone_name, False) + path = '/{}/records/{}/{}'.format(zone_id, record_type, record_id) + self._request('DELETE', path) + + +class ConstellixProvider(BaseProvider): + ''' + Constellix DNS provider + + constellix: + class: octodns.provider.constellix.ConstellixProvider + # Your Contellix api key (required) + api_key: env/CONSTELLIX_API_KEY + # Your Constellix secret key (required) + secret_key: env/CONSTELLIX_SECRET_KEY + # Amount of time to wait between requests to avoid + # ratelimit (optional) + ratelimit_delay: 0.0 + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', + 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) + + def __init__(self, id, api_key, secret_key, ratelimit_delay=0.0, + *args, **kwargs): + self.log = logging.getLogger('ConstellixProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, api_key=***, secret_key=***', id) + super(ConstellixProvider, self).__init__(id, *args, **kwargs) + self._client = ConstellixClient(api_key, secret_key, ratelimit_delay) + self._zone_records = {} + + def _data_for_multiple(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': record['value'] + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + def _data_for_CAA(self, _type, records): + values = [] + record = records[0] + for value in record['value']: + values.append({ + 'flags': value['flag'], + 'tag': value['tag'], + 'value': value['data'] + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_NS(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': [value['value'] for value in record['value']] + } + + def _data_for_ALIAS(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': record['value'][0]['value'] + } + + _data_for_PTR = _data_for_ALIAS + + def _data_for_TXT(self, _type, records): + values = [value['value'].replace(';', '\\;') + for value in records[0]['value']] + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + _data_for_SPF = _data_for_TXT + + def _data_for_MX(self, _type, records): + values = [] + record = records[0] + for value in record['value']: + values.append({ + 'preference': value['level'], + 'exchange': value['value'] + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_single(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': record['value'] + } + + _data_for_CNAME = _data_for_single + + def _data_for_SRV(self, _type, records): + values = [] + record = records[0] + for value in record['value']: + values.append({ + 'port': value['port'], + 'priority': value['priority'], + 'target': value['value'], + 'weight': value['weight'] + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + def zone_records(self, zone): + if zone.name not in self._zone_records: + try: + self._zone_records[zone.name] = \ + self._client.records(zone.name) + except ConstellixClientNotFound: + return [] + + return self._zone_records[zone.name] + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + _type = record['type'] + if _type not in self.SUPPORTS: + self.log.warning('populate: skipping unsupported %s record', + _type) + continue + values[record['name']][record['type']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + + exists = zone.name in self._zone_records + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists + + def _params_for_multiple(self, record): + yield { + 'name': record.name, + 'ttl': record.ttl, + 'roundRobin': [{ + 'value': value + } for value in record.values] + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + + # An A record with this name must exist in this domain for + # this NS record to be valid. Need to handle checking if + # there is an A record before creating NS + _params_for_NS = _params_for_multiple + + def _params_for_single(self, record): + yield { + 'name': record.name, + 'ttl': record.ttl, + 'host': record.value, + } + + _params_for_CNAME = _params_for_single + + def _params_for_ALIAS(self, record): + yield { + 'name': record.name, + 'ttl': record.ttl, + 'roundRobin': [{ + 'value': record.value, + 'disableFlag': False + }] + } + + _params_for_PTR = _params_for_ALIAS + + def _params_for_MX(self, record): + values = [] + for value in record.values: + values.append({ + 'value': value.exchange, + 'level': value.preference + }) + yield { + 'value': value.exchange, + 'name': record.name, + 'ttl': record.ttl, + 'roundRobin': values + } + + def _params_for_SRV(self, record): + values = [] + for value in record.values: + values.append({ + 'value': value.target, + 'priority': value.priority, + 'weight': value.weight, + 'port': value.port + }) + for value in record.values: + yield { + 'name': record.name, + 'ttl': record.ttl, + 'roundRobin': values + } + + def _params_for_TXT(self, record): + # Constellix does not want values escaped + values = [] + for value in record.chunked_values: + values.append({ + 'value': value.replace('\\;', ';') + }) + yield { + 'name': record.name, + 'ttl': record.ttl, + 'roundRobin': values + } + + _params_for_SPF = _params_for_TXT + + def _params_for_CAA(self, record): + values = [] + for value in record.values: + values.append({ + 'tag': value.tag, + 'data': value.value, + 'flag': value.flags, + }) + yield { + 'name': record.name, + 'ttl': record.ttl, + 'roundRobin': values + } + + def _apply_Create(self, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self._client.record_create(new.zone.name, new._type, params) + + def _apply_Update(self, change): + self._apply_Delete(change) + self._apply_Create(change) + + def _apply_Delete(self, change): + existing = change.existing + zone = existing.zone + for record in self.zone_records(zone): + if existing.name == record['name'] and \ + existing._type == record['type']: + self._client.record_delete(zone.name, record['type'], + record['id']) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + try: + self._client.domain(desired.name) + except ConstellixClientNotFound: + self.log.debug('_apply: no matching zone, creating domain') + self._client.domain_create(desired.name[:-1]) + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) diff --git a/tests/fixtures/constellix-domains.json b/tests/fixtures/constellix-domains.json new file mode 100644 index 0000000..4b6392d --- /dev/null +++ b/tests/fixtures/constellix-domains.json @@ -0,0 +1,28 @@ +[{ + "id": 123123, + "name": "unit.tests", + "soa": { + "primaryNameserver": "ns11.constellix.com.", + "email": "dns.constellix.com.", + "ttl": 86400, + "serial": 2015010102, + "refresh": 43200, + "retry": 3600, + "expire": 1209600, + "negCache": 180 + }, + "createdTs": "2019-08-07T03:36:02Z", + "modifiedTs": "2019-08-07T03:36:02Z", + "typeId": 1, + "domainTags": [], + "folder": null, + "hasGtdRegions": false, + "hasGeoIP": false, + "nameserverGroup": 1, + "nameservers": ["ns11.constellix.com.", "ns21.constellix.com.", "ns31.constellix.com.", "ns41.constellix.net.", "ns51.constellix.net.", "ns61.constellix.net."], + "note": "", + "version": 0, + "status": "ACTIVE", + "tags": [], + "contactIds": [] +}] diff --git a/tests/fixtures/constellix-records.json b/tests/fixtures/constellix-records.json new file mode 100644 index 0000000..c1f1fb4 --- /dev/null +++ b/tests/fixtures/constellix-records.json @@ -0,0 +1,598 @@ +[{ + "id": 1808529, + "type": "CAA", + "recordType": "caa", + "name": "", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 3600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149569216, + "value": [{ + "flag": 0, + "tag": "issue", + "data": "ca.unit.tests", + "caaProviderId": 1, + "disableFlag": false + }], + "roundRobin": [{ + "flag": 0, + "tag": "issue", + "data": "ca.unit.tests", + "caaProviderId": 1, + "disableFlag": false + }] +}, { + "id": 1808516, + "type": "A", + "recordType": "a", + "name": "", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149623640, + "value": ["1.2.3.4", "1.2.3.5"], + "roundRobin": [{ + "value": "1.2.3.4", + "disableFlag": false + }, { + "value": "1.2.3.5", + "disableFlag": false + }], + "geolocation": null, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "roundRobinFailover": [], + "pools": [], + "poolsDetail": [] +}, { + "id": 1808527, + "type": "SRV", + "recordType": "srv", + "name": "_srv._tcp", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149714387, + "value": [{ + "value": "foo-1.unit.tests.", + "priority": 10, + "weight": 20, + "port": 30, + "disableFlag": false + }, { + "value": "foo-2.unit.tests.", + "priority": 12, + "weight": 20, + "port": 30, + "disableFlag": false + }], + "roundRobin": [{ + "value": "foo-1.unit.tests.", + "priority": 10, + "weight": 20, + "port": 30, + "disableFlag": false + }, { + "value": "foo-2.unit.tests.", + "priority": 12, + "weight": 20, + "port": 30, + "disableFlag": false + }] +}, { + "id": 1808515, + "type": "AAAA", + "recordType": "aaaa", + "name": "aaaa", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149739464, + "value": ["2601:644:500:e210:62f8:1dff:feb8:947a"], + "roundRobin": [{ + "value": "2601:644:500:e210:62f8:1dff:feb8:947a", + "disableFlag": false + }], + "geolocation": null, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "pools": [], + "poolsDetail": [], + "roundRobinFailover": [] +}, { + "id": 1808530, + "type": "ANAME", + "recordType": "aname", + "name": "", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 1800, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150251379, + "value": [{ + "value": "aname.unit.tests.", + "disableFlag": false + }], + "roundRobin": [{ + "value": "aname.unit.tests.", + "disableFlag": false + }], + "geolocation": null, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "pools": [], + "poolsDetail": [] +}, { + "id": 1808521, + "type": "CNAME", + "recordType": "cname", + "name": "cname", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565152113825, + "value": "", + "roundRobin": [{ + "value": "", + "disableFlag": false + }], + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [{ + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 1, + "markedActive": false + }, { + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 2, + "markedActive": false + }] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [{ + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 1, + "markedActive": false + }, { + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 2, + "markedActive": false + }] + }, + "pools": [], + "poolsDetail": [], + "geolocation": null, + "host": "" +}, { + "id": 1808522, + "type": "CNAME", + "recordType": "cname", + "name": "included", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 3600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565152119137, + "value": "", + "roundRobin": [{ + "value": "", + "disableFlag": false + }], + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [{ + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 1, + "markedActive": false + }, { + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 2, + "markedActive": false + }] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [{ + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 1, + "markedActive": false + }, { + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 2, + "markedActive": false + }] + }, + "pools": [], + "poolsDetail": [], + "geolocation": null, + "host": "" +}, { + "id": 1808523, + "type": "MX", + "recordType": "mx", + "name": "mx", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149879856, + "value": [{ + "value": "smtp-3.unit.tests.", + "level": 30, + "disableFlag": false + }, { + "value": "smtp-2.unit.tests.", + "level": 20, + "disableFlag": false + }, { + "value": "smtp-4.unit.tests.", + "level": 10, + "disableFlag": false + }, { + "value": "smtp-1.unit.tests.", + "level": 40, + "disableFlag": false + }], + "roundRobin": [{ + "value": "smtp-3.unit.tests.", + "level": 30, + "disableFlag": false + }, { + "value": "smtp-2.unit.tests.", + "level": 20, + "disableFlag": false + }, { + "value": "smtp-4.unit.tests.", + "level": 10, + "disableFlag": false + }, { + "value": "smtp-1.unit.tests.", + "level": 40, + "disableFlag": false + }] +}, { + "id": 1808525, + "type": "PTR", + "recordType": "ptr", + "name": "ptr", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150115139, + "value": [{ + "value": "foo.bar.com.", + "disableFlag": false + }], + "roundRobin": [{ + "value": "foo.bar.com.", + "disableFlag": false + }] +}, { + "id": 1808526, + "type": "SPF", + "recordType": "spf", + "name": "spf", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149916132, + "value": [{ + "value": "\"v=spf1 ip4:192.168.0.1/16-all\"", + "disableFlag": false + }], + "roundRobin": [{ + "value": "\"v=spf1 ip4:192.168.0.1/16-all\"", + "disableFlag": false + }] +}, { + "id": 1808528, + "type": "TXT", + "recordType": "txt", + "name": "txt", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149966915, + "value": [{ + "value": "\"Bah bah black sheep\"", + "disableFlag": false + }, { + "value": "\"have you any wool.\"", + "disableFlag": false + }, { + "value": "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"", + "disableFlag": false + }], + "roundRobin": [{ + "value": "\"Bah bah black sheep\"", + "disableFlag": false + }, { + "value": "\"have you any wool.\"", + "disableFlag": false + }, { + "value": "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"", + "disableFlag": false + }] +}, { + "id": 1808524, + "type": "NS", + "recordType": "ns", + "name": "under", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 3600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150062850, + "value": [{ + "value": "ns1.unit.tests.", + "disableFlag": false + }, { + "value": "ns2", + "disableFlag": false + }], + "roundRobin": [{ + "value": "ns1.unit.tests.", + "disableFlag": false + }, { + "value": "ns2", + "disableFlag": false + }] +}, { + "id": 1808531, + "type": "HTTPRedirection", + "recordType": "httpredirection", + "name": "unsupported", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150348154, + "value": "https://redirect.unit.tests", + "roundRobin": [{ + "value": "https://redirect.unit.tests" + }], + "title": "Unsupported Record", + "keywords": "unsupported", + "description": "unsupported record", + "hardlinkFlag": false, + "redirectTypeId": 1, + "url": "https://redirect.unit.tests" +}, { + "id": 1808519, + "type": "A", + "recordType": "a", + "name": "www", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150079027, + "value": ["2.2.3.6"], + "roundRobin": [{ + "value": "2.2.3.6", + "disableFlag": false + }], + "geolocation": null, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "roundRobinFailover": [], + "pools": [], + "poolsDetail": [] +}, { + "id": 1808603, + "type": "ANAME", + "recordType": "aname", + "name": "sub", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 1800, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565153387855, + "value": [{ + "value": "aname.unit.tests.", + "disableFlag": false + }], + "roundRobin": [{ + "value": "aname.unit.tests.", + "disableFlag": false + }], + "geolocation": null, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "pools": [], + "poolsDetail": [] +}, { + "id": 1808520, + "type": "A", + "recordType": "a", + "name": "www.sub", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150090588, + "value": ["2.2.3.6"], + "roundRobin": [{ + "value": "2.2.3.6", + "disableFlag": false + }], + "geolocation": null, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "roundRobinFailover": [], + "pools": [], + "poolsDetail": [] +}] diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py new file mode 100644 index 0000000..346bb17 --- /dev/null +++ b/tests/test_octodns_provider_constellix.py @@ -0,0 +1,218 @@ +# +# +# + + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from mock import Mock, call +from os.path import dirname, join +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.record import Record +from octodns.provider.constellix import ConstellixClientNotFound, \ + ConstellixProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + +import json + + +class TestConstellixProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + + # Our test suite differs a bit, add our NS and remove the simple one + expected.add_record(Record.new(expected, 'under', { + 'ttl': 3600, + 'type': 'NS', + 'values': [ + 'ns1.unit.tests.', + 'ns2.unit.tests.', + ] + })) + + # Add some ALIAS records + expected.add_record(Record.new(expected, '', { + 'ttl': 1800, + 'type': 'ALIAS', + 'value': 'aname.unit.tests.' + })) + + expected.add_record(Record.new(expected, 'sub', { + 'ttl': 1800, + 'type': 'ALIAS', + 'value': 'aname.unit.tests.' + })) + + for record in list(expected.records): + if record.name == 'sub' and record._type == 'NS': + expected._remove_record(record) + break + + def test_populate(self): + provider = ConstellixProvider('test', 'api', 'secret') + + # Bad auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, + text='{"errors": ["Unable to authenticate token"]}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('Unauthorized', ctx.exception.message) + + # Bad request + with requests_mock() as mock: + mock.get(ANY, status_code=400, + text='{"errors": ["\\"unittests\\" is not ' + 'a valid domain name"]}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('\n - "unittests" is not a valid domain name', + ctx.exception.message) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existent zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='') + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # No diffs == no changes + with requests_mock() as mock: + base = 'https://api.dns.constellix.com/v1/domains' + with open('tests/fixtures/constellix-domains.json') as fh: + mock.get('{}{}'.format(base, '/'), text=fh.read()) + with open('tests/fixtures/constellix-records.json') as fh: + mock.get('{}{}'.format(base, '/123123/records'), + text=fh.read()) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(15, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 2nd populate makes no network calls/all from cache + again = Zone('unit.tests.', []) + provider.populate(again) + self.assertEquals(15, len(again.records)) + + # bust the cache + del provider._zone_records[zone.name] + + def test_apply(self): + provider = ConstellixProvider('test', 'api', 'secret') + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + with open('tests/fixtures/constellix-domains.json') as fh: + domains = json.load(fh) + + # non-existent domain, create everything + resp.json.side_effect = [ + ConstellixClientNotFound, # no zone in populate + ConstellixClientNotFound, # no domain during apply + domains + ] + plan = provider.plan(self.expected) + + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected.records) - 5 + self.assertEquals(n, len(plan.changes)) + self.assertEquals(n, provider.apply(plan)) + + provider._client._request.assert_has_calls([ + # created the domain + call('POST', '/', data={'names': ['unit.tests']}), + # get all domains to build the cache + call('GET', '/'), + call('POST', '/123123/records/SRV', data={ + 'roundRobin': [{ + 'priority': 10, + 'weight': 20, + 'value': 'foo-1.unit.tests.', + 'port': 30 + }, { + 'priority': 12, + 'weight': 20, + 'value': 'foo-2.unit.tests.', + 'port': 30 + }], + 'name': '_srv._tcp', + 'ttl': 600, + }), + ]) + + self.assertEquals(20, provider._client._request.call_count) + + provider._client._request.reset_mock() + + provider._client.records = Mock(return_value=[ + { + 'id': 11189897, + 'type': 'A', + 'name': 'www', + 'ttl': 300, + 'value': [ + '1.2.3.4', + '2.2.3.4', + ] + }, { + 'id': 11189898, + 'type': 'A', + 'name': 'ttl', + 'ttl': 600, + 'value': [ + '3.2.3.4' + ] + } + ]) + + # Domain exists, we don't care about return + resp.json.side_effect = ['{}'] + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'ttl', { + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4' + })) + + plan = provider.plan(wanted) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + + # recreate for update, and deletes for the 2 parts of the other + provider._client._request.assert_has_calls([ + call('POST', '/123123/records/A', data={ + 'roundRobin': [{ + 'value': '3.2.3.4' + }], + 'name': 'ttl', + 'ttl': 300 + }), + call('DELETE', '/123123/records/A/11189897'), + call('DELETE', '/123123/records/A/11189898') + ], any_order=True) From dd4238ad6b008b56c1cc6efbba031df943d9ee0e Mon Sep 17 00:00:00 2001 From: William Blackie Date: Mon, 19 Aug 2019 17:06:23 +1000 Subject: [PATCH 70/86] fix split calls in Route53Provider to not specify ' ' parameter so they can tolerate multiple sequential separators --- octodns/provider/route53.py | 8 ++++---- tests/test_octodns_provider_route53.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 7b21573..81e366a 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -723,7 +723,7 @@ class Route53Provider(BaseProvider): def _data_for_CAA(self, rrset): values = [] for rr in rrset['ResourceRecords']: - flags, tag, value = rr['Value'].split(' ') + flags, tag, value = rr['Value'].split() values.append({ 'flags': flags, 'tag': tag, @@ -761,7 +761,7 @@ class Route53Provider(BaseProvider): def _data_for_MX(self, rrset): values = [] for rr in rrset['ResourceRecords']: - preference, exchange = rr['Value'].split(' ') + preference, exchange = rr['Value'].split() values.append({ 'preference': preference, 'exchange': exchange, @@ -776,7 +776,7 @@ class Route53Provider(BaseProvider): values = [] for rr in rrset['ResourceRecords']: order, preference, flags, service, regexp, replacement = \ - rr['Value'].split(' ') + rr['Value'].split() flags = flags[1:-1] service = service[1:-1] regexp = regexp[1:-1] @@ -804,7 +804,7 @@ class Route53Provider(BaseProvider): def _data_for_SRV(self, rrset): values = [] for rr in rrset['ResourceRecords']: - priority, weight, port, target = rr['Value'].split(' ') + priority, weight, port, target = rr['Value'].split() values.append({ 'priority': priority, 'weight': weight, diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 265a0a7..ceafd13 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -503,7 +503,7 @@ class TestRoute53Provider(TestCase): 'ResourceRecords': [{ 'Value': '10 smtp-1.unit.tests.', }, { - 'Value': '20 smtp-2.unit.tests.', + 'Value': '20 smtp-2.unit.tests.', }], 'TTL': 64, }, { From 7d8f181367d540ddc3f99a5a1340e8bd7e5db6ac Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Thu, 19 Sep 2019 10:48:18 -0700 Subject: [PATCH 71/86] Adding Akamai entry to supported providers --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 75a006c..be0a4ab 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic/Geo Support | Notes | |--|--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | +| [Akamai](/octodns/provider/fastdns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, MX, NS, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | From 736d588e86942a215a01607ce3983c3ea1165395 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Fri, 20 Sep 2019 10:18:51 +0200 Subject: [PATCH 72/86] Changed requirements to version 2.22.0 Fixes: ERROR: requests 2.20.0 has requirement urllib3<1.25,>=1.21.1, but you'll have urllib3 1.25.5 which is incompatible. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 75dc1df..f67f2ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ natsort==5.5.0 nsone==0.9.100 ovh==0.4.8 python-dateutil==2.6.1 -requests==2.20.0 +requests==2.22.0 s3transfer==0.1.13 six==1.11.0 setuptools==38.5.2 From bb3f0c0b4a9c0705aad39cfd5aac3cbca8bfab47 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 07:01:06 +0200 Subject: [PATCH 73/86] Added TransIP provider and tests --- octodns/provider/transip.py | 328 +++++++++++++++++++++++++ tests/test_octodns_provider_transip.py | 234 ++++++++++++++++++ 2 files changed, 562 insertions(+) create mode 100644 octodns/provider/transip.py create mode 100644 tests/test_octodns_provider_transip.py diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py new file mode 100644 index 0000000..2ce9180 --- /dev/null +++ b/octodns/provider/transip.py @@ -0,0 +1,328 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from suds import WebFault + +from collections import defaultdict +from .base import BaseProvider +from logging import getLogger +from ..record import Record +from transip.service.domain import DomainService +from transip.service.objects import DnsEntry + + +class TransipProvider(BaseProvider): + ''' + Transip DNS provider + + transip: + class: octodns.provider.transip.TransipProvider + # Your Transip account name (required) + account: yourname + # The api key (required) + key: | + \''' + -----BEGIN PRIVATE KEY----- + ... + -----END PRIVATE KEY----- + \''' + + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set( + ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA')) + # unsupported by OctoDNS: 'TLSA', 'CAA' + MIN_TTL = 120 + TIMEOUT = 15 + ROOT_RECORD = '@' + + def __init__(self, id, account, key, *args, **kwargs): + self.log = getLogger('TransipProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, account=%s, token=***', id, + account) + super(TransipProvider, self).__init__(id, *args, **kwargs) + + self._client = DomainService(account, key) + + self.account = account + self.key = key + + self._zones = None + self._zone_records = {} + + self._currentZone = {} + + def populate(self, zone, target=False, lenient=False): + + exists = False + self._currentZone = zone + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + before = len(zone.records) + try: + zoneInfo = self._client.get_info(zone.name[:-1]) + except WebFault as e: + if e.fault.faultcode == '102' and target == False: + self.log.warning( + 'populate: (%s) Zone %s not found in account ', + e.fault.faultcode, zone.name) + exists = False + return exists + elif e.fault.faultcode == '102' and target == True: + self.log.warning('populate: Transip can\'t create new zones') + raise Exception( + ('populate: ({}) Transip used ' + + 'as target for non-existing zone: {}').format( + e.fault.faultcode, zone.name)) + else: + self.log.error('populate: (%s) %s ', e.fault.faultcode, + e.fault.faultstring) + raise e + + self.log.debug('populate: found %s records for zone %s', + len(zoneInfo.dnsEntries), zone.name) + exists = True + if zoneInfo.dnsEntries: + values = defaultdict(lambda: defaultdict(list)) + for record in zoneInfo.dnsEntries: + name = zone.hostname_from_fqdn(record['name']) + if name == self.ROOT_RECORD: + name = '' + + if record['type'] in self.SUPPORTS: + values[name][record['type']].append(record) + + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + self.log.info('populate: found %s records, exists = %s', + len(zone.records) - before, exists) + + self._currentZone = {} + return exists + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('apply: zone=%s, changes=%d', desired.name, + len(changes)) + # for change in changes: + # class_name = change.__class__.__name__ + # getattr(self, '_apply_{}'.format(class_name))(change) + + self._currentZone = plan.desired + try: + self._client.get_info(plan.desired.name[:-1]) + except WebFault as e: + self.log.warning('_apply: %s ', e.message) + raise e + + _dns_entries = [] + for record in plan.desired.records: + if record._type in self.SUPPORTS: + entries_for = getattr(self, '_entries_for_{}'.format(record._type)) + + # Root records have '@' as name + name = record.name + if name == '': + name = self.ROOT_RECORD + + _dns_entries.extend(entries_for(name, record)) + + try: + self._client.set_dns_entries(plan.desired.name[:-1], _dns_entries) + except WebFault as e: + self.log.warning(('_apply: Set DNS returned ' + + 'one or more errors: {}').format( + e.fault.faultstring)) + raise Exception(200, e.fault.faultstring) + + self._currentZone = {} + + def _entries_for_multiple(self, name, record): + _entries = [] + + for value in record.values: + _entries.append(DnsEntry(name, record.ttl, record._type, value)) + + return _entries + + def _entries_for_single(self, name, record): + + return [DnsEntry(name, record.ttl, record._type, record.value)] + + _entries_for_A = _entries_for_multiple + _entries_for_AAAA = _entries_for_multiple + _entries_for_NS = _entries_for_multiple + _entries_for_SPF = _entries_for_multiple + _entries_for_CNAME = _entries_for_single + + def _entries_for_MX(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {}".format(value.preference, value.exchange) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_SRV(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {} {}".format(value.priority, value.weight, + value.port, value.target) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_SSHFP(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {}".format(value.algorithm, value.fingerprint_type, + value.fingerprint) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_CAA(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {}".format(value.flags, value.tag, + value.value) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_TXT(self, name, record): + _entries = [] + + for value in record.values: + value = value.replace('\\;', ';') + _entries.append(DnsEntry(name, record.ttl, record._type, value)) + + return _entries + + def _parse_to_fqdn(self, value): + + if (value[-1] != '.'): + self.log.debug('parseToFQDN: changed %s to %s', value, + '{}.{}'.format(value, self._currentZone.name)) + value = '{}.{}'.format(value, self._currentZone.name) + + return value + + def _get_lowest_ttl(self, records): + _ttl = 100000 + for record in records: + _ttl = min(_ttl, record['expire']) + return _ttl + + def _data_for_multiple(self, _type, records): + + _values = [] + for record in records: + _values.append(record['content']) + + return { + 'ttl': self._get_lowest_ttl(records), + 'type': _type, + 'values': _values + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_NS = _data_for_multiple + _data_for_SPF = _data_for_multiple + + def _data_for_CNAME(self, _type, records): + return { + 'ttl': records[0]['expire'], + 'type': _type, + 'value': self._parse_to_fqdn(records[0]['content']) + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + preference, exchange = record['content'].split(" ", 1) + values.append({ + 'preference': preference, + 'exchange': self._parse_to_fqdn(exchange) + }) + return { + 'ttl': self._get_lowest_ttl(records), + 'type': _type, + 'values': values + } + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + priority, weight, port, target = record['content'].split(' ', 3) + values.append({ + 'port': port, + 'priority': priority, + 'target': self._parse_to_fqdn(target), + 'weight': weight + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } + + def _data_for_SSHFP(self, _type, records): + values = [] + for record in records: + algorithm, fp_type, fingerprint = record['content'].split(' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint': fingerprint.lower(), + 'fingerprint_type': fp_type + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } + + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + flags, tag, value = record['content'].split(' ', 2) + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } + + def _data_for_TXT(self, _type, records): + values = [] + for record in records: + values.append(record['content'].replace(';', '\\;')) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py new file mode 100644 index 0000000..dbf7eab --- /dev/null +++ b/tests/test_octodns_provider_transip.py @@ -0,0 +1,234 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +# from mock import Mock, call +from os.path import dirname, join + +from suds import WebFault + +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.record import Record +from octodns.provider.transip import TransipProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone +from transip.service.domain import DomainService +from transip.service.objects import DnsEntry + + + + +class MockDomainService(DomainService): + + def __init__(self, *args, **kwargs): + super(MockDomainService, self).__init__('MockDomainService', *args, **kwargs) + self.mockupEntries = [] + + def mockup(self, records): + + provider = TransipProvider('', '', ''); + + _dns_entries = [] + for record in records: + if record._type in provider.SUPPORTS: + entries_for = getattr(provider, '_entries_for_{}'.format(record._type)) + + # Root records have '@' as name + name = record.name + if name == '': + name = provider.ROOT_RECORD + + _dns_entries.extend(entries_for(name, record)) + + _dns_entries.append(DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) + + + self.mockupEntries = _dns_entries + + # Skips authentication layer and returns the entries loaded by "Mockup" + def get_info(self, domain_name): + + if str(domain_name) == str('notfound.unit.tests'): + self.raiseZoneNotFound() + + result = lambda: None + setattr(result, "dnsEntries", self.mockupEntries) + return result + + def set_dns_entries(self, domain_name, dns_entries): + if str(domain_name) == str('failsetdns.unit.tests'): + self.raiseSaveError() + + return True + + def raiseZoneNotFound(self): + fault = lambda: None + setattr(fault, "faultstring", '102 is zone not found') + setattr(fault, "faultcode", str('102')) + document = {} + raise WebFault(fault, document) + + def raiseInvalidAuth(self): + fault = lambda: None + setattr(fault, "faultstring", '200 is invalid auth') + setattr(fault, "faultcode", str('200')) + document = {} + raise WebFault(fault, document) + + def raiseSaveError(self): + fault = lambda: None + setattr(fault, "faultstring", '202 error while saving') + setattr(fault, "faultcode", str('202')) + document = {} + raise WebFault(fault, document) + + + +class TestTransipProvider(TestCase): + bogus_key = str("""-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA0U5HGCkLrz423IyUf3u4cKN2WrNz1x5KNr6PvH2M/zxas+zB +elbxkdT3AQ+wmfcIvOuTmFRTHv35q2um1aBrPxVw+2s+lWo28VwIRttwIB1vIeWu +lSBnkEZQRLyPI2tH0i5QoMX4CVPf9rvij3Uslimi84jdzDfPFIh6jZ6C8nLipOTG +0IMhge1ofVfB0oSy5H+7PYS2858QLAf5ruYbzbAxZRivS402wGmQ0d0Lc1KxraAj +kiMM5yj/CkH/Vm2w9I6+tLFeASE4ub5HCP5G/ig4dbYtqZMQMpqyAbGxd5SOVtyn +UHagAJUxf8DT3I8PyjEHjxdOPUsxNyRtepO/7QIDAQABAoIBAQC7fiZ7gxE/ezjD +2n6PsHFpHVTBLS2gzzZl0dCKZeFvJk6ODJDImaeuHhrh7X8ifMNsEI9XjnojMhl8 +MGPzy88mZHugDNK0H8B19x5G8v1/Fz7dG5WHas660/HFkS+b59cfdXOugYiOOn9O +08HBBpLZNRUOmVUuQfQTjapSwGLG8PocgpyRD4zx0LnldnJcqYCxwCdev+AAsPnq +ibNtOd/MYD37w9MEGcaxLE8wGgkv8yd97aTjkgE+tp4zsM4QE4Rag133tsLLNznT +4Qr/of15M3NW/DXq/fgctyRcJjZpU66eCXLCz2iRTnLyyxxDC2nwlxKbubV+lcS0 +S4hbfd/BAoGBAO8jXxEaiybR0aIhhSR5esEc3ymo8R8vBN3ZMJ+vr5jEPXr/ZuFj +/R4cZ2XV3VoQJG0pvIOYVPZ5DpJM7W+zSXtJ/7bLXy4Bnmh/rc+YYgC+AXQoLSil +iD2OuB2xAzRAK71DVSO0kv8gEEXCersPT2i6+vC2GIlJvLcYbOdRKWGxAoGBAOAQ +aJbRLtKujH+kMdoMI7tRlL8XwI+SZf0FcieEu//nFyerTePUhVgEtcE+7eQ7hyhG +fIXUFx/wALySoqFzdJDLc8U8pTLhbUaoLOTjkwnCTKQVprhnISqQqqh/0U5u47IE +RWzWKN6OHb0CezNTq80Dr6HoxmPCnJHBHn5LinT9AoGAQSpvZpbIIqz8pmTiBl2A +QQ2gFpcuFeRXPClKYcmbXVLkuhbNL1BzEniFCLAt4LQTaRf9ghLJ3FyCxwVlkpHV +zV4N6/8hkcTpKOraL38D/dXJSaEFJVVuee/hZl3tVJjEEpA9rDwx7ooLRSdJEJ6M +ciq55UyKBSdt4KssSiDI2RECgYBL3mJ7xuLy5bWfNsrGiVvD/rC+L928/5ZXIXPw +26oI0Yfun7ulDH4GOroMcDF/GYT/Zzac3h7iapLlR0WYI47xxGI0A//wBZLJ3QIu +krxkDo2C9e3Y/NqnHgsbOQR3aWbiDT4wxydZjIeXS3LKA2fl6Hyc90PN3cTEOb8I +hq2gRQKBgEt0SxhhtyB93SjgTzmUZZ7PiEf0YJatfM6cevmjWHexrZH+x31PB72s +fH2BQyTKKzoCLB1k/6HRaMnZdrWyWSZ7JKz3AHJ8+58d0Hr8LTrzDM1L6BbjeDct +N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd +-----END RSA PRIVATE KEY-----""") + + + def make_expected(self): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + return expected + + def test_populate(self): + + _expected = self.make_expected() + + with self.assertRaises(WebFault) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + zone = Zone('unit.tests.', []) + provider.populate(zone, True) + + self.assertEquals(str('WebFault'), + str(ctx.exception.__class__.__name__)) + + self.assertEquals(str('200'), ctx.exception.fault.faultcode) + + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('notfound.unit.tests.', []) + provider.populate(zone, True) + + self.assertEquals(str('Exception'), + str(ctx.exception.__class__.__name__)) + + self.assertEquals('populate: (102) Transip used as target for non-existing zone: notfound.unit.tests.', ctx.exception.message) + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('notfound.unit.tests.', []) + provider.populate(zone, False) + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + provider._client.mockup(_expected.records) + zone = Zone('unit.tests.', []) + provider.populate(zone, False) + + provider._currentZone = zone + self.assertEquals("www.unit.tests.", provider._parse_to_fqdn("www")) + + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('unit.tests.', []) + exists = provider.populate(zone, True) + self.assertTrue(exists, 'populate should return true') + + + return + + def test_plan(self): + + _expected = self.make_expected() + + print(_expected.name) + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + + self.assertEqual(12, plan.change_counts['Create']) + self.assertEqual(0, plan.change_counts['Update']) + self.assertEqual(0, plan.change_counts['Delete']) + + return + + def test_apply(self): + + _expected = self.make_expected() + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + #self.assertEqual(11, plan.changes) + changes = provider.apply(plan) + + + + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + plan.desired.name = 'notfound.unit.tests.' + changes = provider.apply(plan) + # self.assertEqual(11, changes) + + self.assertEquals(str('WebFault'), + str(ctx.exception.__class__.__name__)) + + _expected = self.make_expected() + + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + plan.desired.name = 'failsetdns.unit.tests.' + changes = provider.apply(plan) + # self.assertEqual(11, changes) + + + #provider = TransipProvider('test', 'unittest', self.bogus_key) + + #plan = provider.plan(_expected) + +# changes = provider.apply(plan) +# self.assertEquals(29, changes) + + From 30c8c4d313d150c2c81324f0b0a7de06f634bdc0 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 08:18:17 +0200 Subject: [PATCH 74/86] Add transip requirement and add provider to readme --- README.md | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index be0a4ab..7124b20 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,7 @@ The above command pulled the existing data out of Route53 and placed the results | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Both | CNAME health checks don't support a Host header | | [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | +| [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, SRV, SPF, TXT, SSHFP, CAA | No | | | [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only | diff --git a/requirements.txt b/requirements.txt index f67f2ee..bb373d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,4 @@ requests==2.22.0 s3transfer==0.1.13 six==1.11.0 setuptools==38.5.2 +transip==2.0.0 From 7056d299072f3c03c446ef8b5435f4339b1bb4b4 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 08:18:59 +0200 Subject: [PATCH 75/86] fixes lint warning. --- octodns/provider/transip.py | 20 +++-- tests/test_octodns_provider_transip.py | 110 +++++++++++++++---------- 2 files changed, 79 insertions(+), 51 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 2ce9180..8014310 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -36,7 +36,7 @@ class TransipProvider(BaseProvider): SUPPORTS_DYNAMIC = False SUPPORTS = set( ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA')) - # unsupported by OctoDNS: 'TLSA', 'CAA' + # unsupported by OctoDNS: 'TLSA' MIN_TTL = 120 TIMEOUT = 15 ROOT_RECORD = '@' @@ -68,17 +68,17 @@ class TransipProvider(BaseProvider): try: zoneInfo = self._client.get_info(zone.name[:-1]) except WebFault as e: - if e.fault.faultcode == '102' and target == False: + if e.fault.faultcode == '102' and target is False: self.log.warning( 'populate: (%s) Zone %s not found in account ', e.fault.faultcode, zone.name) exists = False return exists - elif e.fault.faultcode == '102' and target == True: + elif e.fault.faultcode == '102' and target is True: self.log.warning('populate: Transip can\'t create new zones') raise Exception( ('populate: ({}) Transip used ' + - 'as target for non-existing zone: {}').format( + 'as target for non-existing zone: {}').format( e.fault.faultcode, zone.name)) else: self.log.error('populate: (%s) %s ', e.fault.faultcode, @@ -129,7 +129,8 @@ class TransipProvider(BaseProvider): _dns_entries = [] for record in plan.desired.records: if record._type in self.SUPPORTS: - entries_for = getattr(self, '_entries_for_{}'.format(record._type)) + entries_for = getattr(self, + '_entries_for_{}'.format(record._type)) # Root records have '@' as name name = record.name @@ -143,7 +144,7 @@ class TransipProvider(BaseProvider): except WebFault as e: self.log.warning(('_apply: Set DNS returned ' + 'one or more errors: {}').format( - e.fault.faultstring)) + e.fault.faultstring)) raise Exception(200, e.fault.faultstring) self._currentZone = {} @@ -189,8 +190,9 @@ class TransipProvider(BaseProvider): _entries = [] for value in record.values: - content = "{} {} {}".format(value.algorithm, value.fingerprint_type, - value.fingerprint) + content = "{} {} {}".format(value.algorithm, + value.fingerprint_type, + value.fingerprint) _entries.append(DnsEntry(name, record.ttl, record._type, content)) return _entries @@ -200,7 +202,7 @@ class TransipProvider(BaseProvider): for value in record.values: content = "{} {} {}".format(value.flags, value.tag, - value.value) + value.value) _entries.append(DnsEntry(name, record.ttl, record._type, content)) return _entries diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index dbf7eab..811c1e2 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -10,10 +10,8 @@ from os.path import dirname, join from suds import WebFault -from requests_mock import ANY, mock as requests_mock from unittest import TestCase -from octodns.record import Record from octodns.provider.transip import TransipProvider from octodns.provider.yaml import YamlProvider from octodns.zone import Zone @@ -21,22 +19,35 @@ from transip.service.domain import DomainService from transip.service.objects import DnsEntry +class MockFault(object): + faultstring = "" + faultcode = "" + + def __init__(self, code, string, *args, **kwargs): + self.faultstring = string + self.faultcode = code + + +class MockResponse(object): + dnsEntries = [] class MockDomainService(DomainService): def __init__(self, *args, **kwargs): - super(MockDomainService, self).__init__('MockDomainService', *args, **kwargs) + super(MockDomainService, self).__init__('MockDomainService', *args, + **kwargs) self.mockupEntries = [] def mockup(self, records): - provider = TransipProvider('', '', ''); + provider = TransipProvider('', '', '') _dns_entries = [] for record in records: if record._type in provider.SUPPORTS: - entries_for = getattr(provider, '_entries_for_{}'.format(record._type)) + entries_for = getattr(provider, + '_entries_for_{}'.format(record._type)) # Root records have '@' as name name = record.name @@ -45,50 +56,48 @@ class MockDomainService(DomainService): _dns_entries.extend(entries_for(name, record)) - _dns_entries.append(DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) - + # NS is not supported as a DNS Entry, + # so it should cover the if statement + _dns_entries.append( + DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) self.mockupEntries = _dns_entries # Skips authentication layer and returns the entries loaded by "Mockup" def get_info(self, domain_name): + # Special 'domain' to trigger error if str(domain_name) == str('notfound.unit.tests'): self.raiseZoneNotFound() - result = lambda: None - setattr(result, "dnsEntries", self.mockupEntries) + result = MockResponse() + result.dnsEntries = self.mockupEntries return result def set_dns_entries(self, domain_name, dns_entries): + + # Special 'domain' to trigger error if str(domain_name) == str('failsetdns.unit.tests'): self.raiseSaveError() return True def raiseZoneNotFound(self): - fault = lambda: None - setattr(fault, "faultstring", '102 is zone not found') - setattr(fault, "faultcode", str('102')) + fault = MockFault(str('102'), '102 is zone not found') document = {} raise WebFault(fault, document) def raiseInvalidAuth(self): - fault = lambda: None - setattr(fault, "faultstring", '200 is invalid auth') - setattr(fault, "faultcode", str('200')) + fault = MockFault(str('200'), '200 is invalid auth') document = {} raise WebFault(fault, document) def raiseSaveError(self): - fault = lambda: None - setattr(fault, "faultstring", '202 error while saving') - setattr(fault, "faultcode", str('202')) + fault = MockFault(str('200'), '202 random error') document = {} raise WebFault(fault, document) - class TestTransipProvider(TestCase): bogus_key = str("""-----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA0U5HGCkLrz423IyUf3u4cKN2WrNz1x5KNr6PvH2M/zxas+zB @@ -118,7 +127,6 @@ fH2BQyTKKzoCLB1k/6HRaMnZdrWyWSZ7JKz3AHJ8+58d0Hr8LTrzDM1L6BbjeDct N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd -----END RSA PRIVATE KEY-----""") - def make_expected(self): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) @@ -126,9 +134,10 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd return expected def test_populate(self): - _expected = self.make_expected() + # Unhappy Plan - Not authenticated + # Live test against API, will fail in an unauthorized error with self.assertRaises(WebFault) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) zone = Zone('unit.tests.', []) @@ -139,6 +148,9 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd self.assertEquals(str('200'), ctx.exception.fault.faultcode) + # Unhappy Plan - Zone does not exists + # Will trigger an exception if provider is used as a target for a + # non-existing zone with self.assertRaises(Exception) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) @@ -148,38 +160,48 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd self.assertEquals(str('Exception'), str(ctx.exception.__class__.__name__)) - self.assertEquals('populate: (102) Transip used as target for non-existing zone: notfound.unit.tests.', ctx.exception.message) + self.assertEquals( + 'populate: (102) Transip used as target' + + ' for non-existing zone: notfound.unit.tests.', + ctx.exception.message) + # Happy Plan - Zone does not exists + # Won't trigger an exception if provider is NOT used as a target for a + # non-existing zone. provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) zone = Zone('notfound.unit.tests.', []) provider.populate(zone, False) + # Happy Plan - Populate with mockup records provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) provider._client.mockup(_expected.records) zone = Zone('unit.tests.', []) provider.populate(zone, False) + # Transip allows relative values for types like cname, mx. + # Test is these are correctly appended with the domain provider._currentZone = zone self.assertEquals("www.unit.tests.", provider._parse_to_fqdn("www")) + self.assertEquals("www.unit.tests.", + provider._parse_to_fqdn("www.unit.tests.")) + self.assertEquals("www.sub.sub.sub.unit.tests.", + provider._parse_to_fqdn("www.sub.sub.sub")) - + # Happy Plan - Even if the zone has no records the zone should exist provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) zone = Zone('unit.tests.', []) exists = provider.populate(zone, True) self.assertTrue(exists, 'populate should return true') - return def test_plan(self): - _expected = self.make_expected() - print(_expected.name) - + # Test Happy plan, only create provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) @@ -191,29 +213,38 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd return def test_apply(self): - _expected = self.make_expected() + # Test happy flow. Create all supoorted records provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - #self.assertEqual(11, plan.changes) + self.assertEqual(12, len(plan.changes)) changes = provider.apply(plan) + self.assertEqual(changes, len(plan.changes)) - - + # Test unhappy flow. Trigger 'not found error' in apply stage + # This should normally not happen as populate will capture it first + # but just in case. + changes = [] # reset changes with self.assertRaises(Exception) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) plan.desired.name = 'notfound.unit.tests.' changes = provider.apply(plan) - # self.assertEqual(11, changes) + + # Changes should not be set due to an Exception + self.assertEqual([], changes) self.assertEquals(str('WebFault'), str(ctx.exception.__class__.__name__)) - _expected = self.make_expected() + self.assertEquals(str('102'), ctx.exception.fault.faultcode) + + # Test unhappy flow. Trigger a unrecoverable error while saving + _expected = self.make_expected() # reset expected + changes = [] # reset changes with self.assertRaises(Exception) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) @@ -221,14 +252,9 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd plan = provider.plan(_expected) plan.desired.name = 'failsetdns.unit.tests.' changes = provider.apply(plan) - # self.assertEqual(11, changes) - - - #provider = TransipProvider('test', 'unittest', self.bogus_key) - - #plan = provider.plan(_expected) - -# changes = provider.apply(plan) -# self.assertEquals(29, changes) + # Changes should not be set due to an Exception + self.assertEqual([], changes) + self.assertEquals(str('Exception'), + str(ctx.exception.__class__.__name__)) From 59e44b865cfa5e7d3a1f40d34f41d5b2dd285aac Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 11:24:13 +0200 Subject: [PATCH 76/86] Added detection for edge case that could happen with existing records where the value is '@' TransIP allows '@' as value to alias the root record. '@' was on populate appended with the zone, which trigger an unneeded update. '@' => '@.example.com.' -> 'example.com' This fix will stop the unneeded update --- octodns/provider/transip.py | 7 ++++++- tests/test_octodns_provider_transip.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 8014310..adde617 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -218,7 +218,12 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): - if (value[-1] != '.'): + # TransIP allows '@' as value to alias the root record. + # this provider won't set an '@' value, but can be an existing record + if value == self.ROOT_RECORD: + value = self._currentZone.name + + if value[-1] != '.': self.log.debug('parseToFQDN: changed %s to %s', value, '{}.{}'.format(value, self._currentZone.name)) value = '{}.{}'.format(value, self._currentZone.name) diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index 811c1e2..d6bcaa7 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -188,6 +188,8 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider._parse_to_fqdn("www.unit.tests.")) self.assertEquals("www.sub.sub.sub.unit.tests.", provider._parse_to_fqdn("www.sub.sub.sub")) + self.assertEquals("unit.tests.", + provider._parse_to_fqdn("@")) # Happy Plan - Even if the zone has no records the zone should exist provider = TransipProvider('test', 'unittest', self.bogus_key) From cebc629a06295a9cae3c5821b448e11737eddbbe Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 14:33:49 +0200 Subject: [PATCH 77/86] Enforce values as basic string to fix yaml export error Fixes an exception in combination with the yamlProvider as a target The unmodified value object isn't represented as string while building the yaml output The Exception: yaml.representer.RepresenterError: ('cannot represent an object', 1.1.1.1) yaml/representer.py@249, represent_undefined() --- octodns/provider/transip.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index adde617..050398a 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -218,6 +218,9 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): + # Enforce switch from suds.sax.text.Text to string + value = ''+value + # TransIP allows '@' as value to alias the root record. # this provider won't set an '@' value, but can be an existing record if value == self.ROOT_RECORD: @@ -240,7 +243,8 @@ class TransipProvider(BaseProvider): _values = [] for record in records: - _values.append(record['content']) + # Enforce switch from suds.sax.text.Text to string + _values.append(''+record['content']) return { 'ttl': self._get_lowest_ttl(records), From 9cab94a83a4621334880059b8007fe7bdd2bd8ba Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 14:45:41 +0200 Subject: [PATCH 78/86] Some codestyle review changes. --- octodns/provider/transip.py | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 050398a..aa5b5b3 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -52,9 +52,6 @@ class TransipProvider(BaseProvider): self.account = account self.key = key - self._zones = None - self._zone_records = {} - self._currentZone = {} def populate(self, zone, target=False, lenient=False): @@ -72,7 +69,6 @@ class TransipProvider(BaseProvider): self.log.warning( 'populate: (%s) Zone %s not found in account ', e.fault.faultcode, zone.name) - exists = False return exists elif e.fault.faultcode == '102' and target is True: self.log.warning('populate: Transip can\'t create new zones') @@ -115,9 +111,6 @@ class TransipProvider(BaseProvider): changes = plan.changes self.log.debug('apply: zone=%s, changes=%d', desired.name, len(changes)) - # for change in changes: - # class_name = change.__class__.__name__ - # getattr(self, '_apply_{}'.format(class_name))(change) self._currentZone = plan.desired try: @@ -265,24 +258,24 @@ class TransipProvider(BaseProvider): } def _data_for_MX(self, _type, records): - values = [] + _values = [] for record in records: preference, exchange = record['content'].split(" ", 1) - values.append({ + _values.append({ 'preference': preference, 'exchange': self._parse_to_fqdn(exchange) }) return { 'ttl': self._get_lowest_ttl(records), 'type': _type, - 'values': values + 'values': _values } def _data_for_SRV(self, _type, records): - values = [] + _values = [] for record in records: priority, weight, port, target = record['content'].split(' ', 3) - values.append({ + _values.append({ 'port': port, 'priority': priority, 'target': self._parse_to_fqdn(target), @@ -292,14 +285,14 @@ class TransipProvider(BaseProvider): return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } def _data_for_SSHFP(self, _type, records): - values = [] + _values = [] for record in records: algorithm, fp_type, fingerprint = record['content'].split(' ', 2) - values.append({ + _values.append({ 'algorithm': algorithm, 'fingerprint': fingerprint.lower(), 'fingerprint_type': fp_type @@ -308,14 +301,14 @@ class TransipProvider(BaseProvider): return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } def _data_for_CAA(self, _type, records): - values = [] + _values = [] for record in records: flags, tag, value = record['content'].split(' ', 2) - values.append({ + _values.append({ 'flags': flags, 'tag': tag, 'value': value @@ -324,16 +317,16 @@ class TransipProvider(BaseProvider): return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } def _data_for_TXT(self, _type, records): - values = [] + _values = [] for record in records: - values.append(record['content'].replace(';', '\\;')) + _values.append(record['content'].replace(';', '\\;')) return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } From 71f215932d94b323cddba6ed6e0b8f35ba3b9a4d Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 14:51:53 +0200 Subject: [PATCH 79/86] whitespaces around operators to make /script/lint happy again --- octodns/provider/transip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index aa5b5b3..64692ee 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -212,7 +212,7 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): # Enforce switch from suds.sax.text.Text to string - value = ''+value + value = '' + value # TransIP allows '@' as value to alias the root record. # this provider won't set an '@' value, but can be an existing record @@ -237,7 +237,7 @@ class TransipProvider(BaseProvider): _values = [] for record in records: # Enforce switch from suds.sax.text.Text to string - _values.append(''+record['content']) + _values.append('' + record['content']) return { 'ttl': self._get_lowest_ttl(records), From a035ee8c84955d2dcdf40c0ea15f1901224ae000 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Thu, 26 Sep 2019 14:49:14 +0200 Subject: [PATCH 80/86] Give the option to use a private_key_file. Transip sdk also supports a private_key_file, so forwarding that option to the provider. Could be handy in combination with k8s secrets. --- octodns/provider/transip.py | 14 +++++++++++--- tests/test_octodns_provider_transip.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 64692ee..92d607d 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -23,13 +23,16 @@ class TransipProvider(BaseProvider): class: octodns.provider.transip.TransipProvider # Your Transip account name (required) account: yourname - # The api key (required) + # Path to a private key file (required if key is not used) + key_file: /path/to/file + # The api key as string (required if key_file is not used) key: | \''' -----BEGIN PRIVATE KEY----- ... -----END PRIVATE KEY----- \''' + # if both `key_file` and `key` are presented `key_file` is used ''' SUPPORTS_GEO = False @@ -41,13 +44,18 @@ class TransipProvider(BaseProvider): TIMEOUT = 15 ROOT_RECORD = '@' - def __init__(self, id, account, key, *args, **kwargs): + def __init__(self, id, account, key=None, key_file=None, *args, **kwargs): self.log = getLogger('TransipProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, account=%s, token=***', id, account) super(TransipProvider, self).__init__(id, *args, **kwargs) - self._client = DomainService(account, key) + if key_file is not None: + self._client = DomainService(account, private_key_file=key_file) + elif key is not None: + self._client = DomainService(account, private_key=key) + else: + raise Exception('Missing `key` of `key_file` parameter in config') self.account = account self.key = key diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index d6bcaa7..8d85e1f 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -133,6 +133,19 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd source.populate(expected) return expected + def test_init(self): + with self.assertRaises(Exception) as ctx: + TransipProvider('test', 'unittest') + + self.assertEquals( + str('Missing `key` of `key_file` parameter in config'), + str(ctx.exception)) + + TransipProvider('test', 'unittest', key=self.bogus_key) + + # Existence and content of the key is tested in the SDK on client call + TransipProvider('test', 'unittest', key_file='/fake/path') + def test_populate(self): _expected = self.make_expected() From 637c2547782b15e21473f51960eb1ab4b3a94c1d Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Mon, 30 Sep 2019 13:18:57 +0200 Subject: [PATCH 81/86] Handling PR Review comments. - Added Specific exceptions - str() instead of concatenation - removed zone not found warning --- octodns/provider/transip.py | 29 +++++++++++++++++++------- tests/test_octodns_provider_transip.py | 4 ++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 92d607d..09920a9 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -15,6 +15,18 @@ from transip.service.domain import DomainService from transip.service.objects import DnsEntry +class TransipException(Exception): + pass + + +class TransipConfigException(TransipException): + pass + + +class TransipNewZoneException(TransipException): + pass + + class TransipProvider(BaseProvider): ''' Transip DNS provider @@ -55,7 +67,9 @@ class TransipProvider(BaseProvider): elif key is not None: self._client = DomainService(account, private_key=key) else: - raise Exception('Missing `key` of `key_file` parameter in config') + raise TransipConfigException( + 'Missing `key` of `key_file` parameter in config' + ) self.account = account self.key = key @@ -74,13 +88,12 @@ class TransipProvider(BaseProvider): zoneInfo = self._client.get_info(zone.name[:-1]) except WebFault as e: if e.fault.faultcode == '102' and target is False: - self.log.warning( - 'populate: (%s) Zone %s not found in account ', - e.fault.faultcode, zone.name) + # Zone not found in account, and not a target so just + # leave an empty zone. return exists elif e.fault.faultcode == '102' and target is True: self.log.warning('populate: Transip can\'t create new zones') - raise Exception( + raise TransipNewZoneException( ('populate: ({}) Transip used ' + 'as target for non-existing zone: {}').format( e.fault.faultcode, zone.name)) @@ -146,7 +159,7 @@ class TransipProvider(BaseProvider): self.log.warning(('_apply: Set DNS returned ' + 'one or more errors: {}').format( e.fault.faultstring)) - raise Exception(200, e.fault.faultstring) + raise TransipException(200, e.fault.faultstring) self._currentZone = {} @@ -220,7 +233,7 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): # Enforce switch from suds.sax.text.Text to string - value = '' + value + value = str(value) # TransIP allows '@' as value to alias the root record. # this provider won't set an '@' value, but can be an existing record @@ -245,7 +258,7 @@ class TransipProvider(BaseProvider): _values = [] for record in records: # Enforce switch from suds.sax.text.Text to string - _values.append('' + record['content']) + _values.append(str(record['content'])) return { 'ttl': self._get_lowest_ttl(records), diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index 8d85e1f..c56509a 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -170,7 +170,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd zone = Zone('notfound.unit.tests.', []) provider.populate(zone, True) - self.assertEquals(str('Exception'), + self.assertEquals(str('TransipNewZoneException'), str(ctx.exception.__class__.__name__)) self.assertEquals( @@ -271,5 +271,5 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd # Changes should not be set due to an Exception self.assertEqual([], changes) - self.assertEquals(str('Exception'), + self.assertEquals(str('TransipException'), str(ctx.exception.__class__.__name__)) From 97608b382399b798f7b865f082b943685b3e2fd9 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 09:29:30 -0700 Subject: [PATCH 82/86] v0.9.7 version bump and CHANGELOG updates --- .gitignore | 4 ++++ CHANGELOG.md | 8 ++++++++ octodns/__init__.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1efa084..715b687 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# +# Do not add editor or OS specific ignores here. Have a look at adding +# `excludesfile` to your `~/.gitconfig` to globally ignore such things. +# *.pyc .coverage .env diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f283ec..d80a48f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v0.9.7 - 2019-09-30 - It's about time + +* AkamaiProvider, ConstellixProvider, MythicBeastsProvider, SelectelProvider, + & TransipPovider providers added +* Route53Provider seperator fix +* YamlProvider export error around stringification +* PyPi markdown rendering fix + ## v0.9.6 - 2019-07-16 - The little one that fixes stuff from the big one * Reduced dynamic record value weight range to 0-15 so that Dyn and Route53 diff --git a/octodns/__init__.py b/octodns/__init__.py index 6422577..57300de 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.6' +__VERSION__ = '0.9.7' From a1d2217604fad7ec8c3fccc73f7d6e892071eeb3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 10:17:50 -0700 Subject: [PATCH 83/86] Fix/hack README rendering so that pypi's markdown handling libs are happy --- README.md | 4 ++-- setup.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7124b20..83f0bd1 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,8 @@ Now that we have something to tell OctoDNS about our providers & zones we need t ttl: 60 type: A values: - - 1.2.3.4 - - 1.2.3.5 + - 1.2.3.4 + - 1.2.3.5 ``` Further information can be found in [Records Documentation](/docs/records.md). diff --git a/setup.py b/setup.py index 75a39d7..5cb741b 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +from StringIO import StringIO from os.path import dirname, join import octodns @@ -21,6 +22,39 @@ console_scripts = { for name in cmds } + +def long_description(): + buf = StringIO() + yaml_block = False + supported_providers = False + with open('README.md') as fh: + for line in fh: + if line == '```yaml\n': + yaml_block = True + continue + elif yaml_block and line == '---\n': + # skip the line + continue + elif yaml_block and line == '```\n': + yaml_block = False + continue + elif supported_providers: + if line.startswith('## '): + supported_providers = False + # write this line out, no continue + else: + # We're ignoring this one + continue + elif line == '## Supported providers\n': + supported_providers = True + continue + buf.write(line) + buf = buf.getvalue() + with open('/tmp/mod', 'w') as fh: + fh.write(buf) + return buf + + setup( author='Ross McFarland', author_email='rwmcfa1@gmail.com', @@ -40,7 +74,7 @@ setup( 'requests>=2.20.0' ], license='MIT', - long_description=open('README.md').read(), + long_description=long_description(), long_description_content_type='text/markdown', name='octodns', packages=find_packages(), From e3ad57d15b055db9a441d0a5d7a7b924ff8600d5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 10:18:35 -0700 Subject: [PATCH 84/86] twine check before upload --- script/release | 1 + 1 file changed, 1 insertion(+) diff --git a/script/release b/script/release index dd3e1b1..f2c90bf 100755 --- a/script/release +++ b/script/release @@ -22,5 +22,6 @@ git tag -s "v$VERSION" -m "Release $VERSION" git push origin "v$VERSION" echo "Tagged and pushed v$VERSION" python setup.py sdist +twine check dist/*$VERSION.tar.gz twine upload dist/*$VERSION.tar.gz echo "Uploaded $VERSION" From 659a60de46ad0bb75811266691e57b44e58b46ff Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 10:18:53 -0700 Subject: [PATCH 85/86] v0.9.8 version bump and CHANGELOG updates --- CHANGELOG.md | 4 ++++ octodns/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d80a48f..76ff8b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.9.8 - 2019-09-30 - One with no changes b/c PyPi description problems + +* No material changes + ## v0.9.7 - 2019-09-30 - It's about time * AkamaiProvider, ConstellixProvider, MythicBeastsProvider, SelectelProvider, diff --git a/octodns/__init__.py b/octodns/__init__.py index 57300de..71b5b1a 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.7' +__VERSION__ = '0.9.8' From 3296b55aa62ccbe558dff22fd465d95cd0b7cfb1 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 10:22:05 -0700 Subject: [PATCH 86/86] Include readme_renderer[md] in dev requirements --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 77dd50c..a2833ae 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,5 +5,6 @@ pycodestyle==2.4.0 pycountry>=18.12.8 pycountry_convert>=0.7.2 pyflakes==1.6.0 +readme_renderer[md]==24.0 requests_mock twine==1.13.0