Fixed bug for MX and SRV. Added Azure test suite as well.

octodns/provider/azuredns.py

@@ -4,7 +4,6 @@
 
 from __future__ import absolute_import, division, print_function, \
     unicode_literals
-import sys
 
 from azure.common.credentials import ServicePrincipalCredentials
 from azure.mgmt.dns import DnsManagementClient
@@ -14,53 +13,120 @@ from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \
 from functools import reduce
 
 import logging
-import re
-from ..record import Record, Update
+from ..record import Record
 from .base import BaseProvider
 
+
 class _AzureRecord(object):
-    ''' 
-        Wrapper for OctoDNS record.
-    azuredns.py:
+    ''' Wrapper for OctoDNS record.
+        azuredns.py:
         class: octodns.provider.azuredns._AzureRecord
-        An _AzureRecord is easily accessible to the Azure DNS Management library
-        functions and is used to wrap all relevant data to create a record in 
+        An _AzureRecord is easily accessible to Azure DNS Management library
+        functions and is used to wrap all relevant data to create a record in
         Azure.
     '''
-    
+
     def __init__(self, resource_group, record, values=None):
-        ''' 
+        '''
             :param resource_group: The name of resource group in Azure
-            :type  resource_group: str 
-            :param record: An OctoDNS record 
+            :type  resource_group: str
+            :param record: An OctoDNS record
             :type  record: ..record.Record
             :param values: Parameters for a record. eg IP address, port, domain
                            name, etc. Values usually read from record.data
             :type  values: {'values': [...]} or {'value': [...]}
-            
+
             :type return: _AzureRecord
         '''
         self.resource_group = resource_group
-        self.zone_name = record.zone.name[0:len(record.zone.name)-1]
+        self.zone_name = record.zone.name[0:len(record.zone.name) - 1]
         self.relative_record_set_name = record.name or '@'
         self.record_type = record._type
-        
+
         data = values or record.data
         format_u_s = '' if record._type == 'A' else '_'
-        key_name ='{}{}records'.format(self.record_type, format_u_s).lower()
+        key_name = '{}{}records'.format(self.record_type, format_u_s).lower()
         class_name = '{}'.format(self.record_type).capitalize() + \
                      'Record'.format(self.record_type)
-        self.params = None
-        if not self.record_type == 'CNAME':
-            self.params = self._params(data, key_name, eval(class_name))
-        else:
-            self.params = {'cname_record': CnameRecord(data['value'])}
+
+        self.params = getattr(self, '_params_for_{}'.format(record._type))
+        self.params = self.params(data, key_name, eval(class_name))
         self.params['ttl'] = record.ttl
 
     def _params(self, data, key_name, azure_class):
         return {key_name: [azure_class(v) for v in data['values']]} \
             if 'values' in data else {key_name: [azure_class(data['value'])]}
 
+    _params_for_A = _params
+    _params_for_AAAA = _params
+    _params_for_NS = _params
+    _params_for_PTR = _params
+    _params_for_TXT = _params
+
+    def _params_for_SRV(self, data, key_name, azure_class):
+        params = []
+        if 'values' in data:
+            for vals in data['values']:
+                params.append(azure_class(vals['priority'],
+                                          vals['weight'],
+                                          vals['port'],
+                                          vals['target']))
+        else:
+            params.append(azure_class(data['value']['priority'],
+                                      data['value']['weight'],
+                                      data['value']['port'],
+                                      data['value']['target']))
+        return {key_name: params}
+
+    def _params_for_MX(self, data, key_name, azure_class):
+        params = []
+        if 'values' in data:
+            for vals in data['values']:
+                params.append(azure_class(vals['priority'],
+                                          vals['value']))
+        else:
+            params.append(azure_class(data['value']['priority'],
+                                      data['value']['value']))
+        return {key_name: params}
+
+    def _params_for_CNAME(self, data, key_name, azure_class):
+        return {'cname_record': CnameRecord(data['value'])}
+
+    def _equals(self, b):
+        def parse_dict(params):
+            vals = []
+            for char in params:
+                if char != 'ttl':
+                    list_records = params[char]
+                    try:
+                        for record in list_records:
+                            vals.append(record.__dict__)
+                    except:
+                        vals.append(list_records.__dict__)
+            vals.sort()
+            return vals
+
+        return (self.resource_group == b.resource_group) & \
+               (self.zone_name == b.zone_name) & \
+               (self.record_type == b.record_type) & \
+               (self.params['ttl'] == b.params['ttl']) & \
+               (parse_dict(self.params) == parse_dict(b.params)) & \
+               (self.relative_record_set_name == b.relative_record_set_name)
+
+    def __str__(self):
+        string = 'Zone: {}; '.format(self.zone_name)
+        string += 'Name: {}; '.format(self.relative_record_set_name)
+        string += 'Type: {}; '.format(self.record_type)
+        string += 'Ttl: {}; '.format(self.params['ttl'])
+        for char in self.params:
+            if char != 'ttl':
+                try:
+                    for rec in self.params[char]:
+                        string += 'Record: {}; '.format(rec.__dict__)
+                except:
+                    string += 'Record: {}; '.format(self.params[char].__dict__)
+        return string
+
 
 class AzureProvider(BaseProvider):
     '''
@@ -68,11 +134,11 @@ class AzureProvider(BaseProvider):
 
     azuredns.py:
         class: octodns.provider.azuredns.AzureProvider
-        # Current support of authentication of access to Azure services only 
-        # includes using a Service Principal: 
+        # Current support of authentication of access to Azure services only
+        # includes using a Service Principal:
         # https://docs.microsoft.com/en-us/azure/azure-resource-manager/
         #                        resource-group-create-service-principal-portal
-        # The Azure Active Directory Application ID (referred to client ID) req:
+        # The Azure Active Directory Application ID (aka client ID) req:
         client_id:
         # Authentication Key Value req:
         key:
@@ -82,32 +148,33 @@ class AzureProvider(BaseProvider):
         sub_id:
         # Resource Group name req:
         resource_group:
-        
-        TODO: change the config file to use env variables instead of hard-coded keys?
-        
-        personal notes: testing: test authentication vars located in /home/t-hehwan/vars.txt
+
+        TODO: change config file to use env vars instead of hard-coded keys
+
+        personal notes: testing: test authentication vars located in
+                /home/t-hehwan/vars.txt
     '''
     SUPPORTS_GEO = False
     SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT'))
-    
-    def __init__(self, id, client_id, key, directory_id, sub_id, resource_group,
-                 *args, **kwargs):
+
+    def __init__(self, id, client_id, key, directory_id, sub_id,
+                 resource_group, *args, **kwargs):
         self.log = logging.getLogger('AzureProvider[{}]'.format(id))
         self.log.debug('__init__: id=%s, client_id=%s, '
-                'key=***, directory_id:%s', id, client_id, directory_id)
+                       'key=***, directory_id:%s', id, client_id, directory_id)
         super(AzureProvider, self).__init__(id, *args, **kwargs)
 
         credentials = ServicePrincipalCredentials(
-            client_id, secret = key, tenant = directory_id
+            client_id, secret=key, tenant=directory_id
         )
         self._dns_client = DnsManagementClient(credentials, sub_id)
         self._resource_group = resource_group
         self._azure_zones = set()
-        
+
     def _populate_zones(self):
         self.log.debug('azure_zones: loading')
-        for zone in self._dns_client.zones.list_by_resource_group(
-                                                          self._resource_group):
+        list_zones = self._dns_client.zones.list_by_resource_group
+        for zone in list_zones(self._resource_group):
             self._azure_zones.add(zone.name)
 
     def _check_zone(self, name, create=False):
@@ -115,12 +182,12 @@ class AzureProvider(BaseProvider):
             Checks whether a zone specified in a source exist in Azure server.
             Note that Azure zones omit end '.' eg: contoso.com vs contoso.com.
             Returns the name if it exists.
-            
+
             :param name: Name of a zone to checks
             :type  name: str
             :param create: If True, creates the zone of that name.
             :type  create: bool
-            
+
             :type return: str or None
         '''
         self.log.debug('_check_zone: name=%s', name)
@@ -130,163 +197,151 @@ class AzureProvider(BaseProvider):
             if self._dns_client.zones.get(self._resource_group, name):
                 self._azure_zones.add(name)
                 return name
-        except:
+        except:  # TODO: figure out what location should be
             if create:
                 try:
-                    self.log.debug('_check_zone: no matching zone; creating %s',
-                                                                           name)
-                    if self._dns_client.zones.create_or_update(
-                                    self._resource_group, name, Zone('global')): #TODO: figure out what location should be
+                    self.log.debug('_check_zone:no matching zone; creating %s',
+                                   name)
+                    create_zone = self._dns_client.zones.create_or_update
+                    if create_zone(self._resource_group, name, Zone('global')):
                         return name
                 except:
                     raise
         return None
-        
-    def populate(self, zone, target=False):
+
+    def populate(self, zone, target=False, lenient=False):
         '''
             Required function of manager.py.
-            
+
             Special notes for Azure. Azure zone names omit final '.'
             Azure record names for '' are represented by '@'
             Azure records created through online interface may have null values
             (eg, no IP address for A record). Specific quirks such as these are
             responsible for any strange parsing.
-            
+
             :param zone: A dns zone
             :type  zone: octodns.zone.Zone
-            :param target: Checks if Azure is source or target of config. 
+            :param target: Checks if Azure is source or target of config.
                            Currently only supports as a target. Does not use.
             :type  target: bool
-            
-            
-            TODO: azure interface allows null values. If this attempts to populate with them, will fail. add safety check (simply delete records with null values?)
-            
+
+
+            TODO: azure interface allows null values. If this attempts to
+            populate with them, will fail. add safety check (simply delete
+            records with null values?)
+
             :type return: void
         '''
-        zone_name = zone.name[0:len(zone.name)-1]
+        zone_name = zone.name[0:len(zone.name) - 1]
         self.log.debug('populate: name=%s', zone_name)
         before = len(zone.records)
 
         self._populate_zones()
         if self._check_zone(zone_name):
             for typ in self.SUPPORTS:
-                for azrecord in self._dns_client.record_sets.list_by_type(
-                                          self._resource_group, zone_name, typ):
+                records = self._dns_client.record_sets.list_by_type
+                for azrecord in records(self._resource_group, zone_name, typ):
                     record_name = azrecord.name if azrecord.name != '@' else ''
-                    data = self._type_and_ttl(typ, azrecord.ttl, 
-                           getattr(self, '_data_for_{}'.format(typ))(azrecord))
-                    
+                    data = getattr(self, '_data_for_{}'.format(typ))(azrecord)
+                    data['type'] = typ
+                    data['ttl'] = azrecord.ttl
+
                     record = Record.new(zone, record_name, data, source=self)
                     zone.add_record(record)
 
-        self.log.info('populate: found %s records', len(zone.records)-before)
-
-    def _type_and_ttl(self, typ, ttl, data):
-        ''' Adds type and ttl fields to return dictionary.
-            
-            :param typ: The type of a record
-            :type  typ: str
-            :param ttl: The ttl of a record
-            :type  ttl: int
-            :param data: Dictionary holding values of a record. eg, IP addresses
-            :type  data: {'values': [...]} or {'value': [...]}
-            
-            :type return: {...}
-        '''
-        data['type'] = typ
-        data['ttl'] = ttl
-        return data
-        
+        self.log.info('populate: found %s records', len(zone.records) - before)
+
     def _data_for_A(self, azrecord):
         return {'values': [ar.ipv4_address for ar in azrecord.arecords]}
-        
+
     def _data_for_AAAA(self, azrecord):
         return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]}
-        
+
     def _data_for_TXT(self, azrecord):
-        return {'values': \
-            [reduce((lambda a,b:a+b), ar.value) for ar in azrecord.txt_records]}
+        return {'values': [reduce((lambda a, b: a + b), ar.value)
+                           for ar in azrecord.txt_records]}
 
-    def _data_for_CNAME(self, azrecord): #TODO: see TODO in population comment.
+    def _data_for_CNAME(self, azrecord):  # TODO: see TODO in pop comment.
         try:
             val = azrecord.cname_record.cname
             if not val.endswith('.'):
                 val += '.'
             return {'value': val}
         except:
-            return {'value': '.'} #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value.
-           
-    def _data_for_PTR(self, azrecord): #TODO: see TODO in population comment.
+            return {'value': '.'}  # TODO: this is a bad fix. but octo checks
+            # that cnames have trailing '.' while azure allows creating cnames
+            # on the online interface with no value.
+
+    def _data_for_PTR(self, azrecord):  # TODO: see TODO in population comment.
         try:
             val = azrecord.ptr_records[0].ptdrname
             if not val.endswith('.'):
                 val += '.'
             return {'value': val}
         except:
-            return {'value': '.' } #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value.
-        
+            return {'value': '.'}
+
     def _data_for_MX(self, azrecord):
-        return {'values': [{'priority':ar.preference,
-                            'value':ar.exchange} for ar in azrecord.mx_records]}
-                            
+        return {'values': [{'priority': ar.preference, 'value': ar.exchange}
+                           for ar in azrecord.mx_records]
+                }
+
     def _data_for_SRV(self, azrecord):
-        return {'values': [{'priority': ar.priority,
-                        'weight': ar.weight,
-                        'port': ar.port,
-                        'target': ar.target} for ar in azrecord.srv_records]
-        }
-        
-    def _data_for_NS(self, azrecord): #TODO: see TODO in population comment.
+        return {'values': [{'priority': ar.priority, 'weight': ar.weight,
+                            'port': ar.port, 'target': ar.target}
+                           for ar in azrecord.srv_records]
+                }
+
+    def _data_for_NS(self, azrecord):  # TODO: see TODO in population comment.
         def period_validate(string):
             return string if string.endswith('.') else string + '.'
         vals = [ar.nsdname for ar in azrecord.ns_records]
         return {'values': [period_validate(val) for val in vals]}
 
     def _apply_Create(self, change):
-        ''' A record from change must be created.
-            
+        '''A record from change must be created.
+
             :param change: a change object
             :type  change: octodns.record.Change
-            
+
             :type return: void
         '''
         ar = _AzureRecord(self._resource_group, change.new)
         create = self._dns_client.record_sets.create_or_update
-        
-        create(resource_group_name=ar.resource_group, 
-               zone_name=ar.zone_name, 
-               relative_record_set_name=ar.relative_record_set_name, 
-               record_type=ar.record_type, 
+
+        create(resource_group_name=ar.resource_group,
+               zone_name=ar.zone_name,
+               relative_record_set_name=ar.relative_record_set_name,
+               record_type=ar.record_type,
                parameters=ar.params)
-               
+
     def _apply_Delete(self, change):
         ar = _AzureRecord(self._resource_group, change.existing)
         delete = self._dns_client.record_sets.delete
-        
-        delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, 
+
+        delete(self._resource_group, ar.zone_name, ar.relative_record_set_name,
                ar.record_type)
-    
+
     def _apply_Update(self, change):
         self._apply_Create(change)
-    
+
     def _apply(self, plan):
         '''
             Required function of manager.py
-            
+
             :param plan: Contains the zones and changes to be made
             :type  plan: octodns.provider.base.Plan
-            
+
             :type return: void
         '''
         desired = plan.desired
         changes = plan.changes
-        self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, 
+        self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name,
                        len(changes))
-        
-        azure_zone_name = desired.name[0:len(desired.name)-1]
+
+        azure_zone_name = desired.name[0:len(desired.name) - 1]
         self._check_zone(azure_zone_name, create=True)
-        
+
         for change in changes:
             class_name = change.__class__.__name__
             getattr(self, '_apply_{}'.format(class_name))(change)
-

rb.txt

@@ -1,8 +0,0 @@
-#!/bin/bash
-#script to rebuild octodns quickly
-
-sudo rm -r /home/t-hehwan/GitHub/octodns/build
-sudo rm -r /home/t-hehwan/GitHub/octodns/octodns.egg-info
-sudo python /home/t-hehwan/GitHub/octodns/setup.py -q build
-sudo python /home/t-hehwan/GitHub/octodns/setup.py -q install
-octodns-sync --config-file=./config/production.yaml

tests/test_octodns_provider_azuredns.py

@@ -0,0 +1,171 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from octodns.record import Create, Delete, Record, Update
+from octodns.provider.azuredns import _AzureRecord, AzureProvider
+from octodns.zone import Zone
+
+from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \
+    SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone as AzureZone
+
+from octodns.zone import Zone
+
+from unittest import TestCase
+import sys
+
+
+class Test_AzureRecord(TestCase):
+    zone = Zone(name='unit.tests.', sub_zones=[])
+    octo_records = []
+    octo_records.append(Record.new(zone, '', {
+        'ttl': 0,
+        'type': 'A',
+        'values': ['1.2.3.4', '10.10.10.10']
+    }))
+    octo_records.append(Record.new(zone, 'a', {
+        'ttl': 1,
+        'type': 'A',
+        'values': ['1.2.3.4', '1.1.1.1'],
+    }))
+    octo_records.append(Record.new(zone, 'aa', {
+        'ttl': 9001,
+        'type': 'A',
+        'values': ['1.2.4.3']
+    }))
+    octo_records.append(Record.new(zone, 'aaa', {
+        'ttl': 2,
+        'type': 'A',
+        'values': ['1.1.1.3']
+    }))
+    octo_records.append(Record.new(zone, 'cname', {
+        'ttl': 3,
+        'type': 'CNAME',
+        'value': 'a.unit.tests.',
+    }))
+    octo_records.append(Record.new(zone, '', {
+        'ttl': 3,
+        'type': 'MX',
+        'values': [{
+            'priority': 10,
+            'value': 'mx1.unit.tests.',
+        }, {
+            'priority': 20,
+            'value': 'mx2.unit.tests.',
+        }]
+    }))
+    octo_records.append(Record.new(zone, '', {
+        'ttl': 4,
+        'type': 'NS',
+        'values': ['ns1.unit.tests.', 'ns2.unit.tests.'],
+    }))
+    octo_records.append(Record.new(zone, '', {
+        'ttl': 5,
+        'type': 'NS',
+        'value': 'ns1.unit.tests.',
+    }))
+    octo_records.append(Record.new(zone, '_srv._tcp', {
+        'ttl': 6,
+        'type': 'SRV',
+        'values': [{
+            'priority': 10,
+            'weight': 20,
+            'port': 30,
+            'target': 'foo-1.unit.tests.',
+        }, {
+            'priority': 12,
+            'weight': 30,
+            'port': 30,
+            'target': 'foo-2.unit.tests.',
+        }]
+    }))
+
+    azure_records = []
+    _base0 = _AzureRecord('TestAzure', octo_records[0])
+    _base0.zone_name = 'unit.tests'
+    _base0.relative_record_set_name = '@'
+    _base0.record_type = 'A'
+    _base0.params['ttl'] = 0
+    _base0.params['arecords'] = [ARecord('1.2.3.4'), ARecord('10.10.10.10')]
+    azure_records.append(_base0)
+
+    _base1 = _AzureRecord('TestAzure', octo_records[1])
+    _base1.zone_name = 'unit.tests'
+    _base1.relative_record_set_name = 'a'
+    _base1.record_type = 'A'
+    _base1.params['ttl'] = 1
+    _base1.params['arecords'] = [ARecord('1.2.3.4'), ARecord('1.1.1.1')]
+    azure_records.append(_base1)
+
+    _base2 = _AzureRecord('TestAzure', octo_records[2])
+    _base2.zone_name = 'unit.tests'
+    _base2.relative_record_set_name = 'aa'
+    _base2.record_type = 'A'
+    _base2.params['ttl'] = 9001
+    _base2.params['arecords'] = ARecord('1.2.4.3')
+    azure_records.append(_base2)
+
+    _base3 = _AzureRecord('TestAzure', octo_records[3])
+    _base3.zone_name = 'unit.tests'
+    _base3.relative_record_set_name = 'aaa'
+    _base3.record_type = 'A'
+    _base3.params['ttl'] = 2
+    _base3.params['arecords'] = ARecord('1.1.1.3')
+    azure_records.append(_base3)
+
+    _base4 = _AzureRecord('TestAzure', octo_records[4])
+    _base4.zone_name = 'unit.tests'
+    _base4.relative_record_set_name = 'cname'
+    _base4.record_type = 'CNAME'
+    _base4.params['ttl'] = 3
+    _base4.params['cname_record'] = CnameRecord('a.unit.tests.')
+    azure_records.append(_base4)
+
+    _base5 = _AzureRecord('TestAzure', octo_records[5])
+    _base5.zone_name = 'unit.tests'
+    _base5.relative_record_set_name = '@'
+    _base5.record_type = 'MX'
+    _base5.params['ttl'] = 3
+    _base5.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.'),
+                                   MxRecord(20, 'mx2.unit.tests.')]
+    azure_records.append(_base5)
+
+    _base6 = _AzureRecord('TestAzure', octo_records[6])
+    _base6.zone_name = 'unit.tests'
+    _base6.relative_record_set_name = '@'
+    _base6.record_type = 'NS'
+    _base6.params['ttl'] = 4
+    _base6.params['ns_records'] = [NsRecord('ns1.unit.tests.'),
+                                   NsRecord('ns2.unit.tests.')]
+    azure_records.append(_base6)
+
+    _base7 = _AzureRecord('TestAzure', octo_records[7])
+    _base7.zone_name = 'unit.tests'
+    _base7.relative_record_set_name = '@'
+    _base7.record_type = 'NS'
+    _base7.params['ttl'] = 5
+    _base7.params['ns_records'] = [NsRecord('ns1.unit.tests.')]
+    azure_records.append(_base7)
+
+    _base8 = _AzureRecord('TestAzure', octo_records[8])
+    _base8.zone_name = 'unit.tests'
+    _base8.relative_record_set_name = '_srv._tcp'
+    _base8.record_type = 'SRV'
+    _base8.params['ttl'] = 6
+    _base8.params['srv_records'] = [SrvRecord(10, 20, 30, 'foo-1.unit.tests.'),
+                                    SrvRecord(12, 30, 30, 'foo-2.unit.tests.')]
+    azure_records.append(_base8)
+
+    def test_azure_record(self):
+        assert(len(self.azure_records) == len(self.octo_records))
+        for i in range(len(self.azure_records)):
+            octo = _AzureRecord('TestAzure', self.octo_records[i])
+            assert(self.azure_records[i]._equals(octo))
+
+
+class TestAzureDnsProvider(TestCase):
+    def test_populate(self):
+        pass  # placeholder