Added full support of Azure DNS. TODO: testing.

.gitignore

@@ -9,6 +9,7 @@ nosetests.xml
 octodns.egg-info/
 output/
 tmp/
-Makefile
 build/
-config/
+config/
+./rb.txt
+./doit.txt

doit.txt

@@ -0,0 +1,4 @@
+#!/bin/bash
+#script to rebuild octodns quickly
+
+octodns-sync --config-file=./config/production.yaml --doit

octodns/provider/azuredns.py

@@ -8,54 +8,66 @@ import sys
 
 from azure.common.credentials import ServicePrincipalCredentials
 from azure.mgmt.dns import DnsManagementClient
-from azure.mgmt.dns.models import *
+from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \
+    SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone
+
+from functools import reduce
 
-from collections import defaultdict
-# from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup.
 import logging
 import re
 from ..record import Record, Update
 from .base import BaseProvider
 
-
-#TODO: changes made to master include adding /build, Makefile to .gitignore and 
-# making Makefile.
-# Only made for A records. will have to adjust for more generic params types
 class _AzureRecord(object):
-    def __init__(self, resource_group, record, values=None, ttl=1800):
-        # print('Here4',file=sys.stderr)
+    ''' 
+        Wrapper for OctoDNS record.
+    azuredns.py:
+        class: octodns.provider.azuredns._AzureRecord
+        An _AzureRecord is easily accessible to the Azure DNS Management library
+        functions and is used to wrap all relevant data to create a record in 
+        Azure.
+    '''
+    
+    def __init__(self, resource_group, record, values=None):
+        ''' 
+            :param resource_group: The name of resource group in Azure
+            :type  resource_group: str 
+            :param record: An OctoDNS record 
+            :type  record: ..record.Record
+            :param values: Parameters for a record. eg IP address, port, domain
+                           name, etc. Values usually read from record.data
+            :type  values: {'values': [...]} or {'value': [...]}
+            
+            :type return: _AzureRecord
+        '''
         self.resource_group = resource_group
-        self.zone_name = record.zone.name[0:len(record.zone.name)-1] # strips last period
+        self.zone_name = record.zone.name[0:len(record.zone.name)-1]
         self.relative_record_set_name = record.name or '@'
         self.record_type = record._type
         
-        type_name = '{}records'.format(self.record_type).lower()
+        data = values or record.data
+        format_u_s = '' if record._type == 'A' else '_'
+        key_name ='{}{}records'.format(self.record_type, format_u_s).lower()
         class_name = '{}'.format(self.record_type).capitalize() + \
                      'Record'.format(self.record_type)
-        if values == None:
-            return
-        # TODO: clean up this bit.
-        data = values or record.data #This should fail if it gets to record.data? It only returns ttl. TODO
-        
-        #depending on mult values or not
-        #TODO: import explicitly. eval() uses for example ARecord from azure.mgmt.dns.models
-        self.params = {}
-        try:
-            self.params = {'ttl':record.ttl or ttl, \
-               type_name:[eval(class_name)(ip) for ip in data['values']] or []}
-        except KeyError: # means that doesn't have multiple values but single value
-            self.params = {'ttl':record.ttl or ttl, \
-               type_name:[eval(class_name)(data['value'])] or []}
+        self.params = None
+        if not self.record_type == 'CNAME':
+            self.params = self._params(data, key_name, eval(class_name))
+        else:
+            self.params = {'cname_record': CnameRecord(data['value'])}
+        self.params['ttl'] = record.ttl
 
+    def _params(self, data, key_name, azure_class):
+        return {key_name: [azure_class(v) for v in data['values']]} \
+            if 'values' in data else {key_name: [azure_class(data['value'])]}
 
-    
 
 class AzureProvider(BaseProvider):
     '''
     Azure DNS Provider
-    
-    azure.py:
-        class: octodns.provider.azure.AzureProvider
+
+    azuredns.py:
+        class: octodns.provider.azuredns.AzureProvider
         # Current support of authentication of access to Azure services only 
         # includes using a Service Principal: 
         # https://docs.microsoft.com/en-us/azure/azure-resource-manager/
@@ -71,90 +83,118 @@ class AzureProvider(BaseProvider):
         # Resource Group name req:
         resource_group:
         
-        testing: test authentication vars located in /home/t-hehwan/vars.txt
+        TODO: change the config file to use env variables instead of hard-coded keys?
+        
+        personal notes: testing: test authentication vars located in /home/t-hehwan/vars.txt
     '''
-    SUPPORTS_GEO = False # TODO. Will add support as project progresses.
+    SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT'))
     
-    def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs):
+    def __init__(self, id, client_id, key, directory_id, sub_id, resource_group,
+                 *args, **kwargs):
         self.log = logging.getLogger('AzureProvider[{}]'.format(id))
         self.log.debug('__init__: id=%s, client_id=%s, '
                 'key=***, directory_id:%s', id, client_id, directory_id)
         super(AzureProvider, self).__init__(id, *args, **kwargs)
 
         credentials = ServicePrincipalCredentials(
-        client_id = client_id, secret = key, tenant = directory_id
+            client_id, secret = key, tenant = directory_id
         )
         self._dns_client = DnsManagementClient(credentials, sub_id)
         self._resource_group = resource_group
-
-        self._azure_zones = None # will be a dictionary. key: name. val: id.
-        self._azure_records = {} # will be dict by octodns record, az record
-
-        self._supported_types = ['CNAME', 'A', 'AAAA', 'MX', 'SRV', 'NS', 'PTR']
-        # TODO: add TXT
-
-        # TODO: health checks a la route53.
-
-
+        self._azure_zones = set()
         
-    # TODO: add support for all types. First skeleton: add A.
-    def supports(self, record):
-        return record._type in self._supported_types
-        
-    @property
-    def azure_zones(self):
-        if self._azure_zones is None:
-            self.log.debug('azure_zones: loading')
-            zones = {}
-            for zone in self._dns_client.zones.list_by_resource_group(self._resource_group):
-                zones[zone.name] = zone.id
-            self._azure_zones = zones
-        return self._azure_zones
-        
-    # Given a zone name, returns the zone id. If DNE, creates it.
-    def _get_zone_id(self, name, create=False):
-        self.log.debug('_get_zone_id: name=%s', name)
+    def _populate_zones(self):
+        self.log.debug('azure_zones: loading')
+        for zone in self._dns_client.zones.list_by_resource_group(
+                                                          self._resource_group):
+            self._azure_zones.add(zone.name)
+
+    def _check_zone(self, name, create=False):
+        '''
+            Checks whether a zone specified in a source exist in Azure server.
+            Note that Azure zones omit end '.' eg: contoso.com vs contoso.com.
+            Returns the name if it exists.
+            
+            :param name: Name of a zone to checks
+            :type  name: str
+            :param create: If True, creates the zone of that name.
+            :type  create: bool
+            
+            :type return: str or None
+        '''
+        self.log.debug('_check_zone: name=%s', name)
         try:
-            id = self._dns_client.zones.get(self._resource_group, name)
-            self.log.debug('_get_zone_id: id=%s', id)
-            return id
+            if name in self._azure_zones:
+                return name
+            if self._dns_client.zones.get(self._resource_group, name):
+                self._azure_zones.add(name)
+                return name
         except:
             if create:
-                self.log.debug('_get_zone_id: no matching zone; creating %s', name)
-                #TODO: write
-                return None #placeholder
-            return None
+                try:
+                    self.log.debug('_check_zone: no matching zone; creating %s',
+                                                                           name)
+                    if self._dns_client.zones.create_or_update(
+                                    self._resource_group, name, Zone('global')): #TODO: figure out what location should be
+                        return name
+                except:
+                    raise
+        return None
         
-    # Create a dictionary of record objects by zone and octodns record names
-    # TODO: add geo parsing
-    def populate(self, zone, target):
-        zone_name = zone.name[0:len(zone.name)-1]#Azure zone names do not include suffix .
+    def populate(self, zone, target=False):
+        '''
+            Required function of manager.py.
+            
+            Special notes for Azure. Azure zone names omit final '.'
+            Azure record names for '' are represented by '@'
+            Azure records created through online interface may have null values
+            (eg, no IP address for A record). Specific quirks such as these are
+            responsible for any strange parsing.
+            
+            :param zone: A dns zone
+            :type  zone: octodns.zone.Zone
+            :param target: Checks if Azure is source or target of config. 
+                           Currently only supports as a target. Does not use.
+            :type  target: bool
+            
+            
+            TODO: azure interface allows null values. If this attempts to populate with them, will fail. add safety check (simply delete records with null values?)
+            
+            :type return: void
+        '''
+        zone_name = zone.name[0:len(zone.name)-1]
         self.log.debug('populate: name=%s', zone_name)
         before = len(zone.records)
-        zone_id = self._get_zone_id(zone_name) 
-        if zone_id:
-            #records = defaultdict(list)
-            for type in self._supported_types:
-                # print('populate. type: {}'.format(type),file=sys.stderr)
-                for azrecord in self._dns_client.record_sets.list_by_type(self._resource_group, zone_name, type):
-                    # print(azrecord, file=sys.stderr)
+
+        self._populate_zones()
+        if self._check_zone(zone_name):
+            for typ in self.SUPPORTS:
+                for azrecord in self._dns_client.record_sets.list_by_type(
+                                          self._resource_group, zone_name, typ):
                     record_name = azrecord.name if azrecord.name != '@' else ''
-                    data = self._type_and_ttl(type, azrecord, 
-                           getattr(self, '_data_for_{}'.format(type))(azrecord)) # TODO: azure online interface allows None values. must validate.
+                    data = self._type_and_ttl(typ, azrecord.ttl, 
+                           getattr(self, '_data_for_{}'.format(typ))(azrecord))
+                    
                     record = Record.new(zone, record_name, data, source=self)
-                    # print('HERE0',file=sys.stderr)
                     zone.add_record(record)
-                    self._azure_records[record] = _AzureRecord(self._resource_group, record, data)
-        # print('HERE1',file=sys.stderr)
+
         self.log.info('populate: found %s records', len(zone.records)-before)
-        
-    # might not need
-    def _get_type(azrecord):
-        azrecord['type'].split('/')[-1]
-        
-    def _type_and_ttl(self, type, azrecord, data):
-        data['type'] = type
-        data['ttl'] = azrecord.ttl
+
+    def _type_and_ttl(self, typ, ttl, data):
+        ''' Adds type and ttl fields to return dictionary.
+            
+            :param typ: The type of a record
+            :type  typ: str
+            :param ttl: The ttl of a record
+            :type  ttl: int
+            :param data: Dictionary holding values of a record. eg, IP addresses
+            :type  data: {'values': [...]} or {'value': [...]}
+            
+            :type return: {...}
+        '''
+        data['type'] = typ
+        data['ttl'] = ttl
         return data
         
     def _data_for_A(self, azrecord):
@@ -164,12 +204,10 @@ class AzureProvider(BaseProvider):
         return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]}
         
     def _data_for_TXT(self, azrecord):
-        print('azure',file=sys.stderr)
-        print([ar.value for ar in azrecord.txt_records], file=sys.stderr)
-        print('',file=sys.stderr)
-        return {'values': [ar.value for ar in azrecord.txt_records]}
+        return {'values': \
+            [reduce((lambda a,b:a+b), ar.value) for ar in azrecord.txt_records]}
 
-    def _data_for_CNAME(self, azrecord):
+    def _data_for_CNAME(self, azrecord): #TODO: see TODO in population comment.
         try:
             val = azrecord.cname_record.cname
             if not val.endswith('.'):
@@ -178,7 +216,7 @@ class AzureProvider(BaseProvider):
         except:
             return {'value': '.'} #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value.
            
-    def _data_for_PTR(self, azrecord):
+    def _data_for_PTR(self, azrecord): #TODO: see TODO in population comment.
         try:
             val = azrecord.ptr_records[0].ptdrname
             if not val.endswith('.'):
@@ -198,21 +236,23 @@ class AzureProvider(BaseProvider):
                         'target': ar.target} for ar in azrecord.srv_records]
         }
         
-    def _data_for_NS(self, azrecord):
+    def _data_for_NS(self, azrecord): #TODO: see TODO in population comment.
         def period_validate(string):
             return string if string.endswith('.') else string + '.'
         vals = [ar.nsdname for ar in azrecord.ns_records]
         return {'values': [period_validate(val) for val in vals]}
 
     def _apply_Create(self, change):
-        new = change.new
-
-        #validate that the zone exists.
-        #self._get_zone_id(new.name, create=True)
-        
-        ar = _AzureRecord(self._resource_group, new, new.data)
-        
+        ''' A record from change must be created.
+            
+            :param change: a change object
+            :type  change: octodns.record.Change
+            
+            :type return: void
+        '''
+        ar = _AzureRecord(self._resource_group, change.new)
         create = self._dns_client.record_sets.create_or_update
+        
         create(resource_group_name=ar.resource_group, 
                zone_name=ar.zone_name, 
                relative_record_set_name=ar.relative_record_set_name, 
@@ -220,56 +260,33 @@ class AzureProvider(BaseProvider):
                parameters=ar.params)
                
     def _apply_Delete(self, change):
-        existing = change.existing
-        ar = _AzureRecord(self._resource_group, existing)
+        ar = _AzureRecord(self._resource_group, change.existing)
         delete = self._dns_client.record_sets.delete
+        
         delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, 
                ar.record_type)
     
     def _apply_Update(self, change):
-        self._apply_Delete(change)
         self._apply_Create(change)
     
-    # type plan: Plan class from .base
     def _apply(self, plan):
+        '''
+            Required function of manager.py
+            
+            :param plan: Contains the zones and changes to be made
+            :type  plan: octodns.provider.base.Plan
+            
+            :type return: void
+        '''
         desired = plan.desired
         changes = plan.changes
         self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, 
                        len(changes))
         
-        # validate that the zone exists. function creates zone if DNE.
-        self._get_zone_id(desired.name)
+        azure_zone_name = desired.name[0:len(desired.name)-1]
+        self._check_zone(azure_zone_name, create=True)
         
-        # Some parsing bits to call _mod_Create or _mod_Delete.
-        # changes is a list of Delete and Create objects.
         for change in changes:
             class_name = change.__class__.__name__
             getattr(self, '_apply_{}'.format(class_name))(change)
 
-    
-    
-    # **********
-    # Figuring out what object plan is.
-    
-    # self._executor = ThreadPoolExecutor(max_workers)
-    # futures.append(self._executor.submit(self._populate_and_plan,
-                                     # zone_name, sources, targets))
-    # plans = [p for f in futures for p in f.results()]
-    # type of plans[0] == type of one output of _populate_and_plan
-    
-    # for target, plan in plans:
-        # apply(plan)
-        
-        
-    # type(target) == BaseProvider
-    # type(plan) == Plan()
-    
-    # Plan(existing, desired, changes)
-        # existing.type == desired.type == Zone(desired.name, desired.sub_zones)
-            # Zone(name, sub_zones) (str and set of strs)
-        # changes.type = [Delete/Create]
-    
-
-    # Starts with sync in main() of sync.
-    # {u'values': ['3.3.3.3', '4.4.4.4'], u'type': 'A', u'ttl': 3600}
-    # {u'type': u'A', u'value': [u'3.3.3.3', u'4.4.4.4'], u'ttl': 3600L}