Merge remote-tracking branch 'origin/master' into alias-root-only

README.md

@@ -79,6 +79,9 @@ zones:
     targets:
       - dyn
       - route53
+
+  example.net:
+    alias: example.com.
 ```
 
 `class` is a special key that tells OctoDNS what python class should be loaded. Any other keys will be passed as configuration values to that provider. In general any sensitive or frequently rotated values should come from environmental variables. When OctoDNS sees a value that starts with `env/` it will look for that value in the process's environment and pass the result along.
@@ -87,6 +90,8 @@ Further information can be found in the `docstring` of each source and provider
 
 The `max_workers` key in the `manager` section of the config enables threading to parallelize the planning portion of the sync.
 
+In this example, `example.net` is an alias of zone `example.com`, which means they share the same sources and targets. They will therefore have identical records.
+
 Now that we have something to tell OctoDNS about our providers & zones we need to tell it about or records. We'll keep it simple for now and just create a single `A` record at the top-level of the domain.
 
 `config/example.com.yaml`

octodns/cmds/report.py

@@ -17,7 +17,6 @@ from six import text_type
 
 from octodns.cmds.args import ArgumentParser
 from octodns.manager import Manager
-from octodns.zone import Zone
 
 
 class AsyncResolver(Resolver):
@@ -56,7 +55,7 @@ def main():
     except KeyError as e:
         raise Exception('Unknown source: {}'.format(e.args[0]))
 
-    zone = Zone(args.zone, manager.configured_sub_zones(args.zone))
+    zone = manager.get_zone(args.zone)
     for source in sources:
         source.populate(zone)
 

octodns/manager.py

@@ -222,21 +222,31 @@ class Manager(object):
         self.log.debug('configured_sub_zones: subs=%s', sub_zone_names)
         return set(sub_zone_names)
 
-    def _populate_and_plan(self, zone_name, sources, targets, lenient=False):
+    def _populate_and_plan(self, zone_name, sources, targets, desired=None,
+                           lenient=False):
 
         self.log.debug('sync:   populating, zone=%s, lenient=%s',
                        zone_name, lenient)
         zone = Zone(zone_name,
                     sub_zones=self.configured_sub_zones(zone_name))
-        for source in sources:
-            try:
-                source.populate(zone, lenient=lenient)
-            except TypeError as e:
-                if "keyword argument 'lenient'" not in text_type(e):
-                    raise
-                self.log.warn(': provider %s does not accept lenient param',
-                              source.__class__.__name__)
-                source.populate(zone)
+
+        if desired:
+            # This is an alias zone, rather than populate it we'll copy the
+            # records over from `desired`.
+            for _, records in desired._records.items():
+                for record in records:
+                    zone.add_record(record.copy(zone=zone), lenient=lenient)
+
+        else:
+            for source in sources:
+                try:
+                    source.populate(zone, lenient=lenient)
+                except TypeError as e:
+                    if "keyword argument 'lenient'" not in text_type(e):
+                        raise
+                    self.log.warn(': provider %s does not accept lenient '
+                                  'param', source.__class__.__name__)
+                    source.populate(zone)
 
         self.log.debug('sync:   planning, zone=%s', zone_name)
         plans = []
@@ -253,7 +263,8 @@ class Manager(object):
             if plan:
                 plans.append((target, plan))
 
-        return plans
+        # Return the zone as it's the desired state
+        return plans, zone
 
     def sync(self, eligible_zones=[], eligible_sources=[], eligible_targets=[],
              dry_run=True, force=False):
@@ -265,9 +276,32 @@ class Manager(object):
         if eligible_zones:
             zones = [z for z in zones if z[0] in eligible_zones]
 
+        aliased_zones  = {}
         futures = []
         for zone_name, config in zones:
             self.log.info('sync:   zone=%s', zone_name)
+            if 'alias' in config:
+                source_zone = config['alias']
+
+                # Check that the source zone is defined.
+                if source_zone not in self.config['zones']:
+                    self.log.error('Invalid alias zone {}, target {} does '
+                                   'not exist'.format(zone_name, source_zone))
+                    raise ManagerException('Invalid alias zone {}: '
+                                           'source zone {} does not exist'
+                                           .format(zone_name, source_zone))
+
+                # Check that the source zone is not an alias zone itself.
+                if 'alias' in self.config['zones'][source_zone]:
+                    self.log.error('Invalid alias zone {}, target {} is an '
+                                   'alias zone'.format(zone_name, source_zone))
+                    raise ManagerException('Invalid alias zone {}: source '
+                                           'zone {} is an alias zone'
+                                           .format(zone_name, source_zone))
+
+                aliased_zones[zone_name] = source_zone
+                continue
+
             lenient = config.get('lenient', False)
             try:
                 sources = config['sources']
@@ -327,9 +361,32 @@ class Manager(object):
                                                  zone_name, sources,
                                                  targets, lenient=lenient))
 
-        # Wait on all results and unpack/flatten them in to a list of target &
-        # plan pairs.
-        plans = [p for f in futures for p in f.result()]
+        # Wait on all results and unpack/flatten the plans and store the
+        # desired states in case we need them below
+        plans = []
+        desired = {}
+        for future in futures:
+            ps, d = future.result()
+            desired[d.name] = d
+            for plan in ps:
+                plans.append(plan)
+
+        # Populate aliases zones.
+        futures = []
+        for zone_name, zone_source in aliased_zones.items():
+            source_config = self.config['zones'][zone_source]
+            futures.append(self._executor.submit(
+                self._populate_and_plan,
+                zone_name,
+                [],
+                [self.providers[t] for t in source_config['targets']],
+                desired=desired[zone_source],
+                lenient=lenient
+            ))
+
+        # Wait on results and unpack/flatten the plans, ignore the desired here
+        # as these are aliased zones
+        plans += [p for f in futures for p in f.result()[0]]
 
         # Best effort sort plans children first so that we create/update
         # children zones before parents which should allow us to more safely
@@ -377,12 +434,11 @@ class Manager(object):
         except KeyError as e:
             raise ManagerException('Unknown source: {}'.format(e.args[0]))
 
-        sub_zones = self.configured_sub_zones(zone)
-        za = Zone(zone, sub_zones)
+        za = self.get_zone(zone)
         for source in a:
             source.populate(za)
 
-        zb = Zone(zone, sub_zones)
+        zb = self.get_zone(zone)
         for source in b:
             source.populate(zb)
 
@@ -421,6 +477,25 @@ class Manager(object):
         for zone_name, config in self.config['zones'].items():
             zone = Zone(zone_name, self.configured_sub_zones(zone_name))
 
+            source_zone = config.get('alias')
+            if source_zone:
+                if source_zone not in self.config['zones']:
+                    self.log.exception('Invalid alias zone')
+                    raise ManagerException('Invalid alias zone {}: '
+                                           'source zone {} does not exist'
+                                           .format(zone_name, source_zone))
+
+                if 'alias' in self.config['zones'][source_zone]:
+                    self.log.exception('Invalid alias zone')
+                    raise ManagerException('Invalid alias zone {}: '
+                                           'source zone {} is an alias zone'
+                                           .format(zone_name, source_zone))
+
+                # this is just here to satisfy coverage, see
+                # https://github.com/nedbat/coveragepy/issues/198
+                source_zone = source_zone
+                continue
+
             try:
                 sources = config['sources']
             except KeyError:
@@ -428,9 +503,9 @@ class Manager(object):
                                        .format(zone_name))
 
             try:
-                # rather than using a list comprehension, we break this loop
-                # out so that the `except` block below can reference the
-                # `source`
+                # rather than using a list comprehension, we break this
+                # loop out so that the `except` block below can reference
+                # the `source`
                 collected = []
                 for source in sources:
                     collected.append(self.providers[source])
@@ -442,3 +517,14 @@ class Manager(object):
             for source in sources:
                 if isinstance(source, YamlProvider):
                     source.populate(zone)
+
+    def get_zone(self, zone_name):
+        if not zone_name[-1] == '.':
+            raise ManagerException('Invalid zone name {}, missing ending dot'
+                                   .format(zone_name))
+
+        for name, config in self.config['zones'].items():
+            if name == zone_name:
+                return Zone(name, self.configured_sub_zones(name))
+
+        raise ManagerException('Unknown zone name {}'.format(zone_name))

octodns/record/__init__.py

@@ -219,6 +219,18 @@ class Record(EqualityTupleMixin):
         if self.ttl != other.ttl:
             return Update(self, other)
 
+    def copy(self, zone=None):
+        data = self.data
+        data['type'] = self._type
+
+        return Record.new(
+            zone if zone else self.zone,
+            self.name,
+            data,
+            self.source,
+            lenient=True
+        )
+
     # NOTE: we're using __hash__ and ordering methods that consider Records
     # equivalent if they have the same name & _type. Values are ignored. This
     # is useful when computing diffs/changes.

tests/config/alias-zone-loop.yaml

@@ -0,0 +1,21 @@
+manager:
+  max_workers: 2
+providers:
+  in:
+    class: octodns.provider.yaml.YamlProvider
+    directory: tests/config
+  dump:
+    class: octodns.provider.yaml.YamlProvider
+    directory: env/YAML_TMP_DIR
+zones:
+  unit.tests.:
+    sources:
+      - in
+    targets:
+      - dump
+  
+  alias.tests.:
+    alias: unit.tests.
+
+  alias-loop.tests.:
+    alias: alias.tests.

tests/config/simple-alias-zone.yaml

@@ -0,0 +1,19 @@
+manager:
+  max_workers: 2
+providers:
+  in:
+    class: octodns.provider.yaml.YamlProvider
+    directory: tests/config
+  dump:
+    class: octodns.provider.yaml.YamlProvider
+    directory: env/YAML_TMP_DIR
+zones:
+  unit.tests.:
+    sources:
+      - in
+    targets:
+      - dump
+  
+  alias.tests.:
+    alias: unit.tests.
+

tests/config/unknown-source-zone.yaml

@@ -0,0 +1,18 @@
+manager:
+  max_workers: 2
+providers:
+  in:
+    class: octodns.provider.yaml.YamlProvider
+    directory: tests/config
+  dump:
+    class: octodns.provider.yaml.YamlProvider
+    directory: env/YAML_TMP_DIR
+zones:
+  unit.tests.:
+    sources:
+      - in
+    targets:
+      - dump
+  
+  alias.tests.:
+    alias: does-not-exists.tests.

tests/test_octodns_manager.py

@@ -167,6 +167,30 @@ class TestManager(TestCase):
                 .sync(eligible_targets=['foo'])
             self.assertEquals(0, tc)
 
+    def test_aliases(self):
+        with TemporaryDirectory() as tmpdir:
+            environ['YAML_TMP_DIR'] = tmpdir.dirname
+            # Alias zones with a valid target.
+            tc = Manager(get_config_filename('simple-alias-zone.yaml')) \
+                .sync()
+            self.assertEquals(0, tc)
+
+            # Alias zone with an invalid target.
+            with self.assertRaises(ManagerException) as ctx:
+                tc = Manager(get_config_filename('unknown-source-zone.yaml')) \
+                    .sync()
+            self.assertEquals('Invalid alias zone alias.tests.: source zone '
+                              'does-not-exists.tests. does not exist',
+                              text_type(ctx.exception))
+
+            # Alias zone that points to another alias zone.
+            with self.assertRaises(ManagerException) as ctx:
+                tc = Manager(get_config_filename('alias-zone-loop.yaml')) \
+                    .sync()
+            self.assertEquals('Invalid alias zone alias-loop.tests.: source '
+                              'zone alias.tests. is an alias zone',
+                              text_type(ctx.exception))
+
     def test_compare(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
@@ -286,6 +310,36 @@ class TestManager(TestCase):
                 .validate_configs()
         self.assertTrue('unknown source' in text_type(ctx.exception))
 
+        # Alias zone using an invalid source zone.
+        with self.assertRaises(ManagerException) as ctx:
+            Manager(get_config_filename('unknown-source-zone.yaml')) \
+                .validate_configs()
+        self.assertTrue('does not exist' in
+                        text_type(ctx.exception))
+
+        # Alias zone that points to another alias zone.
+        with self.assertRaises(ManagerException) as ctx:
+            Manager(get_config_filename('alias-zone-loop.yaml')) \
+                .validate_configs()
+        self.assertTrue('is an alias zone' in
+                        text_type(ctx.exception))
+
+        # Valid config file using an alias zone.
+        Manager(get_config_filename('simple-alias-zone.yaml')) \
+            .validate_configs()
+
+    def test_get_zone(self):
+        Manager(get_config_filename('simple.yaml')).get_zone('unit.tests.')
+
+        with self.assertRaises(ManagerException) as ctx:
+            Manager(get_config_filename('simple.yaml')).get_zone('unit.tests')
+        self.assertTrue('missing ending dot' in text_type(ctx.exception))
+
+        with self.assertRaises(ManagerException) as ctx:
+            Manager(get_config_filename('simple.yaml')) \
+                .get_zone('unknown-zone.tests.')
+        self.assertTrue('Unknown zone name' in text_type(ctx.exception))
+
     def test_populate_lenient_fallback(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname

tests/test_octodns_record.py

@@ -813,6 +813,39 @@ class TestRecord(TestCase):
             })
         self.assertTrue('Unknown record type' in text_type(ctx.exception))
 
+    def test_record_copy(self):
+        a = Record.new(self.zone, 'a', {
+            'ttl': 44,
+            'type': 'A',
+            'value': '1.2.3.4',
+        })
+
+        # Identical copy.
+        b = a.copy()
+        self.assertIsInstance(b, ARecord)
+        self.assertEquals('unit.tests.', b.zone.name)
+        self.assertEquals('a', b.name)
+        self.assertEquals('A', b._type)
+        self.assertEquals(['1.2.3.4'], b.values)
+
+        # Copy with another zone object.
+        c_zone = Zone('other.tests.', [])
+        c = a.copy(c_zone)
+        self.assertIsInstance(c, ARecord)
+        self.assertEquals('other.tests.', c.zone.name)
+        self.assertEquals('a', c.name)
+        self.assertEquals('A', c._type)
+        self.assertEquals(['1.2.3.4'], c.values)
+
+        # Record with no record type specified in data.
+        d_data = {
+            'ttl': 600,
+            'values': ['just a test']
+        }
+        d = TxtRecord(self.zone, 'txt', d_data)
+        d.copy()
+        self.assertEquals('TXT', d._type)
+
     def test_change(self):
         existing = Record.new(self.zone, 'txt', {
             'ttl': 44,