Merge pull request #156 from github/plan-outputs

Adds new plan_outputs functionality to allow configuring what to do with plans

.gitignore

@@ -1,6 +1,7 @@
 *.pyc
 .coverage
 .env
+/config/
 coverage.xml
 dist/
 env/
@@ -9,5 +10,3 @@ nosetests.xml
 octodns.egg-info/
 output/
 tmp/
-build/
-config/

octodns/manager.py

@@ -5,13 +5,13 @@
 from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
-from StringIO import StringIO
 from concurrent.futures import ThreadPoolExecutor
 from importlib import import_module
 from os import environ
 import logging
 
-from .provider.base import BaseProvider, Plan
+from .provider.base import BaseProvider
+from .provider.plan import Plan
 from .provider.yaml import YamlProvider
 from .record import Record
 from .yaml import safe_load
@@ -95,23 +95,8 @@ class Manager(object):
                 self.log.exception('Invalid provider class')
                 raise Exception('Provider {} is missing class'
                                 .format(provider_name))
-            _class = self._get_provider_class(_class)
-            # Build up the arguments we need to pass to the provider
-            kwargs = {}
-            for k, v in provider_config.items():
-                try:
-                    if v.startswith('env/'):
-                        try:
-                            env_var = v[4:]
-                            v = environ[env_var]
-                        except KeyError:
-                            self.log.exception('Invalid provider config')
-                            raise Exception('Incorrect provider config, '
-                                            'missing env var {}'
-                                            .format(env_var))
-                except AttributeError:
-                    pass
-                kwargs[k] = v
+            _class = self._get_named_class('provider', _class)
+            kwargs = self._build_kwargs(provider_config)
             try:
                 self.providers[provider_name] = _class(provider_name, **kwargs)
             except TypeError:
@@ -139,20 +124,64 @@ class Manager(object):
                     where = where[piece]
         self.zone_tree = zone_tree
 
-    def _get_provider_class(self, _class):
+        self.plan_outputs = {}
+        plan_outputs = manager_config.get('plan_outputs', {
+            'logger': {
+                'class': 'octodns.provider.plan.PlanLogger',
+                'level': 'info'
+            }
+        })
+        for plan_output_name, plan_output_config in plan_outputs.items():
+            try:
+                _class = plan_output_config.pop('class')
+            except KeyError:
+                self.log.exception('Invalid plan_output class')
+                raise Exception('plan_output {} is missing class'
+                                .format(plan_output_name))
+            _class = self._get_named_class('plan_output', _class)
+            kwargs = self._build_kwargs(plan_output_config)
+            try:
+                self.plan_outputs[plan_output_name] = \
+                    _class(plan_output_name, **kwargs)
+            except TypeError:
+                self.log.exception('Invalid plan_output config')
+                raise Exception('Incorrect plan_output config for {}'
+                                .format(plan_output_name))
+
+    def _get_named_class(self, _type, _class):
         try:
             module_name, class_name = _class.rsplit('.', 1)
             module = import_module(module_name)
         except (ImportError, ValueError):
-            self.log.exception('_get_provider_class: Unable to import '
+            self.log.exception('_get_{}_class: Unable to import '
                                'module %s', _class)
-            raise Exception('Unknown provider class: {}'.format(_class))
+            raise Exception('Unknown {} class: {}'.format(_type, _class))
         try:
             return getattr(module, class_name)
         except AttributeError:
-            self.log.exception('_get_provider_class: Unable to get class %s '
+            self.log.exception('_get_{}_class: Unable to get class %s '
                                'from module %s', class_name, module)
-            raise Exception('Unknown provider class: {}'.format(_class))
+            raise Exception('Unknown {} class: {}'.format(_type, _class))
+
+    def _build_kwargs(self, source):
+        # Build up the arguments we need to pass to the provider
+        kwargs = {}
+        for k, v in source.items():
+            try:
+                if v.startswith('env/'):
+                    try:
+                        env_var = v[4:]
+                        v = environ[env_var]
+                    except KeyError:
+                        self.log.exception('Invalid provider config')
+                        raise Exception('Incorrect provider config, '
+                                        'missing env var {}'
+                                        .format(env_var))
+            except AttributeError:
+                pass
+            kwargs[k] = v
+
+        return kwargs
 
     def configured_sub_zones(self, zone_name):
         # Reversed pieces of the zone name
@@ -259,39 +288,8 @@ class Manager(object):
         # plan pairs.
         plans = [p for f in futures for p in f.result()]
 
-        hr = '*************************************************************' \
-            '*******************\n'
-        buf = StringIO()
-        buf.write('\n')
-        if plans:
-            current_zone = None
-            for target, plan in plans:
-                if plan.desired.name != current_zone:
-                    current_zone = plan.desired.name
-                    buf.write(hr)
-                    buf.write('* ')
-                    buf.write(current_zone)
-                    buf.write('\n')
-                    buf.write(hr)
-
-                buf.write('* ')
-                buf.write(target.id)
-                buf.write(' (')
-                buf.write(target)
-                buf.write(')\n*   ')
-                for change in plan.changes:
-                    buf.write(change.__repr__(leader='* '))
-                    buf.write('\n*   ')
-
-                buf.write('Summary: ')
-                buf.write(plan)
-                buf.write('\n')
-        else:
-            buf.write(hr)
-            buf.write('No changes were planned\n')
-        buf.write(hr)
-        buf.write('\n')
-        self.log.info(buf.getvalue())
+        for output in self.plan_outputs.values():
+            output.run(plans=plans, log=self.log)
 
         if not force:
             self.log.debug('sync:   checking safety')

octodns/provider/base.py

@@ -7,78 +7,7 @@ from __future__ import absolute_import, division, print_function, \
 
 from ..source.base import BaseSource
 from ..zone import Zone
-from logging import getLogger
-
-
-class UnsafePlan(Exception):
-    pass
-
-
-class Plan(object):
-    log = getLogger('Plan')
-
-    MAX_SAFE_UPDATE_PCENT = .3
-    MAX_SAFE_DELETE_PCENT = .3
-    MIN_EXISTING_RECORDS = 10
-
-    def __init__(self, existing, desired, changes,
-                 update_pcent_threshold=MAX_SAFE_UPDATE_PCENT,
-                 delete_pcent_threshold=MAX_SAFE_DELETE_PCENT):
-        self.existing = existing
-        self.desired = desired
-        self.changes = changes
-        self.update_pcent_threshold = update_pcent_threshold
-        self.delete_pcent_threshold = delete_pcent_threshold
-
-        change_counts = {
-            'Create': 0,
-            'Delete': 0,
-            'Update': 0
-        }
-        for change in changes:
-            change_counts[change.__class__.__name__] += 1
-        self.change_counts = change_counts
-
-        try:
-            existing_n = len(self.existing.records)
-        except AttributeError:
-            existing_n = 0
-
-        self.log.debug('__init__: Creates=%d, Updates=%d, Deletes=%d'
-                       'Existing=%d',
-                       self.change_counts['Create'],
-                       self.change_counts['Update'],
-                       self.change_counts['Delete'], existing_n)
-
-    def raise_if_unsafe(self):
-        # TODO: what is safe really?
-        if self.existing and \
-           len(self.existing.records) >= self.MIN_EXISTING_RECORDS:
-
-            existing_record_count = len(self.existing.records)
-            update_pcent = self.change_counts['Update'] / existing_record_count
-            delete_pcent = self.change_counts['Delete'] / existing_record_count
-
-            if update_pcent > self.update_pcent_threshold:
-                raise UnsafePlan('Too many updates, {} is over {} percent'
-                                 '({}/{})'.format(
-                                     update_pcent,
-                                     self.MAX_SAFE_UPDATE_PCENT * 100,
-                                     self.change_counts['Update'],
-                                     existing_record_count))
-            if delete_pcent > self.delete_pcent_threshold:
-                raise UnsafePlan('Too many deletes, {} is over {} percent'
-                                 '({}/{})'.format(
-                                     delete_pcent,
-                                     self.MAX_SAFE_DELETE_PCENT * 100,
-                                     self.change_counts['Delete'],
-                                     existing_record_count))
-
-    def __repr__(self):
-        return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \
-            .format(self.change_counts['Create'], self.change_counts['Update'],
-                    self.change_counts['Delete'],
-                    len(self.existing.records))
+from .plan import Plan
 
 
 class BaseProvider(BaseSource):

octodns/provider/plan.py

@@ -0,0 +1,266 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from StringIO import StringIO
+from logging import DEBUG, ERROR, INFO, WARN, getLogger
+from sys import stdout
+
+
+class UnsafePlan(Exception):
+    pass
+
+
+class Plan(object):
+    log = getLogger('Plan')
+
+    MAX_SAFE_UPDATE_PCENT = .3
+    MAX_SAFE_DELETE_PCENT = .3
+    MIN_EXISTING_RECORDS = 10
+
+    def __init__(self, existing, desired, changes,
+                 update_pcent_threshold=MAX_SAFE_UPDATE_PCENT,
+                 delete_pcent_threshold=MAX_SAFE_DELETE_PCENT):
+        self.existing = existing
+        self.desired = desired
+        self.changes = changes
+        self.update_pcent_threshold = update_pcent_threshold
+        self.delete_pcent_threshold = delete_pcent_threshold
+
+        change_counts = {
+            'Create': 0,
+            'Delete': 0,
+            'Update': 0
+        }
+        for change in changes:
+            change_counts[change.__class__.__name__] += 1
+        self.change_counts = change_counts
+
+        try:
+            existing_n = len(self.existing.records)
+        except AttributeError:
+            existing_n = 0
+
+        self.log.debug('__init__: Creates=%d, Updates=%d, Deletes=%d'
+                       'Existing=%d',
+                       self.change_counts['Create'],
+                       self.change_counts['Update'],
+                       self.change_counts['Delete'], existing_n)
+
+    def raise_if_unsafe(self):
+        # TODO: what is safe really?
+        if self.existing and \
+           len(self.existing.records) >= self.MIN_EXISTING_RECORDS:
+
+            existing_record_count = len(self.existing.records)
+            update_pcent = self.change_counts['Update'] / existing_record_count
+            delete_pcent = self.change_counts['Delete'] / existing_record_count
+
+            if update_pcent > self.update_pcent_threshold:
+                raise UnsafePlan('Too many updates, {} is over {} percent'
+                                 '({}/{})'.format(
+                                     update_pcent,
+                                     self.MAX_SAFE_UPDATE_PCENT * 100,
+                                     self.change_counts['Update'],
+                                     existing_record_count))
+            if delete_pcent > self.delete_pcent_threshold:
+                raise UnsafePlan('Too many deletes, {} is over {} percent'
+                                 '({}/{})'.format(
+                                     delete_pcent,
+                                     self.MAX_SAFE_DELETE_PCENT * 100,
+                                     self.change_counts['Delete'],
+                                     existing_record_count))
+
+    def __repr__(self):
+        return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \
+            .format(self.change_counts['Create'], self.change_counts['Update'],
+                    self.change_counts['Delete'],
+                    len(self.existing.records))
+
+
+class _PlanOutput(object):
+
+    def __init__(self, name):
+        self.name = name
+
+
+class PlanLogger(_PlanOutput):
+
+    def __init__(self, name, level='info'):
+        super(PlanLogger, self).__init__(name)
+        try:
+            self.level = {
+                'debug': DEBUG,
+                'info': INFO,
+                'warn': WARN,
+                'warning': WARN,
+                'error': ERROR
+            }[level.lower()]
+        except (AttributeError, KeyError):
+            raise Exception('Unsupported level: {}'.format(level))
+
+    def run(self, log, plans, *args, **kwargs):
+        hr = '*************************************************************' \
+            '*******************\n'
+        buf = StringIO()
+        buf.write('\n')
+        if plans:
+            current_zone = None
+            for target, plan in plans:
+                if plan.desired.name != current_zone:
+                    current_zone = plan.desired.name
+                    buf.write(hr)
+                    buf.write('* ')
+                    buf.write(current_zone)
+                    buf.write('\n')
+                    buf.write(hr)
+
+                buf.write('* ')
+                buf.write(target.id)
+                buf.write(' (')
+                buf.write(target)
+                buf.write(')\n*   ')
+                for change in plan.changes:
+                    buf.write(change.__repr__(leader='* '))
+                    buf.write('\n*   ')
+
+                buf.write('Summary: ')
+                buf.write(plan)
+                buf.write('\n')
+        else:
+            buf.write(hr)
+            buf.write('No changes were planned\n')
+        buf.write(hr)
+        buf.write('\n')
+        log.log(self.level, buf.getvalue())
+
+
+def _value_stringifier(record, sep):
+    try:
+        values = [str(v) for v in record.values]
+    except AttributeError:
+        values = [record.value]
+    for code, gv in sorted(getattr(record, 'geo', {}).items()):
+        vs = ', '.join([str(v) for v in gv.values])
+        values.append('{}: {}'.format(code, vs))
+    return sep.join(values)
+
+
+class PlanMarkdown(_PlanOutput):
+
+    def run(self, plans, fh=stdout, *args, **kwargs):
+        if plans:
+            current_zone = None
+            for target, plan in plans:
+                if plan.desired.name != current_zone:
+                    current_zone = plan.desired.name
+                    fh.write('## ')
+                    fh.write(current_zone)
+                    fh.write('\n\n')
+
+                fh.write('### ')
+                fh.write(target.id)
+                fh.write('\n\n')
+
+                fh.write('| Operation | Name | Type | TTL | Value | Source |\n'
+                         '|--|--|--|--|--|--|\n')
+
+                for change in plan.changes:
+                    existing = change.existing
+                    new = change.new
+                    record = change.record
+                    fh.write('| ')
+                    fh.write(change.__class__.__name__)
+                    fh.write(' | ')
+                    fh.write(record.name)
+                    fh.write(' | ')
+                    fh.write(record._type)
+                    fh.write(' | ')
+                    # TTL
+                    if existing:
+                        fh.write(str(existing.ttl))
+                        fh.write(' | ')
+                        fh.write(_value_stringifier(existing, '; '))
+                        fh.write(' | |\n')
+                        if new:
+                            fh.write('| | | | ')
+
+                    if new:
+                        fh.write(str(new.ttl))
+                        fh.write(' | ')
+                        fh.write(_value_stringifier(new, '; '))
+                        fh.write(' | ')
+                        fh.write(new.source.id)
+                        fh.write(' |\n')
+
+                fh.write('\nSummary: ')
+                fh.write(str(plan))
+                fh.write('\n\n')
+        else:
+            fh.write('## No changes were planned\n')
+
+
+class PlanHtml(_PlanOutput):
+
+    def run(self, plans, fh=stdout, *args, **kwargs):
+        if plans:
+            current_zone = None
+            for target, plan in plans:
+                if plan.desired.name != current_zone:
+                    current_zone = plan.desired.name
+                    fh.write('<h2>')
+                    fh.write(current_zone)
+                    fh.write('</h2>\n')
+
+                fh.write('<h3>')
+                fh.write(target.id)
+                fh.write('''</h3>
+<table>
+  <tr>
+    <th>Operation</th>
+    <th>Name</th>
+    <th>Type</th>
+    <th>TTL</th>
+    <th>Value</th>
+    <th>Source</th>
+  </tr>
+''')
+
+                for change in plan.changes:
+                    existing = change.existing
+                    new = change.new
+                    record = change.record
+                    fh.write('  <tr>\n    <td>')
+                    fh.write(change.__class__.__name__)
+                    fh.write('</td>\n    <td>')
+                    fh.write(record.name)
+                    fh.write('</td>\n    <td>')
+                    fh.write(record._type)
+                    fh.write('</td>\n')
+                    # TTL
+                    if existing:
+                        fh.write('    <td>')
+                        fh.write(str(existing.ttl))
+                        fh.write('</td>\n    <td>')
+                        fh.write(_value_stringifier(existing, '<br/>'))
+                        fh.write('</td>\n    <td></td>\n  </tr>\n')
+                        if new:
+                            fh.write('  <tr>\n    <td colspan=3></td>\n')
+
+                    if new:
+                        fh.write('    <td>')
+                        fh.write(str(new.ttl))
+                        fh.write('</td>\n    <td>')
+                        fh.write(_value_stringifier(new, '<br/>'))
+                        fh.write('</td>\n    <td>')
+                        fh.write(new.source.id)
+                        fh.write('</td>\n  </tr>\n')
+
+                fh.write('  <tr>\n    <td colspan=6>Summary: ')
+                fh.write(str(plan))
+                fh.write('</td>\n  </tr>\n</table>\n')
+        else:
+            fh.write('<b>No changes were planned</b>')

tests/config/bad-plan-output-config.yaml

@@ -0,0 +1,7 @@
+manager:
+  plan_outputs:
+    'bad': 
+      class: octodns.provider.plan.PlanLogger
+      invalid: config
+providers: {}
+zones: {}

tests/config/bad-plan-output-missing-class.yaml

@@ -0,0 +1,5 @@
+manager:
+  plan_outputs:
+    'bad': {}
+providers: {}
+zones: {}

tests/test_octodns_manager.py

@@ -83,6 +83,19 @@ class TestManager(TestCase):
                 .sync(['unknown.target.'])
         self.assertTrue('unknown target' in ctx.exception.message)
 
+    def test_bad_plan_output_class(self):
+        with self.assertRaises(Exception) as ctx:
+            name = 'bad-plan-output-missing-class.yaml'
+            Manager(get_config_filename(name)).sync()
+        self.assertEquals('plan_output bad is missing class',
+                          ctx.exception.message)
+
+    def test_bad_plan_output_config(self):
+        with self.assertRaises(Exception) as ctx:
+            Manager(get_config_filename('bad-plan-output-config.yaml')).sync()
+        self.assertEqual('Incorrect plan_output config for bad',
+                         ctx.exception.message)
+
     def test_source_only_as_a_target(self):
         with self.assertRaises(Exception) as ctx:
             Manager(get_config_filename('unknown-provider.yaml')) \

tests/test_octodns_plan.py

@@ -0,0 +1,92 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from StringIO import StringIO
+from logging import getLogger
+from unittest import TestCase
+
+from octodns.provider.plan import Plan, PlanHtml, PlanLogger, PlanMarkdown
+from octodns.record import Create, Delete, Record, Update
+from octodns.zone import Zone
+
+from helpers import SimpleProvider
+
+
+class TestPlanLogger(TestCase):
+
+    def test_invalid_level(self):
+        with self.assertRaises(Exception) as ctx:
+            PlanLogger('invalid', 'not-a-level')
+        self.assertEquals('Unsupported level: not-a-level',
+                          ctx.exception.message)
+
+
+simple = SimpleProvider()
+zone = Zone('unit.tests.', [])
+existing = Record.new(zone, 'a', {
+    'ttl': 300,
+    'type': 'A',
+    # This matches the zone data above, one to swap, one to leave
+    'values': ['1.1.1.1', '2.2.2.2'],
+})
+new = Record.new(zone, 'a', {
+    'geo': {
+        'AF': ['5.5.5.5'],
+        'NA-US': ['6.6.6.6']
+    },
+    'ttl': 300,
+    'type': 'A',
+    # This leaves one, swaps ones, and adds one
+    'values': ['2.2.2.2', '3.3.3.3', '4.4.4.4'],
+}, simple)
+create = Create(Record.new(zone, 'b', {
+    'ttl': 60,
+    'type': 'CNAME',
+    'value': 'foo.unit.tests.'
+}, simple))
+update = Update(existing, new)
+delete = Delete(new)
+changes = [create, delete, update]
+plans = [
+    (simple, Plan(zone, zone, changes)),
+    (simple, Plan(zone, zone, changes)),
+]
+
+
+class TestPlanHtml(TestCase):
+    log = getLogger('TestPlanHtml')
+
+    def test_empty(self):
+        out = StringIO()
+        PlanHtml('html').run([], fh=out)
+        self.assertEquals('<b>No changes were planned</b>', out.getvalue())
+
+    def test_simple(self):
+        out = StringIO()
+        PlanHtml('html').run(plans, fh=out)
+        out = out.getvalue()
+        self.assertTrue('    <td colspan=6>Summary: Creates=1, Updates=1, '
+                        'Deletes=1, Existing Records=0</td>' in out)
+
+
+class TestPlanMarkdown(TestCase):
+    log = getLogger('TestPlanMarkdown')
+
+    def test_empty(self):
+        out = StringIO()
+        PlanMarkdown('markdown').run([], fh=out)
+        self.assertEquals('## No changes were planned\n', out.getvalue())
+
+    def test_simple(self):
+        out = StringIO()
+        PlanMarkdown('markdown').run(plans, fh=out)
+        out = out.getvalue()
+        self.assertTrue('## unit.tests.' in out)
+        self.assertTrue('Create | b | CNAME | 60 | foo.unit.tests.' in out)
+        self.assertTrue('Update | a | A | 300 | 1.1.1.1;' in out)
+        self.assertTrue('NA-US: 6.6.6.6 | test' in out)
+        self.assertTrue('Delete | a | A | 300 | 2.2.2.2;' in out)

tests/test_octodns_provider_base.py

@@ -9,7 +9,8 @@ from logging import getLogger
 from unittest import TestCase
 
 from octodns.record import Create, Delete, Record, Update
-from octodns.provider.base import BaseProvider, Plan, UnsafePlan
+from octodns.provider.base import BaseProvider
+from octodns.provider.plan import Plan, UnsafePlan
 from octodns.zone import Zone