From 36b67b8b7aafd3503a2387100dfdba9dbbcbc6cd Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 1 Jun 2018 20:59:03 -0700 Subject: [PATCH 1/3] Implement EtcHostsProvider, content to be used in /etc/hosts ...for testing or emergencies --- CHANGELOG.md | 2 + README.md | 1 + octodns/provider/etc_hosts.py | 123 +++++++++++++++++++++++ tests/test_octodns_provider_etc_hosts.py | 120 ++++++++++++++++++++++ 4 files changed, 246 insertions(+) create mode 100644 octodns/provider/etc_hosts.py create mode 100644 tests/test_octodns_provider_etc_hosts.py diff --git a/CHANGELOG.md b/CHANGELOG.md index eebf7bc..b753fcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## v0.9.2 - Unreleased +* EtcHostsProvider implementation to create static/emergency best effort + content that can be used in /etc/hosts to resolve things. * Add lenient support to Zone.add_record, allows populate from providers that have allowed/created invalid data and situations where a sub-zone is being extracted from a parent, but the records still exist in the remote provider. diff --git a/README.md b/README.md index 39946c1..b263aec 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DnsMadeEasyProvider](/octodns/provider/dnsmadeeasy.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | dyn | All | Yes | | +| [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | nsone | All | Yes | No health checking for GeoDNS | | [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | diff --git a/octodns/provider/etc_hosts.py b/octodns/provider/etc_hosts.py new file mode 100644 index 0000000..b659327 --- /dev/null +++ b/octodns/provider/etc_hosts.py @@ -0,0 +1,123 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from os import makedirs +from os.path import isdir, join +import logging + +from .base import BaseProvider + + +class EtcHostsProvider(BaseProvider): + ''' + Provider that creates a "best effort" static/emergency content that can be + used in /etc/hosts to resolve things. A, AAAA records are supported and + ALIAS and CNAME records will be included when they can be mapped within the + zone. + + config: + class: octodns.provider.etc_hosts.EtcHostsProvider + # The output director for the hosts file .hosts + directory: ./hosts + ''' + SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME')) + + def __init__(self, id, directory, *args, **kwargs): + self.log = logging.getLogger('EtcHostsProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, directory=%s', id, directory) + super(EtcHostsProvider, self).__init__(id, *args, **kwargs) + self.directory = directory + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + # We never act as a source, at least for now, if/when we do we still + # need to noop `if target` + return False + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + cnames = {} + values = {} + for record in sorted([c.new for c in changes]): + # Since we don't have existing we'll only see creates + fqdn = record.fqdn[:-1] + if record._type in ('ALIAS', 'CNAME'): + # Store cnames so we can try and look them up in a minute + cnames[fqdn] = record.value[:-1] + elif record._type == 'AAAA' and fqdn in values: + # We'll prefer A over AAAA, skipping rather than replacing an + # existing A + pass + else: + # If we're here it's and A or AAAA and we want to record it's + # value (maybe replacing if it's an A and we have a AAAA + values[fqdn] = record.values[0] + + if not isdir(self.directory): + makedirs(self.directory) + + filename = '{}hosts'.format(join(self.directory, desired.name)) + self.log.info('_apply: filename=%s', filename) + with open(filename, 'w') as fh: + fh.write('########################################\n') + fh.write('# octoDNS ') + fh.write(self.id) + fh.write('\n') + fh.write('########################################\n\n') + if values: + fh.write('## A & AAAA\n\n') + for fqdn, value in sorted(values.items()): + if fqdn[0] == '*': + fh.write('# ') + fh.write(value) + fh.write('\t') + fh.write(fqdn) + fh.write('\n\n') + + if cnames: + fh.write('\n') + fh.write('## CNAME (mapped)\n\n') + for fqdn, value in sorted(cnames.items()): + # Print out a comment of the first level + fh.write('# ') + fh.write(fqdn) + fh.write(' -> ') + fh.write(value) + fh.write('\n') + # No loop protection :-/ + while True: + try: + value = values[value] + # If we're here we've found the target, print it + # and break the loop + fh.write(value) + fh.write('\t') + fh.write(fqdn) + fh.write('\n') + break + except KeyError: + # Try and step down one level + orig = value + value = cnames.get(value, None) + # Print out this step + fh.write('# ') + fh.write(orig) + fh.write(' -> ') + if value: + fh.write(value) + else: + # Don't have anywhere else to go + fh.write('**unknown**') + break + fh.write('\n') + fh.write('\n') diff --git a/tests/test_octodns_provider_etc_hosts.py b/tests/test_octodns_provider_etc_hosts.py new file mode 100644 index 0000000..9c8f06b --- /dev/null +++ b/tests/test_octodns_provider_etc_hosts.py @@ -0,0 +1,120 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from os.path import dirname, isfile, join +from unittest import TestCase + +from octodns.provider.etc_hosts import EtcHostsProvider +from octodns.provider.plan import Plan +from octodns.record import Record +from octodns.zone import Zone + +from helpers import TemporaryDirectory + + +class TestEtcHostsProvider(TestCase): + + def test_provider(self): + source = EtcHostsProvider('test', join(dirname(__file__), 'config')) + + zone = Zone('unit.tests.', []) + + # We never populate anything, when acting as a source + source.populate(zone, target=source) + self.assertEquals(0, len(zone.records)) + # Same if we're acting as a target + source.populate(zone) + self.assertEquals(0, len(zone.records)) + + record = Record.new(zone, '', { + 'ttl': 60, + 'type': 'ALIAS', + 'value': 'www.unit.tests.' + }) + zone.add_record(record) + + record = Record.new(zone, 'www', { + 'ttl': 60, + 'type': 'AAAA', + 'value': '2001:4860:4860::8888', + }) + zone.add_record(record) + record = Record.new(zone, 'www', { + 'ttl': 60, + 'type': 'A', + 'values': ['1.1.1.1', '2.2.2.2'], + }) + zone.add_record(record) + + record = record.new(zone, 'v6', { + 'ttl': 60, + 'type': 'AAAA', + 'value': '2001:4860:4860::8844', + }) + zone.add_record(record) + + record = record.new(zone, 'start', { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'middle.unit.tests.', + }) + zone.add_record(record) + record = record.new(zone, 'middle', { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'unit.tests.', + }) + zone.add_record(record) + + record = record.new(zone, 'ext', { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'github.com.', + }) + zone.add_record(record) + + record = record.new(zone, '*', { + 'ttl': 60, + 'type': 'A', + 'value': '3.3.3.3', + }) + zone.add_record(record) + + with TemporaryDirectory() as td: + # Add some subdirs to make sure that it can create them + directory = join(td.dirname, 'sub', 'dir') + hosts_file = join(directory, 'unit.tests.hosts') + target = EtcHostsProvider('test', directory) + + # We add everything + plan = target.plan(zone) + self.assertEquals(len(zone.records), len(plan.changes)) + self.assertFalse(isfile(hosts_file)) + + # Now actually do it + self.assertEquals(len(zone.records), target.apply(plan)) + self.assertTrue(isfile(hosts_file)) + + with open(hosts_file) as fh: + data = fh.read() + # v6 + self.assertTrue('2001:4860:4860::8844\tv6.unit.tests') + # www + self.assertTrue('1.1.1.1\twww.unit.tests' in data) + # root ALIAS + self.assertTrue('# unit.tests -> www.unit.tests' in data) + self.assertTrue('1.1.1.1\tunit.tests' in data) + + self.assertTrue('# start.unit.tests -> middle.unit.tests' in + data) + self.assertTrue('# middle.unit.tests -> unit.tests' in data) + self.assertTrue('# unit.tests -> www.unit.tests' in data) + self.assertTrue('1.1.1.1 start.unit.tests' in data) + + # second empty run that won't create dirs and overwrites file + plan = Plan(zone, zone, [], True) + self.assertEquals(0, target.apply(plan)) From 50e1ce3881c130affe617d9221894ef64622bed1 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 1 Jun 2018 21:05:30 -0700 Subject: [PATCH 2/3] Include zone name in the header --- octodns/provider/etc_hosts.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/octodns/provider/etc_hosts.py b/octodns/provider/etc_hosts.py index b659327..92fb2be 100644 --- a/octodns/provider/etc_hosts.py +++ b/octodns/provider/etc_hosts.py @@ -69,11 +69,13 @@ class EtcHostsProvider(BaseProvider): filename = '{}hosts'.format(join(self.directory, desired.name)) self.log.info('_apply: filename=%s', filename) with open(filename, 'w') as fh: - fh.write('########################################\n') + fh.write('##################################################\n') fh.write('# octoDNS ') fh.write(self.id) + fh.write(' ') + fh.write(desired.name) fh.write('\n') - fh.write('########################################\n\n') + fh.write('##################################################\n\n') if values: fh.write('## A & AAAA\n\n') for fqdn, value in sorted(values.items()): From 1e2da343616e88419bdae5b92996972072a1b694 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 9 Jun 2018 16:21:25 -0700 Subject: [PATCH 3/3] Use path.join, add loop detection & tests --- octodns/provider/etc_hosts.py | 50 +++++++---------- tests/test_octodns_provider_etc_hosts.py | 69 ++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 35 deletions(-) diff --git a/octodns/provider/etc_hosts.py b/octodns/provider/etc_hosts.py index 92fb2be..db84ae4 100644 --- a/octodns/provider/etc_hosts.py +++ b/octodns/provider/etc_hosts.py @@ -5,8 +5,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from os import makedirs -from os.path import isdir, join +from os import makedirs, path +from os.path import isdir import logging from .base import BaseProvider @@ -66,60 +66,50 @@ class EtcHostsProvider(BaseProvider): if not isdir(self.directory): makedirs(self.directory) - filename = '{}hosts'.format(join(self.directory, desired.name)) + filename = '{}hosts'.format(path.join(self.directory, desired.name)) self.log.info('_apply: filename=%s', filename) with open(filename, 'w') as fh: fh.write('##################################################\n') - fh.write('# octoDNS ') - fh.write(self.id) - fh.write(' ') - fh.write(desired.name) - fh.write('\n') + fh.write('# octoDNS {} {}\n'.format(self.id, desired.name)) fh.write('##################################################\n\n') if values: fh.write('## A & AAAA\n\n') for fqdn, value in sorted(values.items()): if fqdn[0] == '*': fh.write('# ') - fh.write(value) - fh.write('\t') - fh.write(fqdn) - fh.write('\n\n') + fh.write('{}\t{}\n\n'.format(value, fqdn)) if cnames: - fh.write('\n') - fh.write('## CNAME (mapped)\n\n') + fh.write('\n## CNAME (mapped)\n\n') for fqdn, value in sorted(cnames.items()): # Print out a comment of the first level - fh.write('# ') - fh.write(fqdn) - fh.write(' -> ') - fh.write(value) - fh.write('\n') - # No loop protection :-/ + fh.write('# {} -> {}\n'.format(fqdn, value)) + seen = set() while True: + seen.add(value) try: value = values[value] # If we're here we've found the target, print it # and break the loop - fh.write(value) - fh.write('\t') - fh.write(fqdn) - fh.write('\n') + fh.write('{}\t{}\n'.format(value, fqdn)) break except KeyError: # Try and step down one level orig = value value = cnames.get(value, None) # Print out this step - fh.write('# ') - fh.write(orig) - fh.write(' -> ') if value: - fh.write(value) + if value in seen: + # We'd loop here, break it + fh.write('# {} -> {} **loop**\n' + .format(orig, value)) + break + else: + fh.write('# {} -> {}\n' + .format(orig, value)) else: # Don't have anywhere else to go - fh.write('**unknown**') + fh.write('# {} -> **unknown**\n'.format(orig)) break - fh.write('\n') + fh.write('\n') diff --git a/tests/test_octodns_provider_etc_hosts.py b/tests/test_octodns_provider_etc_hosts.py index 9c8f06b..236164d 100644 --- a/tests/test_octodns_provider_etc_hosts.py +++ b/tests/test_octodns_provider_etc_hosts.py @@ -5,7 +5,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from os.path import dirname, isfile, join +from os import path +from os.path import dirname, isfile from unittest import TestCase from octodns.provider.etc_hosts import EtcHostsProvider @@ -19,7 +20,8 @@ from helpers import TemporaryDirectory class TestEtcHostsProvider(TestCase): def test_provider(self): - source = EtcHostsProvider('test', join(dirname(__file__), 'config')) + source = EtcHostsProvider('test', path.join(dirname(__file__), + 'config')) zone = Zone('unit.tests.', []) @@ -86,8 +88,8 @@ class TestEtcHostsProvider(TestCase): with TemporaryDirectory() as td: # Add some subdirs to make sure that it can create them - directory = join(td.dirname, 'sub', 'dir') - hosts_file = join(directory, 'unit.tests.hosts') + directory = path.join(td.dirname, 'sub', 'dir') + hosts_file = path.join(directory, 'unit.tests.hosts') target = EtcHostsProvider('test', directory) # We add everything @@ -102,7 +104,7 @@ class TestEtcHostsProvider(TestCase): with open(hosts_file) as fh: data = fh.read() # v6 - self.assertTrue('2001:4860:4860::8844\tv6.unit.tests') + self.assertTrue('2001:4860:4860::8844\tv6.unit.tests' in data) # www self.assertTrue('1.1.1.1\twww.unit.tests' in data) # root ALIAS @@ -118,3 +120,60 @@ class TestEtcHostsProvider(TestCase): # second empty run that won't create dirs and overwrites file plan = Plan(zone, zone, [], True) self.assertEquals(0, target.apply(plan)) + + def test_cname_loop(self): + source = EtcHostsProvider('test', path.join(dirname(__file__), + 'config')) + + zone = Zone('unit.tests.', []) + + # We never populate anything, when acting as a source + source.populate(zone, target=source) + self.assertEquals(0, len(zone.records)) + # Same if we're acting as a target + source.populate(zone) + self.assertEquals(0, len(zone.records)) + + record = Record.new(zone, 'start', { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'middle.unit.tests.', + }) + zone.add_record(record) + record = Record.new(zone, 'middle', { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'loop.unit.tests.', + }) + zone.add_record(record) + record = Record.new(zone, 'loop', { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'start.unit.tests.', + }) + zone.add_record(record) + + with TemporaryDirectory() as td: + # Add some subdirs to make sure that it can create them + directory = path.join(td.dirname, 'sub', 'dir') + hosts_file = path.join(directory, 'unit.tests.hosts') + target = EtcHostsProvider('test', directory) + + # We add everything + plan = target.plan(zone) + self.assertEquals(len(zone.records), len(plan.changes)) + self.assertFalse(isfile(hosts_file)) + + # Now actually do it + self.assertEquals(len(zone.records), target.apply(plan)) + self.assertTrue(isfile(hosts_file)) + + with open(hosts_file) as fh: + data = fh.read() + print(data) + self.assertTrue('# loop.unit.tests -> start.unit.tests ' + '**loop**' in data) + self.assertTrue('# middle.unit.tests -> loop.unit.tests ' + '**loop**' in data) + self.assertTrue('# start.unit.tests -> middle.unit.tests ' + '**loop**' in data)