diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b8025f..3cdef09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ * New dynamic zone config support that allows wildcard entries in the octoDNS config to be expanded by the source provider(s). See [Dynamic Zone Config](/README.md#dynamic-zone-config) for more information. +* SplitYamlProvider has been deprecated and will be removed in 2.0. YamlProvider + now includes the ability to process split zones when configured to do so and + allows for more flexibility in how things are laid out than was previously + possible. This includes the ability to split some zones and not others and + even to have partially split zones with some records in the primary zone YAML + and others in a split directory. See YamlProvider documentation for more info. #### Stuff diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 1d495a9..da6fdf1 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -19,18 +19,70 @@ class YamlProvider(BaseProvider): config: class: octodns.provider.yaml.YamlProvider - # The location of yaml config files (required) + + # The location of yaml config files. By default records are defined in a + # file named for the zone in this directory, the zone file, e.g. + # something.com.yaml. + # (required) directory: ./config + # The ttl to use for records when not specified in the data # (optional, default 3600) default_ttl: 3600 - # Whether or not to enforce sorting order on the yaml config + + # Whether or not to enforce sorting order when loading yaml # (optional, default True) enforce_order: true + # Whether duplicate records should replace rather than error - # (optiona, default False) + # (optional, default False) populate_should_replace: false + # The file extension used when loading split style zones, Null means + # disabled. When enabled the provider will search for zone records split + # across multiple YAML files in the directory with split_extension + # appended to the zone name, See "Split Details" below. + # split_extension should include the "." + # (optional, default null, "." is the recommended best practice when + # enabling) + split_extension: null + + # When writing YAML records out to disk with split_extension enabled + # each record is written out into its own file with .yaml appended to + # the name of the record. The two exceptions are for the root and + # wildcard nodes. These records are written into a file named + # `$[zone.name].yaml`. If you would prefer this catchall file not be + # used `split_catchall` can be set to False to instead write those + # records out to `.yaml` and `*.yaml` respectively. Note that some + # operating systems may not allow files with those names. + # (optional, default True) + split_catchall: true + + # Disable loading of the zone .yaml files. + # (optional, default False) + disable_zonefile: false + + Split Details + ------------- + + All files are stored in a subdirectory matching the name of the zone + (including the trailing .) of the directory config. It is a recommended + best practice that the files be named RECORD.yaml, but all files are + sourced and processed ignoring the filenames so it is up to you how to + organize them. + + With `split_extension: .` the directory structure for the zone github.com. + managed under directory "zones/" would look like: + + zones/ + github.com./ + $github.com.yaml + www.yaml + ... + + Overriding Values + ----------------- + Overriding values can be accomplished using multiple yaml providers in the `sources` list where subsequent providers have `populate_should_replace` set to `true`. An example use of this would be a zone that you want to push @@ -98,7 +150,6 @@ class YamlProvider(BaseProvider): You can then sync our records eternally with `--config-file=external.yaml` and internally (with the custom overrides) with `--config-file=internal.yaml` - ''' SUPPORTS_GEO = True @@ -107,6 +158,10 @@ class YamlProvider(BaseProvider): SUPPORTS_DYNAMIC_SUBNETS = True SUPPORTS_MULTIVALUE_PTR = True + # Any record name added to this set will be included in the catch-all file, + # instead of a file matching the record name. + CATCHALL_RECORD_NAMES = ('*', '') + def __init__( self, id, @@ -115,19 +170,25 @@ class YamlProvider(BaseProvider): enforce_order=True, populate_should_replace=False, supports_root_ns=True, + split_extension=False, + split_catchall=True, + disable_zonefile=False, *args, **kwargs, ): klass = self.__class__.__name__ self.log = logging.getLogger(f'{klass}[{id}]') self.log.debug( - '__init__: id=%s, directory=%s, default_ttl=%d, ' - 'enforce_order=%d, populate_should_replace=%d', + '__init__: id=%s, directory=%s, default_ttl=%d, enforce_order=%d, populate_should_replace=%s, supports_root_ns=%s, split_extension=%s, split_catchall=%s, disable_zonefile=%s', id, directory, default_ttl, enforce_order, populate_should_replace, + supports_root_ns, + split_extension, + split_catchall, + disable_zonefile, ) super().__init__(id, *args, **kwargs) self.directory = directory @@ -135,12 +196,15 @@ class YamlProvider(BaseProvider): self.enforce_order = enforce_order self.populate_should_replace = populate_should_replace self.supports_root_ns = supports_root_ns + self.split_extension = split_extension + self.split_catchall = split_catchall + self.disable_zonefile = disable_zonefile def copy(self): - args = dict(self.__dict__) - args['id'] = f'{args["id"]}-copy' - del args['log'] - return self.__class__(**args) + kwargs = dict(self.__dict__) + kwargs['id'] = f'{kwargs["id"]}-copy' + del kwargs['log'] + return YamlProvider(**kwargs) @property def SUPPORTS(self): @@ -162,6 +226,69 @@ class YamlProvider(BaseProvider): def SUPPORTS_ROOT_NS(self): return self.supports_root_ns + def list_zones(self): + self.log.debug('list_zones:') + zones = set() + + extension = self.split_extension + if extension: + # we want to leave the . + trim = len(extension) - 1 + self.log.debug( + 'list_zones: looking for split zones, trim=%d', trim + ) + for dirname in listdir(self.directory): + not_ends_with = not dirname.endswith(extension) + not_dir = not isdir(join(self.directory, dirname)) + if not_dir or not_ends_with: + continue + if trim: + dirname = dirname[:-trim] + zones.add(dirname) + + if not self.disable_zonefile: + self.log.debug('list_zones: looking for zone files') + for filename in listdir(self.directory): + not_ends_with = not filename.endswith('.yaml') + too_few_dots = filename.count('.') < 2 + not_file = not isfile(join(self.directory, filename)) + if not_file or not_ends_with or too_few_dots: + continue + # trim off the yaml, leave the . + zones.add(filename[:-4]) + + return sorted(zones) + + def _split_sources(self, zone): + ext = self.split_extension + utf8 = join(self.directory, f'{zone.decoded_name[:-1]}{ext}') + idna = join(self.directory, f'{zone.name[:-1]}{ext}') + directory = None + if isdir(utf8): + if utf8 != idna and isdir(idna): + raise ProviderException( + f'Both UTF-8 "{utf8}" and IDNA "{idna}" exist for {zone.decoded_name}' + ) + directory = utf8 + else: + directory = idna + + for filename in listdir(directory): + if filename.endswith('.yaml'): + yield join(directory, filename) + + def _zone_sources(self, zone): + utf8 = join(self.directory, f'{zone.decoded_name}yaml') + idna = join(self.directory, f'{zone.name}yaml') + if isfile(utf8): + if utf8 != idna and isfile(idna): + raise ProviderException( + f'Both UTF-8 "{utf8}" and IDNA "{idna}" exist for {zone.decoded_name}' + ) + return utf8 + + return idna + def _populate_from_file(self, filename, zone, lenient): with open(filename, 'r') as fh: yaml_data = safe_load(fh, enforce_order=self.enforce_order) @@ -184,18 +311,6 @@ class YamlProvider(BaseProvider): '_populate_from_file: successfully loaded "%s"', filename ) - def get_filenames(self, zone): - return ( - join(self.directory, f'{zone.decoded_name}yaml'), - join(self.directory, f'{zone.name}yaml'), - ) - - def list_zones(self): - for filename in listdir(self.directory): - if not filename.endswith('.yaml') or filename.count('.') < 2: - continue - yield filename[:-4] - def populate(self, zone, target=False, lenient=False): self.log.debug( 'populate: name=%s, target=%s, lenient=%s', @@ -210,23 +325,21 @@ class YamlProvider(BaseProvider): return False before = len(zone.records) - utf8_filename, idna_filename = self.get_filenames(zone) - # we prefer utf8 - if isfile(utf8_filename): - if utf8_filename != idna_filename and isfile(idna_filename): - raise ProviderException( - f'Both UTF-8 "{utf8_filename}" and IDNA "{idna_filename}" exist for {zone.decoded_name}' - ) - filename = utf8_filename - else: - self.log.warning( - 'populate: "%s" does not exist, falling back to try idna version "%s"', - utf8_filename, - idna_filename, - ) - filename = idna_filename - self._populate_from_file(filename, zone, lenient) + sources = [] + + split_extension = self.split_extension + if split_extension: + sources.extend(self._split_sources(zone)) + + if not self.disable_zonefile: + sources.append(self._zone_sources(zone)) + + # determinstically order our sources + sources.sort() + + for source in sources: + self._populate_from_file(source, zone, lenient) self.log.info( 'populate: found %s records, exists=False', @@ -264,123 +377,75 @@ class YamlProvider(BaseProvider): data[k] = data[k][0] if not isdir(self.directory): + self.log.debug('_apply: creating directory=%s', self.directory) makedirs(self.directory) - self._do_apply(desired, data) - - def _do_apply(self, desired, data): - filename = join(self.directory, f'{desired.decoded_name}yaml') - self.log.debug('_apply: writing filename=%s', filename) - with open(filename, 'w') as fh: - safe_dump(dict(data), fh, allow_unicode=True) + if self.split_extension: + # we're going to do split files + decoded_name = desired.decoded_name[:-1] + directory = join( + self.directory, f'{decoded_name}{self.split_extension}' + ) + if not isdir(directory): + self.log.debug('_apply: creating split directory=%s', directory) + makedirs(directory) + + catchall = {} + for record, config in data.items(): + if self.split_catchall and record in self.CATCHALL_RECORD_NAMES: + catchall[record] = config + continue + filename = join(directory, f'{record}.yaml') + self.log.debug('_apply: writing filename=%s', filename) + + with open(filename, 'w') as fh: + record_data = {record: config} + safe_dump(record_data, fh) + + if catchall: + # Scrub the trailing . to make filenames more sane. + filename = join(directory, f'${decoded_name}.yaml') + self.log.debug( + '_apply: writing catchall filename=%s', filename + ) + with open(filename, 'w') as fh: + safe_dump(catchall, fh) -def _list_all_yaml_files(directory): - yaml_files = set() - for f in listdir(directory): - filename = join(directory, f) - if f.endswith('.yaml') and isfile(filename): - yaml_files.add(filename) - return list(yaml_files) + else: + # single large file + filename = join(self.directory, f'{desired.decoded_name}yaml') + self.log.debug('_apply: writing filename=%s', filename) + with open(filename, 'w') as fh: + safe_dump(dict(data), fh, allow_unicode=True) class SplitYamlProvider(YamlProvider): ''' - Core provider for records configured in multiple YAML files on disk. - - Behaves mostly similarly to YamlConfig, but interacts with multiple YAML - files, instead of a single monolitic one. All files are stored in a - subdirectory matching the name of the zone (including the trailing .) of - the directory config. The files are named RECORD.yaml, except for any - record which cannot be represented easily as a file; these are stored in - the catchall file, which is a YAML file the zone name, prepended with '$'. - For example, a zone, 'github.com.' would have a catch-all file named - '$github.com.yaml'. + DEPRECATED: Use YamlProvider with the split_extension parameter instead. - A full directory structure for the zone github.com. managed under directory - "zones/" would be: + When migrating the following configuration options would result in the same + behavior as SplitYamlProvider - zones/ - github.com./ - $github.com.yaml - www.yaml - ... + config: + class: octodns.provider.yaml.YamlProvider + # extension is configured as split_extension + split_extension: . + split_catchall: true + disable_zonefile: true - config: - class: octodns.provider.yaml.SplitYamlProvider - # The location of yaml config files (required) - directory: ./config - # The ttl to use for records when not specified in the data - # (optional, default 3600) - default_ttl: 3600 - # Whether or not to enforce sorting order on the yaml config - # (optional, default True) - enforce_order: True + TO BE REMOVED: 2.0 ''' - # Any record name added to this set will be included in the catch-all file, - # instead of a file matching the record name. - CATCHALL_RECORD_NAMES = ('*', '') - - def __init__(self, id, directory, extension='.', *args, **kwargs): - super().__init__(id, directory, *args, **kwargs) - self.extension = extension - - def _zone_directory(self, zone): - filename = f'{zone.name[:-1]}{self.extension}' - return join(self.directory, filename) - - def list_zones(self): - n = len(self.extension) - 1 - for filename in listdir(self.directory): - if not filename.endswith(self.extension): - continue - yield filename[:-n] - - def populate(self, zone, target=False, lenient=False): - self.log.debug( - 'populate: name=%s, target=%s, lenient=%s', - zone.name, - target, - lenient, + def __init__(self, id, directory, *args, extension='.', **kwargs): + kwargs.update( + { + 'split_extension': extension, + 'split_catchall': True, + 'disable_zonefile': True, + } ) - - if target: - # When acting as a target we ignore any existing records so that we - # create a completely new copy - return False - - before = len(zone.records) - yaml_filenames = _list_all_yaml_files(self._zone_directory(zone)) - self.log.info('populate: found %s YAML files', len(yaml_filenames)) - for yaml_filename in yaml_filenames: - self._populate_from_file(yaml_filename, zone, lenient) - - self.log.info( - 'populate: found %s records, exists=False', - len(zone.records) - before, + super().__init__(id, directory, *args, **kwargs) + self.log.warning( + '__init__: DEPRECATED use YamlProvider with split_extension, split_catchall, and disable_zonefile instead, will go away in v2.0' ) - return False - - def _do_apply(self, desired, data): - zone_dir = self._zone_directory(desired) - if not isdir(zone_dir): - makedirs(zone_dir) - - catchall = dict() - for record, config in data.items(): - if record in self.CATCHALL_RECORD_NAMES: - catchall[record] = config - continue - filename = join(zone_dir, f'{record}.yaml') - self.log.debug('_apply: writing filename=%s', filename) - with open(filename, 'w') as fh: - record_data = {record: config} - safe_dump(record_data, fh) - if catchall: - # Scrub the trailing . to make filenames more sane. - dname = desired.name[:-1] - filename = join(zone_dir, f'${dname}.yaml') - self.log.debug('_apply: writing catchall filename=%s', filename) - with open(filename, 'w') as fh: - safe_dump(catchall, fh) diff --git a/script/cibuild b/script/cibuild index f50882c..e652b72 100755 --- a/script/cibuild +++ b/script/cibuild @@ -16,7 +16,8 @@ fi echo "## environment & versions ######################################################" python --version pip --version - +echo "## modules: " +pip freeze echo "## clean up ####################################################################" find octodns tests -name "*.pyc" -exec rm {} \; rm -f *.pyc diff --git a/tests/config/split/unit.tests.yaml b/tests/config/split/unit.tests.yaml new file mode 100644 index 0000000..e249c43 --- /dev/null +++ b/tests/config/split/unit.tests.yaml @@ -0,0 +1,5 @@ +--- +only-zone-file: + type: TXT + value: Only included when zone file processing is enabled + diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index af89f78..a41a417 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -2,8 +2,9 @@ # # -from os import makedirs -from os.path import basename, dirname, isdir, isfile, join +from os import makedirs, remove +from os.path import dirname, isdir, isfile, join +from shutil import rmtree from unittest import TestCase from helpers import TemporaryDirectory @@ -12,16 +13,15 @@ from yaml.constructor import ConstructorError from octodns.idna import idna_encode from octodns.provider import ProviderException -from octodns.provider.base import Plan -from octodns.provider.yaml import ( - SplitYamlProvider, - YamlProvider, - _list_all_yaml_files, -) +from octodns.provider.yaml import SplitYamlProvider, YamlProvider from octodns.record import Create, NsValue, Record, ValuesMixin from octodns.zone import SubzoneRecordException, Zone +def touch(filename): + open(filename, 'w').close() + + class TestYamlProvider(TestCase): def test_provider(self): source = YamlProvider('test', join(dirname(__file__), 'config')) @@ -299,6 +299,7 @@ xn--dj-kia8a: self.assertTrue(source.supports(DummyType(self))) def test_list_zones(self): + # test of pre-existing config that lives on disk provider = YamlProvider('test', 'tests/config') self.assertEqual( [ @@ -307,9 +308,159 @@ xn--dj-kia8a: 'subzone.unit.tests.', 'unit.tests.', ], - sorted(provider.list_zones()), + list(provider.list_zones()), ) + # some synthetic tests to explicitly exercise the full functionality + with TemporaryDirectory() as td: + directory = join(td.dirname) + + # noise + touch(join(directory, 'README.txt')) + # not a zone.name.yaml + touch(join(directory, 'production.yaml')) + # non-zone directories + makedirs(join(directory, 'directory')) + makedirs(join(directory, 'never.matches')) + + # basic yaml zone files + touch(join(directory, 'unit.test.yaml')) + touch(join(directory, 'sub.unit.test.yaml')) + touch(join(directory, 'other.tld.yaml')) + touch(join(directory, 'both.tld.yaml')) + + # split zones with . + makedirs(join(directory, 'split.test.')) + makedirs(join(directory, 'sub.split.test.')) + makedirs(join(directory, 'other.split.')) + makedirs(join(directory, 'both.tld.')) + + # split zones with .tst + makedirs(join(directory, 'split-ext.test.tst')) + makedirs(join(directory, 'sub.split-ext.test.tst')) + makedirs(join(directory, 'other-ext.split.tst')) + + provider = YamlProvider('test', directory) + + # basic, should only find zone files + self.assertEqual( + ['both.tld.', 'other.tld.', 'sub.unit.test.', 'unit.test.'], + list(provider.list_zones()), + ) + + # include stuff with . AND basic + provider.split_extension = '.' + self.assertEqual( + [ + 'both.tld.', + 'other.split.', + 'other.tld.', + 'split.test.', + 'sub.split.test.', + 'sub.unit.test.', + 'unit.test.', + ], + list(provider.list_zones()), + ) + + # include stuff with .tst AND basic + provider.split_extension = '.tst' + self.assertEqual( + [ + 'both.tld.', + 'other-ext.split.', + 'other.tld.', + 'split-ext.test.', + 'sub.split-ext.test.', + 'sub.unit.test.', + 'unit.test.', + ], + list(provider.list_zones()), + ) + + # only .tst + provider.disable_zonefile = True + self.assertEqual( + ['other-ext.split.', 'split-ext.test.', 'sub.split-ext.test.'], + list(provider.list_zones()), + ) + + # only . (and both zone) + provider.split_extension = '.' + self.assertEqual( + ['both.tld.', 'other.split.', 'split.test.', 'sub.split.test.'], + list(provider.list_zones()), + ) + + def test_split_sources(self): + with TemporaryDirectory() as td: + directory = join(td.dirname) + + provider = YamlProvider('test', directory, split_extension='.') + + zone = Zone('déjà.vu.', []) + zone_utf8 = join(directory, f'{zone.decoded_name}') + zone_idna = join(directory, f'{zone.name}') + + filenames = ( + '*.yaml', + '.yaml', + 'www.yaml', + f'${zone.decoded_name}yaml', + ) + + # create the utf8 zone dir + makedirs(zone_utf8) + # nothing in it so we should get nothing back + self.assertEqual([], list(provider._split_sources(zone))) + # create some record files + for filename in filenames: + touch(join(zone_utf8, filename)) + # make sure we see them + expected = [join(zone_utf8, f) for f in sorted(filenames)] + self.assertEqual(expected, sorted(provider._split_sources(zone))) + + # add a idna zone directory + makedirs(zone_idna) + for filename in filenames: + touch(join(zone_idna, filename)) + with self.assertRaises(ProviderException) as ctx: + list(provider._split_sources(zone)) + msg = str(ctx.exception) + self.assertTrue('Both UTF-8' in msg) + + # delete the utf8 version + rmtree(zone_utf8) + expected = [join(zone_idna, f) for f in sorted(filenames)] + self.assertEqual(expected, sorted(provider._split_sources(zone))) + + def test_zone_sources(self): + with TemporaryDirectory() as td: + directory = join(td.dirname) + + provider = YamlProvider('test', directory) + + zone = Zone('déjà.vu.', []) + utf8 = join(directory, f'{zone.decoded_name}yaml') + idna = join(directory, f'{zone.name}yaml') + + # create the utf8 version + touch(utf8) + # make sure that's what we get back + self.assertEqual(utf8, provider._zone_sources(zone)) + + # create idna version, both exists + touch(idna) + with self.assertRaises(ProviderException) as ctx: + provider._zone_sources(zone) + msg = str(ctx.exception) + self.assertTrue('Both UTF-8' in msg) + + # delete the utf8 version + remove(utf8) + # make sure that we get the idna one back + self.assertEqual(idna, provider._zone_sources(zone)) + class TestSplitYamlProvider(TestCase): def test_list_all_yaml_files(self): @@ -323,40 +474,16 @@ class TestSplitYamlProvider(TestCase): # Create some files, some of them with a .yaml extension, all of # them empty. for emptyfile in all_files: - open(join(directory, emptyfile), 'w').close() + touch(join(directory, emptyfile)) # Do the same for some fake directories for emptydir in all_dirs: makedirs(join(directory, emptydir)) # This isn't great, but given the variable nature of the temp dir # names, it's necessary. - d = list(basename(f) for f in _list_all_yaml_files(directory)) + d = [join(directory, f) for f in yaml_files] self.assertEqual(len(yaml_files), len(d)) - def test_zone_directory(self): - source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split'), extension='.tst' - ) - - zone = Zone('unit.tests.', []) - - self.assertEqual( - join(dirname(__file__), 'config/split', 'unit.tests.tst'), - source._zone_directory(zone), - ) - - def test_apply_handles_existing_zone_directory(self): - with TemporaryDirectory() as td: - provider = SplitYamlProvider( - 'test', join(td.dirname, 'config'), extension='.tst' - ) - makedirs(join(td.dirname, 'config', 'does.exist.tst')) - - zone = Zone('does.exist.', []) - self.assertTrue(isdir(provider._zone_directory(zone))) - provider.apply(Plan(None, zone, [], True)) - self.assertTrue(isdir(provider._zone_directory(zone))) - def test_provider(self): source = SplitYamlProvider( 'test', @@ -375,9 +502,23 @@ class TestSplitYamlProvider(TestCase): # without it we see everything source.populate(zone) self.assertEqual(20, len(zone.records)) + self.assertFalse([r for r in zone.records if r.name.startswith('only')]) + + # temporarily enable zone file processing too, we should see one extra + # record that came from unit.tests. + source.disable_zonefile = False + zone_both = Zone('unit.tests.', []) + source.populate(zone_both) + self.assertEqual(21, len(zone_both.records)) + n = len([r for r in zone_both.records if r.name == 'only-zone-file']) + self.assertEqual(1, n) + source.disable_zonefile = True source.populate(dynamic_zone) self.assertEqual(5, len(dynamic_zone.records)) + self.assertFalse( + [r for r in dynamic_zone.records if r.name.startswith('only')] + ) with TemporaryDirectory() as td: # Add some subdirs to make sure that it can create them @@ -579,7 +720,7 @@ class TestSplitYamlProvider(TestCase): ) copy = source.copy() self.assertEqual(source.directory, copy.directory) - self.assertEqual(source.extension, copy.extension) + self.assertEqual(source.split_extension, copy.split_extension) self.assertEqual(source.default_ttl, copy.default_ttl) self.assertEqual(source.enforce_order, copy.enforce_order) self.assertEqual(