From 11ddb20005676a95b5367946ffcea8eb8d3437ae Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 12 Aug 2023 11:18:33 -0700
Subject: [PATCH 001/116] Refactory YamlProvider and SplitYamlProvider into a
 unified class

---
 octodns/provider/yaml.py            | 315 +++++++++++++++-------------
 tests/test_octodns_provider_yaml.py |  12 +-
 2 files changed, 176 insertions(+), 151 deletions(-)

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index 1d495a9..e0a727a 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -28,8 +28,35 @@ class YamlProvider(BaseProvider):
         # (optional, default True)
         enforce_order: true
         # Whether duplicate records should replace rather than error
-        # (optiona, default False)
+        # (optional, default False)
         populate_should_replace: false
+        # The filename used to load split style zones, False means disabled.
+        # When enabled the provider will search for zone records split across
+        # multiple YAML files in a directory with the zone name.
+        # See "Split Details" below for more information
+        # (optional, default False, . is the recommended best practice when
+        # enabling)
+        split_extension: false
+
+    Split Details
+    -------------
+
+    All files are stored in a subdirectory matching the name of the zone
+    (including the trailing .) of the directory config. It is a recommended
+    best practice that the files be named RECORD.yaml, but all files are
+    sourced and processed as if they were a single large file.
+
+    A full directory structure for the zone github.com. managed under directory
+    "zones/" would be:
+
+    zones/
+      github.com./
+        .yaml
+        www.yaml
+        ...
+
+    Overriding Values
+    -----------------
 
     Overriding values can be accomplished using multiple yaml providers in the
     `sources` list where subsequent providers have `populate_should_replace`
@@ -98,7 +125,6 @@ class YamlProvider(BaseProvider):
     You can then sync our records eternally with `--config-file=external.yaml`
     and internally (with the custom overrides) with
     `--config-file=internal.yaml`
-
     '''
 
     SUPPORTS_GEO = True
@@ -107,6 +133,10 @@ class YamlProvider(BaseProvider):
     SUPPORTS_DYNAMIC_SUBNETS = True
     SUPPORTS_MULTIVALUE_PTR = True
 
+    # Any record name added to this set will be included in the catch-all file,
+    # instead of a file matching the record name.
+    CATCHALL_RECORD_NAMES = ('*', '')
+
     def __init__(
         self,
         id,
@@ -115,19 +145,25 @@ class YamlProvider(BaseProvider):
         enforce_order=True,
         populate_should_replace=False,
         supports_root_ns=True,
+        split_extension=False,
+        split_only=False,
+        split_catchall=False,
         *args,
         **kwargs,
     ):
         klass = self.__class__.__name__
         self.log = logging.getLogger(f'{klass}[{id}]')
         self.log.debug(
-            '__init__: id=%s, directory=%s, default_ttl=%d, '
-            'enforce_order=%d, populate_should_replace=%d',
+            '__init__: id=%s, directory=%s, default_ttl=%d, enforce_order=%d, populate_should_replace=%s, supports_root_ns=%s, split_extension=%s, split_only=%s, split_catchall=%s',
             id,
             directory,
             default_ttl,
             enforce_order,
             populate_should_replace,
+            supports_root_ns,
+            split_extension,
+            split_only,
+            split_catchall,
         )
         super().__init__(id, *args, **kwargs)
         self.directory = directory
@@ -135,12 +171,15 @@ class YamlProvider(BaseProvider):
         self.enforce_order = enforce_order
         self.populate_should_replace = populate_should_replace
         self.supports_root_ns = supports_root_ns
+        self.split_extension = split_extension
+        self.split_only = split_only
+        self.split_catchall = split_catchall
 
     def copy(self):
-        args = dict(self.__dict__)
-        args['id'] = f'{args["id"]}-copy'
-        del args['log']
-        return self.__class__(**args)
+        kwargs = dict(self.__dict__)
+        kwargs['id'] = f'{kwargs["id"]}-copy'
+        del kwargs['log']
+        return YamlProvider(**kwargs)
 
     @property
     def SUPPORTS(self):
@@ -162,6 +201,37 @@ class YamlProvider(BaseProvider):
     def SUPPORTS_ROOT_NS(self):
         return self.supports_root_ns
 
+    def list_zones(self):
+        self.log.debug('list_zones:')
+        zones = set()
+
+        # TODO: don't allow both utf8 and idna versions of the same zone
+        extension = self.split_extension
+        if extension:
+            self.log.debug('list_zones:   looking for split zones')
+            # look for split
+            # we want to leave the .
+            trim = len(extension) - 1
+            for dirname in listdir(self.directory):
+                if not dirname.endswith(extension) or not isdir(
+                    join(self.directory, dirname)
+                ):
+                    continue
+                zones.add(dirname[:-trim])
+
+        if not self.split_only:
+            self.log.debug('list_zones:   looking for zone files')
+            for filename in listdir(self.directory):
+                if (
+                    not filename.endswith('.yaml')
+                    or filename.count('.') < 2
+                    or not isfile(join(self.directory, filename))
+                ):
+                    continue
+                zones.add(filename[:-4])
+
+        return sorted(zones)
+
     def _populate_from_file(self, filename, zone, lenient):
         with open(filename, 'r') as fh:
             yaml_data = safe_load(fh, enforce_order=self.enforce_order)
@@ -184,18 +254,6 @@ class YamlProvider(BaseProvider):
                 '_populate_from_file: successfully loaded "%s"', filename
             )
 
-    def get_filenames(self, zone):
-        return (
-            join(self.directory, f'{zone.decoded_name}yaml'),
-            join(self.directory, f'{zone.name}yaml'),
-        )
-
-    def list_zones(self):
-        for filename in listdir(self.directory):
-            if not filename.endswith('.yaml') or filename.count('.') < 2:
-                continue
-            yield filename[:-4]
-
     def populate(self, zone, target=False, lenient=False):
         self.log.debug(
             'populate: name=%s, target=%s, lenient=%s',
@@ -210,23 +268,52 @@ class YamlProvider(BaseProvider):
             return False
 
         before = len(zone.records)
-        utf8_filename, idna_filename = self.get_filenames(zone)
 
-        # we prefer utf8
-        if isfile(utf8_filename):
-            if utf8_filename != idna_filename and isfile(idna_filename):
-                raise ProviderException(
-                    f'Both UTF-8 "{utf8_filename}" and IDNA "{idna_filename}" exist for {zone.decoded_name}'
-                )
-            filename = utf8_filename
-        else:
-            self.log.warning(
-                'populate: "%s" does not exist, falling back to try idna version "%s"',
-                utf8_filename,
-                idna_filename,
-            )
-            filename = idna_filename
-        self._populate_from_file(filename, zone, lenient)
+        sources = []
+
+        zone_name_utf8 = zone.name[:-1]
+        zone_name_idna = zone.decoded_name[:-1]
+
+        directory = None
+        split_extension = self.split_extension
+        if split_extension:
+            utf8 = join(self.directory, f'{zone_name_utf8}{split_extension}')
+            idna = join(self.directory, f'{zone_name_idna}{split_extension}')
+            directory = None
+            if isdir(utf8):
+                if utf8 != idna and isdir(idna):
+                    raise ProviderException(
+                        f'Both UTF-8 "{utf8}" and IDNA "{idna}" exist for {zone.decoded_name}'
+                    )
+                directory = utf8
+            else:
+                directory = idna
+
+            for filename in listdir(directory):
+                if filename.endswith('.yaml'):
+                    sources.append(join(directory, filename))
+
+        if not self.split_only:
+            utf8 = join(self.directory, f'{zone_name_utf8}.yaml')
+            idna = join(self.directory, f'{zone_name_idna}.yaml')
+            if isfile(utf8):
+                if utf8 != idna and isfile(idna):
+                    raise ProviderException(
+                        f'Both UTF-8 "{utf8}" and IDNA "{idna}" exist for {zone.decoded_name}'
+                    )
+                sources.append(utf8)
+            else:
+                sources.append(idna)
+
+        if len(sources) == 0:
+            # TODO: what if we don't have any files
+            pass
+
+        # determinstically order our sources
+        sources.sort()
+
+        for source in sources:
+            self._populate_from_file(source, zone, lenient)
 
         self.log.info(
             'populate:   found %s records, exists=False',
@@ -264,123 +351,65 @@ class YamlProvider(BaseProvider):
                 data[k] = data[k][0]
 
         if not isdir(self.directory):
+            self.log.debug('_apply: creating directory=%s', self.directory)
             makedirs(self.directory)
 
-        self._do_apply(desired, data)
-
-    def _do_apply(self, desired, data):
-        filename = join(self.directory, f'{desired.decoded_name}yaml')
-        self.log.debug('_apply:   writing filename=%s', filename)
-        with open(filename, 'w') as fh:
-            safe_dump(dict(data), fh, allow_unicode=True)
+        if self.split_extension:
+            # we're going to do split files
+            decoded_name = desired.decoded_name[:-1]
+            directory = join(
+                self.directory, f'{decoded_name}{self.split_extension}'
+            )
 
+            if not isdir(directory):
+                self.log.debug('_apply: creating split directory=%s', directory)
+                makedirs(directory)
+
+            catchall = {}
+            for record, config in data.items():
+                if self.split_catchall and record in self.CATCHALL_RECORD_NAMES:
+                    catchall[record] = config
+                    continue
+                filename = join(directory, f'{record}.yaml')
+                self.log.debug('_apply:   writing filename=%s', filename)
+
+                with open(filename, 'w') as fh:
+                    record_data = {record: config}
+                    safe_dump(record_data, fh)
+
+            if catchall:
+                # Scrub the trailing . to make filenames more sane.
+                filename = join(directory, f'${decoded_name}.yaml')
+                self.log.debug(
+                    '_apply:   writing catchall filename=%s', filename
+                )
+                with open(filename, 'w') as fh:
+                    safe_dump(catchall, fh)
 
-def _list_all_yaml_files(directory):
-    yaml_files = set()
-    for f in listdir(directory):
-        filename = join(directory, f)
-        if f.endswith('.yaml') and isfile(filename):
-            yaml_files.add(filename)
-    return list(yaml_files)
+        else:
+            # single large file
+            filename = join(self.directory, f'{desired.decoded_name}yaml')
+            self.log.debug('_apply:   writing filename=%s', filename)
+            with open(filename, 'w') as fh:
+                safe_dump(dict(data), fh, allow_unicode=True)
 
 
 class SplitYamlProvider(YamlProvider):
     '''
-    Core provider for records configured in multiple YAML files on disk.
-
-    Behaves mostly similarly to YamlConfig, but interacts with multiple YAML
-    files, instead of a single monolitic one. All files are stored in a
-    subdirectory matching the name of the zone (including the trailing .) of
-    the directory config. The files are named RECORD.yaml, except for any
-    record which cannot be represented easily as a file; these are stored in
-    the catchall file, which is a YAML file the zone name, prepended with '$'.
-    For example, a zone, 'github.com.' would have a catch-all file named
-    '$github.com.yaml'.
+    DEPRECATED: Use YamlProvider with the split_extension parameter instead.
 
-    A full directory structure for the zone github.com. managed under directory
-    "zones/" would be:
-
-    zones/
-      github.com./
-        $github.com.yaml
-        www.yaml
-        ...
-
-    config:
-        class: octodns.provider.yaml.SplitYamlProvider
-        # The location of yaml config files (required)
-        directory: ./config
-        # The ttl to use for records when not specified in the data
-        # (optional, default 3600)
-        default_ttl: 3600
-        # Whether or not to enforce sorting order on the yaml config
-        # (optional, default True)
-        enforce_order: True
+    TO BE REMOVED: 2.0
     '''
 
-    # Any record name added to this set will be included in the catch-all file,
-    # instead of a file matching the record name.
-    CATCHALL_RECORD_NAMES = ('*', '')
-
-    def __init__(self, id, directory, extension='.', *args, **kwargs):
-        super().__init__(id, directory, *args, **kwargs)
-        self.extension = extension
-
-    def _zone_directory(self, zone):
-        filename = f'{zone.name[:-1]}{self.extension}'
-        return join(self.directory, filename)
-
-    def list_zones(self):
-        n = len(self.extension) - 1
-        for filename in listdir(self.directory):
-            if not filename.endswith(self.extension):
-                continue
-            yield filename[:-n]
-
-    def populate(self, zone, target=False, lenient=False):
-        self.log.debug(
-            'populate: name=%s, target=%s, lenient=%s',
-            zone.name,
-            target,
-            lenient,
+    def __init__(self, id, directory, *args, extension='.', **kwargs):
+        kwargs.update(
+            {
+                'split_extension': extension,
+                'split_only': True,
+                'split_catchall': True,
+            }
         )
-
-        if target:
-            # When acting as a target we ignore any existing records so that we
-            # create a completely new copy
-            return False
-
-        before = len(zone.records)
-        yaml_filenames = _list_all_yaml_files(self._zone_directory(zone))
-        self.log.info('populate:   found %s YAML files', len(yaml_filenames))
-        for yaml_filename in yaml_filenames:
-            self._populate_from_file(yaml_filename, zone, lenient)
-
-        self.log.info(
-            'populate:   found %s records, exists=False',
-            len(zone.records) - before,
+        super().__init__(id, directory, *args, **kwargs)
+        self.log.warning(
+            '__init__: DEPRECATED use YamlProvider with split_extension and optionally split_only instead, will go away in v2.0'
         )
-        return False
-
-    def _do_apply(self, desired, data):
-        zone_dir = self._zone_directory(desired)
-        if not isdir(zone_dir):
-            makedirs(zone_dir)
-
-        catchall = dict()
-        for record, config in data.items():
-            if record in self.CATCHALL_RECORD_NAMES:
-                catchall[record] = config
-                continue
-            filename = join(zone_dir, f'{record}.yaml')
-            self.log.debug('_apply:   writing filename=%s', filename)
-            with open(filename, 'w') as fh:
-                record_data = {record: config}
-                safe_dump(record_data, fh)
-        if catchall:
-            # Scrub the trailing . to make filenames more sane.
-            dname = desired.name[:-1]
-            filename = join(zone_dir, f'${dname}.yaml')
-            self.log.debug('_apply:   writing catchall filename=%s', filename)
-            with open(filename, 'w') as fh:
-                safe_dump(catchall, fh)
diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py
index 1cf017a..8f52901 100644
--- a/tests/test_octodns_provider_yaml.py
+++ b/tests/test_octodns_provider_yaml.py
@@ -3,7 +3,7 @@
 #
 
 from os import makedirs
-from os.path import basename, dirname, isdir, isfile, join
+from os.path import dirname, isdir, isfile, join
 from unittest import TestCase
 
 from helpers import TemporaryDirectory
@@ -13,11 +13,7 @@ from yaml.constructor import ConstructorError
 from octodns.idna import idna_encode
 from octodns.provider import ProviderException
 from octodns.provider.base import Plan
-from octodns.provider.yaml import (
-    SplitYamlProvider,
-    YamlProvider,
-    _list_all_yaml_files,
-)
+from octodns.provider.yaml import SplitYamlProvider, YamlProvider
 from octodns.record import Create, NsValue, Record, ValuesMixin
 from octodns.zone import SubzoneRecordException, Zone
 
@@ -327,7 +323,7 @@ class TestSplitYamlProvider(TestCase):
 
             # This isn't great, but given the variable nature of the temp dir
             # names, it's necessary.
-            d = list(basename(f) for f in _list_all_yaml_files(directory))
+            d = [join(directory, f) for f in yaml_files]
             self.assertEqual(len(yaml_files), len(d))
 
     def test_zone_directory(self):
@@ -573,7 +569,7 @@ class TestSplitYamlProvider(TestCase):
         )
         copy = source.copy()
         self.assertEqual(source.directory, copy.directory)
-        self.assertEqual(source.extension, copy.extension)
+        self.assertEqual(source.split_extension, copy.split_extension)
         self.assertEqual(source.default_ttl, copy.default_ttl)
         self.assertEqual(source.enforce_order, copy.enforce_order)
         self.assertEqual(

From 5b8498a550681857477aaaddd6a83f1792857759 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 12 Aug 2023 19:31:38 -0700
Subject: [PATCH 002/116] Refactory yaml source logic out to make it easily
 testable

---
 octodns/provider/yaml.py            | 68 +++++++++++-----------
 tests/test_octodns_provider_yaml.py | 90 ++++++++++++++++++++++-------
 2 files changed, 103 insertions(+), 55 deletions(-)

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index e0a727a..2579414 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -205,11 +205,9 @@ class YamlProvider(BaseProvider):
         self.log.debug('list_zones:')
         zones = set()
 
-        # TODO: don't allow both utf8 and idna versions of the same zone
         extension = self.split_extension
         if extension:
             self.log.debug('list_zones:   looking for split zones')
-            # look for split
             # we want to leave the .
             trim = len(extension) - 1
             for dirname in listdir(self.directory):
@@ -228,10 +226,41 @@ class YamlProvider(BaseProvider):
                     or not isfile(join(self.directory, filename))
                 ):
                     continue
+                # trim off the yaml, leave the .
                 zones.add(filename[:-4])
 
         return sorted(zones)
 
+    def _split_sources(self, zone):
+        ext = self.split_extension
+        utf8 = join(self.directory, f'{zone.decoded_name[:-1]}{ext}')
+        idna = join(self.directory, f'{zone.name[:-1]}{ext}')
+        directory = None
+        if isdir(utf8):
+            if utf8 != idna and isdir(idna):
+                raise ProviderException(
+                    f'Both UTF-8 "{utf8}" and IDNA "{idna}" exist for {zone.decoded_name}'
+                )
+            directory = utf8
+        else:
+            directory = idna
+
+        for filename in listdir(directory):
+            if filename.endswith('.yaml'):
+                yield join(directory, filename)
+
+    def _zone_sources(self, zone):
+        utf8 = join(self.directory, f'{zone.decoded_name}yaml')
+        idna = join(self.directory, f'{zone.name}yaml')
+        if isfile(utf8):
+            if utf8 != idna and isfile(idna):
+                raise ProviderException(
+                    f'Both UTF-8 "{utf8}" and IDNA "{idna}" exist for {zone.decoded_name}'
+                )
+            return utf8
+
+        return idna
+
     def _populate_from_file(self, filename, zone, lenient):
         with open(filename, 'r') as fh:
             yaml_data = safe_load(fh, enforce_order=self.enforce_order)
@@ -271,43 +300,12 @@ class YamlProvider(BaseProvider):
 
         sources = []
 
-        zone_name_utf8 = zone.name[:-1]
-        zone_name_idna = zone.decoded_name[:-1]
-
-        directory = None
         split_extension = self.split_extension
         if split_extension:
-            utf8 = join(self.directory, f'{zone_name_utf8}{split_extension}')
-            idna = join(self.directory, f'{zone_name_idna}{split_extension}')
-            directory = None
-            if isdir(utf8):
-                if utf8 != idna and isdir(idna):
-                    raise ProviderException(
-                        f'Both UTF-8 "{utf8}" and IDNA "{idna}" exist for {zone.decoded_name}'
-                    )
-                directory = utf8
-            else:
-                directory = idna
-
-            for filename in listdir(directory):
-                if filename.endswith('.yaml'):
-                    sources.append(join(directory, filename))
+            sources.extend(self._split_sources(zone))
 
         if not self.split_only:
-            utf8 = join(self.directory, f'{zone_name_utf8}.yaml')
-            idna = join(self.directory, f'{zone_name_idna}.yaml')
-            if isfile(utf8):
-                if utf8 != idna and isfile(idna):
-                    raise ProviderException(
-                        f'Both UTF-8 "{utf8}" and IDNA "{idna}" exist for {zone.decoded_name}'
-                    )
-                sources.append(utf8)
-            else:
-                sources.append(idna)
-
-        if len(sources) == 0:
-            # TODO: what if we don't have any files
-            pass
+            sources.append(self._zone_sources(zone))
 
         # determinstically order our sources
         sources.sort()
diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py
index 8f52901..89b4bb1 100644
--- a/tests/test_octodns_provider_yaml.py
+++ b/tests/test_octodns_provider_yaml.py
@@ -2,8 +2,9 @@
 #
 #
 
-from os import makedirs
+from os import makedirs, remove
 from os.path import dirname, isdir, isfile, join
+from shutil import rmtree
 from unittest import TestCase
 
 from helpers import TemporaryDirectory
@@ -12,12 +13,16 @@ from yaml.constructor import ConstructorError
 
 from octodns.idna import idna_encode
 from octodns.provider import ProviderException
-from octodns.provider.base import Plan
 from octodns.provider.yaml import SplitYamlProvider, YamlProvider
 from octodns.record import Create, NsValue, Record, ValuesMixin
 from octodns.zone import SubzoneRecordException, Zone
 
 
+def touch(filename):
+    with open(filename, 'w'):
+        pass
+
+
 class TestYamlProvider(TestCase):
     def test_provider(self):
         source = YamlProvider('test', join(dirname(__file__), 'config'))
@@ -326,29 +331,74 @@ class TestSplitYamlProvider(TestCase):
             d = [join(directory, f) for f in yaml_files]
             self.assertEqual(len(yaml_files), len(d))
 
-    def test_zone_directory(self):
-        source = SplitYamlProvider(
-            'test', join(dirname(__file__), 'config/split'), extension='.tst'
-        )
+    def test_split_sources(self):
+        with TemporaryDirectory() as td:
+            directory = join(td.dirname)
 
-        zone = Zone('unit.tests.', [])
+            provider = YamlProvider('test', directory, split_extension='.')
 
-        self.assertEqual(
-            join(dirname(__file__), 'config/split', 'unit.tests.tst'),
-            source._zone_directory(zone),
-        )
+            zone = Zone('déjà.vu.', [])
+            zone_utf8 = join(directory, f'{zone.decoded_name}')
+            zone_idna = join(directory, f'{zone.name}')
 
-    def test_apply_handles_existing_zone_directory(self):
-        with TemporaryDirectory() as td:
-            provider = SplitYamlProvider(
-                'test', join(td.dirname, 'config'), extension='.tst'
+            filenames = (
+                '*.yaml',
+                '.yaml',
+                'www.yaml',
+                f'${zone.decoded_name}yaml',
             )
-            makedirs(join(td.dirname, 'config', 'does.exist.tst'))
 
-            zone = Zone('does.exist.', [])
-            self.assertTrue(isdir(provider._zone_directory(zone)))
-            provider.apply(Plan(None, zone, [], True))
-            self.assertTrue(isdir(provider._zone_directory(zone)))
+            # create the utf8 zone dir
+            makedirs(zone_utf8)
+            # nothing in it so we should get nothing back
+            self.assertEqual([], list(provider._split_sources(zone)))
+            # create some record files
+            for filename in filenames:
+                touch(join(zone_utf8, filename))
+            # make sure we see them
+            expected = [join(zone_utf8, f) for f in sorted(filenames)]
+            self.assertEqual(expected, sorted(provider._split_sources(zone)))
+
+            # add a idna zone directory
+            makedirs(zone_idna)
+            for filename in filenames:
+                touch(join(zone_idna, filename))
+            with self.assertRaises(ProviderException) as ctx:
+                list(provider._split_sources(zone))
+            msg = str(ctx.exception)
+            self.assertTrue('Both UTF-8' in msg)
+
+            # delete the utf8 version
+            rmtree(zone_utf8)
+            expected = [join(zone_idna, f) for f in sorted(filenames)]
+            self.assertEqual(expected, sorted(provider._split_sources(zone)))
+
+    def test_zone_sources(self):
+        with TemporaryDirectory() as td:
+            directory = join(td.dirname)
+
+            provider = YamlProvider('test', directory)
+
+            zone = Zone('déjà.vu.', [])
+            utf8 = join(directory, f'{zone.decoded_name}yaml')
+            idna = join(directory, f'{zone.name}yaml')
+
+            # create the utf8 version
+            touch(utf8)
+            # make sure that's what we get back
+            self.assertEqual(utf8, provider._zone_sources(zone))
+
+            # create idna version, both exists
+            touch(idna)
+            with self.assertRaises(ProviderException) as ctx:
+                provider._zone_sources(zone)
+            msg = str(ctx.exception)
+            self.assertTrue('Both UTF-8' in msg)
+
+            # delete the utf8 version
+            remove(utf8)
+            # make sure that we get the idna one back
+            self.assertEqual(idna, provider._zone_sources(zone))
 
     def test_provider(self):
         source = SplitYamlProvider(

From 3c304aa6ee5a3a4366d61e19a8d955012bd52b8d Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 17 Aug 2023 09:27:03 -0700
Subject: [PATCH 003/116] Pass through YamlProvider doc and CHANGELOG entry

---
 CHANGELOG.md             |  6 ++++++
 octodns/provider/yaml.py | 35 +++++++++++++++++++++++++++++++----
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7b8025f..3cdef09 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,12 @@
 * New dynamic zone config support that allows wildcard entries in the octoDNS
   config to be expanded by the source provider(s). See
   [Dynamic Zone Config](/README.md#dynamic-zone-config) for more information.
+* SplitYamlProvider has been deprecated and will be removed in 2.0. YamlProvider
+  now includes the ability to process split zones when configured to do so and
+  allows for more flexibility in how things are laid out than was previously
+  possible. This includes the ability to split some zones and not others and
+  even to have partially split zones with some records in the primary zone YAML
+  and others in a split directory. See YamlProvider documentation for more info.
 
 #### Stuff
 
diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index 2579414..c1c4db8 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -32,11 +32,27 @@ class YamlProvider(BaseProvider):
         populate_should_replace: false
         # The filename used to load split style zones, False means disabled.
         # When enabled the provider will search for zone records split across
-        # multiple YAML files in a directory with the zone name.
+        # multiple YAML files in the directory with split_extension appended to
+        # the zone name. split_extension should include the `.`
         # See "Split Details" below for more information
         # (optional, default False, . is the recommended best practice when
         # enabling)
         split_extension: false
+        # Disable loading of the primary zone .yaml file. If split_extension
+        # is defined both split files and the primary zone .yaml will be loaded
+        # by default. Setting this to true will disable that and rely soley on
+        # split files.
+        # (optional, default False)
+        split_only: false
+        # When writing YAML records out to disk with split_extension enabled
+        # each record is written out into its own file with .yaml appended to
+        # the name of the record. This would result in files like `.yaml` for
+        # the apex and `*.yaml` for a wildcard. If your OS doesn't allow such
+        # filenames or you would prefer to avoid them you can enable
+        # split_catchall to instead write those records into a file named
+        # `$[zone.name].yaml`
+        # (optional, default False)
+        split_catchall: false
 
     Split Details
     -------------
@@ -44,10 +60,11 @@ class YamlProvider(BaseProvider):
     All files are stored in a subdirectory matching the name of the zone
     (including the trailing .) of the directory config. It is a recommended
     best practice that the files be named RECORD.yaml, but all files are
-    sourced and processed as if they were a single large file.
+    sourced and processed ignoring the filenames so it is up to you how to
+    organize them.
 
-    A full directory structure for the zone github.com. managed under directory
-    "zones/" would be:
+    With `split_extension: .` the directory structure for the zone github.com.
+    managed under directory "zones/" would look like:
 
     zones/
       github.com./
@@ -396,6 +413,16 @@ class SplitYamlProvider(YamlProvider):
     '''
     DEPRECATED: Use YamlProvider with the split_extension parameter instead.
 
+    When migrating the following configuration options would result in the same
+    behavior as SplitYamlProvider
+
+       config:
+         class: octodns.provider.yaml.YamlProvider
+         # extension is configured as split_extension
+         split_extension: .
+         split_only: true
+         split_catchall: true
+
     TO BE REMOVED: 2.0
     '''
 

From 10c31e37e7ff8c2eca02af12388a89e8f952d3f1 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 17 Aug 2023 10:31:10 -0700
Subject: [PATCH 004/116] Dump a pip freeze into ci output for ease of seeing
 module verisons

---
 script/cibuild | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/script/cibuild b/script/cibuild
index f50882c..e652b72 100755
--- a/script/cibuild
+++ b/script/cibuild
@@ -16,7 +16,8 @@ fi
 echo "## environment & versions ######################################################"
 python --version
 pip --version
-
+echo "## modules: "
+pip freeze
 echo "## clean up ####################################################################"
 find octodns tests -name "*.pyc" -exec rm {} \;
 rm -f *.pyc

From 608e367a9bad0ecd3236d3004da3e8b2bdef50b6 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 17 Aug 2023 12:13:42 -0700
Subject: [PATCH 005/116] More extensive tests of YamlProvider.list_zones

---
 octodns/provider/yaml.py            |  8 +++-
 tests/test_octodns_provider_yaml.py | 71 +++++++++++++++++++++++++++--
 2 files changed, 73 insertions(+), 6 deletions(-)

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index c1c4db8..bd77b19 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -224,15 +224,19 @@ class YamlProvider(BaseProvider):
 
         extension = self.split_extension
         if extension:
-            self.log.debug('list_zones:   looking for split zones')
             # we want to leave the .
             trim = len(extension) - 1
+            self.log.debug(
+                'list_zones:   looking for split zones, trim=%d', trim
+            )
             for dirname in listdir(self.directory):
                 if not dirname.endswith(extension) or not isdir(
                     join(self.directory, dirname)
                 ):
                     continue
-                zones.add(dirname[:-trim])
+                if trim:
+                    dirname = dirname[:-trim]
+                zones.add(dirname)
 
         if not self.split_only:
             self.log.debug('list_zones:   looking for zone files')
diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py
index 89b4bb1..80ded06 100644
--- a/tests/test_octodns_provider_yaml.py
+++ b/tests/test_octodns_provider_yaml.py
@@ -19,8 +19,7 @@ from octodns.zone import SubzoneRecordException, Zone
 
 
 def touch(filename):
-    with open(filename, 'w'):
-        pass
+    open(filename, 'w').close()
 
 
 class TestYamlProvider(TestCase):
@@ -297,6 +296,7 @@ xn--dj-kia8a:
         self.assertTrue(source.supports(DummyType(self)))
 
     def test_list_zones(self):
+        # test of pre-existing config that lives on disk
         provider = YamlProvider('test', 'tests/config')
         self.assertEqual(
             [
@@ -305,9 +305,72 @@ xn--dj-kia8a:
                 'subzone.unit.tests.',
                 'unit.tests.',
             ],
-            sorted(provider.list_zones()),
+            list(provider.list_zones()),
         )
 
+        # some synthetic tests to explicitly exercise the full functionality
+        with TemporaryDirectory() as td:
+            directory = join(td.dirname)
+
+            # noise
+            touch(join(directory, 'README.txt'))
+            # not a zone.name.yaml
+            touch(join(directory, 'production.yaml'))
+
+            # basic yaml zone files
+            touch(join(directory, 'unit.test.yaml'))
+            touch(join(directory, 'sub.unit.test.yaml'))
+            touch(join(directory, 'other.tld.yaml'))
+            touch(join(directory, 'both.tld.yaml'))
+
+            # split zones with .
+            makedirs(join(directory, 'split.test.'))
+            makedirs(join(directory, 'sub.split.test.'))
+            makedirs(join(directory, 'other.split.'))
+            makedirs(join(directory, 'both.tld.'))
+
+            # split zones with .tst
+            makedirs(join(directory, 'split-ext.test.tst'))
+            makedirs(join(directory, 'sub.split-ext.test.tst'))
+            makedirs(join(directory, 'other-ext.split.tst'))
+
+            provider = YamlProvider('test', directory)
+
+            # basic, should only find zone files
+            self.assertEqual(
+                ['both.tld.', 'other.tld.', 'sub.unit.test.', 'unit.test.'],
+                list(provider.list_zones()),
+            )
+
+            # include stuff with . AND basic
+            provider.split_extension = '.'
+            self.assertEqual(
+                [
+                    'both.tld.',
+                    'other.split.',
+                    'other.tld.',
+                    'split.test.',
+                    'sub.split.test.',
+                    'sub.unit.test.',
+                    'unit.test.',
+                ],
+                list(provider.list_zones()),
+            )
+
+            provider.split_extension = '.tst'
+            self.assertEqual(
+                [
+                    'both.tld.',
+                    'other-ext.split.',
+                    'other.tld.',
+                    'split-ext.test.',
+                    'sub.split-ext.test.',
+                    'sub.unit.test.',
+                    'unit.test.',
+                ],
+                list(provider.list_zones()),
+            )
+
 
 class TestSplitYamlProvider(TestCase):
     def test_list_all_yaml_files(self):
@@ -321,7 +384,7 @@ class TestSplitYamlProvider(TestCase):
             # Create some files, some of them with a .yaml extension, all of
             # them empty.
             for emptyfile in all_files:
-                open(join(directory, emptyfile), 'w').close()
+                touch(join(directory, emptyfile))
             # Do the same for some fake directories
             for emptydir in all_dirs:
                 makedirs(join(directory, emptydir))

From e473c32bfbd2babe6b3fe8f23a6861f601488c0f Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 17 Aug 2023 12:23:43 -0700
Subject: [PATCH 006/116] Add some directories to ignore

---
 tests/test_octodns_provider_yaml.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py
index 80ded06..6e0a615 100644
--- a/tests/test_octodns_provider_yaml.py
+++ b/tests/test_octodns_provider_yaml.py
@@ -316,6 +316,9 @@ xn--dj-kia8a:
             touch(join(directory, 'README.txt'))
             # not a zone.name.yaml
             touch(join(directory, 'production.yaml'))
+            # non-zone directories
+            makedirs(join(directory, 'directory'))
+            makedirs(join(directory, 'never.matches'))
 
             # basic yaml zone files
             touch(join(directory, 'unit.test.yaml'))

From 65096c0f1c8b12f926ed44006cc30d3d1f4dbfd3 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 17 Aug 2023 12:52:13 -0700
Subject: [PATCH 007/116] break up ifs having coverage issues in 3.8/9

---
 octodns/provider/yaml.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index bd77b19..e518efc 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -230,9 +230,9 @@ class YamlProvider(BaseProvider):
                 'list_zones:   looking for split zones, trim=%d', trim
             )
             for dirname in listdir(self.directory):
-                if not dirname.endswith(extension) or not isdir(
-                    join(self.directory, dirname)
-                ):
+                not_ends_with = not dirname.endswith(extension)
+                not_dir = not isdir(join(self.directory, dirname))
+                if not_ends_with or not_dir:
                     continue
                 if trim:
                     dirname = dirname[:-trim]
@@ -241,11 +241,10 @@ class YamlProvider(BaseProvider):
         if not self.split_only:
             self.log.debug('list_zones:   looking for zone files')
             for filename in listdir(self.directory):
-                if (
-                    not filename.endswith('.yaml')
-                    or filename.count('.') < 2
-                    or not isfile(join(self.directory, filename))
-                ):
+                not_ends_with = not filename.endswith('.yaml')
+                too_few_dots = filename.count('.') < 2
+                not_file = not isfile(join(self.directory, filename))
+                if not_ends_with or too_few_dots or not_file:
                     continue
                 # trim off the yaml, leave the .
                 zones.add(filename[:-4])

From b0dff4fe04d81eff9976269a2397b29934bb7c56 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 17 Aug 2023 13:15:15 -0700
Subject: [PATCH 008/116] Try reordering if clauses

---
 octodns/provider/yaml.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index e518efc..a4eceeb 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -232,7 +232,7 @@ class YamlProvider(BaseProvider):
             for dirname in listdir(self.directory):
                 not_ends_with = not dirname.endswith(extension)
                 not_dir = not isdir(join(self.directory, dirname))
-                if not_ends_with or not_dir:
+                if not_dir or not_ends_with:
                     continue
                 if trim:
                     dirname = dirname[:-trim]
@@ -244,7 +244,7 @@ class YamlProvider(BaseProvider):
                 not_ends_with = not filename.endswith('.yaml')
                 too_few_dots = filename.count('.') < 2
                 not_file = not isfile(join(self.directory, filename))
-                if not_ends_with or too_few_dots or not_file:
+                if not_file or not_ends_with or too_few_dots:
                     continue
                 # trim off the yaml, leave the .
                 zones.add(filename[:-4])

From 61d3ed884adf43a74705a155ce67fd3f9ab763a7 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 18 Aug 2023 07:39:47 -0700
Subject: [PATCH 009/116] More YamlProvider details testing

---
 tests/config/split/unit.tests.yaml  |  5 +++++
 tests/test_octodns_provider_yaml.py | 23 +++++++++++++++++++++++
 2 files changed, 28 insertions(+)
 create mode 100644 tests/config/split/unit.tests.yaml

diff --git a/tests/config/split/unit.tests.yaml b/tests/config/split/unit.tests.yaml
new file mode 100644
index 0000000..e249c43
--- /dev/null
+++ b/tests/config/split/unit.tests.yaml
@@ -0,0 +1,5 @@
+---
+only-zone-file:
+  type: TXT
+  value: Only included when zone file processing is enabled
+
diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py
index 6e0a615..4fabb54 100644
--- a/tests/test_octodns_provider_yaml.py
+++ b/tests/test_octodns_provider_yaml.py
@@ -360,6 +360,7 @@ xn--dj-kia8a:
                 list(provider.list_zones()),
             )
 
+            # include stuff with .tst AND basic
             provider.split_extension = '.tst'
             self.assertEqual(
                 [
@@ -374,6 +375,20 @@ xn--dj-kia8a:
                 list(provider.list_zones()),
             )
 
+            # only .tst
+            provider.split_only = True
+            self.assertEqual(
+                ['other-ext.split.', 'split-ext.test.', 'sub.split-ext.test.'],
+                list(provider.list_zones()),
+            )
+
+            # only . (and both zone)
+            provider.split_extension = '.'
+            self.assertEqual(
+                ['both.tld.', 'other.split.', 'split.test.', 'sub.split.test.'],
+                list(provider.list_zones()),
+            )
+
 
 class TestSplitYamlProvider(TestCase):
     def test_list_all_yaml_files(self):
@@ -485,6 +500,14 @@ class TestSplitYamlProvider(TestCase):
         source.populate(zone)
         self.assertEqual(20, len(zone.records))
 
+        # temporarily enable zone file processing too, we should see one extra
+        # record that came from unit.tests.
+        source.split_only = False
+        zone_both = Zone('unit.tests.', [])
+        source.populate(zone_both)
+        self.assertEqual(21, len(zone_both.records))
+        source.split_only = True
+
         source.populate(dynamic_zone)
         self.assertEqual(5, len(dynamic_zone.records))
 

From 3f7234bfd3803fd0525b5d215ea952319e86a6d8 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 18 Aug 2023 08:18:26 -0700
Subject: [PATCH 010/116] Move sources tests into correct class

---
 tests/test_octodns_provider_yaml.py | 46 ++++++++++++++---------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py
index 4fabb54..37df75d 100644
--- a/tests/test_octodns_provider_yaml.py
+++ b/tests/test_octodns_provider_yaml.py
@@ -389,29 +389,6 @@ xn--dj-kia8a:
                 list(provider.list_zones()),
             )
 
-
-class TestSplitYamlProvider(TestCase):
-    def test_list_all_yaml_files(self):
-        yaml_files = ('foo.yaml', '1.yaml', '$unit.tests.yaml')
-        all_files = ('something', 'else', '1', '$$', '-f') + yaml_files
-        all_dirs = ('dir1', 'dir2/sub', 'tricky.yaml')
-
-        with TemporaryDirectory() as td:
-            directory = join(td.dirname)
-
-            # Create some files, some of them with a .yaml extension, all of
-            # them empty.
-            for emptyfile in all_files:
-                touch(join(directory, emptyfile))
-            # Do the same for some fake directories
-            for emptydir in all_dirs:
-                makedirs(join(directory, emptydir))
-
-            # This isn't great, but given the variable nature of the temp dir
-            # names, it's necessary.
-            d = [join(directory, f) for f in yaml_files]
-            self.assertEqual(len(yaml_files), len(d))
-
     def test_split_sources(self):
         with TemporaryDirectory() as td:
             directory = join(td.dirname)
@@ -481,6 +458,29 @@ class TestSplitYamlProvider(TestCase):
             # make sure that we get the idna one back
             self.assertEqual(idna, provider._zone_sources(zone))
 
+
+class TestSplitYamlProvider(TestCase):
+    def test_list_all_yaml_files(self):
+        yaml_files = ('foo.yaml', '1.yaml', '$unit.tests.yaml')
+        all_files = ('something', 'else', '1', '$$', '-f') + yaml_files
+        all_dirs = ('dir1', 'dir2/sub', 'tricky.yaml')
+
+        with TemporaryDirectory() as td:
+            directory = join(td.dirname)
+
+            # Create some files, some of them with a .yaml extension, all of
+            # them empty.
+            for emptyfile in all_files:
+                touch(join(directory, emptyfile))
+            # Do the same for some fake directories
+            for emptydir in all_dirs:
+                makedirs(join(directory, emptydir))
+
+            # This isn't great, but given the variable nature of the temp dir
+            # names, it's necessary.
+            d = [join(directory, f) for f in yaml_files]
+            self.assertEqual(len(yaml_files), len(d))
+
     def test_provider(self):
         source = SplitYamlProvider(
             'test',

From aab868f3455308f9b77b1002a4fbe76ca0f4e788 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 18 Aug 2023 09:25:06 -0700
Subject: [PATCH 011/116] Make sure the only* record isn't showing up when it
 shouldn't

---
 tests/test_octodns_provider_yaml.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py
index 37df75d..0743510 100644
--- a/tests/test_octodns_provider_yaml.py
+++ b/tests/test_octodns_provider_yaml.py
@@ -499,6 +499,7 @@ class TestSplitYamlProvider(TestCase):
         # without it we see everything
         source.populate(zone)
         self.assertEqual(20, len(zone.records))
+        self.assertFalse([r for r in zone.records if r.name.startswith('only')])
 
         # temporarily enable zone file processing too, we should see one extra
         # record that came from unit.tests.
@@ -506,10 +507,15 @@ class TestSplitYamlProvider(TestCase):
         zone_both = Zone('unit.tests.', [])
         source.populate(zone_both)
         self.assertEqual(21, len(zone_both.records))
+        n = len([r for r in zone_both.records if r.name == 'only-zone-file'])
+        self.assertEqual(1, n)
         source.split_only = True
 
         source.populate(dynamic_zone)
         self.assertEqual(5, len(dynamic_zone.records))
+        self.assertFalse(
+            [r for r in dynamic_zone.records if r.name.startswith('only')]
+        )
 
         with TemporaryDirectory() as td:
             # Add some subdirs to make sure that it can create them

From 779f2f44fab4da27736a2c812886ab8ab7080b12 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 18 Aug 2023 15:39:53 -0700
Subject: [PATCH 012/116] Rename split_only to disable_zonefile. More accurate
 and future-proof. Also improve doc a bit

---
 octodns/provider/yaml.py            | 55 ++++++++++++++++-------------
 tests/test_octodns_provider_yaml.py |  6 ++--
 2 files changed, 34 insertions(+), 27 deletions(-)

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index a4eceeb..1be77d2 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -19,31 +19,34 @@ class YamlProvider(BaseProvider):
 
     config:
         class: octodns.provider.yaml.YamlProvider
-        # The location of yaml config files (required)
+
+        # The location of yaml config files. By default records are defined in a
+        # file named for the zone in this directory, the zone file, e.g.
+        # something.com.yaml.
+        # (required)
         directory: ./config
+
         # The ttl to use for records when not specified in the data
         # (optional, default 3600)
         default_ttl: 3600
-        # Whether or not to enforce sorting order on the yaml config
+
+        # Whether or not to enforce sorting order when loading yaml
         # (optional, default True)
         enforce_order: true
+
         # Whether duplicate records should replace rather than error
         # (optional, default False)
         populate_should_replace: false
-        # The filename used to load split style zones, False means disabled.
-        # When enabled the provider will search for zone records split across
-        # multiple YAML files in the directory with split_extension appended to
-        # the zone name. split_extension should include the `.`
-        # See "Split Details" below for more information
-        # (optional, default False, . is the recommended best practice when
+
+        # The file extension used when loading split style zones, Null means
+        # disabled. When enabled the provider will search for zone records split
+        # across multiple YAML files in the directory with split_extension
+        # appended to the zone name, See "Split Details" below.
+        # split_extension should include the "."
+        # (optional, default null, "." is the recommended best practice when
         # enabling)
-        split_extension: false
-        # Disable loading of the primary zone .yaml file. If split_extension
-        # is defined both split files and the primary zone .yaml will be loaded
-        # by default. Setting this to true will disable that and rely soley on
-        # split files.
-        # (optional, default False)
-        split_only: false
+        split_extension: null
+
         # When writing YAML records out to disk with split_extension enabled
         # each record is written out into its own file with .yaml appended to
         # the name of the record. This would result in files like `.yaml` for
@@ -54,6 +57,10 @@ class YamlProvider(BaseProvider):
         # (optional, default False)
         split_catchall: false
 
+        # Disable loading of the zone .yaml files.
+        # (optional, default False)
+        disable_zonefile: false
+
     Split Details
     -------------
 
@@ -163,15 +170,15 @@ class YamlProvider(BaseProvider):
         populate_should_replace=False,
         supports_root_ns=True,
         split_extension=False,
-        split_only=False,
         split_catchall=False,
+        disable_zonefile=False,
         *args,
         **kwargs,
     ):
         klass = self.__class__.__name__
         self.log = logging.getLogger(f'{klass}[{id}]')
         self.log.debug(
-            '__init__: id=%s, directory=%s, default_ttl=%d, enforce_order=%d, populate_should_replace=%s, supports_root_ns=%s, split_extension=%s, split_only=%s, split_catchall=%s',
+            '__init__: id=%s, directory=%s, default_ttl=%d, enforce_order=%d, populate_should_replace=%s, supports_root_ns=%s, split_extension=%s, split_catchall=%s, disable_zonefile=%s',
             id,
             directory,
             default_ttl,
@@ -179,8 +186,8 @@ class YamlProvider(BaseProvider):
             populate_should_replace,
             supports_root_ns,
             split_extension,
-            split_only,
             split_catchall,
+            disable_zonefile,
         )
         super().__init__(id, *args, **kwargs)
         self.directory = directory
@@ -189,8 +196,8 @@ class YamlProvider(BaseProvider):
         self.populate_should_replace = populate_should_replace
         self.supports_root_ns = supports_root_ns
         self.split_extension = split_extension
-        self.split_only = split_only
         self.split_catchall = split_catchall
+        self.disable_zonefile = disable_zonefile
 
     def copy(self):
         kwargs = dict(self.__dict__)
@@ -238,7 +245,7 @@ class YamlProvider(BaseProvider):
                     dirname = dirname[:-trim]
                 zones.add(dirname)
 
-        if not self.split_only:
+        if not self.disable_zonefile:
             self.log.debug('list_zones:   looking for zone files')
             for filename in listdir(self.directory):
                 not_ends_with = not filename.endswith('.yaml')
@@ -324,7 +331,7 @@ class YamlProvider(BaseProvider):
         if split_extension:
             sources.extend(self._split_sources(zone))
 
-        if not self.split_only:
+        if not self.disable_zonefile:
             sources.append(self._zone_sources(zone))
 
         # determinstically order our sources
@@ -423,8 +430,8 @@ class SplitYamlProvider(YamlProvider):
          class: octodns.provider.yaml.YamlProvider
          # extension is configured as split_extension
          split_extension: .
-         split_only: true
          split_catchall: true
+         disable_zonefile: true
 
     TO BE REMOVED: 2.0
     '''
@@ -433,11 +440,11 @@ class SplitYamlProvider(YamlProvider):
         kwargs.update(
             {
                 'split_extension': extension,
-                'split_only': True,
                 'split_catchall': True,
+                'disable_zonefile': True,
             }
         )
         super().__init__(id, directory, *args, **kwargs)
         self.log.warning(
-            '__init__: DEPRECATED use YamlProvider with split_extension and optionally split_only instead, will go away in v2.0'
+            '__init__: DEPRECATED use YamlProvider with split_extension, split_catchall, and disable_zonefile instead, will go away in v2.0'
         )
diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py
index 0743510..ae05b8f 100644
--- a/tests/test_octodns_provider_yaml.py
+++ b/tests/test_octodns_provider_yaml.py
@@ -376,7 +376,7 @@ xn--dj-kia8a:
             )
 
             # only .tst
-            provider.split_only = True
+            provider.disable_zonefile = True
             self.assertEqual(
                 ['other-ext.split.', 'split-ext.test.', 'sub.split-ext.test.'],
                 list(provider.list_zones()),
@@ -503,13 +503,13 @@ class TestSplitYamlProvider(TestCase):
 
         # temporarily enable zone file processing too, we should see one extra
         # record that came from unit.tests.
-        source.split_only = False
+        source.disable_zonefile = False
         zone_both = Zone('unit.tests.', [])
         source.populate(zone_both)
         self.assertEqual(21, len(zone_both.records))
         n = len([r for r in zone_both.records if r.name == 'only-zone-file'])
         self.assertEqual(1, n)
-        source.split_only = True
+        source.disable_zonefile = True
 
         source.populate(dynamic_zone)
         self.assertEqual(5, len(dynamic_zone.records))

From cb28fa0e26885180391af85bc327f40a33b156e4 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 18 Aug 2023 16:20:53 -0700
Subject: [PATCH 013/116] YamlProvider support for shared file, loaded into all
 zones

---
 octodns/provider/yaml.py            | 13 ++++++++++++-
 tests/config/split/unit.tests.yaml  |  1 -
 tests/test_octodns_provider_yaml.py | 17 +++++++++++++++++
 3 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index 1be77d2..3bba1b6 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -57,6 +57,11 @@ class YamlProvider(BaseProvider):
         # (optional, default False)
         split_catchall: false
 
+        # Optional filename with record data to be included in all zones
+        # populated by this provider. Has no effect when used as a target.
+        # (optional, default null)
+        shared_filename: null
+
         # Disable loading of the zone .yaml files.
         # (optional, default False)
         disable_zonefile: false
@@ -171,6 +176,7 @@ class YamlProvider(BaseProvider):
         supports_root_ns=True,
         split_extension=False,
         split_catchall=False,
+        shared_filename=False,
         disable_zonefile=False,
         *args,
         **kwargs,
@@ -178,7 +184,7 @@ class YamlProvider(BaseProvider):
         klass = self.__class__.__name__
         self.log = logging.getLogger(f'{klass}[{id}]')
         self.log.debug(
-            '__init__: id=%s, directory=%s, default_ttl=%d, enforce_order=%d, populate_should_replace=%s, supports_root_ns=%s, split_extension=%s, split_catchall=%s, disable_zonefile=%s',
+            '__init__: id=%s, directory=%s, default_ttl=%d, enforce_order=%d, populate_should_replace=%s, supports_root_ns=%s, split_extension=%s, split_catchall=%s, shared_filename=%s, disable_zonefile=%s',
             id,
             directory,
             default_ttl,
@@ -187,6 +193,7 @@ class YamlProvider(BaseProvider):
             supports_root_ns,
             split_extension,
             split_catchall,
+            shared_filename,
             disable_zonefile,
         )
         super().__init__(id, *args, **kwargs)
@@ -197,6 +204,7 @@ class YamlProvider(BaseProvider):
         self.supports_root_ns = supports_root_ns
         self.split_extension = split_extension
         self.split_catchall = split_catchall
+        self.shared_filename = shared_filename
         self.disable_zonefile = disable_zonefile
 
     def copy(self):
@@ -334,6 +342,9 @@ class YamlProvider(BaseProvider):
         if not self.disable_zonefile:
             sources.append(self._zone_sources(zone))
 
+        if self.shared_filename:
+            sources.append(join(self.directory, self.shared_filename))
+
         # determinstically order our sources
         sources.sort()
 
diff --git a/tests/config/split/unit.tests.yaml b/tests/config/split/unit.tests.yaml
index e249c43..1a25149 100644
--- a/tests/config/split/unit.tests.yaml
+++ b/tests/config/split/unit.tests.yaml
@@ -2,4 +2,3 @@
 only-zone-file:
   type: TXT
   value: Only included when zone file processing is enabled
-
diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py
index ae05b8f..0dc46f3 100644
--- a/tests/test_octodns_provider_yaml.py
+++ b/tests/test_octodns_provider_yaml.py
@@ -511,6 +511,23 @@ class TestSplitYamlProvider(TestCase):
         self.assertEqual(1, n)
         source.disable_zonefile = True
 
+        # temporarily enable shared file processing, we should see one extra
+        # record in the zone
+        source.shared_filename = 'shared.yaml'
+        zone_shared = Zone('unit.tests.', [])
+        source.populate(zone_shared)
+        self.assertEqual(21, len(zone_shared.records))
+        n = len([r for r in zone_shared.records if r.name == 'only-shared'])
+        self.assertEqual(1, n)
+        dynamic_zone_shared = Zone('dynamic.tests.', [])
+        source.populate(dynamic_zone_shared)
+        self.assertEqual(6, len(dynamic_zone_shared.records))
+        n = len(
+            [r for r in dynamic_zone_shared.records if r.name == 'only-shared']
+        )
+        self.assertEqual(1, n)
+        source.shared_filename = None
+
         source.populate(dynamic_zone)
         self.assertEqual(5, len(dynamic_zone.records))
         self.assertFalse(

From e48759791c938482aeb3d121c0f658f08e89dfe9 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 18 Aug 2023 16:35:20 -0700
Subject: [PATCH 014/116] forgot to add shared.yaml test file

---
 tests/config/split/shared.yaml | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 tests/config/split/shared.yaml

diff --git a/tests/config/split/shared.yaml b/tests/config/split/shared.yaml
new file mode 100644
index 0000000..d4c3058
--- /dev/null
+++ b/tests/config/split/shared.yaml
@@ -0,0 +1,4 @@
+---
+only-shared:
+  type: TXT
+  value: Only included when shared file processing is enabled

From c64e279dd2c5a27ab87cdc8ceb6f9261def6c5f6 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 19 Aug 2023 13:45:01 -0700
Subject: [PATCH 015/116] Add support for !include YAML directive

---
 octodns/yaml.py                               | 12 ++++++++++
 tests/config/include/array.yaml               |  5 ++++
 tests/config/include/dict.yaml                |  3 +++
 tests/config/include/empty.yaml               |  1 +
 .../config/include/include-doesnt-exist.yaml  |  2 ++
 tests/config/include/main.yaml                |  8 +++++++
 tests/config/include/nested.yaml              |  2 ++
 tests/config/include/subdir/value.yaml        |  2 ++
 tests/test_octodns_yaml.py                    | 24 +++++++++++++++++++
 9 files changed, 59 insertions(+)
 create mode 100644 tests/config/include/array.yaml
 create mode 100644 tests/config/include/dict.yaml
 create mode 100644 tests/config/include/empty.yaml
 create mode 100644 tests/config/include/include-doesnt-exist.yaml
 create mode 100644 tests/config/include/main.yaml
 create mode 100644 tests/config/include/nested.yaml
 create mode 100644 tests/config/include/subdir/value.yaml

diff --git a/octodns/yaml.py b/octodns/yaml.py
index 09433d9..433486a 100644
--- a/octodns/yaml.py
+++ b/octodns/yaml.py
@@ -2,6 +2,8 @@
 #
 #
 
+from os.path import dirname, join
+
 from natsort import natsort_keygen
 from yaml import SafeDumper, SafeLoader, dump, load
 from yaml.constructor import ConstructorError
@@ -23,7 +25,17 @@ class ContextLoader(SafeLoader):
     def _construct(self, node):
         return self._pairs(node)[0]
 
+    def include(self, node):
+        mark = self.get_mark()
+        directory = dirname(mark.name)
+
+        filename = join(directory, self.construct_scalar(node))
+
+        with open(filename, 'r') as fh:
+            return safe_load(fh, self.__class__)
+
 
+ContextLoader.add_constructor('!include', ContextLoader.include)
 ContextLoader.add_constructor(
     ContextLoader.DEFAULT_MAPPING_TAG, ContextLoader._construct
 )
diff --git a/tests/config/include/array.yaml b/tests/config/include/array.yaml
new file mode 100644
index 0000000..a97221c
--- /dev/null
+++ b/tests/config/include/array.yaml
@@ -0,0 +1,5 @@
+---
+- 14
+- 15
+- 16
+- 72
diff --git a/tests/config/include/dict.yaml b/tests/config/include/dict.yaml
new file mode 100644
index 0000000..da2e22f
--- /dev/null
+++ b/tests/config/include/dict.yaml
@@ -0,0 +1,3 @@
+---
+k: v
+z: 42
diff --git a/tests/config/include/empty.yaml b/tests/config/include/empty.yaml
new file mode 100644
index 0000000..ed97d53
--- /dev/null
+++ b/tests/config/include/empty.yaml
@@ -0,0 +1 @@
+---
diff --git a/tests/config/include/include-doesnt-exist.yaml b/tests/config/include/include-doesnt-exist.yaml
new file mode 100644
index 0000000..7f58025
--- /dev/null
+++ b/tests/config/include/include-doesnt-exist.yaml
@@ -0,0 +1,2 @@
+---
+key: !include does-not-exist.yaml
diff --git a/tests/config/include/main.yaml b/tests/config/include/main.yaml
new file mode 100644
index 0000000..11f6c6c
--- /dev/null
+++ b/tests/config/include/main.yaml
@@ -0,0 +1,8 @@
+---
+included-array: !include array.yaml
+included-dict: !include dict.yaml
+included-empty: !include empty.yaml
+included-nested: !include nested.yaml
+included-subdir: !include subdir/value.yaml
+key: value
+name: main
diff --git a/tests/config/include/nested.yaml b/tests/config/include/nested.yaml
new file mode 100644
index 0000000..02da7f4
--- /dev/null
+++ b/tests/config/include/nested.yaml
@@ -0,0 +1,2 @@
+---
+!include subdir/value.yaml
diff --git a/tests/config/include/subdir/value.yaml b/tests/config/include/subdir/value.yaml
new file mode 100644
index 0000000..6cdc809
--- /dev/null
+++ b/tests/config/include/subdir/value.yaml
@@ -0,0 +1,2 @@
+---
+Hello World!
diff --git a/tests/test_octodns_yaml.py b/tests/test_octodns_yaml.py
index 3a5990f..396d59c 100644
--- a/tests/test_octodns_yaml.py
+++ b/tests/test_octodns_yaml.py
@@ -62,3 +62,27 @@ class TestYaml(TestCase):
         buf = StringIO()
         safe_dump({'45a03129': 42, '45a0392a': 43}, buf)
         self.assertEqual("---\n45a0392a: 43\n45a03129: 42\n", buf.getvalue())
+
+    def test_include(self):
+        with open('tests/config/include/main.yaml') as fh:
+            data = safe_load(fh)
+        self.assertEqual(
+            {
+                'included-array': [14, 15, 16, 72],
+                'included-dict': {'k': 'v', 'z': 42},
+                'included-empty': None,
+                'included-nested': 'Hello World!',
+                'included-subdir': 'Hello World!',
+                'key': 'value',
+                'name': 'main',
+            },
+            data,
+        )
+
+        with open('tests/config/include/include-doesnt-exist.yaml') as fh:
+            with self.assertRaises(FileNotFoundError) as ctx:
+                data = safe_load(fh)
+            self.assertEqual(
+                "[Errno 2] No such file or directory: 'tests/config/include/does-not-exist.yaml'",
+                str(ctx.exception),
+            )

From ca1bae07977966ed96110ef0fe88f541cef5c854 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 19 Aug 2023 14:07:52 -0700
Subject: [PATCH 016/116] CHANGELOG entry for shared_filename

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3cdef09..ce2ebed 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,10 @@
   possible. This includes the ability to split some zones and not others and
   even to have partially split zones with some records in the primary zone YAML
   and others in a split directory. See YamlProvider documentation for more info.
+* YamlProvider now supports a `shared_filename` that can be used to add a set of
+  common records across all zones using the provider. It can be used stand-alone
+  or in combination with zone files and/or split configs to aid in DRYing up DNS
+  configs.
 
 #### Stuff
 

From b5853ddc919fe098bedb427e3a8740b823db851a Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 19 Aug 2023 14:07:52 -0700
Subject: [PATCH 017/116] CHANGELOG entry for shared_filename

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3cdef09..ce2ebed 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,10 @@
   possible. This includes the ability to split some zones and not others and
   even to have partially split zones with some records in the primary zone YAML
   and others in a split directory. See YamlProvider documentation for more info.
+* YamlProvider now supports a `shared_filename` that can be used to add a set of
+  common records across all zones using the provider. It can be used stand-alone
+  or in combination with zone files and/or split configs to aid in DRYing up DNS
+  configs.
 
 #### Stuff
 

From 645b088a386112cda9177be53d0ba9d0eeedfdef Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 19 Aug 2023 14:11:44 -0700
Subject: [PATCH 018/116] CHANGELOG entry for YamlProvider !include support

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ce2ebed..71dcd47 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,10 @@
   common records across all zones using the provider. It can be used stand-alone
   or in combination with zone files and/or split configs to aid in DRYing up DNS
   configs.
+* YamlProvider now supports an `!include` directive which enables shared
+  snippets of config to be reused across many records, e.g. common dynamic rules
+  across a set of services with service-specific pool values or a unified SFP
+  value included in TXT records at the root of all zones.
 
 #### Stuff
 

From 3d719de1a52e28f8a4dd349073fde4a94eb75350 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Wed, 23 Aug 2023 13:28:26 -0700
Subject: [PATCH 019/116] Add octodns-spf to the README

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 9ab50fc..0c1823e 100644
--- a/README.md
+++ b/README.md
@@ -293,6 +293,7 @@ The table below lists the providers octoDNS supports. They are maintained in the
 | [Rackspace](https://www.rackspace.com/library/what-is-dns) | [octodns_rackspace](https://github.com/octodns/octodns-rackspace/) | |
 | [Scaleway](https://www.scaleway.com/en/dns/) | [octodns_scaleway](https://github.com/scaleway/octodns-scaleway) | |
 | [Selectel](https://selectel.ru/en/services/additional/dns/) | [octodns_selectel](https://github.com/octodns/octodns-selectel/) | |
+| [SPF Value Management](https://github.com/octodns/octodns-spf) | [octodns_spf](https://github.com/octodns/octodns-spf/) | |
 | [TransIP](https://www.transip.eu/knowledgebase/entry/155-dns-and-nameservers/) | [octodns_transip](https://github.com/octodns/octodns-transip/) | |
 | [UltraDNS](https://vercara.com/authoritative-dns) | [octodns_ultra](https://github.com/octodns/octodns-ultra/) | |
 | [YamlProvider](/octodns/provider/yaml.py) | built-in | Supports all record types and core functionality |

From 6f39fcc5f7f29d83c8cb6e59904829c89454e032 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 27 Aug 2023 08:37:26 -0700
Subject: [PATCH 020/116] yaml provider is either split or zonefile, not both

---
 octodns/provider/yaml.py            | 30 +++++--------------
 tests/test_octodns_provider_yaml.py | 46 ++---------------------------
 2 files changed, 11 insertions(+), 65 deletions(-)

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index 1be77d2..93b7615 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -54,12 +54,8 @@ class YamlProvider(BaseProvider):
         # filenames or you would prefer to avoid them you can enable
         # split_catchall to instead write those records into a file named
         # `$[zone.name].yaml`
-        # (optional, default False)
-        split_catchall: false
-
-        # Disable loading of the zone .yaml files.
-        # (optional, default False)
-        disable_zonefile: false
+        # (optional, default True)
+        split_catchall: true
 
     Split Details
     -------------
@@ -170,15 +166,14 @@ class YamlProvider(BaseProvider):
         populate_should_replace=False,
         supports_root_ns=True,
         split_extension=False,
-        split_catchall=False,
-        disable_zonefile=False,
+        split_catchall=True,
         *args,
         **kwargs,
     ):
         klass = self.__class__.__name__
         self.log = logging.getLogger(f'{klass}[{id}]')
         self.log.debug(
-            '__init__: id=%s, directory=%s, default_ttl=%d, enforce_order=%d, populate_should_replace=%s, supports_root_ns=%s, split_extension=%s, split_catchall=%s, disable_zonefile=%s',
+            '__init__: id=%s, directory=%s, default_ttl=%d, enforce_order=%d, populate_should_replace=%s, supports_root_ns=%s, split_extension=%s, split_catchall=%s',
             id,
             directory,
             default_ttl,
@@ -187,7 +182,6 @@ class YamlProvider(BaseProvider):
             supports_root_ns,
             split_extension,
             split_catchall,
-            disable_zonefile,
         )
         super().__init__(id, *args, **kwargs)
         self.directory = directory
@@ -197,7 +191,6 @@ class YamlProvider(BaseProvider):
         self.supports_root_ns = supports_root_ns
         self.split_extension = split_extension
         self.split_catchall = split_catchall
-        self.disable_zonefile = disable_zonefile
 
     def copy(self):
         kwargs = dict(self.__dict__)
@@ -245,7 +238,7 @@ class YamlProvider(BaseProvider):
                     dirname = dirname[:-trim]
                 zones.add(dirname)
 
-        if not self.disable_zonefile:
+        if not self.split_extension:
             self.log.debug('list_zones:   looking for zone files')
             for filename in listdir(self.directory):
                 not_ends_with = not filename.endswith('.yaml')
@@ -331,7 +324,7 @@ class YamlProvider(BaseProvider):
         if split_extension:
             sources.extend(self._split_sources(zone))
 
-        if not self.disable_zonefile:
+        if not self.split_extension:
             sources.append(self._zone_sources(zone))
 
         # determinstically order our sources
@@ -431,20 +424,13 @@ class SplitYamlProvider(YamlProvider):
          # extension is configured as split_extension
          split_extension: .
          split_catchall: true
-         disable_zonefile: true
 
     TO BE REMOVED: 2.0
     '''
 
     def __init__(self, id, directory, *args, extension='.', **kwargs):
-        kwargs.update(
-            {
-                'split_extension': extension,
-                'split_catchall': True,
-                'disable_zonefile': True,
-            }
-        )
+        kwargs.update({'split_extension': extension, 'split_catchall': True})
         super().__init__(id, directory, *args, **kwargs)
         self.log.warning(
-            '__init__: DEPRECATED use YamlProvider with split_extension, split_catchall, and disable_zonefile instead, will go away in v2.0'
+            '__init__: DEPRECATED use YamlProvider with split_extension and split_catchall instead, will go away in v2.0'
         )
diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py
index ae05b8f..98eca7f 100644
--- a/tests/test_octodns_provider_yaml.py
+++ b/tests/test_octodns_provider_yaml.py
@@ -345,50 +345,20 @@ xn--dj-kia8a:
                 list(provider.list_zones()),
             )
 
-            # include stuff with . AND basic
+            # split only .
             provider.split_extension = '.'
             self.assertEqual(
-                [
-                    'both.tld.',
-                    'other.split.',
-                    'other.tld.',
-                    'split.test.',
-                    'sub.split.test.',
-                    'sub.unit.test.',
-                    'unit.test.',
-                ],
-                list(provider.list_zones()),
-            )
-
-            # include stuff with .tst AND basic
-            provider.split_extension = '.tst'
-            self.assertEqual(
-                [
-                    'both.tld.',
-                    'other-ext.split.',
-                    'other.tld.',
-                    'split-ext.test.',
-                    'sub.split-ext.test.',
-                    'sub.unit.test.',
-                    'unit.test.',
-                ],
+                ['both.tld.', 'other.split.', 'split.test.', 'sub.split.test.'],
                 list(provider.list_zones()),
             )
 
             # only .tst
-            provider.disable_zonefile = True
+            provider.split_extension = '.tst'
             self.assertEqual(
                 ['other-ext.split.', 'split-ext.test.', 'sub.split-ext.test.'],
                 list(provider.list_zones()),
             )
 
-            # only . (and both zone)
-            provider.split_extension = '.'
-            self.assertEqual(
-                ['both.tld.', 'other.split.', 'split.test.', 'sub.split.test.'],
-                list(provider.list_zones()),
-            )
-
     def test_split_sources(self):
         with TemporaryDirectory() as td:
             directory = join(td.dirname)
@@ -501,16 +471,6 @@ class TestSplitYamlProvider(TestCase):
         self.assertEqual(20, len(zone.records))
         self.assertFalse([r for r in zone.records if r.name.startswith('only')])
 
-        # temporarily enable zone file processing too, we should see one extra
-        # record that came from unit.tests.
-        source.disable_zonefile = False
-        zone_both = Zone('unit.tests.', [])
-        source.populate(zone_both)
-        self.assertEqual(21, len(zone_both.records))
-        n = len([r for r in zone_both.records if r.name == 'only-zone-file'])
-        self.assertEqual(1, n)
-        source.disable_zonefile = True
-
         source.populate(dynamic_zone)
         self.assertEqual(5, len(dynamic_zone.records))
         self.assertFalse(

From 77b1d7a1bab91d06780e4509e5bf3470915b0f5e Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Tue, 29 Aug 2023 12:34:23 -0700
Subject: [PATCH 021/116] Correct split_catchall doc

---
 octodns/provider/yaml.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index 93b7615..2d4ee81 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -49,11 +49,12 @@ class YamlProvider(BaseProvider):
 
         # When writing YAML records out to disk with split_extension enabled
         # each record is written out into its own file with .yaml appended to
-        # the name of the record. This would result in files like `.yaml` for
-        # the apex and `*.yaml` for a wildcard. If your OS doesn't allow such
-        # filenames or you would prefer to avoid them you can enable
-        # split_catchall to instead write those records into a file named
-        # `$[zone.name].yaml`
+        # the name of the record. The two exceptions are for the root and
+        # wildcard nodes. These records are written into a file named
+        # `$[zone.name].yaml`. If you would prefer this catchall file not be
+        # used `split_catchall` can be set to False to instead write those
+        # records out to `.yaml` and `*.yaml` respectively. Note that some
+        # operating systems may not allow files with those names.
         # (optional, default True)
         split_catchall: true
 

From 8177ee6926648bb8dd69788374167b3bf97b4020 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Tue, 29 Aug 2023 12:59:38 -0700
Subject: [PATCH 022/116] Revert "yaml provider is either split or zonefile,
 not both"

This reverts commit 6f39fcc5f7f29d83c8cb6e59904829c89454e032.
---
 octodns/provider/yaml.py            | 24 +++++++++++----
 tests/test_octodns_provider_yaml.py | 46 +++++++++++++++++++++++++++--
 2 files changed, 62 insertions(+), 8 deletions(-)

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index 2d4ee81..d340bc6 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -58,6 +58,10 @@ class YamlProvider(BaseProvider):
         # (optional, default True)
         split_catchall: true
 
+        # Disable loading of the zone .yaml files.
+        # (optional, default False)
+        disable_zonefile: false
+
     Split Details
     -------------
 
@@ -168,13 +172,14 @@ class YamlProvider(BaseProvider):
         supports_root_ns=True,
         split_extension=False,
         split_catchall=True,
+        disable_zonefile=False,
         *args,
         **kwargs,
     ):
         klass = self.__class__.__name__
         self.log = logging.getLogger(f'{klass}[{id}]')
         self.log.debug(
-            '__init__: id=%s, directory=%s, default_ttl=%d, enforce_order=%d, populate_should_replace=%s, supports_root_ns=%s, split_extension=%s, split_catchall=%s',
+            '__init__: id=%s, directory=%s, default_ttl=%d, enforce_order=%d, populate_should_replace=%s, supports_root_ns=%s, split_extension=%s, split_catchall=%s, disable_zonefile=%s',
             id,
             directory,
             default_ttl,
@@ -183,6 +188,7 @@ class YamlProvider(BaseProvider):
             supports_root_ns,
             split_extension,
             split_catchall,
+            disable_zonefile,
         )
         super().__init__(id, *args, **kwargs)
         self.directory = directory
@@ -192,6 +198,7 @@ class YamlProvider(BaseProvider):
         self.supports_root_ns = supports_root_ns
         self.split_extension = split_extension
         self.split_catchall = split_catchall
+        self.disable_zonefile = disable_zonefile
 
     def copy(self):
         kwargs = dict(self.__dict__)
@@ -239,7 +246,7 @@ class YamlProvider(BaseProvider):
                     dirname = dirname[:-trim]
                 zones.add(dirname)
 
-        if not self.split_extension:
+        if not self.disable_zonefile:
             self.log.debug('list_zones:   looking for zone files')
             for filename in listdir(self.directory):
                 not_ends_with = not filename.endswith('.yaml')
@@ -325,7 +332,7 @@ class YamlProvider(BaseProvider):
         if split_extension:
             sources.extend(self._split_sources(zone))
 
-        if not self.split_extension:
+        if not self.disable_zonefile:
             sources.append(self._zone_sources(zone))
 
         # determinstically order our sources
@@ -425,13 +432,20 @@ class SplitYamlProvider(YamlProvider):
          # extension is configured as split_extension
          split_extension: .
          split_catchall: true
+         disable_zonefile: true
 
     TO BE REMOVED: 2.0
     '''
 
     def __init__(self, id, directory, *args, extension='.', **kwargs):
-        kwargs.update({'split_extension': extension, 'split_catchall': True})
+        kwargs.update(
+            {
+                'split_extension': extension,
+                'split_catchall': True,
+                'disable_zonefile': True,
+            }
+        )
         super().__init__(id, directory, *args, **kwargs)
         self.log.warning(
-            '__init__: DEPRECATED use YamlProvider with split_extension and split_catchall instead, will go away in v2.0'
+            '__init__: DEPRECATED use YamlProvider with split_extension, split_catchall, and disable_zonefile instead, will go away in v2.0'
         )
diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py
index 814eb81..a41a417 100644
--- a/tests/test_octodns_provider_yaml.py
+++ b/tests/test_octodns_provider_yaml.py
@@ -348,20 +348,50 @@ xn--dj-kia8a:
                 list(provider.list_zones()),
             )
 
-            # split only .
+            # include stuff with . AND basic
             provider.split_extension = '.'
             self.assertEqual(
-                ['both.tld.', 'other.split.', 'split.test.', 'sub.split.test.'],
+                [
+                    'both.tld.',
+                    'other.split.',
+                    'other.tld.',
+                    'split.test.',
+                    'sub.split.test.',
+                    'sub.unit.test.',
+                    'unit.test.',
+                ],
                 list(provider.list_zones()),
             )
 
-            # only .tst
+            # include stuff with .tst AND basic
             provider.split_extension = '.tst'
+            self.assertEqual(
+                [
+                    'both.tld.',
+                    'other-ext.split.',
+                    'other.tld.',
+                    'split-ext.test.',
+                    'sub.split-ext.test.',
+                    'sub.unit.test.',
+                    'unit.test.',
+                ],
+                list(provider.list_zones()),
+            )
+
+            # only .tst
+            provider.disable_zonefile = True
             self.assertEqual(
                 ['other-ext.split.', 'split-ext.test.', 'sub.split-ext.test.'],
                 list(provider.list_zones()),
             )
 
+            # only . (and both zone)
+            provider.split_extension = '.'
+            self.assertEqual(
+                ['both.tld.', 'other.split.', 'split.test.', 'sub.split.test.'],
+                list(provider.list_zones()),
+            )
+
     def test_split_sources(self):
         with TemporaryDirectory() as td:
             directory = join(td.dirname)
@@ -474,6 +504,16 @@ class TestSplitYamlProvider(TestCase):
         self.assertEqual(20, len(zone.records))
         self.assertFalse([r for r in zone.records if r.name.startswith('only')])
 
+        # temporarily enable zone file processing too, we should see one extra
+        # record that came from unit.tests.
+        source.disable_zonefile = False
+        zone_both = Zone('unit.tests.', [])
+        source.populate(zone_both)
+        self.assertEqual(21, len(zone_both.records))
+        n = len([r for r in zone_both.records if r.name == 'only-zone-file'])
+        self.assertEqual(1, n)
+        source.disable_zonefile = True
+
         source.populate(dynamic_zone)
         self.assertEqual(5, len(dynamic_zone.records))
         self.assertFalse(

From 93b397cbb2f5170a4875dc7357f31404e8700423 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Tue, 29 Aug 2023 13:51:53 -0700
Subject: [PATCH 023/116] Correct Split Details directory example

---
 octodns/provider/yaml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index d340bc6..da6fdf1 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -76,7 +76,7 @@ class YamlProvider(BaseProvider):
 
     zones/
       github.com./
-        .yaml
+        $github.com.yaml
         www.yaml
         ...
 

From 6be12c23b0505801786595f25c5d9fa777d4abf6 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Wed, 6 Sep 2023 14:16:18 -0700
Subject: [PATCH 024/116] Add a Processors list/section to the README

---
 README.md | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/README.md b/README.md
index 0c1823e..9a976e9 100644
--- a/README.md
+++ b/README.md
@@ -25,6 +25,7 @@ The architecture is pluggable and the tooling is flexible to make it applicable
    * [Updating to use extracted providers](#updating-to-use-extracted-providers)
 * [Sources](#sources)
    * [Notes](#notes)
+* [Processors](#processors)
 * [Automatic PTR generation](#automatic-ptr-generation)
 * [Compatibility and Compliance](#compatibility-and-compliance)
    * [`lenient`](#lenient)
@@ -323,6 +324,22 @@ Similar to providers, but can only serve to populate records into a zone, cannot
    * Dnsimple's uses the configured TTL when serving things through the ALIAS, there's also a secondary TXT record created alongside the ALIAS that octoDNS ignores
 * octoDNS itself supports non-ASCII character sets, but in testing Cloudflare is the only provider where that is currently functional end-to-end. Others have failures either in the client libraries or API calls
 
+## Processors
+
+| Processor | Description |
+|--|--|
+| [AcmeMangingProcessor](/octodns/processor/acme.py) | Useful when processes external to octoDNS are managing acme challenge DNS records, e.g. LetsEncrypt |
+| [AutoArpa](/octodns/processor/arpa.py) | See [Automatic PTR generation](#automatic-ptr-generation) below |
+| [IgnoreRootNsFilter](/octodns/processor/filter.py) | Filter that INGORES root/APEX NS records and prevents octoDNS from trying to manage them (where supported.) |
+| [NameAllowlistFilter](/octodns/processor/filter.py) | Filter that ONLY manages records that match specified naming patterns, all others will be ignored |
+| [NameRejectlistFilter](/octodns/processor/filter.py) | Filter that INGORES records that match specified naming patterns, all others will be managed |
+| [OwnershipProcessor](/octodns/processor/ownership.py) | Processor that implements ownership in octoDNS so that it can manage only the records in a zone in sources and will ignore all others. |
+| [SpfDnsLookupProcessor](/octodns/processor/spf.py) | Processor that checks SPF values for violations of DNS query limits |
+| [TtlRestrictionFilter](/octodns/processor/restrict.py) | Processor that restricts the allow TTL values to a specified range or list of specific values |
+| [TypeAllowlistFilter](/octodns/processor/filter.py) | Filter that ONLY manages records of specified types, all others will be ignored |
+| [TypeRejectlistFilter](/octodns/processor/filter.py) | Filter that INGORES records of specified types, all others will be managed |
+| [octodns-spf](https://github.com/octodns/octodns-spf) | SPF Value Management for octoDNS |
+
 ## Automatic PTR generation
 
 octoDNS supports automatically generating PTR records from the `A`/`AAAA` records it manages. For more information see the [auto-arpa documentation](/docs/auto_arpa.md).

From 4256ad6caf4a11f4fe35a203dc6dfebde020f12f Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 9 Sep 2023 14:35:45 -0700
Subject: [PATCH 025/116] Deprecate SpfRecord

---
 CHANGELOG.md          | 2 ++
 octodns/record/spf.py | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ce2ebed..2de5ae0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,8 @@
   common records across all zones using the provider. It can be used stand-alone
   or in combination with zone files and/or split configs to aid in DRYing up DNS
   configs.
+* SpfRecord is formally deprecated with an warning and will become a
+  ValidationError in 2.x
 
 #### Stuff
 
diff --git a/octodns/record/spf.py b/octodns/record/spf.py
index ac59c57..9eb7a4a 100644
--- a/octodns/record/spf.py
+++ b/octodns/record/spf.py
@@ -10,5 +10,11 @@ class SpfRecord(_ChunkedValuesMixin, Record):
     _type = 'SPF'
     _value_type = _ChunkedValue
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.log.warning(
+            'The SPF record type is DEPRECATED in favor of TXT values and will become an ValidationError in 2.0'
+        )
+
 
 Record.register_type(SpfRecord)

From e8a88ac520f74e9f3465cc8890c3fa3ee0f2843d Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 9 Sep 2023 14:46:55 -0700
Subject: [PATCH 026/116] Deprecate SpfDnsLookupProcessor

---
 CHANGELOG.md             | 2 ++
 octodns/processor/spf.py | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2de5ae0..d4f01d0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,8 @@
   configs.
 * SpfRecord is formally deprecated with an warning and will become a
   ValidationError in 2.x
+* SpfDnsLookupProcessor is formally deprcated in favor of the version relocated
+  into https://github.com/octodns/octodns-spf and will be removed in 2.x
 
 #### Stuff
 
diff --git a/octodns/processor/spf.py b/octodns/processor/spf.py
index 6867a91..dee82ed 100644
--- a/octodns/processor/spf.py
+++ b/octodns/processor/spf.py
@@ -55,6 +55,9 @@ class SpfDnsLookupProcessor(BaseProcessor):
 
     def __init__(self, name):
         self.log.debug(f"SpfDnsLookupProcessor: {name}")
+        self.log.warning(
+            'SpfDnsLookupProcessor is DEPRECATED in favor of the version relocated into octodns-spf and will be removed in 2.0'
+        )
         super().__init__(name)
 
     def _get_spf_from_txt_values(

From 557d0eb1cb1ebc562e67e7ca2859bfed832ecbf6 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 10 Sep 2023 07:45:25 -0700
Subject: [PATCH 027/116] Add post_processor Manager configuration
 option/support

---
 CHANGELOG.md       |  3 +++
 octodns/manager.py | 11 ++++++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ce2ebed..33891e2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,9 @@
 * Add --all option to octodns-validate to enable showing all record validation
   errors (as warnings) rather than exiting on the first. Exit code is non-zero
   when there are any validation errors.
+* New `post_processors` manager configuration parameter to add global processors
+  that run AFTER zone-specific processors. This should allow more complete
+  control over when processors are run.
 
 ## v1.0.0 - 2023-07-30 - The One
 
diff --git a/octodns/manager.py b/octodns/manager.py
index 40912dc..a75b9b1 100644
--- a/octodns/manager.py
+++ b/octodns/manager.py
@@ -114,6 +114,11 @@ class Manager(object):
         self.global_processors = manager_config.get('processors', [])
         self.log.info('__init__: global_processors=%s', self.global_processors)
 
+        self.global_post_processors = manager_config.get('post_processors', [])
+        self.log.info(
+            '__init__: global_post_processors=%s', self.global_post_processors
+        )
+
         providers_config = self.config['providers']
         self.providers = self._config_providers(providers_config)
 
@@ -634,7 +639,11 @@ class Manager(object):
 
             try:
                 collected = []
-                for processor in self.global_processors + processors:
+                for processor in (
+                    self.global_processors
+                    + processors
+                    + self.global_post_processors
+                ):
                     collected.append(self.processors[processor])
                 processors = collected
             except KeyError:

From a9467aaebb0ff2cf898ce23df0a04f8a84566f69 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 10 Sep 2023 07:49:02 -0700
Subject: [PATCH 028/116] move auto-arpa to prepend post_processors

Preferable to have it run later after other processors have had their
change to add/remove records. Otherwise there may be PTRs created for
things that processors have filtered out. It's always possible to
manually include it in the appropriate places if you need finger grained
control.
---
 octodns/manager.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/octodns/manager.py b/octodns/manager.py
index a75b9b1..c396440 100644
--- a/octodns/manager.py
+++ b/octodns/manager.py
@@ -127,13 +127,15 @@ class Manager(object):
 
         if self.auto_arpa:
             self.log.info(
-                '__init__: adding auto-arpa to processors and providers, appending it to global_processors list'
+                '__init__: adding auto-arpa to processors and providers, prepending it to global_post_processors list'
             )
             kwargs = self.auto_arpa if isinstance(auto_arpa, dict) else {}
             auto_arpa = AutoArpa('auto-arpa', **kwargs)
             self.providers[auto_arpa.name] = auto_arpa
             self.processors[auto_arpa.name] = auto_arpa
-            self.global_processors.append(auto_arpa.name)
+            self.global_post_processors = [
+                auto_arpa.name
+            ] + self.global_post_processors
 
         plan_outputs_config = manager_config.get(
             'plan_outputs',

From 3343c4ba51eb1dfb09a93026d2fc98e8f294113b Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 10 Sep 2023 12:30:26 -0700
Subject: [PATCH 029/116] MetaProcessor implementation and testing

---
 CHANGELOG.md                         |   3 +
 octodns/processor/meta.py            | 105 ++++++++++++++
 tests/test_octodns_processor_meta.py | 202 +++++++++++++++++++++++++++
 3 files changed, 310 insertions(+)
 create mode 100644 octodns/processor/meta.py
 create mode 100644 tests/test_octodns_processor_meta.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 33891e2..b810b9a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,9 @@
   common records across all zones using the provider. It can be used stand-alone
   or in combination with zone files and/or split configs to aid in DRYing up DNS
   configs.
+* MetaProcessor added to enable some useful/cool options for debugging/tracking
+  DNS changes. Specifically timestamps/uuid so you can track whether changes
+  that have been pushed to providers have propogated/transferred correctly.
 
 #### Stuff
 
diff --git a/octodns/processor/meta.py b/octodns/processor/meta.py
new file mode 100644
index 0000000..9425e7e
--- /dev/null
+++ b/octodns/processor/meta.py
@@ -0,0 +1,105 @@
+#
+#
+#
+
+from datetime import datetime
+from logging import getLogger
+from uuid import uuid4
+
+from .. import __VERSION__
+from ..record import Record
+from .base import BaseProcessor
+
+
+def _keys(values):
+    return set(v.split('=', 1)[0] for v in values)
+
+
+class MetaProcessor(BaseProcessor):
+    @classmethod
+    def now(cls):
+        return datetime.utcnow().isoformat()
+
+    @classmethod
+    def uuid(cls):
+        return str(uuid4())
+
+    def __init__(
+        self,
+        id,
+        record_name='meta',
+        include_time=True,
+        include_uuid=False,
+        include_version=False,
+        include_provider=False,
+        ttl=60,
+    ):
+        self.log = getLogger(f'MetaSource[{id}]')
+        super().__init__(id)
+        self.log.info(
+            '__init__: record_name=%s, include_time=%s, include_uuid=%s, include_version=%s, include_provider=%s, ttl=%d',
+            record_name,
+            include_time,
+            include_uuid,
+            include_version,
+            include_provider,
+            ttl,
+        )
+        self.record_name = record_name
+        values = []
+        if include_time:
+            time = self.now()
+            values.append(f'time={time}')
+        if include_uuid:
+            uuid = self.uuid() if include_uuid else None
+            values.append(f'uuid={uuid}')
+        if include_version:
+            values.append(f'octodns-version={__VERSION__}')
+        self.include_provider = include_provider
+        values.sort()
+        self.values = values
+        self.ttl = ttl
+
+    def process_source_zone(self, desired, sources):
+        meta = Record.new(
+            desired,
+            self.record_name,
+            {'ttl': self.ttl, 'type': 'TXT', 'values': self.values},
+        )
+        desired.add_record(meta)
+        return desired
+
+    def process_target_zone(self, existing, target):
+        if self.include_provider:
+            # look for the meta record
+            for record in sorted(existing.records):
+                if record.name == self.record_name and record._type == 'TXT':
+                    # we've found it, make a copy we can modify
+                    record = record.copy()
+                    record.values = record.values + [f'provider={target.id}']
+                    record.values.sort()
+                    existing.add_record(record, replace=True)
+                    break
+
+        return existing
+
+    def _up_to_date(self, change):
+        # existing state, if there is one
+        existing = getattr(change, 'existing', None)
+        return existing is not None and _keys(existing.values) == _keys(
+            self.values
+        )
+
+    def process_plan(self, plan, sources, target):
+        if (
+            plan
+            and len(plan.changes) == 1
+            and self._up_to_date(plan.changes[0])
+        ):
+            # the only change is the meta record, and it's not meaningfully
+            # changing so we don't actually want to make the change
+            return None
+
+        # There's more than one thing changing so meta should update and/or meta
+        # is meaningfully changing or being created...
+        return plan
diff --git a/tests/test_octodns_processor_meta.py b/tests/test_octodns_processor_meta.py
new file mode 100644
index 0000000..65ed743
--- /dev/null
+++ b/tests/test_octodns_processor_meta.py
@@ -0,0 +1,202 @@
+#
+#
+#
+
+from unittest import TestCase
+from unittest.mock import patch
+
+from octodns import __VERSION__
+from octodns.processor.meta import MetaProcessor
+from octodns.provider.plan import Plan
+from octodns.record import Create, Record, Update
+from octodns.zone import Zone
+
+
+class TestMetaProcessor(TestCase):
+    zone = Zone('unit.tests.', [])
+
+    meta_needs_update = Record.new(
+        zone,
+        'meta',
+        {
+            'type': 'TXT',
+            'ttl': 60,
+            # will always need updating
+            'values': ['uuid'],
+        },
+    )
+
+    meta_up_to_date = Record.new(
+        zone,
+        'meta',
+        {
+            'type': 'TXT',
+            'ttl': 60,
+            # only has time, value should be ignored
+            'values': ['time=xxx'],
+        },
+    )
+
+    not_meta = Record.new(
+        zone,
+        'its-not-meta',
+        {
+            'type': 'TXT',
+            'ttl': 60,
+            # has time, but name is wrong so won't matter
+            'values': ['time=xyz'],
+        },
+    )
+
+    @patch('octodns.processor.meta.MetaProcessor.now')
+    @patch('octodns.processor.meta.MetaProcessor.uuid')
+    def test_args_and_values(self, uuid_mock, now_mock):
+        # defaults, just time
+        uuid_mock.side_effect = [Exception('not used')]
+        now_mock.side_effect = ['the-time']
+        proc = MetaProcessor('test')
+        self.assertEqual(['time=the-time'], proc.values)
+
+        # just uuid
+        uuid_mock.side_effect = ['abcdef-1234567890']
+        now_mock.side_effect = [Exception('not used')]
+        proc = MetaProcessor('test', include_time=False, include_uuid=True)
+        self.assertEqual(['uuid=abcdef-1234567890'], proc.values)
+
+        # just version
+        uuid_mock.side_effect = [Exception('not used')]
+        now_mock.side_effect = [Exception('not used')]
+        proc = MetaProcessor('test', include_time=False, include_version=True)
+        self.assertEqual([f'octodns-version={__VERSION__}'], proc.values)
+
+        # just provider
+        proc = MetaProcessor('test', include_time=False, include_provider=True)
+        self.assertTrue(proc.include_provider)
+        self.assertFalse(proc.values)
+
+        # everything
+        uuid_mock.side_effect = ['abcdef-1234567890']
+        now_mock.side_effect = ['the-time']
+        proc = MetaProcessor(
+            'test',
+            include_time=True,
+            include_uuid=True,
+            include_version=True,
+            include_provider=True,
+        )
+        self.assertEqual(
+            [
+                f'octodns-version={__VERSION__}',
+                'time=the-time',
+                'uuid=abcdef-1234567890',
+            ],
+            proc.values,
+        )
+        self.assertTrue(proc.include_provider)
+
+    def test_uuid(self):
+        proc = MetaProcessor('test', include_time=False, include_uuid=True)
+        self.assertEqual(1, len(proc.values))
+        self.assertTrue(proc.values[0].startswith('uuid'))
+        # uuid's have 4 -
+        self.assertEqual(4, proc.values[0].count('-'))
+
+    def test_up_to_date(self):
+        proc = MetaProcessor('test')
+
+        # Creates always need to happen
+        self.assertFalse(proc._up_to_date(Create(self.meta_needs_update)))
+        self.assertFalse(proc._up_to_date(Create(self.meta_up_to_date)))
+
+        # Updates depend on the contents
+        self.assertFalse(proc._up_to_date(Update(self.meta_needs_update, None)))
+        self.assertTrue(proc._up_to_date(Update(self.meta_up_to_date, None)))
+
+    @patch('octodns.processor.meta.MetaProcessor.now')
+    def test_process_source_zone(self, now_mock):
+        now_mock.side_effect = ['the-time']
+        proc = MetaProcessor('test')
+
+        # meta record was added
+        desired = self.zone.copy()
+        processed = proc.process_source_zone(desired, None)
+        record = next(iter(processed.records))
+        self.assertEqual(self.meta_up_to_date, record)
+        self.assertEqual(['time=the-time'], record.values)
+
+    def test_process_target_zone(self):
+        proc = MetaProcessor('test')
+
+        # with defaults, not enabled
+        zone = self.zone.copy()
+        processed = proc.process_target_zone(zone, None)
+        self.assertFalse(processed.records)
+
+        # enable provider
+        proc = MetaProcessor('test', include_provider=True)
+
+        class DummyTarget:
+            id = 'dummy'
+
+        # enabled provider, no meta record, shouldn't happen, but also shouldn't
+        # blow up
+        processed = proc.process_target_zone(zone, DummyTarget())
+        self.assertFalse(processed.records)
+
+        # enabled provider, should now look for and update the provider value,
+        # - only record so nothing to skip over
+        # - time value in there to be skipped over
+        proc = MetaProcessor('test', include_provider=True)
+        zone = self.zone.copy()
+        meta = self.meta_up_to_date.copy()
+        zone.add_record(meta)
+        processed = proc.process_target_zone(zone, DummyTarget())
+        record = next(iter(processed.records))
+        self.assertEqual(['provider=dummy', 'time=xxx'], record.values)
+
+        # add another unrelated record that needs to be skipped
+        proc = MetaProcessor('test', include_provider=True)
+        zone = self.zone.copy()
+        meta = self.meta_up_to_date.copy()
+        zone.add_record(meta)
+        zone.add_record(self.not_meta)
+        processed = proc.process_target_zone(zone, DummyTarget())
+        self.assertEqual(2, len(processed.records))
+        record = [r for r in processed.records if r.name == proc.record_name][0]
+        self.assertEqual(['provider=dummy', 'time=xxx'], record.values)
+
+    def test_process_plan(self):
+        proc = MetaProcessor('test')
+
+        # no plan, shouldn't happen, but we shouldn't blow up
+        self.assertFalse(proc.process_plan(None, None, None))
+
+        # plan with just an up to date meta record, should kill off the plan
+        plan = Plan(
+            None,
+            None,
+            [Update(self.meta_up_to_date, self.meta_needs_update)],
+            True,
+        )
+        self.assertFalse(proc.process_plan(plan, None, None))
+
+        # plan with an out of date meta record, should leave the plan alone
+        plan = Plan(
+            None,
+            None,
+            [Update(self.meta_needs_update, self.meta_up_to_date)],
+            True,
+        )
+        self.assertEqual(plan, proc.process_plan(plan, None, None))
+
+        # plan with other changes preserved even if meta was somehow up to date
+        plan = Plan(
+            None,
+            None,
+            [
+                Update(self.meta_up_to_date, self.meta_needs_update),
+                Create(self.not_meta),
+            ],
+            True,
+        )
+        self.assertEqual(plan, proc.process_plan(plan, None, None))

From e61363b9105541cf036e5c18e5b5d681ef1e29d3 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 10 Sep 2023 12:48:12 -0700
Subject: [PATCH 030/116] Need to add the meta record with lenient in case it's
 temp empty values

---
 octodns/processor/meta.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/octodns/processor/meta.py b/octodns/processor/meta.py
index 9425e7e..26b4b2e 100644
--- a/octodns/processor/meta.py
+++ b/octodns/processor/meta.py
@@ -65,6 +65,9 @@ class MetaProcessor(BaseProcessor):
             desired,
             self.record_name,
             {'ttl': self.ttl, 'type': 'TXT', 'values': self.values},
+            # we may be passing in empty values here to be filled out later in
+            # process_target_zone
+            lenient=True,
         )
         desired.add_record(meta)
         return desired

From 00cbf2e136a3c1e2bf31b991202866a03ef46cd0 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 10 Sep 2023 12:49:11 -0700
Subject: [PATCH 031/116] processor should use id not name

---
 octodns/processor/base.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/octodns/processor/base.py b/octodns/processor/base.py
index 5279af2..f0890e0 100644
--- a/octodns/processor/base.py
+++ b/octodns/processor/base.py
@@ -9,7 +9,8 @@ class ProcessorException(Exception):
 
 class BaseProcessor(object):
     def __init__(self, name):
-        self.name = name
+        # TODO: name is DEPRECATED, remove in 2.0
+        self.id = self.name = name
 
     def process_source_zone(self, desired, sources):
         '''

From 0ad0c6be716ddbdd642e78e0faccdd3f28966838 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 10 Sep 2023 12:49:49 -0700
Subject: [PATCH 032/116] Update Manager to use MetaProcessor rather than
 special case of adding a meta record

---
 octodns/manager.py | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/octodns/manager.py b/octodns/manager.py
index c396440..65b46a4 100644
--- a/octodns/manager.py
+++ b/octodns/manager.py
@@ -14,10 +14,10 @@ from sys import stdout
 from . import __VERSION__
 from .idna import IdnaDict, idna_decode, idna_encode
 from .processor.arpa import AutoArpa
+from .processor.meta import MetaProcessor
 from .provider.base import BaseProvider
 from .provider.plan import Plan
 from .provider.yaml import SplitYamlProvider, YamlProvider
-from .record import Record
 from .yaml import safe_load
 from .zone import Zone
 
@@ -137,6 +137,19 @@ class Manager(object):
                 auto_arpa.name
             ] + self.global_post_processors
 
+        if self.include_meta:
+            self.log.info(
+                '__init__: adding meta to processors and providers, appending it to global_post_processors list'
+            )
+            meta = MetaProcessor(
+                'meta',
+                record_name='octodns-meta',
+                include_time=False,
+                include_provider=True,
+            )
+            self.processors[meta.id] = meta
+            self.global_post_processors.append(meta.id)
+
         plan_outputs_config = manager_config.get(
             'plan_outputs',
             {
@@ -440,17 +453,6 @@ class Manager(object):
         plans = []
 
         for target in targets:
-            if self.include_meta:
-                meta = Record.new(
-                    zone,
-                    'octodns-meta',
-                    {
-                        'type': 'TXT',
-                        'ttl': 60,
-                        'value': f'provider={target.id}',
-                    },
-                )
-                zone.add_record(meta, replace=True)
             try:
                 plan = target.plan(zone, processors=processors)
             except TypeError as e:

From c0382c3043d509b92a57624864b6ca5c53c40efb Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 10 Sep 2023 14:07:28 -0700
Subject: [PATCH 033/116] Add MetaProcessor documentation

---
 octodns/processor/meta.py | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/octodns/processor/meta.py b/octodns/processor/meta.py
index 26b4b2e..c9e4a05 100644
--- a/octodns/processor/meta.py
+++ b/octodns/processor/meta.py
@@ -16,6 +16,43 @@ def _keys(values):
 
 
 class MetaProcessor(BaseProcessor):
+    '''
+    Add a special metadata record with timestamps, UUIDs, versions, and/or
+    provider name. Will only be updated when there are other changes being made.
+    A useful tool to aid in debugging and monitoring of DNS infrastructure.
+
+    Timestamps or UUIDs can be useful in checking whether changes are
+    propagating, either from a provider's backend to their servers or via AXFRs.
+
+    Provider can be utilized to determine which DNS system responded to a query
+    when things are operating in dual authority or split horizon setups.
+
+    Creates a TXT record with the name configured with values based on processor
+    settings. Values are in the form `key=<value>`, e.g.
+    `time=2023-09-10T05:49:04.246953`
+
+    processors:
+      meta:
+        class: octodns.processor.meta.MetaProcessor
+        # The name to use for the meta record.
+        # (default: meta)
+        record_name: meta
+        # Include a timestamp with a UTC value indicating the timeframe when the
+        # last change was made.
+        # (default: true)
+        include_time: true
+        # Include a UUID that can be utilized to uniquely identify the run
+        # pushing data
+        # (default: false)
+        include_uuid: false
+        # Include the provider id for the target where data is being pushed
+        # (default: false)
+        include_provider: false
+        # Include the octoDNS version being used
+        # (default: false)
+        include_version: false
+    '''
+
     @classmethod
     def now(cls):
         return datetime.utcnow().isoformat()

From 699afbc3dd629eed0a0e83a3ec33cc3a42fa01d4 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 10 Sep 2023 14:43:54 -0700
Subject: [PATCH 034/116] Add MetaProcessor to README list

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 9a976e9..0250890 100644
--- a/README.md
+++ b/README.md
@@ -331,6 +331,7 @@ Similar to providers, but can only serve to populate records into a zone, cannot
 | [AcmeMangingProcessor](/octodns/processor/acme.py) | Useful when processes external to octoDNS are managing acme challenge DNS records, e.g. LetsEncrypt |
 | [AutoArpa](/octodns/processor/arpa.py) | See [Automatic PTR generation](#automatic-ptr-generation) below |
 | [IgnoreRootNsFilter](/octodns/processor/filter.py) | Filter that INGORES root/APEX NS records and prevents octoDNS from trying to manage them (where supported.) |
+| [MetaProcessor](/octodns/processor/meta.py) | Adds a special meta record with timing, UUID, providers, and/or version to aid in debugging and monitoring. |
 | [NameAllowlistFilter](/octodns/processor/filter.py) | Filter that ONLY manages records that match specified naming patterns, all others will be ignored |
 | [NameRejectlistFilter](/octodns/processor/filter.py) | Filter that INGORES records that match specified naming patterns, all others will be managed |
 | [OwnershipProcessor](/octodns/processor/ownership.py) | Processor that implements ownership in octoDNS so that it can manage only the records in a zone in sources and will ignore all others. |

From 9845bd130607fcc4ad37b68a2322423fd6927be5 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Wed, 13 Sep 2023 12:42:38 -0700
Subject: [PATCH 035/116] v1.1.0 version bump & changelog update

---
 CHANGELOG.md        | 2 +-
 octodns/__init__.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7406a7b..96b19f0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,4 @@
-## v1.1.0 - 2023-??-?? - ???
+## v1.1.0 - 2023-09-13 - More than enough for a minor release
 
 #### Noteworthy changes
 
diff --git a/octodns/__init__.py b/octodns/__init__.py
index 9fb2bbb..43b9eb5 100644
--- a/octodns/__init__.py
+++ b/octodns/__init__.py
@@ -1,3 +1,3 @@
 'OctoDNS: DNS as code - Tools for managing DNS across multiple providers'
 
-__VERSION__ = '1.0.0'
+__VERSION__ = '1.1.0'

From b20113e1a9c2d1c1e61d252577cbd9e8cf983675 Mon Sep 17 00:00:00 2001
From: Kiran Naidoo <kiran@kiran.za.org>
Date: Thu, 14 Sep 2023 15:05:54 +0100
Subject: [PATCH 036/116] Fix typo when loading auto-arpa config

Resolved a typo that prevented the auto-arpa configuration options from
loading.
---
 octodns/manager.py            |  2 +-
 tests/config/simple-arpa.yaml |  1 +
 tests/test_octodns_manager.py | 12 ++++++++++++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/octodns/manager.py b/octodns/manager.py
index 65b46a4..95b7ed6 100644
--- a/octodns/manager.py
+++ b/octodns/manager.py
@@ -129,7 +129,7 @@ class Manager(object):
             self.log.info(
                 '__init__: adding auto-arpa to processors and providers, prepending it to global_post_processors list'
             )
-            kwargs = self.auto_arpa if isinstance(auto_arpa, dict) else {}
+            kwargs = self.auto_arpa if isinstance(self.auto_arpa, dict) else {}
             auto_arpa = AutoArpa('auto-arpa', **kwargs)
             self.providers[auto_arpa.name] = auto_arpa
             self.processors[auto_arpa.name] = auto_arpa
diff --git a/tests/config/simple-arpa.yaml b/tests/config/simple-arpa.yaml
index 1056c3b..75669a8 100644
--- a/tests/config/simple-arpa.yaml
+++ b/tests/config/simple-arpa.yaml
@@ -1,6 +1,7 @@
 manager:
   max_workers: 2
   auto_arpa:
+    populate_should_replace: True
     ttl: 1800
 
 providers:
diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py
index 8998610..19dc78f 100644
--- a/tests/test_octodns_manager.py
+++ b/tests/test_octodns_manager.py
@@ -928,6 +928,18 @@ class TestManager(TestCase):
     def test_auto_arpa(self):
         manager = Manager(get_config_filename('simple-arpa.yaml'))
 
+        # provider config
+        self.assertEqual(
+            True, manager.providers.get("auto-arpa").populate_should_replace
+        )
+        self.assertEqual(1800, manager.providers.get("auto-arpa").ttl)
+
+        # processor config
+        self.assertEqual(
+            True, manager.processors.get("auto-arpa").populate_should_replace
+        )
+        self.assertEqual(1800, manager.processors.get("auto-arpa").ttl)
+
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
 

From 2476795a28e942268063e7bb1dbdd96f80952029 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 16 Sep 2023 21:02:11 -0700
Subject: [PATCH 037/116] v1.1.1 version bump and changelog update

---
 CHANGELOG.md        | 4 ++++
 octodns/__init__.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 96b19f0..1fa1428 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## v1.1.0 - 2023-09-16 - Doh! Fix that one little thing
+
+* Address a bug in the handling of loading auto-arpa manager configuration.
+
 ## v1.1.0 - 2023-09-13 - More than enough for a minor release
 
 #### Noteworthy changes
diff --git a/octodns/__init__.py b/octodns/__init__.py
index 43b9eb5..80c6e0e 100644
--- a/octodns/__init__.py
+++ b/octodns/__init__.py
@@ -1,3 +1,3 @@
 'OctoDNS: DNS as code - Tools for managing DNS across multiple providers'
 
-__VERSION__ = '1.1.0'
+__VERSION__ = '1.1.1'

From 33959104b60c1767a289511657ac48da04aab0ef Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 16 Sep 2023 21:05:58 -0700
Subject: [PATCH 038/116] Correct changelog version number

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1fa1428..808a153 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,4 @@
-## v1.1.0 - 2023-09-16 - Doh! Fix that one little thing
+## v1.1.1 - 2023-09-16 - Doh! Fix that one little thing
 
 * Address a bug in the handling of loading auto-arpa manager configuration.
 

From 0181158953f373345b581165a35040a5ebdd2d91 Mon Sep 17 00:00:00 2001
From: Viranch Mehta <email@viranch.me>
Date: Mon, 18 Sep 2023 15:31:16 -0700
Subject: [PATCH 039/116] Include sources only if they exist

---
 octodns/provider/yaml.py                       | 17 ++++++++++++++---
 tests/config/hybrid/one.test.yaml              |  4 ++++
 tests/config/hybrid/two.test./$two.test.yaml   |  4 ++++
 .../hybrid/two.test./split-zone-file.yaml      |  4 ++++
 tests/test_octodns_provider_yaml.py            | 18 ++++++++++++++++++
 5 files changed, 44 insertions(+), 3 deletions(-)
 create mode 100644 tests/config/hybrid/one.test.yaml
 create mode 100644 tests/config/hybrid/two.test./$two.test.yaml
 create mode 100644 tests/config/hybrid/two.test./split-zone-file.yaml

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index 3ca7842..697a29b 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -278,8 +278,10 @@ class YamlProvider(BaseProvider):
                     f'Both UTF-8 "{utf8}" and IDNA "{idna}" exist for {zone.decoded_name}'
                 )
             directory = utf8
-        else:
+        elif isdir(idna):
             directory = idna
+        else:
+            return []
 
         for filename in listdir(directory):
             if filename.endswith('.yaml'):
@@ -294,8 +296,10 @@ class YamlProvider(BaseProvider):
                     f'Both UTF-8 "{utf8}" and IDNA "{idna}" exist for {zone.decoded_name}'
                 )
             return utf8
+        elif isfile(idna):
+            return idna
 
-        return idna
+        return None
 
     def _populate_from_file(self, filename, zone, lenient):
         with open(filename, 'r') as fh:
@@ -341,11 +345,18 @@ class YamlProvider(BaseProvider):
             sources.extend(self._split_sources(zone))
 
         if not self.disable_zonefile:
-            sources.append(self._zone_sources(zone))
+            source = self._zone_sources(zone)
+            if source:
+                sources.append(self._zone_sources(zone))
 
         if self.shared_filename:
             sources.append(join(self.directory, self.shared_filename))
 
+        if not sources:
+            self.log.info(
+                'populate:   no YAMLs found for %s', zone.decoded_name
+            )
+
         # determinstically order our sources
         sources.sort()
 
diff --git a/tests/config/hybrid/one.test.yaml b/tests/config/hybrid/one.test.yaml
new file mode 100644
index 0000000..d2ac6ba
--- /dev/null
+++ b/tests/config/hybrid/one.test.yaml
@@ -0,0 +1,4 @@
+---
+flat-zone-file:
+  type: TXT
+  value: non-split flat zone file
diff --git a/tests/config/hybrid/two.test./$two.test.yaml b/tests/config/hybrid/two.test./$two.test.yaml
new file mode 100644
index 0000000..8019a9d
--- /dev/null
+++ b/tests/config/hybrid/two.test./$two.test.yaml
@@ -0,0 +1,4 @@
+---
+'':
+  type: TXT
+  value: root TXT
diff --git a/tests/config/hybrid/two.test./split-zone-file.yaml b/tests/config/hybrid/two.test./split-zone-file.yaml
new file mode 100644
index 0000000..ea051ba
--- /dev/null
+++ b/tests/config/hybrid/two.test./split-zone-file.yaml
@@ -0,0 +1,4 @@
+---
+split-zone-file:
+  type: TXT
+  value: split zone file
diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py
index c289e6a..901819a 100644
--- a/tests/test_octodns_provider_yaml.py
+++ b/tests/test_octodns_provider_yaml.py
@@ -760,6 +760,24 @@ class TestSplitYamlProvider(TestCase):
             sorted(provider.list_zones()),
         )
 
+    def test_hybrid_directory(self):
+        source = YamlProvider(
+            'test',
+            join(dirname(__file__), 'config/hybrid'),
+            split_extension='.',
+            strict_supports=False,
+        )
+
+        # flat zone file only
+        zone = Zone('one.test.', [])
+        source.populate(zone)
+        self.assertEqual(1, len(zone.records))
+
+        # split zone only
+        zone = Zone('two.test.', [])
+        source.populate(zone)
+        self.assertEqual(2, len(zone.records))
+
 
 class TestOverridingYamlProvider(TestCase):
     def test_provider(self):

From 6042cb0ec545c3ec49e30f1792c3573b7a676b9e Mon Sep 17 00:00:00 2001
From: Viranch Mehta <email@viranch.me>
Date: Mon, 18 Sep 2023 20:34:21 -0700
Subject: [PATCH 040/116] reuse compiled source

---
 octodns/provider/yaml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index 697a29b..50a4c3a 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -347,7 +347,7 @@ class YamlProvider(BaseProvider):
         if not self.disable_zonefile:
             source = self._zone_sources(zone)
             if source:
-                sources.append(self._zone_sources(zone))
+                sources.append(source)
 
         if self.shared_filename:
             sources.append(join(self.directory, self.shared_filename))

From e92c1079c9655b1d2ee190561c371a82d65819d8 Mon Sep 17 00:00:00 2001
From: Viranch Mehta <email@viranch.me>
Date: Mon, 18 Sep 2023 21:16:50 -0700
Subject: [PATCH 041/116] Fix wrong re-use of sources for dynamic zones

---
 octodns/manager.py               |  4 ++--
 tests/config/dynamic-config.yaml | 12 +++++++++++-
 tests/test_octodns_manager.py    |  8 ++++++--
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/octodns/manager.py b/octodns/manager.py
index 95b7ed6..1753269 100644
--- a/octodns/manager.py
+++ b/octodns/manager.py
@@ -516,11 +516,11 @@ class Manager(object):
             # we've found a dynamic config element
 
             # find its sources
-            sources = sources or self._get_sources(
+            found_sources = sources or self._get_sources(
                 name, config, eligible_sources
             )
             self.log.info('sync:   dynamic zone=%s, sources=%s', name, sources)
-            for source in sources:
+            for source in found_sources:
                 if not hasattr(source, 'list_zones'):
                     raise ManagerException(
                         f'dynamic zone={name} includes a source, {source.id}, that does not support `list_zones`'
diff --git a/tests/config/dynamic-config.yaml b/tests/config/dynamic-config.yaml
index 233b831..cf9c460 100644
--- a/tests/config/dynamic-config.yaml
+++ b/tests/config/dynamic-config.yaml
@@ -3,17 +3,27 @@ providers:
     class: octodns.provider.yaml.YamlProvider
     directory: tests/config
 
+  in2:
+    class: octodns.provider.yaml.YamlProvider
+    directory: tests/config/split
+
   dump:
     class: octodns.provider.yaml.YamlProvider
     directory: env/YAML_TMP_DIR
 
 zones:
-  '*':
+  '*.one':
     sources:
     - in
     targets:
     - dump
 
+  '*.two':
+    sources:
+    - in2
+    targets:
+    - dump
+
   subzone.unit.tests.:
     sources:
     - in
diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py
index 19dc78f..ae2f415 100644
--- a/tests/test_octodns_manager.py
+++ b/tests/test_octodns_manager.py
@@ -997,10 +997,14 @@ class TestManager(TestCase):
 
             manager = Manager(get_config_filename('dynamic-config.yaml'))
 
-            # just unit.tests. which should have been dynamically configured via
+            # two zones which should have been dynamically configured via
             # list_zones
             self.assertEqual(
-                23, manager.sync(eligible_zones=['unit.tests.'], dry_run=False)
+                29,
+                manager.sync(
+                    eligible_zones=['unit.tests.', 'dynamic.tests.'],
+                    dry_run=False,
+                ),
             )
 
             # just subzone.unit.tests. which was explicitly configured

From 11118efe93aa87d6ccd166b48c44d6a07aeebbaa Mon Sep 17 00:00:00 2001
From: Viranch Mehta <email@viranch.me>
Date: Mon, 18 Sep 2023 21:41:44 -0700
Subject: [PATCH 042/116] Raise exception when no yamls are found for a zone

---
 octodns/provider/yaml.py            | 4 +---
 tests/test_octodns_provider_yaml.py | 4 ++--
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index 50a4c3a..23e65ef 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -353,9 +353,7 @@ class YamlProvider(BaseProvider):
             sources.append(join(self.directory, self.shared_filename))
 
         if not sources:
-            self.log.info(
-                'populate:   no YAMLs found for %s', zone.decoded_name
-            )
+            raise ProviderException(f'no YAMLs found for {zone.decoded_name}')
 
         # determinstically order our sources
         sources.sort()
diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py
index 901819a..b98c067 100644
--- a/tests/test_octodns_provider_yaml.py
+++ b/tests/test_octodns_provider_yaml.py
@@ -663,8 +663,8 @@ class TestSplitYamlProvider(TestCase):
         zone = Zone('empty.', [])
 
         # without it we see everything
-        source.populate(zone)
-        self.assertEqual(0, len(zone.records))
+        with self.assertRaises(ProviderException):
+            source.populate(zone)
 
     def test_unsorted(self):
         source = SplitYamlProvider(

From 4e26de7a89504be6725ecca473f8837cbfe9a0ba Mon Sep 17 00:00:00 2001
From: Viranch Mehta <email@viranch.me>
Date: Tue, 19 Sep 2023 11:01:38 -0700
Subject: [PATCH 043/116] CHANGELOG entry

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 808a153..2f82be4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+## v1.1.2 - 2023-09-20 - Bunch more bug fixes
+
+* Fix crash bug when using the YamlProvider with a directory that contains a
+  mix of split and non-split zone yamls. See https://github.com/octodns/octodns/issues/1066
+
 ## v1.1.1 - 2023-09-16 - Doh! Fix that one little thing
 
 * Address a bug in the handling of loading auto-arpa manager configuration.

From f599358a2c894d0d9619ab04cc6b1e6b769cd50a Mon Sep 17 00:00:00 2001
From: Viranch Mehta <email@viranch.me>
Date: Tue, 19 Sep 2023 11:17:50 -0700
Subject: [PATCH 044/116] CHANGELOG entry

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2f82be4..09608f9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,8 @@
 
 * Fix crash bug when using the YamlProvider with a directory that contains a
   mix of split and non-split zone yamls. See https://github.com/octodns/octodns/issues/1066
+* Fix discovery of zones from different sources when there are multiple dynamic
+  zones. See https://github.com/octodns/octodns/issues/1068
 
 ## v1.1.1 - 2023-09-16 - Doh! Fix that one little thing
 

From 43d380d76a65ee11c327b4fac918b3005a158a9a Mon Sep 17 00:00:00 2001
From: Martin Frausing <fra@one.com>
Date: Mon, 18 Sep 2023 13:17:30 +0200
Subject: [PATCH 045/116] The record got field names from the DNSKEY record by
 accident, fix this

Basically changing from
https://www.rfc-editor.org/rfc/rfc4034.html#section-2.1 to https://www.rfc-editor.org/rfc/rfc4034.html#section-5.1

So:
flags -> key_tag
protocol -> algorithm
algorithm -> digest_type
public_key -> digest
---
 octodns/record/ds.py            |  86 +++++++-------
 tests/test_octodns_record_ds.py | 192 ++++++++++++++++----------------
 2 files changed, 140 insertions(+), 138 deletions(-)

diff --git a/octodns/record/ds.py b/octodns/record/ds.py
index ad431f2..305facb 100644
--- a/octodns/record/ds.py
+++ b/octodns/record/ds.py
@@ -8,31 +8,31 @@ from .rr import RrParseError
 
 
 class DsValue(EqualityTupleMixin, dict):
-    # https://www.rfc-editor.org/rfc/rfc4034.html#section-2.1
+    # https://www.rfc-editor.org/rfc/rfc4034.html#section-5.1
 
     @classmethod
     def parse_rdata_text(cls, value):
         try:
-            flags, protocol, algorithm, public_key = value.split(' ')
+            key_tag, algorithm, digest_type, digest = value.split(' ')
         except ValueError:
             raise RrParseError()
         try:
-            flags = int(flags)
+            key_tag = int(key_tag)
         except ValueError:
             pass
         try:
-            protocol = int(protocol)
+            algorithm = int(algorithm)
         except ValueError:
             pass
         try:
-            algorithm = int(algorithm)
+            digest_type = int(digest_type)
         except ValueError:
             pass
         return {
-            'flags': flags,
-            'protocol': protocol,
+            'key_tag': key_tag,
             'algorithm': algorithm,
-            'public_key': public_key,
+            'digest_type': digest_type,
+            'digest': digest,
         }
 
     @classmethod
@@ -42,25 +42,25 @@ class DsValue(EqualityTupleMixin, dict):
         reasons = []
         for value in data:
             try:
-                int(value['flags'])
+                int(value['key_tag'])
             except KeyError:
-                reasons.append('missing flags')
+                reasons.append('missing key_tag')
             except ValueError:
-                reasons.append(f'invalid flags "{value["flags"]}"')
-            try:
-                int(value['protocol'])
-            except KeyError:
-                reasons.append('missing protocol')
-            except ValueError:
-                reasons.append(f'invalid protocol "{value["protocol"]}"')
+                reasons.append(f'invalid key_tag "{value["key_tag"]}"')
             try:
                 int(value['algorithm'])
             except KeyError:
                 reasons.append('missing algorithm')
             except ValueError:
                 reasons.append(f'invalid algorithm "{value["algorithm"]}"')
-            if 'public_key' not in value:
-                reasons.append('missing public_key')
+            try:
+                int(value['digest_type'])
+            except KeyError:
+                reasons.append('missing digest_type')
+            except ValueError:
+                reasons.append(f'invalid digest_type "{value["digest_type"]}"')
+            if 'digest' not in value:
+                reasons.append('missing digest')
         return reasons
 
     @classmethod
@@ -70,28 +70,20 @@ class DsValue(EqualityTupleMixin, dict):
     def __init__(self, value):
         super().__init__(
             {
-                'flags': int(value['flags']),
-                'protocol': int(value['protocol']),
+                'key_tag': int(value['key_tag']),
                 'algorithm': int(value['algorithm']),
-                'public_key': value['public_key'],
+                'digest_type': int(value['digest_type']),
+                'digest': value['digest'],
             }
         )
 
     @property
-    def flags(self):
-        return self['flags']
-
-    @flags.setter
-    def flags(self, value):
-        self['flags'] = value
-
-    @property
-    def protocol(self):
-        return self['protocol']
+    def key_tag(self):
+        return self['key_tag']
 
-    @protocol.setter
-    def protocol(self, value):
-        self['protocol'] = value
+    @key_tag.setter
+    def key_tag(self, value):
+        self['key_tag'] = value
 
     @property
     def algorithm(self):
@@ -102,12 +94,20 @@ class DsValue(EqualityTupleMixin, dict):
         self['algorithm'] = value
 
     @property
-    def public_key(self):
-        return self['public_key']
+    def digest_type(self):
+        return self['digest_type']
+
+    @digest_type.setter
+    def digest_type(self, value):
+        self['digest_type'] = value
+
+    @property
+    def digest(self):
+        return self['digest']
 
-    @public_key.setter
-    def public_key(self, value):
-        self['public_key'] = value
+    @digest.setter
+    def digest(self, value):
+        self['digest'] = value
 
     @property
     def data(self):
@@ -116,15 +116,15 @@ class DsValue(EqualityTupleMixin, dict):
     @property
     def rdata_text(self):
         return (
-            f'{self.flags} {self.protocol} {self.algorithm} {self.public_key}'
+            f'{self.key_tag} {self.algorithm} {self.digest_type} {self.digest}'
         )
 
     def _equality_tuple(self):
-        return (self.flags, self.protocol, self.algorithm, self.public_key)
+        return (self.key_tag, self.algorithm, self.digest_type, self.digest)
 
     def __repr__(self):
         return (
-            f'{self.flags} {self.protocol} {self.algorithm} {self.public_key}'
+            f'{self.key_tag} {self.algorithm} {self.digest_type} {self.digest}'
         )
 
 
diff --git a/tests/test_octodns_record_ds.py b/tests/test_octodns_record_ds.py
index 0cb7eed..e4a65fa 100644
--- a/tests/test_octodns_record_ds.py
+++ b/tests/test_octodns_record_ds.py
@@ -12,64 +12,64 @@ from octodns.zone import Zone
 class TestRecordDs(TestCase):
     def test_ds(self):
         for a, b in (
-            # diff flags
+            # diff key_tag
             (
                 {
-                    'flags': 0,
-                    'protocol': 1,
-                    'algorithm': 2,
-                    'public_key': 'abcdef0123456',
+                    'key_tag': 0,
+                    'algorithm': 1,
+                    'digest_type': 2,
+                    'digest': 'abcdef0123456',
                 },
                 {
-                    'flags': 1,
-                    'protocol': 1,
-                    'algorithm': 2,
-                    'public_key': 'abcdef0123456',
+                    'key_tag': 1,
+                    'algorithm': 1,
+                    'digest_type': 2,
+                    'digest': 'abcdef0123456',
                 },
             ),
-            # diff protocol
+            # diff algorithm
             (
                 {
-                    'flags': 0,
-                    'protocol': 1,
-                    'algorithm': 2,
-                    'public_key': 'abcdef0123456',
+                    'key_tag': 0,
+                    'algorithm': 1,
+                    'digest_type': 2,
+                    'digest': 'abcdef0123456',
                 },
                 {
-                    'flags': 0,
-                    'protocol': 2,
+                    'key_tag': 0,
                     'algorithm': 2,
-                    'public_key': 'abcdef0123456',
+                    'digest_type': 2,
+                    'digest': 'abcdef0123456',
                 },
             ),
-            # diff algorithm
+            # diff digest_type
             (
                 {
-                    'flags': 0,
-                    'protocol': 1,
-                    'algorithm': 2,
-                    'public_key': 'abcdef0123456',
+                    'key_tag': 0,
+                    'algorithm': 1,
+                    'digest_type': 2,
+                    'digest': 'abcdef0123456',
                 },
                 {
-                    'flags': 0,
-                    'protocol': 1,
-                    'algorithm': 3,
-                    'public_key': 'abcdef0123456',
+                    'key_tag': 0,
+                    'algorithm': 1,
+                    'digest_type': 3,
+                    'digest': 'abcdef0123456',
                 },
             ),
-            # diff public_key
+            # diff digest
             (
                 {
-                    'flags': 0,
-                    'protocol': 1,
-                    'algorithm': 2,
-                    'public_key': 'abcdef0123456',
+                    'key_tag': 0,
+                    'algorithm': 1,
+                    'digest_type': 2,
+                    'digest': 'abcdef0123456',
                 },
                 {
-                    'flags': 0,
-                    'protocol': 1,
-                    'algorithm': 2,
-                    'public_key': 'bcdef0123456a',
+                    'key_tag': 0,
+                    'algorithm': 1,
+                    'digest_type': 2,
+                    'digest': 'bcdef0123456a',
                 },
             ),
         ):
@@ -104,102 +104,104 @@ class TestRecordDs(TestCase):
         # things ints, will parse
         self.assertEqual(
             {
-                'flags': 'one',
-                'protocol': 'two',
-                'algorithm': 'three',
-                'public_key': 'key',
+                'key_tag': 'one',
+                'algorithm': 'two',
+                'digest_type': 'three',
+                'digest': 'key',
             },
             DsValue.parse_rdata_text('one two three key'),
         )
 
         # valid
         data = {
-            'flags': 0,
-            'protocol': 1,
-            'algorithm': 2,
-            'public_key': '99148c81',
+            'key_tag': 0,
+            'algorithm': 1,
+            'digest_type': 2,
+            'digest': '99148c81',
         }
         self.assertEqual(data, DsValue.parse_rdata_text('0 1 2 99148c81'))
         self.assertEqual([], DsValue.validate(data, 'DS'))
 
-        # missing flags
-        data = {'protocol': 1, 'algorithm': 2, 'public_key': '99148c81'}
-        self.assertEqual(['missing flags'], DsValue.validate(data, 'DS'))
-        # invalid flags
-        data = {
-            'flags': 'a',
-            'protocol': 1,
-            'algorithm': 2,
-            'public_key': '99148c81',
-        }
-        self.assertEqual(['invalid flags "a"'], DsValue.validate(data, 'DS'))
-
-        # missing protocol
-        data = {'flags': 1, 'algorithm': 2, 'public_key': '99148c81'}
-        self.assertEqual(['missing protocol'], DsValue.validate(data, 'DS'))
-        # invalid protocol
+        # missing key_tag
+        data = {'algorithm': 1, 'digest_type': 2, 'digest': '99148c81'}
+        self.assertEqual(['missing key_tag'], DsValue.validate(data, 'DS'))
+        # invalid key_tag
         data = {
-            'flags': 1,
-            'protocol': 'a',
-            'algorithm': 2,
-            'public_key': '99148c81',
+            'key_tag': 'a',
+            'algorithm': 1,
+            'digest_type': 2,
+            'digest': '99148c81',
         }
-        self.assertEqual(['invalid protocol "a"'], DsValue.validate(data, 'DS'))
+        self.assertEqual(['invalid key_tag "a"'], DsValue.validate(data, 'DS'))
 
         # missing algorithm
-        data = {'flags': 1, 'protocol': 2, 'public_key': '99148c81'}
+        data = {'key_tag': 1, 'digest_type': 2, 'digest': '99148c81'}
         self.assertEqual(['missing algorithm'], DsValue.validate(data, 'DS'))
         # invalid algorithm
         data = {
-            'flags': 1,
-            'protocol': 2,
+            'key_tag': 1,
             'algorithm': 'a',
-            'public_key': '99148c81',
+            'digest_type': 2,
+            'digest': '99148c81',
         }
         self.assertEqual(
             ['invalid algorithm "a"'], DsValue.validate(data, 'DS')
         )
 
-        # missing algorithm (list)
-        data = {'flags': 1, 'protocol': 2, 'algorithm': 3}
-        self.assertEqual(['missing public_key'], DsValue.validate([data], 'DS'))
+        # missing digest_type
+        data = {'key_tag': 1, 'algorithm': 2, 'digest': '99148c81'}
+        self.assertEqual(['missing digest_type'], DsValue.validate(data, 'DS'))
+        # invalid digest_type
+        data = {
+            'key_tag': 1,
+            'algorithm': 2,
+            'digest_type': 'a',
+            'digest': '99148c81',
+        }
+        self.assertEqual(
+            ['invalid digest_type "a"'], DsValue.validate(data, 'DS')
+        )
+
+        # missing digest_type (list)
+        data = {'key_tag': 1, 'algorithm': 2, 'digest_type': 3}
+        self.assertEqual(['missing digest'], DsValue.validate([data], 'DS'))
 
         zone = Zone('unit.tests.', [])
         values = [
             {
-                'flags': 0,
-                'protocol': 1,
-                'algorithm': 2,
-                'public_key': '99148c81',
+                'key_tag': 0,
+                'algorithm': 1,
+                'digest_type': 2,
+                'digest': '99148c81',
             },
             {
-                'flags': 1,
-                'protocol': 2,
-                'algorithm': 3,
-                'public_key': '99148c44',
+                'key_tag': 1,
+                'algorithm': 2,
+                'digest_type': 3,
+                'digest': '99148c44',
             },
         ]
         a = DsRecord(zone, 'ds', {'ttl': 32, 'values': values})
-        self.assertEqual(0, a.values[0].flags)
-        a.values[0].flags += 1
-        self.assertEqual(1, a.values[0].flags)
-
-        self.assertEqual(1, a.values[0].protocol)
-        a.values[0].protocol += 1
-        self.assertEqual(2, a.values[0].protocol)
+        self.assertEqual(0, a.values[0].key_tag)
+        a.values[0].key_tag += 1
+        self.assertEqual(1, a.values[0].key_tag)
 
-        self.assertEqual(2, a.values[0].algorithm)
+        self.assertEqual(1, a.values[0].algorithm)
         a.values[0].algorithm += 1
-        self.assertEqual(3, a.values[0].algorithm)
+        self.assertEqual(2, a.values[0].algorithm)
+
+        self.assertEqual(2, a.values[0].digest_type)
+        a.values[0].digest_type += 1
+        self.assertEqual(3, a.values[0].digest_type)
 
-        self.assertEqual('99148c81', a.values[0].public_key)
-        a.values[0].public_key = '99148c42'
-        self.assertEqual('99148c42', a.values[0].public_key)
+        self.assertEqual('99148c81', a.values[0].digest)
+        a.values[0].digest = '99148c42'
+        self.assertEqual('99148c42', a.values[0].digest)
 
-        self.assertEqual(1, a.values[1].flags)
-        self.assertEqual(2, a.values[1].protocol)
-        self.assertEqual(3, a.values[1].algorithm)
-        self.assertEqual('99148c44', a.values[1].public_key)
+        self.assertEqual(1, a.values[1].key_tag)
+        self.assertEqual(2, a.values[1].algorithm)
+        self.assertEqual(3, a.values[1].digest_type)
+        self.assertEqual('99148c44', a.values[1].digest)
 
         self.assertEqual(DsValue(values[1]), a.values[1].data)
         self.assertEqual('1 2 3 99148c44', a.values[1].rdata_text)

From 533cd12128a8619a2055f32abfefaad0c41e1573 Mon Sep 17 00:00:00 2001
From: Martin Frausing <fra@one.com>
Date: Thu, 21 Sep 2023 16:11:46 +0200
Subject: [PATCH 046/116] Support both the both sets of field names for DS

---
 octodns/record/ds.py            | 83 ++++++++++++++++++++++++---------
 tests/test_octodns_record_ds.py | 63 +++++++++++++++++++++++--
 2 files changed, 118 insertions(+), 28 deletions(-)

diff --git a/octodns/record/ds.py b/octodns/record/ds.py
index 305facb..b682ba8 100644
--- a/octodns/record/ds.py
+++ b/octodns/record/ds.py
@@ -41,26 +41,54 @@ class DsValue(EqualityTupleMixin, dict):
             data = (data,)
         reasons = []
         for value in data:
-            try:
-                int(value['key_tag'])
-            except KeyError:
-                reasons.append('missing key_tag')
-            except ValueError:
-                reasons.append(f'invalid key_tag "{value["key_tag"]}"')
-            try:
-                int(value['algorithm'])
-            except KeyError:
-                reasons.append('missing algorithm')
-            except ValueError:
-                reasons.append(f'invalid algorithm "{value["algorithm"]}"')
-            try:
-                int(value['digest_type'])
-            except KeyError:
-                reasons.append('missing digest_type')
-            except ValueError:
-                reasons.append(f'invalid digest_type "{value["digest_type"]}"')
-            if 'digest' not in value:
-                reasons.append('missing digest')
+            # we need to validate both "old" style field names and new
+            # it is safe to assume if public_key or flags are defined then it is "old" style
+            # A DS record without public_key doesn't make any sense and shouldn't have validated previously
+            if "public_key" in value or "flags" in value:
+                try:
+                    int(value['flags'])
+                except KeyError:
+                    reasons.append('missing flags')
+                except ValueError:
+                    reasons.append(f'invalid flags "{value["flags"]}"')
+                try:
+                    int(value['protocol'])
+                except KeyError:
+                    reasons.append('missing protocol')
+                except ValueError:
+                    reasons.append(f'invalid protocol "{value["protocol"]}"')
+                try:
+                    int(value['algorithm'])
+                except KeyError:
+                    reasons.append('missing algorithm')
+                except ValueError:
+                    reasons.append(f'invalid algorithm "{value["algorithm"]}"')
+                if 'public_key' not in value:
+                    reasons.append('missing public_key')
+
+            else:
+                try:
+                    int(value['key_tag'])
+                except KeyError:
+                    reasons.append('missing key_tag')
+                except ValueError:
+                    reasons.append(f'invalid key_tag "{value["key_tag"]}"')
+                try:
+                    int(value['algorithm'])
+                except KeyError:
+                    reasons.append('missing algorithm')
+                except ValueError:
+                    reasons.append(f'invalid algorithm "{value["algorithm"]}"')
+                try:
+                    int(value['digest_type'])
+                except KeyError:
+                    reasons.append('missing digest_type')
+                except ValueError:
+                    reasons.append(
+                        f'invalid digest_type "{value["digest_type"]}"'
+                    )
+                if 'digest' not in value:
+                    reasons.append('missing digest')
         return reasons
 
     @classmethod
@@ -68,14 +96,23 @@ class DsValue(EqualityTupleMixin, dict):
         return [cls(v) for v in values]
 
     def __init__(self, value):
-        super().__init__(
-            {
+        # we need to instantiate both based on "old" style field names and new
+        # it is safe to assume if public_key or flags are defined then it is "old" style
+        if "public_key" in value or "flags" in value:
+            init = {
+                'key_tag': int(value['flags']),
+                'algorithm': int(value['protocol']),
+                'digest_type': int(value['algorithm']),
+                'digest': value['public_key'],
+            }
+        else:
+            init = {
                 'key_tag': int(value['key_tag']),
                 'algorithm': int(value['algorithm']),
                 'digest_type': int(value['digest_type']),
                 'digest': value['digest'],
             }
-        )
+        super().__init__(init)
 
     @property
     def key_tag(self):
diff --git a/tests/test_octodns_record_ds.py b/tests/test_octodns_record_ds.py
index e4a65fa..f0429de 100644
--- a/tests/test_octodns_record_ds.py
+++ b/tests/test_octodns_record_ds.py
@@ -72,6 +72,21 @@ class TestRecordDs(TestCase):
                     'digest': 'bcdef0123456a',
                 },
             ),
+            # diff digest with previously used key names
+            (
+                {
+                    'flags': 0,
+                    'protocol': 1,
+                    'algorithm': 2,
+                    'public_key': 'abcdef0123456',
+                },
+                {
+                    'key_tag': 0,
+                    'algorithm': 1,
+                    'digest_type': 2,
+                    'digest': 'bcdef0123456a',
+                },
+            ),
         ):
             a = DsValue(a)
             self.assertEqual(a, a)
@@ -162,10 +177,48 @@ class TestRecordDs(TestCase):
             ['invalid digest_type "a"'], DsValue.validate(data, 'DS')
         )
 
-        # missing digest_type (list)
+        # missing public_key (list)
         data = {'key_tag': 1, 'algorithm': 2, 'digest_type': 3}
         self.assertEqual(['missing digest'], DsValue.validate([data], 'DS'))
 
+        # do validations again with old field style
+
+        # missing flags (list)
+        data = {'protocol': 2, 'algorithm': 3, 'public_key': '99148c81'}
+        self.assertEqual(['missing flags'], DsValue.validate([data], 'DS'))
+
+        # missing protocol (list)
+        data = {'flags': 1, 'algorithm': 3, 'public_key': '99148c81'}
+        self.assertEqual(['missing protocol'], DsValue.validate([data], 'DS'))
+
+        # missing algorithm (list)
+        data = {'flags': 1, 'protocol': 2, 'public_key': '99148c81'}
+        self.assertEqual(['missing algorithm'], DsValue.validate([data], 'DS'))
+
+        # missing public_key (list)
+        data = {'flags': 1, 'algorithm': 3, 'protocol': 2}
+        self.assertEqual(['missing public_key'], DsValue.validate([data], 'DS'))
+
+        # missing public_key (list)
+        data = {'flags': 1, 'algorithm': 3, 'protocol': 2, 'digest': '99148c81'}
+        self.assertEqual(['missing public_key'], DsValue.validate([data], 'DS'))
+
+        # invalid flags, protocol and algorithm
+        data = {
+            'flags': 'a',
+            'protocol': 'a',
+            'algorithm': 'a',
+            'public_key': '99148c81',
+        }
+        self.assertEqual(
+            [
+                'invalid flags "a"',
+                'invalid protocol "a"',
+                'invalid algorithm "a"',
+            ],
+            DsValue.validate(data, 'DS'),
+        )
+
         zone = Zone('unit.tests.', [])
         values = [
             {
@@ -175,10 +228,10 @@ class TestRecordDs(TestCase):
                 'digest': '99148c81',
             },
             {
-                'key_tag': 1,
-                'algorithm': 2,
-                'digest_type': 3,
-                'digest': '99148c44',
+                'flags': 1,
+                'protocol': 2,
+                'algorithm': 3,
+                'public_key': '99148c44',
             },
         ]
         a = DsRecord(zone, 'ds', {'ttl': 32, 'values': values})

From dfac2da3ecd6ed271a6aaccb3e256a42db89f7f3 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 23 Sep 2023 13:08:59 -0700
Subject: [PATCH 047/116] DEPRECATION warning on DsValue field fixes

---
 octodns/record/ds.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/octodns/record/ds.py b/octodns/record/ds.py
index b682ba8..5760fcf 100644
--- a/octodns/record/ds.py
+++ b/octodns/record/ds.py
@@ -2,6 +2,8 @@
 #
 #
 
+from logging import getLogger
+
 from ..equality import EqualityTupleMixin
 from .base import Record, ValuesMixin
 from .rr import RrParseError
@@ -9,6 +11,7 @@ from .rr import RrParseError
 
 class DsValue(EqualityTupleMixin, dict):
     # https://www.rfc-editor.org/rfc/rfc4034.html#section-5.1
+    log = getLogger('DsValue')
 
     @classmethod
     def parse_rdata_text(cls, value):
@@ -45,6 +48,9 @@ class DsValue(EqualityTupleMixin, dict):
             # it is safe to assume if public_key or flags are defined then it is "old" style
             # A DS record without public_key doesn't make any sense and shouldn't have validated previously
             if "public_key" in value or "flags" in value:
+                self.log.warning(
+                    '"algorithm", "flags", "public_key", and "protocol" support is DEPRECATED and will be removed in 2.0'
+                )
                 try:
                     int(value['flags'])
                 except KeyError:

From 879d8cd5270769bd834e42078e9177f778fd59af Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 23 Sep 2023 13:10:55 -0700
Subject: [PATCH 048/116] cls not self

---
 octodns/record/ds.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/octodns/record/ds.py b/octodns/record/ds.py
index 5760fcf..fe25803 100644
--- a/octodns/record/ds.py
+++ b/octodns/record/ds.py
@@ -48,7 +48,7 @@ class DsValue(EqualityTupleMixin, dict):
             # it is safe to assume if public_key or flags are defined then it is "old" style
             # A DS record without public_key doesn't make any sense and shouldn't have validated previously
             if "public_key" in value or "flags" in value:
-                self.log.warning(
+                cls.log.warning(
                     '"algorithm", "flags", "public_key", and "protocol" support is DEPRECATED and will be removed in 2.0'
                 )
                 try:

From 63c5118bcd236219af3b034fd3152babe02f30b9 Mon Sep 17 00:00:00 2001
From: Viranch Mehta <email@viranch.me>
Date: Sun, 24 Sep 2023 23:20:06 -0700
Subject: [PATCH 049/116] Fix validation for dynamic records with IPv4+IPv6
 subnets

---
 octodns/record/dynamic.py            | 21 ++++++++++----
 tests/test_octodns_record_dynamic.py | 43 ++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+), 5 deletions(-)

diff --git a/octodns/record/dynamic.py b/octodns/record/dynamic.py
index 8423937..c3f807f 100644
--- a/octodns/record/dynamic.py
+++ b/octodns/record/dynamic.py
@@ -3,6 +3,7 @@
 #
 
 import re
+from collections import defaultdict
 from logging import getLogger
 
 from .change import Update
@@ -220,7 +221,7 @@ class _DynamicMixin(object):
         reasons = []
         pools_seen = set()
 
-        subnets_seen = {}
+        subnets_seen = defaultdict(dict)
         geos_seen = {}
 
         if not isinstance(rules, (list, tuple)):
@@ -291,11 +292,21 @@ class _DynamicMixin(object):
                             # previous loop will log any invalid subnets, here we
                             # process only valid ones and skip invalid ones
                             pass
-                    # sort subnets from largest to smallest so that we can
+
+                    # subnets of type IPv4 and IPv6 can't be sorted together
+                    # separately sort them and then combine them
+                    networks_by_type = defaultdict(list)
+                    for network in networks:
+                        networks_by_type[network.__class__].append(network)
+                    sorted_networks = []
+                    for _, networks_of_type in networks_by_type.items():
+                        sorted_networks.extend(sorted(networks_of_type))
+
                     # detect rule that have needlessly targeted a more specific
                     # subnet along with a larger subnet that already contains it
-                    for subnet in sorted(networks):
-                        for seen, where in subnets_seen.items():
+                    for subnet in sorted_networks:
+                        subnets_seen_of_type = subnets_seen[subnet.__class__]
+                        for seen, where in subnets_seen_of_type.items():
                             if subnet == seen:
                                 reasons.append(
                                     f'rule {rule_num} targets subnet {subnet} which has previously been seen in rule {where}'
@@ -305,7 +316,7 @@ class _DynamicMixin(object):
                                     f'rule {rule_num} targets subnet {subnet} which is more specific than the previously seen {seen} in rule {where}'
                                 )
 
-                        subnets_seen[subnet] = rule_num
+                        subnets_seen_of_type[subnet] = rule_num
 
                 if not isinstance(geos, (list, tuple)):
                     reasons.append(f'rule {rule_num} geos must be a list')
diff --git a/tests/test_octodns_record_dynamic.py b/tests/test_octodns_record_dynamic.py
index 589c7d0..22fe911 100644
--- a/tests/test_octodns_record_dynamic.py
+++ b/tests/test_octodns_record_dynamic.py
@@ -1561,3 +1561,46 @@ class TestRecordDynamic(TestCase):
                 ]
             ),
         )
+
+    def test_dynamic_subnet_mixed_versions(self):
+        # mixed IPv4 and IPv6 subnets should not raise a validation error
+        Record.new(
+            self.zone,
+            'good',
+            {
+                'dynamic': {
+                    'pools': {
+                        'one': {'values': [{'value': '1.1.1.1'}]},
+                        'two': {'values': [{'value': '2.2.2.2'}]},
+                    },
+                    'rules': [
+                        {'subnets': ['10.1.0.0/16', '1::/66'], 'pool': 'one'},
+                        {'pool': 'two'},
+                    ],
+                },
+                'ttl': 60,
+                'type': 'A',
+                'values': ['2.2.2.2'],
+            },
+        )
+
+        Record.new(
+            self.zone,
+            'good',
+            {
+                'dynamic': {
+                    'pools': {
+                        'one': {'values': [{'value': '1.1.1.1'}]},
+                        'two': {'values': [{'value': '2.2.2.2'}]},
+                    },
+                    'rules': [
+                        {'subnets': ['10.1.0.0/16'], 'pool': 'one'},
+                        {'subnets': ['1::/66'], 'pool': 'two'},
+                        {'pool': 'two'},
+                    ],
+                },
+                'ttl': 60,
+                'type': 'A',
+                'values': ['2.2.2.2'],
+            },
+        )

From a728a7ca4c5701fbe4203127e79e1e27db9904f6 Mon Sep 17 00:00:00 2001
From: Viranch Mehta <email@viranch.me>
Date: Sun, 24 Sep 2023 23:39:17 -0700
Subject: [PATCH 050/116] Use sorted's key to sort subnets by version

---
 octodns/record/dynamic.py | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/octodns/record/dynamic.py b/octodns/record/dynamic.py
index c3f807f..d261ebd 100644
--- a/octodns/record/dynamic.py
+++ b/octodns/record/dynamic.py
@@ -293,20 +293,15 @@ class _DynamicMixin(object):
                             # process only valid ones and skip invalid ones
                             pass
 
-                    # subnets of type IPv4 and IPv6 can't be sorted together
-                    # separately sort them and then combine them
-                    networks_by_type = defaultdict(list)
-                    for network in networks:
-                        networks_by_type[network.__class__].append(network)
-                    sorted_networks = []
-                    for _, networks_of_type in networks_by_type.items():
-                        sorted_networks.extend(sorted(networks_of_type))
-
+                    # sort subnets from largest to smallest so that we can
                     # detect rule that have needlessly targeted a more specific
                     # subnet along with a larger subnet that already contains it
+                    sorted_networks = sorted(
+                        networks, key=lambda n: (n.version, n)
+                    )
                     for subnet in sorted_networks:
-                        subnets_seen_of_type = subnets_seen[subnet.__class__]
-                        for seen, where in subnets_seen_of_type.items():
+                        subnets_seen_version = subnets_seen[subnet.version]
+                        for seen, where in subnets_seen_version.items():
                             if subnet == seen:
                                 reasons.append(
                                     f'rule {rule_num} targets subnet {subnet} which has previously been seen in rule {where}'
@@ -316,7 +311,7 @@ class _DynamicMixin(object):
                                     f'rule {rule_num} targets subnet {subnet} which is more specific than the previously seen {seen} in rule {where}'
                                 )
 
-                        subnets_seen_of_type[subnet] = rule_num
+                        subnets_seen_version[subnet] = rule_num
 
                 if not isinstance(geos, (list, tuple)):
                     reasons.append(f'rule {rule_num} geos must be a list')

From 76e330a7c373a1ba8f1552ba0ae289ca0e8b3055 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Mon, 25 Sep 2023 12:26:13 -0700
Subject: [PATCH 051/116] Add source parameter to Record.from_rrs

---
 octodns/record/base.py       | 6 ++++--
 tests/test_octodns_record.py | 5 ++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/octodns/record/base.py b/octodns/record/base.py
index a1b26b9..053d8dd 100644
--- a/octodns/record/base.py
+++ b/octodns/record/base.py
@@ -113,7 +113,7 @@ class Record(EqualityTupleMixin):
         return reasons
 
     @classmethod
-    def from_rrs(cls, zone, rrs, lenient=False):
+    def from_rrs(cls, zone, rrs, lenient=False, source=None):
         # group records by name & type so that multiple rdatas can be combined
         # into a single record when needed
         grouped = defaultdict(list)
@@ -128,7 +128,9 @@ class Record(EqualityTupleMixin):
             name = zone.hostname_from_fqdn(rr.name)
             _class = cls._CLASSES[rr._type]
             data = _class.data_from_rrs(rrs)
-            record = Record.new(zone, name, data, lenient=lenient)
+            record = Record.new(
+                zone, name, data, lenient=lenient, source=source
+            )
             records.append(record)
 
         return records
diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py
index 4aaa989..204b4d1 100644
--- a/tests/test_octodns_record.py
+++ b/tests/test_octodns_record.py
@@ -159,10 +159,13 @@ class TestRecord(TestCase):
         )
 
         zone = Zone('unit.tests.', [])
-        records = {(r._type, r.name): r for r in Record.from_rrs(zone, rrs)}
+        records = {
+            (r._type, r.name): r for r in Record.from_rrs(zone, rrs, source=99)
+        }
         record = records[('A', '')]
         self.assertEqual(42, record.ttl)
         self.assertEqual(['1.2.3.4', '2.3.4.5'], record.values)
+        self.assertEqual(99, record.source)
         record = records[('AAAA', '')]
         self.assertEqual(43, record.ttl)
         self.assertEqual(['fc00::1', 'fc00::2'], record.values)

From bca8db6c8f206ce1da581cebd1d7b8a7556280d1 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Mon, 25 Sep 2023 12:59:51 -0700
Subject: [PATCH 052/116] parse_rdata_text supports unquoting things (powerdns)

---
 octodns/record/base.py               |  6 ++++++
 octodns/record/caa.py                |  4 +++-
 octodns/record/loc.py                | 12 +++++++-----
 octodns/record/mx.py                 |  3 ++-
 octodns/record/naptr.py              |  6 +++++-
 octodns/record/srv.py                |  3 ++-
 octodns/record/sshfp.py              |  3 ++-
 octodns/record/tlsa.py               |  3 ++-
 tests/test_octodns_record.py         | 13 +++++++++++++
 tests/test_octodns_record_caa.py     |  6 ++++++
 tests/test_octodns_record_chunked.py |  2 ++
 tests/test_octodns_record_loc.py     | 22 +++++++++++++++++++++-
 tests/test_octodns_record_mx.py      |  6 ++++++
 tests/test_octodns_record_naptr.py   | 13 +++++++++++++
 tests/test_octodns_record_srv.py     | 11 +++++++++++
 tests/test_octodns_record_sshfp.py   |  6 ++++++
 tests/test_octodns_record_tlsa.py    | 11 +++++++++++
 17 files changed, 118 insertions(+), 12 deletions(-)

diff --git a/octodns/record/base.py b/octodns/record/base.py
index 053d8dd..16e6718 100644
--- a/octodns/record/base.py
+++ b/octodns/record/base.py
@@ -12,6 +12,12 @@ from .change import Update
 from .exception import RecordException, ValidationError
 
 
+def unquote(s):
+    if s and s[0] in ('"', "'"):
+        return s[1:-1]
+    return s
+
+
 class Record(EqualityTupleMixin):
     log = getLogger('Record')
 
diff --git a/octodns/record/caa.py b/octodns/record/caa.py
index 02e17cb..e95bb6b 100644
--- a/octodns/record/caa.py
+++ b/octodns/record/caa.py
@@ -3,7 +3,7 @@
 #
 
 from ..equality import EqualityTupleMixin
-from .base import Record, ValuesMixin
+from .base import Record, ValuesMixin, unquote
 from .rr import RrParseError
 
 
@@ -20,6 +20,8 @@ class CaaValue(EqualityTupleMixin, dict):
             flags = int(flags)
         except ValueError:
             pass
+        tag = unquote(tag)
+        value = unquote(value)
         return {'flags': flags, 'tag': tag, 'value': value}
 
     @classmethod
diff --git a/octodns/record/loc.py b/octodns/record/loc.py
index d9b06cd..7c55ecb 100644
--- a/octodns/record/loc.py
+++ b/octodns/record/loc.py
@@ -3,7 +3,7 @@
 #
 
 from ..equality import EqualityTupleMixin
-from .base import Record, ValuesMixin
+from .base import Record, ValuesMixin, unquote
 from .rr import RrParseError
 
 
@@ -58,21 +58,23 @@ class LocValue(EqualityTupleMixin, dict):
         except ValueError:
             pass
         try:
-            altitude = float(altitude)
+            altitude = float(unquote(altitude))
         except ValueError:
             pass
         try:
-            size = float(size)
+            size = float(unquote(size))
         except ValueError:
             pass
         try:
-            precision_horz = float(precision_horz)
+            precision_horz = float(unquote(precision_horz))
         except ValueError:
             pass
         try:
-            precision_vert = float(precision_vert)
+            precision_vert = float(unquote(precision_vert))
         except ValueError:
             pass
+        lat_direction = unquote(lat_direction)
+        long_direction = unquote(long_direction)
         return {
             'lat_degrees': lat_degrees,
             'lat_minutes': lat_minutes,
diff --git a/octodns/record/mx.py b/octodns/record/mx.py
index 77d34a7..d24aa97 100644
--- a/octodns/record/mx.py
+++ b/octodns/record/mx.py
@@ -6,7 +6,7 @@ from fqdn import FQDN
 
 from ..equality import EqualityTupleMixin
 from ..idna import idna_encode
-from .base import Record, ValuesMixin
+from .base import Record, ValuesMixin, unquote
 from .rr import RrParseError
 
 
@@ -21,6 +21,7 @@ class MxValue(EqualityTupleMixin, dict):
             preference = int(preference)
         except ValueError:
             pass
+        exchange = unquote(exchange)
         return {'preference': preference, 'exchange': exchange}
 
     @classmethod
diff --git a/octodns/record/naptr.py b/octodns/record/naptr.py
index 07b4fc0..14d541d 100644
--- a/octodns/record/naptr.py
+++ b/octodns/record/naptr.py
@@ -3,7 +3,7 @@
 #
 
 from ..equality import EqualityTupleMixin
-from .base import Record, ValuesMixin
+from .base import Record, ValuesMixin, unquote
 from .rr import RrParseError
 
 
@@ -28,6 +28,10 @@ class NaptrValue(EqualityTupleMixin, dict):
             preference = int(preference)
         except ValueError:
             pass
+        flags = unquote(flags)
+        service = unquote(service)
+        regexp = unquote(regexp)
+        replacement = unquote(replacement)
         return {
             'order': order,
             'preference': preference,
diff --git a/octodns/record/srv.py b/octodns/record/srv.py
index 33e76ce..058db79 100644
--- a/octodns/record/srv.py
+++ b/octodns/record/srv.py
@@ -8,7 +8,7 @@ from fqdn import FQDN
 
 from ..equality import EqualityTupleMixin
 from ..idna import idna_encode
-from .base import Record, ValuesMixin
+from .base import Record, ValuesMixin, unquote
 from .rr import RrParseError
 
 
@@ -31,6 +31,7 @@ class SrvValue(EqualityTupleMixin, dict):
             port = int(port)
         except ValueError:
             pass
+        target = unquote(target)
         return {
             'priority': priority,
             'weight': weight,
diff --git a/octodns/record/sshfp.py b/octodns/record/sshfp.py
index b3234df..d92cbd2 100644
--- a/octodns/record/sshfp.py
+++ b/octodns/record/sshfp.py
@@ -3,7 +3,7 @@
 #
 
 from ..equality import EqualityTupleMixin
-from .base import Record, ValuesMixin
+from .base import Record, ValuesMixin, unquote
 from .rr import RrParseError
 
 
@@ -25,6 +25,7 @@ class SshfpValue(EqualityTupleMixin, dict):
             fingerprint_type = int(fingerprint_type)
         except ValueError:
             pass
+        fingerprint = unquote(fingerprint)
         return {
             'algorithm': algorithm,
             'fingerprint_type': fingerprint_type,
diff --git a/octodns/record/tlsa.py b/octodns/record/tlsa.py
index 1fa463a..77a0ec5 100644
--- a/octodns/record/tlsa.py
+++ b/octodns/record/tlsa.py
@@ -3,7 +3,7 @@
 #
 
 from ..equality import EqualityTupleMixin
-from .base import Record, ValuesMixin
+from .base import Record, ValuesMixin, unquote
 from .rr import RrParseError
 
 
@@ -31,6 +31,7 @@ class TlsaValue(EqualityTupleMixin, dict):
             matching_type = int(matching_type)
         except ValueError:
             pass
+        certificate_association_data = unquote(certificate_association_data)
         return {
             'certificate_usage': certificate_usage,
             'selector': selector,
diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py
index 204b4d1..a6d27b7 100644
--- a/tests/test_octodns_record.py
+++ b/tests/test_octodns_record.py
@@ -22,6 +22,7 @@ from octodns.record import (
     ValidationError,
     ValuesMixin,
 )
+from octodns.record.base import unquote
 from octodns.yaml import ContextDict
 from octodns.zone import Zone
 
@@ -412,6 +413,18 @@ class TestRecord(TestCase):
             record.rrs,
         )
 
+    def test_unquote(self):
+        s = 'Hello "\'"World!'
+        single = f"'{s}'"
+        double = f'"{s}"'
+        self.assertEqual(s, unquote(s))
+        self.assertEqual(s, unquote(single))
+        self.assertEqual(s, unquote(double))
+
+        # edge cases
+        self.assertEqual(None, unquote(None))
+        self.assertEqual('', unquote(''))
+
 
 class TestRecordValidation(TestCase):
     zone = Zone('unit.tests.', [])
diff --git a/tests/test_octodns_record_caa.py b/tests/test_octodns_record_caa.py
index 17c70dd..d7c020b 100644
--- a/tests/test_octodns_record_caa.py
+++ b/tests/test_octodns_record_caa.py
@@ -105,6 +105,12 @@ class TestRecordCaa(TestCase):
             CaaValue.parse_rdata_text('0 tag 99148c81'),
         )
 
+        # quoted
+        self.assertEqual(
+            {'flags': 0, 'tag': 'tag', 'value': '99148c81'},
+            CaaValue.parse_rdata_text('0 "tag" "99148c81"'),
+        )
+
         zone = Zone('unit.tests.', [])
         a = CaaRecord(
             zone,
diff --git a/tests/test_octodns_record_chunked.py b/tests/test_octodns_record_chunked.py
index 2fa8c0d..ded2e9b 100644
--- a/tests/test_octodns_record_chunked.py
+++ b/tests/test_octodns_record_chunked.py
@@ -21,6 +21,8 @@ class TestRecordChunked(TestCase):
             'some.words.that.here',
             '1.2.word.4',
             '1.2.3.4',
+            # quotes are not removed
+            '"Hello World!"',
         ):
             self.assertEqual(s, _ChunkedValue.parse_rdata_text(s))
 
diff --git a/tests/test_octodns_record_loc.py b/tests/test_octodns_record_loc.py
index 4df81a4..278b816 100644
--- a/tests/test_octodns_record_loc.py
+++ b/tests/test_octodns_record_loc.py
@@ -160,6 +160,26 @@ class TestRecordLoc(TestCase):
             LocValue.parse_rdata_text(s),
         )
 
+        # quoted
+        s = '0 1 2.2 "N" 3 4 5.5 "E" "6.6m" "7.7m" "8.8m" "9.9m"'
+        self.assertEqual(
+            {
+                'altitude': 6.6,
+                'lat_degrees': 0,
+                'lat_direction': 'N',
+                'lat_minutes': 1,
+                'lat_seconds': 2.2,
+                'long_degrees': 3,
+                'long_direction': 'E',
+                'long_minutes': 4,
+                'long_seconds': 5.5,
+                'precision_horz': 8.8,
+                'precision_vert': 9.9,
+                'size': 7.7,
+            },
+            LocValue.parse_rdata_text(s),
+        )
+
         # make sure that the cstor is using parse_rdata_text
         zone = Zone('unit.tests.', [])
         a = LocRecord(
@@ -196,7 +216,7 @@ class TestRecordLoc(TestCase):
         self.assertEqual(7.7, a.values[0].size)
         self.assertEqual(8.8, a.values[0].precision_horz)
         self.assertEqual(9.9, a.values[0].precision_vert)
-        self.assertEqual(s, a.values[0].rdata_text)
+        self.assertEqual(s.replace('"', ''), a.values[0].rdata_text)
 
     def test_loc_value(self):
         a = LocValue(
diff --git a/tests/test_octodns_record_mx.py b/tests/test_octodns_record_mx.py
index 1ae37e6..a2fba19 100644
--- a/tests/test_octodns_record_mx.py
+++ b/tests/test_octodns_record_mx.py
@@ -92,6 +92,12 @@ class TestRecordMx(TestCase):
             MxValue.parse_rdata_text('10 mx.unit.tests.'),
         )
 
+        # quoted
+        self.assertEqual(
+            {'preference': 10, 'exchange': 'mx.unit.tests.'},
+            MxValue.parse_rdata_text('10 "mx.unit.tests."'),
+        )
+
         zone = Zone('unit.tests.', [])
         a = MxRecord(
             zone,
diff --git a/tests/test_octodns_record_naptr.py b/tests/test_octodns_record_naptr.py
index 55c3890..b099de4 100644
--- a/tests/test_octodns_record_naptr.py
+++ b/tests/test_octodns_record_naptr.py
@@ -346,6 +346,19 @@ class TestRecordNaptr(TestCase):
             NaptrValue.parse_rdata_text('1 2 three four five six'),
         )
 
+        # string fields are unquoted if needed
+        self.assertEqual(
+            {
+                'order': 1,
+                'preference': 2,
+                'flags': 'three',
+                'service': 'four',
+                'regexp': 'five',
+                'replacement': 'six',
+            },
+            NaptrValue.parse_rdata_text('1 2 "three" "four" "five" "six"'),
+        )
+
         # make sure that the cstor is using parse_rdata_text
         zone = Zone('unit.tests.', [])
         a = NaptrRecord(
diff --git a/tests/test_octodns_record_srv.py b/tests/test_octodns_record_srv.py
index e774dc5..e525afd 100644
--- a/tests/test_octodns_record_srv.py
+++ b/tests/test_octodns_record_srv.py
@@ -123,6 +123,17 @@ class TestRecordSrv(TestCase):
             SrvValue.parse_rdata_text('1 2 3 srv.unit.tests.'),
         )
 
+        # quoted
+        self.assertEqual(
+            {
+                'priority': 1,
+                'weight': 2,
+                'port': 3,
+                'target': 'srv.unit.tests.',
+            },
+            SrvValue.parse_rdata_text('1 2 3 "srv.unit.tests."'),
+        )
+
         zone = Zone('unit.tests.', [])
         a = SrvRecord(
             zone,
diff --git a/tests/test_octodns_record_sshfp.py b/tests/test_octodns_record_sshfp.py
index 251efca..4e66186 100644
--- a/tests/test_octodns_record_sshfp.py
+++ b/tests/test_octodns_record_sshfp.py
@@ -113,6 +113,12 @@ class TestRecordSshfp(TestCase):
             SshfpValue.parse_rdata_text('1 2 00479b27'),
         )
 
+        # valid
+        self.assertEqual(
+            {'algorithm': 1, 'fingerprint_type': 2, 'fingerprint': '00479b27'},
+            SshfpValue.parse_rdata_text('1 2 "00479b27"'),
+        )
+
         zone = Zone('unit.tests.', [])
         a = SshfpRecord(
             zone,
diff --git a/tests/test_octodns_record_tlsa.py b/tests/test_octodns_record_tlsa.py
index 9739017..26132e8 100644
--- a/tests/test_octodns_record_tlsa.py
+++ b/tests/test_octodns_record_tlsa.py
@@ -160,6 +160,17 @@ class TestRecordTlsa(TestCase):
             TlsaValue.parse_rdata_text('1 2 3 abcd'),
         )
 
+        # valid
+        self.assertEqual(
+            {
+                'certificate_usage': 1,
+                'selector': 2,
+                'matching_type': 3,
+                'certificate_association_data': 'abcd',
+            },
+            TlsaValue.parse_rdata_text('1 2 3 "abcd"'),
+        )
+
         zone = Zone('unit.tests.', [])
         a = TlsaRecord(
             zone,

From 6b681ba8f587d22d5d34a5a57dada92b860b91d0 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Mon, 25 Sep 2023 13:17:24 -0700
Subject: [PATCH 053/116] Changelog entry for RR improvements

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 09608f9..7ef7902 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+## v1.?.? - 2023-??-?? - ???
+
+* Record.from_rrs supports `source` parameter
+* *Record.parse_rdata_text unquotes any quoted (string) values
+
 ## v1.1.2 - 2023-09-20 - Bunch more bug fixes
 
 * Fix crash bug when using the YamlProvider with a directory that contains a

From e36f32a224e13b5a1621ee4311ff69a4810a1cf7 Mon Sep 17 00:00:00 2001
From: Viranch Mehta <email@viranch.me>
Date: Wed, 27 Sep 2023 11:58:30 -0700
Subject: [PATCH 054/116] v1.2.0 bump & changelog updates

---
 CHANGELOG.md        | 7 ++-----
 octodns/__init__.py | 2 +-
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7ef7902..58cd6ba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,10 +1,7 @@
-## v1.?.? - 2023-??-?? - ???
+## v1.2.0 - 2023-09-28 - Bunch more bug fixes
 
 * Record.from_rrs supports `source` parameter
-* *Record.parse_rdata_text unquotes any quoted (string) values
-
-## v1.1.2 - 2023-09-20 - Bunch more bug fixes
-
+* Record.parse_rdata_text unquotes any quoted (string) values
 * Fix crash bug when using the YamlProvider with a directory that contains a
   mix of split and non-split zone yamls. See https://github.com/octodns/octodns/issues/1066
 * Fix discovery of zones from different sources when there are multiple dynamic
diff --git a/octodns/__init__.py b/octodns/__init__.py
index 80c6e0e..75fe888 100644
--- a/octodns/__init__.py
+++ b/octodns/__init__.py
@@ -1,3 +1,3 @@
 'OctoDNS: DNS as code - Tools for managing DNS across multiple providers'
 
-__VERSION__ = '1.1.1'
+__VERSION__ = '1.2.0'

From 3953f1dea5c12422399aef175c2589592af08127 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 29 Sep 2023 12:53:35 -0700
Subject: [PATCH 055/116] Update script/release to do clean room dist builds

---
 script/release | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/script/release b/script/release
index b1c79fc..c100b44 100755
--- a/script/release
+++ b/script/release
@@ -37,7 +37,16 @@ VERSION="$(grep "^__VERSION__" "$ROOT/octodns/__init__.py" | sed -e "s/.* = '//"
 git tag -s "v$VERSION" -m "Release $VERSION"
 git push origin "v$VERSION"
 echo "Tagged and pushed v$VERSION"
-python -m build --sdist --wheel
+
+TMP_DIR=$(mktemp -d -t ci-XXXXXXXXXX)
+git archive --format tar v1.2.0 | tar xv -C $TMP_DIR
+echo "Created clean room $TMP_DIR and archived $VERSION into it"
+
+(cd "$TMP_DIR" && python -m build --sdist --wheel)
+
+cp $TMP_DIR/dist/*$VERSION.tar.gz $TMP_DIR/dist/*$VERSION*.whl dist/
+echo "Copied $TMP_DIR/dists into ./dist"
+
 twine check dist/*$VERSION.tar.gz dist/*$VERSION*.whl
 twine upload dist/*$VERSION.tar.gz dist/*$VERSION*.whl
 echo "Uploaded $VERSION"

From e2ddfa6e481cdce5f55c067f07c3a287771b0515 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 29 Sep 2023 13:01:39 -0700
Subject: [PATCH 056/116] Version from var, not copy-n-paste

---
 script/release | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/script/release b/script/release
index c100b44..ac89a3d 100755
--- a/script/release
+++ b/script/release
@@ -39,7 +39,7 @@ git push origin "v$VERSION"
 echo "Tagged and pushed v$VERSION"
 
 TMP_DIR=$(mktemp -d -t ci-XXXXXXXXXX)
-git archive --format tar v1.2.0 | tar xv -C $TMP_DIR
+git archive --format tar "v$VERSION" | tar xv -C $TMP_DIR
 echo "Created clean room $TMP_DIR and archived $VERSION into it"
 
 (cd "$TMP_DIR" && python -m build --sdist --wheel)

From 32bf55d947970d40af1772675ad80d021c4e7a41 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 29 Sep 2023 13:08:36 -0700
Subject: [PATCH 057/116] v1.2.1 version bump & changelog update

---
 CHANGELOG.md        | 4 ++++
 octodns/__init__.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 58cd6ba..6c4dff5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## v1.2.1 - 2023-09-29 - Now with fewer stale files
+
+* Update script/release to do clean room dist builds
+
 ## v1.2.0 - 2023-09-28 - Bunch more bug fixes
 
 * Record.from_rrs supports `source` parameter
diff --git a/octodns/__init__.py b/octodns/__init__.py
index 75fe888..e106d6c 100644
--- a/octodns/__init__.py
+++ b/octodns/__init__.py
@@ -1,3 +1,3 @@
 'OctoDNS: DNS as code - Tools for managing DNS across multiple providers'
 
-__VERSION__ = '1.2.0'
+__VERSION__ = '1.2.1'

From 519af5b973796448d4ff96d6ce7f13107419c96a Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 29 Sep 2023 13:28:45 -0700
Subject: [PATCH 058/116] Rework dynamic documentation to call using the
 default pool a backup rather than fallback

---
 docs/dynamic_records.md | 55 ++++++++++++++++++++++-------------------
 1 file changed, 29 insertions(+), 26 deletions(-)

diff --git a/docs/dynamic_records.md b/docs/dynamic_records.md
index 1a4dfd1..e3e2faf 100644
--- a/docs/dynamic_records.md
+++ b/docs/dynamic_records.md
@@ -30,7 +30,8 @@ test:
         - value: 4.4.4.4
           weight: 3
       na:
-        # Implicit fallback to the default pool (below)
+        # Implicitly goes to the backup pool (below) if all values are failing
+        # health checks
         values:
         - value: 5.5.5.5
         - value: 6.6.6.6
@@ -59,11 +60,13 @@ test:
     - pool: na
   ttl: 60
   type: A
-  # These values become a non-healthchecked default pool, generally it should be
-  # a superset of the catch-all pool and include enough capacity to try and
-  # serve all global requests (with degraded performance.) The main case they
-  # will come into play is if all dynamic healthchecks are failing, either on
-  # the service side or if the providers systems are expeiencing problems.
+  # These values become a non-healthchecked backup/default pool, generally it
+  # should be a superset of the catch-all pool and include enough capacity to
+  # try and serve all global requests (with degraded performance.) The main
+  # case they will come into play is if all dynamic healthchecks are failing,
+  # either on the service side or if the providers systems are expeiencing
+  # problems. They will also be used for when the record is pushed to a
+  # provider that doesn't support dynamic records.
   values:
   - 3.3.3.3
   - 4.4.4.4
@@ -81,26 +84,26 @@ If you encounter validation errors in dynamic records suggesting best practices
 title: Visual Representation of the Rules and Pools
 ---
 flowchart LR
-  	query((Query)) --> rule_0[Rule 0<br>AF-ZA<br>AS<br>OC]
-	rule_0 --no match--> rule_1[Rule 1<br>AF<br>EU]
-	rule_1 --no match--> rule_2["Rule 2<br>(catch all)"]
-
-	rule_0 --match--> pool_apac[Pool apac<br>1.1.1.1<br>2.2.2.2]
-	pool_apac --fallback--> pool_na
-	rule_1 --match--> pool_eu["Pool eu<br>3.3.3.3 (2/5)<br>4.4.4.4 (3/5)"]
-	pool_eu --fallback--> pool_na
-	rule_2 --> pool_na[Pool na<br>5.5.5.5<br>6.6.6.6<br>7.7.7.7]
-	pool_na --fallback--> values[values<br>3.3.3.3<br>4.4.4.4<br>5.5.5.5<br>6.6.6.6<br>7.7.7.7]
-
-	classDef queryColor fill:#3B67A8,color:#ffffff
-	classDef ruleColor fill:#D8F57A,color:#000000
-	classDef poolColor fill:#F57261,color:#000000
-	classDef valueColor fill:#498FF5,color:#000000
-
-	class query queryColor
-	class rule_0,rule_1,rule_2 ruleColor
-	class pool_apac,pool_eu,pool_na poolColor
-	class values valueColor
+    query((Query)) --> rule_0[Rule 0<br>AF-ZA<br>AS<br>OC]
+    rule_0 --no match--> rule_1[Rule 1<br>AF<br>EU]
+    rule_1 --no match--> rule_2["Rule 2<br>(catch all)"]
+
+    rule_0 --match--> pool_apac[Pool apac<br>1.1.1.1<br>2.2.2.2]
+    pool_apac --fallback--> pool_na
+    rule_1 --match--> pool_eu["Pool eu<br>3.3.3.3 (2/5)<br>4.4.4.4 (3/5)"]
+    pool_eu --fallback--> pool_na
+    rule_2 --> pool_na[Pool na<br>5.5.5.5<br>6.6.6.6<br>7.7.7.7]
+    pool_na --backup--> values[values<br>3.3.3.3<br>4.4.4.4<br>5.5.5.5<br>6.6.6.6<br>7.7.7.7]
+
+    classDef queryColor fill:#3B67A8,color:#ffffff
+    classDef ruleColor fill:#D8F57A,color:#000000
+    classDef poolColor fill:#F57261,color:#000000
+    classDef valueColor fill:#498FF5,color:#000000
+
+    class query queryColor
+    class rule_0,rule_1,rule_2 ruleColor
+    class pool_apac,pool_eu,pool_na poolColor
+    class values valueColor
 ```
 
 

From b61d4d53cbac1a7660f21bc896c11a27ae571366 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Wed, 4 Oct 2023 10:48:23 -0700
Subject: [PATCH 059/116] update requirements*.txt

---
 requirements-dev.txt | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index 40f2926..c0c68ce 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,16 +1,14 @@
 # DO NOT EDIT THIS FILE DIRECTLY - use ./script/update-requirements to update
 Pygments==2.16.1
-black==23.7.0
-bleach==6.0.0
-build==0.10.0
+black==23.9.1
+build==1.0.3
 certifi==2023.7.22
-cffi==1.15.1
-charset-normalizer==3.2.0
-click==8.1.6
+cffi==1.16.0
+charset-normalizer==3.3.0
+click==8.1.7
 cmarkgfm==2022.10.27
-coverage==7.3.0
+coverage==7.3.2
 docutils==0.20.1
-exceptiongroup==1.1.3
 importlib-metadata==6.8.0
 iniconfig==2.0.0
 isort==5.12.0
@@ -20,11 +18,12 @@ markdown-it-py==3.0.0
 mdurl==0.1.2
 more-itertools==10.1.0
 mypy-extensions==1.0.0
-packaging==23.1
+nh3==0.2.14
+packaging==23.2
 pathspec==0.11.2
 pkginfo==1.9.6
-platformdirs==3.10.0
-pluggy==1.2.0
+platformdirs==3.11.0
+pluggy==1.3.0
 pprintpp==0.4.0
 pycountry-convert==0.7.2
 pycountry==22.3.5
@@ -34,15 +33,13 @@ pyproject_hooks==1.0.0
 pytest-cov==4.1.0
 pytest-mock==3.11.1
 pytest-network==0.0.1
-pytest==7.4.0
-readme-renderer==40.0
+pytest==7.4.2
+readme-renderer==42.0
 repoze.lru==0.7
 requests-toolbelt==1.0.0
 requests==2.31.0
 rfc3986==2.0.0
-rich==13.5.2
-tomli==2.0.1
+rich==13.6.0
 twine==4.0.2
-urllib3==2.0.4
-webencodings==0.5.1
-zipp==3.16.2
+urllib3==2.0.6
+zipp==3.17.0

From 0d2953fd06866d17cd5c112c55a498047917d766 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 5 Oct 2023 11:05:52 -0700
Subject: [PATCH 060/116] CI pulls python versions from org secret

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index e96a793..5991f0c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -8,7 +8,7 @@ jobs:
       fail-fast: false
       matrix:
         # Tested versions based on dates in https://devguide.python.org/versions/#versions
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: [${{fromJson(secrets.PYTHON_VERSIONS_ACTIVE)}}]
     steps:
       - uses: actions/checkout@v2
       - name: Setup python

From 752e9742454c96ed89758906f13e3d2d705c4003 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 5 Oct 2023 11:07:52 -0700
Subject: [PATCH 061/116] maybe no bracket

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 5991f0c..d3d4c38 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -8,7 +8,7 @@ jobs:
       fail-fast: false
       matrix:
         # Tested versions based on dates in https://devguide.python.org/versions/#versions
-        python-version: [${{fromJson(secrets.PYTHON_VERSIONS_ACTIVE)}}]
+        python-version: ${{fromJson(secrets.PYTHON_VERSIONS_ACTIVE)}}
     steps:
       - uses: actions/checkout@v2
       - name: Setup python

From 7a238898098544a75b982da9af310fb7687c87e4 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 5 Oct 2023 13:14:44 -0700
Subject: [PATCH 062/116] try an env var instead

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index d3d4c38..7882eef 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -8,7 +8,7 @@ jobs:
       fail-fast: false
       matrix:
         # Tested versions based on dates in https://devguide.python.org/versions/#versions
-        python-version: ${{fromJson(secrets.PYTHON_VERSIONS_ACTIVE)}}
+        python-version: ${{fromJson(env.PYTHON_VERSIONS_ACTIVE)}}
     steps:
       - uses: actions/checkout@v2
       - name: Setup python

From 90764b12a25714589a2efd9720698e8d25e02bb8 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 5 Oct 2023 13:27:29 -0700
Subject: [PATCH 063/116] is it vars

---
 .github/workflows/main.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 7882eef..81fffc8 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -7,8 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Tested versions based on dates in https://devguide.python.org/versions/#versions
-        python-version: ${{fromJson(env.PYTHON_VERSIONS_ACTIVE)}}
+        python-version: ${{fromJson(vars.PYTHON_VERSIONS_ACTIVE)}}
     steps:
       - uses: actions/checkout@v2
       - name: Setup python

From 4ace61de54f26108bb8bc15e8855d2b51e47589e Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 5 Oct 2023 13:29:12 -0700
Subject: [PATCH 064/116] Restore/update comment about active versions

---
 .github/workflows/main.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 81fffc8..8363b87 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -7,6 +7,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # Defined in an org level variable, based on dates in
+        # https://devguide.python.org/versions/#versions
         python-version: ${{fromJson(vars.PYTHON_VERSIONS_ACTIVE)}}
     steps:
       - uses: actions/checkout@v2

From 9f61445a0634e4addeb4e6072b7a7af6c9d0f3fb Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 5 Oct 2023 13:35:37 -0700
Subject: [PATCH 065/116] Drive modules ci job python version off of var too

---
 .github/workflows/modules.yml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/modules.yml b/.github/workflows/modules.yml
index e084a7b..2b6dac5 100644
--- a/.github/workflows/modules.yml
+++ b/.github/workflows/modules.yml
@@ -46,11 +46,7 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v1
         with:
-          # This should generally be the latest stable release of python, but
-          # dyn and ovh don't currently support changes made in 3.10 so we'll
-          # leave it 3.9 for now. Once 3.11 lands though we'll bump to it and
-          # if they haven't updated they'll be removed from the matrix
-          python-version: '3.9'
+          python-version: ${{ vars.PYTHON_VERSION_CURRENT }}
           architecture: x64
       - name: Test Module
         run: |

From cf258cc5a7d1bd20208102b72cdd91eac6fd3d14 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 5 Oct 2023 13:52:21 -0700
Subject: [PATCH 066/116] bump checkout and python action versions

---
 .github/workflows/main.yml    | 4 ++--
 .github/workflows/modules.yml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 8363b87..a3b4e3c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -11,9 +11,9 @@ jobs:
         # https://devguide.python.org/versions/#versions
         python-version: ${{fromJson(vars.PYTHON_VERSIONS_ACTIVE)}}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Setup python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
diff --git a/.github/workflows/modules.yml b/.github/workflows/modules.yml
index 2b6dac5..940abc1 100644
--- a/.github/workflows/modules.yml
+++ b/.github/workflows/modules.yml
@@ -42,9 +42,9 @@ jobs:
           # some point in the future and either re-enable or delete it.
           #- sukiyaki/octodns-netbox
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v4
       - name: Setup python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ vars.PYTHON_VERSION_CURRENT }}
           architecture: x64

From 1c72e75d93b798f4d05dfa1d1c63eca277de3901 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 5 Oct 2023 14:18:42 -0700
Subject: [PATCH 067/116] setup-py python version from var as well

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a3b4e3c..f00534c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -42,7 +42,7 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.11'
+          python-version: ${{ vars.PYTHON_VERSION_CURRENT }}
           architecture: x64
       - name: CI setup.py
         run: |

From f0c29082790aa02af393dc4efc390f20d4ae2a61 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 5 Oct 2023 14:29:48 -0700
Subject: [PATCH 068/116] bump stale to v8 too

---
 .github/workflows/stale.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 0bea4de..d595205 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -6,7 +6,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v4
+      - uses: actions/stale@v8
         with:
           stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
           days-before-stale: 90

From 34a2c55d36d8273a212a2dd3277328f964862c7e Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 6 Oct 2023 11:55:08 -0700
Subject: [PATCH 069/116] missed a couple of action version udpates

---
 .github/workflows/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index f00534c..079730d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -38,9 +38,9 @@ jobs:
   setup-py:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Setup python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ vars.PYTHON_VERSION_CURRENT }}
           architecture: x64

From eec7cadb8664a446b741f94590fcfbd20ba36cfc Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 6 Oct 2023 15:10:47 -0700
Subject: [PATCH 070/116] refactor filter based processors to pull out shared
 logic

---
 octodns/processor/filter.py | 117 ++++++++++++++++++------------------
 1 file changed, 57 insertions(+), 60 deletions(-)

diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py
index f661dbf..078fcf5 100644
--- a/octodns/processor/filter.py
+++ b/octodns/processor/filter.py
@@ -7,7 +7,41 @@ from re import compile as re_compile
 from .base import BaseProcessor
 
 
-class TypeAllowlistFilter(BaseProcessor):
+class AllowsMixin:
+    def matches(self, zone, record):
+        pass
+
+    def doesnt_match(self, zone, record):
+        zone.remove_record(record)
+
+
+class RejectsMixin:
+    def matches(self, zone, record):
+        zone.remove_record(record)
+
+    def doesnt_match(self, zone, record):
+        pass
+
+
+class _TypeBaseFilter(BaseProcessor):
+    def __init__(self, name, _list):
+        super().__init__(name)
+        self._list = set(_list)
+
+    def _process(self, zone, *args, **kwargs):
+        for record in zone.records:
+            if record._type in self._list:
+                self.matches(zone, record)
+            else:
+                self.doesnt_match(zone, record)
+
+        return zone
+
+    process_source_zone = _process
+    process_target_zone = _process
+
+
+class TypeAllowlistFilter(_TypeBaseFilter, AllowsMixin):
     '''Only manage records of the specified type(s).
 
     Example usage:
@@ -30,21 +64,10 @@ class TypeAllowlistFilter(BaseProcessor):
     '''
 
     def __init__(self, name, allowlist):
-        super().__init__(name)
-        self.allowlist = set(allowlist)
-
-    def _process(self, zone, *args, **kwargs):
-        for record in zone.records:
-            if record._type not in self.allowlist:
-                zone.remove_record(record)
-
-        return zone
-
-    process_source_zone = _process
-    process_target_zone = _process
+        super().__init__(name, allowlist)
 
 
-class TypeRejectlistFilter(BaseProcessor):
+class TypeRejectlistFilter(_TypeBaseFilter, RejectsMixin):
     '''Ignore records of the specified type(s).
 
     Example usage:
@@ -66,18 +89,7 @@ class TypeRejectlistFilter(BaseProcessor):
     '''
 
     def __init__(self, name, rejectlist):
-        super().__init__(name)
-        self.rejectlist = set(rejectlist)
-
-    def _process(self, zone, *args, **kwargs):
-        for record in zone.records:
-            if record._type in self.rejectlist:
-                zone.remove_record(record)
-
-        return zone
-
-    process_source_zone = _process
-    process_target_zone = _process
+        super().__init__(name, rejectlist)
 
 
 class _NameBaseFilter(BaseProcessor):
@@ -93,8 +105,25 @@ class _NameBaseFilter(BaseProcessor):
         self.exact = exact
         self.regex = regex
 
+    def _process(self, zone, *args, **kwargs):
+        for record in zone.records:
+            name = record.name
+            if name in self.exact:
+                self.matches(zone, record)
+                continue
+            elif any(r.search(name) for r in self.regex):
+                self.matches(zone, record)
+                continue
+
+            self.doesnt_match(zone, record)
 
-class NameAllowlistFilter(_NameBaseFilter):
+        return zone
+
+    process_source_zone = _process
+    process_target_zone = _process
+
+
+class NameAllowlistFilter(_NameBaseFilter, AllowsMixin):
     '''Only manage records with names that match the provider patterns
 
     Example usage:
@@ -125,23 +154,8 @@ class NameAllowlistFilter(_NameBaseFilter):
     def __init__(self, name, allowlist):
         super().__init__(name, allowlist)
 
-    def _process(self, zone, *args, **kwargs):
-        for record in zone.records:
-            name = record.name
-            if name in self.exact:
-                continue
-            elif any(r.search(name) for r in self.regex):
-                continue
 
-            zone.remove_record(record)
-
-        return zone
-
-    process_source_zone = _process
-    process_target_zone = _process
-
-
-class NameRejectlistFilter(_NameBaseFilter):
+class NameRejectlistFilter(_NameBaseFilter, RejectsMixin):
     '''Reject managing records with names that match the provider patterns
 
     Example usage:
@@ -172,23 +186,6 @@ class NameRejectlistFilter(_NameBaseFilter):
     def __init__(self, name, rejectlist):
         super().__init__(name, rejectlist)
 
-    def _process(self, zone, *args, **kwargs):
-        for record in zone.records:
-            name = record.name
-            if name in self.exact:
-                zone.remove_record(record)
-                continue
-
-            for regex in self.regex:
-                if regex.search(name):
-                    zone.remove_record(record)
-                    break
-
-        return zone
-
-    process_source_zone = _process
-    process_target_zone = _process
-
 
 class IgnoreRootNsFilter(BaseProcessor):
     '''Do not manage Root NS Records.

From da818d12e43c7f35d9183e7d4d93a1d1a0d7ae59 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 6 Oct 2023 16:14:38 -0700
Subject: [PATCH 071/116] ZoneNameFilter to error/ignore when record names end
 with the zone name

---
 CHANGELOG.md                           |  5 ++
 octodns/processor/filter.py            | 51 +++++++++++++++++++
 tests/test_octodns_processor_filter.py | 69 ++++++++++++++++++++++++++
 3 files changed, 125 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6c4dff5..d0ab5d2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+## v1.3.0 - 2023-??-?? - ???
+
+* Added ZoneNameFilter processor to enable ignoring/alerting on type-os like
+  octodns.com.octodns.com
+
 ## v1.2.1 - 2023-09-29 - Now with fewer stale files
 
 * Update script/release to do clean room dist builds
diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py
index 078fcf5..e02bc84 100644
--- a/octodns/processor/filter.py
+++ b/octodns/processor/filter.py
@@ -4,6 +4,7 @@
 
 from re import compile as re_compile
 
+from ..record.exception import ValidationError
 from .base import BaseProcessor
 
 
@@ -215,3 +216,53 @@ class IgnoreRootNsFilter(BaseProcessor):
 
     process_source_zone = _process
     process_target_zone = _process
+
+
+class ZoneNameFilter(BaseProcessor):
+    '''Filter or error on record names that contain the zone name
+
+    Example usage:
+
+    processors:
+      zone-name:
+        class: octodns.processor.filter.ZoneNameFilter
+        # If true a ValidationError will be throw when such records are
+        # encouterd, if false the records will just be ignored/omitted.
+        # (default: false)
+
+    zones:
+      exxampled.com.:
+        sources:
+          - config
+        processors:
+          - zone-name
+        targets:
+          - azure
+    '''
+
+    def __init__(self, name, error=False):
+        super().__init__(name)
+        self.error = error
+
+    def _process(self, zone, *args, **kwargs):
+        zone_name_with_dot = zone.name
+        zone_name_without_dot = zone_name_with_dot[:-1]
+        for record in zone.records:
+            name = record.name
+            if name.endswith(zone_name_with_dot) or name.endswith(
+                zone_name_without_dot
+            ):
+                if self.error:
+                    raise ValidationError(
+                        record.fqdn,
+                        ['record name ends with zone name'],
+                        record.context,
+                    )
+                else:
+                    # just remove it
+                    zone.remove_record(record)
+
+        return zone
+
+    process_source_zone = _process
+    process_target_zone = _process
diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py
index a18eb51..0a1b6dd 100644
--- a/tests/test_octodns_processor_filter.py
+++ b/tests/test_octodns_processor_filter.py
@@ -10,8 +10,10 @@ from octodns.processor.filter import (
     NameRejectlistFilter,
     TypeAllowlistFilter,
     TypeRejectlistFilter,
+    ZoneNameFilter,
 )
 from octodns.record import Record
+from octodns.record.exception import ValidationError
 from octodns.zone import Zone
 
 zone = Zone('unit.tests.', [])
@@ -180,3 +182,70 @@ class TestIgnoreRootNsFilter(TestCase):
             [('A', ''), ('NS', 'sub')],
             sorted([(r._type, r.name) for r in filtered.records]),
         )
+
+
+class TestZoneNameFilter(TestCase):
+    def test_ends_with_zone(self):
+        zone_name_filter = ZoneNameFilter('zone-name')
+
+        zone = Zone('unit.tests.', [])
+
+        # something that doesn't come into play
+        zone.add_record(
+            Record.new(
+                zone, 'www', {'type': 'A', 'ttl': 43, 'value': '1.2.3.4'}
+            )
+        )
+
+        # something that has the zone name, but doesn't end with it
+        zone.add_record(
+            Record.new(
+                zone,
+                f'{zone.name}more',
+                {'type': 'A', 'ttl': 43, 'value': '1.2.3.4'},
+            )
+        )
+
+        self.assertEqual(2, len(zone.records))
+        filtered = zone_name_filter.process_source_zone(zone.copy())
+        # get everything back
+        self.assertEqual(2, len(filtered.records))
+
+        with_dot = zone.copy()
+        with_dot.add_record(
+            Record.new(
+                zone, zone.name, {'type': 'A', 'ttl': 43, 'value': '1.2.3.4'}
+            )
+        )
+        self.assertEqual(3, len(with_dot.records))
+        filtered = zone_name_filter.process_source_zone(with_dot.copy())
+        # don't get the one that ends with the zone name
+        self.assertEqual(2, len(filtered.records))
+
+        without_dot = zone.copy()
+        without_dot.add_record(
+            Record.new(
+                zone,
+                zone.name[:-1],
+                {'type': 'A', 'ttl': 43, 'value': '1.2.3.4'},
+            )
+        )
+        self.assertEqual(3, len(without_dot.records))
+        filtered = zone_name_filter.process_source_zone(without_dot.copy())
+        # don't get the one that ends with the zone name
+        self.assertEqual(2, len(filtered.records))
+
+    def test_error(self):
+        errors = ZoneNameFilter('zone-name', error=True)
+
+        zone = Zone('unit.tests.', [])
+        zone.add_record(
+            Record.new(
+                zone, zone.name, {'type': 'A', 'ttl': 43, 'value': '1.2.3.4'}
+            )
+        )
+        with self.assertRaises(ValidationError) as ctx:
+            errors.process_source_zone(zone)
+        self.assertEqual(
+            ['record name ends with zone name'], ctx.exception.reasons
+        )

From e6ad64f25f59e32f189a0eae6de8c96f4822cb80 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 6 Oct 2023 16:16:48 -0700
Subject: [PATCH 072/116] ZoneNameFilter error defaults to True

---
 octodns/processor/filter.py            | 4 ++--
 tests/test_octodns_processor_filter.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py
index e02bc84..4723073 100644
--- a/octodns/processor/filter.py
+++ b/octodns/processor/filter.py
@@ -228,7 +228,7 @@ class ZoneNameFilter(BaseProcessor):
         class: octodns.processor.filter.ZoneNameFilter
         # If true a ValidationError will be throw when such records are
         # encouterd, if false the records will just be ignored/omitted.
-        # (default: false)
+        # (default: true)
 
     zones:
       exxampled.com.:
@@ -240,7 +240,7 @@ class ZoneNameFilter(BaseProcessor):
           - azure
     '''
 
-    def __init__(self, name, error=False):
+    def __init__(self, name, error=True):
         super().__init__(name)
         self.error = error
 
diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py
index 0a1b6dd..9857926 100644
--- a/tests/test_octodns_processor_filter.py
+++ b/tests/test_octodns_processor_filter.py
@@ -186,7 +186,7 @@ class TestIgnoreRootNsFilter(TestCase):
 
 class TestZoneNameFilter(TestCase):
     def test_ends_with_zone(self):
-        zone_name_filter = ZoneNameFilter('zone-name')
+        zone_name_filter = ZoneNameFilter('zone-name', error=False)
 
         zone = Zone('unit.tests.', [])
 

From 6b9596d4b2e77f66abcb7545a58f30d57915a7fe Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 7 Oct 2023 08:51:27 -0700
Subject: [PATCH 073/116] Try pulling python versions from a json file

---
 .github/workflows/main.yml | 19 +++++++++++++++----
 .python-versions.json      |  4 ++++
 2 files changed, 19 insertions(+), 4 deletions(-)
 create mode 100644 .python-versions.json

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 079730d..580cca5 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -2,14 +2,24 @@ name: OctoDNS
 on: [pull_request]
 
 jobs:
+  config:
+    runs-on: ubuntu-latest
+    outputs:
+      json: ${{ steps.load.outputs.json }}
+    steps:
+    - id: load
+      run: |
+        JSON=$(cat ./.python-versions.json)
+        echo "::set-output name=json::${JSON}"
   ci:
+    needs: config
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        # Defined in an org level variable, based on dates in
-        # https://devguide.python.org/versions/#versions
-        python-version: ${{fromJson(vars.PYTHON_VERSIONS_ACTIVE)}}
+        # Defined in a file that resides in the top level of octodns/octodns,
+        # based on dates in https://devguide.python.org/versions/#versions
+        python-version: ${{ fromJson(needs.config.outputs.json).python_versions_active }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup python
@@ -42,7 +52,8 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ vars.PYTHON_VERSION_CURRENT }}
+          # Most recent release from https://devguide.python.org/versions/#versions
+          python-version: ${{ fromJson(needs.config.outputs.json).python_version_current }}
           architecture: x64
       - name: CI setup.py
         run: |
diff --git a/.python-versions.json b/.python-versions.json
new file mode 100644
index 0000000..0e8eac4
--- /dev/null
+++ b/.python-versions.json
@@ -0,0 +1,4 @@
+{
+  "python_version_current": "3.11",
+  "python_versions_active": ["3.8", "3.9", "3.10", "3.11"]
+}

From 32395331dcb4fb7938f68a704be874559153ae70 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 7 Oct 2023 08:59:15 -0700
Subject: [PATCH 074/116] Guess we have to checkout before we have access to
 files?

---
 .github/workflows/main.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 580cca5..30a8b53 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -8,6 +8,7 @@ jobs:
       json: ${{ steps.load.outputs.json }}
     steps:
     - id: load
+      uses: actions/checkout@v4
       run: |
         JSON=$(cat ./.python-versions.json)
         echo "::set-output name=json::${JSON}"

From 75d8ec53f95bf77c24bee6e138d031fe93a08091 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 7 Oct 2023 08:59:31 -0700
Subject: [PATCH 075/116] setup-py needs config too

---
 .github/workflows/main.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 30a8b53..9c5113a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -47,6 +47,7 @@ jobs:
             coverage.xml
             htmlcov
   setup-py:
+    needs: config
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

From 97e46183134c33b803adba7e67c80ff6bf06e4ef Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 7 Oct 2023 09:00:48 -0700
Subject: [PATCH 076/116] checkout needs to be its own step

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 9c5113a..f0dc707 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -7,8 +7,8 @@ jobs:
     outputs:
       json: ${{ steps.load.outputs.json }}
     steps:
+    - uses: actions/checkout@v4
     - id: load
-      uses: actions/checkout@v4
       run: |
         JSON=$(cat ./.python-versions.json)
         echo "::set-output name=json::${JSON}"

From 97d525df5b2be3008930fca67ce89a2b04f1d66c Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 7 Oct 2023 09:14:48 -0700
Subject: [PATCH 077/116] remove newlines from json

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index f0dc707..73b0344 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -11,7 +11,7 @@ jobs:
     - id: load
       run: |
         JSON=$(cat ./.python-versions.json)
-        echo "::set-output name=json::${JSON}"
+        echo "::set-output name=json::${JSON//'%'/'%25'}"
   ci:
     needs: config
     runs-on: ubuntu-latest

From 973fbd601e08508ce135ad33522cdd66ca24efc7 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 7 Oct 2023 09:25:23 -0700
Subject: [PATCH 078/116] try another approach for multi-line json in an output

---
 .github/workflows/main.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 73b0344..36608b1 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -9,9 +9,13 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - id: load
+      # based on https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
       run: |
-        JSON=$(cat ./.python-versions.json)
-        echo "::set-output name=json::${JSON//'%'/'%25'}"
+        {
+          echo 'json<<EOF'
+          cat ./.python-versions.json
+          echo EOF
+        } >> $GITHUB_OUTPUT
   ci:
     needs: config
     runs-on: ubuntu-latest

From 32c0b268c1da478c6c5860be1280eccb1fcc7485 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 7 Oct 2023 09:30:17 -0700
Subject: [PATCH 079/116] have modules ci job use config for python-version too

---
 .github/workflows/modules.yml | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/modules.yml b/.github/workflows/modules.yml
index 940abc1..1828863 100644
--- a/.github/workflows/modules.yml
+++ b/.github/workflows/modules.yml
@@ -5,7 +5,22 @@ on:
     types: [submitted]
 
 jobs:
+  config:
+    runs-on: ubuntu-latest
+    outputs:
+      json: ${{ steps.load.outputs.json }}
+    steps:
+    - uses: actions/checkout@v4
+    - id: load
+      # based on https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
+      run: |
+        {
+          echo 'json<<EOF'
+          cat ./.python-versions.json
+          echo EOF
+        } >> $GITHUB_OUTPUT
   ci:
+    needs: config
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -46,7 +61,7 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ vars.PYTHON_VERSION_CURRENT }}
+          python-version: ${{ fromJson(needs.config.outputs.json).python_version_current }}
           architecture: x64
       - name: Test Module
         run: |

From 9bd76c81164a576a913949da8c9af9bb77db864e Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 7 Oct 2023 09:34:46 -0700
Subject: [PATCH 080/116] more generic ci config json name

---
 .python-versions.json => .ci-config.json | 0
 .github/workflows/main.yml               | 2 +-
 .github/workflows/modules.yml            | 2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename .python-versions.json => .ci-config.json (100%)

diff --git a/.python-versions.json b/.ci-config.json
similarity index 100%
rename from .python-versions.json
rename to .ci-config.json
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 36608b1..32d3181 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -13,7 +13,7 @@ jobs:
       run: |
         {
           echo 'json<<EOF'
-          cat ./.python-versions.json
+          cat ./.ci-config.json
           echo EOF
         } >> $GITHUB_OUTPUT
   ci:
diff --git a/.github/workflows/modules.yml b/.github/workflows/modules.yml
index 1828863..9d6e767 100644
--- a/.github/workflows/modules.yml
+++ b/.github/workflows/modules.yml
@@ -16,7 +16,7 @@ jobs:
       run: |
         {
           echo 'json<<EOF'
-          cat ./.python-versions.json
+          cat ./.ci-config.json
           echo EOF
         } >> $GITHUB_OUTPUT
   ci:

From c30e21f7e073ee7959c931c5f9128dd475b769b7 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 7 Oct 2023 13:06:46 -0700
Subject: [PATCH 081/116] Add Python 3.12 to the ci test matrix and make it the
 current version

---
 .ci-config.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci-config.json b/.ci-config.json
index 0e8eac4..0e28add 100644
--- a/.ci-config.json
+++ b/.ci-config.json
@@ -1,4 +1,4 @@
 {
-  "python_version_current": "3.11",
-  "python_versions_active": ["3.8", "3.9", "3.10", "3.11"]
+  "python_version_current": "3.12",
+  "python_versions_active": ["3.8", "3.9", "3.10", "3.11", "3.12"]
 }

From 8be2732de442818408ceeed38ac6251e86b4a4ec Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 7 Oct 2023 13:29:35 -0700
Subject: [PATCH 082/116] setuptools is no longer in venvs by default -
 gh-95299

---
 script/cibuild-setup-py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/script/cibuild-setup-py b/script/cibuild-setup-py
index 49f8409..31d1d32 100755
--- a/script/cibuild-setup-py
+++ b/script/cibuild-setup-py
@@ -7,6 +7,7 @@ echo "## create test venv ######################################################
 TMP_DIR=$(mktemp -d -t ci-XXXXXXXXXX)
 python3 -m venv $TMP_DIR
 . "$TMP_DIR/bin/activate"
+pip install setuptools
 echo "## environment & versions ######################################################"
 python --version
 pip --version

From 5a119890ba8684058b410bcbb041643991df5e93 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 7 Oct 2023 13:31:08 -0700
Subject: [PATCH 083/116] explicit setuptools & wheel in requirements now

---
 requirements-dev.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index c0c68ce..e4bfab5 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -40,6 +40,8 @@ requests-toolbelt==1.0.0
 requests==2.31.0
 rfc3986==2.0.0
 rich==13.6.0
+setuptools==68.2.2
 twine==4.0.2
 urllib3==2.0.6
+wheel==0.41.2
 zipp==3.17.0

From 0ba9cb27275fe57fb674d1d06be7ba224336dcd9 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 12 Oct 2023 11:34:16 -0700
Subject: [PATCH 084/116] Rename ./script/test-module to
 ./script/cibuild-module for consistency

---
 .github/workflows/modules.yml          | 2 +-
 script/{test-module => cibuild-module} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename script/{test-module => cibuild-module} (100%)
 mode change 100755 => 100644

diff --git a/.github/workflows/modules.yml b/.github/workflows/modules.yml
index 9d6e767..15e1c3e 100644
--- a/.github/workflows/modules.yml
+++ b/.github/workflows/modules.yml
@@ -65,4 +65,4 @@ jobs:
           architecture: x64
       - name: Test Module
         run: |
-          ./script/test-module ${{ matrix.module }}
+          ./script/cibuild-module ${{ matrix.module }}
diff --git a/script/test-module b/script/cibuild-module
old mode 100755
new mode 100644
similarity index 100%
rename from script/test-module
rename to script/cibuild-module

From 165e32f18dae490a62e48a1b5eece1ca7b09183c Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 12 Oct 2023 11:34:45 -0700
Subject: [PATCH 085/116] Manually install setuptools in script/cibuild-module

---
 script/cibuild-module | 1 +
 1 file changed, 1 insertion(+)

diff --git a/script/cibuild-module b/script/cibuild-module
index 8148d73..b48ba7c 100644
--- a/script/cibuild-module
+++ b/script/cibuild-module
@@ -15,6 +15,7 @@ VENV_PYTHON=$(command -v python3)
 VENV_NAME="${TMP_DIR}/env"
 "$VENV_PYTHON" -m venv "$VENV_NAME"
 . "${VENV_NAME}/bin/activate"
+pip install setuptools
 echo "## environment & versions ######################################################"
 python --version
 pip --version

From 9a5173e252785b94d1f2fe186d95ba3a500619c5 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 12 Oct 2023 11:39:08 -0700
Subject: [PATCH 086/116] doh, cibuild-module doesn't have +x

---
 script/cibuild-module | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 script/cibuild-module

diff --git a/script/cibuild-module b/script/cibuild-module
old mode 100644
new mode 100755

From 5ad3ecb6c08f55c84aefe2c7dfb36f914b3e4edd Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 15 Oct 2023 10:07:34 -0700
Subject: [PATCH 087/116] Update LICENSE details

---
 LICENSE | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/LICENSE b/LICENSE
index acc8e6f..7d64688 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,3 +1,6 @@
+MIT License
+
+Copyright (c) 2021 Ross McFarland & the octoDNS Maintainers
 Copyright (c) 2017 GitHub, Inc.
 
 Permission is hereby granted, free of charge, to any person

From 9a28437b814a7cccec0974a75dc98f8166ce84b7 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 15 Oct 2023 13:20:43 -0700
Subject: [PATCH 088/116] chunked_values should return _value_type not plain
 string

---
 octodns/record/chunked.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/octodns/record/chunked.py b/octodns/record/chunked.py
index 8b115ed..8eef6fb 100644
--- a/octodns/record/chunked.py
+++ b/octodns/record/chunked.py
@@ -18,7 +18,7 @@ class _ChunkedValuesMixin(ValuesMixin):
             for i in range(0, len(value), self.CHUNK_SIZE)
         ]
         vs = '" "'.join(vs)
-        return f'"{vs}"'
+        return self._value_type(f'"{vs}"')
 
     @property
     def chunked_values(self):

From ded53023e7e9b1941fef171d6f86631a20c05f9d Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 15 Oct 2023 13:23:57 -0700
Subject: [PATCH 089/116] add support for chunked values to ValuesMixin.rrs

---
 CHANGELOG.md                     |  2 ++
 octodns/record/base.py           |  5 ++++-
 octodns/record/chunked.py        |  3 +++
 tests/test_octodns_record_txt.py | 36 ++++++++++++++++++++++++++++++++
 4 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d0ab5d2..2155ed8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,8 @@
 
 * Added ZoneNameFilter processor to enable ignoring/alerting on type-os like
   octodns.com.octodns.com
+* Fixed issues with handling of chunking large TXT values for providers that use
+  the in-built `rrs` method
 
 ## v1.2.1 - 2023-09-29 - Now with fewer stale files
 
diff --git a/octodns/record/base.py b/octodns/record/base.py
index 16e6718..30ff3c4 100644
--- a/octodns/record/base.py
+++ b/octodns/record/base.py
@@ -312,13 +312,16 @@ class ValuesMixin(object):
 
         return ret
 
+    def rr_values(self):
+        return self.values
+
     @property
     def rrs(self):
         return (
             self.fqdn,
             self.ttl,
             self._type,
-            [v.rdata_text for v in self.values],
+            [v.rdata_text for v in self.rr_values()],
         )
 
     def __repr__(self):
diff --git a/octodns/record/chunked.py b/octodns/record/chunked.py
index 8eef6fb..021d26e 100644
--- a/octodns/record/chunked.py
+++ b/octodns/record/chunked.py
@@ -27,6 +27,9 @@ class _ChunkedValuesMixin(ValuesMixin):
             values.append(self.chunked_value(v))
         return values
 
+    def rr_values(self):
+        return self.chunked_values
+
 
 class _ChunkedValue(str):
     _unescaped_semicolon_re = re.compile(r'\w;')
diff --git a/tests/test_octodns_record_txt.py b/tests/test_octodns_record_txt.py
index 1d46426..3d96352 100644
--- a/tests/test_octodns_record_txt.py
+++ b/tests/test_octodns_record_txt.py
@@ -142,3 +142,39 @@ class TestRecordTxt(TestCase):
         self.assertEqual(single.values, chunked.values)
         # should be chunked values, with quoting
         self.assertEqual(single.chunked_values, chunked.chunked_values)
+
+    def test_rr(self):
+        zone = Zone('unit.tests.', [])
+
+        # simple TXT
+        record = Record.new(
+            zone,
+            'txt',
+            {'ttl': 42, 'type': 'TXT', 'values': ['short 1', 'short 2']},
+        )
+        self.assertEqual(
+            ('txt.unit.tests.', 42, 'TXT', ['"short 1"', '"short 2"']),
+            record.rrs,
+        )
+
+        # long chunked text
+        record = Record.new(
+            zone,
+            'txt',
+            {
+                'ttl': 42,
+                'type': 'TXT',
+                'values': [
+                    'before',
+                    'v=DKIM1\\; h=sha256\\; k=rsa\\; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx78E7PtJvr8vpoNgHdIAe+llFKoy8WuTXDd6Z5mm3D4AUva9MBt5fFetxg/kcRy3KMDnMw6kDybwbpS/oPw1ylk6DL1xit7Cr5xeYYSWKukxXURAlHwT2K72oUsFKRUvN1X9lVysAeo+H8H/22Z9fJ0P30sOuRIRqCaiz+OiUYicxy4xrpfH2s9a+o3yRwX3zhlp8GjRmmmyK5mf7CkQTCfjnKVsYtB7mabXXmClH9tlcymnBMoN9PeXxaS5JRRysVV8RBCC9/wmfp9y//cck8nvE/MavFpSUHvv+TfTTdVKDlsXPjKX8iZQv0nO3xhspgkqFquKjydiR8nf4meHhwIDAQAB',
+                    'z after',
+                ],
+            },
+        )
+        vals = [
+            '"before"',
+            '"v=DKIM1\\; h=sha256\\; k=rsa\\; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx78E7PtJvr8vpoNgHdIAe+llFKoy8WuTXDd6Z5mm3D4AUva9MBt5fFetxg/kcRy3KMDnMw6kDybwbpS/oPw1ylk6DL1xit7Cr5xeYYSWKukxXURAlHwT2K72oUsFKRUvN1X9lVysAeo+H8H/22Z9fJ0P30sOuRIRqCaiz+OiUYicxy4xrpfH" '
+            '"2s9a+o3yRwX3zhlp8GjRmmmyK5mf7CkQTCfjnKVsYtB7mabXXmClH9tlcymnBMoN9PeXxaS5JRRysVV8RBCC9/wmfp9y//cck8nvE/MavFpSUHvv+TfTTdVKDlsXPjKX8iZQv0nO3xhspgkqFquKjydiR8nf4meHhwIDAQAB"',
+            '"z after"',
+        ]
+        self.assertEqual(('txt.unit.tests.', 42, 'TXT', vals), record.rrs)

From 65f4a48bc1a1a14cd14042d9041916134a886c82 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 15 Oct 2023 17:51:06 -0700
Subject: [PATCH 090/116] rr_values as a property for consistency

---
 octodns/record/base.py    | 3 ++-
 octodns/record/chunked.py | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/octodns/record/base.py b/octodns/record/base.py
index 30ff3c4..9bbb9aa 100644
--- a/octodns/record/base.py
+++ b/octodns/record/base.py
@@ -312,6 +312,7 @@ class ValuesMixin(object):
 
         return ret
 
+    @property
     def rr_values(self):
         return self.values
 
@@ -321,7 +322,7 @@ class ValuesMixin(object):
             self.fqdn,
             self.ttl,
             self._type,
-            [v.rdata_text for v in self.rr_values()],
+            [v.rdata_text for v in self.rr_values],
         )
 
     def __repr__(self):
diff --git a/octodns/record/chunked.py b/octodns/record/chunked.py
index 021d26e..976baea 100644
--- a/octodns/record/chunked.py
+++ b/octodns/record/chunked.py
@@ -27,6 +27,7 @@ class _ChunkedValuesMixin(ValuesMixin):
             values.append(self.chunked_value(v))
         return values
 
+    @property
     def rr_values(self):
         return self.chunked_values
 

From 1b293253d9c5c03499ba0b7dad5a04fa11121756 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Tue, 31 Oct 2023 18:56:59 -0700
Subject: [PATCH 091/116] ExcludeRootNsChanges added w/tests

---
 CHANGELOG.md                           |  2 +
 README.md                              |  1 +
 octodns/processor/filter.py            | 51 ++++++++++++++++++++++++
 tests/test_octodns_processor_filter.py | 55 +++++++++++++++++++++++++-
 4 files changed, 108 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2155ed8..a9e1f79 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,8 @@
   octodns.com.octodns.com
 * Fixed issues with handling of chunking large TXT values for providers that use
   the in-built `rrs` method
+* ExcludeRootNsChanges processor that will error (or warn) if plan includes a
+  change to root NS records
 
 ## v1.2.1 - 2023-09-29 - Now with fewer stale files
 
diff --git a/README.md b/README.md
index 0250890..4c5cac0 100644
--- a/README.md
+++ b/README.md
@@ -330,6 +330,7 @@ Similar to providers, but can only serve to populate records into a zone, cannot
 |--|--|
 | [AcmeMangingProcessor](/octodns/processor/acme.py) | Useful when processes external to octoDNS are managing acme challenge DNS records, e.g. LetsEncrypt |
 | [AutoArpa](/octodns/processor/arpa.py) | See [Automatic PTR generation](#automatic-ptr-generation) below |
+| [ExcludeRootNsChanges](/octodns/processor/filter.py) | Filter that errors or warns on planned root/APEX NS records changes. |
 | [IgnoreRootNsFilter](/octodns/processor/filter.py) | Filter that INGORES root/APEX NS records and prevents octoDNS from trying to manage them (where supported.) |
 | [MetaProcessor](/octodns/processor/meta.py) | Adds a special meta record with timing, UUID, providers, and/or version to aid in debugging and monitoring. |
 | [NameAllowlistFilter](/octodns/processor/filter.py) | Filter that ONLY manages records that match specified naming patterns, all others will be ignored |
diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py
index 4723073..2de3b23 100644
--- a/octodns/processor/filter.py
+++ b/octodns/processor/filter.py
@@ -2,6 +2,7 @@
 #
 #
 
+from logging import getLogger
 from re import compile as re_compile
 
 from ..record.exception import ValidationError
@@ -218,6 +219,56 @@ class IgnoreRootNsFilter(BaseProcessor):
     process_target_zone = _process
 
 
+class ExcludeRootNsChanges(BaseProcessor):
+    '''Do not allow root NS record changes
+
+    Example usage:
+
+    processors:
+      exclude-root-ns-changes:
+        class: octodns.processor.filter.ExcludeRootNsChanges
+        # If true an a change for a root NS is seen an error will be thrown. If
+        # false a warning will be printed and the change will be removed from
+        # the plan.
+        # (default: true)
+        error: true
+
+    zones:
+      exxampled.com.:
+        sources:
+          - config
+        processors:
+          - exclude-root-ns-changes
+        targets:
+          - ns1
+    '''
+
+    def __init__(self, name, error=True):
+        self.log = getLogger(f'ExcludeRootNsChanges[{name}]')
+        super().__init__(name)
+        self.error = error
+
+    def process_plan(self, plan, sources, target):
+        if plan:
+            for change in list(plan.changes):
+                record = change.record
+                if record._type == 'NS' and record.name == '':
+                    self.log.warning(
+                        'root NS changes are disallowed, fqdn=%s', record.fqdn
+                    )
+                    if self.error:
+                        raise ValidationError(
+                            record.fqdn,
+                            ['root NS changes are disallowed'],
+                            record.context,
+                        )
+                    plan.changes.remove(change)
+
+            print(len(plan.changes))
+
+        return plan
+
+
 class ZoneNameFilter(BaseProcessor):
     '''Filter or error on record names that contain the zone name
 
diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py
index 9857926..2d9b881 100644
--- a/tests/test_octodns_processor_filter.py
+++ b/tests/test_octodns_processor_filter.py
@@ -5,6 +5,7 @@
 from unittest import TestCase
 
 from octodns.processor.filter import (
+    ExcludeRootNsChanges,
     IgnoreRootNsFilter,
     NameAllowlistFilter,
     NameRejectlistFilter,
@@ -12,7 +13,8 @@ from octodns.processor.filter import (
     TypeRejectlistFilter,
     ZoneNameFilter,
 )
-from octodns.record import Record
+from octodns.provider.plan import Plan
+from octodns.record import Record, Update
 from octodns.record.exception import ValidationError
 from octodns.zone import Zone
 
@@ -184,6 +186,57 @@ class TestIgnoreRootNsFilter(TestCase):
         )
 
 
+class TestExcludeRootNsChanges(TestCase):
+    zone = Zone('unit.tests.', [])
+    root = Record.new(
+        zone, '', {'type': 'NS', 'ttl': 42, 'value': 'ns1.unit.tests.'}
+    )
+    zone.add_record(root)
+    not_root = Record.new(
+        zone, 'sub', {'type': 'NS', 'ttl': 43, 'value': 'ns2.unit.tests.'}
+    )
+    zone.add_record(not_root)
+    not_ns = Record.new(zone, '', {'type': 'A', 'ttl': 42, 'value': '3.4.5.6'})
+    zone.add_record(not_ns)
+    changes_with_root = [
+        Update(root, root),
+        Update(not_root, not_root),
+        Update(not_ns, not_ns),
+    ]
+    plan_with_root = Plan(zone, zone, changes_with_root, True)
+    changes_without_root = [Update(not_root, not_root), Update(not_ns, not_ns)]
+    plan_without_root = Plan(zone, zone, changes_without_root, True)
+
+    def test_no_plan(self):
+        proc = ExcludeRootNsChanges('exclude-root')
+        self.assertFalse(proc.process_plan(None, None, None))
+
+    def test_error(self):
+        proc = ExcludeRootNsChanges('exclude-root')
+
+        with self.assertRaises(ValidationError) as ctx:
+            proc.process_plan(self.plan_with_root, None, None)
+        self.assertEqual(
+            ['root NS changes are disallowed'], ctx.exception.reasons
+        )
+
+        self.assertEqual(
+            self.plan_without_root,
+            proc.process_plan(self.plan_without_root, None, None),
+        )
+
+    def test_warning(self):
+        proc = ExcludeRootNsChanges('exclude-root', error=False)
+
+        filtered_plan = proc.process_plan(self.plan_with_root, None, None)
+        self.assertEqual(self.plan_without_root.changes, filtered_plan.changes)
+
+        self.assertEqual(
+            self.plan_without_root,
+            proc.process_plan(self.plan_without_root, None, None),
+        )
+
+
 class TestZoneNameFilter(TestCase):
     def test_ends_with_zone(self):
         zone_name_filter = ZoneNameFilter('zone-name', error=False)

From f5fd68bb7e847aaee098f6e1c4e3f291c53506ed Mon Sep 17 00:00:00 2001
From: Solvik Blum <solvik@numberly.com>
Date: Wed, 8 Nov 2023 17:40:34 +0100
Subject: [PATCH 092/116] add NetworkValueRejectlistFilter and
 NetworkValueAllowlistFilter processors

---
 octodns/processor/filter.py            | 85 ++++++++++++++++++++++++++
 tests/test_octodns_processor_filter.py | 34 +++++++++++
 2 files changed, 119 insertions(+)

diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py
index 2de3b23..db14c85 100644
--- a/octodns/processor/filter.py
+++ b/octodns/processor/filter.py
@@ -2,6 +2,8 @@
 #
 #
 
+from ipaddress import ip_address, ip_network
+from itertools import product
 from logging import getLogger
 from re import compile as re_compile
 
@@ -125,6 +127,35 @@ class _NameBaseFilter(BaseProcessor):
     process_target_zone = _process
 
 
+class _NetworkValueBaseFilter(BaseProcessor):
+    def __init__(self, name, _list):
+        super().__init__(name)
+        self.networks = []
+        for value in _list:
+            try:
+                self.networks.append(ip_network(value))
+            except ValueError:
+                raise ValueError(f'{value} is not a valid CIDR to use')
+
+    def _process(self, zone, *args, **kwargs):
+        for record in zone.records:
+            if record._type not in ['A', 'AAAA']:
+                continue
+
+            if any(
+                ip_address(value) in network
+                for value, network in product(record.values, self.networks)
+            ):
+                self.matches(zone, record)
+            else:
+                self.doesnt_match(zone, record)
+
+        return zone
+
+    process_source_zone = _process
+    process_target_zone = _process
+
+
 class NameAllowlistFilter(_NameBaseFilter, AllowsMixin):
     '''Only manage records with names that match the provider patterns
 
@@ -189,6 +220,60 @@ class NameRejectlistFilter(_NameBaseFilter, RejectsMixin):
         super().__init__(name, rejectlist)
 
 
+class NetworkValueAllowlistFilter(_NetworkValueBaseFilter, AllowsMixin):
+    '''Only manage records with values that match the provider patterns
+
+    Example usage:
+
+    processors:
+      only-these:
+        class: octodns.processor.filter.NetworkValueAllowlistFilter
+        allowlist:
+          - 127.0.0.1/32
+          - 192.168.0.0/16
+          - fd00::/8
+
+    zones:
+      exxampled.com.:
+        sources:
+          - config
+        processors:
+          - only-these
+        targets:
+          - route53
+    '''
+
+    def __init__(self, name, allowlist):
+        super().__init__(name, allowlist)
+
+
+class NetworkValueRejectlistFilter(_NetworkValueBaseFilter, RejectsMixin):
+    '''Reject managing records with value matching a that match the provider patterns
+
+    Example usage:
+
+    processors:
+      not-these:
+        class: octodns.processor.filter.NetworkValueRejectlistFilter
+        rejectlist:
+          - 127.0.0.1/32
+          - 192.168.0.0/16
+          - fd00::/8
+
+    zones:
+      exxampled.com.:
+        sources:
+          - config
+        processors:
+          - not-these
+        targets:
+          - route53
+    '''
+
+    def __init__(self, name, rejectlist):
+        super().__init__(name, rejectlist)
+
+
 class IgnoreRootNsFilter(BaseProcessor):
     '''Do not manage Root NS Records.
 
diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py
index 2d9b881..4ecef63 100644
--- a/tests/test_octodns_processor_filter.py
+++ b/tests/test_octodns_processor_filter.py
@@ -9,6 +9,8 @@ from octodns.processor.filter import (
     IgnoreRootNsFilter,
     NameAllowlistFilter,
     NameRejectlistFilter,
+    NetworkValueAllowlistFilter,
+    NetworkValueRejectlistFilter,
     TypeAllowlistFilter,
     TypeRejectlistFilter,
     ZoneNameFilter,
@@ -161,6 +163,38 @@ class TestNameRejectListFilter(TestCase):
         )
 
 
+class TestNetworkValueFilter(TestCase):
+    zone = Zone('unit.tests.', [])
+    matches = Record.new(
+        zone, 'private-ipv4', {'type': 'A', 'ttl': 42, 'value': '10.42.42.42'}
+    )
+    zone.add_record(matches)
+    doesnt = Record.new(
+        zone, 'public-ipv4', {'type': 'A', 'ttl': 42, 'value': '42.42.42.42'}
+    )
+    zone.add_record(doesnt)
+    matchable1 = Record.new(
+        zone, 'private-ipv6', {'type': 'AAAA', 'ttl': 42, 'value': 'fd12:3456:789a:1::1'}
+    )
+    zone.add_record(matchable1)
+    matchable2 = Record.new(
+        zone, 'public-ipv6', {'type': 'AAAA', 'ttl': 42, 'value': 'dead:beef:cafe::1'}
+    )
+    zone.add_record(matchable2)
+
+    def test_reject(self):
+        filter_private = NetworkValueRejectlistFilter('rejectlist', set(('10.0.0.0/8', 'fd00::/8')))
+
+        got = filter_private.process_source_zone(self.zone.copy())
+        self.assertEqual(['public-ipv4', 'public-ipv6'], sorted([r.name for r in got.records]))
+
+    def test_allow(self):
+        filter_private = NetworkValueAllowlistFilter('allowlist', set(('10.0.0.0/8', 'fd00::/8')))
+
+        got = filter_private.process_source_zone(self.zone.copy())
+        self.assertEqual(['private-ipv4', 'private-ipv6'], sorted([r.name for r in got.records]))
+
+
 class TestIgnoreRootNsFilter(TestCase):
     zone = Zone('unit.tests.', [])
     root = Record.new(

From 354b8c2967532f3941ed1dabb7e89831a4570067 Mon Sep 17 00:00:00 2001
From: Solvik <solvik@solvik.fr>
Date: Thu, 9 Nov 2023 17:47:55 +0100
Subject: [PATCH 093/116] do not recreate the ip list for each network test

Co-authored-by: Ross McFarland <rwmcfa1@gmail.com>
---
 octodns/processor/filter.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py
index db14c85..34115d5 100644
--- a/octodns/processor/filter.py
+++ b/octodns/processor/filter.py
@@ -142,9 +142,10 @@ class _NetworkValueBaseFilter(BaseProcessor):
             if record._type not in ['A', 'AAAA']:
                 continue
 
+            ips = [ip_address(value) for value in record.values]
             if any(
-                ip_address(value) in network
-                for value, network in product(record.values, self.networks)
+                ip in network
+                for ip, network in product(ips, self.networks)
             ):
                 self.matches(zone, record)
             else:

From f9cb31b602c99761f5e9fc4cc02e31866a4e8494 Mon Sep 17 00:00:00 2001
From: Solvik Blum <solvik@numberly.com>
Date: Thu, 9 Nov 2023 19:01:59 +0100
Subject: [PATCH 094/116] add a txt in tests so we can see the filter
 effectively only handles A/AAAA

---
 tests/test_octodns_processor_filter.py | 62 +++++++++++++++++---------
 1 file changed, 42 insertions(+), 20 deletions(-)

diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py
index 4ecef63..409ec50 100644
--- a/tests/test_octodns_processor_filter.py
+++ b/tests/test_octodns_processor_filter.py
@@ -165,34 +165,56 @@ class TestNameRejectListFilter(TestCase):
 
 class TestNetworkValueFilter(TestCase):
     zone = Zone('unit.tests.', [])
-    matches = Record.new(
-        zone, 'private-ipv4', {'type': 'A', 'ttl': 42, 'value': '10.42.42.42'}
-    )
-    zone.add_record(matches)
-    doesnt = Record.new(
-        zone, 'public-ipv4', {'type': 'A', 'ttl': 42, 'value': '42.42.42.42'}
-    )
-    zone.add_record(doesnt)
-    matchable1 = Record.new(
-        zone, 'private-ipv6', {'type': 'AAAA', 'ttl': 42, 'value': 'fd12:3456:789a:1::1'}
-    )
-    zone.add_record(matchable1)
-    matchable2 = Record.new(
-        zone, 'public-ipv6', {'type': 'AAAA', 'ttl': 42, 'value': 'dead:beef:cafe::1'}
-    )
-    zone.add_record(matchable2)
+    for record in [
+        Record.new(
+            zone,
+            'private-ipv4',
+            {'type': 'A', 'ttl': 42, 'value': '10.42.42.42'},
+        ),
+        Record.new(
+            zone,
+            'public-ipv4',
+            {'type': 'A', 'ttl': 42, 'value': '42.42.42.42'},
+        ),
+        Record.new(
+            zone,
+            'private-ipv6',
+            {'type': 'AAAA', 'ttl': 42, 'value': 'fd12:3456:789a:1::1'},
+        ),
+        Record.new(
+            zone,
+            'public-ipv6',
+            {'type': 'AAAA', 'ttl': 42, 'value': 'dead:beef:cafe::1'},
+        ),
+        Record.new(
+            zone,
+            'keep-me',
+            {'ttl': 30, 'type': 'TXT', 'value': 'this should always be here'},
+        ),
+    ]:
+        zone.add_record(record)
 
     def test_reject(self):
-        filter_private = NetworkValueRejectlistFilter('rejectlist', set(('10.0.0.0/8', 'fd00::/8')))
+        filter_private = NetworkValueRejectlistFilter(
+            'rejectlist', set(('10.0.0.0/8', 'fd00::/8'))
+        )
 
         got = filter_private.process_source_zone(self.zone.copy())
-        self.assertEqual(['public-ipv4', 'public-ipv6'], sorted([r.name for r in got.records]))
+        self.assertEqual(
+            ['keep-me', 'public-ipv4', 'public-ipv6'],
+            sorted([r.name for r in got.records]),
+        )
 
     def test_allow(self):
-        filter_private = NetworkValueAllowlistFilter('allowlist', set(('10.0.0.0/8', 'fd00::/8')))
+        filter_private = NetworkValueAllowlistFilter(
+            'allowlist', set(('10.0.0.0/8', 'fd00::/8'))
+        )
 
         got = filter_private.process_source_zone(self.zone.copy())
-        self.assertEqual(['private-ipv4', 'private-ipv6'], sorted([r.name for r in got.records]))
+        self.assertEqual(
+            ['keep-me', 'private-ipv4', 'private-ipv6'],
+            sorted([r.name for r in got.records]),
+        )
 
 
 class TestIgnoreRootNsFilter(TestCase):

From 010e5039cca36eced58d4ad926f21770565e3f36 Mon Sep 17 00:00:00 2001
From: Solvik Blum <solvik@numberly.com>
Date: Thu, 9 Nov 2023 19:20:46 +0100
Subject: [PATCH 095/116] add mention to docstring that the NetworkValue
 filters won't touch anything except A/AAAA

---
 octodns/processor/filter.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py
index 34115d5..77f66ea 100644
--- a/octodns/processor/filter.py
+++ b/octodns/processor/filter.py
@@ -222,7 +222,8 @@ class NameRejectlistFilter(_NameBaseFilter, RejectsMixin):
 
 
 class NetworkValueAllowlistFilter(_NetworkValueBaseFilter, AllowsMixin):
-    '''Only manage records with values that match the provider patterns
+    '''Only manage A and AAAA records with values that match the provider patterns
+    All other types will be left as-is.
 
     Example usage:
 
@@ -249,7 +250,8 @@ class NetworkValueAllowlistFilter(_NetworkValueBaseFilter, AllowsMixin):
 
 
 class NetworkValueRejectlistFilter(_NetworkValueBaseFilter, RejectsMixin):
-    '''Reject managing records with value matching a that match the provider patterns
+    '''Reject managing A and AAAA records with value matching a that match the provider patterns
+    All other types will be left as-is.
 
     Example usage:
 

From fa56dfaffddb1c5e183fabf10cd8b0d3d2619baf Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Thu, 9 Nov 2023 11:34:16 -0800
Subject: [PATCH 096/116] fix minor formatting failure

---
 octodns/processor/filter.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py
index 77f66ea..ddc35d0 100644
--- a/octodns/processor/filter.py
+++ b/octodns/processor/filter.py
@@ -144,8 +144,7 @@ class _NetworkValueBaseFilter(BaseProcessor):
 
             ips = [ip_address(value) for value in record.values]
             if any(
-                ip in network
-                for ip, network in product(ips, self.networks)
+                ip in network for ip, network in product(ips, self.networks)
             ):
                 self.matches(zone, record)
             else:

From 3ed7a88e343c89b7153efea25db1b6287b2f0823 Mon Sep 17 00:00:00 2001
From: Solvik Blum <solvik@numberly.com>
Date: Fri, 10 Nov 2023 13:58:08 +0100
Subject: [PATCH 097/116] add test to cover CIDR validation in config for
 filters

---
 tests/test_octodns_processor_filter.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py
index 409ec50..6525900 100644
--- a/tests/test_octodns_processor_filter.py
+++ b/tests/test_octodns_processor_filter.py
@@ -194,6 +194,12 @@ class TestNetworkValueFilter(TestCase):
     ]:
         zone.add_record(record)
 
+    def test_bad_config(self):
+        with self.assertRaises(ValueError):
+            filter_private = NetworkValueRejectlistFilter(
+                'rejectlist', set(('string', '42.42.42.42/43'))
+            )
+
     def test_reject(self):
         filter_private = NetworkValueRejectlistFilter(
             'rejectlist', set(('10.0.0.0/8', 'fd00::/8'))

From abdab8f6d83894e9f37eb3213efa828dfc741cd0 Mon Sep 17 00:00:00 2001
From: Solvik Blum <solvik@numberly.com>
Date: Fri, 10 Nov 2023 15:28:01 +0100
Subject: [PATCH 098/116] fix lint

---
 tests/test_octodns_processor_filter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py
index 6525900..7ee98a8 100644
--- a/tests/test_octodns_processor_filter.py
+++ b/tests/test_octodns_processor_filter.py
@@ -196,7 +196,7 @@ class TestNetworkValueFilter(TestCase):
 
     def test_bad_config(self):
         with self.assertRaises(ValueError):
-            filter_private = NetworkValueRejectlistFilter(
+            NetworkValueRejectlistFilter(
                 'rejectlist', set(('string', '42.42.42.42/43'))
             )
 

From e9cdacdd13f15ca3e67487f6695b0d4f9878a2bf Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 10 Nov 2023 16:01:14 -0800
Subject: [PATCH 099/116] Include octodns special section in record __repr__

---
 octodns/record/base.py       | 10 ++++++--
 tests/test_octodns_record.py | 48 ++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/octodns/record/base.py b/octodns/record/base.py
index 9bbb9aa..3b9b36e 100644
--- a/octodns/record/base.py
+++ b/octodns/record/base.py
@@ -328,7 +328,10 @@ class ValuesMixin(object):
     def __repr__(self):
         values = "', '".join([str(v) for v in self.values])
         klass = self.__class__.__name__
-        return f"<{klass} {self._type} {self.ttl}, {self.decoded_fqdn}, ['{values}']>"
+        octodns = ''
+        if self._octodns:
+            octodns = f', {self._octodns}'
+        return f"<{klass} {self._type} {self.ttl}, {self.decoded_fqdn}, ['{values}']{octodns}>"
 
 
 class ValueMixin(object):
@@ -371,4 +374,7 @@ class ValueMixin(object):
 
     def __repr__(self):
         klass = self.__class__.__name__
-        return f'<{klass} {self._type} {self.ttl}, {self.decoded_fqdn}, {self.value}>'
+        octodns = ''
+        if self._octodns:
+            octodns = f', {self._octodns}'
+        return f'<{klass} {self._type} {self.ttl}, {self.decoded_fqdn}, {self.value}{octodns}>'
diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py
index a6d27b7..abd886d 100644
--- a/tests/test_octodns_record.py
+++ b/tests/test_octodns_record.py
@@ -654,3 +654,51 @@ class TestRecordValidation(TestCase):
             ),
         )
         self.assertEqual('needle', record.context)
+
+    def test_values_mixin_repr(self):
+        # ValuesMixin
+        record = Record.new(
+            self.zone,
+            'www',
+            {
+                'ttl': 42,
+                'type': 'A',
+                'values': ['1.2.3.4', '2.3.4.5'],
+                'octodns': {'key': 'value'},
+            },
+        )
+        # has the octodns special section
+        self.assertEqual(
+            "<ARecord A 42, www.unit.tests., ['1.2.3.4', '2.3.4.5'], {'key': 'value'}>",
+            record.__repr__(),
+        )
+        # no special section
+        record._octodns = {}
+        self.assertEqual(
+            "<ARecord A 42, www.unit.tests., ['1.2.3.4', '2.3.4.5']>",
+            record.__repr__(),
+        )
+
+    def test_value_mixin_repr(self):
+        # ValueMixin
+        record = Record.new(
+            self.zone,
+            'pointer',
+            {
+                'ttl': 43,
+                'type': 'CNAME',
+                'value': 'unit.tests.',
+                'octodns': {'key': 42},
+            },
+        )
+        # has the octodns special section
+        self.assertEqual(
+            "<CnameRecord CNAME 43, pointer.unit.tests., unit.tests., {'key': 42}>",
+            record.__repr__(),
+        )
+        # no special section
+        record._octodns = {}
+        self.assertEqual(
+            '<CnameRecord CNAME 43, pointer.unit.tests., unit.tests.>',
+            record.__repr__(),
+        )

From b55b29af316cfc7530f656a01666cc2edbf152cd Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 10 Nov 2023 16:02:53 -0800
Subject: [PATCH 100/116] changelog entry for __repr__ improvement

---
 CHANGELOG.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a9e1f79..8dde47b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,9 @@
   the in-built `rrs` method
 * ExcludeRootNsChanges processor that will error (or warn) if plan includes a
   change to root NS records
+* Include the octodns special section info in Record __repr__, makes it easier
+  to debug things with providers that have special functionality configured
+  there.
 
 ## v1.2.1 - 2023-09-29 - Now with fewer stale files
 

From 731eb56ab91cf01fd4f324ad303530a8beba84a9 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 10 Nov 2023 16:08:22 -0800
Subject: [PATCH 101/116] CHANGELOG entry for network cidr filters

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8dde47b..180b245 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,11 @@
   octodns.com.octodns.com
 * Fixed issues with handling of chunking large TXT values for providers that use
   the in-built `rrs` method
+* NetworkValueAllowlistFilter/NetworkValueRejectlistFilter added to
+  processors.filter to enable filtering A/AAAA records based on value. Can be
+  useful if you have records with non-routable values in an internal copy of a
+  zone, but want to exclude them when pushing the same zone publically (split
+  horizon)
 * ExcludeRootNsChanges processor that will error (or warn) if plan includes a
   change to root NS records
 * Include the octodns special section info in Record __repr__, makes it easier

From 6cd933a83439bbab07c944198783c2dbc5e1f514 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 10 Nov 2023 16:57:01 -0800
Subject: [PATCH 102/116] Add include_target option to most processor.filter
 processors

---
 CHANGELOG.md                           |  4 +++
 octodns/processor/filter.py            | 49 ++++++++++++++------------
 tests/test_octodns_processor_filter.py | 16 +++++++++
 3 files changed, 47 insertions(+), 22 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8dde47b..b4cd240 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,10 @@
 * Include the octodns special section info in Record __repr__, makes it easier
   to debug things with providers that have special functionality configured
   there.
+* Most processor.filter processors now support an include_target flag that can
+  be set to False to leave the target zone data untouched, thus remove any
+  existing filtered records. Default behavior is unchanged and filtered records
+  will be completely invisible to octoDNS
 
 ## v1.2.1 - 2023-09-29 - Now with fewer stale files
 
diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py
index 2de3b23..eeab039 100644
--- a/octodns/processor/filter.py
+++ b/octodns/processor/filter.py
@@ -9,6 +9,20 @@ from ..record.exception import ValidationError
 from .base import BaseProcessor
 
 
+class _FilterProcessor(BaseProcessor):
+    def __init__(self, name, include_target=True, **kwargs):
+        super().__init__(name, **kwargs)
+        self.include_target = include_target
+
+    def process_source_zone(self, *args, **kwargs):
+        return self._process(*args, **kwargs)
+
+    def process_target_zone(self, existing, *args, **kwargs):
+        if self.include_target:
+            return self._process(existing, *args, **kwargs)
+        return existing
+
+
 class AllowsMixin:
     def matches(self, zone, record):
         pass
@@ -25,9 +39,9 @@ class RejectsMixin:
         pass
 
 
-class _TypeBaseFilter(BaseProcessor):
-    def __init__(self, name, _list):
-        super().__init__(name)
+class _TypeBaseFilter(_FilterProcessor):
+    def __init__(self, name, _list, **kwargs):
+        super().__init__(name, **kwargs)
         self._list = set(_list)
 
     def _process(self, zone, *args, **kwargs):
@@ -39,9 +53,6 @@ class _TypeBaseFilter(BaseProcessor):
 
         return zone
 
-    process_source_zone = _process
-    process_target_zone = _process
-
 
 class TypeAllowlistFilter(_TypeBaseFilter, AllowsMixin):
     '''Only manage records of the specified type(s).
@@ -65,8 +76,8 @@ class TypeAllowlistFilter(_TypeBaseFilter, AllowsMixin):
           - ns1
     '''
 
-    def __init__(self, name, allowlist):
-        super().__init__(name, allowlist)
+    def __init__(self, name, allowlist, **kwargs):
+        super().__init__(name, allowlist, **kwargs)
 
 
 class TypeRejectlistFilter(_TypeBaseFilter, RejectsMixin):
@@ -90,13 +101,13 @@ class TypeRejectlistFilter(_TypeBaseFilter, RejectsMixin):
           - route53
     '''
 
-    def __init__(self, name, rejectlist):
-        super().__init__(name, rejectlist)
+    def __init__(self, name, rejectlist, **kwargs):
+        super().__init__(name, rejectlist, **kwargs)
 
 
-class _NameBaseFilter(BaseProcessor):
-    def __init__(self, name, _list):
-        super().__init__(name)
+class _NameBaseFilter(_FilterProcessor):
+    def __init__(self, name, _list, **kwargs):
+        super().__init__(name, **kwargs)
         exact = set()
         regex = []
         for pattern in _list:
@@ -121,9 +132,6 @@ class _NameBaseFilter(BaseProcessor):
 
         return zone
 
-    process_source_zone = _process
-    process_target_zone = _process
-
 
 class NameAllowlistFilter(_NameBaseFilter, AllowsMixin):
     '''Only manage records with names that match the provider patterns
@@ -269,7 +277,7 @@ class ExcludeRootNsChanges(BaseProcessor):
         return plan
 
 
-class ZoneNameFilter(BaseProcessor):
+class ZoneNameFilter(_FilterProcessor):
     '''Filter or error on record names that contain the zone name
 
     Example usage:
@@ -291,8 +299,8 @@ class ZoneNameFilter(BaseProcessor):
           - azure
     '''
 
-    def __init__(self, name, error=True):
-        super().__init__(name)
+    def __init__(self, name, error=True, **kwargs):
+        super().__init__(name, **kwargs)
         self.error = error
 
     def _process(self, zone, *args, **kwargs):
@@ -314,6 +322,3 @@ class ZoneNameFilter(BaseProcessor):
                     zone.remove_record(record)
 
         return zone
-
-    process_source_zone = _process
-    process_target_zone = _process
diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py
index 2d9b881..d4c36ef 100644
--- a/tests/test_octodns_processor_filter.py
+++ b/tests/test_octodns_processor_filter.py
@@ -54,6 +54,22 @@ class TestTypeAllowListFilter(TestCase):
             ['a', 'a2', 'aaaa'], sorted([r.name for r in got.records])
         )
 
+    def test_include_target(self):
+        filter_txt = TypeAllowlistFilter(
+            'only-txt', ['TXT'], include_target=False
+        )
+
+        # as a source we don't see them
+        got = filter_txt.process_source_zone(zone.copy())
+        self.assertEqual(['txt', 'txt2'], sorted([r.name for r in got.records]))
+
+        # but as a target we do b/c it's not included
+        got = filter_txt.process_target_zone(zone.copy())
+        self.assertEqual(
+            ['a', 'a2', 'aaaa', 'txt', 'txt2'],
+            sorted([r.name for r in got.records]),
+        )
+
 
 class TestTypeRejectListFilter(TestCase):
     def test_basics(self):

From be5d28dc56e2fc3e7a67fe1d2db080baceff6d02 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 10 Nov 2023 17:00:24 -0800
Subject: [PATCH 103/116] doc new include_target filter prarams

---
 octodns/processor/filter.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py
index eeab039..31c12c9 100644
--- a/octodns/processor/filter.py
+++ b/octodns/processor/filter.py
@@ -65,6 +65,10 @@ class TypeAllowlistFilter(_TypeBaseFilter, AllowsMixin):
         allowlist:
           - A
           - AAAA
+        # Optional param that can be set to False to leave the target zone
+        # alone, thus allowing deletion of existing records
+        # (default: true)
+        # include_target: True
 
     zones:
       exxampled.com.:
@@ -90,6 +94,10 @@ class TypeRejectlistFilter(_TypeBaseFilter, RejectsMixin):
         class: octodns.processor.filter.TypeRejectlistFilter
         rejectlist:
           - CNAME
+        # Optional param that can be set to False to leave the target zone
+        # alone, thus allowing deletion of existing records
+        # (default: true)
+        # include_target: True
 
     zones:
       exxampled.com.:
@@ -150,6 +158,10 @@ class NameAllowlistFilter(_NameBaseFilter, AllowsMixin):
           - /some-pattern-\\d\\+/
           # regex - anchored so has to match start to end
           - /^start-.+-end$/
+        # Optional param that can be set to False to leave the target zone
+        # alone, thus allowing deletion of existing records
+        # (default: true)
+        # include_target: True
 
     zones:
       exxampled.com.:
@@ -182,6 +194,10 @@ class NameRejectlistFilter(_NameBaseFilter, RejectsMixin):
           - /some-pattern-\\d\\+/
           # regex - anchored so has to match start to end
           - /^start-.+-end$/
+        # Optional param that can be set to False to leave the target zone
+        # alone, thus allowing deletion of existing records
+        # (default: true)
+        # include_target: True
 
     zones:
       exxampled.com.:
@@ -288,6 +304,10 @@ class ZoneNameFilter(_FilterProcessor):
         # If true a ValidationError will be throw when such records are
         # encouterd, if false the records will just be ignored/omitted.
         # (default: true)
+        # Optional param that can be set to False to leave the target zone
+        # alone, thus allowing deletion of existing records
+        # (default: true)
+        # include_target: True
 
     zones:
       exxampled.com.:

From de3ec8e094d85faa45667509fd286d8aa874d6d8 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 10 Nov 2023 17:02:20 -0800
Subject: [PATCH 104/116] Move _NetworkValueBaseFilter down with it's children

---
 octodns/processor/filter.py | 58 ++++++++++++++++++-------------------
 1 file changed, 29 insertions(+), 29 deletions(-)

diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py
index ddc35d0..e7913d8 100644
--- a/octodns/processor/filter.py
+++ b/octodns/processor/filter.py
@@ -127,35 +127,6 @@ class _NameBaseFilter(BaseProcessor):
     process_target_zone = _process
 
 
-class _NetworkValueBaseFilter(BaseProcessor):
-    def __init__(self, name, _list):
-        super().__init__(name)
-        self.networks = []
-        for value in _list:
-            try:
-                self.networks.append(ip_network(value))
-            except ValueError:
-                raise ValueError(f'{value} is not a valid CIDR to use')
-
-    def _process(self, zone, *args, **kwargs):
-        for record in zone.records:
-            if record._type not in ['A', 'AAAA']:
-                continue
-
-            ips = [ip_address(value) for value in record.values]
-            if any(
-                ip in network for ip, network in product(ips, self.networks)
-            ):
-                self.matches(zone, record)
-            else:
-                self.doesnt_match(zone, record)
-
-        return zone
-
-    process_source_zone = _process
-    process_target_zone = _process
-
-
 class NameAllowlistFilter(_NameBaseFilter, AllowsMixin):
     '''Only manage records with names that match the provider patterns
 
@@ -220,6 +191,35 @@ class NameRejectlistFilter(_NameBaseFilter, RejectsMixin):
         super().__init__(name, rejectlist)
 
 
+class _NetworkValueBaseFilter(BaseProcessor):
+    def __init__(self, name, _list):
+        super().__init__(name)
+        self.networks = []
+        for value in _list:
+            try:
+                self.networks.append(ip_network(value))
+            except ValueError:
+                raise ValueError(f'{value} is not a valid CIDR to use')
+
+    def _process(self, zone, *args, **kwargs):
+        for record in zone.records:
+            if record._type not in ['A', 'AAAA']:
+                continue
+
+            ips = [ip_address(value) for value in record.values]
+            if any(
+                ip in network for ip, network in product(ips, self.networks)
+            ):
+                self.matches(zone, record)
+            else:
+                self.doesnt_match(zone, record)
+
+        return zone
+
+    process_source_zone = _process
+    process_target_zone = _process
+
+
 class NetworkValueAllowlistFilter(_NetworkValueBaseFilter, AllowsMixin):
     '''Only manage A and AAAA records with values that match the provider patterns
     All other types will be left as-is.

From 9b7341fee2c32138563143c919648709a860c57e Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 11 Nov 2023 09:32:21 -0800
Subject: [PATCH 105/116] Add preferred __version__, per pep-8

---
 CHANGELOG.md        | 11 +++++++++--
 octodns/__init__.py |  3 ++-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8dde47b..a1b59b1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,9 +1,16 @@
 ## v1.3.0 - 2023-??-?? - ???
 
-* Added ZoneNameFilter processor to enable ignoring/alerting on type-os like
-  octodns.com.octodns.com
+#### Noteworthy changes
+
+* Added octodns.__version__ to replace octodns.__VERSION__ as the former is more
+  of a standard, per pep-8. __VERSION__ is deprecated and will go away in 2.x
 * Fixed issues with handling of chunking large TXT values for providers that use
   the in-built `rrs` method
+
+#### Stuff
+
+* Added ZoneNameFilter processor to enable ignoring/alerting on type-os like
+  octodns.com.octodns.com
 * ExcludeRootNsChanges processor that will error (or warn) if plan includes a
   change to root NS records
 * Include the octodns special section info in Record __repr__, makes it easier
diff --git a/octodns/__init__.py b/octodns/__init__.py
index e106d6c..1ae31b0 100644
--- a/octodns/__init__.py
+++ b/octodns/__init__.py
@@ -1,3 +1,4 @@
 'OctoDNS: DNS as code - Tools for managing DNS across multiple providers'
 
-__VERSION__ = '1.2.1'
+# TODO: remove __VERSION__ w/2.x
+__version__ = __VERSION__ = '1.2.1'

From c9a2c8f72bb81eb81573eea5d21381c5f88aeeb4 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 11 Nov 2023 09:33:48 -0800
Subject: [PATCH 106/116] Simplify versioning, just use __version__

---
 CHANGELOG.md |  2 ++
 setup.py     | 15 ++-------------
 2 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a1b59b1..efa7ba5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,8 @@
   of a standard, per pep-8. __VERSION__ is deprecated and will go away in 2.x
 * Fixed issues with handling of chunking large TXT values for providers that use
   the in-built `rrs` method
+* Removed code that included sha in module version number when installing from
+  repo a it caused problems with non-binary installs.
 
 #### Stuff
 
diff --git a/setup.py b/setup.py
index aa02a1a..c75618d 100644
--- a/setup.py
+++ b/setup.py
@@ -1,9 +1,7 @@
 #!/usr/bin/env python
 
 from io import StringIO
-from os import environ
 from os.path import dirname, join
-from subprocess import CalledProcessError, check_output
 
 import octodns
 
@@ -50,16 +48,7 @@ def long_description():
 
 
 def version():
-    # pep440 style public & local version numbers
-    if environ.get('OCTODNS_RELEASE', False):
-        # public
-        return octodns.__VERSION__
-    try:
-        sha = check_output(['git', 'rev-parse', 'HEAD']).decode('utf-8')[:8]
-    except (CalledProcessError, FileNotFoundError):
-        sha = 'unknown'
-    # local
-    return f'{octodns.__VERSION__}+{sha}'
+    return octodns.__VERSION__
 
 
 tests_require = ('pytest>=6.2.5', 'pytest-cov>=3.0.0', 'pytest-network>=0.0.1')
@@ -102,5 +91,5 @@ setup(
     python_requires='>=3.8',
     tests_require=tests_require,
     url='https://github.com/octodns/octodns',
-    version=version(),
+    version=octodns.__version__,
 )

From 44499a996e6198c1765f9a5d9cfe36768114889f Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 11 Nov 2023 12:38:10 -0800
Subject: [PATCH 107/116] Remove dead version function

---
 setup.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/setup.py b/setup.py
index c75618d..a3e9ec5 100644
--- a/setup.py
+++ b/setup.py
@@ -47,10 +47,6 @@ def long_description():
     return buf.getvalue()
 
 
-def version():
-    return octodns.__VERSION__
-
-
 tests_require = ('pytest>=6.2.5', 'pytest-cov>=3.0.0', 'pytest-network>=0.0.1')
 
 setup(

From 45900a861dc99ef3cc0a756c0ebbe84843defd69 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sat, 11 Nov 2023 13:57:04 -0800
Subject: [PATCH 108/116] Correct a it -> as it type-o in changelog

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index efa7ba5..8ec139a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,7 +7,7 @@
 * Fixed issues with handling of chunking large TXT values for providers that use
   the in-built `rrs` method
 * Removed code that included sha in module version number when installing from
-  repo a it caused problems with non-binary installs.
+  repo as it caused problems with non-binary installs.
 
 #### Stuff
 

From eb4979d40b4a2e33aa737b8beb8ab3c09f65a305 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 12 Nov 2023 15:04:22 -0800
Subject: [PATCH 109/116] __VERSION__ -> __version__ in script/release

---
 script/release | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/script/release b/script/release
index ac89a3d..da5d216 100755
--- a/script/release
+++ b/script/release
@@ -32,7 +32,7 @@ fi
 # Set so that setup.py will create a public release style version number
 export OCTODNS_RELEASE=1
 
-VERSION="$(grep "^__VERSION__" "$ROOT/octodns/__init__.py" | sed -e "s/.* = '//" -e "s/'$//")"
+VERSION="$(grep "^__version__" "$ROOT/octodns/__init__.py" | sed -e "s/.* = '//" -e "s/'$//")"
 
 git tag -s "v$VERSION" -m "Release $VERSION"
 git push origin "v$VERSION"

From 344bc2de5f98d31298c35243fc092024cbb69893 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 12 Nov 2023 15:45:43 -0800
Subject: [PATCH 110/116] Remove stale script/sdist script

---
 octodns/cmds/args.py      |  4 ++--
 octodns/manager.py        |  9 ++++++---
 octodns/processor/meta.py |  4 ++--
 script/changelog          |  2 +-
 script/sdist              | 15 ---------------
 5 files changed, 11 insertions(+), 23 deletions(-)
 delete mode 100755 script/sdist

diff --git a/octodns/cmds/args.py b/octodns/cmds/args.py
index 52b42da..1aefbea 100644
--- a/octodns/cmds/args.py
+++ b/octodns/cmds/args.py
@@ -10,7 +10,7 @@ from sys import stderr, stdout
 
 from yaml import safe_load
 
-from octodns import __VERSION__
+from octodns import __version__
 
 
 class ArgumentParser(_Base):
@@ -24,7 +24,7 @@ class ArgumentParser(_Base):
         super().__init__(*args, **kwargs)
 
     def parse_args(self, default_log_level=INFO):
-        version = f'octoDNS {__VERSION__}'
+        version = f'octoDNS {__version__}'
         self.add_argument(
             '--version',
             action='version',
diff --git a/octodns/manager.py b/octodns/manager.py
index 1753269..3ec163c 100644
--- a/octodns/manager.py
+++ b/octodns/manager.py
@@ -11,7 +11,7 @@ from logging import getLogger
 from os import environ
 from sys import stdout
 
-from . import __VERSION__
+from . import __version__
 from .idna import IdnaDict, idna_decode, idna_encode
 from .processor.arpa import AutoArpa
 from .processor.meta import MetaProcessor
@@ -89,7 +89,7 @@ class Manager(object):
     def __init__(
         self, config_file, max_workers=None, include_meta=False, auto_arpa=False
     ):
-        version = self._try_version('octodns', version=__VERSION__)
+        version = self._try_version('octodns', version=__version__)
         self.log.info(
             '__init__: config_file=%s, (octoDNS %s)', config_file, version
         )
@@ -308,7 +308,10 @@ class Manager(object):
         # finally try and import the module and see if it has a __VERSION__
         if module is None:
             module = import_module(module_name)
-        return getattr(module, '__VERSION__', None)
+        # TODO: remove the __VERSION__ fallback eventually?
+        return getattr(
+            module, '__version__', getattr(module, '__VERSION__', None)
+        )
 
     def _import_module(self, module_name):
         current = module_name
diff --git a/octodns/processor/meta.py b/octodns/processor/meta.py
index c9e4a05..ee56ef8 100644
--- a/octodns/processor/meta.py
+++ b/octodns/processor/meta.py
@@ -6,7 +6,7 @@ from datetime import datetime
 from logging import getLogger
 from uuid import uuid4
 
-from .. import __VERSION__
+from .. import __version__
 from ..record import Record
 from .base import BaseProcessor
 
@@ -91,7 +91,7 @@ class MetaProcessor(BaseProcessor):
             uuid = self.uuid() if include_uuid else None
             values.append(f'uuid={uuid}')
         if include_version:
-            values.append(f'octodns-version={__VERSION__}')
+            values.append(f'octodns-version={__version__}')
         self.include_provider = include_provider
         values.sort()
         self.values = values
diff --git a/script/changelog b/script/changelog
index 4257b67..c270fb9 100755
--- a/script/changelog
+++ b/script/changelog
@@ -2,6 +2,6 @@
 
 set -e
 
-VERSION=v$(grep __VERSION__ octodns/__init__.py | sed -e "s/^[^']*'//" -e "s/'$//")
+VERSION=v$(grep __version__ octodns/__init__.py | sed -e "s/^[^']*'//" -e "s/'$//")
 echo $VERSION
 git log --pretty="%h - %cr - %s (%an)" "${VERSION}..HEAD"
diff --git a/script/sdist b/script/sdist
deleted file mode 100755
index 1ab0949..0000000
--- a/script/sdist
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-set -e
-
-if ! git diff-index --quiet HEAD --; then
-    echo "Changes in local directory, commit or clear" >&2
-    exit 1
-fi
-
-SHA=$(git rev-parse HEAD)
-python setup.py sdist
-TARBALL="dist/octodns-$SHA.tar.gz"
-mv dist/octodns-0.*.tar.gz "$TARBALL"
-
-echo "Created $TARBALL"

From d2baf6db8f132f5f7517fa509717ca0a0188ef89 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Sun, 12 Nov 2023 15:46:28 -0800
Subject: [PATCH 111/116] Prefer __version__ over __VERSION__ internally,
 fallback when referencing modules

---
 tests/test_octodns_manager.py        | 6 +++---
 tests/test_octodns_processor_meta.py | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py
index ae2f415..2bec87e 100644
--- a/tests/test_octodns_manager.py
+++ b/tests/test_octodns_manager.py
@@ -16,7 +16,7 @@ from helpers import (
     TemporaryDirectory,
 )
 
-from octodns import __VERSION__
+from octodns import __version__
 from octodns.idna import IdnaDict, idna_encode
 from octodns.manager import (
     MainThreadExecutor,
@@ -746,13 +746,13 @@ class TestManager(TestCase):
         manager = Manager(get_config_filename('simple.yaml'))
 
         class DummyModule(object):
-            __VERSION__ = '2.3.4'
+            __version__ = '2.3.4'
 
         dummy_module = DummyModule()
 
         # use importlib.metadata.version
         self.assertTrue(
-            __VERSION__,
+            __version__,
             manager._try_version(
                 'octodns', module=dummy_module, version='1.2.3'
             ),
diff --git a/tests/test_octodns_processor_meta.py b/tests/test_octodns_processor_meta.py
index 65ed743..c5af501 100644
--- a/tests/test_octodns_processor_meta.py
+++ b/tests/test_octodns_processor_meta.py
@@ -5,7 +5,7 @@
 from unittest import TestCase
 from unittest.mock import patch
 
-from octodns import __VERSION__
+from octodns import __version__
 from octodns.processor.meta import MetaProcessor
 from octodns.provider.plan import Plan
 from octodns.record import Create, Record, Update
@@ -67,7 +67,7 @@ class TestMetaProcessor(TestCase):
         uuid_mock.side_effect = [Exception('not used')]
         now_mock.side_effect = [Exception('not used')]
         proc = MetaProcessor('test', include_time=False, include_version=True)
-        self.assertEqual([f'octodns-version={__VERSION__}'], proc.values)
+        self.assertEqual([f'octodns-version={__version__}'], proc.values)
 
         # just provider
         proc = MetaProcessor('test', include_time=False, include_provider=True)
@@ -86,7 +86,7 @@ class TestMetaProcessor(TestCase):
         )
         self.assertEqual(
             [
-                f'octodns-version={__VERSION__}',
+                f'octodns-version={__version__}',
                 'time=the-time',
                 'uuid=abcdef-1234567890',
             ],

From 0d10ae02a193447634c29616b7d33ff1ec12e54c Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Mon, 13 Nov 2023 09:32:32 -0800
Subject: [PATCH 112/116] Update the modules action job's module list

---
 .github/workflows/modules.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/modules.yml b/.github/workflows/modules.yml
index 15e1c3e..c44dc74 100644
--- a/.github/workflows/modules.yml
+++ b/.github/workflows/modules.yml
@@ -35,10 +35,11 @@ jobs:
           - octodns/octodns-digitalocean
           - octodns/octodns-dnsimple
           - octodns/octodns-dnsmadeeasy
-          - octodns/octodns-dyn
           - octodns/octodns-easydns
+          - octodns/octodns-edgecenter
           - octodns/octodns-edgedns
           - octodns/octodns-etchosts
+          - octodns/octodns-fastly
           - octodns/octodns-gandi
           - octodns/octodns-gcore
           - octodns/octodns-googlecloud
@@ -50,12 +51,10 @@ jobs:
           - octodns/octodns-rackspace
           - octodns/octodns-route53
           - octodns/octodns-selectel
+          - octodns/octodns-spf
           - octodns/octodns-transip
           - octodns/octodns-ultra
-          # has been failing for a while now and afaict not related to octoDNS
-          # changes commenting out on 2023-07-30, will check on it again in at
-          # some point in the future and either re-enable or delete it.
-          #- sukiyaki/octodns-netbox
+          - sukiyaki/octodns-netbox
     steps:
       - uses: actions/checkout@v4
       - name: Setup python

From 8ab3384e512775c53f911c0a0324d9a97052a13e Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Mon, 13 Nov 2023 09:39:08 -0800
Subject: [PATCH 113/116] octodns-netbox is still failing, just removing it

---
 .github/workflows/modules.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/modules.yml b/.github/workflows/modules.yml
index c44dc74..3262c1a 100644
--- a/.github/workflows/modules.yml
+++ b/.github/workflows/modules.yml
@@ -54,7 +54,6 @@ jobs:
           - octodns/octodns-spf
           - octodns/octodns-transip
           - octodns/octodns-ultra
-          - sukiyaki/octodns-netbox
     steps:
       - uses: actions/checkout@v4
       - name: Setup python

From 16489d004a5f97f7bc0f8fa15493ac19b0d97303 Mon Sep 17 00:00:00 2001
From: Viranch Mehta <email@viranch.me>
Date: Tue, 14 Nov 2023 10:19:10 -0800
Subject: [PATCH 114/116] v1.3.0 bump & changelog update

---
 CHANGELOG.md        | 2 +-
 octodns/__init__.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 19c4aa1..56829fa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,4 @@
-## v1.3.0 - 2023-??-?? - ???
+## v1.3.0 - 2023-11-14 - New and improved processors
 
 #### Noteworthy changes
 
diff --git a/octodns/__init__.py b/octodns/__init__.py
index 1ae31b0..083cfbf 100644
--- a/octodns/__init__.py
+++ b/octodns/__init__.py
@@ -1,4 +1,4 @@
 'OctoDNS: DNS as code - Tools for managing DNS across multiple providers'
 
 # TODO: remove __VERSION__ w/2.x
-__version__ = __VERSION__ = '1.2.1'
+__version__ = __VERSION__ = '1.3.0'

From 2696e67fed6c5161eab3534ee033951ce12995a8 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Tue, 14 Nov 2023 10:42:50 -0800
Subject: [PATCH 115/116] Quote dunders in CHANGELOG so markdown doesn't bold
 them

---
 CHANGELOG.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 56829fa..c5c1d58 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,8 +2,9 @@
 
 #### Noteworthy changes
 
-* Added octodns.__version__ to replace octodns.__VERSION__ as the former is more
-  of a standard, per pep-8. __VERSION__ is deprecated and will go away in 2.x
+* Added `octodns.__version__` to replace `octodns.__VERSION__` as the former is
+  more of a standard, per pep-8. `__VERSION__` is deprecated and will go away
+  in 2.x
 * Fixed issues with handling of chunking large TXT values for providers that use
   the in-built `rrs` method
 * Removed code that included sha in module version number when installing from
@@ -20,7 +21,7 @@
   horizon)
 * ExcludeRootNsChanges processor that will error (or warn) if plan includes a
   change to root NS records
-* Include the octodns special section info in Record __repr__, makes it easier
+* Include the octodns special section info in `Record.__repr__`, makes it easier
   to debug things with providers that have special functionality configured
   there.
 * Most processor.filter processors now support an include_target flag that can

From 7be540b86c890723c83da479b437e4dce4d25262 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Tue, 14 Nov 2023 14:19:14 -0800
Subject: [PATCH 116/116] Record.lenient property added similar to other
 common/standard _octodns data

---
 CHANGELOG.md                  | 4 ++++
 octodns/processor/restrict.py | 2 +-
 octodns/processor/spf.py      | 2 +-
 octodns/record/base.py        | 4 ++++
 4 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c5c1d58..830c7f1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## v1.?.0 - 2023-??-?? -
+
+* Record.lenient property added similar to other common/standard _octodns data
+
 ## v1.3.0 - 2023-11-14 - New and improved processors
 
 #### Noteworthy changes
diff --git a/octodns/processor/restrict.py b/octodns/processor/restrict.py
index e585eeb..c11a21b 100644
--- a/octodns/processor/restrict.py
+++ b/octodns/processor/restrict.py
@@ -59,7 +59,7 @@ class TtlRestrictionFilter(BaseProcessor):
 
     def process_source_zone(self, zone, *args, **kwargs):
         for record in zone.records:
-            if record._octodns.get('lenient'):
+            if record.lenient:
                 continue
             if self.allowed_ttls and record.ttl not in self.allowed_ttls:
                 raise RestrictionException(
diff --git a/octodns/processor/spf.py b/octodns/processor/spf.py
index dee82ed..0d86d5e 100644
--- a/octodns/processor/spf.py
+++ b/octodns/processor/spf.py
@@ -137,7 +137,7 @@ class SpfDnsLookupProcessor(BaseProcessor):
             if record._type != 'TXT':
                 continue
 
-            if record._octodns.get('lenient'):
+            if record.lenient:
                 continue
 
             self._check_dns_lookups(record, record.values, 0)
diff --git a/octodns/record/base.py b/octodns/record/base.py
index 3b9b36e..700b65d 100644
--- a/octodns/record/base.py
+++ b/octodns/record/base.py
@@ -231,6 +231,10 @@ class Record(EqualityTupleMixin):
         except KeyError:
             return 443
 
+    @property
+    def lenient(self):
+        return self._octodns.get('lenient', False)
+
     def changes(self, other, target):
         # We're assuming we have the same name and type if we're being compared
         if self.ttl != other.ttl: