From fac662a9ec09697e7c5a95724ab1255e7fff1a27 Mon Sep 17 00:00:00 2001 From: Tobias Mueller Date: Thu, 16 Oct 2025 22:03:09 -0700 Subject: [PATCH 1/8] processors: Added a simple TTL clamping processor This will come in handy for APIs not supporting TTL outside a certain range. The Spaceship API, for example, only allows TTLs in the range of 5..3600 but rewriting a whole zone for Spaceship only seems not as convenient as clamping the values as they flow through OctoDNS. The code is coming from Claude. My prompt was: Write a simple OctoDNS processor that clamps TTL values. I followed up with: AttributeError: 'TtlClampProcessor' object has no attribute 'log' Where should self.log come from? And finally: Hm. I suspect the code is not running, somehow. It doesn't seem to be getting control. I put a raise in the process_record method but it doesn't fail. And I don't see any clamped values nor log output. So it took three attempts to make it produce something useful. --- octodns/processor/clamp.py | 63 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 octodns/processor/clamp.py diff --git a/octodns/processor/clamp.py b/octodns/processor/clamp.py new file mode 100644 index 0000000..4bfaeb5 --- /dev/null +++ b/octodns/processor/clamp.py @@ -0,0 +1,63 @@ +from logging import getLogger +from octodns.processor.base import BaseProcessor + + +class TtlClampProcessor(BaseProcessor): + """ + Processor that clamps TTL values to a specified range. + + Configuration: + min_ttl: Minimum TTL value (default: 300 seconds / 5 minutes) + max_ttl: Maximum TTL value (default: 86400 seconds / 24 hours) + + Example config.yaml: + processors: + clamp: + class: octodns.processor.clamp.TtlClampProcessor + min_ttl: 300 + max_ttl: 3600 + + zones: + example.com.: + sources: + - config + processors: + - clamp + targets: + - route53 + """ + + def __init__(self, id, min_ttl=300, max_ttl=86400): + super().__init__(id) + self.log = getLogger(f'{self.__class__.__module__}.{self.__class__.__name__}') + self.min_ttl = min_ttl + self.max_ttl = max_ttl + self.log.info( + f'TtlClampProcessor initialized: min={min_ttl}s, max={max_ttl}s' + ) + + def process_source_zone(self, desired, sources): + """ + Process records from source zone(s). + + Args: + desired: Zone object containing the desired records + sources: List of source names + + Returns: + The modified zone + """ + self.log.debug(f'Processing source zone: {desired.name}') + + for record in desired.records: + original_ttl = record.ttl + clamped_ttl = max(self.min_ttl, min(self.max_ttl, original_ttl)) + + if clamped_ttl != original_ttl: + self.log.info( + f'Clamping TTL for {record.fqdn} ({record._type}): ' + f'{original_ttl}s -> {clamped_ttl}s' + ) + record.ttl = clamped_ttl + + return desired From 6ebba0411132b805c1b28bf1dd372c5ff49fb4d8 Mon Sep 17 00:00:00 2001 From: Tobias Mueller Date: Fri, 17 Oct 2025 07:08:47 -0700 Subject: [PATCH 2/8] clamp: formatted I ran both isort and black. Let's hope that this makes the CI happy. --- octodns/processor/clamp.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/octodns/processor/clamp.py b/octodns/processor/clamp.py index 4bfaeb5..31c9e89 100644 --- a/octodns/processor/clamp.py +++ b/octodns/processor/clamp.py @@ -1,22 +1,23 @@ from logging import getLogger + from octodns.processor.base import BaseProcessor class TtlClampProcessor(BaseProcessor): """ Processor that clamps TTL values to a specified range. - + Configuration: min_ttl: Minimum TTL value (default: 300 seconds / 5 minutes) max_ttl: Maximum TTL value (default: 86400 seconds / 24 hours) - + Example config.yaml: processors: clamp: class: octodns.processor.clamp.TtlClampProcessor min_ttl: 300 max_ttl: 3600 - + zones: example.com.: sources: @@ -26,38 +27,40 @@ class TtlClampProcessor(BaseProcessor): targets: - route53 """ - + def __init__(self, id, min_ttl=300, max_ttl=86400): super().__init__(id) - self.log = getLogger(f'{self.__class__.__module__}.{self.__class__.__name__}') + self.log = getLogger( + f'{self.__class__.__module__}.{self.__class__.__name__}' + ) self.min_ttl = min_ttl self.max_ttl = max_ttl self.log.info( f'TtlClampProcessor initialized: min={min_ttl}s, max={max_ttl}s' ) - + def process_source_zone(self, desired, sources): """ Process records from source zone(s). - + Args: desired: Zone object containing the desired records sources: List of source names - + Returns: The modified zone """ self.log.debug(f'Processing source zone: {desired.name}') - + for record in desired.records: original_ttl = record.ttl clamped_ttl = max(self.min_ttl, min(self.max_ttl, original_ttl)) - + if clamped_ttl != original_ttl: self.log.info( f'Clamping TTL for {record.fqdn} ({record._type}): ' f'{original_ttl}s -> {clamped_ttl}s' ) record.ttl = clamped_ttl - + return desired From 890808464d809a8701dd9c8651e1136f8c784d96 Mon Sep 17 00:00:00 2001 From: Tobias Mueller Date: Fri, 17 Oct 2025 09:32:21 -0700 Subject: [PATCH 3/8] added changelog --- .changelog/2809d288040441ccb8e6633f514b09b0.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changelog/2809d288040441ccb8e6633f514b09b0.md diff --git a/.changelog/2809d288040441ccb8e6633f514b09b0.md b/.changelog/2809d288040441ccb8e6633f514b09b0.md new file mode 100644 index 0000000..91ca407 --- /dev/null +++ b/.changelog/2809d288040441ccb8e6633f514b09b0.md @@ -0,0 +1,5 @@ +--- +type: minor +--- + +Add processor for clamping TTLs From 5d9fd7e789c32657152b3795b39894973ae6203b Mon Sep 17 00:00:00 2001 From: Tobias Mueller Date: Fri, 17 Oct 2025 10:48:03 -0700 Subject: [PATCH 4/8] processor: clamp: Raise if TTL arguments are not logical We must not have min TTLs that are not smaller than the max TTL. --- octodns/processor/clamp.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/octodns/processor/clamp.py b/octodns/processor/clamp.py index 31c9e89..49dd241 100644 --- a/octodns/processor/clamp.py +++ b/octodns/processor/clamp.py @@ -1,6 +1,9 @@ from logging import getLogger -from octodns.processor.base import BaseProcessor +from .base import BaseProcessor, ProcessorException + +class TTLArgumentException(ProcessorException): + pass class TtlClampProcessor(BaseProcessor): @@ -33,6 +36,8 @@ class TtlClampProcessor(BaseProcessor): self.log = getLogger( f'{self.__class__.__module__}.{self.__class__.__name__}' ) + if not min_ttl <= max_ttl: + raise TTLArgumentException(f'Min TTL {min_ttl} is not lower than max TTL {max_ttl}') self.min_ttl = min_ttl self.max_ttl = max_ttl self.log.info( From d606318fdc2d9c59e169912cd37c0996631e4bf8 Mon Sep 17 00:00:00 2001 From: Tobias Mueller Date: Fri, 17 Oct 2025 10:50:50 -0700 Subject: [PATCH 5/8] clamp: Added test cases --- tests/test_octodns_processor_clamp.py | 120 ++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 tests/test_octodns_processor_clamp.py diff --git a/tests/test_octodns_processor_clamp.py b/tests/test_octodns_processor_clamp.py new file mode 100644 index 0000000..63c2de1 --- /dev/null +++ b/tests/test_octodns_processor_clamp.py @@ -0,0 +1,120 @@ +from unittest import TestCase + +from octodns.processor.clamp import TTLArgumentException, TtlClampProcessor +from octodns.record.base import Record +from octodns.zone import Zone + + +class TestClampProcessor(TestCase): + + def test_processor_min(self): + "Test the processor for clamping to the minimum" + min_ttl = 42 + processor = TtlClampProcessor('test', min_ttl=min_ttl) + + too_low_ttl = 23 + self.assertLess(too_low_ttl, min_ttl) + + zone = Zone('unit.tests.', []) + zone.add_record( + Record.new( + zone, '', {'type': 'TXT', 'ttl': too_low_ttl, 'value': 'foo'} + ) + ) + + processed_zone = processor.process_source_zone(zone.copy(), None) + self.assertNotEqual(zone, processed_zone) + + self.assertEqual(len(processed_zone.records), len(zone.records)) + self.assertEqual(len(processed_zone.records), 1) + self.assertEqual(processed_zone.records.pop().ttl, min_ttl) + + def test_processor_max(self): + "Test the processor for clamping to the maximum" + max_ttl = 4711 + processor = TtlClampProcessor('test', max_ttl=max_ttl) + + too_high_ttl = max_ttl + 1 + self.assertLess(max_ttl, too_high_ttl) + + zone = Zone('unit.tests.', []) + zone.add_record( + Record.new( + zone, '', {'type': 'TXT', 'ttl': too_high_ttl, 'value': 'foo'} + ) + ) + + processed_zone = processor.process_source_zone(zone.copy(), None) + self.assertNotEqual(zone, processed_zone) + + self.assertEqual(len(processed_zone.records), len(zone.records)) + self.assertEqual(len(processed_zone.records), 1) + self.assertEqual(processed_zone.records.pop().ttl, max_ttl) + + def test_processor_maxmin(self): + "Test the processor for unlogical arguments" + min_ttl = 42 + max_ttl = 23 + self.assertRaises( + TTLArgumentException, + TtlClampProcessor, + 'test', + min_ttl=min_ttl, + max_ttl=max_ttl, + ) + + def test_processor_minmax(self): + "Test the processor for clamping both min and max values" + min_ttl = 42 + max_ttl = 4711 + processor = TtlClampProcessor('test', min_ttl=min_ttl, max_ttl=max_ttl) + + too_low_ttl = min_ttl - 1 + too_high_ttl = max_ttl + 1 + self.assertLess(too_low_ttl, min_ttl) + self.assertLess(too_low_ttl, min_ttl) + self.assertLess(max_ttl, too_high_ttl) + + zone = Zone('unit.tests.', []) + zone.add_record( + Record.new( + zone, + 'high', + {'type': 'TXT', 'ttl': too_high_ttl, 'value': 'high'}, + ) + ) + zone.add_record( + Record.new( + zone, 'low', {'type': 'TXT', 'ttl': too_low_ttl, 'value': 'low'} + ) + ) + + processed_zone = processor.process_source_zone(zone.copy(), None) + self.assertNotEqual(zone, processed_zone) + + processed_records = sorted( + list(processed_zone.records), key=lambda r: r.ttl + ) + self.assertEqual(len(processed_records), 2) + + self.assertEqual(processed_records[0].ttl, min_ttl) + self.assertEqual(processed_records[1].ttl, max_ttl) + + def test_processor_noclamp(self): + "Test the processor for working with TTLs not requiring any clamping" + min_ttl = 23 + max_ttl = 4711 + processor = TtlClampProcessor('test', min_ttl=min_ttl, max_ttl=max_ttl) + + ttl = 42 + + self.assertLess(min_ttl, ttl) + self.assertLess(ttl, max_ttl) + + zone = Zone('unit.tests.', []) + zone.add_record( + Record.new(zone, '', {'type': 'TXT', 'ttl': ttl, 'value': 'foo'}) + ) + + processed_zone = processor.process_source_zone(zone.copy(), None) + self.assertEqual(processed_zone.records.pop().ttl, ttl) From 24881168c1b9504fbde6e306016cd6e1d4d2ef7e Mon Sep 17 00:00:00 2001 From: Tobias Mueller Date: Sat, 18 Oct 2025 12:04:24 -0700 Subject: [PATCH 6/8] clamp: let log format in place rather than f-strings --- octodns/processor/clamp.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/octodns/processor/clamp.py b/octodns/processor/clamp.py index 49dd241..acb3876 100644 --- a/octodns/processor/clamp.py +++ b/octodns/processor/clamp.py @@ -2,6 +2,7 @@ from logging import getLogger from .base import BaseProcessor, ProcessorException + class TTLArgumentException(ProcessorException): pass @@ -33,15 +34,17 @@ class TtlClampProcessor(BaseProcessor): def __init__(self, id, min_ttl=300, max_ttl=86400): super().__init__(id) - self.log = getLogger( - f'{self.__class__.__module__}.{self.__class__.__name__}' - ) + self.log = getLogger(self.__class__.__name__) if not min_ttl <= max_ttl: - raise TTLArgumentException(f'Min TTL {min_ttl} is not lower than max TTL {max_ttl}') + raise TTLArgumentException( + f'Min TTL {min_ttl} is not lower than max TTL {max_ttl}' + ) self.min_ttl = min_ttl self.max_ttl = max_ttl self.log.info( - f'TtlClampProcessor initialized: min={min_ttl}s, max={max_ttl}s' + 'TtlClampProcessor initialized: min=%ds, max=%ds', + self.min_ttl, + self.max_ttl, ) def process_source_zone(self, desired, sources): @@ -55,7 +58,7 @@ class TtlClampProcessor(BaseProcessor): Returns: The modified zone """ - self.log.debug(f'Processing source zone: {desired.name}') + self.log.debug('Processing source zone: %s', desired.name) for record in desired.records: original_ttl = record.ttl @@ -63,8 +66,11 @@ class TtlClampProcessor(BaseProcessor): if clamped_ttl != original_ttl: self.log.info( - f'Clamping TTL for {record.fqdn} ({record._type}): ' - f'{original_ttl}s -> {clamped_ttl}s' + 'Clamping TTL for %s (%s) %s -> %s', + record.fqdn, + record._type, + original_ttl, + clamped_ttl, ) record.ttl = clamped_ttl From 1a69d344b8c39e923909f3e593bebbc291242c6f Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 18 Oct 2025 13:36:50 -0700 Subject: [PATCH 7/8] std logging prefix, func names --- octodns/processor/clamp.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/octodns/processor/clamp.py b/octodns/processor/clamp.py index acb3876..68a3fe2 100644 --- a/octodns/processor/clamp.py +++ b/octodns/processor/clamp.py @@ -41,11 +41,7 @@ class TtlClampProcessor(BaseProcessor): ) self.min_ttl = min_ttl self.max_ttl = max_ttl - self.log.info( - 'TtlClampProcessor initialized: min=%ds, max=%ds', - self.min_ttl, - self.max_ttl, - ) + self.log.info('__init__: min=%ds, max=%ds', self.min_ttl, self.max_ttl) def process_source_zone(self, desired, sources): """ @@ -58,7 +54,7 @@ class TtlClampProcessor(BaseProcessor): Returns: The modified zone """ - self.log.debug('Processing source zone: %s', desired.name) + self.log.debug('process_source_zone: desired=%s', desired.name) for record in desired.records: original_ttl = record.ttl @@ -66,7 +62,7 @@ class TtlClampProcessor(BaseProcessor): if clamped_ttl != original_ttl: self.log.info( - 'Clamping TTL for %s (%s) %s -> %s', + 'process_source_zone: clamping TTL for %s (%s) %s -> %s', record.fqdn, record._type, original_ttl, From ba24a88d0a9ce05c6e32600a18a33f12807673f3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 18 Oct 2025 15:04:53 -0700 Subject: [PATCH 8/8] Add a pass at documenting the life-cycle of zones during a sync --- .../b1e000a850584f8fa14a300538a85f4f.md | 4 + docs/api.rst | 1 + docs/zone_lifecycle.rst | 154 ++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 .changelog/b1e000a850584f8fa14a300538a85f4f.md create mode 100644 docs/zone_lifecycle.rst diff --git a/.changelog/b1e000a850584f8fa14a300538a85f4f.md b/.changelog/b1e000a850584f8fa14a300538a85f4f.md new file mode 100644 index 0000000..2e9eed4 --- /dev/null +++ b/.changelog/b1e000a850584f8fa14a300538a85f4f.md @@ -0,0 +1,4 @@ +--- +type: none +--- +Add a pass at documenting the life-cycle of zones during a sync \ No newline at end of file diff --git a/docs/api.rst b/docs/api.rst index 16cd8b1..82350db 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -8,3 +8,4 @@ Developer Interface :glob: api/* + zone_lifecycle.rst \ No newline at end of file diff --git a/docs/zone_lifecycle.rst b/docs/zone_lifecycle.rst new file mode 100644 index 0000000..b64372f --- /dev/null +++ b/docs/zone_lifecycle.rst @@ -0,0 +1,154 @@ +Zone Lifecycle During Sync +========================== + +This document describes the lifecycle of a :py:class:`~octodns.zone.Zone` +object during the sync process in octoDNS. The +:py:meth:`octodns.manager.Manager.sync` method is the entry point for this +process. + +Zone Creation and Population +---------------------------- + +* **Zone object creation**: :py:class:`~octodns.zone.Zone` objects are created + by :py:meth:`octodns.manager.Manager.get_zone` with the zone name, configured + sub-zones, and threshold values from the configuration + +* **Source population**: The + :py:meth:`~octodns.source.base.BaseSource.populate` method is called for each + source to add records to the zone + + * Sources iterate through their data and call + :py:meth:`~octodns.zone.Zone.add_record` to add each record + +* **Source zone processing**: + + * :py:meth:`~octodns.processor.base.BaseProcessor.process_source_zone` is + then called for each configured processor allowing them to modify or filter + the populated zone + +Planning Phase +-------------- + +* **Plan creation**: Each target provider's + :py:meth:`~octodns.provider.base.BaseProvider.plan` method is called with the + final the desired (source) zone + +* **Existing zone population**: A new empty :py:class:`~octodns.zone.Zone` is + created to represent the target's current state + + * The target provider populates this zone via + :py:meth:`~octodns.source.base.BaseSource.populate` with ``target=True`` + and ``lenient=True`` + * This additonally return whether the zone exists in the target + +* **Desired zone copy**: A shallow copy of the desired zone is created via + :py:meth:`~octodns.zone.Zone.copy` + + * Uses copy-on-write semantics for efficiency + * Actual record copying is deferred until modifications are needed + +* **Desired zone processing**: The target provider calls + :py:meth:`~octodns.provider.base.BaseProvider._process_desired_zone` to adapt + records for the target + + * Removes unsupported record types + * Handles dynamic record support/fallback + * Handles multi-value PTR record support + * Handles root NS record support + * May warn or raise exceptions based on ``strict_supports`` setting + * Providers may overide this method to add additional checks or + modifications, they must always call super to allow the above processing + +* **Existing zone processing**: The target provider calls + :py:meth:`~octodns.provider.base.BaseProvider._process_existing_zone` to + normalize existing records + + * Filters out existing root NS records if not supported or not in desired + +* **Target zone processing**: Each processor's + :py:meth:`~octodns.processor.base.BaseProcessor.process_target_zone` is + called to modify the existing (target) zone for this provider + + * Processors can filter or modify what octoDNS sees as the current state + +* **Source and target zone processing**: Each processor calls + :py:meth:`~octodns.processor.base.BaseProcessor.process_source_and_target_zones` + with both zones + + * Allows processors to make coordinated changes to both desired and existing + states + +* **Change detection**: The existing zone's + :py:meth:`~octodns.zone.Zone.changes` method compares existing records to + desired records + + * Identifies records to create, update, or delete + * Honors record-level ``ignored``, ``included``, and ``excluded`` flags + * Skips records not supported by the target + +* **Change filtering**: The target provider's + :py:meth:`~octodns.provider.base.BaseProvider._include_change` method filters + false positive changes + + * Providers can exclude changes due to implementation details (e.g., minimum + TTL enforcement) + +* **Extra changes**: The target provider's + :py:meth:`~octodns.provider.base.BaseProvider._extra_changes` method adds + provider-specific changes + + * Allows providers to add changes for ancillary records or zone configuration + +* **Meta changes**: The target provider's + :py:meth:`~octodns.provider.base.BaseProvider._plan_meta` method provides + additional non-record change information + + * Used for zone-level settings or metadata + +* **Plan processing**: Each processor calls + :py:meth:`~octodns.processor.base.BaseProcessor.process_plan` to modify or + filter the plan + + * Processors can add, modify, or remove changes from the plan + +* **Plan finalization**: A :py:class:`~octodns.provider.plan.Plan` object is + created if changes exist + + * Contains the existing zone, desired zone, list of changes, and metadata + * Returns ``None`` if no changes are needed + +Plan Output and Safety Checks +----------------------------- + +* **Plan output**: All configured plan outputs run to display or record the + plan + + * Default is :py:class:`~octodns.provider.plan.PlanLogger` which logs the + plan + * Other outputs include :py:class:`~octodns.provider.plan.PlanJson`, + :py:class:`~octodns.provider.plan.PlanMarkdown`, and + :py:class:`~octodns.provider.plan.PlanHtml` + +* **Safety validation**: Each plan's + :py:meth:`~octodns.provider.plan.Plan.raise_if_unsafe` method checks for + dangerous/numerous changes (unless ``force=True``) + + * Validates update and delete percentages against thresholds + * Requires force for root NS record changes + * Raises :py:exc:`~octodns.provider.plan.UnsafePlan` if thresholds exceeded + +Apply Phase +----------- + +* **Change application**: Each target provider's + :py:meth:`~octodns.provider.base.BaseProvider.apply` method is called if not + in dry-run mode + + * Calls the provider's :py:meth:`~octodns.provider.base.BaseProvider._apply` + method to submit changes + * The ``_apply`` implementation is provider-specific and interacts with the + DNS provider's API + * Returns the number of changes applied + +* **Completion**: The sync process completes and returns the total number of + changes made across all zones and targets