diff --git a/.changelog/2809d288040441ccb8e6633f514b09b0.md b/.changelog/2809d288040441ccb8e6633f514b09b0.md new file mode 100644 index 0000000..91ca407 --- /dev/null +++ b/.changelog/2809d288040441ccb8e6633f514b09b0.md @@ -0,0 +1,5 @@ +--- +type: minor +--- + +Add processor for clamping TTLs diff --git a/.changelog/b1e000a850584f8fa14a300538a85f4f.md b/.changelog/b1e000a850584f8fa14a300538a85f4f.md new file mode 100644 index 0000000..2e9eed4 --- /dev/null +++ b/.changelog/b1e000a850584f8fa14a300538a85f4f.md @@ -0,0 +1,4 @@ +--- +type: none +--- +Add a pass at documenting the life-cycle of zones during a sync \ No newline at end of file diff --git a/docs/api.rst b/docs/api.rst index 16cd8b1..82350db 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -8,3 +8,4 @@ Developer Interface :glob: api/* + zone_lifecycle.rst \ No newline at end of file diff --git a/docs/zone_lifecycle.rst b/docs/zone_lifecycle.rst new file mode 100644 index 0000000..b64372f --- /dev/null +++ b/docs/zone_lifecycle.rst @@ -0,0 +1,154 @@ +Zone Lifecycle During Sync +========================== + +This document describes the lifecycle of a :py:class:`~octodns.zone.Zone` +object during the sync process in octoDNS. The +:py:meth:`octodns.manager.Manager.sync` method is the entry point for this +process. + +Zone Creation and Population +---------------------------- + +* **Zone object creation**: :py:class:`~octodns.zone.Zone` objects are created + by :py:meth:`octodns.manager.Manager.get_zone` with the zone name, configured + sub-zones, and threshold values from the configuration + +* **Source population**: The + :py:meth:`~octodns.source.base.BaseSource.populate` method is called for each + source to add records to the zone + + * Sources iterate through their data and call + :py:meth:`~octodns.zone.Zone.add_record` to add each record + +* **Source zone processing**: + + * :py:meth:`~octodns.processor.base.BaseProcessor.process_source_zone` is + then called for each configured processor allowing them to modify or filter + the populated zone + +Planning Phase +-------------- + +* **Plan creation**: Each target provider's + :py:meth:`~octodns.provider.base.BaseProvider.plan` method is called with the + final the desired (source) zone + +* **Existing zone population**: A new empty :py:class:`~octodns.zone.Zone` is + created to represent the target's current state + + * The target provider populates this zone via + :py:meth:`~octodns.source.base.BaseSource.populate` with ``target=True`` + and ``lenient=True`` + * This additonally return whether the zone exists in the target + +* **Desired zone copy**: A shallow copy of the desired zone is created via + :py:meth:`~octodns.zone.Zone.copy` + + * Uses copy-on-write semantics for efficiency + * Actual record copying is deferred until modifications are needed + +* **Desired zone processing**: The target provider calls + :py:meth:`~octodns.provider.base.BaseProvider._process_desired_zone` to adapt + records for the target + + * Removes unsupported record types + * Handles dynamic record support/fallback + * Handles multi-value PTR record support + * Handles root NS record support + * May warn or raise exceptions based on ``strict_supports`` setting + * Providers may overide this method to add additional checks or + modifications, they must always call super to allow the above processing + +* **Existing zone processing**: The target provider calls + :py:meth:`~octodns.provider.base.BaseProvider._process_existing_zone` to + normalize existing records + + * Filters out existing root NS records if not supported or not in desired + +* **Target zone processing**: Each processor's + :py:meth:`~octodns.processor.base.BaseProcessor.process_target_zone` is + called to modify the existing (target) zone for this provider + + * Processors can filter or modify what octoDNS sees as the current state + +* **Source and target zone processing**: Each processor calls + :py:meth:`~octodns.processor.base.BaseProcessor.process_source_and_target_zones` + with both zones + + * Allows processors to make coordinated changes to both desired and existing + states + +* **Change detection**: The existing zone's + :py:meth:`~octodns.zone.Zone.changes` method compares existing records to + desired records + + * Identifies records to create, update, or delete + * Honors record-level ``ignored``, ``included``, and ``excluded`` flags + * Skips records not supported by the target + +* **Change filtering**: The target provider's + :py:meth:`~octodns.provider.base.BaseProvider._include_change` method filters + false positive changes + + * Providers can exclude changes due to implementation details (e.g., minimum + TTL enforcement) + +* **Extra changes**: The target provider's + :py:meth:`~octodns.provider.base.BaseProvider._extra_changes` method adds + provider-specific changes + + * Allows providers to add changes for ancillary records or zone configuration + +* **Meta changes**: The target provider's + :py:meth:`~octodns.provider.base.BaseProvider._plan_meta` method provides + additional non-record change information + + * Used for zone-level settings or metadata + +* **Plan processing**: Each processor calls + :py:meth:`~octodns.processor.base.BaseProcessor.process_plan` to modify or + filter the plan + + * Processors can add, modify, or remove changes from the plan + +* **Plan finalization**: A :py:class:`~octodns.provider.plan.Plan` object is + created if changes exist + + * Contains the existing zone, desired zone, list of changes, and metadata + * Returns ``None`` if no changes are needed + +Plan Output and Safety Checks +----------------------------- + +* **Plan output**: All configured plan outputs run to display or record the + plan + + * Default is :py:class:`~octodns.provider.plan.PlanLogger` which logs the + plan + * Other outputs include :py:class:`~octodns.provider.plan.PlanJson`, + :py:class:`~octodns.provider.plan.PlanMarkdown`, and + :py:class:`~octodns.provider.plan.PlanHtml` + +* **Safety validation**: Each plan's + :py:meth:`~octodns.provider.plan.Plan.raise_if_unsafe` method checks for + dangerous/numerous changes (unless ``force=True``) + + * Validates update and delete percentages against thresholds + * Requires force for root NS record changes + * Raises :py:exc:`~octodns.provider.plan.UnsafePlan` if thresholds exceeded + +Apply Phase +----------- + +* **Change application**: Each target provider's + :py:meth:`~octodns.provider.base.BaseProvider.apply` method is called if not + in dry-run mode + + * Calls the provider's :py:meth:`~octodns.provider.base.BaseProvider._apply` + method to submit changes + * The ``_apply`` implementation is provider-specific and interacts with the + DNS provider's API + * Returns the number of changes applied + +* **Completion**: The sync process completes and returns the total number of + changes made across all zones and targets diff --git a/octodns/processor/clamp.py b/octodns/processor/clamp.py new file mode 100644 index 0000000..68a3fe2 --- /dev/null +++ b/octodns/processor/clamp.py @@ -0,0 +1,73 @@ +from logging import getLogger + +from .base import BaseProcessor, ProcessorException + + +class TTLArgumentException(ProcessorException): + pass + + +class TtlClampProcessor(BaseProcessor): + """ + Processor that clamps TTL values to a specified range. + + Configuration: + min_ttl: Minimum TTL value (default: 300 seconds / 5 minutes) + max_ttl: Maximum TTL value (default: 86400 seconds / 24 hours) + + Example config.yaml: + processors: + clamp: + class: octodns.processor.clamp.TtlClampProcessor + min_ttl: 300 + max_ttl: 3600 + + zones: + example.com.: + sources: + - config + processors: + - clamp + targets: + - route53 + """ + + def __init__(self, id, min_ttl=300, max_ttl=86400): + super().__init__(id) + self.log = getLogger(self.__class__.__name__) + if not min_ttl <= max_ttl: + raise TTLArgumentException( + f'Min TTL {min_ttl} is not lower than max TTL {max_ttl}' + ) + self.min_ttl = min_ttl + self.max_ttl = max_ttl + self.log.info('__init__: min=%ds, max=%ds', self.min_ttl, self.max_ttl) + + def process_source_zone(self, desired, sources): + """ + Process records from source zone(s). + + Args: + desired: Zone object containing the desired records + sources: List of source names + + Returns: + The modified zone + """ + self.log.debug('process_source_zone: desired=%s', desired.name) + + for record in desired.records: + original_ttl = record.ttl + clamped_ttl = max(self.min_ttl, min(self.max_ttl, original_ttl)) + + if clamped_ttl != original_ttl: + self.log.info( + 'process_source_zone: clamping TTL for %s (%s) %s -> %s', + record.fqdn, + record._type, + original_ttl, + clamped_ttl, + ) + record.ttl = clamped_ttl + + return desired diff --git a/tests/test_octodns_processor_clamp.py b/tests/test_octodns_processor_clamp.py new file mode 100644 index 0000000..63c2de1 --- /dev/null +++ b/tests/test_octodns_processor_clamp.py @@ -0,0 +1,120 @@ +from unittest import TestCase + +from octodns.processor.clamp import TTLArgumentException, TtlClampProcessor +from octodns.record.base import Record +from octodns.zone import Zone + + +class TestClampProcessor(TestCase): + + def test_processor_min(self): + "Test the processor for clamping to the minimum" + min_ttl = 42 + processor = TtlClampProcessor('test', min_ttl=min_ttl) + + too_low_ttl = 23 + self.assertLess(too_low_ttl, min_ttl) + + zone = Zone('unit.tests.', []) + zone.add_record( + Record.new( + zone, '', {'type': 'TXT', 'ttl': too_low_ttl, 'value': 'foo'} + ) + ) + + processed_zone = processor.process_source_zone(zone.copy(), None) + self.assertNotEqual(zone, processed_zone) + + self.assertEqual(len(processed_zone.records), len(zone.records)) + self.assertEqual(len(processed_zone.records), 1) + self.assertEqual(processed_zone.records.pop().ttl, min_ttl) + + def test_processor_max(self): + "Test the processor for clamping to the maximum" + max_ttl = 4711 + processor = TtlClampProcessor('test', max_ttl=max_ttl) + + too_high_ttl = max_ttl + 1 + self.assertLess(max_ttl, too_high_ttl) + + zone = Zone('unit.tests.', []) + zone.add_record( + Record.new( + zone, '', {'type': 'TXT', 'ttl': too_high_ttl, 'value': 'foo'} + ) + ) + + processed_zone = processor.process_source_zone(zone.copy(), None) + self.assertNotEqual(zone, processed_zone) + + self.assertEqual(len(processed_zone.records), len(zone.records)) + self.assertEqual(len(processed_zone.records), 1) + self.assertEqual(processed_zone.records.pop().ttl, max_ttl) + + def test_processor_maxmin(self): + "Test the processor for unlogical arguments" + min_ttl = 42 + max_ttl = 23 + self.assertRaises( + TTLArgumentException, + TtlClampProcessor, + 'test', + min_ttl=min_ttl, + max_ttl=max_ttl, + ) + + def test_processor_minmax(self): + "Test the processor for clamping both min and max values" + min_ttl = 42 + max_ttl = 4711 + processor = TtlClampProcessor('test', min_ttl=min_ttl, max_ttl=max_ttl) + + too_low_ttl = min_ttl - 1 + too_high_ttl = max_ttl + 1 + self.assertLess(too_low_ttl, min_ttl) + self.assertLess(too_low_ttl, min_ttl) + self.assertLess(max_ttl, too_high_ttl) + + zone = Zone('unit.tests.', []) + zone.add_record( + Record.new( + zone, + 'high', + {'type': 'TXT', 'ttl': too_high_ttl, 'value': 'high'}, + ) + ) + zone.add_record( + Record.new( + zone, 'low', {'type': 'TXT', 'ttl': too_low_ttl, 'value': 'low'} + ) + ) + + processed_zone = processor.process_source_zone(zone.copy(), None) + self.assertNotEqual(zone, processed_zone) + + processed_records = sorted( + list(processed_zone.records), key=lambda r: r.ttl + ) + self.assertEqual(len(processed_records), 2) + + self.assertEqual(processed_records[0].ttl, min_ttl) + self.assertEqual(processed_records[1].ttl, max_ttl) + + def test_processor_noclamp(self): + "Test the processor for working with TTLs not requiring any clamping" + min_ttl = 23 + max_ttl = 4711 + processor = TtlClampProcessor('test', min_ttl=min_ttl, max_ttl=max_ttl) + + ttl = 42 + + self.assertLess(min_ttl, ttl) + self.assertLess(ttl, max_ttl) + + zone = Zone('unit.tests.', []) + zone.add_record( + Record.new(zone, '', {'type': 'TXT', 'ttl': ttl, 'value': 'foo'}) + ) + + processed_zone = processor.process_source_zone(zone.copy(), None) + self.assertEqual(processed_zone.records.pop().ttl, ttl)