diff --git a/.changelog/2809d288040441ccb8e6633f514b09b0.md b/.changelog/2809d288040441ccb8e6633f514b09b0.md new file mode 100644 index 0000000..91ca407 --- /dev/null +++ b/.changelog/2809d288040441ccb8e6633f514b09b0.md @@ -0,0 +1,5 @@ +--- +type: minor +--- + +Add processor for clamping TTLs diff --git a/.changelog/3deb7273fd0b4516a6c66636e3c1ed34.md b/.changelog/3deb7273fd0b4516a6c66636e3c1ed34.md new file mode 100644 index 0000000..455e625 --- /dev/null +++ b/.changelog/3deb7273fd0b4516a6c66636e3c1ed34.md @@ -0,0 +1,4 @@ +--- +type: none +--- +Remove really old secret clearing code that doesn't apply anymore \ No newline at end of file diff --git a/.changelog/3f8fbaa8c6324b649fa64ccbec8f8f5c.md b/.changelog/3f8fbaa8c6324b649fa64ccbec8f8f5c.md new file mode 100644 index 0000000..d0f75ed --- /dev/null +++ b/.changelog/3f8fbaa8c6324b649fa64ccbec8f8f5c.md @@ -0,0 +1,4 @@ +--- +type: none +--- +Add comprehensive API documentation to Zone class and related exceptions diff --git a/.changelog/6fa199b732dc40c58481d369ce992eb3.md b/.changelog/6fa199b732dc40c58481d369ce992eb3.md new file mode 100644 index 0000000..217573b --- /dev/null +++ b/.changelog/6fa199b732dc40c58481d369ce992eb3.md @@ -0,0 +1,4 @@ +--- +type: none +--- +Spelling/type-o corrections in docs \ No newline at end of file diff --git a/.changelog/99fb6721f2d340ed933d7407c91622ca.md b/.changelog/99fb6721f2d340ed933d7407c91622ca.md new file mode 100644 index 0000000..2ba6b0d --- /dev/null +++ b/.changelog/99fb6721f2d340ed933d7407c91622ca.md @@ -0,0 +1,4 @@ +--- +type: none +--- +Pass at adding AI-assisted API documentation \ No newline at end of file diff --git a/.changelog/b1e000a850584f8fa14a300538a85f4f.md b/.changelog/b1e000a850584f8fa14a300538a85f4f.md new file mode 100644 index 0000000..2e9eed4 --- /dev/null +++ b/.changelog/b1e000a850584f8fa14a300538a85f4f.md @@ -0,0 +1,4 @@ +--- +type: none +--- +Add a pass at documenting the life-cycle of zones during a sync \ No newline at end of file diff --git a/.changelog/bb94cc6d9dde44b38d875cf17be4bbce.md b/.changelog/bb94cc6d9dde44b38d875cf17be4bbce.md new file mode 100644 index 0000000..da44967 --- /dev/null +++ b/.changelog/bb94cc6d9dde44b38d875cf17be4bbce.md @@ -0,0 +1,4 @@ +--- +type: minor +--- +Full rewrite of octodns-report: support for IPv6 resolvers, async names resolution and JSON output \ No newline at end of file diff --git a/.changelog/cbeaa629126c4d7bbb6f8b59e76b8835.md b/.changelog/cbeaa629126c4d7bbb6f8b59e76b8835.md new file mode 100644 index 0000000..338be56 --- /dev/null +++ b/.changelog/cbeaa629126c4d7bbb6f8b59e76b8835.md @@ -0,0 +1,4 @@ +--- +type: none +--- +Fix venv activation not working when using a custom VENV_NAME \ No newline at end of file diff --git a/.git_hooks_pre-commit b/.git_hooks_pre-commit index 3b5f6ac..cb9fe46 100755 --- a/.git_hooks_pre-commit +++ b/.git_hooks_pre-commit @@ -1,13 +1,14 @@ -#!/bin/sh +#!/bin/bash -set -e +# Scripts path +SCRIPT_PATH="$( dirname -- "$( readlink -f -- "${0}"; )"; )/script" +# Activate OctoDNS Python venv +source "${SCRIPT_PATH}/common.sh" -HOOKS=$(dirname "$0") -GIT=$(dirname "$HOOKS") -ROOT=$(dirname "$GIT") - -. "$ROOT/env/bin/activate" -"$ROOT/script/changelog" check -"$ROOT/script/lint" -"$ROOT/script/format" --check --quiet || (echo "Formatting check failed, run ./script/format" && exit 1) -"$ROOT/script/coverage" +./script/changelog check +./script/lint +./script/format --check --quiet || ( + echo "Formatting check failed, run ./script/format" && + exit 1 +) +./script/coverage diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 7dbc6c7..044ea29 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -15,10 +15,10 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Install dependencies + - name: Setup Environment run: | python -m pip install --upgrade pip - pip install -r requirements.txt -r requirements-dev.txt + ./script/bootstrap - name: Changelog Check run: | ./script/changelog check diff --git a/docs/api.rst b/docs/api.rst index 16cd8b1..82350db 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -8,3 +8,4 @@ Developer Interface :glob: api/* + zone_lifecycle.rst \ No newline at end of file diff --git a/docs/dynamic_records.rst b/docs/dynamic_records.rst index ca85a2e..b4bb88f 100644 --- a/docs/dynamic_records.rst +++ b/docs/dynamic_records.rst @@ -59,14 +59,14 @@ An Annotated Example - geos: # AF-ZA was sent to apac above and the rest of AF else goes to eu here, # sub-locations (e.g. AF-ZA) should come before their parents (AF.) If a - # more specific geo occured after a general one requests in that + # more specific geo occurred after a general one requests in that # location would have already matched the previous rule. For the same # reasons locations may not be repeated in multiple rules. - AF - EU pool: eu # No geos means match all queries, the final rule should generally be a - # "catch-all", served to any requests that didn't match the preceeding + # "catch-all", served to any requests that didn't match the preceding # rules. The catch-all is the only case where a pool may be re-used. - pool: na ttl: 60 @@ -75,7 +75,7 @@ An Annotated Example # should be a superset of the catch-all pool and include enough capacity to # try and serve all global requests (with degraded performance.) The main # case they will come into play is if all dynamic healthchecks are failing, - # either on the service side or if the providers systems are expeiencing + # either on the service side or if the providers systems are experiencing # problems. They will also be used for when the record is pushed to a # provider that doesn't support dynamic records. values: diff --git a/docs/dynamic_zone_config.rst b/docs/dynamic_zone_config.rst index c6cd36b..78adf31 100644 --- a/docs/dynamic_zone_config.rst +++ b/docs/dynamic_zone_config.rst @@ -69,13 +69,13 @@ The following octoDNS configuration would match them as described in comments:: zones: # the names here do not really matter beyond starting with a *, it is a - # reccomended best practice to match the glob, but not required. It will be + # recommended best practice to match the glob, but not required. It will be # used in logging to aid in debugging. # they are applied in the order defined and once claimed a zone is no # longer available for matching - # everytyhing is available for matching + # everything is available for matching '*internal.net': # we only want the private zones here and they are all under # internet.net. so this glob will claim them. @@ -142,7 +142,7 @@ The following octoDNS configuration would match them as described in comments:: # regexes are too ugly to use as names, so these have useful info for # logging/debugging - # everytyhing is available for matching + # everything is available for matching '*us-east-1': # we only want the private zones here and they are all under # internet.net. So this regex will claim them, yes this could be done @@ -154,7 +154,7 @@ The following octoDNS configuration would match them as described in comments:: # only push it to the us-east-1 provider - us-east-1 - # everytyhing with the exception of the us-east-1 .net zones are available + # everything with the exception of the us-east-1 .net zones are available '*us-west-2': regex: '^.*us-west-2.*.net.$' sources: diff --git a/docs/zone_lifecycle.rst b/docs/zone_lifecycle.rst new file mode 100644 index 0000000..b64372f --- /dev/null +++ b/docs/zone_lifecycle.rst @@ -0,0 +1,154 @@ +Zone Lifecycle During Sync +========================== + +This document describes the lifecycle of a :py:class:`~octodns.zone.Zone` +object during the sync process in octoDNS. The +:py:meth:`octodns.manager.Manager.sync` method is the entry point for this +process. + +Zone Creation and Population +---------------------------- + +* **Zone object creation**: :py:class:`~octodns.zone.Zone` objects are created + by :py:meth:`octodns.manager.Manager.get_zone` with the zone name, configured + sub-zones, and threshold values from the configuration + +* **Source population**: The + :py:meth:`~octodns.source.base.BaseSource.populate` method is called for each + source to add records to the zone + + * Sources iterate through their data and call + :py:meth:`~octodns.zone.Zone.add_record` to add each record + +* **Source zone processing**: + + * :py:meth:`~octodns.processor.base.BaseProcessor.process_source_zone` is + then called for each configured processor allowing them to modify or filter + the populated zone + +Planning Phase +-------------- + +* **Plan creation**: Each target provider's + :py:meth:`~octodns.provider.base.BaseProvider.plan` method is called with the + final the desired (source) zone + +* **Existing zone population**: A new empty :py:class:`~octodns.zone.Zone` is + created to represent the target's current state + + * The target provider populates this zone via + :py:meth:`~octodns.source.base.BaseSource.populate` with ``target=True`` + and ``lenient=True`` + * This additonally return whether the zone exists in the target + +* **Desired zone copy**: A shallow copy of the desired zone is created via + :py:meth:`~octodns.zone.Zone.copy` + + * Uses copy-on-write semantics for efficiency + * Actual record copying is deferred until modifications are needed + +* **Desired zone processing**: The target provider calls + :py:meth:`~octodns.provider.base.BaseProvider._process_desired_zone` to adapt + records for the target + + * Removes unsupported record types + * Handles dynamic record support/fallback + * Handles multi-value PTR record support + * Handles root NS record support + * May warn or raise exceptions based on ``strict_supports`` setting + * Providers may overide this method to add additional checks or + modifications, they must always call super to allow the above processing + +* **Existing zone processing**: The target provider calls + :py:meth:`~octodns.provider.base.BaseProvider._process_existing_zone` to + normalize existing records + + * Filters out existing root NS records if not supported or not in desired + +* **Target zone processing**: Each processor's + :py:meth:`~octodns.processor.base.BaseProcessor.process_target_zone` is + called to modify the existing (target) zone for this provider + + * Processors can filter or modify what octoDNS sees as the current state + +* **Source and target zone processing**: Each processor calls + :py:meth:`~octodns.processor.base.BaseProcessor.process_source_and_target_zones` + with both zones + + * Allows processors to make coordinated changes to both desired and existing + states + +* **Change detection**: The existing zone's + :py:meth:`~octodns.zone.Zone.changes` method compares existing records to + desired records + + * Identifies records to create, update, or delete + * Honors record-level ``ignored``, ``included``, and ``excluded`` flags + * Skips records not supported by the target + +* **Change filtering**: The target provider's + :py:meth:`~octodns.provider.base.BaseProvider._include_change` method filters + false positive changes + + * Providers can exclude changes due to implementation details (e.g., minimum + TTL enforcement) + +* **Extra changes**: The target provider's + :py:meth:`~octodns.provider.base.BaseProvider._extra_changes` method adds + provider-specific changes + + * Allows providers to add changes for ancillary records or zone configuration + +* **Meta changes**: The target provider's + :py:meth:`~octodns.provider.base.BaseProvider._plan_meta` method provides + additional non-record change information + + * Used for zone-level settings or metadata + +* **Plan processing**: Each processor calls + :py:meth:`~octodns.processor.base.BaseProcessor.process_plan` to modify or + filter the plan + + * Processors can add, modify, or remove changes from the plan + +* **Plan finalization**: A :py:class:`~octodns.provider.plan.Plan` object is + created if changes exist + + * Contains the existing zone, desired zone, list of changes, and metadata + * Returns ``None`` if no changes are needed + +Plan Output and Safety Checks +----------------------------- + +* **Plan output**: All configured plan outputs run to display or record the + plan + + * Default is :py:class:`~octodns.provider.plan.PlanLogger` which logs the + plan + * Other outputs include :py:class:`~octodns.provider.plan.PlanJson`, + :py:class:`~octodns.provider.plan.PlanMarkdown`, and + :py:class:`~octodns.provider.plan.PlanHtml` + +* **Safety validation**: Each plan's + :py:meth:`~octodns.provider.plan.Plan.raise_if_unsafe` method checks for + dangerous/numerous changes (unless ``force=True``) + + * Validates update and delete percentages against thresholds + * Requires force for root NS record changes + * Raises :py:exc:`~octodns.provider.plan.UnsafePlan` if thresholds exceeded + +Apply Phase +----------- + +* **Change application**: Each target provider's + :py:meth:`~octodns.provider.base.BaseProvider.apply` method is called if not + in dry-run mode + + * Calls the provider's :py:meth:`~octodns.provider.base.BaseProvider._apply` + method to submit changes + * The ``_apply`` implementation is provider-specific and interacts with the + DNS provider's API + * Returns the number of changes applied + +* **Completion**: The sync process completes and returns the total number of + changes made across all zones and targets diff --git a/octodns/cmds/report.py b/octodns/cmds/report.py index 80de6b0..60180c0 100755 --- a/octodns/cmds/report.py +++ b/octodns/cmds/report.py @@ -3,25 +3,48 @@ Octo-DNS Reporter ''' -import re -from concurrent.futures import ThreadPoolExecutor +from asyncio import Semaphore, new_event_loop, wait +from collections import defaultdict +from csv import QUOTE_NONE, writer +from io import StringIO +from ipaddress import ip_address +from json import dump from logging import getLogger -from sys import stdout - -from dns.exception import Timeout -from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers, Resolver, query +from sys import exit + +from dns.asyncresolver import Resolver as AsyncResolver +from dns.resolver import ( + NXDOMAIN, + YXDOMAIN, + LifetimeTimeout, + NoAnswer, + NoNameservers, + resolve, +) from octodns.cmds.args import ArgumentParser from octodns.manager import Manager -class AsyncResolver(Resolver): - def __init__(self, num_workers, *args, **kwargs): - super().__init__(*args, **kwargs) - self.executor = ThreadPoolExecutor(max_workers=num_workers) +async def async_resolve(record, resolver, timeout, limit): + async with limit: + r = AsyncResolver(configure=False) + r.lifetime = timeout + r.nameservers = [resolver] + + try: + query = await r.resolve(qname=record.fqdn, rdtype=record._type) + answer = sorted([str(a) for a in query]) + except (NoAnswer, NoNameservers): + answer = ['*no answer*'] + except NXDOMAIN: + answer = ['*does not exist*'] + except YXDOMAIN: + answer = ['*should not exist*'] + except LifetimeTimeout: + answer = ['*timeout*'] - def query(self, *args, **kwargs): - return self.executor.submit(super().query, *args, **kwargs) + return [record, resolver, answer] def main(): @@ -41,10 +64,22 @@ def main(): help='Source(s) to pull data from', ) parser.add_argument( - '--num-workers', default=4, help='Number of background workers' + '--concurrency', + type=int, + default=4, + help='Maximum number of concurrent DNS queries', + ) + parser.add_argument( + '--timeout', + type=float, + default=1, + help='Number seconds to wait for an answer', ) parser.add_argument( - '--timeout', default=1, help='Number seconds to wait for an answer' + '--output-format', + choices=['csv', 'json'], + default='csv', + help='Output format', ) parser.add_argument( '--lenient', @@ -52,13 +87,17 @@ def main(): default=False, help='Ignore record validations and do a best effort dump', ) - parser.add_argument('server', nargs='+', help='Servers to query') + parser.add_argument('server', nargs='+', help='DNS resolver to query') args = parser.parse_args() + concurrency = args.concurrency + timeout = args.timeout + output_format = args.output_format manager = Manager(args.config_file) log = getLogger('report') + log.info(f'concurrency={concurrency} timeout={timeout}') try: sources = [manager.providers[source] for source in args.source] @@ -69,49 +108,104 @@ def main(): for source in sources: source.populate(zone, lenient=args.lenient) - servers = ','.join(args.server) - print(f'name,type,ttl,{servers},consistent') + servers = args.server resolvers = [] - ip_addr_re = re.compile(r'^[\d\.]+$') - for server in args.server: - resolver = AsyncResolver( - configure=False, num_workers=int(args.num_workers) - ) - if not ip_addr_re.match(server): - server = str(query(server, 'A')[0]) - log.info('server=%s', server) - resolver.nameservers = [server] - resolver.lifetime = int(args.timeout) - resolvers.append(resolver) - - queries = {} + for server in servers: + resolver = None + is_hostname = False + + try: + ip = ip_address(server) + # "2001:4860:4860:0:0:0:0:8888" => "2001:4860:4860::8888" + resolver = ip.compressed + + # The specified server isn't a valid IP address, maybe it's a valid + # hostname? So we try to resolve it. + except ValueError: + # IPv4 first, then IPv6. + for rrtype in ['A', 'AAAA']: + try: + query = resolve(server, rrtype) + resolver = str(query.rrset[0]) + is_hostname = True + # Exit on first IP address found. + break + + # NXDOMAIN, NoAnswer, NoNameservers... + except Exception: + continue + + if resolver and resolver not in resolvers: + if not is_hostname: + log.info(f'server={resolver}') + else: + log.info(f'server={resolver} ({server})') + + resolvers.append(resolver) + + if not resolvers: + print(f'Error: No valid resolver specified ({", ".join(servers)})') + exit(1) + + loop = new_event_loop() + limit = Semaphore(concurrency) + tasks = [] for record in sorted(zone.records): - queries[record] = [ - r.query(record.fqdn, record._type) for r in resolvers - ] - - for record, futures in sorted(queries.items(), key=lambda d: d[0]): - stdout.write(record.decoded_fqdn) - stdout.write(',') - stdout.write(record._type) - stdout.write(',') - stdout.write(str(record.ttl)) - compare = {} - for future in futures: - stdout.write(',') - try: - answers = [str(r) for r in future.result()] - except (NoAnswer, NoNameservers): - answers = ['*no answer*'] - except NXDOMAIN: - answers = ['*does not exist*'] - except Timeout: - answers = ['*timeout*'] - stdout.write(' '.join(answers)) - # sorting to ignore order - answers = '*:*'.join(sorted(answers)).lower() - compare[answers] = True - stdout.write(',True\n' if len(compare) == 1 else ',False\n') + for resolver in resolvers: + tasks.append( + loop.create_task( + async_resolve(record, resolver, timeout, limit) + ) + ) + + queries = defaultdict(dict) + done, _ = loop.run_until_complete(wait(tasks)) + for task in done: + _record, _resolver, _answer = task.result() + queries[_record][_resolver] = _answer + + loop.close() + + output = StringIO() + if output_format == 'csv': + csvout = writer(output, quoting=QUOTE_NONE, quotechar=None) + csvheader = ['Name', 'Type', 'TTL'] + csvheader = [*csvheader, *resolvers] + csvheader.append('Consistent') + csvout.writerow(csvheader) + + for record, answers in sorted(queries.items()): + csvrow = [record.decoded_fqdn, record._type, record.ttl] + values_check = {} + + for resolver in resolvers: + answer = ' '.join(answers.get(resolver, [])) + values_check[answer.lower()] = True + csvrow.append(answer) + + csvrow.append(bool(len(values_check) == 1)) + csvout.writerow(csvrow) + + elif output_format == 'json': + jsonout = defaultdict(lambda: defaultdict(dict)) + for record, answers in sorted(queries.items()): + values_check = {} + + for resolver in resolvers: + # Stripping the surrounding quotes of TXT records values to + # avoid them being unnecessarily escaped by JSON module. + answer = [a.strip('"') for a in answers.get(resolver, [])] + jsonout[record.decoded_fqdn][record._type][resolver] = answer + values_check[' '.join(answer).lower()] = True + + jsonout[record.fqdn][record._type]['consistent'] = bool( + len(values_check) == 1 + ) + + dump(jsonout, output) + + print(output.getvalue()) + output.close() if __name__ == '__main__': diff --git a/octodns/processor/base.py b/octodns/processor/base.py index 277d0af..d2f3d97 100644 --- a/octodns/processor/base.py +++ b/octodns/processor/base.py @@ -4,93 +4,207 @@ class ProcessorException(Exception): + ''' + Exception raised when a processor encounters an error during processing. + + A subclass of this exception can be raised by processors when they + encounter invalid configurations, unsupported operations, or other + processing errors. + ''' + pass class BaseProcessor(object): + ''' + Base class for all octoDNS processors. + + Processors provide hooks into the octoDNS sync process to modify zones and + records at various stages. They can be used to filter, transform, or + validate DNS records before planning and applying changes. + + Processors are executed in the order they are configured and can modify: + + - **Source zones** (after sources populate, before planning) + - **Target zones** (after target populates, before planning) + - **Source and target zones** (just before computing changes) + - **Plans** (after planning, before applying) + + Subclasses should override one or more of the ``process_*`` methods to + implement custom processing logic. + + Example usage:: + + processors: + my-processor: + class: my.custom.processor.MyProcessor + # processor-specific configuration + + zones: + example.com.: + sources: + - config + processors: + - my-processor + targets: + - route53 + + See Also: + - :class:`octodns.processor.filter.TypeAllowlistFilter` + - :class:`octodns.processor.ownership.OwnershipProcessor` + - :class:`octodns.processor.acme.AcmeMangingProcessor` + ''' + def __init__(self, name): + ''' + Initialize the processor. + + :param name: Unique identifier for this processor instance. Used in + logging and configuration references. + :type name: str + + .. note:: + The ``name`` parameter is deprecated and will be removed in + version 2.0. Use ``id`` instead. + ''' # TODO: name is DEPRECATED, remove in 2.0 self.id = self.name = name def process_source_zone(self, desired, sources): ''' + Process the desired zone after all sources have populated. + Called after all sources have completed populate. Provides an - opportunity for the processor to modify the desired `Zone` that targets + opportunity for the processor to modify the desired zone that targets will receive. - - Will see `desired` after any modifications done by - `Provider._process_desired_zone` and processors configured to run - before this one. - - May modify `desired` directly. - - Must return `desired` which will normally be the `desired` param. - - Must not modify records directly, `record.copy` should be called, - the results of which can be modified, and then `Zone.add_record` may - be used with `replace=True`. - - May call `Zone.remove_record` to remove records from `desired`. - - Sources may be empty, as will be the case for aliased zones. + :param desired: The desired zone state after all sources have populated. + This zone will be used as the target state for planning. + :type desired: octodns.zone.Zone + :param sources: List of source providers that populated the zone. May be + empty for aliased zones. + :type sources: list[octodns.provider.base.BaseProvider] + + :return: The modified desired zone, typically the same object passed in. + :rtype: octodns.zone.Zone + + .. important:: + - Will see ``desired`` after any modifications done by + ``Provider._process_desired_zone`` and processors configured to run + before this one. + - May modify ``desired`` directly. + - Must return ``desired`` which will normally be the ``desired`` param. + - Must not modify records directly; ``record.copy`` should be called, + the results of which can be modified, and then ``Zone.add_record`` + may be used with ``replace=True``. + - May call ``Zone.remove_record`` to remove records from ``desired``. + - Sources may be empty, as will be the case for aliased zones. ''' return desired def process_target_zone(self, existing, target): ''' - Called after a target has completed `populate`, before changes are - computed between `existing` and `desired`. This provides an opportunity - to modify the `existing` `Zone`. - - - Will see `existing` after any modifications done by processors - configured to run before this one. - - May modify `existing` directly. - - Must return `existing` which will normally be the `existing` param. - - Must not modify records directly, `record.copy` should be called, - the results of which can be modified, and then `Zone.add_record` may - be used with `replace=True`. - - May call `Zone.remove_record` to remove records from `existing`. + Process the existing zone after the target has populated. + + Called after a target has completed ``populate``, before changes are + computed between ``existing`` and ``desired``. This provides an + opportunity to modify the existing zone state. + + :param existing: The current zone state from the target provider. + :type existing: octodns.zone.Zone + :param target: The target provider that populated the existing zone. + :type target: octodns.provider.base.BaseProvider + + :return: The modified existing zone, typically the same object passed in. + :rtype: octodns.zone.Zone + + .. important:: + - Will see ``existing`` after any modifications done by processors + configured to run before this one. + - May modify ``existing`` directly. + - Must return ``existing`` which will normally be the ``existing`` param. + - Must not modify records directly; ``record.copy`` should be called, + the results of which can be modified, and then ``Zone.add_record`` + may be used with ``replace=True``. + - May call ``Zone.remove_record`` to remove records from ``existing``. ''' return existing def process_source_and_target_zones(self, desired, existing, target): ''' - Called just prior to computing changes for ``target`` between - ``desired`` and `existing`. Provides an opportunity for the processor - to modify either the desired or existing ``Zone`` that will be used to - compute the changes and create the initial plan. - - - Will see ``desired`` after any modifications done by - ``Provider._process_desired_zone`` and all processors via - ``Processor.process_source_zone`` - - Will see ``existing`` after any modifications done by all processors - via ``Processor.process_target_zone`` - - Will see both ``desired`` and ``existing`` after any modifications - done by any processors configured to run before this one via - ``Processor.process_source_and_target_zones``. - - May modify ``desired`` directly. - - Must return ``desired`` which will normally be the ``desired`` param. - - May modify ``existing`` directly. - - Must return ``existing`` which will normally be the ``existing`` - param. - - Must not modify records directly, ``record.copy`` should be called, - the results of which can be modified, and then ``Zone.add_record`` - may be used with ``replace=True``. - - May call ``Zone.remove_record`` to remove records from ``desired``. - - May call ``Zone.remove_record`` to remove records from ``existing``. + Process both desired and existing zones before computing changes. + + Called just prior to computing changes for the target provider between + ``desired`` and ``existing``. Provides an opportunity for the processor + to modify either or both zones that will be used to compute the changes + and create the initial plan. + + :param desired: The desired zone state after all source processing. + :type desired: octodns.zone.Zone + :param existing: The existing zone state after all target processing. + :type existing: octodns.zone.Zone + :param target: The target provider for which changes will be computed. + :type target: octodns.provider.base.BaseProvider + + :return: A tuple of (desired, existing) zones, typically the same + objects passed in. + :rtype: tuple[octodns.zone.Zone, octodns.zone.Zone] + + .. important:: + - Will see ``desired`` after any modifications done by + ``Provider._process_desired_zone`` and all processors via + ``Processor.process_source_zone``. + - Will see ``existing`` after any modifications done by all processors + via ``Processor.process_target_zone``. + - Will see both ``desired`` and ``existing`` after any modifications + done by any processors configured to run before this one via + ``Processor.process_source_and_target_zones``. + - May modify ``desired`` directly. + - Must return ``desired`` which will normally be the ``desired`` param. + - May modify ``existing`` directly. + - Must return ``existing`` which will normally be the ``existing`` + param. + - Must not modify records directly; ``record.copy`` should be called, + the results of which can be modified, and then ``Zone.add_record`` + may be used with ``replace=True``. + - May call ``Zone.remove_record`` to remove records from ``desired``. + - May call ``Zone.remove_record`` to remove records from ``existing``. ''' return desired, existing def process_plan(self, plan, sources, target): ''' + Process the plan after it has been computed. + Called after the planning phase has completed. Provides an opportunity - for the processors to modify the plan thus changing the actions that + for the processor to modify the plan, thus changing the actions that will be displayed and potentially applied. - - `plan` may be None if no changes were detected, if so a `Plan` may - still be created and returned. - - May modify `plan.changes` directly or create a new `Plan`. - - Does not have to modify `plan.desired` and/or `plan.existing` to line - up with any modifications made to `plan.changes`. - - Should copy over `plan.exists`, `plan.update_pcent_threshold`, and - `plan.delete_pcent_threshold` when creating a new `Plan`. - - Must return a `Plan` which may be `plan` or can be a newly created - one `plan.desired` and `plan.existing` copied over as-is or modified. + :param plan: The computed plan containing the changes to be applied. + May be None if no changes were detected. + :type plan: octodns.provider.plan.Plan or None + :param sources: List of source providers for this zone. May be empty + for aliased zones. + :type sources: list[octodns.provider.base.BaseProvider] + :param target: The target provider for which the plan was created. + :type target: octodns.provider.base.BaseProvider + + :return: The modified plan, which may be the same object passed in, + a newly created Plan, or None if no changes are needed. + :rtype: octodns.provider.plan.Plan or None + + .. important:: + - ``plan`` may be None if no changes were detected; if so, a ``Plan`` + may still be created and returned. + - May modify ``plan.changes`` directly or create a new ``Plan``. + - Does not have to modify ``plan.desired`` and/or ``plan.existing`` to + line up with any modifications made to ``plan.changes``. + - Should copy over ``plan.exists``, ``plan.update_pcent_threshold``, + and ``plan.delete_pcent_threshold`` when creating a new ``Plan``. + - Must return a ``Plan`` which may be ``plan`` or can be a newly + created one with ``plan.desired`` and ``plan.existing`` copied over + as-is or modified. + - Sources may be empty, as will be the case for aliased zones. ''' # plan may be None if no changes were detected up until now, the # process may still create a plan. diff --git a/octodns/processor/clamp.py b/octodns/processor/clamp.py new file mode 100644 index 0000000..68a3fe2 --- /dev/null +++ b/octodns/processor/clamp.py @@ -0,0 +1,73 @@ +from logging import getLogger + +from .base import BaseProcessor, ProcessorException + + +class TTLArgumentException(ProcessorException): + pass + + +class TtlClampProcessor(BaseProcessor): + """ + Processor that clamps TTL values to a specified range. + + Configuration: + min_ttl: Minimum TTL value (default: 300 seconds / 5 minutes) + max_ttl: Maximum TTL value (default: 86400 seconds / 24 hours) + + Example config.yaml: + processors: + clamp: + class: octodns.processor.clamp.TtlClampProcessor + min_ttl: 300 + max_ttl: 3600 + + zones: + example.com.: + sources: + - config + processors: + - clamp + targets: + - route53 + """ + + def __init__(self, id, min_ttl=300, max_ttl=86400): + super().__init__(id) + self.log = getLogger(self.__class__.__name__) + if not min_ttl <= max_ttl: + raise TTLArgumentException( + f'Min TTL {min_ttl} is not lower than max TTL {max_ttl}' + ) + self.min_ttl = min_ttl + self.max_ttl = max_ttl + self.log.info('__init__: min=%ds, max=%ds', self.min_ttl, self.max_ttl) + + def process_source_zone(self, desired, sources): + """ + Process records from source zone(s). + + Args: + desired: Zone object containing the desired records + sources: List of source names + + Returns: + The modified zone + """ + self.log.debug('process_source_zone: desired=%s', desired.name) + + for record in desired.records: + original_ttl = record.ttl + clamped_ttl = max(self.min_ttl, min(self.max_ttl, original_ttl)) + + if clamped_ttl != original_ttl: + self.log.info( + 'process_source_zone: clamping TTL for %s (%s) %s -> %s', + record.fqdn, + record._type, + original_ttl, + clamped_ttl, + ) + record.ttl = clamped_ttl + + return desired diff --git a/octodns/provider/base.py b/octodns/provider/base.py index b089ffd..8910131 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -9,6 +9,56 @@ from .plan import Plan class BaseProvider(BaseSource): + ''' + Base class for all octoDNS providers. + + Providers extend :class:`octodns.source.base.BaseSource` to add the ability + to apply DNS changes to a target system. While sources only need to + implement ``populate()`` to read DNS data, providers also implement + ``plan()`` and ``apply()`` to manage the complete sync workflow. + + The provider workflow: + + 1. **Populate**: Load current state from the provider via ``populate()`` + 2. **Process**: Modify zones through ``_process_desired_zone()`` and + ``_process_existing_zone()`` to handle provider-specific limitations + 3. **Plan**: Compute changes between desired and existing state via ``plan()`` + 4. **Apply**: Submit approved changes to the provider via ``apply()`` + + Subclasses must implement: + + - **_apply(plan)**: Actually submit changes to the provider's API/backend + + Subclasses should override as needed: + + - **_process_desired_zone(desired)**: Modify desired state before planning + - **_process_existing_zone(existing, desired)**: Modify existing state before planning + - **_include_change(change)**: Filter out false positive changes + - **_extra_changes(existing, desired, changes)**: Add provider-specific changes + - **_plan_meta(existing, desired, changes)**: Add metadata to the plan + + Example provider configuration:: + + providers: + route53: + class: octodns_route53.Route53Provider + access_key_id: env/AWS_ACCESS_KEY_ID + secret_access_key: env/AWS_SECRET_ACCESS_KEY + + zones: + example.com.: + sources: + - config + targets: + - route53 + + See Also: + - :class:`octodns.source.base.BaseSource` + - :class:`octodns.provider.plan.Plan` + - :class:`octodns.provider.yaml.YamlProvider` + - :doc:`/zone_lifecycle` + ''' + def __init__( self, id, @@ -18,6 +68,30 @@ class BaseProvider(BaseSource): strict_supports=True, root_ns_warnings=True, ): + ''' + Initialize the provider. + + :param id: Unique identifier for this provider instance. + :type id: str + :param apply_disabled: If True, the provider will plan changes but not + apply them. Useful for read-only/validation mode. + :type apply_disabled: bool + :param update_pcent_threshold: Maximum percentage of existing records + that can be updated in one sync before + requiring ``--force``. Default: 0.3 (30%). + :type update_pcent_threshold: float + :param delete_pcent_threshold: Maximum percentage of existing records + that can be deleted in one sync before + requiring ``--force``. Default: 0.3 (30%). + :type delete_pcent_threshold: float + :param strict_supports: If True, raise exceptions when unsupported + features are encountered. If False, log warnings + and attempt to work around limitations. + :type strict_supports: bool + :param root_ns_warnings: If True, log warnings about root NS record + handling. If False, silently handle root NS. + :type root_ns_warnings: bool + ''' super().__init__(id) self.log.debug( '__init__: id=%s, apply_disabled=%s, ' @@ -40,21 +114,33 @@ class BaseProvider(BaseSource): def _process_desired_zone(self, desired): ''' - An opportunity for providers to modify the desired zone records before - planning. `desired` is a "shallow" copy, see `Zone.copy` for more - information - - - Must call `super` at an appropriate point for their work, generally - that means as the final step of the method, returning the result of - the `super` call. - - May modify `desired` directly. - - Must not modify records directly, `record.copy` should be called, - the results of which can be modified, and then `Zone.add_record` may - be used with `replace=True`. - - May call `Zone.remove_record` to remove records from `desired`. - - Must call supports_warn_or_except with information about any changes - that are made to have them logged or throw errors depending on the - provider configuration. + Process the desired zone before planning. + + Called during the planning phase to modify the desired zone records + before changes are computed. This is where providers handle their + limitations by removing or modifying records that aren't supported. The + parent method will deal with "standard" unsupported cases like types, + dynamic, and root NS handling. The ``desired`` zone is a shallow copy + (see :meth:`octodns.zone.Zone.copy`). + + :param desired: The desired zone state to be processed. This is a shallow + copy that can be modified. + :type desired: octodns.zone.Zone + + :return: The processed desired zone, typically the same object passed in. + :rtype: octodns.zone.Zone + + .. important:: + - Must call ``super()`` at an appropriate point, generally as the + final step of the method, returning the result of the super call. + - May modify ``desired`` directly. + - Must not modify records directly; ``record.copy()`` should be called, + the results of which can be modified, and then ``Zone.add_record()`` + may be used with ``replace=True``. + - May call ``Zone.remove_record()`` to remove records from ``desired``. + - Must call :meth:`supports_warn_or_except` with information about any + changes that are made to have them logged or throw errors depending + on the provider configuration. ''' for record in desired.records: @@ -178,22 +264,35 @@ class BaseProvider(BaseSource): def _process_existing_zone(self, existing, desired): ''' - An opportunity for providers to modify the existing zone records before - planning. `existing` is a "shallow" copy, see `Zone.copy` for more - information - - - `desired` must not be modified in anyway, it is only for reference - - Must call `super` at an appropriate point for their work, generally - that means as the final step of the method, returning the result of - the `super` call. - - May modify `existing` directly. - - Must not modify records directly, `record.copy` should be called, - the results of which can be modified, and then `Zone.add_record` may - be used with `replace=True`. - - May call `Zone.remove_record` to remove records from `existing`. - - Must call supports_warn_or_except with information about any changes - that are made to have them logged or throw errors depending on the - provider configuration. + Process the existing zone before planning. + + Called during the planning phase to modify the existing zone records + before changes are computed. This allows providers to normalize or filter + the current state from the provider. The ``existing`` zone is a shallow + copy (see :meth:`octodns.zone.Zone.copy`). + + :param existing: The existing zone state from the provider. This is a + shallow copy that can be modified. + :type existing: octodns.zone.Zone + :param desired: The desired zone state. This is for reference only and + must not be modified. + :type desired: octodns.zone.Zone + + :return: The processed existing zone, typically the same object passed in. + :rtype: octodns.zone.Zone + + .. important:: + - ``desired`` must not be modified in any way; it is only for reference. + - Must call ``super()`` at an appropriate point, generally as the + final step of the method, returning the result of the super call. + - May modify ``existing`` directly. + - Must not modify records directly; ``record.copy()`` should be called, + the results of which can be modified, and then ``Zone.add_record()`` + may be used with ``replace=True``. + - May call ``Zone.remove_record()`` to remove records from ``existing``. + - Must call :meth:`supports_warn_or_except` with information about any + changes that are made to have them logged or throw errors depending + on the provider configuration. ''' existing_root_ns = existing.root_ns @@ -211,35 +310,116 @@ class BaseProvider(BaseSource): def _include_change(self, change): ''' - An opportunity for providers to filter out false positives due to - peculiarities in their implementation. E.g. minimum TTLs. + Filter out false positive changes. + + Called during planning to allow providers to filter out changes that + are false positives due to peculiarities in their implementation (e.g., + providers that enforce minimum TTLs). + + :param change: A change being considered for inclusion in the plan. + :type change: octodns.record.change.Change + + :return: True if the change should be included in the plan, False to + filter it out. + :rtype: bool ''' return True def _extra_changes(self, existing, desired, changes): ''' - An opportunity for providers to add extra changes to the plan that are - necessary to update ancillary record data or configure the zone. E.g. - base NS records. + Add provider-specific extra changes to the plan. + + Called during planning to allow providers to add extra changes that are + necessary to update ancillary record data or configure the zone (e.g., + base NS records that must be managed separately). + + :param existing: The existing zone state. + :type existing: octodns.zone.Zone + :param desired: The desired zone state. + :type desired: octodns.zone.Zone + :param changes: The list of changes already computed. + :type changes: list[octodns.record.change.Change] + + :return: A list of additional changes to add to the plan. Return an + empty list if no extra changes are needed. + :rtype: list[octodns.record.change.Change] ''' return [] def _plan_meta(self, existing, desired, changes): ''' - An opportunity for providers to indicate they have "meta" changes - to the zone which are unrelated to records. Examples may include service - plan changes, replication settings, and notes. The returned data is - arbitrary/opaque to octoDNS, with the only requirement being that - pprint.pformat can display it. A dict is recommended. + Indicate provider-specific metadata changes to the zone. + + Called during planning to allow providers to indicate they have "meta" + changes to the zone which are unrelated to records. Examples may include + service plan changes, replication settings, and notes. + + :param existing: The existing zone state. + :type existing: octodns.zone.Zone + :param desired: The desired zone state. + :type desired: octodns.zone.Zone + :param changes: The list of changes computed for this plan. + :type changes: list[octodns.record.change.Change] + + :return: Arbitrary metadata about zone-level changes. The only + requirement is that ``pprint.pformat`` can display it. A dict + is recommended. Return None if no meta changes. + :rtype: dict or None ''' return None def supports_warn_or_except(self, msg, fallback): + ''' + Handle unsupported features based on strict_supports setting. + + If ``strict_supports`` is True, raises a :class:`SupportsException`. + Otherwise, logs a warning with the message and fallback behavior. + + :param msg: Description of the unsupported feature or limitation. + :type msg: str + :param fallback: Description of the fallback behavior being used. + :type fallback: str + + :raises SupportsException: If ``strict_supports`` is True. + ''' if self.strict_supports: raise SupportsException(f'{self.id}: {msg}') self.log.warning('%s; %s', msg, fallback) def plan(self, desired, processors=[]): + ''' + Compute a plan of changes needed to sync the desired state to this provider. + + This is the main planning method that orchestrates the entire planning + workflow. It populates the current state, processes both desired and + existing zones, runs processors, computes changes, and returns a + :class:`Plan` object. + + The planning workflow: + + 1. Populate existing state from the provider via :meth:`populate` + 2. Process desired zone via :meth:`_process_desired_zone` + 3. Process existing zone via :meth:`_process_existing_zone` + 4. Run target zone processors + 5. Run source and target zone processors + 6. Compute changes between existing and desired + 7. Filter changes via :meth:`_include_change` + 8. Add extra changes via :meth:`_extra_changes` + 9. Add metadata via :meth:`_plan_meta` + 10. Create and return a Plan (or None if no changes) + + :param desired: The desired zone state to sync to this provider. + :type desired: octodns.zone.Zone + :param processors: List of processors to run during planning. + :type processors: list[octodns.processor.base.BaseProcessor] + + :return: A Plan containing the computed changes, or None if no changes + are needed. + :rtype: octodns.provider.plan.Plan or None + + See Also: + - :doc:`/zone_lifecycle` for details on the complete sync workflow + ''' self.log.info('plan: desired=%s', desired.decoded_name) existing = Zone(desired.name, desired.sub_zones) @@ -310,8 +490,21 @@ class BaseProvider(BaseSource): def apply(self, plan): ''' - Submits actual planned changes to the provider. Returns the number of - changes made + Apply the planned changes to the provider. + + This is the main apply method that submits the approved plan to the + provider's backend. If ``apply_disabled`` is True, this method does + nothing and returns 0. + + :param plan: The plan containing changes to apply. + :type plan: octodns.provider.plan.Plan + + :return: The number of changes that were applied. + :rtype: int + + See Also: + - :meth:`_apply` for the provider-specific implementation + - :doc:`/zone_lifecycle` for details on the complete sync workflow ''' if self.apply_disabled: self.log.info('apply: disabled') @@ -324,4 +517,28 @@ class BaseProvider(BaseSource): return len(plan.changes) def _apply(self, plan): + ''' + Actually submit the changes to the provider's backend. + + This is an abstract method that must be implemented by all provider + subclasses. It should take the changes in the plan and apply them to + the provider's API or backend system. + + :param plan: The plan containing changes to apply. + :type plan: octodns.provider.plan.Plan + + :raises NotImplementedError: This base class method must be overridden + by subclasses. + + .. important:: + - Must implement the actual logic to submit changes to the provider. + - Should handle errors appropriately (log, raise exceptions, etc.). + - May apply changes in any order that makes sense for the provider + with as much safety as possible given the API methods available. + Often the order of changes should apply deletes before adds to + avoid comflicts during type changes, specidically **CNAME** <-> + other types. If the provider's API supports batching or atomic + changes they should be used. + - Should be idempotent where possible. + ''' raise NotImplementedError('Abstract base class, _apply method missing') diff --git a/octodns/source/base.py b/octodns/source/base.py index d790213..40cec09 100644 --- a/octodns/source/base.py +++ b/octodns/source/base.py @@ -4,6 +4,48 @@ class BaseSource(object): + ''' + Base class for all octoDNS sources and providers. + + Sources are responsible for loading DNS records from various backends into + octoDNS zones. They implement the ``populate`` method to read DNS data from + their respective data stores (YAML files, APIs, databases, etc.) and add + records to the provided zone. + + Subclasses must define the following class attributes either statically or + prior to calling ``super().__init__``: + + - **SUPPORTS**: Set of supported record types (e.g., ``{'A', 'AAAA', 'CNAME'}``) + - **SUPPORTS_GEO**: Boolean indicating if the source supports GeoDNS records + - **log**: Logger instance for the source + + Optional class attributes: + + - **SUPPORTS_MULTIVALUE_PTR**: Support for multiple PTR records (default: False) + - **SUPPORTS_POOL_VALUE_STATUS**: Support for pool value status flags (default: False) + - **SUPPORTS_ROOT_NS**: Support for root NS records (default: False) + - **SUPPORTS_DYNAMIC_SUBNETS**: Support for dynamic subnet-based routing (default: False) + + Example usage:: + + sources: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config + + zones: + example.com.: + sources: + - config + targets: + - route53 + + See Also: + - :class:`octodns.provider.yaml.YamlProvider` + - :class:`octodns.source.tinydns.TinyDnsFileSource` + - :class:`octodns.source.envvar.EnvVarSource` + ''' + SUPPORTS_MULTIVALUE_PTR = False SUPPORTS_POOL_VALUE_STATUS = False SUPPORTS_ROOT_NS = False @@ -11,8 +53,15 @@ class BaseSource(object): def __init__(self, id): ''' - :param id: unique identifier for the provider or source + Initialize the source. + + :param id: Unique identifier for this source instance. Used in logging + and configuration references. :type id: str + + :raises NotImplementedError: If required class attributes (``log``, + ``SUPPORTS_GEO``, or ``SUPPORTS``) are not + defined in the subclass. ''' self.id = id @@ -31,29 +80,75 @@ class BaseSource(object): @property def SUPPORTS_DYNAMIC(self): + ''' + Indicates whether this source supports dynamic records. + + Dynamic records include advanced routing features like GeoDNS pools, + health checks, and weighted responses. Most sources do not support + dynamic records. + + :return: True if dynamic records are supported, False otherwise. + :rtype: bool + ''' return False def populate(self, zone, target=False, lenient=False): ''' - Loads all records the provider knows about for the provided zone + Load DNS records from the source into the provided zone. - When `target` is True the populate call is being made to load the - current state of the provider. + This method is responsible for reading DNS data from the source's + backend and adding records to the zone using ``zone.add_record()``. + Subclasses must implement this method. - When `lenient` is True the populate call may skip record validation and - do a "best effort" load of data. That will allow through some common, - but not best practices stuff that we otherwise would reject. E.g. no - trailing . or missing escapes for ;. + :param zone: The zone to populate with records from this source. + :type zone: octodns.zone.Zone + :param target: If True, the populate call is loading the current state + from a target provider (for comparison during sync). If + False, loading desired state from a source. + :type target: bool + :param lenient: If True, skip strict record validation and do a "best + effort" load of data. This allows some non-best-practice + configurations through (e.g., missing trailing dots or + unescaped semicolons). + :type lenient: bool - When target is True (loading current state) this method should return - True if the zone exists or False if it does not. + :return: When ``target`` is True (loading current state), should return + True if the zone exists in the target or False if it does not. + When ``target`` is False (loading desired state), return value + is ignored and may be None. + :rtype: bool or None + + :raises NotImplementedError: This base class method must be overridden + by subclasses. + + .. important:: + - Must use ``zone.add_record()`` to add records to the zone. + - Should not modify the zone name or other zone properties. + - When ``target=True``, must return a boolean indicating zone existence. + - When ``lenient=True``, should relax validation to handle common + non-standard configurations. ''' raise NotImplementedError( 'Abstract base class, populate method missing' ) def supports(self, record): + ''' + Check if this source supports the given record type. + + :param record: The DNS record to check for support. + :type record: octodns.record.base.Record + + :return: True if the record type is supported, False otherwise. + :rtype: bool + ''' return record._type in self.SUPPORTS def __repr__(self): + ''' + Return a string representation of this source. + + :return: The class name of this source instance. + :rtype: str + ''' return self.__class__.__name__ diff --git a/octodns/zone.py b/octodns/zone.py index 9d975cd..f6bb150 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -12,6 +12,17 @@ from .record import Create, Delete class SubzoneRecordException(Exception): + ''' + Exception raised when a record belongs in a sub-zone but is added to the parent. + + This exception is raised when attempting to add a record to a zone that + should actually be managed in a configured sub-zone. Only NS and DS records + are allowed at the sub-zone boundary. + + :param record: The record that caused the exception. + :type record: octodns.record.base.Record + ''' + def __init__(self, msg, record): self.record = record @@ -22,6 +33,19 @@ class SubzoneRecordException(Exception): class DuplicateRecordException(Exception): + ''' + Exception raised when attempting to add a duplicate record to a zone. + + A duplicate is defined as a record with the same name and type as an + existing record in the zone. The exception includes references to both + the existing and new records for debugging. + + :param existing: The existing record in the zone. + :type existing: octodns.record.base.Record + :param new: The new record being added. + :type new: octodns.record.base.Record + ''' + def __init__(self, msg, existing, new): self.existing = existing self.new = new @@ -44,6 +68,17 @@ class DuplicateRecordException(Exception): class InvalidNodeException(Exception): + ''' + Exception raised when CNAME records coexist with other records at a node. + + Per DNS standards, CNAME records cannot coexist with other record types + at the same node. This exception is raised when such an invalid + configuration is detected. + + :param record: The record that caused the exception. + :type record: octodns.record.base.Record + ''' + def __init__(self, msg, record): self.record = record @@ -54,10 +89,59 @@ class InvalidNodeException(Exception): class InvalidNameError(Exception): + ''' + Exception raised when a zone name is invalid. + + Zone names must: + - End with a dot (.) + - Not contain double dots (..) + - Not contain whitespace + ''' + pass class Zone(object): + ''' + Container for DNS records belonging to a single DNS zone. + + A Zone represents a DNS zone and manages all the records within it. It + provides methods for adding, removing, and querying records, as well as + computing changes between zones and applying those changes. + + Zones support copy-on-write semantics via the :meth:`copy` method, which + creates shallow copies that are hydrated on first modification. This allows + for efficient processing of zones through multiple stages without + unnecessary copying. + + Key features: + + - **Record management**: Add, remove, and query DNS records + - **Validation**: Enforce DNS standards (CNAME restrictions, sub-zone rules) + - **IDNA support**: Handle internationalized domain names + - **Sub-zone awareness**: Respect configured sub-zone boundaries + - **Change tracking**: Compute differences between desired and existing state + - **Copy-on-write**: Efficient shallow copying with lazy hydration + + Example usage:: + + from octodns.zone import Zone + from octodns.record import Record + + zone = Zone('example.com.', []) + record = Record.new(zone, 'www', {'type': 'A', 'ttl': 300, 'value': '1.2.3.4'}) + zone.add_record(record) + + # Create a shallow copy + copy = zone.copy() + # Modifications to copy don't affect the original until hydrated + + See Also: + - :doc:`/zone_lifecycle` for details on zone processing workflow + - :class:`octodns.record.base.Record` + - :class:`octodns.provider.base.BaseProvider` + ''' + log = getLogger('Zone') def __init__( @@ -67,6 +151,31 @@ class Zone(object): update_pcent_threshold=None, delete_pcent_threshold=None, ): + ''' + Initialize a DNS zone. + + :param name: The zone name (must end with a dot). Internationalized + domain names (IDN) are automatically encoded to IDNA format. + :type name: str + :param sub_zones: List of sub-zone names managed separately. Records + belonging to sub-zones will be rejected (except NS/DS + at the boundary). + :type sub_zones: list[str] + :param update_pcent_threshold: Override for maximum update percentage + threshold. If None, uses provider default. + :type update_pcent_threshold: float or None + :param delete_pcent_threshold: Override for maximum delete percentage + threshold. If None, uses provider default. + :type delete_pcent_threshold: float or None + + :raises InvalidNameError: If the zone name is invalid (missing trailing + dot, contains double dots, or has whitespace). + + .. important:: + - Zone names must end with a dot (.) + - Zone names are automatically encoded to IDNA format internally + - Sub-zones prevent records from being added to the parent zone + ''' if not name[-1] == '.': raise InvalidNameError( f'Invalid zone name {name}, missing ending dot' @@ -108,17 +217,53 @@ class Zone(object): @property def records(self): + ''' + Get all records in this zone. + + Returns a set of all DNS records in the zone. If this is a shallow copy + (not yet hydrated), returns records from the origin zone. + + :return: Set of all records in the zone. + :rtype: set[octodns.record.base.Record] + ''' if self._origin: return self._origin.records return set([r for _, node in self._records.items() for r in node]) @property def root_ns(self): + ''' + Get the root NS record for this zone. + + The root NS record is the NS record at the zone apex (empty hostname). + Returns None if no root NS record exists. + + :return: The root NS record, or None if not present. + :rtype: octodns.record.ns.NsRecord or None + ''' if self._origin: return self._origin.root_ns return self._root_ns def hostname_from_fqdn(self, fqdn): + ''' + Extract the hostname portion from a fully qualified domain name. + + Strips the zone name from the FQDN to get just the hostname portion. + Handles both IDNA-encoded and UTF-8 domain names correctly. + + :param fqdn: Fully qualified domain name. + :type fqdn: str + + :return: The hostname portion (without the zone name). + :rtype: str + + Example:: + + zone = Zone('example.com.', []) + zone.hostname_from_fqdn('www.example.com.') # Returns 'www' + zone.hostname_from_fqdn('example.com.') # Returns '' + ''' try: fqdn.encode('ascii') # it's non-idna or idna encoded @@ -128,6 +273,27 @@ class Zone(object): return self._utf8_name_re.sub('', fqdn) def owns(self, _type, fqdn): + ''' + Determine if this zone owns a given FQDN for a specific record type. + + Checks whether a record with the given FQDN and type should be managed + by this zone, taking into account sub-zone boundaries. Records under + sub-zones are not owned by the parent (except NS records at the exact + sub-zone boundary). + + :param _type: The DNS record type (e.g., 'A', 'CNAME', 'NS'). + :type _type: str + :param fqdn: Fully qualified domain name to check. + :type fqdn: str + + :return: True if this zone owns the FQDN for this type, False otherwise. + :rtype: bool + + .. important:: + - NS records at sub-zone boundaries are owned by the parent zone + - All other records under sub-zones are not owned by the parent + - FQDNs are automatically normalized (trailing dot added if missing) + ''' if fqdn[-1] != '.': fqdn = f'{fqdn}.' @@ -154,6 +320,38 @@ class Zone(object): return True def add_record(self, record, replace=False, lenient=False): + ''' + Add a DNS record to this zone. + + Adds the provided record to the zone with validation. If this is a + shallow copy (has an origin), it will be hydrated before adding. + + :param record: The DNS record to add to the zone. + :type record: octodns.record.base.Record + :param replace: If True, replace any existing record with the same name + and type. If False, raise an exception if a duplicate + exists. + :type replace: bool + :param lenient: If True, skip some validation checks (sub-zone checks, + CNAME coexistence checks). Useful when loading existing + data that may not be standards-compliant. + :type lenient: bool + + :raises SubzoneRecordException: If the record belongs in a configured + sub-zone (unless it's an NS/DS record + at the boundary). + :raises DuplicateRecordException: If a record with the same name and type + already exists and ``replace=False``. + :raises InvalidNodeException: If adding the record would create an + invalid CNAME coexistence situation. + + .. important:: + - Automatically hydrates shallow copies on first modification + - NS/DS records are allowed at sub-zone boundaries + - CNAME records cannot coexist with other records at the same node + - Use ``replace=True`` to update existing records + - Use ``lenient=True`` when loading potentially non-compliant data + ''' if self._origin: self.hydrate() @@ -218,6 +416,21 @@ class Zone(object): node.add(record) def remove_record(self, record): + ''' + Remove a DNS record from this zone. + + Removes the provided record from the zone. If this is a shallow copy + (has an origin), it will be hydrated before removing. + + :param record: The DNS record to remove from the zone. + :type record: octodns.record.base.Record + + .. important:: + - Automatically hydrates shallow copies on first modification + - Clearing the root NS record (empty name) also clears the cached + ``root_ns`` property + - Silently succeeds if the record doesn't exist in the zone + ''' if self._origin: self.hydrate() @@ -235,6 +448,29 @@ class Zone(object): return self.remove_record(record) def changes(self, desired, target): + ''' + Compute the changes needed to transform this zone into the desired state. + + Compares this zone (existing state) with the desired zone and returns + a list of changes (Creates, Updates, Deletes) required to make this + zone match the desired state. Respects record-level include/exclude + filtering and provider support. + + :param desired: The desired zone state to compare against. + :type desired: Zone + :param target: The target provider that will apply these changes. Used + to check record support and apply include/exclude rules. + :type target: octodns.provider.base.BaseProvider + + :return: List of changes needed to transform this zone to the desired state. + :rtype: list[octodns.record.change.Change] + + .. important:: + - Skips records marked as ``ignored`` + - Respects record-level ``included`` and ``excluded`` lists + - Only includes changes for record types the target supports + - Returns Creates, Updates (via record.changes), and Deletes + ''' self.log.debug('changes: zone=%s, target=%s', self, target) # Build up a hash of the desired records, thanks to our special @@ -351,7 +587,20 @@ class Zone(object): def apply(self, changes): ''' - Apply the provided changes to the zone. + Apply a list of changes to this zone. + + Applies the provided changes by adding new/updated records and removing + deleted records. Uses ``replace=True`` and ``lenient=True`` to handle + updates and non-standard records gracefully. + + :param changes: List of changes to apply to the zone. + :type changes: list[octodns.record.change.Change] + + .. important:: + - Delete changes remove the existing record + - Create and Update changes add the new record with ``replace=True`` + - All adds use ``lenient=True`` to skip validation + - Changes are applied in the order provided ''' for change in changes: if isinstance(change, Delete): @@ -361,14 +610,25 @@ class Zone(object): def hydrate(self): ''' - Take a shallow copy Zone and make it a deeper copy holding its own - reference to records. These records will still be the originals and - they should not be modified. Changes should be made by calling - `add_record`, often with `replace=True`, and/or `remove_record`. - - Note: This method does not need to be called under normal circumstances - as `add_record` and `remove_record` will automatically call it when - appropriate. + Convert a shallow copy into a hydrated copy with its own record references. + + Hydration copies all records from the origin zone into this zone, + making it independent. The records themselves are still the original + objects and should not be modified directly. Use :meth:`add_record` + with ``replace=True`` or :meth:`remove_record` to make changes. + + :return: True if hydration occurred, False if already hydrated. + :rtype: bool + + .. note:: + This method is automatically called by :meth:`add_record` and + :meth:`remove_record` when needed, so manual calls are rarely necessary. + + .. important:: + - Only hydrates if this is a shallow copy (has an ``_origin``) + - Clears the ``_origin`` reference after hydration + - Uses ``lenient=True`` when adding records from origin + - Records are still shared with the origin (not deep copied) ''' origin = self._origin if origin is None: @@ -383,14 +643,33 @@ class Zone(object): def copy(self): ''' - Copy-on-write semantics support. This method will create a shallow - clone of the zone which will be hydrated the first time `add_record` or - `remove_record` is called. - - This allows low-cost copies of things to be made in situations where - changes are unlikely and only incurs the "expense" of actually - copying the records when required. The actual record copy will not be - "deep" meaning that records should not be modified directly. + Create a shallow copy of this zone using copy-on-write semantics. + + Creates a new zone that shares records with this zone until the copy + is modified. When :meth:`add_record` or :meth:`remove_record` is called + on the copy, it will be automatically hydrated with its own record + references. + + :return: A shallow copy of this zone. + :rtype: Zone + + .. important:: + - The copy shares records with the original until hydrated + - Hydration happens automatically on first modification + - Records in the hydrated copy are still the same objects (not deep copied) + - Modifying records directly affects both zones; use ``record.copy()`` + and ``add_record(..., replace=True)`` instead + + Example:: + + original = Zone('example.com.', []) + # ... add records to original ... + + copy = original.copy() # Shallow copy, shares records + # No copying has occurred yet + + copy.add_record(new_record) # Triggers hydration, copies record refs + # Now copy has its own record references ''' copy = Zone( self.name, @@ -402,4 +681,10 @@ class Zone(object): return copy def __repr__(self): + ''' + Return a string representation of this zone. + + :return: String in the format ``Zone`` using the decoded name. + :rtype: str + ''' return f'Zone<{self.decoded_name}>' diff --git a/script/changelog b/script/changelog index f981f5a..5b4a296 100755 --- a/script/changelog +++ b/script/changelog @@ -1,16 +1,8 @@ -#!/bin/sh -set -e +#!/bin/bash -cd "$(dirname "$0")/.." -ROOT=$(pwd) - -if [ -z "$VENV_NAME" ]; then - VENV_NAME="env" -fi - -ACTIVATE="$VENV_NAME/bin/activate" -if [ -f "$ACTIVATE" ]; then - . "$ACTIVATE" -fi +# Get current script path +SCRIPT_PATH="$( dirname -- "$( readlink -f -- "${0}"; )"; )" +# Activate OctoDNS Python venv +source "${SCRIPT_PATH}/common.sh" changelet "$@" diff --git a/script/common.sh b/script/common.sh new file mode 100644 index 0000000..6b57b34 --- /dev/null +++ b/script/common.sh @@ -0,0 +1,27 @@ +# This script contains Python's venv management features common to all shell +# scripts located in this directory and to the repository pre-commit hook. +# This script is *not* meant to be run directly. + +# Exit on any error +set -e + +# Path to OctoDNS base directory +OCTODNS_PATH="$( dirname -- "${SCRIPT_PATH}"; )" + +# Change to path OctoDNS base directory +cd "${OCTODNS_PATH}" + +# If no venv name is set, set it to "env" +if [ -z "${VENV_NAME}" ]; then + VENV_NAME="${OCTODNS_PATH}/env" +fi + +ACTIVATE="${VENV_NAME}/bin/activate" +# Check that [venv_directory]/bin/activate exists. +if [ ! -f "${ACTIVATE}" ]; then + echo "${ACTIVATE} does not exist. Run ./script/bootstrap" >&2 + exit 1 +fi + +# Activate OctoDNS venv. +source "${ACTIVATE}" diff --git a/script/coverage b/script/coverage index 914a042..c47d2ca 100755 --- a/script/coverage +++ b/script/coverage @@ -1,34 +1,9 @@ -#!/bin/sh -set -e +#!/bin/bash -cd "$(dirname "$0")/.." - -if [ -z "$VENV_NAME" ]; then - VENV_NAME="env" -fi - -ACTIVATE="$VENV_NAME/bin/activate" -if [ ! -f "$ACTIVATE" ]; then - echo "$ACTIVATE does not exist, run ./script/bootstrap" >&2 - exit 1 -fi -. "$ACTIVATE" - -# Just to be sure/safe -export AWS_ACCESS_KEY_ID= -export AWS_SECRET_ACCESS_KEY= -export CLOUDFLARE_EMAIL= -export CLOUDFLARE_TOKEN= -export DNSIMPLE_ACCOUNT= -export DNSIMPLE_TOKEN= -export DYN_CUSTOMER= -export DYN_PASSWORD= -export DYN_USERNAME= -export GOOGLE_APPLICATION_CREDENTIALS= -export ARM_CLIENT_ID= -export ARM_CLIENT_SECRET= -export ARM_TENANT_ID= -export ARM_SUBSCRIPTION_ID= +# Get current script path +SCRIPT_PATH="$( dirname -- "$( readlink -f -- "${0}"; )"; )" +# Activate OctoDNS Python venv +source "${SCRIPT_PATH}/common.sh" SOURCE_DIR="octodns/" diff --git a/script/format b/script/format index a0d3499..4a7229b 100755 --- a/script/format +++ b/script/format @@ -1,10 +1,11 @@ #!/bin/bash -set -e +# Get current script path +SCRIPT_PATH="$( dirname -- "$( readlink -f -- "${0}"; )"; )" +# Activate OctoDNS Python venv +source "${SCRIPT_PATH}/common.sh" SOURCES="$(find *.py octodns tests docs -name '*.py') $(grep --files-with-matches '^#!.*python' script/*)" -. env/bin/activate - isort "$@" $SOURCES black "$@" $SOURCES diff --git a/script/generate-docs b/script/generate-docs index 8d73b76..51b5746 100755 --- a/script/generate-docs +++ b/script/generate-docs @@ -1,19 +1,9 @@ -#!/bin/sh -set -e +#!/bin/bash -cd "$(dirname "$0")/.." -ROOT=$(pwd) - -if [ -z "$VENV_NAME" ]; then - VENV_NAME="env" -fi - -ACTIVATE="$VENV_NAME/bin/activate" -if [ ! -f "$ACTIVATE" ]; then - echo "$ACTIVATE does not exist, run ./script/bootstrap" >&2 - exit 1 -fi -. "$ACTIVATE" +# Get current script path +SCRIPT_PATH="$( dirname -- "$( readlink -f -- "${0}"; )"; )" +# Activate OctoDNS Python venv +source "${SCRIPT_PATH}/common.sh" cd docs diff --git a/script/lint b/script/lint index f68f54d..cd42d3a 100755 --- a/script/lint +++ b/script/lint @@ -1,19 +1,9 @@ -#!/bin/sh -set -e +#!/bin/bash -cd "$(dirname "$0")/.." -ROOT=$(pwd) - -if [ -z "$VENV_NAME" ]; then - VENV_NAME="env" -fi - -ACTIVATE="$VENV_NAME/bin/activate" -if [ ! -f "$ACTIVATE" ]; then - echo "$ACTIVATE does not exist, run ./script/bootstrap" >&2 - exit 1 -fi -. "$ACTIVATE" +# Get current script path +SCRIPT_PATH="$( dirname -- "$( readlink -f -- "${0}"; )"; )" +# Activate OctoDNS Python venv +source "${SCRIPT_PATH}/common.sh" SOURCES="$(find *.py octodns tests docs -name '*.py') $(grep --files-with-matches '^#!.*python' script/*)" diff --git a/script/markdown-toc b/script/markdown-toc deleted file mode 100755 index 2b192ef..0000000 --- a/script/markdown-toc +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python - -from re import compile -from sys import stdin - -splitter = compile(r'\s+') - -in_pre = False -headings = [] -for line in stdin: - if line.startswith('```'): - in_pre = not in_pre - if in_pre or not line.startswith('#'): - continue - level, heading = splitter.split(line, 1) - if 'Table of Contents' in heading: - continue - headings.append((len(level), heading.strip())) - -# ignore the first one, it's more of a title -headings.pop(0) - -print('\n## Table of Contents\n') -min_level = min(h[0] for h in headings) -for heading in headings: - level = heading[0] - min_level - pre = ' ' * (level * 3) - title = heading[1] - link = title.lower().replace(' ', '-').replace('`', '') - print(f'{pre}* [{title}](#{link})') -print() diff --git a/script/release b/script/release index 1fab0eb..f3c18a2 100755 --- a/script/release +++ b/script/release @@ -1,13 +1,9 @@ #!/bin/bash -set -e - -cd "$(dirname "$0")"/.. -ROOT=$(pwd) - -if [ -z "$VENV_NAME" ]; then - VENV_NAME="env" -fi +# Get current script path +SCRIPT_PATH="$( dirname -- "$( readlink -f -- "${0}"; )"; )" +# Activate OctoDNS Python venv +source "${SCRIPT_PATH}/common.sh" PYPYRC="$HOME/.pypirc" if [ ! -e "$PYPYRC" ]; then @@ -22,13 +18,6 @@ EndOfMessage exit 1 fi -ACTIVATE="$VENV_NAME/bin/activate" -if [ ! -f "$ACTIVATE" ]; then - echo "$ACTIVATE does not exist, run ./script/bootstrap" >&2 - exit 1 -fi -. "$ACTIVATE" - # Set so that setup.py will create a public release style version number export OCTODNS_RELEASE=1 diff --git a/script/test b/script/test index f5e4d8e..0bde9a7 100755 --- a/script/test +++ b/script/test @@ -1,33 +1,8 @@ -#!/bin/sh -set -e +#!/bin/bash -cd "$(dirname "$0")/.." - -if [ -z "$VENV_NAME" ]; then - VENV_NAME="env" -fi - -ACTIVATE="$VENV_NAME/bin/activate" -if [ ! -f "$ACTIVATE" ]; then - echo "$ACTIVATE does not exist, run ./script/bootstrap" >&2 - exit 1 -fi -. "$ACTIVATE" - -# Just to be sure/safe -export AWS_ACCESS_KEY_ID= -export AWS_SECRET_ACCESS_KEY= -export CLOUDFLARE_EMAIL= -export CLOUDFLARE_TOKEN= -export DNSIMPLE_ACCOUNT= -export DNSIMPLE_TOKEN= -export DYN_CUSTOMER= -export DYN_PASSWORD= -export DYN_USERNAME= -export GOOGLE_APPLICATION_CREDENTIALS= -export ARM_CLIENT_ID= -export ARM_CLIENT_SECRET= -export ARM_TENANT_ID= -export ARM_SUBSCRIPTION_ID= +# Get current script path +SCRIPT_PATH="$( dirname -- "$( readlink -f -- "${0}"; )"; )" +# Activate OctoDNS Python venv +source "${SCRIPT_PATH}/common.sh" pytest --disable-network "$@" diff --git a/tests/test_octodns_processor_clamp.py b/tests/test_octodns_processor_clamp.py new file mode 100644 index 0000000..63c2de1 --- /dev/null +++ b/tests/test_octodns_processor_clamp.py @@ -0,0 +1,120 @@ +from unittest import TestCase + +from octodns.processor.clamp import TTLArgumentException, TtlClampProcessor +from octodns.record.base import Record +from octodns.zone import Zone + + +class TestClampProcessor(TestCase): + + def test_processor_min(self): + "Test the processor for clamping to the minimum" + min_ttl = 42 + processor = TtlClampProcessor('test', min_ttl=min_ttl) + + too_low_ttl = 23 + self.assertLess(too_low_ttl, min_ttl) + + zone = Zone('unit.tests.', []) + zone.add_record( + Record.new( + zone, '', {'type': 'TXT', 'ttl': too_low_ttl, 'value': 'foo'} + ) + ) + + processed_zone = processor.process_source_zone(zone.copy(), None) + self.assertNotEqual(zone, processed_zone) + + self.assertEqual(len(processed_zone.records), len(zone.records)) + self.assertEqual(len(processed_zone.records), 1) + self.assertEqual(processed_zone.records.pop().ttl, min_ttl) + + def test_processor_max(self): + "Test the processor for clamping to the maximum" + max_ttl = 4711 + processor = TtlClampProcessor('test', max_ttl=max_ttl) + + too_high_ttl = max_ttl + 1 + self.assertLess(max_ttl, too_high_ttl) + + zone = Zone('unit.tests.', []) + zone.add_record( + Record.new( + zone, '', {'type': 'TXT', 'ttl': too_high_ttl, 'value': 'foo'} + ) + ) + + processed_zone = processor.process_source_zone(zone.copy(), None) + self.assertNotEqual(zone, processed_zone) + + self.assertEqual(len(processed_zone.records), len(zone.records)) + self.assertEqual(len(processed_zone.records), 1) + self.assertEqual(processed_zone.records.pop().ttl, max_ttl) + + def test_processor_maxmin(self): + "Test the processor for unlogical arguments" + min_ttl = 42 + max_ttl = 23 + self.assertRaises( + TTLArgumentException, + TtlClampProcessor, + 'test', + min_ttl=min_ttl, + max_ttl=max_ttl, + ) + + def test_processor_minmax(self): + "Test the processor for clamping both min and max values" + min_ttl = 42 + max_ttl = 4711 + processor = TtlClampProcessor('test', min_ttl=min_ttl, max_ttl=max_ttl) + + too_low_ttl = min_ttl - 1 + too_high_ttl = max_ttl + 1 + self.assertLess(too_low_ttl, min_ttl) + self.assertLess(too_low_ttl, min_ttl) + self.assertLess(max_ttl, too_high_ttl) + + zone = Zone('unit.tests.', []) + zone.add_record( + Record.new( + zone, + 'high', + {'type': 'TXT', 'ttl': too_high_ttl, 'value': 'high'}, + ) + ) + zone.add_record( + Record.new( + zone, 'low', {'type': 'TXT', 'ttl': too_low_ttl, 'value': 'low'} + ) + ) + + processed_zone = processor.process_source_zone(zone.copy(), None) + self.assertNotEqual(zone, processed_zone) + + processed_records = sorted( + list(processed_zone.records), key=lambda r: r.ttl + ) + self.assertEqual(len(processed_records), 2) + + self.assertEqual(processed_records[0].ttl, min_ttl) + self.assertEqual(processed_records[1].ttl, max_ttl) + + def test_processor_noclamp(self): + "Test the processor for working with TTLs not requiring any clamping" + min_ttl = 23 + max_ttl = 4711 + processor = TtlClampProcessor('test', min_ttl=min_ttl, max_ttl=max_ttl) + + ttl = 42 + + self.assertLess(min_ttl, ttl) + self.assertLess(ttl, max_ttl) + + zone = Zone('unit.tests.', []) + zone.add_record( + Record.new(zone, '', {'type': 'TXT', 'ttl': ttl, 'value': 'foo'}) + ) + + processed_zone = processor.process_source_zone(zone.copy(), None) + self.assertEqual(processed_zone.records.pop().ttl, ttl)