From 414a80b670fe2a05ef19b6e7b120250ea688ea13 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 28 Sep 2025 17:55:58 -0700 Subject: [PATCH] Documentation of dynamic config glob and regex matching --- docs/configuration.rst | 37 ++++++- docs/dynamic_zone_config.rst | 181 +++++++++++++++++++++++++++++++++++ docs/getting-started.rst | 4 +- docs/index.rst | 1 + octodns/manager.py | 24 +++-- 5 files changed, 237 insertions(+), 10 deletions(-) create mode 100644 docs/dynamic_zone_config.rst diff --git a/docs/configuration.rst b/docs/configuration.rst index 08b3647..4a9a07b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -13,10 +13,45 @@ YamlProvider :py:mod:`octodns.provider.yaml` lays out the options for configuring the most commonly used source of record data. +Dynamic Zone Config +------------------- + +In many cases octoDNS's dynamic zone configuration is the best option for +configuring octoDNS to manage your zones. In its simplest form that would look +something like:: + + --- + providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config + default_ttl: 3600 + enforce_order: True + ns1: + class: octodns_ns1.Ns1Provider + api_key: env/NS1_API_KEY + route53: + class: octodns_route53.Route53Provider + access_key_id: env/AWS_ACCESS_KEY_ID + secret_access_key: env/AWS_SECRET_ACCESS_KEY + + zones: + '*': + sources: + - config + targets: + - ns1 + - route53 + +This configuration will query both ns1 and route53 for the list of zones they +are managing and dynamically add them to the list being managed using the +sources and targets corresponding to the '*' section. See +:ref:`dynamic-zone-config` for details. + Static Zone Config ------------------ -In cases where finer grained control is desired and the configuration of +In cases where fine grained control is desired and the configuration of individual zones varies ``zones`` can be an explicit list with each configured zone listed along with its specific setup. As exemplified below ``alias`` zones can be useful when two zones are exact copies of each other, with the same diff --git a/docs/dynamic_zone_config.rst b/docs/dynamic_zone_config.rst new file mode 100644 index 0000000..c6cd36b --- /dev/null +++ b/docs/dynamic_zone_config.rst @@ -0,0 +1,181 @@ +.. _dynamic-zone-config: + +Dynamic Zone Config +=================== + +Dynamic zone configuration is a powerful tool for reducing the +configuration required to run octoDNS, specifically the *zones* section. Rather +than an exhaustive list of every zone and its corresponding sources and targets +it's possible to define the pattern once with a wildcard. + +This is most commonly done with a `YamlProvider`_ which will result in building +the list of zones managed at runtime from the yaml zone files in it's +directory, but any provider that supports the +:py:meth:`octodns.provider.yaml.YamlProvider.list_zones` method can be used. + +Any zone name configured in the *zones* section with a leading * is considered +dynamic and the information in this document applies. It is possible to include +multiple dynamic zone configurations in advanced setups utilizing +distinct sources and/or carefully crafted matching as described below. + +Matching +-------- + +There are three types of matching supported: legacy, file-glob, and regular +expression. This ultimately results in very flexible and powerful options, but +makes it pretty easy to build a foot-gun. The matching process has thorough +info and debug logging that can be enabled with **--debug** and should be the +first step in debugging a dynamic zone configuration. + +Legacy +...... + +This is the default mode and the only one supported in versions prior to +1.14.0. It is in effect a catch-all in that any zones returned by the sources' +:py:meth:`octodns.provider.yaml.YamlProvider.list_zones`. + +This generally means that it only makes sense to have multiple legacy matchers +when they have distinct sources, otherwise the first one configured will claim +all the zones leaving nothing available. + +.. _file-glob: + +File-glob +......... + +This mode uses Unix shell style matching using the `fnmatch`_ module and is +generally the place to start when trying to apply configs to zones in a single +source or set of sources as it's relatively easy to understand and predict the +behavior of it. + +A public and private setup where the public zones are also pushed internally is +a good starting example. If the following zone YAML files are in the *config* +provider's directory:: + + company.com. + foundation.org + internal.net. + jobs.company.com. + other.com + support.company.com. + us-east-1.internal.net. + us-west-2.internal.net. + +The following octoDNS configuration would match them as described in comments:: + + --- + ... + + zones: + + # the names here do not really matter beyond starting with a *, it is a + # reccomended best practice to match the glob, but not required. It will be + # used in logging to aid in debugging. + + # they are applied in the order defined and once claimed a zone is no + # longer available for matching + + # everytyhing is available for matching + '*internal.net': + # we only want the private zones here and they are all under + # internet.net. so this glob will claim them. + glob: '*internal.net.' + sources: + - config + targets: + # only push it to the private provider + - private + + # legacy style match everything that's left, all our various public zones + '*': + # legacy style match everything that's left, all our various public zones + sources: + - config + targets: + # push it to the public dns + - public + # and private + - private + +This does mean that things are public by default so care would need to be taken +if a new internal zone naming pattern is added. + +.. _fnmatch: https://docs.python.org/3/library/fnmatch.html + +.. versionadded:: 1.14.0 + File-glob matching support was added in 1.14.0 + +.. _regular-expression: + +Regular Expression +.................. + +Regular expression mode works similarly to :ref:`file-glob` with the matching +performed by the python regular expression engine `re`_. It enables much more +complex and powerful matching logic with the trade-off of having to work with +regular expressions. + +Continuing on with the public/private split, adding in the wrinkles of multiple +internal domain names and the desire to split the regions pushing only to the +co-located DNS servers. All of our internal zones end in .net., anything else +is public:: + + company.com. + foundation.org + jobs.company.com. + other.com + support.company.com. + us-east-1.hosts.net. + us-east-1.network.net. + us-east-1.services.net. + us-west-2.hosts.net. + us-west-2.network.net. + us-west-2.services.net. + +The following octoDNS configuration would match them as described in comments:: + + --- + ... + + zones: + + # regexes are too ugly to use as names, so these have useful info for + # logging/debugging + + # everytyhing is available for matching + '*us-east-1': + # we only want the private zones here and they are all under + # internet.net. So this regex will claim them, yes this could be done + # with a glob, but ... + regex: '^.*us-east-1.*.net.$' + sources: + - config + targets: + # only push it to the us-east-1 provider + - us-east-1 + + # everytyhing with the exception of the us-east-1 .net zones are available + '*us-west-2': + regex: '^.*us-west-2.*.net.$' + sources: + - config + targets: + # only push it to the us-east-1 provider + - us-west-2 + + # legacy style match everything that's left, all our various public zones + '*': + sources: + - config + targets: + # push it to the public dns + - public + # and private + - private + +.. _re: https://docs.python.org/3/library/re.html + +.. versionadded:: 1.14.0 + Regular expression matching support was added in 1.14.0 + +.. _YamlProvider: /octodns/provider/yaml.py diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 681e413..6bffa79 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -41,9 +41,7 @@ separate accounts and each manage a distinct set of zones. A good example of this this might be ``./config/staging.yaml`` & ``./config/production.yaml``. We'll focus on a ``config/production.yaml``. -.. _dynamic-zone-config: - -Dynamic Zone Config +Zone Config ................... octoDNS supports dynamically building the list of zones it will work with when diff --git a/docs/index.rst b/docs/index.rst index 372d80d..983a5ca 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,6 +28,7 @@ Documentation getting-started.rst records.md configuration.rst + dynamic_zone_config.rst dynamic_records.rst auto_arpa.rst examples/README.rst diff --git a/octodns/manager.py b/octodns/manager.py index 4478c71..bcdb849 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -630,25 +630,37 @@ class Manager(object): # add this source's zones to the candidates candidates |= source_zones[source.id] - self.log.debug('_preprocess_zones: candidates=%s', candidates) + self.log.debug( + '_preprocess_zones: name=%s, candidates=%s', name, candidates + ) # remove any zones that are already configured, either explicitly or # from a previous dyanmic config candidates -= set(zones.keys()) if glob := config.pop('glob', None): - self.log.debug('_preprocess_zones: glob=%s', glob) + self.log.debug( + '_preprocess_zones: name=%s, glob=%s', name, glob + ) candidates = set(fnmatch_filter(candidates, glob)) elif regex := config.pop('regex', None): - self.log.debug('_preprocess_zones: regex=%s', regex) + self.log.debug( + '_preprocess_zones: name=%s, regex=%s', name, regex + ) regex = re_compile(regex) - self.log.debug('_preprocess_zones: compiled=%s', regex) + self.log.debug( + '_preprocess_zones: name=%s, compiled=%s', name, regex + ) candidates = set(z for z in candidates if regex.search(z)) else: # old-style wildcard that uses everything - self.log.debug('_preprocess_zones: old semantics, catch all') + self.log.debug( + '_preprocess_zones: name=%s, old semantics, catch all', name + ) - self.log.debug('_preprocess_zones: matches=%s', candidates) + self.log.debug( + '_preprocess_zones: name=%s, matches=%s', name, candidates + ) for match in candidates: zones[match] = config