Ross McFarland
5 months ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
43 changed files with 956 additions and 69 deletions
Split View
Diff Options
-
+4 -0.changelog/80f748502d3e4258bcbf4a05d5f5bdb0.md
-
+0 -4.changelog/acc2596fb367494db070e6c06abf705a.md
-
+4 -0.changelog/c4f025d1c23c40dd98380e6d3496364d.md
-
+1 -1.git_hooks_pre-commit
-
+51 -0.github/workflows/geo-data.yml
-
+6 -3CHANGELOG.md
-
+24 -22README.md
-
+59 -0docs/processors.md
-
+1 -1octodns/__init__.py
-
+113 -0octodns/processor/templating.py
-
+7 -0octodns/record/caa.py
-
+5 -0octodns/record/chunked.py
-
+7 -0octodns/record/ds.py
-
+1 -1octodns/record/geo_data.py
-
+3 -0octodns/record/ip.py
-
+3 -0octodns/record/loc.py
-
+7 -0octodns/record/mx.py
-
+13 -0octodns/record/naptr.py
-
+7 -0octodns/record/srv.py
-
+7 -0octodns/record/sshfp.py
-
+8 -0octodns/record/svcb.py
-
+10 -0octodns/record/target.py
-
+9 -0octodns/record/tlsa.py
-
+8 -0octodns/record/urlfwd.py
-
+30 -28requirements-dev.txt
-
+1 -1requirements.txt
-
+38 -7script/changelog
-
+4 -0script/release
-
+12 -0script/update-requirements
-
+227 -0tests/test_octodns_processor_templating.py
-
+17 -0tests/test_octodns_record_caa.py
-
+12 -0tests/test_octodns_record_chunked.py
-
+27 -0tests/test_octodns_record_ds.py
-
+8 -0tests/test_octodns_record_ip.py
-
+23 -0tests/test_octodns_record_loc.py
-
+13 -0tests/test_octodns_record_mx.py
-
+33 -0tests/test_octodns_record_naptr.py
-
+27 -0tests/test_octodns_record_srv.py
-
+21 -0tests/test_octodns_record_sshfp.py
-
+15 -0tests/test_octodns_record_svcb.py
-
+31 -1tests/test_octodns_record_target.py
-
+29 -0tests/test_octodns_record_tlsa.py
-
+30 -0tests/test_octodns_record_urlfwd.py
+ 4
- 0
.changelog/80f748502d3e4258bcbf4a05d5f5bdb0.md
View File
| @ -0,0 +1,4 @@ | |||
| --- | |||
| type: none | |||
| --- | |||
| Create dist directory in scripte/release if it doesn't exist | |||
+ 0
- 4
.changelog/acc2596fb367494db070e6c06abf705a.md
View File
| @ -1,4 +0,0 @@ | |||
| --- | |||
| type: none | |||
| --- | |||
| Adding changelog management infra and doc | |||
+ 4
- 0
.changelog/c4f025d1c23c40dd98380e6d3496364d.md
View File
| @ -0,0 +1,4 @@ | |||
| --- | |||
| type: none | |||
| --- | |||
| Documentation for processors | |||
+ 1
- 1
.git_hooks_pre-commit
View File
+ 51
- 0
.github/workflows/geo-data.yml
View File
| @ -0,0 +1,51 @@ | |||
| name: Update geo_data.py | |||
| on: | |||
| workflow_dispatch: # option to run manually if/when needed | |||
| schedule: | |||
| - cron: "42 3 * * 6" # sat @ 3:42am | |||
| jobs: | |||
| config: | |||
| runs-on: ubuntu-latest | |||
| outputs: | |||
| json: ${{ steps.load.outputs.json }} | |||
| steps: | |||
| - uses: actions/checkout@v4 | |||
| - id: load | |||
| # based on https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings | |||
| run: | | |||
| { | |||
| echo 'json<<EOF' | |||
| cat ./.ci-config.json | |||
| echo EOF | |||
| } >> $GITHUB_OUTPUT | |||
| update-geo-data: | |||
| needs: config | |||
| runs-on: ubuntu-latest | |||
| steps: | |||
| - uses: actions/checkout@v4 | |||
| - name: Setup python | |||
| uses: actions/setup-python@v4 | |||
| with: | |||
| python-version: ${{ fromJson(needs.config.outputs.json).python_version_current }} | |||
| architecture: x64 | |||
| - name: Install dependencies | |||
| run: | | |||
| python -m pip install --upgrade pip | |||
| pip install -r requirements.txt | |||
| pip install virtualenv | |||
| - name: Generate geo_data.py | |||
| run: | | |||
| ./script/bootstrap | |||
| source env/bin/activate | |||
| ./script/generate-geo-data > octodns/record/geo_data.py | |||
| git diff | |||
| [ `git status --porcelain=1 | wc -l` -ne 0 ] && ./script/changelog create -t minor Periodic updates to geo_data.py || true | |||
| - name: Create Pull Request | |||
| id: cpr | |||
| uses: peter-evans/create-pull-request@v7 | |||
| with: | |||
| commit-message: Periodic updates to geo_data.py | |||
| branch: update-geo-data | |||
| title: Update geo_data.py to reflect recent changes | |||
| body: Auto-generated with https://github.com/peter-evans/create-pull-request | |||
+ 6
- 3
CHANGELOG.md
View File
+ 24
- 22
README.md
View File
+ 59
- 0
docs/processors.md
View File
| @ -0,0 +1,59 @@ | |||
| # octoDNS processors | |||
| ## Available processors | |||
| These are listed in the main [`README`](../README.md#processors) | |||
| ## Configuring processors | |||
| Configuring processors is done in the main config file. | |||
| ### Defining a processor configuration | |||
| This is done under the top-level `processors` key in the octoDNS config file (for example `config.yaml`), as a sibling to the `manager` key. | |||
| The `processors` key contains YAML objects, where the key is the name of the processor, and the `class` value within that object refers to the processor name. | |||
| For example, to define a provider called `custom_meta` using the [`MetaProcessor`](../octodns/processor/meta.py) in order to extend the default `include_meta` behaviour: | |||
| ```yaml | |||
| manager: | |||
| include_meta: false # disable default, basic `meta` records | |||
| processors: | |||
| custom_meta: | |||
| class: octodns.processor.meta.MetaProcessor | |||
| record_name: meta | |||
| include_time: true | |||
| include_uuid: true | |||
| include_provider: true | |||
| include_version: false | |||
| ``` | |||
| **NOTE:** the specific parameters for each processor are only documented within [the code](../octodns/processor/) | |||
| ### Utilising the processor configuration | |||
| #### On **individual** domains | |||
| Each domain can utilise the processor independently by adding the name of the defined processor to a `processors` key beneath a `zone`: | |||
| ```yaml | |||
| zones: | |||
| example.com.: | |||
| source: | |||
| - yaml_config | |||
| target: | |||
| - hetzner | |||
| processors: | |||
| - custom_meta | |||
| ``` | |||
| #### On **all** domains | |||
| To utilise the processor on **all** domains automatically, including new domains added to the `zones` config in future then you can add this to the `processors` key under the `manager` section of the configuration: | |||
| ```yaml | |||
| manager: | |||
| processors: | |||
| - custom_meta | |||
| ``` | |||
+ 1
- 1
octodns/__init__.py
View File
| @ -1,4 +1,4 @@ | |||
| 'OctoDNS: DNS as code - Tools for managing DNS across multiple providers' | |||
| # TODO: remove __VERSION__ w/2.x | |||
| __version__ = __VERSION__ = '1.11.0' | |||
| __version__ = __VERSION__ = '1.12.0' | |||
+ 113
- 0
octodns/processor/templating.py
View File
| @ -0,0 +1,113 @@ | |||
| # | |||
| # | |||
| # | |||
| from octodns.processor.base import BaseProcessor | |||
| class Templating(BaseProcessor): | |||
| ''' | |||
| Record templating using python format. For simple records like TXT and CAA | |||
| that is the value itself. For multi-field records like MX or SRV it's the | |||
| text portions, exchange and target respectively. | |||
| Example Processor Config: | |||
| templating: | |||
| class: octodns.processor.templating.Templating | |||
| # Any k/v present in context will be passed into the .format method and | |||
| # thus be available as additional variables in the template. This is all | |||
| # optional. | |||
| context: | |||
| key: value | |||
| another: 42 | |||
| Example Records: | |||
| foo: | |||
| type: TXT | |||
| value: The zone this record lives in is {zone_name}. There are {zone_num_records} record(s). | |||
| bar: | |||
| type: MX | |||
| values: | |||
| - preference: 1 | |||
| exchange: mx1.{zone_name}.mail.mx. | |||
| - preference: 1 | |||
| exchange: mx2.{zone_name}.mail.mx. | |||
| Note that validations for some types reject values with {}. When | |||
| encountered the best option is to use record level `lenient: true` | |||
| https://github.com/octodns/octodns/blob/main/docs/records.md#lenience | |||
| Note that if you need to add dynamic context you can create a custom | |||
| processor that inherits from Templating and passes them into the call to | |||
| super, e.g. | |||
| class MyTemplating(Templating): | |||
| def __init__(self, *args, context={}, **kwargs): | |||
| context['year'] = lambda desired, sources: datetime.now().strftime('%Y') | |||
| super().__init__(*args, context, **kwargs) | |||
| See https://docs.python.org/3/library/string.html#custom-string-formatting | |||
| for details on formatting options. Anything possible in an `f-string` or | |||
| `.format` should work here. | |||
| ''' | |||
| def __init__(self, id, *args, context={}, **kwargs): | |||
| super().__init__(id, *args, **kwargs) | |||
| self.context = context | |||
| def process_source_zone(self, desired, sources): | |||
| sources = sources or [] | |||
| zone_params = { | |||
| 'zone_name': desired.decoded_name, | |||
| 'zone_decoded_name': desired.decoded_name, | |||
| 'zone_encoded_name': desired.name, | |||
| 'zone_num_records': len(desired.records), | |||
| 'zone_source_ids': ', '.join(s.id for s in sources), | |||
| # add any extra context provided to us, if the value is a callable | |||
| # object call it passing our params so that arbitrary dynamic | |||
| # context can be added for use in formatting | |||
| **{ | |||
| k: (v(desired, sources) if callable(v) else v) | |||
| for k, v in self.context.items() | |||
| }, | |||
| } | |||
| def params(record): | |||
| return { | |||
| 'record_name': record.decoded_name, | |||
| 'record_decoded_name': record.decoded_name, | |||
| 'record_encoded_name': record.name, | |||
| 'record_fqdn': record.decoded_fqdn, | |||
| 'record_decoded_fqdn': record.decoded_fqdn, | |||
| 'record_encoded_fqdn': record.fqdn, | |||
| 'record_type': record._type, | |||
| 'record_ttl': record.ttl, | |||
| 'record_source_id': record.source.id if record.source else None, | |||
| **zone_params, | |||
| } | |||
| for record in desired.records: | |||
| if hasattr(record, 'values'): | |||
| if record.values and not hasattr(record.values[0], 'template'): | |||
| # the (custom) value type does not support templating | |||
| continue | |||
| new_values = [v.template(params(record)) for v in record.values] | |||
| if record.values != new_values: | |||
| new = record.copy() | |||
| new.values = new_values | |||
| desired.add_record(new, replace=True) | |||
| else: | |||
| if not hasattr(record.value, 'template'): | |||
| # the (custom) value type does not support templating | |||
| continue | |||
| new_value = record.value.template(params(record)) | |||
| if record.value != new_value: | |||
| new = record.copy() | |||
| new.value = new_value | |||
| desired.add_record(new, replace=True) | |||
| return desired | |||
+ 7
- 0
octodns/record/caa.py
View File
+ 5
- 0
octodns/record/chunked.py
View File
+ 7
- 0
octodns/record/ds.py
View File
+ 1
- 1
octodns/record/geo_data.py
View File
+ 3
- 0
octodns/record/ip.py
View File
+ 3
- 0
octodns/record/loc.py
View File
+ 7
- 0
octodns/record/mx.py
View File
+ 13
- 0
octodns/record/naptr.py
View File
+ 7
- 0
octodns/record/srv.py
View File
+ 7
- 0
octodns/record/sshfp.py
View File
+ 8
- 0
octodns/record/svcb.py
View File
+ 10
- 0
octodns/record/target.py
View File
+ 9
- 0
octodns/record/tlsa.py
View File
+ 8
- 0
octodns/record/urlfwd.py
View File
+ 30
- 28
requirements-dev.txt
View File
| @ -1,48 +1,50 @@ | |||
| # DO NOT EDIT THIS FILE DIRECTLY - use ./script/update-requirements to update | |||
| Pygments==2.18.0 | |||
| Pygments==2.19.2 | |||
| SecretStorage==3.3.3 | |||
| black==24.10.0 | |||
| build==1.2.2.post1 | |||
| certifi==2024.8.30 | |||
| certifi==2025.6.15 | |||
| cffi==1.17.1 | |||
| charset-normalizer==3.3.2 | |||
| click==8.1.7 | |||
| cmarkgfm==2024.1.14 | |||
| coverage==7.6.1 | |||
| charset-normalizer==3.4.2 | |||
| click==8.1.8; python_version<'3.10' | |||
| click==8.2.1; python_version>='3.10' | |||
| cmarkgfm==2024.11.20 | |||
| coverage==7.9.1 | |||
| cryptography==45.0.4 | |||
| docutils==0.21.2 | |||
| importlib_metadata==8.5.0 | |||
| iniconfig==2.0.0 | |||
| isort==5.13.2 | |||
| id==1.5.0 | |||
| iniconfig==2.1.0 | |||
| isort==6.0.1 | |||
| jaraco.classes==3.4.0 | |||
| jaraco.context==6.0.1 | |||
| jaraco.functools==4.1.0 | |||
| keyring==25.4.1 | |||
| jaraco.functools==4.2.1 | |||
| jeepney==0.9.0 | |||
| keyring==25.6.0 | |||
| markdown-it-py==3.0.0 | |||
| mdurl==0.1.2 | |||
| more-itertools==10.5.0 | |||
| mypy-extensions==1.0.0 | |||
| nh3==0.2.18 | |||
| packaging==24.1 | |||
| more-itertools==10.7.0 | |||
| mypy_extensions==1.1.0 | |||
| nh3==0.2.21 | |||
| packaging==25.0 | |||
| pathspec==0.12.1 | |||
| pkginfo==1.10.0 | |||
| platformdirs==4.3.6 | |||
| pluggy==1.5.0 | |||
| platformdirs==4.3.8 | |||
| pluggy==1.6.0 | |||
| pprintpp==0.4.0 | |||
| pycountry-convert==0.7.2 | |||
| pycountry==24.6.1 | |||
| pycparser==2.22 | |||
| pyflakes==3.2.0 | |||
| pyflakes==3.4.0 | |||
| pyproject_hooks==1.2.0 | |||
| pytest-cov==5.0.0 | |||
| pytest-mock==3.14.0 | |||
| pytest==8.3.3 | |||
| pytest-cov==6.2.1 | |||
| pytest-mock==3.14.1 | |||
| pytest==8.4.1 | |||
| pytest_network==0.0.1 | |||
| readme_renderer==44.0 | |||
| repoze.lru==0.7 | |||
| requests-toolbelt==1.0.0 | |||
| requests==2.32.3 | |||
| requests==2.32.4 | |||
| rfc3986==2.0.0 | |||
| rich==13.9.2 | |||
| twine==5.1.1 | |||
| urllib3==2.2.3 | |||
| wheel==0.44.0 | |||
| zipp==3.20.2 | |||
| rich==14.0.0 | |||
| twine==6.1.0 | |||
| urllib3==2.5.0 | |||
| wheel==0.45.1 | |||
+ 1
- 1
requirements.txt
View File
+ 38
- 7
script/changelog
View File
+ 4
- 0
script/release
View File
+ 12
- 0
script/update-requirements
View File
+ 227
- 0
tests/test_octodns_processor_templating.py
View File
| @ -0,0 +1,227 @@ | |||
| # | |||
| # | |||
| # | |||
| from unittest import TestCase | |||
| from unittest.mock import call, patch | |||
| from octodns.processor.templating import Templating | |||
| from octodns.record import Record, ValueMixin, ValuesMixin | |||
| from octodns.zone import Zone | |||
| class DummySource: | |||
| def __init__(self, id): | |||
| self.id = str(id) | |||
| class CustomValue(str): | |||
| @classmethod | |||
| def validate(cls, *args, **kwargs): | |||
| return [] | |||
| @classmethod | |||
| def process(cls, v): | |||
| if isinstance(v, (list, tuple)): | |||
| return (CustomValue(i) for i in v) | |||
| return CustomValue(v) | |||
| @classmethod | |||
| def parse_rdata_text(cls, *args, **kwargs): | |||
| pass | |||
| def __init__(self, *args, **kwargs): | |||
| self._asked_for = set() | |||
| def rdata_text(self): | |||
| pass | |||
| def __getattr__(self, item): | |||
| self._asked_for.add(item) | |||
| raise AttributeError('nope') | |||
| class Single(ValueMixin, Record): | |||
| _type = 'S' | |||
| _value_type = CustomValue | |||
| Record.register_type(Single, 'S') | |||
| class Multiple(ValuesMixin, Record): | |||
| _type = 'M' | |||
| _value_type = CustomValue | |||
| Record.register_type(Multiple, 'M') | |||
| def _find(zone, name): | |||
| return next(r for r in zone.records if r.name == name) | |||
| class TemplatingTest(TestCase): | |||
| def test_cname(self): | |||
| templ = Templating('test') | |||
| zone = Zone('unit.tests.', []) | |||
| cname = Record.new( | |||
| zone, | |||
| 'cname', | |||
| { | |||
| 'type': 'CNAME', | |||
| 'ttl': 42, | |||
| 'value': '_cname.{zone_name}something.else.', | |||
| }, | |||
| lenient=True, | |||
| ) | |||
| zone.add_record(cname) | |||
| noop = Record.new( | |||
| zone, | |||
| 'noop', | |||
| { | |||
| 'type': 'CNAME', | |||
| 'ttl': 42, | |||
| 'value': '_noop.nothing_to_do.something.else.', | |||
| }, | |||
| lenient=True, | |||
| ) | |||
| zone.add_record(noop) | |||
| got = templ.process_source_zone(zone, None) | |||
| cname = _find(got, 'cname') | |||
| self.assertEqual('_cname.unit.tests.something.else.', cname.value) | |||
| noop = _find(got, 'noop') | |||
| self.assertEqual('_noop.nothing_to_do.something.else.', noop.value) | |||
| def test_txt(self): | |||
| templ = Templating('test') | |||
| zone = Zone('unit.tests.', []) | |||
| txt = Record.new( | |||
| zone, | |||
| 'txt', | |||
| { | |||
| 'type': 'TXT', | |||
| 'ttl': 42, | |||
| 'value': 'There are {zone_num_records} record(s) in {zone_name}', | |||
| }, | |||
| ) | |||
| zone.add_record(txt) | |||
| noop = Record.new( | |||
| zone, | |||
| 'noop', | |||
| {'type': 'TXT', 'ttl': 43, 'value': 'Nothing to template here.'}, | |||
| ) | |||
| zone.add_record(noop) | |||
| got = templ.process_source_zone(zone, None) | |||
| txt = _find(got, 'txt') | |||
| self.assertEqual('There are 2 record(s) in unit.tests.', txt.values[0]) | |||
| noop = _find(got, 'noop') | |||
| self.assertEqual('Nothing to template here.', noop.values[0]) | |||
| def test_no_template(self): | |||
| templ = Templating('test') | |||
| zone = Zone('unit.tests.', []) | |||
| s = Record.new(zone, 's', {'type': 'S', 'ttl': 42, 'value': 'string'}) | |||
| zone.add_record(s) | |||
| m = Record.new( | |||
| zone, 'm', {'type': 'M', 'ttl': 43, 'values': ('string', 'another')} | |||
| ) | |||
| zone.add_record(m) | |||
| # this should check for the template method on our values that don't | |||
| # have one | |||
| templ.process_source_zone(zone, None) | |||
| # and these should make sure that the value types were asked if they | |||
| # have a template method | |||
| self.assertEqual({'template'}, s.value._asked_for) | |||
| self.assertEqual({'template'}, m.values[0]._asked_for) | |||
| @patch('octodns.record.TxtValue.template') | |||
| def test_params(self, mock_template): | |||
| templ = Templating('test') | |||
| zone = Zone('unit.tests.', []) | |||
| record_source = DummySource('record') | |||
| txt = Record.new( | |||
| zone, | |||
| 'txt', | |||
| { | |||
| 'type': 'TXT', | |||
| 'ttl': 42, | |||
| 'value': 'There are {zone_num_records} record(s) in {zone_name}', | |||
| }, | |||
| source=record_source, | |||
| ) | |||
| zone.add_record(txt) | |||
| templ.process_source_zone( | |||
| zone, sources=[record_source, DummySource('other')] | |||
| ) | |||
| mock_template.assert_called_once() | |||
| self.assertEqual( | |||
| call( | |||
| { | |||
| 'record_name': 'txt', | |||
| 'record_decoded_name': 'txt', | |||
| 'record_encoded_name': 'txt', | |||
| 'record_fqdn': 'txt.unit.tests.', | |||
| 'record_decoded_fqdn': 'txt.unit.tests.', | |||
| 'record_encoded_fqdn': 'txt.unit.tests.', | |||
| 'record_type': 'TXT', | |||
| 'record_ttl': 42, | |||
| 'record_source_id': 'record', | |||
| 'zone_name': 'unit.tests.', | |||
| 'zone_decoded_name': 'unit.tests.', | |||
| 'zone_encoded_name': 'unit.tests.', | |||
| 'zone_num_records': 1, | |||
| 'zone_source_ids': 'record, other', | |||
| } | |||
| ), | |||
| mock_template.call_args, | |||
| ) | |||
| def test_context(self): | |||
| templ = Templating( | |||
| 'test', | |||
| context={ | |||
| # static | |||
| 'the_answer': 42, | |||
| # dynamic | |||
| 'the_date': lambda _, __: 'today', | |||
| # uses a param | |||
| 'num_sources': lambda z, ss: len(ss), | |||
| }, | |||
| ) | |||
| zone = Zone('unit.tests.', []) | |||
| txt = Record.new( | |||
| zone, | |||
| 'txt', | |||
| { | |||
| 'type': 'TXT', | |||
| 'ttl': 42, | |||
| 'values': ( | |||
| 'the_answer: {the_answer}', | |||
| 'the_date: {the_date}', | |||
| 'num_sources: {num_sources}', | |||
| ), | |||
| }, | |||
| ) | |||
| zone.add_record(txt) | |||
| got = templ.process_source_zone( | |||
| zone, tuple(DummySource(i) for i in range(3)) | |||
| ) | |||
| txt = _find(got, 'txt') | |||
| self.assertEqual(3, len(txt.values)) | |||
| self.assertEqual('num_sources: 3', txt.values[0]) | |||
| self.assertEqual('the_answer: 42', txt.values[1]) | |||
| self.assertEqual('the_date: today', txt.values[2]) | |||