From f5fd68bb7e847aaee098f6e1c4e3f291c53506ed Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Wed, 8 Nov 2023 17:40:34 +0100 Subject: [PATCH 1/9] add NetworkValueRejectlistFilter and NetworkValueAllowlistFilter processors --- octodns/processor/filter.py | 85 ++++++++++++++++++++++++++ tests/test_octodns_processor_filter.py | 34 +++++++++++ 2 files changed, 119 insertions(+) diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py index 2de3b23..db14c85 100644 --- a/octodns/processor/filter.py +++ b/octodns/processor/filter.py @@ -2,6 +2,8 @@ # # +from ipaddress import ip_address, ip_network +from itertools import product from logging import getLogger from re import compile as re_compile @@ -125,6 +127,35 @@ class _NameBaseFilter(BaseProcessor): process_target_zone = _process +class _NetworkValueBaseFilter(BaseProcessor): + def __init__(self, name, _list): + super().__init__(name) + self.networks = [] + for value in _list: + try: + self.networks.append(ip_network(value)) + except ValueError: + raise ValueError(f'{value} is not a valid CIDR to use') + + def _process(self, zone, *args, **kwargs): + for record in zone.records: + if record._type not in ['A', 'AAAA']: + continue + + if any( + ip_address(value) in network + for value, network in product(record.values, self.networks) + ): + self.matches(zone, record) + else: + self.doesnt_match(zone, record) + + return zone + + process_source_zone = _process + process_target_zone = _process + + class NameAllowlistFilter(_NameBaseFilter, AllowsMixin): '''Only manage records with names that match the provider patterns @@ -189,6 +220,60 @@ class NameRejectlistFilter(_NameBaseFilter, RejectsMixin): super().__init__(name, rejectlist) +class NetworkValueAllowlistFilter(_NetworkValueBaseFilter, AllowsMixin): + '''Only manage records with values that match the provider patterns + + Example usage: + + processors: + only-these: + class: octodns.processor.filter.NetworkValueAllowlistFilter + allowlist: + - 127.0.0.1/32 + - 192.168.0.0/16 + - fd00::/8 + + zones: + exxampled.com.: + sources: + - config + processors: + - only-these + targets: + - route53 + ''' + + def __init__(self, name, allowlist): + super().__init__(name, allowlist) + + +class NetworkValueRejectlistFilter(_NetworkValueBaseFilter, RejectsMixin): + '''Reject managing records with value matching a that match the provider patterns + + Example usage: + + processors: + not-these: + class: octodns.processor.filter.NetworkValueRejectlistFilter + rejectlist: + - 127.0.0.1/32 + - 192.168.0.0/16 + - fd00::/8 + + zones: + exxampled.com.: + sources: + - config + processors: + - not-these + targets: + - route53 + ''' + + def __init__(self, name, rejectlist): + super().__init__(name, rejectlist) + + class IgnoreRootNsFilter(BaseProcessor): '''Do not manage Root NS Records. diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py index 2d9b881..4ecef63 100644 --- a/tests/test_octodns_processor_filter.py +++ b/tests/test_octodns_processor_filter.py @@ -9,6 +9,8 @@ from octodns.processor.filter import ( IgnoreRootNsFilter, NameAllowlistFilter, NameRejectlistFilter, + NetworkValueAllowlistFilter, + NetworkValueRejectlistFilter, TypeAllowlistFilter, TypeRejectlistFilter, ZoneNameFilter, @@ -161,6 +163,38 @@ class TestNameRejectListFilter(TestCase): ) +class TestNetworkValueFilter(TestCase): + zone = Zone('unit.tests.', []) + matches = Record.new( + zone, 'private-ipv4', {'type': 'A', 'ttl': 42, 'value': '10.42.42.42'} + ) + zone.add_record(matches) + doesnt = Record.new( + zone, 'public-ipv4', {'type': 'A', 'ttl': 42, 'value': '42.42.42.42'} + ) + zone.add_record(doesnt) + matchable1 = Record.new( + zone, 'private-ipv6', {'type': 'AAAA', 'ttl': 42, 'value': 'fd12:3456:789a:1::1'} + ) + zone.add_record(matchable1) + matchable2 = Record.new( + zone, 'public-ipv6', {'type': 'AAAA', 'ttl': 42, 'value': 'dead:beef:cafe::1'} + ) + zone.add_record(matchable2) + + def test_reject(self): + filter_private = NetworkValueRejectlistFilter('rejectlist', set(('10.0.0.0/8', 'fd00::/8'))) + + got = filter_private.process_source_zone(self.zone.copy()) + self.assertEqual(['public-ipv4', 'public-ipv6'], sorted([r.name for r in got.records])) + + def test_allow(self): + filter_private = NetworkValueAllowlistFilter('allowlist', set(('10.0.0.0/8', 'fd00::/8'))) + + got = filter_private.process_source_zone(self.zone.copy()) + self.assertEqual(['private-ipv4', 'private-ipv6'], sorted([r.name for r in got.records])) + + class TestIgnoreRootNsFilter(TestCase): zone = Zone('unit.tests.', []) root = Record.new( From 354b8c2967532f3941ed1dabb7e89831a4570067 Mon Sep 17 00:00:00 2001 From: Solvik Date: Thu, 9 Nov 2023 17:47:55 +0100 Subject: [PATCH 2/9] do not recreate the ip list for each network test Co-authored-by: Ross McFarland --- octodns/processor/filter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py index db14c85..34115d5 100644 --- a/octodns/processor/filter.py +++ b/octodns/processor/filter.py @@ -142,9 +142,10 @@ class _NetworkValueBaseFilter(BaseProcessor): if record._type not in ['A', 'AAAA']: continue + ips = [ip_address(value) for value in record.values] if any( - ip_address(value) in network - for value, network in product(record.values, self.networks) + ip in network + for ip, network in product(ips, self.networks) ): self.matches(zone, record) else: From f9cb31b602c99761f5e9fc4cc02e31866a4e8494 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Thu, 9 Nov 2023 19:01:59 +0100 Subject: [PATCH 3/9] add a txt in tests so we can see the filter effectively only handles A/AAAA --- tests/test_octodns_processor_filter.py | 62 +++++++++++++++++--------- 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py index 4ecef63..409ec50 100644 --- a/tests/test_octodns_processor_filter.py +++ b/tests/test_octodns_processor_filter.py @@ -165,34 +165,56 @@ class TestNameRejectListFilter(TestCase): class TestNetworkValueFilter(TestCase): zone = Zone('unit.tests.', []) - matches = Record.new( - zone, 'private-ipv4', {'type': 'A', 'ttl': 42, 'value': '10.42.42.42'} - ) - zone.add_record(matches) - doesnt = Record.new( - zone, 'public-ipv4', {'type': 'A', 'ttl': 42, 'value': '42.42.42.42'} - ) - zone.add_record(doesnt) - matchable1 = Record.new( - zone, 'private-ipv6', {'type': 'AAAA', 'ttl': 42, 'value': 'fd12:3456:789a:1::1'} - ) - zone.add_record(matchable1) - matchable2 = Record.new( - zone, 'public-ipv6', {'type': 'AAAA', 'ttl': 42, 'value': 'dead:beef:cafe::1'} - ) - zone.add_record(matchable2) + for record in [ + Record.new( + zone, + 'private-ipv4', + {'type': 'A', 'ttl': 42, 'value': '10.42.42.42'}, + ), + Record.new( + zone, + 'public-ipv4', + {'type': 'A', 'ttl': 42, 'value': '42.42.42.42'}, + ), + Record.new( + zone, + 'private-ipv6', + {'type': 'AAAA', 'ttl': 42, 'value': 'fd12:3456:789a:1::1'}, + ), + Record.new( + zone, + 'public-ipv6', + {'type': 'AAAA', 'ttl': 42, 'value': 'dead:beef:cafe::1'}, + ), + Record.new( + zone, + 'keep-me', + {'ttl': 30, 'type': 'TXT', 'value': 'this should always be here'}, + ), + ]: + zone.add_record(record) def test_reject(self): - filter_private = NetworkValueRejectlistFilter('rejectlist', set(('10.0.0.0/8', 'fd00::/8'))) + filter_private = NetworkValueRejectlistFilter( + 'rejectlist', set(('10.0.0.0/8', 'fd00::/8')) + ) got = filter_private.process_source_zone(self.zone.copy()) - self.assertEqual(['public-ipv4', 'public-ipv6'], sorted([r.name for r in got.records])) + self.assertEqual( + ['keep-me', 'public-ipv4', 'public-ipv6'], + sorted([r.name for r in got.records]), + ) def test_allow(self): - filter_private = NetworkValueAllowlistFilter('allowlist', set(('10.0.0.0/8', 'fd00::/8'))) + filter_private = NetworkValueAllowlistFilter( + 'allowlist', set(('10.0.0.0/8', 'fd00::/8')) + ) got = filter_private.process_source_zone(self.zone.copy()) - self.assertEqual(['private-ipv4', 'private-ipv6'], sorted([r.name for r in got.records])) + self.assertEqual( + ['keep-me', 'private-ipv4', 'private-ipv6'], + sorted([r.name for r in got.records]), + ) class TestIgnoreRootNsFilter(TestCase): From 010e5039cca36eced58d4ad926f21770565e3f36 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Thu, 9 Nov 2023 19:20:46 +0100 Subject: [PATCH 4/9] add mention to docstring that the NetworkValue filters won't touch anything except A/AAAA --- octodns/processor/filter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py index 34115d5..77f66ea 100644 --- a/octodns/processor/filter.py +++ b/octodns/processor/filter.py @@ -222,7 +222,8 @@ class NameRejectlistFilter(_NameBaseFilter, RejectsMixin): class NetworkValueAllowlistFilter(_NetworkValueBaseFilter, AllowsMixin): - '''Only manage records with values that match the provider patterns + '''Only manage A and AAAA records with values that match the provider patterns + All other types will be left as-is. Example usage: @@ -249,7 +250,8 @@ class NetworkValueAllowlistFilter(_NetworkValueBaseFilter, AllowsMixin): class NetworkValueRejectlistFilter(_NetworkValueBaseFilter, RejectsMixin): - '''Reject managing records with value matching a that match the provider patterns + '''Reject managing A and AAAA records with value matching a that match the provider patterns + All other types will be left as-is. Example usage: From fa56dfaffddb1c5e183fabf10cd8b0d3d2619baf Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 9 Nov 2023 11:34:16 -0800 Subject: [PATCH 5/9] fix minor formatting failure --- octodns/processor/filter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py index 77f66ea..ddc35d0 100644 --- a/octodns/processor/filter.py +++ b/octodns/processor/filter.py @@ -144,8 +144,7 @@ class _NetworkValueBaseFilter(BaseProcessor): ips = [ip_address(value) for value in record.values] if any( - ip in network - for ip, network in product(ips, self.networks) + ip in network for ip, network in product(ips, self.networks) ): self.matches(zone, record) else: From 3ed7a88e343c89b7153efea25db1b6287b2f0823 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Fri, 10 Nov 2023 13:58:08 +0100 Subject: [PATCH 6/9] add test to cover CIDR validation in config for filters --- tests/test_octodns_processor_filter.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py index 409ec50..6525900 100644 --- a/tests/test_octodns_processor_filter.py +++ b/tests/test_octodns_processor_filter.py @@ -194,6 +194,12 @@ class TestNetworkValueFilter(TestCase): ]: zone.add_record(record) + def test_bad_config(self): + with self.assertRaises(ValueError): + filter_private = NetworkValueRejectlistFilter( + 'rejectlist', set(('string', '42.42.42.42/43')) + ) + def test_reject(self): filter_private = NetworkValueRejectlistFilter( 'rejectlist', set(('10.0.0.0/8', 'fd00::/8')) From abdab8f6d83894e9f37eb3213efa828dfc741cd0 Mon Sep 17 00:00:00 2001 From: Solvik Blum Date: Fri, 10 Nov 2023 15:28:01 +0100 Subject: [PATCH 7/9] fix lint --- tests/test_octodns_processor_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py index 6525900..7ee98a8 100644 --- a/tests/test_octodns_processor_filter.py +++ b/tests/test_octodns_processor_filter.py @@ -196,7 +196,7 @@ class TestNetworkValueFilter(TestCase): def test_bad_config(self): with self.assertRaises(ValueError): - filter_private = NetworkValueRejectlistFilter( + NetworkValueRejectlistFilter( 'rejectlist', set(('string', '42.42.42.42/43')) ) From 731eb56ab91cf01fd4f324ad303530a8beba84a9 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 10 Nov 2023 16:08:22 -0800 Subject: [PATCH 8/9] CHANGELOG entry for network cidr filters --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dde47b..180b245 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ octodns.com.octodns.com * Fixed issues with handling of chunking large TXT values for providers that use the in-built `rrs` method +* NetworkValueAllowlistFilter/NetworkValueRejectlistFilter added to + processors.filter to enable filtering A/AAAA records based on value. Can be + useful if you have records with non-routable values in an internal copy of a + zone, but want to exclude them when pushing the same zone publically (split + horizon) * ExcludeRootNsChanges processor that will error (or warn) if plan includes a change to root NS records * Include the octodns special section info in Record __repr__, makes it easier From de3ec8e094d85faa45667509fd286d8aa874d6d8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 10 Nov 2023 17:02:20 -0800 Subject: [PATCH 9/9] Move _NetworkValueBaseFilter down with it's children --- octodns/processor/filter.py | 58 ++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py index ddc35d0..e7913d8 100644 --- a/octodns/processor/filter.py +++ b/octodns/processor/filter.py @@ -127,35 +127,6 @@ class _NameBaseFilter(BaseProcessor): process_target_zone = _process -class _NetworkValueBaseFilter(BaseProcessor): - def __init__(self, name, _list): - super().__init__(name) - self.networks = [] - for value in _list: - try: - self.networks.append(ip_network(value)) - except ValueError: - raise ValueError(f'{value} is not a valid CIDR to use') - - def _process(self, zone, *args, **kwargs): - for record in zone.records: - if record._type not in ['A', 'AAAA']: - continue - - ips = [ip_address(value) for value in record.values] - if any( - ip in network for ip, network in product(ips, self.networks) - ): - self.matches(zone, record) - else: - self.doesnt_match(zone, record) - - return zone - - process_source_zone = _process - process_target_zone = _process - - class NameAllowlistFilter(_NameBaseFilter, AllowsMixin): '''Only manage records with names that match the provider patterns @@ -220,6 +191,35 @@ class NameRejectlistFilter(_NameBaseFilter, RejectsMixin): super().__init__(name, rejectlist) +class _NetworkValueBaseFilter(BaseProcessor): + def __init__(self, name, _list): + super().__init__(name) + self.networks = [] + for value in _list: + try: + self.networks.append(ip_network(value)) + except ValueError: + raise ValueError(f'{value} is not a valid CIDR to use') + + def _process(self, zone, *args, **kwargs): + for record in zone.records: + if record._type not in ['A', 'AAAA']: + continue + + ips = [ip_address(value) for value in record.values] + if any( + ip in network for ip, network in product(ips, self.networks) + ): + self.matches(zone, record) + else: + self.doesnt_match(zone, record) + + return zone + + process_source_zone = _process + process_target_zone = _process + + class NetworkValueAllowlistFilter(_NetworkValueBaseFilter, AllowsMixin): '''Only manage A and AAAA records with values that match the provider patterns All other types will be left as-is.