From e9d8b023650448de275034b0d3414096bf0fc78f Mon Sep 17 00:00:00 2001 From: Nullreff Date: Sun, 14 Jan 2024 08:56:51 -0800 Subject: [PATCH 1/2] Add filters for checking record values Create two new filters, ValueAllowlistFilter and ValueRejectlistFilter that allow checing the value(s) of records to include or exclude, similar to the name filters that alread exist --- README.md | 2 + octodns/processor/filter.py | 105 ++++++++++++++++++ tests/test_octodns_processor_filter.py | 142 +++++++++++++++++++++++++ 3 files changed, 249 insertions(+) diff --git a/README.md b/README.md index 85bf0c6..df00bf2 100644 --- a/README.md +++ b/README.md @@ -335,6 +335,8 @@ Similar to providers, but can only serve to populate records into a zone, cannot | [MetaProcessor](/octodns/processor/meta.py) | Adds a special meta record with timing, UUID, providers, and/or version to aid in debugging and monitoring. | | [NameAllowlistFilter](/octodns/processor/filter.py) | Filter that ONLY manages records that match specified naming patterns, all others will be ignored | | [NameRejectlistFilter](/octodns/processor/filter.py) | Filter that INGORES records that match specified naming patterns, all others will be managed | +| [ValueAllowlistFilter](/octodns/processor/filter.py) | Filter that ONLY manages records that match specified value patterns, all others will be ignored | +| [ValueRejectlistFilter](/octodns/processor/filter.py) | Filter that INGORES records that match specified value patterns, all others will be managed | | [OwnershipProcessor](/octodns/processor/ownership.py) | Processor that implements ownership in octoDNS so that it can manage only the records in a zone in sources and will ignore all others. | | [SpfDnsLookupProcessor](/octodns/processor/spf.py) | Processor that checks SPF values for violations of DNS query limits | | [TtlRestrictionFilter](/octodns/processor/restrict.py) | Processor that restricts the allow TTL values to a specified range or list of specific values | diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py index cd1c82d..43dd515 100644 --- a/octodns/processor/filter.py +++ b/octodns/processor/filter.py @@ -215,6 +215,111 @@ class NameRejectlistFilter(_NameBaseFilter, RejectsMixin): super().__init__(name, rejectlist) +class _ValueBaseFilter(_FilterProcessor): + def __init__(self, name, _list, **kwargs): + super().__init__(name, **kwargs) + exact = set() + regex = [] + for pattern in _list: + if pattern.startswith('/'): + regex.append(re_compile(pattern[1:-1])) + else: + exact.add(pattern) + self.exact = exact + self.regex = regex + + def _process(self, zone, *args, **kwargs): + for record in zone.records: + values = [] + if hasattr(record, 'values'): + values = [str(value) for value in record.values] + else: + values = [str(record.value)] + + if any(value in self.exact for value in values): + self.matches(zone, record) + continue + elif any(r.search(value) for r in self.regex for value in values): + self.matches(zone, record) + continue + + self.doesnt_match(zone, record) + + return zone + + +class ValueAllowlistFilter(_ValueBaseFilter, AllowsMixin): + '''Only manage records with values that match the provider patterns + + Example usage: + + processors: + only-these: + class: octodns.processor.filter.ValueAllowlistFilter + allowlist: + # exact string match + - www + # contains/substring match + - /substring/ + # regex pattern match + - /some-pattern-\\d\\+/ + # regex - anchored so has to match start to end + - /^start-.+-end$/ + # Optional param that can be set to False to leave the target zone + # alone, thus allowing deletion of existing records + # (default: true) + # include_target: True + + zones: + exxampled.com.: + sources: + - config + processors: + - only-these + targets: + - route53 + ''' + + def __init__(self, name, allowlist): + super().__init__(name, allowlist) + + +class ValueRejectlistFilter(_ValueBaseFilter, RejectsMixin): + '''Reject managing records with names that match the provider patterns + + Example usage: + + processors: + not-these: + class: octodns.processor.filter.ValueRejectlistFilter + rejectlist: + # exact string match + - www + # contains/substring match + - /substring/ + # regex pattern match + - /some-pattern-\\d\\+/ + # regex - anchored so has to match start to end + - /^start-.+-end$/ + # Optional param that can be set to False to leave the target zone + # alone, thus allowing deletion of existing records + # (default: true) + # include_target: True + + zones: + exxampled.com.: + sources: + - config + processors: + - not-these + targets: + - route53 + ''' + + def __init__(self, name, rejectlist): + super().__init__(name, rejectlist) + + class _NetworkValueBaseFilter(BaseProcessor): def __init__(self, name, _list): super().__init__(name) diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py index 1880a16..0c00d19 100644 --- a/tests/test_octodns_processor_filter.py +++ b/tests/test_octodns_processor_filter.py @@ -13,6 +13,8 @@ from octodns.processor.filter import ( NetworkValueRejectlistFilter, TypeAllowlistFilter, TypeRejectlistFilter, + ValueAllowlistFilter, + ValueRejectlistFilter, ZoneNameFilter, ) from octodns.provider.plan import Plan @@ -179,6 +181,146 @@ class TestNameRejectListFilter(TestCase): ) +class TestValueAllowListFilter(TestCase): + zone = Zone('unit.tests.', []) + matches = Record.new( + zone, + 'good.exact', + {'type': 'CNAME', 'ttl': 42, 'value': 'matches.example.com.'}, + ) + zone.add_record(matches) + doesnt = Record.new( + zone, + 'bad.exact', + {'type': 'CNAME', 'ttl': 42, 'value': 'doesnt.example.com.'}, + ) + zone.add_record(doesnt) + matches_many = Record.new( + zone, + 'good.values', + { + 'type': 'TXT', + 'ttl': 42, + 'values': ['matches.example.com.', 'another'], + }, + ) + zone.add_record(matches_many) + doesnt_many = Record.new( + zone, + 'bad.values', + { + 'type': 'TXT', + 'ttl': 42, + 'values': ['doesnt.example.com.', 'another'], + }, + ) + zone.add_record(doesnt_many) + matchable1 = Record.new( + zone, + 'first.regex', + {'type': 'CNAME', 'ttl': 42, 'value': 'start.f43ad96.end.'}, + ) + zone.add_record(matchable1) + matchable2 = Record.new( + zone, + 'second.regex', + {'type': 'CNAME', 'ttl': 42, 'value': 'start.a3b444c.end.'}, + ) + zone.add_record(matchable2) + + def test_exact(self): + allows = ValueAllowlistFilter('exact', ('matches.example.com.',)) + + self.assertEqual(6, len(self.zone.records)) + filtered = allows.process_source_zone(self.zone.copy()) + self.assertEqual(2, len(filtered.records)) + self.assertEqual( + ['good.exact', 'good.values'], + sorted([r.name for r in filtered.records]), + ) + + def test_regex(self): + allows = ValueAllowlistFilter('exact', ('/^start\\..+\\.end\\.$/',)) + + self.assertEqual(6, len(self.zone.records)) + filtered = allows.process_source_zone(self.zone.copy()) + self.assertEqual(2, len(filtered.records)) + self.assertEqual( + ['first.regex', 'second.regex'], + sorted([r.name for r in filtered.records]), + ) + + +class TestValueRejectListFilter(TestCase): + zone = Zone('unit.tests.', []) + matches = Record.new( + zone, + 'good.compare', + {'type': 'CNAME', 'ttl': 42, 'value': 'matches.example.com.'}, + ) + zone.add_record(matches) + doesnt = Record.new( + zone, + 'bad.compare', + {'type': 'CNAME', 'ttl': 42, 'value': 'doesnt.example.com.'}, + ) + zone.add_record(doesnt) + matches_many = Record.new( + zone, + 'good.values', + { + 'type': 'TXT', + 'ttl': 42, + 'values': ['matches.example.com.', 'another'], + }, + ) + zone.add_record(matches_many) + doesnt_many = Record.new( + zone, + 'bad.values', + { + 'type': 'TXT', + 'ttl': 42, + 'values': ['doesnt.example.com.', 'another'], + }, + ) + zone.add_record(doesnt_many) + matchable1 = Record.new( + zone, + 'first.regex', + {'type': 'CNAME', 'ttl': 42, 'value': 'start.f43ad96.end.'}, + ) + zone.add_record(matchable1) + matchable2 = Record.new( + zone, + 'second.regex', + {'type': 'CNAME', 'ttl': 42, 'value': 'start.a3b444c.end.'}, + ) + zone.add_record(matchable2) + + def test_exact(self): + rejects = ValueRejectlistFilter('exact', ('matches.example.com.',)) + + self.assertEqual(6, len(self.zone.records)) + filtered = rejects.process_source_zone(self.zone.copy()) + self.assertEqual(4, len(filtered.records)) + self.assertEqual( + ['bad.compare', 'bad.values', 'first.regex', 'second.regex'], + sorted([r.name for r in filtered.records]), + ) + + def test_regex(self): + rejects = ValueRejectlistFilter('exact', ('/^start\\..+\\.end\\.$/',)) + + self.assertEqual(6, len(self.zone.records)) + filtered = rejects.process_source_zone(self.zone.copy()) + self.assertEqual(4, len(filtered.records)) + self.assertEqual( + ['bad.compare', 'bad.values', 'good.compare', 'good.values'], + sorted([r.name for r in filtered.records]), + ) + + class TestNetworkValueFilter(TestCase): zone = Zone('unit.tests.', []) for record in [ From 7a5512f601caf3796434bb5b12f99792bb52ac25 Mon Sep 17 00:00:00 2001 From: Nullreff Date: Mon, 15 Jan 2024 06:02:48 -0800 Subject: [PATCH 2/2] Remove redundant code and use rdata_text --- README.md | 4 ++-- octodns/processor/filter.py | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index df00bf2..7ef0aff 100644 --- a/README.md +++ b/README.md @@ -335,8 +335,8 @@ Similar to providers, but can only serve to populate records into a zone, cannot | [MetaProcessor](/octodns/processor/meta.py) | Adds a special meta record with timing, UUID, providers, and/or version to aid in debugging and monitoring. | | [NameAllowlistFilter](/octodns/processor/filter.py) | Filter that ONLY manages records that match specified naming patterns, all others will be ignored | | [NameRejectlistFilter](/octodns/processor/filter.py) | Filter that INGORES records that match specified naming patterns, all others will be managed | -| [ValueAllowlistFilter](/octodns/processor/filter.py) | Filter that ONLY manages records that match specified value patterns, all others will be ignored | -| [ValueRejectlistFilter](/octodns/processor/filter.py) | Filter that INGORES records that match specified value patterns, all others will be managed | +| [ValueAllowlistFilter](/octodns/processor/filter.py) | Filter that ONLY manages records that match specified value patterns based on `rdata_text`, all others will be ignored | +| [ValueRejectlistFilter](/octodns/processor/filter.py) | Filter that INGORES records that match specified value patterns based on `rdata_text`, all others will be managed | | [OwnershipProcessor](/octodns/processor/ownership.py) | Processor that implements ownership in octoDNS so that it can manage only the records in a zone in sources and will ignore all others. | | [SpfDnsLookupProcessor](/octodns/processor/spf.py) | Processor that checks SPF values for violations of DNS query limits | | [TtlRestrictionFilter](/octodns/processor/restrict.py) | Processor that restricts the allow TTL values to a specified range or list of specific values | diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py index 43dd515..183f173 100644 --- a/octodns/processor/filter.py +++ b/octodns/processor/filter.py @@ -232,9 +232,9 @@ class _ValueBaseFilter(_FilterProcessor): for record in zone.records: values = [] if hasattr(record, 'values'): - values = [str(value) for value in record.values] + values = [value.rdata_text for value in record.values] else: - values = [str(record.value)] + values = [record.value.rdata_text] if any(value in self.exact for value in values): self.matches(zone, record) @@ -280,9 +280,6 @@ class ValueAllowlistFilter(_ValueBaseFilter, AllowsMixin): - route53 ''' - def __init__(self, name, allowlist): - super().__init__(name, allowlist) - class ValueRejectlistFilter(_ValueBaseFilter, RejectsMixin): '''Reject managing records with names that match the provider patterns @@ -316,9 +313,6 @@ class ValueRejectlistFilter(_ValueBaseFilter, RejectsMixin): - route53 ''' - def __init__(self, name, rejectlist): - super().__init__(name, rejectlist) - class _NetworkValueBaseFilter(BaseProcessor): def __init__(self, name, _list):