Merge branch 'main' into global-processors

CHANGELOG.md

@@ -1,5 +1,21 @@
 ## v0.9.19 - 2022-??-?? - ???
 
+#### Noteworthy changes
+
+* Added support for automatic handling of IDNA (utf-8) zones. Everything is
+  stored IDNA encoded internally. For ASCII zones that's a noop. For zones with
+  utf-8 chars they will be converted and all internals/providers will see the
+  encoded version and work with it without any knowledge of it having been
+  converted. This means that all providers will automatically support IDNA as of
+  this version. IDNA zones will generally be displayed in the logs in their
+  decoded form. Both forms should be accepted in command line arguments.
+  Providers may need to be updated to display the decoded form in their logs,
+  until then they'd display the IDNA version.
+* Support for configuring global processors that apply to all zones with
+  `manager.processors`
+
+#### Stuff
+
 * Addressed shortcomings with YamlProvider.SUPPORTS in that it didn't include
   dynamically registered types, was a static list that could have drifted over
   time even ignoring 3rd party types.
@@ -11,8 +27,9 @@
 * Now that it's used as it needed to be YamlProvider overrides
   Provider.supports and just always says Yes so that any dynamically registered
   types will be supported.
-* Support for configuring global processors that apply to all zones with
-  `manager.processors`
+* Add TtlRestrictionFilter processor for adding ttl restriction/checking
+* NameAllowlistFilter & NameRejectlistFilter implementations to support
+  filtering on record names to include/exclude records from management.
 
 ## v0.9.18 - 2022-08-14 - Subzone handling
 

octodns/processor/base.py

@@ -10,6 +10,10 @@ from __future__ import (
 )
 
 
+class ProcessorException(Exception):
+    pass
+
+
 class BaseProcessor(object):
     def __init__(self, name):
         self.name = name

octodns/processor/filter.py

@@ -9,6 +9,8 @@ from __future__ import (
     unicode_literals,
 )
 
+from re import compile as re_compile
+
 from .base import BaseProcessor
 
 
@@ -19,8 +21,8 @@ class TypeAllowlistFilter(BaseProcessor):
 
     processors:
       only-a-and-aaaa:
-        class: octodns.processor.filter.TypeRejectlistFilter
-        rejectlist:
+        class: octodns.processor.filter.TypeAllowlistFilter
+        allowlist:
           - A
           - AAAA
 
@@ -35,7 +37,7 @@ class TypeAllowlistFilter(BaseProcessor):
     '''
 
     def __init__(self, name, allowlist):
-        super(TypeAllowlistFilter, self).__init__(name)
+        super().__init__(name)
         self.allowlist = set(allowlist)
 
     def _process(self, zone, *args, **kwargs):
@@ -71,7 +73,7 @@ class TypeRejectlistFilter(BaseProcessor):
     '''
 
     def __init__(self, name, rejectlist):
-        super(TypeRejectlistFilter, self).__init__(name)
+        super().__init__(name)
         self.rejectlist = set(rejectlist)
 
     def _process(self, zone, *args, **kwargs):
@@ -83,3 +85,113 @@ class TypeRejectlistFilter(BaseProcessor):
 
     process_source_zone = _process
     process_target_zone = _process
+
+
+class _NameBaseFilter(BaseProcessor):
+    def __init__(self, name, _list):
+        super().__init__(name)
+        exact = set()
+        regex = []
+        for pattern in _list:
+            if pattern.startswith('/'):
+                regex.append(re_compile(pattern[1:-1]))
+            else:
+                exact.add(pattern)
+        self.exact = exact
+        self.regex = regex
+
+
+class NameAllowlistFilter(_NameBaseFilter):
+    '''Only manage records with names that match the provider patterns
+
+    Example usage:
+
+    processors:
+      only-these:
+        class: octodns.processor.filter.NameAllowlistFilter
+        allowlist:
+          # exact string match
+          - www
+          # contains/substring match
+          - /substring/
+          # regex pattern match
+          - /some-pattern-\\d\\+/
+          # regex - anchored so has to match start to end
+          - /^start-.+-end$/
+
+    zones:
+      exxampled.com.:
+        sources:
+          - config
+        processors:
+          - only-these
+        targets:
+          - route53
+    '''
+
+    def __init__(self, name, allowlist):
+        super().__init__(name, allowlist)
+
+    def _process(self, zone, *args, **kwargs):
+        for record in zone.records:
+            name = record.name
+            if name in self.exact:
+                continue
+            elif any(r.search(name) for r in self.regex):
+                continue
+
+            zone.remove_record(record)
+
+        return zone
+
+    process_source_zone = _process
+    process_target_zone = _process
+
+
+class NameRejectlistFilter(_NameBaseFilter):
+    '''Reject managing records with names that match the provider patterns
+
+    Example usage:
+
+    processors:
+      not-these:
+        class: octodns.processor.filter.NameRejectlistFilter
+        rejectlist:
+          # exact string match
+          - www
+          # contains/substring match
+          - /substring/
+          # regex pattern match
+          - /some-pattern-\\d\\+/
+          # regex - anchored so has to match start to end
+          - /^start-.+-end$/
+
+    zones:
+      exxampled.com.:
+        sources:
+          - config
+        processors:
+          - not-these
+        targets:
+          - route53
+    '''
+
+    def __init__(self, name, rejectlist):
+        super().__init__(name, rejectlist)
+
+    def _process(self, zone, *args, **kwargs):
+        for record in zone.records:
+            name = record.name
+            if name in self.exact:
+                zone.remove_record(record)
+                continue
+
+            for regex in self.regex:
+                if regex.search(name):
+                    zone.remove_record(record)
+                    break
+
+        return zone
+
+    process_source_zone = _process
+    process_target_zone = _process

octodns/processor/restrict.py

@@ -0,0 +1,83 @@
+#
+#
+#
+
+from __future__ import (
+    absolute_import,
+    division,
+    print_function,
+    unicode_literals,
+)
+
+from .base import BaseProcessor, ProcessorException
+
+
+class RestrictionException(ProcessorException):
+    pass
+
+
+class TtlRestrictionFilter(BaseProcessor):
+    '''
+    Ensure that configured TTLs are between a configured minimum and maximum or
+    in an allowed set of values.
+
+    The default minimum is 1 (the behavior of 0 is undefined spec-wise) and the
+    default maximum is 604800 (seven days.) allowed_ttls is only used when
+    explicitly configured and min and max are ignored in that case.
+
+    Example usage:
+
+    processors:
+      min-max-ttl:
+        class: octodns.processor.restrict.TtlRestrictionFilter
+        min_ttl: 60
+        max_ttl: 3600
+        # allowed_ttls: [300, 900, 3600]
+
+    zones:
+      exxampled.com.:
+        sources:
+          - config
+        processors:
+          - min-max-ttl
+        targets:
+          - azure
+
+    The restriction can be skipped for specific records by setting the lenient
+    flag, e.g.
+
+    a:
+      octodns:
+        lenient: true
+      ttl: 0
+      value: 1.2.3.4
+
+    The higher level lenient flags are not checked as it would make more sense
+    to just avoid enabling the processor in those cases.
+    '''
+
+    SEVEN_DAYS = 60 * 60 * 24 * 7
+
+    def __init__(self, name, min_ttl=1, max_ttl=SEVEN_DAYS, allowed_ttls=None):
+        super().__init__(name)
+        self.min_ttl = min_ttl
+        self.max_ttl = max_ttl
+        self.allowed_ttls = set(allowed_ttls) if allowed_ttls else None
+
+    def process_source_zone(self, zone, *args, **kwargs):
+        for record in zone.records:
+            if record._octodns.get('lenient'):
+                continue
+            if self.allowed_ttls and record.ttl not in self.allowed_ttls:
+                raise RestrictionException(
+                    f'{record.fqdn} ttl={record.ttl} not an allowed value, allowed_ttls={self.allowed_ttls}'
+                )
+            elif record.ttl < self.min_ttl:
+                raise RestrictionException(
+                    f'{record.fqdn} ttl={record.ttl} too low, min_ttl={self.min_ttl}'
+                )
+            elif record.ttl > self.max_ttl:
+                raise RestrictionException(
+                    f'{record.fqdn} ttl={record.ttl} too high, max_ttl={self.max_ttl}'
+                )
+        return zone

tests/test_octodns_processor_filter.py

@@ -11,7 +11,12 @@ from __future__ import (
 
 from unittest import TestCase
 
-from octodns.processor.filter import TypeAllowlistFilter, TypeRejectlistFilter
+from octodns.processor.filter import (
+    NameAllowlistFilter,
+    NameRejectlistFilter,
+    TypeAllowlistFilter,
+    TypeRejectlistFilter,
+)
 from octodns.record import Record
 from octodns.zone import Zone
 
@@ -76,3 +81,83 @@ class TestTypeRejectListFilter(TestCase):
         filter_a_aaaa = TypeRejectlistFilter('not-a-aaaa', set(('A', 'AAAA')))
         got = filter_a_aaaa.process_target_zone(zone.copy())
         self.assertEqual(['txt', 'txt2'], sorted([r.name for r in got.records]))
+
+
+class TestNameAllowListFilter(TestCase):
+    zone = Zone('unit.tests.', [])
+    matches = Record.new(
+        zone, 'matches', {'type': 'A', 'ttl': 42, 'value': '1.2.3.4'}
+    )
+    zone.add_record(matches)
+    doesnt = Record.new(
+        zone, 'doesnt', {'type': 'A', 'ttl': 42, 'value': '2.3.4.5'}
+    )
+    zone.add_record(doesnt)
+    matchable1 = Record.new(
+        zone, 'start-f43ad96-end', {'type': 'A', 'ttl': 42, 'value': '3.4.5.6'}
+    )
+    zone.add_record(matchable1)
+    matchable2 = Record.new(
+        zone, 'start-a3b444c-end', {'type': 'A', 'ttl': 42, 'value': '4.5.6.7'}
+    )
+    zone.add_record(matchable2)
+
+    def test_exact(self):
+        allows = NameAllowlistFilter('exact', ('matches',))
+
+        self.assertEqual(4, len(self.zone.records))
+        filtered = allows.process_source_zone(self.zone.copy())
+        self.assertEqual(1, len(filtered.records))
+        self.assertEqual(['matches'], [r.name for r in filtered.records])
+
+    def test_regex(self):
+        allows = NameAllowlistFilter('exact', ('/^start-.+-end$/',))
+
+        self.assertEqual(4, len(self.zone.records))
+        filtered = allows.process_source_zone(self.zone.copy())
+        self.assertEqual(2, len(filtered.records))
+        self.assertEqual(
+            ['start-a3b444c-end', 'start-f43ad96-end'],
+            sorted([r.name for r in filtered.records]),
+        )
+
+
+class TestNameRejectListFilter(TestCase):
+    zone = Zone('unit.tests.', [])
+    matches = Record.new(
+        zone, 'matches', {'type': 'A', 'ttl': 42, 'value': '1.2.3.4'}
+    )
+    zone.add_record(matches)
+    doesnt = Record.new(
+        zone, 'doesnt', {'type': 'A', 'ttl': 42, 'value': '2.3.4.5'}
+    )
+    zone.add_record(doesnt)
+    matchable1 = Record.new(
+        zone, 'start-f43ad96-end', {'type': 'A', 'ttl': 42, 'value': '3.4.5.6'}
+    )
+    zone.add_record(matchable1)
+    matchable2 = Record.new(
+        zone, 'start-a3b444c-end', {'type': 'A', 'ttl': 42, 'value': '4.5.6.7'}
+    )
+    zone.add_record(matchable2)
+
+    def test_exact(self):
+        rejects = NameRejectlistFilter('exact', ('matches',))
+
+        self.assertEqual(4, len(self.zone.records))
+        filtered = rejects.process_source_zone(self.zone.copy())
+        self.assertEqual(3, len(filtered.records))
+        self.assertEqual(
+            ['doesnt', 'start-a3b444c-end', 'start-f43ad96-end'],
+            sorted([r.name for r in filtered.records]),
+        )
+
+    def test_regex(self):
+        rejects = NameRejectlistFilter('exact', ('/^start-.+-end$/',))
+
+        self.assertEqual(4, len(self.zone.records))
+        filtered = rejects.process_source_zone(self.zone.copy())
+        self.assertEqual(2, len(filtered.records))
+        self.assertEqual(
+            ['doesnt', 'matches'], sorted([r.name for r in filtered.records])
+        )

tests/test_octodns_processor_restrict.py

@@ -0,0 +1,113 @@
+from unittest import TestCase
+
+from octodns.processor.restrict import (
+    RestrictionException,
+    TtlRestrictionFilter,
+)
+from octodns.record import Record
+from octodns.zone import Zone
+
+
+class TestTtlRestrictionFilter(TestCase):
+    def test_restrict_ttl(self):
+        # configured values
+        restrictor = TtlRestrictionFilter('test', min_ttl=32, max_ttl=1024)
+
+        zone = Zone('unit.tests.', [])
+        good = Record.new(
+            zone, 'good', {'type': 'A', 'ttl': 42, 'value': '1.2.3.4'}
+        )
+        zone.add_record(good)
+
+        restricted = restrictor.process_source_zone(zone)
+        self.assertEqual(zone.records, restricted.records)
+
+        # too low
+        low = Record.new(
+            zone, 'low', {'type': 'A', 'ttl': 16, 'value': '1.2.3.4'}
+        )
+        copy = zone.copy()
+        copy.add_record(low)
+        with self.assertRaises(RestrictionException) as ctx:
+            restrictor.process_source_zone(copy)
+        self.assertEqual(
+            'low.unit.tests. ttl=16 too low, min_ttl=32', str(ctx.exception)
+        )
+
+        # with lenient set, we can go lower
+        lenient = Record.new(
+            zone,
+            'low',
+            {
+                'octodns': {'lenient': True},
+                'type': 'A',
+                'ttl': 16,
+                'value': '1.2.3.4',
+            },
+        )
+        copy = zone.copy()
+        copy.add_record(lenient)
+        restricted = restrictor.process_source_zone(copy)
+        self.assertEqual(copy.records, restricted.records)
+
+        # too high
+        high = Record.new(
+            zone, 'high', {'type': 'A', 'ttl': 2048, 'value': '1.2.3.4'}
+        )
+        copy = zone.copy()
+        copy.add_record(high)
+        with self.assertRaises(RestrictionException) as ctx:
+            restrictor.process_source_zone(copy)
+        self.assertEqual(
+            'high.unit.tests. ttl=2048 too high, max_ttl=1024',
+            str(ctx.exception),
+        )
+
+        # too low defaults
+        restrictor = TtlRestrictionFilter('test')
+        low = Record.new(
+            zone, 'low', {'type': 'A', 'ttl': 0, 'value': '1.2.3.4'}
+        )
+        copy = zone.copy()
+        copy.add_record(low)
+        with self.assertRaises(RestrictionException) as ctx:
+            restrictor.process_source_zone(copy)
+        self.assertEqual(
+            'low.unit.tests. ttl=0 too low, min_ttl=1', str(ctx.exception)
+        )
+
+        # too high defaults
+        high = Record.new(
+            zone, 'high', {'type': 'A', 'ttl': 999999, 'value': '1.2.3.4'}
+        )
+        copy = zone.copy()
+        copy.add_record(high)
+        with self.assertRaises(RestrictionException) as ctx:
+            restrictor.process_source_zone(copy)
+        self.assertEqual(
+            'high.unit.tests. ttl=999999 too high, max_ttl=604800',
+            str(ctx.exception),
+        )
+
+        # allowed_ttls
+        restrictor = TtlRestrictionFilter('test', allowed_ttls=[42, 300])
+
+        # add 300 (42 is already there)
+        another = Record.new(
+            zone, 'another', {'type': 'A', 'ttl': 300, 'value': '4.5.6.7'}
+        )
+        zone.add_record(another)
+
+        # 42 and 300 are allowed through
+        restricted = restrictor.process_source_zone(zone)
+        self.assertEqual(zone.records, restricted.records)
+
+        # 16 is not
+        copy = zone.copy()
+        copy.add_record(low)
+        with self.assertRaises(RestrictionException) as ctx:
+            restrictor.process_source_zone(copy)
+        self.assertEqual(
+            'low.unit.tests. ttl=0 not an allowed value, allowed_ttls={42, 300}',
+            str(ctx.exception),
+        )