diff --git a/CHANGELOG.md b/CHANGELOG.md index 3aaf549..a5fd39c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## v1.6.x - 2024-??-?? - ??? * Add EnsureTrailingDots processor +* Beta support for custom secret providers added to Manager. ## v1.5.1 - 2024-03-08 - env/* type conversion fix diff --git a/octodns/manager.py b/octodns/manager.py index e34d3a0..903f797 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -10,7 +10,6 @@ from importlib.metadata import PackageNotFoundError from importlib.metadata import version as module_version from json import dumps from logging import getLogger -from os import environ from sys import stdout from . import __version__ @@ -20,6 +19,7 @@ from .processor.meta import MetaProcessor from .provider.base import BaseProvider from .provider.plan import Plan from .provider.yaml import SplitYamlProvider, YamlProvider +from .secret.environ import EnvironSecrets from .yaml import safe_load from .zone import Zone @@ -119,6 +119,14 @@ class Manager(object): manager_config, enable_checksum ) + # add our hard-coded environ handler first so that other secret + # providers can pull in env variables w/it + self.secret_handlers = {'env': EnvironSecrets('env')} + secret_handlers_config = self.config.get('secret_handlers', {}) + self.secret_handlers.update( + self._config_secret_handlers(secret_handlers_config) + ) + self.auto_arpa = self._config_auto_arpa(manager_config, auto_arpa) self.global_processors = manager_config.get('processors', []) @@ -219,6 +227,38 @@ class Manager(object): self.log.info('_config_auto_arpa: auto_arpa=%s', auto_arpa) return auto_arpa + def _config_secret_handlers(self, secret_handlers_config): + self.log.debug('_config_secret_handlers: configuring secret_handlers') + secret_handlers = {} + for sh_name, sh_config in secret_handlers_config.items(): + # Get our class and remove it from the secret handler config + try: + _class = sh_config.pop('class') + except KeyError: + self.log.exception('Invalid secret handler class') + raise ManagerException( + f'Secret Handler {sh_name} is missing class, {sh_config.context}' + ) + _class, module, version = self._get_named_class( + 'secret handler', _class, sh_config.context + ) + kwargs = self._build_kwargs(sh_config) + try: + secret_handlers[sh_name] = _class(sh_name, **kwargs) + self.log.info( + '__init__: secret_handler=%s (%s %s)', + sh_name, + module, + version, + ) + except TypeError: + self.log.exception('Invalid secret handler config') + raise ManagerException( + f'Incorrect secret handler config for {sh_name}, {sh_config.context}' + ) + + return secret_handlers + def _config_providers(self, providers_config): self.log.debug('_config_providers: configuring providers') providers = {} @@ -362,7 +402,7 @@ class Manager(object): return getattr(module, class_name), module_name, version except AttributeError: self.log.exception( - '_get_{}_class: Unable to get class %s from module %s', + '_get_named_class: Unable to get class %s from module %s', class_name, module, ) @@ -377,26 +417,34 @@ class Manager(object): if isinstance(v, dict): v = self._build_kwargs(v) elif isinstance(v, str): - if v.startswith('env/'): - # expand env variables + if '/' in v: + handler, name = v.split('/', 1) try: - env_var = v[4:] - v = environ[env_var] + handler = self.secret_handlers[handler] except KeyError: - self.log.exception('Invalid provider config') - raise ManagerException( - f'Incorrect provider config, missing env var {env_var}, {source.context}' + # we don't have a matching handler, but don't want to + # make that an error b/c config values will often + # contain /. We don't want to print the values in case + # they're sensitive so just provide the key, and even + # that only at debug level. + self.log.debug( + '_build_kwargs: failed to find handler for key "%sp ', + k, ) - try: - if '.' in v: - # has a dot, try converting it to a float - v = float(v) - else: - # no dot, try converting it to an int - v = int(v) - except ValueError: - # just leave it as a string - pass + else: + v = handler.fetch(name, source) + + if isinstance(v, str): + try: + if '.' in v: + # has a dot, try converting it to a float + v = float(v) + else: + # no dot, try converting it to an int + v = int(v) + except ValueError: + # just leave it as a string + pass kwargs[k] = v diff --git a/octodns/secret/__init__.py b/octodns/secret/__init__.py new file mode 100644 index 0000000..407eb4e --- /dev/null +++ b/octodns/secret/__init__.py @@ -0,0 +1,3 @@ +# +# +# diff --git a/octodns/secret/base.py b/octodns/secret/base.py new file mode 100644 index 0000000..bb7a93b --- /dev/null +++ b/octodns/secret/base.py @@ -0,0 +1,11 @@ +# +# +# + +from logging import getLogger + + +class BaseSecrets: + def __init__(self, name): + self.log = getLogger(f'{self.__class__.__name__}[{name}]') + self.name = name diff --git a/octodns/secret/environ.py b/octodns/secret/environ.py new file mode 100644 index 0000000..22c4b9d --- /dev/null +++ b/octodns/secret/environ.py @@ -0,0 +1,32 @@ +# +# +# + +from os import environ + +from .base import BaseSecrets +from .exception import SecretsException + + +class EnvironSecretsException(SecretsException): + pass + + +class EnvironSecrets(BaseSecrets): + def fetch(self, name, source): + # expand env variables + try: + v = environ[name] + except KeyError: + self.log.exception('Invalid provider config') + raise EnvironSecretsException( + f'Incorrect provider config, missing env var {name}, {source.context}' + ) + try: + # try converting the value to a number to see if it + # converts + v = float(v) + except ValueError: + pass + + return v diff --git a/octodns/secret/exception.py b/octodns/secret/exception.py new file mode 100644 index 0000000..53077bc --- /dev/null +++ b/octodns/secret/exception.py @@ -0,0 +1,7 @@ +# +# +# + + +class SecretsException(Exception): + pass diff --git a/tests/config/secrets.yaml b/tests/config/secrets.yaml new file mode 100644 index 0000000..d46e74f --- /dev/null +++ b/tests/config/secrets.yaml @@ -0,0 +1,19 @@ +--- +secret_handlers: + dummy: + class: helpers.DummySecrets + prefix: in_config/ + requires-env: + class: helpers.DummySecrets + # things can pull from env, it prexists + prefix: env/FROM_ENV_WILL_WORK + requires-dummy: + class: helpers.DummySecrets + # things can't pull from other handlers, the order they're configured in is + # indeterminent so it's not safe, they're also all added at once + prefix: dummy/FROM_DUMMY_WONT_WORK + +# Not needed, but required key +providers: {} +# Not needed, but required key +zones: {} diff --git a/tests/helpers.py b/tests/helpers.py index 019395b..b19d236 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -9,6 +9,7 @@ from tempfile import mkdtemp from octodns.processor.base import BaseProcessor from octodns.provider.base import BaseProvider from octodns.provider.yaml import YamlProvider +from octodns.secret.base import BaseSecrets class SimpleSource(object): @@ -134,3 +135,13 @@ class CountingProcessor(BaseProcessor): def process_source_zone(self, zone, *args, **kwargs): self.count += len(zone.records) return zone + + +class DummySecrets(BaseSecrets): + def __init__(self, name, prefix): + super().__init__(name) + self.log.info('__init__: name=%s, prefix=%s', name, prefix) + self.prefix = prefix + + def fetch(self, name, source): + return f'{self.prefix}{name}' diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index b0d4019..05d20aa 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -8,6 +8,7 @@ from unittest import TestCase from unittest.mock import MagicMock, patch from helpers import ( + DummySecrets, DynamicProvider, GeoProvider, NoSshFpProvider, @@ -17,6 +18,7 @@ from helpers import ( ) from octodns import __version__ +from octodns.context import ContextDict from octodns.idna import IdnaDict, idna_encode from octodns.manager import ( MainThreadExecutor, @@ -26,6 +28,7 @@ from octodns.manager import ( ) from octodns.processor.base import BaseProcessor from octodns.record import Create, Delete, Record, Update +from octodns.secret.environ import EnvironSecretsException from octodns.yaml import safe_load from octodns.zone import Zone @@ -68,7 +71,8 @@ class TestManager(TestCase): self.assertTrue('provider config' in str(ctx.exception)) def test_missing_env_config(self): - with self.assertRaises(ManagerException) as ctx: + # details of the EnvironSecrets will be tested in dedicated tests + with self.assertRaises(EnvironSecretsException) as ctx: Manager(get_config_filename('missing-provider-env.yaml')).sync() self.assertTrue('missing env var' in str(ctx.exception)) @@ -1215,6 +1219,81 @@ class TestManager(TestCase): ), ) + def test_config_secret_handlers(self): + # config doesn't matter here + manager = Manager(get_config_filename('simple.yaml')) + + # no config + self.assertEqual({}, manager._config_secret_handlers({})) + + # missing class + with self.assertRaises(ManagerException) as ctx: + cfg = {'secr3t': ContextDict({}, context='xyz')} + manager._config_secret_handlers(cfg) + self.assertEqual( + 'Secret Handler secr3t is missing class, xyz', str(ctx.exception) + ) + + # bad param + with self.assertRaises(ManagerException) as ctx: + cfg = { + 'secr3t': ContextDict( + { + 'class': 'octodns.secret.environ.EnvironSecrets', + 'bad': 'param', + }, + context='xyz', + ) + } + manager._config_secret_handlers(cfg) + self.assertEqual( + 'Incorrect secret handler config for secr3t, xyz', + str(ctx.exception), + ) + + # valid with a param that gets used/tested + cfg = { + 'secr3t': ContextDict( + {'class': 'helpers.DummySecrets', 'prefix': 'pre-'}, + context='xyz', + ) + } + shs = manager._config_secret_handlers(cfg) + sh = shs.get('secr3t') + self.assertTrue(sh) + self.assertEqual('pre-thing', sh.fetch('thing', None)) + + # test configuring secret handlers + environ['FROM_ENV_WILL_WORK'] = 'fetched_from_env/' + manager = Manager(get_config_filename('secrets.yaml')) + + # dummy was configured + self.assertTrue('dummy' in manager.secret_handlers) + dummy = manager.secret_handlers['dummy'] + self.assertIsInstance(dummy, DummySecrets) + # and has the prefix value explicitly stated in the yaml + self.assertEqual('in_config/hello', dummy.fetch('hello', None)) + + # requires-env was configured + self.assertTrue('requires-env' in manager.secret_handlers) + requires_env = manager.secret_handlers['requires-env'] + self.assertIsInstance(requires_env, DummySecrets) + # and successfully pulled a value from env as its prefix + self.assertEqual( + 'fetched_from_env/hello', requires_env.fetch('hello', None) + ) + + # requires-dummy was created + self.assertTrue('requires-dummy' in manager.secret_handlers) + requires_dummy = manager.secret_handlers['requires-dummy'] + self.assertIsInstance(requires_dummy, DummySecrets) + # but failed to fetch a secret from dummy so we just get the configured + # value as it was in the yaml for prefix + self.assertEqual( + 'dummy/FROM_DUMMY_WONT_WORK:hello', + requires_dummy.fetch(':hello', None), + ) + class TestMainThreadExecutor(TestCase): def test_success(self): diff --git a/tests/test_octodns_secret_environ.py b/tests/test_octodns_secret_environ.py new file mode 100644 index 0000000..ff09c32 --- /dev/null +++ b/tests/test_octodns_secret_environ.py @@ -0,0 +1,31 @@ +# +# +# + +from os import environ +from unittest import TestCase + +from octodns.context import ContextDict +from octodns.secret.environ import EnvironSecrets, EnvironSecretsException + + +class TestEnvironSecrets(TestCase): + def test_environ_secrets(self): + # put some secrets into our env + environ['THIS_EXISTS'] = 'and has a val' + environ['THIS_IS_AN_INT'] = '42' + environ['THIS_IS_A_FLOAT'] = '43.44' + + es = EnvironSecrets('env') + + source = ContextDict({}, context='xyz') + self.assertEqual('and has a val', es.fetch('THIS_EXISTS', source)) + self.assertEqual(42, es.fetch('THIS_IS_AN_INT', source)) + self.assertEqual(43.44, es.fetch('THIS_IS_A_FLOAT', source)) + + with self.assertRaises(EnvironSecretsException) as ctx: + es.fetch('DOES_NOT_EXIST', source) + self.assertEqual( + 'Incorrect provider config, missing env var DOES_NOT_EXIST, xyz', + str(ctx.exception), + )