Merge pull request #1283 from octodns/gen-doc

Documentation generation via Sphinx

.changelog/88a37c98bbcf4ea58b57854afb46b73c.md

@@ -0,0 +1,4 @@
+---
+type: none
+---
+Add sphinx doc generation

.gitignore

@@ -2,18 +2,20 @@
 # Do not add editor or OS specific ignores here. Have a look at adding
 # `excludesfile` to your `~/.gitconfig` to globally ignore such things.
 #
-*.pyc
-.coverage
-.env
 /build/
 /config/
+.coverage
 coverage.xml
 dist/
+docs/_build/
+docs/modules/
+.env
 env/
 examples/migrating-to-octodns/config/
 htmlcov/
 nosetests.xml
 octodns.egg-info/
 output/
+*.pyc
 tests/zones/unit.tests.
 tmp/

docs/conf.py

@@ -0,0 +1,96 @@
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent.resolve()))
+
+print(f"SYS.PATH={sys.path}")
+
+from octodns.__init__ import __version__
+
+### sphinx config ###
+
+project = "octodns"
+copyright = "2017-present"  # noqa
+author = "Ross McFarland"
+release = __version__
+
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.coverage",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.todo",
+    "sphinx.ext.viewcode",
+    "myst_parser",
+    "sphinx_copybutton",
+]
+
+
+### autodoc ###
+
+autodoc_default_options = {
+    "members": True,
+    "undoc-members": True,
+    "private-members": True,
+    "special-members": "__init__, __repr__",
+    # "inherited-members": True,
+    "exclude-members": "__weakref__",
+    "show-inheritance": True,
+}
+autodoc_typehints = "both"
+autodoc_typehints_description_target = "all"
+autodoc_member_order = "bysource"
+
+### extlinks ###
+
+extlinks = {
+    "github": ("https://github.com/%s", "%s"),
+    "pypi": ("https://pypi.org/project/%s/", "%s"),
+}
+extlinks_detect_hardcoded_links = True
+
+
+### intersphinx ###
+
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
+    "sphinx": ("https://www.sphinx-doc.org/en/master", None),
+    "dnspython": ("https://dnspython.readthedocs.io/en/stable/", None),
+    "six": ("https://six.readthedocs.io/", None),
+    "python-dateutil": ("https://dateutil.readthedocs.io/en/stable/", None),
+    "fqdn": ("https://fqdn.readthedocs.io/en/latest/", None),
+}
+
+
+### todo ###
+
+todo_include_todos = True
+
+
+### myst ###
+
+myst_enable_extensions = [
+    "colon_fence",
+    "deflist",
+]
+myst_heading_anchors = 3
+
+
+### content ###
+
+master_doc = "index"
+
+templates_path = ["_templates"]
+html_static_path = ["_static"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+### theme ###
+
+# html_theme = "alabaster"
+html_theme = "furo"
+html_theme_options = {
+    "source_repository": "https://github.com/octodns/octodns/",
+    "source_branch": "main",
+    "source_directory": "docs/",
+}

docs/index.md

@@ -0,0 +1,88 @@
+# octodns documentation
+
+```{include} ../README.md
+---
+end-before: '## Table of Contents'
+---
+```
+
+______________________________________________________________________
+
+## User documentation
+
+```{toctree}
+:caption: Getting Started:
+:maxdepth: 1
+
+examples/basic/README.md
+examples/migrating-to-octodns/README.md
+records.md
+```
+
+```{toctree}
+:caption: Guides:
+:maxdepth: 1
+:glob:
+
+[a-q]*
+#records.md
+[s-z]*
+```
+
+______________________________________________________________________
+
+## Module documentation
+
+```{toctree}
+:caption: Providers:
+:maxdepth: 2
+:glob:
+
+modules/provider/*
+```
+
+```{toctree}
+:caption: Sources:
+:maxdepth: 2
+:glob:
+
+modules/source/*
+```
+
+```{toctree}
+:caption: Records:
+:maxdepth: 2
+:glob:
+
+modules/record/*
+```
+
+```{toctree}
+:caption: Processors:
+:maxdepth: 2
+:glob:
+
+modules/processor/*
+```
+
+```{toctree}
+:caption: Other modules:
+:titlesonly:
+:glob:
+
+modules/*
+modules/cmds/*
+modules/secret/*
+```
+
+______________________________________________________________________
+
+## Indices and tables
+
+- {ref}`genindex`
+- {ref}`modindex`
+
+### Project info
+
+- [License](info/license.md)
+- [Changelog](info/changelog.md)

docs/info/changelog.md

@@ -0,0 +1,5 @@
+# Changelog
+
+```{include} ../../CHANGELOG.md
+
+```

docs/info/license.md

@@ -0,0 +1,5 @@
+# License
+
+```{include} ../../LICENSE
+
+```

requirements-dev.txt

@@ -2,15 +2,15 @@
 Pygments==2.19.2
 SecretStorage==3.3.3
 black==24.10.0
-build==1.2.2.post1
-certifi==2025.7.14
+build==1.3.0
+certifi==2025.8.3
 cffi==1.17.1
 changelet==0.1.0
 charset-normalizer==3.4.2
 click==8.1.8; python_version<'3.10'
 click==8.2.1; python_version>='3.10'
 cmarkgfm==2024.11.20
-coverage==7.10.0
+coverage==7.10.2
 cryptography==45.0.5
 docutils==0.21.2
 id==1.5.0
@@ -31,19 +31,19 @@ pathspec==0.12.1
 platformdirs==4.3.8
 pluggy==1.6.0
 pprintpp==0.4.0
-pycountry-convert==0.7.2
 pycountry==24.6.1
+pycountry-convert==0.7.2
 pycparser==2.22
 pyflakes==3.4.0
 pyproject_hooks==1.2.0
+pytest==8.4.1
 pytest-cov==6.2.1
 pytest-mock==3.14.1
-pytest==8.4.1
 pytest_network==0.0.1
 readme_renderer==44.0
 repoze.lru==0.7
-requests-toolbelt==1.0.0
 requests==2.32.4
+requests-toolbelt==1.0.0
 rfc3986==2.0.0
 rich==14.1.0
 semver==3.0.4

requirements-docs.txt

@@ -0,0 +1,28 @@
+# DO NOT EDIT THIS FILE DIRECTLY - use ./script/update-requirements to update
+Jinja2==3.1.6
+MarkupSafe==3.0.2
+Sphinx==7.4.7; python_version=='3.9'
+Sphinx==8.1.3; python_version=='3.10'
+Sphinx==8.2.3; python_version>='3.11'
+accessible-pygments==0.0.5
+alabaster==0.7.16; python_version<'3.10'
+alabaster==1.0.0; python_version>='3.10'
+babel==2.17.0
+beautifulsoup4==4.13.4
+furo==2025.7.19
+imagesize==1.4.1
+mdit-py-plugins==0.4.2
+myst-parser==3.0.1; python_version<'3.10'
+myst-parser==4.0.1; python_version>='3.10'
+roman-numerals-py==3.1.0
+snowballstemmer==3.0.1
+soupsieve==2.7
+sphinx-basic-ng==1.0.0b2
+sphinx-copybutton==0.5.2
+sphinxcontrib-applehelp==2.0.0
+sphinxcontrib-devhelp==2.0.0
+sphinxcontrib-htmlhelp==2.1.0
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==2.0.0
+sphinxcontrib-serializinghtml==2.0.0
+typing_extensions==4.14.1

script/bootstrap

@@ -26,7 +26,7 @@ python -m pip install -U 'pip>=10.0.1'
 python -m pip install -r requirements.txt
 
 if [ "$ENV" != "production" ]; then
-    python -m pip install -r requirements-dev.txt
+    python -m pip install -r requirements-dev.txt -r requirements-docs.txt
 fi
 
 if [ -d ".git" ]; then

script/cibuild

@@ -29,4 +29,6 @@ echo "## formatting ############################################################
 script/format --check || (echo "Formatting check failed, run ./script/format" && exit 1)
 echo "## tests/coverage ##############################################################"
 script/coverage
+echo "## documentation ###############################################################"
+script/generate-docs
 echo "## complete ####################################################################"

script/generate-docs

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+cd "$(dirname "$0")/.."
+ROOT=$(pwd)
+cd "$ROOT"
+
+rm -rf ./docs/_build ./docs/modules
+
+for f in $(find octodns -name "*.py" | grep -v __init__.py); do
+    module_name=$(echo $f | sed -e 's#/#.#g' -e 's/.py//')
+    outdir="docs/modules$(dirname $f | sed -e 's#octodns##')"
+    mkdir -p "$outdir"
+    cat <<EOL > "${outdir}/${module_name}.rst"
+====================================
+``${module_name}``
+====================================
+.. automodule:: ${module_name}
+EOL
+done
+
+sphinx-build -M html "${ROOT}/docs" "${ROOT}/docs/_build" "$@"

script/update-requirements

@@ -1,70 +1,89 @@
 #!/usr/bin/env python3
 
-import re
 from os.path import join
 from subprocess import check_call, check_output
-from sys import argv
+from sys import argv, stdout
 from tempfile import TemporaryDirectory
 
-
-def print_packages(packages, heading):
-    print(f'{heading}:')
-    print('  ', end='')
-    print('\n  '.join(packages))
-
-
-# would  be nice if there was a cleaner way to get this, but I've not found a
-# more reliable one.
-with open('setup.py') as fh:
-    match = re.search(r"name='(?P<pkg>[\w-]+)',", fh.read())
-    if not match:
-        raise Exception('failed to determine our package name')
-    our_package_name = match.group('pkg')
-    print(f'our_package_name: {our_package_name}')
-
 with TemporaryDirectory() as tmpdir:
     check_call(['python3', '-m', 'venv', tmpdir])
 
+    def install_and_list(target):
+        check_call([join(tmpdir, 'bin', 'pip'), 'install', target])
+        frozen = (
+            check_output([join(tmpdir, 'bin', 'pip'), 'freeze'])
+            .decode('utf-8')
+            .strip()
+            .split('\n')
+        )
+        # file bit skips ourself
+        return set(p.split('=', 1)[0] for p in frozen if 'file' not in p)
+
     # base needs
-    check_call([join(tmpdir, 'bin', 'pip'), 'install', '.'])
-    frozen = check_output([join(tmpdir, 'bin', 'pip'), 'freeze'])
-    frozen = set(frozen.decode('utf-8').strip().split('\n'))
+    frozen = install_and_list('.')
+    # dev needs
+    dev_frozen = install_and_list('.[dev]') - frozen
+    # docs needs
+    docs_frozen = install_and_list('.[docs]') - dev_frozen - frozen
 
-    # dev additions
-    check_call([join(tmpdir, 'bin', 'pip'), 'install', '.[dev]'])
-    dev_frozen = check_output([join(tmpdir, 'bin', 'pip'), 'freeze'])
-    dev_frozen = set(dev_frozen.decode('utf-8').strip().split('\n')) - frozen
+    # find the installed version for each package
+    versions = {}
+    for pv in (
+        check_output([join(tmpdir, 'bin', 'pip'), 'freeze'])
+        .decode('utf-8')
+        .strip()
+        .split('\n')
+    ):
+        if 'file' in pv:
+            # skip ourself
+            continue
+        p, v = pv.split('==')
+        versions[p] = (v,)
 
-# pip installs the module itself along with deps so we need to get that out of
-# our list by finding the thing that was file installed during dev
-frozen = sorted([p for p in frozen if not p.startswith(our_package_name)])
-dev_frozen = sorted(
-    [p for p in dev_frozen if not p.startswith(our_package_name)]
-)
 
-# special handling for click until python 3.9 is gone due to it dropping
-# support for active versions early
-i = [i for i, r in enumerate(dev_frozen) if r.startswith('click==')][0]
-dev_frozen = (
-    dev_frozen[:i]
-    + [
-        "click==8.1.8; python_version<'3.10'",
-        f"{dev_frozen[i]}; python_version>='3.10'",
-    ]
-    + dev_frozen[i + 1 :]
+# special handling for older python versions due to libraries dropping support
+# early
+versions['alabaster'] = (
+    "0.7.16; python_version<'3.10'",
+    f"{versions['alabaster'][0]}; python_version>='3.10'",
+)
+versions['click'] = (
+    "8.1.8; python_version<'3.10'",
+    f"{versions['click'][0]}; python_version>='3.10'",
 )
+versions['myst-parser'] = (
+    "3.0.1; python_version<'3.10'",
+    f"{versions['myst-parser'][0]}; python_version>='3.10'",
+)
+versions['Sphinx'] = (
+    "7.4.7; python_version=='3.9'",
+    "8.1.3; python_version=='3.10'",
+    f"{versions['Sphinx'][0]}; python_version>='3.11'",
+)
+
 
-print_packages(frozen, 'frozen')
-print_packages(dev_frozen, 'dev_frozen')
+def write_packages(fh, packages, header, prefix=''):
+    fh.write(header)
+    for p in sorted(packages):
+        for v in versions[p]:
+            fh.write(prefix)
+            fh.write(p)
+            fh.write('==')
+            fh.write(v)
+            fh.write('\n')
 
-script = argv[0]
+
+write_packages(stdout, frozen, header='base\n', prefix='  ')
+write_packages(stdout, dev_frozen, header='dev\n', prefix='  ')
+write_packages(stdout, docs_frozen, header='docs\n', prefix='  ')
+
+header = f'# DO NOT EDIT THIS FILE DIRECTLY - use {argv[0]} to update\n'
 
 with open('requirements.txt', 'w') as fh:
-    fh.write(f'# DO NOT EDIT THIS FILE DIRECTLY - use {script} to update\n')
-    fh.write('\n'.join(frozen))
-    fh.write('\n')
+    write_packages(fh, frozen, header=header)
 
 with open('requirements-dev.txt', 'w') as fh:
-    fh.write(f'# DO NOT EDIT THIS FILE DIRECTLY - use {script} to update\n')
-    fh.write('\n'.join(dev_frozen))
-    fh.write('\n')
+    write_packages(fh, dev_frozen, header=header)
+
+with open('requirements-docs.txt', 'w') as fh:
+    write_packages(fh, docs_frozen, header=header)

setup.py

@@ -70,7 +70,13 @@ setup(
             'pyflakes>=2.2.0',
             'readme_renderer[md]>=26.0',
             'twine>=3.4.2',
-        )
+        ),
+        'docs': (
+            'Sphinx>=8.2.3',
+            'furo>=2024.8.6',
+            'myst-parser>=4.0.1',
+            'sphinx-copybutton>=0.5.2',
+        ),
     },
     install_requires=(
         'PyYaml>=4.2b1',