diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 431860278..4a8c84d3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,7 @@ jobs: dashboard: ${{ steps.filter.outputs.dashboard }} packaging: ${{ steps.filter.outputs.packaging }} workflows: ${{ steps.filter.outputs.workflows }} + env_docs: ${{ steps.filter.outputs.env_docs }} steps: - uses: actions/checkout@v7 - uses: dorny/paths-filter@v4 @@ -99,6 +100,27 @@ jobs: - '.github/workflows/**' workflows: - '.github/workflows/**' + env_docs: + - 'headroom/**' + - 'crates/**' + - 'sdk/**' + - 'plugins/**' + - 'deploy/**' + - 'docker/**' + - 'scripts/**' + + env-doc-consistency: + needs: changes + if: needs.changes.outputs.env_docs == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: ${{ env.PY_VERSION }} + - name: Verify documented environment variables + run: python scripts/ci/verify_documented_env_vars.py lint: needs: changes diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c574cf636..bc6bd33c9 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,6 +22,9 @@ on: branches: [main] paths: - 'docs/**' + - 'README.md' + - 'SECURITY.md' + - 'scripts/ci/verify_documented_env_vars.py' - '.github/workflows/docs.yml' workflow_dispatch: @@ -36,6 +39,13 @@ jobs: steps: - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Verify documented environment variables + run: python scripts/ci/verify_documented_env_vars.py + - name: Setup Node.js uses: actions/setup-node@v6 with: diff --git a/scripts/ci/verify_documented_env_vars.py b/scripts/ci/verify_documented_env_vars.py new file mode 100644 index 000000000..4a5128a3f --- /dev/null +++ b/scripts/ci/verify_documented_env_vars.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""Verify that documented environment variables exist in implementation source. + +The check is deliberately text-based and dependency-free so it can run before +either the Python package or the documentation application is installed. It +guards against documentation for a misspelled or removed setting silently +surviving after the implementation changes. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path + +# Match uppercase snake-case names without assuming who owns them. Wildcard +# families may be written as ``VENDOR_*`` or ``VENDOR_``; both are +# normalized to ``VENDOR_*`` before comparison with concrete source names. +# Lowercase letters are part of the boundary so fragments such as ``DS_S`` in +# ``.DS_Store`` are not mistaken for environment variables. +ENV_VAR_PATTERN = re.compile( + r"(?" + r"(?:[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)*_(?:\*|<[A-Z][A-Z0-9_]*>))" + r"|(?:[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+)" + r")" + r"(?![A-Za-z0-9_])" +) + +# Single-word names are too ambiguous to extract from prose, but shell syntax is +# unambiguous. This covers POSIX expansion (``$HOME`` / ``${HOME}``), +# PowerShell (``$env:PATH``), and Windows expansion (``%USERPROFILE%``). +CONTEXTUAL_SINGLE_ENV_VAR_PATTERN = re.compile( + r"(?:\$env:|\$\{?|%)" + r"(?P[A-Z][A-Z0-9]*)" + r"(?:\}|%|(?![A-Za-z0-9_]))" +) + +# Source can read a single-word variable through language APIs without shell +# expansion. Keep the accessors explicit so arbitrary quoted constants do not +# become implementation evidence. +SOURCE_SINGLE_ENV_ACCESS_PATTERN = re.compile( + r"(?:" + r"(?:os\.)?environ\.get|(?:os\.)?getenv|env\.get|getEnv|env_path|" + r"std::env::(?:var|var_os)" + r")\(\s*[\"'](?P[A-Z][A-Z0-9]*)[\"']" + r"|(?:os\.)?environ\[\s*[\"'](?P[A-Z][A-Z0-9]*)[\"']\s*\]" +) + +# Uppercase snake-case is also used for enum members, error identifiers, and +# explanatory path aliases. These reviewed names occur in the scanned docs but +# are not configuration. Keep this list narrow: adding an entry opts that name +# out of the documentation/source consistency guarantee. +DOCUMENTED_NON_ENV_IDENTIFIERS = frozenset( + { + # Content types and pipeline stages. + "BUILD_OUTPUT", + "FIRST_LINE", + "INPUT_CACHED", + "INPUT_COMPRESSED", + "INPUT_RECEIVED", + "INPUT_REMEMBERED", + "INPUT_ROUTED", + "PLAIN_TEXT", + "POST_SEND", + "POST_START", + "PRE_SEND", + "PRE_START", + "RESPONSE_RECEIVED", + "SEARCH_RESULTS", + # Errors and library constants shown in troubleshooting guidance. + "CERTIFICATE_VERIFY_FAILED", + "MALFORMED_FUNCTION_CALL", + "UNSUPPORTED_METRIC_TYPE_MONOTONIC_CUMULATIVE_SUM", + "VERIFY_X509_STRICT", + # Symbolic bucket names used only in filesystem diagrams. + "CONFIG_DIR", + "WORKSPACE_DIR", + } +) + +# Some documented variables are read by libraries or tools that Headroom invokes +# rather than by a literal read site in this repository. Listing them here keeps +# typo detection strict while making that external ownership explicit and +# reviewable. The verifier itself is excluded from source scanning so this +# policy cannot accidentally count as an implementation reference. +EXTERNALLY_CONSUMED_VARIABLES = frozenset( + { + # Standard proxy and certificate variables read by HTTP clients. + "ALL_PROXY", + "CURL_CA_BUNDLE", + "HF_ENDPOINT", + "HTTPS_PROXY", + "HTTP_PROXY", + "NO_PROXY", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_FILE", + # Standard shell / operating-system environment. + "HOME", + "PATH", + "PWD", + "TMPDIR", + "USERPROFILE", + # AWS SDK / Bedrock credential and endpoint discovery. + "AWS_ENDPOINT_URL_BEDROCK_RUNTIME", + # LiteLLM provider configuration. + "DEEPSEEK_API_KEY", + "GOOGLE_CLOUD_LOCATION", + "GOOGLE_CLOUD_PROJECT", + "GROK_CODE_XAI_API_KEY", + "VERTEXAI_LOCATION", + "VERTEXAI_PROJECT", + "XAI_API_KEY", + # ONNX Runtime build and dynamic-link controls. + "ORT_LIB_LOCATION", + "ORT_PREFER_DYNAMIC_LINK", + "ORT_STRATEGY", + # OpenTelemetry SDK autoconfiguration. + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_HEADERS", + "OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE", + "OTEL_EXPORTER_OTLP_PROTOCOL", + } +) + +DOCUMENTATION_FILES = (Path("README.md"), Path("SECURITY.md")) +DOCUMENTATION_GLOB = "docs/content/docs/**/*.mdx" + +# These are implementation surfaces, not tests or examples. Docker and install +# sources are included because a few documented host-side variables are consumed +# before the Python or Rust process starts. +SOURCE_PATHS = ( + Path(".github"), + Path("headroom"), + Path("crates"), + Path("sdk"), + Path("plugins"), + Path("deploy"), + Path("docker"), + Path("scripts"), +) +EXCLUDED_SOURCE_FILES = {Path("scripts/ci/verify_documented_env_vars.py")} +SOURCE_SUFFIXES = { + ".cjs", + ".js", + ".json", + ".mjs", + ".ps1", + ".py", + ".rs", + ".sh", + ".toml", + ".ts", + ".tsx", + ".yaml", + ".yml", +} +EXCLUDED_SOURCE_PARTS = { + "__pycache__", + "examples", + "fixtures", + "node_modules", + "target", + "tests", +} + + +@dataclass(frozen=True) +class Occurrence: + """One documented environment-variable reference.""" + + path: Path + line: int + + +def _read_text(path: Path) -> str: + return path.read_text(encoding="utf-8", errors="ignore") + + +def _documentation_paths(root: Path) -> list[Path]: + paths = [root / relative_path for relative_path in DOCUMENTATION_FILES] + paths.extend(root.glob(DOCUMENTATION_GLOB)) + return sorted(path for path in paths if path.is_file()) + + +def _source_files(root: Path) -> list[Path]: + files: list[Path] = [] + for relative_path in SOURCE_PATHS: + path = root / relative_path + if path.is_file(): + candidates = [path] + elif path.is_dir(): + candidates = path.rglob("*") + else: + continue + + for candidate in candidates: + if not candidate.is_file() or candidate.suffix not in SOURCE_SUFFIXES: + continue + relative_path = candidate.relative_to(root) + if relative_path in EXCLUDED_SOURCE_FILES: + continue + relative_parts = relative_path.parts + if any(part in EXCLUDED_SOURCE_PARTS for part in relative_parts): + continue + files.append(candidate) + return sorted(files) + + +def _normalized_name(match: re.Match[str]) -> str: + """Return a concrete name or a normalized ``PREFIX_*`` family.""" + + name = match.group("name") + if "_<" in name: + return f"{name.split('_<', maxsplit=1)[0]}_*" + return name + + +def _is_dotted_member(line: str, start: int) -> bool: + """Return whether a candidate is a code member such as ``Stage.PRE_SEND``.""" + + return re.search(r"[A-Za-z_][A-Za-z0-9_]*\.$", line[:start]) is not None + + +def _documented_names_in_line(line: str) -> set[str]: + """Extract environment-variable references from one documentation line.""" + + names: set[str] = set() + for match in ENV_VAR_PATTERN.finditer(line): + name = _normalized_name(match) + if name in DOCUMENTED_NON_ENV_IDENTIFIERS: + continue + # Dotted uppercase members are code constants, not process settings. + # Explicit exclusions above still cover prose/table mentions of the same + # constants where the owning type is not present. + if _is_dotted_member(line, match.start()): + continue + names.add(name) + names.update(match.group("name") for match in CONTEXTUAL_SINGLE_ENV_VAR_PATTERN.finditer(line)) + return names + + +def documented_variables(root: Path) -> dict[str, list[Occurrence]]: + """Return documented variables and the locations that mention them.""" + + variables: dict[str, list[Occurrence]] = defaultdict(list) + for path in _documentation_paths(root): + for line_number, line in enumerate(_read_text(path).splitlines(), start=1): + for name in _documented_names_in_line(line): + variables[name].append(Occurrence(path=path.relative_to(root), line=line_number)) + return dict(variables) + + +def source_variables(root: Path) -> set[str]: + """Return environment-variable-shaped identifiers found in implementation source.""" + + variables: set[str] = set() + for path in _source_files(root): + text = _read_text(path) + variables.update(_normalized_name(match) for match in ENV_VAR_PATTERN.finditer(text)) + variables.update( + match.group("name") for match in CONTEXTUAL_SINGLE_ENV_VAR_PATTERN.finditer(text) + ) + variables.update( + match.group("name") or match.group("subscript_name") + for match in SOURCE_SINGLE_ENV_ACCESS_PATTERN.finditer(text) + ) + return variables + + +def _is_resolved(name: str, available: set[str] | frozenset[str]) -> bool: + """Return whether a concrete name or wildcard family is available.""" + + if name.endswith("*"): + prefix = name[:-1] + return any( + candidate.startswith(prefix) and not candidate.endswith("*") for candidate in available + ) + return name in available + + +def missing_variables(root: Path) -> dict[str, list[Occurrence]]: + """Return documented variables that have no implementation reference.""" + + documented = documented_variables(root) + implemented = source_variables(root) + available = implemented | EXTERNALLY_CONSUMED_VARIABLES + missing: dict[str, list[Occurrence]] = {} + + for name, occurrences in documented.items(): + if not _is_resolved(name, available): + missing[name] = occurrences + + return missing + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--root", + type=Path, + default=Path(__file__).resolve().parents[2], + help="Repository root (defaults to the checkout containing this script)", + ) + args = parser.parse_args(argv) + root = args.root.resolve() + + documented = documented_variables(root) + missing = missing_variables(root) + if missing: + for name, occurrences in sorted(missing.items()): + first = occurrences[0] + print( + f"::error file={first.path},line={first.line}::" + f"Documented environment variable {name} has no implementation reference", + file=sys.stderr, + ) + print( + f"Found {len(missing)} documented environment variable(s) with no source reference.", + file=sys.stderr, + ) + return 1 + + implemented = source_variables(root) + external_count = sum( + not _is_resolved(name, implemented) and _is_resolved(name, EXTERNALLY_CONSUMED_VARIABLES) + for name in documented + ) + print( + f"Verified {len(documented)} documented environment variables " + f"({len(documented) - external_count} against repository source; " + f"{external_count} against the approved external-consumer policy)." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_ci_workflow.py b/scripts/tests/test_ci_workflow.py index c38c399b9..56bbc01f0 100644 --- a/scripts/tests/test_ci_workflow.py +++ b/scripts/tests/test_ci_workflow.py @@ -21,3 +21,19 @@ def test_sharded_ci_uploads_only_explicit_coverage_reports() -> None: assert "files: coverage-${{ matrix.shard }}.xml" in upload_step assert "disable_search: true" in upload_step + + +def test_ci_checks_documented_environment_variables_for_source_changes() -> None: + workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") + + assert "env_docs: ${{ steps.filter.outputs.env_docs }}" in workflow + assert "env-doc-consistency:" in workflow + assert "python scripts/ci/verify_documented_env_vars.py" in workflow + + +def test_docs_workflow_checks_environment_variables_for_documentation_changes() -> None: + workflow = Path(".github/workflows/docs.yml").read_text(encoding="utf-8") + + assert "- 'README.md'" in workflow + assert "- 'SECURITY.md'" in workflow + assert "python scripts/ci/verify_documented_env_vars.py" in workflow diff --git a/scripts/tests/test_verify_documented_env_vars.py b/scripts/tests/test_verify_documented_env_vars.py new file mode 100644 index 000000000..d599f28e4 --- /dev/null +++ b/scripts/tests/test_verify_documented_env_vars.py @@ -0,0 +1,255 @@ +"""Tests for the documented environment-variable consistency check.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + + +def _load_module(): + script = Path(__file__).parent.parent / "ci" / "verify_documented_env_vars.py" + spec = importlib.util.spec_from_file_location("verify_documented_env_vars", script) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _write_minimal_project(root: Path, *, docs: str, source: str) -> None: + docs_path = root / "docs" / "content" / "docs" / "configuration.mdx" + docs_path.parent.mkdir(parents=True) + docs_path.write_text(docs, encoding="utf-8") + source_path = root / "headroom" / "config.py" + source_path.parent.mkdir(parents=True) + source_path.write_text(source, encoding="utf-8") + + +def test_exact_documented_variable_must_exist_in_source(tmp_path: Path) -> None: + module = _load_module() + _write_minimal_project( + tmp_path, + docs="Use `HEADROOM_REAL_SETTING` and `HEADROOM_MISSPELLED_SETTING`.\n", + source='REAL_SETTING = "HEADROOM_REAL_SETTING"\n', + ) + + missing = module.missing_variables(tmp_path) + + assert list(missing) == ["HEADROOM_MISSPELLED_SETTING"] + assert missing["HEADROOM_MISSPELLED_SETTING"][0].line == 1 + + +def test_documented_wildcard_matches_a_concrete_source_variable(tmp_path: Path) -> None: + module = _load_module() + _write_minimal_project( + tmp_path, + docs="Leave `HEADROOM_OTEL_*` unset to use the ambient provider.\n", + source='ENABLED = "HEADROOM_OTEL_METRICS_ENABLED"\n', + ) + + assert module.missing_variables(tmp_path) == {} + + +def test_placeholder_wildcard_is_normalized_and_matches_source(tmp_path: Path) -> None: + module = _load_module() + _write_minimal_project( + tmp_path, + docs="Set `DISABLE_PROMPT_CACHING_` for one model family.\n", + source='SETTING = "DISABLE_PROMPT_CACHING_OPUS"\n', + ) + + documented = module.documented_variables(tmp_path) + + assert set(documented) == {"DISABLE_PROMPT_CACHING_*"} + assert module.missing_variables(tmp_path) == {} + + +def test_single_word_variable_is_discovered_from_shell_context(tmp_path: Path) -> None: + module = _load_module() + _write_minimal_project( + tmp_path, + docs="The wrapper reads `$ACME` and a typo such as `$ACMEE` must fail.\n", + source='value = os.environ.get("ACME")\n', + ) + + assert set(module.documented_variables(tmp_path)) == {"ACME", "ACMEE"} + assert set(module.missing_variables(tmp_path)) == {"ACMEE"} + + +def test_host_side_variable_can_be_implemented_by_install_source(tmp_path: Path) -> None: + module = _load_module() + _write_minimal_project( + tmp_path, + docs="Set `HEADROOM_DOCKER_IMAGE` before installation.\n", + source="", + ) + install_script = tmp_path / "scripts" / "install.sh" + install_script.parent.mkdir(parents=True) + install_script.write_text('IMAGE="${HEADROOM_DOCKER_IMAGE:-latest}"\n', encoding="utf-8") + + assert module.missing_variables(tmp_path) == {} + + +def test_root_docs_and_generic_environment_variable_names_are_scanned( + tmp_path: Path, +) -> None: + module = _load_module() + (tmp_path / "README.md").write_text( + "Use `ANTHROPIC_BASE_URL`, `OPENAI_BASE_URL`, or `ACME_SERVICE_TOKEN`.\n", + encoding="utf-8", + ) + (tmp_path / "SECURITY.md").write_text("Set `DO_NOT_TRACK=1`.\n", encoding="utf-8") + source_path = tmp_path / "headroom" / "proxy.py" + source_path.parent.mkdir(parents=True) + source_path.write_text( + 'VARIABLES = ("ANTHROPIC_BASE_URL", "OPENAI_BASE_URL", ' + '"ACME_SERVICE_TOKEN", "DO_NOT_TRACK")\n', + encoding="utf-8", + ) + + documented = module.documented_variables(tmp_path) + + assert set(documented) == { + "ACME_SERVICE_TOKEN", + "ANTHROPIC_BASE_URL", + "DO_NOT_TRACK", + "OPENAI_BASE_URL", + } + assert module.missing_variables(tmp_path) == {} + + +def test_code_constants_and_lowercase_name_fragments_are_not_variables( + tmp_path: Path, +) -> None: + module = _load_module() + _write_minimal_project( + tmp_path, + docs=( + "Use `HEADROOM_REAL_SETTING`.\n" + "Compare PipelineStage.PRE_SEND with `FIRST_LINE`.\n" + "An install may contain .DS_Store or report CERTIFICATE_VERIFY_FAILED.\n" + ), + source='SETTING = "HEADROOM_REAL_SETTING"\n', + ) + + assert set(module.documented_variables(tmp_path)) == {"HEADROOM_REAL_SETTING"} + + +def test_external_consumer_policy_is_explicit_and_does_not_hide_typos( + tmp_path: Path, +) -> None: + module = _load_module() + _write_minimal_project( + tmp_path, + docs="Set `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_ENDPONT`.\n", + source="", + ) + + missing = module.missing_variables(tmp_path) + + assert set(missing) == {"OTEL_EXPORTER_OTLP_ENDPONT"} + + +def test_cli_success_reports_source_and_external_policy_scope(tmp_path: Path, capsys) -> None: + module = _load_module() + _write_minimal_project( + tmp_path, + docs="Set `HEADROOM_REAL_SETTING` and `OTEL_EXPORTER_OTLP_ENDPOINT`.\n", + source='SETTING = "HEADROOM_REAL_SETTING"\n', + ) + + exit_code = module.main(["--root", str(tmp_path)]) + + captured = capsys.readouterr() + assert exit_code == 0 + assert ( + "Verified 2 documented environment variables " + "(1 against repository source; 1 against the approved external-consumer policy)." + in captured.out + ) + + +def test_verifier_policy_does_not_count_as_an_implementation_reference( + tmp_path: Path, +) -> None: + module = _load_module() + _write_minimal_project( + tmp_path, + docs="Set `ACME_EXTERNAL_SETTING`.\n", + source="", + ) + verifier = tmp_path / "scripts" / "ci" / "verify_documented_env_vars.py" + verifier.parent.mkdir(parents=True) + verifier.write_text('POLICY = {"ACME_EXTERNAL_SETTING"}\n', encoding="utf-8") + + assert set(module.missing_variables(tmp_path)) == {"ACME_EXTERNAL_SETTING"} + + +def test_verifier_tests_do_not_count_as_implementation_references(tmp_path: Path) -> None: + module = _load_module() + _write_minimal_project( + tmp_path, + docs="Set `ACME_TEST_ONLY_SETTING`.\n", + source="", + ) + verifier_test = tmp_path / "scripts" / "tests" / "test_verify_documented_env_vars.py" + verifier_test.parent.mkdir(parents=True) + verifier_test.write_text( + 'REPRESENTATIVE_VARIABLE = "ACME_TEST_ONLY_SETTING"\n', + encoding="utf-8", + ) + + assert "ACME_TEST_ONLY_SETTING" not in module.source_variables(tmp_path) + assert set(module.missing_variables(tmp_path)) == {"ACME_TEST_ONLY_SETTING"} + + +def test_repository_docs_discover_representative_environment_variable_families() -> None: + module = _load_module() + repository_root = Path(__file__).resolve().parents[2] + + documented = set(module.documented_variables(repository_root)) + + assert { + # AWS and Bedrock. + "AWS_ACCESS_KEY_ID", + "AWS_ENDPOINT_URL_BEDROCK_RUNTIME", + "BEDROCK_TARGET_API_URL", + # Claude's alternate cloud runtimes. + "CLAUDE_CODE_USE_BEDROCK", + "CLAUDE_CODE_USE_FOUNDRY", + "CLAUDE_CODE_USE_VERTEX", + # Google and Vertex. + "GOOGLE_APPLICATION_CREDENTIALS", + "VERTEXAI_PROJECT", + # GitHub Copilot. + "GITHUB_COPILOT_ENTERPRISE_URL", + # Process-wide proxy and TLS controls. + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_FILE", + # OpenTelemetry. + "OTEL_EXPORTER_OTLP_ENDPOINT", + # Provider keys outside the original prefix policy. + "OPENROUTER_API_KEY", + "XAI_API_KEY", + } <= documented + + +def test_cli_fails_with_a_github_annotation_for_missing_variable(tmp_path: Path, capsys) -> None: + module = _load_module() + _write_minimal_project( + tmp_path, + docs="Configuration: `HEADROOM_REMOVED_SETTING`\n", + source="", + ) + + exit_code = module.main(["--root", str(tmp_path)]) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "::error file=docs/content/docs/configuration.mdx,line=1::" in captured.err + assert "HEADROOM_REMOVED_SETTING" in captured.err