mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
## Description Hardens two routine PR-review pain points from the recent open-PR sweep: - PR Governance reruns could keep validating the stale `pull_request_target` event body even after the live PR description had been fixed. - Main CI model-cache misses could surface as dozens of unrelated memory-test failures instead of one clear cache-preflight failure. This intentionally avoids PyPI/package-bloat and release/nightly workflow changes so the PR stays scoped to review and CI stabilization. ## Type of Change - [x] Bug fix - [ ] New feature - [ ] Documentation - [x] Refactor - [x] Tests only ## Changes Made - Added `--body-file` support to `scripts/pr-governance.py` so workflows can validate the current PR body rather than stale rerun payloads. - Updated PR Governance to fetch the live PR body via the GitHub API before validating template fields. - Added a CI preflight script that loads the default sentence-transformer model in offline mode and verifies the expected embedding dimension. - Wired that preflight into the sharded CI job before pytest starts, turning missing/corrupt Hugging Face caches into one early, actionable failure. - Added workflow/script regression tests for the live-body override and model-cache preflight placement. ## Testing - [x] Unit tests - [x] Lint/static checks - [ ] Integration tests - [ ] Manual testing ### Test Output ```text uv run --with pytest --with pytest-asyncio python -m pytest scripts/tests/test_pr_governance.py scripts/tests/test_pr_health_workflow.py scripts/tests/test_ci_workflow.py -q 9 passed in 0.04s uv run ruff check scripts/pr-governance.py scripts/ci/verify_hf_model_cache.py scripts/tests/test_pr_governance.py scripts/tests/test_pr_health_workflow.py scripts/tests/test_ci_workflow.py All checks passed! python -m py_compile scripts\ci\verify_hf_model_cache.py scripts\pr-governance.py # passed ``` ## Real Behavior Proof - Environment: Windows 11, Python 3.13.3, isolated worktree `C:\git\headroom\.worktrees\stabilization-hardening`. - Exact command / steps: Ran the focused governance/workflow tests, ruff on touched Python files, and `py_compile` for the executable scripts. - Observed result: Governance tests prove a stale event body can be overridden by the live PR body; workflow tests prove CI validates live PR body and runs the Hugging Face offline-cache preflight before pytest shards. - Not tested: Full GitHub CI before PR creation; that will run on this PR. The new Hugging Face preflight itself is intentionally not run locally because it depends on the CI-warmed offline model cache. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review
214 lines
5.6 KiB
Python
214 lines
5.6 KiB
Python
"""Tests for pr-governance.py."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
def _load_module():
|
|
script = Path(__file__).parent.parent / "pr-governance.py"
|
|
spec = importlib.util.spec_from_file_location("pr_governance", script)
|
|
assert spec is not None
|
|
assert spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _event(body: str, *, draft: bool = False, login: str = "octocat") -> dict[str, object]:
|
|
return {
|
|
"pull_request": {
|
|
"number": 42,
|
|
"draft": draft,
|
|
"body": body,
|
|
"user": {"login": login},
|
|
}
|
|
}
|
|
|
|
|
|
VALID_BODY = """## Description
|
|
|
|
Add a required PR-governance gate for template validation and review readiness.
|
|
|
|
Closes #123
|
|
|
|
## Type of Change
|
|
|
|
- [x] New feature (non-breaking change that adds functionality)
|
|
- [ ] Documentation update
|
|
|
|
## Changes Made
|
|
|
|
- Added a workflow-backed PR template validator.
|
|
- Added local commit message linting in the commit-msg hook.
|
|
|
|
## Testing
|
|
|
|
- [x] Unit tests pass (`pytest`)
|
|
- [x] Manual testing performed
|
|
|
|
### Test Output
|
|
|
|
```text
|
|
pytest scripts/tests/test_pr_governance.py -q
|
|
```
|
|
|
|
## Real Behavior Proof
|
|
|
|
- Environment: Ubuntu runner, Python 3.12
|
|
- Exact command / steps: Open a PR, remove the ready checkbox, re-run the workflow.
|
|
- Observed result: The governance check fails and the PR gets a needs-author-action label.
|
|
- Not tested: Automatic Copilot review rulesets in repository settings.
|
|
|
|
## Review Readiness
|
|
|
|
- [x] I have performed a self-review
|
|
- [x] This PR is ready for human review
|
|
|
|
## Additional Notes
|
|
|
|
- Maintainers can optionally enable Copilot code review from repository rulesets.
|
|
"""
|
|
|
|
|
|
def test_validate_pull_request_marks_ready_pr_valid() -> None:
|
|
module = _load_module()
|
|
|
|
report = module.validate_pull_request(_event(VALID_BODY))
|
|
|
|
assert report.valid is True
|
|
assert report.ready_for_review is True
|
|
assert report.needs_author_action is False
|
|
assert report.problems == []
|
|
assert report.labels_to_add == [module.READY_LABEL]
|
|
assert module.AUTHOR_ACTION_LABEL in report.labels_to_remove
|
|
|
|
|
|
def test_validate_pull_request_accepts_crlf_test_output_code_block() -> None:
|
|
module = _load_module()
|
|
body = VALID_BODY.replace("\n", "\r\n")
|
|
|
|
report = module.validate_pull_request(_event(body))
|
|
|
|
assert report.valid is True
|
|
assert report.problems == []
|
|
|
|
|
|
def test_validate_pull_request_body_override_uses_live_body() -> None:
|
|
module = _load_module()
|
|
stale_event_body = ""
|
|
|
|
report = module.validate_pull_request_body(_event(stale_event_body), VALID_BODY)
|
|
|
|
assert report.valid is True
|
|
assert report.ready_for_review is True
|
|
assert report.problems == []
|
|
|
|
|
|
def test_cli_body_file_override_uses_live_body(tmp_path: Path, monkeypatch) -> None:
|
|
module = _load_module()
|
|
event_path = tmp_path / "event.json"
|
|
body_path = tmp_path / "body.md"
|
|
report_path = tmp_path / "report.json"
|
|
event_path.write_text(
|
|
json.dumps(_event("")),
|
|
encoding="utf-8",
|
|
)
|
|
body_path.write_text(VALID_BODY, encoding="utf-8")
|
|
monkeypatch.delenv("GITHUB_OUTPUT", raising=False)
|
|
|
|
exit_code = module.main(
|
|
[
|
|
"--event",
|
|
str(event_path),
|
|
"--body-file",
|
|
str(body_path),
|
|
"--report",
|
|
str(report_path),
|
|
]
|
|
)
|
|
|
|
assert exit_code == 0
|
|
report = json.loads(report_path.read_text(encoding="utf-8"))
|
|
assert report["valid"] is True
|
|
assert report["ready_for_review"] is True
|
|
|
|
|
|
def test_validate_pull_request_allows_draft_without_ready_checkboxes() -> None:
|
|
module = _load_module()
|
|
body = VALID_BODY.replace(
|
|
"- [x] I have performed a self-review", "- [ ] I have performed a self-review"
|
|
)
|
|
body = body.replace(
|
|
"- [x] This PR is ready for human review",
|
|
"- [ ] This PR is ready for human review",
|
|
)
|
|
|
|
report = module.validate_pull_request(_event(body, draft=True))
|
|
|
|
assert report.valid is True
|
|
assert report.ready_for_review is False
|
|
assert report.needs_author_action is False
|
|
assert report.labels_to_add == []
|
|
assert module.READY_LABEL in report.labels_to_remove
|
|
|
|
|
|
def test_validate_pull_request_fails_on_missing_required_content() -> None:
|
|
module = _load_module()
|
|
body = """## Description
|
|
|
|
Fixes #123
|
|
|
|
## Type of Change
|
|
|
|
- [ ] New feature (non-breaking change that adds functionality)
|
|
|
|
## Changes Made
|
|
|
|
- Change 1
|
|
|
|
## Testing
|
|
|
|
### Test Output
|
|
|
|
```text
|
|
# Paste relevant command output or artifact links here
|
|
```
|
|
|
|
## Real Behavior Proof
|
|
|
|
- Environment:
|
|
- Exact command / steps:
|
|
- Observed result:
|
|
- Not tested:
|
|
|
|
## Review Readiness
|
|
|
|
- [ ] I have performed a self-review
|
|
- [ ] This PR is ready for human review
|
|
"""
|
|
|
|
report = module.validate_pull_request(_event(body))
|
|
|
|
assert report.valid is False
|
|
assert report.needs_author_action is True
|
|
assert module.AUTHOR_ACTION_LABEL in report.labels_to_add
|
|
assert any("Description" in problem for problem in report.problems)
|
|
assert any("Type of Change" in problem for problem in report.problems)
|
|
assert any("Test Output" in problem for problem in report.problems)
|
|
assert any("Real Behavior Proof" in problem for problem in report.problems)
|
|
|
|
|
|
def test_validate_pull_request_skips_bot_authored_prs() -> None:
|
|
module = _load_module()
|
|
|
|
report = module.validate_pull_request(_event("", login="dependabot[bot]"))
|
|
|
|
assert report.valid is True
|
|
assert report.is_bot_pr is True
|
|
assert report.needs_author_action is False
|
|
assert report.labels_to_add == []
|