mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-10 14:27:00 -04:00
84f9871e30
## Description Adds the Phase 0 `agent-evals/` nested project for benchmarking coding-agent task accuracy with and without Headroom's proxy/compression path. The project is intentionally separate from the published `headroom-ai` package and provides the shared A/B framework used by the stacked Phase 1 PR #1040. ## Type of Change - [x] New feature (non-breaking change that adds functionality) - [x] Tests only - [x] Documentation update ## Changes Made - Added a three-arm experiment model for direct provider calls, Headroom passthrough, and Headroom compression. - Added the resumable orchestrator, run manifest/config models, JSON logging, and append-only journal handling. - Added savings capture from Headroom response headers plus scorecard reporting for resolved rate and savings. - Added unit tests and live-test markers for provider-key dependent validation. - Kept the benchmark project isolated from the product package and normal Headroom release wheel. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check`) - [x] Type checking passes (`mypy`) - [x] New tests added for new functionality ### Test Output ```text agent-evals Phase 0 validation from original PR: 74 unit tests passed ruff clean mypy clean CI on this PR: changes and commitlint pass; product CI jobs are skipped because this only changes the nested agent-evals project. ``` ## Real Behavior Proof - Environment: local agent-evals development environment with provider-key dependent live tests skipped unless credentials are present. - Exact command / steps: Ran the Phase 0 unit suite, ruff, and mypy for the nested `agent-evals` project; GitHub CI also ran the repository change detection and commitlint jobs for this PR. - Observed result: The Phase 0 framework tests passed locally, static checks were clean, and GitHub CI reported passing change detection/commitlint for the PR. - Not tested: live provider accuracy claims; those require provider keys and larger benchmark runs and are intentionally covered by live-marked tests and later stacked phases. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
79 lines
2.1 KiB
TOML
79 lines
2.1 KiB
TOML
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[project]
|
|
name = "agent-evals"
|
|
version = "0.0.1"
|
|
description = "End-to-end accuracy A/B framework: coding-agent benchmarks WITH vs WITHOUT Headroom compression"
|
|
readme = "README.md"
|
|
license = "Apache-2.0"
|
|
requires-python = ">=3.10"
|
|
dependencies = [
|
|
"pydantic>=2.0.0",
|
|
"pydantic-settings>=2.0.0",
|
|
"httpx>=0.24.0",
|
|
"numpy>=1.24.0",
|
|
"click>=8.1.0",
|
|
"rich>=13.0.0",
|
|
"pyyaml>=6.0",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
# Dev/test toolchain (the agent-evals gate). Pure Python, no heavy benchmark deps.
|
|
dev = [
|
|
"pytest>=7.4",
|
|
"pytest-asyncio>=0.23",
|
|
"ruff>=0.4",
|
|
"mypy>=1.8",
|
|
"types-PyYAML",
|
|
]
|
|
# Statistics (Phase 1+): paired bootstrap, TOST, Wilson, power.
|
|
stats = ["scipy>=1.10", "statsmodels>=0.14", "pandas>=2.0"]
|
|
# Rigorous mixed-effects estimator (Phase 2).
|
|
glmm = ["bambi>=0.13"]
|
|
# Layer-2 fidelity probes (Phase 2).
|
|
fidelity = ["rouge-score>=0.1.2", "bert-score>=0.3.13", "sentence-transformers>=2.2.0,<6.0"]
|
|
# Heavy benchmark graders — kept OUT of the default install so they never bloat anything.
|
|
swebench = ["swebench>=2.0", "docker>=7.0"]
|
|
aider = ["aider-chat>=0.50"]
|
|
|
|
[project.scripts]
|
|
agent-evals = "agent_evals.cli:main"
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["src/agent_evals"]
|
|
|
|
[tool.ruff]
|
|
target-version = "py310"
|
|
line-length = 100
|
|
|
|
[tool.ruff.lint]
|
|
select = ["E", "W", "F", "I", "B", "C4", "UP"]
|
|
ignore = ["E501", "B008", "B905"]
|
|
|
|
[tool.ruff.lint.isort]
|
|
known-first-party = ["agent_evals"]
|
|
|
|
[tool.ruff.format]
|
|
quote-style = "double"
|
|
indent-style = "space"
|
|
|
|
[tool.mypy]
|
|
python_version = "3.10"
|
|
warn_unused_configs = true
|
|
disallow_untyped_defs = true
|
|
ignore_missing_imports = true
|
|
files = ["src"]
|
|
|
|
[tool.pytest.ini_options]
|
|
testpaths = ["tests"]
|
|
python_files = ["test_*.py"]
|
|
python_functions = ["test_*"]
|
|
addopts = "-v --tb=short"
|
|
asyncio_mode = "auto"
|
|
markers = [
|
|
"slow: slow tests (subprocess spawns, large fixtures)",
|
|
"real_llm: tests that hit real LLM APIs; skipped unless keys present",
|
|
"live: opt-in tests that spawn a real headroom proxy / hit upstreams; require keys",
|
|
]
|