mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
84f9871e30
## Description Adds the Phase 0 `agent-evals/` nested project for benchmarking coding-agent task accuracy with and without Headroom's proxy/compression path. The project is intentionally separate from the published `headroom-ai` package and provides the shared A/B framework used by the stacked Phase 1 PR #1040. ## Type of Change - [x] New feature (non-breaking change that adds functionality) - [x] Tests only - [x] Documentation update ## Changes Made - Added a three-arm experiment model for direct provider calls, Headroom passthrough, and Headroom compression. - Added the resumable orchestrator, run manifest/config models, JSON logging, and append-only journal handling. - Added savings capture from Headroom response headers plus scorecard reporting for resolved rate and savings. - Added unit tests and live-test markers for provider-key dependent validation. - Kept the benchmark project isolated from the product package and normal Headroom release wheel. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check`) - [x] Type checking passes (`mypy`) - [x] New tests added for new functionality ### Test Output ```text agent-evals Phase 0 validation from original PR: 74 unit tests passed ruff clean mypy clean CI on this PR: changes and commitlint pass; product CI jobs are skipped because this only changes the nested agent-evals project. ``` ## Real Behavior Proof - Environment: local agent-evals development environment with provider-key dependent live tests skipped unless credentials are present. - Exact command / steps: Ran the Phase 0 unit suite, ruff, and mypy for the nested `agent-evals` project; GitHub CI also ran the repository change detection and commitlint jobs for this PR. - Observed result: The Phase 0 framework tests passed locally, static checks were clean, and GitHub CI reported passing change detection/commitlint for the PR. - Not tested: live provider accuracy claims; those require provider keys and larger benchmark runs and are intentionally covered by live-marked tests and later stacked phases. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
43 lines
1.5 KiB
Python
43 lines
1.5 KiB
Python
"""Tests for the configuration surface (defaults + env overrides)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from agent_evals.config import Settings
|
|
from agent_evals.models import Provider
|
|
|
|
|
|
def test_defaults() -> None:
|
|
s = Settings()
|
|
assert s.provider == Provider.ANTHROPIC
|
|
assert s.stats.k_runs == 10
|
|
assert s.stats.margin_lossy_pp == pytest.approx(2.0)
|
|
assert s.stats.margin_ccr_pp == pytest.approx(0.0)
|
|
assert s.proxy.port_range_start < s.proxy.port_range_end
|
|
assert s.proxy.headroom_cmd == ["headroom", "proxy"]
|
|
|
|
|
|
def test_env_override_flat(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("AGENT_EVALS_CONCURRENCY", "8")
|
|
monkeypatch.setenv("AGENT_EVALS_MODEL_SNAPSHOT", "gpt-5.2")
|
|
monkeypatch.setenv("AGENT_EVALS_PROVIDER", "openai")
|
|
s = Settings()
|
|
assert s.concurrency == 8
|
|
assert s.model_snapshot == "gpt-5.2"
|
|
assert s.provider == Provider.OPENAI
|
|
|
|
|
|
def test_env_override_nested(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("AGENT_EVALS_STATS__K_RUNS", "20")
|
|
monkeypatch.setenv("AGENT_EVALS_STATS__MARGIN_LOSSY_PP", "1.5")
|
|
monkeypatch.setenv("AGENT_EVALS_PROXY__READYZ_TIMEOUT_S", "45")
|
|
s = Settings()
|
|
assert s.stats.k_runs == 20
|
|
assert s.stats.margin_lossy_pp == pytest.approx(1.5)
|
|
assert s.proxy.readyz_timeout_s == pytest.approx(45.0)
|
|
|
|
|
|
def test_alpha_bounds_validated() -> None:
|
|
with pytest.raises(ValueError):
|
|
Settings(stats={"alpha": 1.5}) # type: ignore[arg-type]
|