mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-10 14:27:00 -04:00
84f9871e30
## Description Adds the Phase 0 `agent-evals/` nested project for benchmarking coding-agent task accuracy with and without Headroom's proxy/compression path. The project is intentionally separate from the published `headroom-ai` package and provides the shared A/B framework used by the stacked Phase 1 PR #1040. ## Type of Change - [x] New feature (non-breaking change that adds functionality) - [x] Tests only - [x] Documentation update ## Changes Made - Added a three-arm experiment model for direct provider calls, Headroom passthrough, and Headroom compression. - Added the resumable orchestrator, run manifest/config models, JSON logging, and append-only journal handling. - Added savings capture from Headroom response headers plus scorecard reporting for resolved rate and savings. - Added unit tests and live-test markers for provider-key dependent validation. - Kept the benchmark project isolated from the product package and normal Headroom release wheel. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check`) - [x] Type checking passes (`mypy`) - [x] New tests added for new functionality ### Test Output ```text agent-evals Phase 0 validation from original PR: 74 unit tests passed ruff clean mypy clean CI on this PR: changes and commitlint pass; product CI jobs are skipped because this only changes the nested agent-evals project. ``` ## Real Behavior Proof - Environment: local agent-evals development environment with provider-key dependent live tests skipped unless credentials are present. - Exact command / steps: Ran the Phase 0 unit suite, ruff, and mypy for the nested `agent-evals` project; GitHub CI also ran the repository change detection and commitlint jobs for this PR. - Observed result: The Phase 0 framework tests passed locally, static checks were clean, and GitHub CI reported passing change detection/commitlint for the PR. - Not tested: live provider accuracy claims; those require provider keys and larger benchmark runs and are intentionally covered by live-marked tests and later stacked phases. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
87 lines
2.9 KiB
Python
87 lines
2.9 KiB
Python
"""Tests for the core data-model contracts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
|
|
import pytest
|
|
|
|
from agent_evals.models import (
|
|
ArmName,
|
|
DeltaEstimate,
|
|
EquivalenceVerdict,
|
|
Pricing,
|
|
RunSavings,
|
|
TaskResult,
|
|
TaskSavings,
|
|
)
|
|
|
|
|
|
def test_savings_from_token_counts_basic(pricing: Pricing) -> None:
|
|
s = TaskSavings.from_token_counts(tokens_before=1000, tokens_after=400, pricing=pricing)
|
|
assert s.tokens_saved == 600
|
|
assert s.savings_percent == pytest.approx(60.0)
|
|
assert s.ratio == pytest.approx(0.4)
|
|
# cost = tokens / 1e6 * input_usd_per_1m (=2.0)
|
|
assert s.cost_usd_before == pytest.approx(1000 / 1_000_000 * 2.0)
|
|
assert s.cost_usd_after == pytest.approx(400 / 1_000_000 * 2.0)
|
|
assert s.cost_usd_saved == pytest.approx(s.cost_usd_before - s.cost_usd_after)
|
|
assert s.source == "headers"
|
|
|
|
|
|
def test_savings_zero_tokens_before_is_safe(pricing: Pricing) -> None:
|
|
s = TaskSavings.from_token_counts(tokens_before=0, tokens_after=0, pricing=pricing)
|
|
assert s.tokens_saved == 0
|
|
assert s.savings_percent == 0.0
|
|
assert s.ratio == 1.0
|
|
assert s.cost_usd_saved == 0.0
|
|
assert math.isfinite(s.ratio)
|
|
|
|
|
|
def test_savings_no_compression_ratio_one(pricing: Pricing) -> None:
|
|
s = TaskSavings.from_token_counts(tokens_before=500, tokens_after=500, pricing=pricing)
|
|
assert s.tokens_saved == 0
|
|
assert s.ratio == pytest.approx(1.0)
|
|
assert s.savings_percent == pytest.approx(0.0)
|
|
|
|
|
|
def test_savings_carries_flags(pricing: Pricing) -> None:
|
|
s = TaskSavings.from_token_counts(
|
|
tokens_before=100,
|
|
tokens_after=90,
|
|
pricing=pricing,
|
|
transforms=["smart_crusher", "read_lifecycle"],
|
|
cached=True,
|
|
compression_failed=False,
|
|
source="stats_delta",
|
|
)
|
|
assert s.transforms == ["smart_crusher", "read_lifecycle"]
|
|
assert s.cached is True
|
|
assert s.source == "stats_delta"
|
|
|
|
|
|
def test_task_result_cell_key() -> None:
|
|
tr = TaskResult(task_id="t1", arm=ArmName.B_HEADROOM, run_index=3, resolved=True)
|
|
assert tr.cell_key == ("t1", "b_headroom", 3)
|
|
|
|
|
|
def test_equivalence_verdict_roundtrip() -> None:
|
|
v = EquivalenceVerdict(
|
|
delta=DeltaEstimate(point=-0.5, ci_low=-1.9, ci_high=0.9, method="paired_bootstrap"),
|
|
margin=2.0,
|
|
verdict="equivalent",
|
|
)
|
|
again = EquivalenceVerdict.model_validate_json(v.model_dump_json())
|
|
assert again.verdict == "equivalent"
|
|
assert again.delta.ci_low == pytest.approx(-1.9)
|
|
|
|
|
|
def test_run_savings_optional_preserved_tokens() -> None:
|
|
rs = RunSavings(cache_read_tokens=120, prefix_freeze_busts_avoided=3)
|
|
assert rs.prefix_freeze_tokens_preserved is None
|
|
|
|
|
|
def test_arm_name_values_are_stable() -> None:
|
|
# Journal keys depend on these string values; guard against accidental renames.
|
|
assert ArmName.A1_PASSTHROUGH.value == "a1_passthrough"
|
|
assert ArmName.B_HEADROOM.value == "b_headroom"
|