mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
a2b2ec5463
Evaluation Suite: - Tiered eval framework (Tier 1 ~$3/15min, Tier 2/3 for extended coverage) - 16 benchmarks across 3 tiers: GSM8K, TruthfulQA, MMLU, ARC, HumanEval, SQuAD v2, BFCL, Tool Outputs, CCR needle retention, HotpotQA, and more - Before/After runner with full proxy support (compression + CCR retrieval) - LLM-as-judge for ground-truth comparison (BFCL function calling) - Zero-cost compression-only runner (CCR needle retention, info retention) - Cost tracker with per-model pricing and budget enforcement - Report card generator (Markdown, JSON, HTML) - Suite CLI: python -m headroom.evals suite --tier 1 - Fix BFCL dataset loader for current HuggingFace schema - CI workflow: PR smoke test + weekly full Tier 1 Results: SQuAD 97%, BFCL 97%, Tool Outputs 100%, CCR 100% SmartCrusher: - Universal JSON crush for heterogeneous arrays - Fix mypy redefinition warning in _crush_string_array Other: - Latency benchmark suite with docs - Known limitations doc - Prompt comparison evaluator - Config updates for new features
62 lines
2.1 KiB
YAML
62 lines
2.1 KiB
YAML
name: Evaluation Suite
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 6 * * 1' # Weekly on Monday 6am UTC
|
|
workflow_dispatch: # Manual trigger
|
|
pull_request:
|
|
paths:
|
|
- 'headroom/transforms/**'
|
|
- 'headroom/evals/**'
|
|
- 'headroom/compress.py'
|
|
|
|
jobs:
|
|
# Fast smoke test on PRs touching compression code (~$0.05, ~2 min)
|
|
smoke-test:
|
|
if: github.event_name == 'pull_request'
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.11"
|
|
- name: Install dependencies
|
|
run: pip install -e ".[all]"
|
|
- name: Run CCR round-trip (zero cost)
|
|
run: |
|
|
python -c "
|
|
from headroom.evals.runners.compression_only import CompressionOnlyRunner
|
|
runner = CompressionOnlyRunner()
|
|
cases = runner.generate_ccr_test_cases(n=50)
|
|
result = runner.evaluate_ccr_lossless(cases)
|
|
print(f'CCR Round-trip: {result.passed_cases}/{result.total_cases} passed')
|
|
assert result.passed, f'CCR failures: {result.errors}'
|
|
"
|
|
- name: Run built-in tool output eval
|
|
run: python -m headroom.evals quick -n 8 --provider openai --model gpt-4o-mini
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
|
|
# Full Tier 1 suite, weekly or manual (~$3-5, ~30-45 min)
|
|
weekly-suite:
|
|
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 60
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.11"
|
|
- name: Install dependencies
|
|
run: pip install -e ".[all]"
|
|
- name: Run Tier 1 evaluation suite
|
|
run: python -m headroom.evals suite --tier 1 --ci -o eval_results/
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
- name: Upload results
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: eval-results-${{ github.run_number }}
|
|
path: eval_results/
|