mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
beec0789ed
The action was set to @stable, which installs whatever the latest stable is (1.95.0 right now). Then maturin invokes cargo, which reads rust-toolchain.toml and re-resolves to "1.95.0 + clippy + rustfmt". rustup treats stable and 1.95.0 as distinct toolchain identities and refuses the second install with: failed to install component 'clippy-preview-x86_64-unknown-linux-gnu', detected conflict: 'bin/cargo-clippy' This was intermittent across the matrix (only test (3.10) tripped on the most recent run; others got lucky on cache state). Pinning the action ref to 1.95.0 makes both sides ask for the exact same toolchain identity, so the second install is a no-op and the conflict can't fire. Bump procedure stays the same: when rust-toolchain.toml's channel changes, update these refs in lock-step. Plugin manifests auto-bumped 0.11.0 -> 0.13.2 by sync-plugin-versions hook (unrelated to the workflow fix).
92 lines
3.4 KiB
YAML
92 lines
3.4 KiB
YAML
name: Evaluation Suite
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 6 * * 1' # Weekly on Monday 6am UTC
|
|
workflow_dispatch: # Manual trigger
|
|
pull_request:
|
|
paths:
|
|
- 'headroom/transforms/**'
|
|
- 'headroom/evals/**'
|
|
- 'headroom/compress.py'
|
|
|
|
jobs:
|
|
# Fast smoke test on PRs touching compression code (~$0.05, ~2 min)
|
|
smoke-test:
|
|
if: github.event_name == 'pull_request'
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.11"
|
|
- name: Install dependencies
|
|
run: pip install -e ".[all]"
|
|
|
|
# `compression_only.evaluate_ccr_lossless` constructs a SmartCrusher
|
|
# which now hard-imports `headroom._core` (Stage 3c.1b). Build the
|
|
# Rust extension before running the smoke test or every call raises
|
|
# `ModuleNotFoundError`. Mirrors the main CI `test` job pattern.
|
|
- name: Install Rust toolchain
|
|
uses: dtolnay/rust-toolchain@1.95.0
|
|
|
|
- name: Cache cargo registry + build
|
|
uses: Swatinem/rust-cache@v2
|
|
with:
|
|
workspaces: ". -> target"
|
|
|
|
- name: Install maturin
|
|
run: pip install 'maturin>=1.5,<2.0'
|
|
|
|
- name: Build Rust extension (headroom._core)
|
|
run: |
|
|
set -euo pipefail
|
|
maturin build --release -m crates/headroom-py/Cargo.toml --out dist
|
|
pip install --force-reinstall --no-deps dist/headroom_core_py-*.whl
|
|
SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])")
|
|
SO_FILE=$(find "$SITE_PACKAGES/headroom" -maxdepth 1 -name "_core.cpython-*.so" -print -quit 2>/dev/null)
|
|
if [[ -z "$SO_FILE" ]]; then
|
|
echo "error: could not find _core.cpython-*.so under $SITE_PACKAGES/headroom/" >&2
|
|
exit 1
|
|
fi
|
|
ln -sf "$SO_FILE" "headroom/$(basename "$SO_FILE")"
|
|
python -c "from headroom._core import SmartCrusher; print('headroom._core OK:', SmartCrusher)"
|
|
|
|
- name: Run CCR round-trip (zero cost)
|
|
run: |
|
|
python -c "
|
|
from headroom.evals.runners.compression_only import CompressionOnlyRunner
|
|
runner = CompressionOnlyRunner()
|
|
cases = runner.generate_ccr_test_cases(n=50)
|
|
result = runner.evaluate_ccr_lossless(cases)
|
|
print(f'CCR Round-trip: {result.passed_cases}/{result.total_cases} passed')
|
|
assert result.passed, f'CCR failures: {result.errors}'
|
|
"
|
|
- name: Run built-in tool output eval
|
|
run: python -m headroom.evals quick -n 8 --provider openai --model gpt-4o-mini
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
|
|
# Full Tier 1 suite, weekly or manual (~$3-5, ~30-45 min)
|
|
weekly-suite:
|
|
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 60
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.11"
|
|
- name: Install dependencies
|
|
run: pip install -e ".[all]"
|
|
- name: Run Tier 1 evaluation suite
|
|
run: python -m headroom.evals suite --tier 1 --ci -o eval_results/
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
- name: Upload results
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: eval-results-${{ github.run_number }}
|
|
path: eval_results/
|