headroom/.github/workflows/eval.yml

name: Evaluation Suite

on:
  schedule:
    - cron: '0 6 * * 1'  # Weekly on Monday 6am UTC
  workflow_dispatch:       # Manual trigger
  pull_request:
    paths:
      - 'headroom/transforms/**'
      - 'headroom/evals/**'
      - 'headroom/compress.py'

jobs:
  # Fast smoke test on PRs touching compression code (~$0.05, ~2 min)
  smoke-test:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install dependencies
        run: pip install -e ".[all]"

      # `compression_only.evaluate_ccr_lossless` constructs a SmartCrusher
      # which now hard-imports `headroom._core` (Stage 3c.1b). Build the
      # Rust extension before running the smoke test or every call raises
      # `ModuleNotFoundError`. Mirrors the main CI `test` job pattern.
      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable

      - name: Cache cargo registry + build
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: ". -> target"

      - name: Install maturin
        run: pip install 'maturin>=1.5,<2.0'

      - name: Build Rust extension (headroom._core)
        run: |
          set -euo pipefail
          maturin build --release -m crates/headroom-py/Cargo.toml --out dist
          pip install --force-reinstall --no-deps dist/headroom_core_py-*.whl
          SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])")
          SO_FILE=$(find "$SITE_PACKAGES/headroom" -maxdepth 1 -name "_core.cpython-*.so" -print -quit 2>/dev/null)
          if [[ -z "$SO_FILE" ]]; then
            echo "error: could not find _core.cpython-*.so under $SITE_PACKAGES/headroom/" >&2
            exit 1
          fi
          ln -sf "$SO_FILE" "headroom/$(basename "$SO_FILE")"
          python -c "from headroom._core import SmartCrusher; print('headroom._core OK:', SmartCrusher)"

      - name: Run CCR round-trip (zero cost)
        run: |
          python -c "
          from headroom.evals.runners.compression_only import CompressionOnlyRunner
          runner = CompressionOnlyRunner()
          cases = runner.generate_ccr_test_cases(n=50)
          result = runner.evaluate_ccr_lossless(cases)
          print(f'CCR Round-trip: {result.passed_cases}/{result.total_cases} passed')
          assert result.passed, f'CCR failures: {result.errors}'
          "
      - name: Run built-in tool output eval
        run: python -m headroom.evals quick -n 8 --provider openai --model gpt-4o-mini
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

  # Full Tier 1 suite, weekly or manual (~$3-5, ~30-45 min)
  weekly-suite:
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install dependencies
        run: pip install -e ".[all]"
      - name: Run Tier 1 evaluation suite
        run: python -m headroom.evals suite --tier 1 --ci -o eval_results/
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: eval-results-${{ github.run_number }}
          path: eval_results/