From 8062e7f7f5b89128cef53136a5a8908d902822d7 Mon Sep 17 00:00:00 2001 From: Tejas Chopra Date: Thu, 4 Jun 2026 10:35:54 -0700 Subject: [PATCH 1/7] ci: add experimental intelligent+parallel pipeline (ci-fast.yml) Runs alongside ci.yml (does not touch required checks) so it can be validated and timed on a real PR before cutover. - changes: paths-filter skips everything for docs-only PRs - build-wheel: compile the Rust ext ONCE via maturin, share via artifact (today ci.yml rebuilds it ~7x across the matrix/extras/agno/build jobs) - lint: ruff + mypy once - test: 4 parallel shards via pytest-split, each a fresh runner VM so the suite's shared-state tests (repo-root db, port 8787) can't collide - CPU-only torch (drops the ~2.5GB CUDA stack) + cached HF model Verified locally: YAML valid; pytest-split partitions the suite cleanly. --- .github/workflows/ci-fast.yml | 155 ++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 .github/workflows/ci-fast.yml diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml new file mode 100644 index 000000000..32bf0071b --- /dev/null +++ b/.github/workflows/ci-fast.yml @@ -0,0 +1,155 @@ +name: CI Fast (experimental) + +# Intelligent + parallel CI, running ALONGSIDE the existing ci.yml so it can be +# validated/measured without touching required status checks. Once it's proven +# green + faster, fold it into ci.yml and update branch protection. +# +# Design: +# * changes — paths-filter: skip the heavy work for docs-only PRs. +# * build-wheel — compile the Rust extension ONCE (maturin), share via artifact. +# * lint — ruff + mypy, once (3.12). +# * test — 4 parallel shards (pytest-split), each a fresh runner VM so +# the suite's shared-state tests (repo-root db, port 8787) don't +# collide. CPU-only torch + cached HF model keep installs fast. + +on: + pull_request: + branches: [main] + workflow_dispatch: + +concurrency: + group: ci-fast-${{ github.ref }} + cancel-in-progress: true + +env: + PY_VERSION: "3.12" + # CPU-only torch index — CI runners have no GPU; the default CUDA wheels pull + # ~2.5 GB of nvidia libs we never use. + PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + code: ${{ steps.filter.outputs.code }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + code: + - 'headroom/**' + - 'crates/**' + - '**/*.rs' + - 'pyproject.toml' + - 'Cargo.toml' + - 'Cargo.lock' + - 'tests/**' + - 'scripts/tests/**' + - '.github/workflows/ci-fast.yml' + + build-wheel: + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PY_VERSION }} + - uses: dtolnay/rust-toolchain@1.95.0 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: ". -> target" + - name: Build wheel once (maturin) + run: | + python -m pip install --upgrade pip maturin + maturin build --release --out dist --interpreter "python${PY_VERSION}" + - uses: actions/upload-artifact@v4 + with: + name: headroom-wheel + path: dist/*.whl + retention-days: 1 + + lint: + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PY_VERSION }} + - name: Install ruff + mypy + run: python -m pip install --upgrade pip ruff mypy + - name: ruff check + run: ruff check . + - name: ruff format --check + run: ruff format --check . + - name: mypy + run: mypy headroom --ignore-missing-imports + + test: + needs: [changes, build-wheel] + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4] + env: + FASTEMBED_CACHE_PATH: ${{ github.workspace }}/.fastembed-cache + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PY_VERSION }} + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pipfast-${{ env.PY_VERSION }}-${{ hashFiles('pyproject.toml') }} + restore-keys: ${{ runner.os }}-pipfast-${{ env.PY_VERSION }}- + + - name: Cache HuggingFace + fastembed models + uses: actions/cache@v4 + with: + path: | + ~/.cache/huggingface + ${{ github.workspace }}/.fastembed-cache + key: ${{ runner.os }}-models-allMiniLM-v1 + + - name: Download prebuilt wheel + uses: actions/download-artifact@v4 + with: + name: headroom-wheel + path: dist + + - name: Install (CPU torch + prebuilt wheel + dev deps, no cargo rebuild) + run: | + python -m pip install --upgrade pip + # Install torch CPU explicitly first so the editable/extra resolve reuses it. + pip install torch --index-url https://download.pytorch.org/whl/cpu + WHEEL="$(ls dist/*.whl)" + pip install "${WHEEL}[dev]" + pip install pytest-split + python -c "from headroom._core import DiffCompressor; print('headroom._core OK')" + + - name: Pre-fetch embedding model (resilient to HF rate limits) + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HF_HUB_DISABLE_TELEMETRY: "1" + run: | + for i in 1 2 3 4 5; do + if python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2')"; then exit 0; fi + echo "::warning::model fetch attempt $i failed; backing off"; sleep $((i * 20)) + done + echo "::error::could not fetch all-MiniLM-L6-v2 from HuggingFace"; exit 1 + + - name: Run test shard ${{ matrix.shard }}/4 + run: | + pytest tests scripts/tests \ + --splits 4 --group ${{ matrix.shard }} \ + --tb=short -q From 322d02efad2700b7c6ef9df76da998b0d609e3f6 Mon Sep 17 00:00:00 2001 From: Tejas Chopra Date: Thu, 4 Jun 2026 10:52:51 -0700 Subject: [PATCH 2/7] ci: copy built _core.so into source tree so sharded tests import it The prebuilt wheel installs headroom into site-packages, but tests run from the repo root where ./headroom shadows it and has no compiled extension. Copy the built _core.*.so into the source tree (no second cargo build) so 'import headroom._core' resolves. --- .github/workflows/ci-fast.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 32bf0071b..ac80073fb 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -130,11 +130,17 @@ jobs: - name: Install (CPU torch + prebuilt wheel + dev deps, no cargo rebuild) run: | python -m pip install --upgrade pip - # Install torch CPU explicitly first so the editable/extra resolve reuses it. + # Install torch CPU explicitly first so the extra-resolve reuses it. pip install torch --index-url https://download.pytorch.org/whl/cpu WHEEL="$(ls dist/*.whl)" - pip install "${WHEEL}[dev]" - pip install pytest-split + pip install "${WHEEL}[dev]" pytest-split + # The tests run against the repo's ./headroom source tree (they import + # headroom AND read source files), and cwd shadows the installed wheel's + # package — so the compiled extension from site-packages is invisible. + # Copy the built _core.*.so into the source tree so imports resolve, + # without a second cargo build. + SITE="$(python -c 'import sysconfig; print(sysconfig.get_path("platlib"))')" + cp "${SITE}/headroom/"_core*.so headroom/ python -c "from headroom._core import DiffCompressor; print('headroom._core OK')" - name: Pre-fetch embedding model (resilient to HF rate limits) From 9a16a59df4321ca5522b928db50fb115a829491a Mon Sep 17 00:00:00 2001 From: Tejas Chopra Date: Thu, 4 Jun 2026 11:10:02 -0700 Subject: [PATCH 3/7] ci: prefetch HF model once + run shards offline (kill the 429 herd) Cold-cache run had all 4 shards download all-MiniLM in parallel -> HF 429'd a shard. Add a prefetch-model job that fetches it once (huggingface_hub, no torch) and warms the shared cache; shards then run with HF_HUB_OFFLINE=1 so they load from cache with zero HF network calls (the 429 was on a cache-validation HEAD). --- .github/workflows/ci-fast.yml | 56 ++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index ac80073fb..28c9307d5 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -90,16 +90,49 @@ jobs: - name: mypy run: mypy headroom --ignore-missing-imports + # Download the embedding model ONCE and warm the shared cache, so the 4 test + # shards don't all hammer HuggingFace in parallel (that 429'd a shard on a + # cold cache). Uses huggingface_hub directly — no torch needed just to fetch. + prefetch-model: + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PY_VERSION }} + - name: Cache HuggingFace model + id: hfcache + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface + key: ${{ runner.os }}-models-allMiniLM-v2 + - name: Fetch all-MiniLM-L6-v2 (resilient to HF rate limits) + if: steps.hfcache.outputs.cache-hit != 'true' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HF_HUB_DISABLE_TELEMETRY: "1" + run: | + python -m pip install --upgrade pip huggingface_hub + for i in 1 2 3 4 5 6; do + if python -c "from huggingface_hub import snapshot_download; snapshot_download('sentence-transformers/all-MiniLM-L6-v2')"; then exit 0; fi + echo "::warning::model fetch attempt $i failed; backing off"; sleep $((i * 30)) + done + echo "::error::could not fetch all-MiniLM-L6-v2 from HuggingFace"; exit 1 + test: - needs: [changes, build-wheel] + needs: [changes, build-wheel, prefetch-model] if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest strategy: fail-fast: false matrix: shard: [1, 2, 3, 4] + # Shards read the model purely from the cache warmed by prefetch-model — no + # HF network calls at all (so no rate-limit HEAD requests can fail them). env: - FASTEMBED_CACHE_PATH: ${{ github.workspace }}/.fastembed-cache + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -113,13 +146,11 @@ jobs: key: ${{ runner.os }}-pipfast-${{ env.PY_VERSION }}-${{ hashFiles('pyproject.toml') }} restore-keys: ${{ runner.os }}-pipfast-${{ env.PY_VERSION }}- - - name: Cache HuggingFace + fastembed models + - name: Restore HuggingFace model cache (warmed by prefetch-model) uses: actions/cache@v4 with: - path: | - ~/.cache/huggingface - ${{ github.workspace }}/.fastembed-cache - key: ${{ runner.os }}-models-allMiniLM-v1 + path: ~/.cache/huggingface + key: ${{ runner.os }}-models-allMiniLM-v2 - name: Download prebuilt wheel uses: actions/download-artifact@v4 @@ -143,17 +174,6 @@ jobs: cp "${SITE}/headroom/"_core*.so headroom/ python -c "from headroom._core import DiffCompressor; print('headroom._core OK')" - - name: Pre-fetch embedding model (resilient to HF rate limits) - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - HF_HUB_DISABLE_TELEMETRY: "1" - run: | - for i in 1 2 3 4 5; do - if python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2')"; then exit 0; fi - echo "::warning::model fetch attempt $i failed; backing off"; sleep $((i * 20)) - done - echo "::error::could not fetch all-MiniLM-L6-v2 from HuggingFace"; exit 1 - - name: Run test shard ${{ matrix.shard }}/4 run: | pytest tests scripts/tests \ From a08a0befa72e99a2ae60bacced68ac7a44af0fcf Mon Sep 17 00:00:00 2001 From: Tejas Chopra Date: Thu, 4 Jun 2026 11:38:30 -0700 Subject: [PATCH 4/7] ci: build the CI test wheel with a fast cargo profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release profile (lto=thin, codegen-units=1) is great for the shipped wheel but slow to compile — it was the build-wheel long pole (~3m38s) gating the test shards. Add [profile.ci] (no LTO, codegen-units=256, opt-level=1) and build the CI wheel with --profile ci. Does not affect --release / shipped wheels. --- .github/workflows/ci-fast.yml | 6 ++++-- Cargo.toml | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 28c9307d5..a5751fdf8 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -62,10 +62,12 @@ jobs: - uses: Swatinem/rust-cache@v2 with: workspaces: ". -> target" - - name: Build wheel once (maturin) + - name: Build wheel once (maturin, fast CI profile) run: | python -m pip install --upgrade pip maturin - maturin build --release --out dist --interpreter "python${PY_VERSION}" + # `--profile ci`: no LTO + parallel codegen-units → much faster compile + # than `--release` (lto + codegen-units=1). Tests don't need the optimized ext. + maturin build --profile ci --out dist --interpreter "python${PY_VERSION}" - uses: actions/upload-artifact@v4 with: name: headroom-wheel diff --git a/Cargo.toml b/Cargo.toml index c2c14f83a..4adfcbaa7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,3 +106,16 @@ gcp_auth = "0.12" strip = "symbols" lto = "thin" codegen-units = 1 + +# Fast-to-compile profile for CI test wheels. The shipped wheel uses +# `release` (lto + codegen-units=1) for runtime/size; CI only needs a working +# extension, so trade runtime perf for ~parallel, lto-free compilation. Used +# via `maturin build --profile ci`. Does NOT affect `--release` builds. +[profile.ci] +inherits = "release" +lto = false +codegen-units = 256 +opt-level = 1 +strip = "none" +debug = false +incremental = false From 6f0dc32680bd363e6969b8051f299a73a44104fc Mon Sep 17 00:00:00 2001 From: Tejas Chopra Date: Thu, 4 Jun 2026 11:42:03 -0700 Subject: [PATCH 5/7] ci: pin least-privilege GITHUB_TOKEN permissions (contents: read) Resolves the CodeQL 'workflow does not contain permissions' advisory. No job in this workflow writes contents/PRs/releases, so read-only is sufficient. --- .github/workflows/ci-fast.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index a5751fdf8..cbb5543db 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -17,6 +17,13 @@ on: branches: [main] workflow_dispatch: +# Least-privilege GITHUB_TOKEN: every job here only reads the repo (checkout, +# paths-filter, build, test). No job writes contents/PRs/releases, so read-only +# is sufficient and satisfies the CodeQL "workflow does not contain permissions" +# advisory. +permissions: + contents: read + concurrency: group: ci-fast-${{ github.ref }} cancel-in-progress: true From 51af24f7bf55b848db9ea359f2f8bab5750b0985 Mon Sep 17 00:00:00 2001 From: Tejas Chopra Date: Thu, 4 Jun 2026 12:14:36 -0700 Subject: [PATCH 6/7] ci: cut over to the intelligent+parallel pipeline in ci.yml Fold ci-fast.yml into ci.yml: change-detection (paths-filter), build the Rust ext once (fast cargo profile) shared via artifact, lint once, prefetch the embedding model once (authenticated) into a shared cache, and run the suite as 4 offline shards. Preserve commitlint, build smoke, workflow-validation, and the docker/windows/macos e2e jobs (heavy ones gated on paths-filter). CPU-only torch throughout; least-privilege permissions. Removes ci-fast.yml. PRs run one Python version x 4 shards; multi-version on main is a follow-up. --- .github/workflows/ci-fast.yml | 190 -------------- .github/workflows/ci.yml | 460 +++++++++++++++++++++------------- 2 files changed, 279 insertions(+), 371 deletions(-) delete mode 100644 .github/workflows/ci-fast.yml diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml deleted file mode 100644 index cbb5543db..000000000 --- a/.github/workflows/ci-fast.yml +++ /dev/null @@ -1,190 +0,0 @@ -name: CI Fast (experimental) - -# Intelligent + parallel CI, running ALONGSIDE the existing ci.yml so it can be -# validated/measured without touching required status checks. Once it's proven -# green + faster, fold it into ci.yml and update branch protection. -# -# Design: -# * changes — paths-filter: skip the heavy work for docs-only PRs. -# * build-wheel — compile the Rust extension ONCE (maturin), share via artifact. -# * lint — ruff + mypy, once (3.12). -# * test — 4 parallel shards (pytest-split), each a fresh runner VM so -# the suite's shared-state tests (repo-root db, port 8787) don't -# collide. CPU-only torch + cached HF model keep installs fast. - -on: - pull_request: - branches: [main] - workflow_dispatch: - -# Least-privilege GITHUB_TOKEN: every job here only reads the repo (checkout, -# paths-filter, build, test). No job writes contents/PRs/releases, so read-only -# is sufficient and satisfies the CodeQL "workflow does not contain permissions" -# advisory. -permissions: - contents: read - -concurrency: - group: ci-fast-${{ github.ref }} - cancel-in-progress: true - -env: - PY_VERSION: "3.12" - # CPU-only torch index — CI runners have no GPU; the default CUDA wheels pull - # ~2.5 GB of nvidia libs we never use. - PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu - -jobs: - changes: - runs-on: ubuntu-latest - outputs: - code: ${{ steps.filter.outputs.code }} - steps: - - uses: actions/checkout@v4 - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - code: - - 'headroom/**' - - 'crates/**' - - '**/*.rs' - - 'pyproject.toml' - - 'Cargo.toml' - - 'Cargo.lock' - - 'tests/**' - - 'scripts/tests/**' - - '.github/workflows/ci-fast.yml' - - build-wheel: - needs: changes - if: needs.changes.outputs.code == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PY_VERSION }} - - uses: dtolnay/rust-toolchain@1.95.0 - - uses: Swatinem/rust-cache@v2 - with: - workspaces: ". -> target" - - name: Build wheel once (maturin, fast CI profile) - run: | - python -m pip install --upgrade pip maturin - # `--profile ci`: no LTO + parallel codegen-units → much faster compile - # than `--release` (lto + codegen-units=1). Tests don't need the optimized ext. - maturin build --profile ci --out dist --interpreter "python${PY_VERSION}" - - uses: actions/upload-artifact@v4 - with: - name: headroom-wheel - path: dist/*.whl - retention-days: 1 - - lint: - needs: changes - if: needs.changes.outputs.code == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PY_VERSION }} - - name: Install ruff + mypy - run: python -m pip install --upgrade pip ruff mypy - - name: ruff check - run: ruff check . - - name: ruff format --check - run: ruff format --check . - - name: mypy - run: mypy headroom --ignore-missing-imports - - # Download the embedding model ONCE and warm the shared cache, so the 4 test - # shards don't all hammer HuggingFace in parallel (that 429'd a shard on a - # cold cache). Uses huggingface_hub directly — no torch needed just to fetch. - prefetch-model: - needs: changes - if: needs.changes.outputs.code == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PY_VERSION }} - - name: Cache HuggingFace model - id: hfcache - uses: actions/cache@v4 - with: - path: ~/.cache/huggingface - key: ${{ runner.os }}-models-allMiniLM-v2 - - name: Fetch all-MiniLM-L6-v2 (resilient to HF rate limits) - if: steps.hfcache.outputs.cache-hit != 'true' - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - HF_HUB_DISABLE_TELEMETRY: "1" - run: | - python -m pip install --upgrade pip huggingface_hub - for i in 1 2 3 4 5 6; do - if python -c "from huggingface_hub import snapshot_download; snapshot_download('sentence-transformers/all-MiniLM-L6-v2')"; then exit 0; fi - echo "::warning::model fetch attempt $i failed; backing off"; sleep $((i * 30)) - done - echo "::error::could not fetch all-MiniLM-L6-v2 from HuggingFace"; exit 1 - - test: - needs: [changes, build-wheel, prefetch-model] - if: needs.changes.outputs.code == 'true' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4] - # Shards read the model purely from the cache warmed by prefetch-model — no - # HF network calls at all (so no rate-limit HEAD requests can fail them). - env: - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PY_VERSION }} - - - name: Cache pip - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pipfast-${{ env.PY_VERSION }}-${{ hashFiles('pyproject.toml') }} - restore-keys: ${{ runner.os }}-pipfast-${{ env.PY_VERSION }}- - - - name: Restore HuggingFace model cache (warmed by prefetch-model) - uses: actions/cache@v4 - with: - path: ~/.cache/huggingface - key: ${{ runner.os }}-models-allMiniLM-v2 - - - name: Download prebuilt wheel - uses: actions/download-artifact@v4 - with: - name: headroom-wheel - path: dist - - - name: Install (CPU torch + prebuilt wheel + dev deps, no cargo rebuild) - run: | - python -m pip install --upgrade pip - # Install torch CPU explicitly first so the extra-resolve reuses it. - pip install torch --index-url https://download.pytorch.org/whl/cpu - WHEEL="$(ls dist/*.whl)" - pip install "${WHEEL}[dev]" pytest-split - # The tests run against the repo's ./headroom source tree (they import - # headroom AND read source files), and cwd shadows the installed wheel's - # package — so the compiled extension from site-packages is invisible. - # Copy the built _core.*.so into the source tree so imports resolve, - # without a second cargo build. - SITE="$(python -c 'import sysconfig; print(sysconfig.get_path("platlib"))')" - cp "${SITE}/headroom/"_core*.so headroom/ - python -c "from headroom._core import DiffCompressor; print('headroom._core OK')" - - - name: Run test shard ${{ matrix.shard }}/4 - run: | - pytest tests scripts/tests \ - --splits 4 --group ${{ matrix.shard }} \ - --tb=short -q diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46ba3b367..1cde45243 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,16 @@ name: CI +# Intelligent + parallel pipeline (cutover from the old 4-version matrix): +# changes — paths-filter; skips heavy work for docs-only changes +# build-wheel — compile the Rust ext ONCE (fast `ci` cargo profile), share via artifact +# lint — ruff + mypy, once +# prefetch-model — download the embedding model ONCE (authenticated), warm shared cache +# test — 4 parallel shards (pytest-split), each a fresh runner VM; run offline +# test-extras / test-agno / build / commitlint / workflow-validation / *-e2e — preserved +# +# Notes: CPU-only torch everywhere (no CUDA stack); test shards run HF_HUB_OFFLINE. +# Multi-version (3.10/3.11/3.13) coverage on main is a planned follow-up. + on: push: branches: [main] @@ -7,151 +18,317 @@ on: branches: [main] workflow_dispatch: +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + # Cancel superseded runs on PRs/branches, but never cancel a main build. + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + PY_VERSION: "3.12" + # CPU-only torch — runners have no GPU; the default CUDA wheels pull ~2.5 GB. + PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu + jobs: + changes: + runs-on: ubuntu-latest + outputs: + code: ${{ steps.filter.outputs.code }} + e2e: ${{ steps.filter.outputs.e2e }} + workflows: ${{ steps.filter.outputs.workflows }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + code: + - 'headroom/**' + - 'crates/**' + - '**/*.rs' + - 'pyproject.toml' + - 'Cargo.toml' + - 'Cargo.lock' + - 'tests/**' + - 'scripts/**' + - '.github/workflows/ci.yml' + e2e: + - 'headroom/**' + - 'crates/**' + - 'docker/**' + - 'Dockerfile' + - 'e2e/**' + - 'scripts/install*' + - 'pyproject.toml' + workflows: + - '.github/workflows/**' + + lint: + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PY_VERSION }} + - run: python -m pip install --upgrade pip ruff mypy + - name: ruff check + run: ruff check . + - name: ruff format --check + run: ruff format --check . + - name: mypy + run: mypy headroom --ignore-missing-imports + + build-wheel: + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PY_VERSION }} + - uses: dtolnay/rust-toolchain@1.95.0 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: ". -> target" + - name: Build wheel once (fast CI cargo profile) + run: | + python -m pip install --upgrade pip maturin + maturin build --profile ci --out dist --interpreter "python${PY_VERSION}" + - uses: actions/upload-artifact@v4 + with: + name: headroom-wheel + path: dist/*.whl + retention-days: 1 + + prefetch-model: + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PY_VERSION }} + - name: Cache HuggingFace model + id: hfcache + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface + key: ${{ runner.os }}-models-allMiniLM-v2 + - name: Fetch all-MiniLM-L6-v2 once (authenticated, resilient) + if: steps.hfcache.outputs.cache-hit != 'true' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HF_HUB_DISABLE_TELEMETRY: "1" + run: | + python -m pip install --upgrade pip huggingface_hub + for i in 1 2 3 4 5 6; do + if python -c "from huggingface_hub import snapshot_download; snapshot_download('sentence-transformers/all-MiniLM-L6-v2')"; then exit 0; fi + echo "::warning::model fetch attempt $i failed; backing off"; sleep $((i * 30)) + done + echo "::error::could not fetch all-MiniLM-L6-v2 from HuggingFace"; exit 1 + test: + needs: [changes, build-wheel, prefetch-model] + if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] - + shard: [1, 2, 3, 4] + env: + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" steps: - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ env.PY_VERSION }} - # `pip install -e .` invokes maturin (declared in `[build-system] - # requires`) under the hood, which calls cargo to build the Rust - # extension. The toolchain has to be available before the install - # step, otherwise build-isolation pulls maturin but `cargo` is - # missing. - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@1.95.0 - - - name: Cache cargo registry + build - uses: Swatinem/rust-cache@v2 - with: - workspaces: ". -> target" - - - name: Cache pip packages + - name: Cache pip uses: actions/cache@v4 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip-${{ matrix.python-version }}- + key: ${{ runner.os }}-pip-${{ env.PY_VERSION }}-${{ hashFiles('pyproject.toml') }} + restore-keys: ${{ runner.os }}-pip-${{ env.PY_VERSION }}- - - name: Install dependencies (builds Rust extension via maturin) + - name: Restore HuggingFace model cache (warmed by prefetch-model) + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface + key: ${{ runner.os }}-models-allMiniLM-v2 + + - name: Download prebuilt wheel + uses: actions/download-artifact@v4 + with: + name: headroom-wheel + path: dist + + - name: Install (CPU torch + prebuilt wheel + dev deps, no cargo rebuild) run: | python -m pip install --upgrade pip - pip install -e ".[dev]" - python -c "from headroom._core import DiffCompressor; print('headroom._core OK:', DiffCompressor)" + pip install torch --index-url https://download.pytorch.org/whl/cpu + WHEEL="$(ls dist/*.whl)" + pip install "${WHEEL}[dev]" pytest-split + # cwd's ./headroom source tree shadows the installed wheel; copy the + # compiled extension in so tests import it (no second cargo build). + SITE="$(python -c 'import sysconfig; print(sysconfig.get_path("platlib"))')" + cp "${SITE}/headroom/"_core*.so headroom/ + python -c "from headroom._core import DiffCompressor; print('headroom._core OK')" - - name: Run linting - if: matrix.python-version == '3.12' + - name: Run test shard ${{ matrix.shard }}/4 run: | - ruff check . - ruff format --check . - - - name: Run type checking - if: matrix.python-version == '3.12' - run: | - mypy headroom --ignore-missing-imports - - - name: Run tests - run: | - pytest -v --tb=short tests scripts/tests - - - name: Run tests with coverage - if: matrix.python-version == '3.11' - run: | - pytest tests scripts/tests --cov=headroom --cov-report=xml --cov-report=term-missing - - - name: Upload coverage to Codecov - if: matrix.python-version == '3.11' - uses: codecov/codecov-action@v4 - with: - file: ./coverage.xml - fail_ci_if_error: false + pytest tests scripts/tests \ + --splits 4 --group ${{ matrix.shard }} \ + --tb=short -q test-extras: + needs: [changes, build-wheel] + if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest + env: + FASTEMBED_CACHE_PATH: ${{ github.workspace }}/.fastembed-cache steps: - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: - python-version: "3.11" - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@1.95.0 - - - name: Cache cargo registry + build - uses: Swatinem/rust-cache@v2 + python-version: ${{ env.PY_VERSION }} + - name: Cache pip + uses: actions/cache@v4 with: - workspaces: ". -> target" - - - name: Install with relevance extras (builds Rust extension via maturin) + path: ~/.cache/pip + key: ${{ runner.os }}-pip-extras-${{ hashFiles('pyproject.toml') }} + restore-keys: ${{ runner.os }}-pip-extras- + - name: Cache fastembed model + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.fastembed-cache + key: ${{ runner.os }}-fastembed-bge-small-v1 + - name: Download prebuilt wheel + uses: actions/download-artifact@v4 + with: + name: headroom-wheel + path: dist + - name: Install (CPU torch + wheel[dev,relevance]) run: | python -m pip install --upgrade pip - pip install -e ".[dev,relevance]" - python -c "from headroom._core import SmartCrusher; print('headroom._core OK:', SmartCrusher)" - - - name: Run relevance tests + pip install torch --index-url https://download.pytorch.org/whl/cpu + WHEEL="$(ls dist/*.whl)" + pip install "${WHEEL}[dev,relevance]" + SITE="$(python -c 'import sysconfig; print(sysconfig.get_path("platlib"))')" + cp "${SITE}/headroom/"_core*.so headroom/ + python -c "from headroom._core import SmartCrusher; print('headroom._core OK')" + - name: Pre-fetch fastembed model (authenticated, resilient) + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HF_HUB_DISABLE_TELEMETRY: "1" run: | - pytest tests/test_relevance.py -v + for i in 1 2 3 4 5; do + if python -c "from fastembed import TextEmbedding; TextEmbedding('BAAI/bge-small-en-v1.5')"; then exit 0; fi + echo "::warning::fastembed fetch attempt $i failed; backing off"; sleep $((i * 20)) + done + echo "::error::could not fetch fastembed model from HuggingFace"; exit 1 + - name: Run relevance tests + run: pytest tests/test_relevance.py -v test-agno: + needs: [changes, build-wheel] + if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: - python-version: "3.11" - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@1.95.0 - - - name: Cache cargo registry + build - uses: Swatinem/rust-cache@v2 + python-version: ${{ env.PY_VERSION }} + - name: Download prebuilt wheel + uses: actions/download-artifact@v4 with: - workspaces: ". -> target" - - - name: Install with agno extras (builds Rust extension via maturin) + name: headroom-wheel + path: dist + - name: Install (CPU torch + wheel[dev,agno]) run: | python -m pip install --upgrade pip - pip install -e ".[dev,agno]" - + pip install torch --index-url https://download.pytorch.org/whl/cpu + WHEEL="$(ls dist/*.whl)" + pip install "${WHEEL}[dev,agno]" + SITE="$(python -c 'import sysconfig; print(sysconfig.get_path("platlib"))')" + cp "${SITE}/headroom/"_core*.so headroom/ - name: Run agno tests - run: | - pytest tests/test_integrations/agno/ -v + run: pytest tests/test_integrations/agno/ -v - docker-native-e2e: + commitlint: + if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'Merge pull request ') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: wagoid/commitlint-github-action@v5 + with: + configFile: .commitlintrc.json - - name: Set up Python - uses: actions/setup-python@v5 + build: + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" - - - name: Build local Headroom image + - uses: dtolnay/rust-toolchain@1.95.0 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: ". -> target" + # Smoke check that the SHIPPED build (release profile) + sdist are wired + # right; release.yml's matrix is what actually publishes to PyPI. + - name: Install build tools run: | - docker build -t headroom-native-e2e:latest . + python -m pip install --upgrade pip + pip install 'maturin>=1.5,<2.0' twine + - name: Build wheel + sdist + run: | + maturin sdist --out dist + maturin build --release --out dist + - name: Check package + run: twine check dist/* + workflow-validation: + needs: changes + if: needs.changes.outputs.workflows == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install actionlint + run: | + curl -fsSL https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash | bash + sudo mv ./actionlint /usr/local/bin/actionlint + - name: Install act + run: | + curl -fsSL https://raw.githubusercontent.com/nektos/act/master/install.sh | sudo bash + sudo install ./bin/act /usr/local/bin/act + - name: Validate workflow files + run: bash scripts/validate-workflows.sh + + docker-native-e2e: + needs: changes + if: needs.changes.outputs.e2e == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Build local Headroom image + run: docker build -t headroom-native-e2e:latest . - name: Run Docker-native installer e2e env: HEADROOM_DOCKER_IMAGE: headroom-native-e2e:latest - run: | - bash e2e/docker-native-install.sh - + run: bash e2e/docker-native-install.sh - name: Run Docker-native compose smoke test env: HEADROOM_IMAGE: headroom-native-e2e:latest @@ -171,126 +348,47 @@ jobs: fi sleep 1 done - - name: Run Docker-native wrap e2e run: | docker build -f e2e/wrap/Dockerfile -t headroom-wrap-e2e . docker run --rm headroom-wrap-e2e - - name: Run Docker-native init e2e run: | docker build -f e2e/init/Dockerfile -t headroom-init-e2e . docker run --rm headroom-init-e2e windows-native-wrapper: + needs: changes + if: needs.changes.outputs.e2e == 'true' runs-on: windows-latest steps: - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install test dependencies run: | python -m pip install --upgrade pip pip install pytest - - name: Run native installer wrapper tests - run: | - pytest tests/test_install/test_native_installers.py -q + run: pytest tests/test_install/test_native_installers.py -q macos-native-wrapper: + needs: changes + if: needs.changes.outputs.e2e == 'true' runs-on: macos-latest steps: - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: "3.11" - - name: Install bash and test dependencies run: | brew install bash python -m pip install --upgrade pip python -m pip install --retries 10 --timeout 60 pytest - - name: Run native installer wrapper tests run: | BASH_PREFIX="$(brew --prefix bash)" export PATH="$BASH_PREFIX/bin:$PATH" pytest tests/test_install/test_native_installers.py -q - - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@1.95.0 - - - name: Cache cargo registry + build - uses: Swatinem/rust-cache@v2 - with: - workspaces: ". -> target" - - # Single-wheel build via maturin: produces both the linux wheel and - # the platform-independent sdist in one shot. release.yml's matrix - # is what builds per-platform wheels for PyPI; this `build` job is - # a smoke check that the build system is wired right. - - name: Install build tools - run: | - python -m pip install --upgrade pip - pip install 'maturin>=1.5,<2.0' twine - - - name: Build wheel + sdist - run: | - maturin sdist --out dist - maturin build --release --out dist - - - name: Check package - run: | - twine check dist/* - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: dist - path: dist/ - - commitlint: - if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'Merge pull request ') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: wagoid/commitlint-github-action@v5 - with: - configFile: .commitlintrc.json - - workflow-validation: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install actionlint - run: | - curl -fsSL https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash | bash - sudo mv ./actionlint /usr/local/bin/actionlint - - - name: Install act - run: | - curl -fsSL https://raw.githubusercontent.com/nektos/act/master/install.sh | sudo bash - sudo install ./bin/act /usr/local/bin/act - - - name: Validate workflow files - run: | - bash scripts/validate-workflows.sh From 2ea548a86dc23536c0c6c954e46cf53eafd02ce1 Mon Sep 17 00:00:00 2001 From: Tejas Chopra Date: Thu, 4 Jun 2026 12:16:04 -0700 Subject: [PATCH 7/7] ci: run relevance tests offline so fastembed doesn't 429 on cache HEAD --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cde45243..a2767c556 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -233,6 +233,11 @@ jobs: done echo "::error::could not fetch fastembed model from HuggingFace"; exit 1 - name: Run relevance tests + # Offline so fastembed reads the cache the prefetch step just warmed, + # without an unauthenticated cache-validation HEAD that could 429. + env: + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" run: pytest tests/test_relevance.py -v test-agno: