diff --git a/.agents/skills/manage-ci/SKILL.md b/.agents/skills/manage-ci/SKILL.md index 057175ddc..2a2b928d2 100644 --- a/.agents/skills/manage-ci/SKILL.md +++ b/.agents/skills/manage-ci/SKILL.md @@ -303,6 +303,11 @@ update the skill resources in the same change. image. CUDA/ROCm/Vulkan device absence is not a skip condition for `--version`, runtime discovery/listing, or client startup. Hardware-qualified serving tests are additional coverage. +- The hermetic readiness smoke starts its client as a noninteractive background + service. Use bounded SIGTERM shutdown on Unix and CTRL_BREAK_EVENT on Windows; + do not use Unix SIGINT for this service probe because asynchronous + noninteractive shell children may inherit it as ignored. Keep interactive + Ctrl-C behavior covered by runtime and console tests instead. - Namespace cache keys and include every compatibility boundary that can make reuse unsafe: OS, architecture, backend/toolchain, relevant lockfiles, `.github/cache-version.txt`, and build inputs. Do not broaden restore keys @@ -324,6 +329,13 @@ update the skill resources in the same change. `mozilla-actions/sccache-action` users through `.github/actions/configure-sccache-gha`; do not let a reusable workflow silently restore read-write PR publication. +- Keep GitHub-hosted main `rust_crate_tests` shards on writable job-local + sccache. Their distinct bulk Cargo target caches own cross-run reuse; four + concurrent per-object GHA writers caused repository-wide write contention + without improving the two worst shards. Do not extend this opt-out to + producer or grouped-test jobs without measured evidence. The configure + action evaluates an explicitly authorized Depot WebDAV cache before the GHA + opt-out, so a future trusted Depot rollout may still use `disk,webdav`. - Depot's GitHub cache namespace is repository-scoped and has no branch isolation. With automatic Depot Cache enabled, its authority is injected into the whole runner job and cannot be contained by sccache disk-only mode or diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index b0b3b11df..4f7e78ce4 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -48,7 +48,9 @@ The host producer attests the binary, writes `host-imports.json`, and publishes its checksum. Product consumers verify that immutable input before composition; they must not re-stamp or otherwise mutate the host per backend alias. Release CPU and backend product consumers also perform a noninteractive JSON client -readiness smoke from the verified host/runtime inputs before publication; CI +readiness smoke from the verified host/runtime inputs before publication. The +background client must exit cleanly after bounded SIGTERM on Unix or +CTRL_BREAK_EVENT on Windows; interactive Ctrl-C coverage remains separate. CI and packaging consumers must not rebuild either input. Portable bundles place runtimes at `mesh-bundle/native-runtimes/`; Debian/Arch packages use `/usr/local/lib/mesh-llm//native-runtimes`; Homebrew uses @@ -361,7 +363,12 @@ sccache server a credential-free environment and uses job-local disk only. GitHub-hosted trusted jobs retain `disk,gha` or explicit disk-only mode. PR events use job-local disk only, including direct sccache-action users routed through the configure action, while trusted main, release, warmer, and dispatch -paths may seed the GHA tier. Swift restores a +paths may seed the GHA tier. The high-fanout main `rust_crate_tests` matrix is +explicitly disk-only on GitHub-hosted runners because its four concurrent +per-object writers caused 94% of cold-control GHA write errors; its distinct +bulk Cargo target caches retain cross-run reuse. Other producer and grouped-test +jobs remain remote-enabled. An explicitly authorized Depot call selects +`disk,webdav` before that GHA opt-out. Swift restores a mode-independent Rust dependency cache that only trusted main pushes save. Persistent Cargo target and ABI reuse remains owned by `Swatinem/rust-cache` and `actions/cache`. Current PR jobs use the normal diff --git a/.github/actions/configure-sccache-gha/action.yml b/.github/actions/configure-sccache-gha/action.yml index b4efa29e7..b6612a277 100644 --- a/.github/actions/configure-sccache-gha/action.yml +++ b/.github/actions/configure-sccache-gha/action.yml @@ -179,15 +179,17 @@ runs: // Honor an explicit job-level opt-out of the GitHub Actions cache // backend. Long-running lanes (CUDA/ROCm/Vulkan release builds that - // exceed the ephemeral Actions cache token lifetime) set - // SCCACHE_GHA_ENABLED=false. Keep those lanes disk-only: an sccache - // server started after the token expires probes the gha tier at - // startup, and that storage read fails permanently (HTTP 400), - // aborting the build before any fail-open policy applies. Clearing the - // cache URL/token for the server process as well prevents any - // residual gha configuration from re-enabling the remote tier. Do - // not clear them job-wide: later actions/cache consumers still need - // their credentials. + // exceed the ephemeral Actions cache token lifetime) and high-fanout + // GitHub-hosted compiler shards set SCCACHE_GHA_ENABLED=false. For + // long-running lanes, an sccache server started after the token + // expires probes the gha tier at startup and that storage read fails + // permanently (HTTP 400), aborting the build before any fail-open + // policy applies. High-fanout shards avoid a repository-wide storm of + // duplicate per-object writes while their bulk target caches retain + // persistent reuse. Clearing the cache URL/token for the server + // process also prevents residual gha configuration from re-enabling + // the remote tier. Do not clear them job-wide: later actions/cache + // consumers still need their credentials. if ((process.env.SCCACHE_GHA_ENABLED || '').toLowerCase() === 'false') { core.info( 'SCCACHE_GHA_ENABLED=false is set; using baked sccache with disk-only cache.', diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d705fb2a9..2377cb6ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -412,6 +412,11 @@ jobs: LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static MESH_LLM_SKIP_UI: "1" RUSTFLAGS: "-C link-arg=-fuse-ld=lld" + # Four concurrent crate shards caused 94% of cold-main GHA compiler-cache + # write errors. Their per-shard Cargo target caches own persistent reuse; + # keep sccache writable and job-local on GitHub-hosted runners instead. + # Authorized Depot jobs still select disk,webdav before this GHA opt-out. + SCCACHE_GHA_ENABLED: "false" RUSTC_WRAPPER: sccache steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 diff --git a/ci/ci.md b/ci/ci.md index f8d6ae490..98194114e 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -505,7 +505,12 @@ to misses, cache write failures only warn, and a failed remote probe restarts `sccache` with disk-only storage. PR crate-test shards restore the existing `main-rust-crate-tests-` Cargo target caches read-only (`save-if: false`), so trusted main owns the cache while PRs avoid recompiling -the same workspace graph. +the same workspace graph. GitHub-hosted main crate-test shards also use +writable job-local sccache because four concurrent remote per-object writers +caused 94% of cold-control GHA write errors; their distinct bulk Cargo target +caches own persistent reuse. Other trusted producers and grouped tests retain +remote sccache. An explicitly authorized Depot call selects `disk,webdav` +before this GHA-only opt-out. Native ABI cache keys and llama build stamps share one resolved toolchain epoch. Digest-pinned Linux jobs use the immutable runner-image digest. Hosted macOS diff --git a/docs/design/TESTING.md b/docs/design/TESTING.md index 7519d1a8f..7affe6812 100644 --- a/docs/design/TESTING.md +++ b/docs/design/TESTING.md @@ -521,7 +521,9 @@ just build `./target/debug/mesh-llm --log-format json runtime list`; CI also starts the composed client noninteractively, observes either the JSON `Client ready` message or the structured `passive_mode`/`status=ready`/`role=client` event, - and requires a clean SIGINT shutdown. + and requires bounded graceful SIGTERM shutdown on Unix or CTRL_BREAK_EVENT + on Windows. Interactive Ctrl-C/SIGINT behavior is tested separately because + noninteractive background shell children may inherit SIGINT as ignored. ## Mesh Identity @@ -1186,7 +1188,8 @@ Required evidence: - A product with an incompatible MeshLLM version or Skippy ABI is rejected before loading. - `client --auto --log-format json --no-console` starts without a native - runtime or GPU driver, emits a real ready event, and stops cleanly on SIGINT. + runtime or GPU driver, emits a real ready event, and the noninteractive + service probe stops cleanly on SIGTERM (Unix) or CTRL_BREAK_EVENT (Windows). Run the command-surface smoke on every product platform without device passthrough. Separately qualify backend loading and a minimal operation on diff --git a/scripts/ci-client-readiness-smoke.sh b/scripts/ci-client-readiness-smoke.sh index f402b411e..4fed0862b 100755 --- a/scripts/ci-client-readiness-smoke.sh +++ b/scripts/ci-client-readiness-smoke.sh @@ -46,7 +46,11 @@ shutdown_client_unix() { local shutdown_timed_out="$STATE_DIR/shutdown-timed-out" rm -f "$shutdown_done" "$shutdown_timed_out" - kill -INT "$pid" 2>/dev/null || true + # This client is an asynchronous child of a noninteractive shell. POSIX + # shells may start such children with SIGINT ignored, so SIGINT is not a + # reliable graceful-shutdown probe here. SIGTERM is handled by mesh-llm's + # normal shutdown path and is the platform-appropriate CI service signal. + kill -TERM "$pid" 2>/dev/null || true ( for ((attempt = 0; attempt < SHUTDOWN_MAX_WAIT; attempt++)); do sleep 1 @@ -54,9 +58,7 @@ shutdown_client_unix() { done if [[ ! -e "$shutdown_done" ]] && kill -0 "$pid" 2>/dev/null; then : >"$shutdown_timed_out" - kill -TERM "$pid" 2>/dev/null || true - sleep 1 - [[ -e "$shutdown_done" ]] || kill -KILL "$pid" 2>/dev/null || true + kill -KILL "$pid" 2>/dev/null || true fi ) /dev/null 2>&1 & deadline_pid=$! @@ -71,11 +73,11 @@ shutdown_client_unix() { wait "$deadline_pid" 2>/dev/null || true if [[ -e "$shutdown_timed_out" ]]; then - echo "client did not stop cleanly after SIGINT within ${SHUTDOWN_MAX_WAIT}s" >&2 + echo "client did not stop cleanly after SIGTERM within ${SHUTDOWN_MAX_WAIT}s" >&2 return 1 fi if [[ "$child_status" -ne 0 ]]; then - echo "client exited non-cleanly after SIGINT: $child_status" >&2 + echo "client exited non-cleanly after SIGTERM: $child_status" >&2 return 1 fi return 0 diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index b19198149..09dec47c5 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -1765,7 +1765,7 @@ class CiArtifactActionTests(unittest.TestCase): main = ( ROOT / ".github" / "workflows" / "ci.yml" ).read_text(encoding="utf-8") - self.assertNotIn('SCCACHE_GHA_ENABLED: "false"', main) + self.assertEqual(main.count('SCCACHE_GHA_ENABLED: "false"'), 1) quality = ( ROOT / ".github" / "workflows" / "pr_quality.yml" diff --git a/scripts/tests/test_ci_client_readiness_smoke.py b/scripts/tests/test_ci_client_readiness_smoke.py index ee867d188..e58a7a7c3 100644 --- a/scripts/tests/test_ci_client_readiness_smoke.py +++ b/scripts/tests/test_ci_client_readiness_smoke.py @@ -23,7 +23,8 @@ with open(marker, "w", encoding="utf-8") as fh: fh.write(f"start:{{os.getpid()}}\\n") def stop(signum, frame): - del signum, frame + del frame + signal_name = signal.Signals(signum).name {handler} signal.signal(signal.SIGINT, stop) @@ -83,7 +84,7 @@ class CiClientReadinessSmokeTests(unittest.TestCase): with self.assertRaises(ProcessLookupError): os.kill(pid, 0) - def test_clean_sigint_shutdown_succeeds_and_reaps_runtime(self): + def test_clean_sigterm_shutdown_succeeds_and_reaps_runtime(self): with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) runtime = root / "mesh-llm" @@ -91,7 +92,7 @@ class CiClientReadinessSmokeTests(unittest.TestCase): runtime, """\ with open(marker, "a", encoding="utf-8") as fh: - fh.write(f"signal:{os.getpid()}\\n") + fh.write(f"signal:{signal_name}:{os.getpid()}\\n") raise SystemExit(0) """, ) @@ -101,10 +102,14 @@ class CiClientReadinessSmokeTests(unittest.TestCase): self.assertEqual(result.returncode, 0, result.stdout + result.stderr) marker = root / "events" pids = [ - int(line.split(":")[1]) + int(line.rsplit(":", 1)[1]) for line in marker.read_text(encoding="utf-8").splitlines() ] self.assertEqual(pids, [pids[0], pids[0]]) + self.assertIn( + f"signal:SIGTERM:{pids[0]}", + marker.read_text(encoding="utf-8").splitlines(), + ) self.assert_process_absent(marker) self.assertEqual(list((root / "state").iterdir()), []) @@ -118,7 +123,7 @@ class CiClientReadinessSmokeTests(unittest.TestCase): self.assertNotEqual(result.returncode, 0) self.assertIn( - "client did not stop cleanly after SIGINT within 1s", result.stderr + "client did not stop cleanly after SIGTERM within 1s", result.stderr ) self.assert_process_absent(root / "events") self.assertEqual(list((root / "state").iterdir()), []) diff --git a/scripts/tests/test_ci_workflow_artifacts.py b/scripts/tests/test_ci_workflow_artifacts.py index 84d768536..910383ad9 100644 --- a/scripts/tests/test_ci_workflow_artifacts.py +++ b/scripts/tests/test_ci_workflow_artifacts.py @@ -225,6 +225,22 @@ class CiWorkflowArtifactTests(unittest.TestCase): self.assertNotIn("run: scripts/build-llama.sh", consumer) self.assertNotIn("Cache patched llama.cpp ABI build", consumer) + def test_main_crate_shards_avoid_shared_gha_write_contention(self) -> None: + crate_tests = job_section(self.workflow, "rust_crate_tests") + grouped_tests = job_section(self.workflow, "linux_test_groups") + + self.assertIn('SCCACHE_GHA_ENABLED: "false"', crate_tests) + self.assertIn( + "shared-key: main-rust-crate-tests-${{ matrix.batch.idx }}", + crate_tests, + ) + self.assertIn("uses: ./.github/actions/configure-sccache-gha", crate_tests) + self.assertNotIn('SCCACHE_GHA_ENABLED: "false"', grouped_tests) + self.assertEqual( + self.workflow.count('SCCACHE_GHA_ENABLED: "false"'), + 1, + ) + def test_macos_host_and_runtime_are_independent_producers(self) -> None: route = ( "if: ${{ needs.changes.outputs."