From 39a075918a64d9bc07944b15f7bcf5dd09904cc2 Mon Sep 17 00:00:00 2001 From: maziggy Date: Sun, 24 May 2026 13:08:43 +0200 Subject: [PATCH] ci(docker): drop -v, -n auto instead of -n 30, pip cache mount MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three things were making the Docker test runs noisier and slower than they needed to be: 1. -v was hardcoded in Dockerfile.test:35 CMD and in docker-compose. test.yml's integration-test-runner command. The ci.yml change to drop -v from the bare pytest call missed both — Docker runs use the image's CMD, not the workflow's. 2. -n 30 was hardcoded as the xdist worker count. On a 2-vCPU CI box that's 30 Python processes fighting over 2 cores — mostly IPC and import-thrash overhead. -n auto adapts to the host: 2 on CI, 30 on a 30-core dev box. Same final-result throughput on the dev box, much better on small runners. 3. pip install had --no-cache-dir and no BuildKit cache mount, so every Docker build re-fetched ~50 packages from PyPI (~60-90s on a cold pip cache). Adding `RUN --mount=type=cache,target= /root/.cache/pip` (with the `# syntax=docker/dockerfile:1.7` directive that enables it) makes subsequent builds re-use the download cache so they only do install work, ~5s instead of ~90s. DOCKER_BUILDKIT=1 is already exported in test_docker.sh and is the GHA default since runner image 2023, so the cache mount is always honoured. Verified locally: Docker build is 19s warm (was ~90s cold each time), test run is 102s with 5287 passed / 1 skipped (the by-design spoolbuddy importorskip) — clean output, no [gwN] worker spam, no "created: 30/30 workers" startup line. GHA-side per-run cold-build slowness still happens because GHA runners are ephemeral; a follow-up using docker/build-push-action with type=gha cache backend would persist the BuildKit cache across CI runs but that's a bigger workflow change. --- Dockerfile.test | 20 ++++++++++++++++---- docker-compose.test.yml | 4 +++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Dockerfile.test b/Dockerfile.test index 4a77a00db..2ff586185 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,4 +1,5 @@ # Test image for running backend and frontend tests +# syntax=docker/dockerfile:1.7 FROM python:3.13-slim AS backend-test WORKDIR /app @@ -8,10 +9,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ && rm -rf /var/lib/apt/lists/* -# Install Python dependencies including test dependencies +# Install Python dependencies including test dependencies. +# BuildKit cache mount makes subsequent builds re-use the pip download +# cache so the second build only does install work — the slow ~60-90s +# fetch step from the first build becomes ~5s. Requires DOCKER_BUILDKIT=1 +# (already set in test_docker.sh). COPY requirements.txt ./ COPY requirements-dev.txt ./ -RUN pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install -r requirements.txt -r requirements-dev.txt # Copy backend code COPY backend/ ./backend/ @@ -31,8 +37,14 @@ ENV PYTHONUNBUFFERED=1 ENV DATA_DIR=/app/data ENV TESTING=1 -# Default command runs pytest (excluding docker integration tests) -CMD ["pytest", "backend/tests/", "-v", "--tb=short", "-p", "no:cacheprovider", "-n", "30"] +# Default command runs pytest (excluding docker integration tests). +# -v dropped: 5300+ "PASSED foo::bar" lines per worker eat noticeable +# stdout I/O time and clutter test_docker.sh output. --tb=short still +# gives full failure tracebacks when something breaks. +# -n auto adapts to the host's vCPU count instead of hard-coding 30 — +# on a 2-vCPU CI / VM runner, -n 30 spawns 30 Python processes fighting +# for 2 cores, which is mostly IPC + import-thrash overhead. +CMD ["pytest", "backend/tests/", "--tb=short", "-p", "no:cacheprovider", "-n", "auto"] # ------------------------------------------- # Frontend test stage diff --git a/docker-compose.test.yml b/docker-compose.test.yml index d5c8cf520..cf52e33ef 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -56,7 +56,9 @@ services: environment: - BAMBUDDY_TEST_URL=http://integration:8000 - TESTING=1 - command: ["pytest", "backend/tests/integration/", "-v", "--tb=short", "-p", "no:cacheprovider", "-n", "30"] + # -v dropped + -n auto so integration tests inherit the same noise / + # parallelism profile as the backend-test image (see Dockerfile.test CMD). + command: ["pytest", "backend/tests/integration/", "--tb=short", "-p", "no:cacheprovider", "-n", "auto"] volumes: - ./backend:/app/backend:ro