diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5236106f..dd1c1c53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,6 +147,9 @@ jobs: - name: Checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 + - name: Smoke-test package bloat checker + run: bash scripts/ci/verify-package-contents-smoke.sh + - name: Set up development environment uses: ./.github/actions/setup-dev-environment with: @@ -169,6 +172,9 @@ jobs: - name: Verify bleak bundled in frozen backend run: bash scripts/ci/github-verify-frozen-bleak.sh build/exe + - name: Verify frozen backend has no package bloat + run: bash scripts/ci/verify-package-contents.sh frozen build/exe + - name: Verify workspace clean uses: ./.github/actions/verify-workspace-clean @@ -208,6 +214,8 @@ jobs: test -n "$(ls -A .artifacts/linux-build-check/build/exe)" bash scripts/ci/github-verify-frozen-bleak.sh \ .artifacts/linux-build-check/build/exe + bash scripts/ci/verify-package-contents.sh frozen \ + .artifacts/linux-build-check/build/exe echo "Linux build artifact download + content validation passed." native-build: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index dff1f790..4152b159 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -219,6 +219,13 @@ jobs: OCI_VERSION=${{ steps.oci.outputs.version }} OCI_CREATED=${{ steps.oci.outputs.created }} + - name: Verify image has no package bloat + run: | + set -euo pipefail + img="${{ steps.image.outputs.name }}@${{ steps.build.outputs.digest }}" + docker pull --platform linux/amd64 "$img" + bash scripts/ci/verify-package-contents.sh docker "$img" + - name: Install Cosign uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 diff --git a/Taskfile.yml b/Taskfile.yml index 2c685248..6808a4ae 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -637,6 +637,26 @@ tasks: cmds: - sh scripts/ci/verify-workspace-clean.sh "${RNS_INVENTORY_OUT:-/tmp/meshchatx-tree-inventory.txt}" + verify:package:frozen: + desc: Fail if frozen backend contains forbidden bloat paths + cmds: + - bash scripts/ci/verify-package-contents.sh frozen "{{.CLI_ARGS}}" + + verify:package:wheel: + desc: Fail if Python wheel contains forbidden bloat paths + cmds: + - bash scripts/ci/verify-package-contents.sh wheel "{{.CLI_ARGS}}" + + verify:package:docker: + desc: Fail if Docker image contains forbidden bloat paths (pass image as CLI_ARGS) + cmds: + - bash scripts/ci/verify-package-contents.sh docker "{{.CLI_ARGS}}" + + verify:package:smoke: + desc: Self-test the package bloat checker with temp trees + cmds: + - bash scripts/ci/verify-package-contents-smoke.sh + # --- Maintenance --- dist: diff --git a/meshchatx.rsm b/meshchatx.rsm index 63d9c449..21dc6b40 100644 Binary files a/meshchatx.rsm and b/meshchatx.rsm differ diff --git a/scripts/ci/verify-package-contents-smoke.sh b/scripts/ci/verify-package-contents-smoke.sh new file mode 100755 index 00000000..ef10c03e --- /dev/null +++ b/scripts/ci/verify-package-contents-smoke.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Smoke tests for verify-package-contents.sh (no full freeze required). +set -euo pipefail + +ROOT="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)" +SCRIPT="$ROOT/scripts/ci/verify-package-contents.sh" +tmp="$(mktemp -d "${TMPDIR:-/tmp}/pkg-bloat-test.XXXXXX")" +trap 'rm -rf "$tmp"' EXIT INT + +mkdir -p "$tmp/clean/lib/meshchatx/src/backend" +echo ok >"$tmp/clean/lib/meshchatx/src/backend/x.py" + +mkdir -p "$tmp/dirty/lib/meshchatx/src/frontend/components" +mkdir -p "$tmp/dirty/lib/numpy/tests" +mkdir -p "$tmp/dirty/node_modules/left-pad" +echo vue >"$tmp/dirty/lib/meshchatx/src/frontend/components/App.vue" +echo junk >"$tmp/dirty/lib/numpy/tests/test_x.py" +echo junk >"$tmp/dirty/node_modules/left-pad/index.js" + +echo "expect clean dir to pass" +bash "$SCRIPT" dir "$tmp/clean" + +echo "expect clean frozen tree to pass" +bash "$SCRIPT" frozen "$tmp/clean" + +echo "expect dirty frozen tree to fail" +if bash "$SCRIPT" frozen "$tmp/dirty"; then + echo "expected failure for dirty frozen tree" >&2 + exit 1 +fi + +echo "expect dirty dir (node_modules) to fail" +if bash "$SCRIPT" dir "$tmp/dirty"; then + echo "expected failure for dirty dir" >&2 + exit 1 +fi + +echo "verify-package-contents smoke OK" diff --git a/scripts/ci/verify-package-contents.sh b/scripts/ci/verify-package-contents.sh new file mode 100755 index 00000000..fbd277b1 --- /dev/null +++ b/scripts/ci/verify-package-contents.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash +# Fail if release artifacts contain paths that should not ship +# (dev trees, duplicate frontend sources, vendor offline caches, etc.). +# +# Usage: +# verify-package-contents.sh frozen [build/exe] +# verify-package-contents.sh docker [image:tag] +# verify-package-contents.sh wheel [path/to.whl] +# verify-package-contents.sh dir [path] +# verify-package-contents.sh appimage [path/to.AppImage] +# verify-package-contents.sh deb [path/to.deb] +# verify-package-contents.sh apk [path/to.apk] +# +# Env: +# PACKAGE_BLOAT_MAX_HITS stop after N hits (default 40) +set -euo pipefail + +ROOT="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)" +MODE="${1:-}" +TARGET="${2:-}" +MAX_HITS="${PACKAGE_BLOAT_MAX_HITS:-40}" + +usage() { + cat <<'EOF' >&2 +Usage: + verify-package-contents.sh frozen [build/exe] + verify-package-contents.sh docker [image:tag] + verify-package-contents.sh wheel [file.whl] + verify-package-contents.sh dir [path] + verify-package-contents.sh appimage [file.AppImage] + verify-package-contents.sh deb [file.deb] + verify-package-contents.sh apk [file.apk] +EOF + exit 2 +} + +[ -n "$MODE" ] || usage + +hits=0 +hit_lines=() + +record_hit() { + hits=$((hits + 1)) + hit_lines+=("$1") + if [ "$hits" -ge "$MAX_HITS" ]; then + echo "verify-package-contents.sh: hit cap ($MAX_HITS) reached" >&2 + return 1 + fi + return 0 +} + +# Shared denylist patterns (grep -E against relative paths). +COMMON_DENY_RE='(^|/)\.git(/|$)|(^|/)node_modules(/|$)|(^|/)\.pnpm-store(/|$)|(^|/)\.venv(/|$)|(^|/)vendor/offline(/|$)|(^|/)vendor/lxmfy/tests(/|$)|(^|/)vendor/lxmfy/docs(/|$)|(^|/)vendor/lxmfy/docker(/|$)|(^|/)\.github(/|$)|(^|/)docs/agents(/|$)|(^|/)screenshots(/|$)|(^|/)__pycache__(/|$)|(^|/)\.pytest_cache(/|$)|(^|/)mutants(/|$)|(^|/)coverage(/|$)' + +FROZEN_DENY_RE="${COMMON_DENY_RE}|(^|/)lib/meshchatx/public(/|$)|(^|/)lib/meshchatx/src/frontend/.+\.vue$|(^|/)lib/meshchatx/src/frontend/.+\.css$|(^|/)lib/setuptools(/|$)|(^|/)lib/pydoc_data(/|$)|(^|/)lib/numpy/.*/tests(/|$)|(^|/)lib/numpy/tests(/|$)" + +DOCKER_DENY_RE="${COMMON_DENY_RE}|(^|/)meshchatx/src/frontend/.+\.vue$|(^|/)meshchatx/src/frontend/.+\.css$|(^|/)tests(/|$)|(^|/)electron(/|$)|(^|/)android(/|$)" + +WHEEL_DENY_RE="${COMMON_DENY_RE}|(^|/)meshchatx/src/frontend/.+\.vue$|(^|/)meshchatx/src/frontend/.+\.css$|(^|/)tests(/|$)" + +APK_DENY_RE="${COMMON_DENY_RE}|(^|/)tests(/|$)|(^|/)electron(/|$)|(^|/)\.github(/|$)" + +# Read relative paths from stdin. Must not run in a pipe subshell so hits persist. +scan_path_list() { + deny_re="$1" + while IFS= read -r rel || [ -n "${rel:-}" ]; do + [ -n "$rel" ] || continue + rel="${rel#./}" + case "$rel" in + "") continue ;; + esac + if printf '%s\n' "$rel" | grep -Eq "$deny_re"; then + if ! record_hit "$rel"; then + break + fi + fi + done +} + +resolve_frozen_root() { + base="${1:-}" + if [ -z "$base" ]; then + base="$ROOT/build/exe" + fi + if [ ! -d "$base" ]; then + echo "verify-package-contents.sh: frozen root missing: $base" >&2 + exit 1 + fi + if [ -d "$base/lib" ]; then + printf '%s\n' "$base" + return 0 + fi + for sub in "$base"/*; do + if [ -d "$sub/lib" ]; then + printf '%s\n' "$sub" + return 0 + fi + done + echo "verify-package-contents.sh: no lib/ under $base" >&2 + exit 1 +} + +scan_directory_tree() { + root="$1" + deny_re="$2" + # Process substitution keeps scan_path_list in this shell (hits accumulate). + scan_path_list "$deny_re" < <( + CDPATH= cd -- "$root" || exit 1 + find . -print 2>/dev/null | sed 's|^\./||' + ) +} + +scan_archive_listing() { + archive="$1" + deny_re="$2" + if command -v unzip >/dev/null 2>&1; then + scan_path_list "$deny_re" < <(unzip -Z1 "$archive" 2>/dev/null) + return 0 + fi + echo "verify-package-contents.sh: unzip required to inspect $archive" >&2 + exit 1 +} + +scan_frozen() { + root="$(resolve_frozen_root "${TARGET:-}")" + echo "verify-package-contents.sh: scanning frozen tree $root" + scan_directory_tree "$root" "$FROZEN_DENY_RE" +} + +scan_dir() { + root="${TARGET:-}" + [ -n "$root" ] || usage + [ -d "$root" ] || { + echo "verify-package-contents.sh: missing dir $root" >&2 + exit 1 + } + echo "verify-package-contents.sh: scanning dir $root" + scan_directory_tree "$root" "$COMMON_DENY_RE" +} + +scan_wheel() { + whl="${TARGET:-}" + if [ -z "$whl" ]; then + whl="$(ls -1 "$ROOT"/python-dist/*.whl 2>/dev/null | head -n 1 || true)" + fi + [ -n "$whl" ] && [ -f "$whl" ] || { + echo "verify-package-contents.sh: wheel not found" >&2 + exit 1 + } + echo "verify-package-contents.sh: scanning wheel $whl" + scan_archive_listing "$whl" "$WHEEL_DENY_RE" +} + +scan_apk() { + apk="${TARGET:-}" + [ -n "$apk" ] && [ -f "$apk" ] || { + echo "verify-package-contents.sh: apk not found" >&2 + exit 1 + } + echo "verify-package-contents.sh: scanning apk $apk" + scan_archive_listing "$apk" "$APK_DENY_RE" +} + +scan_deb() { + deb="${TARGET:-}" + [ -n "$deb" ] && [ -f "$deb" ] || { + echo "verify-package-contents.sh: deb not found" >&2 + exit 1 + } + tmp="$(mktemp -d "${TMPDIR:-/tmp}/pkg-bloat-deb.XXXXXX")" + trap 'rm -rf "$tmp"' EXIT INT + echo "verify-package-contents.sh: extracting deb $deb" + dpkg-deb -x "$deb" "$tmp" + scan_directory_tree "$tmp" "$FROZEN_DENY_RE" +} + +scan_appimage() { + ai="${TARGET:-}" + [ -n "$ai" ] && [ -f "$ai" ] || { + echo "verify-package-contents.sh: AppImage not found" >&2 + exit 1 + } + tmp="$(mktemp -d "${TMPDIR:-/tmp}/pkg-bloat-ai.XXXXXX")" + trap 'rm -rf "$tmp"' EXIT INT + echo "verify-package-contents.sh: extracting AppImage $ai" + chmod +x "$ai" || true + ( + CDPATH= cd -- "$tmp" + "$ai" --appimage-extract >/dev/null + ) + scan_directory_tree "$tmp/squashfs-root" "$FROZEN_DENY_RE" +} + +scan_docker() { + image="${TARGET:-}" + [ -n "$image" ] || { + echo "verify-package-contents.sh: docker image tag required" >&2 + exit 1 + } + if ! command -v docker >/dev/null 2>&1; then + echo "verify-package-contents.sh: docker not available" >&2 + exit 1 + fi + echo "verify-package-contents.sh: scanning docker image $image (/opt/venv)" + # Use python (always present) so Alpine and Chainguard/hardened images both work. + list="$( + docker run --rm --user 0 --entrypoint python "$image" -c ' +import os +roots = [p for p in ("/opt/venv", "/app") if os.path.isdir(p)] +if not roots: + roots = ["/"] +skip = {"/proc", "/sys", "/dev", "/tmp", "/run"} +for root in roots: + for dirpath, dirnames, filenames in os.walk(root): + if root == "/": + dirnames[:] = [d for d in dirnames if os.path.join(dirpath, d) not in skip] + print(dirpath) + for name in filenames: + print(os.path.join(dirpath, name)) +' 2>/dev/null || true + )" + if [ -z "$list" ]; then + echo "verify-package-contents.sh: could not list files in $image" >&2 + exit 1 + fi + scan_path_list "$DOCKER_DENY_RE" < <( + printf '%s\n' "$list" | sed 's|^/opt/venv/||;s|^/app/||;s|^/||' + ) +} + +case "$MODE" in +frozen) + scan_frozen + ;; +dir) + scan_dir + ;; +wheel) + scan_wheel + ;; +apk) + scan_apk + ;; +deb) + scan_deb + ;; +appimage) + scan_appimage + ;; +docker) + scan_docker + ;; +*) + usage + ;; +esac + +if [ "$hits" -gt 0 ]; then + echo "verify-package-contents.sh: FAIL ($hits forbidden path(s))" >&2 + for line in "${hit_lines[@]}"; do + echo " - $line" >&2 + done + exit 1 +fi + +echo "verify-package-contents.sh: OK ($MODE)"