mirror of
https://github.com/Quad4-Software/MeshChatX.git
synced 2026-08-18 09:49:09 -04:00
feat: add package bloat verification scripts and integrate into CI workflows
This commit is contained in:
Ivan
parent
7a02eae651
commit
61c8ca9ba8
6 changed files with 339 additions and 0 deletions
8
.github/workflows/ci.yml
vendored
8
.github/workflows/ci.yml
vendored
|
|
@ -147,6 +147,9 @@ jobs:
|
|||
- name: Checkout
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
|
||||
- name: Smoke-test package bloat checker
|
||||
run: bash scripts/ci/verify-package-contents-smoke.sh
|
||||
|
||||
- name: Set up development environment
|
||||
uses: ./.github/actions/setup-dev-environment
|
||||
with:
|
||||
|
|
@ -169,6 +172,9 @@ jobs:
|
|||
- name: Verify bleak bundled in frozen backend
|
||||
run: bash scripts/ci/github-verify-frozen-bleak.sh build/exe
|
||||
|
||||
- name: Verify frozen backend has no package bloat
|
||||
run: bash scripts/ci/verify-package-contents.sh frozen build/exe
|
||||
|
||||
- name: Verify workspace clean
|
||||
uses: ./.github/actions/verify-workspace-clean
|
||||
|
||||
|
|
@ -208,6 +214,8 @@ jobs:
|
|||
test -n "$(ls -A .artifacts/linux-build-check/build/exe)"
|
||||
bash scripts/ci/github-verify-frozen-bleak.sh \
|
||||
.artifacts/linux-build-check/build/exe
|
||||
bash scripts/ci/verify-package-contents.sh frozen \
|
||||
.artifacts/linux-build-check/build/exe
|
||||
echo "Linux build artifact download + content validation passed."
|
||||
|
||||
native-build:
|
||||
|
|
|
|||
7
.github/workflows/docker.yml
vendored
7
.github/workflows/docker.yml
vendored
|
|
@ -219,6 +219,13 @@ jobs:
|
|||
OCI_VERSION=${{ steps.oci.outputs.version }}
|
||||
OCI_CREATED=${{ steps.oci.outputs.created }}
|
||||
|
||||
- name: Verify image has no package bloat
|
||||
run: |
|
||||
set -euo pipefail
|
||||
img="${{ steps.image.outputs.name }}@${{ steps.build.outputs.digest }}"
|
||||
docker pull --platform linux/amd64 "$img"
|
||||
bash scripts/ci/verify-package-contents.sh docker "$img"
|
||||
|
||||
- name: Install Cosign
|
||||
uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5
|
||||
|
||||
|
|
|
|||
20
Taskfile.yml
20
Taskfile.yml
|
|
@ -637,6 +637,26 @@ tasks:
|
|||
cmds:
|
||||
- sh scripts/ci/verify-workspace-clean.sh "${RNS_INVENTORY_OUT:-/tmp/meshchatx-tree-inventory.txt}"
|
||||
|
||||
verify:package:frozen:
|
||||
desc: Fail if frozen backend contains forbidden bloat paths
|
||||
cmds:
|
||||
- bash scripts/ci/verify-package-contents.sh frozen "{{.CLI_ARGS}}"
|
||||
|
||||
verify:package:wheel:
|
||||
desc: Fail if Python wheel contains forbidden bloat paths
|
||||
cmds:
|
||||
- bash scripts/ci/verify-package-contents.sh wheel "{{.CLI_ARGS}}"
|
||||
|
||||
verify:package:docker:
|
||||
desc: Fail if Docker image contains forbidden bloat paths (pass image as CLI_ARGS)
|
||||
cmds:
|
||||
- bash scripts/ci/verify-package-contents.sh docker "{{.CLI_ARGS}}"
|
||||
|
||||
verify:package:smoke:
|
||||
desc: Self-test the package bloat checker with temp trees
|
||||
cmds:
|
||||
- bash scripts/ci/verify-package-contents-smoke.sh
|
||||
|
||||
# --- Maintenance ---
|
||||
|
||||
dist:
|
||||
|
|
|
|||
BIN
meshchatx.rsm
BIN
meshchatx.rsm
Binary file not shown.
38
scripts/ci/verify-package-contents-smoke.sh
Executable file
38
scripts/ci/verify-package-contents-smoke.sh
Executable file
|
|
@ -0,0 +1,38 @@
|
|||
#!/usr/bin/env bash
|
||||
# Smoke tests for verify-package-contents.sh (no full freeze required).
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)"
|
||||
SCRIPT="$ROOT/scripts/ci/verify-package-contents.sh"
|
||||
tmp="$(mktemp -d "${TMPDIR:-/tmp}/pkg-bloat-test.XXXXXX")"
|
||||
trap 'rm -rf "$tmp"' EXIT INT
|
||||
|
||||
mkdir -p "$tmp/clean/lib/meshchatx/src/backend"
|
||||
echo ok >"$tmp/clean/lib/meshchatx/src/backend/x.py"
|
||||
|
||||
mkdir -p "$tmp/dirty/lib/meshchatx/src/frontend/components"
|
||||
mkdir -p "$tmp/dirty/lib/numpy/tests"
|
||||
mkdir -p "$tmp/dirty/node_modules/left-pad"
|
||||
echo vue >"$tmp/dirty/lib/meshchatx/src/frontend/components/App.vue"
|
||||
echo junk >"$tmp/dirty/lib/numpy/tests/test_x.py"
|
||||
echo junk >"$tmp/dirty/node_modules/left-pad/index.js"
|
||||
|
||||
echo "expect clean dir to pass"
|
||||
bash "$SCRIPT" dir "$tmp/clean"
|
||||
|
||||
echo "expect clean frozen tree to pass"
|
||||
bash "$SCRIPT" frozen "$tmp/clean"
|
||||
|
||||
echo "expect dirty frozen tree to fail"
|
||||
if bash "$SCRIPT" frozen "$tmp/dirty"; then
|
||||
echo "expected failure for dirty frozen tree" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "expect dirty dir (node_modules) to fail"
|
||||
if bash "$SCRIPT" dir "$tmp/dirty"; then
|
||||
echo "expected failure for dirty dir" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "verify-package-contents smoke OK"
|
||||
266
scripts/ci/verify-package-contents.sh
Executable file
266
scripts/ci/verify-package-contents.sh
Executable file
|
|
@ -0,0 +1,266 @@
|
|||
#!/usr/bin/env bash
|
||||
# Fail if release artifacts contain paths that should not ship
|
||||
# (dev trees, duplicate frontend sources, vendor offline caches, etc.).
|
||||
#
|
||||
# Usage:
|
||||
# verify-package-contents.sh frozen [build/exe]
|
||||
# verify-package-contents.sh docker [image:tag]
|
||||
# verify-package-contents.sh wheel [path/to.whl]
|
||||
# verify-package-contents.sh dir [path]
|
||||
# verify-package-contents.sh appimage [path/to.AppImage]
|
||||
# verify-package-contents.sh deb [path/to.deb]
|
||||
# verify-package-contents.sh apk [path/to.apk]
|
||||
#
|
||||
# Env:
|
||||
# PACKAGE_BLOAT_MAX_HITS stop after N hits (default 40)
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)"
|
||||
MODE="${1:-}"
|
||||
TARGET="${2:-}"
|
||||
MAX_HITS="${PACKAGE_BLOAT_MAX_HITS:-40}"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF' >&2
|
||||
Usage:
|
||||
verify-package-contents.sh frozen [build/exe]
|
||||
verify-package-contents.sh docker [image:tag]
|
||||
verify-package-contents.sh wheel [file.whl]
|
||||
verify-package-contents.sh dir [path]
|
||||
verify-package-contents.sh appimage [file.AppImage]
|
||||
verify-package-contents.sh deb [file.deb]
|
||||
verify-package-contents.sh apk [file.apk]
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
[ -n "$MODE" ] || usage
|
||||
|
||||
hits=0
|
||||
hit_lines=()
|
||||
|
||||
record_hit() {
|
||||
hits=$((hits + 1))
|
||||
hit_lines+=("$1")
|
||||
if [ "$hits" -ge "$MAX_HITS" ]; then
|
||||
echo "verify-package-contents.sh: hit cap ($MAX_HITS) reached" >&2
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Shared denylist patterns (grep -E against relative paths).
|
||||
COMMON_DENY_RE='(^|/)\.git(/|$)|(^|/)node_modules(/|$)|(^|/)\.pnpm-store(/|$)|(^|/)\.venv(/|$)|(^|/)vendor/offline(/|$)|(^|/)vendor/lxmfy/tests(/|$)|(^|/)vendor/lxmfy/docs(/|$)|(^|/)vendor/lxmfy/docker(/|$)|(^|/)\.github(/|$)|(^|/)docs/agents(/|$)|(^|/)screenshots(/|$)|(^|/)__pycache__(/|$)|(^|/)\.pytest_cache(/|$)|(^|/)mutants(/|$)|(^|/)coverage(/|$)'
|
||||
|
||||
FROZEN_DENY_RE="${COMMON_DENY_RE}|(^|/)lib/meshchatx/public(/|$)|(^|/)lib/meshchatx/src/frontend/.+\.vue$|(^|/)lib/meshchatx/src/frontend/.+\.css$|(^|/)lib/setuptools(/|$)|(^|/)lib/pydoc_data(/|$)|(^|/)lib/numpy/.*/tests(/|$)|(^|/)lib/numpy/tests(/|$)"
|
||||
|
||||
DOCKER_DENY_RE="${COMMON_DENY_RE}|(^|/)meshchatx/src/frontend/.+\.vue$|(^|/)meshchatx/src/frontend/.+\.css$|(^|/)tests(/|$)|(^|/)electron(/|$)|(^|/)android(/|$)"
|
||||
|
||||
WHEEL_DENY_RE="${COMMON_DENY_RE}|(^|/)meshchatx/src/frontend/.+\.vue$|(^|/)meshchatx/src/frontend/.+\.css$|(^|/)tests(/|$)"
|
||||
|
||||
APK_DENY_RE="${COMMON_DENY_RE}|(^|/)tests(/|$)|(^|/)electron(/|$)|(^|/)\.github(/|$)"
|
||||
|
||||
# Read relative paths from stdin. Must not run in a pipe subshell so hits persist.
|
||||
scan_path_list() {
|
||||
deny_re="$1"
|
||||
while IFS= read -r rel || [ -n "${rel:-}" ]; do
|
||||
[ -n "$rel" ] || continue
|
||||
rel="${rel#./}"
|
||||
case "$rel" in
|
||||
"") continue ;;
|
||||
esac
|
||||
if printf '%s\n' "$rel" | grep -Eq "$deny_re"; then
|
||||
if ! record_hit "$rel"; then
|
||||
break
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
resolve_frozen_root() {
|
||||
base="${1:-}"
|
||||
if [ -z "$base" ]; then
|
||||
base="$ROOT/build/exe"
|
||||
fi
|
||||
if [ ! -d "$base" ]; then
|
||||
echo "verify-package-contents.sh: frozen root missing: $base" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -d "$base/lib" ]; then
|
||||
printf '%s\n' "$base"
|
||||
return 0
|
||||
fi
|
||||
for sub in "$base"/*; do
|
||||
if [ -d "$sub/lib" ]; then
|
||||
printf '%s\n' "$sub"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
echo "verify-package-contents.sh: no lib/ under $base" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
scan_directory_tree() {
|
||||
root="$1"
|
||||
deny_re="$2"
|
||||
# Process substitution keeps scan_path_list in this shell (hits accumulate).
|
||||
scan_path_list "$deny_re" < <(
|
||||
CDPATH= cd -- "$root" || exit 1
|
||||
find . -print 2>/dev/null | sed 's|^\./||'
|
||||
)
|
||||
}
|
||||
|
||||
scan_archive_listing() {
|
||||
archive="$1"
|
||||
deny_re="$2"
|
||||
if command -v unzip >/dev/null 2>&1; then
|
||||
scan_path_list "$deny_re" < <(unzip -Z1 "$archive" 2>/dev/null)
|
||||
return 0
|
||||
fi
|
||||
echo "verify-package-contents.sh: unzip required to inspect $archive" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
scan_frozen() {
|
||||
root="$(resolve_frozen_root "${TARGET:-}")"
|
||||
echo "verify-package-contents.sh: scanning frozen tree $root"
|
||||
scan_directory_tree "$root" "$FROZEN_DENY_RE"
|
||||
}
|
||||
|
||||
scan_dir() {
|
||||
root="${TARGET:-}"
|
||||
[ -n "$root" ] || usage
|
||||
[ -d "$root" ] || {
|
||||
echo "verify-package-contents.sh: missing dir $root" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "verify-package-contents.sh: scanning dir $root"
|
||||
scan_directory_tree "$root" "$COMMON_DENY_RE"
|
||||
}
|
||||
|
||||
scan_wheel() {
|
||||
whl="${TARGET:-}"
|
||||
if [ -z "$whl" ]; then
|
||||
whl="$(ls -1 "$ROOT"/python-dist/*.whl 2>/dev/null | head -n 1 || true)"
|
||||
fi
|
||||
[ -n "$whl" ] && [ -f "$whl" ] || {
|
||||
echo "verify-package-contents.sh: wheel not found" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "verify-package-contents.sh: scanning wheel $whl"
|
||||
scan_archive_listing "$whl" "$WHEEL_DENY_RE"
|
||||
}
|
||||
|
||||
scan_apk() {
|
||||
apk="${TARGET:-}"
|
||||
[ -n "$apk" ] && [ -f "$apk" ] || {
|
||||
echo "verify-package-contents.sh: apk not found" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "verify-package-contents.sh: scanning apk $apk"
|
||||
scan_archive_listing "$apk" "$APK_DENY_RE"
|
||||
}
|
||||
|
||||
scan_deb() {
|
||||
deb="${TARGET:-}"
|
||||
[ -n "$deb" ] && [ -f "$deb" ] || {
|
||||
echo "verify-package-contents.sh: deb not found" >&2
|
||||
exit 1
|
||||
}
|
||||
tmp="$(mktemp -d "${TMPDIR:-/tmp}/pkg-bloat-deb.XXXXXX")"
|
||||
trap 'rm -rf "$tmp"' EXIT INT
|
||||
echo "verify-package-contents.sh: extracting deb $deb"
|
||||
dpkg-deb -x "$deb" "$tmp"
|
||||
scan_directory_tree "$tmp" "$FROZEN_DENY_RE"
|
||||
}
|
||||
|
||||
scan_appimage() {
|
||||
ai="${TARGET:-}"
|
||||
[ -n "$ai" ] && [ -f "$ai" ] || {
|
||||
echo "verify-package-contents.sh: AppImage not found" >&2
|
||||
exit 1
|
||||
}
|
||||
tmp="$(mktemp -d "${TMPDIR:-/tmp}/pkg-bloat-ai.XXXXXX")"
|
||||
trap 'rm -rf "$tmp"' EXIT INT
|
||||
echo "verify-package-contents.sh: extracting AppImage $ai"
|
||||
chmod +x "$ai" || true
|
||||
(
|
||||
CDPATH= cd -- "$tmp"
|
||||
"$ai" --appimage-extract >/dev/null
|
||||
)
|
||||
scan_directory_tree "$tmp/squashfs-root" "$FROZEN_DENY_RE"
|
||||
}
|
||||
|
||||
scan_docker() {
|
||||
image="${TARGET:-}"
|
||||
[ -n "$image" ] || {
|
||||
echo "verify-package-contents.sh: docker image tag required" >&2
|
||||
exit 1
|
||||
}
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "verify-package-contents.sh: docker not available" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "verify-package-contents.sh: scanning docker image $image (/opt/venv)"
|
||||
# Use python (always present) so Alpine and Chainguard/hardened images both work.
|
||||
list="$(
|
||||
docker run --rm --user 0 --entrypoint python "$image" -c '
|
||||
import os
|
||||
roots = [p for p in ("/opt/venv", "/app") if os.path.isdir(p)]
|
||||
if not roots:
|
||||
roots = ["/"]
|
||||
skip = {"/proc", "/sys", "/dev", "/tmp", "/run"}
|
||||
for root in roots:
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
if root == "/":
|
||||
dirnames[:] = [d for d in dirnames if os.path.join(dirpath, d) not in skip]
|
||||
print(dirpath)
|
||||
for name in filenames:
|
||||
print(os.path.join(dirpath, name))
|
||||
' 2>/dev/null || true
|
||||
)"
|
||||
if [ -z "$list" ]; then
|
||||
echo "verify-package-contents.sh: could not list files in $image" >&2
|
||||
exit 1
|
||||
fi
|
||||
scan_path_list "$DOCKER_DENY_RE" < <(
|
||||
printf '%s\n' "$list" | sed 's|^/opt/venv/||;s|^/app/||;s|^/||'
|
||||
)
|
||||
}
|
||||
|
||||
case "$MODE" in
|
||||
frozen)
|
||||
scan_frozen
|
||||
;;
|
||||
dir)
|
||||
scan_dir
|
||||
;;
|
||||
wheel)
|
||||
scan_wheel
|
||||
;;
|
||||
apk)
|
||||
scan_apk
|
||||
;;
|
||||
deb)
|
||||
scan_deb
|
||||
;;
|
||||
appimage)
|
||||
scan_appimage
|
||||
;;
|
||||
docker)
|
||||
scan_docker
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "$hits" -gt 0 ]; then
|
||||
echo "verify-package-contents.sh: FAIL ($hits forbidden path(s))" >&2
|
||||
for line in "${hit_lines[@]}"; do
|
||||
echo " - $line" >&2
|
||||
done
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "verify-package-contents.sh: OK ($MODE)"
|
||||
Loading…
Add table
Add a link
Reference in a new issue