mirror of
https://github.com/brazilofmux/tinymux
synced 2026-08-13 00:23:11 -04:00
c3973f1642
With ctime() covered, FnCoverage still reported 10 uncovered builtins -- the SQL/recordset family. They are not untested. They do not exist: SQL, MAPSQL #if defined(INLINESQL) RSERROR RSNEXT RSPREV RSREC #if defined(STUB_SLAVE) RSRECNEXT RSRECPREV RSRELEASE RSROWS Neither macro is defined in the configuration the corpus runs in -- CLAUDE.md has --enable-stubslave as a release-build flag, omitted for smoke -- and calling one answers "#-1 FUNCTION (SQL) NOT FOUND". Measured, not inferred: all ten return that under muxscript. The tool greps the table with a flat regex and has no notion of #if, so it was reporting work no test could do. That is the wrong direction for a coverage number to be wrong in: it invents a backlog, and #1380 has been carrying these as "needs live session / modules / side effects" when the first problem is that they are compiled out. Guards are now resolved against what this build actually defines, from the two places the macros come from -- -D flags in the engine Makefile (REALITY_LVLS, WOD_REALMS, TINYMUX_JIT) and autoconf's header. Generic rather than a STUB_SLAVE special case, because there were already two different guards and hardcoding one would only move the error. before: 503 builtins, 493 referenced (98%), 10 with no reference after: 493 builtins, 493 referenced (100%), 0 with no reference, 10 excluded as absent from this build Excluded rather than dropped, and listed with their guard under --list: "cannot be covered in this build" is a different fact from "nobody wrote a test", and a reader chasing #1160 needs to tell them apart. The single-macro, non-nested assumption is verified against the table as it stands and written into the comment, because a compound guard would be read as its first macro and fail silently. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
162 lines
6.1 KiB
Bash
Executable file
162 lines
6.1 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
#
|
|
# FnCoverage — which registered builtins have no reference in the corpus.
|
|
#
|
|
# #1160 measured 196 uncovered builtins. A day later it was 144, because
|
|
# several test files landed in between. A number that moves that fast is
|
|
# only useful if it is cheap to re-measure, so this does the measuring
|
|
# rather than leaving it in an issue body to go stale.
|
|
#
|
|
# "Covered" here means the name appears in call position — NAME( — in some
|
|
# testcases/*.mux. That is a weak notion deliberately: it answers "does
|
|
# anything reference this at all", which is the question #1160 asks. It
|
|
# says nothing about whether the reference asserts anything, so treat it as
|
|
# a floor, not a measure of test quality.
|
|
#
|
|
# Usage:
|
|
# ./tools/FnCoverage summary only
|
|
# ./tools/FnCoverage --list summary plus the uncovered names
|
|
# ./tools/FnCoverage --tier1 exit 1 if any Tier 1 name is uncovered
|
|
#
|
|
# Exit: 0 normally; 1 if --tier1 is given and a Tier 1 name is uncovered.
|
|
|
|
set -u
|
|
|
|
HERE=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
|
TESTCASES=$(CDPATH= cd -- "$HERE/.." && pwd)
|
|
REPO=$(CDPATH= cd -- "$TESTCASES/.." && pwd)
|
|
FUNCTIONS="$REPO/mux/modules/engine/functions.cpp"
|
|
|
|
MODE=${1:-}
|
|
|
|
if [ ! -f "$FUNCTIONS" ]; then
|
|
echo "FnCoverage: cannot find $FUNCTIONS" >&2
|
|
exit 2
|
|
fi
|
|
|
|
WORK=$(mktemp -d "${TMPDIR:-/tmp}/fncov.XXXXXX") || exit 2
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
|
|
# Registered builtins: the { T("NAME"), fun_... } table entries.
|
|
#
|
|
# The table is not flat. Some entries sit inside #if blocks and are absent
|
|
# from builds that do not define the guard, and counting those as gaps makes
|
|
# the number wrong in the direction that matters: it reports work no test
|
|
# could do, because in this configuration the function does not exist and
|
|
# answers "#-1 FUNCTION (NAME) NOT FOUND".
|
|
#
|
|
# So resolve each guard against what this build actually defines rather than
|
|
# assuming. Two sources, because the macros come from two places: -D flags
|
|
# in the engine Makefile (REALITY_LVLS, WOD_REALMS, TINYMUX_JIT) and
|
|
# autoconf's header (HAVE_*, and STUB_SLAVE when --enable-stubslave is on).
|
|
#
|
|
# Today this splits out ten names -- the SQL/recordset family behind
|
|
# STUB_SLAVE and INLINESQL, neither of which the smoke build defines. They
|
|
# are reported separately rather than dropped: "cannot be covered in this
|
|
# build" is a different fact from "nobody wrote a test", and a reader chasing
|
|
# #1160 needs to tell them apart.
|
|
#
|
|
# Guards are assumed non-nested and single-macro. Verified against the
|
|
# table as it stands: every #if inside it is one of #if defined(INLINESQL),
|
|
# #if defined(STUB_SLAVE), #if defined(TINYMUX_JIT), #ifdef REALITY_LVLS,
|
|
# #ifdef TINYMUX_JIT. A compound guard such as #if defined(A) || defined(B)
|
|
# would be read as A alone, so if one is ever added here this needs
|
|
# revisiting -- the failure would be silent, which is why it is written down
|
|
# rather than left to be rediscovered.
|
|
{
|
|
grep -ohE '[-]D[A-Za-z_][A-Za-z0-9_]*' \
|
|
"$REPO/mux/modules/engine/Makefile" 2>/dev/null | sed 's/^-D//'
|
|
sed -n 's/^#define[[:space:]]\{1,\}\([A-Za-z_][A-Za-z0-9_]*\).*/\1/p' \
|
|
"$REPO/mux/src/autoconf.h" 2>/dev/null
|
|
} | sort -u > "$WORK/defined"
|
|
|
|
awk '
|
|
/^#if[[:space:]]+defined\(/ {
|
|
g = $0; sub(/.*defined\(/, "", g); sub(/\).*/, "", g); guard = g; next
|
|
}
|
|
/^#ifdef[[:space:]]/ { guard = $2; next }
|
|
/^#if[[:space:]]/ { guard = "?"; next }
|
|
/^#endif/ { guard = ""; next }
|
|
match($0, /\{[[:space:]]*T\("[A-Z0-9_]+"\)[[:space:]]*,[[:space:]]*fun_/) {
|
|
n = $0; sub(/.*T\("/, "", n); sub(/"\).*/, "", n)
|
|
print (guard == "" ? "-" : guard) " " n
|
|
}
|
|
' "$FUNCTIONS" | sort -u > "$WORK/tagged"
|
|
|
|
: > "$WORK/all"; : > "$WORK/guarded"
|
|
while read -r g n; do
|
|
if [ "$g" = "-" ] || grep -qx "$g" "$WORK/defined"; then
|
|
printf '%s\n' "$n" >> "$WORK/all"
|
|
else
|
|
printf '%s (%s)\n' "$n" "$g" >> "$WORK/guarded"
|
|
fi
|
|
done < "$WORK/tagged"
|
|
sort -u -o "$WORK/all" "$WORK/all"
|
|
sort -u -o "$WORK/guarded" "$WORK/guarded"
|
|
|
|
# Names beginning with _ are internal helpers reached only from generated
|
|
# softcode, never written by a user, so they are excluded from the ratio
|
|
# rather than counted as gaps.
|
|
grep '^_' "$WORK/all" > "$WORK/internal" || true
|
|
grep -v '^_' "$WORK/all" > "$WORK/user"
|
|
|
|
cat "$TESTCASES"/*.mux 2>/dev/null | tr 'A-Z' 'a-z' > "$WORK/corpus"
|
|
|
|
: > "$WORK/uncovered"
|
|
while read -r fn; do
|
|
low=$(printf '%s' "$fn" | tr 'A-Z' 'a-z')
|
|
if ! grep -qE "(^|[^a-z0-9_])${low}[[:space:]]*\(" "$WORK/corpus"; then
|
|
printf '%s\n' "$fn" >> "$WORK/uncovered"
|
|
fi
|
|
done < "$WORK/user"
|
|
|
|
n_all=$(wc -l < "$WORK/user")
|
|
n_int=$(wc -l < "$WORK/internal")
|
|
n_grd=$(wc -l < "$WORK/guarded")
|
|
n_unc=$(wc -l < "$WORK/uncovered")
|
|
n_cov=$((n_all - n_unc))
|
|
pct=$(( n_cov * 100 / (n_all > 0 ? n_all : 1) ))
|
|
|
|
echo "=== builtin corpus coverage (#1160) ==="
|
|
printf ' user-callable builtins : %d\n' "$n_all"
|
|
printf ' referenced in a test : %d (%d%%)\n' "$n_cov" "$pct"
|
|
printf ' no reference at all : %d\n' "$n_unc"
|
|
printf ' internal helpers (excluded): %d\n' "$n_int"
|
|
printf ' absent from this build (excluded): %d\n' "$n_grd"
|
|
|
|
# Tier 1 — the eval/composition family. #1160 singled these out because
|
|
# the JIT does its most aggressive work here (nested compilation, carg
|
|
# marshalling, result-type handoff) and #1159 lived in exactly this set.
|
|
TIER1="ULOCAL UDEFAULT DEFAULT EDEFAULT ZFUN SUBEVAL FOREACH MUNGE LMATH FILTERBOOL"
|
|
t1_missing=""
|
|
for f in $TIER1; do
|
|
if grep -qx "$f" "$WORK/uncovered"; then
|
|
t1_missing="$t1_missing $f"
|
|
fi
|
|
done
|
|
|
|
echo
|
|
if [ -n "$t1_missing" ]; then
|
|
echo " TIER 1 UNCOVERED:$t1_missing"
|
|
else
|
|
echo " Tier 1 (eval/composition): all covered"
|
|
fi
|
|
|
|
if [ "$MODE" = "--list" ]; then
|
|
echo
|
|
echo "Uncovered:"
|
|
fold -s -w 72 < "$WORK/uncovered" | sed 's/^/ /'
|
|
tr '\n' ' ' < "$WORK/uncovered" | fold -s -w 72 | sed 's/^/ /'
|
|
echo
|
|
if [ "$n_grd" -gt 0 ]; then
|
|
echo
|
|
echo "Excluded -- guarded by a macro this build does not define, so"
|
|
echo "no test in the corpus can reference them:"
|
|
sed 's/^/ /' < "$WORK/guarded"
|
|
fi
|
|
fi
|
|
|
|
if [ "$MODE" = "--tier1" ] && [ -n "$t1_missing" ]; then
|
|
exit 1
|
|
fi
|
|
exit 0
|