/*! \file jit_compiler.cpp
 * \brief Top-level JIT compiler pipeline.
 *
 * JIT statistics, Tier 2 blob management, compile cache,
 * compile_expression(), ECALL handler, and the public entry
 * points: jit_eval(), fun_jitstats(), fun_rveval(), fun_rvbench().
 */

#include "jit_tier1_stamp.h"
#include "copyright.h"
#include "autoconf.h"
#include "config.h"
#include "externs.h"
#include "sqlite_backend.h"
#include "ast.h"

#include "dbt_compile.h"
#include "dbt.h"
#include "dbt_decoder.h"
#include "engine_api.h"
#include "hir_lower_lua.h"
#include "sha1.h"

#include "../../rv64/rv64blob.h"

extern "C" {
#include <lua.h>
#include <lauxlib.h>
}

#include <cstdio>
#include <cstring>
#include <cstdlib>
#include <cmath>
#include <cerrno>
#include <ctime>
#include <algorithm>
#include <vector>
#include <string>
#include <string_view>
#include <map>
#include <unordered_map>
#include <unordered_set>
#include <list>

jit_stats_t s_jit_stats = {};

// ---------------------------------------------------------------
// JIT Arena Management (Tier B)
//
// Uses shared_ptr<RegBuffer> for buffer lifecycle — unified with the
// engine's register packing system.
// ---------------------------------------------------------------

class JITArena {
public:
    struct Arena {
        std::shared_ptr<RegBuffer> buf;
        size_t    used;
        uint32_t  id;
    };

    static Arena *Alloc(size_t n) {
        if (n > LBUF_SIZE) return nullptr;

        if (!s_current || s_current->used + n > LBUF_SIZE) {
            s_current = Create();
        }

        s_current->used += n;
        return s_current;
    }

    static void AddRef(uint32_t id) {
        // No-op: shared_ptr refcount is managed via reg_ref copies.
        UNUSED_PARAMETER(id);
    }

    static void Release(uint32_t id) {
        auto it = s_arenas.find(id);
        if (it != s_arenas.end()) {
            Arena *a = it->second;
            // If only the arena itself holds a reference, clean up.
            if (a->buf.use_count() <= 1) {
                if (s_current && s_current->id == id) s_current = nullptr;
                delete a;
                s_arenas.erase(it);
            }
        }
    }

    static Arena *Get(uint32_t id) {
        auto it = s_arenas.find(id);
        return (it != s_arenas.end()) ? it->second : nullptr;
    }

    // Release arenas that have no external references.
    static void gc() {
        s_current = nullptr;
        auto it = s_arenas.begin();
        while (it != s_arenas.end()) {
            Arena *a = it->second;
            if (a->buf.use_count() <= 1) {
                delete a;
                it = s_arenas.erase(it);
            } else {
                ++it;
            }
        }
    }

private:
    static Arena *Create() {
        Arena *a = new Arena;
        a->buf = std::make_shared<RegBuffer>();
        a->used = 0;
        a->id = ++s_next_id;
        s_arenas[a->id] = a;
        return a;
    }

    static inline thread_local uint32_t s_next_id = 0;
    static inline thread_local Arena *s_current = nullptr;
    static inline thread_local std::unordered_map<uint32_t, Arena *> s_arenas;
};

// ---------------------------------------------------------------
// Tier 2: pre-compiled RV64 library blob
// ---------------------------------------------------------------

tier2_state s_tier2 = { false, {}, 0, 0, 0, 0, {}, 0 };

// Tier 1 compiler version.  Bump this whenever the HIR lowering,
// codegen, constant folding, NOEVAL handlers, or any other tier 1
// logic changes in a way that could produce different RV64 output
// for the same softcode input.
//
static const char JIT_COMPILER_VERSION[] = "jit-t1-003";

// Build stamp backstop.  s_blob_version (below) feeds the SQLite
// code_cache staleness key (blob_hash); a persisted entry is only reused
// when its blob_hash still matches.  Relying solely on a hand-maintained
// JIT_COMPILER_VERSION is fragile: a codegen change in another TU (e.g.
// hir_codegen.cpp) with no version bump leaves stale entries matching and
// being served — which manifests as a "poisoned" compiled result that
// survives an upgrade until the cache is cleared by hand.  Because the
// JIT requires a clean rebuild to take effect, this file is recompiled
// every such build, so its __DATE__/__TIME__ stamp changes and folds into
// the hash, invalidating every previously persisted entry automatically.
//
static const char JIT_BUILD_STAMP[] = __DATE__ " " __TIME__;

// Blob content hash for cache invalidation.  Incorporates both the
// tier 2 blob and the tier 1 compiler version so that upgrading
// either one invalidates stale cached entries.
//
std::string s_blob_version = "none";

static bool tier2_allowed(const std::string &mux_name) {
    static constexpr std::string_view s_allowlist[] = {
        // co_* wrappers: cross-compiled from the same Ragel color_ops
        // source the server uses.  Semantics-matched by construction.
        //
        "STRLEN",
        "LCSTR",
        "UCSTR",
        "CAPSTR",
        "REVERSE",
        "ESCAPE",
        "STRIPANSI",
        "COMPRESS",
        "FIRST",
        "REST",
        "LAST",
        "WORDS",
        "MID",
        "POS",
        "REPEAT",
        "TRIM",
        "MEMBER",
        "EXTRACT",
        // Cursor walk backing the ITER lowering (#2052).  Not reachable as a
        // softcode function -- there is no SPLIT_TOKEN() builtin -- so this
        // entry exists only so hir_lower can resolve it.
        "SPLIT_TOKEN",
        // Same story: ITER's in-place accumulator (#2072), unreachable
        // from softcode.
        "APPEND",
        // Integer-ABI successors of the two above (#2132) -- same
        // unreachable-from-softcode story.  Forgetting THIS list is the
        // silent way to ship the fallback: tier2_lookup returns 0, the
        // lowering keeps the string route, and every test still passes.
        "SPLIT_STEP",
        "APPEND_I",
        "BYTELEN_I",
        // And MAP's element-size guard (#2080).
        "BYTELEN",
        "LEFT",
        "RIGHT",
        "LPOS",
        "LDELETE",      // co_ldelete_wrap → co_delete_at (word-list delete,
                        // mirrors co_replace_at/co_insert_at).  See #768.
        "REPLACE",
        "INSERT",
        "LJUST",
        "RJUST",
        "CENTER",
        "EDIT",
        "SPLICE",
        "SETUNION",
        "SETDIFF",
        "SETINTER",

        // rv64_* hand-written: only trivial ops where the blob
        // implementation is demonstrably equivalent to the server.
        //
        "CAT",
        "STRCAT",
        "SPACE",

        // Tier 2 math — MATH_WRAP transcendentals.
        // These are simple strtod → libm → fval wrappers, semantics-
        // matched to the server by construction.
        //
        "SIN", "COS", "TAN",
        "ASIN", "ACOS", "ATAN", "ATAN2",
        "EXP", "LOG", "LOG10",
        "SQRT", "CEIL", "FLOOR",
        "ABS",          // rv64_fabs: fabs() via MATH_WRAP_1
        "FMOD",         // rv64_fmod: fmod() via MATH_WRAP_2
        "POWER",        // rv64_power: pow() via MATH_WRAP_2

        // Tier 2 arithmetic.
        "ADD", "SUB",
        "MUL",          // rv64_mul: NearestPretty intrinsic
        "FDIV",         // rv64_fdiv: IEEE div, fval handles Inf/NaN
        "MOD",          // rv64_mod: atoi64 % atoi64
        "SIGN",         // rv64_sign: -1/0/1
        "MIN", "MAX",   // rv64_min/max: strtod compare → fval
        "INC", "DEC",   // rv64_inc/dec: atoi64 ± 1
        "TRUNC",        // rv64_trunc: MATH_WRAP via libm ::trunc (#827 — the
                        // old (val>=0)?floor():ceil() pattern got inlined to a
                        // DBT-mistranslated fcvt and rounded away from zero)
        "ROUND",        // rv64_round: ftoa_round intrinsic

        // Tier 2 list/string ops — parity-tested via smoke suite.
        //
        "BEFORE", "AFTER",
        "WORDPOS",      // rv64_wordpos: char-position → word number, mirroring
                        // fun_wordpos (strip color, split words, map byte
                        // offset to 1-based word).  See #768.
        "DELETE", "ELEMENTS", "REMOVE",
        "REVWORDS",
        "LNUM",
        // LADD reproduces fun_ladd's AddDoubles (|x|-sorted, error-
        // compensated, NearestPretty) via the rv64_add_doubles host
        // intrinsic — exact parity by construction.  See #813.
        "LADD",
        "LMAX", "LMIN", "LAND", "LOR",
        "ISNUM", "ISINT",
        "DEC2HEX", "HEX2DEC",
        // ISDBREF is deliberately absent: parse_dbref accepts the objid
        // #<dbref>:<timestamp> form, validated against the object's
        // creation_seconds — engine state the blob cannot reach.
        "CHR", "ORD",
        "SECURE", "SQUISH",
        "TRANSLATE",
        "STRMATCH", "MATCH", "GRAB", "GRABALL",
        "SORT",
    };
    const std::string_view name(mux_name);
    for (std::string_view allowed : s_allowlist) {
        if (name == allowed) {
            return true;
        }
    }
    return false;
}

static std::string sha1_hex_parts(const void *const *parts, const size_t *sizes,
                                  int count) {
    static constexpr unsigned int SHA1_DIGEST_LEN = 20;
    std::vector<const UTF8 *> digest_parts;
    std::vector<size_t> digest_sizes;
    digest_parts.reserve(count);
    digest_sizes.reserve(count);

    for (int i = 0; i < count; i++) {
        if (parts[i] && sizes[i] > 0) {
            digest_parts.push_back(reinterpret_cast<const UTF8 *>(parts[i]));
            digest_sizes.push_back(sizes[i]);
        }
    }

    uint8_t digest[SHA1_DIGEST_LEN];
    unsigned int digest_len = 0;
    if (!mux_sha1_digest(digest_parts.data(), digest_sizes.data(),
                         static_cast<int>(digest_parts.size()),
                         digest, &digest_len)
        || digest_len != SHA1_DIGEST_LEN) {
        return "none";
    }

    static const char hex[] = "0123456789abcdef";
    std::string out;
    out.resize(SHA1_DIGEST_LEN * 2);
    for (size_t i = 0; i < SHA1_DIGEST_LEN; i++) {
        out[i * 2] = hex[digest[i] >> 4];
        out[i * 2 + 1] = hex[digest[i] & 0x0F];
    }
    return out;
}

// Map MUX function names (uppercase) to Tier 2 blob entry names.
// The blob uses rv64_ prefixed names; MUX uses plain uppercase.
//
static const struct { const char *mux_name; const char *blob_name; } s_tier2_map[] = {
    // --- Color-aware co_* wrappers (Ragel, PUA color, Unicode 16) ---
    // These replace the ASCII-only rv64_* versions for functions where
    // color preservation and grapheme-cluster-aware word boundaries matter.
    //
    { "FIRST",       "co_first_wrap" },
    { "REST",        "co_rest_wrap" },
    { "LAST",        "co_last_wrap" },
    { "WORDS",       "co_words_wrap" },
    { "EXTRACT",     "co_extract_wrap" },
    { "SPLIT_TOKEN", "rv64_split_token" },  // cursor walk for ITER (#2052)
    { "APPEND",      "rv64_append" },       // in-place accumulator for ITER (#2072)
    { "SPLIT_STEP",  "rv64_split_step" },   // integer-ABI cursor walk (#2132)
    { "APPEND_I",    "rv64_append_i" },     // integer-ABI accumulator (#2132)
    { "BYTELEN_I",   "rv64_bytelen_i" },    // integer-ABI CARGS guard (#2152)
    { "BYTELEN",     "rv64_bytelen" },      // CARGS-slot fit guard for MAP (#2080)
    { "MEMBER",      "co_member_wrap" },
    { "TRIM",        "co_trim_wrap" },
    { "REPEAT",      "co_repeat_wrap" },
    { "MID",         "co_mid_wrap" },
    { "POS",         "co_pos_wrap" },
    { "SORT",        "co_sort_wrap" },
    { "SETUNION",    "co_setunion_wrap" },
    { "SETDIFF",     "co_setdiff_wrap" },
    { "SETINTER",    "co_setinter_wrap" },
    { "LDELETE",     "co_ldelete_wrap" },
    { "REPLACE",     "co_replace_wrap" },
    { "INSERT",      "co_insert_wrap" },

    // --- ASCII-only rv64_* (no color, byte-level operations) ---
    // These are fine for functions that don't handle colored text.
    //
    { "CAT",         "rv64_cat" },
    { "STRCAT",      "rv64_strcat" },
    { "BEFORE",      "rv64_before" },
    { "AFTER",       "rv64_after" },

    // --- Batch 2: case, reverse, escape, left/right, compress, lpos ---
    //
    { "STRLEN",      "co_strlen_wrap" },
    { "LCSTR",       "co_lcstr_wrap" },
    { "UCSTR",       "co_ucstr_wrap" },
    { "REVERSE",     "co_reverse_wrap" },
    { "ESCAPE",      "co_escape_wrap" },
    { "LEFT",        "co_left_wrap" },
    { "RIGHT",       "co_right_wrap" },
    { "COMPRESS",    "co_compress_wrap" },
    { "LPOS",        "co_lpos_wrap" },

    // --- Batch 3: justify, edit, splice, totitle, stripansi, vislen ---
    { "LJUST",       "co_ljust_wrap" },
    { "RJUST",       "co_rjust_wrap" },
    { "CENTER",      "co_center_wrap" },
    { "EDIT",        "co_edit_wrap" },
    { "SPLICE",      "co_splice_wrap" },
    { "CAPSTR",      "co_totitle_wrap" },
    { "STRIPANSI",   "co_stripansi_wrap" },

    // --- Batch 4: space, secure, squish, delete, elements ---
    { "SPACE",       "rv64_space" },
    { "SECURE",      "co_secure_wrap" },
    { "SQUISH",      "co_compress_wrap" },
    { "DELETE",      "rv64_delete" },
    { "ELEMENTS",    "rv64_elements" },
    { "TRANSLATE",   "rv64_translate" },

    // --- Batch 5: wildcard matching ---
    { "STRMATCH",   "rv64_strmatch" },
    { "MATCH",       "rv64_match" },
    { "GRAB",        "rv64_grab" },
    { "GRABALL",     "rv64_graball" },

    // --- Batch 6: numbers, chars, base conversion ---
    { "LNUM",        "rv64_lnum" },
    { "ISNUM",       "rv64_isnum" },
    { "ISINT",       "rv64_isint" },
    { "CHR",         "rv64_chr" },
    { "ORD",         "rv64_ord" },
    { "DEC2HEX",     "rv64_dec2hex" },
    { "HEX2DEC",     "rv64_hex2dec" },

    // --- Batch 7: wordpos, remove ---
    { "WORDPOS",     "rv64_wordpos" },
    { "REMOVE",      "rv64_remove" },

    // --- Batch 8: list aggregation, reversal, type checks ---
    { "LADD",        "rv64_ladd" },   // sum via rv64_add_doubles intrinsic (#813)
    { "LMAX",        "rv64_lmax" },
    { "LMIN",        "rv64_lmin" },
    { "LAND",        "rv64_land" },
    { "LOR",         "rv64_lor" },
    { "REVWORDS",    "rv64_revwords" },
    { "FLIP",        "rv64_revwords" },   // alias
    // (no ISDBREF: objid colon form needs creation_seconds; always ECALLs)

    // --- Batch 8: math via intrinsics (string↔double + platform libm) ---
    { "SIN",         "rv64_sin" },
    { "COS",         "rv64_cos" },
    { "TAN",         "rv64_tan" },
    { "ASIN",        "rv64_asin" },
    { "ACOS",        "rv64_acos" },
    { "ATAN",        "rv64_atan" },
    { "ATAN2",       "rv64_atan2" },
    { "EXP",         "rv64_exp" },
    { "LOG",         "rv64_log10" },  // MUX log() defaults to common (base 10)
    { "LOG10",       "rv64_log10" },
    { "SQRT",        "rv64_sqrt" },
    { "CEIL",        "rv64_ceil" },
    { "FLOOR",       "rv64_floor" },
    { "ABS",         "rv64_fabs" },
    { "FMOD",        "rv64_fmod" },
    { "POWER",       "rv64_power" },

    // --- Batch 9: arithmetic ---
    { "ADD",         "rv64_add" },
    { "SUB",         "rv64_sub" },
    { "MUL",         "rv64_mul" },
    { "FDIV",        "rv64_fdiv" },
    { "MOD",         "rv64_mod" },
    { "SIGN",        "rv64_sign" },
    { "MIN",         "rv64_min" },
    { "MAX",         "rv64_max" },
    { "INC",         "rv64_inc" },
    { "DEC",         "rv64_dec" },
    { "TRUNC",       "rv64_trunc" },

    // Deliberately exclude ROUND for now. Constant ROUND calls fold
    // correctly, but the Tier 2 runtime path still misbehaves for
    // dynamic inputs such as `round(sqrt(2),6)`. Fall back to the
    // interpreter until the rv64 implementation is parity-correct.

    { nullptr, nullptr }
};

// Load the Tier 2 blob from a file.
// Called once at init time.  Guest base address is where the blob's
// code section will be mapped in each program's guest memory.
//
static bool tier2_load(const char *path, uint64_t guest_base) {
    FILE *f = fopen(path, "rb");
    if (!f) return false;

    rv64_blob_header hdr;
    memset(&hdr, 0, sizeof(hdr));
    // Read at least the v1 header (32 bytes), then the v2 extension.
    if (fread(&hdr, 32, 1, f) != 1) { fclose(f); return false; }
    if (hdr.magic != RV64_BLOB_MAGIC
        || (hdr.version != 1 && hdr.version != 2)) {
        fclose(f);
        return false;
    }
    if (hdr.version >= 2) {
        // Read the v2 extension fields (bytes 32-47).
        if (fread(&hdr.data_offset, sizeof(hdr) - 32, 1, f) != 1) {
            fclose(f);
            return false;
        }
    }

    // Read code section directly into the install image.
    s_tier2.code_size = hdr.code_size;
    s_tier2.bss_size = (hdr.version >= 2) ? hdr.bss_size : 0;

    // The blob (code + rodata + data + BSS) is mapped at BLOB_BASE and must
    // fit within the blob window; otherwise it would overflow into the
    // string pool.  tier2_install only checks against the full MEM_SIZE, so
    // enforce the tighter window bound here at load time.
    if (s_tier2.code_size + s_tier2.bss_size
        > rv_compiler::BLOB_LIMIT - rv_compiler::BLOB_BASE) {
        fclose(f);
        return false;
    }

    s_tier2.image.resize(hdr.code_size + s_tier2.bss_size, 0);
    fseek(f, hdr.code_offset, SEEK_SET);
    if (fread(s_tier2.image.data(), hdr.code_size, 1, f) != 1) {
        fclose(f);
        return false;
    }

    // Record writable data offset within the flat image for runtime reset.
    if (hdr.version >= 2 && hdr.data_size > 0) {
        s_tier2.data_image_offset = hdr.data_offset - hdr.code_offset;
        s_tier2.data_image_size = hdr.data_size;
    } else {
        s_tier2.data_image_offset = 0;
        s_tier2.data_image_size = 0;
    }

    // Read entry table.
    std::vector<rv64_blob_entry> entries(hdr.entry_count);
    fseek(f, hdr.entry_offset, SEEK_SET);
    if (fread(entries.data(), sizeof(rv64_blob_entry), hdr.entry_count, f)
        != hdr.entry_count) {
        fclose(f);
        return false;
    }
    fclose(f);

    // Build lookup table.
    s_tier2.guest_base = guest_base;
    s_tier2.funcs.clear();
    for (uint32_t i = 0; i < hdr.entry_count; i++) {
        tier2_entry te;
        te.code_off = entries[i].code_off;
        te.guest_addr = static_cast<uint32_t>(guest_base) + entries[i].code_off;
        s_tier2.funcs[entries[i].name] = te;
    }

    // Build MUX name → blob mapping.
    for (int i = 0; s_tier2_map[i].mux_name; i++) {
        auto it = s_tier2.funcs.find(s_tier2_map[i].blob_name);
        if (it != s_tier2.funcs.end()) {
            s_tier2.funcs[s_tier2_map[i].mux_name] = it->second;
        }
    }

    s_tier2.loaded = true;
    // Tier 1 leg: every unit that can emit different RV64 for the same
    // softcode, each carrying its own __DATE__/__TIME__ (#2061).  JIT_BUILD_STAMP
    // alone covered only this file, so a codegen change in any other unit left
    // the key unmoved and the cache served the previous build's output.
    // Deliberately excludes the dbt_* units: they execute the stored RV64
    // rather than produce it, so invalidating on their account would discard
    // the cache for no gain.  See jit_tier1_stamp.h.
    // Layout constants are part of the key (#2107): a guest-address
    // map change (STR/FARGS bases) rewrites every baked pointer in the
    // code and pool blobs, and must not reuse a row written under the
    // previous map even when __DATE__/__TIME__ happened not to move.
    static const uint64_t layout[] = {
        rv_compiler::CODE_BASE, rv_compiler::CODE_LIMIT,
        rv_compiler::STR_BASE,  rv_compiler::STR_LIMIT,
        rv_compiler::FARGS_BASE, rv_compiler::FARGS_LIMIT,
        rv_compiler::BLOB_BASE, rv_compiler::MEM_SIZE,
    };
    const void *parts[] = {
        JIT_COMPILER_VERSION,
        JIT_BUILD_STAMP,
        TIER1_STAMP_AST,
        TIER1_STAMP_HIR_LOWER,
        TIER1_STAMP_HIR_SSA,
        TIER1_STAMP_HIR_OPT,
        TIER1_STAMP_HIR_CODEGEN,
        &hdr,
        s_tier2.image.data(),
        entries.empty() ? nullptr : entries.data(),
        layout,
    };
    const size_t sizes[] = {
        sizeof(JIT_COMPILER_VERSION) - 1,
        sizeof(JIT_BUILD_STAMP) - 1,
        // strlen, not sizeof: these are extern arrays of unknown bound here.
        strlen(TIER1_STAMP_AST),
        strlen(TIER1_STAMP_HIR_LOWER),
        strlen(TIER1_STAMP_HIR_SSA),
        strlen(TIER1_STAMP_HIR_OPT),
        strlen(TIER1_STAMP_HIR_CODEGEN),
        sizeof(hdr),
        s_tier2.code_size,
        entries.size() * sizeof(rv64_blob_entry),
        sizeof(layout),
    };
    s_blob_version = sha1_hex_parts(parts, sizes, 11);
    return true;
}

// Look up a function by MUX name (uppercase).
// Returns guest address, or 0 if not found.
//
uint64_t tier2_lookup(const std::string &mux_name) {
    if (!s_tier2.loaded) return 0;
    if (!tier2_allowed(mux_name)) return 0;
    auto it = s_tier2.funcs.find(mux_name);
    if (it != s_tier2.funcs.end()) return it->second.guest_addr;
    return 0;
}

// Look up a raw blob symbol by name (e.g., "sin", "cos", "rv64_strtod").
// Bypasses the tier2_allowed() gate — used for direct FP intrinsic calls
// from the type-propagated lowering path.
// Returns guest address, or 0 if not found.
//
uint64_t tier2_sym_addr(const char *blob_name) {
    if (!s_tier2.loaded) return 0;
    auto it = s_tier2.funcs.find(blob_name);
    if (it != s_tier2.funcs.end()) return it->second.guest_addr;
    return 0;
}

// Pre-translate all Tier 2 blob entry points so that superblocks
// can use native CALL continuation for Tier 2 function calls.
// Called after dbt_init/dbt_reset, before dbt_run.
//
// Helper: register a blob symbol as an intrinsic if it exists.
//
static void reg_intrinsic(dbt_state_t *dbt, const char *blob_name,
                           dbt_emitter_id eid, void *host_fn = nullptr) {
    auto it = s_tier2.funcs.find(blob_name);
    if (it != s_tier2.funcs.end()) {
        dbt_register_intrinsic(dbt, it->second.guest_addr, eid, host_fn);
    }
}


// Host-side wrappers for string↔double conversion intrinsics.
// These are called by the DBT stubs with host pointers and FP values.
//
static double host_strtod(const char *s) {
    return mux_atof(reinterpret_cast<const UTF8 *>(s));
}

// Guest heap bump cursor and the rv64_alloc intrinsic backing it.
//
// The heap is a per-evaluation arena: the guest bump-allocates scratch via
// rv64_alloc and never frees; the cursor is reset to HEAP_BASE before each
// evaluation (alongside the blob .data/.bss reset).  host_alloc returns a
// *guest* address (offset into the guest image), not a host pointer, so the
// DBT stub must not host<->guest convert the return (DBT_EMIT_ALLOC uses
// ptr_mask=0).  0 is returned on exhaustion → the guest sees NULL.
//
static uint64_t s_heap_next = rv_compiler::HEAP_BASE;

static uint64_t host_alloc(uint64_t size) {
    uint64_t aligned = (size + 15) & ~15ULL;
    uint64_t addr = s_heap_next;
    if (aligned > rv_compiler::HEAP_LIMIT - rv_compiler::HEAP_BASE
        || addr + aligned > rv_compiler::HEAP_LIMIT) {
        return 0;  // out of heap → guest NULL
    }
    s_heap_next = addr + aligned;
    return addr;
}

// rv64_add_doubles intrinsic — error-compensated list sum for fun_ladd
// parity.  The guest fills a doubles array (the fixed DSCRATCH region) and
// calls this; the host does the order-sensitive arithmetic (|x|-sorted
// qsort, TwoSum chain, NearestPretty) so the result is byte-identical to
// fun_ladd by construction.  AddDoubles already applies NearestPretty and
// sorts vals[] in place (harmless — it is throwaway scratch).
//
static double host_add_doubles(double *vals, int n) {
    return AddDoubles(n, vals);
}

static int host_fval(char *buf, double val) {
    UTF8 *bufc = reinterpret_cast<UTF8 *>(buf);
    UTF8 *start = bufc;
    fval(reinterpret_cast<UTF8 *>(buf), &bufc, val);
    *bufc = '\0';
    return static_cast<int>(bufc - start);
}

// Host-side wrapper for rv64_ftoa_round intrinsic.
// Matches fun_round: mux_fpclass check + mux_ftoa(r, true, frac).
//
static int host_ftoa_round(char *buf, double val, int frac) {
#ifdef HAVE_IEEE_FP_FORMAT
    int fpc = mux_fpclass(val);
    if (MUX_FPGROUP(fpc) != MUX_FPGROUP_PASS
        && MUX_FPGROUP(fpc) != MUX_FPGROUP_ZERO) {
        const UTF8 *s = mux_FPStrings[MUX_FPCLASS(fpc)];
        size_t len = strlen(reinterpret_cast<const char *>(s));
        memcpy(buf, s, len);
        buf[len] = '\0';
        return static_cast<int>(len);
    }
    if (MUX_FPGROUP(fpc) == MUX_FPGROUP_ZERO) {
        val = 0.0;
    }
#endif
    UTF8 *result = mux_ftoa(val, true, frac);
    size_t len = strlen(reinterpret_cast<const char *>(result));
    memcpy(buf, result, len);
    buf[len] = '\0';
    return static_cast<int>(len);
}

void pretranslate_tier2(dbt_state_t *dbt) {
    if (!s_tier2.loaded) return;

    // Register intrinsics FIRST — translate_block() checks these
    // addresses and emits native x86-64 stubs instead of translating
    // the RV64 bodies.  Must happen before pretranslation.

    // Block-level intrinsics (custom emitters).
    //
    reg_intrinsic(dbt, "rv64_slen",  DBT_EMIT_SLEN);
    reg_intrinsic(dbt, "rv64_scopy", DBT_EMIT_SCOPY);
    reg_intrinsic(dbt, "memcpy",     DBT_EMIT_MEMCPY);
    reg_intrinsic(dbt, "memcmp",     DBT_EMIT_MEMCMP);
    reg_intrinsic(dbt, "memset",     DBT_EMIT_MEMSET);
    reg_intrinsic(dbt, "memswap",    DBT_EMIT_MEMSWAP);

    // co_* Ragel functions → native host calls.
    // The wrapper does fargs unpacking in RV64 (cheap to translate),
    // then JALs to co_first/co_rest/etc.  The intrinsic intercepts
    // the JAL and calls the host's native Ragel implementation directly.
    //
    // 4 args: (out:ptr, p:ptr, len:int, delim:int)
    reg_intrinsic(dbt, "co_first",  DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_first));
    reg_intrinsic(dbt, "co_rest",   DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_rest));
    reg_intrinsic(dbt, "co_last",   DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_last));
    reg_intrinsic(dbt, "co_repeat", DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_repeat));

    // 3 args: (p:ptr, len:int, delim:int)
    reg_intrinsic(dbt, "co_words_count", DBT_EMIT_CO_3P, reinterpret_cast<void *>(co_words_count));

    // 4 args: (haystack:ptr, hlen:int, needle:ptr, nlen:int)
    reg_intrinsic(dbt, "co_pos", DBT_EMIT_CO_POS, reinterpret_cast<void *>(co_pos));

    // 5 args: (out:ptr, p:ptr, len:int, start:int, count:int)
    reg_intrinsic(dbt, "co_mid",  DBT_EMIT_CO_5PP, reinterpret_cast<void *>(co_mid));
    reg_intrinsic(dbt, "co_trim", DBT_EMIT_CO_5PP, reinterpret_cast<void *>(co_trim));

    // 5 args: (target:ptr, tlen:int, list:ptr, llen:int, delim:int)
    reg_intrinsic(dbt, "co_member", DBT_EMIT_CO_MEMBER, reinterpret_cast<void *>(co_member));

    // 6 args: (out:ptr, list:ptr, llen:int, x:int, y:int, z:int)
    reg_intrinsic(dbt, "co_delete",     DBT_EMIT_CO_6PP, reinterpret_cast<void *>(co_delete));
    reg_intrinsic(dbt, "co_sort_words", DBT_EMIT_CO_6PP, reinterpret_cast<void *>(co_sort_words));

    // 7 args: (out:ptr, p:ptr, len:int, iFirst:int, nWords:int, delim:int, osep:int)
    reg_intrinsic(dbt, "co_extract", DBT_EMIT_CO_7PP, reinterpret_cast<void *>(co_extract));

    // 8 args: (out:ptr, list1:ptr, len1:int, list2:ptr, len2:int, delim:int, osep:int, sort_type:int)
    reg_intrinsic(dbt, "co_setunion", DBT_EMIT_CO_8PPP, reinterpret_cast<void *>(co_setunion));
    reg_intrinsic(dbt, "co_setdiff",  DBT_EMIT_CO_8PPP, reinterpret_cast<void *>(co_setdiff));
    reg_intrinsic(dbt, "co_setinter", DBT_EMIT_CO_8PPP, reinterpret_cast<void *>(co_setinter));

    // Batch 2: inner co_* functions called by wrappers.
    //
    // 2 args: (data:ptr, len:int)
    reg_intrinsic(dbt, "co_cluster_count", DBT_EMIT_CO_2P, reinterpret_cast<void *>(co_cluster_count));

    // 3 args: (out:ptr, p:ptr, len:int)
    reg_intrinsic(dbt, "co_tolower",  DBT_EMIT_CO_3PP, reinterpret_cast<void *>(co_tolower));
    reg_intrinsic(dbt, "co_toupper",  DBT_EMIT_CO_3PP, reinterpret_cast<void *>(co_toupper));
    reg_intrinsic(dbt, "co_reverse",  DBT_EMIT_CO_3PP, reinterpret_cast<void *>(co_reverse));
    reg_intrinsic(dbt, "co_escape",   DBT_EMIT_CO_3PP, reinterpret_cast<void *>(co_escape));

    // 4 args: (out:ptr, p:ptr, len:int, n:int)
    reg_intrinsic(dbt, "co_left",     DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_left));
    reg_intrinsic(dbt, "co_right",    DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_right));
    reg_intrinsic(dbt, "co_compress", DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_compress));
    reg_intrinsic(dbt, "co_lpos",     DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_lpos));

    // Batch 3: totitle, strip_color, visible_length.
    // ljust/rjust/center/edit/splice need new emitter patterns (7+ args with
    // complex pointer layouts) — registered as Tier 2 but not intrinsics yet.
    reg_intrinsic(dbt, "co_totitle",         DBT_EMIT_CO_3PP, reinterpret_cast<void *>(co_totitle));
    reg_intrinsic(dbt, "co_strip_color",     DBT_EMIT_CO_3PP, reinterpret_cast<void *>(co_strip_color));
    reg_intrinsic(dbt, "co_visible_length",  DBT_EMIT_CO_2P,  reinterpret_cast<void *>(co_visible_length));

    // FP math intrinsics — double→double via platform libm.
    // Explicit casts resolve C++ overload ambiguity (float/double/long double).
    //
    using fn_d_d = double(*)(double);
    using fn_dd_d = double(*)(double, double);

    reg_intrinsic(dbt, "sin",   DBT_EMIT_FP_D_D,  reinterpret_cast<void *>(static_cast<fn_d_d>(::sin)));
    reg_intrinsic(dbt, "cos",   DBT_EMIT_FP_D_D,  reinterpret_cast<void *>(static_cast<fn_d_d>(::cos)));
    reg_intrinsic(dbt, "tan",   DBT_EMIT_FP_D_D,  reinterpret_cast<void *>(static_cast<fn_d_d>(::tan)));
    reg_intrinsic(dbt, "asin",  DBT_EMIT_FP_D_D,  reinterpret_cast<void *>(static_cast<fn_d_d>(::asin)));
    reg_intrinsic(dbt, "acos",  DBT_EMIT_FP_D_D,  reinterpret_cast<void *>(static_cast<fn_d_d>(::acos)));
    reg_intrinsic(dbt, "atan",  DBT_EMIT_FP_D_D,  reinterpret_cast<void *>(static_cast<fn_d_d>(::atan)));
    reg_intrinsic(dbt, "exp",   DBT_EMIT_FP_D_D,  reinterpret_cast<void *>(static_cast<fn_d_d>(::exp)));
    reg_intrinsic(dbt, "log",   DBT_EMIT_FP_D_D,  reinterpret_cast<void *>(static_cast<fn_d_d>(::log)));
    reg_intrinsic(dbt, "log10", DBT_EMIT_FP_D_D,  reinterpret_cast<void *>(static_cast<fn_d_d>(::log10)));
    reg_intrinsic(dbt, "ceil",  DBT_EMIT_FP_D_D,  reinterpret_cast<void *>(static_cast<fn_d_d>(::ceil)));
    reg_intrinsic(dbt, "floor", DBT_EMIT_FP_D_D,  reinterpret_cast<void *>(static_cast<fn_d_d>(::floor)));
    reg_intrinsic(dbt, "fabs",  DBT_EMIT_FP_D_D,  reinterpret_cast<void *>(static_cast<fn_d_d>(::fabs)));
    reg_intrinsic(dbt, "trunc", DBT_EMIT_FP_D_D,  reinterpret_cast<void *>(static_cast<fn_d_d>(::trunc)));

    // FP math intrinsics — (double,double)→double via platform libm.
    //
    reg_intrinsic(dbt, "pow",   DBT_EMIT_FP_DD_D, reinterpret_cast<void *>(static_cast<fn_dd_d>(::pow)));
    reg_intrinsic(dbt, "atan2", DBT_EMIT_FP_DD_D, reinterpret_cast<void *>(static_cast<fn_dd_d>(::atan2)));
    reg_intrinsic(dbt, "fmod",  DBT_EMIT_FP_DD_D, reinterpret_cast<void *>(static_cast<fn_dd_d>(::fmod)));
    // max()/min() float path (#1273) — host fmax/fmin intercept the blob stubs.
    //
    reg_intrinsic(dbt, "fmax",  DBT_EMIT_FP_DD_D, reinterpret_cast<void *>(static_cast<fn_dd_d>(::fmax)));
    reg_intrinsic(dbt, "fmin",  DBT_EMIT_FP_DD_D, reinterpret_cast<void *>(static_cast<fn_dd_d>(::fmin)));

    // Rounding intrinsic — NearestPretty (double→double).
    //
    reg_intrinsic(dbt, "rv64_nearest_pretty", DBT_EMIT_FP_D_D,
                  reinterpret_cast<void *>(static_cast<fn_d_d>(NearestPretty)));

    // String↔double conversion intrinsics.
    //
    reg_intrinsic(dbt, "rv64_strtod", DBT_EMIT_STRTOD, reinterpret_cast<void *>(host_strtod));
    reg_intrinsic(dbt, "rv64_fval",   DBT_EMIT_FVAL,   reinterpret_cast<void *>(host_fval));
    reg_intrinsic(dbt, "rv64_alloc",  DBT_EMIT_ALLOC,  reinterpret_cast<void *>(host_alloc));
    reg_intrinsic(dbt, "rv64_add_doubles", DBT_EMIT_ADD_DOUBLES,
                  reinterpret_cast<void *>(host_add_doubles));

    // Round-to-precision intrinsic.
    reg_intrinsic(dbt, "rv64_ftoa_round", DBT_EMIT_FTOA_ROUND,
                  reinterpret_cast<void *>(host_ftoa_round));

    // Pre-translate all intrinsic stubs into the cache BEFORE any
    // function pretranslation.  Intrinsics are leaf functions (strlen,
    // memcpy, sitoa, co_* wrappers) that blob functions call internally.
    // By caching them first, every subsequent translate_block that
    // encounters a JAL to an intrinsic address will find it in the
    // cache and emit an inline CALL — regardless of worklist order
    // or superblock extension boundaries.
    //
    for (int i = 0; i < dbt->num_intrinsics; i++) {
        uint64_t addr = dbt->intrinsics[i].guest_addr;
        if (addr) {
            dbt_pretranslate(dbt, addr);
        }
    }

    // Pretranslate all Tier 2 functions.  With intrinsics already cached,
    // inline CALLs to intrinsic targets will fire on first encounter.
    //
    for (auto &kv : s_tier2.funcs) {
        dbt_pretranslate(dbt, kv.second.guest_addr);
    }

    // Resolve cross-function chains: block A (from function X)
    // exits to block B (from function Y) — the backpatch during Y's
    // pretranslation won't find A's patch site.  This pass fixes them.
    dbt_resolve_chains(dbt);
}

// Copy blob code into a program's guest memory.
// Called during compile_expression() before codegen.
//
template<typename Vec>
void tier2_install(Vec &memory, uint64_t guest_base) {
    if (!s_tier2.loaded) return;

    // Copy the prebuilt flat image (code + rodata + data + zeroed BSS)
    // as one contiguous block.  This preserves the exact ELF layout
    // while avoiding a second memset on each install.
    uint64_t total = s_tier2.image.size();
    if (guest_base + total > memory.size()) return;

    memcpy(memory.data() + guest_base,
           s_tier2.image.data(), total);
}

template void tier2_install(guest_memory_t &, uint64_t);
template void tier2_install(std::vector<uint8_t> &, uint64_t);

// Reset only the writable portions of the Tier 2 blob (data + BSS).
// Code and rodata are immutable and don't need re-copying at runtime.
//
template<typename Vec>
static void tier2_reset_writable(Vec &memory, uint64_t guest_base) {
    if (!s_tier2.loaded) return;

    // Re-copy initialized writable data (.sdata/.data).
    if (s_tier2.data_image_size > 0) {
        uint64_t dst = guest_base + s_tier2.data_image_offset;
        if (dst + s_tier2.data_image_size <= memory.size()) {
            memcpy(memory.data() + dst,
                   s_tier2.image.data() + s_tier2.data_image_offset,
                   s_tier2.data_image_size);
        }
    }

    // Zero-fill BSS.
    if (s_tier2.bss_size > 0) {
        uint64_t bss_start = guest_base + s_tier2.code_size;
        if (bss_start + s_tier2.bss_size <= memory.size()) {
            memset(memory.data() + bss_start, 0, s_tier2.bss_size);
        }
    }
}

// Lazy-init Tier 2 blob on first compile.
static bool s_tier2_init = false;
static void tier2_lazy_init() {
    if (s_tier2_init) return;
    s_tier2_init = true;

    // Try to load from the game's bin directory (where engine.so lives).
    const char *paths[] = {
        "bin/softlib.rv64",
        "./softlib.rv64",
        nullptr
    };
    for (int i = 0; paths[i]; i++) {
        if (tier2_load(paths[i], rv_compiler::BLOB_BASE)) {
            return;
        }
    }
    // No blob found — the JIT declines every expression (jit_eval
    // checks s_tier2.loaded) and the interpreter handles everything.
    // Say so once, loudly: this is a degraded state on a JIT build,
    // not a silent mode of normal operation (#875).
    fprintf(stderr, "tier2: softlib.rv64 missing or unloadable -- "
            "JIT disabled, falling back to the interpreter\n");
    // Still hash the compiler version so tier 1 upgrades invalidate.  Same
    // per-unit stamps as the loaded path (#2061) -- without them this leg has
    // the identical hole, and it is the leg a blob-less build runs on.
    // Layout is in the key for the same reason as the blob-loaded path (#2107).
    static const uint64_t layout[] = {
        rv_compiler::CODE_BASE, rv_compiler::CODE_LIMIT,
        rv_compiler::STR_BASE,  rv_compiler::STR_LIMIT,
        rv_compiler::FARGS_BASE, rv_compiler::FARGS_LIMIT,
        rv_compiler::BLOB_BASE, rv_compiler::MEM_SIZE,
    };
    const void *parts[] = {
        JIT_COMPILER_VERSION,
        JIT_BUILD_STAMP,
        TIER1_STAMP_AST,
        TIER1_STAMP_HIR_LOWER,
        TIER1_STAMP_HIR_SSA,
        TIER1_STAMP_HIR_OPT,
        TIER1_STAMP_HIR_CODEGEN,
        layout,
    };
    const size_t sizes[] = {
        sizeof(JIT_COMPILER_VERSION) - 1,
        sizeof(JIT_BUILD_STAMP) - 1,
        strlen(TIER1_STAMP_AST),
        strlen(TIER1_STAMP_HIR_LOWER),
        strlen(TIER1_STAMP_HIR_SSA),
        strlen(TIER1_STAMP_HIR_OPT),
        strlen(TIER1_STAMP_HIR_CODEGEN),
        sizeof(layout),
    };
    s_blob_version = sha1_hex_parts(parts, sizes, 8);
}

void tier2_ensure(void) {
    tier2_lazy_init();
}

// Maximum AST_FUNCCALL nesting depth of a parse tree (#1002).
static int ast_max_funccall_depth(const ASTNode *node)
{
    if (!node) return 0;
    int child_max = 0;
    for (const auto &c : node->children) {
        int d = ast_max_funccall_depth(c.get());
        if (d > child_max) child_max = d;
    }
    return child_max + (node->type == AST_FUNCCALL ? 1 : 0);
}

// Total AST_FUNCCALL nodes in a parse tree (invocation-count watermark).
static int ast_funccall_count(const ASTNode *node)
{
    if (!node) return 0;
    int n = (node->type == AST_FUNCCALL) ? 1 : 0;
    for (const auto &c : node->children) {
        n += ast_funccall_count(c.get());
    }
    return n;
}

// Bounded strlen over guest memory (#1057).  Returns false when `addr`
// is out of range or the region [addr, memory_size) has no NUL — callers
// must not treat the pointer as a C string in that case.
//
static bool guest_strnlen(const uint8_t *memory, size_t memory_size,
                          uint64_t addr, size_t *out_len)
{
    if (!memory || addr >= memory_size) {
        return false;
    }
    const size_t maxn = memory_size - static_cast<size_t>(addr);
    const char *p = reinterpret_cast<const char *>(memory + addr);
    const void *nul = memchr(p, '\0', maxn);
    if (!nul) {
        return false;
    }
    *out_len = static_cast<size_t>(static_cast<const char *>(nul) - p);
    return true;
}

// #1071: guest pointer as a host C string only after guest_strnlen succeeds.
//
static const char *guest_cstr(const uint8_t *memory, size_t memory_size,
                              uint64_t addr)
{
    size_t len = 0;
    if (!guest_strnlen(memory, memory_size, addr, &len)) {
        return nullptr;
    }
    return reinterpret_cast<const char *>(memory + addr);
}

// Load fargs[i] pointer from the guest fargs table; returns false if the
// slot is out of range.  Does not validate the pointed-to string.
//
static bool guest_farg_addr(const uint8_t *memory, size_t memory_size,
                            uint64_t fargs_addr, int idx, uint64_t *out_addr)
{
    if (!memory || !out_addr || idx < 0) {
        return false;
    }
    if (memory_size < 8u) {
        return false;
    }
    const uint64_t slot = fargs_addr + static_cast<uint64_t>(idx) * 8u;
    // Overflow-safe bound: reject a wrapped slot (slot < fargs_addr, e.g. a
    // near-2^64 fargs_addr) and any slot whose 8-byte read runs past the
    // guest region -- without adding to slot, which could itself wrap.
    if (slot < fargs_addr || slot > memory_size - 8u) {
        return false;
    }
    uint64_t p = 0;
    memcpy(&p, memory + slot, 8);
    *out_addr = p;
    return true;
}

// #1078: overflow-safe [addr, addr+nbytes) wholly inside guest memory.
// nbytes == 0 is allowed only when addr <= memory_size.
//
static bool guest_range_ok(uint64_t addr, uint64_t nbytes, size_t memory_size)
{
    if (nbytes == 0) {
        return addr <= memory_size;
    }
    if (addr >= memory_size) {
        return false;
    }
    // Equivalent to addr + nbytes <= memory_size without wrap.
    return nbytes <= memory_size - static_cast<size_t>(addr);
}

// Shared helper for dbt_run/dbt_resume nonzero status: count wall-clock
// aborts and, when requested, write the AST-shaped diagnostic so callers
// that treat the run as handled do not fall through to an empty result.
static bool handle_dbt_run_status(int rc, UTF8 *out, size_t out_size,
                                  bool emit_cpu_limited)
{
    if (rc == 0) {
        return true;
    }
    if (rc == -3) {
        s_jit_stats.bail_alarm++;
        if (emit_cpu_limited && nullptr != out && 0 < out_size) {
            // Mirror AST's alarm path (ast_eval_function): emit
            // "#-1 CPU LIMITED" rather than returning false and letting
            // the AST short-circuit produce an empty string.
            const UTF8 *kMsg = S_("#-1 CPU LIMITED");
            size_t n = strlen(reinterpret_cast<const char *>(kMsg));
            if (n >= out_size) {
                n = out_size - 1;
            }
            memcpy(out, kMsg, n);
            out[n] = '\0';
            return true;  // handled
        }
    }
    return false;
}

static compiled_program compile_expression(const UTF8 *expr, size_t nLen,
                                            int eval = EV_FCHECK | EV_EVAL,
                                            uint64_t code_base = 0,
                                            uint64_t str_start = rv_compiler::STR_BASE,
                                            uint64_t str_lim = rv_compiler::STR_LIMIT,
                                            uint64_t fargs_start = rv_compiler::FARGS_BASE,
                                            uint64_t fargs_lim = rv_compiler::FARGS_LIMIT,
                                            uint64_t out_start = 0) {
    tier2_lazy_init();

    compiled_program prog;
    prog.ok = false;
    prog.out_used = 0;
    prog.entry_pc = code_base;
    prog.folds = 0;
    prog.ecalls = 0;
    prog.tier2_calls = 0;
    prog.native_ops = 0;
    prog.needs_jit = false;

    // Parse the expression.
    auto ast = ast_parse_string(expr, nLen);
    if (!ast) {
        s_jit_stats.compile_fail++;
        return prog;
    }

    // Static function-nesting depth watermark (#1002): the maximum
    // AST_FUNCCALL nesting the AST evaluator would reach.  jit_eval
    // declines the run when live func_nest_lev + this watermark would
    // trip function_recursion_limit, so the AST reproduces the limit
    // error the flattened compiled code cannot.
    //
    // Outer-AST-only for now; inlined u()/ulocal() body watermarks are
    // accumulated during lowering and added after (#1056).
    prog.max_func_depth = ast_max_funccall_depth(ast.get());

    // Static invocation-count watermark: total FUNCCALL nodes.  Flattened
    // JIT does not maintain func_invk_ctr for sequential calls, so decline
    // when live ctr + this count would trip function_invocation_limit.
    prog.n_func_calls = ast_funccall_count(ast.get());

    // --- HIR pipeline ---

    // Phase 1: Lower AST → HIR.
    rv_compiler rc(code_base, str_start, str_lim, fargs_start, fargs_lim, out_start);

    // Tier 2 blob is NOT installed here — compilation never reads
    // the blob region.  Callers that use prog.memory for runtime
    // execution install it after compile_expression() returns.

    // Set compile-time eval flags for unknown-function resolution.
    s_compile_eval = eval;
    s_fcheck_available = (eval & EV_FCHECK) != 0;

    hir_program h;
    h.init();
    qreg_init();

    // Set up Tier 3 compile-time deps collector.
    std::vector<compiled_program::inline_dep> deps;
    s_compile_deps = &deps;
    s_inline_depth = 0;

    h.result = hir_lower_node(h, rc, ast.get());

    // Fold inlined-body watermarks into the program stats (#1056).
    prog.max_func_depth += h.inline_extra_depth;
    prog.n_func_calls   += h.inline_extra_calls;

    // Convert native scalar results to strings for MUX output.
    // This is the boundary where non-string values escape to
    // top-level string context.
    if (h.result >= 0 && h.ty[h.result] == TY_INT) {
        h.result = h.emit(HIR_ITOA, TY_STRING, h.result);
    } else if (h.result >= 0 && h.ty[h.result] == TY_FLOAT) {
        h.result = h.emit(HIR_FTOA, TY_STRING, h.result);
    }

    s_compile_deps = nullptr;

    // If any HIR capacity limit was hit during lowering, the program
    // contains -1 instruction/block indices that later phases would
    // dereference (#859).  Bail out — the AST evaluator handles it.
    // refused_index() is the consumer-side half of the same condition (#1501):
    // overflowed says a producer handed out -1, refused_index() says something
    // then used it as a subscript.  Either way this program is not safe to
    // codegen, and the AST evaluator answers instead.
    //
    if (h.overflowed || h.refused_index()) {
        s_jit_stats.compile_fail++;
        return prog;  // prog.ok is still false
    }

    const char *dump_env = getenv("TINYMUX_DUMP_HIR");
    bool bDump = (dump_env && *dump_env != '0');

    if (bDump) {
        printf("\n--- JIT Compilation: %.*s ---\n", static_cast<int>(nLen), expr);
        printf("Phase 1: HIR Lowering\n");
        hir_dump(h);
    }

    // Phase 2: SSA construction (for multi-block programs, M4+).
    // For single-block programs this is a no-op but builds the CFG.
    hir_build_cfg(h);
    if (h.n_blocks > 1) {
        hir_superblock(h);
        if (bDump) {
            printf("Phase 1b: Superblock Formation\n");
            hir_dump(h);
        }

        hir_ssa_construct(h);
        if (bDump) {
            printf("Phase 2: SSA Construction\n");
            hir_dump(h);
        }

        // SSA construction can exhaust capacity on its own, and the
        // check at the end of lowering (above) has already run by this
        // point.  hir_insert_phis() sets overflowed when it cannot
        // reserve parg slots (#1149), and the emit()/emit_phi() guards
        // in hir.h fire from here too.  Without this check the flag is
        // written and never read: renaming and codegen proceed over a
        // program whose PHI insertion stopped partway.
        //
        if (h.overflowed || h.refused_index()) {
            s_jit_stats.compile_fail++;
            return prog;  // prog.ok is still false
        }
    }

    // Phase 3: SSA optimization (constant fold, copy prop, DCE).
    hir_optimize(h);
    if (bDump) {
        printf("Phase 3: SSA Optimization\n");
        hir_dump(h);
    }

    // Phase 4: Codegen HIR → RV64.
    hir_codegen(h, rc);

    // Record what this compile WANTED, before any ceiling can reject it.
    //
    // code_bytes_max below is taken after the checks and so is censored at
    // CODE_LIMIT: it cannot report a program that did not fit, which is
    // precisely the program a resize decision needs to see (#2074).
    //
    {
        const uint64_t want_code =
            static_cast<uint64_t>(rc.code.size()) * 4;
        if (want_code > s_jit_stats.want_code_max) {
            s_jit_stats.want_code_max = want_code;
        }
        if (rc.str_want > s_jit_stats.want_strpool_max) {
            s_jit_stats.want_strpool_max = rc.str_want;
        }
        if (rc.fargs_want > s_jit_stats.want_fargs_max) {
            s_jit_stats.want_fargs_max = rc.fargs_want;
        }
        if (rc.out_want > s_jit_stats.want_outslots_max) {
            s_jit_stats.want_outslots_max = rc.out_want;
        }
    }

    // Check for code overflow before copying.
    if (rc.code.size() * 4 > rv_compiler::CODE_LIMIT) {
        s_jit_stats.compile_fail++;
        s_jit_stats.bail_code++;
        s_jit_stats.bail_slots++;   // imprecise; kept for compatibility
        return prog;  // prog.ok is still false
    }

    // Copy code to guest memory at the configured code_base.
    for (size_t i = 0; i < rc.code.size(); i++) {
        memcpy(rc.memory.data() + rc.code_base + i * 4, &rc.code[i], 4);
    }

    // If any resource was exhausted during compilation, the generated
    // code references address 0 and would corrupt guest memory.
    // Bail out — the AST evaluator will handle this expression.
    if (rc.out_exhausted || rc.pool_exhausted) {
        s_jit_stats.compile_fail++;
        // Attribute to the ceiling that actually ran out.  More than one
        // can be set in a single compile, so these are counted
        // independently rather than as an if/else chain -- the totals are
        // per-ceiling occurrences, not a partition of compile_fail.
        if (rc.str_exhausted)   { s_jit_stats.bail_strpool++; }
        if (rc.fargs_exhausted) { s_jit_stats.bail_fargs++; }
        if (rc.out_exhausted)   { s_jit_stats.bail_outslots++; }
        if (!rc.bail_was_noeval) {
            s_jit_stats.bail_slots++;   // imprecise; kept for compatibility
        }
        return prog;  // prog.ok is still false
    }

    prog.memory = std::move(rc.memory);
    prog.memory_size = rv_compiler::MEM_SIZE;
    prog.out_addr = rc.final_out;
    prog.out_used = (rv_compiler::STACK_TOP - 8) - rc.out_pool;
    prog.entry_pc = rc.code_base;
    prog.code_size = rc.code.size() * 4;
    prog.str_pool_end = rc.str_pool;
    prog.fargs_pool_end = rc.fargs_pool;
    prog.out_pool_end = rc.out_pool;
    prog.ok = true;
    s_jit_stats.compile_ok++;

    // Code size tracking.
    uint64_t code_bytes = rc.code.size() * 4;
    s_jit_stats.code_bytes_total += code_bytes;
    if (code_bytes > s_jit_stats.code_bytes_max)
        s_jit_stats.code_bytes_max = code_bytes;
    s_jit_stats.hir_insns_total += static_cast<uint64_t>(h.n_insns);
    if (static_cast<uint64_t>(h.n_insns) > s_jit_stats.hir_insns_max)
        s_jit_stats.hir_insns_max = static_cast<uint64_t>(h.n_insns);
    s_jit_stats.spills_total += static_cast<uint64_t>(rc.spills);

    prog.folds = h.folds;
    prog.ecalls = h.ecalls;
    prog.tier2_calls = h.tier2_calls;
    prog.native_ops = h.native_ops;
    prog.needs_jit = h.needs_jit || rc.needs_jit;

    // Classify every runtime substitution/carg reference emitted during
    // lowering into a per-program mask, so run_cached_program populates only
    // the guest CARGS/SUBST slots this program actually reads.  emit_sref is
    // the single choke point for these references, so this set is complete.
    prog.subst_mask = 0;
    prog.cargs_used = 0;
    for (uint64_t a : h.sref_addrs) {
        if (a >= rv_compiler::CARGS_BASE
            && a < rv_compiler::CARGS_BASE
                   + static_cast<uint64_t>(rv_compiler::MAX_CARGS)
                     * rv_compiler::CARGS_SLOT) {
            int idx = static_cast<int>(
                (a - rv_compiler::CARGS_BASE) / rv_compiler::CARGS_SLOT);
            if (idx + 1 > prog.cargs_used) prog.cargs_used = idx + 1;
        } else if (a >= rv_compiler::SUBST_BASE
            && a < rv_compiler::SUBST_BASE
                   + static_cast<uint64_t>(rv_compiler::SUBST_COUNT)
                     * rv_compiler::SUBST_SLOT) {
            int slot = static_cast<int>(
                (a - rv_compiler::SUBST_BASE) / rv_compiler::SUBST_SLOT);
            prog.subst_mask |= (UINT64_C(1) << slot);
        }
    }

    prog.deps = std::move(deps);
    return prog;
}

static uint64_t resolve_runtime_out_addr(uint64_t out_addr, uint64_t entry_sp) {
    return rv_compiler::resolve_output_addr(out_addr, entry_sp);
}

// ---------------------------------------------------------------
// fun_rveval: softcode function.
//
// rveval(<expression>)
//
// Parses and compiles <expression> to RV64, runs through the JIT,
// returns the result.
//
// Examples:
//   think rveval(add(1,2))           → 3   (constant folded)
//   think rveval(add(mul(3,4),5))    → 17  (fully folded)
//   think rveval(strlen(hello))      → 5   (folded)
// ---------------------------------------------------------------

// ---------------------------------------------------------------
// Persistent DBT state — avoids mmap/munmap per call.
//
// The 64 MB mmap + 1 MB block cache allocation dominated the
// ECALL path.  By keeping the dbt_state_t alive, we amortize
// the cost to one-time initialization.
// ---------------------------------------------------------------


static int dbt_trace_mask_from_env() {
    const char *env = getenv("TINYMUX_DBT_TRACE");
    if (!env || !env[0]) return 0;

    if (strcmp(env, "1") == 0 || strcmp(env, "all") == 0) {
        return DBT_TRACE_EXEC | DBT_TRACE_TRANSLATE;
    }

    int mask = 0;
    if (strstr(env, "exec")) mask |= DBT_TRACE_EXEC;
    if (strstr(env, "xlate") || strstr(env, "translate")) {
        mask |= DBT_TRACE_TRANSLATE;
    }
    return mask;
}

static void dbt_configure_trace_from_env(dbt_state_t *dbt) {
    dbt->trace = dbt_trace_mask_from_env();
    dbt->trace_guest_pc = 0;
    dbt->trace_guest_pc_filter = false;

    const char *env = getenv("TINYMUX_DBT_TRACE_PC");
    if (!env || !env[0]) return;

    errno = 0;
    char *end = nullptr;
    unsigned long long value = strtoull(env, &end, 0);
    if (errno != 0 || end == env || *end != '\0') {
        fprintf(stderr,
                "dbt: ignoring invalid TINYMUX_DBT_TRACE_PC='%s'\n", env);
        return;
    }

    dbt->trace_guest_pc = static_cast<uint64_t>(value);
    dbt->trace_guest_pc_filter = true;
}

// One complete execution context for a cached program: the guest memory the
// program is materialized into, and the DBT that translates and runs it.
//
// There are two (#1326).  Everything in here is per-execution state that a
// nested run would otherwise destroy for the run above it: a nested program
// materializes its own code/strings over the outer program's, and resetting
// the shared DBT throws away the outer program's translated blocks while its
// frames are still live.  That is why run_cached_program used to refuse
// outright, which made softcode JIT -> fun_lua -> Lua JIT fall back to the
// Lua interpreter for every nested call.
//
// Index is the nesting depth, so depth 0 is softcode's and depth 1 is the
// nested (in practice Lua) one.  Depth 2 still refuses: a second nesting
// would need a third context, and nothing reaches it today.
//
// The cost is nothing until nesting actually happens.  Both the buffer
// (MEM_SIZE) and the DBT's code buffer (CODE_BUF_SIZE) are allocated on first
// use -- vector::resize and dbt_init respectively -- so an installation whose
// softcode never calls lua() never pays for the second context.
//
// Guest code slots (#2129).
//
// Every cached program is compiled at the same canonical base, so two
// programs collide at the same guest PCs and the DBT had to reset (throw
// away all translated blocks) on every program switch.  One uint64_t of
// "which program was last" made the translated-block cache hold exactly ONE
// program: two expressions alternating re-translated on every evaluation,
// a fixed +22..76 µs per command that no single-expression benchmark could
// see.
//
// The fix is to make the guest PC itself the disambiguator, the same model
// the shared heap and persistent_vm already use: materialize each program's
// CODE at its own 16 KB slot, so the PC-keyed block cache (and the inline
// lookup emitted into host code, which must not grow a tag) holds all
// resident programs at once.  Only the code moves — str/fargs DATA keeps
// swapping at its canonical addresses, because data does not affect
// translation validity, only the code bytes at translate time do.
//
// Slot addresses live in the two spans this arena leaves unallocated, both
// within JAL range (±1 MB) of the blob so the relocated Tier 2 calls still
// encode:
//
//   slot 0            0x00000            canonical (delta 0, no relocation;
//                                        the only slot PINNED programs use)
//   slots 1-4         0x40000..0x50000   between BLOB_LIMIT and LUA_ARRAY
//   slots 5-6         0x60000..0x68000   between LUA_ARRAY_LIMIT and CARGS
//
// (persistent_vm places its pools at 0x40000 too, but in its OWN arena —
// reusing addresses across arenas is free; only overlap within one image
// matters.)
//
static constexpr int JIT_CODE_SLOTS = 7;
static constexpr uint64_t JIT_SLOT_BASE[JIT_CODE_SLOTS] = {
    rv_compiler::CODE_BASE,
    0x40000, 0x44000, 0x48000, 0x4C000,
    0x60000, 0x64000,
};

static_assert(rv_compiler::CODE_LIMIT == 0x4000,
              "slot stride below assumes 16 KB code regions");
static_assert(0x40000 >= rv_compiler::BLOB_LIMIT
              && 0x4C000 + 0x4000 <= rv_compiler::LUA_ARRAY_BASE,
              "slots 1-4 must fit between the blob and the Lua array");
static_assert(0x60000 >= rv_compiler::LUA_ARRAY_LIMIT
              && 0x64000 + 0x4000 <= rv_compiler::CARGS_BASE,
              "slots 5-6 must fit between the Lua array and CARGS");
static_assert(0x64000 + 0x4000 <= rv_compiler::BLOB_BASE + (1 << 20),
              "every slot PC must keep the blob within JAL range");

struct jit_run_vm {
    // Guest memory a program is materialized into.  Tier 2 is installed once,
    // when the buffer is first sized.
    guest_memory_t buffer;
    bool           buffer_ready = false;

    // program_id whose str/fargs DATA blobs currently occupy the canonical
    // pool addresses.  materialize_data sets it; run_cached_program skips
    // the data memcpy when the pools already hold the program being run.
    // 0 = unknown/none (program_ids start at 1).
    uint64_t       buffer_program_id = 0;

    // The DBT translating out of `buffer`.  Translated blocks for every
    // slot-resident program coexist in it, keyed by their slotted PCs.
    dbt_state_t    dbt;
    bool           dbt_ready = false;

    // True while the DBT is bound to some other guest image entirely
    // (run_compiled points it at a program's own uncompacted memory).
    // The next run_cached_program must dbt_reset back to `buffer` and
    // forget every slot rather than trust translations made against
    // foreign bytes.
    bool           dbt_foreign = false;

    // Which program's code occupies each slot.  Residency means only that
    // the code BYTES are materialized in the guest slot; translations are
    // rebuilt lazily via block-cache misses, so DBT-level reclaims need no
    // bookkeeping here.
    //
    // Admission policy (#2130 campaign, from Kagura's #2139 review): slot 0
    // is the PROBATION lane — always claimable, where first-touch and
    // pinned programs run — while slots 1+ are PROTECTED: evictable only
    // when cold (their `hot` bit was not set since the last sweep).  A
    // strict LRU admitted every first-touch program straight into a
    // protected slot, so round-robin one past capacity evicted the entry
    // needed next on every single evaluation — 100% miss, ~8% WORSE than
    // the old unconditional reset.  With probation, a cold storm thrashes
    // only slot 0 (the old per-switch cost, no worse) and the hot working
    // set keeps its translations through it.  The sweep runs every
    // JIT_SLOT_SWEEP_PERIOD misses, so "hot" means "used within the last
    // couple of windows" and a program that went idle becomes evictable
    // instead of squatting.
    struct code_slot {
        uint64_t program_id = 0;   // 0 = free
        uint64_t stamp = 0;        // LRU tick of last use
        bool     hot = false;      // used since the last sweep
    };
    code_slot      slots[JIT_CODE_SLOTS];
    uint64_t       slot_stamp = 0;
    uint32_t       misses_since_sweep = 0;

    void release_all_slots() {
        for (int i = 0; i < JIT_CODE_SLOTS; i++) {
            slots[i] = code_slot{};
        }
        misses_since_sweep = 0;
        buffer_program_id = 0;
    }
};

// Deepest nesting level that has its own context.  run_cached_program refuses
// beyond this.  Sizes s_vm below, which s_run_cached_depth indexes -- raising
// this must not be able to leave the array behind it.
//
static constexpr int JIT_MAX_RUN_DEPTH = 2;

static jit_run_vm s_vm[JIT_MAX_RUN_DEPTH];

// Forget a program in every context that has it slot-resident — called when
// its compiled_program is dropped (compile-cache eviction, staleness).  The
// slot's stale block-cache entries need no eviction here: program_ids are
// unique for the life of the process, so nothing can claim the residency,
// and reassigning the slot to any other program invalidates its PC range
// before trusting it (pick_code_slot).
//
static void release_program_slots(uint64_t program_id) {
    for (int vi = 0; vi < JIT_MAX_RUN_DEPTH; vi++) {
        for (int si = 0; si < JIT_CODE_SLOTS; si++) {
            if (s_vm[vi].slots[si].program_id == program_id) {
                s_vm[vi].slots[si] = jit_run_vm::code_slot{};
            }
        }
        if (s_vm[vi].buffer_program_id == program_id) {
            s_vm[vi].buffer_program_id = 0;
        }
    }
}

// Current nesting depth of run_cached_program, and therefore the index of the
// first context NOT in use by a live run.  Declared here rather than beside
// run_cached_program because the compile path needs it too: materializing into
// a context that a shallower run owns overwrites the memory that run is
// executing out of (#1326).
static int s_run_cached_depth = 0;

// Defined with the Tier 3 helper save stacks below; called from
// run_cached_program at top-level entry.
static void jit_helper_stacks_reset();

// Claims the current context for the duration of a run, so anything the run
// re-enters picks the next one instead.  Hoisted to file scope because BOTH
// entry points into the DBT need it: run_cached_program (jit_eval's route)
// and run_compiled (rvbench's).  run_compiled owning s_vm[0] without
// claiming it was #2106 -- a nested run then reset the very DBT the outer
// run's frames were executing from.
//
struct RunDepthGuard {
    int &depth;
    explicit RunDepthGuard(int &d) : depth(d) { ++depth; }
    ~RunDepthGuard() { --depth; }
    RunDepthGuard(const RunDepthGuard &) = delete;
    RunDepthGuard &operator=(const RunDepthGuard &) = delete;
};

// Release the persistent DBT state on shutdown.
//
void dbt_compile_cleanup(void) {
    for (int i = 0; i < JIT_MAX_RUN_DEPTH; i++) {
        if (s_vm[i].dbt_ready) {
            dbt_cleanup(&s_vm[i].dbt);
            s_vm[i].dbt_ready = false;
        }
    }
}

// Get a reset DBT state bound to an arbitrary guest image, initializing on
// first use.  Returns nullptr on allocation failure.
//
// This is run_compiled's binding (rvbench runs straight out of a program's
// own uncompacted memory).  It marks the context FOREIGN: every slot claim
// and translation now refers to someone else's bytes, so the next slotted
// run must reset back to the vm's own buffer and forget the slots.
//
static dbt_state_t *get_dbt(jit_run_vm *vm,
                             uint8_t *memory, size_t memory_size,
                             int (*ecall_fn)(rv64_ctx_t *, void *),
                             void *ecall_user) {
    dbt_state_t *dbt = &vm->dbt;
    vm->dbt_foreign = true;
    vm->release_all_slots();
    if (!vm->dbt_ready) {
        if (dbt_init(dbt, memory, memory_size, ecall_fn, ecall_user) != 0) {
            return nullptr;
        }
        dbt_configure_trace_from_env(dbt);

        // Dispatch limit: safety net during development.
        // TINYMUX_DBT_MAX_DISPATCH overrides; 0 = unlimited.
        const char *md_env = getenv("TINYMUX_DBT_MAX_DISPATCH");
        dbt->max_dispatch = md_env ? strtoull(md_env, nullptr, 0) : 10000000;
        dbt->alarm_flag = &alarm_clock.alarmed;  // wall-clock abort (#JIT-alarm)

        vm->dbt_ready = true;
        return dbt;
    }
    // Reset for new program: keep mmap'd code buffer + cache allocation.
    dbt_reset(dbt, memory, memory_size, ecall_fn, ecall_user);
    dbt_configure_trace_from_env(dbt);
    return dbt;
}

// ECALL handler context and forward declaration.
//

static int eval_ecall(rv64_ctx_t *ctx, void *user_data);
static int poc_ecall(rv64_ctx_t *ctx, void *user_data);

// Bind the vm's DBT to its own runtime buffer for a slotted run (#2129).
// Unlike get_dbt this does NOT reset on every program switch — that reset
// is exactly what made the block cache one-program-deep.  It initializes
// on first use, resets only to recover from a foreign binding, and
// otherwise leaves every resident program's translations alone; the caller
// sets the per-run ECALL context via dbt_rerun.
//
// The caller must have runtime_buffer_init'd the vm first: the blob bytes
// have to be in the buffer before anything is translated out of it.
//
static dbt_state_t *bind_run_dbt(jit_run_vm *vm) {
    dbt_state_t *dbt = &vm->dbt;
    if (!vm->dbt_ready) {
        if (dbt_init(dbt, vm->buffer.data(), rv_compiler::MEM_SIZE,
                     eval_ecall, nullptr) != 0) {
            return nullptr;
        }
        dbt_configure_trace_from_env(dbt);

        const char *md_env = getenv("TINYMUX_DBT_MAX_DISPATCH");
        dbt->max_dispatch = md_env ? strtoull(md_env, nullptr, 0) : 10000000;
        dbt->alarm_flag = &alarm_clock.alarmed;  // wall-clock abort (#JIT-alarm)

        vm->dbt_ready = true;
        vm->dbt_foreign = false;
        vm->release_all_slots();
        return dbt;
    }
    if (vm->dbt_foreign) {
        dbt_reset(dbt, vm->buffer.data(), rv_compiler::MEM_SIZE,
                  eval_ecall, nullptr);
        dbt_configure_trace_from_env(dbt);
        vm->dbt_foreign = false;
        vm->release_all_slots();
    }
    return dbt;
}

struct persistent_vm_t {
    // Pool layout within THIS arena's image (#2124).
    //
    //   code   0x00004 .. 0x10000   bump-allocated, bounded by BLOB_BASE
    //   blob   0x10000 .. 0x40000   installed once
    //   str    0x40000 .. 0x48000   (32 KB)
    //   fargs  0x48000 .. 0x4C000   (16 KB)
    //
    // These are deliberately NOT the rv_compiler one-shot constants, which
    // is what they used to be.  The code heap here bump-allocates upward
    // from 0x0004 across many compilations, while STR_BASE/FARGS_BASE sit
    // at 0x8000/0x4000 -- inside the range the code heap grows through.
    // Nothing bounded that: install_code() wrote unchecked, and
    // arena_nearly_full() called the code heap's capacity
    // BLOB_BASE - 0x0004, which is only true once the pools are elsewhere.
    //
    // Each arena owns a separate MEM_SIZE image (persistent_vm_t,
    // shared_heap_t, and each jit_run_vm::buffer), so reusing addresses
    // ACROSS arenas is free; only overlap within one image matters.  These
    // sit in the span left unallocated between the blob and LUA_ARRAY,
    // which is why they can be disjoint without moving anything else.
    //
    static constexpr uint64_t PVM_CODE_START  = 0x0004;   // avoid PC=0
    static constexpr uint64_t PVM_CODE_LIMIT  = rv_compiler::BLOB_BASE;
    static constexpr uint64_t PVM_STR_BASE    = 0x40000;
    static constexpr uint64_t PVM_STR_LIMIT   = 0x48000;
    static constexpr uint64_t PVM_FARGS_BASE  = 0x48000;
    static constexpr uint64_t PVM_FARGS_LIMIT = 0x4C000;

    static_assert(PVM_CODE_LIMIT <= rv_compiler::BLOB_BASE,
                  "pvm code heap must not reach the blob");
    static_assert(PVM_STR_BASE >= rv_compiler::BLOB_LIMIT,
                  "pvm string pool must sit above the blob");
    static_assert(PVM_STR_LIMIT == PVM_FARGS_BASE,
                  "pvm string pool must abut its fargs pool");
    static_assert(PVM_FARGS_LIMIT <= rv_compiler::LUA_ARRAY_BASE,
                  "pvm pools must not reach the pinned Lua array region");

    std::vector<uint8_t> memory;
    dbt_state_t dbt;
    bool dbt_ready;
    uint32_t run_count;
    uint32_t dbt_buffer_resets;   // times the x86 translation buffer was reclaimed

    // Re-entrancy guard for the code arena. compile_attr() may only reclaim
    // (reset) the arena when no compiled code from it is on the call stack,
    // i.e. exec_depth == 0. Pressure detected mid-run is deferred via
    // reset_pending and serviced when the outermost run() unwinds.
    int exec_depth;
    bool reset_pending;

    // Code heap: bump allocator for code sections.
    uint64_t code_heap_next;

    // Shared pool cursors — advance across compilations so each
    // function gets its own non-overlapping region.
    uint64_t str_pool_next;
    uint64_t fargs_pool_next;
    uint64_t out_pool_next;

    // Compiled attribute cache: maps (obj, attr_num) → compiled info.
    // Checked for staleness via mod_count on each lookup.
    //
    struct attr_cache_entry {
        uint32_t mod_count;
        uint64_t entry_pc;
        uint64_t out_addr;
        bool     needs_jit;
    };
    static uint64_t attr_cache_key(dbref obj, int attr_num) {
        return (static_cast<uint64_t>(static_cast<uint32_t>(obj)) << 32)
             | static_cast<uint32_t>(attr_num);
    }
    std::unordered_map<uint64_t, attr_cache_entry> attr_cache;

    // Track the worst-case output allocation across all compilations.
    uint64_t worst_out_pool;

    persistent_vm_t()
        : memory(rv_compiler::MEM_SIZE, 0),
          dbt_ready(false), run_count(0), dbt_buffer_resets(0),
          exec_depth(0), reset_pending(false),
          code_heap_next(PVM_CODE_START),  // avoid PC=0 (cache sentinel)
          str_pool_next(PVM_STR_BASE),
          fargs_pool_next(PVM_FARGS_BASE),
          out_pool_next(rv_compiler::STACK_TOP - 8),
          worst_out_pool(rv_compiler::STACK_TOP - 8) {}

    // Compile an expression and install it into persistent memory.
    // Returns {entry_pc, out_addr} on success, {0, 0} on failure.
    //
    struct compile_result {
        uint64_t entry_pc;
        uint64_t out_addr;
        bool     needs_jit;
    };

    compile_result compile(const UTF8 *expr, size_t len,
                           int eval = EV_FCHECK | EV_EVAL) {
        tier2_lazy_init();

        // No blob, no persistent-VM compiles — same clean decline as
        // jit_eval (#875).
        if (!s_tier2.loaded) {
            return {0, 0, false};
        }

        // Bounds check: code heap must not reach blob region.
        if (code_heap_next >= PVM_CODE_LIMIT) {
            return {0, 0, false};
        }

        // Reset the output pool for each compilation.  Output slots
        // are stack-allocated at runtime via the prologue, so each
        // expression can reuse the same output addresses.
        uint64_t out_start = rv_compiler::STACK_TOP - 8;

        compiled_program prog = compile_expression(
            expr, len, eval,
            code_heap_next,
            str_pool_next, PVM_STR_LIMIT,
            fargs_pool_next, PVM_FARGS_LIMIT,
            out_start);

        if (!prog.ok) return {0, 0, false};

        // Post-compilation overflow check.
        uint64_t code_end = prog.entry_pc + prog.code_size;
        if (code_end > rv_compiler::BLOB_BASE) {
            return {0, 0, false};
        }

        install(prog);
        return {prog.entry_pc, prog.out_addr, prog.needs_jit};
    }

    // Compile an attribute body, with caching and staleness checks.
    // Returns {entry_pc, out_addr} on success, {0, 0} on failure.
    //
    compile_result compile_attr(dbref obj, int attr_num,
                                const UTF8 *body, size_t body_len,
                                int eval = EV_FCHECK | EV_EVAL) {
        // Check cache.
        uint32_t mc = attr_mod_count_get(obj, attr_num);
        uint64_t key = attr_cache_key(obj, attr_num);
        auto it = attr_cache.find(key);
        if (it != attr_cache.end() && it->second.mod_count == mc) {
            return {it->second.entry_pc, it->second.out_addr,
                    it->second.needs_jit};
        }
        // Stale or missing. The previous compilation of this attribute (if
        // any) leaves its code/string/fargs regions stranded — recompiling
        // bump-allocates fresh space and never reclaims the old region. Before
        // that pushes the arena toward exhaustion (after which the JIT bails
        // for good), reclaim it wholesale: immediately if nothing is executing
        // on this VM, otherwise defer to the next safe point (the outermost
        // run() unwind) via reset_pending.
        if (arena_nearly_full())
        {
            if (0 == exec_depth)
            {
                reset_arena();
            }
            else
            {
                reset_pending = true;
            }
        }

        // Compile.
        compile_result cr = compile(body, body_len, eval);
        if (!cr.entry_pc) return {0, 0, false};

        // Cache (insert or overwrite).
        attr_cache_entry entry;
        entry.mod_count = mc;
        entry.entry_pc = cr.entry_pc;
        entry.out_addr = cr.out_addr;
        entry.needs_jit = cr.needs_jit;
        attr_cache[key] = entry;

        return cr;
    }

    // Install a hand-assembled code blob at the current code heap
    // position.  Returns the entry_pc.
    //
    uint64_t install_code(const std::vector<uint32_t> &code) {
        uint64_t entry = code_heap_next;
        // The memcpy below is otherwise unbounded -- it wrote wherever the
        // cursor pointed, and the only check on this path lived in
        // compile(), which install_code() does not go through (#2124).
        //
        if (entry + code.size() * 4 > PVM_CODE_LIMIT) {
            return 0;   // 0 is the caller's "no entry" sentinel
        }
        for (size_t i = 0; i < code.size(); i++) {
            memcpy(memory.data() + entry + i * 4, &code[i], 4);
        }
        code_heap_next = entry + code.size() * 4;
        code_heap_next = (code_heap_next + 15) & ~15ULL;
        return entry;
    }

    // Initialize the DBT (first time) or update ECALL handler.
    //
    bool ensure_dbt() {
        if (!dbt_ready) {
            if (dbt_init(&dbt, memory.data(), memory.size(),
                         poc_ecall, &dbt) != 0) {
                return false;
            }
            dbt_ready = true;
            tier2_install(memory, rv_compiler::BLOB_BASE);
            pretranslate_tier2(&dbt);
            dbt.blob_code_end = dbt.code_used;
        }
        return true;
    }

    // Prepare for execution: clear output buffers and reset blob
    // writable state.  Code and rodata are immutable — only the
    // data section and BSS need resetting between runs.
    //
    void prepare_run() {
        if (worst_out_pool < rv_compiler::STACK_TOP - 8) {
            memset(memory.data() + worst_out_pool, 0,
                   (rv_compiler::STACK_TOP - 8) - worst_out_pool);
        }
        if (s_tier2.loaded) {
            tier2_reset_writable(memory, rv_compiler::BLOB_BASE);
        }
        s_heap_next = rv_compiler::HEAP_BASE;  // reset per-eval heap arena
    }

    // Run a compiled function.  Returns 0 on success.
    // First call uses dbt_run (zeroes ctx); subsequent use dbt_resume.
    //
    int run(uint64_t entry_pc) {
        // PC=0 is the reserved "no entry" sentinel -- the code heap starts
        // at PVM_CODE_START precisely to keep 0 free.  install_code() and
        // compile() both return it on failure, so running it would execute
        // whatever happens to sit at guest address 0 (#2124).
        //
        if (0 == entry_pc) {
            return -1;
        }
        // Track that compiled code from this arena is on the stack so a
        // re-entrant compile_attr() does not reclaim the arena out from under
        // a running program. A reclaim requested mid-run is serviced once the
        // outermost run() unwinds (exec_depth back to 0).
        exec_depth++;
        dbt_rerun(&dbt, poc_ecall, &dbt);
        int rc;
        if (run_count == 0) {
            run_count++;
            rc = dbt_run(&dbt, entry_pc, rv_compiler::STACK_TOP);
        } else {
            dbt.ctx.x[2] = rv_compiler::STACK_TOP;
            run_count++;
            rc = dbt_resume(&dbt, entry_pc);
        }
        exec_depth--;
        if (0 == exec_depth) {
            if (reset_pending) {
                // Guest arena pressure (or a deferred request) — full reclaim,
                // which also clears the DBT translation buffer.
                reset_arena();
            } else if (dbt_buffer_nearly_full()) {
                // The DBT's x86 translation buffer filled at RUN time: new
                // guest blocks (data-dependent paths through already-compiled
                // attributes) keep translating even when no new compilation
                // happens, so the compile-time arena check never sees it.  Once
                // code_used pins at full, translate_block bails and those paths
                // silently degrade to the interpreter forever.  Reclaim just the
                // translation buffer (the guest arena + attr cache stay valid;
                // blocks re-translate lazily) at this safe (exec_depth == 0)
                // point.
                reset_dbt_buffer();
            }
        }
        return rc;
    }

    // True when the DBT's x86-64 translation buffer (separate from the guest
    // RV64 arena pools) has crossed a 7/8 high-water mark and holds program
    // blocks worth reclaiming.  Guarded so we never thrash: there must be
    // translation above the permanent blob region, and a reset (which rewinds
    // to blob_code_end) must leave real headroom.
    bool dbt_buffer_nearly_full() const {
        if (!dbt_ready) return false;
        uint64_t used = dbt.code_used;
        uint64_t blob = dbt.blob_code_end;
        uint64_t cap  = CODE_BUF_SIZE;
        return used * 8 >= cap * 7
            && used > blob
            && blob * 8 < cap * 7;
    }

    // Reclaim ONLY the DBT translation buffer — clears the x86 block cache and
    // rewinds code_used to blob_code_end (blob translations preserved), leaving
    // the guest code arena and attr cache intact.  The guest programs stay
    // valid at their PCs and re-translate lazily on next run.  Safe ONLY at
    // exec_depth == 0 (no translated block is on the call stack).
    void reset_dbt_buffer() {
        if (!dbt_ready) return;
        dbt_reset(&dbt, memory.data(), memory.size(), poc_ecall, &dbt);
        run_count = 0;
        dbt_buffer_resets++;
    }

    // True when any arena pool has crossed a 7/8 high-water mark and should be
    // reclaimed before it exhausts.
    bool arena_nearly_full() const {
        uint64_t code_used  = code_heap_next - PVM_CODE_START;
        uint64_t code_cap   = PVM_CODE_LIMIT - PVM_CODE_START;
        uint64_t str_used   = str_pool_next - PVM_STR_BASE;
        uint64_t str_cap    = PVM_STR_LIMIT - PVM_STR_BASE;
        uint64_t fargs_used = fargs_pool_next - PVM_FARGS_BASE;
        uint64_t fargs_cap  = PVM_FARGS_LIMIT - PVM_FARGS_BASE;
        return code_used  * 8 >= code_cap  * 7
            || str_used   * 8 >= str_cap   * 7
            || fargs_used * 8 >= fargs_cap * 7;
    }

    // Reclaim the code arena wholesale. Safe ONLY when exec_depth == 0 (no
    // compiled code from this arena is executing): it rewinds the bump
    // allocators and drops every cached compilation, so all attributes are
    // recompiled lazily on next use. The reused code-heap PCs would otherwise
    // hit stale entries in the DBT block cache, so dbt_reset() evicts the
    // program-code translations (blob translations are preserved) and the
    // run sequence restarts from dbt_run.
    void reset_arena() {
        attr_cache.clear();
        code_heap_next  = PVM_CODE_START;
        str_pool_next   = PVM_STR_BASE;
        fargs_pool_next = PVM_FARGS_BASE;
        worst_out_pool  = rv_compiler::STACK_TOP - 8;
        reset_pending   = false;
        if (dbt_ready) {
            dbt_reset(&dbt, memory.data(), memory.size(), poc_ecall, &dbt);
            run_count = 0;
        }
    }

    // Reset blob BSS between function runs (Tier 2 writable data).
    //
    void reset_blob_bss() {
        if (s_tier2.loaded && s_tier2.bss_size > 0) {
            uint64_t bss_start = rv_compiler::BLOB_BASE
                               + s_tier2.code_size;
            if (bss_start + s_tier2.bss_size <= memory.size()) {
                memset(memory.data() + bss_start, 0, s_tier2.bss_size);
            }
        }
    }

    // Read a NUL-terminated result string from guest memory.
    //
    const char *result(uint64_t out_addr,
                       uint64_t entry_sp = rv_compiler::STACK_TOP) const {
        uint64_t resolved = resolve_runtime_out_addr(out_addr, entry_sp);
        if (resolved == 0 || resolved >= memory.size()) return "";
        return reinterpret_cast<const char *>(memory.data() + resolved);
    }

private:
    // Install a compiled program's regions into persistent memory
    // and advance pool cursors.
    //
    void install(const compiled_program &prog) {
        // Code.
        memcpy(memory.data() + prog.entry_pc,
               prog.memory.data() + prog.entry_pc, prog.code_size);

        // Strings.
        if (prog.str_pool_end > str_pool_next) {
            memcpy(memory.data() + str_pool_next,
                   prog.memory.data() + str_pool_next,
                   prog.str_pool_end - str_pool_next);
        }

        // Fargs.
        if (prog.fargs_pool_end > fargs_pool_next) {
            memcpy(memory.data() + fargs_pool_next,
                   prog.memory.data() + fargs_pool_next,
                   prog.fargs_pool_end - fargs_pool_next);
        }

        // Output is stack-allocated at runtime via the prologue;
        // no install-time copy needed.  Track worst-case for clearing.
        if (prog.out_pool_end < worst_out_pool) {
            worst_out_pool = prog.out_pool_end;
        }

        // Advance code and data cursors only.
        code_heap_next = prog.entry_pc + prog.code_size;
        code_heap_next = (code_heap_next + 15) & ~15ULL;
        str_pool_next = prog.str_pool_end;
        fargs_pool_next = prog.fargs_pool_end;
    }
};


// ---------------------------------------------------------------
// Compile cache — LRU cache of compiled programs.
//
// Keyed by expression text.  Cache hits skip compilation entirely.
// Combined with DBT block cache persistence (dbt_rerun), repeated
// evaluation of the same expression does zero compilation and zero
// JIT translation — just runs the cached native code.
//
// The DBT tracks which program it was last set up for.  Same
// program → dbt_rerun (keep translated blocks).  Different
// program → dbt_reset (re-translate).
// ---------------------------------------------------------------

struct compile_cache_entry {
    compiled_program prog;
    std::list<std::string>::iterator lru_it;
};

static std::unordered_map<std::string, compile_cache_entry> s_compile_cache;
static std::list<std::string> s_compile_lru;

// Capacity is a config knob (jit_compile_cache_max), read at insert time so
// runtime @admin takes effect on the next compile.  The old fixed 256 was a
// cliff, not a slope (#2130): round-robin over an LRU is its worst case, so
// ONE expression past capacity took the hit rate from 99.9% to 0.16% and
// every lookup became a SQLite fetch + full program deserialization
// (~19.4us on the issue's box).  The default covers the ~1500 distinct
// programs a live-workload profile actually showed (#2129 measurement) —
// entries are compact blobs, typically a few KB each.
//
static constexpr int COMPILE_CACHE_DEFAULT = 2048;
static constexpr size_t COMPILE_CACHE_MIN_LEN = 8;

static size_t compile_cache_max(void) {
    int n = mudconf.jit_compile_cache_max;
    if (n < 8) {
        n = 8;
    } else if (n > 65536) {
        n = 65536;
    }
    return static_cast<size_t>(n);
}

// Decline memo (#2130): bail_noop is a pure function of the compiled code,
// so once a shape has been declined there is no reason to ever fetch or
// reconstruct its program again just to look at four integers and not run
// it.  Keyed by the same compile-cache key; checked in jit_eval BEFORE
// compile_cached, so a memoized decline touches neither the memory LRU nor
// SQLite.  Only dep-free programs are memoized — a program with inline
// deps can recompile into a different shape when an attribute changes.
//
// Bounded by wholesale clear rather than an LRU: an entry is ~the
// expression text, a false eviction costs exactly one fetch+decline to
// re-memoize, and the flush hook clears it with the other caches.
//
static std::unordered_set<std::string> s_decline_memo;
static constexpr size_t DECLINE_MEMO_MAX = 8192;

// Drop memoized declines when the function table changes (#2140 review).
// A shape that was bail_noop can recompile into a different shape once a
// softcode @function or builtin registration moves; leaving the memo
// would refuse forever without re-fetching.  Same invalidation trigger as
// the #2068 gate epoch.
//
void jit_decline_memo_invalidate(void)
{
    s_decline_memo.clear();
}

// Track which program the DBT was last set up for, so we can
// use dbt_rerun (fast) instead of dbt_reset (slow) on cache hits.
//
static uint64_t s_next_program_id = 1;

static void runtime_buffer_init(jit_run_vm *vm) {
    if (vm->buffer_ready) return;
    vm->buffer.resize(rv_compiler::MEM_SIZE);
    tier2_install(vm->buffer, rv_compiler::BLOB_BASE);
    vm->buffer_ready = true;
}

// Compact a compiled_program: extract the occupied regions into
// small blob vectors, extract the folded result if constant-folded,
// assign a unique program_id, then release the 4MB memory.
//
static void compact_program(compiled_program &prog) {
    prog.program_id = s_next_program_id++;

    // Extract code blob.
    if (prog.code_size > 0 && prog.entry_pc + prog.code_size <= prog.memory.size()) {
        prog.code_blob.assign(
            prog.memory.data() + prog.entry_pc,
            prog.memory.data() + prog.entry_pc + prog.code_size);
    }

    // Extract string pool blob.
    if (prog.str_pool_end > rv_compiler::STR_BASE) {
        size_t len = static_cast<size_t>(prog.str_pool_end - rv_compiler::STR_BASE);
        prog.str_blob.assign(
            prog.memory.data() + rv_compiler::STR_BASE,
            prog.memory.data() + rv_compiler::STR_BASE + len);
    }

    // Extract fargs pool blob.
    if (prog.fargs_pool_end > rv_compiler::FARGS_BASE) {
        size_t len = static_cast<size_t>(prog.fargs_pool_end - rv_compiler::FARGS_BASE);
        prog.fargs_blob.assign(
            prog.memory.data() + rv_compiler::FARGS_BASE,
            prog.memory.data() + rv_compiler::FARGS_BASE + len);
    }

    // Extract folded result for constant-folded programs.
    if (!prog.needs_jit) {
        uint64_t out_addr = rv_compiler::resolve_output_addr(
            prog.out_addr, rv_compiler::STACK_TOP);
        if (out_addr < prog.memory.size()) {
            prog.folded_result = reinterpret_cast<const char *>(
                prog.memory.data() + out_addr);
        }
    }

    // Release the 4MB vector.
    prog.memory.clear();
    prog.memory.shrink_to_fit();
}

// Materialize a compact program's DATA — string pool and fargs blobs — at
// their canonical addresses.  Data placement is independent of which code
// slot the program occupies: compiled code references the pools absolutely,
// the pools never move, and swapping their CONTENT does not invalidate any
// translation (only code bytes matter at translate time).
//
static void materialize_data(jit_run_vm *vm, const compiled_program &prog) {
    runtime_buffer_init(vm);

    // Record which program's data now occupies the pools so a consecutive
    // re-run of the same program can skip this copy (see run_cached_program).
    vm->buffer_program_id = prog.program_id;

    if (!prog.str_blob.empty()) {
        memcpy(vm->buffer.data() + rv_compiler::STR_BASE,
               prog.str_blob.data(), prog.str_blob.size());
    }

    if (!prog.fargs_blob.empty()) {
        memcpy(vm->buffer.data() + rv_compiler::FARGS_BASE,
               prog.fargs_blob.data(), prog.fargs_blob.size());
    }
}

// Materialize a compact program's CODE at `slot_base` (#2129), re-aiming
// its blob-call JALs by the placement delta.  Classifies the program on
// first use; a PINNED program only ever materializes at the canonical base.
// Returns false if the program cannot be placed at this base.
//
static void classify_reloc(compiled_program &prog) {
    if (RV_RELOC_UNSCANNED != prog.reloc_class) {
        return;
    }
    prog.reloc_class = rv_scan_extern_jals(
        prog.code_blob.data(), prog.code_blob.size(),
        prog.entry_pc, rv_compiler::CODE_LIMIT,
        rv_compiler::BLOB_BASE, rv_compiler::BLOB_LIMIT,
        prog.extern_jals);
    if (RV_RELOC_OK != prog.reloc_class) {
        s_jit_stats.slot_pinned++;
    }
}

static bool materialize_code_slot(jit_run_vm *vm, compiled_program &prog,
                                  uint64_t slot_base) {
    runtime_buffer_init(vm);
    classify_reloc(prog);

    const int64_t delta = static_cast<int64_t>(slot_base)
                        - static_cast<int64_t>(rv_compiler::CODE_BASE);
    if (0 != delta && RV_RELOC_OK != prog.reloc_class) {
        return false;
    }

    if (prog.code_blob.empty()) {
        return true;
    }
    uint8_t *dst = vm->buffer.data() + slot_base + prog.entry_pc;
    memcpy(dst, prog.code_blob.data(), prog.code_blob.size());

    if (0 == delta) {
        return true;
    }
    for (uint32_t off : prog.extern_jals) {
        if (off + 4 > prog.code_blob.size()) {
            return false;   // corrupt metadata; caller pins
        }
        uint32_t w;
        memcpy(&w, dst + off, 4);
        if (!rv_jal_relocate(&w, delta)) {
            return false;   // displacement not encodable at this base
        }
        memcpy(dst + off, &w, 4);
    }
    return true;
}

// Materialize a compact program into the shared runtime buffer at its
// canonical addresses — code and data.  Used by the fold-result reader
// (s_fold_vm), which only needs somewhere to read a string back out of.
//
static void materialize_program(jit_run_vm *vm, const compiled_program &prog) {
    runtime_buffer_init(vm);
    materialize_data(vm, prog);

    if (!prog.code_blob.empty()) {
        memcpy(vm->buffer.data() + prog.entry_pc,
               prog.code_blob.data(), prog.code_blob.size());
    }
}

// How many misses between hot-bit sweeps.  A protected slot whose owner
// did not run within a full window becomes evictable at the sweep; a
// program hitting at least once per window keeps its slot indefinitely.
// Counted in misses, not runs, so a stable all-hits workload never pays a
// sweep and never loses a slot.
//
static constexpr uint32_t JIT_SLOT_SWEEP_PERIOD = 64;

// Pick (or keep) a guest code slot for `prog` (#2129, admission #2130).
//
// A hit means the program's code is already materialized at *slot_base and
// every translation the DBT made for it is still keyed under those PCs —
// the program switch costs a str/fargs memcpy instead of a re-translation.
//
// A miss claims a victim in strict preference order:
//
//   1. a FREE protected slot (1..n-1) — a small working set gets protection
//      immediately, no probation served;
//   2. the free probation slot (0);
//   3. the coldest protected slot that is EVICTABLE — its hot bit not set
//      since the last sweep, i.e. its owner has gone idle;
//   4. slot 0, the probation lane, unconditionally.
//
// Rule 4 is what removes the cliff Kagura measured at working sets one
// past the slot count: a first-touch program cannot displace a hot
// resident, so a round-robin storm over any number of cold programs
// churns only slot 0 (the old per-switch cost) while the hot set keeps
// its translations.  PINNED programs (relocation scan refused) contend
// for slot 0 only — the pre-#2129 behaviour for exactly those programs.
// jit_code_slots=1 collapses everything onto slot 0, the A/B lever.
//
static bool pick_code_slot(jit_run_vm *vm, dbt_state_t *dbt,
                           compiled_program &prog, uint64_t *slot_base) {
    classify_reloc(prog);

    int nslots = mudconf.jit_code_slots;
    if (nslots < 1) {
        nslots = 1;
    } else if (nslots > JIT_CODE_SLOTS) {
        nslots = JIT_CODE_SLOTS;
    }
    if (RV_RELOC_OK != prog.reloc_class) {
        nslots = 1;
    }

    for (int i = 0; i < nslots; i++) {
        if (vm->slots[i].program_id == prog.program_id) {
            vm->slots[i].stamp = ++vm->slot_stamp;
            vm->slots[i].hot = true;
            s_jit_stats.slot_hit++;
            *slot_base = JIT_SLOT_BASE[i];
            return true;
        }
    }

    // Sweep: age the protection.  Runs on miss traffic only — misses are
    // when eviction decisions happen, and a workload with no misses has
    // no reason to decay anything.
    if (++vm->misses_since_sweep >= JIT_SLOT_SWEEP_PERIOD) {
        vm->misses_since_sweep = 0;
        for (int i = 1; i < nslots; i++) {
            vm->slots[i].hot = false;
        }
    }

    int victim = -1;
    for (int i = 1; i < nslots; i++) {          // 1: free protected
        if (0 == vm->slots[i].program_id) {
            victim = i;
            break;
        }
    }
    if (victim < 0 && 0 == vm->slots[0].program_id) {
        victim = 0;                              // 2: free probation
    }
    if (victim < 0) {
        for (int i = 1; i < nslots; i++) {      // 3: coldest evictable
            if (!vm->slots[i].hot
                && (victim < 0
                    || vm->slots[i].stamp < vm->slots[victim].stamp)) {
                victim = i;
            }
        }
    }
    if (victim < 0) {
        victim = 0;                              // 4: probation lane
    }

    if (0 != vm->slots[victim].program_id) {
        if (0 == victim) {
            s_jit_stats.slot_churn0++;
        } else {
            s_jit_stats.slot_evict++;
        }
    }

    // Stale translations at this slot's PCs must be gone before new code
    // claims them — this is the range-scoped version of what dbt_reset did
    // for the whole program region on every switch.
    dbt_invalidate_guest_range(dbt, JIT_SLOT_BASE[victim],
                               JIT_SLOT_BASE[victim] + rv_compiler::CODE_LIMIT);
    vm->slots[victim] = jit_run_vm::code_slot{};

    if (!materialize_code_slot(vm, prog, JIT_SLOT_BASE[victim])) {
        // Placement refused (relocation not encodable).  Retry once at the
        // canonical base, where no relocation is needed.
        if (0 == victim) {
            return false;
        }
        if (0 != vm->slots[0].program_id) {
            s_jit_stats.slot_churn0++;
        }
        dbt_invalidate_guest_range(dbt, JIT_SLOT_BASE[0],
                                   JIT_SLOT_BASE[0] + rv_compiler::CODE_LIMIT);
        vm->slots[0] = jit_run_vm::code_slot{};
        if (!materialize_code_slot(vm, prog, JIT_SLOT_BASE[0])) {
            return false;
        }
        victim = 0;
    }

    vm->slots[victim].program_id = prog.program_id;
    vm->slots[victim].stamp = ++vm->slot_stamp;
    vm->slots[victim].hot = true;
    s_jit_stats.slot_miss++;
    *slot_base = JIT_SLOT_BASE[victim];
    return true;
}

// Reconstruct a compiled_program from a SQLite code cache record.
// Populates compact blob vectors directly — no full 4MB allocation.
//
static compiled_program reconstruct_from_cache(
    const CSQLiteDB::CodeCacheRecord &rec) {
    compiled_program prog;

    const bool has_compact_image =
        rec.code_len > 0
        || rec.str_len > 0
        || rec.fargs_len > 0
        || rec.code_size > 0
        || rec.out_pool_end > 0;

    if (has_compact_image) {
        const int64_t code_end = rec.entry_pc + rec.code_len;
        const int64_t str_end = static_cast<int64_t>(rv_compiler::STR_BASE)
            + rec.str_len;
        const int64_t fargs_end = static_cast<int64_t>(rv_compiler::FARGS_BASE)
            + rec.fargs_len;
        const bool valid_code =
            rec.code_len >= 0
            && rec.code_size >= 0
            && rec.entry_pc >= static_cast<int64_t>(rv_compiler::CODE_BASE)
            && rec.entry_pc <= static_cast<int64_t>(rv_compiler::CODE_LIMIT)
            && code_end >= rec.entry_pc
            && code_end <= static_cast<int64_t>(rv_compiler::CODE_LIMIT)
            && rec.code_size >= rec.code_len;
        const bool valid_str =
            rec.str_len >= 0
            && rec.str_pool_end >= static_cast<int64_t>(rv_compiler::STR_BASE)
            && rec.str_pool_end <= static_cast<int64_t>(rv_compiler::STR_LIMIT)
            && str_end >= static_cast<int64_t>(rv_compiler::STR_BASE)
            && str_end <= static_cast<int64_t>(rv_compiler::STR_LIMIT)
            && rec.str_pool_end >= str_end;
        const bool valid_fargs =
            rec.fargs_len >= 0
            && rec.fargs_pool_end >= static_cast<int64_t>(rv_compiler::FARGS_BASE)
            && rec.fargs_pool_end <= static_cast<int64_t>(rv_compiler::FARGS_LIMIT)
            && fargs_end >= static_cast<int64_t>(rv_compiler::FARGS_BASE)
            && fargs_end <= static_cast<int64_t>(rv_compiler::FARGS_LIMIT)
            && rec.fargs_pool_end >= fargs_end;
        const bool valid_out =
            rec.out_pool_end >= static_cast<int64_t>(rv_compiler::OUT_STACK_LIMIT)
            && rec.out_pool_end <= static_cast<int64_t>(rv_compiler::STACK_TOP - 8);

        if (!valid_code || !valid_str || !valid_fargs || !valid_out) {
            return prog;
        }

        // Populate compact blob vectors directly from the SQLite record.
        if (rec.code_blob && rec.code_len > 0) {
            prog.code_blob.assign(
                static_cast<const uint8_t *>(rec.code_blob),
                static_cast<const uint8_t *>(rec.code_blob) + rec.code_len);
        }

        if (rec.str_blob && rec.str_len > 0) {
            prog.str_blob.assign(
                static_cast<const uint8_t *>(rec.str_blob),
                static_cast<const uint8_t *>(rec.str_blob) + rec.str_len);
        }

        if (rec.fargs_blob && rec.fargs_len > 0) {
            prog.fargs_blob.assign(
                static_cast<const uint8_t *>(rec.fargs_blob),
                static_cast<const uint8_t *>(rec.fargs_blob) + rec.fargs_len);
        }
    } else if (rec.memory_blob && rec.memory_len > 0) {
        // Legacy format: single memory blob covering code+str+fargs.
        // Extract the occupied regions into compact vectors.
        // Cap at the highest of the three pool ends (#2107: STR now sits
        // above FARGS, so FARGS_LIMIT is no longer the image high water).
        int copy_len = rec.memory_len;
        const int layout_end = static_cast<int>(
            (std::max)(rv_compiler::STR_LIMIT, rv_compiler::FARGS_LIMIT));
        if (copy_len > layout_end) {
            copy_len = layout_end;
        }
        const auto *base = static_cast<const uint8_t *>(rec.memory_blob);

        // Code region: [CODE_BASE..CODE_LIMIT)
        if (copy_len > static_cast<int>(rv_compiler::CODE_BASE)) {
            int code_end = (std::min)(copy_len,
                static_cast<int>(rv_compiler::CODE_LIMIT));
            prog.code_blob.assign(base + rv_compiler::CODE_BASE,
                                  base + code_end);
        }

        // String pool: [STR_BASE..STR_LIMIT)
        if (copy_len > static_cast<int>(rv_compiler::STR_BASE)) {
            int str_end = (std::min)(copy_len,
                static_cast<int>(rv_compiler::STR_LIMIT));
            prog.str_blob.assign(base + rv_compiler::STR_BASE,
                                 base + str_end);
        }

        // Fargs pool: [FARGS_BASE..FARGS_LIMIT)
        if (copy_len > static_cast<int>(rv_compiler::FARGS_BASE)) {
            int fargs_end = (std::min)(copy_len,
                static_cast<int>(rv_compiler::FARGS_LIMIT));
            prog.fargs_blob.assign(base + rv_compiler::FARGS_BASE,
                                   base + fargs_end);
        }
    }

    prog.memory_size = rv_compiler::MEM_SIZE;
    prog.out_addr = static_cast<uint64_t>(rec.out_addr);
    prog.out_used = 0;
    prog.entry_pc = has_compact_image
        ? static_cast<uint64_t>(rec.entry_pc)
        : rv_compiler::CODE_BASE;
    prog.code_size = has_compact_image
        ? static_cast<uint64_t>(rec.code_size)
        : 0;
    prog.str_pool_end = has_compact_image && rec.str_pool_end > 0
        ? static_cast<uint64_t>(rec.str_pool_end)
        : rv_compiler::STR_BASE;
    prog.fargs_pool_end = has_compact_image && rec.fargs_pool_end > 0
        ? static_cast<uint64_t>(rec.fargs_pool_end)
        : rv_compiler::FARGS_BASE;
    if (has_compact_image && rec.out_pool_end > 0) {
        prog.out_pool_end = static_cast<uint64_t>(rec.out_pool_end);
    } else {
        prog.out_pool_end = rec.needs_jit
            ? rv_compiler::OUT_STACK_LIMIT
            : rv_compiler::STACK_TOP - 8;
    }
    prog.ok = true;
    prog.needs_jit = rec.needs_jit != 0;
    prog.folds = rec.folds;
    prog.ecalls = rec.ecalls;
    prog.tier2_calls = rec.tier2_calls;
    prog.native_ops = rec.native_ops;
    prog.max_func_depth = static_cast<int>(rec.max_func_depth);
    prog.n_func_calls   = static_cast<int>(rec.n_func_calls);

    // Restore inline dependencies from BLOB.
    if (rec.deps_blob && rec.deps_len > 0)
    {
        int ndeps = rec.deps_len / static_cast<int>(
            sizeof(compiled_program::inline_dep));
        prog.deps.resize(ndeps);
        memcpy(prog.deps.data(), rec.deps_blob,
               ndeps * sizeof(compiled_program::inline_dep));
    }

    // Extract folded result for constant-folded programs.
    // Materialize blobs into the runtime buffer to read the result string.
    //
    // A folded result always lives in the string pool — hir_codegen sets
    // final_out via pool_str() (or an interned str-pool address) precisely
    // so it survives SQLite cache persistence.  Validate the resolved
    // address falls inside the materialized string pool and bound the NUL
    // scan to that region: out_addr comes straight from the cache record,
    // so a malformed/corrupt row could otherwise point it into the high
    // runtime buffer (e.g. the non-NUL DSCRATCH doubles area) and walk the
    // string copy's strlen past the end of the 4 MB buffer (OOB read).
    if (!prog.needs_jit && prog.ok) {
        // A dedicated context, never one of the run contexts (#1326).
        //
        // This path only needs somewhere to read a folded string back out of,
        // but it is reachable *while runs are live*: compile_cached() is called
        // from jit_eval(), jit_eval() from the AST evaluator, and the AST
        // evaluator from ECALLs inside a running program.  Materializing into a
        // context a live run owns overwrites the code and string pool that run
        // is executing out of, and the symptom is not a wrong answer -- the
        // corrupted guest code loops inside a single translated block, so
        // neither max_dispatch nor the ECALL path nor the alarm ever notices.
        // It just stops.
        //
        // Picking "the first context not in use" does not work either: at the
        // depth where runs are refused there is no free context to pick, and
        // clamping lands on one that is live.  So this owns its own, allocated
        // on first use like the others.
        static jit_run_vm s_fold_vm;
        jit_run_vm *vm = &s_fold_vm;
        runtime_buffer_init(vm);
        materialize_program(vm, prog);
        uint64_t out_addr = rv_compiler::resolve_output_addr(
            prog.out_addr, rv_compiler::STACK_TOP);
        if (out_addr < static_cast<uint64_t>(rv_compiler::STR_BASE)
            || out_addr >= prog.str_pool_end
            || prog.str_pool_end > vm->buffer.size()) {
            // Folded out_addr outside the materialized string pool — the
            // record is corrupt; reject it so the caller recompiles.
            prog.ok = false;
            return prog;
        }
        const char *p = reinterpret_cast<const char *>(
            vm->buffer.data() + out_addr);
        size_t maxlen = static_cast<size_t>(prog.str_pool_end - out_addr);
        const void *nul = memchr(p, '\0', maxlen);
        size_t n = nul ? static_cast<size_t>(
            static_cast<const char *>(nul) - p) : maxlen;
        prog.folded_result.assign(p, n);
    }

    // Assign a program ID.
    prog.program_id = s_next_program_id++;

    return prog;
}

static std::string compile_cache_key(const UTF8 *expr, size_t nLen, int eval) {
    // Include every eval flag that changes compile-time semantics.
    int eval_key = eval & (EV_FCHECK | EV_FMAND | EV_STRIP_CURLY);

    std::string key(reinterpret_cast<const char *>(expr), nLen);
    key += '\0';
    key += static_cast<char>(eval_key & 0xFF);
    key += static_cast<char>((eval_key >> 8) & 0xFF);
    return key;
}

// Persist a compiled_program to the SQLite code cache.
//
// Queued for batched execution via cache_flush_writes() — the same
// demand-driven mechanism used for attribute writes.  This avoids
// leaving a transaction open across arbitrary game operations.
//
static void store_to_sqlite_cache(const std::string &key,
                                   const compiled_program &prog) {
    if (!g_pSQLiteBackend) return;

    int persist_len = 0;
    const char *legacy_blob = "";
    const void *code_blob = nullptr;
    int code_len = static_cast<int>(prog.code_size);
    if (code_len > 0) {
        code_blob = prog.memory.data() + prog.entry_pc;
    }

    const void *str_blob = nullptr;
    int str_len = 0;
    if (prog.str_pool_end > rv_compiler::STR_BASE) {
        str_blob = prog.memory.data() + rv_compiler::STR_BASE;
        str_len = static_cast<int>(prog.str_pool_end - rv_compiler::STR_BASE);
    }

    const void *fargs_blob = nullptr;
    int fargs_len = 0;
    if (prog.fargs_pool_end > rv_compiler::FARGS_BASE) {
        fargs_blob = prog.memory.data() + rv_compiler::FARGS_BASE;
        fargs_len = static_cast<int>(prog.fargs_pool_end - rv_compiler::FARGS_BASE);
    }

    cache_queue_code_cache_put(
        key.data(), static_cast<int>(key.size()),
        s_blob_version.data(), static_cast<int>(s_blob_version.size()),
        legacy_blob, persist_len,
        code_blob, code_len,
        static_cast<int64_t>(prog.entry_pc),
        static_cast<int64_t>(prog.code_size),
        str_blob, str_len,
        static_cast<int64_t>(prog.str_pool_end),
        fargs_blob, fargs_len,
        static_cast<int64_t>(prog.fargs_pool_end),
        static_cast<int64_t>(prog.out_pool_end),
        static_cast<int64_t>(prog.out_addr),
        prog.needs_jit ? 1 : 0,
        prog.folds, prog.ecalls,
        prog.tier2_calls, prog.native_ops,
        static_cast<int64_t>(prog.max_func_depth),
        static_cast<int64_t>(prog.n_func_calls),
        prog.deps.data(),
        static_cast<int>(prog.deps.size()
            * sizeof(compiled_program::inline_dep)));
}

// Check if a compiled program's inlined dependencies are still fresh.
// Returns true if all deps match their current mod_counts (or no deps).
//
static bool deps_are_fresh(const compiled_program &prog) {
    for (const auto &dep : prog.deps) {
        uint32_t current = attr_mod_count_get(
            static_cast<dbref>(dep.obj), dep.attr_num);
        if (current != dep.mod_count) {
            return false;
        }
    }
    return true;
}

// ---------------------------------------------------------------
// Public wrappers for SQLite code cache (shared with Lua JIT).
// ---------------------------------------------------------------

std::string jit_sha1_hex(const void *data, size_t len) {
    const void *parts[] = { data };
    size_t sizes[] = { len };
    return sha1_hex_parts(parts, sizes, 1);
}

void jit_store_to_sqlite(const std::string &key, const compiled_program &prog) {
    store_to_sqlite_cache(key, prog);
}

void jit_compact_program(compiled_program &prog) {
    compact_program(prog);
}

bool jit_load_from_sqlite(const std::string &key, compiled_program &out) {
    if (!g_pSQLiteBackend) return false;
    CSQLiteDB &db = g_pSQLiteBackend->GetDB();
    CSQLiteDB::CodeCacheRecord rec;
    if (!db.CodeCacheGet(key.data(), static_cast<int>(key.size()),
                         s_blob_version.data(),
                         static_cast<int>(s_blob_version.size()), rec)) {
        return false;
    }
    out = reconstruct_from_cache(rec);
    return out.ok;
}

// Marshal %q register slots from mudstate.global_regs into guest SUBST
// memory.  `subst_mask` selects which registers to copy (bit
// SUBST_QREG0+i, matching compiled_program::subst_mask); pass
// ~UINT64_C(0) to copy all of them.  This is the single authority for
// q-register slot population: program entry uses it today, and the
// scope-restore / post-ECALL resync points reuse it so the slots can
// never drift from global_regs at a sync boundary
// (docs/plan-jit-evalbracket-lift.md, Phases 2-3).
//
// The subst_mask bits covering the %q register slots
// (SUBST_QREG0 .. SUBST_QREG0 + MAX_GLOBAL_REGS - 1).
static constexpr uint64_t QREG_SLOT_BITS =
    ((MAX_GLOBAL_REGS < 64
        ? (UINT64_C(1) << MAX_GLOBAL_REGS) - 1
        : ~UINT64_C(0))) << rv_compiler::SUBST_QREG0;

// Also maintains the long-register bitmap (#996 step 2): the whole
// QREG_LONGBITS u64 is rewritten on EVERY call — computed bits for
// masked registers, zero elsewhere.  The runtime buffer is reused
// across programs, so a partial update would leak a previous program's
// stale bits; the whole-word write makes freshness true by
// construction.  A truncated slot no longer declines the run: %q reads
// branch on the bit and fetch long values via the fun_r ECALL.
static void marshal_qregs_to_slots(uint8_t *mem, uint64_t subst_mask)
{
    uint64_t longbits = 0;
    for (int i = 0; i < MAX_GLOBAL_REGS; i++) {
        const int slot_idx = rv_compiler::SUBST_QREG0 + i;
        if (!((subst_mask >> slot_idx) & UINT64_C(1))) {
            continue;
        }
        const uint64_t slot = rv_compiler::SUBST_BASE
            + static_cast<uint64_t>(slot_idx) * rv_compiler::SUBST_SLOT;
        const UTF8 *value = nullptr;
        if (mudstate.global_regs[i] && mudstate.global_regs[i]->reg_ptr) {
            value = mudstate.global_regs[i]->reg_ptr;
        }
        if (value && value[0]) {
            size_t len = strlen(reinterpret_cast<const char *>(value));
            if (len >= static_cast<size_t>(rv_compiler::SUBST_SLOT)) {
                len = rv_compiler::SUBST_SLOT - 1;
                longbits |= UINT64_C(1) << i;
            }
            memcpy(mem + slot, value, len);
            mem[slot + len] = 0;
        } else {
            mem[slot] = 0;
        }
    }
    memcpy(mem + rv_compiler::QREG_LONGBITS, &longbits, sizeof(longbits));
}

// Set/clear one register's long bit after an ECALL_SETQ write (#996).
static void qreg_longbit_update(eval_ctx *ec, int regnum, size_t vlen)
{
    if (rv_compiler::QREG_LONGBITS + sizeof(uint64_t) > ec->memory_size) {
        return;
    }
    uint64_t bits;
    memcpy(&bits, ec->memory + rv_compiler::QREG_LONGBITS, sizeof(bits));
    const uint64_t bit = UINT64_C(1) << regnum;
    if (vlen >= static_cast<size_t>(rv_compiler::SUBST_SLOT)) {
        bits |= bit;
    } else {
        bits &= ~bit;
    }
    memcpy(ec->memory + rv_compiler::QREG_LONGBITS, &bits, sizeof(bits));
}

// Look up or compile an expression.  Returns a pointer to the
// cached compiled_program (owned by the cache — do not free).
// Returns nullptr on compilation failure.
//
// Perform a jitstats(flush) that had to wait for the JIT to become quiet.
// Defined below with s_run_cached_depth; called here because this is the
// only place a caller obtains a compiled_program * for the softcode route,
// so it is the last moment at which clearing the cache frees nothing that
// is already spoken for (#1316).
//
static void jit_drain_pending_flush(void);

static compiled_program *compile_cached(const UTF8 *expr, size_t nLen,
                                         int eval = EV_FCHECK | EV_EVAL) {
    jit_drain_pending_flush();

    std::string key = compile_cache_key(expr, nLen, eval);

    auto it = s_compile_cache.find(key);
    if (it != s_compile_cache.end()) {
        // Staleness check: if this entry has inlined deps and any
        // attr has been modified since compilation, evict and recompile.
        if (!it->second.prog.deps.empty()
            && !deps_are_fresh(it->second.prog))
        {
            release_program_slots(it->second.prog.program_id);
            s_compile_lru.erase(it->second.lru_it);
            s_compile_cache.erase(it);
            // Fall through to recompile below.
        }
        else
        {
            // Memory cache hit — move to front of LRU.
            s_compile_lru.splice(s_compile_lru.begin(), s_compile_lru,
                                 it->second.lru_it);
            s_jit_stats.cache_hit_mem++;
            return &it->second.prog;
        }
    }

    // Memory cache miss — check SQLite persistent cache.
    compiled_program prog;
    bool from_sqlite = false;

    if (g_pSQLiteBackend && nLen >= COMPILE_CACHE_MIN_LEN) {
        CSQLiteDB &db = g_pSQLiteBackend->GetDB();
        CSQLiteDB::CodeCacheRecord rec;
        if (db.CodeCacheGet(key.data(), static_cast<int>(key.size()),
                            s_blob_version.data(),
                            static_cast<int>(s_blob_version.size()), rec)) {
            prog = reconstruct_from_cache(rec);
            db.CodeCacheReset();

            // Check staleness of SQLite-cached entry too.
            if (!prog.deps.empty() && !deps_are_fresh(prog))
            {
                // Stale — discard and recompile.
                prog.ok = false;
            }
            else
            {
                from_sqlite = true;
                s_jit_stats.cache_hit_sqlite++;
            }
        }
    }

    if (!from_sqlite) {
        // Full cache miss — compile from scratch.
        s_jit_stats.cache_miss++;
        prog = compile_expression(expr, nLen, eval);
        if (!prog.ok) return nullptr;

        // Persist to SQLite while prog.memory still exists.
        if (nLen >= COMPILE_CACHE_MIN_LEN) {
            store_to_sqlite_cache(key, prog);
        }

        // Compact: extract blobs, release 4MB memory.
        compact_program(prog);
    }

    // Insert into memory LRU cache.
    while (s_compile_cache.size() >= compile_cache_max()) {
        auto &victim_key = s_compile_lru.back();
        auto vit = s_compile_cache.find(victim_key);
        if (vit != s_compile_cache.end()) {
            // Evicting a program must release its slot residency in every
            // context (#1326, adapted for #2129's slots).
            release_program_slots(vit->second.prog.program_id);
        }
        s_compile_cache.erase(victim_key);
        s_compile_lru.pop_back();
    }

    s_compile_lru.push_front(key);
    auto [ins_it, _] = s_compile_cache.emplace(
        key, compile_cache_entry{std::move(prog), s_compile_lru.begin()});
    return &ins_it->second.prog;
}

// ---------------------------------------------------------------
// Shared code heap — persistent guest memory for re-entrant JIT.
//
// All compiled expressions deposit code, strings, and fargs into
// a single 4MB guest memory image.  The blob is installed once.
// Code accumulates via bump allocation; the DBT block cache
// persists across all expressions.
//
// Pool layout within shared memory:
//   0x0004-0x0FFFF  Code heap (64KB)
//   0x10000-0x3FFFF Blob (installed once)
//   0x40000-0x5FFFF String pool (128KB)
//   0x60000-0x67FFF Fargs pool (32KB)
//   0x68000+        CARGS/SUBST/DMA/output (per-execution)
// ---------------------------------------------------------------

struct shared_heap_t {
    guest_memory_t memory;
    bool ready;

    uint64_t code_next;        // next free code address
    uint64_t str_next;         // next free string pool address
    uint64_t fargs_next;       // next free fargs address

    static constexpr uint64_t CODE_START  = 0x0004;  // avoid PC=0
    static constexpr uint64_t CODE_LIMIT  = rv_compiler::BLOB_BASE;
    static constexpr uint64_t STR_START   = 0x40000;
    static constexpr uint64_t STR_LIMIT   = 0x60000;
    static constexpr uint64_t FARGS_START = 0x60000;
    static constexpr uint64_t FARGS_LIMIT = 0x68000;

    // Compile result — lightweight, references shared memory.
    struct entry {
        uint64_t entry_pc;
        uint64_t out_addr;
        bool needs_jit;
        int ecalls;
        int tier2_calls;
        std::vector<compiled_program::inline_dep> deps;
    };

    // Expression cache.
    std::unordered_map<std::string, entry> cache;

    shared_heap_t()
        : memory(rv_compiler::MEM_SIZE),
          ready(false),
          code_next(CODE_START),
          str_next(STR_START),
          fargs_next(FARGS_START) {}

    // Initialize: install blob once.
    bool init() {
        if (ready) return true;
        tier2_lazy_init();
        if (!s_tier2.loaded) return false;
        tier2_install(memory, rv_compiler::BLOB_BASE);
        ready = true;
        return true;
    }

    // Compile an expression into the shared heap.
    // Returns {0,0,false} on failure.
    entry compile(const UTF8 *expr, size_t nLen,
                  int eval = EV_FCHECK | EV_EVAL) {

        if (!init()) return {0, 0, false, 0, 0};

        // Bounds check: code heap must not overflow.
        if (code_next >= CODE_LIMIT) return {0, 0, false, 0, 0};

        compiled_program prog = compile_expression(
            expr, nLen, eval,
            code_next,
            str_next, STR_LIMIT,
            fargs_next, FARGS_LIMIT,
            rv_compiler::STACK_TOP - 8);

        if (!prog.ok) return {0, 0, false, 0, 0};

        // Install compiled regions into shared memory.
        //
        // Code.
        if (prog.entry_pc + prog.code_size > CODE_LIMIT) {
            return {0, 0, false, 0, 0};
        }
        memcpy(memory.data() + prog.entry_pc,
               prog.memory.data() + prog.entry_pc, prog.code_size);

        // Strings (if any new strings were added).
        if (prog.str_pool_end > str_next) {
            memcpy(memory.data() + str_next,
                   prog.memory.data() + str_next,
                   prog.str_pool_end - str_next);
        }

        // Fargs (if any new fargs were added).
        if (prog.fargs_pool_end > fargs_next) {
            memcpy(memory.data() + fargs_next,
                   prog.memory.data() + fargs_next,
                   prog.fargs_pool_end - fargs_next);
        }

        // Advance cursors.
        code_next = prog.entry_pc + prog.code_size;
        code_next = (code_next + 15) & ~15ULL;  // align
        str_next = prog.str_pool_end;
        fargs_next = prog.fargs_pool_end;

        return {prog.entry_pc, prog.out_addr, prog.needs_jit,
                prog.ecalls, prog.tier2_calls, std::move(prog.deps)};
    }

    // Look up or compile an expression.  Returns nullptr on failure.
    // Validates inline dependencies on cache hit — if any inlined
    // attribute body has changed, the cached entry is stale and
    // must be evicted.  (Code heap space is leaked; the shared heap
    // has no reclamation yet.)
    const entry *lookup(const UTF8 *expr, size_t nLen, int eval) {
        std::string key = compile_cache_key(expr, nLen, eval);
        auto it = cache.find(key);
        if (it != cache.end()) {
            // Check dependency freshness.
            bool fresh = true;
            for (const auto &dep : it->second.deps) {
                uint32_t current = attr_mod_count_get(
                    static_cast<dbref>(dep.obj), dep.attr_num);
                if (current != dep.mod_count) {
                    fresh = false;
                    break;
                }
            }
            if (fresh) return &it->second;

            // Stale — evict and recompile.
            cache.erase(it);
        }

        entry e = compile(expr, nLen, eval);
        if (!e.entry_pc) return nullptr;

        auto [ins, _] = cache.emplace(key, std::move(e));
        return &ins->second;
    }

    // ---------------------------------------------------------------
    // Evaluate an expression via the shared heap's own DBT.
    //
    // Compiles (or cache-hits), populates CARGS/SUBST, runs via
    // dbt_run, extracts result.  Returns true if handled.
    //
    // The shared heap DBT is independent of the outer expression's
    // DBT, so this is safe to call from within an ECALL handler.
    //
    // It is NOT independent of *itself* (#1994).  There is one dbt and
    // one guest register context here, so a second eval() entered from a
    // host ECALL of a suspended eval() reuses both.  run_depth declines
    // that case; see the guard in eval().
    // ---------------------------------------------------------------

    dbt_state_t dbt;
    bool dbt_ready = false;
    uint32_t run_count = 0;
    int run_depth = 0;         // >0 while a run is live (#1994)

    bool eval(const UTF8 *expr, size_t nLen,
              UTF8 *out, size_t out_size,
              dbref executor, dbref caller, dbref enactor,
              int eval_flags,
              const UTF8 *cargs[], int ncargs,
              int *out_ecalls  = nullptr,
              int *out_tier2   = nullptr,
              bool *out_folded = nullptr) {

        const entry *e = lookup(expr, nLen, eval_flags);
        if (!e) return false;

        // Copy fields we need after guest execution before dbt_run() —
        // a re-entrant eval() during guest code can call lookup() and
        // erase this entry if the same key goes stale (recursive
        // self-modifying softcode), leaving e dangling (#1940).  Stats
        // were already snapshotted for #1938; out_addr is the remaining
        // post-run read.
        const int entry_ecalls     = e->ecalls;
        const int entry_tier2      = e->tier2_calls;
        const bool entry_needs_jit = e->needs_jit;
        const uint64_t entry_out_addr = e->out_addr;

        // Constant-folded: result is in shared memory.
        if (!entry_needs_jit) {
            uint64_t out_addr = resolve_runtime_out_addr(
                entry_out_addr, rv_compiler::STACK_TOP);
            size_t n = 0;
            if (!guest_strnlen(memory.data(), memory.size(), out_addr, &n)) {
                return false;
            }
            if (n >= out_size) n = out_size - 1;
            memcpy(out, memory.data() + out_addr, n);
            out[n] = '\0';
            if (out_ecalls)  *out_ecalls  = entry_ecalls;
            if (out_tier2)   *out_tier2   = entry_tier2;
            if (out_folded)  *out_folded  = true;
            return true;
        }

        // Runtime execution.  Everything from here on touches state that
        // is shared across every eval() on this heap: the DBT's register
        // context and stack pointer, the writable blob region, and the
        // per-eval heap arena.  A nested eval() -- reached when this
        // heap's own program ECALLs into u(), whose body evaluates
        // another bracket -- would reset all three underneath the
        // suspended outer run, which then resumes at a program counter
        // that is not its own.  The backend refuses to translate there,
        // dbt_resume returns -1, and jit_eval hands the whole subtree
        // back to mux_exec to redo through the AST.  Since that subtree
        // contains the next recursion level, each level costs two
        // evaluations and terminating a runaway recursion becomes
        // 2^depth (#1994).
        //
        // Declining up front costs nothing measurable: instrumented over
        // a runaway recursion, every re-entrant run failed this way
        // (511 of 511 at function_recursion_limit 12) and not one ever
        // produced a result.  The AST handles the nested bracket instead,
        // exactly as it does for the #1002 depth watermark above -- one
        // evaluation per level rather than two.
        //
        if (0 < run_depth) {
            s_jit_stats.bail_shared_busy++;
            return false;
        }
        struct run_depth_guard {
            int &d;
            explicit run_depth_guard(int &r) : d(r) { d++; }
            ~run_depth_guard() { d--; }
        } depth_guard(run_depth);

        // Initialize DBT on first use.
        if (!dbt_ready) {
            if (dbt_init(&dbt, memory.data(), memory.size(),
                         eval_ecall, nullptr) != 0) {
                return false;
            }
            dbt_ready = true;

            const char *md_env = getenv("TINYMUX_DBT_MAX_DISPATCH");
            dbt.max_dispatch = md_env
                ? strtoull(md_env, nullptr, 0) : 10000000;
            dbt.alarm_flag = &alarm_clock.alarmed;  // wall-clock (#JIT-alarm)

            pretranslate_tier2(&dbt);
            dbt.blob_code_end = dbt.code_used;
        }

        // Reset writable blob state and the per-eval heap arena.
        tier2_reset_writable(memory, rv_compiler::BLOB_BASE);
        s_heap_next = rv_compiler::HEAP_BASE;

        // NUL-sentinel output slots.
        {
            uint64_t addr = rv_compiler::STACK_TOP - 8
                          - rv_compiler::OUT_SLOT;
            while (addr >= rv_compiler::OUT_STACK_LIMIT) {
                memory[addr] = 0;
                addr -= rv_compiler::OUT_SLOT;
            }
        }

        // Populate CARGS.  Slot is CARGS_SLOT bytes including the trailing
        // NUL.  Truncating a long carg would change softcode results vs the
        // AST path (LBUF-sized), so decline instead and let the AST evaluator
        // handle it — the nested-eval twin of run_cached_program's fix (#1055).
        // Nothing has run yet (memory is reset each eval), so an early return
        // here is a clean bail.
        for (int i = 0; i < rv_compiler::MAX_CARGS; i++) {
            uint64_t slot = rv_compiler::CARGS_BASE
                          + static_cast<uint64_t>(i) * rv_compiler::CARGS_SLOT;
            if (i < ncargs && cargs && cargs[i]) {
                size_t len = strlen(
                    reinterpret_cast<const char *>(cargs[i]));
                if (len >= static_cast<size_t>(rv_compiler::CARGS_SLOT)) {
                    return false;
                }
                memcpy(memory.data() + slot, cargs[i], len);
                memory[slot + len] = 0;
            } else {
                memory[slot] = 0;
            }
        }

        // Populate SUBST slots.
        auto copy_subst = [&](int slot_idx, const UTF8 *value) {
            uint64_t slot = rv_compiler::SUBST_BASE
                + static_cast<uint64_t>(slot_idx) * rv_compiler::SUBST_SLOT;
            if (value && value[0]) {
                size_t len = strlen(
                    reinterpret_cast<const char *>(value));
                if (len >= static_cast<size_t>(rv_compiler::SUBST_SLOT))
                    len = rv_compiler::SUBST_SLOT - 1;
                memcpy(memory.data() + slot, value, len);
                memory[slot + len] = 0;
            } else {
                memory[slot] = 0;
            }
        };

        {
            UTF8 dbref_buf[32];
            mux_sprintf(dbref_buf, sizeof(dbref_buf), T("#%d"), enactor);
            copy_subst(rv_compiler::SUBST_ENACTOR, dbref_buf);
            mux_sprintf(dbref_buf, sizeof(dbref_buf), T("#%d"), executor);
            copy_subst(rv_compiler::SUBST_EXECUTOR, dbref_buf);
        }
        if (Good_obj(enactor)) {
            copy_subst(rv_compiler::SUBST_NAME, Name(enactor));
            UTF8 dbref_buf[32];
            mux_sprintf(dbref_buf, sizeof(dbref_buf), T("#%d"),
                        Location(enactor));
            copy_subst(rv_compiler::SUBST_LOCATION, dbref_buf);
            copy_subst(rv_compiler::SUBST_MONIKER, Moniker(enactor));
        } else {
            copy_subst(rv_compiler::SUBST_NAME, nullptr);
            copy_subst(rv_compiler::SUBST_LOCATION, nullptr);
            copy_subst(rv_compiler::SUBST_MONIKER, nullptr);
        }
        marshal_qregs_to_slots(memory.data(), ~UINT64_C(0));
        copy_subst(rv_compiler::SUBST_LASTCMD, mudstate.curr_cmd);
        copy_subst(rv_compiler::SUBST_POUT, mudstate.pout);
        {
            UTF8 ncbuf[32];
            mux_sprintf(ncbuf, sizeof(ncbuf), T("%d"), ncargs);
            copy_subst(rv_compiler::SUBST_NCARGS, ncbuf);
        }

        // Set up ECALL context.
        eval_ctx ec;
        ec.memory = memory.data();
        ec.memory_size = memory.size();
        ec.executor = executor;
        ec.caller = caller;
        ec.enactor = enactor;
        ec.eval = eval_flags;
        ec.cargs = cargs;
        ec.ncargs = ncargs;
        ec.qreg_mask = ~UINT64_C(0);
        ec.lua_result_base = 0;
        ec.lua_result_count = 0;
        ec.lua_state = nullptr;
        ec.host_ecalls = 0;
        ec.dbt = &dbt;
        ec.pvm = nullptr;

        // Run via the shared heap's DBT.
        dbt_rerun(&dbt, eval_ecall, &ec);
        int rc;
        if (run_count == 0) {
            run_count++;
            rc = dbt_run(&dbt, e->entry_pc, rv_compiler::STACK_TOP);
        } else {
            dbt.ctx.x[2] = rv_compiler::STACK_TOP;
            run_count++;
            rc = dbt_resume(&dbt, e->entry_pc);
        }

        // Alarm abort: count + emit #-1 CPU LIMITED (handled). Other
        // failures fall through to the outer AST path.
        if (!handle_dbt_run_status(rc, out, out_size, true)) {
            return false;
        }
        if (rc == -3) {
            return true;
        }

        // Extract result (#1057: bound guest NUL scan).  Use the
        // snapshotted out_addr — not e->out_addr — so a mid-run
        // eviction cannot UAF here (#1940).
        uint64_t out_addr = resolve_runtime_out_addr(
            entry_out_addr, rv_compiler::STACK_TOP);
        size_t n = 0;
        if (!guest_strnlen(memory.data(), memory.size(), out_addr, &n)) {
            return false;
        }
        if (n >= out_size) n = out_size - 1;
        memcpy(out, memory.data() + out_addr, n);
        out[n] = '\0';
        if (out_ecalls)  *out_ecalls  = entry_ecalls;
        if (out_tier2)   *out_tier2   = entry_tier2;
        if (out_folded)  *out_folded  = false;
        return true;
    }
};

static shared_heap_t s_shared_heap;

// Run a cached program.  Uses dbt_rerun if the DBT already has
// translated blocks for this program, otherwise dbt_reset.
//
// Nesting depth of run_cached_program.  Softcode JIT ECALL → fun_lua →
// Lua TryJIT must not dbt_reset/rerun the shared persistent DBT while the
// outer softcode program is live (#1309 nested corruption / hang).
//

// Deferred half of jitstats(flush) (#1316).
//
// s_compile_cache holds compiled_program by value and compile_cached() hands
// back a pointer into it, which run_cached_program holds across the whole
// execution -- and jit_eval still reads prog->ecalls after that returns.
// Compiled code reaches the interpreter through ECALLs (u(), ufuns), so
// fun_jitstats can run with a live program on the stack; clearing the cache
// there frees the program that is mid-flight.  It showed up as jitstats(flush)
// returning an empty string instead of OK on the compiled route, because the
// post-run harvest reads prog->out_addr out of freed memory.
//
// So the SQLite DELETE and the write-queue drop happen immediately -- neither
// frees anything a running program points at -- while the in-memory caches
// wait until no program is live.  Draining at the top of compile_cached()
// (and of the Lua Run/Compile entries) is what makes that safe: those are the
// points where the caller does not yet hold a pointer, and s_run_cached_depth
// == 0 means no outer run_cached_program frame holds one either.
//
static bool s_code_cache_flush_pending = false;

static void jit_flush_memory_caches(void)
{
    s_compile_cache.clear();
    s_compile_lru.clear();
    s_decline_memo.clear();
    for (int i = 0; i < JIT_MAX_RUN_DEPTH; i++) {
        s_vm[i].release_all_slots();
    }
    jit_lua_clear_cache();
}

static void jit_drain_pending_flush(void)
{
    if (  s_code_cache_flush_pending
       && 0 == s_run_cached_depth)
    {
        s_code_cache_flush_pending = false;
        jit_flush_memory_caches();
    }
}

// Same drain, reachable from jit_lua.cpp, which owns the other program cache
// and takes a pointer into it the same way.
//
void jit_flush_pending_caches(void)
{
    jit_drain_pending_flush();
}

// eval_ecall status for Lua (#1423 / #1751).
// dbt_run continues on a negative return and stops on a non-negative one;
// 0 is success.  ECALL_LUA_ERROR commits an interpreter-class message.
// ECALL_DECLINE is retained only as a residual safety net (Phase 4): any
// remaining site is rewritten to LUA ERROR text and must not re-run.
// Pre-entry fallback (compile refuse / cache miss) is unchanged.
//
static constexpr int ECALL_DECLINE = 1;
static constexpr int ECALL_LUA_ERROR = 2;

static thread_local const char *s_lua_decline_site = nullptr;
static thread_local char s_lua_ecall_error[256];

// Forward: residual post-entry soft declines become committed errors.
//
static int ecall_lua_error_cstr(const char *msg);

// Phase 4: no soft decline.  Name kept for greppability of residual sites.
//
static int lua_ecall_decline(const char *site)
{
    s_lua_decline_site = (nullptr != site) ? site : "UNKNOWN";
    char buf[160];
    mux_snprintf(reinterpret_cast<UTF8 *>(buf), sizeof(buf),
        T("post-entry residual decline (%s)"),
        reinterpret_cast<const UTF8 *>(s_lua_decline_site));
    return ecall_lua_error_cstr(buf);
}

bool run_cached_program(compiled_program *prog,
                        dbref executor, dbref caller_db,
                        dbref enactor,
                        UTF8 *out, size_t out_size,
                        const UTF8 *cargs[],
                        int ncargs,
                        int eval,
                        void *lua_state) {
    // Each nesting level gets its own guest buffer and DBT (#1326), so a
    // nested run no longer has to be refused.  It used to be: softcode JIT
    // ECALL -> fun_lua -> Lua JIT would have materialized its program over the
    // outer program's memory and reset the shared DBT under the outer
    // program's live frames, so run_cached_program declined and Lua fell back
    // to its interpreter for every nested call -- which is every call, since
    // eval brackets are compiled by default.
    //
    // Beyond the contexts we have, still refuse.  The fallback is correct, so
    // running out of depth costs speed and nothing else.
    //
    if (s_run_cached_depth >= JIT_MAX_RUN_DEPTH) {
        return false;
    }
    jit_run_vm *vm = &s_vm[s_run_cached_depth];

    // #1002 depth watermark (see jit_eval; repeated here for callers
    // that bypass it, e.g. rvbench).
    if (mudstate.func_nest_lev + prog->max_func_depth
        >= mudconf.func_nest_lim) {
        s_jit_stats.bail_depth++;
        return false;
    }

    // Invocation-count watermark (0 means unknown/pre-v13 cache row —
    // skip until that program recompiles and is rewritten with a count).
    if (  0 < prog->n_func_calls
       && mudstate.func_invk_ctr + prog->n_func_calls
          >= mudconf.func_invk_lim) {
        s_jit_stats.bail_invk++;
        return false;
    }

    if (!prog->needs_jit) {
        size_t n = prog->folded_result.size();
        if (n >= out_size) n = out_size - 1;
        memcpy(out, prog->folded_result.data(), n);
        out[n] = '\0';
        return true;
    }

    // A top-level entry proves no outer program holds a live handle into
    // the Tier 3 helper save stacks — reclaim slots leaked by abandoned
    // runs (see jit_helper_stacks_reset).
    if (0 == s_run_cached_depth) {
        jit_helper_stacks_reset();
    }

    // Hold the reentrancy lock for the whole DBT path (including early
    // declines that still touch shared runtime state).
    //
    RunDepthGuard run_depth_guard(s_run_cached_depth);

    // Bind the DBT and place the program (#2129).  Binding comes first —
    // slot eviction must be able to invalidate stale translations before
    // new code claims their PCs — and a fresh binding pretranslates the
    // blob exactly once for the life of the context, not once per program.
    runtime_buffer_init(vm);
    dbt_state_t *dbt = bind_run_dbt(vm);
    if (nullptr == dbt) {
        return false;
    }
    if (0 == dbt->blob_code_end) {
        pretranslate_tier2(dbt);
        dbt->blob_code_end = dbt->code_used;
    }

    // Code: keep the slot the program already occupies (its translations
    // are still live under those PCs), or claim one.
    uint64_t slot_base = 0;
    if (!pick_code_slot(vm, dbt, *prog, &slot_base)) {
        return false;
    }

    // Data: str/fargs pools are shared across slots at canonical addresses,
    // so they need refreshing whenever any other program ran since — unless
    // the pools already hold this exact program (consecutive re-run, the
    // common hot path).  Execution does not dirty these regions: the string
    // pool is read-only and frame-relative fargs entries are re-patched by
    // the program's own code on every run.
    //
    // "Read-only" here rests on the #2136 const-fargs contract: FUNCTION
    // bodies take `const UTF8 * const fargs[]`, so an ECALL callee cannot
    // tokenize the pool constants it is handed (that corruption was #2128,
    // and this very skip is what made it permanent instead of one-run).
    // #2135's per-call argument copy used to shield this; the contract
    // replaced it, and the compiler now enforces what the copy papered
    // over.
    if (vm->buffer_program_id != prog->program_id) {
        materialize_data(vm, *prog);
    }

    // Reset writable blob state (data + BSS) for clean re-run.
    if (s_tier2.loaded) {
        tier2_reset_writable(vm->buffer, rv_compiler::BLOB_BASE);
    }

    // Clear output buffers: NUL the first byte of each slot.
    {
        uint64_t addr = rv_compiler::STACK_TOP - 8 - rv_compiler::OUT_SLOT;
        while (addr >= prog->out_pool_end) {
            vm->buffer[addr] = 0;
            addr -= rv_compiler::OUT_SLOT;
        }
    }

    // Zero the loop-context depth (#2171): the VM buffer is shared
    // across programs, so a program that never runs its loop prologue
    // must publish no levels instead of whatever the last program left.
    memset(vm->buffer.data() + rv_compiler::LOOPCTX_BASE, 0, 8);

    // Populate CARGS: copy each arg, NUL-terminate unused slots.  Only the
    // slots this program actually reads (%0..%N) need populating; functions
    // reached via ECALL receive cargs through the host pointer array, not
    // these guest slots.
    //
    // Slot is CARGS_SLOT bytes including the trailing NUL.  Silently
    // truncating a long carg would change softcode results vs the AST
    // path (LBUF-sized); decline so the AST evaluator handles it (#1055).
    for (int i = 0; i < prog->cargs_used; i++) {
        uint64_t slot = rv_compiler::CARGS_BASE
                      + static_cast<uint64_t>(i) * rv_compiler::CARGS_SLOT;
        if (i < ncargs && cargs && cargs[i]) {
            size_t len = strlen(reinterpret_cast<const char *>(cargs[i]));
            if (len >= static_cast<size_t>(rv_compiler::CARGS_SLOT)) {
                return false;
            }
            memcpy(vm->buffer.data() + slot, cargs[i], len);
            vm->buffer[slot + len] = 0;
        } else {
            vm->buffer[slot] = 0;
        }
    }

    // Copy %-substitution runtime values into SUBST slots.
    auto copy_subst = [&](int slot_idx, const UTF8 *value) {
        uint64_t slot = rv_compiler::SUBST_BASE
                      + static_cast<uint64_t>(slot_idx) * rv_compiler::SUBST_SLOT;
        if (value && value[0]) {
            size_t len = strlen(reinterpret_cast<const char *>(value));
            if (len >= static_cast<size_t>(rv_compiler::SUBST_SLOT))
                len = rv_compiler::SUBST_SLOT - 1;
            memcpy(vm->buffer.data() + slot, value, len);
            vm->buffer[slot + len] = 0;
        } else {
            vm->buffer[slot] = 0;
        }
    };

    // Populate only the SUBST slots this program actually reads.  Each
    // substitution's value (and several of the lookups below — Name/Location/
    // Moniker, mux_sprintf) is computed only when its slot is referenced.
    auto subst_used = [&](int slot) -> bool {
        return (prog->subst_mask >> slot) & UINT64_C(1);
    };

    // %# — enactor dbref as string.
    if (subst_used(rv_compiler::SUBST_ENACTOR)) {
        UTF8 dbref_buf[32];
        mux_sprintf(dbref_buf, sizeof(dbref_buf), T("#%d"), enactor);
        copy_subst(rv_compiler::SUBST_ENACTOR, dbref_buf);
    }

    // %! — executor dbref as string.
    if (subst_used(rv_compiler::SUBST_EXECUTOR)) {
        UTF8 dbref_buf[32];
        mux_sprintf(dbref_buf, sizeof(dbref_buf), T("#%d"), executor);
        copy_subst(rv_compiler::SUBST_EXECUTOR, dbref_buf);
    }

    // %n — enactor name.
    if (subst_used(rv_compiler::SUBST_NAME)) {
        copy_subst(rv_compiler::SUBST_NAME,
                   Good_obj(enactor) ? Name(enactor) : nullptr);
    }

    // %l — enactor location.
    if (subst_used(rv_compiler::SUBST_LOCATION)) {
        if (Good_obj(enactor)) {
            dbref loc = Location(enactor);
            UTF8 dbref_buf[32];
            mux_sprintf(dbref_buf, sizeof(dbref_buf), T("#%d"), loc);
            copy_subst(rv_compiler::SUBST_LOCATION, dbref_buf);
        } else {
            copy_subst(rv_compiler::SUBST_LOCATION, nullptr);
        }
    }

    // %q global registers (+ the long-register bitmap, #996).
    marshal_qregs_to_slots(vm->buffer.data(), prog->subst_mask);

    // %m — last command.
    if (subst_used(rv_compiler::SUBST_LASTCMD)) {
        copy_subst(rv_compiler::SUBST_LASTCMD, mudstate.curr_cmd);
    }

    // %k — moniker (enactor name with color).
    if (subst_used(rv_compiler::SUBST_MONIKER)) {
        copy_subst(rv_compiler::SUBST_MONIKER,
                   Good_obj(enactor) ? Moniker(enactor) : nullptr);
    }

    // %| — piped command output.
    if (subst_used(rv_compiler::SUBST_POUT)) {
        copy_subst(rv_compiler::SUBST_POUT, mudstate.pout);
    }

    // %+ — number of cargs.
    if (subst_used(rv_compiler::SUBST_NCARGS)) {
        UTF8 ncbuf[32];
        mux_sprintf(ncbuf, sizeof(ncbuf), T("%d"), ncargs);
        copy_subst(rv_compiler::SUBST_NCARGS, ncbuf);
    }

    eval_ctx ec;
    ec.memory = vm->buffer.data();
    ec.memory_size = rv_compiler::MEM_SIZE;
    ec.executor = executor;
    ec.caller = caller_db;
    ec.enactor = enactor;
    ec.eval = eval;
    ec.cargs = cargs;
    ec.ncargs = ncargs;
    ec.qreg_mask = prog->subst_mask;
    ec.lua_state = lua_state;
    ec.host_ecalls = 0;
    ec.dbt = nullptr;
    ec.pvm = nullptr;

    // The DBT was bound (and the program placed) before marshalling; only
    // the per-run ECALL context remains to be set.  No reset on program
    // switch — that reset was #2129, the one-program translated-block cache.
    dbt_rerun(dbt, eval_ecall, &ec);

    ec.dbt = dbt;

    // Clear Phase 0 post-entry bookkeeping for this run.
    //
    s_lua_decline_site = nullptr;

    int rc = dbt_run(dbt, slot_base + prog->entry_pc, rv_compiler::STACK_TOP);

    // #1751 Phase 0–4: after dbt_run for a Lua program, never return false
    // (that re-runs the whole chunk in the Lua VM).  Commit an error text.
    //
    if (rc == ECALL_DECLINE && nullptr != lua_state) {
        // Residual only: lua_ecall_decline now returns ECALL_LUA_ERROR.
        // Keep this arm so a stray return ECALL_DECLINE cannot re-run.
        //
        const char *site = (nullptr != s_lua_decline_site)
            ? s_lua_decline_site : "UNKNOWN";
        if (nullptr != out && 0 < out_size) {
            mux_snprintf(out, out_size,
                T("#-1 LUA JIT POST-ENTRY DECLINE (%s)"),
                reinterpret_cast<const UTF8 *>(site));
        }
        STARTLOG(LOG_ALWAYS, "JIT", "RETRY");
        log_text(T("Lua residual ECALL_DECLINE (no re-run): "));
        log_text(reinterpret_cast<const UTF8 *>(site));
        ENDLOG;
        return true;
    }

    if (rc == ECALL_LUA_ERROR && nullptr != lua_state) {
        if (nullptr != out && 0 < out_size) {
            mux_strncpy(out,
                reinterpret_cast<const UTF8 *>(s_lua_ecall_error),
                out_size - 1);
        }
        return true;
    }

    if (!handle_dbt_run_status(rc, out, out_size, true)) {
        // Phase 4: any other post-run failure on the Lua path is committed,
        // not a silent interpreter re-run.
        //
        if (nullptr != lua_state) {
            if (nullptr != out && 0 < out_size
                && (out[0] == '\0'
                    || 0 != strncmp(reinterpret_cast<const char *>(out),
                                    "#-1", 3))) {
                mux_snprintf(out, out_size, T("#-1 LUA JIT RUN FAIL"));
            }
            return true;
        }
        // Softcode (#1791): CALL_FUNC never soft-declines, but mid-run
        // DBT infrastructure failure (code buffer full, …) used to
        // return false → full AST re-run.  After any host ECALL, that
        // doubles effects.  Commit a loud fail instead.
        //
        if (0 < ec.host_ecalls) {
            if (nullptr != out && 0 < out_size
                && (out[0] == '\0'
                    || 0 != strncmp(reinterpret_cast<const char *>(out),
                                    "#-1", 3))) {
                mux_snprintf(out, out_size, T("#-1 JIT POST-ENTRY FAIL"));
            }
            STARTLOG(LOG_ALWAYS, "JIT", "SOFT");
            log_text(T("Softcode mid-run DBT fail after host ECALL (no AST re-run)"));
            ENDLOG;
            return true;
        }
        return false;
    }
    if (rc == -3) {
        return true;  // CPU LIMITED already written
    }

    // Harvest result from guest memory (#1057).
    uint64_t out_addr = resolve_runtime_out_addr(
        prog->out_addr, rv_compiler::STACK_TOP);
    size_t n = 0;
    if (!guest_strnlen(vm->buffer.data(), vm->buffer.size(),
                       out_addr, &n)) {
        // Successful dbt_run then a harvest miss is still post-entry if
        // host ECALLs ran — do not AST-re-run (#1791).
        //
        if (nullptr == lua_state && 0 < ec.host_ecalls) {
            if (nullptr != out && 0 < out_size) {
                mux_snprintf(out, out_size, T("#-1 JIT POST-ENTRY FAIL"));
            }
            return true;
        }
        return false;
    }
    if (n >= out_size) n = out_size - 1;
    memcpy(out, vm->buffer.data() + out_addr, n);
    out[n] = '\0';
    return true;
}

// ECALL handler implementation.
//
// Common helper: call a FUN* with guest-memory arguments and write
// result to guest output buffer.  Returns bytes written.
//
// Thread-local pointer to the current
// ECALL execution context.  Set during ecall_invoke_fun so that
// Tier 3 helper functions (_write_carg, _save_cargs, _restore_cargs)
// can access JIT guest memory.
//
static thread_local eval_ctx *s_current_ecall_ctx = nullptr;

// #1989: keeps the softcode call counters correct across an ECALL.
// func_invk_ctr is monotonic for the evaluation and is not given back,
// matching ast.cpp, which decrements only func_nest_lev when the call
// returns.  RAII so the early returns for a tripped limit, a bad argument
// count, or a guest-memory rejection all unwind the nesting level.
//
class CallCounter
{
public:
    explicit CallCounter(bool bCount) : m_bCount(bCount)
    {
        if (m_bCount)
        {
            mudstate.func_nest_lev++;
            mudstate.func_invk_ctr++;
        }
    }
    ~CallCounter()
    {
        if (m_bCount)
        {
            mudstate.func_nest_lev--;
        }
    }
private:
    const bool m_bCount;
};

// Invoke a global user function (@function) from an ECALL.
//
// A global is an attribute reference rather than a C entry point, so it
// cannot go through ecall_invoke_fun.  This mirrors ast.cpp's ufun arm
// (permissions, FN_PRIV executor swap, AF_NOEVAL raw copy, FN_PRES
// register preservation) while keeping this file's guest-memory
// conventions for arguments and the result.
//

// ECALL arguments are guest-memory pointers handed to callees DIRECTLY,
// including pointers into regions that outlive the call (pooled string
// constants, blob rodata, code-embedded constants below CARGS_BASE).
//
// That is safe because of the #2136 const-fargs contract: FUNCTION/
// XFUNCTION bodies take `const UTF8 * const fargs[]`, so a builtin cannot
// tokenize or otherwise scribble on its argument text — the compiler
// enforces it, and the campaign's conversion left zero const_casts in
// function bodies.  Handlers that legitimately mutate text (do_link,
// do_trigger, ...) receive FargCopy/FargVec copies made by their wrappers.
// The ufun path hands the pointers to mux_exec as cargs, which have the
// same const contract.
//
// History: before #2136 this file copied every below-CARGS_BASE argument
// into an LBuf per call (#2135, after #2128's silent pool corruption:
// first run right, all later runs wrong).  The copy and its
// `ptr < CARGS_BASE` predicate are gone — deleting them, not the const
// keywords, was the campaign's payoff.

static int ecall_invoke_ufun(UFUN *ufp, eval_ctx *ec, rv64_ctx_t *ctx,
                             uint64_t fargs_addr, int nfargs,
                             uint64_t out_addr, uint64_t out_size) {
    // Host softcode work — counts for post-entry no-re-run (#1791).
    //
    ++ec->host_ecalls;

    // #1079: the out buffer must be a valid guest range before any write.
    //
    if (!guest_range_ok(out_addr, out_size, ec->memory_size) || out_size == 0) {
        ctx->x[10] = 0;
        return -1;
    }

    // #1124: the same permission gates ast.cpp applies.  A global can be
    // registered with restrictive perms, and compiled softcode must not
    // be a way around them.
    //
    if (  !check_access(ec->executor, ufp->perms)
       || (  (ufp->flags & FN_RESTRICT)
          && !Wizard(ec->executor)))
    {
        // mux_snprintf returns what it WROTE, not what it would have, so
        // snprintf's <0 and >=out_size clamps are gone rather than unused.
        size_t n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
            out_size, T("%s"), FUNC_NOPERM_MESSAGE);
        ctx->x[10] = static_cast<uint64_t>(n);
        return -1;
    }

    // #1989: count this call, as ast.cpp's ufun arm does.  A global whose
    // body reaches itself needs the live counters to stop it; nothing in
    // the compiled route maintained them.
    //
    CallCounter call_counter(true);
    {
        const UTF8 *pLimit = nullptr;
        if (mudconf.func_nest_lim <= mudstate.func_nest_lev) {
            pLimit = T("#-1 FUNCTION RECURSION LIMIT EXCEEDED");
        } else if (mudconf.func_invk_lim <= mudstate.func_invk_ctr) {
            pLimit = T("#-1 FUNCTION INVOCATION LIMIT EXCEEDED");
        }
        if (nullptr != pLimit) {
            size_t n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
                out_size, T("%s"), pLimit);
            ctx->x[10] = static_cast<uint64_t>(n);
            return -1;
        }
    }

    if (nfargs > MAX_ARG) nfargs = MAX_ARG;
    if (nfargs < 0) nfargs = 0;

    const UTF8 *fargs[MAX_ARG];
    memset(fargs, 0, sizeof(fargs));
    uint64_t frame_top = ctx->x[8];  // s0 = frame pointer
    for (int i = 0; i < nfargs; i++) {
        uint64_t ptr = 0;
        if (!guest_farg_addr(ec->memory, ec->memory_size, fargs_addr, i, &ptr)) {
            ctx->x[10] = 0;
            return -1;
        }
        if (rv_compiler::is_output_frame_ref(ptr)) {
            ptr = rv_compiler::resolve_output_addr(ptr, frame_top);
        }
        // Bounds-validate the guest C string even though the length itself
        // is no longer needed for a copy.
        size_t slen = 0;
        if (!guest_strnlen(ec->memory, ec->memory_size, ptr, &slen)) {
            ctx->x[10] = 0;
            return -1;
        }
        fargs[i] = ec->memory + ptr;
    }

    dbref aowner;
    int aflags;
    UTF8 *tbuf = atr_get("ecall.ufun", ufp->obj, ufp->atr, &aowner, &aflags);
    dbref obj = (ufp->flags & FN_PRIV) ? ufp->obj : ec->executor;

    LBuf buff = LBuf_Src("eval_ecall_ufun");
    UTF8 *bufc = buff.get();

    eval_ctx *saved_ctx = s_current_ecall_ctx;
    s_current_ecall_ctx = ec;

    if (  (aflags & AF_NOEVAL)
       || NoEval(ufp->obj))
    {
        size_t nLen = strlen(reinterpret_cast<const char *>(tbuf));
        safe_copy_buf(tbuf, nLen, buff, &bufc);
    }
    else
    {
        reg_ref **preserve = nullptr;
        if (ufp->flags & FN_PRES) {
            preserve = PushRegisters(MAX_GLOBAL_REGS);
            save_global_regs(preserve);
        }

        int feval = ec->eval & ~(EV_TOP | EV_FMAND);
        mux_exec(tbuf, LBUF_SIZE-1, buff, &bufc, obj, ec->executor, ec->enactor,
                 AttrTrace(aflags, feval),
                 fargs, nfargs);

        if (ufp->flags & FN_PRES) {
            restore_global_regs(preserve);
            PopRegisters(preserve, MAX_GLOBAL_REGS);
        }
    }

    s_current_ecall_ctx = saved_ctx;
    free_lbuf(tbuf);

    // A user-function body is arbitrary softcode and may setq, so the %q
    // slots this program reads have to be re-marshalled from the
    // authoritative global_regs — the same resync ecall_invoke_fun does
    // after a builtin.  Unconditional here: there is no pure-read case
    // to exempt.
    //
    if ((ec->qreg_mask & QREG_SLOT_BITS) != 0) {
        marshal_qregs_to_slots(ec->memory, ec->qreg_mask);
        s_jit_stats.qreg_resyncs++;
    }

    *bufc = '\0';
    size_t result_len = static_cast<size_t>(bufc - buff);
    if (result_len >= out_size) result_len = out_size - 1;
    memcpy(ec->memory + out_addr, buff, result_len);
    ec->memory[out_addr + result_len] = '\0';

    ctx->x[10] = static_cast<uint64_t>(result_len);
    return -1;
}

// Push the calling program's live compiled iter levels onto the
// interpreter's itext/inum stack for the duration of a host callee
// (#2171).  Compiled loops keep the guest loop-context table at
// LOOPCTX_BASE current (HIR_LCTX_* stores); without this push, any
// callee that evaluates softcode — fun_u's mux_exec, fun_itext, ilev()
// — saw an EMPTY stack and %i0 inside u() called from a compiled iter
// came back blank.  The pushed itext pointers point into guest memory,
// which outlives the ECALL (the arena is stable for the eval), and
// every consumer copies out.  RAII so all the invoke paths' returns
// unwind it.
//
class GuestLoopContext
{
public:
    explicit GuestLoopContext(eval_ctx *ec) : m_pushed(0)
    {
        const uint64_t base = rv_compiler::LOOPCTX_BASE;
        if (base + 8 > ec->memory_size) {
            return;
        }
        uint64_t depth;
        memcpy(&depth, ec->memory + base, 8);
        if (0 == depth || depth > rv_compiler::LOOPCTX_MAX_LEVELS
            || base + (1 + 2 * depth) * 8 > ec->memory_size) {
            return;
        }
        for (uint64_t k = 0; k < depth; k++) {
            if (mudstate.in_loop < 0 || mudstate.in_loop >= MAX_ITEXT) {
                break;
            }
            uint64_t eaddr, in1;
            memcpy(&eaddr, ec->memory + base + (1 + 2 * k) * 8, 8);
            memcpy(&in1,   ec->memory + base + (2 + 2 * k) * 8, 8);
            size_t slen = 0;
            if (!guest_strnlen(ec->memory, ec->memory_size, eaddr, &slen)) {
                break;
            }
            mudstate.itext[mudstate.in_loop] = ec->memory + eaddr;
            mudstate.inum[mudstate.in_loop] =
                static_cast<int>(in1);
            mudstate.in_loop++;
            m_pushed++;
        }
    }
    ~GuestLoopContext()
    {
        mudstate.in_loop -= m_pushed;
    }
    GuestLoopContext(const GuestLoopContext &) = delete;
    GuestLoopContext &operator=(const GuestLoopContext &) = delete;
private:
    int m_pushed;
};

static int ecall_invoke_fun(FUN *fp, eval_ctx *ec, rv64_ctx_t *ctx,
                            uint64_t fargs_addr, int nfargs,
                            uint64_t out_addr, uint64_t out_size) {
    // Host softcode work — counts for post-entry no-re-run (#1791).
    // Count even on early validation failure: a partial path that later
    // grows side effects must still be treated as entry.
    //
    ++ec->host_ecalls;

    // #1079: out buffer must be a valid guest range before any write.
    //
    if (!guest_range_ok(out_addr, out_size, ec->memory_size) || out_size == 0) {
        ctx->x[10] = 0;
        return -1;
    }

    // #1124: mirror AST's check_access (ast.cpp) so CA_WIZARD / CA_GOD
    // builtins cannot be invoked from JIT-compiled softcode by mortals.
    // engine_api_table includes every builtin; without this gate, ECALL
    // would call fp->fun with no perms test.
    //
    // Exemption: underscore-prefixed names are JIT internal helpers
    // (_SAVE_QREGS, _WRITE_CARG, _CHECK_U_PERM, …) deliberately marked
    // CA_GOD so softcode cannot call them, but the compiler must.  They
    // never appear as softcode-visible symbols softcode can type.
    //
    if (  fp->name[0] != '_'
       && !check_access(ec->executor, fp->perms))
    {
        // mux_snprintf returns what it WROTE, not what it would have, so
        // snprintf's <0 and >=out_size clamps are gone rather than unused.
        size_t n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
            out_size, T("%s"), FUNC_NOPERM_MESSAGE);
        ctx->x[10] = static_cast<uint64_t>(n);
        return -1;
    }

    // #1989: mirror AST's call accounting (ast.cpp) the same way the perms
    // gate above mirrors its check_access.  ast.cpp bumps func_nest_lev /
    // func_invk_ctr and tests both limits immediately before its own
    // fp->fun dispatch; the compiled route reaches the builtin without
    // passing through that branch, so nothing counted a u() that calls
    // itself -- it recursed until the stack was gone, at any limit.  The
    // static max_func_depth watermark cannot catch this: it is a
    // compile-time property that does not grow with runtime recursion.
    //
    // Underscore-prefixed names are JIT internal helpers, not softcode
    // calls, and must not spend the player's budget -- the same exemption
    // check_access uses above.
    //
    const bool bCount = (fp->name[0] != '_');
    CallCounter call_counter(bCount);
    if (bCount)
    {
        const UTF8 *pLimit = nullptr;
        if (mudconf.func_nest_lim <= mudstate.func_nest_lev) {
            pLimit = T("#-1 FUNCTION RECURSION LIMIT EXCEEDED");
        } else if (mudconf.func_invk_lim <= mudstate.func_invk_ctr) {
            pLimit = T("#-1 FUNCTION INVOCATION LIMIT EXCEEDED");
        }
        if (nullptr != pLimit) {
            size_t n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
                out_size, T("%s"), pLimit);
            ctx->x[10] = static_cast<uint64_t>(n);
            return -1;
        }
    }

    // Validate argument count against function's declared limits.
    // Return the same error string the AST evaluator would.
    if (nfargs < fp->minArgs) {
        int n;
        if (fp->minArgs == fp->maxArgs) {
            n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
                out_size, T("#-1 FUNCTION (%s) EXPECTS %d ARGUMENTS"),
                fp->name, fp->minArgs);
        } else if (fp->minArgs + 1 == fp->maxArgs) {
            n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
                out_size, T("#-1 FUNCTION (%s) EXPECTS %d OR %d ARGUMENTS"),
                fp->name, fp->minArgs, fp->maxArgs);
        } else {
            n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
                out_size, T("#-1 FUNCTION (%s) EXPECTS BETWEEN %d AND %d ARGUMENTS"),
                fp->name, fp->minArgs, fp->maxArgs);
        }
        if (n < 0) n = 0;
        if (static_cast<size_t>(n) >= out_size) n = static_cast<int>(out_size - 1);
        ctx->x[10] = static_cast<uint64_t>(n);
        return -1;
    }
    if (fp->maxArgs >= 0 && nfargs > fp->maxArgs) {
        nfargs = fp->maxArgs;
    }

    const UTF8 *fargs[MAX_ARG];
    if (nfargs > MAX_ARG) nfargs = MAX_ARG;
    uint64_t frame_top = ctx->x[8];  // s0 = frame pointer
    for (int i = 0; i < nfargs; i++) {
        uint64_t ptr = 0;
        // #1079: overflow-safe fargs table slot load + guest C-string bound.
        //
        if (!guest_farg_addr(ec->memory, ec->memory_size, fargs_addr, i, &ptr)) {
            ctx->x[10] = 0;
            return -1;
        }
        // Resolve frame-relative output references.
        if (rv_compiler::is_output_frame_ref(ptr)) {
            ptr = rv_compiler::resolve_output_addr(ptr, frame_top);
        }
        size_t slen = 0;
        if (!guest_strnlen(ec->memory, ec->memory_size, ptr, &slen)) {
            ctx->x[10] = 0;
            return -1;
        }
        // Direct guest pointer — safe under the #2136 const contract, and
        // exactly what the `_`-prefixed ABI helpers need (fun__write_carg
        // converts fargs[1] back to a guest address).
        fargs[i] = ec->memory + ptr;
    }

    LBuf buff = LBuf_Src("eval_ecall");
    UTF8 *bufc = buff.get();

    eval_ctx *saved_ctx = s_current_ecall_ctx;
    s_current_ecall_ctx = ec;

    // Builtins receive only the trace bit, mirroring the AST dispatch
    // (ast.cpp fp->fun(..., feval & EV_TRACE, ...)).  Passing the full
    // program flags diverged for re-evaluating handlers: fun_eval's
    // inner mux_exec inherited EV_STRIP_CURLY and stripped nested
    // braces the AST route preserves (#991 parser TC010).
    fp->fun(fp, buff, &bufc, ec->executor, ec->caller, ec->enactor,
            ec->eval & EV_TRACE, fargs, nfargs, ec->cargs, ec->ncargs);

    s_current_ecall_ctx = saved_ctx;

    // Conservative post-ECALL resync: any host callee may have mutated
    // mudstate.global_regs (a scope _RESTORE_QREGS reverting them, or
    // an interpreter-side setq — which writes only global_regs, never
    // the guest slots), so re-marshal the %q slots this program reads
    // from the authoritative global_regs.  Masked by the program's
    // subst_mask: %q-free programs skip at the bit test.  A pure
    // callee makes this a semantic no-op (slots already match).
    // (docs/plan-jit-evalbracket-lift.md, Phases 2-3.)
    // fun_r is a pure register read — the long-path %q ECALL (#996) —
    // and must not itself trigger a full masked re-marshal.
    static FUN *s_fun_r_fp = nullptr;
    if (s_fun_r_fp == nullptr) {
        int r_idx = engine_api_lookup("R");
        if (r_idx > 0 && r_idx < ENGINE_API_MAX_FUNCS) {
            s_fun_r_fp = engine_api_table[r_idx];
        }
    }
    if ((ec->qreg_mask & QREG_SLOT_BITS) != 0
        && fp != s_fun_r_fp) {
        // Re-marshal slots AND the long-register bitmap from the
        // authoritative global_regs (#996: a register grown past the
        // slot by the callee sets its bit here, so later %q reads
        // take the fun_r path instead of reading a truncated slot).
        marshal_qregs_to_slots(ec->memory, ec->qreg_mask);
        s_jit_stats.qreg_resyncs++;
    }

    *bufc = '\0';
    size_t result_len = static_cast<size_t>(bufc - buff);

    if (result_len >= out_size) result_len = out_size - 1;
    memcpy(ec->memory + out_addr, buff, result_len);
    ec->memory[out_addr + result_len] = '\0';

    ctx->x[10] = static_cast<uint64_t>(result_len);
    return -1;
}

// ---------------------------------------------------------------
// JIT DMA Controller (Tier C)
// ---------------------------------------------------------------

class jit_dma_controller {
public:
    static constexpr int MAX_WINDOWS = rv_compiler::DMA_WINDOW_COUNT;

    static void submit(int window, size_t length, int op, eval_ctx *ec) {
        if (window < 0 || window >= MAX_WINDOWS) return;

        uint64_t addr = rv_compiler::DMA_BASE + window * rv_compiler::DMA_WINDOW_SIZE;
        if (addr + length > ec->memory_size) return;

        const UTF8 *data = ec->memory + addr;

        // Implementation of ops (FINALIZE, etc.)
        // For now, simple registration into an arena.
        if (op == 1) { // DMA_OP_FINALIZE
            auto *a = JITArena::Alloc(length + 1);
            if (a) {
                size_t off = a->used - (length + 1);
                memcpy(a->buf->data + off, data, length);
                a->buf->data[off + length] = '\0';
                // Success: queue for ACK.
                s_ack_queue.push_back(window);
                s_window_busy |= (1 << window);
            }
        }
    }

    static int get_next_ack() {
        if (s_ack_queue.empty()) return -1;
        int window = s_ack_queue.front();
        s_ack_queue.pop_front();
        s_window_busy &= ~(1 << window);
        return window;
    }

    static void reset() {
        s_window_busy = 0;
        s_ack_queue.clear();
    }

private:
    static inline uint32_t s_window_busy = 0;
    static inline std::list<int> s_ack_queue;
};

// A Lua error raised inside an ECALL has no protected call frame anywhere
// above it, so luaD_throw() reaches the default panic handler and abort()s
// the process (#1423).  Full table ops (lua_geti / lua_settable / luaL_len)
// can raise through metamethods or type errors.
//
// #1751 Phase 1: GET/SET/LEN are *total* — never ECALL_DECLINE for policy
// (metatable / type surprise / missing global).  Use the real VM ops under
// pcall; on error commit an interpreter-class LUA ERROR (not a silent
// re-run, not a POST-ENTRY DECLINE).  Typed result claims that fail after
// a successful get return ok=0 and continue (same as GETI_INT already did
// for non-integers) so the guest can take a typed alternate path.
//
// ECALL_LUA_ERROR and s_lua_ecall_error live next to ECALL_DECLINE above.
//
static int ecall_lua_commit_error(lua_State *L)
{
    const char *msg = lua_tostring(L, -1);
    if (nullptr == msg) {
        msg = "unknown Lua error";
    }
    // Match the interpreter's softcode framing for raised Lua errors.
    //
    mux_snprintf(reinterpret_cast<UTF8 *>(s_lua_ecall_error),
        sizeof(s_lua_ecall_error),
        T("#-1 LUA ERROR: %s"),
        reinterpret_cast<const UTF8 *>(msg));
    lua_pop(L, 1);
    return ECALL_LUA_ERROR;
}

// Commit a C-string error without a Lua stack message (#1751 Phase 3).
//
static int ecall_lua_error_cstr(const char *msg)
{
    if (nullptr == msg) {
        msg = "unknown Lua error";
    }
    mux_snprintf(reinterpret_cast<UTF8 *>(s_lua_ecall_error),
        sizeof(s_lua_ecall_error),
        T("#-1 LUA ERROR: %s"),
        reinterpret_cast<const UTF8 *>(msg));
    return ECALL_LUA_ERROR;
}

// Marshal one Lua value into a guest buffer the way fun_lua does for a
// chunk result (nil→"", bool→"0"/"1", else lua_tolstring).  Truncates to
// out_size rather than declining.  #1751 follow-up: CALL_STR result path
// must not soft-decline after the callee has run (string.find returns an
// integer as the first multi-value; both routes must keep it).
//
static size_t ecall_lua_marshal_to_guest(lua_State *L, int idx,
                                         uint8_t *out, size_t out_size)
{
    if (0 == out_size) {
        return 0;
    }
    size_t len = 0;
    const char *result = nullptr;
    if (lua_isnil(L, idx) || lua_isnone(L, idx)) {
        result = "";
        len = 0;
    } else if (lua_isboolean(L, idx)) {
        result = lua_toboolean(L, idx) ? "1" : "0";
        len = 1;
    } else {
        result = lua_tolstring(L, idx, &len);
        if (nullptr == result) {
            result = "";
            len = 0;
        }
    }
    if (len >= out_size) {
        len = out_size - 1;
    }
    if (len > 0) {
        memcpy(out, result, len);
    }
    out[len] = '\0';
    return len;
}

// pcall helpers: stack protocol is (table, key[, value]) via absolute
// indices copied onto the stack before the call.
//
static int ecall_aux_gettable(lua_State *L)
{
    lua_gettable(L, 1);
    return 1;
}

static int ecall_aux_settable(lua_State *L)
{
    lua_settable(L, 1);
    return 0;
}

static int ecall_aux_len(lua_State *L)
{
    lua_pushinteger(L, static_cast<lua_Integer>(luaL_len(L, 1)));
    return 1;
}

// #1836: GETGLOBAL and lua_compare can raise through metamethods
// (__index on _ENV, __eq).  Same unprotected-ECALL class as #1423.
//
static int ecall_aux_getglobal(lua_State *L)
{
    const char *key = lua_tostring(L, 1);
    if (nullptr == key) {
        lua_pushnil(L);
        return 1;
    }
    lua_getglobal(L, key);
    return 1;
}

static int ecall_aux_compare_eq(lua_State *L)
{
    // Stack: lhs, rhs.  Result: boolean.
    //
    const int eq = lua_compare(L, 1, 2, LUA_OPEQ);
    lua_pushboolean(L, eq);
    return 1;
}

// Absolute stack index must refer to a table (or something indexable via
// metamethods).  Out-of-range is a soft miss (ok=0), not a decline.
//
static bool ecall_lua_stack_index_ok(lua_State *L, int idx)
{
    return idx > 0 && idx <= lua_gettop(L);
}

// Protected get: pushes the value at table[key].  Returns 0 on success
// (value on stack), ECALL_LUA_ERROR on raise.  key_is_str: key is a C
// string pushed by the caller as... actually we push inside.
//
static int ecall_lua_pget_intkey(lua_State *L, int tbl_idx, lua_Integer key)
{
    lua_pushcfunction(L, ecall_aux_gettable);
    lua_pushvalue(L, tbl_idx);
    lua_pushinteger(L, key);
    if (LUA_OK != lua_pcall(L, 2, 1, 0)) {
        return ecall_lua_commit_error(L);
    }
    return 0;
}

static int ecall_lua_pget_strkey(lua_State *L, int tbl_idx, const char *key)
{
    lua_pushcfunction(L, ecall_aux_gettable);
    lua_pushvalue(L, tbl_idx);
    lua_pushstring(L, key);
    if (LUA_OK != lua_pcall(L, 2, 1, 0)) {
        return ecall_lua_commit_error(L);
    }
    return 0;
}

static int ecall_lua_pset_intkey(lua_State *L, int tbl_idx, lua_Integer key,
                                 lua_Integer val)
{
    lua_pushcfunction(L, ecall_aux_settable);
    lua_pushvalue(L, tbl_idx);
    lua_pushinteger(L, key);
    lua_pushinteger(L, val);
    if (LUA_OK != lua_pcall(L, 3, 0, 0)) {
        return ecall_lua_commit_error(L);
    }
    return 0;
}

static int ecall_lua_pset_strkey(lua_State *L, int tbl_idx, const char *key,
                                 lua_Integer val)
{
    lua_pushcfunction(L, ecall_aux_settable);
    lua_pushvalue(L, tbl_idx);
    lua_pushstring(L, key);
    lua_pushinteger(L, val);
    if (LUA_OK != lua_pcall(L, 3, 0, 0)) {
        return ecall_lua_commit_error(L);
    }
    return 0;
}

static int ecall_lua_plen(lua_State *L, int tbl_idx)
{
    lua_pushcfunction(L, ecall_aux_len);
    lua_pushvalue(L, tbl_idx);
    if (LUA_OK != lua_pcall(L, 1, 1, 0)) {
        return ecall_lua_commit_error(L);
    }
    return 0;
}

static int ecall_lua_pgetglobal(lua_State *L, const char *key)
{
    lua_pushcfunction(L, ecall_aux_getglobal);
    lua_pushstring(L, key);
    if (LUA_OK != lua_pcall(L, 1, 1, 0)) {
        return ecall_lua_commit_error(L);
    }
    return 0;
}

// lhs and rhs must already be on the stack at absolute indices.
// Pushes nothing; returns 0 with *peq set, or ECALL_LUA_ERROR.
//
static int ecall_lua_pcompare_eq(lua_State *L, int lhs, int rhs, int *peq)
{
    lua_pushcfunction(L, ecall_aux_compare_eq);
    lua_pushvalue(L, lhs);
    lua_pushvalue(L, rhs);
    if (LUA_OK != lua_pcall(L, 2, 1, 0)) {
        return ecall_lua_commit_error(L);
    }
    *peq = lua_toboolean(L, -1) ? 1 : 0;
    lua_pop(L, 1);
    return 0;
}

// Push the arguments of a compiled Lua call (ECALL_LUA_CALL_INT/_STR).
// The two result variants share one argument encoding: nargs in a1's low
// byte, then TWO BITS per argument in the next byte -- 0 is an integer in
// x[12+j], 1 is the guest address of a NUL-terminated string in x[12+j],
// 2 is a double as raw bits in x[12+j] (the FMV.X.D lane ECALL_LUA_FTOA
// already proved on both execution routes).  One decoder shared by both
// handlers, because two near-identical loops is the drift shape (#1457)
// the lowering's twin call branches had to be merged out of.
//
// Returns false -- caller declines -- on a bad address or an unknown kind.
// The caller owns stack cleanup, as it also must for a failed pcall.
//
static bool ecall_lua_push_call_args(lua_State *L, const rv64_ctx_t *ctx,
                                     const eval_ctx *ec,
                                     int nargs, int kinds)
{
    for (int j = 0; j < nargs; j++) {
        const uint64_t raw = ctx->x[12 + j];
        switch ((kinds >> (2 * j)) & 3) {
        case 0:
            lua_pushinteger(L, static_cast<lua_Integer>(raw));
            break;
        case 1: {
            const char *sarg = guest_cstr(ec->memory, ec->memory_size, raw);
            if (nullptr == sarg) {
                return false;
            }
            lua_pushstring(L, sarg);
            break;
        }
        case 2: {
            double d;
            memcpy(&d, &raw, 8);
            lua_pushnumber(L, d);
            break;
        }
        default: {
            // Kind 3: a Lua stack reference -- the register holds the
            // stack index a handle-typed value carries.  lua_pushvalue is
            // the one honest use of a handle as an argument: the VALUE it
            // refers to is pushed, never the index as a number (#1579).
            // Absolute indices stay valid as later arguments push.
            const int idx = static_cast<int>(raw);
            if (idx <= 0 || idx > lua_gettop(L)) {
                return false;
            }
            lua_pushvalue(L, idx);
            break;
        }
        }
    }
    return true;
}

// The Lua lowering's bridge ECALLs are dispatched by name and all live in a
// reserved "__lua_" namespace (hir_lower_lua.cpp).  Recognising the namespace
// -- rather than just a leading "__" -- is what lets the dispatch below fail
// closed on an unimplemented bridge name without swallowing a softcode
// function that happens to start with an underscore, which @function permits.
//
// Case-insensitive on purpose.  The names the lowering emits are lower case
// and the comparisons in the dispatch are upper case, which is the defect
// behind #1512; a prefix test that agreed with only one of the two spellings
// would re-create the same trap for the next name added.
//
static bool is_lua_bridge_name(const UTF8 *s) {
    static const char prefix[] = "__lua_";
    for (size_t i = 0; i < sizeof(prefix) - 1; i++) {
        // A short name fails on the NUL, before reading past it.
        if (tolower(static_cast<unsigned char>(s[i])) != prefix[i]) {
            return false;
        }
    }
    return true;
}

static int eval_ecall(rv64_ctx_t *ctx, void *user_data) {
    eval_ctx *ec = static_cast<eval_ctx *>(user_data);
    uint64_t syscall_num = ctx->x[17];

    switch (syscall_num) {
    case ECALL_EXIT:
        return static_cast<int>(ctx->x[10]);

    case ECALL_CALL_INDEX: {
        // Indexed dispatch: a0 = function index, a1 = fargs,
        // a2 = nfargs, a3 = output, a4 = outsize.
        int func_idx = static_cast<int>(ctx->x[10]);
        uint64_t fargs_addr = ctx->x[11];
        int nfargs = static_cast<int>(ctx->x[12]);
        uint64_t out_addr = ctx->x[13];
        uint64_t out_size = ctx->x[14];

        if (func_idx <= 0 || func_idx >= engine_api_count ||
            !guest_range_ok(out_addr, out_size, ec->memory_size)) {
            ctx->x[10] = 0;
            return -1;
        }

        FUN *fp = engine_api_table[func_idx];

        GuestLoopContext glc(ec);  // (#2171)
        int rc = ecall_invoke_fun(fp, ec, ctx, fargs_addr, nfargs,
                                out_addr, out_size);
        return rc;
    }

    case ECALL_CALL_FUNC: {
        // String-based dispatch (fallback): a0 = name ptr.
        uint64_t name_addr = ctx->x[10];
        uint64_t fargs_addr = ctx->x[11];
        int nfargs = static_cast<int>(ctx->x[12]);
        uint64_t out_addr = ctx->x[13];
        uint64_t out_size = ctx->x[14];

        GuestLoopContext glc(ec);  // (#2171) — covers builtin AND ufun paths

        size_t name_len = 0;
        if (  !guest_strnlen(ec->memory, ec->memory_size, name_addr, &name_len)
           || !guest_range_ok(out_addr, out_size, ec->memory_size)) {
            ctx->x[10] = 0;
            return -1;
        }

        const UTF8 *func_name = ec->memory + name_addr;

        // Intercept __lua_* internal functions for Lua VM table ops.
        if (ec->lua_state && is_lua_bridge_name(func_name)) {
            // The gate above is deliberately case-insensitive, but every
            // dispatch below is a case-sensitive strcmp against an upper-case
            // literal -- and hir_lower_lua emits these names in lower case
            // ("__lua_call").  So a name could get past the gate, match none
            // of the thirteen comparisons, and fall out to the bail at the
            // bottom.  Normalise once here so the two halves agree (#1519).
            //
            // This was unobservable until a chunk could reach the ECALL at
            // all: consumers of a bridge result declined at lowering, so the
            // comparisons were never run against a real name.
            char fnbuf[64];
            size_t fnlen = name_len < sizeof(fnbuf) - 1
                         ? name_len : sizeof(fnbuf) - 1;
            for (size_t i = 0; i < fnlen; i++) {
                fnbuf[i] = static_cast<char>(toupper(
                    static_cast<unsigned char>(func_name[i])));
            }
            fnbuf[fnlen] = '\0';
            const char *fn = fnbuf;
            // Which bridge ECALLs a chunk actually reaches is otherwise
            // invisible: a decline inside any handler just surfaces as
            // lua_run_fail, indistinguishable from every other cause.
            // TINYMUX_TRACE_LUA_ECALL=1 names them in order, which is how the
            // remaining #1519 blocker was located.
            if (getenv("TINYMUX_TRACE_LUA_ECALL")) {
                fprintf(stderr, "LUAECALL: enter %s\n", fn);
            }
            // The named __lua_* bridge is gone (#1519 / #1309).
            //
            // Thirteen handlers lived here and none had ever executed in any
            // build: the lowering emitted lower case, this compared upper.
            // Correcting the case was measured on #1519 -- 16 of 21 chunks
            // compiled then failed at run time, and 5 of 21 returned the
            // EMPTY STRING with lua_run_ok=1, wrong with no signal.
            //
            // They were also the wrong shape.  Every one passed a Lua stack
            // INDEX through guest memory as a decimal string, which is what
            // made #t answer 22: an index flowed on as though it were the
            // value it points at.  The four numbered ECALLs that do work
            // (FTOA, GETI_INT, NEWTABLE, SETI_INT) pass VALUES in registers
            // instead, and that is the direction #1309 settled on.
            //
            // Deleting rather than leaving them: a mechanism that looks
            // available and is not is what let this hide for so long.  Every
            // __lua_* name now reaches the fail-closed path below, which is
            // what already happened in practice.


            // Fail closed on an unrecognised __lua_* name (#1512).
            //
            // Falling through from here reaches the softcode function
            // dispatch below, which upper-cases the name and looks it up in
            // builtin_functions/ufunc_htab.  No __lua_* name is a softcode
            // function, so the lookup misses and the ECALL hands back the
            // *string* "#-1 FUNCTION NOT FOUND" as this call's value -- and
            // that value then flows on as data.  `local x=tonumber(a) return
            // x+1` produced 1, and `return x*2` produced 0, because the error
            // string coerces to zero in arithmetic: wrong answers, not error
            // markers.
            //
            // It is not hypothetical.  Every name the lowering emits is lower
            // case ("__lua_getglobal") and every comparison above is upper
            // case ("__LUA_GETGLOBAL"), so the entire named bridge --
            // getglobal/setglobal/getenv, call/get_result, getfield/setfield,
            // geti/seti, newtable, pin_array, tfor_call -- has never once
            // been reached.  The case mismatch alone was survivable; what
            // made it produce silent corruption instead of a visible failure
            // is this fall-through.
            //
            // ECALL_DECLINE stops dbt_run with a failure status, so
            // run_cached_program returns false without harvesting output and
            // the caller re-evaluates on the interpreter -- the same
            // containment contract the table ops use (#1423).  A name this
            // block does not implement now costs the JIT, never correctness.
            //
            if (getenv("TINYMUX_TRACE_LUA_ECALL")) {
                fprintf(stderr, "LUAECALL: UNHANDLED %s\n", fn);
            }
            // Phase 4: committed error, not soft decline / re-run.
            //
            return ecall_lua_error_cstr("unhandled Lua bridge ECALL");
        }
        size_t nCased;
        UTF8 *pCased = mux_strupr(func_name, nCased);
        std::vector<UTF8> key(pCased, pCased + nCased);
        auto it = mudstate.builtin_functions.find(key);
        if (it == mudstate.builtin_functions.end()) {
            // Not a builtin — try the global user-function table before
            // giving up (#1231).  @function registers into ufunc_htab,
            // and consulting only builtin_functions made every global
            // unreachable from compiled softcode while the AST route
            // resolved it fine.  2.13 checks both in the same order
            // (mux/src/eval.cpp:1487), as does ast.cpp.
            //
            auto it_ufunc = mudstate.ufunc_htab.find(key);
            if (it_ufunc != mudstate.ufunc_htab.end()) {
                return ecall_invoke_ufun(
                    static_cast<UFUN *>(it_ufunc->second),
                    ec, ctx, fargs_addr, nfargs, out_addr, out_size);
            }
            const char *err = "#-1 FUNCTION NOT FOUND";
            size_t elen = strlen(err);
            if (elen >= out_size) elen = out_size - 1;
            memcpy(ec->memory + out_addr, err, elen);
            ec->memory[out_addr + elen] = '\0';
            ctx->x[10] = static_cast<uint64_t>(elen);
            return -1;
        }

        return ecall_invoke_fun(it->second, ec, ctx, fargs_addr, nfargs,
                                out_addr, out_size);
    }

    case ECALL_SETQ: {
        // Traditional write-through: a0=reg, a1=addr
        // Writes to both SUBST slot (for JIT %q reads) and
        // mudstate.global_regs (for ECALL reads).
        ++ec->host_ecalls;
        int regnum = static_cast<int>(ctx->x[10]);
        uint64_t val_addr = ctx->x[11];
        size_t vlen = 0;
        if (regnum >= 0 && regnum < MAX_GLOBAL_REGS
            && guest_strnlen(ec->memory, ec->memory_size, val_addr, &vlen)) {
            const UTF8 *value = ec->memory + val_addr;

            // Write to SUBST slot in guest memory.
            uint64_t slot = rv_compiler::SUBST_BASE
                + (rv_compiler::SUBST_QREG0 + regnum)
                  * rv_compiler::SUBST_SLOT;
            if (slot + rv_compiler::SUBST_SLOT <= ec->memory_size) {
                size_t cplen = vlen;
                if (cplen >= static_cast<size_t>(rv_compiler::SUBST_SLOT))
                    cplen = rv_compiler::SUBST_SLOT - 1;
                memcpy(ec->memory + slot, value, cplen);
                ec->memory[slot + cplen] = 0;
            }
            qreg_longbit_update(ec, regnum, vlen);

            RegAssign(&mudstate.global_regs[regnum], vlen, value);
            ctx->x[10] = vlen;
        } else {
            ctx->x[10] = 0;
        }
        return -1;
    }

    case ECALL_SETQ_PACK: {
        // Fast path: a0=reg, a1=addr, a2=len.
        // Packs into a JIT Arena.
        ++ec->host_ecalls;
        int regnum = static_cast<int>(ctx->x[10]);
        uint64_t val_addr = ctx->x[11];
        size_t vlen = static_cast<size_t>(ctx->x[12]);

        if (0 == vlen) {
            // Length 0 means "use strlen" — bound the scan to guest
            // memory so a missing NUL cannot walk off the buffer (#1057).
            if (!guest_strnlen(ec->memory, ec->memory_size, val_addr, &vlen)) {
                ctx->x[10] = 0;
                return -1;
            }
        }

        if (regnum >= 0 && regnum < MAX_GLOBAL_REGS && val_addr + vlen <= ec->memory_size) {
            const UTF8 *value = ec->memory + val_addr;

            // Write to SUBST slot in guest memory (for JIT %q reads).
            uint64_t slot = rv_compiler::SUBST_BASE
                + (rv_compiler::SUBST_QREG0 + regnum)
                  * rv_compiler::SUBST_SLOT;
            if (slot + rv_compiler::SUBST_SLOT <= ec->memory_size) {
                size_t cplen = vlen;
                if (cplen >= static_cast<size_t>(rv_compiler::SUBST_SLOT))
                    cplen = rv_compiler::SUBST_SLOT - 1;
                memcpy(ec->memory + slot, value, cplen);
                ec->memory[slot + cplen] = 0;
            }
            qreg_longbit_update(ec, regnum, vlen);

            // Allocate from Arena (Tier B).
            auto *a = JITArena::Alloc(vlen + 1);
            if (a) {
                size_t off = a->used - (vlen + 1);
                memcpy(a->buf->data + off, value, vlen);
                a->buf->data[off + vlen] = '\0';

                // Bind to register.
                if (mudstate.global_regs[regnum]) {
                    RegRelease(mudstate.global_regs[regnum]);
                }
                reg_ref *rr = new reg_ref();
                rr->refcount = 1;
                rr->buf = a->buf;
                rr->reg_ptr = a->buf->data + off;
                rr->reg_len = vlen;
                mudstate.global_regs[regnum] = rr;
                ctx->x[10] = vlen;
            } else {
                // Fallback if arena alloc fails (shouldn't happen for < 8KB).
                RegAssign(&mudstate.global_regs[regnum], vlen, value);
                ctx->x[10] = vlen;
            }
        } else {
            ctx->x[10] = 0;
        }
        return -1;
    }

    case ECALL_ARENA_ALLOC: {
        size_t size = static_cast<size_t>(ctx->x[10]);
        auto *a = JITArena::Alloc(size);
        if (a) {
            ctx->x[10] = a->id;
            ctx->x[11] = a->used - size;
        } else {
            ctx->x[10] = 0;
        }
        return -1;
    }

    case ECALL_ARENA_REF:
        JITArena::AddRef(static_cast<uint32_t>(ctx->x[10]));
        return -1;

    case ECALL_ARENA_RELEASE:
        JITArena::Release(static_cast<uint32_t>(ctx->x[10]));
        return -1;

    case ECALL_DMA_SUBMIT: {
        // a0=window, a1=length, a2=op
        jit_dma_controller::submit(static_cast<int>(ctx->x[10]),
                                   static_cast<size_t>(ctx->x[11]),
                                   static_cast<int>(ctx->x[12]),
                                   ec);
        return -1;
    }

    case ECALL_DMA_ACK: {
        ctx->x[10] = static_cast<uint64_t>(jit_dma_controller::get_next_ack());
        return -1;
    }

    case ECALL_FTOA: {
        // a0 = double bits (via FMV.X.D), a1 = output guest address.
        double val;
        uint64_t bits = ctx->x[10];
        memcpy(&val, &bits, 8);
        uint64_t out_addr = ctx->x[11];
        if (out_addr < ec->memory_size - 64) {
            char *out = reinterpret_cast<char *>(ec->memory + out_addr);
            LBuf buf = LBuf_Src("ecall.fmvxd");
            UTF8 *bufc = buf;
            fval(buf, &bufc, val);
            *bufc = '\0';
            size_t len = bufc - buf;
            if (len > 63) len = 63;
            memcpy(out, buf, len);
            out[len] = '\0';
        }
        return -1;
    }

    case ECALL_LUA_FTOA: {
        // a0 = double bits (via FMV.X.D), a1 = output guest address.
        // Same shape as ECALL_FTOA, but rendered Lua's way: "%.14g" plus a
        // trailing ".0" when the result looks like an integer, so a Lua float
        // keeps its subtype in the result string (#1488).
        double val;
        uint64_t bits = ctx->x[10];
        memcpy(&val, &bits, 8);
        uint64_t out_addr = ctx->x[11];
        if (out_addr < ec->memory_size - 64) {
            char *out = reinterpret_cast<char *>(ec->memory + out_addr);
            char buf[64];
            lua_format_double(val, buf, sizeof(buf));
            size_t len = strlen(buf);
            if (len > 63) len = 63;
            memcpy(out, buf, len);
            out[len] = '\0';
        }
        return -1;
    }

    case ECALL_ATOF: {
        // a0 = guest address of string → store double in fa0 (f[10]).
        uint64_t str_addr = ctx->x[10];
        double val = 0.0;
        if (str_addr < ec->memory_size - 1) {
            const char *s = reinterpret_cast<const char *>(ec->memory + str_addr);
            val = mux_atof(reinterpret_cast<const UTF8 *>(s));
        }
        memcpy(&ctx->f[10], &val, 8);
        return -1;
    }

    case ECALL_GOOD_OBJ: {
        // a0 = dbref integer → a0 = 1 if Good_obj, 0 otherwise.
        // This is a leaf database lookup — no softcode evaluation,
        // no re-entrancy risk.
        dbref obj = static_cast<dbref>(ctx->x[10]);
        ctx->x[10] = Good_obj(obj) ? 1 : 0;
        return -1;
    }

    case ECALL_CHR: {
        // a0 = guest addr of input string (space-separated codepoints)
        // a1 = guest addr of output buffer
        // Returns a0 = 0 on success, -1 on error (output holds error msg).
        uint64_t in_addr  = ctx->x[10];
        uint64_t out_addr = ctx->x[11];
        if (in_addr >= ec->memory_size || out_addr >= ec->memory_size - 64) {
            ctx->x[10] = static_cast<uint64_t>(-1);
            return -1;
        }
        const UTF8 *pArg = ec->memory + in_addr;
        char *out = reinterpret_cast<char *>(ec->memory + out_addr);
        size_t out_max = 7999;

        // Build raw UTF-8 from space-separated codepoints.
        LBuf raw = LBuf_Src("ecall.chr");
        UTF8 *pRaw = raw;
        const UTF8 *pEnd = raw.get() + LBUF_SIZE - 5;
        bool bAny = false;

        while ('\0' != *pArg) {
            while (mux_isspace(*pArg)) pArg++;
            if ('\0' == *pArg) break;

            bool bNeg = ('-' == *pArg);
            if ('-' == *pArg || '+' == *pArg) pArg++;
            if (!mux_isdigit(*pArg)) {
                memcpy(out, "#-1 ARGUMENT MUST BE A NUMBER", 30);
                ctx->x[10] = static_cast<uint64_t>(-1);
                return -1;
            }
            uint64_t uv = 0;
            while (mux_isdigit(*pArg)) {
                const uint64_t digit = static_cast<uint64_t>(*pArg - '0');
                if (uv > (UINT64_MAX - digit) / 10ULL) {
                    memcpy(out, "#-1 ARGUMENT OUT OF RANGE", 26);
                    ctx->x[10] = static_cast<uint64_t>(-1);
                    return -1;
                }
                uv = 10ULL * uv + digit;
                pArg++;
            }
            if ('\0' != *pArg && !mux_isspace(*pArg)) {
                memcpy(out, "#-1 ARGUMENT MUST BE A NUMBER", 30);
                ctx->x[10] = static_cast<uint64_t>(-1);
                return -1;
            }
            int64_t iv = bNeg ? -static_cast<int64_t>(uv) : static_cast<int64_t>(uv);
            if (iv < 0 || iv > static_cast<int64_t>(UNI_MAX_LEGAL_UTF32)
                || (static_cast<UTF32>(iv) >= UNI_SUR_HIGH_START
                    && static_cast<UTF32>(iv) <= UNI_SUR_LOW_END)) {
                memcpy(out, "#-1 ARGUMENT OUT OF RANGE", 26);
                ctx->x[10] = static_cast<uint64_t>(-1);
                return -1;
            }
            UTF32 ch = static_cast<UTF32>(iv);
            UTF8 *p = ConvertToUTF8(ch);
            if (!mux_isprint(p)) {
                memcpy(out, "#-1 UNPRINTABLE CHARACTER", 26);
                ctx->x[10] = static_cast<uint64_t>(-1);
                return -1;
            }
            size_t nb = strlen(reinterpret_cast<const char *>(p));
            if (pRaw + nb <= pEnd) {
                memcpy(pRaw, p, nb);
                pRaw += nb;
            }
            bAny = true;
        }
        *pRaw = '\0';

        if (!bAny) {
            memcpy(out, "#-1 ARGUMENT MUST BE A NUMBER", 30);
            ctx->x[10] = static_cast<uint64_t>(-1);
            return -1;
        }

        // NFC normalize.
        size_t nRaw = pRaw - raw;
        LBuf nfc = LBuf_Src("ecall.chr.nfc");
        size_t nNfc;
        utf8_normalize_nfc(raw, nRaw, nfc, LBUF_SIZE - 1, &nNfc);
        nfc[nNfc] = '\0';

        if (nNfc > out_max) nNfc = out_max;
        memcpy(out, nfc, nNfc);
        out[nNfc] = '\0';
        ctx->x[10] = 0;
        return -1;
    }

    case ECALL_ORD: {
        // a0 = guest addr of input string
        // a1 = guest addr of output buffer
        // Returns a0 = 0 on success, -1 on error.
        uint64_t in_addr  = ctx->x[10];
        uint64_t out_addr = ctx->x[11];
        if (in_addr >= ec->memory_size || out_addr >= ec->memory_size - 64) {
            ctx->x[10] = static_cast<uint64_t>(-1);
            return -1;
        }
        const UTF8 *pIn = ec->memory + in_addr;
        char *out = reinterpret_cast<char *>(ec->memory + out_addr);

        // Strip color.
        size_t nBytes = 0;
        UTF8 *p = strip_color(pIn, &nBytes, nullptr);
        if (0 == nBytes) {
            memcpy(out, "#-1 FUNCTION EXPECTS ONE CHARACTER", 35);
            ctx->x[10] = static_cast<uint64_t>(-1);
            return -1;
        }

        // First grapheme cluster.
        mux_cursor cluster = utf8_next_grapheme(p, nBytes);
        if (0 == cluster.m_byte) {
            memcpy(out, "#-1 STRING IS INVALID", 22);
            ctx->x[10] = static_cast<uint64_t>(-1);
            return -1;
        }

        // Exactly one cluster.
        if (cluster.m_byte < nBytes) {
            mux_cursor second = utf8_next_grapheme(p + cluster.m_byte,
                                                    nBytes - cluster.m_byte);
            if (0 < second.m_byte) {
                memcpy(out, "#-1 FUNCTION EXPECTS ONE CHARACTER", 35);
                ctx->x[10] = static_cast<uint64_t>(-1);
                return -1;
            }
            memcpy(out, "#-1 STRING IS INVALID", 22);
            ctx->x[10] = static_cast<uint64_t>(-1);
            return -1;
        }

        // Decode codepoints.
        //
        // This loop is fun_ord()'s, transliterated -- but where the
        // interpreter writes through safe_chr/safe_ltoa, which stop at the
        // buffer end, this had a bare `op += sprintf(op, ...)` and no bound
        // at all.  The only check above is `out_addr >= memory_size - 64`,
        // which reads like it bounds the write and does not: it guarantees
        // 64 bytes of headroom against a loop that emits up to 8 bytes per
        // CODEPOINT, and a grapheme cluster is one cluster but any number of
        // codepoints.
        //
        // Measured before the fix, ord() on 'a' followed by N combining
        // acutes -- one cluster, N+1 codepoints -- wrote linearly with no
        // cap: N=400 produced 1602 bytes, N=1200 produced 4802, N=3800
        // produced ~15k.  It did not crash only because out_addr happens to
        // sit far enough from the end of guest memory; everything past the
        // result slot was overwritten regardless.  Input is player-supplied.
        //
        // Bound by the space that actually exists and stop cleanly, which is
        // also what the interpreter does when the LBUF fills.
        //
        const size_t out_avail = ec->memory_size - out_addr;
        char *op = out;
        size_t used = 0;
        const UTF8 *q = p;
        const UTF8 *qEnd = p + cluster.m_byte;
        bool bFirst = true;
        while (q < qEnd) {
            UTF32 ch = ConvertFromUTF8(q);
            if (UNI_EOF == ch) {
                memcpy(out, "#-1 STRING IS INVALID", 22);
                ctx->x[10] = static_cast<uint64_t>(-1);
                return -1;
            }
            if (!bFirst) {
                if (used + 2 > out_avail) break;   // separator plus NUL
                *op++ = ' ';
                used++;
            }
            size_t k = mux_snprintf(reinterpret_cast<UTF8 *>(op),
                out_avail - used, T("%ld"), static_cast<long>(ch));
            if (0 == k) break;
            op += k;
            used += k;
            bFirst = false;
            size_t nAdv = utf8_FirstByte[static_cast<unsigned char>(*q)];
            if (nAdv < 1 || nAdv >= UTF8_CONTINUE) nAdv = 1;
            q += nAdv;
        }
        *op = '\0';
        ctx->x[10] = 0;
        return -1;
    }

    case ECALL_TRANSLATE: {
        // a0 = guest addr of input string
        // a1 = type (0=spaces, 1=percent substitutions)
        // a2 = guest addr of output buffer
        uint64_t in_addr  = ctx->x[10];
        int type           = static_cast<int>(ctx->x[11]);
        uint64_t out_addr = ctx->x[12];
        if (in_addr >= ec->memory_size || out_addr >= ec->memory_size - 64) {
            ctx->x[10] = static_cast<uint64_t>(-1);
            return -1;
        }
        const UTF8 *pIn = ec->memory + in_addr;
        char *out = reinterpret_cast<char *>(ec->memory + out_addr);

        UTF8 *result = translate_string(pIn, type != 0);
        size_t len = strlen(reinterpret_cast<const char *>(result));
        if (len > 7999) len = 7999;
        memcpy(out, result, len);
        out[len] = '\0';
        ctx->x[10] = 0;
        return -1;
    }

    case ECALL_QUICK_WILD: {
        // a0 = guest addr of pattern, a1 = guest addr of data string
        // Returns a0 = 1 on match, 0 on no match.
        // Uses quick_wild() which pre-lowercases the pattern with
        // mux_strlwr() for Unicode-aware case-insensitive matching.
        uint64_t pat_addr  = ctx->x[10];
        uint64_t data_addr = ctx->x[11];
        if (pat_addr >= ec->memory_size || data_addr >= ec->memory_size) {
            ctx->x[10] = 0;
            return -1;
        }
        const UTF8 *pat  = ec->memory + pat_addr;
        const UTF8 *data = ec->memory + data_addr;
        mudstate.wild_invk_ctr = 0;
        ctx->x[10] = quick_wild(pat, data) ? 1 : 0;
        return -1;
    }

    case ECALL_SORT: {
        // a0 = guest addr of list string
        // a1 = sort_type char (e.g. 'a', 'n', 'u')
        // a2 = delim char
        // a3 = osep char
        // a4 = guest addr of output buffer
        uint64_t list_addr = ctx->x[10];
        char sort_type     = static_cast<char>(ctx->x[11]);
        unsigned char delim = static_cast<unsigned char>(ctx->x[12]);
        unsigned char osep  = static_cast<unsigned char>(ctx->x[13]);
        uint64_t out_addr  = ctx->x[14];
        if (list_addr >= ec->memory_size ||
            out_addr >= ec->memory_size - 64) {
            ctx->x[10] = 0;
            return -1;
        }
        const UTF8 *list_in = ec->memory + list_addr;
        UTF8 *out = ec->memory + out_addr;
        size_t n = sort_to_buffer(list_in, sort_type, delim, osep,
                                  out, LBUF_SIZE - 1);
        ctx->x[10] = n;
        return -1;
    }

    // ---- Lua VM ECALLs ----
    // These call back into the Lua interpreter via lua_State *L.
    // They handle table operations that the JIT can't do natively.

    case ECALL_LUA_NEWTABLE: {
        if (!ec->lua_state) { ctx->x[10] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int narr = static_cast<int>(ctx->x[10]);
        int nrec = static_cast<int>(ctx->x[11]);
        lua_createtable(L, narr, nrec);
        // Return the absolute stack index of the new table.
        ctx->x[10] = static_cast<uint64_t>(lua_gettop(L));
        return -1;
    }






    case ECALL_LUA_GETI_INT: {
        // Integer-keyed table get (#1751 Phase 1, revised).  The lowering
        // only emits this against a PLAIN-PROVEN table (built by this
        // chunk's NEWTABLE, never escaped), so no metamethod can fire and
        // every stored value is an integer by construction.  The one
        // honest miss left is an ABSENT key: the value is Lua nil, which
        // a typed integer slot cannot carry.  Continuing with 0 was the
        // pre-#1751 behavior and a measured silent wrong answer
        // (`return t[2]` gave "0" where the interpreter gave "") -- so an
        // absent key is a loud post-entry fail until nil is
        // representable.  The stack-index and integer checks are belts:
        // the proof should make them unreachable, and their firing means
        // a compiler bug, which is exactly what loud is for.
        //
        if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int tbl_idx = static_cast<int>(ctx->x[10]);
        lua_Integer key = static_cast<lua_Integer>(ctx->x[11]);
        if (!ecall_lua_stack_index_ok(L, tbl_idx)) {
            return lua_ecall_decline("ECALL_LUA_GETI_INT_BADIDX");
        }
        int pr = ecall_lua_pget_intkey(L, tbl_idx, key);
        if (0 != pr) {
            return pr;
        }
        if (!lua_isinteger(L, -1)) {
            lua_pop(L, 1);
            return lua_ecall_decline("ECALL_LUA_GETI_INT_NONINT");
        }
        ctx->x[10] = static_cast<uint64_t>(lua_tointeger(L, -1));
        ctx->x[11] = 1;
        lua_pop(L, 1);
        return -1;
    }

    case ECALL_LUA_GETGLOBAL: {
        // a0 = guest addr of name -> a0 = stack index of the global (handle).
        // Nil globals stay on the stack like the interpreter (#1751 Phase 1);
        // never decline for missing names.
        //
        // #1836: unprotected lua_getglobal can abort via a raising
        // _ENV __index metamethod.  Route through pcall like GET/SET/LEN.
        //
        if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        const char *key = guest_cstr(ec->memory, ec->memory_size, ctx->x[10]);
        if (nullptr == key) {
            ctx->x[10] = 0;
            ctx->x[11] = 0;
            return -1;
        }
        int pr = ecall_lua_pgetglobal(L, key);
        if (0 != pr) {
            return pr;
        }
        ctx->x[10] = static_cast<uint64_t>(lua_gettop(L));
        ctx->x[11] = 1;
        return -1;
    }

    case ECALL_LUA_GETFIELD_REF: {
        // a0=tbl_idx, a1=key addr -> a0=stack index of the field (handle).
        // Total: real gettable under pcall; nil fields stay as handles.
        //
        if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int tbl_idx = static_cast<int>(ctx->x[10]);
        const char *key = guest_cstr(ec->memory, ec->memory_size, ctx->x[11]);
        if (nullptr == key || !ecall_lua_stack_index_ok(L, tbl_idx)) {
            ctx->x[10] = 0;
            ctx->x[11] = 0;
            return -1;
        }
        int pr = ecall_lua_pget_strkey(L, tbl_idx, key);
        if (0 != pr) {
            return pr;
        }
        ctx->x[10] = static_cast<uint64_t>(lua_gettop(L));
        ctx->x[11] = 1;
        return -1;
    }

    case ECALL_LUA_CALL_STR: {
        // a0=fn stack idx, a1 = nargs | (argkind bits << 8), a2..args,
        // out addr/size in a5/a6.  #1751: after the callee may have run,
        // never soft-decline.  Marshal the first result like fun_lua
        // (numbers become decimal text — string.find's first value is an
        // integer; the chunk pcall keeps one result on both routes).
        //
        if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int fn_idx  = static_cast<int>(ctx->x[10]);
        int nargs   = static_cast<int>(ctx->x[11] & 0xFF);
        int kinds   = static_cast<int>((ctx->x[11] >> 8) & 0xFF);
        uint64_t out_addr = ctx->x[15];
        uint64_t out_size = ctx->x[16];

        if (nargs < 0 || nargs > 3
            || 0 == out_size
            || !guest_range_ok(out_addr, out_size, ec->memory_size)) {
            return ecall_lua_error_cstr("invalid call encoding");
        }
        if (fn_idx <= 0 || fn_idx > lua_gettop(L)
            || !lua_isfunction(L, fn_idx)) {
            return ecall_lua_error_cstr("attempt to call a non-function value");
        }

        int base = lua_gettop(L);
        lua_pushvalue(L, fn_idx);
        if (!ecall_lua_push_call_args(L, ctx, ec, nargs, kinds)) {
            lua_settop(L, base);
            return ecall_lua_error_cstr("invalid call argument");
        }
        if (LUA_OK != lua_pcall(L, nargs, 1, 0)) {
            int er = ecall_lua_commit_error(L);
            lua_settop(L, base);
            return er;
        }

        size_t slen = ecall_lua_marshal_to_guest(L, -1,
            ec->memory + out_addr, static_cast<size_t>(out_size));
        lua_settop(L, base);
        ctx->x[10] = static_cast<uint64_t>(slen);
        ctx->x[11] = 1;
        return -1;
    }

    case ECALL_LUA_CALL_VAL: {
        // Typed call result (#1764 shape 2): leave the first pcall value
        // on the Lua stack and return its absolute index as a handle.
        // Marshal only at the softcode boundary (HIR_LUA_MARSHAL / RET).
        //
        if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int fn_idx  = static_cast<int>(ctx->x[10]);
        int nargs   = static_cast<int>(ctx->x[11] & 0xFF);
        int kinds   = static_cast<int>((ctx->x[11] >> 8) & 0xFF);

        if (nargs < 0 || nargs > 3) {
            return ecall_lua_error_cstr("invalid call encoding");
        }
        if (fn_idx <= 0 || fn_idx > lua_gettop(L)
            || !lua_isfunction(L, fn_idx)) {
            return ecall_lua_error_cstr("attempt to call a non-function value");
        }

        int base = lua_gettop(L);
        lua_pushvalue(L, fn_idx);
        if (!ecall_lua_push_call_args(L, ctx, ec, nargs, kinds)) {
            lua_settop(L, base);
            return ecall_lua_error_cstr("invalid call argument");
        }
        if (LUA_OK != lua_pcall(L, nargs, 1, 0)) {
            int er = ecall_lua_commit_error(L);
            lua_settop(L, base);
            return er;
        }
        // Result remains at top; do not pop.
        ctx->x[10] = static_cast<uint64_t>(lua_gettop(L));
        ctx->x[11] = 1;
        return -1;
    }

    case ECALL_LUA_MARSHAL: {
        // a0=stack idx, a1=out addr, a2=out size → a0=len.  fun_lua rules.
        if (!ec->lua_state) { ctx->x[10] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int idx = static_cast<int>(ctx->x[10]);
        uint64_t out_addr = ctx->x[11];
        uint64_t out_size = ctx->x[12];
        if (0 == out_size
            || !guest_range_ok(out_addr, out_size, ec->memory_size)
            || !ecall_lua_stack_index_ok(L, idx)) {
            return ecall_lua_error_cstr("invalid marshal encoding");
        }
        size_t slen = ecall_lua_marshal_to_guest(L, idx,
            ec->memory + out_addr, static_cast<size_t>(out_size));
        ctx->x[10] = static_cast<uint64_t>(slen);
        return -1;
    }

    case ECALL_LUA_TOBOOL: {
        // a0=stack idx → a0 = 0/1 under Lua truthiness (only nil/false falsy).
        if (!ec->lua_state) { ctx->x[10] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int idx = static_cast<int>(ctx->x[10]);
        if (!ecall_lua_stack_index_ok(L, idx)) {
            ctx->x[10] = 0;
            return -1;
        }
        ctx->x[10] = lua_toboolean(L, idx) ? 1 : 0;
        return -1;
    }

    case ECALL_LUA_EQ: {
        // a0=lhs stack idx, a1=rhs kind, a2=rhs payload → a0=0/1 under Lua ==.
        // Kind: 0=int, 1=string guest addr, 2=handle stack idx, 3=nil, 4=bool.
        // Keeps type distinctions that a marshal-then-STRCMP would erase
        // (tonumber("17") == "17" is false; tostring(0) == "0" is true).
        //
        if (!ec->lua_state) { ctx->x[10] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int lhs = static_cast<int>(ctx->x[10]);
        int kind = static_cast<int>(ctx->x[11]);
        uint64_t rhs = ctx->x[12];
        if (!ecall_lua_stack_index_ok(L, lhs)) {
            ctx->x[10] = 0;
            return -1;
        }
        const int base = lua_gettop(L);
        lua_pushvalue(L, lhs);
        switch (kind) {
        case 0:
            lua_pushinteger(L, static_cast<lua_Integer>(
                static_cast<int64_t>(rhs)));
            break;
        case 1: {
            const char *s = guest_cstr(ec->memory, ec->memory_size, rhs);
            if (nullptr == s) {
                lua_settop(L, base);
                ctx->x[10] = 0;
                return -1;
            }
            lua_pushstring(L, s);
            break;
        }
        case 2: {
            int ridx = static_cast<int>(rhs);
            if (!ecall_lua_stack_index_ok(L, ridx)) {
                lua_settop(L, base);
                ctx->x[10] = 0;
                return -1;
            }
            lua_pushvalue(L, ridx);
            break;
        }
        case 3:
            lua_pushnil(L);
            break;
        case 4:
            lua_pushboolean(L, rhs != 0);
            break;
        default:
            lua_settop(L, base);
            ctx->x[10] = 0;
            return -1;
        }
        // Absolute indices: pushvalue after the cfunction would shift
        // relative slots.  #1836: __eq can raise; protect like GETGLOBAL.
        //
        const int lhs_abs = base + 1;
        const int rhs_abs = base + 2;
        int eq = 0;
        int pr = ecall_lua_pcompare_eq(L, lhs_abs, rhs_abs, &eq);
        lua_settop(L, base);
        if (0 != pr) {
            return pr;
        }
        ctx->x[10] = eq ? 1 : 0;
        return -1;
    }

    case ECALL_LUA_CALL_INT: {
        // a0=fn stack idx, a1=nargs (0..2), a2=arg0, a3=arg1, integers.
        // -> a0 = integer result, a1 = ok.
        //
        // Deliberately narrow: integer in, integer out, so this exercises
        // the whole global->field->call chain without also deciding how a
        // string result is marshalled.  Anything else declines and the
        // interpreter answers.
        if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int fn_idx = static_cast<int>(ctx->x[10]);
        int nargs  = static_cast<int>(ctx->x[11] & 0xFF);
        int kinds  = static_cast<int>((ctx->x[11] >> 8) & 0xFF);
        if (fn_idx <= 0 || fn_idx > lua_gettop(L)
            || !lua_isfunction(L, fn_idx) || nargs < 0 || nargs > 3) {
            ctx->x[11] = 0;
            return lua_ecall_decline("ECALL_LUA_CALL_INT");
        }
        int base = lua_gettop(L);
        lua_pushvalue(L, fn_idx);
        // Same argument encoding as CALL_STR, one decoder for both: the two
        // differ only in the RESULT type, so they have no business
        // differing in how arguments arrive -- tonumber("17") returns an
        // integer from a string argument and needs both halves.
        if (!ecall_lua_push_call_args(L, ctx, ec, nargs, kinds)) {
            lua_settop(L, base);
            ctx->x[11] = 0;
            return lua_ecall_decline("ECALL_LUA_CALL_INT");
        }
        if (LUA_OK != lua_pcall(L, nargs, 1, 0)) {
            lua_settop(L, base);
            ctx->x[11] = 0;
            return lua_ecall_decline("ECALL_LUA_CALL_INT");
        }
        if (!lua_isinteger(L, -1)) {
            lua_settop(L, base);
            ctx->x[11] = 0;
            return lua_ecall_decline("ECALL_LUA_CALL_INT");
        }
        ctx->x[10] = static_cast<uint64_t>(lua_tointeger(L, -1));
        ctx->x[11] = 1;
        lua_settop(L, base);
        return -1;
    }

    case ECALL_LUA_CALL_VOID: {
        // Call for effect: same encoding as CALL_INT/CALL_STR, result
        // discarded.  pcall is asked for zero results.  After the callee
        // may have run, do not soft-decline: a raised error is a real Lua
        // error (same as CALL_STR).  Pre-entry encoding misses still use
        // residual decline.
        //
        if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int fn_idx = static_cast<int>(ctx->x[10]);
        int nargs  = static_cast<int>(ctx->x[11] & 0xFF);
        int kinds  = static_cast<int>((ctx->x[11] >> 8) & 0xFF);
        if (fn_idx <= 0 || fn_idx > lua_gettop(L)
            || !lua_isfunction(L, fn_idx) || nargs < 0 || nargs > 3) {
            ctx->x[11] = 0;
            return lua_ecall_decline("ECALL_LUA_CALL_VOID");
        }
        int base = lua_gettop(L);
        lua_pushvalue(L, fn_idx);
        if (!ecall_lua_push_call_args(L, ctx, ec, nargs, kinds)) {
            lua_settop(L, base);
            return ecall_lua_error_cstr("invalid call argument");
        }
        if (LUA_OK != lua_pcall(L, nargs, 0, 0)) {
            int er = ecall_lua_commit_error(L);
            lua_settop(L, base);
            return er;
        }
        lua_settop(L, base);
        ctx->x[11] = 1;
        return -1;
    }

    case ECALL_LUA_LIMITED: {
        // Back-edge budget exhausted (#1732 / #1751 Phase 3).  Commit the
        // same text InsnCountHook raises on the interpreter route
        // (luaL_error "instruction limit exceeded" → softcode framing).
        // No re-run: the compiled path already spent the budget, and a
        // silent interpreter replay would double side effects.
        //
        return ecall_lua_error_cstr("instruction limit exceeded");
    }

    case ECALL_LUA_GETFIELD_INT: {
        // String-keyed int read via real gettable (#1751 Phase 1).
        // Typed claim: integer.  Non-integer → ok=0, no decline.
        //
        if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int tbl_idx = static_cast<int>(ctx->x[10]);
        const char *key = guest_cstr(ec->memory, ec->memory_size, ctx->x[11]);
        if (nullptr == key || !ecall_lua_stack_index_ok(L, tbl_idx)) {
            ctx->x[10] = 0;
            ctx->x[11] = 0;
            return -1;
        }
        int pr = ecall_lua_pget_strkey(L, tbl_idx, key);
        if (0 != pr) {
            return pr;
        }
        if (lua_isinteger(L, -1)) {
            ctx->x[10] = static_cast<uint64_t>(lua_tointeger(L, -1));
            ctx->x[11] = 1;
        } else {
            ctx->x[10] = 0;
            ctx->x[11] = 0;
        }
        lua_pop(L, 1);
        return -1;
    }

    case ECALL_LUA_GETFIELD_FLT: {
        // String-keyed float read via real gettable (#1751 Phase 1).
        // Only a genuine non-integer number fills the FP slot; else ok=0.
        //
        if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int tbl_idx = static_cast<int>(ctx->x[10]);
        const char *key = guest_cstr(ec->memory, ec->memory_size, ctx->x[11]);
        if (nullptr == key || !ecall_lua_stack_index_ok(L, tbl_idx)) {
            ctx->x[10] = 0;
            ctx->x[11] = 0;
            return -1;
        }
        int pr = ecall_lua_pget_strkey(L, tbl_idx, key);
        if (0 != pr) {
            return pr;
        }
        if (LUA_TNUMBER == lua_type(L, -1) && !lua_isinteger(L, -1)) {
            double d = lua_tonumber(L, -1);
            uint64_t bits;
            memcpy(&bits, &d, 8);
            ctx->x[10] = bits;
            ctx->x[11] = 1;
        } else {
            ctx->x[10] = 0;
            ctx->x[11] = 0;
        }
        lua_pop(L, 1);
        return -1;
    }

    case ECALL_LUA_SETFIELD_INT: {
        // a0=tbl_idx, a1=guest key addr, a2=integer value.  Total settable.
        //
        if (!ec->lua_state) return -1;
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int tbl_idx = static_cast<int>(ctx->x[10]);
        const char *key = guest_cstr(ec->memory, ec->memory_size, ctx->x[11]);
        lua_Integer val = static_cast<lua_Integer>(ctx->x[12]);
        if (nullptr == key || !ecall_lua_stack_index_ok(L, tbl_idx)) {
            return -1;
        }
        int pr = ecall_lua_pset_strkey(L, tbl_idx, key, val);
        if (0 != pr) {
            return pr;
        }
        return -1;
    }

    case ECALL_LUA_INSN_BUDGET: {
        // a0 = the CURRENT lua_instruction_limit.  The whole point is that
        // this reads mudconf at run time: the program carries no config
        // value, so @admin lua_instruction_limit takes effect on the next
        // run of every cached and persisted blob alike (#1745, #1613).
        int64_t lim = static_cast<int64_t>(mudconf.lua_instruction_limit);
        if (lim < 1)
        {
            lim = 1;   // a zero/negative limit must abort loops, not arm an
                       // effectively-unbounded unsigned countdown
        }
        ctx->x[10] = static_cast<uint64_t>(lim);
        return -1;
    }

    case ECALL_LUA_LEN_INT: {
        // Lua's # via luaL_len under pcall (#1751 Phase 1 / #1424 root).
        // Honours __len; raises as the interpreter would (committed error).
        //
        if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int tbl_idx = static_cast<int>(ctx->x[10]);
        if (!ecall_lua_stack_index_ok(L, tbl_idx)) {
            ctx->x[10] = 0;
            ctx->x[11] = 0;
            return -1;
        }
        int pr = ecall_lua_plen(L, tbl_idx);
        if (0 != pr) {
            return pr;
        }
        ctx->x[10] = static_cast<uint64_t>(lua_tointeger(L, -1));
        ctx->x[11] = 1;
        lua_pop(L, 1);
        return -1;
    }

    case ECALL_LUA_SETI_INT: {
        // Integer-keyed table set via real settable (#1751 Phase 1).
        //
        if (!ec->lua_state) return -1;
        lua_State *L = static_cast<lua_State *>(ec->lua_state);
        int tbl_idx = static_cast<int>(ctx->x[10]);
        lua_Integer key = static_cast<lua_Integer>(ctx->x[11]);
        lua_Integer val = static_cast<lua_Integer>(ctx->x[12]);
        if (!ecall_lua_stack_index_ok(L, tbl_idx)) {
            return -1;
        }
        int pr = ecall_lua_pset_intkey(L, tbl_idx, key, val);
        if (0 != pr) {
            return pr;
        }
        return -1;
    }



    case ECALL_CALL_COMPILED: {
        // Re-entrant call into a compiled function within the
        // persistent VM.  Saves the full CPU context, runs the
        // inner function via dbt_resume, restores context.
        //
        // a0 = entry_pc of target function
        // a1 = guest addr of target's output buffer
        // Returns: a0 = result string length (0 on failure)
        //
        if (!ec->dbt) {
            ctx->x[10] = 0;
            return -1;
        }

        uint64_t target_pc = ctx->x[10];
        uint64_t out_ref   = ctx->x[11];

        // Save outer CPU context.
        rv64_ctx_t saved_ctx = *ctx;

        // Run inner function.
        int inner_rc = dbt_resume(ec->dbt, target_pc);

        // Extract result length (#1057: bound guest NUL scan).
        uint64_t result_len = 0;
        uint64_t resolved_out = resolve_runtime_out_addr(out_ref, saved_ctx.x[2]);
        size_t n = 0;
        if (inner_rc == 0 && resolved_out > 0
            && guest_strnlen(ec->memory, ec->memory_size, resolved_out, &n)) {
            result_len = n;
        }

        // Restore outer context.
        *ctx = saved_ctx;
        ctx->x[10] = result_len;
        ctx->x[11] = resolved_out;
        return -1;  // continue outer execution
    }

    case ECALL_COMPILE_ATTR: {
        // Resolve an attribute, compile its body into the persistent
        // VM, and return the entry point.
        //
        // a0 = dbref of target object
        // a1 = guest addr of attribute name string
        // Returns: a0 = entry_pc (0 on failure)
        //          a1 = out_addr
        //          a2 = aflags
        //
        persistent_vm_t *pvm =
            static_cast<persistent_vm_t *>(ec->pvm);
        if (!pvm) {
            ctx->x[10] = 0;
            return -1;
        }

        dbref obj = static_cast<dbref>(ctx->x[10]);
        uint64_t name_addr = ctx->x[11];

        if (!Good_obj(obj) || name_addr >= ec->memory_size) {
            ctx->x[10] = 0;
            return -1;
        }

        const UTF8 *attr_name = ec->memory + name_addr;
        ATTR *ap = atr_str(attr_name);
        if (!ap) {
            ctx->x[10] = 0;
            return -1;
        }

        if (!See_attr(ec->executor, obj, ap)) {
            ctx->x[10] = 0;
            return -1;
        }

        dbref aowner;
        int aflags;
        UTF8 *atext = atr_pget(obj, ap->number, &aowner, &aflags);
        if (!atext || !*atext) {
            if (atext) free_lbuf(atext);
            ctx->x[10] = 0;
            return -1;
        }

        // Skip compilation if NOEVAL.
        if ((aflags & AF_NOEVAL) || NoEval(obj)) {
            free_lbuf(atext);
            ctx->x[10] = 0;
            ctx->x[12] = static_cast<uint64_t>(aflags);
            return -1;
        }

        // Compile (with caching).
        size_t alen = strlen(reinterpret_cast<const char *>(atext));
        auto cr = pvm->compile_attr(obj, ap->number, atext, alen);
        free_lbuf(atext);

        ctx->x[10] = cr.entry_pc;
        ctx->x[11] = cr.out_addr;
        ctx->x[12] = static_cast<uint64_t>(aflags);
        return -1;  // continue
    }

    default:
        ctx->x[10] = 0;
        return -1;
    }
}

// ---------------------------------------------------------------
// jit_eval: try to compile and execute an expression via the JIT.
//
// Returns true if the JIT handled it (result written to buff/bufc).
// ---------------------------------------------------------------
// Tier 3 helper functions: guest-memory CARGS operations.
//
// These are FUNCTION() handlers registered in the standard function
// table.  They use s_current_ecall_ctx (set by ecall_invoke_fun)
// to access JIT guest memory for CARGS slot operations.
// ---------------------------------------------------------------

static constexpr int MAX_CARG_SAVE_DEPTH = 16;
static struct {
    uint8_t data[10 * rv_compiler::CARGS_SLOT];
    int ncargs;
    bool in_use;
} s_carg_save_stack[MAX_CARG_SAVE_DEPTH];

// Executor-context stack for inlined u()/ulocal() bodies (#2179).
// fun_u evaluates the attribute body with executor = the object holding
// the attribute (and caller = the previous executor); an inlined body
// runs inside the CALLER's program, so without this swap every
// executor-derived thing inside it was the caller's: %! (SUBST slot),
// %va-%vz / %=<name> (xget against the SUBST slot), and every ECALL's
// ec->executor (v(), bare-name u(), name(me), permission checks).
//
static constexpr int MAX_UEXEC_SAVE_DEPTH = 16;
static struct {
    dbref executor;
    dbref caller;
    bool  in_use;
} s_uexec_save_stack[MAX_UEXEC_SAVE_DEPTH];

// Both save stacks leak their slot if a program is abandoned between a
// save/push and its restore/pop (DBT decline, error unwind): nothing
// runs the paired helper, in_use stays set, and the stack eventually
// fills — after which _SAVE_CARGS returns -1 (restore skipped, CARGS
// clobbered for the caller) and _PUSH_UEXEC falls back to the fun_u
// ECALL.  A top-level entry proves no outer program holds a live
// handle, so everything still marked in-use is leaked garbage.
// run_cached_program calls this when s_run_cached_depth == 0.
//
static void jit_helper_stacks_reset()
{
    for (int i = 0; i < MAX_CARG_SAVE_DEPTH; i++) {
        s_carg_save_stack[i].in_use = false;
    }
    for (int i = 0; i < MAX_UEXEC_SAVE_DEPTH; i++) {
        s_uexec_save_stack[i].in_use = false;
    }
}

// Render "#<dbref>" into the guest %! substitution slot so inlined-body
// reads of %! / %va-%vz / %=<name> see the swapped executor.  Writing
// when the program never reads the slot is harmless.
//
static void uexec_write_subst(eval_ctx *ec, dbref executor)
{
    uint64_t slot = rv_compiler::SUBST_BASE
        + static_cast<uint64_t>(rv_compiler::SUBST_EXECUTOR)
        * rv_compiler::SUBST_SLOT;
    if (slot + rv_compiler::SUBST_SLOT <= ec->memory_size)
    {
        mux_sprintf(ec->memory + slot, rv_compiler::SUBST_SLOT,
                    T("#%d"), executor);
    }
}

// _PUSH_UEXEC(thing_dbref_str): enter an inlined u() body's executor
// context (#2179) — save executor/caller, set caller = old executor and
// executor = thing (mirroring fun_u's mux_exec call), refresh the guest
// %! slot.  Returns a handle for _POP_UEXEC, or "-1" on failure — the
// lowering branches a failed push to the fun_u ECALL fallback, which
// establishes its own context, so exhaustion costs the inline, never
// correctness.
//
FUNCTION(fun__push_uexec)
{
    UNUSED_PARAMETER(executor);
    UNUSED_PARAMETER(caller);
    UNUSED_PARAMETER(enactor);
    UNUSED_PARAMETER(eval);
    UNUSED_PARAMETER(cargs);
    UNUSED_PARAMETER(ncargs);

    eval_ctx *ec = s_current_ecall_ctx;
    dbref thing = (nfargs >= 1) ? mux_atoi64(fargs[0]) : NOTHING;
    if (!ec || !Good_obj(thing))
    {
        safe_str(T("-1"), buff, bufc);
        return;
    }

    for (int i = 0; i < MAX_UEXEC_SAVE_DEPTH; i++)
    {
        if (!s_uexec_save_stack[i].in_use)
        {
            s_uexec_save_stack[i].executor = ec->executor;
            s_uexec_save_stack[i].caller = ec->caller;
            s_uexec_save_stack[i].in_use = true;
            ec->caller = ec->executor;
            ec->executor = thing;
            uexec_write_subst(ec, thing);
            safe_ltoa(i, buff, bufc);
            return;
        }
    }
    safe_str(T("-1"), buff, bufc);
}

// _POP_UEXEC(handle_str): restore the pre-inline executor context and
// the guest %! slot.
//
FUNCTION(fun__pop_uexec)
{
    UNUSED_PARAMETER(executor);
    UNUSED_PARAMETER(caller);
    UNUSED_PARAMETER(enactor);
    UNUSED_PARAMETER(eval);
    UNUSED_PARAMETER(cargs);
    UNUSED_PARAMETER(ncargs);
    UNUSED_PARAMETER(buff);
    UNUSED_PARAMETER(bufc);

    eval_ctx *ec = s_current_ecall_ctx;
    if (!ec || nfargs < 1) return;

    int64_t idx = mux_atoi64(fargs[0]);
    if (idx >= 0 && idx < MAX_UEXEC_SAVE_DEPTH
        && s_uexec_save_stack[idx].in_use)
    {
        ec->executor = s_uexec_save_stack[idx].executor;
        ec->caller = s_uexec_save_stack[idx].caller;
        s_uexec_save_stack[idx].in_use = false;
        uexec_write_subst(ec, ec->executor);
    }
}

// _SAVE_CARGS(): save the CARGS region of guest memory.
// Returns a handle string (index into save stack).
//
FUNCTION(fun__save_cargs)
{
    UNUSED_PARAMETER(executor);
    UNUSED_PARAMETER(caller);
    UNUSED_PARAMETER(enactor);
    UNUSED_PARAMETER(eval);
    UNUSED_PARAMETER(nfargs);
    UNUSED_PARAMETER(cargs);
    UNUSED_PARAMETER(ncargs);

    if (!s_current_ecall_ctx)
    {
        safe_str(T("-1"), buff, bufc);
        return;
    }

    for (int i = 0; i < MAX_CARG_SAVE_DEPTH; i++)
    {
        if (!s_carg_save_stack[i].in_use)
        {
            uint64_t base = rv_compiler::CARGS_BASE;
            size_t region = 10 * rv_compiler::CARGS_SLOT;
            if (base + region <= s_current_ecall_ctx->memory_size)
            {
                memcpy(s_carg_save_stack[i].data,
                       s_current_ecall_ctx->memory + base, region);
            }
            s_carg_save_stack[i].ncargs = s_current_ecall_ctx->ncargs;
            s_carg_save_stack[i].in_use = true;
            safe_ltoa(i, buff, bufc);
            return;
        }
    }
    safe_str(T("-1"), buff, bufc);
}

// _RESTORE_CARGS(handle_str): restore saved CARGS region.
//
FUNCTION(fun__restore_cargs)
{
    UNUSED_PARAMETER(executor);
    UNUSED_PARAMETER(caller);
    UNUSED_PARAMETER(enactor);
    UNUSED_PARAMETER(eval);
    UNUSED_PARAMETER(cargs);
    UNUSED_PARAMETER(ncargs);

    if (!s_current_ecall_ctx || nfargs < 1) return;

    int64_t idx = mux_atoi64(fargs[0]);
    if (idx >= 0 && idx < MAX_CARG_SAVE_DEPTH
        && s_carg_save_stack[idx].in_use)
    {
        uint64_t base = rv_compiler::CARGS_BASE;
        size_t region = 10 * rv_compiler::CARGS_SLOT;
        if (base + region <= s_current_ecall_ctx->memory_size)
        {
            memcpy(s_current_ecall_ctx->memory + base,
                   s_carg_save_stack[idx].data, region);
        }
        s_current_ecall_ctx->ncargs = s_carg_save_stack[idx].ncargs;
        s_carg_save_stack[idx].in_use = false;

        // Refresh the guest %+ substitution slot.
        uint64_t nslot = rv_compiler::SUBST_BASE
            + static_cast<uint64_t>(rv_compiler::SUBST_NCARGS)
            * rv_compiler::SUBST_SLOT;
        if (nslot + 4 <= s_current_ecall_ctx->memory_size)
        {
            char nbuf[16];
            size_t len = mux_snprintf(reinterpret_cast<UTF8 *>(nbuf), sizeof(nbuf),
                               T("%d"), s_carg_save_stack[idx].ncargs);
            memcpy(s_current_ecall_ctx->memory + nslot, nbuf, len + 1);
        }
    }
}

// _SET_NCARGS(n_str): update the %+ substitution slot and ncargs.
// Must be called after writing CARGS slots for inlined u() bodies
// so that %+ in the body reflects the callee's argument count.
//
FUNCTION(fun__set_ncargs)
{
    UNUSED_PARAMETER(executor);
    UNUSED_PARAMETER(caller);
    UNUSED_PARAMETER(enactor);
    UNUSED_PARAMETER(eval);
    UNUSED_PARAMETER(cargs);
    UNUSED_PARAMETER(ncargs);

    if (!s_current_ecall_ctx || nfargs < 1) return;

    int64_t n = mux_atoi64(fargs[0]);
    if (n < 0) n = 0;
    if (n > 10) n = 10;

    s_current_ecall_ctx->ncargs = n;

    // Update the guest %+ substitution slot.
    uint64_t slot = rv_compiler::SUBST_BASE
        + static_cast<uint64_t>(rv_compiler::SUBST_NCARGS)
        * rv_compiler::SUBST_SLOT;
    if (slot + 4 <= s_current_ecall_ctx->memory_size)
    {
        char nbuf[16];
        size_t len = mux_snprintf(reinterpret_cast<UTF8 *>(nbuf), sizeof(nbuf), T("%d"), n);
        memcpy(s_current_ecall_ctx->memory + slot, nbuf, len + 1);
    }
}

// _WRITE_CARG(idx_str, value_str): write value to a CARGS slot.
//
FUNCTION(fun__write_carg)
{
    UNUSED_PARAMETER(executor);
    UNUSED_PARAMETER(caller);
    UNUSED_PARAMETER(enactor);
    UNUSED_PARAMETER(eval);
    UNUSED_PARAMETER(cargs);
    UNUSED_PARAMETER(ncargs);

    if (!s_current_ecall_ctx || nfargs < 2) return;

    int64_t idx = mux_atoi64(fargs[0]);
    if (idx < 0 || idx >= 10) return;

    uint64_t dst = rv_compiler::CARGS_BASE
                 + static_cast<uint64_t>(idx) * rv_compiler::CARGS_SLOT;
    if (dst + rv_compiler::CARGS_SLOT > s_current_ecall_ctx->memory_size) return;

    // fargs[1] is a guest-memory pointer (from ecall_invoke_fun).
    // Bound the NUL scan; reject if missing.  Values that do not fit
    // the slot are also rejected rather than silently truncated
    // (#1055 / #1057) — the host has no clean way to decline the whole
    // program from here, so leave the slot untouched.
    eval_ctx *ec = s_current_ecall_ctx;
    if (fargs[1] < ec->memory
        || fargs[1] >= ec->memory + ec->memory_size) {
        return;
    }
    uint64_t val_addr = static_cast<uint64_t>(fargs[1] - ec->memory);
    size_t len = 0;
    if (!guest_strnlen(ec->memory, ec->memory_size, val_addr, &len)) {
        return;
    }
    if (len >= static_cast<size_t>(rv_compiler::CARGS_SLOT)) {
        return;
    }

    memcpy(ec->memory + dst, fargs[1], len);
    ec->memory[dst + len] = 0;
}

// ---------------------------------------------------------------
// Returns false if compilation failed — caller should fall back to
// the AST evaluator.
//
// This is the entry point for --enable-jit's mux_exec integration.
// ---------------------------------------------------------------

bool jit_eval(const UTF8 *expr, size_t nLen,
              UTF8 *buff, UTF8 **bufc,
              dbref executor, dbref caller, dbref enactor,
              int eval,
              const UTF8 *cargs[], int ncargs) {
    // sandbox() sets bSandboxActive to force AST-only evaluation,
    // which checks fp->perms (CA_DISABLED) before each function call.
    //
    if (mudstate.bSandboxActive)
    {
        return false;
    }

    // Re-entrancy depth tracking.  When JIT code ECALLs into a
    // function like u() which calls mux_exec(), the nested mux_exec
    // re-enters jit_eval().
    //
    // Depth 0 (top-level): full JIT — compile, constant-fold, or
    //   DBT-execute.
    // Depth 1+ (nested): compile and return constant-folded results
    //   (no DBT needed), but fall back to AST for programs that
    //   require runtime execution.  This is step 1 toward full
    //   re-entrant JIT: inner expressions that fold at compile time
    //   get the JIT result without touching the DBT.
    //
    static int s_jit_depth = 0;
    s_jit_depth++;

    struct jit_depth_guard {
        ~jit_depth_guard() { s_jit_depth--; }
    } depth_guard;

    if (s_jit_depth == 1) {
        JITArena::gc();
        jit_dma_controller::reset();
    }

    // Don't JIT until the Tier 2 blob is loaded and the persistent
    // DBT state is initialized.  compile_cached calls tier2_lazy_init,
    // but the DBT infrastructure (mmap, block cache) may not be safe
    // to initialize during early startup (config loading, @startup).
    // The per-context dbt_ready flag is set after the first successful
    // get_dbt.
    //
    // The loaded check must be OUTSIDE the init-once conditional: with
    // it inside, only the first call declined when the blob was missing
    // and every later call ran the JIT blob-less, producing silently
    // wrong results for some compiled shapes (#875).
    if (!s_tier2_init) {
        tier2_lazy_init();
    }
    if (!s_tier2.loaded) return false;

    s_jit_stats.eval_attempts++;

    // Memoized decline (#2130): a shape already known to be bail_noop is
    // refused here, before the compile cache and before any SQLite fetch.
    // The server used to do a SELECT plus a full program deserialization
    // per evaluation in order to read four integers and not run the result.
    {
        std::string memo_key = compile_cache_key(expr, nLen, eval);
        if (s_decline_memo.count(memo_key)) {
            s_jit_stats.bail_noop++;
            s_jit_stats.noop_memo++;
            s_jit_stats.eval_bailout++;
            return false;
        }
    }

    compiled_program *prog = compile_cached(expr, nLen, eval);
    if (!prog) {
        s_jit_stats.eval_bailout++;
        return false;
    }

    // #1002 static-depth watermark: the AST evaluator errors a call at
    // nesting level L when func_nest_lim <= L; compiled code flattens
    // the nest (and folding pre-computes it), so decline to the AST
    // whenever this evaluation could reach the limit.  Uses the LIVE
    // func_nest_lev so nested entry contexts count, and the live limit
    // so @admin changes apply to cached programs.
    if (mudstate.func_nest_lev + prog->max_func_depth
        >= mudconf.func_nest_lim) {
        s_jit_stats.bail_depth++;
        s_jit_stats.eval_bailout++;
        return false;
    }

    // Static invocation-count watermark: flattened sequential calls do
    // not maintain func_invk_ctr.  Decline when the live counter plus
    // this program's FUNCCALL count would trip function_invocation_limit.
    if (  0 < prog->n_func_calls
       && mudstate.func_invk_ctr + prog->n_func_calls
          >= mudconf.func_invk_lim) {
        s_jit_stats.bail_invk++;
        s_jit_stats.eval_bailout++;
        return false;
    }

    if (!prog->needs_jit) {
        // Constant-folded — result was extracted at compaction time.
        // Safe at any nesting depth (no DBT involved).
        s_jit_stats.folded_total++;
        s_jit_stats.eval_handled++;
        safe_str(reinterpret_cast<const UTF8 *>(prog->folded_result.c_str()),
                 buff, bufc);
        return true;
    }

    // Decline a program that does no work (#2086).
    //
    // When a lowering is a single host ECALL with nothing computed around it
    // -- no native arithmetic, no constant folding, no tier 2 call -- running
    // it performs exactly the host call the AST evaluator would have made,
    // and adds guest entry, argument marshalling and exit on top.  There is no
    // arrangement of those costs that comes out ahead; the compiled path can
    // only be the interpreter's cost plus a constant.
    //
    // Measured for get(perfobj/perf0): 108 bytes of RV64, one ECALL per
    // evaluation, zero tier2/folds, and 1.35x the no-JIT path on
    // Linux/aarch64 (1.36x on Windows/MSVC via #2083's attrread phase).
    // The shape covers get/v/u and most attribute and database accessors,
    // which is what live games actually call (#2064).
    //
    // Correctness is identical either way -- both routes call the same host
    // function -- so this only chooses the cheaper route to it.
    //
    // Deliberately conservative: ONE ecall and nothing else.  A program with
    // two ECALLs has already saved one AST dispatch, and anything with
    // native_ops, folds or tier2_calls is doing work the interpreter would
    // otherwise repeat.  Widening this predicate would start declining
    // programs the JIT genuinely wins on, and that failure mode looks like a
    // speedup on every benchmark that is not measuring the thing it broke.
    //
    if (   0 == prog->native_ops
        && 0 == prog->folds
        && 0 == prog->tier2_calls
        && 1 >= prog->ecalls) {
        // Memoize the verdict (#2130) so later evaluations refuse before
        // any cache machinery runs.  Dep-free only: a program with inline
        // deps can recompile into a different shape when an attr changes,
        // and noop shapes are dep-free in practice (deps come from
        // u()-inlining, which emits code).
        if (prog->deps.empty()) {
            if (s_decline_memo.size() >= DECLINE_MEMO_MAX) {
                s_decline_memo.clear();
            }
            s_decline_memo.insert(compile_cache_key(expr, nLen, eval));
        }
        s_jit_stats.bail_noop++;
        s_jit_stats.eval_bailout++;
        return false;
    }

    // Depth > 1: try executing via the shared heap's independent DBT.
    // The shared heap compiles into persistent memory and runs in its
    // own DBT context, so this is safe during an outer ECALL.
    if (s_jit_depth > 1) {
        LBuf shresult = LBuf_Src("jit_eval.shared");
        int  sh_ecalls = 0;
        int  sh_tier2  = 0;
        bool sh_folded = false;
        if (s_shared_heap.eval(expr, nLen, shresult, LBUF_SIZE,
                               executor, caller, enactor, eval,
                               cargs, ncargs,
                               &sh_ecalls, &sh_tier2, &sh_folded)) {
            s_jit_stats.eval_handled++;
            s_jit_stats.ecall_total  += sh_ecalls;
            s_jit_stats.tier2_total  += sh_tier2;
            if (sh_folded) s_jit_stats.folded_total++;
            safe_str(shresult, buff, bufc);
            return true;
        }
        s_jit_stats.eval_bailout++;
        return false;
    }

    LBuf result = LBuf_Src("jit_eval");
    if (!run_cached_program(prog, executor, caller, enactor,
                             result, LBUF_SIZE,
                             cargs, ncargs, eval)) {
        s_jit_stats.eval_bailout++;
        return false;  // JIT execution error — fall back to AST.
    }
    s_jit_stats.eval_handled++;
    s_jit_stats.ecall_total += prog->ecalls;
    s_jit_stats.tier2_total += prog->tier2_calls;
    safe_str(result, buff, bufc);
    return true;
}

// ---------------------------------------------------------------
// jitstats() — wizard-only function returning JIT profiling counters.
//
// Returns a space-separated key=value list suitable for parsing.
// With argument "reset", clears all counters.
// ---------------------------------------------------------------

// Sample jit_eval()'s attempt/handled counters (#2133 item 5).
//
// A narrow accessor rather than exporting s_jit_stats: functions.cpp cannot
// include dbt_compile.h (it collides with color_ops.h's C-linkage
// declarations), and benchmark() needs exactly these two numbers to report
// whether the JIT actually ran during a timed loop.
//
void jit_eval_counters(uint64_t *pAttempts, uint64_t *pHandled)
{
    if (nullptr != pAttempts) *pAttempts = s_jit_stats.eval_attempts;
    if (nullptr != pHandled)  *pHandled  = s_jit_stats.eval_handled;
}

FUNCTION(fun_jitstats)
{
    UNUSED_PARAMETER(fp);
    UNUSED_PARAMETER(caller);
    UNUSED_PARAMETER(enactor);
    UNUSED_PARAMETER(eval);
    UNUSED_PARAMETER(cargs);
    UNUSED_PARAMETER(ncargs);

    if (!Wizard(executor)) {
        safe_str(S_("#-1 PERMISSION DENIED"), buff, bufc);
        return;
    }

    if (nfargs >= 1) {
        const char *arg = reinterpret_cast<const char *>(fargs[0]);
        if (strcmp(arg, "reset") == 0) {
            memset(&s_jit_stats, 0, sizeof(s_jit_stats));
            jit_lua_reset_stats();
            safe_str(T("OK"), buff, bufc);
            return;
        }
        if (strcmp(arg, "flush") == 0) {
            // Wipe the persisted JIT code_cache and every in-process
            // compiled program that would still serve the old codegen.
            // Without this, a later build can load a code_cache row written
            // by an earlier one (blob_hash only covers the Tier-2 blob, not
            // engine codegen changes) — the A/B trap from #1315 / #1316.
            //
            // 1. Drop pending OP_CODE_CACHE_PUT so a later write-queue flush
            //    cannot re-insert what we are about to DELETE.
            // 2. DELETE FROM code_cache.
            // 3. Clear softcode and Lua in-memory compile caches.
            //
#if defined(HAVE_WORKING_FORK)
            if (mudstate.write_protect) {
                safe_str(S_("#-1 PERMISSION DENIED"), buff, bufc);
                return;
            }
#endif
            cache_discard_code_cache_writes();
            bool ok = true;
            if (g_pSQLiteBackend) {
                ok = g_pSQLiteBackend->GetDB().CodeCacheFlush();
            }
            // The in-memory caches can only be cleared when no compiled
            // program is live: an ECALL can reach this function from inside
            // one, and clearing would free the program still executing.  See
            // s_code_cache_flush_pending.  Nothing is lost by waiting -- the
            // drain runs before the next program is looked up, so no stale
            // native code can be served in between.
            //
            if (0 == s_run_cached_depth) {
                jit_flush_memory_caches();
            } else {
                s_code_cache_flush_pending = true;
            }
            if (!ok) {
                safe_str(S_("#-1 CODE CACHE FLUSH FAILED"), buff, bufc);
                return;
            }
            safe_str(T("OK"), buff, bufc);
            return;
        }
    }

    // Format: key=value pairs, newline-separated for readability.
    LBuf tmp = LBuf_Src("jitstats");
    size_t n = mux_snprintf(tmp.get(), LBUF_SIZE,
        T("eval_attempts=%llu "
        "eval_handled=%llu "
        "eval_bailout=%llu "
        "cache_hit_mem=%llu "
        "cache_hit_sqlite=%llu "
        "cache_miss=%llu "
        "compile_ok=%llu "
        "compile_fail=%llu "
        "bail_noeval=%llu "
        "bail_slots=%llu "
        "folded=%llu "
        "ecalls=%llu "
        "tier2=%llu "
        "code_bytes=%llu "
        "code_max=%llu "
        "hir_insns=%llu "
        "hir_max=%llu "
        "spills=%llu "
        "qreg_resyncs=%llu "
        "bail_longreg=%llu "
        "bail_depth=%llu "
        "bail_invk=%llu "
        "bail_alarm=%llu "
        "bail_shared_busy=%llu "
        "bail_noop=%llu "
        "bail_code=%llu "
        "bail_strpool=%llu "
        "bail_fargs=%llu "
        "bail_outslots=%llu "
        "want_code_max=%llu "
        "want_strpool_max=%llu "
        "want_fargs_max=%llu "
        "want_outslots_max=%llu "
        "slot_hit=%llu "
        "slot_miss=%llu "
        "slot_evict=%llu "
        "slot_churn0=%llu "
        "slot_pinned=%llu "
        "noop_memo=%llu"),
        (unsigned long long)s_jit_stats.eval_attempts,
        (unsigned long long)s_jit_stats.eval_handled,
        (unsigned long long)s_jit_stats.eval_bailout,
        (unsigned long long)s_jit_stats.cache_hit_mem,
        (unsigned long long)s_jit_stats.cache_hit_sqlite,
        (unsigned long long)s_jit_stats.cache_miss,
        (unsigned long long)s_jit_stats.compile_ok,
        (unsigned long long)s_jit_stats.compile_fail,
        (unsigned long long)s_jit_stats.bail_noeval,
        (unsigned long long)s_jit_stats.bail_slots,
        (unsigned long long)s_jit_stats.folded_total,
        (unsigned long long)s_jit_stats.ecall_total,
        (unsigned long long)s_jit_stats.tier2_total,
        (unsigned long long)s_jit_stats.code_bytes_total,
        (unsigned long long)s_jit_stats.code_bytes_max,
        (unsigned long long)s_jit_stats.hir_insns_total,
        (unsigned long long)s_jit_stats.hir_insns_max,
        (unsigned long long)s_jit_stats.spills_total,
        (unsigned long long)s_jit_stats.qreg_resyncs,
        (unsigned long long)s_jit_stats.bail_longreg,
        (unsigned long long)s_jit_stats.bail_depth,
        (unsigned long long)s_jit_stats.bail_invk,
        (unsigned long long)s_jit_stats.bail_alarm,
        (unsigned long long)s_jit_stats.bail_shared_busy,
        (unsigned long long)s_jit_stats.bail_noop,
        (unsigned long long)s_jit_stats.bail_code,
        (unsigned long long)s_jit_stats.bail_strpool,
        (unsigned long long)s_jit_stats.bail_fargs,
        (unsigned long long)s_jit_stats.bail_outslots,
        (unsigned long long)s_jit_stats.want_code_max,
        (unsigned long long)s_jit_stats.want_strpool_max,
        (unsigned long long)s_jit_stats.want_fargs_max,
        (unsigned long long)s_jit_stats.want_outslots_max,
        (unsigned long long)s_jit_stats.slot_hit,
        (unsigned long long)s_jit_stats.slot_miss,
        (unsigned long long)s_jit_stats.slot_evict,
        (unsigned long long)s_jit_stats.slot_churn0,
        (unsigned long long)s_jit_stats.slot_pinned,
        (unsigned long long)s_jit_stats.noop_memo);

    // Append Lua JIT counters (#1316).  lua_run_fail incrementing while
    // softcode still returns correct answers is the signature of a Lua JIT
    // that compiles and then silently falls back to the interpreter.
    if (n < static_cast<int>(LBUF_SIZE) - 256) {
        lua_jit_counters lj = {};
        jit_lua_get_stats(&lj);
        n += mux_snprintf(tmp.get() + n, LBUF_SIZE - n,
            T(" lua_compile_ok=%llu"
            " lua_compile_fail=%llu"
            " lua_run_ok=%llu"
            " lua_run_fail=%llu"
            " lua_cache_hits=%llu"
            " lua_invalidations=%llu"
            " lua_post_entry_decline=%llu"),
            (unsigned long long)lj.compile_ok,
            (unsigned long long)lj.compile_fail,
            (unsigned long long)lj.run_ok,
            (unsigned long long)lj.run_fail,
            (unsigned long long)lj.cache_hits,
            (unsigned long long)lj.invalidations,
            (unsigned long long)lj.post_entry_decline);
    }

    // Append NOEVAL breakdown.
    for (int i = 0; i < s_jit_stats.noeval_top_used && n < static_cast<int>(LBUF_SIZE) - 64; i++) {
        n += mux_snprintf(tmp.get() + n, LBUF_SIZE - n, T(" noeval_%s=%llu"),
            s_jit_stats.noeval_top[i].name,
            (unsigned long long)s_jit_stats.noeval_top[i].count);
    }

    // Append DBT code-buffer occupancy (#1315).  The Tier-2 blob is
    // pretranslated once and preserved across every dbt_reset, so
    // dbt_blob_bytes is a permanent reservation out of dbt_code_cap and
    // what remains is all any program will ever get.  That cost is
    // backend-specific -- the same blob is not the same number of host
    // bytes on Win64, x64 SysV and aarch64 -- so it wants measuring per
    // platform rather than assuming one constant suits every backend.
    //
    // dbt_code_full counting up means translations are being declined for
    // want of space; dbt_code_reclaims counts the mid-run recoveries that
    // keep such a decline local to one program instead of permanent.
    //
    // Summed over every initialized run context, not just depth 0 (#1326).
    // Each nesting depth owns a separate DBT with a separate code buffer, so
    // reading only s_vm[0] hides a nested Lua run filling its own -- and
    // dbt_code_full is precisely the signal that would be hidden.  cap is
    // summed the same way so used/cap stays a ratio of the same population;
    // a context that has not been initialized has no buffer to report.
    //
    unsigned long long dbt_cap = 0, dbt_blob = 0, dbt_used = 0;
    unsigned long long dbt_reclaims = 0, dbt_full = 0;
    for (int i = 0; i < JIT_MAX_RUN_DEPTH; i++) {
        if (!s_vm[i].dbt_ready) {
            continue;
        }
        dbt_cap      += CODE_BUF_SIZE;
        dbt_blob     += s_vm[i].dbt.blob_code_end;
        dbt_used     += s_vm[i].dbt.code_used;
        dbt_reclaims += s_vm[i].dbt.code_reclaims;
        dbt_full     += s_vm[i].dbt.code_full;
    }

    if (n < static_cast<int>(LBUF_SIZE) - 256) {
        n += mux_snprintf(tmp.get() + n, LBUF_SIZE - n,
            T(" dbt_code_cap=%u"
            " dbt_blob_bytes=%u"
            " dbt_code_used=%u"
            " dbt_code_reclaims=%llu"
            " dbt_code_full=%llu"),
            static_cast<unsigned>(dbt_cap),
            static_cast<unsigned>(dbt_blob),
            static_cast<unsigned>(dbt_used),
            static_cast<unsigned long long>(dbt_reclaims),
            static_cast<unsigned long long>(dbt_full));
    }

    safe_str(tmp, buff, bufc);
}

// ---------------------------------------------------------------
// fun_rvbench: benchmark JIT vs native mux_exec.
//
// rvbench(<expression>, <iterations>)
//
// Runs the expression through three paths:
//   1. Native mux_exec (AST eval) — the current production path
//   2. rveval compile-every-time
//   3. rveval compile-once, run N times (amortized)
//
// Returns a multi-line report with timings in microseconds.
// ---------------------------------------------------------------

#ifdef WIN32
static double elapsed_us(const LARGE_INTEGER &start,
                          const LARGE_INTEGER &end) {
    LARGE_INTEGER freq;
    QueryPerformanceFrequency(&freq);
    return static_cast<double>(end.QuadPart - start.QuadPart) * 1e6
         / static_cast<double>(freq.QuadPart);
}
#else
static double elapsed_us(const struct timespec &start,
                          const struct timespec &end) {
    double s = static_cast<double>(end.tv_sec - start.tv_sec);
    double ns = static_cast<double>(end.tv_nsec - start.tv_nsec);
    return (s * 1e6) + (ns / 1e3);
}
#endif

#ifdef WIN32
#define BENCH_TIMER LARGE_INTEGER
#define BENCH_NOW(t) QueryPerformanceCounter(&(t))
#else
#define BENCH_TIMER struct timespec
#define BENCH_NOW(t) clock_gettime(CLOCK_MONOTONIC, &(t))
#endif

// Run the compiled program through the JIT.  Returns the result
// string (written into caller-provided buffer).
//
// If reuse_dbt is true, skip the full DBT reset and only update the
// ECALL callback — keeps translated blocks cached.  Caller must
// ensure the guest code region is unchanged.
//
static bool run_compiled(compiled_program &prog,
                          dbref executor, dbref caller_db, dbref enactor,
                          UTF8 *out, size_t out_size,
                          bool reuse_dbt = false) {
    if (!prog.needs_jit) {
        // Fully folded — result is already in guest memory.
        uint64_t out_addr = resolve_runtime_out_addr(
            prog.out_addr, rv_compiler::STACK_TOP);
        size_t n = 0;
        if (!guest_strnlen(prog.memory.data(), prog.memory_size,
                           out_addr, &n)) {
            return false;
        }
        if (n >= out_size) n = out_size - 1;
        memcpy(out, prog.memory.data() + out_addr, n);
        out[n] = '\0';
        return true;
    }

    eval_ctx ec;
    ec.memory = prog.memory.data();
    ec.memory_size = prog.memory_size;
    ec.executor = executor;
    ec.caller = caller_db;
    ec.enactor = enactor;
    ec.eval = EV_FCHECK | EV_EVAL;
    ec.cargs = nullptr;
    ec.ncargs = 0;
    ec.qreg_mask = prog.subst_mask;
    ec.lua_result_base = 0;
    ec.lua_result_count = 0;
    ec.lua_state = nullptr;
    ec.host_ecalls = 0;
    ec.dbt = nullptr;
    ec.pvm = nullptr;

    // This path runs out of prog.memory rather than the context's own buffer
    // (dbt_reset rebinds it), so it must CLAIM the context for the duration
    // of the run (#2106).  The old code took s_vm[0] unconditionally without
    // touching s_run_cached_depth, on the reasoning that run_compiled is
    // never reached FROM a nested run -- true, and beside the point.  It
    // makes one: a program whose ECALL re-enters mux_exec per element
    // (fun_map, fun_filter) lands in run_cached_program, which reads a depth
    // still at 0, picks this same s_vm[0], and calls get_dbt on it --
    // rebinding the DBT from prog.memory to the context's own buffer while
    // the outer run's frames are executing out of it.  That is precisely the
    // hazard the depth check above run_cached_program's slot pick describes.
    //
    // Both buffers are live and the same size, so on glibc the outer run
    // simply continues against the wrong memory and returns a wrong answer;
    // it took a platform whose allocator left the region unmapped to turn it
    // into the SIGSEGV that got it noticed.
    //
    if (s_run_cached_depth >= JIT_MAX_RUN_DEPTH) {
        return false;
    }
    jit_run_vm *vm = &s_vm[s_run_cached_depth];
    RunDepthGuard run_depth_guard(s_run_cached_depth);

    dbt_state_t *dbt;
    if (reuse_dbt && vm->dbt_ready) {
        dbt = &vm->dbt;
        dbt_rerun(dbt, eval_ecall, &ec);
    } else {
        dbt = get_dbt(vm, prog.memory.data(), prog.memory_size,
                       eval_ecall, &ec);
        if (!dbt) return false;
        if (dbt->blob_code_end == 0) {
            pretranslate_tier2(dbt);
            dbt->blob_code_end = dbt->code_used;
        }
    }

    int rc = dbt_run(dbt, prog.entry_pc, rv_compiler::STACK_TOP);

    if (!handle_dbt_run_status(rc, out, out_size, true)) {
        return false;
    }
    if (rc == -3) {
        return true;
    }

    uint64_t out_addr = resolve_runtime_out_addr(
        prog.out_addr, rv_compiler::STACK_TOP);
    size_t n = 0;
    if (!guest_strnlen(prog.memory.data(), prog.memory_size, out_addr, &n)) {
        return false;
    }
    if (n >= out_size) n = out_size - 1;
    memcpy(out, prog.memory.data() + out_addr, n);
    out[n] = '\0';
    return true;
}

FUNCTION(fun_rvbench)
{
    UNUSED_PARAMETER(fp);
    UNUSED_PARAMETER(eval);
    UNUSED_PARAMETER(cargs);
    UNUSED_PARAMETER(ncargs);

    JITArena::gc();

    if (nfargs < 2) {
        safe_str(S_("#-1 TOO FEW ARGUMENTS"), buff, bufc);
        return;
    }

    const UTF8 *expr = fargs[0];
    size_t nLen = strlen(reinterpret_cast<const char *>(expr));
    // Clamp in 64-bit before narrowing; see the note in fun_astbench.
    // rvbench(expr, 4294967296) otherwise truncated to 0 and then got
    // bumped to 1 by the floor below -- a request 4000x above the cap
    // silently becoming the smallest legal run (#1402).
    //
    int64_t iRequested = mux_atoi64(fargs[1]);
    if (iRequested < 1) iRequested = 1;
    if (iRequested > 1000000) iRequested = 1000000;
    int iterations = static_cast<int>(iRequested);

    // Verify both paths produce the same result.
    compiled_program prog = compile_expression(expr, nLen);
    if (!prog.ok) {
        safe_str(S_("#-1 COMPILATION FAILED"), buff, bufc);
        return;
    }
    tier2_install(prog.memory, rv_compiler::BLOB_BASE);

    // --- Benchmark 1: Native mux_exec ---
    BENCH_TIMER t0, t1;
    int eval_flags = EV_FCHECK | EV_EVAL;

    BENCH_NOW(t0);
    for (int i = 0; i < iterations; i++) {
        LBuf tbuf = LBuf_Src("rvbench.native");
        UTF8 *tbufc = tbuf.get();
        mux_exec(expr, nLen, tbuf, &tbufc, executor, caller, enactor,
                 eval_flags, nullptr, 0);
        *tbufc = '\0';
    }
    BENCH_NOW(t1);
    double native_us = elapsed_us(t0, t1);

    // --- Benchmark 2: rveval compile-every-time ---
    BENCH_NOW(t0);
    for (int i = 0; i < iterations; i++) {
        compiled_program p = compile_expression(expr, nLen);
        if (p.ok) {
            tier2_install(p.memory, rv_compiler::BLOB_BASE);
            UTF8 result[256];
            run_compiled(p, executor, caller, enactor, result, sizeof(result));
        }
    }
    BENCH_NOW(t1);
    double compile_each_us = elapsed_us(t0, t1);

    // --- Benchmark 3: production path (compile cache + block cache) ---
    // Uses compile_cached (LRU) + run_cached_program (dbt_rerun).
    // First iteration is a cache miss (compiles + JIT translates);
    // subsequent iterations hit both caches — zero compilation,
    // zero JIT translation.
    //
    // Invalidate the compile cache entry for this expression first
    // so the first iteration is a genuine miss.
    {
        std::string key = compile_cache_key(expr, nLen, EV_FMAND | EV_EVAL);
        auto cit = s_compile_cache.find(key);
        if (cit != s_compile_cache.end()) {
            release_program_slots(cit->second.prog.program_id);
            s_compile_lru.erase(cit->second.lru_it);
            s_compile_cache.erase(cit);
        }
    }
    BENCH_NOW(t0);
    for (int i = 0; i < iterations; i++) {
        compiled_program *cp = compile_cached(expr, nLen, EV_FMAND | EV_EVAL);
        if (cp) {
            UTF8 result[256];
            run_cached_program(cp, executor, caller, enactor,
                               result, sizeof(result));
        }
    }
    BENCH_NOW(t1);
    double cached_us = elapsed_us(t0, t1);

    // Format report.
    double per_native = native_us / iterations;
    double per_compile = compile_each_us / iterations;
    double per_cached = cached_us / iterations;
    uint64_t disp = s_vm[0].dbt.dispatch_count;
    uint64_t sb = s_vm[0].dbt.superblock_count;
    uint64_t se = s_vm[0].dbt.side_exits_total;
    uint64_t ic = s_vm[0].dbt.inline_calls;
    uint64_t ih = s_vm[0].dbt.intrinsic_hits;
    uint64_t ce = s_vm[0].dbt.cold_exit_count;
    uint64_t ce_actual = s_vm[0].dbt.cold_exit_actual;
    uint64_t ce_expected = s_vm[0].dbt.cold_exit_expected;
    uint64_t ce_from = s_vm[0].dbt.last_exit_from;

    // Timings report ns/call, not us (#2046).  The cached path runs at
    // 10-50ns/call, which %.2fus rendered as "0.01" or "0.02" -- one or two
    // significant digits, so consecutive identical runs looked like 100%
    // swings and any regression under about 2x was invisible.  A number that
    // cannot be resolved cannot be gated on.
    //
    // Keep the format one unbroken literal: check_formats.py reads the source
    // and requires mux_snprintf()'s format to be a constant, so a comment
    // interleaved with the concatenation trips the guard.
    //
    LBuf report = LBuf_Src("rvbench");
    mux_snprintf(report.get(), LBUF_SIZE,
        T("expr=%s iters=%d folds=%d ecalls=%d tier2=%d nativ=%d disp=%llu sb=%llu/%llu ic=%llu ih=%llu ce=%llu(a=0x%llX,e=0x%llX,from=0x%llX) | "
        "native=%.1fns/call | "
        "compile-each=%.1fns/call (%.1fx) | "
        "cached=%.1fns/call (%.1fx)"),
        reinterpret_cast<const char *>(expr),
        iterations, prog.folds, prog.ecalls, prog.tier2_calls, prog.native_ops,
        (unsigned long long)disp,
        (unsigned long long)sb, (unsigned long long)se, (unsigned long long)ic,
        (unsigned long long)ih, (unsigned long long)ce, (unsigned long long)ce_actual, (unsigned long long)ce_expected, (unsigned long long)ce_from,
        per_native * 1000.0,
        per_compile * 1000.0, per_compile / per_native,
        per_cached * 1000.0, per_cached / per_native);

    safe_str(report, buff, bufc);
}

// ---------------------------------------------------------------
// Persistent VM proof-of-concept.
//
// Demonstrates the core concept: a single RV64 guest memory that
// lives across multiple evaluations, with compiled functions
// calling each other via JAL — no DBT reset, no context zeroing.
//
// Layout in persistent guest memory:
//   0x0000: Main stub (calls func_add42 via JAL, writes result, exits)
//   0x0100: func_add42 — adds 42 to a0, returns via JALR ra
//   0x1000: String pool (output buffer)
//
// On each invocation, the block cache retains translations from
// all previous runs.  The second and subsequent calls should show
// zero cache misses.
// ---------------------------------------------------------------

// ---------------------------------------------------------------
// RV64 instruction encoders for hand-assembled persistent VM stubs.
// Subset of the encoders in hir_codegen.cpp (separate TU).
// ---------------------------------------------------------------

namespace rv64_asm {

static uint32_t i_type(uint8_t op, uint8_t rd, uint8_t f3,
                        uint8_t rs1, int32_t imm) {
    return op | (rd << 7) | (f3 << 12) | (rs1 << 15)
         | ((static_cast<uint32_t>(imm) & 0xFFF) << 20);
}

static uint32_t ADDI(uint8_t rd, uint8_t rs1, int32_t imm) {
    return i_type(OP_IMM, rd, 0, rs1, imm);
}

static uint32_t LUI(uint8_t rd, int32_t imm) {
    return OP_LUI | (rd << 7) | (static_cast<uint32_t>(imm) & 0xFFFFF000);
}

static uint32_t SB(uint8_t base, uint8_t src, int32_t off) {
    return OP_STORE | ((off & 0x1F) << 7) | (0 << 12)
         | (base << 15) | (src << 20)
         | (((off >> 5) & 0x7F) << 25);
}

static uint32_t ECALL() { return 0x00000073; }

static uint32_t BNE(uint8_t rs1, uint8_t rs2, int32_t off) {
    uint32_t u = static_cast<uint32_t>(off);
    return OP_BRANCH
         | (((u >> 11) & 1) << 7)
         | (((u >> 1) & 0xF) << 8)
         | (1 << 12)
         | (rs1 << 15) | (rs2 << 20)
         | (((u >> 5) & 0x3F) << 25)
         | (((u >> 12) & 1) << 31);
}

static void load_val(std::vector<uint32_t> &code, uint8_t rd, uint64_t val) {
    if (val == 0) {
        code.push_back(ADDI(rd, 0, 0));
        return;
    }
    int32_t sval = static_cast<int32_t>(val);
    if (sval >= -2048 && sval <= 2047
        && val == static_cast<uint64_t>(static_cast<uint32_t>(sval)))
    {
        code.push_back(ADDI(rd, 0, sval));
        return;
    }
    uint32_t hi = static_cast<uint32_t>(val) & 0xFFFFF000;
    int32_t lo = static_cast<int32_t>(val & 0xFFF);
    if (lo & 0x800) { hi += 0x1000; lo -= 0x1000; }
    code.push_back(LUI(rd, hi));
    if (lo) code.push_back(ADDI(rd, rd, lo));
}

static uint32_t SUB(uint8_t rd, uint8_t rs1, uint8_t rs2) {
    return OP_REG | (rd << 7) | (0 << 12) | (rs1 << 15)
         | (rs2 << 20) | (0x20u << 25);
}

} // namespace rv64_asm

// Persistent VM ECALL handler.
// Handles ECALL_EXIT and ECALL_CALL_COMPILED (re-entrant calls).
//
static int poc_ecall(rv64_ctx_t *ctx, void *user_data) {
    uint64_t nr = ctx->x[17];  // a7
    if (nr == ECALL_EXIT) {
        return static_cast<int>(ctx->x[10]);  // a0 = exit code
    }

    if (nr == ECALL_CALL_COMPILED) {
        // Re-entrant call into a compiled function.
        // a0 = entry_pc, a1 = output buffer addr, a2 = fargs addr, a3 = nfargs
        dbt_state_t *dbt = static_cast<dbt_state_t *>(user_data);
        if (!dbt) {
            ctx->x[10] = 0;
            return -1;  // continue
        }

        uint64_t target_pc = ctx->x[10];
        uint64_t out_ref   = ctx->x[11];

        // Save outer execution's full CPU context.
        rv64_ctx_t saved_ctx = *ctx;

        // Set up for inner call: SP already points to available
        // stack space (below outer's frame).
        // The inner function's prologue will decrement SP further.
        ctx->x[2] = saved_ctx.x[2];  // preserve SP

        // Run inner function via dbt_resume.
        int inner_rc = dbt_resume(dbt, target_pc);

        // Extract inner result length (#1057: bound guest NUL scan).
        uint64_t result_len = 0;
        uint64_t resolved_out = resolve_runtime_out_addr(out_ref, saved_ctx.x[2]);
        size_t n = 0;
        if (inner_rc == 0 && resolved_out > 0
            && guest_strnlen(dbt->memory, dbt->memory_size, resolved_out, &n)) {
            result_len = n;
        }

        // Restore outer CPU context.
        *ctx = saved_ctx;

        // Return result info in a0.
        ctx->x[10] = result_len;
        ctx->x[11] = resolved_out;
        return -1;  // continue outer execution
    }

    // Unknown ECALL — error.
    fprintf(stderr, "pocvm: unknown ECALL %llu\n",
            static_cast<unsigned long long>(nr));
    return -1;
}

// ---------------------------------------------------------------
// Persistent VM: compile real MUX expressions at different code
// offsets in shared 4MB guest memory, execute via dbt_run/dbt_resume,
// and demonstrate re-entrant calls via ECALL_CALL_COMPILED.

// ---------------------------------------------------------------


static persistent_vm_t s_pvm;

FUNCTION(fun_pocvm2)
{
    UNUSED_PARAMETER(fp);
    UNUSED_PARAMETER(caller);
    UNUSED_PARAMETER(enactor);
    UNUSED_PARAMETER(eval);
    UNUSED_PARAMETER(fargs);
    UNUSED_PARAMETER(nfargs);
    UNUSED_PARAMETER(cargs);
    UNUSED_PARAMETER(ncargs);

    if (!Wizard(executor)) {
        safe_str(S_("#-1 PERMISSION DENIED"), buff, bufc);
        return;
    }

    // One-time compilation of test functions.
    static uint64_t func_a_entry = 0, func_a_out = 0;
    static uint64_t func_b_entry = 0, func_b_out = 0;
    static uint64_t func_c_entry = 0, func_c_out = 0;
    static bool compiled = false;

    if (!compiled) {
        // Function A: lnum(3) → "0 1 2"
        // lnum() is Tier 2 (no ECALL) and not in try_fold,
        // so the persistent VM produces executable JIT code.
        const char *expr_a = "lnum(3)";
        auto a = s_pvm.compile(
            reinterpret_cast<const UTF8 *>(expr_a), strlen(expr_a));
        if (!a.entry_pc) {
            safe_str(S_("#-1 FUNC A COMPILE FAILED"), buff, bufc);
            return;
        }
        func_a_entry = a.entry_pc;
        func_a_out = a.out_addr;

        // Function B: lnum(5) → "0 1 2 3 4"
        const char *expr_b = "lnum(5)";
        auto b = s_pvm.compile(
            reinterpret_cast<const UTF8 *>(expr_b), strlen(expr_b));
        if (!b.entry_pc) {
            safe_str(S_("#-1 FUNC B COMPILE FAILED"), buff, bufc);
            return;
        }
        func_b_entry = b.entry_pc;
        func_b_out = b.out_addr;

        // Function C: hand-assembled re-entrant call stub.
        // Calls A via ECALL_CALL_COMPILED, writes "C:" + A's result.
        {
            std::vector<uint32_t> code;
            constexpr uint8_t a0 = 10, a1 = 11, a7 = 17;
            constexpr uint8_t t0 = 5, t3 = 28, t4 = 29;

            func_c_out = 0x3000;  // fixed output address

            rv64_asm::load_val(code, a0, func_a_entry);
            rv64_asm::load_val(code, a1, func_a_out);
            code.push_back(rv64_asm::ADDI(a7, 0,
                static_cast<int32_t>(ECALL_CALL_COMPILED)));
            code.push_back(rv64_asm::ECALL());

            rv64_asm::load_val(code, t4, func_c_out);
            code.push_back(rv64_asm::ADDI(t3, 0, 'C'));
            code.push_back(rv64_asm::SB(t4, t3, 0));
            code.push_back(rv64_asm::ADDI(t3, 0, ':'));
            code.push_back(rv64_asm::SB(t4, t3, 1));
            code.push_back(rv64_asm::ADDI(t4, t4, 2));

            // After ECALL_CALL_COMPILED, a1 (x11) holds the resolved
            // output address of the inner function.
            code.push_back(rv64_asm::ADDI(t3, a1, 0));  // t3 = a1
            size_t copy_loop = code.size();
            code.push_back(rv64_asm::i_type(OP_LOAD, t0, 4, t3, 0));
            code.push_back(rv64_asm::SB(t4, t0, 0));
            code.push_back(rv64_asm::ADDI(t3, t3, 1));
            code.push_back(rv64_asm::ADDI(t4, t4, 1));
            int32_t off = -static_cast<int32_t>(
                (code.size() - copy_loop) * 4);
            code.push_back(rv64_asm::BNE(t0, 0, off));

            code.push_back(rv64_asm::ADDI(a7, 0, ECALL_EXIT));
            code.push_back(rv64_asm::ADDI(a0, 0, 0));
            code.push_back(rv64_asm::ECALL());

            func_c_entry = s_pvm.install_code(code);
        }

        compiled = true;
    }

    if (!s_pvm.ensure_dbt()) {
        safe_str(S_("#-1 DBT INIT FAILED"), buff, bufc);
        return;
    }

    // Run A.
    s_pvm.prepare_run();
    int rc_a = s_pvm.run(func_a_entry);
    const char *result_a = (rc_a == 0)
        ? s_pvm.result(func_a_out) : "#-1 RUN A FAILED";

    // Run B.  Copy A's result first — B shares the same output slot.
    std::string str_a(result_a);
    s_pvm.reset_blob_bss();
    int rc_b = s_pvm.run(func_b_entry);
    const char *result_b = (rc_b == 0)
        ? s_pvm.result(func_b_out) : "#-1 RUN B FAILED";
    std::string str_b(result_b);

    // Run C (re-entrant: calls A internally).
    s_pvm.reset_blob_bss();
    uint64_t c_out_abs = rv_compiler::resolve_output_addr(
        func_c_out, rv_compiler::STACK_TOP);
    uint64_t a_out_abs = rv_compiler::resolve_output_addr(
        func_a_out, rv_compiler::STACK_TOP);
    if (c_out_abs && c_out_abs < s_pvm.memory.size())
        memset(s_pvm.memory.data() + c_out_abs, 0, 256);
    if (a_out_abs && a_out_abs < s_pvm.memory.size())
        memset(s_pvm.memory.data() + a_out_abs, 0, 256);
    int rc_c = s_pvm.run(func_c_entry);
    const char *result_c = (rc_c == 0)
        ? s_pvm.result(func_c_out) : "#-1 RUN C FAILED";

    LBuf tmp = LBuf_Src("pocvm2");
    mux_snprintf(tmp.get(), LBUF_SIZE,
        T("a=%s b=%s c=%s a_pc=0x%llX b_pc=0x%llX c_pc=0x%llX calls=%u"),
        str_a.c_str(), str_b.c_str(), result_c,
        (unsigned long long)func_a_entry,
        (unsigned long long)func_b_entry,
        (unsigned long long)func_c_entry,
        s_pvm.run_count);

    safe_str(tmp, buff, bufc);
}