mmo-ref/tinymux
1
0
Fork 0
mirror of https://github.com/brazilofmux/tinymux synced 2026-08-13 00:23:11 -04:00
Code Issues Projects Releases Packages Wiki Activity
tinymux/mux/modules/engine/jit_compiler.cpp
Stephen Dennis d433d5f5f0 fix(#2179): u()-inline runs the body in the target's executor context 
fun_u evaluates the attribute body with executor = the object holding
the attribute and caller = the previous executor; the Tier 3 inline
compiled the body into the caller's program without that swap, so
everything executor-derived inside u(#obj/attr) resolved against the
CALLER: %! (SUBST slot), %va-%vz and %=<name> (xget against that slot),
and every ECALL's ec->executor — v(), bare-name u(), name(me), and
permission checks, in both directions.  The MAP/FILTER/FOLD inline arms
documented this exact hole and guard it with a runtime executor==thing
STRCMP fallback ("the u()-inline lives with this hole"); #2092's arm
un-reversal then made the u() hole reachable for the first time.

Fix: _PUSH_UEXEC/_POP_UEXEC helpers bracket the inlined body.  Push
saves ec->executor/ec->caller to a handle stack, sets caller = old
executor and executor = thing (mirroring fun_u's mux_exec call), and
rewrites the guest %! substitution slot; pop restores all three.  Both
executor-derived channels — the ECALL context and the SUBST slot — go
through exactly this pair, so v(), bare-name u() with parent
inheritance, name(me), %!, %va-%vz, %=<name> and See_attr checks all
compose, including across nested inlines (each pair strictly brackets
its body).  A failed push (stack exhausted) returns -1 and the lowering
BRANCHES it to the existing fun_u ECALL fallback, which establishes its
own context — exhaustion costs the inline, never correctness.

The helper save stacks (this one and the existing CARGS stack) leaked
their slot when a program was abandoned between save and restore (DBT
decline, error unwind); a top-level run_cached_program entry proves no
outer program holds a live handle, so both stacks reset there.  Nested
entries never reset: both entry points have claimed the depth counter
since #2106.

Verified against the interpreter oracle (the [num(#obj)] spelling that
never reaches the inline gate): the issue's repro plus nested inline
(#2 -> #3 -> back), executor restored after the call, ulocal, and the
real-world iter(u(bare-name)) shape all byte-match, jit_handled=5/5.
jit_route_parity_fn TC012 pins the shape in smoke — its trigger context
runs as the test object, so caller != target (#1) makes the
discrimination live on every run.  Full suite 35/0.

The farm's 2x +jobs regression (wrong-executor denials flooding the
ECALL error-format path) should collapse with this; if utf8_strlen
under mux_vsnprintf stays hot afterwards it deserves its own issue.
MAP/FILTER/FOLD can now adopt the same pair and drop their conservative
executor==thing gate to open inlining to executor != thing — filed as a
perf follow-up in the arm's comment.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-06 23:07:46 -06:00

7014 lines
266 KiB
C++
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/*! \file jit_compiler.cpp
* \brief Top-level JIT compiler pipeline.
*
* JIT statistics, Tier 2 blob management, compile cache,
* compile_expression(), ECALL handler, and the public entry
* points: jit_eval(), fun_jitstats(), fun_rveval(), fun_rvbench().
*/
#include "jit_tier1_stamp.h"
#include "copyright.h"
#include "autoconf.h"
#include "config.h"
#include "externs.h"
#include "sqlite_backend.h"
#include "ast.h"
#include "dbt_compile.h"
#include "dbt.h"
#include "dbt_decoder.h"
#include "engine_api.h"
#include "hir_lower_lua.h"
#include "sha1.h"
#include "../../rv64/rv64blob.h"
extern "C" {
#include <lua.h>
#include <lauxlib.h>
}
#include <cstdio>
#include <cstring>
#include <cstdlib>
#include <cmath>
#include <cerrno>
#include <ctime>
#include <algorithm>
#include <vector>
#include <string>
#include <string_view>
#include <map>
#include <unordered_map>
#include <unordered_set>
#include <list>
jit_stats_t s_jit_stats = {};
// ---------------------------------------------------------------
// JIT Arena Management (Tier B)
//
// Uses shared_ptr<RegBuffer> for buffer lifecycle — unified with the
// engine's register packing system.
// ---------------------------------------------------------------
class JITArena {
public:
struct Arena {
std::shared_ptr<RegBuffer> buf;
size_t used;
uint32_t id;
};
static Arena *Alloc(size_t n) {
if (n > LBUF_SIZE) return nullptr;
if (!s_current || s_current->used + n > LBUF_SIZE) {
s_current = Create();
}
s_current->used += n;
return s_current;
}
static void AddRef(uint32_t id) {
// No-op: shared_ptr refcount is managed via reg_ref copies.
UNUSED_PARAMETER(id);
}
static void Release(uint32_t id) {
auto it = s_arenas.find(id);
if (it != s_arenas.end()) {
Arena *a = it->second;
// If only the arena itself holds a reference, clean up.
if (a->buf.use_count() <= 1) {
if (s_current && s_current->id == id) s_current = nullptr;
delete a;
s_arenas.erase(it);
}
}
}
static Arena *Get(uint32_t id) {
auto it = s_arenas.find(id);
return (it != s_arenas.end()) ? it->second : nullptr;
}
// Release arenas that have no external references.
static void gc() {
s_current = nullptr;
auto it = s_arenas.begin();
while (it != s_arenas.end()) {
Arena *a = it->second;
if (a->buf.use_count() <= 1) {
delete a;
it = s_arenas.erase(it);
} else {
++it;
}
}
}
private:
static Arena *Create() {
Arena *a = new Arena;
a->buf = std::make_shared<RegBuffer>();
a->used = 0;
a->id = ++s_next_id;
s_arenas[a->id] = a;
return a;
}
static inline thread_local uint32_t s_next_id = 0;
static inline thread_local Arena *s_current = nullptr;
static inline thread_local std::unordered_map<uint32_t, Arena *> s_arenas;
};
// ---------------------------------------------------------------
// Tier 2: pre-compiled RV64 library blob
// ---------------------------------------------------------------
tier2_state s_tier2 = { false, {}, 0, 0, 0, 0, {}, 0 };
// Tier 1 compiler version. Bump this whenever the HIR lowering,
// codegen, constant folding, NOEVAL handlers, or any other tier 1
// logic changes in a way that could produce different RV64 output
// for the same softcode input.
//
static const char JIT_COMPILER_VERSION[] = "jit-t1-003";
// Build stamp backstop. s_blob_version (below) feeds the SQLite
// code_cache staleness key (blob_hash); a persisted entry is only reused
// when its blob_hash still matches. Relying solely on a hand-maintained
// JIT_COMPILER_VERSION is fragile: a codegen change in another TU (e.g.
// hir_codegen.cpp) with no version bump leaves stale entries matching and
// being served — which manifests as a "poisoned" compiled result that
// survives an upgrade until the cache is cleared by hand. Because the
// JIT requires a clean rebuild to take effect, this file is recompiled
// every such build, so its __DATE__/__TIME__ stamp changes and folds into
// the hash, invalidating every previously persisted entry automatically.
//
static const char JIT_BUILD_STAMP[] = __DATE__ " " __TIME__;
// Blob content hash for cache invalidation. Incorporates both the
// tier 2 blob and the tier 1 compiler version so that upgrading
// either one invalidates stale cached entries.
//
std::string s_blob_version = "none";
static bool tier2_allowed(const std::string &mux_name) {
static constexpr std::string_view s_allowlist[] = {
// co_* wrappers: cross-compiled from the same Ragel color_ops
// source the server uses. Semantics-matched by construction.
//
"STRLEN",
"LCSTR",
"UCSTR",
"CAPSTR",
"REVERSE",
"ESCAPE",
"STRIPANSI",
"COMPRESS",
"FIRST",
"REST",
"LAST",
"WORDS",
"MID",
"POS",
"REPEAT",
"TRIM",
"MEMBER",
"EXTRACT",
// Cursor walk backing the ITER lowering (#2052). Not reachable as a
// softcode function -- there is no SPLIT_TOKEN() builtin -- so this
// entry exists only so hir_lower can resolve it.
"SPLIT_TOKEN",
// Same story: ITER's in-place accumulator (#2072), unreachable
// from softcode.
"APPEND",
// Integer-ABI successors of the two above (#2132) -- same
// unreachable-from-softcode story. Forgetting THIS list is the
// silent way to ship the fallback: tier2_lookup returns 0, the
// lowering keeps the string route, and every test still passes.
"SPLIT_STEP",
"APPEND_I",
"BYTELEN_I",
// And MAP's element-size guard (#2080).
"BYTELEN",
"LEFT",
"RIGHT",
"LPOS",
"LDELETE", // co_ldelete_wrap → co_delete_at (word-list delete,
// mirrors co_replace_at/co_insert_at). See #768.
"REPLACE",
"INSERT",
"LJUST",
"RJUST",
"CENTER",
"EDIT",
"SPLICE",
"SETUNION",
"SETDIFF",
"SETINTER",
// rv64_* hand-written: only trivial ops where the blob
// implementation is demonstrably equivalent to the server.
//
"CAT",
"STRCAT",
"SPACE",
// Tier 2 math — MATH_WRAP transcendentals.
// These are simple strtod → libm → fval wrappers, semantics-
// matched to the server by construction.
//
"SIN", "COS", "TAN",
"ASIN", "ACOS", "ATAN", "ATAN2",
"EXP", "LOG", "LOG10",
"SQRT", "CEIL", "FLOOR",
"ABS", // rv64_fabs: fabs() via MATH_WRAP_1
"FMOD", // rv64_fmod: fmod() via MATH_WRAP_2
"POWER", // rv64_power: pow() via MATH_WRAP_2
// Tier 2 arithmetic.
"ADD", "SUB",
"MUL", // rv64_mul: NearestPretty intrinsic
"FDIV", // rv64_fdiv: IEEE div, fval handles Inf/NaN
"MOD", // rv64_mod: atoi64 % atoi64
"SIGN", // rv64_sign: -1/0/1
"MIN", "MAX", // rv64_min/max: strtod compare → fval
"INC", "DEC", // rv64_inc/dec: atoi64 ± 1
"TRUNC", // rv64_trunc: MATH_WRAP via libm ::trunc (#827 — the
// old (val>=0)?floor():ceil() pattern got inlined to a
// DBT-mistranslated fcvt and rounded away from zero)
"ROUND", // rv64_round: ftoa_round intrinsic
// Tier 2 list/string ops — parity-tested via smoke suite.
//
"BEFORE", "AFTER",
"WORDPOS", // rv64_wordpos: char-position → word number, mirroring
// fun_wordpos (strip color, split words, map byte
// offset to 1-based word). See #768.
"DELETE", "ELEMENTS", "REMOVE",
"REVWORDS",
"LNUM",
// LADD reproduces fun_ladd's AddDoubles (|x|-sorted, error-
// compensated, NearestPretty) via the rv64_add_doubles host
// intrinsic — exact parity by construction. See #813.
"LADD",
"LMAX", "LMIN", "LAND", "LOR",
"ISNUM", "ISINT",
"DEC2HEX", "HEX2DEC",
// ISDBREF is deliberately absent: parse_dbref accepts the objid
// #<dbref>:<timestamp> form, validated against the object's
// creation_seconds — engine state the blob cannot reach.
"CHR", "ORD",
"SECURE", "SQUISH",
"TRANSLATE",
"STRMATCH", "MATCH", "GRAB", "GRABALL",
"SORT",
};
const std::string_view name(mux_name);
for (std::string_view allowed : s_allowlist) {
if (name == allowed) {
return true;
}
}
return false;
}
static std::string sha1_hex_parts(const void *const *parts, const size_t *sizes,
int count) {
static constexpr unsigned int SHA1_DIGEST_LEN = 20;
std::vector<const UTF8 *> digest_parts;
std::vector<size_t> digest_sizes;
digest_parts.reserve(count);
digest_sizes.reserve(count);
for (int i = 0; i < count; i++) {
if (parts[i] && sizes[i] > 0) {
digest_parts.push_back(reinterpret_cast<const UTF8 *>(parts[i]));
digest_sizes.push_back(sizes[i]);
}
}
uint8_t digest[SHA1_DIGEST_LEN];
unsigned int digest_len = 0;
if (!mux_sha1_digest(digest_parts.data(), digest_sizes.data(),
static_cast<int>(digest_parts.size()),
digest, &digest_len)
|| digest_len != SHA1_DIGEST_LEN) {
return "none";
}
static const char hex[] = "0123456789abcdef";
std::string out;
out.resize(SHA1_DIGEST_LEN * 2);
for (size_t i = 0; i < SHA1_DIGEST_LEN; i++) {
out[i * 2] = hex[digest[i] >> 4];
out[i * 2 + 1] = hex[digest[i] & 0x0F];
}
return out;
}
// Map MUX function names (uppercase) to Tier 2 blob entry names.
// The blob uses rv64_ prefixed names; MUX uses plain uppercase.
//
static const struct { const char *mux_name; const char *blob_name; } s_tier2_map[] = {
// --- Color-aware co_* wrappers (Ragel, PUA color, Unicode 16) ---
// These replace the ASCII-only rv64_* versions for functions where
// color preservation and grapheme-cluster-aware word boundaries matter.
//
{ "FIRST", "co_first_wrap" },
{ "REST", "co_rest_wrap" },
{ "LAST", "co_last_wrap" },
{ "WORDS", "co_words_wrap" },
{ "EXTRACT", "co_extract_wrap" },
{ "SPLIT_TOKEN", "rv64_split_token" }, // cursor walk for ITER (#2052)
{ "APPEND", "rv64_append" }, // in-place accumulator for ITER (#2072)
{ "SPLIT_STEP", "rv64_split_step" }, // integer-ABI cursor walk (#2132)
{ "APPEND_I", "rv64_append_i" }, // integer-ABI accumulator (#2132)
{ "BYTELEN_I", "rv64_bytelen_i" }, // integer-ABI CARGS guard (#2152)
{ "BYTELEN", "rv64_bytelen" }, // CARGS-slot fit guard for MAP (#2080)
{ "MEMBER", "co_member_wrap" },
{ "TRIM", "co_trim_wrap" },
{ "REPEAT", "co_repeat_wrap" },
{ "MID", "co_mid_wrap" },
{ "POS", "co_pos_wrap" },
{ "SORT", "co_sort_wrap" },
{ "SETUNION", "co_setunion_wrap" },
{ "SETDIFF", "co_setdiff_wrap" },
{ "SETINTER", "co_setinter_wrap" },
{ "LDELETE", "co_ldelete_wrap" },
{ "REPLACE", "co_replace_wrap" },
{ "INSERT", "co_insert_wrap" },
// --- ASCII-only rv64_* (no color, byte-level operations) ---
// These are fine for functions that don't handle colored text.
//
{ "CAT", "rv64_cat" },
{ "STRCAT", "rv64_strcat" },
{ "BEFORE", "rv64_before" },
{ "AFTER", "rv64_after" },
// --- Batch 2: case, reverse, escape, left/right, compress, lpos ---
//
{ "STRLEN", "co_strlen_wrap" },
{ "LCSTR", "co_lcstr_wrap" },
{ "UCSTR", "co_ucstr_wrap" },
{ "REVERSE", "co_reverse_wrap" },
{ "ESCAPE", "co_escape_wrap" },
{ "LEFT", "co_left_wrap" },
{ "RIGHT", "co_right_wrap" },
{ "COMPRESS", "co_compress_wrap" },
{ "LPOS", "co_lpos_wrap" },
// --- Batch 3: justify, edit, splice, totitle, stripansi, vislen ---
{ "LJUST", "co_ljust_wrap" },
{ "RJUST", "co_rjust_wrap" },
{ "CENTER", "co_center_wrap" },
{ "EDIT", "co_edit_wrap" },
{ "SPLICE", "co_splice_wrap" },
{ "CAPSTR", "co_totitle_wrap" },
{ "STRIPANSI", "co_stripansi_wrap" },
// --- Batch 4: space, secure, squish, delete, elements ---
{ "SPACE", "rv64_space" },
{ "SECURE", "co_secure_wrap" },
{ "SQUISH", "co_compress_wrap" },
{ "DELETE", "rv64_delete" },
{ "ELEMENTS", "rv64_elements" },
{ "TRANSLATE", "rv64_translate" },
// --- Batch 5: wildcard matching ---
{ "STRMATCH", "rv64_strmatch" },
{ "MATCH", "rv64_match" },
{ "GRAB", "rv64_grab" },
{ "GRABALL", "rv64_graball" },
// --- Batch 6: numbers, chars, base conversion ---
{ "LNUM", "rv64_lnum" },
{ "ISNUM", "rv64_isnum" },
{ "ISINT", "rv64_isint" },
{ "CHR", "rv64_chr" },
{ "ORD", "rv64_ord" },
{ "DEC2HEX", "rv64_dec2hex" },
{ "HEX2DEC", "rv64_hex2dec" },
// --- Batch 7: wordpos, remove ---
{ "WORDPOS", "rv64_wordpos" },
{ "REMOVE", "rv64_remove" },
// --- Batch 8: list aggregation, reversal, type checks ---
{ "LADD", "rv64_ladd" }, // sum via rv64_add_doubles intrinsic (#813)
{ "LMAX", "rv64_lmax" },
{ "LMIN", "rv64_lmin" },
{ "LAND", "rv64_land" },
{ "LOR", "rv64_lor" },
{ "REVWORDS", "rv64_revwords" },
{ "FLIP", "rv64_revwords" }, // alias
// (no ISDBREF: objid colon form needs creation_seconds; always ECALLs)
// --- Batch 8: math via intrinsics (string↔double + platform libm) ---
{ "SIN", "rv64_sin" },
{ "COS", "rv64_cos" },
{ "TAN", "rv64_tan" },
{ "ASIN", "rv64_asin" },
{ "ACOS", "rv64_acos" },
{ "ATAN", "rv64_atan" },
{ "ATAN2", "rv64_atan2" },
{ "EXP", "rv64_exp" },
{ "LOG", "rv64_log10" }, // MUX log() defaults to common (base 10)
{ "LOG10", "rv64_log10" },
{ "SQRT", "rv64_sqrt" },
{ "CEIL", "rv64_ceil" },
{ "FLOOR", "rv64_floor" },
{ "ABS", "rv64_fabs" },
{ "FMOD", "rv64_fmod" },
{ "POWER", "rv64_power" },
// --- Batch 9: arithmetic ---
{ "ADD", "rv64_add" },
{ "SUB", "rv64_sub" },
{ "MUL", "rv64_mul" },
{ "FDIV", "rv64_fdiv" },
{ "MOD", "rv64_mod" },
{ "SIGN", "rv64_sign" },
{ "MIN", "rv64_min" },
{ "MAX", "rv64_max" },
{ "INC", "rv64_inc" },
{ "DEC", "rv64_dec" },
{ "TRUNC", "rv64_trunc" },
// Deliberately exclude ROUND for now. Constant ROUND calls fold
// correctly, but the Tier 2 runtime path still misbehaves for
// dynamic inputs such as `round(sqrt(2),6)`. Fall back to the
// interpreter until the rv64 implementation is parity-correct.
{ nullptr, nullptr }
};
// Load the Tier 2 blob from a file.
// Called once at init time. Guest base address is where the blob's
// code section will be mapped in each program's guest memory.
//
static bool tier2_load(const char *path, uint64_t guest_base) {
FILE *f = fopen(path, "rb");
if (!f) return false;
rv64_blob_header hdr;
memset(&hdr, 0, sizeof(hdr));
// Read at least the v1 header (32 bytes), then the v2 extension.
if (fread(&hdr, 32, 1, f) != 1) { fclose(f); return false; }
if (hdr.magic != RV64_BLOB_MAGIC
|| (hdr.version != 1 && hdr.version != 2)) {
fclose(f);
return false;
}
if (hdr.version >= 2) {
// Read the v2 extension fields (bytes 32-47).
if (fread(&hdr.data_offset, sizeof(hdr) - 32, 1, f) != 1) {
fclose(f);
return false;
}
}
// Read code section directly into the install image.
s_tier2.code_size = hdr.code_size;
s_tier2.bss_size = (hdr.version >= 2) ? hdr.bss_size : 0;
// The blob (code + rodata + data + BSS) is mapped at BLOB_BASE and must
// fit within the blob window; otherwise it would overflow into the
// string pool. tier2_install only checks against the full MEM_SIZE, so
// enforce the tighter window bound here at load time.
if (s_tier2.code_size + s_tier2.bss_size
> rv_compiler::BLOB_LIMIT - rv_compiler::BLOB_BASE) {
fclose(f);
return false;
}
s_tier2.image.resize(hdr.code_size + s_tier2.bss_size, 0);
fseek(f, hdr.code_offset, SEEK_SET);
if (fread(s_tier2.image.data(), hdr.code_size, 1, f) != 1) {
fclose(f);
return false;
}
// Record writable data offset within the flat image for runtime reset.
if (hdr.version >= 2 && hdr.data_size > 0) {
s_tier2.data_image_offset = hdr.data_offset - hdr.code_offset;
s_tier2.data_image_size = hdr.data_size;
} else {
s_tier2.data_image_offset = 0;
s_tier2.data_image_size = 0;
}
// Read entry table.
std::vector<rv64_blob_entry> entries(hdr.entry_count);
fseek(f, hdr.entry_offset, SEEK_SET);
if (fread(entries.data(), sizeof(rv64_blob_entry), hdr.entry_count, f)
!= hdr.entry_count) {
fclose(f);
return false;
}
fclose(f);
// Build lookup table.
s_tier2.guest_base = guest_base;
s_tier2.funcs.clear();
for (uint32_t i = 0; i < hdr.entry_count; i++) {
tier2_entry te;
te.code_off = entries[i].code_off;
te.guest_addr = static_cast<uint32_t>(guest_base) + entries[i].code_off;
s_tier2.funcs[entries[i].name] = te;
}
// Build MUX name → blob mapping.
for (int i = 0; s_tier2_map[i].mux_name; i++) {
auto it = s_tier2.funcs.find(s_tier2_map[i].blob_name);
if (it != s_tier2.funcs.end()) {
s_tier2.funcs[s_tier2_map[i].mux_name] = it->second;
}
}
s_tier2.loaded = true;
// Tier 1 leg: every unit that can emit different RV64 for the same
// softcode, each carrying its own __DATE__/__TIME__ (#2061). JIT_BUILD_STAMP
// alone covered only this file, so a codegen change in any other unit left
// the key unmoved and the cache served the previous build's output.
// Deliberately excludes the dbt_* units: they execute the stored RV64
// rather than produce it, so invalidating on their account would discard
// the cache for no gain. See jit_tier1_stamp.h.
// Layout constants are part of the key (#2107): a guest-address
// map change (STR/FARGS bases) rewrites every baked pointer in the
// code and pool blobs, and must not reuse a row written under the
// previous map even when __DATE__/__TIME__ happened not to move.
static const uint64_t layout[] = {
rv_compiler::CODE_BASE, rv_compiler::CODE_LIMIT,
rv_compiler::STR_BASE, rv_compiler::STR_LIMIT,
rv_compiler::FARGS_BASE, rv_compiler::FARGS_LIMIT,
rv_compiler::BLOB_BASE, rv_compiler::MEM_SIZE,
};
const void *parts[] = {
JIT_COMPILER_VERSION,
JIT_BUILD_STAMP,
TIER1_STAMP_AST,
TIER1_STAMP_HIR_LOWER,
TIER1_STAMP_HIR_SSA,
TIER1_STAMP_HIR_OPT,
TIER1_STAMP_HIR_CODEGEN,
&hdr,
s_tier2.image.data(),
entries.empty() ? nullptr : entries.data(),
layout,
};
const size_t sizes[] = {
sizeof(JIT_COMPILER_VERSION) - 1,
sizeof(JIT_BUILD_STAMP) - 1,
// strlen, not sizeof: these are extern arrays of unknown bound here.
strlen(TIER1_STAMP_AST),
strlen(TIER1_STAMP_HIR_LOWER),
strlen(TIER1_STAMP_HIR_SSA),
strlen(TIER1_STAMP_HIR_OPT),
strlen(TIER1_STAMP_HIR_CODEGEN),
sizeof(hdr),
s_tier2.code_size,
entries.size() * sizeof(rv64_blob_entry),
sizeof(layout),
};
s_blob_version = sha1_hex_parts(parts, sizes, 11);
return true;
}
// Look up a function by MUX name (uppercase).
// Returns guest address, or 0 if not found.
//
uint64_t tier2_lookup(const std::string &mux_name) {
if (!s_tier2.loaded) return 0;
if (!tier2_allowed(mux_name)) return 0;
auto it = s_tier2.funcs.find(mux_name);
if (it != s_tier2.funcs.end()) return it->second.guest_addr;
return 0;
}
// Look up a raw blob symbol by name (e.g., "sin", "cos", "rv64_strtod").
// Bypasses the tier2_allowed() gate — used for direct FP intrinsic calls
// from the type-propagated lowering path.
// Returns guest address, or 0 if not found.
//
uint64_t tier2_sym_addr(const char *blob_name) {
if (!s_tier2.loaded) return 0;
auto it = s_tier2.funcs.find(blob_name);
if (it != s_tier2.funcs.end()) return it->second.guest_addr;
return 0;
}
// Pre-translate all Tier 2 blob entry points so that superblocks
// can use native CALL continuation for Tier 2 function calls.
// Called after dbt_init/dbt_reset, before dbt_run.
//
// Helper: register a blob symbol as an intrinsic if it exists.
//
static void reg_intrinsic(dbt_state_t *dbt, const char *blob_name,
dbt_emitter_id eid, void *host_fn = nullptr) {
auto it = s_tier2.funcs.find(blob_name);
if (it != s_tier2.funcs.end()) {
dbt_register_intrinsic(dbt, it->second.guest_addr, eid, host_fn);
}
}
// Host-side wrappers for string↔double conversion intrinsics.
// These are called by the DBT stubs with host pointers and FP values.
//
static double host_strtod(const char *s) {
return mux_atof(reinterpret_cast<const UTF8 *>(s));
}
// Guest heap bump cursor and the rv64_alloc intrinsic backing it.
//
// The heap is a per-evaluation arena: the guest bump-allocates scratch via
// rv64_alloc and never frees; the cursor is reset to HEAP_BASE before each
// evaluation (alongside the blob .data/.bss reset). host_alloc returns a
// *guest* address (offset into the guest image), not a host pointer, so the
// DBT stub must not host<->guest convert the return (DBT_EMIT_ALLOC uses
// ptr_mask=0). 0 is returned on exhaustion → the guest sees NULL.
//
static uint64_t s_heap_next = rv_compiler::HEAP_BASE;
static uint64_t host_alloc(uint64_t size) {
uint64_t aligned = (size + 15) & ~15ULL;
uint64_t addr = s_heap_next;
if (aligned > rv_compiler::HEAP_LIMIT - rv_compiler::HEAP_BASE
|| addr + aligned > rv_compiler::HEAP_LIMIT) {
return 0; // out of heap → guest NULL
}
s_heap_next = addr + aligned;
return addr;
}
// rv64_add_doubles intrinsic — error-compensated list sum for fun_ladd
// parity. The guest fills a doubles array (the fixed DSCRATCH region) and
// calls this; the host does the order-sensitive arithmetic (|x|-sorted
// qsort, TwoSum chain, NearestPretty) so the result is byte-identical to
// fun_ladd by construction. AddDoubles already applies NearestPretty and
// sorts vals[] in place (harmless — it is throwaway scratch).
//
static double host_add_doubles(double *vals, int n) {
return AddDoubles(n, vals);
}
static int host_fval(char *buf, double val) {
UTF8 *bufc = reinterpret_cast<UTF8 *>(buf);
UTF8 *start = bufc;
fval(reinterpret_cast<UTF8 *>(buf), &bufc, val);
*bufc = '\0';
return static_cast<int>(bufc - start);
}
// Host-side wrapper for rv64_ftoa_round intrinsic.
// Matches fun_round: mux_fpclass check + mux_ftoa(r, true, frac).
//
static int host_ftoa_round(char *buf, double val, int frac) {
#ifdef HAVE_IEEE_FP_FORMAT
int fpc = mux_fpclass(val);
if (MUX_FPGROUP(fpc) != MUX_FPGROUP_PASS
&& MUX_FPGROUP(fpc) != MUX_FPGROUP_ZERO) {
const UTF8 *s = mux_FPStrings[MUX_FPCLASS(fpc)];
size_t len = strlen(reinterpret_cast<const char *>(s));
memcpy(buf, s, len);
buf[len] = '\0';
return static_cast<int>(len);
}
if (MUX_FPGROUP(fpc) == MUX_FPGROUP_ZERO) {
val = 0.0;
}
#endif
UTF8 *result = mux_ftoa(val, true, frac);
size_t len = strlen(reinterpret_cast<const char *>(result));
memcpy(buf, result, len);
buf[len] = '\0';
return static_cast<int>(len);
}
void pretranslate_tier2(dbt_state_t *dbt) {
if (!s_tier2.loaded) return;
// Register intrinsics FIRST — translate_block() checks these
// addresses and emits native x86-64 stubs instead of translating
// the RV64 bodies. Must happen before pretranslation.
// Block-level intrinsics (custom emitters).
//
reg_intrinsic(dbt, "rv64_slen", DBT_EMIT_SLEN);
reg_intrinsic(dbt, "rv64_scopy", DBT_EMIT_SCOPY);
reg_intrinsic(dbt, "memcpy", DBT_EMIT_MEMCPY);
reg_intrinsic(dbt, "memcmp", DBT_EMIT_MEMCMP);
reg_intrinsic(dbt, "memset", DBT_EMIT_MEMSET);
reg_intrinsic(dbt, "memswap", DBT_EMIT_MEMSWAP);
// co_* Ragel functions → native host calls.
// The wrapper does fargs unpacking in RV64 (cheap to translate),
// then JALs to co_first/co_rest/etc. The intrinsic intercepts
// the JAL and calls the host's native Ragel implementation directly.
//
// 4 args: (out:ptr, p:ptr, len:int, delim:int)
reg_intrinsic(dbt, "co_first", DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_first));
reg_intrinsic(dbt, "co_rest", DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_rest));
reg_intrinsic(dbt, "co_last", DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_last));
reg_intrinsic(dbt, "co_repeat", DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_repeat));
// 3 args: (p:ptr, len:int, delim:int)
reg_intrinsic(dbt, "co_words_count", DBT_EMIT_CO_3P, reinterpret_cast<void *>(co_words_count));
// 4 args: (haystack:ptr, hlen:int, needle:ptr, nlen:int)
reg_intrinsic(dbt, "co_pos", DBT_EMIT_CO_POS, reinterpret_cast<void *>(co_pos));
// 5 args: (out:ptr, p:ptr, len:int, start:int, count:int)
reg_intrinsic(dbt, "co_mid", DBT_EMIT_CO_5PP, reinterpret_cast<void *>(co_mid));
reg_intrinsic(dbt, "co_trim", DBT_EMIT_CO_5PP, reinterpret_cast<void *>(co_trim));
// 5 args: (target:ptr, tlen:int, list:ptr, llen:int, delim:int)
reg_intrinsic(dbt, "co_member", DBT_EMIT_CO_MEMBER, reinterpret_cast<void *>(co_member));
// 6 args: (out:ptr, list:ptr, llen:int, x:int, y:int, z:int)
reg_intrinsic(dbt, "co_delete", DBT_EMIT_CO_6PP, reinterpret_cast<void *>(co_delete));
reg_intrinsic(dbt, "co_sort_words", DBT_EMIT_CO_6PP, reinterpret_cast<void *>(co_sort_words));
// 7 args: (out:ptr, p:ptr, len:int, iFirst:int, nWords:int, delim:int, osep:int)
reg_intrinsic(dbt, "co_extract", DBT_EMIT_CO_7PP, reinterpret_cast<void *>(co_extract));
// 8 args: (out:ptr, list1:ptr, len1:int, list2:ptr, len2:int, delim:int, osep:int, sort_type:int)
reg_intrinsic(dbt, "co_setunion", DBT_EMIT_CO_8PPP, reinterpret_cast<void *>(co_setunion));
reg_intrinsic(dbt, "co_setdiff", DBT_EMIT_CO_8PPP, reinterpret_cast<void *>(co_setdiff));
reg_intrinsic(dbt, "co_setinter", DBT_EMIT_CO_8PPP, reinterpret_cast<void *>(co_setinter));
// Batch 2: inner co_* functions called by wrappers.
//
// 2 args: (data:ptr, len:int)
reg_intrinsic(dbt, "co_cluster_count", DBT_EMIT_CO_2P, reinterpret_cast<void *>(co_cluster_count));
// 3 args: (out:ptr, p:ptr, len:int)
reg_intrinsic(dbt, "co_tolower", DBT_EMIT_CO_3PP, reinterpret_cast<void *>(co_tolower));
reg_intrinsic(dbt, "co_toupper", DBT_EMIT_CO_3PP, reinterpret_cast<void *>(co_toupper));
reg_intrinsic(dbt, "co_reverse", DBT_EMIT_CO_3PP, reinterpret_cast<void *>(co_reverse));
reg_intrinsic(dbt, "co_escape", DBT_EMIT_CO_3PP, reinterpret_cast<void *>(co_escape));
// 4 args: (out:ptr, p:ptr, len:int, n:int)
reg_intrinsic(dbt, "co_left", DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_left));
reg_intrinsic(dbt, "co_right", DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_right));
reg_intrinsic(dbt, "co_compress", DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_compress));
reg_intrinsic(dbt, "co_lpos", DBT_EMIT_CO_4PP, reinterpret_cast<void *>(co_lpos));
// Batch 3: totitle, strip_color, visible_length.
// ljust/rjust/center/edit/splice need new emitter patterns (7+ args with
// complex pointer layouts) — registered as Tier 2 but not intrinsics yet.
reg_intrinsic(dbt, "co_totitle", DBT_EMIT_CO_3PP, reinterpret_cast<void *>(co_totitle));
reg_intrinsic(dbt, "co_strip_color", DBT_EMIT_CO_3PP, reinterpret_cast<void *>(co_strip_color));
reg_intrinsic(dbt, "co_visible_length", DBT_EMIT_CO_2P, reinterpret_cast<void *>(co_visible_length));
// FP math intrinsics — double→double via platform libm.
// Explicit casts resolve C++ overload ambiguity (float/double/long double).
//
using fn_d_d = double(*)(double);
using fn_dd_d = double(*)(double, double);
reg_intrinsic(dbt, "sin", DBT_EMIT_FP_D_D, reinterpret_cast<void *>(static_cast<fn_d_d>(::sin)));
reg_intrinsic(dbt, "cos", DBT_EMIT_FP_D_D, reinterpret_cast<void *>(static_cast<fn_d_d>(::cos)));
reg_intrinsic(dbt, "tan", DBT_EMIT_FP_D_D, reinterpret_cast<void *>(static_cast<fn_d_d>(::tan)));
reg_intrinsic(dbt, "asin", DBT_EMIT_FP_D_D, reinterpret_cast<void *>(static_cast<fn_d_d>(::asin)));
reg_intrinsic(dbt, "acos", DBT_EMIT_FP_D_D, reinterpret_cast<void *>(static_cast<fn_d_d>(::acos)));
reg_intrinsic(dbt, "atan", DBT_EMIT_FP_D_D, reinterpret_cast<void *>(static_cast<fn_d_d>(::atan)));
reg_intrinsic(dbt, "exp", DBT_EMIT_FP_D_D, reinterpret_cast<void *>(static_cast<fn_d_d>(::exp)));
reg_intrinsic(dbt, "log", DBT_EMIT_FP_D_D, reinterpret_cast<void *>(static_cast<fn_d_d>(::log)));
reg_intrinsic(dbt, "log10", DBT_EMIT_FP_D_D, reinterpret_cast<void *>(static_cast<fn_d_d>(::log10)));
reg_intrinsic(dbt, "ceil", DBT_EMIT_FP_D_D, reinterpret_cast<void *>(static_cast<fn_d_d>(::ceil)));
reg_intrinsic(dbt, "floor", DBT_EMIT_FP_D_D, reinterpret_cast<void *>(static_cast<fn_d_d>(::floor)));
reg_intrinsic(dbt, "fabs", DBT_EMIT_FP_D_D, reinterpret_cast<void *>(static_cast<fn_d_d>(::fabs)));
reg_intrinsic(dbt, "trunc", DBT_EMIT_FP_D_D, reinterpret_cast<void *>(static_cast<fn_d_d>(::trunc)));
// FP math intrinsics — (double,double)→double via platform libm.
//
reg_intrinsic(dbt, "pow", DBT_EMIT_FP_DD_D, reinterpret_cast<void *>(static_cast<fn_dd_d>(::pow)));
reg_intrinsic(dbt, "atan2", DBT_EMIT_FP_DD_D, reinterpret_cast<void *>(static_cast<fn_dd_d>(::atan2)));
reg_intrinsic(dbt, "fmod", DBT_EMIT_FP_DD_D, reinterpret_cast<void *>(static_cast<fn_dd_d>(::fmod)));
// max()/min() float path (#1273) — host fmax/fmin intercept the blob stubs.
//
reg_intrinsic(dbt, "fmax", DBT_EMIT_FP_DD_D, reinterpret_cast<void *>(static_cast<fn_dd_d>(::fmax)));
reg_intrinsic(dbt, "fmin", DBT_EMIT_FP_DD_D, reinterpret_cast<void *>(static_cast<fn_dd_d>(::fmin)));
// Rounding intrinsic — NearestPretty (double→double).
//
reg_intrinsic(dbt, "rv64_nearest_pretty", DBT_EMIT_FP_D_D,
reinterpret_cast<void *>(static_cast<fn_d_d>(NearestPretty)));
// String↔double conversion intrinsics.
//
reg_intrinsic(dbt, "rv64_strtod", DBT_EMIT_STRTOD, reinterpret_cast<void *>(host_strtod));
reg_intrinsic(dbt, "rv64_fval", DBT_EMIT_FVAL, reinterpret_cast<void *>(host_fval));
reg_intrinsic(dbt, "rv64_alloc", DBT_EMIT_ALLOC, reinterpret_cast<void *>(host_alloc));
reg_intrinsic(dbt, "rv64_add_doubles", DBT_EMIT_ADD_DOUBLES,
reinterpret_cast<void *>(host_add_doubles));
// Round-to-precision intrinsic.
reg_intrinsic(dbt, "rv64_ftoa_round", DBT_EMIT_FTOA_ROUND,
reinterpret_cast<void *>(host_ftoa_round));
// Pre-translate all intrinsic stubs into the cache BEFORE any
// function pretranslation. Intrinsics are leaf functions (strlen,
// memcpy, sitoa, co_* wrappers) that blob functions call internally.
// By caching them first, every subsequent translate_block that
// encounters a JAL to an intrinsic address will find it in the
// cache and emit an inline CALL — regardless of worklist order
// or superblock extension boundaries.
//
for (int i = 0; i < dbt->num_intrinsics; i++) {
uint64_t addr = dbt->intrinsics[i].guest_addr;
if (addr) {
dbt_pretranslate(dbt, addr);
}
}
// Pretranslate all Tier 2 functions. With intrinsics already cached,
// inline CALLs to intrinsic targets will fire on first encounter.
//
for (auto &kv : s_tier2.funcs) {
dbt_pretranslate(dbt, kv.second.guest_addr);
}
// Resolve cross-function chains: block A (from function X)
// exits to block B (from function Y) — the backpatch during Y's
// pretranslation won't find A's patch site. This pass fixes them.
dbt_resolve_chains(dbt);
}
// Copy blob code into a program's guest memory.
// Called during compile_expression() before codegen.
//
template<typename Vec>
void tier2_install(Vec &memory, uint64_t guest_base) {
if (!s_tier2.loaded) return;
// Copy the prebuilt flat image (code + rodata + data + zeroed BSS)
// as one contiguous block. This preserves the exact ELF layout
// while avoiding a second memset on each install.
uint64_t total = s_tier2.image.size();
if (guest_base + total > memory.size()) return;
memcpy(memory.data() + guest_base,
s_tier2.image.data(), total);
}
template void tier2_install(guest_memory_t &, uint64_t);
template void tier2_install(std::vector<uint8_t> &, uint64_t);
// Reset only the writable portions of the Tier 2 blob (data + BSS).
// Code and rodata are immutable and don't need re-copying at runtime.
//
template<typename Vec>
static void tier2_reset_writable(Vec &memory, uint64_t guest_base) {
if (!s_tier2.loaded) return;
// Re-copy initialized writable data (.sdata/.data).
if (s_tier2.data_image_size > 0) {
uint64_t dst = guest_base + s_tier2.data_image_offset;
if (dst + s_tier2.data_image_size <= memory.size()) {
memcpy(memory.data() + dst,
s_tier2.image.data() + s_tier2.data_image_offset,
s_tier2.data_image_size);
}
}
// Zero-fill BSS.
if (s_tier2.bss_size > 0) {
uint64_t bss_start = guest_base + s_tier2.code_size;
if (bss_start + s_tier2.bss_size <= memory.size()) {
memset(memory.data() + bss_start, 0, s_tier2.bss_size);
}
}
}
// Lazy-init Tier 2 blob on first compile.
static bool s_tier2_init = false;
static void tier2_lazy_init() {
if (s_tier2_init) return;
s_tier2_init = true;
// Try to load from the game's bin directory (where engine.so lives).
const char *paths[] = {
"bin/softlib.rv64",
"./softlib.rv64",
nullptr
};
for (int i = 0; paths[i]; i++) {
if (tier2_load(paths[i], rv_compiler::BLOB_BASE)) {
return;
}
}
// No blob found — the JIT declines every expression (jit_eval
// checks s_tier2.loaded) and the interpreter handles everything.
// Say so once, loudly: this is a degraded state on a JIT build,
// not a silent mode of normal operation (#875).
fprintf(stderr, "tier2: softlib.rv64 missing or unloadable -- "
"JIT disabled, falling back to the interpreter\n");
// Still hash the compiler version so tier 1 upgrades invalidate. Same
// per-unit stamps as the loaded path (#2061) -- without them this leg has
// the identical hole, and it is the leg a blob-less build runs on.
// Layout is in the key for the same reason as the blob-loaded path (#2107).
static const uint64_t layout[] = {
rv_compiler::CODE_BASE, rv_compiler::CODE_LIMIT,
rv_compiler::STR_BASE, rv_compiler::STR_LIMIT,
rv_compiler::FARGS_BASE, rv_compiler::FARGS_LIMIT,
rv_compiler::BLOB_BASE, rv_compiler::MEM_SIZE,
};
const void *parts[] = {
JIT_COMPILER_VERSION,
JIT_BUILD_STAMP,
TIER1_STAMP_AST,
TIER1_STAMP_HIR_LOWER,
TIER1_STAMP_HIR_SSA,
TIER1_STAMP_HIR_OPT,
TIER1_STAMP_HIR_CODEGEN,
layout,
};
const size_t sizes[] = {
sizeof(JIT_COMPILER_VERSION) - 1,
sizeof(JIT_BUILD_STAMP) - 1,
strlen(TIER1_STAMP_AST),
strlen(TIER1_STAMP_HIR_LOWER),
strlen(TIER1_STAMP_HIR_SSA),
strlen(TIER1_STAMP_HIR_OPT),
strlen(TIER1_STAMP_HIR_CODEGEN),
sizeof(layout),
};
s_blob_version = sha1_hex_parts(parts, sizes, 8);
}
void tier2_ensure(void) {
tier2_lazy_init();
}
// Maximum AST_FUNCCALL nesting depth of a parse tree (#1002).
static int ast_max_funccall_depth(const ASTNode *node)
{
if (!node) return 0;
int child_max = 0;
for (const auto &c : node->children) {
int d = ast_max_funccall_depth(c.get());
if (d > child_max) child_max = d;
}
return child_max + (node->type == AST_FUNCCALL ? 1 : 0);
}
// Total AST_FUNCCALL nodes in a parse tree (invocation-count watermark).
static int ast_funccall_count(const ASTNode *node)
{
if (!node) return 0;
int n = (node->type == AST_FUNCCALL) ? 1 : 0;
for (const auto &c : node->children) {
n += ast_funccall_count(c.get());
}
return n;
}
// Bounded strlen over guest memory (#1057). Returns false when `addr`
// is out of range or the region [addr, memory_size) has no NUL — callers
// must not treat the pointer as a C string in that case.
//
static bool guest_strnlen(const uint8_t *memory, size_t memory_size,
uint64_t addr, size_t *out_len)
{
if (!memory || addr >= memory_size) {
return false;
}
const size_t maxn = memory_size - static_cast<size_t>(addr);
const char *p = reinterpret_cast<const char *>(memory + addr);
const void *nul = memchr(p, '\0', maxn);
if (!nul) {
return false;
}
*out_len = static_cast<size_t>(static_cast<const char *>(nul) - p);
return true;
}
// #1071: guest pointer as a host C string only after guest_strnlen succeeds.
//
static const char *guest_cstr(const uint8_t *memory, size_t memory_size,
uint64_t addr)
{
size_t len = 0;
if (!guest_strnlen(memory, memory_size, addr, &len)) {
return nullptr;
}
return reinterpret_cast<const char *>(memory + addr);
}
// Load fargs[i] pointer from the guest fargs table; returns false if the
// slot is out of range. Does not validate the pointed-to string.
//
static bool guest_farg_addr(const uint8_t *memory, size_t memory_size,
uint64_t fargs_addr, int idx, uint64_t *out_addr)
{
if (!memory || !out_addr || idx < 0) {
return false;
}
if (memory_size < 8u) {
return false;
}
const uint64_t slot = fargs_addr + static_cast<uint64_t>(idx) * 8u;
// Overflow-safe bound: reject a wrapped slot (slot < fargs_addr, e.g. a
// near-2^64 fargs_addr) and any slot whose 8-byte read runs past the
// guest region -- without adding to slot, which could itself wrap.
if (slot < fargs_addr || slot > memory_size - 8u) {
return false;
}
uint64_t p = 0;
memcpy(&p, memory + slot, 8);
*out_addr = p;
return true;
}
// #1078: overflow-safe [addr, addr+nbytes) wholly inside guest memory.
// nbytes == 0 is allowed only when addr <= memory_size.
//
static bool guest_range_ok(uint64_t addr, uint64_t nbytes, size_t memory_size)
{
if (nbytes == 0) {
return addr <= memory_size;
}
if (addr >= memory_size) {
return false;
}
// Equivalent to addr + nbytes <= memory_size without wrap.
return nbytes <= memory_size - static_cast<size_t>(addr);
}
// Shared helper for dbt_run/dbt_resume nonzero status: count wall-clock
// aborts and, when requested, write the AST-shaped diagnostic so callers
// that treat the run as handled do not fall through to an empty result.
static bool handle_dbt_run_status(int rc, UTF8 *out, size_t out_size,
bool emit_cpu_limited)
{
if (rc == 0) {
return true;
}
if (rc == -3) {
s_jit_stats.bail_alarm++;
if (emit_cpu_limited && nullptr != out && 0 < out_size) {
// Mirror AST's alarm path (ast_eval_function): emit
// "#-1 CPU LIMITED" rather than returning false and letting
// the AST short-circuit produce an empty string.
const UTF8 *kMsg = S_("#-1 CPU LIMITED");
size_t n = strlen(reinterpret_cast<const char *>(kMsg));
if (n >= out_size) {
n = out_size - 1;
}
memcpy(out, kMsg, n);
out[n] = '\0';
return true; // handled
}
}
return false;
}
static compiled_program compile_expression(const UTF8 *expr, size_t nLen,
int eval = EV_FCHECK | EV_EVAL,
uint64_t code_base = 0,
uint64_t str_start = rv_compiler::STR_BASE,
uint64_t str_lim = rv_compiler::STR_LIMIT,
uint64_t fargs_start = rv_compiler::FARGS_BASE,
uint64_t fargs_lim = rv_compiler::FARGS_LIMIT,
uint64_t out_start = 0) {
tier2_lazy_init();
compiled_program prog;
prog.ok = false;
prog.out_used = 0;
prog.entry_pc = code_base;
prog.folds = 0;
prog.ecalls = 0;
prog.tier2_calls = 0;
prog.native_ops = 0;
prog.needs_jit = false;
// Parse the expression.
auto ast = ast_parse_string(expr, nLen);
if (!ast) {
s_jit_stats.compile_fail++;
return prog;
}
// Static function-nesting depth watermark (#1002): the maximum
// AST_FUNCCALL nesting the AST evaluator would reach. jit_eval
// declines the run when live func_nest_lev + this watermark would
// trip function_recursion_limit, so the AST reproduces the limit
// error the flattened compiled code cannot.
//
// Outer-AST-only for now; inlined u()/ulocal() body watermarks are
// accumulated during lowering and added after (#1056).
prog.max_func_depth = ast_max_funccall_depth(ast.get());
// Static invocation-count watermark: total FUNCCALL nodes. Flattened
// JIT does not maintain func_invk_ctr for sequential calls, so decline
// when live ctr + this count would trip function_invocation_limit.
prog.n_func_calls = ast_funccall_count(ast.get());
// --- HIR pipeline ---
// Phase 1: Lower AST → HIR.
rv_compiler rc(code_base, str_start, str_lim, fargs_start, fargs_lim, out_start);
// Tier 2 blob is NOT installed here — compilation never reads
// the blob region. Callers that use prog.memory for runtime
// execution install it after compile_expression() returns.
// Set compile-time eval flags for unknown-function resolution.
s_compile_eval = eval;
s_fcheck_available = (eval & EV_FCHECK) != 0;
hir_program h;
h.init();
qreg_init();
// Set up Tier 3 compile-time deps collector.
std::vector<compiled_program::inline_dep> deps;
s_compile_deps = &deps;
s_inline_depth = 0;
h.result = hir_lower_node(h, rc, ast.get());
// Fold inlined-body watermarks into the program stats (#1056).
prog.max_func_depth += h.inline_extra_depth;
prog.n_func_calls += h.inline_extra_calls;
// Convert native scalar results to strings for MUX output.
// This is the boundary where non-string values escape to
// top-level string context.
if (h.result >= 0 && h.ty[h.result] == TY_INT) {
h.result = h.emit(HIR_ITOA, TY_STRING, h.result);
} else if (h.result >= 0 && h.ty[h.result] == TY_FLOAT) {
h.result = h.emit(HIR_FTOA, TY_STRING, h.result);
}
s_compile_deps = nullptr;
// If any HIR capacity limit was hit during lowering, the program
// contains -1 instruction/block indices that later phases would
// dereference (#859). Bail out — the AST evaluator handles it.
// refused_index() is the consumer-side half of the same condition (#1501):
// overflowed says a producer handed out -1, refused_index() says something
// then used it as a subscript. Either way this program is not safe to
// codegen, and the AST evaluator answers instead.
//
if (h.overflowed || h.refused_index()) {
s_jit_stats.compile_fail++;
return prog; // prog.ok is still false
}
const char *dump_env = getenv("TINYMUX_DUMP_HIR");
bool bDump = (dump_env && *dump_env != '0');
if (bDump) {
printf("\n--- JIT Compilation: %.*s ---\n", static_cast<int>(nLen), expr);
printf("Phase 1: HIR Lowering\n");
hir_dump(h);
}
// Phase 2: SSA construction (for multi-block programs, M4+).
// For single-block programs this is a no-op but builds the CFG.
hir_build_cfg(h);
if (h.n_blocks > 1) {
hir_superblock(h);
if (bDump) {
printf("Phase 1b: Superblock Formation\n");
hir_dump(h);
}
hir_ssa_construct(h);
if (bDump) {
printf("Phase 2: SSA Construction\n");
hir_dump(h);
}
// SSA construction can exhaust capacity on its own, and the
// check at the end of lowering (above) has already run by this
// point. hir_insert_phis() sets overflowed when it cannot
// reserve parg slots (#1149), and the emit()/emit_phi() guards
// in hir.h fire from here too. Without this check the flag is
// written and never read: renaming and codegen proceed over a
// program whose PHI insertion stopped partway.
//
if (h.overflowed || h.refused_index()) {
s_jit_stats.compile_fail++;
return prog; // prog.ok is still false
}
}
// Phase 3: SSA optimization (constant fold, copy prop, DCE).
hir_optimize(h);
if (bDump) {
printf("Phase 3: SSA Optimization\n");
hir_dump(h);
}
// Phase 4: Codegen HIR → RV64.
hir_codegen(h, rc);
// Record what this compile WANTED, before any ceiling can reject it.
//
// code_bytes_max below is taken after the checks and so is censored at
// CODE_LIMIT: it cannot report a program that did not fit, which is
// precisely the program a resize decision needs to see (#2074).
//
{
const uint64_t want_code =
static_cast<uint64_t>(rc.code.size()) * 4;
if (want_code > s_jit_stats.want_code_max) {
s_jit_stats.want_code_max = want_code;
}
if (rc.str_want > s_jit_stats.want_strpool_max) {
s_jit_stats.want_strpool_max = rc.str_want;
}
if (rc.fargs_want > s_jit_stats.want_fargs_max) {
s_jit_stats.want_fargs_max = rc.fargs_want;
}
if (rc.out_want > s_jit_stats.want_outslots_max) {
s_jit_stats.want_outslots_max = rc.out_want;
}
}
// Check for code overflow before copying.
if (rc.code.size() * 4 > rv_compiler::CODE_LIMIT) {
s_jit_stats.compile_fail++;
s_jit_stats.bail_code++;
s_jit_stats.bail_slots++; // imprecise; kept for compatibility
return prog; // prog.ok is still false
}
// Copy code to guest memory at the configured code_base.
for (size_t i = 0; i < rc.code.size(); i++) {
memcpy(rc.memory.data() + rc.code_base + i * 4, &rc.code[i], 4);
}
// If any resource was exhausted during compilation, the generated
// code references address 0 and would corrupt guest memory.
// Bail out — the AST evaluator will handle this expression.
if (rc.out_exhausted || rc.pool_exhausted) {
s_jit_stats.compile_fail++;
// Attribute to the ceiling that actually ran out. More than one
// can be set in a single compile, so these are counted
// independently rather than as an if/else chain -- the totals are
// per-ceiling occurrences, not a partition of compile_fail.
if (rc.str_exhausted) { s_jit_stats.bail_strpool++; }
if (rc.fargs_exhausted) { s_jit_stats.bail_fargs++; }
if (rc.out_exhausted) { s_jit_stats.bail_outslots++; }
if (!rc.bail_was_noeval) {
s_jit_stats.bail_slots++; // imprecise; kept for compatibility
}
return prog; // prog.ok is still false
}
prog.memory = std::move(rc.memory);
prog.memory_size = rv_compiler::MEM_SIZE;
prog.out_addr = rc.final_out;
prog.out_used = (rv_compiler::STACK_TOP - 8) - rc.out_pool;
prog.entry_pc = rc.code_base;
prog.code_size = rc.code.size() * 4;
prog.str_pool_end = rc.str_pool;
prog.fargs_pool_end = rc.fargs_pool;
prog.out_pool_end = rc.out_pool;
prog.ok = true;
s_jit_stats.compile_ok++;
// Code size tracking.
uint64_t code_bytes = rc.code.size() * 4;
s_jit_stats.code_bytes_total += code_bytes;
if (code_bytes > s_jit_stats.code_bytes_max)
s_jit_stats.code_bytes_max = code_bytes;
s_jit_stats.hir_insns_total += static_cast<uint64_t>(h.n_insns);
if (static_cast<uint64_t>(h.n_insns) > s_jit_stats.hir_insns_max)
s_jit_stats.hir_insns_max = static_cast<uint64_t>(h.n_insns);
s_jit_stats.spills_total += static_cast<uint64_t>(rc.spills);
prog.folds = h.folds;
prog.ecalls = h.ecalls;
prog.tier2_calls = h.tier2_calls;
prog.native_ops = h.native_ops;
prog.needs_jit = h.needs_jit || rc.needs_jit;
// Classify every runtime substitution/carg reference emitted during
// lowering into a per-program mask, so run_cached_program populates only
// the guest CARGS/SUBST slots this program actually reads. emit_sref is
// the single choke point for these references, so this set is complete.
prog.subst_mask = 0;
prog.cargs_used = 0;
for (uint64_t a : h.sref_addrs) {
if (a >= rv_compiler::CARGS_BASE
&& a < rv_compiler::CARGS_BASE
+ static_cast<uint64_t>(rv_compiler::MAX_CARGS)
* rv_compiler::CARGS_SLOT) {
int idx = static_cast<int>(
(a - rv_compiler::CARGS_BASE) / rv_compiler::CARGS_SLOT);
if (idx + 1 > prog.cargs_used) prog.cargs_used = idx + 1;
} else if (a >= rv_compiler::SUBST_BASE
&& a < rv_compiler::SUBST_BASE
+ static_cast<uint64_t>(rv_compiler::SUBST_COUNT)
* rv_compiler::SUBST_SLOT) {
int slot = static_cast<int>(
(a - rv_compiler::SUBST_BASE) / rv_compiler::SUBST_SLOT);
prog.subst_mask |= (UINT64_C(1) << slot);
}
}
prog.deps = std::move(deps);
return prog;
}
static uint64_t resolve_runtime_out_addr(uint64_t out_addr, uint64_t entry_sp) {
return rv_compiler::resolve_output_addr(out_addr, entry_sp);
}
// ---------------------------------------------------------------
// fun_rveval: softcode function.
//
// rveval(<expression>)
//
// Parses and compiles <expression> to RV64, runs through the JIT,
// returns the result.
//
// Examples:
// think rveval(add(1,2)) → 3 (constant folded)
// think rveval(add(mul(3,4),5)) → 17 (fully folded)
// think rveval(strlen(hello)) → 5 (folded)
// ---------------------------------------------------------------
// ---------------------------------------------------------------
// Persistent DBT state — avoids mmap/munmap per call.
//
// The 64 MB mmap + 1 MB block cache allocation dominated the
// ECALL path. By keeping the dbt_state_t alive, we amortize
// the cost to one-time initialization.
// ---------------------------------------------------------------
static int dbt_trace_mask_from_env() {
const char *env = getenv("TINYMUX_DBT_TRACE");
if (!env || !env[0]) return 0;
if (strcmp(env, "1") == 0 || strcmp(env, "all") == 0) {
return DBT_TRACE_EXEC | DBT_TRACE_TRANSLATE;
}
int mask = 0;
if (strstr(env, "exec")) mask |= DBT_TRACE_EXEC;
if (strstr(env, "xlate") || strstr(env, "translate")) {
mask |= DBT_TRACE_TRANSLATE;
}
return mask;
}
static void dbt_configure_trace_from_env(dbt_state_t *dbt) {
dbt->trace = dbt_trace_mask_from_env();
dbt->trace_guest_pc = 0;
dbt->trace_guest_pc_filter = false;
const char *env = getenv("TINYMUX_DBT_TRACE_PC");
if (!env || !env[0]) return;
errno = 0;
char *end = nullptr;
unsigned long long value = strtoull(env, &end, 0);
if (errno != 0 || end == env || *end != '\0') {
fprintf(stderr,
"dbt: ignoring invalid TINYMUX_DBT_TRACE_PC='%s'\n", env);
return;
}
dbt->trace_guest_pc = static_cast<uint64_t>(value);
dbt->trace_guest_pc_filter = true;
}
// One complete execution context for a cached program: the guest memory the
// program is materialized into, and the DBT that translates and runs it.
//
// There are two (#1326). Everything in here is per-execution state that a
// nested run would otherwise destroy for the run above it: a nested program
// materializes its own code/strings over the outer program's, and resetting
// the shared DBT throws away the outer program's translated blocks while its
// frames are still live. That is why run_cached_program used to refuse
// outright, which made softcode JIT -> fun_lua -> Lua JIT fall back to the
// Lua interpreter for every nested call.
//
// Index is the nesting depth, so depth 0 is softcode's and depth 1 is the
// nested (in practice Lua) one. Depth 2 still refuses: a second nesting
// would need a third context, and nothing reaches it today.
//
// The cost is nothing until nesting actually happens. Both the buffer
// (MEM_SIZE) and the DBT's code buffer (CODE_BUF_SIZE) are allocated on first
// use -- vector::resize and dbt_init respectively -- so an installation whose
// softcode never calls lua() never pays for the second context.
//
// Guest code slots (#2129).
//
// Every cached program is compiled at the same canonical base, so two
// programs collide at the same guest PCs and the DBT had to reset (throw
// away all translated blocks) on every program switch. One uint64_t of
// "which program was last" made the translated-block cache hold exactly ONE
// program: two expressions alternating re-translated on every evaluation,
// a fixed +22..76 µs per command that no single-expression benchmark could
// see.
//
// The fix is to make the guest PC itself the disambiguator, the same model
// the shared heap and persistent_vm already use: materialize each program's
// CODE at its own 16 KB slot, so the PC-keyed block cache (and the inline
// lookup emitted into host code, which must not grow a tag) holds all
// resident programs at once. Only the code moves — str/fargs DATA keeps
// swapping at its canonical addresses, because data does not affect
// translation validity, only the code bytes at translate time do.
//
// Slot addresses live in the two spans this arena leaves unallocated, both
// within JAL range (±1 MB) of the blob so the relocated Tier 2 calls still
// encode:
//
// slot 0 0x00000 canonical (delta 0, no relocation;
// the only slot PINNED programs use)
// slots 1-4 0x40000..0x50000 between BLOB_LIMIT and LUA_ARRAY
// slots 5-6 0x60000..0x68000 between LUA_ARRAY_LIMIT and CARGS
//
// (persistent_vm places its pools at 0x40000 too, but in its OWN arena —
// reusing addresses across arenas is free; only overlap within one image
// matters.)
//
static constexpr int JIT_CODE_SLOTS = 7;
static constexpr uint64_t JIT_SLOT_BASE[JIT_CODE_SLOTS] = {
rv_compiler::CODE_BASE,
0x40000, 0x44000, 0x48000, 0x4C000,
0x60000, 0x64000,
};
static_assert(rv_compiler::CODE_LIMIT == 0x4000,
"slot stride below assumes 16 KB code regions");
static_assert(0x40000 >= rv_compiler::BLOB_LIMIT
&& 0x4C000 + 0x4000 <= rv_compiler::LUA_ARRAY_BASE,
"slots 1-4 must fit between the blob and the Lua array");
static_assert(0x60000 >= rv_compiler::LUA_ARRAY_LIMIT
&& 0x64000 + 0x4000 <= rv_compiler::CARGS_BASE,
"slots 5-6 must fit between the Lua array and CARGS");
static_assert(0x64000 + 0x4000 <= rv_compiler::BLOB_BASE + (1 << 20),
"every slot PC must keep the blob within JAL range");
struct jit_run_vm {
// Guest memory a program is materialized into. Tier 2 is installed once,
// when the buffer is first sized.
guest_memory_t buffer;
bool buffer_ready = false;
// program_id whose str/fargs DATA blobs currently occupy the canonical
// pool addresses. materialize_data sets it; run_cached_program skips
// the data memcpy when the pools already hold the program being run.
// 0 = unknown/none (program_ids start at 1).
uint64_t buffer_program_id = 0;
// The DBT translating out of `buffer`. Translated blocks for every
// slot-resident program coexist in it, keyed by their slotted PCs.
dbt_state_t dbt;
bool dbt_ready = false;
// True while the DBT is bound to some other guest image entirely
// (run_compiled points it at a program's own uncompacted memory).
// The next run_cached_program must dbt_reset back to `buffer` and
// forget every slot rather than trust translations made against
// foreign bytes.
bool dbt_foreign = false;
// Which program's code occupies each slot. Residency means only that
// the code BYTES are materialized in the guest slot; translations are
// rebuilt lazily via block-cache misses, so DBT-level reclaims need no
// bookkeeping here.
//
// Admission policy (#2130 campaign, from Kagura's #2139 review): slot 0
// is the PROBATION lane — always claimable, where first-touch and
// pinned programs run — while slots 1+ are PROTECTED: evictable only
// when cold (their `hot` bit was not set since the last sweep). A
// strict LRU admitted every first-touch program straight into a
// protected slot, so round-robin one past capacity evicted the entry
// needed next on every single evaluation — 100% miss, ~8% WORSE than
// the old unconditional reset. With probation, a cold storm thrashes
// only slot 0 (the old per-switch cost, no worse) and the hot working
// set keeps its translations through it. The sweep runs every
// JIT_SLOT_SWEEP_PERIOD misses, so "hot" means "used within the last
// couple of windows" and a program that went idle becomes evictable
// instead of squatting.
struct code_slot {
uint64_t program_id = 0; // 0 = free
uint64_t stamp = 0; // LRU tick of last use
bool hot = false; // used since the last sweep
};
code_slot slots[JIT_CODE_SLOTS];
uint64_t slot_stamp = 0;
uint32_t misses_since_sweep = 0;
void release_all_slots() {
for (int i = 0; i < JIT_CODE_SLOTS; i++) {
slots[i] = code_slot{};
}
misses_since_sweep = 0;
buffer_program_id = 0;
}
};
// Deepest nesting level that has its own context. run_cached_program refuses
// beyond this. Sizes s_vm below, which s_run_cached_depth indexes -- raising
// this must not be able to leave the array behind it.
//
static constexpr int JIT_MAX_RUN_DEPTH = 2;
static jit_run_vm s_vm[JIT_MAX_RUN_DEPTH];
// Forget a program in every context that has it slot-resident — called when
// its compiled_program is dropped (compile-cache eviction, staleness). The
// slot's stale block-cache entries need no eviction here: program_ids are
// unique for the life of the process, so nothing can claim the residency,
// and reassigning the slot to any other program invalidates its PC range
// before trusting it (pick_code_slot).
//
static void release_program_slots(uint64_t program_id) {
for (int vi = 0; vi < JIT_MAX_RUN_DEPTH; vi++) {
for (int si = 0; si < JIT_CODE_SLOTS; si++) {
if (s_vm[vi].slots[si].program_id == program_id) {
s_vm[vi].slots[si] = jit_run_vm::code_slot{};
}
}
if (s_vm[vi].buffer_program_id == program_id) {
s_vm[vi].buffer_program_id = 0;
}
}
}
// Current nesting depth of run_cached_program, and therefore the index of the
// first context NOT in use by a live run. Declared here rather than beside
// run_cached_program because the compile path needs it too: materializing into
// a context that a shallower run owns overwrites the memory that run is
// executing out of (#1326).
static int s_run_cached_depth = 0;
// Defined with the Tier 3 helper save stacks below; called from
// run_cached_program at top-level entry.
static void jit_helper_stacks_reset();
// Claims the current context for the duration of a run, so anything the run
// re-enters picks the next one instead. Hoisted to file scope because BOTH
// entry points into the DBT need it: run_cached_program (jit_eval's route)
// and run_compiled (rvbench's). run_compiled owning s_vm[0] without
// claiming it was #2106 -- a nested run then reset the very DBT the outer
// run's frames were executing from.
//
struct RunDepthGuard {
int &depth;
explicit RunDepthGuard(int &d) : depth(d) { ++depth; }
~RunDepthGuard() { --depth; }
RunDepthGuard(const RunDepthGuard &) = delete;
RunDepthGuard &operator=(const RunDepthGuard &) = delete;
};
// Release the persistent DBT state on shutdown.
//
void dbt_compile_cleanup(void) {
for (int i = 0; i < JIT_MAX_RUN_DEPTH; i++) {
if (s_vm[i].dbt_ready) {
dbt_cleanup(&s_vm[i].dbt);
s_vm[i].dbt_ready = false;
}
}
}
// Get a reset DBT state bound to an arbitrary guest image, initializing on
// first use. Returns nullptr on allocation failure.
//
// This is run_compiled's binding (rvbench runs straight out of a program's
// own uncompacted memory). It marks the context FOREIGN: every slot claim
// and translation now refers to someone else's bytes, so the next slotted
// run must reset back to the vm's own buffer and forget the slots.
//
static dbt_state_t *get_dbt(jit_run_vm *vm,
uint8_t *memory, size_t memory_size,
int (*ecall_fn)(rv64_ctx_t *, void *),
void *ecall_user) {
dbt_state_t *dbt = &vm->dbt;
vm->dbt_foreign = true;
vm->release_all_slots();
if (!vm->dbt_ready) {
if (dbt_init(dbt, memory, memory_size, ecall_fn, ecall_user) != 0) {
return nullptr;
}
dbt_configure_trace_from_env(dbt);
// Dispatch limit: safety net during development.
// TINYMUX_DBT_MAX_DISPATCH overrides; 0 = unlimited.
const char *md_env = getenv("TINYMUX_DBT_MAX_DISPATCH");
dbt->max_dispatch = md_env ? strtoull(md_env, nullptr, 0) : 10000000;
dbt->alarm_flag = &alarm_clock.alarmed; // wall-clock abort (#JIT-alarm)
vm->dbt_ready = true;
return dbt;
}
// Reset for new program: keep mmap'd code buffer + cache allocation.
dbt_reset(dbt, memory, memory_size, ecall_fn, ecall_user);
dbt_configure_trace_from_env(dbt);
return dbt;
}
// ECALL handler context and forward declaration.
//
static int eval_ecall(rv64_ctx_t *ctx, void *user_data);
static int poc_ecall(rv64_ctx_t *ctx, void *user_data);
// Bind the vm's DBT to its own runtime buffer for a slotted run (#2129).
// Unlike get_dbt this does NOT reset on every program switch — that reset
// is exactly what made the block cache one-program-deep. It initializes
// on first use, resets only to recover from a foreign binding, and
// otherwise leaves every resident program's translations alone; the caller
// sets the per-run ECALL context via dbt_rerun.
//
// The caller must have runtime_buffer_init'd the vm first: the blob bytes
// have to be in the buffer before anything is translated out of it.
//
static dbt_state_t *bind_run_dbt(jit_run_vm *vm) {
dbt_state_t *dbt = &vm->dbt;
if (!vm->dbt_ready) {
if (dbt_init(dbt, vm->buffer.data(), rv_compiler::MEM_SIZE,
eval_ecall, nullptr) != 0) {
return nullptr;
}
dbt_configure_trace_from_env(dbt);
const char *md_env = getenv("TINYMUX_DBT_MAX_DISPATCH");
dbt->max_dispatch = md_env ? strtoull(md_env, nullptr, 0) : 10000000;
dbt->alarm_flag = &alarm_clock.alarmed; // wall-clock abort (#JIT-alarm)
vm->dbt_ready = true;
vm->dbt_foreign = false;
vm->release_all_slots();
return dbt;
}
if (vm->dbt_foreign) {
dbt_reset(dbt, vm->buffer.data(), rv_compiler::MEM_SIZE,
eval_ecall, nullptr);
dbt_configure_trace_from_env(dbt);
vm->dbt_foreign = false;
vm->release_all_slots();
}
return dbt;
}
struct persistent_vm_t {
// Pool layout within THIS arena's image (#2124).
//
// code 0x00004 .. 0x10000 bump-allocated, bounded by BLOB_BASE
// blob 0x10000 .. 0x40000 installed once
// str 0x40000 .. 0x48000 (32 KB)
// fargs 0x48000 .. 0x4C000 (16 KB)
//
// These are deliberately NOT the rv_compiler one-shot constants, which
// is what they used to be. The code heap here bump-allocates upward
// from 0x0004 across many compilations, while STR_BASE/FARGS_BASE sit
// at 0x8000/0x4000 -- inside the range the code heap grows through.
// Nothing bounded that: install_code() wrote unchecked, and
// arena_nearly_full() called the code heap's capacity
// BLOB_BASE - 0x0004, which is only true once the pools are elsewhere.
//
// Each arena owns a separate MEM_SIZE image (persistent_vm_t,
// shared_heap_t, and each jit_run_vm::buffer), so reusing addresses
// ACROSS arenas is free; only overlap within one image matters. These
// sit in the span left unallocated between the blob and LUA_ARRAY,
// which is why they can be disjoint without moving anything else.
//
static constexpr uint64_t PVM_CODE_START = 0x0004; // avoid PC=0
static constexpr uint64_t PVM_CODE_LIMIT = rv_compiler::BLOB_BASE;
static constexpr uint64_t PVM_STR_BASE = 0x40000;
static constexpr uint64_t PVM_STR_LIMIT = 0x48000;
static constexpr uint64_t PVM_FARGS_BASE = 0x48000;
static constexpr uint64_t PVM_FARGS_LIMIT = 0x4C000;
static_assert(PVM_CODE_LIMIT <= rv_compiler::BLOB_BASE,
"pvm code heap must not reach the blob");
static_assert(PVM_STR_BASE >= rv_compiler::BLOB_LIMIT,
"pvm string pool must sit above the blob");
static_assert(PVM_STR_LIMIT == PVM_FARGS_BASE,
"pvm string pool must abut its fargs pool");
static_assert(PVM_FARGS_LIMIT <= rv_compiler::LUA_ARRAY_BASE,
"pvm pools must not reach the pinned Lua array region");
std::vector<uint8_t> memory;
dbt_state_t dbt;
bool dbt_ready;
uint32_t run_count;
uint32_t dbt_buffer_resets; // times the x86 translation buffer was reclaimed
// Re-entrancy guard for the code arena. compile_attr() may only reclaim
// (reset) the arena when no compiled code from it is on the call stack,
// i.e. exec_depth == 0. Pressure detected mid-run is deferred via
// reset_pending and serviced when the outermost run() unwinds.
int exec_depth;
bool reset_pending;
// Code heap: bump allocator for code sections.
uint64_t code_heap_next;
// Shared pool cursors — advance across compilations so each
// function gets its own non-overlapping region.
uint64_t str_pool_next;
uint64_t fargs_pool_next;
uint64_t out_pool_next;
// Compiled attribute cache: maps (obj, attr_num) → compiled info.
// Checked for staleness via mod_count on each lookup.
//
struct attr_cache_entry {
uint32_t mod_count;
uint64_t entry_pc;
uint64_t out_addr;
bool needs_jit;
};
static uint64_t attr_cache_key(dbref obj, int attr_num) {
return (static_cast<uint64_t>(static_cast<uint32_t>(obj)) << 32)
| static_cast<uint32_t>(attr_num);
}
std::unordered_map<uint64_t, attr_cache_entry> attr_cache;
// Track the worst-case output allocation across all compilations.
uint64_t worst_out_pool;
persistent_vm_t()
: memory(rv_compiler::MEM_SIZE, 0),
dbt_ready(false), run_count(0), dbt_buffer_resets(0),
exec_depth(0), reset_pending(false),
code_heap_next(PVM_CODE_START), // avoid PC=0 (cache sentinel)
str_pool_next(PVM_STR_BASE),
fargs_pool_next(PVM_FARGS_BASE),
out_pool_next(rv_compiler::STACK_TOP - 8),
worst_out_pool(rv_compiler::STACK_TOP - 8) {}
// Compile an expression and install it into persistent memory.
// Returns {entry_pc, out_addr} on success, {0, 0} on failure.
//
struct compile_result {
uint64_t entry_pc;
uint64_t out_addr;
bool needs_jit;
};
compile_result compile(const UTF8 *expr, size_t len,
int eval = EV_FCHECK | EV_EVAL) {
tier2_lazy_init();
// No blob, no persistent-VM compiles — same clean decline as
// jit_eval (#875).
if (!s_tier2.loaded) {
return {0, 0, false};
}
// Bounds check: code heap must not reach blob region.
if (code_heap_next >= PVM_CODE_LIMIT) {
return {0, 0, false};
}
// Reset the output pool for each compilation. Output slots
// are stack-allocated at runtime via the prologue, so each
// expression can reuse the same output addresses.
uint64_t out_start = rv_compiler::STACK_TOP - 8;
compiled_program prog = compile_expression(
expr, len, eval,
code_heap_next,
str_pool_next, PVM_STR_LIMIT,
fargs_pool_next, PVM_FARGS_LIMIT,
out_start);
if (!prog.ok) return {0, 0, false};
// Post-compilation overflow check.
uint64_t code_end = prog.entry_pc + prog.code_size;
if (code_end > rv_compiler::BLOB_BASE) {
return {0, 0, false};
}
install(prog);
return {prog.entry_pc, prog.out_addr, prog.needs_jit};
}
// Compile an attribute body, with caching and staleness checks.
// Returns {entry_pc, out_addr} on success, {0, 0} on failure.
//
compile_result compile_attr(dbref obj, int attr_num,
const UTF8 *body, size_t body_len,
int eval = EV_FCHECK | EV_EVAL) {
// Check cache.
uint32_t mc = attr_mod_count_get(obj, attr_num);
uint64_t key = attr_cache_key(obj, attr_num);
auto it = attr_cache.find(key);
if (it != attr_cache.end() && it->second.mod_count == mc) {
return {it->second.entry_pc, it->second.out_addr,
it->second.needs_jit};
}
// Stale or missing. The previous compilation of this attribute (if
// any) leaves its code/string/fargs regions stranded — recompiling
// bump-allocates fresh space and never reclaims the old region. Before
// that pushes the arena toward exhaustion (after which the JIT bails
// for good), reclaim it wholesale: immediately if nothing is executing
// on this VM, otherwise defer to the next safe point (the outermost
// run() unwind) via reset_pending.
if (arena_nearly_full())
{
if (0 == exec_depth)
{
reset_arena();
}
else
{
reset_pending = true;
}
}
// Compile.
compile_result cr = compile(body, body_len, eval);
if (!cr.entry_pc) return {0, 0, false};
// Cache (insert or overwrite).
attr_cache_entry entry;
entry.mod_count = mc;
entry.entry_pc = cr.entry_pc;
entry.out_addr = cr.out_addr;
entry.needs_jit = cr.needs_jit;
attr_cache[key] = entry;
return cr;
}
// Install a hand-assembled code blob at the current code heap
// position. Returns the entry_pc.
//
uint64_t install_code(const std::vector<uint32_t> &code) {
uint64_t entry = code_heap_next;
// The memcpy below is otherwise unbounded -- it wrote wherever the
// cursor pointed, and the only check on this path lived in
// compile(), which install_code() does not go through (#2124).
//
if (entry + code.size() * 4 > PVM_CODE_LIMIT) {
return 0; // 0 is the caller's "no entry" sentinel
}
for (size_t i = 0; i < code.size(); i++) {
memcpy(memory.data() + entry + i * 4, &code[i], 4);
}
code_heap_next = entry + code.size() * 4;
code_heap_next = (code_heap_next + 15) & ~15ULL;
return entry;
}
// Initialize the DBT (first time) or update ECALL handler.
//
bool ensure_dbt() {
if (!dbt_ready) {
if (dbt_init(&dbt, memory.data(), memory.size(),
poc_ecall, &dbt) != 0) {
return false;
}
dbt_ready = true;
tier2_install(memory, rv_compiler::BLOB_BASE);
pretranslate_tier2(&dbt);
dbt.blob_code_end = dbt.code_used;
}
return true;
}
// Prepare for execution: clear output buffers and reset blob
// writable state. Code and rodata are immutable — only the
// data section and BSS need resetting between runs.
//
void prepare_run() {
if (worst_out_pool < rv_compiler::STACK_TOP - 8) {
memset(memory.data() + worst_out_pool, 0,
(rv_compiler::STACK_TOP - 8) - worst_out_pool);
}
if (s_tier2.loaded) {
tier2_reset_writable(memory, rv_compiler::BLOB_BASE);
}
s_heap_next = rv_compiler::HEAP_BASE; // reset per-eval heap arena
}
// Run a compiled function. Returns 0 on success.
// First call uses dbt_run (zeroes ctx); subsequent use dbt_resume.
//
int run(uint64_t entry_pc) {
// PC=0 is the reserved "no entry" sentinel -- the code heap starts
// at PVM_CODE_START precisely to keep 0 free. install_code() and
// compile() both return it on failure, so running it would execute
// whatever happens to sit at guest address 0 (#2124).
//
if (0 == entry_pc) {
return -1;
}
// Track that compiled code from this arena is on the stack so a
// re-entrant compile_attr() does not reclaim the arena out from under
// a running program. A reclaim requested mid-run is serviced once the
// outermost run() unwinds (exec_depth back to 0).
exec_depth++;
dbt_rerun(&dbt, poc_ecall, &dbt);
int rc;
if (run_count == 0) {
run_count++;
rc = dbt_run(&dbt, entry_pc, rv_compiler::STACK_TOP);
} else {
dbt.ctx.x[2] = rv_compiler::STACK_TOP;
run_count++;
rc = dbt_resume(&dbt, entry_pc);
}
exec_depth--;
if (0 == exec_depth) {
if (reset_pending) {
// Guest arena pressure (or a deferred request) — full reclaim,
// which also clears the DBT translation buffer.
reset_arena();
} else if (dbt_buffer_nearly_full()) {
// The DBT's x86 translation buffer filled at RUN time: new
// guest blocks (data-dependent paths through already-compiled
// attributes) keep translating even when no new compilation
// happens, so the compile-time arena check never sees it. Once
// code_used pins at full, translate_block bails and those paths
// silently degrade to the interpreter forever. Reclaim just the
// translation buffer (the guest arena + attr cache stay valid;
// blocks re-translate lazily) at this safe (exec_depth == 0)
// point.
reset_dbt_buffer();
}
}
return rc;
}
// True when the DBT's x86-64 translation buffer (separate from the guest
// RV64 arena pools) has crossed a 7/8 high-water mark and holds program
// blocks worth reclaiming. Guarded so we never thrash: there must be
// translation above the permanent blob region, and a reset (which rewinds
// to blob_code_end) must leave real headroom.
bool dbt_buffer_nearly_full() const {
if (!dbt_ready) return false;
uint64_t used = dbt.code_used;
uint64_t blob = dbt.blob_code_end;
uint64_t cap = CODE_BUF_SIZE;
return used * 8 >= cap * 7
&& used > blob
&& blob * 8 < cap * 7;
}
// Reclaim ONLY the DBT translation buffer — clears the x86 block cache and
// rewinds code_used to blob_code_end (blob translations preserved), leaving
// the guest code arena and attr cache intact. The guest programs stay
// valid at their PCs and re-translate lazily on next run. Safe ONLY at
// exec_depth == 0 (no translated block is on the call stack).
void reset_dbt_buffer() {
if (!dbt_ready) return;
dbt_reset(&dbt, memory.data(), memory.size(), poc_ecall, &dbt);
run_count = 0;
dbt_buffer_resets++;
}
// True when any arena pool has crossed a 7/8 high-water mark and should be
// reclaimed before it exhausts.
bool arena_nearly_full() const {
uint64_t code_used = code_heap_next - PVM_CODE_START;
uint64_t code_cap = PVM_CODE_LIMIT - PVM_CODE_START;
uint64_t str_used = str_pool_next - PVM_STR_BASE;
uint64_t str_cap = PVM_STR_LIMIT - PVM_STR_BASE;
uint64_t fargs_used = fargs_pool_next - PVM_FARGS_BASE;
uint64_t fargs_cap = PVM_FARGS_LIMIT - PVM_FARGS_BASE;
return code_used * 8 >= code_cap * 7
|| str_used * 8 >= str_cap * 7
|| fargs_used * 8 >= fargs_cap * 7;
}
// Reclaim the code arena wholesale. Safe ONLY when exec_depth == 0 (no
// compiled code from this arena is executing): it rewinds the bump
// allocators and drops every cached compilation, so all attributes are
// recompiled lazily on next use. The reused code-heap PCs would otherwise
// hit stale entries in the DBT block cache, so dbt_reset() evicts the
// program-code translations (blob translations are preserved) and the
// run sequence restarts from dbt_run.
void reset_arena() {
attr_cache.clear();
code_heap_next = PVM_CODE_START;
str_pool_next = PVM_STR_BASE;
fargs_pool_next = PVM_FARGS_BASE;
worst_out_pool = rv_compiler::STACK_TOP - 8;
reset_pending = false;
if (dbt_ready) {
dbt_reset(&dbt, memory.data(), memory.size(), poc_ecall, &dbt);
run_count = 0;
}
}
// Reset blob BSS between function runs (Tier 2 writable data).
//
void reset_blob_bss() {
if (s_tier2.loaded && s_tier2.bss_size > 0) {
uint64_t bss_start = rv_compiler::BLOB_BASE
+ s_tier2.code_size;
if (bss_start + s_tier2.bss_size <= memory.size()) {
memset(memory.data() + bss_start, 0, s_tier2.bss_size);
}
}
}
// Read a NUL-terminated result string from guest memory.
//
const char *result(uint64_t out_addr,
uint64_t entry_sp = rv_compiler::STACK_TOP) const {
uint64_t resolved = resolve_runtime_out_addr(out_addr, entry_sp);
if (resolved == 0 || resolved >= memory.size()) return "";
return reinterpret_cast<const char *>(memory.data() + resolved);
}
private:
// Install a compiled program's regions into persistent memory
// and advance pool cursors.
//
void install(const compiled_program &prog) {
// Code.
memcpy(memory.data() + prog.entry_pc,
prog.memory.data() + prog.entry_pc, prog.code_size);
// Strings.
if (prog.str_pool_end > str_pool_next) {
memcpy(memory.data() + str_pool_next,
prog.memory.data() + str_pool_next,
prog.str_pool_end - str_pool_next);
}
// Fargs.
if (prog.fargs_pool_end > fargs_pool_next) {
memcpy(memory.data() + fargs_pool_next,
prog.memory.data() + fargs_pool_next,
prog.fargs_pool_end - fargs_pool_next);
}
// Output is stack-allocated at runtime via the prologue;
// no install-time copy needed. Track worst-case for clearing.
if (prog.out_pool_end < worst_out_pool) {
worst_out_pool = prog.out_pool_end;
}
// Advance code and data cursors only.
code_heap_next = prog.entry_pc + prog.code_size;
code_heap_next = (code_heap_next + 15) & ~15ULL;
str_pool_next = prog.str_pool_end;
fargs_pool_next = prog.fargs_pool_end;
}
};
// ---------------------------------------------------------------
// Compile cache — LRU cache of compiled programs.
//
// Keyed by expression text. Cache hits skip compilation entirely.
// Combined with DBT block cache persistence (dbt_rerun), repeated
// evaluation of the same expression does zero compilation and zero
// JIT translation — just runs the cached native code.
//
// The DBT tracks which program it was last set up for. Same
// program → dbt_rerun (keep translated blocks). Different
// program → dbt_reset (re-translate).
// ---------------------------------------------------------------
struct compile_cache_entry {
compiled_program prog;
std::list<std::string>::iterator lru_it;
};
static std::unordered_map<std::string, compile_cache_entry> s_compile_cache;
static std::list<std::string> s_compile_lru;
// Capacity is a config knob (jit_compile_cache_max), read at insert time so
// runtime @admin takes effect on the next compile. The old fixed 256 was a
// cliff, not a slope (#2130): round-robin over an LRU is its worst case, so
// ONE expression past capacity took the hit rate from 99.9% to 0.16% and
// every lookup became a SQLite fetch + full program deserialization
// (~19.4us on the issue's box). The default covers the ~1500 distinct
// programs a live-workload profile actually showed (#2129 measurement) —
// entries are compact blobs, typically a few KB each.
//
static constexpr int COMPILE_CACHE_DEFAULT = 2048;
static constexpr size_t COMPILE_CACHE_MIN_LEN = 8;
static size_t compile_cache_max(void) {
int n = mudconf.jit_compile_cache_max;
if (n < 8) {
n = 8;
} else if (n > 65536) {
n = 65536;
}
return static_cast<size_t>(n);
}
// Decline memo (#2130): bail_noop is a pure function of the compiled code,
// so once a shape has been declined there is no reason to ever fetch or
// reconstruct its program again just to look at four integers and not run
// it. Keyed by the same compile-cache key; checked in jit_eval BEFORE
// compile_cached, so a memoized decline touches neither the memory LRU nor
// SQLite. Only dep-free programs are memoized — a program with inline
// deps can recompile into a different shape when an attribute changes.
//
// Bounded by wholesale clear rather than an LRU: an entry is ~the
// expression text, a false eviction costs exactly one fetch+decline to
// re-memoize, and the flush hook clears it with the other caches.
//
static std::unordered_set<std::string> s_decline_memo;
static constexpr size_t DECLINE_MEMO_MAX = 8192;
// Drop memoized declines when the function table changes (#2140 review).
// A shape that was bail_noop can recompile into a different shape once a
// softcode @function or builtin registration moves; leaving the memo
// would refuse forever without re-fetching. Same invalidation trigger as
// the #2068 gate epoch.
//
void jit_decline_memo_invalidate(void)
{
s_decline_memo.clear();
}
// Track which program the DBT was last set up for, so we can
// use dbt_rerun (fast) instead of dbt_reset (slow) on cache hits.
//
static uint64_t s_next_program_id = 1;
static void runtime_buffer_init(jit_run_vm *vm) {
if (vm->buffer_ready) return;
vm->buffer.resize(rv_compiler::MEM_SIZE);
tier2_install(vm->buffer, rv_compiler::BLOB_BASE);
vm->buffer_ready = true;
}
// Compact a compiled_program: extract the occupied regions into
// small blob vectors, extract the folded result if constant-folded,
// assign a unique program_id, then release the 4MB memory.
//
static void compact_program(compiled_program &prog) {
prog.program_id = s_next_program_id++;
// Extract code blob.
if (prog.code_size > 0 && prog.entry_pc + prog.code_size <= prog.memory.size()) {
prog.code_blob.assign(
prog.memory.data() + prog.entry_pc,
prog.memory.data() + prog.entry_pc + prog.code_size);
}
// Extract string pool blob.
if (prog.str_pool_end > rv_compiler::STR_BASE) {
size_t len = static_cast<size_t>(prog.str_pool_end - rv_compiler::STR_BASE);
prog.str_blob.assign(
prog.memory.data() + rv_compiler::STR_BASE,
prog.memory.data() + rv_compiler::STR_BASE + len);
}
// Extract fargs pool blob.
if (prog.fargs_pool_end > rv_compiler::FARGS_BASE) {
size_t len = static_cast<size_t>(prog.fargs_pool_end - rv_compiler::FARGS_BASE);
prog.fargs_blob.assign(
prog.memory.data() + rv_compiler::FARGS_BASE,
prog.memory.data() + rv_compiler::FARGS_BASE + len);
}
// Extract folded result for constant-folded programs.
if (!prog.needs_jit) {
uint64_t out_addr = rv_compiler::resolve_output_addr(
prog.out_addr, rv_compiler::STACK_TOP);
if (out_addr < prog.memory.size()) {
prog.folded_result = reinterpret_cast<const char *>(
prog.memory.data() + out_addr);
}
}
// Release the 4MB vector.
prog.memory.clear();
prog.memory.shrink_to_fit();
}
// Materialize a compact program's DATA — string pool and fargs blobs — at
// their canonical addresses. Data placement is independent of which code
// slot the program occupies: compiled code references the pools absolutely,
// the pools never move, and swapping their CONTENT does not invalidate any
// translation (only code bytes matter at translate time).
//
static void materialize_data(jit_run_vm *vm, const compiled_program &prog) {
runtime_buffer_init(vm);
// Record which program's data now occupies the pools so a consecutive
// re-run of the same program can skip this copy (see run_cached_program).
vm->buffer_program_id = prog.program_id;
if (!prog.str_blob.empty()) {
memcpy(vm->buffer.data() + rv_compiler::STR_BASE,
prog.str_blob.data(), prog.str_blob.size());
}
if (!prog.fargs_blob.empty()) {
memcpy(vm->buffer.data() + rv_compiler::FARGS_BASE,
prog.fargs_blob.data(), prog.fargs_blob.size());
}
}
// Materialize a compact program's CODE at `slot_base` (#2129), re-aiming
// its blob-call JALs by the placement delta. Classifies the program on
// first use; a PINNED program only ever materializes at the canonical base.
// Returns false if the program cannot be placed at this base.
//
static void classify_reloc(compiled_program &prog) {
if (RV_RELOC_UNSCANNED != prog.reloc_class) {
return;
}
prog.reloc_class = rv_scan_extern_jals(
prog.code_blob.data(), prog.code_blob.size(),
prog.entry_pc, rv_compiler::CODE_LIMIT,
rv_compiler::BLOB_BASE, rv_compiler::BLOB_LIMIT,
prog.extern_jals);
if (RV_RELOC_OK != prog.reloc_class) {
s_jit_stats.slot_pinned++;
}
}
static bool materialize_code_slot(jit_run_vm *vm, compiled_program &prog,
uint64_t slot_base) {
runtime_buffer_init(vm);
classify_reloc(prog);
const int64_t delta = static_cast<int64_t>(slot_base)
- static_cast<int64_t>(rv_compiler::CODE_BASE);
if (0 != delta && RV_RELOC_OK != prog.reloc_class) {
return false;
}
if (prog.code_blob.empty()) {
return true;
}
uint8_t *dst = vm->buffer.data() + slot_base + prog.entry_pc;
memcpy(dst, prog.code_blob.data(), prog.code_blob.size());
if (0 == delta) {
return true;
}
for (uint32_t off : prog.extern_jals) {
if (off + 4 > prog.code_blob.size()) {
return false; // corrupt metadata; caller pins
}
uint32_t w;
memcpy(&w, dst + off, 4);
if (!rv_jal_relocate(&w, delta)) {
return false; // displacement not encodable at this base
}
memcpy(dst + off, &w, 4);
}
return true;
}
// Materialize a compact program into the shared runtime buffer at its
// canonical addresses — code and data. Used by the fold-result reader
// (s_fold_vm), which only needs somewhere to read a string back out of.
//
static void materialize_program(jit_run_vm *vm, const compiled_program &prog) {
runtime_buffer_init(vm);
materialize_data(vm, prog);
if (!prog.code_blob.empty()) {
memcpy(vm->buffer.data() + prog.entry_pc,
prog.code_blob.data(), prog.code_blob.size());
}
}
// How many misses between hot-bit sweeps. A protected slot whose owner
// did not run within a full window becomes evictable at the sweep; a
// program hitting at least once per window keeps its slot indefinitely.
// Counted in misses, not runs, so a stable all-hits workload never pays a
// sweep and never loses a slot.
//
static constexpr uint32_t JIT_SLOT_SWEEP_PERIOD = 64;
// Pick (or keep) a guest code slot for `prog` (#2129, admission #2130).
//
// A hit means the program's code is already materialized at *slot_base and
// every translation the DBT made for it is still keyed under those PCs —
// the program switch costs a str/fargs memcpy instead of a re-translation.
//
// A miss claims a victim in strict preference order:
//
// 1. a FREE protected slot (1..n-1) — a small working set gets protection
// immediately, no probation served;
// 2. the free probation slot (0);
// 3. the coldest protected slot that is EVICTABLE — its hot bit not set
// since the last sweep, i.e. its owner has gone idle;
// 4. slot 0, the probation lane, unconditionally.
//
// Rule 4 is what removes the cliff Kagura measured at working sets one
// past the slot count: a first-touch program cannot displace a hot
// resident, so a round-robin storm over any number of cold programs
// churns only slot 0 (the old per-switch cost) while the hot set keeps
// its translations. PINNED programs (relocation scan refused) contend
// for slot 0 only — the pre-#2129 behaviour for exactly those programs.
// jit_code_slots=1 collapses everything onto slot 0, the A/B lever.
//
static bool pick_code_slot(jit_run_vm *vm, dbt_state_t *dbt,
compiled_program &prog, uint64_t *slot_base) {
classify_reloc(prog);
int nslots = mudconf.jit_code_slots;
if (nslots < 1) {
nslots = 1;
} else if (nslots > JIT_CODE_SLOTS) {
nslots = JIT_CODE_SLOTS;
}
if (RV_RELOC_OK != prog.reloc_class) {
nslots = 1;
}
for (int i = 0; i < nslots; i++) {
if (vm->slots[i].program_id == prog.program_id) {
vm->slots[i].stamp = ++vm->slot_stamp;
vm->slots[i].hot = true;
s_jit_stats.slot_hit++;
*slot_base = JIT_SLOT_BASE[i];
return true;
}
}
// Sweep: age the protection. Runs on miss traffic only — misses are
// when eviction decisions happen, and a workload with no misses has
// no reason to decay anything.
if (++vm->misses_since_sweep >= JIT_SLOT_SWEEP_PERIOD) {
vm->misses_since_sweep = 0;
for (int i = 1; i < nslots; i++) {
vm->slots[i].hot = false;
}
}
int victim = -1;
for (int i = 1; i < nslots; i++) { // 1: free protected
if (0 == vm->slots[i].program_id) {
victim = i;
break;
}
}
if (victim < 0 && 0 == vm->slots[0].program_id) {
victim = 0; // 2: free probation
}
if (victim < 0) {
for (int i = 1; i < nslots; i++) { // 3: coldest evictable
if (!vm->slots[i].hot
&& (victim < 0
|| vm->slots[i].stamp < vm->slots[victim].stamp)) {
victim = i;
}
}
}
if (victim < 0) {
victim = 0; // 4: probation lane
}
if (0 != vm->slots[victim].program_id) {
if (0 == victim) {
s_jit_stats.slot_churn0++;
} else {
s_jit_stats.slot_evict++;
}
}
// Stale translations at this slot's PCs must be gone before new code
// claims them — this is the range-scoped version of what dbt_reset did
// for the whole program region on every switch.
dbt_invalidate_guest_range(dbt, JIT_SLOT_BASE[victim],
JIT_SLOT_BASE[victim] + rv_compiler::CODE_LIMIT);
vm->slots[victim] = jit_run_vm::code_slot{};
if (!materialize_code_slot(vm, prog, JIT_SLOT_BASE[victim])) {
// Placement refused (relocation not encodable). Retry once at the
// canonical base, where no relocation is needed.
if (0 == victim) {
return false;
}
if (0 != vm->slots[0].program_id) {
s_jit_stats.slot_churn0++;
}
dbt_invalidate_guest_range(dbt, JIT_SLOT_BASE[0],
JIT_SLOT_BASE[0] + rv_compiler::CODE_LIMIT);
vm->slots[0] = jit_run_vm::code_slot{};
if (!materialize_code_slot(vm, prog, JIT_SLOT_BASE[0])) {
return false;
}
victim = 0;
}
vm->slots[victim].program_id = prog.program_id;
vm->slots[victim].stamp = ++vm->slot_stamp;
vm->slots[victim].hot = true;
s_jit_stats.slot_miss++;
*slot_base = JIT_SLOT_BASE[victim];
return true;
}
// Reconstruct a compiled_program from a SQLite code cache record.
// Populates compact blob vectors directly — no full 4MB allocation.
//
static compiled_program reconstruct_from_cache(
const CSQLiteDB::CodeCacheRecord &rec) {
compiled_program prog;
const bool has_compact_image =
rec.code_len > 0
|| rec.str_len > 0
|| rec.fargs_len > 0
|| rec.code_size > 0
|| rec.out_pool_end > 0;
if (has_compact_image) {
const int64_t code_end = rec.entry_pc + rec.code_len;
const int64_t str_end = static_cast<int64_t>(rv_compiler::STR_BASE)
+ rec.str_len;
const int64_t fargs_end = static_cast<int64_t>(rv_compiler::FARGS_BASE)
+ rec.fargs_len;
const bool valid_code =
rec.code_len >= 0
&& rec.code_size >= 0
&& rec.entry_pc >= static_cast<int64_t>(rv_compiler::CODE_BASE)
&& rec.entry_pc <= static_cast<int64_t>(rv_compiler::CODE_LIMIT)
&& code_end >= rec.entry_pc
&& code_end <= static_cast<int64_t>(rv_compiler::CODE_LIMIT)
&& rec.code_size >= rec.code_len;
const bool valid_str =
rec.str_len >= 0
&& rec.str_pool_end >= static_cast<int64_t>(rv_compiler::STR_BASE)
&& rec.str_pool_end <= static_cast<int64_t>(rv_compiler::STR_LIMIT)
&& str_end >= static_cast<int64_t>(rv_compiler::STR_BASE)
&& str_end <= static_cast<int64_t>(rv_compiler::STR_LIMIT)
&& rec.str_pool_end >= str_end;
const bool valid_fargs =
rec.fargs_len >= 0
&& rec.fargs_pool_end >= static_cast<int64_t>(rv_compiler::FARGS_BASE)
&& rec.fargs_pool_end <= static_cast<int64_t>(rv_compiler::FARGS_LIMIT)
&& fargs_end >= static_cast<int64_t>(rv_compiler::FARGS_BASE)
&& fargs_end <= static_cast<int64_t>(rv_compiler::FARGS_LIMIT)
&& rec.fargs_pool_end >= fargs_end;
const bool valid_out =
rec.out_pool_end >= static_cast<int64_t>(rv_compiler::OUT_STACK_LIMIT)
&& rec.out_pool_end <= static_cast<int64_t>(rv_compiler::STACK_TOP - 8);
if (!valid_code || !valid_str || !valid_fargs || !valid_out) {
return prog;
}
// Populate compact blob vectors directly from the SQLite record.
if (rec.code_blob && rec.code_len > 0) {
prog.code_blob.assign(
static_cast<const uint8_t *>(rec.code_blob),
static_cast<const uint8_t *>(rec.code_blob) + rec.code_len);
}
if (rec.str_blob && rec.str_len > 0) {
prog.str_blob.assign(
static_cast<const uint8_t *>(rec.str_blob),
static_cast<const uint8_t *>(rec.str_blob) + rec.str_len);
}
if (rec.fargs_blob && rec.fargs_len > 0) {
prog.fargs_blob.assign(
static_cast<const uint8_t *>(rec.fargs_blob),
static_cast<const uint8_t *>(rec.fargs_blob) + rec.fargs_len);
}
} else if (rec.memory_blob && rec.memory_len > 0) {
// Legacy format: single memory blob covering code+str+fargs.
// Extract the occupied regions into compact vectors.
// Cap at the highest of the three pool ends (#2107: STR now sits
// above FARGS, so FARGS_LIMIT is no longer the image high water).
int copy_len = rec.memory_len;
const int layout_end = static_cast<int>(
(std::max)(rv_compiler::STR_LIMIT, rv_compiler::FARGS_LIMIT));
if (copy_len > layout_end) {
copy_len = layout_end;
}
const auto *base = static_cast<const uint8_t *>(rec.memory_blob);
// Code region: [CODE_BASE..CODE_LIMIT)
if (copy_len > static_cast<int>(rv_compiler::CODE_BASE)) {
int code_end = (std::min)(copy_len,
static_cast<int>(rv_compiler::CODE_LIMIT));
prog.code_blob.assign(base + rv_compiler::CODE_BASE,
base + code_end);
}
// String pool: [STR_BASE..STR_LIMIT)
if (copy_len > static_cast<int>(rv_compiler::STR_BASE)) {
int str_end = (std::min)(copy_len,
static_cast<int>(rv_compiler::STR_LIMIT));
prog.str_blob.assign(base + rv_compiler::STR_BASE,
base + str_end);
}
// Fargs pool: [FARGS_BASE..FARGS_LIMIT)
if (copy_len > static_cast<int>(rv_compiler::FARGS_BASE)) {
int fargs_end = (std::min)(copy_len,
static_cast<int>(rv_compiler::FARGS_LIMIT));
prog.fargs_blob.assign(base + rv_compiler::FARGS_BASE,
base + fargs_end);
}
}
prog.memory_size = rv_compiler::MEM_SIZE;
prog.out_addr = static_cast<uint64_t>(rec.out_addr);
prog.out_used = 0;
prog.entry_pc = has_compact_image
? static_cast<uint64_t>(rec.entry_pc)
: rv_compiler::CODE_BASE;
prog.code_size = has_compact_image
? static_cast<uint64_t>(rec.code_size)
: 0;
prog.str_pool_end = has_compact_image && rec.str_pool_end > 0
? static_cast<uint64_t>(rec.str_pool_end)
: rv_compiler::STR_BASE;
prog.fargs_pool_end = has_compact_image && rec.fargs_pool_end > 0
? static_cast<uint64_t>(rec.fargs_pool_end)
: rv_compiler::FARGS_BASE;
if (has_compact_image && rec.out_pool_end > 0) {
prog.out_pool_end = static_cast<uint64_t>(rec.out_pool_end);
} else {
prog.out_pool_end = rec.needs_jit
? rv_compiler::OUT_STACK_LIMIT
: rv_compiler::STACK_TOP - 8;
}
prog.ok = true;
prog.needs_jit = rec.needs_jit != 0;
prog.folds = rec.folds;
prog.ecalls = rec.ecalls;
prog.tier2_calls = rec.tier2_calls;
prog.native_ops = rec.native_ops;
prog.max_func_depth = static_cast<int>(rec.max_func_depth);
prog.n_func_calls = static_cast<int>(rec.n_func_calls);
// Restore inline dependencies from BLOB.
if (rec.deps_blob && rec.deps_len > 0)
{
int ndeps = rec.deps_len / static_cast<int>(
sizeof(compiled_program::inline_dep));
prog.deps.resize(ndeps);
memcpy(prog.deps.data(), rec.deps_blob,
ndeps * sizeof(compiled_program::inline_dep));
}
// Extract folded result for constant-folded programs.
// Materialize blobs into the runtime buffer to read the result string.
//
// A folded result always lives in the string pool — hir_codegen sets
// final_out via pool_str() (or an interned str-pool address) precisely
// so it survives SQLite cache persistence. Validate the resolved
// address falls inside the materialized string pool and bound the NUL
// scan to that region: out_addr comes straight from the cache record,
// so a malformed/corrupt row could otherwise point it into the high
// runtime buffer (e.g. the non-NUL DSCRATCH doubles area) and walk the
// string copy's strlen past the end of the 4 MB buffer (OOB read).
if (!prog.needs_jit && prog.ok) {
// A dedicated context, never one of the run contexts (#1326).
//
// This path only needs somewhere to read a folded string back out of,
// but it is reachable *while runs are live*: compile_cached() is called
// from jit_eval(), jit_eval() from the AST evaluator, and the AST
// evaluator from ECALLs inside a running program. Materializing into a
// context a live run owns overwrites the code and string pool that run
// is executing out of, and the symptom is not a wrong answer -- the
// corrupted guest code loops inside a single translated block, so
// neither max_dispatch nor the ECALL path nor the alarm ever notices.
// It just stops.
//
// Picking "the first context not in use" does not work either: at the
// depth where runs are refused there is no free context to pick, and
// clamping lands on one that is live. So this owns its own, allocated
// on first use like the others.
static jit_run_vm s_fold_vm;
jit_run_vm *vm = &s_fold_vm;
runtime_buffer_init(vm);
materialize_program(vm, prog);
uint64_t out_addr = rv_compiler::resolve_output_addr(
prog.out_addr, rv_compiler::STACK_TOP);
if (out_addr < static_cast<uint64_t>(rv_compiler::STR_BASE)
|| out_addr >= prog.str_pool_end
|| prog.str_pool_end > vm->buffer.size()) {
// Folded out_addr outside the materialized string pool — the
// record is corrupt; reject it so the caller recompiles.
prog.ok = false;
return prog;
}
const char *p = reinterpret_cast<const char *>(
vm->buffer.data() + out_addr);
size_t maxlen = static_cast<size_t>(prog.str_pool_end - out_addr);
const void *nul = memchr(p, '\0', maxlen);
size_t n = nul ? static_cast<size_t>(
static_cast<const char *>(nul) - p) : maxlen;
prog.folded_result.assign(p, n);
}
// Assign a program ID.
prog.program_id = s_next_program_id++;
return prog;
}
static std::string compile_cache_key(const UTF8 *expr, size_t nLen, int eval) {
// Include every eval flag that changes compile-time semantics.
int eval_key = eval & (EV_FCHECK | EV_FMAND | EV_STRIP_CURLY);
std::string key(reinterpret_cast<const char *>(expr), nLen);
key += '\0';
key += static_cast<char>(eval_key & 0xFF);
key += static_cast<char>((eval_key >> 8) & 0xFF);
return key;
}
// Persist a compiled_program to the SQLite code cache.
//
// Queued for batched execution via cache_flush_writes() — the same
// demand-driven mechanism used for attribute writes. This avoids
// leaving a transaction open across arbitrary game operations.
//
static void store_to_sqlite_cache(const std::string &key,
const compiled_program &prog) {
if (!g_pSQLiteBackend) return;
int persist_len = 0;
const char *legacy_blob = "";
const void *code_blob = nullptr;
int code_len = static_cast<int>(prog.code_size);
if (code_len > 0) {
code_blob = prog.memory.data() + prog.entry_pc;
}
const void *str_blob = nullptr;
int str_len = 0;
if (prog.str_pool_end > rv_compiler::STR_BASE) {
str_blob = prog.memory.data() + rv_compiler::STR_BASE;
str_len = static_cast<int>(prog.str_pool_end - rv_compiler::STR_BASE);
}
const void *fargs_blob = nullptr;
int fargs_len = 0;
if (prog.fargs_pool_end > rv_compiler::FARGS_BASE) {
fargs_blob = prog.memory.data() + rv_compiler::FARGS_BASE;
fargs_len = static_cast<int>(prog.fargs_pool_end - rv_compiler::FARGS_BASE);
}
cache_queue_code_cache_put(
key.data(), static_cast<int>(key.size()),
s_blob_version.data(), static_cast<int>(s_blob_version.size()),
legacy_blob, persist_len,
code_blob, code_len,
static_cast<int64_t>(prog.entry_pc),
static_cast<int64_t>(prog.code_size),
str_blob, str_len,
static_cast<int64_t>(prog.str_pool_end),
fargs_blob, fargs_len,
static_cast<int64_t>(prog.fargs_pool_end),
static_cast<int64_t>(prog.out_pool_end),
static_cast<int64_t>(prog.out_addr),
prog.needs_jit ? 1 : 0,
prog.folds, prog.ecalls,
prog.tier2_calls, prog.native_ops,
static_cast<int64_t>(prog.max_func_depth),
static_cast<int64_t>(prog.n_func_calls),
prog.deps.data(),
static_cast<int>(prog.deps.size()
* sizeof(compiled_program::inline_dep)));
}
// Check if a compiled program's inlined dependencies are still fresh.
// Returns true if all deps match their current mod_counts (or no deps).
//
static bool deps_are_fresh(const compiled_program &prog) {
for (const auto &dep : prog.deps) {
uint32_t current = attr_mod_count_get(
static_cast<dbref>(dep.obj), dep.attr_num);
if (current != dep.mod_count) {
return false;
}
}
return true;
}
// ---------------------------------------------------------------
// Public wrappers for SQLite code cache (shared with Lua JIT).
// ---------------------------------------------------------------
std::string jit_sha1_hex(const void *data, size_t len) {
const void *parts[] = { data };
size_t sizes[] = { len };
return sha1_hex_parts(parts, sizes, 1);
}
void jit_store_to_sqlite(const std::string &key, const compiled_program &prog) {
store_to_sqlite_cache(key, prog);
}
void jit_compact_program(compiled_program &prog) {
compact_program(prog);
}
bool jit_load_from_sqlite(const std::string &key, compiled_program &out) {
if (!g_pSQLiteBackend) return false;
CSQLiteDB &db = g_pSQLiteBackend->GetDB();
CSQLiteDB::CodeCacheRecord rec;
if (!db.CodeCacheGet(key.data(), static_cast<int>(key.size()),
s_blob_version.data(),
static_cast<int>(s_blob_version.size()), rec)) {
return false;
}
out = reconstruct_from_cache(rec);
return out.ok;
}
// Marshal %q register slots from mudstate.global_regs into guest SUBST
// memory. `subst_mask` selects which registers to copy (bit
// SUBST_QREG0+i, matching compiled_program::subst_mask); pass
// ~UINT64_C(0) to copy all of them. This is the single authority for
// q-register slot population: program entry uses it today, and the
// scope-restore / post-ECALL resync points reuse it so the slots can
// never drift from global_regs at a sync boundary
// (docs/plan-jit-evalbracket-lift.md, Phases 2-3).
//
// The subst_mask bits covering the %q register slots
// (SUBST_QREG0 .. SUBST_QREG0 + MAX_GLOBAL_REGS - 1).
static constexpr uint64_t QREG_SLOT_BITS =
((MAX_GLOBAL_REGS < 64
? (UINT64_C(1) << MAX_GLOBAL_REGS) - 1
: ~UINT64_C(0))) << rv_compiler::SUBST_QREG0;
// Also maintains the long-register bitmap (#996 step 2): the whole
// QREG_LONGBITS u64 is rewritten on EVERY call — computed bits for
// masked registers, zero elsewhere. The runtime buffer is reused
// across programs, so a partial update would leak a previous program's
// stale bits; the whole-word write makes freshness true by
// construction. A truncated slot no longer declines the run: %q reads
// branch on the bit and fetch long values via the fun_r ECALL.
static void marshal_qregs_to_slots(uint8_t *mem, uint64_t subst_mask)
{
uint64_t longbits = 0;
for (int i = 0; i < MAX_GLOBAL_REGS; i++) {
const int slot_idx = rv_compiler::SUBST_QREG0 + i;
if (!((subst_mask >> slot_idx) & UINT64_C(1))) {
continue;
}
const uint64_t slot = rv_compiler::SUBST_BASE
+ static_cast<uint64_t>(slot_idx) * rv_compiler::SUBST_SLOT;
const UTF8 *value = nullptr;
if (mudstate.global_regs[i] && mudstate.global_regs[i]->reg_ptr) {
value = mudstate.global_regs[i]->reg_ptr;
}
if (value && value[0]) {
size_t len = strlen(reinterpret_cast<const char *>(value));
if (len >= static_cast<size_t>(rv_compiler::SUBST_SLOT)) {
len = rv_compiler::SUBST_SLOT - 1;
longbits |= UINT64_C(1) << i;
}
memcpy(mem + slot, value, len);
mem[slot + len] = 0;
} else {
mem[slot] = 0;
}
}
memcpy(mem + rv_compiler::QREG_LONGBITS, &longbits, sizeof(longbits));
}
// Set/clear one register's long bit after an ECALL_SETQ write (#996).
static void qreg_longbit_update(eval_ctx *ec, int regnum, size_t vlen)
{
if (rv_compiler::QREG_LONGBITS + sizeof(uint64_t) > ec->memory_size) {
return;
}
uint64_t bits;
memcpy(&bits, ec->memory + rv_compiler::QREG_LONGBITS, sizeof(bits));
const uint64_t bit = UINT64_C(1) << regnum;
if (vlen >= static_cast<size_t>(rv_compiler::SUBST_SLOT)) {
bits |= bit;
} else {
bits &= ~bit;
}
memcpy(ec->memory + rv_compiler::QREG_LONGBITS, &bits, sizeof(bits));
}
// Look up or compile an expression. Returns a pointer to the
// cached compiled_program (owned by the cache — do not free).
// Returns nullptr on compilation failure.
//
// Perform a jitstats(flush) that had to wait for the JIT to become quiet.
// Defined below with s_run_cached_depth; called here because this is the
// only place a caller obtains a compiled_program * for the softcode route,
// so it is the last moment at which clearing the cache frees nothing that
// is already spoken for (#1316).
//
static void jit_drain_pending_flush(void);
static compiled_program *compile_cached(const UTF8 *expr, size_t nLen,
int eval = EV_FCHECK | EV_EVAL) {
jit_drain_pending_flush();
std::string key = compile_cache_key(expr, nLen, eval);
auto it = s_compile_cache.find(key);
if (it != s_compile_cache.end()) {
// Staleness check: if this entry has inlined deps and any
// attr has been modified since compilation, evict and recompile.
if (!it->second.prog.deps.empty()
&& !deps_are_fresh(it->second.prog))
{
release_program_slots(it->second.prog.program_id);
s_compile_lru.erase(it->second.lru_it);
s_compile_cache.erase(it);
// Fall through to recompile below.
}
else
{
// Memory cache hit — move to front of LRU.
s_compile_lru.splice(s_compile_lru.begin(), s_compile_lru,
it->second.lru_it);
s_jit_stats.cache_hit_mem++;
return &it->second.prog;
}
}
// Memory cache miss — check SQLite persistent cache.
compiled_program prog;
bool from_sqlite = false;
if (g_pSQLiteBackend && nLen >= COMPILE_CACHE_MIN_LEN) {
CSQLiteDB &db = g_pSQLiteBackend->GetDB();
CSQLiteDB::CodeCacheRecord rec;
if (db.CodeCacheGet(key.data(), static_cast<int>(key.size()),
s_blob_version.data(),
static_cast<int>(s_blob_version.size()), rec)) {
prog = reconstruct_from_cache(rec);
db.CodeCacheReset();
// Check staleness of SQLite-cached entry too.
if (!prog.deps.empty() && !deps_are_fresh(prog))
{
// Stale — discard and recompile.
prog.ok = false;
}
else
{
from_sqlite = true;
s_jit_stats.cache_hit_sqlite++;
}
}
}
if (!from_sqlite) {
// Full cache miss — compile from scratch.
s_jit_stats.cache_miss++;
prog = compile_expression(expr, nLen, eval);
if (!prog.ok) return nullptr;
// Persist to SQLite while prog.memory still exists.
if (nLen >= COMPILE_CACHE_MIN_LEN) {
store_to_sqlite_cache(key, prog);
}
// Compact: extract blobs, release 4MB memory.
compact_program(prog);
}
// Insert into memory LRU cache.
while (s_compile_cache.size() >= compile_cache_max()) {
auto &victim_key = s_compile_lru.back();
auto vit = s_compile_cache.find(victim_key);
if (vit != s_compile_cache.end()) {
// Evicting a program must release its slot residency in every
// context (#1326, adapted for #2129's slots).
release_program_slots(vit->second.prog.program_id);
}
s_compile_cache.erase(victim_key);
s_compile_lru.pop_back();
}
s_compile_lru.push_front(key);
auto [ins_it, _] = s_compile_cache.emplace(
key, compile_cache_entry{std::move(prog), s_compile_lru.begin()});
return &ins_it->second.prog;
}
// ---------------------------------------------------------------
// Shared code heap — persistent guest memory for re-entrant JIT.
//
// All compiled expressions deposit code, strings, and fargs into
// a single 4MB guest memory image. The blob is installed once.
// Code accumulates via bump allocation; the DBT block cache
// persists across all expressions.
//
// Pool layout within shared memory:
// 0x0004-0x0FFFF Code heap (64KB)
// 0x10000-0x3FFFF Blob (installed once)
// 0x40000-0x5FFFF String pool (128KB)
// 0x60000-0x67FFF Fargs pool (32KB)
// 0x68000+ CARGS/SUBST/DMA/output (per-execution)
// ---------------------------------------------------------------
struct shared_heap_t {
guest_memory_t memory;
bool ready;
uint64_t code_next; // next free code address
uint64_t str_next; // next free string pool address
uint64_t fargs_next; // next free fargs address
static constexpr uint64_t CODE_START = 0x0004; // avoid PC=0
static constexpr uint64_t CODE_LIMIT = rv_compiler::BLOB_BASE;
static constexpr uint64_t STR_START = 0x40000;
static constexpr uint64_t STR_LIMIT = 0x60000;
static constexpr uint64_t FARGS_START = 0x60000;
static constexpr uint64_t FARGS_LIMIT = 0x68000;
// Compile result — lightweight, references shared memory.
struct entry {
uint64_t entry_pc;
uint64_t out_addr;
bool needs_jit;
int ecalls;
int tier2_calls;
std::vector<compiled_program::inline_dep> deps;
};
// Expression cache.
std::unordered_map<std::string, entry> cache;
shared_heap_t()
: memory(rv_compiler::MEM_SIZE),
ready(false),
code_next(CODE_START),
str_next(STR_START),
fargs_next(FARGS_START) {}
// Initialize: install blob once.
bool init() {
if (ready) return true;
tier2_lazy_init();
if (!s_tier2.loaded) return false;
tier2_install(memory, rv_compiler::BLOB_BASE);
ready = true;
return true;
}
// Compile an expression into the shared heap.
// Returns {0,0,false} on failure.
entry compile(const UTF8 *expr, size_t nLen,
int eval = EV_FCHECK | EV_EVAL) {
if (!init()) return {0, 0, false, 0, 0};
// Bounds check: code heap must not overflow.
if (code_next >= CODE_LIMIT) return {0, 0, false, 0, 0};
compiled_program prog = compile_expression(
expr, nLen, eval,
code_next,
str_next, STR_LIMIT,
fargs_next, FARGS_LIMIT,
rv_compiler::STACK_TOP - 8);
if (!prog.ok) return {0, 0, false, 0, 0};
// Install compiled regions into shared memory.
//
// Code.
if (prog.entry_pc + prog.code_size > CODE_LIMIT) {
return {0, 0, false, 0, 0};
}
memcpy(memory.data() + prog.entry_pc,
prog.memory.data() + prog.entry_pc, prog.code_size);
// Strings (if any new strings were added).
if (prog.str_pool_end > str_next) {
memcpy(memory.data() + str_next,
prog.memory.data() + str_next,
prog.str_pool_end - str_next);
}
// Fargs (if any new fargs were added).
if (prog.fargs_pool_end > fargs_next) {
memcpy(memory.data() + fargs_next,
prog.memory.data() + fargs_next,
prog.fargs_pool_end - fargs_next);
}
// Advance cursors.
code_next = prog.entry_pc + prog.code_size;
code_next = (code_next + 15) & ~15ULL; // align
str_next = prog.str_pool_end;
fargs_next = prog.fargs_pool_end;
return {prog.entry_pc, prog.out_addr, prog.needs_jit,
prog.ecalls, prog.tier2_calls, std::move(prog.deps)};
}
// Look up or compile an expression. Returns nullptr on failure.
// Validates inline dependencies on cache hit — if any inlined
// attribute body has changed, the cached entry is stale and
// must be evicted. (Code heap space is leaked; the shared heap
// has no reclamation yet.)
const entry *lookup(const UTF8 *expr, size_t nLen, int eval) {
std::string key = compile_cache_key(expr, nLen, eval);
auto it = cache.find(key);
if (it != cache.end()) {
// Check dependency freshness.
bool fresh = true;
for (const auto &dep : it->second.deps) {
uint32_t current = attr_mod_count_get(
static_cast<dbref>(dep.obj), dep.attr_num);
if (current != dep.mod_count) {
fresh = false;
break;
}
}
if (fresh) return &it->second;
// Stale — evict and recompile.
cache.erase(it);
}
entry e = compile(expr, nLen, eval);
if (!e.entry_pc) return nullptr;
auto [ins, _] = cache.emplace(key, std::move(e));
return &ins->second;
}
// ---------------------------------------------------------------
// Evaluate an expression via the shared heap's own DBT.
//
// Compiles (or cache-hits), populates CARGS/SUBST, runs via
// dbt_run, extracts result. Returns true if handled.
//
// The shared heap DBT is independent of the outer expression's
// DBT, so this is safe to call from within an ECALL handler.
//
// It is NOT independent of *itself* (#1994). There is one dbt and
// one guest register context here, so a second eval() entered from a
// host ECALL of a suspended eval() reuses both. run_depth declines
// that case; see the guard in eval().
// ---------------------------------------------------------------
dbt_state_t dbt;
bool dbt_ready = false;
uint32_t run_count = 0;
int run_depth = 0; // >0 while a run is live (#1994)
bool eval(const UTF8 *expr, size_t nLen,
UTF8 *out, size_t out_size,
dbref executor, dbref caller, dbref enactor,
int eval_flags,
const UTF8 *cargs[], int ncargs,
int *out_ecalls = nullptr,
int *out_tier2 = nullptr,
bool *out_folded = nullptr) {
const entry *e = lookup(expr, nLen, eval_flags);
if (!e) return false;
// Copy fields we need after guest execution before dbt_run() —
// a re-entrant eval() during guest code can call lookup() and
// erase this entry if the same key goes stale (recursive
// self-modifying softcode), leaving e dangling (#1940). Stats
// were already snapshotted for #1938; out_addr is the remaining
// post-run read.
const int entry_ecalls = e->ecalls;
const int entry_tier2 = e->tier2_calls;
const bool entry_needs_jit = e->needs_jit;
const uint64_t entry_out_addr = e->out_addr;
// Constant-folded: result is in shared memory.
if (!entry_needs_jit) {
uint64_t out_addr = resolve_runtime_out_addr(
entry_out_addr, rv_compiler::STACK_TOP);
size_t n = 0;
if (!guest_strnlen(memory.data(), memory.size(), out_addr, &n)) {
return false;
}
if (n >= out_size) n = out_size - 1;
memcpy(out, memory.data() + out_addr, n);
out[n] = '\0';
if (out_ecalls) *out_ecalls = entry_ecalls;
if (out_tier2) *out_tier2 = entry_tier2;
if (out_folded) *out_folded = true;
return true;
}
// Runtime execution. Everything from here on touches state that
// is shared across every eval() on this heap: the DBT's register
// context and stack pointer, the writable blob region, and the
// per-eval heap arena. A nested eval() -- reached when this
// heap's own program ECALLs into u(), whose body evaluates
// another bracket -- would reset all three underneath the
// suspended outer run, which then resumes at a program counter
// that is not its own. The backend refuses to translate there,
// dbt_resume returns -1, and jit_eval hands the whole subtree
// back to mux_exec to redo through the AST. Since that subtree
// contains the next recursion level, each level costs two
// evaluations and terminating a runaway recursion becomes
// 2^depth (#1994).
//
// Declining up front costs nothing measurable: instrumented over
// a runaway recursion, every re-entrant run failed this way
// (511 of 511 at function_recursion_limit 12) and not one ever
// produced a result. The AST handles the nested bracket instead,
// exactly as it does for the #1002 depth watermark above -- one
// evaluation per level rather than two.
//
if (0 < run_depth) {
s_jit_stats.bail_shared_busy++;
return false;
}
struct run_depth_guard {
int &d;
explicit run_depth_guard(int &r) : d(r) { d++; }
~run_depth_guard() { d--; }
} depth_guard(run_depth);
// Initialize DBT on first use.
if (!dbt_ready) {
if (dbt_init(&dbt, memory.data(), memory.size(),
eval_ecall, nullptr) != 0) {
return false;
}
dbt_ready = true;
const char *md_env = getenv("TINYMUX_DBT_MAX_DISPATCH");
dbt.max_dispatch = md_env
? strtoull(md_env, nullptr, 0) : 10000000;
dbt.alarm_flag = &alarm_clock.alarmed; // wall-clock (#JIT-alarm)
pretranslate_tier2(&dbt);
dbt.blob_code_end = dbt.code_used;
}
// Reset writable blob state and the per-eval heap arena.
tier2_reset_writable(memory, rv_compiler::BLOB_BASE);
s_heap_next = rv_compiler::HEAP_BASE;
// NUL-sentinel output slots.
{
uint64_t addr = rv_compiler::STACK_TOP - 8
- rv_compiler::OUT_SLOT;
while (addr >= rv_compiler::OUT_STACK_LIMIT) {
memory[addr] = 0;
addr -= rv_compiler::OUT_SLOT;
}
}
// Populate CARGS. Slot is CARGS_SLOT bytes including the trailing
// NUL. Truncating a long carg would change softcode results vs the
// AST path (LBUF-sized), so decline instead and let the AST evaluator
// handle it — the nested-eval twin of run_cached_program's fix (#1055).
// Nothing has run yet (memory is reset each eval), so an early return
// here is a clean bail.
for (int i = 0; i < rv_compiler::MAX_CARGS; i++) {
uint64_t slot = rv_compiler::CARGS_BASE
+ static_cast<uint64_t>(i) * rv_compiler::CARGS_SLOT;
if (i < ncargs && cargs && cargs[i]) {
size_t len = strlen(
reinterpret_cast<const char *>(cargs[i]));
if (len >= static_cast<size_t>(rv_compiler::CARGS_SLOT)) {
return false;
}
memcpy(memory.data() + slot, cargs[i], len);
memory[slot + len] = 0;
} else {
memory[slot] = 0;
}
}
// Populate SUBST slots.
auto copy_subst = [&](int slot_idx, const UTF8 *value) {
uint64_t slot = rv_compiler::SUBST_BASE
+ static_cast<uint64_t>(slot_idx) * rv_compiler::SUBST_SLOT;
if (value && value[0]) {
size_t len = strlen(
reinterpret_cast<const char *>(value));
if (len >= static_cast<size_t>(rv_compiler::SUBST_SLOT))
len = rv_compiler::SUBST_SLOT - 1;
memcpy(memory.data() + slot, value, len);
memory[slot + len] = 0;
} else {
memory[slot] = 0;
}
};
{
UTF8 dbref_buf[32];
mux_sprintf(dbref_buf, sizeof(dbref_buf), T("#%d"), enactor);
copy_subst(rv_compiler::SUBST_ENACTOR, dbref_buf);
mux_sprintf(dbref_buf, sizeof(dbref_buf), T("#%d"), executor);
copy_subst(rv_compiler::SUBST_EXECUTOR, dbref_buf);
}
if (Good_obj(enactor)) {
copy_subst(rv_compiler::SUBST_NAME, Name(enactor));
UTF8 dbref_buf[32];
mux_sprintf(dbref_buf, sizeof(dbref_buf), T("#%d"),
Location(enactor));
copy_subst(rv_compiler::SUBST_LOCATION, dbref_buf);
copy_subst(rv_compiler::SUBST_MONIKER, Moniker(enactor));
} else {
copy_subst(rv_compiler::SUBST_NAME, nullptr);
copy_subst(rv_compiler::SUBST_LOCATION, nullptr);
copy_subst(rv_compiler::SUBST_MONIKER, nullptr);
}
marshal_qregs_to_slots(memory.data(), ~UINT64_C(0));
copy_subst(rv_compiler::SUBST_LASTCMD, mudstate.curr_cmd);
copy_subst(rv_compiler::SUBST_POUT, mudstate.pout);
{
UTF8 ncbuf[32];
mux_sprintf(ncbuf, sizeof(ncbuf), T("%d"), ncargs);
copy_subst(rv_compiler::SUBST_NCARGS, ncbuf);
}
// Set up ECALL context.
eval_ctx ec;
ec.memory = memory.data();
ec.memory_size = memory.size();
ec.executor = executor;
ec.caller = caller;
ec.enactor = enactor;
ec.eval = eval_flags;
ec.cargs = cargs;
ec.ncargs = ncargs;
ec.qreg_mask = ~UINT64_C(0);
ec.lua_result_base = 0;
ec.lua_result_count = 0;
ec.lua_state = nullptr;
ec.host_ecalls = 0;
ec.dbt = &dbt;
ec.pvm = nullptr;
// Run via the shared heap's DBT.
dbt_rerun(&dbt, eval_ecall, &ec);
int rc;
if (run_count == 0) {
run_count++;
rc = dbt_run(&dbt, e->entry_pc, rv_compiler::STACK_TOP);
} else {
dbt.ctx.x[2] = rv_compiler::STACK_TOP;
run_count++;
rc = dbt_resume(&dbt, e->entry_pc);
}
// Alarm abort: count + emit #-1 CPU LIMITED (handled). Other
// failures fall through to the outer AST path.
if (!handle_dbt_run_status(rc, out, out_size, true)) {
return false;
}
if (rc == -3) {
return true;
}
// Extract result (#1057: bound guest NUL scan). Use the
// snapshotted out_addr — not e->out_addr — so a mid-run
// eviction cannot UAF here (#1940).
uint64_t out_addr = resolve_runtime_out_addr(
entry_out_addr, rv_compiler::STACK_TOP);
size_t n = 0;
if (!guest_strnlen(memory.data(), memory.size(), out_addr, &n)) {
return false;
}
if (n >= out_size) n = out_size - 1;
memcpy(out, memory.data() + out_addr, n);
out[n] = '\0';
if (out_ecalls) *out_ecalls = entry_ecalls;
if (out_tier2) *out_tier2 = entry_tier2;
if (out_folded) *out_folded = false;
return true;
}
};
static shared_heap_t s_shared_heap;
// Run a cached program. Uses dbt_rerun if the DBT already has
// translated blocks for this program, otherwise dbt_reset.
//
// Nesting depth of run_cached_program. Softcode JIT ECALL → fun_lua →
// Lua TryJIT must not dbt_reset/rerun the shared persistent DBT while the
// outer softcode program is live (#1309 nested corruption / hang).
//
// Deferred half of jitstats(flush) (#1316).
//
// s_compile_cache holds compiled_program by value and compile_cached() hands
// back a pointer into it, which run_cached_program holds across the whole
// execution -- and jit_eval still reads prog->ecalls after that returns.
// Compiled code reaches the interpreter through ECALLs (u(), ufuns), so
// fun_jitstats can run with a live program on the stack; clearing the cache
// there frees the program that is mid-flight. It showed up as jitstats(flush)
// returning an empty string instead of OK on the compiled route, because the
// post-run harvest reads prog->out_addr out of freed memory.
//
// So the SQLite DELETE and the write-queue drop happen immediately -- neither
// frees anything a running program points at -- while the in-memory caches
// wait until no program is live. Draining at the top of compile_cached()
// (and of the Lua Run/Compile entries) is what makes that safe: those are the
// points where the caller does not yet hold a pointer, and s_run_cached_depth
// == 0 means no outer run_cached_program frame holds one either.
//
static bool s_code_cache_flush_pending = false;
static void jit_flush_memory_caches(void)
{
s_compile_cache.clear();
s_compile_lru.clear();
s_decline_memo.clear();
for (int i = 0; i < JIT_MAX_RUN_DEPTH; i++) {
s_vm[i].release_all_slots();
}
jit_lua_clear_cache();
}
static void jit_drain_pending_flush(void)
{
if ( s_code_cache_flush_pending
&& 0 == s_run_cached_depth)
{
s_code_cache_flush_pending = false;
jit_flush_memory_caches();
}
}
// Same drain, reachable from jit_lua.cpp, which owns the other program cache
// and takes a pointer into it the same way.
//
void jit_flush_pending_caches(void)
{
jit_drain_pending_flush();
}
// eval_ecall status for Lua (#1423 / #1751).
// dbt_run continues on a negative return and stops on a non-negative one;
// 0 is success. ECALL_LUA_ERROR commits an interpreter-class message.
// ECALL_DECLINE is retained only as a residual safety net (Phase 4): any
// remaining site is rewritten to LUA ERROR text and must not re-run.
// Pre-entry fallback (compile refuse / cache miss) is unchanged.
//
static constexpr int ECALL_DECLINE = 1;
static constexpr int ECALL_LUA_ERROR = 2;
static thread_local const char *s_lua_decline_site = nullptr;
static thread_local char s_lua_ecall_error[256];
// Forward: residual post-entry soft declines become committed errors.
//
static int ecall_lua_error_cstr(const char *msg);
// Phase 4: no soft decline. Name kept for greppability of residual sites.
//
static int lua_ecall_decline(const char *site)
{
s_lua_decline_site = (nullptr != site) ? site : "UNKNOWN";
char buf[160];
mux_snprintf(reinterpret_cast<UTF8 *>(buf), sizeof(buf),
T("post-entry residual decline (%s)"),
reinterpret_cast<const UTF8 *>(s_lua_decline_site));
return ecall_lua_error_cstr(buf);
}
bool run_cached_program(compiled_program *prog,
dbref executor, dbref caller_db,
dbref enactor,
UTF8 *out, size_t out_size,
const UTF8 *cargs[],
int ncargs,
int eval,
void *lua_state) {
// Each nesting level gets its own guest buffer and DBT (#1326), so a
// nested run no longer has to be refused. It used to be: softcode JIT
// ECALL -> fun_lua -> Lua JIT would have materialized its program over the
// outer program's memory and reset the shared DBT under the outer
// program's live frames, so run_cached_program declined and Lua fell back
// to its interpreter for every nested call -- which is every call, since
// eval brackets are compiled by default.
//
// Beyond the contexts we have, still refuse. The fallback is correct, so
// running out of depth costs speed and nothing else.
//
if (s_run_cached_depth >= JIT_MAX_RUN_DEPTH) {
return false;
}
jit_run_vm *vm = &s_vm[s_run_cached_depth];
// #1002 depth watermark (see jit_eval; repeated here for callers
// that bypass it, e.g. rvbench).
if (mudstate.func_nest_lev + prog->max_func_depth
>= mudconf.func_nest_lim) {
s_jit_stats.bail_depth++;
return false;
}
// Invocation-count watermark (0 means unknown/pre-v13 cache row —
// skip until that program recompiles and is rewritten with a count).
if ( 0 < prog->n_func_calls
&& mudstate.func_invk_ctr + prog->n_func_calls
>= mudconf.func_invk_lim) {
s_jit_stats.bail_invk++;
return false;
}
if (!prog->needs_jit) {
size_t n = prog->folded_result.size();
if (n >= out_size) n = out_size - 1;
memcpy(out, prog->folded_result.data(), n);
out[n] = '\0';
return true;
}
// A top-level entry proves no outer program holds a live handle into
// the Tier 3 helper save stacks — reclaim slots leaked by abandoned
// runs (see jit_helper_stacks_reset).
if (0 == s_run_cached_depth) {
jit_helper_stacks_reset();
}
// Hold the reentrancy lock for the whole DBT path (including early
// declines that still touch shared runtime state).
//
RunDepthGuard run_depth_guard(s_run_cached_depth);
// Bind the DBT and place the program (#2129). Binding comes first —
// slot eviction must be able to invalidate stale translations before
// new code claims their PCs — and a fresh binding pretranslates the
// blob exactly once for the life of the context, not once per program.
runtime_buffer_init(vm);
dbt_state_t *dbt = bind_run_dbt(vm);
if (nullptr == dbt) {
return false;
}
if (0 == dbt->blob_code_end) {
pretranslate_tier2(dbt);
dbt->blob_code_end = dbt->code_used;
}
// Code: keep the slot the program already occupies (its translations
// are still live under those PCs), or claim one.
uint64_t slot_base = 0;
if (!pick_code_slot(vm, dbt, *prog, &slot_base)) {
return false;
}
// Data: str/fargs pools are shared across slots at canonical addresses,
// so they need refreshing whenever any other program ran since — unless
// the pools already hold this exact program (consecutive re-run, the
// common hot path). Execution does not dirty these regions: the string
// pool is read-only and frame-relative fargs entries are re-patched by
// the program's own code on every run.
//
// "Read-only" here rests on the #2136 const-fargs contract: FUNCTION
// bodies take `const UTF8 * const fargs[]`, so an ECALL callee cannot
// tokenize the pool constants it is handed (that corruption was #2128,
// and this very skip is what made it permanent instead of one-run).
// #2135's per-call argument copy used to shield this; the contract
// replaced it, and the compiler now enforces what the copy papered
// over.
if (vm->buffer_program_id != prog->program_id) {
materialize_data(vm, *prog);
}
// Reset writable blob state (data + BSS) for clean re-run.
if (s_tier2.loaded) {
tier2_reset_writable(vm->buffer, rv_compiler::BLOB_BASE);
}
// Clear output buffers: NUL the first byte of each slot.
{
uint64_t addr = rv_compiler::STACK_TOP - 8 - rv_compiler::OUT_SLOT;
while (addr >= prog->out_pool_end) {
vm->buffer[addr] = 0;
addr -= rv_compiler::OUT_SLOT;
}
}
// Zero the loop-context depth (#2171): the VM buffer is shared
// across programs, so a program that never runs its loop prologue
// must publish no levels instead of whatever the last program left.
memset(vm->buffer.data() + rv_compiler::LOOPCTX_BASE, 0, 8);
// Populate CARGS: copy each arg, NUL-terminate unused slots. Only the
// slots this program actually reads (%0..%N) need populating; functions
// reached via ECALL receive cargs through the host pointer array, not
// these guest slots.
//
// Slot is CARGS_SLOT bytes including the trailing NUL. Silently
// truncating a long carg would change softcode results vs the AST
// path (LBUF-sized); decline so the AST evaluator handles it (#1055).
for (int i = 0; i < prog->cargs_used; i++) {
uint64_t slot = rv_compiler::CARGS_BASE
+ static_cast<uint64_t>(i) * rv_compiler::CARGS_SLOT;
if (i < ncargs && cargs && cargs[i]) {
size_t len = strlen(reinterpret_cast<const char *>(cargs[i]));
if (len >= static_cast<size_t>(rv_compiler::CARGS_SLOT)) {
return false;
}
memcpy(vm->buffer.data() + slot, cargs[i], len);
vm->buffer[slot + len] = 0;
} else {
vm->buffer[slot] = 0;
}
}
// Copy %-substitution runtime values into SUBST slots.
auto copy_subst = [&](int slot_idx, const UTF8 *value) {
uint64_t slot = rv_compiler::SUBST_BASE
+ static_cast<uint64_t>(slot_idx) * rv_compiler::SUBST_SLOT;
if (value && value[0]) {
size_t len = strlen(reinterpret_cast<const char *>(value));
if (len >= static_cast<size_t>(rv_compiler::SUBST_SLOT))
len = rv_compiler::SUBST_SLOT - 1;
memcpy(vm->buffer.data() + slot, value, len);
vm->buffer[slot + len] = 0;
} else {
vm->buffer[slot] = 0;
}
};
// Populate only the SUBST slots this program actually reads. Each
// substitution's value (and several of the lookups below — Name/Location/
// Moniker, mux_sprintf) is computed only when its slot is referenced.
auto subst_used = [&](int slot) -> bool {
return (prog->subst_mask >> slot) & UINT64_C(1);
};
// %# — enactor dbref as string.
if (subst_used(rv_compiler::SUBST_ENACTOR)) {
UTF8 dbref_buf[32];
mux_sprintf(dbref_buf, sizeof(dbref_buf), T("#%d"), enactor);
copy_subst(rv_compiler::SUBST_ENACTOR, dbref_buf);
}
// %! — executor dbref as string.
if (subst_used(rv_compiler::SUBST_EXECUTOR)) {
UTF8 dbref_buf[32];
mux_sprintf(dbref_buf, sizeof(dbref_buf), T("#%d"), executor);
copy_subst(rv_compiler::SUBST_EXECUTOR, dbref_buf);
}
// %n — enactor name.
if (subst_used(rv_compiler::SUBST_NAME)) {
copy_subst(rv_compiler::SUBST_NAME,
Good_obj(enactor) ? Name(enactor) : nullptr);
}
// %l — enactor location.
if (subst_used(rv_compiler::SUBST_LOCATION)) {
if (Good_obj(enactor)) {
dbref loc = Location(enactor);
UTF8 dbref_buf[32];
mux_sprintf(dbref_buf, sizeof(dbref_buf), T("#%d"), loc);
copy_subst(rv_compiler::SUBST_LOCATION, dbref_buf);
} else {
copy_subst(rv_compiler::SUBST_LOCATION, nullptr);
}
}
// %q global registers (+ the long-register bitmap, #996).
marshal_qregs_to_slots(vm->buffer.data(), prog->subst_mask);
// %m — last command.
if (subst_used(rv_compiler::SUBST_LASTCMD)) {
copy_subst(rv_compiler::SUBST_LASTCMD, mudstate.curr_cmd);
}
// %k — moniker (enactor name with color).
if (subst_used(rv_compiler::SUBST_MONIKER)) {
copy_subst(rv_compiler::SUBST_MONIKER,
Good_obj(enactor) ? Moniker(enactor) : nullptr);
}
// %| — piped command output.
if (subst_used(rv_compiler::SUBST_POUT)) {
copy_subst(rv_compiler::SUBST_POUT, mudstate.pout);
}
// %+ — number of cargs.
if (subst_used(rv_compiler::SUBST_NCARGS)) {
UTF8 ncbuf[32];
mux_sprintf(ncbuf, sizeof(ncbuf), T("%d"), ncargs);
copy_subst(rv_compiler::SUBST_NCARGS, ncbuf);
}
eval_ctx ec;
ec.memory = vm->buffer.data();
ec.memory_size = rv_compiler::MEM_SIZE;
ec.executor = executor;
ec.caller = caller_db;
ec.enactor = enactor;
ec.eval = eval;
ec.cargs = cargs;
ec.ncargs = ncargs;
ec.qreg_mask = prog->subst_mask;
ec.lua_state = lua_state;
ec.host_ecalls = 0;
ec.dbt = nullptr;
ec.pvm = nullptr;
// The DBT was bound (and the program placed) before marshalling; only
// the per-run ECALL context remains to be set. No reset on program
// switch — that reset was #2129, the one-program translated-block cache.
dbt_rerun(dbt, eval_ecall, &ec);
ec.dbt = dbt;
// Clear Phase 0 post-entry bookkeeping for this run.
//
s_lua_decline_site = nullptr;
int rc = dbt_run(dbt, slot_base + prog->entry_pc, rv_compiler::STACK_TOP);
// #1751 Phase 0–4: after dbt_run for a Lua program, never return false
// (that re-runs the whole chunk in the Lua VM). Commit an error text.
//
if (rc == ECALL_DECLINE && nullptr != lua_state) {
// Residual only: lua_ecall_decline now returns ECALL_LUA_ERROR.
// Keep this arm so a stray return ECALL_DECLINE cannot re-run.
//
const char *site = (nullptr != s_lua_decline_site)
? s_lua_decline_site : "UNKNOWN";
if (nullptr != out && 0 < out_size) {
mux_snprintf(out, out_size,
T("#-1 LUA JIT POST-ENTRY DECLINE (%s)"),
reinterpret_cast<const UTF8 *>(site));
}
STARTLOG(LOG_ALWAYS, "JIT", "RETRY");
log_text(T("Lua residual ECALL_DECLINE (no re-run): "));
log_text(reinterpret_cast<const UTF8 *>(site));
ENDLOG;
return true;
}
if (rc == ECALL_LUA_ERROR && nullptr != lua_state) {
if (nullptr != out && 0 < out_size) {
mux_strncpy(out,
reinterpret_cast<const UTF8 *>(s_lua_ecall_error),
out_size - 1);
}
return true;
}
if (!handle_dbt_run_status(rc, out, out_size, true)) {
// Phase 4: any other post-run failure on the Lua path is committed,
// not a silent interpreter re-run.
//
if (nullptr != lua_state) {
if (nullptr != out && 0 < out_size
&& (out[0] == '\0'
|| 0 != strncmp(reinterpret_cast<const char *>(out),
"#-1", 3))) {
mux_snprintf(out, out_size, T("#-1 LUA JIT RUN FAIL"));
}
return true;
}
// Softcode (#1791): CALL_FUNC never soft-declines, but mid-run
// DBT infrastructure failure (code buffer full, …) used to
// return false → full AST re-run. After any host ECALL, that
// doubles effects. Commit a loud fail instead.
//
if (0 < ec.host_ecalls) {
if (nullptr != out && 0 < out_size
&& (out[0] == '\0'
|| 0 != strncmp(reinterpret_cast<const char *>(out),
"#-1", 3))) {
mux_snprintf(out, out_size, T("#-1 JIT POST-ENTRY FAIL"));
}
STARTLOG(LOG_ALWAYS, "JIT", "SOFT");
log_text(T("Softcode mid-run DBT fail after host ECALL (no AST re-run)"));
ENDLOG;
return true;
}
return false;
}
if (rc == -3) {
return true; // CPU LIMITED already written
}
// Harvest result from guest memory (#1057).
uint64_t out_addr = resolve_runtime_out_addr(
prog->out_addr, rv_compiler::STACK_TOP);
size_t n = 0;
if (!guest_strnlen(vm->buffer.data(), vm->buffer.size(),
out_addr, &n)) {
// Successful dbt_run then a harvest miss is still post-entry if
// host ECALLs ran — do not AST-re-run (#1791).
//
if (nullptr == lua_state && 0 < ec.host_ecalls) {
if (nullptr != out && 0 < out_size) {
mux_snprintf(out, out_size, T("#-1 JIT POST-ENTRY FAIL"));
}
return true;
}
return false;
}
if (n >= out_size) n = out_size - 1;
memcpy(out, vm->buffer.data() + out_addr, n);
out[n] = '\0';
return true;
}
// ECALL handler implementation.
//
// Common helper: call a FUN* with guest-memory arguments and write
// result to guest output buffer. Returns bytes written.
//
// Thread-local pointer to the current
// ECALL execution context. Set during ecall_invoke_fun so that
// Tier 3 helper functions (_write_carg, _save_cargs, _restore_cargs)
// can access JIT guest memory.
//
static thread_local eval_ctx *s_current_ecall_ctx = nullptr;
// #1989: keeps the softcode call counters correct across an ECALL.
// func_invk_ctr is monotonic for the evaluation and is not given back,
// matching ast.cpp, which decrements only func_nest_lev when the call
// returns. RAII so the early returns for a tripped limit, a bad argument
// count, or a guest-memory rejection all unwind the nesting level.
//
class CallCounter
{
public:
explicit CallCounter(bool bCount) : m_bCount(bCount)
{
if (m_bCount)
{
mudstate.func_nest_lev++;
mudstate.func_invk_ctr++;
}
}
~CallCounter()
{
if (m_bCount)
{
mudstate.func_nest_lev--;
}
}
private:
const bool m_bCount;
};
// Invoke a global user function (@function) from an ECALL.
//
// A global is an attribute reference rather than a C entry point, so it
// cannot go through ecall_invoke_fun. This mirrors ast.cpp's ufun arm
// (permissions, FN_PRIV executor swap, AF_NOEVAL raw copy, FN_PRES
// register preservation) while keeping this file's guest-memory
// conventions for arguments and the result.
//
// ECALL arguments are guest-memory pointers handed to callees DIRECTLY,
// including pointers into regions that outlive the call (pooled string
// constants, blob rodata, code-embedded constants below CARGS_BASE).
//
// That is safe because of the #2136 const-fargs contract: FUNCTION/
// XFUNCTION bodies take `const UTF8 * const fargs[]`, so a builtin cannot
// tokenize or otherwise scribble on its argument text — the compiler
// enforces it, and the campaign's conversion left zero const_casts in
// function bodies. Handlers that legitimately mutate text (do_link,
// do_trigger, ...) receive FargCopy/FargVec copies made by their wrappers.
// The ufun path hands the pointers to mux_exec as cargs, which have the
// same const contract.
//
// History: before #2136 this file copied every below-CARGS_BASE argument
// into an LBuf per call (#2135, after #2128's silent pool corruption:
// first run right, all later runs wrong). The copy and its
// `ptr < CARGS_BASE` predicate are gone — deleting them, not the const
// keywords, was the campaign's payoff.
static int ecall_invoke_ufun(UFUN *ufp, eval_ctx *ec, rv64_ctx_t *ctx,
uint64_t fargs_addr, int nfargs,
uint64_t out_addr, uint64_t out_size) {
// Host softcode work — counts for post-entry no-re-run (#1791).
//
++ec->host_ecalls;
// #1079: the out buffer must be a valid guest range before any write.
//
if (!guest_range_ok(out_addr, out_size, ec->memory_size) || out_size == 0) {
ctx->x[10] = 0;
return -1;
}
// #1124: the same permission gates ast.cpp applies. A global can be
// registered with restrictive perms, and compiled softcode must not
// be a way around them.
//
if ( !check_access(ec->executor, ufp->perms)
|| ( (ufp->flags & FN_RESTRICT)
&& !Wizard(ec->executor)))
{
// mux_snprintf returns what it WROTE, not what it would have, so
// snprintf's <0 and >=out_size clamps are gone rather than unused.
size_t n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
out_size, T("%s"), FUNC_NOPERM_MESSAGE);
ctx->x[10] = static_cast<uint64_t>(n);
return -1;
}
// #1989: count this call, as ast.cpp's ufun arm does. A global whose
// body reaches itself needs the live counters to stop it; nothing in
// the compiled route maintained them.
//
CallCounter call_counter(true);
{
const UTF8 *pLimit = nullptr;
if (mudconf.func_nest_lim <= mudstate.func_nest_lev) {
pLimit = T("#-1 FUNCTION RECURSION LIMIT EXCEEDED");
} else if (mudconf.func_invk_lim <= mudstate.func_invk_ctr) {
pLimit = T("#-1 FUNCTION INVOCATION LIMIT EXCEEDED");
}
if (nullptr != pLimit) {
size_t n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
out_size, T("%s"), pLimit);
ctx->x[10] = static_cast<uint64_t>(n);
return -1;
}
}
if (nfargs > MAX_ARG) nfargs = MAX_ARG;
if (nfargs < 0) nfargs = 0;
const UTF8 *fargs[MAX_ARG];
memset(fargs, 0, sizeof(fargs));
uint64_t frame_top = ctx->x[8]; // s0 = frame pointer
for (int i = 0; i < nfargs; i++) {
uint64_t ptr = 0;
if (!guest_farg_addr(ec->memory, ec->memory_size, fargs_addr, i, &ptr)) {
ctx->x[10] = 0;
return -1;
}
if (rv_compiler::is_output_frame_ref(ptr)) {
ptr = rv_compiler::resolve_output_addr(ptr, frame_top);
}
// Bounds-validate the guest C string even though the length itself
// is no longer needed for a copy.
size_t slen = 0;
if (!guest_strnlen(ec->memory, ec->memory_size, ptr, &slen)) {
ctx->x[10] = 0;
return -1;
}
fargs[i] = ec->memory + ptr;
}
dbref aowner;
int aflags;
UTF8 *tbuf = atr_get("ecall.ufun", ufp->obj, ufp->atr, &aowner, &aflags);
dbref obj = (ufp->flags & FN_PRIV) ? ufp->obj : ec->executor;
LBuf buff = LBuf_Src("eval_ecall_ufun");
UTF8 *bufc = buff.get();
eval_ctx *saved_ctx = s_current_ecall_ctx;
s_current_ecall_ctx = ec;
if ( (aflags & AF_NOEVAL)
|| NoEval(ufp->obj))
{
size_t nLen = strlen(reinterpret_cast<const char *>(tbuf));
safe_copy_buf(tbuf, nLen, buff, &bufc);
}
else
{
reg_ref **preserve = nullptr;
if (ufp->flags & FN_PRES) {
preserve = PushRegisters(MAX_GLOBAL_REGS);
save_global_regs(preserve);
}
int feval = ec->eval & ~(EV_TOP | EV_FMAND);
mux_exec(tbuf, LBUF_SIZE-1, buff, &bufc, obj, ec->executor, ec->enactor,
AttrTrace(aflags, feval),
fargs, nfargs);
if (ufp->flags & FN_PRES) {
restore_global_regs(preserve);
PopRegisters(preserve, MAX_GLOBAL_REGS);
}
}
s_current_ecall_ctx = saved_ctx;
free_lbuf(tbuf);
// A user-function body is arbitrary softcode and may setq, so the %q
// slots this program reads have to be re-marshalled from the
// authoritative global_regs — the same resync ecall_invoke_fun does
// after a builtin. Unconditional here: there is no pure-read case
// to exempt.
//
if ((ec->qreg_mask & QREG_SLOT_BITS) != 0) {
marshal_qregs_to_slots(ec->memory, ec->qreg_mask);
s_jit_stats.qreg_resyncs++;
}
*bufc = '\0';
size_t result_len = static_cast<size_t>(bufc - buff);
if (result_len >= out_size) result_len = out_size - 1;
memcpy(ec->memory + out_addr, buff, result_len);
ec->memory[out_addr + result_len] = '\0';
ctx->x[10] = static_cast<uint64_t>(result_len);
return -1;
}
// Push the calling program's live compiled iter levels onto the
// interpreter's itext/inum stack for the duration of a host callee
// (#2171). Compiled loops keep the guest loop-context table at
// LOOPCTX_BASE current (HIR_LCTX_* stores); without this push, any
// callee that evaluates softcode — fun_u's mux_exec, fun_itext, ilev()
// — saw an EMPTY stack and %i0 inside u() called from a compiled iter
// came back blank. The pushed itext pointers point into guest memory,
// which outlives the ECALL (the arena is stable for the eval), and
// every consumer copies out. RAII so all the invoke paths' returns
// unwind it.
//
class GuestLoopContext
{
public:
explicit GuestLoopContext(eval_ctx *ec) : m_pushed(0)
{
const uint64_t base = rv_compiler::LOOPCTX_BASE;
if (base + 8 > ec->memory_size) {
return;
}
uint64_t depth;
memcpy(&depth, ec->memory + base, 8);
if (0 == depth || depth > rv_compiler::LOOPCTX_MAX_LEVELS
|| base + (1 + 2 * depth) * 8 > ec->memory_size) {
return;
}
for (uint64_t k = 0; k < depth; k++) {
if (mudstate.in_loop < 0 || mudstate.in_loop >= MAX_ITEXT) {
break;
}
uint64_t eaddr, in1;
memcpy(&eaddr, ec->memory + base + (1 + 2 * k) * 8, 8);
memcpy(&in1, ec->memory + base + (2 + 2 * k) * 8, 8);
size_t slen = 0;
if (!guest_strnlen(ec->memory, ec->memory_size, eaddr, &slen)) {
break;
}
mudstate.itext[mudstate.in_loop] = ec->memory + eaddr;
mudstate.inum[mudstate.in_loop] =
static_cast<int>(in1);
mudstate.in_loop++;
m_pushed++;
}
}
~GuestLoopContext()
{
mudstate.in_loop -= m_pushed;
}
GuestLoopContext(const GuestLoopContext &) = delete;
GuestLoopContext &operator=(const GuestLoopContext &) = delete;
private:
int m_pushed;
};
static int ecall_invoke_fun(FUN *fp, eval_ctx *ec, rv64_ctx_t *ctx,
uint64_t fargs_addr, int nfargs,
uint64_t out_addr, uint64_t out_size) {
// Host softcode work — counts for post-entry no-re-run (#1791).
// Count even on early validation failure: a partial path that later
// grows side effects must still be treated as entry.
//
++ec->host_ecalls;
// #1079: out buffer must be a valid guest range before any write.
//
if (!guest_range_ok(out_addr, out_size, ec->memory_size) || out_size == 0) {
ctx->x[10] = 0;
return -1;
}
// #1124: mirror AST's check_access (ast.cpp) so CA_WIZARD / CA_GOD
// builtins cannot be invoked from JIT-compiled softcode by mortals.
// engine_api_table includes every builtin; without this gate, ECALL
// would call fp->fun with no perms test.
//
// Exemption: underscore-prefixed names are JIT internal helpers
// (_SAVE_QREGS, _WRITE_CARG, _CHECK_U_PERM, …) deliberately marked
// CA_GOD so softcode cannot call them, but the compiler must. They
// never appear as softcode-visible symbols softcode can type.
//
if ( fp->name[0] != '_'
&& !check_access(ec->executor, fp->perms))
{
// mux_snprintf returns what it WROTE, not what it would have, so
// snprintf's <0 and >=out_size clamps are gone rather than unused.
size_t n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
out_size, T("%s"), FUNC_NOPERM_MESSAGE);
ctx->x[10] = static_cast<uint64_t>(n);
return -1;
}
// #1989: mirror AST's call accounting (ast.cpp) the same way the perms
// gate above mirrors its check_access. ast.cpp bumps func_nest_lev /
// func_invk_ctr and tests both limits immediately before its own
// fp->fun dispatch; the compiled route reaches the builtin without
// passing through that branch, so nothing counted a u() that calls
// itself -- it recursed until the stack was gone, at any limit. The
// static max_func_depth watermark cannot catch this: it is a
// compile-time property that does not grow with runtime recursion.
//
// Underscore-prefixed names are JIT internal helpers, not softcode
// calls, and must not spend the player's budget -- the same exemption
// check_access uses above.
//
const bool bCount = (fp->name[0] != '_');
CallCounter call_counter(bCount);
if (bCount)
{
const UTF8 *pLimit = nullptr;
if (mudconf.func_nest_lim <= mudstate.func_nest_lev) {
pLimit = T("#-1 FUNCTION RECURSION LIMIT EXCEEDED");
} else if (mudconf.func_invk_lim <= mudstate.func_invk_ctr) {
pLimit = T("#-1 FUNCTION INVOCATION LIMIT EXCEEDED");
}
if (nullptr != pLimit) {
size_t n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
out_size, T("%s"), pLimit);
ctx->x[10] = static_cast<uint64_t>(n);
return -1;
}
}
// Validate argument count against function's declared limits.
// Return the same error string the AST evaluator would.
if (nfargs < fp->minArgs) {
int n;
if (fp->minArgs == fp->maxArgs) {
n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
out_size, T("#-1 FUNCTION (%s) EXPECTS %d ARGUMENTS"),
fp->name, fp->minArgs);
} else if (fp->minArgs + 1 == fp->maxArgs) {
n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
out_size, T("#-1 FUNCTION (%s) EXPECTS %d OR %d ARGUMENTS"),
fp->name, fp->minArgs, fp->maxArgs);
} else {
n = mux_snprintf(reinterpret_cast<UTF8 *>(ec->memory + out_addr),
out_size, T("#-1 FUNCTION (%s) EXPECTS BETWEEN %d AND %d ARGUMENTS"),
fp->name, fp->minArgs, fp->maxArgs);
}
if (n < 0) n = 0;
if (static_cast<size_t>(n) >= out_size) n = static_cast<int>(out_size - 1);
ctx->x[10] = static_cast<uint64_t>(n);
return -1;
}
if (fp->maxArgs >= 0 && nfargs > fp->maxArgs) {
nfargs = fp->maxArgs;
}
const UTF8 *fargs[MAX_ARG];
if (nfargs > MAX_ARG) nfargs = MAX_ARG;
uint64_t frame_top = ctx->x[8]; // s0 = frame pointer
for (int i = 0; i < nfargs; i++) {
uint64_t ptr = 0;
// #1079: overflow-safe fargs table slot load + guest C-string bound.
//
if (!guest_farg_addr(ec->memory, ec->memory_size, fargs_addr, i, &ptr)) {
ctx->x[10] = 0;
return -1;
}
// Resolve frame-relative output references.
if (rv_compiler::is_output_frame_ref(ptr)) {
ptr = rv_compiler::resolve_output_addr(ptr, frame_top);
}
size_t slen = 0;
if (!guest_strnlen(ec->memory, ec->memory_size, ptr, &slen)) {
ctx->x[10] = 0;
return -1;
}
// Direct guest pointer — safe under the #2136 const contract, and
// exactly what the `_`-prefixed ABI helpers need (fun__write_carg
// converts fargs[1] back to a guest address).
fargs[i] = ec->memory + ptr;
}
LBuf buff = LBuf_Src("eval_ecall");
UTF8 *bufc = buff.get();
eval_ctx *saved_ctx = s_current_ecall_ctx;
s_current_ecall_ctx = ec;
// Builtins receive only the trace bit, mirroring the AST dispatch
// (ast.cpp fp->fun(..., feval & EV_TRACE, ...)). Passing the full
// program flags diverged for re-evaluating handlers: fun_eval's
// inner mux_exec inherited EV_STRIP_CURLY and stripped nested
// braces the AST route preserves (#991 parser TC010).
fp->fun(fp, buff, &bufc, ec->executor, ec->caller, ec->enactor,
ec->eval & EV_TRACE, fargs, nfargs, ec->cargs, ec->ncargs);
s_current_ecall_ctx = saved_ctx;
// Conservative post-ECALL resync: any host callee may have mutated
// mudstate.global_regs (a scope _RESTORE_QREGS reverting them, or
// an interpreter-side setq — which writes only global_regs, never
// the guest slots), so re-marshal the %q slots this program reads
// from the authoritative global_regs. Masked by the program's
// subst_mask: %q-free programs skip at the bit test. A pure
// callee makes this a semantic no-op (slots already match).
// (docs/plan-jit-evalbracket-lift.md, Phases 2-3.)
// fun_r is a pure register read — the long-path %q ECALL (#996) —
// and must not itself trigger a full masked re-marshal.
static FUN *s_fun_r_fp = nullptr;
if (s_fun_r_fp == nullptr) {
int r_idx = engine_api_lookup("R");
if (r_idx > 0 && r_idx < ENGINE_API_MAX_FUNCS) {
s_fun_r_fp = engine_api_table[r_idx];
}
}
if ((ec->qreg_mask & QREG_SLOT_BITS) != 0
&& fp != s_fun_r_fp) {
// Re-marshal slots AND the long-register bitmap from the
// authoritative global_regs (#996: a register grown past the
// slot by the callee sets its bit here, so later %q reads
// take the fun_r path instead of reading a truncated slot).
marshal_qregs_to_slots(ec->memory, ec->qreg_mask);
s_jit_stats.qreg_resyncs++;
}
*bufc = '\0';
size_t result_len = static_cast<size_t>(bufc - buff);
if (result_len >= out_size) result_len = out_size - 1;
memcpy(ec->memory + out_addr, buff, result_len);
ec->memory[out_addr + result_len] = '\0';
ctx->x[10] = static_cast<uint64_t>(result_len);
return -1;
}
// ---------------------------------------------------------------
// JIT DMA Controller (Tier C)
// ---------------------------------------------------------------
class jit_dma_controller {
public:
static constexpr int MAX_WINDOWS = rv_compiler::DMA_WINDOW_COUNT;
static void submit(int window, size_t length, int op, eval_ctx *ec) {
if (window < 0 || window >= MAX_WINDOWS) return;
uint64_t addr = rv_compiler::DMA_BASE + window * rv_compiler::DMA_WINDOW_SIZE;
if (addr + length > ec->memory_size) return;
const UTF8 *data = ec->memory + addr;
// Implementation of ops (FINALIZE, etc.)
// For now, simple registration into an arena.
if (op == 1) { // DMA_OP_FINALIZE
auto *a = JITArena::Alloc(length + 1);
if (a) {
size_t off = a->used - (length + 1);
memcpy(a->buf->data + off, data, length);
a->buf->data[off + length] = '\0';
// Success: queue for ACK.
s_ack_queue.push_back(window);
s_window_busy |= (1 << window);
}
}
}
static int get_next_ack() {
if (s_ack_queue.empty()) return -1;
int window = s_ack_queue.front();
s_ack_queue.pop_front();
s_window_busy &= ~(1 << window);
return window;
}
static void reset() {
s_window_busy = 0;
s_ack_queue.clear();
}
private:
static inline uint32_t s_window_busy = 0;
static inline std::list<int> s_ack_queue;
};
// A Lua error raised inside an ECALL has no protected call frame anywhere
// above it, so luaD_throw() reaches the default panic handler and abort()s
// the process (#1423). Full table ops (lua_geti / lua_settable / luaL_len)
// can raise through metamethods or type errors.
//
// #1751 Phase 1: GET/SET/LEN are *total* — never ECALL_DECLINE for policy
// (metatable / type surprise / missing global). Use the real VM ops under
// pcall; on error commit an interpreter-class LUA ERROR (not a silent
// re-run, not a POST-ENTRY DECLINE). Typed result claims that fail after
// a successful get return ok=0 and continue (same as GETI_INT already did
// for non-integers) so the guest can take a typed alternate path.
//
// ECALL_LUA_ERROR and s_lua_ecall_error live next to ECALL_DECLINE above.
//
static int ecall_lua_commit_error(lua_State *L)
{
const char *msg = lua_tostring(L, -1);
if (nullptr == msg) {
msg = "unknown Lua error";
}
// Match the interpreter's softcode framing for raised Lua errors.
//
mux_snprintf(reinterpret_cast<UTF8 *>(s_lua_ecall_error),
sizeof(s_lua_ecall_error),
T("#-1 LUA ERROR: %s"),
reinterpret_cast<const UTF8 *>(msg));
lua_pop(L, 1);
return ECALL_LUA_ERROR;
}
// Commit a C-string error without a Lua stack message (#1751 Phase 3).
//
static int ecall_lua_error_cstr(const char *msg)
{
if (nullptr == msg) {
msg = "unknown Lua error";
}
mux_snprintf(reinterpret_cast<UTF8 *>(s_lua_ecall_error),
sizeof(s_lua_ecall_error),
T("#-1 LUA ERROR: %s"),
reinterpret_cast<const UTF8 *>(msg));
return ECALL_LUA_ERROR;
}
// Marshal one Lua value into a guest buffer the way fun_lua does for a
// chunk result (nil→"", bool→"0"/"1", else lua_tolstring). Truncates to
// out_size rather than declining. #1751 follow-up: CALL_STR result path
// must not soft-decline after the callee has run (string.find returns an
// integer as the first multi-value; both routes must keep it).
//
static size_t ecall_lua_marshal_to_guest(lua_State *L, int idx,
uint8_t *out, size_t out_size)
{
if (0 == out_size) {
return 0;
}
size_t len = 0;
const char *result = nullptr;
if (lua_isnil(L, idx) || lua_isnone(L, idx)) {
result = "";
len = 0;
} else if (lua_isboolean(L, idx)) {
result = lua_toboolean(L, idx) ? "1" : "0";
len = 1;
} else {
result = lua_tolstring(L, idx, &len);
if (nullptr == result) {
result = "";
len = 0;
}
}
if (len >= out_size) {
len = out_size - 1;
}
if (len > 0) {
memcpy(out, result, len);
}
out[len] = '\0';
return len;
}
// pcall helpers: stack protocol is (table, key[, value]) via absolute
// indices copied onto the stack before the call.
//
static int ecall_aux_gettable(lua_State *L)
{
lua_gettable(L, 1);
return 1;
}
static int ecall_aux_settable(lua_State *L)
{
lua_settable(L, 1);
return 0;
}
static int ecall_aux_len(lua_State *L)
{
lua_pushinteger(L, static_cast<lua_Integer>(luaL_len(L, 1)));
return 1;
}
// #1836: GETGLOBAL and lua_compare can raise through metamethods
// (__index on _ENV, __eq). Same unprotected-ECALL class as #1423.
//
static int ecall_aux_getglobal(lua_State *L)
{
const char *key = lua_tostring(L, 1);
if (nullptr == key) {
lua_pushnil(L);
return 1;
}
lua_getglobal(L, key);
return 1;
}
static int ecall_aux_compare_eq(lua_State *L)
{
// Stack: lhs, rhs. Result: boolean.
//
const int eq = lua_compare(L, 1, 2, LUA_OPEQ);
lua_pushboolean(L, eq);
return 1;
}
// Absolute stack index must refer to a table (or something indexable via
// metamethods). Out-of-range is a soft miss (ok=0), not a decline.
//
static bool ecall_lua_stack_index_ok(lua_State *L, int idx)
{
return idx > 0 && idx <= lua_gettop(L);
}
// Protected get: pushes the value at table[key]. Returns 0 on success
// (value on stack), ECALL_LUA_ERROR on raise. key_is_str: key is a C
// string pushed by the caller as... actually we push inside.
//
static int ecall_lua_pget_intkey(lua_State *L, int tbl_idx, lua_Integer key)
{
lua_pushcfunction(L, ecall_aux_gettable);
lua_pushvalue(L, tbl_idx);
lua_pushinteger(L, key);
if (LUA_OK != lua_pcall(L, 2, 1, 0)) {
return ecall_lua_commit_error(L);
}
return 0;
}
static int ecall_lua_pget_strkey(lua_State *L, int tbl_idx, const char *key)
{
lua_pushcfunction(L, ecall_aux_gettable);
lua_pushvalue(L, tbl_idx);
lua_pushstring(L, key);
if (LUA_OK != lua_pcall(L, 2, 1, 0)) {
return ecall_lua_commit_error(L);
}
return 0;
}
static int ecall_lua_pset_intkey(lua_State *L, int tbl_idx, lua_Integer key,
lua_Integer val)
{
lua_pushcfunction(L, ecall_aux_settable);
lua_pushvalue(L, tbl_idx);
lua_pushinteger(L, key);
lua_pushinteger(L, val);
if (LUA_OK != lua_pcall(L, 3, 0, 0)) {
return ecall_lua_commit_error(L);
}
return 0;
}
static int ecall_lua_pset_strkey(lua_State *L, int tbl_idx, const char *key,
lua_Integer val)
{
lua_pushcfunction(L, ecall_aux_settable);
lua_pushvalue(L, tbl_idx);
lua_pushstring(L, key);
lua_pushinteger(L, val);
if (LUA_OK != lua_pcall(L, 3, 0, 0)) {
return ecall_lua_commit_error(L);
}
return 0;
}
static int ecall_lua_plen(lua_State *L, int tbl_idx)
{
lua_pushcfunction(L, ecall_aux_len);
lua_pushvalue(L, tbl_idx);
if (LUA_OK != lua_pcall(L, 1, 1, 0)) {
return ecall_lua_commit_error(L);
}
return 0;
}
static int ecall_lua_pgetglobal(lua_State *L, const char *key)
{
lua_pushcfunction(L, ecall_aux_getglobal);
lua_pushstring(L, key);
if (LUA_OK != lua_pcall(L, 1, 1, 0)) {
return ecall_lua_commit_error(L);
}
return 0;
}
// lhs and rhs must already be on the stack at absolute indices.
// Pushes nothing; returns 0 with *peq set, or ECALL_LUA_ERROR.
//
static int ecall_lua_pcompare_eq(lua_State *L, int lhs, int rhs, int *peq)
{
lua_pushcfunction(L, ecall_aux_compare_eq);
lua_pushvalue(L, lhs);
lua_pushvalue(L, rhs);
if (LUA_OK != lua_pcall(L, 2, 1, 0)) {
return ecall_lua_commit_error(L);
}
*peq = lua_toboolean(L, -1) ? 1 : 0;
lua_pop(L, 1);
return 0;
}
// Push the arguments of a compiled Lua call (ECALL_LUA_CALL_INT/_STR).
// The two result variants share one argument encoding: nargs in a1's low
// byte, then TWO BITS per argument in the next byte -- 0 is an integer in
// x[12+j], 1 is the guest address of a NUL-terminated string in x[12+j],
// 2 is a double as raw bits in x[12+j] (the FMV.X.D lane ECALL_LUA_FTOA
// already proved on both execution routes). One decoder shared by both
// handlers, because two near-identical loops is the drift shape (#1457)
// the lowering's twin call branches had to be merged out of.
//
// Returns false -- caller declines -- on a bad address or an unknown kind.
// The caller owns stack cleanup, as it also must for a failed pcall.
//
static bool ecall_lua_push_call_args(lua_State *L, const rv64_ctx_t *ctx,
const eval_ctx *ec,
int nargs, int kinds)
{
for (int j = 0; j < nargs; j++) {
const uint64_t raw = ctx->x[12 + j];
switch ((kinds >> (2 * j)) & 3) {
case 0:
lua_pushinteger(L, static_cast<lua_Integer>(raw));
break;
case 1: {
const char *sarg = guest_cstr(ec->memory, ec->memory_size, raw);
if (nullptr == sarg) {
return false;
}
lua_pushstring(L, sarg);
break;
}
case 2: {
double d;
memcpy(&d, &raw, 8);
lua_pushnumber(L, d);
break;
}
default: {
// Kind 3: a Lua stack reference -- the register holds the
// stack index a handle-typed value carries. lua_pushvalue is
// the one honest use of a handle as an argument: the VALUE it
// refers to is pushed, never the index as a number (#1579).
// Absolute indices stay valid as later arguments push.
const int idx = static_cast<int>(raw);
if (idx <= 0 || idx > lua_gettop(L)) {
return false;
}
lua_pushvalue(L, idx);
break;
}
}
}
return true;
}
// The Lua lowering's bridge ECALLs are dispatched by name and all live in a
// reserved "__lua_" namespace (hir_lower_lua.cpp). Recognising the namespace
// -- rather than just a leading "__" -- is what lets the dispatch below fail
// closed on an unimplemented bridge name without swallowing a softcode
// function that happens to start with an underscore, which @function permits.
//
// Case-insensitive on purpose. The names the lowering emits are lower case
// and the comparisons in the dispatch are upper case, which is the defect
// behind #1512; a prefix test that agreed with only one of the two spellings
// would re-create the same trap for the next name added.
//
static bool is_lua_bridge_name(const UTF8 *s) {
static const char prefix[] = "__lua_";
for (size_t i = 0; i < sizeof(prefix) - 1; i++) {
// A short name fails on the NUL, before reading past it.
if (tolower(static_cast<unsigned char>(s[i])) != prefix[i]) {
return false;
}
}
return true;
}
static int eval_ecall(rv64_ctx_t *ctx, void *user_data) {
eval_ctx *ec = static_cast<eval_ctx *>(user_data);
uint64_t syscall_num = ctx->x[17];
switch (syscall_num) {
case ECALL_EXIT:
return static_cast<int>(ctx->x[10]);
case ECALL_CALL_INDEX: {
// Indexed dispatch: a0 = function index, a1 = fargs,
// a2 = nfargs, a3 = output, a4 = outsize.
int func_idx = static_cast<int>(ctx->x[10]);
uint64_t fargs_addr = ctx->x[11];
int nfargs = static_cast<int>(ctx->x[12]);
uint64_t out_addr = ctx->x[13];
uint64_t out_size = ctx->x[14];
if (func_idx <= 0 || func_idx >= engine_api_count ||
!guest_range_ok(out_addr, out_size, ec->memory_size)) {
ctx->x[10] = 0;
return -1;
}
FUN *fp = engine_api_table[func_idx];
GuestLoopContext glc(ec); // (#2171)
int rc = ecall_invoke_fun(fp, ec, ctx, fargs_addr, nfargs,
out_addr, out_size);
return rc;
}
case ECALL_CALL_FUNC: {
// String-based dispatch (fallback): a0 = name ptr.
uint64_t name_addr = ctx->x[10];
uint64_t fargs_addr = ctx->x[11];
int nfargs = static_cast<int>(ctx->x[12]);
uint64_t out_addr = ctx->x[13];
uint64_t out_size = ctx->x[14];
GuestLoopContext glc(ec); // (#2171) — covers builtin AND ufun paths
size_t name_len = 0;
if ( !guest_strnlen(ec->memory, ec->memory_size, name_addr, &name_len)
|| !guest_range_ok(out_addr, out_size, ec->memory_size)) {
ctx->x[10] = 0;
return -1;
}
const UTF8 *func_name = ec->memory + name_addr;
// Intercept __lua_* internal functions for Lua VM table ops.
if (ec->lua_state && is_lua_bridge_name(func_name)) {
// The gate above is deliberately case-insensitive, but every
// dispatch below is a case-sensitive strcmp against an upper-case
// literal -- and hir_lower_lua emits these names in lower case
// ("__lua_call"). So a name could get past the gate, match none
// of the thirteen comparisons, and fall out to the bail at the
// bottom. Normalise once here so the two halves agree (#1519).
//
// This was unobservable until a chunk could reach the ECALL at
// all: consumers of a bridge result declined at lowering, so the
// comparisons were never run against a real name.
char fnbuf[64];
size_t fnlen = name_len < sizeof(fnbuf) - 1
? name_len : sizeof(fnbuf) - 1;
for (size_t i = 0; i < fnlen; i++) {
fnbuf[i] = static_cast<char>(toupper(
static_cast<unsigned char>(func_name[i])));
}
fnbuf[fnlen] = '\0';
const char *fn = fnbuf;
// Which bridge ECALLs a chunk actually reaches is otherwise
// invisible: a decline inside any handler just surfaces as
// lua_run_fail, indistinguishable from every other cause.
// TINYMUX_TRACE_LUA_ECALL=1 names them in order, which is how the
// remaining #1519 blocker was located.
if (getenv("TINYMUX_TRACE_LUA_ECALL")) {
fprintf(stderr, "LUAECALL: enter %s\n", fn);
}
// The named __lua_* bridge is gone (#1519 / #1309).
//
// Thirteen handlers lived here and none had ever executed in any
// build: the lowering emitted lower case, this compared upper.
// Correcting the case was measured on #1519 -- 16 of 21 chunks
// compiled then failed at run time, and 5 of 21 returned the
// EMPTY STRING with lua_run_ok=1, wrong with no signal.
//
// They were also the wrong shape. Every one passed a Lua stack
// INDEX through guest memory as a decimal string, which is what
// made #t answer 22: an index flowed on as though it were the
// value it points at. The four numbered ECALLs that do work
// (FTOA, GETI_INT, NEWTABLE, SETI_INT) pass VALUES in registers
// instead, and that is the direction #1309 settled on.
//
// Deleting rather than leaving them: a mechanism that looks
// available and is not is what let this hide for so long. Every
// __lua_* name now reaches the fail-closed path below, which is
// what already happened in practice.
// Fail closed on an unrecognised __lua_* name (#1512).
//
// Falling through from here reaches the softcode function
// dispatch below, which upper-cases the name and looks it up in
// builtin_functions/ufunc_htab. No __lua_* name is a softcode
// function, so the lookup misses and the ECALL hands back the
// *string* "#-1 FUNCTION NOT FOUND" as this call's value -- and
// that value then flows on as data. `local x=tonumber(a) return
// x+1` produced 1, and `return x*2` produced 0, because the error
// string coerces to zero in arithmetic: wrong answers, not error
// markers.
//
// It is not hypothetical. Every name the lowering emits is lower
// case ("__lua_getglobal") and every comparison above is upper
// case ("__LUA_GETGLOBAL"), so the entire named bridge --
// getglobal/setglobal/getenv, call/get_result, getfield/setfield,
// geti/seti, newtable, pin_array, tfor_call -- has never once
// been reached. The case mismatch alone was survivable; what
// made it produce silent corruption instead of a visible failure
// is this fall-through.
//
// ECALL_DECLINE stops dbt_run with a failure status, so
// run_cached_program returns false without harvesting output and
// the caller re-evaluates on the interpreter -- the same
// containment contract the table ops use (#1423). A name this
// block does not implement now costs the JIT, never correctness.
//
if (getenv("TINYMUX_TRACE_LUA_ECALL")) {
fprintf(stderr, "LUAECALL: UNHANDLED %s\n", fn);
}
// Phase 4: committed error, not soft decline / re-run.
//
return ecall_lua_error_cstr("unhandled Lua bridge ECALL");
}
size_t nCased;
UTF8 *pCased = mux_strupr(func_name, nCased);
std::vector<UTF8> key(pCased, pCased + nCased);
auto it = mudstate.builtin_functions.find(key);
if (it == mudstate.builtin_functions.end()) {
// Not a builtin — try the global user-function table before
// giving up (#1231). @function registers into ufunc_htab,
// and consulting only builtin_functions made every global
// unreachable from compiled softcode while the AST route
// resolved it fine. 2.13 checks both in the same order
// (mux/src/eval.cpp:1487), as does ast.cpp.
//
auto it_ufunc = mudstate.ufunc_htab.find(key);
if (it_ufunc != mudstate.ufunc_htab.end()) {
return ecall_invoke_ufun(
static_cast<UFUN *>(it_ufunc->second),
ec, ctx, fargs_addr, nfargs, out_addr, out_size);
}
const char *err = "#-1 FUNCTION NOT FOUND";
size_t elen = strlen(err);
if (elen >= out_size) elen = out_size - 1;
memcpy(ec->memory + out_addr, err, elen);
ec->memory[out_addr + elen] = '\0';
ctx->x[10] = static_cast<uint64_t>(elen);
return -1;
}
return ecall_invoke_fun(it->second, ec, ctx, fargs_addr, nfargs,
out_addr, out_size);
}
case ECALL_SETQ: {
// Traditional write-through: a0=reg, a1=addr
// Writes to both SUBST slot (for JIT %q reads) and
// mudstate.global_regs (for ECALL reads).
++ec->host_ecalls;
int regnum = static_cast<int>(ctx->x[10]);
uint64_t val_addr = ctx->x[11];
size_t vlen = 0;
if (regnum >= 0 && regnum < MAX_GLOBAL_REGS
&& guest_strnlen(ec->memory, ec->memory_size, val_addr, &vlen)) {
const UTF8 *value = ec->memory + val_addr;
// Write to SUBST slot in guest memory.
uint64_t slot = rv_compiler::SUBST_BASE
+ (rv_compiler::SUBST_QREG0 + regnum)
* rv_compiler::SUBST_SLOT;
if (slot + rv_compiler::SUBST_SLOT <= ec->memory_size) {
size_t cplen = vlen;
if (cplen >= static_cast<size_t>(rv_compiler::SUBST_SLOT))
cplen = rv_compiler::SUBST_SLOT - 1;
memcpy(ec->memory + slot, value, cplen);
ec->memory[slot + cplen] = 0;
}
qreg_longbit_update(ec, regnum, vlen);
RegAssign(&mudstate.global_regs[regnum], vlen, value);
ctx->x[10] = vlen;
} else {
ctx->x[10] = 0;
}
return -1;
}
case ECALL_SETQ_PACK: {
// Fast path: a0=reg, a1=addr, a2=len.
// Packs into a JIT Arena.
++ec->host_ecalls;
int regnum = static_cast<int>(ctx->x[10]);
uint64_t val_addr = ctx->x[11];
size_t vlen = static_cast<size_t>(ctx->x[12]);
if (0 == vlen) {
// Length 0 means "use strlen" — bound the scan to guest
// memory so a missing NUL cannot walk off the buffer (#1057).
if (!guest_strnlen(ec->memory, ec->memory_size, val_addr, &vlen)) {
ctx->x[10] = 0;
return -1;
}
}
if (regnum >= 0 && regnum < MAX_GLOBAL_REGS && val_addr + vlen <= ec->memory_size) {
const UTF8 *value = ec->memory + val_addr;
// Write to SUBST slot in guest memory (for JIT %q reads).
uint64_t slot = rv_compiler::SUBST_BASE
+ (rv_compiler::SUBST_QREG0 + regnum)
* rv_compiler::SUBST_SLOT;
if (slot + rv_compiler::SUBST_SLOT <= ec->memory_size) {
size_t cplen = vlen;
if (cplen >= static_cast<size_t>(rv_compiler::SUBST_SLOT))
cplen = rv_compiler::SUBST_SLOT - 1;
memcpy(ec->memory + slot, value, cplen);
ec->memory[slot + cplen] = 0;
}
qreg_longbit_update(ec, regnum, vlen);
// Allocate from Arena (Tier B).
auto *a = JITArena::Alloc(vlen + 1);
if (a) {
size_t off = a->used - (vlen + 1);
memcpy(a->buf->data + off, value, vlen);
a->buf->data[off + vlen] = '\0';
// Bind to register.
if (mudstate.global_regs[regnum]) {
RegRelease(mudstate.global_regs[regnum]);
}
reg_ref *rr = new reg_ref();
rr->refcount = 1;
rr->buf = a->buf;
rr->reg_ptr = a->buf->data + off;
rr->reg_len = vlen;
mudstate.global_regs[regnum] = rr;
ctx->x[10] = vlen;
} else {
// Fallback if arena alloc fails (shouldn't happen for < 8KB).
RegAssign(&mudstate.global_regs[regnum], vlen, value);
ctx->x[10] = vlen;
}
} else {
ctx->x[10] = 0;
}
return -1;
}
case ECALL_ARENA_ALLOC: {
size_t size = static_cast<size_t>(ctx->x[10]);
auto *a = JITArena::Alloc(size);
if (a) {
ctx->x[10] = a->id;
ctx->x[11] = a->used - size;
} else {
ctx->x[10] = 0;
}
return -1;
}
case ECALL_ARENA_REF:
JITArena::AddRef(static_cast<uint32_t>(ctx->x[10]));
return -1;
case ECALL_ARENA_RELEASE:
JITArena::Release(static_cast<uint32_t>(ctx->x[10]));
return -1;
case ECALL_DMA_SUBMIT: {
// a0=window, a1=length, a2=op
jit_dma_controller::submit(static_cast<int>(ctx->x[10]),
static_cast<size_t>(ctx->x[11]),
static_cast<int>(ctx->x[12]),
ec);
return -1;
}
case ECALL_DMA_ACK: {
ctx->x[10] = static_cast<uint64_t>(jit_dma_controller::get_next_ack());
return -1;
}
case ECALL_FTOA: {
// a0 = double bits (via FMV.X.D), a1 = output guest address.
double val;
uint64_t bits = ctx->x[10];
memcpy(&val, &bits, 8);
uint64_t out_addr = ctx->x[11];
if (out_addr < ec->memory_size - 64) {
char *out = reinterpret_cast<char *>(ec->memory + out_addr);
LBuf buf = LBuf_Src("ecall.fmvxd");
UTF8 *bufc = buf;
fval(buf, &bufc, val);
*bufc = '\0';
size_t len = bufc - buf;
if (len > 63) len = 63;
memcpy(out, buf, len);
out[len] = '\0';
}
return -1;
}
case ECALL_LUA_FTOA: {
// a0 = double bits (via FMV.X.D), a1 = output guest address.
// Same shape as ECALL_FTOA, but rendered Lua's way: "%.14g" plus a
// trailing ".0" when the result looks like an integer, so a Lua float
// keeps its subtype in the result string (#1488).
double val;
uint64_t bits = ctx->x[10];
memcpy(&val, &bits, 8);
uint64_t out_addr = ctx->x[11];
if (out_addr < ec->memory_size - 64) {
char *out = reinterpret_cast<char *>(ec->memory + out_addr);
char buf[64];
lua_format_double(val, buf, sizeof(buf));
size_t len = strlen(buf);
if (len > 63) len = 63;
memcpy(out, buf, len);
out[len] = '\0';
}
return -1;
}
case ECALL_ATOF: {
// a0 = guest address of string → store double in fa0 (f[10]).
uint64_t str_addr = ctx->x[10];
double val = 0.0;
if (str_addr < ec->memory_size - 1) {
const char *s = reinterpret_cast<const char *>(ec->memory + str_addr);
val = mux_atof(reinterpret_cast<const UTF8 *>(s));
}
memcpy(&ctx->f[10], &val, 8);
return -1;
}
case ECALL_GOOD_OBJ: {
// a0 = dbref integer → a0 = 1 if Good_obj, 0 otherwise.
// This is a leaf database lookup — no softcode evaluation,
// no re-entrancy risk.
dbref obj = static_cast<dbref>(ctx->x[10]);
ctx->x[10] = Good_obj(obj) ? 1 : 0;
return -1;
}
case ECALL_CHR: {
// a0 = guest addr of input string (space-separated codepoints)
// a1 = guest addr of output buffer
// Returns a0 = 0 on success, -1 on error (output holds error msg).
uint64_t in_addr = ctx->x[10];
uint64_t out_addr = ctx->x[11];
if (in_addr >= ec->memory_size || out_addr >= ec->memory_size - 64) {
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
const UTF8 *pArg = ec->memory + in_addr;
char *out = reinterpret_cast<char *>(ec->memory + out_addr);
size_t out_max = 7999;
// Build raw UTF-8 from space-separated codepoints.
LBuf raw = LBuf_Src("ecall.chr");
UTF8 *pRaw = raw;
const UTF8 *pEnd = raw.get() + LBUF_SIZE - 5;
bool bAny = false;
while ('\0' != *pArg) {
while (mux_isspace(*pArg)) pArg++;
if ('\0' == *pArg) break;
bool bNeg = ('-' == *pArg);
if ('-' == *pArg || '+' == *pArg) pArg++;
if (!mux_isdigit(*pArg)) {
memcpy(out, "#-1 ARGUMENT MUST BE A NUMBER", 30);
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
uint64_t uv = 0;
while (mux_isdigit(*pArg)) {
const uint64_t digit = static_cast<uint64_t>(*pArg - '0');
if (uv > (UINT64_MAX - digit) / 10ULL) {
memcpy(out, "#-1 ARGUMENT OUT OF RANGE", 26);
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
uv = 10ULL * uv + digit;
pArg++;
}
if ('\0' != *pArg && !mux_isspace(*pArg)) {
memcpy(out, "#-1 ARGUMENT MUST BE A NUMBER", 30);
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
int64_t iv = bNeg ? -static_cast<int64_t>(uv) : static_cast<int64_t>(uv);
if (iv < 0 || iv > static_cast<int64_t>(UNI_MAX_LEGAL_UTF32)
|| (static_cast<UTF32>(iv) >= UNI_SUR_HIGH_START
&& static_cast<UTF32>(iv) <= UNI_SUR_LOW_END)) {
memcpy(out, "#-1 ARGUMENT OUT OF RANGE", 26);
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
UTF32 ch = static_cast<UTF32>(iv);
UTF8 *p = ConvertToUTF8(ch);
if (!mux_isprint(p)) {
memcpy(out, "#-1 UNPRINTABLE CHARACTER", 26);
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
size_t nb = strlen(reinterpret_cast<const char *>(p));
if (pRaw + nb <= pEnd) {
memcpy(pRaw, p, nb);
pRaw += nb;
}
bAny = true;
}
*pRaw = '\0';
if (!bAny) {
memcpy(out, "#-1 ARGUMENT MUST BE A NUMBER", 30);
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
// NFC normalize.
size_t nRaw = pRaw - raw;
LBuf nfc = LBuf_Src("ecall.chr.nfc");
size_t nNfc;
utf8_normalize_nfc(raw, nRaw, nfc, LBUF_SIZE - 1, &nNfc);
nfc[nNfc] = '\0';
if (nNfc > out_max) nNfc = out_max;
memcpy(out, nfc, nNfc);
out[nNfc] = '\0';
ctx->x[10] = 0;
return -1;
}
case ECALL_ORD: {
// a0 = guest addr of input string
// a1 = guest addr of output buffer
// Returns a0 = 0 on success, -1 on error.
uint64_t in_addr = ctx->x[10];
uint64_t out_addr = ctx->x[11];
if (in_addr >= ec->memory_size || out_addr >= ec->memory_size - 64) {
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
const UTF8 *pIn = ec->memory + in_addr;
char *out = reinterpret_cast<char *>(ec->memory + out_addr);
// Strip color.
size_t nBytes = 0;
UTF8 *p = strip_color(pIn, &nBytes, nullptr);
if (0 == nBytes) {
memcpy(out, "#-1 FUNCTION EXPECTS ONE CHARACTER", 35);
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
// First grapheme cluster.
mux_cursor cluster = utf8_next_grapheme(p, nBytes);
if (0 == cluster.m_byte) {
memcpy(out, "#-1 STRING IS INVALID", 22);
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
// Exactly one cluster.
if (cluster.m_byte < nBytes) {
mux_cursor second = utf8_next_grapheme(p + cluster.m_byte,
nBytes - cluster.m_byte);
if (0 < second.m_byte) {
memcpy(out, "#-1 FUNCTION EXPECTS ONE CHARACTER", 35);
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
memcpy(out, "#-1 STRING IS INVALID", 22);
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
// Decode codepoints.
//
// This loop is fun_ord()'s, transliterated -- but where the
// interpreter writes through safe_chr/safe_ltoa, which stop at the
// buffer end, this had a bare `op += sprintf(op, ...)` and no bound
// at all. The only check above is `out_addr >= memory_size - 64`,
// which reads like it bounds the write and does not: it guarantees
// 64 bytes of headroom against a loop that emits up to 8 bytes per
// CODEPOINT, and a grapheme cluster is one cluster but any number of
// codepoints.
//
// Measured before the fix, ord() on 'a' followed by N combining
// acutes -- one cluster, N+1 codepoints -- wrote linearly with no
// cap: N=400 produced 1602 bytes, N=1200 produced 4802, N=3800
// produced ~15k. It did not crash only because out_addr happens to
// sit far enough from the end of guest memory; everything past the
// result slot was overwritten regardless. Input is player-supplied.
//
// Bound by the space that actually exists and stop cleanly, which is
// also what the interpreter does when the LBUF fills.
//
const size_t out_avail = ec->memory_size - out_addr;
char *op = out;
size_t used = 0;
const UTF8 *q = p;
const UTF8 *qEnd = p + cluster.m_byte;
bool bFirst = true;
while (q < qEnd) {
UTF32 ch = ConvertFromUTF8(q);
if (UNI_EOF == ch) {
memcpy(out, "#-1 STRING IS INVALID", 22);
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
if (!bFirst) {
if (used + 2 > out_avail) break; // separator plus NUL
*op++ = ' ';
used++;
}
size_t k = mux_snprintf(reinterpret_cast<UTF8 *>(op),
out_avail - used, T("%ld"), static_cast<long>(ch));
if (0 == k) break;
op += k;
used += k;
bFirst = false;
size_t nAdv = utf8_FirstByte[static_cast<unsigned char>(*q)];
if (nAdv < 1 || nAdv >= UTF8_CONTINUE) nAdv = 1;
q += nAdv;
}
*op = '\0';
ctx->x[10] = 0;
return -1;
}
case ECALL_TRANSLATE: {
// a0 = guest addr of input string
// a1 = type (0=spaces, 1=percent substitutions)
// a2 = guest addr of output buffer
uint64_t in_addr = ctx->x[10];
int type = static_cast<int>(ctx->x[11]);
uint64_t out_addr = ctx->x[12];
if (in_addr >= ec->memory_size || out_addr >= ec->memory_size - 64) {
ctx->x[10] = static_cast<uint64_t>(-1);
return -1;
}
const UTF8 *pIn = ec->memory + in_addr;
char *out = reinterpret_cast<char *>(ec->memory + out_addr);
UTF8 *result = translate_string(pIn, type != 0);
size_t len = strlen(reinterpret_cast<const char *>(result));
if (len > 7999) len = 7999;
memcpy(out, result, len);
out[len] = '\0';
ctx->x[10] = 0;
return -1;
}
case ECALL_QUICK_WILD: {
// a0 = guest addr of pattern, a1 = guest addr of data string
// Returns a0 = 1 on match, 0 on no match.
// Uses quick_wild() which pre-lowercases the pattern with
// mux_strlwr() for Unicode-aware case-insensitive matching.
uint64_t pat_addr = ctx->x[10];
uint64_t data_addr = ctx->x[11];
if (pat_addr >= ec->memory_size || data_addr >= ec->memory_size) {
ctx->x[10] = 0;
return -1;
}
const UTF8 *pat = ec->memory + pat_addr;
const UTF8 *data = ec->memory + data_addr;
mudstate.wild_invk_ctr = 0;
ctx->x[10] = quick_wild(pat, data) ? 1 : 0;
return -1;
}
case ECALL_SORT: {
// a0 = guest addr of list string
// a1 = sort_type char (e.g. 'a', 'n', 'u')
// a2 = delim char
// a3 = osep char
// a4 = guest addr of output buffer
uint64_t list_addr = ctx->x[10];
char sort_type = static_cast<char>(ctx->x[11]);
unsigned char delim = static_cast<unsigned char>(ctx->x[12]);
unsigned char osep = static_cast<unsigned char>(ctx->x[13]);
uint64_t out_addr = ctx->x[14];
if (list_addr >= ec->memory_size ||
out_addr >= ec->memory_size - 64) {
ctx->x[10] = 0;
return -1;
}
const UTF8 *list_in = ec->memory + list_addr;
UTF8 *out = ec->memory + out_addr;
size_t n = sort_to_buffer(list_in, sort_type, delim, osep,
out, LBUF_SIZE - 1);
ctx->x[10] = n;
return -1;
}
// ---- Lua VM ECALLs ----
// These call back into the Lua interpreter via lua_State *L.
// They handle table operations that the JIT can't do natively.
case ECALL_LUA_NEWTABLE: {
if (!ec->lua_state) { ctx->x[10] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int narr = static_cast<int>(ctx->x[10]);
int nrec = static_cast<int>(ctx->x[11]);
lua_createtable(L, narr, nrec);
// Return the absolute stack index of the new table.
ctx->x[10] = static_cast<uint64_t>(lua_gettop(L));
return -1;
}
case ECALL_LUA_GETI_INT: {
// Integer-keyed table get (#1751 Phase 1, revised). The lowering
// only emits this against a PLAIN-PROVEN table (built by this
// chunk's NEWTABLE, never escaped), so no metamethod can fire and
// every stored value is an integer by construction. The one
// honest miss left is an ABSENT key: the value is Lua nil, which
// a typed integer slot cannot carry. Continuing with 0 was the
// pre-#1751 behavior and a measured silent wrong answer
// (`return t[2]` gave "0" where the interpreter gave "") -- so an
// absent key is a loud post-entry fail until nil is
// representable. The stack-index and integer checks are belts:
// the proof should make them unreachable, and their firing means
// a compiler bug, which is exactly what loud is for.
//
if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int tbl_idx = static_cast<int>(ctx->x[10]);
lua_Integer key = static_cast<lua_Integer>(ctx->x[11]);
if (!ecall_lua_stack_index_ok(L, tbl_idx)) {
return lua_ecall_decline("ECALL_LUA_GETI_INT_BADIDX");
}
int pr = ecall_lua_pget_intkey(L, tbl_idx, key);
if (0 != pr) {
return pr;
}
if (!lua_isinteger(L, -1)) {
lua_pop(L, 1);
return lua_ecall_decline("ECALL_LUA_GETI_INT_NONINT");
}
ctx->x[10] = static_cast<uint64_t>(lua_tointeger(L, -1));
ctx->x[11] = 1;
lua_pop(L, 1);
return -1;
}
case ECALL_LUA_GETGLOBAL: {
// a0 = guest addr of name -> a0 = stack index of the global (handle).
// Nil globals stay on the stack like the interpreter (#1751 Phase 1);
// never decline for missing names.
//
// #1836: unprotected lua_getglobal can abort via a raising
// _ENV __index metamethod. Route through pcall like GET/SET/LEN.
//
if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
const char *key = guest_cstr(ec->memory, ec->memory_size, ctx->x[10]);
if (nullptr == key) {
ctx->x[10] = 0;
ctx->x[11] = 0;
return -1;
}
int pr = ecall_lua_pgetglobal(L, key);
if (0 != pr) {
return pr;
}
ctx->x[10] = static_cast<uint64_t>(lua_gettop(L));
ctx->x[11] = 1;
return -1;
}
case ECALL_LUA_GETFIELD_REF: {
// a0=tbl_idx, a1=key addr -> a0=stack index of the field (handle).
// Total: real gettable under pcall; nil fields stay as handles.
//
if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int tbl_idx = static_cast<int>(ctx->x[10]);
const char *key = guest_cstr(ec->memory, ec->memory_size, ctx->x[11]);
if (nullptr == key || !ecall_lua_stack_index_ok(L, tbl_idx)) {
ctx->x[10] = 0;
ctx->x[11] = 0;
return -1;
}
int pr = ecall_lua_pget_strkey(L, tbl_idx, key);
if (0 != pr) {
return pr;
}
ctx->x[10] = static_cast<uint64_t>(lua_gettop(L));
ctx->x[11] = 1;
return -1;
}
case ECALL_LUA_CALL_STR: {
// a0=fn stack idx, a1 = nargs | (argkind bits << 8), a2..args,
// out addr/size in a5/a6. #1751: after the callee may have run,
// never soft-decline. Marshal the first result like fun_lua
// (numbers become decimal text — string.find's first value is an
// integer; the chunk pcall keeps one result on both routes).
//
if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int fn_idx = static_cast<int>(ctx->x[10]);
int nargs = static_cast<int>(ctx->x[11] & 0xFF);
int kinds = static_cast<int>((ctx->x[11] >> 8) & 0xFF);
uint64_t out_addr = ctx->x[15];
uint64_t out_size = ctx->x[16];
if (nargs < 0 || nargs > 3
|| 0 == out_size
|| !guest_range_ok(out_addr, out_size, ec->memory_size)) {
return ecall_lua_error_cstr("invalid call encoding");
}
if (fn_idx <= 0 || fn_idx > lua_gettop(L)
|| !lua_isfunction(L, fn_idx)) {
return ecall_lua_error_cstr("attempt to call a non-function value");
}
int base = lua_gettop(L);
lua_pushvalue(L, fn_idx);
if (!ecall_lua_push_call_args(L, ctx, ec, nargs, kinds)) {
lua_settop(L, base);
return ecall_lua_error_cstr("invalid call argument");
}
if (LUA_OK != lua_pcall(L, nargs, 1, 0)) {
int er = ecall_lua_commit_error(L);
lua_settop(L, base);
return er;
}
size_t slen = ecall_lua_marshal_to_guest(L, -1,
ec->memory + out_addr, static_cast<size_t>(out_size));
lua_settop(L, base);
ctx->x[10] = static_cast<uint64_t>(slen);
ctx->x[11] = 1;
return -1;
}
case ECALL_LUA_CALL_VAL: {
// Typed call result (#1764 shape 2): leave the first pcall value
// on the Lua stack and return its absolute index as a handle.
// Marshal only at the softcode boundary (HIR_LUA_MARSHAL / RET).
//
if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int fn_idx = static_cast<int>(ctx->x[10]);
int nargs = static_cast<int>(ctx->x[11] & 0xFF);
int kinds = static_cast<int>((ctx->x[11] >> 8) & 0xFF);
if (nargs < 0 || nargs > 3) {
return ecall_lua_error_cstr("invalid call encoding");
}
if (fn_idx <= 0 || fn_idx > lua_gettop(L)
|| !lua_isfunction(L, fn_idx)) {
return ecall_lua_error_cstr("attempt to call a non-function value");
}
int base = lua_gettop(L);
lua_pushvalue(L, fn_idx);
if (!ecall_lua_push_call_args(L, ctx, ec, nargs, kinds)) {
lua_settop(L, base);
return ecall_lua_error_cstr("invalid call argument");
}
if (LUA_OK != lua_pcall(L, nargs, 1, 0)) {
int er = ecall_lua_commit_error(L);
lua_settop(L, base);
return er;
}
// Result remains at top; do not pop.
ctx->x[10] = static_cast<uint64_t>(lua_gettop(L));
ctx->x[11] = 1;
return -1;
}
case ECALL_LUA_MARSHAL: {
// a0=stack idx, a1=out addr, a2=out size → a0=len. fun_lua rules.
if (!ec->lua_state) { ctx->x[10] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int idx = static_cast<int>(ctx->x[10]);
uint64_t out_addr = ctx->x[11];
uint64_t out_size = ctx->x[12];
if (0 == out_size
|| !guest_range_ok(out_addr, out_size, ec->memory_size)
|| !ecall_lua_stack_index_ok(L, idx)) {
return ecall_lua_error_cstr("invalid marshal encoding");
}
size_t slen = ecall_lua_marshal_to_guest(L, idx,
ec->memory + out_addr, static_cast<size_t>(out_size));
ctx->x[10] = static_cast<uint64_t>(slen);
return -1;
}
case ECALL_LUA_TOBOOL: {
// a0=stack idx → a0 = 0/1 under Lua truthiness (only nil/false falsy).
if (!ec->lua_state) { ctx->x[10] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int idx = static_cast<int>(ctx->x[10]);
if (!ecall_lua_stack_index_ok(L, idx)) {
ctx->x[10] = 0;
return -1;
}
ctx->x[10] = lua_toboolean(L, idx) ? 1 : 0;
return -1;
}
case ECALL_LUA_EQ: {
// a0=lhs stack idx, a1=rhs kind, a2=rhs payload → a0=0/1 under Lua ==.
// Kind: 0=int, 1=string guest addr, 2=handle stack idx, 3=nil, 4=bool.
// Keeps type distinctions that a marshal-then-STRCMP would erase
// (tonumber("17") == "17" is false; tostring(0) == "0" is true).
//
if (!ec->lua_state) { ctx->x[10] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int lhs = static_cast<int>(ctx->x[10]);
int kind = static_cast<int>(ctx->x[11]);
uint64_t rhs = ctx->x[12];
if (!ecall_lua_stack_index_ok(L, lhs)) {
ctx->x[10] = 0;
return -1;
}
const int base = lua_gettop(L);
lua_pushvalue(L, lhs);
switch (kind) {
case 0:
lua_pushinteger(L, static_cast<lua_Integer>(
static_cast<int64_t>(rhs)));
break;
case 1: {
const char *s = guest_cstr(ec->memory, ec->memory_size, rhs);
if (nullptr == s) {
lua_settop(L, base);
ctx->x[10] = 0;
return -1;
}
lua_pushstring(L, s);
break;
}
case 2: {
int ridx = static_cast<int>(rhs);
if (!ecall_lua_stack_index_ok(L, ridx)) {
lua_settop(L, base);
ctx->x[10] = 0;
return -1;
}
lua_pushvalue(L, ridx);
break;
}
case 3:
lua_pushnil(L);
break;
case 4:
lua_pushboolean(L, rhs != 0);
break;
default:
lua_settop(L, base);
ctx->x[10] = 0;
return -1;
}
// Absolute indices: pushvalue after the cfunction would shift
// relative slots. #1836: __eq can raise; protect like GETGLOBAL.
//
const int lhs_abs = base + 1;
const int rhs_abs = base + 2;
int eq = 0;
int pr = ecall_lua_pcompare_eq(L, lhs_abs, rhs_abs, &eq);
lua_settop(L, base);
if (0 != pr) {
return pr;
}
ctx->x[10] = eq ? 1 : 0;
return -1;
}
case ECALL_LUA_CALL_INT: {
// a0=fn stack idx, a1=nargs (0..2), a2=arg0, a3=arg1, integers.
// -> a0 = integer result, a1 = ok.
//
// Deliberately narrow: integer in, integer out, so this exercises
// the whole global->field->call chain without also deciding how a
// string result is marshalled. Anything else declines and the
// interpreter answers.
if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int fn_idx = static_cast<int>(ctx->x[10]);
int nargs = static_cast<int>(ctx->x[11] & 0xFF);
int kinds = static_cast<int>((ctx->x[11] >> 8) & 0xFF);
if (fn_idx <= 0 || fn_idx > lua_gettop(L)
|| !lua_isfunction(L, fn_idx) || nargs < 0 || nargs > 3) {
ctx->x[11] = 0;
return lua_ecall_decline("ECALL_LUA_CALL_INT");
}
int base = lua_gettop(L);
lua_pushvalue(L, fn_idx);
// Same argument encoding as CALL_STR, one decoder for both: the two
// differ only in the RESULT type, so they have no business
// differing in how arguments arrive -- tonumber("17") returns an
// integer from a string argument and needs both halves.
if (!ecall_lua_push_call_args(L, ctx, ec, nargs, kinds)) {
lua_settop(L, base);
ctx->x[11] = 0;
return lua_ecall_decline("ECALL_LUA_CALL_INT");
}
if (LUA_OK != lua_pcall(L, nargs, 1, 0)) {
lua_settop(L, base);
ctx->x[11] = 0;
return lua_ecall_decline("ECALL_LUA_CALL_INT");
}
if (!lua_isinteger(L, -1)) {
lua_settop(L, base);
ctx->x[11] = 0;
return lua_ecall_decline("ECALL_LUA_CALL_INT");
}
ctx->x[10] = static_cast<uint64_t>(lua_tointeger(L, -1));
ctx->x[11] = 1;
lua_settop(L, base);
return -1;
}
case ECALL_LUA_CALL_VOID: {
// Call for effect: same encoding as CALL_INT/CALL_STR, result
// discarded. pcall is asked for zero results. After the callee
// may have run, do not soft-decline: a raised error is a real Lua
// error (same as CALL_STR). Pre-entry encoding misses still use
// residual decline.
//
if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int fn_idx = static_cast<int>(ctx->x[10]);
int nargs = static_cast<int>(ctx->x[11] & 0xFF);
int kinds = static_cast<int>((ctx->x[11] >> 8) & 0xFF);
if (fn_idx <= 0 || fn_idx > lua_gettop(L)
|| !lua_isfunction(L, fn_idx) || nargs < 0 || nargs > 3) {
ctx->x[11] = 0;
return lua_ecall_decline("ECALL_LUA_CALL_VOID");
}
int base = lua_gettop(L);
lua_pushvalue(L, fn_idx);
if (!ecall_lua_push_call_args(L, ctx, ec, nargs, kinds)) {
lua_settop(L, base);
return ecall_lua_error_cstr("invalid call argument");
}
if (LUA_OK != lua_pcall(L, nargs, 0, 0)) {
int er = ecall_lua_commit_error(L);
lua_settop(L, base);
return er;
}
lua_settop(L, base);
ctx->x[11] = 1;
return -1;
}
case ECALL_LUA_LIMITED: {
// Back-edge budget exhausted (#1732 / #1751 Phase 3). Commit the
// same text InsnCountHook raises on the interpreter route
// (luaL_error "instruction limit exceeded" → softcode framing).
// No re-run: the compiled path already spent the budget, and a
// silent interpreter replay would double side effects.
//
return ecall_lua_error_cstr("instruction limit exceeded");
}
case ECALL_LUA_GETFIELD_INT: {
// String-keyed int read via real gettable (#1751 Phase 1).
// Typed claim: integer. Non-integer → ok=0, no decline.
//
if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int tbl_idx = static_cast<int>(ctx->x[10]);
const char *key = guest_cstr(ec->memory, ec->memory_size, ctx->x[11]);
if (nullptr == key || !ecall_lua_stack_index_ok(L, tbl_idx)) {
ctx->x[10] = 0;
ctx->x[11] = 0;
return -1;
}
int pr = ecall_lua_pget_strkey(L, tbl_idx, key);
if (0 != pr) {
return pr;
}
if (lua_isinteger(L, -1)) {
ctx->x[10] = static_cast<uint64_t>(lua_tointeger(L, -1));
ctx->x[11] = 1;
} else {
ctx->x[10] = 0;
ctx->x[11] = 0;
}
lua_pop(L, 1);
return -1;
}
case ECALL_LUA_GETFIELD_FLT: {
// String-keyed float read via real gettable (#1751 Phase 1).
// Only a genuine non-integer number fills the FP slot; else ok=0.
//
if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int tbl_idx = static_cast<int>(ctx->x[10]);
const char *key = guest_cstr(ec->memory, ec->memory_size, ctx->x[11]);
if (nullptr == key || !ecall_lua_stack_index_ok(L, tbl_idx)) {
ctx->x[10] = 0;
ctx->x[11] = 0;
return -1;
}
int pr = ecall_lua_pget_strkey(L, tbl_idx, key);
if (0 != pr) {
return pr;
}
if (LUA_TNUMBER == lua_type(L, -1) && !lua_isinteger(L, -1)) {
double d = lua_tonumber(L, -1);
uint64_t bits;
memcpy(&bits, &d, 8);
ctx->x[10] = bits;
ctx->x[11] = 1;
} else {
ctx->x[10] = 0;
ctx->x[11] = 0;
}
lua_pop(L, 1);
return -1;
}
case ECALL_LUA_SETFIELD_INT: {
// a0=tbl_idx, a1=guest key addr, a2=integer value. Total settable.
//
if (!ec->lua_state) return -1;
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int tbl_idx = static_cast<int>(ctx->x[10]);
const char *key = guest_cstr(ec->memory, ec->memory_size, ctx->x[11]);
lua_Integer val = static_cast<lua_Integer>(ctx->x[12]);
if (nullptr == key || !ecall_lua_stack_index_ok(L, tbl_idx)) {
return -1;
}
int pr = ecall_lua_pset_strkey(L, tbl_idx, key, val);
if (0 != pr) {
return pr;
}
return -1;
}
case ECALL_LUA_INSN_BUDGET: {
// a0 = the CURRENT lua_instruction_limit. The whole point is that
// this reads mudconf at run time: the program carries no config
// value, so @admin lua_instruction_limit takes effect on the next
// run of every cached and persisted blob alike (#1745, #1613).
int64_t lim = static_cast<int64_t>(mudconf.lua_instruction_limit);
if (lim < 1)
{
lim = 1; // a zero/negative limit must abort loops, not arm an
// effectively-unbounded unsigned countdown
}
ctx->x[10] = static_cast<uint64_t>(lim);
return -1;
}
case ECALL_LUA_LEN_INT: {
// Lua's # via luaL_len under pcall (#1751 Phase 1 / #1424 root).
// Honours __len; raises as the interpreter would (committed error).
//
if (!ec->lua_state) { ctx->x[11] = 0; return -1; }
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int tbl_idx = static_cast<int>(ctx->x[10]);
if (!ecall_lua_stack_index_ok(L, tbl_idx)) {
ctx->x[10] = 0;
ctx->x[11] = 0;
return -1;
}
int pr = ecall_lua_plen(L, tbl_idx);
if (0 != pr) {
return pr;
}
ctx->x[10] = static_cast<uint64_t>(lua_tointeger(L, -1));
ctx->x[11] = 1;
lua_pop(L, 1);
return -1;
}
case ECALL_LUA_SETI_INT: {
// Integer-keyed table set via real settable (#1751 Phase 1).
//
if (!ec->lua_state) return -1;
lua_State *L = static_cast<lua_State *>(ec->lua_state);
int tbl_idx = static_cast<int>(ctx->x[10]);
lua_Integer key = static_cast<lua_Integer>(ctx->x[11]);
lua_Integer val = static_cast<lua_Integer>(ctx->x[12]);
if (!ecall_lua_stack_index_ok(L, tbl_idx)) {
return -1;
}
int pr = ecall_lua_pset_intkey(L, tbl_idx, key, val);
if (0 != pr) {
return pr;
}
return -1;
}
case ECALL_CALL_COMPILED: {
// Re-entrant call into a compiled function within the
// persistent VM. Saves the full CPU context, runs the
// inner function via dbt_resume, restores context.
//
// a0 = entry_pc of target function
// a1 = guest addr of target's output buffer
// Returns: a0 = result string length (0 on failure)
//
if (!ec->dbt) {
ctx->x[10] = 0;
return -1;
}
uint64_t target_pc = ctx->x[10];
uint64_t out_ref = ctx->x[11];
// Save outer CPU context.
rv64_ctx_t saved_ctx = *ctx;
// Run inner function.
int inner_rc = dbt_resume(ec->dbt, target_pc);
// Extract result length (#1057: bound guest NUL scan).
uint64_t result_len = 0;
uint64_t resolved_out = resolve_runtime_out_addr(out_ref, saved_ctx.x[2]);
size_t n = 0;
if (inner_rc == 0 && resolved_out > 0
&& guest_strnlen(ec->memory, ec->memory_size, resolved_out, &n)) {
result_len = n;
}
// Restore outer context.
*ctx = saved_ctx;
ctx->x[10] = result_len;
ctx->x[11] = resolved_out;
return -1; // continue outer execution
}
case ECALL_COMPILE_ATTR: {
// Resolve an attribute, compile its body into the persistent
// VM, and return the entry point.
//
// a0 = dbref of target object
// a1 = guest addr of attribute name string
// Returns: a0 = entry_pc (0 on failure)
// a1 = out_addr
// a2 = aflags
//
persistent_vm_t *pvm =
static_cast<persistent_vm_t *>(ec->pvm);
if (!pvm) {
ctx->x[10] = 0;
return -1;
}
dbref obj = static_cast<dbref>(ctx->x[10]);
uint64_t name_addr = ctx->x[11];
if (!Good_obj(obj) || name_addr >= ec->memory_size) {
ctx->x[10] = 0;
return -1;
}
const UTF8 *attr_name = ec->memory + name_addr;
ATTR *ap = atr_str(attr_name);
if (!ap) {
ctx->x[10] = 0;
return -1;
}
if (!See_attr(ec->executor, obj, ap)) {
ctx->x[10] = 0;
return -1;
}
dbref aowner;
int aflags;
UTF8 *atext = atr_pget(obj, ap->number, &aowner, &aflags);
if (!atext || !*atext) {
if (atext) free_lbuf(atext);
ctx->x[10] = 0;
return -1;
}
// Skip compilation if NOEVAL.
if ((aflags & AF_NOEVAL) || NoEval(obj)) {
free_lbuf(atext);
ctx->x[10] = 0;
ctx->x[12] = static_cast<uint64_t>(aflags);
return -1;
}
// Compile (with caching).
size_t alen = strlen(reinterpret_cast<const char *>(atext));
auto cr = pvm->compile_attr(obj, ap->number, atext, alen);
free_lbuf(atext);
ctx->x[10] = cr.entry_pc;
ctx->x[11] = cr.out_addr;
ctx->x[12] = static_cast<uint64_t>(aflags);
return -1; // continue
}
default:
ctx->x[10] = 0;
return -1;
}
}
// ---------------------------------------------------------------
// jit_eval: try to compile and execute an expression via the JIT.
//
// Returns true if the JIT handled it (result written to buff/bufc).
// ---------------------------------------------------------------
// Tier 3 helper functions: guest-memory CARGS operations.
//
// These are FUNCTION() handlers registered in the standard function
// table. They use s_current_ecall_ctx (set by ecall_invoke_fun)
// to access JIT guest memory for CARGS slot operations.
// ---------------------------------------------------------------
static constexpr int MAX_CARG_SAVE_DEPTH = 16;
static struct {
uint8_t data[10 * rv_compiler::CARGS_SLOT];
int ncargs;
bool in_use;
} s_carg_save_stack[MAX_CARG_SAVE_DEPTH];
// Executor-context stack for inlined u()/ulocal() bodies (#2179).
// fun_u evaluates the attribute body with executor = the object holding
// the attribute (and caller = the previous executor); an inlined body
// runs inside the CALLER's program, so without this swap every
// executor-derived thing inside it was the caller's: %! (SUBST slot),
// %va-%vz / %=<name> (xget against the SUBST slot), and every ECALL's
// ec->executor (v(), bare-name u(), name(me), permission checks).
//
static constexpr int MAX_UEXEC_SAVE_DEPTH = 16;
static struct {
dbref executor;
dbref caller;
bool in_use;
} s_uexec_save_stack[MAX_UEXEC_SAVE_DEPTH];
// Both save stacks leak their slot if a program is abandoned between a
// save/push and its restore/pop (DBT decline, error unwind): nothing
// runs the paired helper, in_use stays set, and the stack eventually
// fills — after which _SAVE_CARGS returns -1 (restore skipped, CARGS
// clobbered for the caller) and _PUSH_UEXEC falls back to the fun_u
// ECALL. A top-level entry proves no outer program holds a live
// handle, so everything still marked in-use is leaked garbage.
// run_cached_program calls this when s_run_cached_depth == 0.
//
static void jit_helper_stacks_reset()
{
for (int i = 0; i < MAX_CARG_SAVE_DEPTH; i++) {
s_carg_save_stack[i].in_use = false;
}
for (int i = 0; i < MAX_UEXEC_SAVE_DEPTH; i++) {
s_uexec_save_stack[i].in_use = false;
}
}
// Render "#<dbref>" into the guest %! substitution slot so inlined-body
// reads of %! / %va-%vz / %=<name> see the swapped executor. Writing
// when the program never reads the slot is harmless.
//
static void uexec_write_subst(eval_ctx *ec, dbref executor)
{
uint64_t slot = rv_compiler::SUBST_BASE
+ static_cast<uint64_t>(rv_compiler::SUBST_EXECUTOR)
* rv_compiler::SUBST_SLOT;
if (slot + rv_compiler::SUBST_SLOT <= ec->memory_size)
{
mux_sprintf(ec->memory + slot, rv_compiler::SUBST_SLOT,
T("#%d"), executor);
}
}
// _PUSH_UEXEC(thing_dbref_str): enter an inlined u() body's executor
// context (#2179) — save executor/caller, set caller = old executor and
// executor = thing (mirroring fun_u's mux_exec call), refresh the guest
// %! slot. Returns a handle for _POP_UEXEC, or "-1" on failure — the
// lowering branches a failed push to the fun_u ECALL fallback, which
// establishes its own context, so exhaustion costs the inline, never
// correctness.
//
FUNCTION(fun__push_uexec)
{
UNUSED_PARAMETER(executor);
UNUSED_PARAMETER(caller);
UNUSED_PARAMETER(enactor);
UNUSED_PARAMETER(eval);
UNUSED_PARAMETER(cargs);
UNUSED_PARAMETER(ncargs);
eval_ctx *ec = s_current_ecall_ctx;
dbref thing = (nfargs >= 1) ? mux_atoi64(fargs[0]) : NOTHING;
if (!ec || !Good_obj(thing))
{
safe_str(T("-1"), buff, bufc);
return;
}
for (int i = 0; i < MAX_UEXEC_SAVE_DEPTH; i++)
{
if (!s_uexec_save_stack[i].in_use)
{
s_uexec_save_stack[i].executor = ec->executor;
s_uexec_save_stack[i].caller = ec->caller;
s_uexec_save_stack[i].in_use = true;
ec->caller = ec->executor;
ec->executor = thing;
uexec_write_subst(ec, thing);
safe_ltoa(i, buff, bufc);
return;
}
}
safe_str(T("-1"), buff, bufc);
}
// _POP_UEXEC(handle_str): restore the pre-inline executor context and
// the guest %! slot.
//
FUNCTION(fun__pop_uexec)
{
UNUSED_PARAMETER(executor);
UNUSED_PARAMETER(caller);
UNUSED_PARAMETER(enactor);
UNUSED_PARAMETER(eval);
UNUSED_PARAMETER(cargs);
UNUSED_PARAMETER(ncargs);
UNUSED_PARAMETER(buff);
UNUSED_PARAMETER(bufc);
eval_ctx *ec = s_current_ecall_ctx;
if (!ec || nfargs < 1) return;
int64_t idx = mux_atoi64(fargs[0]);
if (idx >= 0 && idx < MAX_UEXEC_SAVE_DEPTH
&& s_uexec_save_stack[idx].in_use)
{
ec->executor = s_uexec_save_stack[idx].executor;
ec->caller = s_uexec_save_stack[idx].caller;
s_uexec_save_stack[idx].in_use = false;
uexec_write_subst(ec, ec->executor);
}
}
// _SAVE_CARGS(): save the CARGS region of guest memory.
// Returns a handle string (index into save stack).
//
FUNCTION(fun__save_cargs)
{
UNUSED_PARAMETER(executor);
UNUSED_PARAMETER(caller);
UNUSED_PARAMETER(enactor);
UNUSED_PARAMETER(eval);
UNUSED_PARAMETER(nfargs);
UNUSED_PARAMETER(cargs);
UNUSED_PARAMETER(ncargs);
if (!s_current_ecall_ctx)
{
safe_str(T("-1"), buff, bufc);
return;
}
for (int i = 0; i < MAX_CARG_SAVE_DEPTH; i++)
{
if (!s_carg_save_stack[i].in_use)
{
uint64_t base = rv_compiler::CARGS_BASE;
size_t region = 10 * rv_compiler::CARGS_SLOT;
if (base + region <= s_current_ecall_ctx->memory_size)
{
memcpy(s_carg_save_stack[i].data,
s_current_ecall_ctx->memory + base, region);
}
s_carg_save_stack[i].ncargs = s_current_ecall_ctx->ncargs;
s_carg_save_stack[i].in_use = true;
safe_ltoa(i, buff, bufc);
return;
}
}
safe_str(T("-1"), buff, bufc);
}
// _RESTORE_CARGS(handle_str): restore saved CARGS region.
//
FUNCTION(fun__restore_cargs)
{
UNUSED_PARAMETER(executor);
UNUSED_PARAMETER(caller);
UNUSED_PARAMETER(enactor);
UNUSED_PARAMETER(eval);
UNUSED_PARAMETER(cargs);
UNUSED_PARAMETER(ncargs);
if (!s_current_ecall_ctx || nfargs < 1) return;
int64_t idx = mux_atoi64(fargs[0]);
if (idx >= 0 && idx < MAX_CARG_SAVE_DEPTH
&& s_carg_save_stack[idx].in_use)
{
uint64_t base = rv_compiler::CARGS_BASE;
size_t region = 10 * rv_compiler::CARGS_SLOT;
if (base + region <= s_current_ecall_ctx->memory_size)
{
memcpy(s_current_ecall_ctx->memory + base,
s_carg_save_stack[idx].data, region);
}
s_current_ecall_ctx->ncargs = s_carg_save_stack[idx].ncargs;
s_carg_save_stack[idx].in_use = false;
// Refresh the guest %+ substitution slot.
uint64_t nslot = rv_compiler::SUBST_BASE
+ static_cast<uint64_t>(rv_compiler::SUBST_NCARGS)
* rv_compiler::SUBST_SLOT;
if (nslot + 4 <= s_current_ecall_ctx->memory_size)
{
char nbuf[16];
size_t len = mux_snprintf(reinterpret_cast<UTF8 *>(nbuf), sizeof(nbuf),
T("%d"), s_carg_save_stack[idx].ncargs);
memcpy(s_current_ecall_ctx->memory + nslot, nbuf, len + 1);
}
}
}
// _SET_NCARGS(n_str): update the %+ substitution slot and ncargs.
// Must be called after writing CARGS slots for inlined u() bodies
// so that %+ in the body reflects the callee's argument count.
//
FUNCTION(fun__set_ncargs)
{
UNUSED_PARAMETER(executor);
UNUSED_PARAMETER(caller);
UNUSED_PARAMETER(enactor);
UNUSED_PARAMETER(eval);
UNUSED_PARAMETER(cargs);
UNUSED_PARAMETER(ncargs);
if (!s_current_ecall_ctx || nfargs < 1) return;
int64_t n = mux_atoi64(fargs[0]);
if (n < 0) n = 0;
if (n > 10) n = 10;
s_current_ecall_ctx->ncargs = n;
// Update the guest %+ substitution slot.
uint64_t slot = rv_compiler::SUBST_BASE
+ static_cast<uint64_t>(rv_compiler::SUBST_NCARGS)
* rv_compiler::SUBST_SLOT;
if (slot + 4 <= s_current_ecall_ctx->memory_size)
{
char nbuf[16];
size_t len = mux_snprintf(reinterpret_cast<UTF8 *>(nbuf), sizeof(nbuf), T("%d"), n);
memcpy(s_current_ecall_ctx->memory + slot, nbuf, len + 1);
}
}
// _WRITE_CARG(idx_str, value_str): write value to a CARGS slot.
//
FUNCTION(fun__write_carg)
{
UNUSED_PARAMETER(executor);
UNUSED_PARAMETER(caller);
UNUSED_PARAMETER(enactor);
UNUSED_PARAMETER(eval);
UNUSED_PARAMETER(cargs);
UNUSED_PARAMETER(ncargs);
if (!s_current_ecall_ctx || nfargs < 2) return;
int64_t idx = mux_atoi64(fargs[0]);
if (idx < 0 || idx >= 10) return;
uint64_t dst = rv_compiler::CARGS_BASE
+ static_cast<uint64_t>(idx) * rv_compiler::CARGS_SLOT;
if (dst + rv_compiler::CARGS_SLOT > s_current_ecall_ctx->memory_size) return;
// fargs[1] is a guest-memory pointer (from ecall_invoke_fun).
// Bound the NUL scan; reject if missing. Values that do not fit
// the slot are also rejected rather than silently truncated
// (#1055 / #1057) — the host has no clean way to decline the whole
// program from here, so leave the slot untouched.
eval_ctx *ec = s_current_ecall_ctx;
if (fargs[1] < ec->memory
|| fargs[1] >= ec->memory + ec->memory_size) {
return;
}
uint64_t val_addr = static_cast<uint64_t>(fargs[1] - ec->memory);
size_t len = 0;
if (!guest_strnlen(ec->memory, ec->memory_size, val_addr, &len)) {
return;
}
if (len >= static_cast<size_t>(rv_compiler::CARGS_SLOT)) {
return;
}
memcpy(ec->memory + dst, fargs[1], len);
ec->memory[dst + len] = 0;
}
// ---------------------------------------------------------------
// Returns false if compilation failed — caller should fall back to
// the AST evaluator.
//
// This is the entry point for --enable-jit's mux_exec integration.
// ---------------------------------------------------------------
bool jit_eval(const UTF8 *expr, size_t nLen,
UTF8 *buff, UTF8 **bufc,
dbref executor, dbref caller, dbref enactor,
int eval,
const UTF8 *cargs[], int ncargs) {
// sandbox() sets bSandboxActive to force AST-only evaluation,
// which checks fp->perms (CA_DISABLED) before each function call.
//
if (mudstate.bSandboxActive)
{
return false;
}
// Re-entrancy depth tracking. When JIT code ECALLs into a
// function like u() which calls mux_exec(), the nested mux_exec
// re-enters jit_eval().
//
// Depth 0 (top-level): full JIT — compile, constant-fold, or
// DBT-execute.
// Depth 1+ (nested): compile and return constant-folded results
// (no DBT needed), but fall back to AST for programs that
// require runtime execution. This is step 1 toward full
// re-entrant JIT: inner expressions that fold at compile time
// get the JIT result without touching the DBT.
//
static int s_jit_depth = 0;
s_jit_depth++;
struct jit_depth_guard {
~jit_depth_guard() { s_jit_depth--; }
} depth_guard;
if (s_jit_depth == 1) {
JITArena::gc();
jit_dma_controller::reset();
}
// Don't JIT until the Tier 2 blob is loaded and the persistent
// DBT state is initialized. compile_cached calls tier2_lazy_init,
// but the DBT infrastructure (mmap, block cache) may not be safe
// to initialize during early startup (config loading, @startup).
// The per-context dbt_ready flag is set after the first successful
// get_dbt.
//
// The loaded check must be OUTSIDE the init-once conditional: with
// it inside, only the first call declined when the blob was missing
// and every later call ran the JIT blob-less, producing silently
// wrong results for some compiled shapes (#875).
if (!s_tier2_init) {
tier2_lazy_init();
}
if (!s_tier2.loaded) return false;
s_jit_stats.eval_attempts++;
// Memoized decline (#2130): a shape already known to be bail_noop is
// refused here, before the compile cache and before any SQLite fetch.
// The server used to do a SELECT plus a full program deserialization
// per evaluation in order to read four integers and not run the result.
{
std::string memo_key = compile_cache_key(expr, nLen, eval);
if (s_decline_memo.count(memo_key)) {
s_jit_stats.bail_noop++;
s_jit_stats.noop_memo++;
s_jit_stats.eval_bailout++;
return false;
}
}
compiled_program *prog = compile_cached(expr, nLen, eval);
if (!prog) {
s_jit_stats.eval_bailout++;
return false;
}
// #1002 static-depth watermark: the AST evaluator errors a call at
// nesting level L when func_nest_lim <= L; compiled code flattens
// the nest (and folding pre-computes it), so decline to the AST
// whenever this evaluation could reach the limit. Uses the LIVE
// func_nest_lev so nested entry contexts count, and the live limit
// so @admin changes apply to cached programs.
if (mudstate.func_nest_lev + prog->max_func_depth
>= mudconf.func_nest_lim) {
s_jit_stats.bail_depth++;
s_jit_stats.eval_bailout++;
return false;
}
// Static invocation-count watermark: flattened sequential calls do
// not maintain func_invk_ctr. Decline when the live counter plus
// this program's FUNCCALL count would trip function_invocation_limit.
if ( 0 < prog->n_func_calls
&& mudstate.func_invk_ctr + prog->n_func_calls
>= mudconf.func_invk_lim) {
s_jit_stats.bail_invk++;
s_jit_stats.eval_bailout++;
return false;
}
if (!prog->needs_jit) {
// Constant-folded — result was extracted at compaction time.
// Safe at any nesting depth (no DBT involved).
s_jit_stats.folded_total++;
s_jit_stats.eval_handled++;
safe_str(reinterpret_cast<const UTF8 *>(prog->folded_result.c_str()),
buff, bufc);
return true;
}
// Decline a program that does no work (#2086).
//
// When a lowering is a single host ECALL with nothing computed around it
// -- no native arithmetic, no constant folding, no tier 2 call -- running
// it performs exactly the host call the AST evaluator would have made,
// and adds guest entry, argument marshalling and exit on top. There is no
// arrangement of those costs that comes out ahead; the compiled path can
// only be the interpreter's cost plus a constant.
//
// Measured for get(perfobj/perf0): 108 bytes of RV64, one ECALL per
// evaluation, zero tier2/folds, and 1.35x the no-JIT path on
// Linux/aarch64 (1.36x on Windows/MSVC via #2083's attrread phase).
// The shape covers get/v/u and most attribute and database accessors,
// which is what live games actually call (#2064).
//
// Correctness is identical either way -- both routes call the same host
// function -- so this only chooses the cheaper route to it.
//
// Deliberately conservative: ONE ecall and nothing else. A program with
// two ECALLs has already saved one AST dispatch, and anything with
// native_ops, folds or tier2_calls is doing work the interpreter would
// otherwise repeat. Widening this predicate would start declining
// programs the JIT genuinely wins on, and that failure mode looks like a
// speedup on every benchmark that is not measuring the thing it broke.
//
if ( 0 == prog->native_ops
&& 0 == prog->folds
&& 0 == prog->tier2_calls
&& 1 >= prog->ecalls) {
// Memoize the verdict (#2130) so later evaluations refuse before
// any cache machinery runs. Dep-free only: a program with inline
// deps can recompile into a different shape when an attr changes,
// and noop shapes are dep-free in practice (deps come from
// u()-inlining, which emits code).
if (prog->deps.empty()) {
if (s_decline_memo.size() >= DECLINE_MEMO_MAX) {
s_decline_memo.clear();
}
s_decline_memo.insert(compile_cache_key(expr, nLen, eval));
}
s_jit_stats.bail_noop++;
s_jit_stats.eval_bailout++;
return false;
}
// Depth > 1: try executing via the shared heap's independent DBT.
// The shared heap compiles into persistent memory and runs in its
// own DBT context, so this is safe during an outer ECALL.
if (s_jit_depth > 1) {
LBuf shresult = LBuf_Src("jit_eval.shared");
int sh_ecalls = 0;
int sh_tier2 = 0;
bool sh_folded = false;
if (s_shared_heap.eval(expr, nLen, shresult, LBUF_SIZE,
executor, caller, enactor, eval,
cargs, ncargs,
&sh_ecalls, &sh_tier2, &sh_folded)) {
s_jit_stats.eval_handled++;
s_jit_stats.ecall_total += sh_ecalls;
s_jit_stats.tier2_total += sh_tier2;
if (sh_folded) s_jit_stats.folded_total++;
safe_str(shresult, buff, bufc);
return true;
}
s_jit_stats.eval_bailout++;
return false;
}
LBuf result = LBuf_Src("jit_eval");
if (!run_cached_program(prog, executor, caller, enactor,
result, LBUF_SIZE,
cargs, ncargs, eval)) {
s_jit_stats.eval_bailout++;
return false; // JIT execution error — fall back to AST.
}
s_jit_stats.eval_handled++;
s_jit_stats.ecall_total += prog->ecalls;
s_jit_stats.tier2_total += prog->tier2_calls;
safe_str(result, buff, bufc);
return true;
}
// ---------------------------------------------------------------
// jitstats() — wizard-only function returning JIT profiling counters.
//
// Returns a space-separated key=value list suitable for parsing.
// With argument "reset", clears all counters.
// ---------------------------------------------------------------
// Sample jit_eval()'s attempt/handled counters (#2133 item 5).
//
// A narrow accessor rather than exporting s_jit_stats: functions.cpp cannot
// include dbt_compile.h (it collides with color_ops.h's C-linkage
// declarations), and benchmark() needs exactly these two numbers to report
// whether the JIT actually ran during a timed loop.
//
void jit_eval_counters(uint64_t *pAttempts, uint64_t *pHandled)
{
if (nullptr != pAttempts) *pAttempts = s_jit_stats.eval_attempts;
if (nullptr != pHandled) *pHandled = s_jit_stats.eval_handled;
}
FUNCTION(fun_jitstats)
{
UNUSED_PARAMETER(fp);
UNUSED_PARAMETER(caller);
UNUSED_PARAMETER(enactor);
UNUSED_PARAMETER(eval);
UNUSED_PARAMETER(cargs);
UNUSED_PARAMETER(ncargs);
if (!Wizard(executor)) {
safe_str(S_("#-1 PERMISSION DENIED"), buff, bufc);
return;
}
if (nfargs >= 1) {
const char *arg = reinterpret_cast<const char *>(fargs[0]);
if (strcmp(arg, "reset") == 0) {
memset(&s_jit_stats, 0, sizeof(s_jit_stats));
jit_lua_reset_stats();
safe_str(T("OK"), buff, bufc);
return;
}
if (strcmp(arg, "flush") == 0) {
// Wipe the persisted JIT code_cache and every in-process
// compiled program that would still serve the old codegen.
// Without this, a later build can load a code_cache row written
// by an earlier one (blob_hash only covers the Tier-2 blob, not
// engine codegen changes) — the A/B trap from #1315 / #1316.
//
// 1. Drop pending OP_CODE_CACHE_PUT so a later write-queue flush
// cannot re-insert what we are about to DELETE.
// 2. DELETE FROM code_cache.
// 3. Clear softcode and Lua in-memory compile caches.
//
#if defined(HAVE_WORKING_FORK)
if (mudstate.write_protect) {
safe_str(S_("#-1 PERMISSION DENIED"), buff, bufc);
return;
}
#endif
cache_discard_code_cache_writes();
bool ok = true;
if (g_pSQLiteBackend) {
ok = g_pSQLiteBackend->GetDB().CodeCacheFlush();
}
// The in-memory caches can only be cleared when no compiled
// program is live: an ECALL can reach this function from inside
// one, and clearing would free the program still executing. See
// s_code_cache_flush_pending. Nothing is lost by waiting -- the
// drain runs before the next program is looked up, so no stale
// native code can be served in between.
//
if (0 == s_run_cached_depth) {
jit_flush_memory_caches();
} else {
s_code_cache_flush_pending = true;
}
if (!ok) {
safe_str(S_("#-1 CODE CACHE FLUSH FAILED"), buff, bufc);
return;
}
safe_str(T("OK"), buff, bufc);
return;
}
}
// Format: key=value pairs, newline-separated for readability.
LBuf tmp = LBuf_Src("jitstats");
size_t n = mux_snprintf(tmp.get(), LBUF_SIZE,
T("eval_attempts=%llu "
"eval_handled=%llu "
"eval_bailout=%llu "
"cache_hit_mem=%llu "
"cache_hit_sqlite=%llu "
"cache_miss=%llu "
"compile_ok=%llu "
"compile_fail=%llu "
"bail_noeval=%llu "
"bail_slots=%llu "
"folded=%llu "
"ecalls=%llu "
"tier2=%llu "
"code_bytes=%llu "
"code_max=%llu "
"hir_insns=%llu "
"hir_max=%llu "
"spills=%llu "
"qreg_resyncs=%llu "
"bail_longreg=%llu "
"bail_depth=%llu "
"bail_invk=%llu "
"bail_alarm=%llu "
"bail_shared_busy=%llu "
"bail_noop=%llu "
"bail_code=%llu "
"bail_strpool=%llu "
"bail_fargs=%llu "
"bail_outslots=%llu "
"want_code_max=%llu "
"want_strpool_max=%llu "
"want_fargs_max=%llu "
"want_outslots_max=%llu "
"slot_hit=%llu "
"slot_miss=%llu "
"slot_evict=%llu "
"slot_churn0=%llu "
"slot_pinned=%llu "
"noop_memo=%llu"),
(unsigned long long)s_jit_stats.eval_attempts,
(unsigned long long)s_jit_stats.eval_handled,
(unsigned long long)s_jit_stats.eval_bailout,
(unsigned long long)s_jit_stats.cache_hit_mem,
(unsigned long long)s_jit_stats.cache_hit_sqlite,
(unsigned long long)s_jit_stats.cache_miss,
(unsigned long long)s_jit_stats.compile_ok,
(unsigned long long)s_jit_stats.compile_fail,
(unsigned long long)s_jit_stats.bail_noeval,
(unsigned long long)s_jit_stats.bail_slots,
(unsigned long long)s_jit_stats.folded_total,
(unsigned long long)s_jit_stats.ecall_total,
(unsigned long long)s_jit_stats.tier2_total,
(unsigned long long)s_jit_stats.code_bytes_total,
(unsigned long long)s_jit_stats.code_bytes_max,
(unsigned long long)s_jit_stats.hir_insns_total,
(unsigned long long)s_jit_stats.hir_insns_max,
(unsigned long long)s_jit_stats.spills_total,
(unsigned long long)s_jit_stats.qreg_resyncs,
(unsigned long long)s_jit_stats.bail_longreg,
(unsigned long long)s_jit_stats.bail_depth,
(unsigned long long)s_jit_stats.bail_invk,
(unsigned long long)s_jit_stats.bail_alarm,
(unsigned long long)s_jit_stats.bail_shared_busy,
(unsigned long long)s_jit_stats.bail_noop,
(unsigned long long)s_jit_stats.bail_code,
(unsigned long long)s_jit_stats.bail_strpool,
(unsigned long long)s_jit_stats.bail_fargs,
(unsigned long long)s_jit_stats.bail_outslots,
(unsigned long long)s_jit_stats.want_code_max,
(unsigned long long)s_jit_stats.want_strpool_max,
(unsigned long long)s_jit_stats.want_fargs_max,
(unsigned long long)s_jit_stats.want_outslots_max,
(unsigned long long)s_jit_stats.slot_hit,
(unsigned long long)s_jit_stats.slot_miss,
(unsigned long long)s_jit_stats.slot_evict,
(unsigned long long)s_jit_stats.slot_churn0,
(unsigned long long)s_jit_stats.slot_pinned,
(unsigned long long)s_jit_stats.noop_memo);
// Append Lua JIT counters (#1316). lua_run_fail incrementing while
// softcode still returns correct answers is the signature of a Lua JIT
// that compiles and then silently falls back to the interpreter.
if (n < static_cast<int>(LBUF_SIZE) - 256) {
lua_jit_counters lj = {};
jit_lua_get_stats(&lj);
n += mux_snprintf(tmp.get() + n, LBUF_SIZE - n,
T(" lua_compile_ok=%llu"
" lua_compile_fail=%llu"
" lua_run_ok=%llu"
" lua_run_fail=%llu"
" lua_cache_hits=%llu"
" lua_invalidations=%llu"
" lua_post_entry_decline=%llu"),
(unsigned long long)lj.compile_ok,
(unsigned long long)lj.compile_fail,
(unsigned long long)lj.run_ok,
(unsigned long long)lj.run_fail,
(unsigned long long)lj.cache_hits,
(unsigned long long)lj.invalidations,
(unsigned long long)lj.post_entry_decline);
}
// Append NOEVAL breakdown.
for (int i = 0; i < s_jit_stats.noeval_top_used && n < static_cast<int>(LBUF_SIZE) - 64; i++) {
n += mux_snprintf(tmp.get() + n, LBUF_SIZE - n, T(" noeval_%s=%llu"),
s_jit_stats.noeval_top[i].name,
(unsigned long long)s_jit_stats.noeval_top[i].count);
}
// Append DBT code-buffer occupancy (#1315). The Tier-2 blob is
// pretranslated once and preserved across every dbt_reset, so
// dbt_blob_bytes is a permanent reservation out of dbt_code_cap and
// what remains is all any program will ever get. That cost is
// backend-specific -- the same blob is not the same number of host
// bytes on Win64, x64 SysV and aarch64 -- so it wants measuring per
// platform rather than assuming one constant suits every backend.
//
// dbt_code_full counting up means translations are being declined for
// want of space; dbt_code_reclaims counts the mid-run recoveries that
// keep such a decline local to one program instead of permanent.
//
// Summed over every initialized run context, not just depth 0 (#1326).
// Each nesting depth owns a separate DBT with a separate code buffer, so
// reading only s_vm[0] hides a nested Lua run filling its own -- and
// dbt_code_full is precisely the signal that would be hidden. cap is
// summed the same way so used/cap stays a ratio of the same population;
// a context that has not been initialized has no buffer to report.
//
unsigned long long dbt_cap = 0, dbt_blob = 0, dbt_used = 0;
unsigned long long dbt_reclaims = 0, dbt_full = 0;
for (int i = 0; i < JIT_MAX_RUN_DEPTH; i++) {
if (!s_vm[i].dbt_ready) {
continue;
}
dbt_cap += CODE_BUF_SIZE;
dbt_blob += s_vm[i].dbt.blob_code_end;
dbt_used += s_vm[i].dbt.code_used;
dbt_reclaims += s_vm[i].dbt.code_reclaims;
dbt_full += s_vm[i].dbt.code_full;
}
if (n < static_cast<int>(LBUF_SIZE) - 256) {
n += mux_snprintf(tmp.get() + n, LBUF_SIZE - n,
T(" dbt_code_cap=%u"
" dbt_blob_bytes=%u"
" dbt_code_used=%u"
" dbt_code_reclaims=%llu"
" dbt_code_full=%llu"),
static_cast<unsigned>(dbt_cap),
static_cast<unsigned>(dbt_blob),
static_cast<unsigned>(dbt_used),
static_cast<unsigned long long>(dbt_reclaims),
static_cast<unsigned long long>(dbt_full));
}
safe_str(tmp, buff, bufc);
}
// ---------------------------------------------------------------
// fun_rvbench: benchmark JIT vs native mux_exec.
//
// rvbench(<expression>, <iterations>)
//
// Runs the expression through three paths:
// 1. Native mux_exec (AST eval) — the current production path
// 2. rveval compile-every-time
// 3. rveval compile-once, run N times (amortized)
//
// Returns a multi-line report with timings in microseconds.
// ---------------------------------------------------------------
#ifdef WIN32
static double elapsed_us(const LARGE_INTEGER &start,
const LARGE_INTEGER &end) {
LARGE_INTEGER freq;
QueryPerformanceFrequency(&freq);
return static_cast<double>(end.QuadPart - start.QuadPart) * 1e6
/ static_cast<double>(freq.QuadPart);
}
#else
static double elapsed_us(const struct timespec &start,
const struct timespec &end) {
double s = static_cast<double>(end.tv_sec - start.tv_sec);
double ns = static_cast<double>(end.tv_nsec - start.tv_nsec);
return (s * 1e6) + (ns / 1e3);
}
#endif
#ifdef WIN32
#define BENCH_TIMER LARGE_INTEGER
#define BENCH_NOW(t) QueryPerformanceCounter(&(t))
#else
#define BENCH_TIMER struct timespec
#define BENCH_NOW(t) clock_gettime(CLOCK_MONOTONIC, &(t))
#endif
// Run the compiled program through the JIT. Returns the result
// string (written into caller-provided buffer).
//
// If reuse_dbt is true, skip the full DBT reset and only update the
// ECALL callback — keeps translated blocks cached. Caller must
// ensure the guest code region is unchanged.
//
static bool run_compiled(compiled_program &prog,
dbref executor, dbref caller_db, dbref enactor,
UTF8 *out, size_t out_size,
bool reuse_dbt = false) {
if (!prog.needs_jit) {
// Fully folded — result is already in guest memory.
uint64_t out_addr = resolve_runtime_out_addr(
prog.out_addr, rv_compiler::STACK_TOP);
size_t n = 0;
if (!guest_strnlen(prog.memory.data(), prog.memory_size,
out_addr, &n)) {
return false;
}
if (n >= out_size) n = out_size - 1;
memcpy(out, prog.memory.data() + out_addr, n);
out[n] = '\0';
return true;
}
eval_ctx ec;
ec.memory = prog.memory.data();
ec.memory_size = prog.memory_size;
ec.executor = executor;
ec.caller = caller_db;
ec.enactor = enactor;
ec.eval = EV_FCHECK | EV_EVAL;
ec.cargs = nullptr;
ec.ncargs = 0;
ec.qreg_mask = prog.subst_mask;
ec.lua_result_base = 0;
ec.lua_result_count = 0;
ec.lua_state = nullptr;
ec.host_ecalls = 0;
ec.dbt = nullptr;
ec.pvm = nullptr;
// This path runs out of prog.memory rather than the context's own buffer
// (dbt_reset rebinds it), so it must CLAIM the context for the duration
// of the run (#2106). The old code took s_vm[0] unconditionally without
// touching s_run_cached_depth, on the reasoning that run_compiled is
// never reached FROM a nested run -- true, and beside the point. It
// makes one: a program whose ECALL re-enters mux_exec per element
// (fun_map, fun_filter) lands in run_cached_program, which reads a depth
// still at 0, picks this same s_vm[0], and calls get_dbt on it --
// rebinding the DBT from prog.memory to the context's own buffer while
// the outer run's frames are executing out of it. That is precisely the
// hazard the depth check above run_cached_program's slot pick describes.
//
// Both buffers are live and the same size, so on glibc the outer run
// simply continues against the wrong memory and returns a wrong answer;
// it took a platform whose allocator left the region unmapped to turn it
// into the SIGSEGV that got it noticed.
//
if (s_run_cached_depth >= JIT_MAX_RUN_DEPTH) {
return false;
}
jit_run_vm *vm = &s_vm[s_run_cached_depth];
RunDepthGuard run_depth_guard(s_run_cached_depth);
dbt_state_t *dbt;
if (reuse_dbt && vm->dbt_ready) {
dbt = &vm->dbt;
dbt_rerun(dbt, eval_ecall, &ec);
} else {
dbt = get_dbt(vm, prog.memory.data(), prog.memory_size,
eval_ecall, &ec);
if (!dbt) return false;
if (dbt->blob_code_end == 0) {
pretranslate_tier2(dbt);
dbt->blob_code_end = dbt->code_used;
}
}
int rc = dbt_run(dbt, prog.entry_pc, rv_compiler::STACK_TOP);
if (!handle_dbt_run_status(rc, out, out_size, true)) {
return false;
}
if (rc == -3) {
return true;
}
uint64_t out_addr = resolve_runtime_out_addr(
prog.out_addr, rv_compiler::STACK_TOP);
size_t n = 0;
if (!guest_strnlen(prog.memory.data(), prog.memory_size, out_addr, &n)) {
return false;
}
if (n >= out_size) n = out_size - 1;
memcpy(out, prog.memory.data() + out_addr, n);
out[n] = '\0';
return true;
}
FUNCTION(fun_rvbench)
{
UNUSED_PARAMETER(fp);
UNUSED_PARAMETER(eval);
UNUSED_PARAMETER(cargs);
UNUSED_PARAMETER(ncargs);
JITArena::gc();
if (nfargs < 2) {
safe_str(S_("#-1 TOO FEW ARGUMENTS"), buff, bufc);
return;
}
const UTF8 *expr = fargs[0];
size_t nLen = strlen(reinterpret_cast<const char *>(expr));
// Clamp in 64-bit before narrowing; see the note in fun_astbench.
// rvbench(expr, 4294967296) otherwise truncated to 0 and then got
// bumped to 1 by the floor below -- a request 4000x above the cap
// silently becoming the smallest legal run (#1402).
//
int64_t iRequested = mux_atoi64(fargs[1]);
if (iRequested < 1) iRequested = 1;
if (iRequested > 1000000) iRequested = 1000000;
int iterations = static_cast<int>(iRequested);
// Verify both paths produce the same result.
compiled_program prog = compile_expression(expr, nLen);
if (!prog.ok) {
safe_str(S_("#-1 COMPILATION FAILED"), buff, bufc);
return;
}
tier2_install(prog.memory, rv_compiler::BLOB_BASE);
// --- Benchmark 1: Native mux_exec ---
BENCH_TIMER t0, t1;
int eval_flags = EV_FCHECK | EV_EVAL;
BENCH_NOW(t0);
for (int i = 0; i < iterations; i++) {
LBuf tbuf = LBuf_Src("rvbench.native");
UTF8 *tbufc = tbuf.get();
mux_exec(expr, nLen, tbuf, &tbufc, executor, caller, enactor,
eval_flags, nullptr, 0);
*tbufc = '\0';
}
BENCH_NOW(t1);
double native_us = elapsed_us(t0, t1);
// --- Benchmark 2: rveval compile-every-time ---
BENCH_NOW(t0);
for (int i = 0; i < iterations; i++) {
compiled_program p = compile_expression(expr, nLen);
if (p.ok) {
tier2_install(p.memory, rv_compiler::BLOB_BASE);
UTF8 result[256];
run_compiled(p, executor, caller, enactor, result, sizeof(result));
}
}
BENCH_NOW(t1);
double compile_each_us = elapsed_us(t0, t1);
// --- Benchmark 3: production path (compile cache + block cache) ---
// Uses compile_cached (LRU) + run_cached_program (dbt_rerun).
// First iteration is a cache miss (compiles + JIT translates);
// subsequent iterations hit both caches — zero compilation,
// zero JIT translation.
//
// Invalidate the compile cache entry for this expression first
// so the first iteration is a genuine miss.
{
std::string key = compile_cache_key(expr, nLen, EV_FMAND | EV_EVAL);
auto cit = s_compile_cache.find(key);
if (cit != s_compile_cache.end()) {
release_program_slots(cit->second.prog.program_id);
s_compile_lru.erase(cit->second.lru_it);
s_compile_cache.erase(cit);
}
}
BENCH_NOW(t0);
for (int i = 0; i < iterations; i++) {
compiled_program *cp = compile_cached(expr, nLen, EV_FMAND | EV_EVAL);
if (cp) {
UTF8 result[256];
run_cached_program(cp, executor, caller, enactor,
result, sizeof(result));
}
}
BENCH_NOW(t1);
double cached_us = elapsed_us(t0, t1);
// Format report.
double per_native = native_us / iterations;
double per_compile = compile_each_us / iterations;
double per_cached = cached_us / iterations;
uint64_t disp = s_vm[0].dbt.dispatch_count;
uint64_t sb = s_vm[0].dbt.superblock_count;
uint64_t se = s_vm[0].dbt.side_exits_total;
uint64_t ic = s_vm[0].dbt.inline_calls;
uint64_t ih = s_vm[0].dbt.intrinsic_hits;
uint64_t ce = s_vm[0].dbt.cold_exit_count;
uint64_t ce_actual = s_vm[0].dbt.cold_exit_actual;
uint64_t ce_expected = s_vm[0].dbt.cold_exit_expected;
uint64_t ce_from = s_vm[0].dbt.last_exit_from;
// Timings report ns/call, not us (#2046). The cached path runs at
// 10-50ns/call, which %.2fus rendered as "0.01" or "0.02" -- one or two
// significant digits, so consecutive identical runs looked like 100%
// swings and any regression under about 2x was invisible. A number that
// cannot be resolved cannot be gated on.
//
// Keep the format one unbroken literal: check_formats.py reads the source
// and requires mux_snprintf()'s format to be a constant, so a comment
// interleaved with the concatenation trips the guard.
//
LBuf report = LBuf_Src("rvbench");
mux_snprintf(report.get(), LBUF_SIZE,
T("expr=%s iters=%d folds=%d ecalls=%d tier2=%d nativ=%d disp=%llu sb=%llu/%llu ic=%llu ih=%llu ce=%llu(a=0x%llX,e=0x%llX,from=0x%llX) | "
"native=%.1fns/call | "
"compile-each=%.1fns/call (%.1fx) | "
"cached=%.1fns/call (%.1fx)"),
reinterpret_cast<const char *>(expr),
iterations, prog.folds, prog.ecalls, prog.tier2_calls, prog.native_ops,
(unsigned long long)disp,
(unsigned long long)sb, (unsigned long long)se, (unsigned long long)ic,
(unsigned long long)ih, (unsigned long long)ce, (unsigned long long)ce_actual, (unsigned long long)ce_expected, (unsigned long long)ce_from,
per_native * 1000.0,
per_compile * 1000.0, per_compile / per_native,
per_cached * 1000.0, per_cached / per_native);
safe_str(report, buff, bufc);
}
// ---------------------------------------------------------------
// Persistent VM proof-of-concept.
//
// Demonstrates the core concept: a single RV64 guest memory that
// lives across multiple evaluations, with compiled functions
// calling each other via JAL — no DBT reset, no context zeroing.
//
// Layout in persistent guest memory:
// 0x0000: Main stub (calls func_add42 via JAL, writes result, exits)
// 0x0100: func_add42 — adds 42 to a0, returns via JALR ra
// 0x1000: String pool (output buffer)
//
// On each invocation, the block cache retains translations from
// all previous runs. The second and subsequent calls should show
// zero cache misses.
// ---------------------------------------------------------------
// ---------------------------------------------------------------
// RV64 instruction encoders for hand-assembled persistent VM stubs.
// Subset of the encoders in hir_codegen.cpp (separate TU).
// ---------------------------------------------------------------
namespace rv64_asm {
static uint32_t i_type(uint8_t op, uint8_t rd, uint8_t f3,
uint8_t rs1, int32_t imm) {
return op | (rd << 7) | (f3 << 12) | (rs1 << 15)
| ((static_cast<uint32_t>(imm) & 0xFFF) << 20);
}
static uint32_t ADDI(uint8_t rd, uint8_t rs1, int32_t imm) {
return i_type(OP_IMM, rd, 0, rs1, imm);
}
static uint32_t LUI(uint8_t rd, int32_t imm) {
return OP_LUI | (rd << 7) | (static_cast<uint32_t>(imm) & 0xFFFFF000);
}
static uint32_t SB(uint8_t base, uint8_t src, int32_t off) {
return OP_STORE | ((off & 0x1F) << 7) | (0 << 12)
| (base << 15) | (src << 20)
| (((off >> 5) & 0x7F) << 25);
}
static uint32_t ECALL() { return 0x00000073; }
static uint32_t BNE(uint8_t rs1, uint8_t rs2, int32_t off) {
uint32_t u = static_cast<uint32_t>(off);
return OP_BRANCH
| (((u >> 11) & 1) << 7)
| (((u >> 1) & 0xF) << 8)
| (1 << 12)
| (rs1 << 15) | (rs2 << 20)
| (((u >> 5) & 0x3F) << 25)
| (((u >> 12) & 1) << 31);
}
static void load_val(std::vector<uint32_t> &code, uint8_t rd, uint64_t val) {
if (val == 0) {
code.push_back(ADDI(rd, 0, 0));
return;
}
int32_t sval = static_cast<int32_t>(val);
if (sval >= -2048 && sval <= 2047
&& val == static_cast<uint64_t>(static_cast<uint32_t>(sval)))
{
code.push_back(ADDI(rd, 0, sval));
return;
}
uint32_t hi = static_cast<uint32_t>(val) & 0xFFFFF000;
int32_t lo = static_cast<int32_t>(val & 0xFFF);
if (lo & 0x800) { hi += 0x1000; lo -= 0x1000; }
code.push_back(LUI(rd, hi));
if (lo) code.push_back(ADDI(rd, rd, lo));
}
static uint32_t SUB(uint8_t rd, uint8_t rs1, uint8_t rs2) {
return OP_REG | (rd << 7) | (0 << 12) | (rs1 << 15)
| (rs2 << 20) | (0x20u << 25);
}
} // namespace rv64_asm
// Persistent VM ECALL handler.
// Handles ECALL_EXIT and ECALL_CALL_COMPILED (re-entrant calls).
//
static int poc_ecall(rv64_ctx_t *ctx, void *user_data) {
uint64_t nr = ctx->x[17]; // a7
if (nr == ECALL_EXIT) {
return static_cast<int>(ctx->x[10]); // a0 = exit code
}
if (nr == ECALL_CALL_COMPILED) {
// Re-entrant call into a compiled function.
// a0 = entry_pc, a1 = output buffer addr, a2 = fargs addr, a3 = nfargs
dbt_state_t *dbt = static_cast<dbt_state_t *>(user_data);
if (!dbt) {
ctx->x[10] = 0;
return -1; // continue
}
uint64_t target_pc = ctx->x[10];
uint64_t out_ref = ctx->x[11];
// Save outer execution's full CPU context.
rv64_ctx_t saved_ctx = *ctx;
// Set up for inner call: SP already points to available
// stack space (below outer's frame).
// The inner function's prologue will decrement SP further.
ctx->x[2] = saved_ctx.x[2]; // preserve SP
// Run inner function via dbt_resume.
int inner_rc = dbt_resume(dbt, target_pc);
// Extract inner result length (#1057: bound guest NUL scan).
uint64_t result_len = 0;
uint64_t resolved_out = resolve_runtime_out_addr(out_ref, saved_ctx.x[2]);
size_t n = 0;
if (inner_rc == 0 && resolved_out > 0
&& guest_strnlen(dbt->memory, dbt->memory_size, resolved_out, &n)) {
result_len = n;
}
// Restore outer CPU context.
*ctx = saved_ctx;
// Return result info in a0.
ctx->x[10] = result_len;
ctx->x[11] = resolved_out;
return -1; // continue outer execution
}
// Unknown ECALL — error.
fprintf(stderr, "pocvm: unknown ECALL %llu\n",
static_cast<unsigned long long>(nr));
return -1;
}
// ---------------------------------------------------------------
// Persistent VM: compile real MUX expressions at different code
// offsets in shared 4MB guest memory, execute via dbt_run/dbt_resume,
// and demonstrate re-entrant calls via ECALL_CALL_COMPILED.
// ---------------------------------------------------------------
static persistent_vm_t s_pvm;
FUNCTION(fun_pocvm2)
{
UNUSED_PARAMETER(fp);
UNUSED_PARAMETER(caller);
UNUSED_PARAMETER(enactor);
UNUSED_PARAMETER(eval);
UNUSED_PARAMETER(fargs);
UNUSED_PARAMETER(nfargs);
UNUSED_PARAMETER(cargs);
UNUSED_PARAMETER(ncargs);
if (!Wizard(executor)) {
safe_str(S_("#-1 PERMISSION DENIED"), buff, bufc);
return;
}
// One-time compilation of test functions.
static uint64_t func_a_entry = 0, func_a_out = 0;
static uint64_t func_b_entry = 0, func_b_out = 0;
static uint64_t func_c_entry = 0, func_c_out = 0;
static bool compiled = false;
if (!compiled) {
// Function A: lnum(3) → "0 1 2"
// lnum() is Tier 2 (no ECALL) and not in try_fold,
// so the persistent VM produces executable JIT code.
const char *expr_a = "lnum(3)";
auto a = s_pvm.compile(
reinterpret_cast<const UTF8 *>(expr_a), strlen(expr_a));
if (!a.entry_pc) {
safe_str(S_("#-1 FUNC A COMPILE FAILED"), buff, bufc);
return;
}
func_a_entry = a.entry_pc;
func_a_out = a.out_addr;
// Function B: lnum(5) → "0 1 2 3 4"
const char *expr_b = "lnum(5)";
auto b = s_pvm.compile(
reinterpret_cast<const UTF8 *>(expr_b), strlen(expr_b));
if (!b.entry_pc) {
safe_str(S_("#-1 FUNC B COMPILE FAILED"), buff, bufc);
return;
}
func_b_entry = b.entry_pc;
func_b_out = b.out_addr;
// Function C: hand-assembled re-entrant call stub.
// Calls A via ECALL_CALL_COMPILED, writes "C:" + A's result.
{
std::vector<uint32_t> code;
constexpr uint8_t a0 = 10, a1 = 11, a7 = 17;
constexpr uint8_t t0 = 5, t3 = 28, t4 = 29;
func_c_out = 0x3000; // fixed output address
rv64_asm::load_val(code, a0, func_a_entry);
rv64_asm::load_val(code, a1, func_a_out);
code.push_back(rv64_asm::ADDI(a7, 0,
static_cast<int32_t>(ECALL_CALL_COMPILED)));
code.push_back(rv64_asm::ECALL());
rv64_asm::load_val(code, t4, func_c_out);
code.push_back(rv64_asm::ADDI(t3, 0, 'C'));
code.push_back(rv64_asm::SB(t4, t3, 0));
code.push_back(rv64_asm::ADDI(t3, 0, ':'));
code.push_back(rv64_asm::SB(t4, t3, 1));
code.push_back(rv64_asm::ADDI(t4, t4, 2));
// After ECALL_CALL_COMPILED, a1 (x11) holds the resolved
// output address of the inner function.
code.push_back(rv64_asm::ADDI(t3, a1, 0)); // t3 = a1
size_t copy_loop = code.size();
code.push_back(rv64_asm::i_type(OP_LOAD, t0, 4, t3, 0));
code.push_back(rv64_asm::SB(t4, t0, 0));
code.push_back(rv64_asm::ADDI(t3, t3, 1));
code.push_back(rv64_asm::ADDI(t4, t4, 1));
int32_t off = -static_cast<int32_t>(
(code.size() - copy_loop) * 4);
code.push_back(rv64_asm::BNE(t0, 0, off));
code.push_back(rv64_asm::ADDI(a7, 0, ECALL_EXIT));
code.push_back(rv64_asm::ADDI(a0, 0, 0));
code.push_back(rv64_asm::ECALL());
func_c_entry = s_pvm.install_code(code);
}
compiled = true;
}
if (!s_pvm.ensure_dbt()) {
safe_str(S_("#-1 DBT INIT FAILED"), buff, bufc);
return;
}
// Run A.
s_pvm.prepare_run();
int rc_a = s_pvm.run(func_a_entry);
const char *result_a = (rc_a == 0)
? s_pvm.result(func_a_out) : "#-1 RUN A FAILED";
// Run B. Copy A's result first — B shares the same output slot.
std::string str_a(result_a);
s_pvm.reset_blob_bss();
int rc_b = s_pvm.run(func_b_entry);
const char *result_b = (rc_b == 0)
? s_pvm.result(func_b_out) : "#-1 RUN B FAILED";
std::string str_b(result_b);
// Run C (re-entrant: calls A internally).
s_pvm.reset_blob_bss();
uint64_t c_out_abs = rv_compiler::resolve_output_addr(
func_c_out, rv_compiler::STACK_TOP);
uint64_t a_out_abs = rv_compiler::resolve_output_addr(
func_a_out, rv_compiler::STACK_TOP);
if (c_out_abs && c_out_abs < s_pvm.memory.size())
memset(s_pvm.memory.data() + c_out_abs, 0, 256);
if (a_out_abs && a_out_abs < s_pvm.memory.size())
memset(s_pvm.memory.data() + a_out_abs, 0, 256);
int rc_c = s_pvm.run(func_c_entry);
const char *result_c = (rc_c == 0)
? s_pvm.result(func_c_out) : "#-1 RUN C FAILED";
LBuf tmp = LBuf_Src("pocvm2");
mux_snprintf(tmp.get(), LBUF_SIZE,
T("a=%s b=%s c=%s a_pc=0x%llX b_pc=0x%llX c_pc=0x%llX calls=%u"),
str_a.c_str(), str_b.c_str(), result_c,
(unsigned long long)func_a_entry,
(unsigned long long)func_b_entry,
(unsigned long long)func_c_entry,
s_pvm.run_count);
safe_str(tmp, buff, bufc);
}
Reference in a new issue View git blame Copy permalink