mmo-ref/tinymux
1
0
Fork 0
mirror of https://github.com/brazilofmux/tinymux synced 2026-08-13 00:23:11 -04:00
Code Issues Projects Releases Packages Wiki Activity
tinymux/mux/modules/engine/dbt.cpp
Stephen Dennis 02446bf4e7 fix(dbt): include <string> for CODEDUMP diagnostics (#2151) 
libstdc++ does not pull string transitively; libc++ does. Unblocks
Linux/GCC. Soften CHANGES: residual superlinearity is largely closed,
not gone (Kagura re-measure).
2026-08-06 17:36:42 +00:00

1025 lines
41 KiB
C++
Raw Permalink Blame History

/*! \file dbt.cpp
* \brief RV64IMD dynamic binary translator — shared code.
*
* Platform-independent parts of the DBT: block cache, trace helpers,
* dispatch loop, and public API. The per-platform translation backend
* (trampoline, instruction translation, intrinsic stubs) is in a
* separate file selected at configure time (e.g. dbt_x64_sysv.cpp).
*
* See docs/DBT-PORTABILITY.md for the multi-platform design.
*/
#include "dbt.h"
#include "dbt_host.h"
#include "dbt_internal.h"
#include "dbt_jit_mem.h"
#include "dbt_decoder.h"
#include <cstdarg>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <new>
#include <string>
// ---------------------------------------------------------------
// Instruction cache coherency
// ---------------------------------------------------------------
// Flush the instruction cache for newly generated or modified code.
// On AArch64, the I-cache and D-cache are not coherent — writes to
// executable memory require an explicit cache maintenance operation
// before the CPU will fetch the new instructions. On x86-64, this
// is a no-op (coherent I-cache).
//
static inline void dbt_flush_code(dbt_state_t *dbt, uint32_t from_offset) {
jit_write_end(dbt->code_buf + from_offset,
dbt->code_used - from_offset);
}
// ---------------------------------------------------------------
// Trace helpers
// ---------------------------------------------------------------
bool dbt_trace_translate_enabled(const dbt_state_t *dbt,
uint64_t guest_pc) {
if ((dbt->trace & DBT_TRACE_TRANSLATE) == 0) return false;
return !dbt->trace_guest_pc_filter || dbt->trace_guest_pc == guest_pc;
}
void dbt_trace_translate_pc(dbt_state_t *dbt, uint64_t guest_pc,
const char *fmt, ...) {
if (!dbt_trace_translate_enabled(dbt, guest_pc)) return;
va_list ap;
va_start(ap, fmt);
fputs("[dbt-xlate] ", stderr);
vfprintf(stderr, fmt, ap);
fputc('\n', stderr);
va_end(ap);
}
void dbt_trace_translate(dbt_state_t *dbt, const char *fmt, ...) {
if ((dbt->trace & DBT_TRACE_TRANSLATE) == 0) return;
va_list ap;
va_start(ap, fmt);
fputs("[dbt-xlate] ", stderr);
vfprintf(stderr, fmt, ap);
fputc('\n', stderr);
va_end(ap);
}
void dbt_trace_fusion(dbt_state_t *dbt, uint64_t pc, const char *kind) {
dbt_trace_translate_pc(dbt, pc, "fusion guest_pc=0x%llX kind=%s",
static_cast<unsigned long long>(pc), kind);
}
// ---------------------------------------------------------------
// Direct JALR target resolution (pure computation)
// ---------------------------------------------------------------
bool dbt_resolve_direct_jalr_target(uint64_t pc,
const rv64_insn_t &insn,
const rv64_insn_t &next,
uint64_t *target_out,
uint64_t *return_pc_out) {
if (!insn.rd) return false;
if (insn.opcode != OP_LUI && insn.opcode != OP_AUIPC) return false;
if (next.opcode != OP_JALR || next.rs1 != insn.rd) return false;
int64_t base = (insn.opcode == OP_AUIPC) ? static_cast<int64_t>(pc) : 0;
int64_t target = base + static_cast<int64_t>(insn.imm)
+ static_cast<int64_t>(next.imm);
target &= ~1LL; // clear bit 0 per JALR spec
if (target < 0) return false;
if (target_out) *target_out = static_cast<uint64_t>(target);
if (return_pc_out) *return_pc_out = pc + 8;
return true;
}
// ---------------------------------------------------------------
// Out-of-range guest pointer sink (#1151)
// ---------------------------------------------------------------
//
// Intrinsic stubs convert a guest offset to a host pointer and hand it
// straight to strlen/strcpy/memcpy/memset with lengths the guest chose.
// The interpreter bounds-checks every access (dbt_interp.cpp mem_check);
// the DBT did not, so a bad guest pointer became a host read or write at
// an arbitrary address.
//
// A conversion that fails the bound is redirected here instead of being
// trapped. That follows the interpreter, which logs and returns 0 rather
// than halting -- there is no fault path in the DBT to unwind to. A
// clamped strlen sees an immediate NUL, a clamped memcpy reads zeros, and
// nothing touches host memory outside this buffer. Writes through a
// clamped pointer land here too and may dirty it; that is contained, which
// is the property being bought. ctx.mem_clamps counts the redirects so a
// live clamp is visible rather than silent.
//
alignas(16) uint8_t g_dbt_safe_page[DBT_SAFE_PAGE_SIZE];
// ---------------------------------------------------------------
// Block cache
// ---------------------------------------------------------------
static inline uint32_t cache_set(uint64_t pc) {
uint32_t h = static_cast<uint32_t>(pc >> 2);
h ^= (h >> 10);
return h & BLOCK_CACHE_MASK;
}
block_entry_t *dbt_cache_lookup(dbt_state_t *dbt, uint64_t pc) {
uint32_t set = cache_set(pc);
block_entry_t *base = &dbt->cache[set * BLOCK_CACHE_WAYS];
for (size_t w = 0; w < BLOCK_CACHE_WAYS; w++) {
if (base[w].guest_pc == pc && base[w].native_code) {
dbt->cache_hits++;
return &base[w];
}
}
dbt->cache_misses++;
return nullptr;
}
void dbt_cache_insert(dbt_state_t *dbt, uint64_t pc, uint8_t *code) {
uint32_t set = cache_set(pc);
block_entry_t *base = &dbt->cache[set * BLOCK_CACHE_WAYS];
// An entry for this pc already exists — update it in place rather than
// consuming a second way (#1153). Intrinsic blocks are inserted twice:
// try_emit_intrinsic() in the backend inserts before returning, and
// every caller of dbt_backend_translate_block() inserts the result
// again. Without this, one intrinsic occupied two of the four ways in
// its set, and a set holding two such duplicates could FIFO-evict a
// live block that still had room. The match condition mirrors
// dbt_cache_lookup so an empty way (guest_pc 0, native_code null) is
// never mistaken for an entry.
//
for (size_t w = 0; w < BLOCK_CACHE_WAYS; w++) {
if (base[w].guest_pc == pc && base[w].native_code) {
base[w].native_code = code;
return;
}
}
// Use first empty way.
for (size_t w = 0; w < BLOCK_CACHE_WAYS; w++) {
if (base[w].guest_pc == 0) {
base[w].guest_pc = pc;
base[w].native_code = code;
return;
}
}
// All ways occupied — evict way 0 (FIFO).
base[0].guest_pc = pc;
base[0].native_code = code;
}
// ---------------------------------------------------------------
// Block chaining
// ---------------------------------------------------------------
// Translate one block, publishing its entry PC for the duration so the
// backend can tell a back-edge from a forward exit (#1571).
//
static uint8_t *dbt_translate_block_at(dbt_state_t *dbt, uint64_t pc) {
dbt->translating_pc = pc;
uint8_t *code = dbt_backend_translate_block(dbt, pc);
dbt->translating_pc = DBT_NO_TRANSLATION;
return code;
}
// Backpatch all pending exits that target the given guest PC.
//
void dbt_backpatch_chains(dbt_state_t *dbt, uint64_t guest_pc,
uint8_t *native_code) {
const auto it = dbt->pending_patch_targets.find(guest_pc);
if (it == dbt->pending_patch_targets.end()) {
return;
}
for (size_t patch_index : it->second) {
dbt_backend_backpatch_jmp(dbt->code_buf,
dbt->patches[patch_index].jmp_offset,
native_code);
dbt->chain_hits++;
}
dbt->pending_patch_targets.erase(it);
}
// ---------------------------------------------------------------
// Public API
// ---------------------------------------------------------------
int dbt_init(dbt_state_t *dbt, uint8_t *memory, size_t memory_size,
int (*ecall_fn)(rv64_ctx_t *, void *), void *ecall_user) {
*dbt = dbt_state_t();
dbt->memory = memory;
dbt->memory_size = memory_size;
dbt->ecall_fn = ecall_fn;
dbt->ecall_user = ecall_user;
// Allocate block cache.
try
{
dbt->cache.assign(BLOCK_CACHE_SIZE, block_entry_t{});
}
catch (const std::bad_alloc &)
{
fprintf(stderr, "dbt: cannot allocate block cache\n");
return -1;
}
// Allocate JIT code buffer (RWX).
dbt->code_buf = jit_alloc(CODE_BUF_SIZE);
if (!dbt->code_buf) {
fprintf(stderr, "dbt: cannot allocate JIT code buffer\n");
dbt->cache.clear();
return -1;
}
// Emit trampoline at the start of the code buffer.
jit_write_begin();
dbt_backend_emit_trampoline(dbt);
dbt_flush_code(dbt, 0);
return 0;
}
void dbt_reset(dbt_state_t *dbt, uint8_t *memory, size_t memory_size,
int (*ecall_fn)(rv64_ctx_t *, void *), void *ecall_user) {
// Update program pointers.
dbt->memory = memory;
dbt->memory_size = memory_size;
dbt->ecall_fn = ecall_fn;
dbt->ecall_user = ecall_user;
if (dbt->blob_code_end > 0) {
// Preserve blob translations: only clear program blocks.
// Evict cache entries whose native code is beyond the blob region.
uint8_t *blob_end = dbt->code_buf + dbt->blob_code_end;
for (size_t i = 0; i < BLOCK_CACHE_SIZE; i++) {
if (dbt->cache[i].native_code >= blob_end) {
dbt->cache[i].guest_pc = 0;
dbt->cache[i].native_code = nullptr;
}
}
dbt->patches.clear();
dbt->pending_patch_targets.clear();
dbt->code_used = dbt->blob_code_end;
// Optional NOP sled for alignment experiments.
// TINYMUX_DBT_PAD=N inserts N bytes of NOP padding before
// program code. On x86-64, each NOP is 1 byte (0x90).
// On AArch64, NOPs are 4 bytes so N is rounded up to a
// multiple of 4.
jit_write_begin();
{
static int pad = -1;
if (pad < 0) {
const char *env = getenv("TINYMUX_DBT_PAD");
// Full-width env parse; pad is a NOP count (#1402).
//
pad = env ? static_cast<int>(strtoll(env, nullptr, 10)) : 0;
}
uint32_t p = static_cast<uint32_t>(pad);
#if defined(__aarch64__)
// AArch64 NOP: 0xD503201F (4 bytes each).
uint32_t n_nops = (p + 3) / 4;
uint32_t pad_bytes = n_nops * 4;
if (n_nops > 0 && dbt->code_used + pad_bytes < CODE_BUF_SIZE) {
for (uint32_t i = 0; i < n_nops; i++) {
uint32_t nop = 0xD503201F;
memcpy(dbt->code_buf + dbt->code_used + i * 4, &nop, 4);
}
dbt->code_used += pad_bytes;
}
#else
// x86-64 NOP: 0x90 (1 byte each).
if (p > 0 && dbt->code_used + p < CODE_BUF_SIZE) {
memset(dbt->code_buf + dbt->code_used, 0x90, p);
dbt->code_used += p;
}
#endif
}
// Keep intrinsics — they're blob-related.
dbt_flush_code(dbt, dbt->blob_code_end);
} else {
// No blob — full reset.
for (auto &entry : dbt->cache) {
entry = {};
}
dbt->patches.clear();
dbt->pending_patch_targets.clear();
dbt->code_used = 0;
jit_write_begin();
dbt_backend_emit_trampoline(dbt);
dbt->num_intrinsics = 0;
memset(dbt->intrinsics, 0, sizeof(dbt->intrinsics));
dbt_flush_code(dbt, 0);
}
// Reset statistics.
dbt->blocks_translated = 0;
dbt->cache_hits = 0;
dbt->cache_misses = 0;
dbt->insns_translated = 0;
dbt->ras_hits = 0;
dbt->ras_misses = 0;
dbt->chain_hits = 0;
dbt->chain_misses = 0;
dbt->insns_fused = 0;
dbt->trace = 0;
dbt->trace_guest_pc = 0;
dbt->trace_guest_pc_filter = false;
}
// Reclaim the program region of the code buffer after a translation was
// declined for want of space.
//
// This is the same reclaim dbt_reset() performs on its preserve-blob branch
// (evict the cache entries above the blob, drop pending chains, rewind
// code_used), made available mid-run. Without it a full buffer is terminal:
// dbt_run returns -1 and nothing ever rewinds code_used, so every later
// program fails to translate too and the JIT is dead for the life of the
// process — including for programs that have nothing to do with whatever
// filled the buffer (#1315).
//
// Safe at the point it is called from — the dispatch loop, after the
// trampoline has returned and before the next block is entered, so no
// translated block is live on the host stack. Blob translations and
// intrinsics are preserved, guest state is untouched, and execution resumes
// from ctx.next_pc; the only cost is re-translating program blocks. The
// exposure to a blob block that was backpatched directly into a program
// block is the same one dbt_reset() already carries between programs.
//
// Bounded to MAX_RECLAIMS_PER_RUN per dbt_run. One reclaim clears whatever
// earlier programs left behind, which is the case worth recovering from. If
// the same run fills the buffer again afterwards then its own live blocks do
// not fit, and no further reclaim can help — it would just re-translate the
// same code and exhaust again. Measured: without this bound a Lua program
// that does not fit reclaimed 2217 times in a single run and burned ~16s of
// CPU before the dispatch limit stopped it. Declining promptly instead lets
// the caller fall back to the interpreter, which is both correct and fast.
//
// Returns false when there is nothing to reclaim (no blob boundary yet,
// already rewound, or the per-run budget is spent) so the caller retries at
// most once per translation and cannot spin.
//
static constexpr uint32_t MAX_RECLAIMS_PER_RUN = 1;
static bool dbt_reclaim_program_code(dbt_state_t *dbt) {
if ( 0 == dbt->blob_code_end
|| dbt->code_used <= dbt->blob_code_end
|| dbt->reclaims_this_run >= MAX_RECLAIMS_PER_RUN) {
return false;
}
uint8_t *blob_end = dbt->code_buf + dbt->blob_code_end;
for (size_t i = 0; i < BLOCK_CACHE_SIZE; i++) {
if (dbt->cache[i].native_code >= blob_end) {
dbt->cache[i].guest_pc = 0;
dbt->cache[i].native_code = nullptr;
}
}
dbt->patches.clear();
dbt->pending_patch_targets.clear();
dbt->code_used = dbt->blob_code_end;
dbt->code_reclaims++;
dbt->reclaims_this_run++;
jit_write_begin();
dbt_flush_code(dbt, dbt->blob_code_end);
return true;
}
// Evict every translation whose block STARTS in [lo, hi), plus any pending
// chain patches waiting on a target in that range. For reusing a guest code
// range for different code while keeping everything translated outside it —
// the per-program code slots (#2129) reassign a 16 KB slot this way.
//
// What this deliberately does NOT do, and why that is sound for slots:
//
// - It does not touch already-applied chains. An applied chain jumps from
// one translated block to another; the only blocks that ever direct-jump
// to a program-slot PC are blocks of the same slot (program code reaches
// other code only via PC-relative branches/JALs within its own slot, or
// JALs into the blob — never into another slot). Evicting the slot's
// cache entries makes its whole translated subgraph unreachable: entry
// is only ever via a cache lookup, and every intra-subgraph chain source
// dies with the subgraph. This is the same reachability argument
// dbt_reset's preserve-blob branch relies on.
//
// - It does not reclaim code_buf bytes. Dead translations accumulate
// until the buffer fills; dbt_reclaim_program_code already handles that
// wholesale, and cache misses re-translate lazily afterwards.
//
// - It does not clear ctx/RAS. dbt_run wipes the whole guest context at
// entry, so stale predictions cannot cross runs.
//
// Blocks that start below `lo` cannot extend into the range and cover a
// reused PC: a translated block's guest span is contiguous from its entry,
// and slot ranges are aligned regions that program code never straddles
// (a program's code lives entirely inside its own slot).
//
void dbt_invalidate_guest_range(dbt_state_t *dbt, uint64_t lo, uint64_t hi) {
for (size_t i = 0; i < BLOCK_CACHE_SIZE; i++) {
uint64_t pc = dbt->cache[i].guest_pc;
if (pc >= lo && pc < hi) {
dbt->cache[i].guest_pc = 0;
dbt->cache[i].native_code = nullptr;
}
}
for (auto it = dbt->pending_patch_targets.begin();
it != dbt->pending_patch_targets.end(); ) {
if (it->first >= lo && it->first < hi) {
it = dbt->pending_patch_targets.erase(it);
} else {
++it;
}
}
}
// Lightweight re-run: update only the ECALL callback and clear the CPU
// context. Keeps the block cache and translated code intact — safe when
// the guest code region is unchanged between runs.
//
void dbt_rerun(dbt_state_t *dbt,
int (*ecall_fn)(rv64_ctx_t *, void *), void *ecall_user) {
dbt->ecall_fn = ecall_fn;
dbt->ecall_user = ecall_user;
}
// Pre-translate all reachable blocks from a guest address.
// Used to ensure Tier 2 blob functions are fully translated and chained
// before superblocks try to inline-call them.
//
void dbt_pretranslate(dbt_state_t *dbt, uint64_t guest_pc) {
// Visited set: prevents re-scanning already-processed blocks.
// Direct-mapped hash — collisions just cause redundant work, not bugs.
static constexpr int VISITED_SIZE = 4096;
static constexpr int VISITED_MASK = VISITED_SIZE - 1;
uint64_t visited_map[VISITED_SIZE];
memset(visited_map, 0, sizeof(visited_map));
auto is_visited = [&](uint64_t pc) -> bool {
return visited_map[(pc >> 2) & VISITED_MASK] == pc;
};
auto mark_visited = [&](uint64_t pc) {
visited_map[(pc >> 2) & VISITED_MASK] = pc;
};
static constexpr int MAX_WORKLIST = 4096;
uint64_t worklist[MAX_WORKLIST];
int wl_count = 0;
auto enqueue_pc = [&](uint64_t pc) -> bool {
if (wl_count >= MAX_WORKLIST || is_visited(pc)) return false;
mark_visited(pc);
worklist[wl_count++] = pc;
return true;
};
enqueue_pc(guest_pc);
while (wl_count > 0) {
uint64_t pc = worklist[--wl_count];
// Scan RV64 code FIRST to discover successors.
// For function calls (JAL rd=1), recursively pretranslate the
// call target BEFORE translating this block. This ensures the
// call target is in the cache when translate_block runs, so the
// inline CALL optimization fires.
uint64_t scan_pc = pc;
for (int i = 0; i < MAX_BLOCK_INSNS && dbt_guest_range_ok(scan_pc, 4, dbt->memory_size); i++) {
uint32_t w;
memcpy(&w, dbt->memory + scan_pc, 4);
rv64_insn_t si;
rv64_decode(w, &si);
rv64_insn_t next_si;
bool have_next = false;
if (dbt_guest_range_ok(scan_pc, 8, dbt->memory_size)) {
uint32_t next_w;
memcpy(&next_w, dbt->memory + scan_pc + 4, 4);
rv64_decode(next_w, &next_si);
have_next = true;
}
if (si.opcode == OP_BRANCH) {
uint64_t target = scan_pc + static_cast<int64_t>(si.imm);
enqueue_pc(target);
enqueue_pc(scan_pc + 4);
break;
}
if (si.opcode == OP_JAL) {
uint64_t target = scan_pc + static_cast<int64_t>(si.imm);
if (si.rd != 0 && !is_visited(target)) {
// Function call: pretranslate callee FIRST so inline
// CALL can find it in the cache.
mark_visited(target);
dbt_pretranslate(dbt, target);
}
if (si.rd == 0) {
// Unconditional jump: follow target, stop scanning.
enqueue_pc(target);
break;
}
// Function call: enqueue both target and fall-through,
// then CONTINUE scanning. The block may absorb this
// call via inline CALL and continue translating past
// it — subsequent calls in the same block must also
// be discovered and pretranslated.
enqueue_pc(target);
enqueue_pc(scan_pc + 4);
scan_pc += 4;
continue;
}
uint64_t target;
uint64_t return_pc;
if (have_next
&& dbt_resolve_direct_jalr_target(scan_pc, si, next_si,
&target, &return_pc)
&& target < dbt->memory_size) {
if (next_si.rd != 0 && !is_visited(target)) {
// Direct call encoded as AUIPC/LUI + JALR: pretranslate
// callee first so shared inline CALL can find it.
mark_visited(target);
dbt_pretranslate(dbt, target);
}
if (next_si.rd == 0) {
// Unconditional indirect jump: follow, stop.
enqueue_pc(target);
break;
}
// Direct call: enqueue and continue scanning past
// the 2-instruction pair (same rationale as JAL rd=1).
enqueue_pc(target);
enqueue_pc(return_pc);
scan_pc += 8; // skip LUI/AUIPC + JALR
continue;
}
if (si.opcode == OP_JALR || si.opcode == OP_SYSTEM) {
break;
}
scan_pc += 4;
}
// Translate if not already cached.
// Brackets are per-iteration so dbt_pretranslate's recursion
// (function-call discovery) doesn't nest write-mode toggles.
if (!dbt_cache_lookup(dbt, pc)) {
jit_write_begin();
uint8_t *code = dbt_translate_block_at(dbt, pc);
if (!code) {
dbt_flush_code(dbt, dbt->code_used);
continue;
}
dbt_cache_insert(dbt, pc, code);
dbt_backpatch_chains(dbt, pc, code);
dbt_flush_code(dbt, static_cast<uint32_t>(code - dbt->code_buf));
}
}
}
void dbt_resolve_chains(dbt_state_t *dbt) {
// Second pass: resolve any patch sites whose targets are now in cache
// but weren't when the JMP was emitted. Only patches still pointing
// to their slow-path stub (unresolved) are updated — already-resolved
// patches have been backpatched by backpatch_chains and must not be
// touched again.
//
uint32_t resolved = 0;
uint32_t already_ok = 0;
uint32_t unresolvable = 0;
jit_write_begin();
for (size_t i = 0; i < dbt->patches.size(); i++) {
uint64_t target = dbt->patches[i].target_pc;
if (target == 0) continue;
// Check: is the JMP still pointing to the slow-path stub?
uint32_t jmp_off = dbt->patches[i].jmp_offset;
// Stale failed-translate sites may point past the arena (#1147).
if (static_cast<size_t>(jmp_off) + 4 > CODE_BUF_SIZE
|| jmp_off >= dbt->code_used) {
unresolvable++;
continue;
}
// Ask the backend where this site currently branches to. The
// decode used to be an inline rel32 read, which is x86-64's
// format; on AArch64 the site is a B imm26 word, so cur_target
// came out as noise, never matched stub_offset, and every site
// fell into the already_ok path below — leaving this pass
// unable to resolve anything on A64 (#1152).
//
uint32_t cur_target =
dbt_backend_decode_jmp_target(dbt->code_buf, jmp_off);
if (cur_target != dbt->patches[i].stub_offset) {
dbt->pending_patch_targets.erase(target);
already_ok++;
continue; // already resolved by backpatch_chains
}
block_entry_t *be = dbt_cache_lookup(dbt, target);
if (be) {
dbt_backend_backpatch_jmp(dbt->code_buf, jmp_off,
be->native_code);
dbt->chain_hits++;
resolved++;
dbt->pending_patch_targets.erase(target);
} else {
unresolvable++;
dbt_trace_translate(dbt, "unresolved chain: target=0x%llX",
static_cast<unsigned long long>(target));
}
}
dbt_trace_translate(dbt,
"resolve_chains: %u resolved, %u already_ok, %u unresolvable of %u total",
resolved, already_ok, unresolvable,
static_cast<unsigned>(dbt->patches.size()));
// Flush I-cache for any backpatched JMP targets, and balance the
// jit_write_begin above (always required so write protection is
// restored on Apple Silicon even when no patches resolved).
dbt_flush_code(dbt, resolved > 0 ? 0 : dbt->code_used);
}
int dbt_run(dbt_state_t *dbt, uint64_t entry_pc, uint64_t stack_top) {
typedef void (*trampoline_fn_t)(rv64_ctx_t *ctx, uint8_t *mem,
void *block, void *cache);
trampoline_fn_t trampoline =
reinterpret_cast<trampoline_fn_t>(static_cast<void *>(dbt->code_buf));
dbt->ctx = {};
dbt->ctx.next_pc = entry_pc;
dbt->ctx.x[2] = stack_top; // SP
dbt->reclaims_this_run = 0;
// Publish the guest bound for the intrinsic stubs' pointer check
// (#1151). Set here, after the ctx wipe, because dbt_reset can change
// memory_size between runs while blob translations survive.
dbt->ctx.mem_size = dbt->memory_size;
uint64_t dispatch_count = 0;
for (;;) {
dispatch_count++;
// Refill the self-loop back-edge budget (#1571). A block with a
// native back-edge decrements this per iteration and returns here at
// zero, so the two guards below actually get a chance to run.
dbt->ctx.loop_budget = DBT_LOOP_BUDGET;
if (dbt->max_dispatch && dispatch_count > dbt->max_dispatch) {
dbt->dispatch_count = dispatch_count;
fprintf(stderr, "dbt: dispatch limit exceeded (%llu)\n",
static_cast<unsigned long long>(dbt->max_dispatch));
return -2;
}
// Wall-clock abort: the per-command alarm has fired. Return so the
// caller aborts the run. Polled only at the top of this dispatch
// loop — after an ECALL return and before each trampoline entry.
// Block chaining / pretranslated softlib can keep execution in
// native code across many guest branches without returning here, so
// pure tier-2 compute is best-effort between host ECALLs (same class
// of mid-function gap a long C builtin has on the AST path). A
// relaxed load is a plain read.
if ( dbt->alarm_flag
&& dbt->alarm_flag->load(std::memory_order_relaxed)) {
dbt->dispatch_count = dispatch_count;
return -3;
}
uint64_t pc = dbt->ctx.next_pc;
// ECALL signal: bit 0 set.
if (pc & 1) {
if (dbt->trace & DBT_TRACE_EXEC) {
fprintf(stderr, "[dbt] disp=%llu ECALL pc=0x%llX\n",
static_cast<unsigned long long>(dispatch_count),
static_cast<unsigned long long>(pc & ~3ULL));
}
dbt->ctx.next_pc = (pc & ~3ULL) + 4;
int rc = dbt->ecall_fn(&dbt->ctx, dbt->ecall_user);
if (rc >= 0) {
dbt->dispatch_count = dispatch_count;
return rc;
}
dbt->ctx.x[0] = 0;
continue;
}
// EBREAK signal: bit 1 set.
if (pc & 2) {
dbt->dispatch_count = dispatch_count;
fprintf(stderr, "dbt: EBREAK at 0x%llX\n",
static_cast<unsigned long long>(pc & ~3ULL));
return -1;
}
// #1864: refuse a guest-controlled next_pc that cannot hold a 4-byte
// instruction (including wrap cases near UINT64_MAX). Backends also
// guard before each fetch; the dispatch path must not treat a wild
// PC as a normal cache miss / translate entry.
//
if (!dbt_guest_range_ok(pc, 4, dbt->memory_size)) {
dbt->dispatch_count = dispatch_count;
fprintf(stderr, "dbt: fetch out of bounds at PC=0x%llX\n",
static_cast<unsigned long long>(pc));
return -1;
}
// Look up or translate block.
block_entry_t *be = dbt_cache_lookup(dbt, pc);
uint8_t *code;
if (be) {
code = be->native_code;
if (dbt->trace & DBT_TRACE_EXEC) {
fprintf(stderr, "[dbt] disp=%llu HIT pc=0x%llX\n",
static_cast<unsigned long long>(dispatch_count),
static_cast<unsigned long long>(pc));
}
} else {
jit_write_begin();
dbt->xlate_fail = dbt_state_t::XLATE_OK;
code = dbt_translate_block_at(dbt, pc);
// Reclaim only on buffer-full. Refuse (#1323) also returns
// nullptr; reclaiming then would wipe live program blocks and
// mis-count code_full (#1331 review).
//
if ( !code
&& dbt->xlate_fail == dbt_state_t::XLATE_FULL
&& dbt_reclaim_program_code(dbt)) {
// Buffer filled with program translations. Reclaim them
// and retry once before declining (#1315).
jit_write_begin();
dbt->xlate_fail = dbt_state_t::XLATE_OK;
code = dbt_translate_block_at(dbt, pc);
}
if (!code) {
if (dbt->xlate_fail == dbt_state_t::XLATE_FULL) {
dbt->code_full++;
if (1 == dbt->code_full) {
// Say so once. Declining is otherwise invisible:
// the caller falls back to the interpreter and
// still produces correct output.
fprintf(stderr, "dbt: code buffer full at pc=0x%llX "
"(used=%u blob=%u cap=%u); JIT declining\n",
static_cast<unsigned long long>(pc),
static_cast<unsigned>(dbt->code_used),
static_cast<unsigned>(dbt->blob_code_end),
static_cast<unsigned>(CODE_BUF_SIZE));
}
}
dbt_flush_code(dbt, dbt->code_used);
dbt->dispatch_count = dispatch_count;
return -1; // full (after reclaim) or refuse
}
dbt_cache_insert(dbt, pc, code);
// Diagnostic map for profiling anonymous JIT frames (#2132):
// TINYMUX_DBT_MAP=<file> appends guest-pc -> host-address lines.
// One shared handle, never closed: a per-block fopen/fclose costs
// more than the translation it annotates once compile-each style
// loops re-translate thousands of times.
{
static FILE *mf = []() -> FILE * {
const char *p = getenv("TINYMUX_DBT_MAP");
return p ? fopen(p, "a") : nullptr;
}();
if (mf) {
fprintf(mf, "block pc=0x%llx host=%p end=%p\n",
static_cast<unsigned long long>(pc),
static_cast<void *>(code),
static_cast<void *>(dbt->code_buf + dbt->code_used));
}
}
// Backpatch any chained exits that were waiting for this block.
dbt_backpatch_chains(dbt, pc, code);
// Flush I-cache for newly generated and backpatched code.
dbt_flush_code(dbt, static_cast<uint32_t>(code - dbt->code_buf));
if (dbt->trace & DBT_TRACE_EXEC) {
fprintf(stderr, "[dbt] disp=%llu MISS pc=0x%llX\n",
static_cast<unsigned long long>(dispatch_count),
static_cast<unsigned long long>(pc));
}
}
// Execute.
trampoline(&dbt->ctx, dbt->memory, code, dbt->cache.data());
dbt->ctx.x[0] = 0;
}
}
int dbt_resume(dbt_state_t *dbt, uint64_t entry_pc) {
typedef void (*trampoline_fn_t)(rv64_ctx_t *ctx, uint8_t *mem,
void *block, void *cache);
trampoline_fn_t trampoline =
reinterpret_cast<trampoline_fn_t>(static_cast<void *>(dbt->code_buf));
dbt->ctx.next_pc = entry_pc;
uint64_t dispatch_count = 0;
for (;;) {
dispatch_count++;
// Refill the self-loop back-edge budget (#1571). A block with a
// native back-edge decrements this per iteration and returns here at
// zero, so the two guards below actually get a chance to run.
dbt->ctx.loop_budget = DBT_LOOP_BUDGET;
if (dbt->max_dispatch && dispatch_count > dbt->max_dispatch) {
dbt->dispatch_count = dispatch_count;
fprintf(stderr, "dbt: dispatch limit exceeded (%llu)\n",
static_cast<unsigned long long>(dbt->max_dispatch));
return -2;
}
// Wall-clock abort: the per-command alarm has fired. Return so the
// caller aborts the run. Polled only at the top of this dispatch
// loop — after an ECALL return and before each trampoline entry.
// Block chaining / pretranslated softlib can keep execution in
// native code across many guest branches without returning here, so
// pure tier-2 compute is best-effort between host ECALLs (same class
// of mid-function gap a long C builtin has on the AST path). A
// relaxed load is a plain read.
if ( dbt->alarm_flag
&& dbt->alarm_flag->load(std::memory_order_relaxed)) {
dbt->dispatch_count = dispatch_count;
return -3;
}
uint64_t pc = dbt->ctx.next_pc;
// ECALL signal: bit 0 set.
if (pc & 1) {
if (dbt->trace & DBT_TRACE_EXEC) {
fprintf(stderr, "[dbt] disp=%llu ECALL pc=0x%llX\n",
static_cast<unsigned long long>(dispatch_count),
static_cast<unsigned long long>(pc & ~3ULL));
}
dbt->ctx.next_pc = (pc & ~3ULL) + 4;
int rc = dbt->ecall_fn(&dbt->ctx, dbt->ecall_user);
if (rc >= 0) {
dbt->dispatch_count = dispatch_count;
return rc;
}
dbt->ctx.x[0] = 0;
continue;
}
// EBREAK signal: bit 1 set.
if (pc & 2) {
dbt->dispatch_count = dispatch_count;
fprintf(stderr, "dbt: EBREAK at 0x%llX\n",
static_cast<unsigned long long>(pc & ~3ULL));
return -1;
}
// #1864: refuse a guest-controlled next_pc that cannot hold a 4-byte
// instruction (including wrap cases near UINT64_MAX). Backends also
// guard before each fetch; the dispatch path must not treat a wild
// PC as a normal cache miss / translate entry.
//
if (!dbt_guest_range_ok(pc, 4, dbt->memory_size)) {
dbt->dispatch_count = dispatch_count;
fprintf(stderr, "dbt: fetch out of bounds at PC=0x%llX\n",
static_cast<unsigned long long>(pc));
return -1;
}
// Look up or translate block.
block_entry_t *be = dbt_cache_lookup(dbt, pc);
uint8_t *code;
if (be) {
code = be->native_code;
if (dbt->trace & DBT_TRACE_EXEC) {
fprintf(stderr, "[dbt] disp=%llu HIT pc=0x%llX\n",
static_cast<unsigned long long>(dispatch_count),
static_cast<unsigned long long>(pc));
}
} else {
jit_write_begin();
dbt->xlate_fail = dbt_state_t::XLATE_OK;
code = dbt_translate_block_at(dbt, pc);
if ( !code
&& dbt->xlate_fail == dbt_state_t::XLATE_FULL
&& dbt_reclaim_program_code(dbt)) {
// Buffer filled with program translations. Reclaim them
// and retry once before declining (#1315 / #1331).
jit_write_begin();
dbt->xlate_fail = dbt_state_t::XLATE_OK;
code = dbt_translate_block_at(dbt, pc);
}
if (!code) {
if (dbt->xlate_fail == dbt_state_t::XLATE_FULL) {
dbt->code_full++;
if (1 == dbt->code_full) {
// Say so once. Declining is otherwise invisible:
// the caller falls back to the interpreter and
// still produces correct output.
fprintf(stderr, "dbt: code buffer full at pc=0x%llX "
"(used=%u blob=%u cap=%u); JIT declining\n",
static_cast<unsigned long long>(pc),
static_cast<unsigned>(dbt->code_used),
static_cast<unsigned>(dbt->blob_code_end),
static_cast<unsigned>(CODE_BUF_SIZE));
}
}
dbt_flush_code(dbt, dbt->code_used);
dbt->dispatch_count = dispatch_count;
return -1; // full (after reclaim) or refuse
}
dbt_cache_insert(dbt, pc, code);
// Diagnostic map for profiling anonymous JIT frames (#2132):
// TINYMUX_DBT_MAP=<file> appends guest-pc -> host-address lines.
// One shared handle, never closed: a per-block fopen/fclose costs
// more than the translation it annotates once compile-each style
// loops re-translate thousands of times.
{
static FILE *mf = []() -> FILE * {
const char *p = getenv("TINYMUX_DBT_MAP");
return p ? fopen(p, "a") : nullptr;
}();
if (mf) {
fprintf(mf, "block pc=0x%llx host=%p end=%p\n",
static_cast<unsigned long long>(pc),
static_cast<void *>(code),
static_cast<void *>(dbt->code_buf + dbt->code_used));
}
}
// Backpatch any chained exits that were waiting for this block.
dbt_backpatch_chains(dbt, pc, code);
// Flush I-cache for newly generated and backpatched code.
dbt_flush_code(dbt, static_cast<uint32_t>(code - dbt->code_buf));
if (dbt->trace & DBT_TRACE_EXEC) {
fprintf(stderr, "[dbt] disp=%llu MISS pc=0x%llX\n",
static_cast<unsigned long long>(dispatch_count),
static_cast<unsigned long long>(pc));
}
}
// Execute.
trampoline(&dbt->ctx, dbt->memory, code, dbt->cache.data());
dbt->ctx.x[0] = 0;
}
}
// Diagnostic dump of the translated code for offline disassembly (#2132):
// TINYMUX_DBT_CODEDUMP=<file> writes code_buf[0..code_used] plus a sidecar
// <file>.base with the runtime base address, at cleanup.
static void dbt_maybe_dump_code(dbt_state_t *dbt)
{
const char *dump_path = getenv("TINYMUX_DBT_CODEDUMP");
if (!dump_path || !dbt->code_buf) {
return;
}
FILE *df = fopen(dump_path, "wb");
if (df) {
fwrite(dbt->code_buf, 1, dbt->code_used, df);
fclose(df);
}
const std::string side_base = std::string(dump_path) + ".base";
FILE *bf = fopen(side_base.c_str(), "w");
if (bf) {
fprintf(bf, "%p %u\n", static_cast<void *>(dbt->code_buf),
dbt->code_used);
fclose(bf);
}
// Every cached block (covers pretranslated blob blocks the dbt_run map
// never sees).
const std::string side_cache = std::string(dump_path) + ".cache";
FILE *cf = fopen(side_cache.c_str(), "w");
if (cf) {
for (size_t i = 0; i < BLOCK_CACHE_SIZE; i++) {
if (dbt->cache[i].guest_pc) {
fprintf(cf, "entry pc=0x%llx host=%p\n",
static_cast<unsigned long long>(dbt->cache[i].guest_pc),
static_cast<void *>(dbt->cache[i].native_code));
}
}
fclose(cf);
}
}
void dbt_cleanup(dbt_state_t *dbt) {
dbt_maybe_dump_code(dbt);
if (dbt->code_buf) {
jit_free(dbt->code_buf, CODE_BUF_SIZE);
dbt->code_buf = nullptr;
}
dbt->cache.clear();
dbt->cache.shrink_to_fit();
dbt->patches.clear();
dbt->patches.shrink_to_fit();
dbt->pending_patch_targets.clear();
}
Reference in a new issue View git blame Copy permalink