/*! \file dbt.cpp * \brief RV64IMD dynamic binary translator — shared code. * * Platform-independent parts of the DBT: block cache, trace helpers, * dispatch loop, and public API. The per-platform translation backend * (trampoline, instruction translation, intrinsic stubs) is in a * separate file selected at configure time (e.g. dbt_x64_sysv.cpp). * * See docs/DBT-PORTABILITY.md for the multi-platform design. */ #include "dbt.h" #include "dbt_host.h" #include "dbt_internal.h" #include "dbt_jit_mem.h" #include "dbt_decoder.h" #include #include #include #include #include #include // --------------------------------------------------------------- // Instruction cache coherency // --------------------------------------------------------------- // Flush the instruction cache for newly generated or modified code. // On AArch64, the I-cache and D-cache are not coherent — writes to // executable memory require an explicit cache maintenance operation // before the CPU will fetch the new instructions. On x86-64, this // is a no-op (coherent I-cache). // static inline void dbt_flush_code(dbt_state_t *dbt, uint32_t from_offset) { jit_write_end(dbt->code_buf + from_offset, dbt->code_used - from_offset); } // --------------------------------------------------------------- // Trace helpers // --------------------------------------------------------------- bool dbt_trace_translate_enabled(const dbt_state_t *dbt, uint64_t guest_pc) { if ((dbt->trace & DBT_TRACE_TRANSLATE) == 0) return false; return !dbt->trace_guest_pc_filter || dbt->trace_guest_pc == guest_pc; } void dbt_trace_translate_pc(dbt_state_t *dbt, uint64_t guest_pc, const char *fmt, ...) { if (!dbt_trace_translate_enabled(dbt, guest_pc)) return; va_list ap; va_start(ap, fmt); fputs("[dbt-xlate] ", stderr); vfprintf(stderr, fmt, ap); fputc('\n', stderr); va_end(ap); } void dbt_trace_translate(dbt_state_t *dbt, const char *fmt, ...) { if ((dbt->trace & DBT_TRACE_TRANSLATE) == 0) return; va_list ap; va_start(ap, fmt); fputs("[dbt-xlate] ", stderr); vfprintf(stderr, fmt, ap); fputc('\n', stderr); va_end(ap); } void dbt_trace_fusion(dbt_state_t *dbt, uint64_t pc, const char *kind) { dbt_trace_translate_pc(dbt, pc, "fusion guest_pc=0x%llX kind=%s", static_cast(pc), kind); } // --------------------------------------------------------------- // Direct JALR target resolution (pure computation) // --------------------------------------------------------------- bool dbt_resolve_direct_jalr_target(uint64_t pc, const rv64_insn_t &insn, const rv64_insn_t &next, uint64_t *target_out, uint64_t *return_pc_out) { if (!insn.rd) return false; if (insn.opcode != OP_LUI && insn.opcode != OP_AUIPC) return false; if (next.opcode != OP_JALR || next.rs1 != insn.rd) return false; int64_t base = (insn.opcode == OP_AUIPC) ? static_cast(pc) : 0; int64_t target = base + static_cast(insn.imm) + static_cast(next.imm); target &= ~1LL; // clear bit 0 per JALR spec if (target < 0) return false; if (target_out) *target_out = static_cast(target); if (return_pc_out) *return_pc_out = pc + 8; return true; } // --------------------------------------------------------------- // Out-of-range guest pointer sink (#1151) // --------------------------------------------------------------- // // Intrinsic stubs convert a guest offset to a host pointer and hand it // straight to strlen/strcpy/memcpy/memset with lengths the guest chose. // The interpreter bounds-checks every access (dbt_interp.cpp mem_check); // the DBT did not, so a bad guest pointer became a host read or write at // an arbitrary address. // // A conversion that fails the bound is redirected here instead of being // trapped. That follows the interpreter, which logs and returns 0 rather // than halting -- there is no fault path in the DBT to unwind to. A // clamped strlen sees an immediate NUL, a clamped memcpy reads zeros, and // nothing touches host memory outside this buffer. Writes through a // clamped pointer land here too and may dirty it; that is contained, which // is the property being bought. ctx.mem_clamps counts the redirects so a // live clamp is visible rather than silent. // alignas(16) uint8_t g_dbt_safe_page[DBT_SAFE_PAGE_SIZE]; // --------------------------------------------------------------- // Block cache // --------------------------------------------------------------- static inline uint32_t cache_set(uint64_t pc) { uint32_t h = static_cast(pc >> 2); h ^= (h >> 10); return h & BLOCK_CACHE_MASK; } block_entry_t *dbt_cache_lookup(dbt_state_t *dbt, uint64_t pc) { uint32_t set = cache_set(pc); block_entry_t *base = &dbt->cache[set * BLOCK_CACHE_WAYS]; for (size_t w = 0; w < BLOCK_CACHE_WAYS; w++) { if (base[w].guest_pc == pc && base[w].native_code) { dbt->cache_hits++; return &base[w]; } } dbt->cache_misses++; return nullptr; } void dbt_cache_insert(dbt_state_t *dbt, uint64_t pc, uint8_t *code) { uint32_t set = cache_set(pc); block_entry_t *base = &dbt->cache[set * BLOCK_CACHE_WAYS]; // An entry for this pc already exists — update it in place rather than // consuming a second way (#1153). Intrinsic blocks are inserted twice: // try_emit_intrinsic() in the backend inserts before returning, and // every caller of dbt_backend_translate_block() inserts the result // again. Without this, one intrinsic occupied two of the four ways in // its set, and a set holding two such duplicates could FIFO-evict a // live block that still had room. The match condition mirrors // dbt_cache_lookup so an empty way (guest_pc 0, native_code null) is // never mistaken for an entry. // for (size_t w = 0; w < BLOCK_CACHE_WAYS; w++) { if (base[w].guest_pc == pc && base[w].native_code) { base[w].native_code = code; return; } } // Use first empty way. for (size_t w = 0; w < BLOCK_CACHE_WAYS; w++) { if (base[w].guest_pc == 0) { base[w].guest_pc = pc; base[w].native_code = code; return; } } // All ways occupied — evict way 0 (FIFO). base[0].guest_pc = pc; base[0].native_code = code; } // --------------------------------------------------------------- // Block chaining // --------------------------------------------------------------- // Translate one block, publishing its entry PC for the duration so the // backend can tell a back-edge from a forward exit (#1571). // static uint8_t *dbt_translate_block_at(dbt_state_t *dbt, uint64_t pc) { dbt->translating_pc = pc; uint8_t *code = dbt_backend_translate_block(dbt, pc); dbt->translating_pc = DBT_NO_TRANSLATION; return code; } // Backpatch all pending exits that target the given guest PC. // void dbt_backpatch_chains(dbt_state_t *dbt, uint64_t guest_pc, uint8_t *native_code) { const auto it = dbt->pending_patch_targets.find(guest_pc); if (it == dbt->pending_patch_targets.end()) { return; } for (size_t patch_index : it->second) { dbt_backend_backpatch_jmp(dbt->code_buf, dbt->patches[patch_index].jmp_offset, native_code); dbt->chain_hits++; } dbt->pending_patch_targets.erase(it); } // --------------------------------------------------------------- // Public API // --------------------------------------------------------------- int dbt_init(dbt_state_t *dbt, uint8_t *memory, size_t memory_size, int (*ecall_fn)(rv64_ctx_t *, void *), void *ecall_user) { *dbt = dbt_state_t(); dbt->memory = memory; dbt->memory_size = memory_size; dbt->ecall_fn = ecall_fn; dbt->ecall_user = ecall_user; // Allocate block cache. try { dbt->cache.assign(BLOCK_CACHE_SIZE, block_entry_t{}); } catch (const std::bad_alloc &) { fprintf(stderr, "dbt: cannot allocate block cache\n"); return -1; } // Allocate JIT code buffer (RWX). dbt->code_buf = jit_alloc(CODE_BUF_SIZE); if (!dbt->code_buf) { fprintf(stderr, "dbt: cannot allocate JIT code buffer\n"); dbt->cache.clear(); return -1; } // Emit trampoline at the start of the code buffer. jit_write_begin(); dbt_backend_emit_trampoline(dbt); dbt_flush_code(dbt, 0); return 0; } void dbt_reset(dbt_state_t *dbt, uint8_t *memory, size_t memory_size, int (*ecall_fn)(rv64_ctx_t *, void *), void *ecall_user) { // Update program pointers. dbt->memory = memory; dbt->memory_size = memory_size; dbt->ecall_fn = ecall_fn; dbt->ecall_user = ecall_user; if (dbt->blob_code_end > 0) { // Preserve blob translations: only clear program blocks. // Evict cache entries whose native code is beyond the blob region. uint8_t *blob_end = dbt->code_buf + dbt->blob_code_end; for (size_t i = 0; i < BLOCK_CACHE_SIZE; i++) { if (dbt->cache[i].native_code >= blob_end) { dbt->cache[i].guest_pc = 0; dbt->cache[i].native_code = nullptr; } } dbt->patches.clear(); dbt->pending_patch_targets.clear(); dbt->code_used = dbt->blob_code_end; // Optional NOP sled for alignment experiments. // TINYMUX_DBT_PAD=N inserts N bytes of NOP padding before // program code. On x86-64, each NOP is 1 byte (0x90). // On AArch64, NOPs are 4 bytes so N is rounded up to a // multiple of 4. jit_write_begin(); { static int pad = -1; if (pad < 0) { const char *env = getenv("TINYMUX_DBT_PAD"); // Full-width env parse; pad is a NOP count (#1402). // pad = env ? static_cast(strtoll(env, nullptr, 10)) : 0; } uint32_t p = static_cast(pad); #if defined(__aarch64__) // AArch64 NOP: 0xD503201F (4 bytes each). uint32_t n_nops = (p + 3) / 4; uint32_t pad_bytes = n_nops * 4; if (n_nops > 0 && dbt->code_used + pad_bytes < CODE_BUF_SIZE) { for (uint32_t i = 0; i < n_nops; i++) { uint32_t nop = 0xD503201F; memcpy(dbt->code_buf + dbt->code_used + i * 4, &nop, 4); } dbt->code_used += pad_bytes; } #else // x86-64 NOP: 0x90 (1 byte each). if (p > 0 && dbt->code_used + p < CODE_BUF_SIZE) { memset(dbt->code_buf + dbt->code_used, 0x90, p); dbt->code_used += p; } #endif } // Keep intrinsics — they're blob-related. dbt_flush_code(dbt, dbt->blob_code_end); } else { // No blob — full reset. for (auto &entry : dbt->cache) { entry = {}; } dbt->patches.clear(); dbt->pending_patch_targets.clear(); dbt->code_used = 0; jit_write_begin(); dbt_backend_emit_trampoline(dbt); dbt->num_intrinsics = 0; memset(dbt->intrinsics, 0, sizeof(dbt->intrinsics)); dbt_flush_code(dbt, 0); } // Reset statistics. dbt->blocks_translated = 0; dbt->cache_hits = 0; dbt->cache_misses = 0; dbt->insns_translated = 0; dbt->ras_hits = 0; dbt->ras_misses = 0; dbt->chain_hits = 0; dbt->chain_misses = 0; dbt->insns_fused = 0; dbt->trace = 0; dbt->trace_guest_pc = 0; dbt->trace_guest_pc_filter = false; } // Reclaim the program region of the code buffer after a translation was // declined for want of space. // // This is the same reclaim dbt_reset() performs on its preserve-blob branch // (evict the cache entries above the blob, drop pending chains, rewind // code_used), made available mid-run. Without it a full buffer is terminal: // dbt_run returns -1 and nothing ever rewinds code_used, so every later // program fails to translate too and the JIT is dead for the life of the // process — including for programs that have nothing to do with whatever // filled the buffer (#1315). // // Safe at the point it is called from — the dispatch loop, after the // trampoline has returned and before the next block is entered, so no // translated block is live on the host stack. Blob translations and // intrinsics are preserved, guest state is untouched, and execution resumes // from ctx.next_pc; the only cost is re-translating program blocks. The // exposure to a blob block that was backpatched directly into a program // block is the same one dbt_reset() already carries between programs. // // Bounded to MAX_RECLAIMS_PER_RUN per dbt_run. One reclaim clears whatever // earlier programs left behind, which is the case worth recovering from. If // the same run fills the buffer again afterwards then its own live blocks do // not fit, and no further reclaim can help — it would just re-translate the // same code and exhaust again. Measured: without this bound a Lua program // that does not fit reclaimed 2217 times in a single run and burned ~16s of // CPU before the dispatch limit stopped it. Declining promptly instead lets // the caller fall back to the interpreter, which is both correct and fast. // // Returns false when there is nothing to reclaim (no blob boundary yet, // already rewound, or the per-run budget is spent) so the caller retries at // most once per translation and cannot spin. // static constexpr uint32_t MAX_RECLAIMS_PER_RUN = 1; static bool dbt_reclaim_program_code(dbt_state_t *dbt) { if ( 0 == dbt->blob_code_end || dbt->code_used <= dbt->blob_code_end || dbt->reclaims_this_run >= MAX_RECLAIMS_PER_RUN) { return false; } uint8_t *blob_end = dbt->code_buf + dbt->blob_code_end; for (size_t i = 0; i < BLOCK_CACHE_SIZE; i++) { if (dbt->cache[i].native_code >= blob_end) { dbt->cache[i].guest_pc = 0; dbt->cache[i].native_code = nullptr; } } dbt->patches.clear(); dbt->pending_patch_targets.clear(); dbt->code_used = dbt->blob_code_end; dbt->code_reclaims++; dbt->reclaims_this_run++; jit_write_begin(); dbt_flush_code(dbt, dbt->blob_code_end); return true; } // Evict every translation whose block STARTS in [lo, hi), plus any pending // chain patches waiting on a target in that range. For reusing a guest code // range for different code while keeping everything translated outside it — // the per-program code slots (#2129) reassign a 16 KB slot this way. // // What this deliberately does NOT do, and why that is sound for slots: // // - It does not touch already-applied chains. An applied chain jumps from // one translated block to another; the only blocks that ever direct-jump // to a program-slot PC are blocks of the same slot (program code reaches // other code only via PC-relative branches/JALs within its own slot, or // JALs into the blob — never into another slot). Evicting the slot's // cache entries makes its whole translated subgraph unreachable: entry // is only ever via a cache lookup, and every intra-subgraph chain source // dies with the subgraph. This is the same reachability argument // dbt_reset's preserve-blob branch relies on. // // - It does not reclaim code_buf bytes. Dead translations accumulate // until the buffer fills; dbt_reclaim_program_code already handles that // wholesale, and cache misses re-translate lazily afterwards. // // - It does not clear ctx/RAS. dbt_run wipes the whole guest context at // entry, so stale predictions cannot cross runs. // // Blocks that start below `lo` cannot extend into the range and cover a // reused PC: a translated block's guest span is contiguous from its entry, // and slot ranges are aligned regions that program code never straddles // (a program's code lives entirely inside its own slot). // void dbt_invalidate_guest_range(dbt_state_t *dbt, uint64_t lo, uint64_t hi) { for (size_t i = 0; i < BLOCK_CACHE_SIZE; i++) { uint64_t pc = dbt->cache[i].guest_pc; if (pc >= lo && pc < hi) { dbt->cache[i].guest_pc = 0; dbt->cache[i].native_code = nullptr; } } for (auto it = dbt->pending_patch_targets.begin(); it != dbt->pending_patch_targets.end(); ) { if (it->first >= lo && it->first < hi) { it = dbt->pending_patch_targets.erase(it); } else { ++it; } } } // Lightweight re-run: update only the ECALL callback and clear the CPU // context. Keeps the block cache and translated code intact — safe when // the guest code region is unchanged between runs. // void dbt_rerun(dbt_state_t *dbt, int (*ecall_fn)(rv64_ctx_t *, void *), void *ecall_user) { dbt->ecall_fn = ecall_fn; dbt->ecall_user = ecall_user; } // Pre-translate all reachable blocks from a guest address. // Used to ensure Tier 2 blob functions are fully translated and chained // before superblocks try to inline-call them. // void dbt_pretranslate(dbt_state_t *dbt, uint64_t guest_pc) { // Visited set: prevents re-scanning already-processed blocks. // Direct-mapped hash — collisions just cause redundant work, not bugs. static constexpr int VISITED_SIZE = 4096; static constexpr int VISITED_MASK = VISITED_SIZE - 1; uint64_t visited_map[VISITED_SIZE]; memset(visited_map, 0, sizeof(visited_map)); auto is_visited = [&](uint64_t pc) -> bool { return visited_map[(pc >> 2) & VISITED_MASK] == pc; }; auto mark_visited = [&](uint64_t pc) { visited_map[(pc >> 2) & VISITED_MASK] = pc; }; static constexpr int MAX_WORKLIST = 4096; uint64_t worklist[MAX_WORKLIST]; int wl_count = 0; auto enqueue_pc = [&](uint64_t pc) -> bool { if (wl_count >= MAX_WORKLIST || is_visited(pc)) return false; mark_visited(pc); worklist[wl_count++] = pc; return true; }; enqueue_pc(guest_pc); while (wl_count > 0) { uint64_t pc = worklist[--wl_count]; // Scan RV64 code FIRST to discover successors. // For function calls (JAL rd=1), recursively pretranslate the // call target BEFORE translating this block. This ensures the // call target is in the cache when translate_block runs, so the // inline CALL optimization fires. uint64_t scan_pc = pc; for (int i = 0; i < MAX_BLOCK_INSNS && dbt_guest_range_ok(scan_pc, 4, dbt->memory_size); i++) { uint32_t w; memcpy(&w, dbt->memory + scan_pc, 4); rv64_insn_t si; rv64_decode(w, &si); rv64_insn_t next_si; bool have_next = false; if (dbt_guest_range_ok(scan_pc, 8, dbt->memory_size)) { uint32_t next_w; memcpy(&next_w, dbt->memory + scan_pc + 4, 4); rv64_decode(next_w, &next_si); have_next = true; } if (si.opcode == OP_BRANCH) { uint64_t target = scan_pc + static_cast(si.imm); enqueue_pc(target); enqueue_pc(scan_pc + 4); break; } if (si.opcode == OP_JAL) { uint64_t target = scan_pc + static_cast(si.imm); if (si.rd != 0 && !is_visited(target)) { // Function call: pretranslate callee FIRST so inline // CALL can find it in the cache. mark_visited(target); dbt_pretranslate(dbt, target); } if (si.rd == 0) { // Unconditional jump: follow target, stop scanning. enqueue_pc(target); break; } // Function call: enqueue both target and fall-through, // then CONTINUE scanning. The block may absorb this // call via inline CALL and continue translating past // it — subsequent calls in the same block must also // be discovered and pretranslated. enqueue_pc(target); enqueue_pc(scan_pc + 4); scan_pc += 4; continue; } uint64_t target; uint64_t return_pc; if (have_next && dbt_resolve_direct_jalr_target(scan_pc, si, next_si, &target, &return_pc) && target < dbt->memory_size) { if (next_si.rd != 0 && !is_visited(target)) { // Direct call encoded as AUIPC/LUI + JALR: pretranslate // callee first so shared inline CALL can find it. mark_visited(target); dbt_pretranslate(dbt, target); } if (next_si.rd == 0) { // Unconditional indirect jump: follow, stop. enqueue_pc(target); break; } // Direct call: enqueue and continue scanning past // the 2-instruction pair (same rationale as JAL rd=1). enqueue_pc(target); enqueue_pc(return_pc); scan_pc += 8; // skip LUI/AUIPC + JALR continue; } if (si.opcode == OP_JALR || si.opcode == OP_SYSTEM) { break; } scan_pc += 4; } // Translate if not already cached. // Brackets are per-iteration so dbt_pretranslate's recursion // (function-call discovery) doesn't nest write-mode toggles. if (!dbt_cache_lookup(dbt, pc)) { jit_write_begin(); uint8_t *code = dbt_translate_block_at(dbt, pc); if (!code) { dbt_flush_code(dbt, dbt->code_used); continue; } dbt_cache_insert(dbt, pc, code); dbt_backpatch_chains(dbt, pc, code); dbt_flush_code(dbt, static_cast(code - dbt->code_buf)); } } } void dbt_resolve_chains(dbt_state_t *dbt) { // Second pass: resolve any patch sites whose targets are now in cache // but weren't when the JMP was emitted. Only patches still pointing // to their slow-path stub (unresolved) are updated — already-resolved // patches have been backpatched by backpatch_chains and must not be // touched again. // uint32_t resolved = 0; uint32_t already_ok = 0; uint32_t unresolvable = 0; jit_write_begin(); for (size_t i = 0; i < dbt->patches.size(); i++) { uint64_t target = dbt->patches[i].target_pc; if (target == 0) continue; // Check: is the JMP still pointing to the slow-path stub? uint32_t jmp_off = dbt->patches[i].jmp_offset; // Stale failed-translate sites may point past the arena (#1147). if (static_cast(jmp_off) + 4 > CODE_BUF_SIZE || jmp_off >= dbt->code_used) { unresolvable++; continue; } // Ask the backend where this site currently branches to. The // decode used to be an inline rel32 read, which is x86-64's // format; on AArch64 the site is a B imm26 word, so cur_target // came out as noise, never matched stub_offset, and every site // fell into the already_ok path below — leaving this pass // unable to resolve anything on A64 (#1152). // uint32_t cur_target = dbt_backend_decode_jmp_target(dbt->code_buf, jmp_off); if (cur_target != dbt->patches[i].stub_offset) { dbt->pending_patch_targets.erase(target); already_ok++; continue; // already resolved by backpatch_chains } block_entry_t *be = dbt_cache_lookup(dbt, target); if (be) { dbt_backend_backpatch_jmp(dbt->code_buf, jmp_off, be->native_code); dbt->chain_hits++; resolved++; dbt->pending_patch_targets.erase(target); } else { unresolvable++; dbt_trace_translate(dbt, "unresolved chain: target=0x%llX", static_cast(target)); } } dbt_trace_translate(dbt, "resolve_chains: %u resolved, %u already_ok, %u unresolvable of %u total", resolved, already_ok, unresolvable, static_cast(dbt->patches.size())); // Flush I-cache for any backpatched JMP targets, and balance the // jit_write_begin above (always required so write protection is // restored on Apple Silicon even when no patches resolved). dbt_flush_code(dbt, resolved > 0 ? 0 : dbt->code_used); } int dbt_run(dbt_state_t *dbt, uint64_t entry_pc, uint64_t stack_top) { typedef void (*trampoline_fn_t)(rv64_ctx_t *ctx, uint8_t *mem, void *block, void *cache); trampoline_fn_t trampoline = reinterpret_cast(static_cast(dbt->code_buf)); dbt->ctx = {}; dbt->ctx.next_pc = entry_pc; dbt->ctx.x[2] = stack_top; // SP dbt->reclaims_this_run = 0; // Publish the guest bound for the intrinsic stubs' pointer check // (#1151). Set here, after the ctx wipe, because dbt_reset can change // memory_size between runs while blob translations survive. dbt->ctx.mem_size = dbt->memory_size; uint64_t dispatch_count = 0; for (;;) { dispatch_count++; // Refill the self-loop back-edge budget (#1571). A block with a // native back-edge decrements this per iteration and returns here at // zero, so the two guards below actually get a chance to run. dbt->ctx.loop_budget = DBT_LOOP_BUDGET; if (dbt->max_dispatch && dispatch_count > dbt->max_dispatch) { dbt->dispatch_count = dispatch_count; fprintf(stderr, "dbt: dispatch limit exceeded (%llu)\n", static_cast(dbt->max_dispatch)); return -2; } // Wall-clock abort: the per-command alarm has fired. Return so the // caller aborts the run. Polled only at the top of this dispatch // loop — after an ECALL return and before each trampoline entry. // Block chaining / pretranslated softlib can keep execution in // native code across many guest branches without returning here, so // pure tier-2 compute is best-effort between host ECALLs (same class // of mid-function gap a long C builtin has on the AST path). A // relaxed load is a plain read. if ( dbt->alarm_flag && dbt->alarm_flag->load(std::memory_order_relaxed)) { dbt->dispatch_count = dispatch_count; return -3; } uint64_t pc = dbt->ctx.next_pc; // ECALL signal: bit 0 set. if (pc & 1) { if (dbt->trace & DBT_TRACE_EXEC) { fprintf(stderr, "[dbt] disp=%llu ECALL pc=0x%llX\n", static_cast(dispatch_count), static_cast(pc & ~3ULL)); } dbt->ctx.next_pc = (pc & ~3ULL) + 4; int rc = dbt->ecall_fn(&dbt->ctx, dbt->ecall_user); if (rc >= 0) { dbt->dispatch_count = dispatch_count; return rc; } dbt->ctx.x[0] = 0; continue; } // EBREAK signal: bit 1 set. if (pc & 2) { dbt->dispatch_count = dispatch_count; fprintf(stderr, "dbt: EBREAK at 0x%llX\n", static_cast(pc & ~3ULL)); return -1; } // #1864: refuse a guest-controlled next_pc that cannot hold a 4-byte // instruction (including wrap cases near UINT64_MAX). Backends also // guard before each fetch; the dispatch path must not treat a wild // PC as a normal cache miss / translate entry. // if (!dbt_guest_range_ok(pc, 4, dbt->memory_size)) { dbt->dispatch_count = dispatch_count; fprintf(stderr, "dbt: fetch out of bounds at PC=0x%llX\n", static_cast(pc)); return -1; } // Look up or translate block. block_entry_t *be = dbt_cache_lookup(dbt, pc); uint8_t *code; if (be) { code = be->native_code; if (dbt->trace & DBT_TRACE_EXEC) { fprintf(stderr, "[dbt] disp=%llu HIT pc=0x%llX\n", static_cast(dispatch_count), static_cast(pc)); } } else { jit_write_begin(); dbt->xlate_fail = dbt_state_t::XLATE_OK; code = dbt_translate_block_at(dbt, pc); // Reclaim only on buffer-full. Refuse (#1323) also returns // nullptr; reclaiming then would wipe live program blocks and // mis-count code_full (#1331 review). // if ( !code && dbt->xlate_fail == dbt_state_t::XLATE_FULL && dbt_reclaim_program_code(dbt)) { // Buffer filled with program translations. Reclaim them // and retry once before declining (#1315). jit_write_begin(); dbt->xlate_fail = dbt_state_t::XLATE_OK; code = dbt_translate_block_at(dbt, pc); } if (!code) { if (dbt->xlate_fail == dbt_state_t::XLATE_FULL) { dbt->code_full++; if (1 == dbt->code_full) { // Say so once. Declining is otherwise invisible: // the caller falls back to the interpreter and // still produces correct output. fprintf(stderr, "dbt: code buffer full at pc=0x%llX " "(used=%u blob=%u cap=%u); JIT declining\n", static_cast(pc), static_cast(dbt->code_used), static_cast(dbt->blob_code_end), static_cast(CODE_BUF_SIZE)); } } dbt_flush_code(dbt, dbt->code_used); dbt->dispatch_count = dispatch_count; return -1; // full (after reclaim) or refuse } dbt_cache_insert(dbt, pc, code); // Diagnostic map for profiling anonymous JIT frames (#2132): // TINYMUX_DBT_MAP= appends guest-pc -> host-address lines. // One shared handle, never closed: a per-block fopen/fclose costs // more than the translation it annotates once compile-each style // loops re-translate thousands of times. { static FILE *mf = []() -> FILE * { const char *p = getenv("TINYMUX_DBT_MAP"); return p ? fopen(p, "a") : nullptr; }(); if (mf) { fprintf(mf, "block pc=0x%llx host=%p end=%p\n", static_cast(pc), static_cast(code), static_cast(dbt->code_buf + dbt->code_used)); } } // Backpatch any chained exits that were waiting for this block. dbt_backpatch_chains(dbt, pc, code); // Flush I-cache for newly generated and backpatched code. dbt_flush_code(dbt, static_cast(code - dbt->code_buf)); if (dbt->trace & DBT_TRACE_EXEC) { fprintf(stderr, "[dbt] disp=%llu MISS pc=0x%llX\n", static_cast(dispatch_count), static_cast(pc)); } } // Execute. trampoline(&dbt->ctx, dbt->memory, code, dbt->cache.data()); dbt->ctx.x[0] = 0; } } int dbt_resume(dbt_state_t *dbt, uint64_t entry_pc) { typedef void (*trampoline_fn_t)(rv64_ctx_t *ctx, uint8_t *mem, void *block, void *cache); trampoline_fn_t trampoline = reinterpret_cast(static_cast(dbt->code_buf)); dbt->ctx.next_pc = entry_pc; uint64_t dispatch_count = 0; for (;;) { dispatch_count++; // Refill the self-loop back-edge budget (#1571). A block with a // native back-edge decrements this per iteration and returns here at // zero, so the two guards below actually get a chance to run. dbt->ctx.loop_budget = DBT_LOOP_BUDGET; if (dbt->max_dispatch && dispatch_count > dbt->max_dispatch) { dbt->dispatch_count = dispatch_count; fprintf(stderr, "dbt: dispatch limit exceeded (%llu)\n", static_cast(dbt->max_dispatch)); return -2; } // Wall-clock abort: the per-command alarm has fired. Return so the // caller aborts the run. Polled only at the top of this dispatch // loop — after an ECALL return and before each trampoline entry. // Block chaining / pretranslated softlib can keep execution in // native code across many guest branches without returning here, so // pure tier-2 compute is best-effort between host ECALLs (same class // of mid-function gap a long C builtin has on the AST path). A // relaxed load is a plain read. if ( dbt->alarm_flag && dbt->alarm_flag->load(std::memory_order_relaxed)) { dbt->dispatch_count = dispatch_count; return -3; } uint64_t pc = dbt->ctx.next_pc; // ECALL signal: bit 0 set. if (pc & 1) { if (dbt->trace & DBT_TRACE_EXEC) { fprintf(stderr, "[dbt] disp=%llu ECALL pc=0x%llX\n", static_cast(dispatch_count), static_cast(pc & ~3ULL)); } dbt->ctx.next_pc = (pc & ~3ULL) + 4; int rc = dbt->ecall_fn(&dbt->ctx, dbt->ecall_user); if (rc >= 0) { dbt->dispatch_count = dispatch_count; return rc; } dbt->ctx.x[0] = 0; continue; } // EBREAK signal: bit 1 set. if (pc & 2) { dbt->dispatch_count = dispatch_count; fprintf(stderr, "dbt: EBREAK at 0x%llX\n", static_cast(pc & ~3ULL)); return -1; } // #1864: refuse a guest-controlled next_pc that cannot hold a 4-byte // instruction (including wrap cases near UINT64_MAX). Backends also // guard before each fetch; the dispatch path must not treat a wild // PC as a normal cache miss / translate entry. // if (!dbt_guest_range_ok(pc, 4, dbt->memory_size)) { dbt->dispatch_count = dispatch_count; fprintf(stderr, "dbt: fetch out of bounds at PC=0x%llX\n", static_cast(pc)); return -1; } // Look up or translate block. block_entry_t *be = dbt_cache_lookup(dbt, pc); uint8_t *code; if (be) { code = be->native_code; if (dbt->trace & DBT_TRACE_EXEC) { fprintf(stderr, "[dbt] disp=%llu HIT pc=0x%llX\n", static_cast(dispatch_count), static_cast(pc)); } } else { jit_write_begin(); dbt->xlate_fail = dbt_state_t::XLATE_OK; code = dbt_translate_block_at(dbt, pc); if ( !code && dbt->xlate_fail == dbt_state_t::XLATE_FULL && dbt_reclaim_program_code(dbt)) { // Buffer filled with program translations. Reclaim them // and retry once before declining (#1315 / #1331). jit_write_begin(); dbt->xlate_fail = dbt_state_t::XLATE_OK; code = dbt_translate_block_at(dbt, pc); } if (!code) { if (dbt->xlate_fail == dbt_state_t::XLATE_FULL) { dbt->code_full++; if (1 == dbt->code_full) { // Say so once. Declining is otherwise invisible: // the caller falls back to the interpreter and // still produces correct output. fprintf(stderr, "dbt: code buffer full at pc=0x%llX " "(used=%u blob=%u cap=%u); JIT declining\n", static_cast(pc), static_cast(dbt->code_used), static_cast(dbt->blob_code_end), static_cast(CODE_BUF_SIZE)); } } dbt_flush_code(dbt, dbt->code_used); dbt->dispatch_count = dispatch_count; return -1; // full (after reclaim) or refuse } dbt_cache_insert(dbt, pc, code); // Diagnostic map for profiling anonymous JIT frames (#2132): // TINYMUX_DBT_MAP= appends guest-pc -> host-address lines. // One shared handle, never closed: a per-block fopen/fclose costs // more than the translation it annotates once compile-each style // loops re-translate thousands of times. { static FILE *mf = []() -> FILE * { const char *p = getenv("TINYMUX_DBT_MAP"); return p ? fopen(p, "a") : nullptr; }(); if (mf) { fprintf(mf, "block pc=0x%llx host=%p end=%p\n", static_cast(pc), static_cast(code), static_cast(dbt->code_buf + dbt->code_used)); } } // Backpatch any chained exits that were waiting for this block. dbt_backpatch_chains(dbt, pc, code); // Flush I-cache for newly generated and backpatched code. dbt_flush_code(dbt, static_cast(code - dbt->code_buf)); if (dbt->trace & DBT_TRACE_EXEC) { fprintf(stderr, "[dbt] disp=%llu MISS pc=0x%llX\n", static_cast(dispatch_count), static_cast(pc)); } } // Execute. trampoline(&dbt->ctx, dbt->memory, code, dbt->cache.data()); dbt->ctx.x[0] = 0; } } // Diagnostic dump of the translated code for offline disassembly (#2132): // TINYMUX_DBT_CODEDUMP= writes code_buf[0..code_used] plus a sidecar // .base with the runtime base address, at cleanup. static void dbt_maybe_dump_code(dbt_state_t *dbt) { const char *dump_path = getenv("TINYMUX_DBT_CODEDUMP"); if (!dump_path || !dbt->code_buf) { return; } FILE *df = fopen(dump_path, "wb"); if (df) { fwrite(dbt->code_buf, 1, dbt->code_used, df); fclose(df); } const std::string side_base = std::string(dump_path) + ".base"; FILE *bf = fopen(side_base.c_str(), "w"); if (bf) { fprintf(bf, "%p %u\n", static_cast(dbt->code_buf), dbt->code_used); fclose(bf); } // Every cached block (covers pretranslated blob blocks the dbt_run map // never sees). const std::string side_cache = std::string(dump_path) + ".cache"; FILE *cf = fopen(side_cache.c_str(), "w"); if (cf) { for (size_t i = 0; i < BLOCK_CACHE_SIZE; i++) { if (dbt->cache[i].guest_pc) { fprintf(cf, "entry pc=0x%llx host=%p\n", static_cast(dbt->cache[i].guest_pc), static_cast(dbt->cache[i].native_code)); } } fclose(cf); } } void dbt_cleanup(dbt_state_t *dbt) { dbt_maybe_dump_code(dbt); if (dbt->code_buf) { jit_free(dbt->code_buf, CODE_BUF_SIZE); dbt->code_buf = nullptr; } dbt->cache.clear(); dbt->cache.shrink_to_fit(); dbt->patches.clear(); dbt->patches.shrink_to_fit(); dbt->pending_patch_targets.clear(); }