From f70bd5f0e139e31f49593df494ee11f23716095c Mon Sep 17 00:00:00 2001 From: angr-bot Date: Wed, 15 Jul 2026 10:17:19 +0000 Subject: [PATCH 001/122] Update version to 9.3.1.dev0 [ci skip] --- angr/__init__.py | 2 +- pyproject.toml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/angr/__init__.py b/angr/__init__.py index 231644083..159465753 100644 --- a/angr/__init__.py +++ b/angr/__init__.py @@ -1,7 +1,7 @@ # pylint: disable=wrong-import-position from __future__ import annotations -__version__ = "9.3.0.dev0" +__version__ = "9.3.1.dev0" if bytes is str: raise Exception(""" diff --git a/pyproject.toml b/pyproject.toml index 3eaf6af81..4d63fe874 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=77.0.0", "setuptools-rust", "pyvex==9.3.0.dev0"] +requires = ["setuptools>=77.0.0", "setuptools-rust", "pyvex==9.3.1.dev0"] build-backend = "setuptools.build_meta" [project] @@ -19,12 +19,12 @@ dependencies = [ "cxxheaderparser", "GitPython", "angr-data~=0.1.0", - "archinfo==9.3.0.dev0", + "archinfo==9.3.1.dev0", "cachetools", "capstone==5.0.6", "cffi>=1.14.0", - "claripy==9.3.0.dev0", - "cle==9.3.0.dev0", + "claripy==9.3.1.dev0", + "cle==9.3.1.dev0", "lmdb==2.1.1", "msgspec; implementation_name == 'cpython'", "mulpyplexer", @@ -35,7 +35,7 @@ dependencies = [ "platformdirs", "pydemumble==0.0.1", "pypcode~=4.0", - "pyvex==9.3.0.dev0", + "pyvex==9.3.1.dev0", "rich>=13.1.0", "rust-demangler==1.0", "sortedcontainers", From d9071a160e9f21f8e938f8fd53fda04b9182359f Mon Sep 17 00:00:00 2001 From: Fish Date: Thu, 16 Jul 2026 17:25:45 -0700 Subject: [PATCH 002/122] Decompiler: Fix "unsupported instructions" when rbp is used as GPR. (#6627) * VariableRecoveryFast: Do not seed bp with a stack address when bp is a GPR. * RegisterSaveAreaSimplifierAdvanced: Fix extern check and handle shrink-wrapped spills. --- .../register_save_area_simplifier_adv.py | 100 +++++++++++------- .../variable_recovery_fast.py | 6 +- tests/analyses/decompiler/test_decompiler.py | 26 +++++ 3 files changed, 93 insertions(+), 39 deletions(-) diff --git a/angr/analyses/decompiler/optimization_passes/register_save_area_simplifier_adv.py b/angr/analyses/decompiler/optimization_passes/register_save_area_simplifier_adv.py index 214b2040f..1d0110e1e 100644 --- a/angr/analyses/decompiler/optimization_passes/register_save_area_simplifier_adv.py +++ b/angr/analyses/decompiler/optimization_passes/register_save_area_simplifier_adv.py @@ -6,7 +6,7 @@ import logging from angr.ailment.expression import VirtualVariable from angr.ailment.statement import Assignment from angr.analyses.decompiler.stack_item import StackItem, StackItemType -from angr.code_location import CodeLocation, ExternalCodeLocation +from angr.code_location import CodeLocation from angr.utils.ail import is_phi_assignment from .optimization_pass import OptimizationPass, OptimizationPassStage @@ -60,17 +60,16 @@ class RegisterSaveAreaSimplifierAdvanced(OptimizationPass): if cache is None: return - info: list[tuple[int, CodeLocation, int, CodeLocation, int]] = cache["info"] + info: list[tuple[list[CodeLocation], int]] = cache["info"] updated_blocks = {} - for _regvar, regvar_loc, _stackvar, stackvar_loc, _ in info: - # remove storing statements - old_block = self._get_block(regvar_loc.block_addr, idx=regvar_loc.block_idx) - assert regvar_loc.stmt_idx is not None - self._modify_statement(old_block, regvar_loc.stmt_idx, updated_blocks) - old_block = self._get_block(stackvar_loc.block_addr, idx=stackvar_loc.block_idx) - assert stackvar_loc.stmt_idx is not None - self._modify_statement(old_block, stackvar_loc.stmt_idx, updated_blocks) + for locs, _ in info: + # remove all statements involved in this save (the store plus its matching restore, or the store plus the + # dead phi statements it feeds) + for loc in locs: + old_block = self._get_block(loc.block_addr, idx=loc.block_idx) + assert old_block is not None and loc.stmt_idx is not None + self._modify_statement(old_block, loc.stmt_idx, updated_blocks) for old_block, new_block in updated_blocks.items(): # remove all statements that are None @@ -80,20 +79,23 @@ class RegisterSaveAreaSimplifierAdvanced(OptimizationPass): if updated_blocks: # update stack_items - for _, _, _, _, stack_offset in info: + for _, stack_offset in info: self.stack_items[stack_offset] = StackItem( stack_offset, self.project.arch.bytes, "regs", StackItemType.SAVED_REGS ) - def _find_reg_store_and_restore_locations(self) -> list[tuple[int, CodeLocation, int, CodeLocation, int]]: - results = [] + def _find_reg_store_and_restore_locations(self) -> list[tuple[list[CodeLocation], int]]: + results: list[tuple[list[CodeLocation], int]] = [] assert self._srda is not None srda_model = self._srda.model # find all registers that are defined externally and used exactly once saved_vvars: set[tuple[int, CodeLocation]] = set() for vvar_id, loc in srda_model.all_vvar_definitions.items(): - if isinstance(loc, ExternalCodeLocation): + # SReachingDefinitions records externally-defined (function live-in) vvars via + # AILCodeLocation.make_extern(). These are AILCodeLocation instances (not ExternalCodeLocation), so the + # extern check must go through AILCodeLocation.is_extern. + if loc.is_extern: uses = srda_model.all_vvar_uses.get(vvar_id, []) if len(uses) == 1: vvar, used_loc = next(iter(uses)) @@ -105,8 +107,13 @@ class RegisterSaveAreaSimplifierAdvanced(OptimizationPass): # for each candidate, we check to ensure: # - it is stored onto the stack (into a stack virtual variable) - # - the stack virtual variable is only used once and restores the value to the same register - # - the restore location is in the dominance frontier of the store location + # - either + # (a) the stack virtual variable is used exactly once (ignoring phi uses) to restore the value to the same + # register, and the restore location is in the dominance frontier of the store location; or + # (b) the stack virtual variable has no non-phi uses and only feeds phi nodes whose results are dead. This + # happens with shrink-wrapped prologues (e.g. MSVC drivers) where the callee-saved register is + # conditionally spilled and the matching restore has already been removed as a dead assignment; what + # remains is a dead store whose stack vvar merges with an undefined value at a loop/branch join. for vvar_id, used_loc in saved_vvars: def_block = self._get_block(used_loc.block_addr, idx=used_loc.block_idx) assert def_block is not None and used_loc.stmt_idx is not None @@ -122,40 +129,59 @@ class RegisterSaveAreaSimplifierAdvanced(OptimizationPass): continue stack_vvar = stmt.dst all_stack_vvar_uses = srda_model.all_vvar_uses.get(stack_vvar.varid, []) - # eliminate the use location if it's a phi statement + # partition the uses into phi uses and non-phi uses stack_vvar_uses = set() + phi_use_locs: list[CodeLocation] = [] for vvar_, loc_ in all_stack_vvar_uses: use_block = self._get_block(loc_.block_addr, idx=loc_.block_idx) if use_block is None or loc_.stmt_idx is None: continue use_stmt = use_block.statements[loc_.stmt_idx] if is_phi_assignment(use_stmt): + phi_use_locs.append(loc_) continue stack_vvar_uses.add((vvar_, loc_)) - if len(stack_vvar_uses) != 1: - continue - _, stack_vvar_use_loc = next(iter(stack_vvar_uses)) - restore_block = self._get_block(stack_vvar_use_loc.block_addr, idx=stack_vvar_use_loc.block_idx) - assert restore_block is not None - restore_stmt = restore_block.statements[stack_vvar_use_loc.stmt_idx] - if not ( - isinstance(restore_stmt, Assignment) - and isinstance(restore_stmt.src, VirtualVariable) - and restore_stmt.src.varid == stack_vvar.varid - and isinstance(restore_stmt.dst, VirtualVariable) - and restore_stmt.dst.was_reg - and restore_stmt.dst.reg_offset == stmt.src.reg_offset - ): - continue - # this is the dumb version of the dominance frontier check - if self._within_dominance_frontier(def_block, restore_block, True, True): - results.append( - (stmt.src.varid, used_loc, stack_vvar.varid, stack_vvar_use_loc, stack_vvar.stack_offset) - ) + if len(stack_vvar_uses) == 1: + # case (a): a genuine store/restore pair + _, stack_vvar_use_loc = next(iter(stack_vvar_uses)) + restore_block = self._get_block(stack_vvar_use_loc.block_addr, idx=stack_vvar_use_loc.block_idx) + assert restore_block is not None + restore_stmt = restore_block.statements[stack_vvar_use_loc.stmt_idx] + + if not ( + isinstance(restore_stmt, Assignment) + and isinstance(restore_stmt.src, VirtualVariable) + and restore_stmt.src.varid == stack_vvar.varid + and isinstance(restore_stmt.dst, VirtualVariable) + and restore_stmt.dst.was_reg + and restore_stmt.dst.reg_offset == stmt.src.reg_offset + ): + continue + # this is the dumb version of the dominance frontier check + if self._within_dominance_frontier(def_block, restore_block, True, True): + results.append(([used_loc, stack_vvar_use_loc], stack_vvar.stack_offset)) + elif not stack_vvar_uses and phi_use_locs and self._phi_uses_are_dead(srda_model, phi_use_locs): + # case (b): a dead spill whose stack vvar only feeds dead phi nodes + results.append(([used_loc, *phi_use_locs], stack_vvar.stack_offset)) return results + def _phi_uses_are_dead(self, srda_model, phi_use_locs: list[CodeLocation]) -> bool: + """Return True iff every phi statement at ``phi_use_locs`` defines a virtual variable that has no uses. Such a + phi is dead and can be removed together with the store that feeds it.""" + + for loc in phi_use_locs: + block = self._get_block(loc.block_addr, idx=loc.block_idx) + if block is None or loc.stmt_idx is None: + return False + phi_stmt = block.statements[loc.stmt_idx] + if not (isinstance(phi_stmt, Assignment) and isinstance(phi_stmt.dst, VirtualVariable)): + return False + if srda_model.all_vvar_uses.get(phi_stmt.dst.varid, []): + return False + return True + def _within_dominance_frontier(self, dom_node, node, use_preds: bool, use_succs: bool) -> bool: if use_succs: # scan forward diff --git a/angr/analyses/variable_recovery/variable_recovery_fast.py b/angr/analyses/variable_recovery/variable_recovery_fast.py index a99cc7725..a30178553 100644 --- a/angr/analyses/variable_recovery/variable_recovery_fast.py +++ b/angr/analyses/variable_recovery/variable_recovery_fast.py @@ -403,8 +403,10 @@ class VariableRecoveryFast(ForwardAnalysis, VariableRecoveryBase): # pylint:dis initial_sp = state.stack_address(self.project.arch.bytes if self.project.arch.call_pushes_ret else 0) if self.project.arch.sp_offset is not None: state.register_region.store(self.project.arch.sp_offset, initial_sp) - # give it enough stack space - if self.project.arch.bp_offset is not None: + # give it enough stack space, but only when bp is used as a frame pointer; when bp is a general-purpose + # register, seeding it with a stack address would misclassify bp-based memory accesses as stack accesses + # and suppress the creation of a register variable for the initial value of bp. + if self.project.arch.bp_offset is not None and not self.function.info.get("bp_as_gpr", False): state.register_region.store(self.project.arch.bp_offset, initial_sp + 0x100000) internal_manager = self.variable_manager[self.function.addr] diff --git a/tests/analyses/decompiler/test_decompiler.py b/tests/analyses/decompiler/test_decompiler.py index 6b7e9fc4b..6f3d66727 100755 --- a/tests/analyses/decompiler/test_decompiler.py +++ b/tests/analyses/decompiler/test_decompiler.py @@ -3866,6 +3866,32 @@ class TestDecompiler(unittest.TestCase): f"the decompilation output contains struct typedef(s) that nothing references: {unreferenced}" ) + @structuring_algo("sailr") + def test_decompiling_03fb29da_132b0_bp_as_gpr_reg_spill(self, decompiler_options=None): + # Regression test covering two bugs in sub_132b0, which uses bp as a general-purpose register, + # conditionally spills the callee-saved registers rbp/rsi/rdi/r12 into its local frame at 0x13365 + # ("shrink-wrapped" prologue), and restores them at 0x134e5. + bin_path = os.path.join( + test_location, "x86_64", "windows", "03fb29dab8ab848f15852a37a1c04aa65289c0160d9200dceff64d890b3290dd" + ) + proj, _ = load_project_with_scoped_cfg(bin_path, 0x132B0) + f = proj.kb.functions[0x132B0] + assert f.info.get("bp_as_gpr", None) is True + d = proj.analyses[Decompiler].prep(fail_fast=True)(f, options=decompiler_options) + assert d.codegen is not None and d.codegen.text is not None + print_decompilation_result(d) + + assert "unsupported instruction" not in d.codegen.text + + # The shrink-wrapped callee-saved spill previously showed up as a run of four consecutive dead + # "vX = vY;" register-copy statements. After the fix no such spill block survives. + copy_re = re.compile(r"^\s*v\d+ = v\d+;\s*$") + max_run = run = 0 + for line in d.codegen.text.splitlines(): + run = run + 1 if copy_re.match(line) else 0 + max_run = max(max_run, run) + assert max_run < 3, f"a callee-saved register spill block survived: {max_run} consecutive copy statements" + def test_test_binop_ret_dup(self, decompiler_options=None): bin_path = os.path.join(test_location, "x86_64", "decompiler", "test.o") proj = angr.Project(bin_path, auto_load_libs=False) From 07115bb2ad9b5ccc8a19944155040b87054f66ef Mon Sep 17 00:00:00 2001 From: Vedant Soni <83280635+tedanvosin@users.noreply.github.com> Date: Fri, 17 Jul 2026 02:06:11 -0700 Subject: [PATCH 003/122] RustStructuredCodeGenerator: Make indent_delta configurable (#6626) * add indent_size kwarg to rustcodegen * make indent_delta variable --- .../decompiler/structured_codegen/rust.py | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/angr/analyses/decompiler/structured_codegen/rust.py b/angr/analyses/decompiler/structured_codegen/rust.py index a79019c83..1a06ca187 100644 --- a/angr/analyses/decompiler/structured_codegen/rust.py +++ b/angr/analyses/decompiler/structured_codegen/rust.py @@ -611,8 +611,8 @@ class RustFunction(RustConstruct): # pylint:disable=abstract-method yield " ", None yield "{", brace yield "\n", None - yield from self.variable_list_repr_chunks(indent=indent + INDENT_DELTA) - yield from self.statements.c_repr_chunks(indent=indent + INDENT_DELTA) + yield from self.variable_list_repr_chunks(indent=indent + self.codegen.indent_delta) + yield from self.statements.c_repr_chunks(indent=indent + self.codegen.indent_delta) yield indent_str, None yield "}", brace yield "\n", None @@ -756,7 +756,7 @@ class RustInfiniteLoop(RustLoop): else: yield "{", brace yield "\n", None - yield from self.body.c_repr_chunks(indent=indent + INDENT_DELTA) + yield from self.body.c_repr_chunks(indent=indent + self.codegen.indent_delta) yield indent_str, None yield "}", brace yield "\n", None @@ -804,7 +804,7 @@ class RustWhileLoop(RustLoop): else: yield "{", brace yield "\n", None - yield from self.body.c_repr_chunks(indent=indent + INDENT_DELTA) + yield from self.body.c_repr_chunks(indent=indent + self.codegen.indent_delta) yield indent_str, None yield "}", brace yield "\n", None @@ -843,7 +843,7 @@ class RustDoWhileLoop(RustLoop): if self.body is not None: yield "{", brace yield "\n", None - yield from self.body.c_repr_chunks(indent=indent + INDENT_DELTA) + yield from self.body.c_repr_chunks(indent=indent + self.codegen.indent_delta) yield indent_str, None yield "}", brace else: @@ -905,7 +905,7 @@ class RustForLoop(RustStatement): yield "{", brace yield "\n", None - yield from self.body.c_repr_chunks(indent=indent + INDENT_DELTA) + yield from self.body.c_repr_chunks(indent=indent + self.codegen.indent_delta) yield indent_str, None yield "}", brace else: @@ -976,7 +976,7 @@ class RustIfElse(RustStatement): yield "\n", None if node is not None: - yield from node.c_repr_chunks(indent=INDENT_DELTA + indent) + yield from node.c_repr_chunks(indent=self.codegen.indent_delta + indent) yield indent_str, None yield "}", brace @@ -1002,7 +1002,7 @@ class RustIfElse(RustStatement): yield "{", brace yield "\n", None - yield from self.else_node.c_repr_chunks(indent=indent + INDENT_DELTA) + yield from self.else_node.c_repr_chunks(indent=indent + self.codegen.indent_delta) yield indent_str, None yield "}", brace @@ -1044,12 +1044,12 @@ class RustIfBreak(RustStatement): else: yield " ", None if self.cstyle_ifs: - yield self.indent_str(indent=INDENT_DELTA), self + yield self.indent_str(indent=self.codegen.indent_delta), self yield "break;\n", self else: yield "{", brace yield "\n", None - yield self.indent_str(indent=indent + INDENT_DELTA), self + yield self.indent_str(indent=indent + self.codegen.indent_delta), self yield "break;\n", self yield indent_str, None yield "}", brace @@ -1127,7 +1127,7 @@ class RustSwitchCase(RustStatement): yield "\n", None # cases - indent_str = self.indent_str(indent=indent + INDENT_DELTA) + indent_str = self.indent_str(indent=indent + self.codegen.indent_delta) for id_or_ids, case in self.cases: yield indent_str, None if isinstance(id_or_ids, int): @@ -1155,14 +1155,14 @@ class RustSwitchCase(RustStatement): if i != len(ranges) - 1: yield "| ", None yield "=> {\n", None - yield from case.c_repr_chunks(indent=indent + INDENT_DELTA * 2) + yield from case.c_repr_chunks(indent=indent + self.codegen.indent_delta * 2) yield indent_str, None yield "}\n", None if self.default is not None: yield indent_str, None yield "_ => {\n", self - yield from self.default.c_repr_chunks(indent=indent + INDENT_DELTA * 2) + yield from self.default.c_repr_chunks(indent=indent + self.codegen.indent_delta * 2) yield indent_str, None yield "}\n", None @@ -1199,7 +1199,7 @@ class RustPatternMatch(RustStatement): yield "{", brace yield "\n", None - arm_indent_str = self.indent_str(indent=indent + INDENT_DELTA) + arm_indent_str = self.indent_str(indent=indent + self.codegen.indent_delta) # arms for (variant, bound_vars), arm in self.arms: @@ -1216,14 +1216,14 @@ class RustPatternMatch(RustStatement): yield "_", None yield ")", paren yield " => {\n", self - yield from arm.c_repr_chunks(indent=indent + 2 * INDENT_DELTA) + yield from arm.c_repr_chunks(indent=indent + 2 * self.codegen.indent_delta) yield arm_indent_str, None yield "},\n", self if self.default is not None: yield arm_indent_str, None yield "_ => {\n", self - yield from self.default.c_repr_chunks(indent=indent + 2 * INDENT_DELTA) + yield from self.default.c_repr_chunks(indent=indent + 2 * self.codegen.indent_delta) yield arm_indent_str, None yield "}\n", self @@ -1274,7 +1274,7 @@ class RustIfLet(RustStatement): yield " ", None yield "{", brace yield "\n", None - yield from self.true_node.c_repr_chunks(indent=indent + INDENT_DELTA) + yield from self.true_node.c_repr_chunks(indent=indent + self.codegen.indent_delta) yield indent_str, None yield "}", brace @@ -1282,7 +1282,7 @@ class RustIfLet(RustStatement): yield " else ", self yield "{", brace yield "\n", None - yield from self.false_node.c_repr_chunks(indent=indent + INDENT_DELTA) + yield from self.false_node.c_repr_chunks(indent=indent + self.codegen.indent_delta) yield indent_str, None yield "}", brace @@ -1657,7 +1657,7 @@ class RustStruct(RustExpression): yield "...", self return indent_str = self.indent_str(indent=indent) - field_indent_str = self.indent_str(indent=indent + INDENT_DELTA) + field_indent_str = self.indent_str(indent=indent + self.codegen.indent_delta) yield str(self.name), self if not self.field_names: yield " {", brace @@ -1669,7 +1669,9 @@ class RustStruct(RustExpression): yield name, self yield ": ", self if offset in self.fields: - yield from RustExpression._try_c_repr_chunks(self.fields[offset], indent + INDENT_DELTA) + yield from RustExpression._try_c_repr_chunks( + self.fields[offset], indent + self.codegen.indent_delta + ) else: yield "", None yield "\n", None @@ -2755,6 +2757,7 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): notes=None, display_notes: bool = True, variable_map: VariableMap | None = None, + indent_size: int = INDENT_DELTA, ): super().__init__(flavor=flavor, notes=notes) @@ -2843,6 +2846,7 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): self.rust_func: RustFunction | None = None self.cexterns: set[RustVariable] | None = None self.display_notes = display_notes + self.indent_delta = indent_size self._analyze() @@ -2868,6 +2872,8 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): self.simplify_else_scope = value elif option.param == "cstyle_ifs": self.cstyle_ifs = value + elif option.param == "indent_size": + self.indent_delta = value def _translate_prototype_to_rust(self, prototype: SimTypeFunction): translator = RustTypeTranslator(self.project.arch) From 352d642363d9cc6bdfe4ea0ae33c7d0b36ea8c29 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Fri, 17 Jul 2026 10:41:08 -0700 Subject: [PATCH 004/122] identifier: Deduplicate 22 copies of rand_str into shared helpers (#6628) --- angr/analyses/identifier/func.py | 20 +++++++++++++++++++ angr/analyses/identifier/functions/atoi.py | 9 ++------- .../identifier/functions/based_atoi.py | 5 ----- .../analyses/identifier/functions/fdprintf.py | 10 ++-------- angr/analyses/identifier/functions/int2str.py | 20 ------------------- angr/analyses/identifier/functions/memcmp.py | 8 +------- angr/analyses/identifier/functions/memcpy.py | 8 +------- angr/analyses/identifier/functions/memset.py | 8 +------- angr/analyses/identifier/functions/printf.py | 10 ++-------- .../identifier/functions/recv_until.py | 8 +------- .../identifier/functions/skip_recv_n.py | 8 +------- .../analyses/identifier/functions/snprintf.py | 12 +++-------- angr/analyses/identifier/functions/sprintf.py | 12 +++-------- .../identifier/functions/strcasecmp.py | 8 -------- angr/analyses/identifier/functions/strcmp.py | 10 +--------- angr/analyses/identifier/functions/strcpy.py | 12 +++-------- angr/analyses/identifier/functions/strlen.py | 9 ++------- angr/analyses/identifier/functions/strncmp.py | 10 +--------- angr/analyses/identifier/functions/strncpy.py | 12 +++-------- angr/analyses/identifier/functions/strtol.py | 9 ++------- 20 files changed, 49 insertions(+), 159 deletions(-) diff --git a/angr/analyses/identifier/func.py b/angr/analyses/identifier/func.py index e9d3eb095..a88accfe8 100644 --- a/angr/analyses/identifier/func.py +++ b/angr/analyses/identifier/func.py @@ -1,5 +1,25 @@ from __future__ import annotations +import random + + +def rand_str(length, byte_list=None) -> str: + """ + Generate a random string of `length` characters, drawn from `byte_list` (or all 256 characters if not provided). + """ + if byte_list is None: + return "".join(chr(random.randint(0, 255)) for _ in range(length)) + return "".join(random.choice(byte_list) for _ in range(length)) + + +def rand_bytes(length, byte_list=None) -> bytes: + """ + Generate `length` random bytes, drawn from `byte_list` (or all 256 byte values if not provided). + """ + if byte_list is None: + return bytes(random.randint(0, 255) for _ in range(length)) + return bytes(random.choice(byte_list) for _ in range(length)) + class TestData: def __init__( diff --git a/angr/analyses/identifier/functions/atoi.py b/angr/analyses/identifier/functions/atoi.py index 79fb71494..7bdcaa480 100644 --- a/angr/analyses/identifier/functions/atoi.py +++ b/angr/analyses/identifier/functions/atoi.py @@ -3,7 +3,7 @@ from __future__ import annotations import random import string -from angr.analyses.identifier.func import Func, TestData +from angr.analyses.identifier.func import Func, TestData, rand_str class atoi(Func): @@ -12,11 +12,6 @@ class atoi(Func): self.skips_whitespace = False self.allows_negative = True - def rand_str(self, length, byte_list=None): # pylint disable=no-self-use - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) - def num_args(self): return 1 @@ -53,7 +48,7 @@ class atoi(Func): return False s = str(num) - s = self.rand_str(10, string.whitespace) + s + s = rand_str(10, string.whitespace) + s test_input = [s] test_output = [s] return_val = num diff --git a/angr/analyses/identifier/functions/based_atoi.py b/angr/analyses/identifier/functions/based_atoi.py index b407d218a..195f8b546 100644 --- a/angr/analyses/identifier/functions/based_atoi.py +++ b/angr/analyses/identifier/functions/based_atoi.py @@ -51,11 +51,6 @@ class based_atoi(Func): self.allows_negative = True self.base = None - def rand_str(self, length, byte_list=None): # pylint disable=no-self-use - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) - def num_args(self): return OneTwoOrThree() diff --git a/angr/analyses/identifier/functions/fdprintf.py b/angr/analyses/identifier/functions/fdprintf.py index 80db51a86..8f35ccafa 100644 --- a/angr/analyses/identifier/functions/fdprintf.py +++ b/angr/analyses/identifier/functions/fdprintf.py @@ -1,12 +1,11 @@ from __future__ import annotations import logging -import random import string import claripy -from angr.analyses.identifier.func import Func, TestData +from angr.analyses.identifier.func import Func, TestData, rand_str l = logging.getLogger(name=__name__) @@ -20,11 +19,6 @@ class fdprintf(Func): self.string_spec_char = None self.allows_n = False - def rand_str(self, length, byte_list=None): # pylint disable=no-self-use - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) - def num_args(self): return 2 @@ -44,7 +38,7 @@ class fdprintf(Func): def pre_test(self, func, runner): # make sure it prints alphanumeric stuff length = 10 - test_str = self.rand_str(length, string.ascii_letters + string.digits) + test_str = rand_str(length, string.ascii_letters + string.digits) test_input = [1, test_str] test_output = [None, test_str] max_steps = len(test_str) * 3 + 20 diff --git a/angr/analyses/identifier/functions/int2str.py b/angr/analyses/identifier/functions/int2str.py index 73e6bac0b..6f7cbb5d6 100644 --- a/angr/analyses/identifier/functions/int2str.py +++ b/angr/analyses/identifier/functions/int2str.py @@ -49,11 +49,6 @@ class int2str(Func): super().__init__() self.is_signed = False - def rand_str(self, length, byte_list=None): # pylint disable=no-self-use - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) - def num_args(self): return 3 @@ -109,11 +104,6 @@ class int2str_v2(Func): super().__init__() self.is_signed = False - def rand_str(self, length, byte_list=None): # pylint disable=no-self-use - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) - def num_args(self): return TwoOrThree() @@ -169,11 +159,6 @@ class int2str_v3(Func): super().__init__() self.is_signed = False - def rand_str(self, length, byte_list=None): # pylint disable=no-self-use - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) - def num_args(self): return TwoOrThree() @@ -229,11 +214,6 @@ class int2str_v4(Func): super().__init__() self.is_signed = False - def rand_str(self, length, byte_list=None): # pylint disable=no-self-use - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) - def num_args(self): return ThreeOrFour() diff --git a/angr/analyses/identifier/functions/memcmp.py b/angr/analyses/identifier/functions/memcmp.py index 2ddfd18e9..50025754f 100644 --- a/angr/analyses/identifier/functions/memcmp.py +++ b/angr/analyses/identifier/functions/memcmp.py @@ -2,13 +2,7 @@ from __future__ import annotations import random -from angr.analyses.identifier.func import Func, TestData - - -def rand_str(length, byte_list=None): - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) +from angr.analyses.identifier.func import Func, TestData, rand_str class memcmp(Func): diff --git a/angr/analyses/identifier/functions/memcpy.py b/angr/analyses/identifier/functions/memcpy.py index 0a98c9089..bc5d2702d 100644 --- a/angr/analyses/identifier/functions/memcpy.py +++ b/angr/analyses/identifier/functions/memcpy.py @@ -3,16 +3,10 @@ from __future__ import annotations import random from angr.analyses.identifier.custom_callable import IdentifierCallable -from angr.analyses.identifier.func import Func, TestData +from angr.analyses.identifier.func import Func, TestData, rand_str from angr.sim_type import SimTypeFunction, SimTypeInt -def rand_str(length, byte_list=None): - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) - - class memcpy(Func): non_null = [chr(i) for i in range(1, 256)] diff --git a/angr/analyses/identifier/functions/memset.py b/angr/analyses/identifier/functions/memset.py index 4d474b96c..ee8c99726 100644 --- a/angr/analyses/identifier/functions/memset.py +++ b/angr/analyses/identifier/functions/memset.py @@ -2,13 +2,7 @@ from __future__ import annotations import random -from angr.analyses.identifier.func import Func, TestData - - -def rand_str(length, byte_list=None): - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) +from angr.analyses.identifier.func import Func, TestData, rand_str class memset(Func): diff --git a/angr/analyses/identifier/functions/printf.py b/angr/analyses/identifier/functions/printf.py index 38d6a548f..f71f79991 100644 --- a/angr/analyses/identifier/functions/printf.py +++ b/angr/analyses/identifier/functions/printf.py @@ -1,12 +1,11 @@ from __future__ import annotations import logging -import random import string import claripy -from angr.analyses.identifier.func import Func, TestData +from angr.analyses.identifier.func import Func, TestData, rand_str l = logging.getLogger(name=__name__) @@ -20,11 +19,6 @@ class printf(Func): self.string_spec_char = None self.allows_n = False - def rand_str(self, length, byte_list=None): # pylint disable=no-self-use - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) - def num_args(self): return 1 @@ -44,7 +38,7 @@ class printf(Func): def pre_test(self, func, runner): # make sure it prints alphanumeric stuff length = 10 - test_str = self.rand_str(length, string.ascii_letters + string.digits) + test_str = rand_str(length, string.ascii_letters + string.digits) test_input = [test_str] test_output = [test_str] max_steps = len(test_str) * 3 + 20 diff --git a/angr/analyses/identifier/functions/recv_until.py b/angr/analyses/identifier/functions/recv_until.py index f56c7d5fb..47e49b401 100644 --- a/angr/analyses/identifier/functions/recv_until.py +++ b/angr/analyses/identifier/functions/recv_until.py @@ -4,13 +4,7 @@ import itertools import random from angr.analyses.identifier.errors import FunctionNotInitialized -from angr.analyses.identifier.func import Func, TestData - - -def rand_str(length, byte_list=None): - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) +from angr.analyses.identifier.func import Func, TestData, rand_str # FIXME this can fail test 2 diff --git a/angr/analyses/identifier/functions/skip_recv_n.py b/angr/analyses/identifier/functions/skip_recv_n.py index 949310087..6bfb404bc 100644 --- a/angr/analyses/identifier/functions/skip_recv_n.py +++ b/angr/analyses/identifier/functions/skip_recv_n.py @@ -3,13 +3,7 @@ from __future__ import annotations import random import struct -from angr.analyses.identifier.func import Func, TestData - - -def rand_str(length, byte_list=None): - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) +from angr.analyses.identifier.func import Func, TestData, rand_str class receive_n4(Func): diff --git a/angr/analyses/identifier/functions/snprintf.py b/angr/analyses/identifier/functions/snprintf.py index b2aa2c119..3c0d5165a 100644 --- a/angr/analyses/identifier/functions/snprintf.py +++ b/angr/analyses/identifier/functions/snprintf.py @@ -1,11 +1,10 @@ from __future__ import annotations -import random import string import claripy -from angr.analyses.identifier.func import Func, TestData +from angr.analyses.identifier.func import Func, TestData, rand_str class snprintf(Func): @@ -17,11 +16,6 @@ class snprintf(Func): self.string_spec_char = None self.allows_n = False - def rand_str(self, length, byte_list=None): # pylint disable=no-self-use - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) - def num_args(self): return 3 @@ -42,8 +36,8 @@ class snprintf(Func): def pre_test(self, func, runner): # make sure it prints alphanumeric stuff length = 10 - test_str = self.rand_str(length, string.ascii_letters + string.digits) - outbuf = self.rand_str(length + 2) + test_str = rand_str(length, string.ascii_letters + string.digits) + outbuf = rand_str(length + 2) test_input = [outbuf, length, test_str] test_output = [test_str[: length - 1] + "\x00" + outbuf[length:], None, test_str] max_steps = 20 diff --git a/angr/analyses/identifier/functions/sprintf.py b/angr/analyses/identifier/functions/sprintf.py index 7d53d3630..86fddf663 100644 --- a/angr/analyses/identifier/functions/sprintf.py +++ b/angr/analyses/identifier/functions/sprintf.py @@ -1,11 +1,10 @@ from __future__ import annotations -import random import string import claripy -from angr.analyses.identifier.func import Func, TestData +from angr.analyses.identifier.func import Func, TestData, rand_str class sprintf(Func): @@ -17,11 +16,6 @@ class sprintf(Func): self.string_spec_char = None self.allows_n = False - def rand_str(self, length, byte_list=None): # pylint disable=no-self-use - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) - def num_args(self): return 2 @@ -42,8 +36,8 @@ class sprintf(Func): def pre_test(self, func, runner): # make sure it prints alphanumeric stuff length = 10 - test_str = self.rand_str(length, string.ascii_letters + string.digits) - outbuf = self.rand_str(length + 2) + test_str = rand_str(length, string.ascii_letters + string.digits) + outbuf = rand_str(length + 2) test_input = [outbuf, test_str] test_output = [test_str + "\x00", test_str] max_steps = 20 diff --git a/angr/analyses/identifier/functions/strcasecmp.py b/angr/analyses/identifier/functions/strcasecmp.py index 8bfbd0745..b337054c5 100644 --- a/angr/analyses/identifier/functions/strcasecmp.py +++ b/angr/analyses/identifier/functions/strcasecmp.py @@ -1,16 +1,8 @@ from __future__ import annotations -import random - from .strcmp import strcmp -def rand_str(length, byte_list=None): - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) - - class strcasecmp(strcmp): non_null = [chr(i) for i in range(1, 256)] diff --git a/angr/analyses/identifier/functions/strcmp.py b/angr/analyses/identifier/functions/strcmp.py index 985eabba6..e015354de 100644 --- a/angr/analyses/identifier/functions/strcmp.py +++ b/angr/analyses/identifier/functions/strcmp.py @@ -1,14 +1,6 @@ from __future__ import annotations -import random - -from angr.analyses.identifier.func import Func, TestData - - -def rand_str(length, byte_list=None): - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) +from angr.analyses.identifier.func import Func, TestData, rand_str class strcmp(Func): diff --git a/angr/analyses/identifier/functions/strcpy.py b/angr/analyses/identifier/functions/strcpy.py index 7d1ca8c32..25b4ad41b 100644 --- a/angr/analyses/identifier/functions/strcpy.py +++ b/angr/analyses/identifier/functions/strcpy.py @@ -2,13 +2,7 @@ from __future__ import annotations import random -from angr.analyses.identifier.func import Func, TestData - - -def rand_str(length, byte_list=None): - if byte_list is None: - return bytes(random.randint(0, 255) for _ in range(length)) - return bytes(random.choice(byte_list) for _ in range(length)) +from angr.analyses.identifier.func import Func, TestData, rand_bytes class strcpy(Func): @@ -32,8 +26,8 @@ class strcpy(Func): def gen_input_output_pair(self): # TODO we don't check the return val, some cases I saw char * strcpy, some size_t strcpy strlen = random.randint(1, 80) - buf = rand_str(strlen, byte_list=strcpy.non_null) + b"\x00" - result_buf = rand_str(strlen + 1) + buf = rand_bytes(strlen, byte_list=strcpy.non_null) + b"\x00" + result_buf = rand_bytes(strlen + 1) test_input = [result_buf, buf] test_output = [buf, buf] max_steps = 20 diff --git a/angr/analyses/identifier/functions/strlen.py b/angr/analyses/identifier/functions/strlen.py index ec19107ef..1bcc7c9fd 100644 --- a/angr/analyses/identifier/functions/strlen.py +++ b/angr/analyses/identifier/functions/strlen.py @@ -2,17 +2,12 @@ from __future__ import annotations import random -from angr.analyses.identifier.func import Func, TestData +from angr.analyses.identifier.func import Func, TestData, rand_bytes class strlen(Func): non_null = list(range(1, 256)) - def rand_str(self, length, byte_list=None): # pylint disable=no-self-use - if byte_list is None: - return bytes(random.randint(0, 255) for _ in range(length)) - return bytes(random.choice(byte_list) for _ in range(length)) - def num_args(self): return 1 @@ -21,7 +16,7 @@ class strlen(Func): def gen_input_output_pair(self): length = random.randint(2, 100) - s = self.rand_str(length, strlen.non_null) + b"\x00" + self.rand_str(length) + s = rand_bytes(length, strlen.non_null) + b"\x00" + rand_bytes(length) test_input = [s] test_output = [s] max_steps = 20 diff --git a/angr/analyses/identifier/functions/strncmp.py b/angr/analyses/identifier/functions/strncmp.py index 3c7cd324a..453329af4 100644 --- a/angr/analyses/identifier/functions/strncmp.py +++ b/angr/analyses/identifier/functions/strncmp.py @@ -1,14 +1,6 @@ from __future__ import annotations -import random - -from angr.analyses.identifier.func import Func, TestData - - -def rand_str(length, byte_list=None): - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) +from angr.analyses.identifier.func import Func, TestData, rand_str class strncmp(Func): diff --git a/angr/analyses/identifier/functions/strncpy.py b/angr/analyses/identifier/functions/strncpy.py index 95b7d8b9a..a970e80e1 100644 --- a/angr/analyses/identifier/functions/strncpy.py +++ b/angr/analyses/identifier/functions/strncpy.py @@ -2,13 +2,7 @@ from __future__ import annotations import random -from angr.analyses.identifier.func import Func, TestData - - -def rand_str(length, byte_list=None): - if byte_list is None: - return bytes(random.randint(0, 255) for _ in range(length)) - return bytes(random.choice(byte_list) for _ in range(length)) +from angr.analyses.identifier.func import Func, TestData, rand_bytes class strncpy(Func): @@ -36,8 +30,8 @@ class strncpy(Func): # TODO we don't check the return val, some cases I saw char * strcpy, some size_t strcpy strlen = random.randint(1, 20) max_len = random.randint(1, 10) - buf = rand_str(strlen, byte_list=strncpy.non_null) + b"\x00" - result_buf = rand_str(strlen + 1) + buf = rand_bytes(strlen, byte_list=strncpy.non_null) + b"\x00" + result_buf = rand_bytes(strlen + 1) test_input = [result_buf, buf, max_len] outlen = min(max_len, strlen + 1) test_output = [buf[:outlen], buf, None] diff --git a/angr/analyses/identifier/functions/strtol.py b/angr/analyses/identifier/functions/strtol.py index 28aa7e9cb..10f8bced3 100644 --- a/angr/analyses/identifier/functions/strtol.py +++ b/angr/analyses/identifier/functions/strtol.py @@ -3,7 +3,7 @@ from __future__ import annotations import random import string -from angr.analyses.identifier.func import Func, TestData +from angr.analyses.identifier.func import Func, TestData, rand_str digs = string.digits + string.ascii_letters @@ -32,11 +32,6 @@ class strtol(Func): self.skips_whitespace = False self.version = "" - def rand_str(self, length, byte_list=None): # pylint disable=no-self-use - if byte_list is None: - return "".join(chr(random.randint(0, 255)) for _ in range(length)) - return "".join(random.choice(byte_list) for _ in range(length)) - def num_args(self): # pylint disable=no-self-use return 3 @@ -70,7 +65,7 @@ class strtol(Func): return False s = str(num) - s = self.rand_str(10, string.whitespace) + s + s = rand_str(10, string.whitespace) + s test_input = [s, 0, 10] test_output = [s, None, None] return_val = num From c213b234b32c4c55276287e774ae35ca616cf8b6 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Fri, 17 Jul 2026 11:09:09 -0700 Subject: [PATCH 005/122] Pin a recent pydantic-ai (#6630) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4d63fe874..35df4db15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ Repository = "https://github.com/angr/angr" angrdb = ["sqlalchemy"] keystone = ["keystone-engine"] telemetry = ["opentelemetry-api"] -llm = ["pydantic-ai", "mcp>=1.0.0", "fastmcp>=2.3.0"] +llm = ["pydantic-ai>=2.12.0", "mcp>=1.0.0", "fastmcp>=2.3.0"] unicorn = ["unicorn==2.1.4"] [project.scripts] From a6f88cd76f101b5256b98f66fbda016ff3037542 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Fri, 17 Jul 2026 11:15:50 -0700 Subject: [PATCH 006/122] Move STRONGREF_STATE logic entirely within history plugin (#6629) --- angr/sim_state.py | 4 ---- angr/state_plugins/history.py | 11 +++++------ angr/state_plugins/plugin.py | 5 ----- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/angr/sim_state.py b/angr/sim_state.py index f9cc54ab2..4222139d6 100644 --- a/angr/sim_state.py +++ b/angr/sim_state.py @@ -288,8 +288,6 @@ class SimState[IPTypeConc, IPTypeSym](PluginHub[SimStatePlugin]): self.__dict__.update(s) for p in self.plugins.values(): p.set_state(self) - if p.STRONGREF_STATE: - p.set_strongref_state(self) def _get_weakref(self): return weakref.proxy(self) @@ -459,8 +457,6 @@ class SimState[IPTypeConc, IPTypeSym](PluginHub[SimStatePlugin]): def _set_plugin_state(self, plugin: SimStatePlugin, inhibit_init: bool = False): plugin.set_state(self) - if plugin.STRONGREF_STATE: - plugin.set_strongref_state(self) if not inhibit_init: plugin.init_state() diff --git a/angr/state_plugins/history.py b/angr/state_plugins/history.py index 8207ad9a8..6e11b422b 100644 --- a/angr/state_plugins/history.py +++ b/angr/state_plugins/history.py @@ -24,8 +24,6 @@ class SimStateHistory(SimStatePlugin): This class keeps track of historically-relevant information for paths. """ - STRONGREF_STATE = True - def __init__(self, parent=None, clone=None): SimStatePlugin.__init__(self) @@ -72,6 +70,11 @@ class SimStateHistory(SimStatePlugin): self.strongref_state = None if clone is None else clone.strongref_state self.arch = None + def set_state(self, state): + super().set_state(state) + if sim_options.EFFICIENT_STATE_MERGING in state.options: + self.strongref_state = state + def init_state(self): self.successor_ip = self.state._ip self.arch = self.state.arch @@ -128,10 +131,6 @@ class SimStateHistory(SimStatePlugin): return f"" - def set_strongref_state(self, state): - if sim_options.EFFICIENT_STATE_MERGING in state.options: - self.strongref_state = state - @property def addr(self): if not self.recent_bbl_addrs: diff --git a/angr/state_plugins/plugin.py b/angr/state_plugins/plugin.py index 8904bdd36..d276c4862 100644 --- a/angr/state_plugins/plugin.py +++ b/angr/state_plugins/plugin.py @@ -30,8 +30,6 @@ class SimStatePlugin: storage and persistence for SimProcedures. """ - STRONGREF_STATE = False - def __init__(self) -> None: self.state: SimState[Any, Any] = cast("SimState[Any, Any]", None) @@ -41,9 +39,6 @@ class SimStatePlugin: """ self.state = state._get_weakref() - def set_strongref_state(self, state) -> None: - pass - def __getstate__(self) -> dict[str, Any]: d = dict(self.__dict__) d["state"] = None From bd58a30e6e1194ce4106a12d41e2581d663efecd Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Fri, 17 Jul 2026 11:50:28 -0700 Subject: [PATCH 007/122] Update installation CI run (#6631) --- .github/workflows/ci.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70934deab..ba0bbe804 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,20 +16,26 @@ jobs: ci: uses: angr/ci-settings/.github/workflows/angr-ci.yml@master - smoketest: - name: Test installation + installation: + name: Test installation (${{ matrix.environment.os }}, py${{ matrix.environment.python-version }}) strategy: matrix: - os: [windows-2022, macos-15-intel] + environment: + - os: windows-2025 + python-version: 3.12 + - os: macos-26 + python-version: 3.12 + - os: ubuntu-24.04 + python-version: 3.14 fail-fast: false - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.environment.os }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 if: startsWith(runner.os, 'windows') - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v6 - name: Sync dependencies - run: uv sync -p 3.12 + run: uv sync -p ${{ matrix.environment.python-version }} - name: Collect tests run: uv run pytest --collect-only tests From 1944e72e90622517a0401018b0bb35b5809cd5ba Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Fri, 17 Jul 2026 13:23:53 -0700 Subject: [PATCH 008/122] Remove widen() from state api (#6632) --- .../forward_analysis/forward_analysis.py | 3 --- angr/analyses/stack_pointer_tracker.py | 8 ------ angr/analyses/vfg.py | 25 ++---------------- angr/sim_state.py | 26 ------------------- angr/state_plugins/callstack.py | 3 --- angr/state_plugins/cgc.py | 3 --- angr/state_plugins/edge_hitmap.py | 10 ------- angr/state_plugins/filesystem.py | 9 ------- angr/state_plugins/globals.py | 4 --- angr/state_plugins/heap/heap_brk.py | 3 --- angr/state_plugins/heap/heap_ptmalloc.py | 3 --- angr/state_plugins/history.py | 4 --- angr/state_plugins/icicle.py | 3 --- angr/state_plugins/inspect.py | 4 --- angr/state_plugins/javavm_classloader.py | 4 --- angr/state_plugins/jni_references.py | 4 --- angr/state_plugins/libc.py | 3 --- angr/state_plugins/log.py | 3 --- angr/state_plugins/loop_data.py | 4 --- angr/state_plugins/plugin.py | 15 +---------- angr/state_plugins/posix.py | 9 ------- angr/state_plugins/preconstrainer.py | 4 --- angr/state_plugins/scratch.py | 3 --- angr/state_plugins/solver.py | 6 ----- angr/state_plugins/trace_additions.py | 6 ----- angr/state_plugins/unicorn_engine.py | 3 --- angr/state_plugins/view.py | 6 ----- angr/storage/file.py | 15 ----------- .../memory_mixins/javavm_memory_mixin.py | 3 --- angr/storage/memory_mixins/memory_mixin.py | 2 -- .../regioned_memory/region_meta_mixin.py | 7 ----- docs/extending-angr/state_plugins.rst | 8 ------ tests/sim/test_icicle.py | 5 ++-- 33 files changed, 5 insertions(+), 213 deletions(-) diff --git a/angr/analyses/forward_analysis/forward_analysis.py b/angr/analyses/forward_analysis/forward_analysis.py index 5996cb8e3..676e63c97 100644 --- a/angr/analyses/forward_analysis/forward_analysis.py +++ b/angr/analyses/forward_analysis/forward_analysis.py @@ -235,9 +235,6 @@ class ForwardAnalysis[AnalysisState, NodeType, JobType, JobKey, SuccessorType]: _, has_no_changes = self._merge_states(node, old_state, new_state) return has_no_changes - def _widen_states(self, *states: AnalysisState) -> AnalysisState: - raise NotImplementedError("_widen_states() is not implemented.") - # Special interfaces for non-graph-traversal mode def _merge_jobs(self, *jobs: JobType): diff --git a/angr/analyses/stack_pointer_tracker.py b/angr/analyses/stack_pointer_tracker.py index 9f03f9dbd..dbfea52d1 100644 --- a/angr/analyses/stack_pointer_tracker.py +++ b/angr/analyses/stack_pointer_tracker.py @@ -988,14 +988,6 @@ class StackPointerTracker(Analysis, ForwardAnalysis): return curr_stmt_start_addr - def _widen_states(self, *states: FrozenStackPointerTrackerState): - assert len(states) == 2 - merged, _ = self._merge_states(None, *states) - if len(merged.memory) > 5: - _l.info("Encountered too many memory writes in stack pointer tracking. Abandoning memory tracking.") - merged = merged.unfreeze().give_up_on_memory_tracking().freeze() - return merged - def _merge_states(self, node, *states: FrozenStackPointerTrackerState): merged_state = states[0] for other in states[1:]: diff --git a/angr/analyses/vfg.py b/angr/analyses/vfg.py index 88d74d7e0..4e994e4ff 100644 --- a/angr/analyses/vfg.py +++ b/angr/analyses/vfg.py @@ -1123,7 +1123,8 @@ class VFG(ForwardAnalysis[SimState, VFGNode, VFGJob, BlockID, SimState], Analysi l.debug("Widening %s", job_1) - new_state, _ = self._widen_states(job_0.state, job_1.state) + new_state = job_0.state.copy() + new_state.memory.merge([job_1.state.memory], None) new_job = VFGJob( jobs[0].addr, @@ -1200,28 +1201,6 @@ class VFG(ForwardAnalysis[SimState, VFGNode, VFGJob, BlockID, SimState], Analysi return merged, merging_occurred - @staticmethod - def _widen_states(old_state, new_state): - """ - Perform widen operation on the given states, and return a new one. - - :param old_state: - :param new_state: - :returns: The widened state, and whether widening has occurred - """ - - # print old_state.dbg_print_stack() - # print new_state.dbg_print_stack() - - l.debug("Widening state at IP %s", old_state.ip) - - widened_state, widening_occurred = old_state.widen(new_state) - - # print "Widened: " - # print widened_state.dbg_print_stack() - - return widened_state, widening_occurred - @staticmethod def _narrow_states(node, old_state, new_state, previously_widened_state): # pylint:disable=unused-argument,no-self-use """ diff --git a/angr/sim_state.py b/angr/sim_state.py index 4222139d6..5d2ec7fee 100644 --- a/angr/sim_state.py +++ b/angr/sim_state.py @@ -674,32 +674,6 @@ class SimState[IPTypeConc, IPTypeSym](PluginHub[SimStatePlugin]): merged.add_constraints(claripy.Or(*merge_conditions)) return merged, merge_conditions, merging_occurred - def widen(self, *others): - """ - Perform a widening between self and other states - :param others: - :return: - """ - - if len({frozenset(o.plugins.keys()) for o in others}) != 1: - raise SimMergeError("Unable to widen due to different sets of plugins.") - if len({o.arch.name for o in others}) != 1: - raise SimMergeError("Unable to widen due to different architectures.") - - widened = self.copy() - widening_occurred = False - - # plugins - for p in self.plugins: - if p in ("solver", "unicorn"): - continue - plugin_state_widened = widened.plugins[p].widen([_.plugins[p] for _ in others]) - if plugin_state_widened: - l.debug("Widening occurred in %s", p) - widening_occurred = True - - return widened, widening_occurred - ############################################# ### Accessors for tmps, registers, memory ### ############################################# diff --git a/angr/state_plugins/callstack.py b/angr/state_plugins/callstack.py index ff48a496a..2110ced06 100644 --- a/angr/state_plugins/callstack.py +++ b/angr/state_plugins/callstack.py @@ -97,9 +97,6 @@ class CallStack(SimStatePlugin): if o != self: l.error("Trying to merge states with disparate callstacks!") - def widen(self, others): # pylint: disable=unused-argument - l.warning("Widening not implemented for callstacks") - def __iter__(self) -> Iterator[CallStack]: """ Iterate through the callstack, from top to bottom diff --git a/angr/state_plugins/cgc.py b/angr/state_plugins/cgc.py index d9c874561..b3e4d0982 100644 --- a/angr/state_plugins/cgc.py +++ b/angr/state_plugins/cgc.py @@ -106,9 +106,6 @@ class SimStateCGC(SimStatePlugin): def merge(self, others, merge_conditions, common_ancestor=None): # pylint: disable=unused-argument return self._combine(others) - def widen(self, others): - return self._combine(others) - ### HEAP MANAGEMENT def get_max_sinkhole(self, length): diff --git a/angr/state_plugins/edge_hitmap.py b/angr/state_plugins/edge_hitmap.py index 04f02e3ec..9eeb76e32 100644 --- a/angr/state_plugins/edge_hitmap.py +++ b/angr/state_plugins/edge_hitmap.py @@ -66,13 +66,3 @@ class SimStateEdgeHitmap(SimStatePlugin): else: log.warning("Cannot merge edge hitmaps of different sizes") return True - - def widen(self, others): # pylint: disable=unused-argument - """ - Widening operation for static analysis. - - :param others: Other plugin instances. - :return: False, widening is not meaningful for hitmaps. - """ - log.warning("Widening is not implemented for edge hitmap") - return False diff --git a/angr/state_plugins/filesystem.py b/angr/state_plugins/filesystem.py index 85fda0288..cf2435dad 100644 --- a/angr/state_plugins/filesystem.py +++ b/angr/state_plugins/filesystem.py @@ -5,7 +5,6 @@ import os from collections import namedtuple from angr.errors import SimMergeError -from angr.misc.ux import once from angr.storage.file import SimFile from .plugin import SimStatePlugin @@ -142,10 +141,6 @@ class SimFilesystem(SimStatePlugin): # pretends links don't exist return True - def widen(self, others): # pylint: disable=unused-argument - if once("fs_widen_warning"): - l.warning("Filesystems can't be widened yet - beware unsoundness") - def _normalize_path(self, path): """ Takes a path and returns a simple absolute path as a list of directories from the root @@ -396,10 +391,6 @@ class SimConcreteFilesystem(SimMount): merging_occurred |= subdeck[0].merge(subdeck[1:], merge_conditions, common_ancestor=common_simfile) return merging_occurred - def widen(self, others): # pylint: disable=unused-argument - if once("host_fs_widen_warning"): - l.warning("The host filesystem mount can't be widened yet - beware unsoundness") - def _join_chunks(self, keys): """ Takes a list of directories from the root and joins them into a string path diff --git a/angr/state_plugins/globals.py b/angr/state_plugins/globals.py index e72cea399..9bb318971 100644 --- a/angr/state_plugins/globals.py +++ b/angr/state_plugins/globals.py @@ -25,10 +25,6 @@ class SimStateGlobals(SimStatePlugin): return True - def widen(self, others): # pylint: disable=unused-argument - l.warning("Widening is unimplemented for globals") - return False - def __iter__(self): return iter(self._backer) diff --git a/angr/state_plugins/heap/heap_brk.py b/angr/state_plugins/heap/heap_brk.py index 6ddb910a1..1046bdb5f 100644 --- a/angr/state_plugins/heap/heap_brk.py +++ b/angr/state_plugins/heap/heap_brk.py @@ -131,8 +131,5 @@ class SimHeapBrk(SimHeapBase): def merge(self, others, merge_conditions, common_ancestor=None): # pylint:disable=unused-argument return self._combine(others) - def widen(self, others): - return self._combine(others) - SimState.register_default("heap", SimHeapBrk) diff --git a/angr/state_plugins/heap/heap_ptmalloc.py b/angr/state_plugins/heap/heap_ptmalloc.py index 6b4a26722..c35a6b8e2 100644 --- a/angr/state_plugins/heap/heap_ptmalloc.py +++ b/angr/state_plugins/heap/heap_ptmalloc.py @@ -587,9 +587,6 @@ class SimHeapPTMalloc(SimHeapFreelist): def merge(self, others, merge_conditions, common_ancestor=None): # pylint:disable=unused-argument return self._combine(others) - def widen(self, others): - return self._combine(others) - def init_state(self): super().init_state() diff --git a/angr/state_plugins/history.py b/angr/state_plugins/history.py index 6e11b422b..19898396e 100644 --- a/angr/state_plugins/history.py +++ b/angr/state_plugins/history.py @@ -175,10 +175,6 @@ class SimStateHistory(SimStatePlugin): return True - def widen(self, others): # pylint: disable=unused-argument - l.warning("history widening is not implemented!") - return # TODO - @SimStatePlugin.memo def copy(self, memo): # pylint: disable=unused-argument return SimStateHistory(clone=self) diff --git a/angr/state_plugins/icicle.py b/angr/state_plugins/icicle.py index 4a89249ab..d1976ae01 100644 --- a/angr/state_plugins/icicle.py +++ b/angr/state_plugins/icicle.py @@ -84,8 +84,5 @@ class SimStateIcicle(SimStatePlugin): def merge(self, others, merge_conditions, common_ancestor=None): return False - def widen(self, others): - return False - SimState.register_default("icicle", SimStateIcicle) diff --git a/angr/state_plugins/inspect.py b/angr/state_plugins/inspect.py index 54f3fc04f..dcddb22a5 100644 --- a/angr/state_plugins/inspect.py +++ b/angr/state_plugins/inspect.py @@ -41,7 +41,6 @@ class EventType(enum.StrEnum): SYSCALL = "syscall" CFG_HANDLE_JOB = "cfg_handle_job" VFG_HANDLE_SUCCESSOR = "vfg_handle_successor" - VFG_WIDEN_STATE = "vfg_widen_state" ENGINE_PROCESS = "engine_process" MEMORY_PAGE_MAP = "memory_page_map" @@ -416,9 +415,6 @@ class SimInspector(SimStatePlugin): def merge(self, others, merge_conditions, common_ancestor=None): # pylint: disable=unused-argument return self._combine(others) - def widen(self, others): - return self._combine(others) - def set_state(self, state): super().set_state(state) state.supports_inspect = True diff --git a/angr/state_plugins/javavm_classloader.py b/angr/state_plugins/javavm_classloader.py index a5b66f64d..217990ca5 100644 --- a/angr/state_plugins/javavm_classloader.py +++ b/angr/state_plugins/javavm_classloader.py @@ -126,10 +126,6 @@ class SimJavaVmClassloader(SimStatePlugin): l.warning("Merging is not implemented for JavaVM classloader!") return False - def widen(self, others): # pylint: disable=unused-argument - l.warning("Widening is not implemented for JavaVM classloader!") - return False - # TODO use a default JavaVM preset # see for reference: angr/engines/__init__.py diff --git a/angr/state_plugins/jni_references.py b/angr/state_plugins/jni_references.py index 8cbeefb78..5c0dff6ed 100644 --- a/angr/state_plugins/jni_references.py +++ b/angr/state_plugins/jni_references.py @@ -87,10 +87,6 @@ class SimStateJNIReferences(SimStatePlugin): l.warning("Merging is not implemented for JNI references!") return False - def widen(self, others): # pylint: disable=unused-argument - l.warning("Widening is not implemented for JNI references!") - return False - # TODO use a default JavaVM preset # see for reference: angr/engines/__init__.py diff --git a/angr/state_plugins/libc.py b/angr/state_plugins/libc.py index 29a6d1d65..90501e838 100644 --- a/angr/state_plugins/libc.py +++ b/angr/state_plugins/libc.py @@ -1240,9 +1240,6 @@ class SimStateLibc(SimStatePlugin): def merge(self, others, merge_conditions, common_ancestor=None): # pylint: disable=unused-argument return False - def widen(self, others): - return False - @property def errno(self): return self.state.mem[self.errno_location].int.resolved diff --git a/angr/state_plugins/log.py b/angr/state_plugins/log.py index c5ac8b30e..37913ae53 100644 --- a/angr/state_plugins/log.py +++ b/angr/state_plugins/log.py @@ -68,9 +68,6 @@ class SimStateLog(SimStatePlugin): def merge(self, others, merge_conditions, common_ancestor=None): # pylint: disable=unused-argument return self._combine(others) - def widen(self, others): - return self._combine(others) - def clear(self): s = self.state self.__init__() diff --git a/angr/state_plugins/loop_data.py b/angr/state_plugins/loop_data.py index 0d251fbee..3385b125f 100644 --- a/angr/state_plugins/loop_data.py +++ b/angr/state_plugins/loop_data.py @@ -77,10 +77,6 @@ class SimStateLoopData(SimStatePlugin): l.warning("Merging is not implemented for loop data!") return False - def widen(self, others): # pylint: disable=unused-argument - l.warning("Widening is not implemented for loop data!") - return False - @SimStatePlugin.memo def copy(self, memo): # pylint: disable=unused-argument return SimStateLoopData( diff --git a/angr/state_plugins/plugin.py b/angr/state_plugins/plugin.py index d276c4862..720509a5f 100644 --- a/angr/state_plugins/plugin.py +++ b/angr/state_plugins/plugin.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from collections.abc import Callable, Iterable +from collections.abc import Callable from functools import wraps from typing import TYPE_CHECKING, Any, Protocol, cast @@ -120,19 +120,6 @@ class SimStatePlugin: """ raise NotImplementedError(f"merge() not implement for {self.__class__.__name__}") - def widen(self, others: Iterable[SimStatePlugin]) -> bool: # pylint:disable=unused-argument - """ - The widening operation for plugins. Widening is a special kind of merging that produces a more general state - from several more specific states. It is used only during intensive static analysis. The same behavior - regarding copying and mutation from ``merge`` should be followed. - - :param others: the other state plugins to widen with - - :returns: True if the state plugin is actually widened. - :rtype: bool - """ - raise NotImplementedError(f"widen() not implemented for {self.__class__.__name__}") - @classmethod def register_default(cls, name: str, xtr: type[SimStatePlugin] | str | None = None) -> None: if cls is SimStatePlugin: diff --git a/angr/state_plugins/posix.py b/angr/state_plugins/posix.py index 0e3bdb5a2..216755fbf 100644 --- a/angr/state_plugins/posix.py +++ b/angr/state_plugins/posix.py @@ -39,9 +39,6 @@ class PosixDevFS(SimMount): # this'll be mounted at /dev def merge(self, others, conditions, common_ancestor=None): # pylint: disable=unused-argument, arguments-differ return False - def widen(self, others): # pylint: disable=unused-argument - return False - def copy(self, _): return self # this holds no state! @@ -68,9 +65,6 @@ class PosixProcFS(SimMount): def merge(self, others, conditions, common_ancestor=None): # pylint: disable=unused-argument, arguments-differ return False - def widen(self, others): # pylint: disable=unused-argument - return False - def copy(self, _): return self # this holds no state! @@ -665,9 +659,6 @@ class SimSystemPosix(SimStatePlugin): return merging_occurred - def widen(self, _): - raise SimMergeError("Widening the system state is unsupported") - def dump_file_by_path(self, path, **kwargs): """ Returns the concrete content for a file by path. diff --git a/angr/state_plugins/preconstrainer.py b/angr/state_plugins/preconstrainer.py index dfa85c260..997b2208a 100644 --- a/angr/state_plugins/preconstrainer.py +++ b/angr/state_plugins/preconstrainer.py @@ -34,10 +34,6 @@ class SimStatePreconstrainer(SimStatePlugin): l.warning("Merging is not implemented for preconstrainer!") return False - def widen(self, others): # pylint: disable=unused-argument - l.warning("Widening is not implemented for preconstrainer!") - return False - @SimStatePlugin.memo def copy(self, memo): # pylint: disable=unused-argument c = SimStatePreconstrainer(constrained_addrs=self._constrained_addrs) diff --git a/angr/state_plugins/scratch.py b/angr/state_plugins/scratch.py index 4c846dbb8..bc7dc7981 100644 --- a/angr/state_plugins/scratch.py +++ b/angr/state_plugins/scratch.py @@ -180,9 +180,6 @@ class SimStateScratch(SimStatePlugin): def merge(self, others, merge_conditions, common_ancestor=None): # pylint: disable=unused-argument return False - def widen(self, others): # pylint: disable=unused-argument - return False - def clear(self): s = self.state j = self.jumpkind diff --git a/angr/state_plugins/solver.py b/angr/state_plugins/solver.py index 481d22f02..e02e8e77c 100644 --- a/angr/state_plugins/solver.py +++ b/angr/state_plugins/solver.py @@ -474,12 +474,6 @@ class SimSolver(SimStatePlugin): ) return merging_occurred - @error_converter - def widen(self, others): - c = claripy.BVS("random_widen_condition", 32) - merge_conditions = [[c == i] for i in range(len(others) + 1)] - return self.merge(others, merge_conditions) - # # Frontend passthroughs # diff --git a/angr/state_plugins/trace_additions.py b/angr/state_plugins/trace_additions.py index 16ecb10af..f77491044 100644 --- a/angr/state_plugins/trace_additions.py +++ b/angr/state_plugins/trace_additions.py @@ -358,9 +358,6 @@ class ChallRespInfo(angr.state_plugins.SimStatePlugin): def merge(self, others, merge_conditions, common_ancestor=None): # pylint: disable=unused-argument raise angr.errors.SimMergeError("Can't merge ChallRespInfo - what on earth are you doing?") - def widen(self, others): # pylint: disable=unused-argument - raise angr.errors.SimMergeError("Can't widen ChallRespInfo - what on earth are you doing?") - @staticmethod def get_byte(var_name): ## XXX TODO FIXME DO NOT DO THIS HOLY SHIT WHAT THE FUCK @@ -687,9 +684,6 @@ class ZenPlugin(angr.state_plugins.SimStatePlugin): def merge(self, others, merge_conditions, common_ancestor=None): # pylint: disable=unused-argument raise angr.errors.SimMergeError("Can't merge ZenPlugin - what on earth are you doing?") - def widen(self, others): # pylint: disable=unused-argument - raise angr.errors.SimMergeError("Can't widen ZenPlugin - what on earth are you doing?") - def get_flag_bytes(self, ast): flag_args = self.get_flag_rand_args(ast) flag_arg_vars = set.union(*[set(v.variables) for v in flag_args]) diff --git a/angr/state_plugins/unicorn_engine.py b/angr/state_plugins/unicorn_engine.py index 0324f1841..b9d455d65 100644 --- a/angr/state_plugins/unicorn_engine.py +++ b/angr/state_plugins/unicorn_engine.py @@ -800,9 +800,6 @@ class Unicorn(SimStatePlugin): # I guess always lie to the static analysis? return False - def widen(self, others): # pylint: disable=unused-argument - l.warning("Can't widen the unicorn plugin!") - def __getstate__(self): d = dict(self.__dict__) del d["_bullshit_cb"] diff --git a/angr/state_plugins/view.py b/angr/state_plugins/view.py index 18f1b748e..4acccf6af 100644 --- a/angr/state_plugins/view.py +++ b/angr/state_plugins/view.py @@ -109,9 +109,6 @@ class SimRegNameView(SimStatePlugin): def merge(self, others, merge_conditions, common_ancestor=None): # pylint: disable=unused-argument return False - def widen(self, others): # pylint: disable=unused-argument - return False - def get(self, reg_name): return self.__getattr__(reg_name) @@ -261,9 +258,6 @@ class SimMemView(SimStatePlugin): def merge(self, others, merge_conditions, common_ancestor=None): # pylint: disable=unused-argument return False - def widen(self, others): # pylint: disable=unused-argument - return False - @property def resolvable(self): return self._type is not None and self._addr is not None diff --git a/angr/storage/file.py b/angr/storage/file.py index 809006241..e35255540 100644 --- a/angr/storage/file.py +++ b/angr/storage/file.py @@ -355,9 +355,6 @@ class SimFile(SimFileBase, DefaultMemory): # TODO: pick a better base class omg return super().merge(others, merge_conditions, common_ancestor=common_ancestor) - def widen(self, _): - raise SimMergeError("Widening the filesystem is unsupported") - class SimFileStream(SimFile): """ @@ -657,9 +654,6 @@ class SimPackets(SimFileBase): return True - def widen(self, _): - raise SimMergeError("Widening the filesystem is unsupported") - class SimPacketsStream(SimPackets): """ @@ -1010,9 +1004,6 @@ class SimFileDescriptor(SimFileDescriptorBase): return True - def widen(self, _): - raise SimMergeError("Widening the filesystem is unsupported") - class SimFileDescriptorDuplex(SimFileDescriptorBase): """ @@ -1129,9 +1120,6 @@ class SimFileDescriptorDuplex(SimFileDescriptorBase): return True - def widen(self, _): - raise SimMergeError("Widening the filesystem is unsupported") - class SimPacketsSlots(SimFileBase): """ @@ -1209,6 +1197,3 @@ class SimPacketsSlots(SimFileBase): ) return True - - def widen(self, _): - raise SimMergeError("Widening the filesystem is unsupported") diff --git a/angr/storage/memory_mixins/javavm_memory_mixin.py b/angr/storage/memory_mixins/javavm_memory_mixin.py index 3330c59db..eca09b459 100644 --- a/angr/storage/memory_mixins/javavm_memory_mixin.py +++ b/angr/storage/memory_mixins/javavm_memory_mixin.py @@ -382,9 +382,6 @@ class JavaVmMemoryMixin(MemoryMixin): def merge(self, others, merge_conditions, common_ancestor=None): # pylint: disable=unused-argument l.warning("Merging is not implemented for JavaVM memory!") - def widen(self, others): # pylint: disable=no-self-use,unused-argument - l.warning("Widening is not implemented for JavaVM memory!") - # pylint: disable=no-self-use,unused-argument def _find(self, addr, what, max_search=None, max_symbolic_bytes=None, default=None): l.warning("Find is not implemented for JavaVM memory!") diff --git a/angr/storage/memory_mixins/memory_mixin.py b/angr/storage/memory_mixins/memory_mixin.py index 8f13f6d59..84492bd0d 100644 --- a/angr/storage/memory_mixins/memory_mixin.py +++ b/angr/storage/memory_mixins/memory_mixin.py @@ -80,8 +80,6 @@ class MemoryMixin[InData, OutData, Addr](SimStatePlugin): def compare(self, other: Self) -> bool: ... - def widen(self, others: list[Self]) -> bool: ... - def permissions(self, addr: Addr, permissions: int | claripy.ast.BV | None = None, **kwargs) -> claripy.ast.BV: ... def map_region( diff --git a/angr/storage/memory_mixins/regioned_memory/region_meta_mixin.py b/angr/storage/memory_mixins/regioned_memory/region_meta_mixin.py index 9d69c5089..1f745367b 100644 --- a/angr/storage/memory_mixins/regioned_memory/region_meta_mixin.py +++ b/angr/storage/memory_mixins/regioned_memory/region_meta_mixin.py @@ -224,13 +224,6 @@ class MemoryRegionMetaMixin(MemoryMixin): r |= super().merge([other_region], merge_conditions, common_ancestor=common_ancestor) return r - def widen(self, others): - result = False - for other_region in others: - self._merge_alocs(other_region) - result |= super().widen([other_region.memory]) - return result - def dbg_print(self, indent=0): """ Print out debugging information diff --git a/docs/extending-angr/state_plugins.rst b/docs/extending-angr/state_plugins.rst index f3121a4d1..b77663a3f 100644 --- a/docs/extending-angr/state_plugins.rst +++ b/docs/extending-angr/state_plugins.rst @@ -150,14 +150,6 @@ which case it will be None. There are no rules for how exactly you should use this to improve the quality of your merges, but you may find it useful in more complex setups. -Widening --------- - -There is another kind of merging called *widening* which takes several states -and produces a more general state. It is used during static analysis. - -.. todo:: Explain what this means - Serialization ------------- diff --git a/tests/sim/test_icicle.py b/tests/sim/test_icicle.py index 5769d3dc4..e58e8508e 100644 --- a/tests/sim/test_icicle.py +++ b/tests/sim/test_icicle.py @@ -929,8 +929,8 @@ class TestSimStateIciclePlugin(TestCase): copied.dirty_pages.add(6) assert 6 not in plugin.dirty_pages - def test_plugin_merge_and_widen(self): - """Test that merge and widen return False (not mergeable).""" + def test_plugin_merge(self): + """Test that merge returns False (not mergeable).""" dummy_td = cast(IcicleStateTranslationData, None) plugin = SimStateIcicle( generation=1, @@ -938,7 +938,6 @@ class TestSimStateIciclePlugin(TestCase): dirty_pages=set(), ) assert plugin.merge([], [], None) is False - assert plugin.widen([]) is False class TestContinuation(TestCase): From 3984a816b606e1459d01e5427fb4b1e304b16828 Mon Sep 17 00:00:00 2001 From: Ati Priya Date: Mon, 20 Jul 2026 02:12:23 -0700 Subject: [PATCH 009/122] Fix bswap32 intrinsic name: __buildin_ -> __builtin_ (#6635) gcc has no __buildin_bswap32, so the emitted call never resolved. --- .../peephole_optimizations/bswap.py | 2 +- .../decompiler/test_peephole_optimizations.py | 107 +++++++++++++++++- 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/angr/analyses/decompiler/peephole_optimizations/bswap.py b/angr/analyses/decompiler/peephole_optimizations/bswap.py index 7fe6eeff0..b93521d9c 100644 --- a/angr/analyses/decompiler/peephole_optimizations/bswap.py +++ b/angr/analyses/decompiler/peephole_optimizations/bswap.py @@ -100,7 +100,7 @@ class Bswap(PeepholeOptimizationExprBase): (">>", 8, 0xFF00), }: core_expr = next(iter(cores)) - return Call(expr.idx, "__buildin_bswap32", args=[core_expr], bits=expr.bits, **expr.tags) + return Call(expr.idx, "__builtin_bswap32", args=[core_expr], bits=expr.bits, **expr.tags) return None diff --git a/tests/analyses/decompiler/test_peephole_optimizations.py b/tests/analyses/decompiler/test_peephole_optimizations.py index 298f2a4df..33e86cce2 100755 --- a/tests/analyses/decompiler/test_peephole_optimizations.py +++ b/tests/analyses/decompiler/test_peephole_optimizations.py @@ -11,10 +11,11 @@ import archinfo import angr from angr import ailment -from angr.ailment.expression import BinaryOp, Const, Convert, Extract, Insert, Register +from angr.ailment.expression import BinaryOp, Call, Const, Convert, Extract, Insert, Register from angr.ailment.manager import Manager from angr.analyses.decompiler.peephole_optimizations import ( EXPR_OPTS, + Bswap, CmpMaskedShift, CmpSubConst, ConstantDereferences, @@ -233,6 +234,110 @@ class TestPeepholeOptimizations(unittest.TestCase): divisor_operand = r.operand.operands[1] assert isinstance(divisor_operand, Const) and divisor_operand.value == divisor + def test_bswap32_intrinsic_name(self): + proj = angr.load_shellcode(b"\x90", "AMD64") + manager = Manager() + opt = Bswap(proj, proj.kb, manager) + + # (Conv(64->32, x) << 0x18) | + # ((Conv(64->32, x) << 8) & 0xff0000) | + # ((Conv(64->32, x) >> 8) & 0xff00) | + # ((Conv(64->32, x) >> 0x18) & 0xff) + # => __builtin_bswap32(Conv(64->32, x)) + conv = Convert(manager.next_atom(), 64, 32, False, Register(manager.next_atom(), 16, 64)) + p0 = BinaryOp(manager.next_atom(), "Shl", [conv, Const(manager.next_atom(), 0x18, 8)], False, bits=32) + p1 = BinaryOp( + manager.next_atom(), + "And", + [ + BinaryOp(manager.next_atom(), "Shl", [conv, Const(manager.next_atom(), 8, 8)], False, bits=32), + Const(manager.next_atom(), 0xFF0000, 32), + ], + False, + bits=32, + ) + p2 = BinaryOp( + manager.next_atom(), + "And", + [ + BinaryOp(manager.next_atom(), "Shr", [conv, Const(manager.next_atom(), 8, 8)], False, bits=32), + Const(manager.next_atom(), 0xFF00, 32), + ], + False, + bits=32, + ) + p3 = BinaryOp( + manager.next_atom(), + "And", + [ + BinaryOp(manager.next_atom(), "Shr", [conv, Const(manager.next_atom(), 0x18, 8)], False, bits=32), + Const(manager.next_atom(), 0xFF, 32), + ], + False, + bits=32, + ) + expr = BinaryOp( + manager.next_atom(), + "Or", + [ + p0, + BinaryOp( + manager.next_atom(), + "Or", + [p1, BinaryOp(manager.next_atom(), "Or", [p2, p3], False, bits=32)], + False, + bits=32, + ), + ], + False, + bits=32, + ) + + out = opt.optimize(expr) + assert isinstance(out, Call) + assert out.target == "__builtin_bswap32" + assert len(out.args) == 1 and out.args[0].likes(conv) + assert out.bits == 32 + + def test_bswap16_intrinsic_name(self): + proj = angr.load_shellcode(b"\x90", "AMD64") + manager = Manager() + opt = Bswap(proj, proj.kb, manager) + + # ((((Conv(16->32, a) << 8) & 0xff00ff00) | ((Conv(16->32, a) >> 8) & 0xff00ff)) & 0xffff) + # => __builtin_bswap16(a) + reg = Register(manager.next_atom(), 16, 16) + shl = BinaryOp( + manager.next_atom(), + "Shl", + [Convert(manager.next_atom(), 16, 32, False, reg), Const(manager.next_atom(), 8, 8)], + False, + bits=32, + ) + shr = BinaryOp( + manager.next_atom(), + "Shr", + [Convert(manager.next_atom(), 16, 32, False, reg), Const(manager.next_atom(), 8, 8)], + False, + bits=32, + ) + inner = BinaryOp( + manager.next_atom(), + "Or", + [ + BinaryOp(manager.next_atom(), "And", [shl, Const(manager.next_atom(), 0xFF00FF00, 32)], False, bits=32), + BinaryOp(manager.next_atom(), "And", [shr, Const(manager.next_atom(), 0x00FF00FF, 32)], False, bits=32), + ], + False, + bits=32, + ) + expr = BinaryOp(manager.next_atom(), "And", [inner, Const(manager.next_atom(), 0xFFFF, 32)], False, bits=32) + + out = opt.optimize(expr) + assert isinstance(out, Call) + assert out.target == "__builtin_bswap16" + assert len(out.args) == 1 and out.args[0].likes(reg) + def test_bitwise_inserts(self): proj = angr.load_shellcode(b"\x90", "AMD64") manager = Manager() From d145bd41fd7c37e9721c184b77f82fe5ce1cc68c Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Mon, 20 Jul 2026 10:06:54 -0700 Subject: [PATCH 010/122] Remove global condition from SimState (#6641) --- angr/engines/successors.py | 2 +- angr/sim_state.py | 41 ---------- angr/state_plugins/solver.py | 76 +++++-------------- .../memory_mixins/conditional_store_mixin.py | 2 - tests/sim/test_state.py | 34 --------- 5 files changed, 22 insertions(+), 133 deletions(-) diff --git a/angr/engines/successors.py b/angr/engines/successors.py index 6524dc63b..0d08fa27e 100644 --- a/angr/engines/successors.py +++ b/angr/engines/successors.py @@ -294,7 +294,7 @@ class SimSuccessors: skip_max_targets_warning = True # don't warn elif o.KEEP_IP_SYMBOLIC in state.options: s = claripy.Solver() - addrs = s.eval(target, _max_targets + 1, extra_constraints=tuple(state.ip_constraints)) + addrs = s.eval(target, _max_targets + 1) if len(addrs) > _max_targets: # It is not a library l.debug("It is not a Library") diff --git a/angr/sim_state.py b/angr/sim_state.py index 5d2ec7fee..424a89651 100644 --- a/angr/sim_state.py +++ b/angr/sim_state.py @@ -1,6 +1,5 @@ from __future__ import annotations -import contextlib import functools import itertools import logging @@ -166,10 +165,6 @@ class SimState[IPTypeConc, IPTypeSym](PluginHub[SimStatePlugin]): self.uninitialized_access_handler = None self._special_memory_filler = special_memory_filler - # this is a global condition, applied to all added constraints, memory reads, etc - self._global_condition = None - self.ip_constraints = [] - # plugins. lord help us if plugin_preset is not None: self.use_plugin_preset(plugin_preset) @@ -562,9 +557,6 @@ class SimState[IPTypeConc, IPTypeSym](PluginHub[SimStatePlugin]): Returns a copy of the state. """ - if self._global_condition is not None: - raise SimStateError("global condition was not cleared before state.copy().") - c_plugins = self._copy_plugins() state = SimState( project=self.project, @@ -581,7 +573,6 @@ class SimState[IPTypeConc, IPTypeSym](PluginHub[SimStatePlugin]): state.uninitialized_access_handler = self.uninitialized_access_handler state._special_memory_filler = self._special_memory_filler - state.ip_constraints = self.ip_constraints return state @@ -846,38 +837,6 @@ class SimState[IPTypeConc, IPTypeSym](PluginHub[SimStatePlugin]): concrete_ip = self.solver.eval(self.regs.ip) return concrete_ip % 2 == 1 - # - # Some pretty fancy global condition stuff! - # - - @property - def with_condition(self): - @contextlib.contextmanager - def ctx(c): - old_condition = self._global_condition - try: - new_condition = c if old_condition is None else claripy.And(old_condition, c) - self._global_condition = new_condition - yield - finally: - self._global_condition = old_condition - - return ctx - - def _adjust_condition(self, c): - if self._global_condition is None: - return c - if c is None: - return self._global_condition - return claripy.And(self._global_condition, c) - - def _adjust_condition_list(self, conditions): - if self._global_condition is None: - return conditions - if len(conditions) == 0: - return conditions.__class__((self._global_condition,)) - return conditions.__class__((self._adjust_condition(claripy.And(*conditions)),)) - default_state_plugin_preset = PluginPreset() SimState.register_preset("default", default_state_plugin_preset) diff --git a/angr/state_plugins/solver.py b/angr/state_plugins/solver.py index e02e8e77c..b2bf3cb58 100644 --- a/angr/state_plugins/solver.py +++ b/angr/state_plugins/solver.py @@ -492,21 +492,6 @@ class SimSolver(SimStatePlugin): """ return self._solver.constraints - def _adjust_constraint(self, c): - if self.state._global_condition is None: - return c - if c is None: # this should never happen - l.critical("PLEASE REPORT THIS MESSAGE, AND WHAT YOU WERE DOING, TO YAN") - return self.state._global_condition - return claripy.Or(claripy.Not(self.state._global_condition), c) - - def _adjust_constraint_list(self, constraints): - if self.state._global_condition is None: - return constraints - if len(constraints) == 0: - return constraints.__class__((self.state._global_condition,)) - return constraints.__class__((self._adjust_constraint(claripy.And(*constraints)),)) - @timed_function @ast_stripping_decorator @error_converter @@ -521,9 +506,7 @@ class SimSolver(SimStatePlugin): :return: a tuple of the solutions, in the form of claripy AST nodes :rtype: tuple """ - return self._solver.eval_to_ast( - e, n, extra_constraints=self._adjust_constraint_list(extra_constraints), exact=exact - ) + return self._solver.eval_to_ast(e, n, extra_constraints=extra_constraints, exact=exact) @concrete_path_tuple @timed_function @@ -540,7 +523,7 @@ class SimSolver(SimStatePlugin): :return: a tuple of the solutions, in the form of Python primitives :rtype: tuple """ - return self._solver.eval(e, n, extra_constraints=self._adjust_constraint_list(extra_constraints), exact=exact) + return self._solver.eval(e, n, extra_constraints=extra_constraints, exact=exact) @concrete_path_scalar @timed_function @@ -557,15 +540,11 @@ class SimSolver(SimStatePlugin): :return: the maximum possible value of e (backend object) """ if exact is False and o.VALIDATE_APPROXIMATIONS in self.state.options: - ar = self._solver.max( - e, extra_constraints=self._adjust_constraint_list(extra_constraints), exact=False, signed=signed - ) - er = self._solver.max(e, extra_constraints=self._adjust_constraint_list(extra_constraints), signed=signed) + ar = self._solver.max(e, extra_constraints=extra_constraints, exact=False, signed=signed) + er = self._solver.max(e, extra_constraints=extra_constraints, signed=signed) assert er <= ar return ar - return self._solver.max( - e, extra_constraints=self._adjust_constraint_list(extra_constraints), exact=exact, signed=signed - ) + return self._solver.max(e, extra_constraints=extra_constraints, exact=exact, signed=signed) @concrete_path_scalar @timed_function @@ -582,15 +561,11 @@ class SimSolver(SimStatePlugin): :return: the minimum possible value of e (backend object) """ if exact is False and o.VALIDATE_APPROXIMATIONS in self.state.options: - ar = self._solver.min( - e, extra_constraints=self._adjust_constraint_list(extra_constraints), exact=False, signed=signed - ) - er = self._solver.min(e, extra_constraints=self._adjust_constraint_list(extra_constraints), signed=signed) + ar = self._solver.min(e, extra_constraints=extra_constraints, exact=False, signed=signed) + er = self._solver.min(e, extra_constraints=extra_constraints, signed=signed) assert ar <= er return ar - return self._solver.min( - e, extra_constraints=self._adjust_constraint_list(extra_constraints), exact=exact, signed=signed - ) + return self._solver.min(e, extra_constraints=extra_constraints, exact=exact, signed=signed) @timed_function @ast_stripping_decorator @@ -606,16 +581,12 @@ class SimSolver(SimStatePlugin): :return: True if `v` is a solution of `expr`, False otherwise """ if exact is False and o.VALIDATE_APPROXIMATIONS in self.state.options: - ar = self._solver.solution( - e, v, extra_constraints=self._adjust_constraint_list(extra_constraints), exact=False - ) - er = self._solver.solution(e, v, extra_constraints=self._adjust_constraint_list(extra_constraints)) + ar = self._solver.solution(e, v, extra_constraints=extra_constraints, exact=False) + er = self._solver.solution(e, v, extra_constraints=extra_constraints) if er is True: assert ar is True return ar - return self._solver.solution( - e, v, extra_constraints=self._adjust_constraint_list(extra_constraints), exact=exact - ) + return self._solver.solution(e, v, extra_constraints=extra_constraints, exact=exact) @concrete_path_bool @timed_function @@ -633,12 +604,12 @@ class SimSolver(SimStatePlugin): :return: True if `v` is definitely true, False otherwise """ if exact is False and o.VALIDATE_APPROXIMATIONS in self.state.options: - ar = self._solver.is_true(e, extra_constraints=self._adjust_constraint_list(extra_constraints), exact=False) - er = self._solver.is_true(e, extra_constraints=self._adjust_constraint_list(extra_constraints)) + ar = self._solver.is_true(e, extra_constraints=extra_constraints, exact=False) + er = self._solver.is_true(e, extra_constraints=extra_constraints) if er is False: assert ar is False return ar - return self._solver.is_true(e, extra_constraints=self._adjust_constraint_list(extra_constraints), exact=exact) + return self._solver.is_true(e, extra_constraints=extra_constraints, exact=exact) @concrete_path_not_bool @timed_function @@ -656,14 +627,12 @@ class SimSolver(SimStatePlugin): :return: True if `v` is definitely false, False otherwise """ if exact is False and o.VALIDATE_APPROXIMATIONS in self.state.options: - ar = self._solver.is_false( - e, extra_constraints=self._adjust_constraint_list(extra_constraints), exact=False - ) - er = self._solver.is_false(e, extra_constraints=self._adjust_constraint_list(extra_constraints)) + ar = self._solver.is_false(e, extra_constraints=extra_constraints, exact=False) + er = self._solver.is_false(e, extra_constraints=extra_constraints) if er is False: assert ar is False return ar - return self._solver.is_false(e, extra_constraints=self._adjust_constraint_list(extra_constraints), exact=exact) + return self._solver.is_false(e, extra_constraints=extra_constraints, exact=exact) @timed_function @ast_stripping_decorator @@ -695,14 +664,12 @@ class SimSolver(SimStatePlugin): return all(not self.is_false(e) for e in extra_constraints) if exact is False and o.VALIDATE_APPROXIMATIONS in self.state.options: - er = self._solver.satisfiable(extra_constraints=self._adjust_constraint_list(extra_constraints)) - ar = self._solver.satisfiable( - extra_constraints=self._adjust_constraint_list(extra_constraints), exact=False - ) + er = self._solver.satisfiable(extra_constraints=extra_constraints) + ar = self._solver.satisfiable(extra_constraints=extra_constraints, exact=False) if er is True: assert ar is True return ar - return self._solver.satisfiable(extra_constraints=self._adjust_constraint_list(extra_constraints), exact=exact) + return self._solver.satisfiable(extra_constraints=extra_constraints, exact=exact) @timed_function @ast_stripping_decorator @@ -723,8 +690,7 @@ class SimSolver(SimStatePlugin): self.state._inspect("constraints", BP_BEFORE, added_constraints=constraints) constraints = self.state._inspect_getattr("added_constraints", constraints) - cc = self._adjust_constraint_list(constraints) - added = self._solver.add(cc) + added = self._solver.add(constraints) self.state._inspect("constraints", BP_AFTER) # add actions for the added constraints diff --git a/angr/storage/memory_mixins/conditional_store_mixin.py b/angr/storage/memory_mixins/conditional_store_mixin.py index e78c9b5f3..d92585eae 100644 --- a/angr/storage/memory_mixins/conditional_store_mixin.py +++ b/angr/storage/memory_mixins/conditional_store_mixin.py @@ -13,8 +13,6 @@ class ConditionalMixin(MemoryMixin): return res def store(self, addr, data, size=None, *, condition=None, **kwargs): - condition = self.state._adjust_condition(condition) - if condition is None or self.state.solver.is_true(condition): super().store(addr, data, size=size, **kwargs) return diff --git a/tests/sim/test_state.py b/tests/sim/test_state.py index ff9e3df14..df0627463 100755 --- a/tests/sim/test_state.py +++ b/tests/sim/test_state.py @@ -211,40 +211,6 @@ class TestState(unittest.TestCase): s = pickle.loads(sp) assert s.solver.eval(s.memory.load(100, 10), cast_to=bytes) == b"AAABAABABC" - def test_global_condition(self): - s = SimState(arch="AMD64") - - s.regs.rax = 10 - old_rax = s.regs.rax - with s.with_condition(False): - assert not s.solver.satisfiable() - s.regs.rax = 20 - assert s._global_condition is None - assert old_rax is s.regs.rax - - with s.with_condition(True): - s.regs.rax = 20 - assert s._global_condition is None - assert old_rax is not s.regs.rax - assert claripy.BVV(20, s.arch.bits) is s.regs.rax - - with s.with_condition(s.regs.rbx != 0): - s.regs.rax = 25 - assert s._global_condition is None - assert claripy.BVV(25, s.arch.bits) is not s.regs.rax - - with s.with_condition(s.regs.rbx != 1): - s.regs.rax = 30 - assert s._global_condition is None - assert claripy.BVV(30, s.arch.bits) is not s.regs.rax - - with s.with_condition(s.regs.rbx == 0): - assert s.solver.eval_upto(s.regs.rbx, 10) == [0] - assert s.solver.eval_upto(s.regs.rax, 10) == [30] - with s.with_condition(s.regs.rbx == 1): - assert s.solver.eval_upto(s.regs.rbx, 10) == [1] - assert s.solver.eval_upto(s.regs.rax, 10) == [25] - def test_successors_catch_arbitrary_interrupts(self): # int 0xd2 should fail on x86/amd64 since it's an unsupported interrupt block_bytes = b"\xcd\xd2" From e9d795193a23db5e456825e190e6fe2a19b9d102 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:07:39 -0700 Subject: [PATCH 011/122] ci: bump actions/checkout from 7.0.0 to 7.0.1 (#6638) Bumps [actions/checkout](https://github.com/actions/checkout) from 7.0.0 to 7.0.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0...3d3c42e5aac5ba805825da76410c181273ba90b1) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 6 +++--- .github/workflows/coverage.yml | 6 +++--- .github/workflows/nightly-ci.yml | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba0bbe804..b7a2c8420 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: fail-fast: false runs-on: ${{ matrix.environment.os }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 if: startsWith(runner.os, 'windows') - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v6 @@ -43,7 +43,7 @@ jobs: name: Rust Check runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1 with: components: clippy, rustfmt @@ -56,7 +56,7 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v5 with: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ea303037d..bf76d332b 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -23,7 +23,7 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v6 - name: Restore test durations cache @@ -61,7 +61,7 @@ jobs: tar -xpf $PWD/env.tzst -C / rm env.tzst - name: Download test binaries - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 with: repository: angr/binaries path: binaries @@ -96,7 +96,7 @@ jobs: name: Test Rust packages runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1 - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2 diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 006d2f785..56d6dc45d 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -28,10 +28,10 @@ jobs: runner_id: [1, 2, 3, 4, 5] fail-fast: false steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 with: path: angr - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 with: repository: angr/binaries path: binaries @@ -54,10 +54,10 @@ jobs: runner_id: [1, 2, 3] fail-fast: false steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 with: path: angr - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 with: repository: angr/binaries path: binaries From 635ddcd134d173b5ac0ebbe85e7eec8a25aaff46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:07:54 -0700 Subject: [PATCH 012/122] ci: bump taiki-e/install-action from 2.83.2 to 2.84.0 (#6636) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.83.2 to 2.84.0. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/43aecc8d72668fbcfe75c31400bc4f890f1c5853...a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.84.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index bf76d332b..d9d4a5307 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -99,7 +99,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1 - - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2 + - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2 with: tool: cargo-llvm-cov - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v5 From ada9fcd58586f20b31ed3a329e9a093e47a56de1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:08:07 -0700 Subject: [PATCH 013/122] rust: bump serde from 1.0.228 to 1.0.229 (#6639) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.228 to 1.0.229. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.228...v1.0.229) --- updated-dependencies: - dependency-name: serde dependency-version: 1.0.229 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1368a9447..f27a0ea88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1497,9 +1497,9 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -1529,22 +1529,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 3.0.2", ] [[package]] @@ -1669,6 +1669,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "target-lexicon" version = "0.13.5" From 41baf181f3935f3cd79be1dd61c6da89fe533662 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:08:14 -0700 Subject: [PATCH 014/122] rust: bump regex from 1.12.2 to 1.13.1 (#6640) Bumps [regex](https://github.com/rust-lang/regex) from 1.12.2 to 1.13.1. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.12.2...1.13.1) --- updated-dependencies: - dependency-name: regex dependency-version: 1.13.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f27a0ea88..c61d6df5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1367,9 +1367,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -1379,9 +1379,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -1390,9 +1390,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "region" From 7d74ec8924110c1354858a0a99d25d88f59f6d48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:08:44 -0700 Subject: [PATCH 015/122] ci: bump actions/setup-python from 6.3.0 to 7.0.0 (#6637) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6.3.0 to 7.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/ece7cb06caefa5fff74198d8649806c4678c61a1...5fda3b95a4ea91299a34e894583c3862153e4b97) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- .github/workflows/coverage.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7a2c8420..47d12c130 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v5 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v5 with: python-version: "3.12" - run: cargo test --release diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index d9d4a5307..60ebc0333 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -102,7 +102,7 @@ jobs: - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2 with: tool: cargo-llvm-cov - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v5 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v5 with: python-version: "3.12" - name: Run tests From 2c6c986f66252b18dfff5c8a7dc45e0cd5c534f3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:49:02 -0700 Subject: [PATCH 016/122] [pre-commit.ci] pre-commit autoupdate (#6644) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.15.21 → v0.15.22](https://github.com/astral-sh/ruff-pre-commit/compare/v0.15.21...v0.15.22) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3e4de2471..f3b926357 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,7 +62,7 @@ repos: args: [--py310-plus] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.21 + rev: v0.15.22 hooks: - id: ruff-check args: [--fix, --exit-non-zero-on-fix] From d099a933bf9a458c400d06a18c4a2055deb56829 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Mon, 20 Jul 2026 11:00:28 -0700 Subject: [PATCH 017/122] Remove ConcreteBackerMixin (#6643) --- angr/storage/memory_mixins/__init__.py | 4 +- .../paged_memory/page_backer_mixins.py | 44 ------------------- 2 files changed, 1 insertion(+), 47 deletions(-) diff --git a/angr/storage/memory_mixins/__init__.py b/angr/storage/memory_mixins/__init__.py index b04333cae..602651818 100644 --- a/angr/storage/memory_mixins/__init__.py +++ b/angr/storage/memory_mixins/__init__.py @@ -16,7 +16,7 @@ from .keyvalue_memory_mixin import KeyValueMemoryMixin from .label_merger_mixin import LabelMergerMixin from .multi_value_merger_mixin import MultiValueMergerMixin from .name_resolution_mixin import NameResolutionMixin -from .paged_memory.page_backer_mixins import ClemoryBackerMixin, ConcreteBackerMixin, DictBackerMixin +from .paged_memory.page_backer_mixins import ClemoryBackerMixin, DictBackerMixin from .paged_memory.paged_memory_mixin import ( ListPagesMixin, ListPagesWithLabelsMixin, @@ -81,7 +81,6 @@ class DefaultMemory( DirtyAddrsMixin, # ----- StackAllocationMixin, - ConcreteBackerMixin, ClemoryBackerMixin, DictBackerMixin, PrivilegedPagingMixin, @@ -251,7 +250,6 @@ __all__ = ( "ActionsMixinLow", "AddressConcretizationMixin", "ClemoryBackerMixin", - "ConcreteBackerMixin", "ConditionalMixin", "ConvenientMappingsMixin", "CooperationBase", diff --git a/angr/storage/memory_mixins/paged_memory/page_backer_mixins.py b/angr/storage/memory_mixins/paged_memory/page_backer_mixins.py index 9858a7e99..345a7ff35 100644 --- a/angr/storage/memory_mixins/paged_memory/page_backer_mixins.py +++ b/angr/storage/memory_mixins/paged_memory/page_backer_mixins.py @@ -183,50 +183,6 @@ class ClemoryBackerMixin(PagedMemoryMixin): return out -class ConcreteBackerMixin(ClemoryBackerMixin): - def _initialize_page(self, pageno, permissions=None, *, force_default=False, **kwargs): - if self._clemory_backer is None or force_default: - return super()._initialize_page(pageno, permissions=permissions, **kwargs) - - addr = pageno * self.page_size - - try: - backer_iter = self._clemory_backer.backers(addr) - backer_start, _backer = next(backer_iter) - except StopIteration: - return super()._initialize_page(pageno, permissions=permissions, **kwargs) - - if backer_start >= addr + self.page_size: - return super()._initialize_page(pageno, permissions=permissions, **kwargs) - - if self.state.project.concrete_target: - l.debug("Fetching data from concrete target") - data = claripy.BVV( - bytearray(self.state.project.concrete_target.read_memory(pageno * self.page_size, self.page_size)), - self.page_size * 8, - ) - else: - # the concrete backer only is here to support concrete loading, defer back to the CleMemoryBacker - return super()._initialize_page(pageno, permissions=permissions, **kwargs) - - permissions = self._cle_permissions_lookup(addr) - - # see if this page supports creating without copying - if type(data) is NotMemoryview: - try: - new_from_shared = self.PAGE_TYPE.new_from_shared - except AttributeError: - data = claripy.BVV(bytes(data[:])) - else: - return new_from_shared(data, **self._page_kwargs(pageno, permissions)) - - new_page = PagedMemoryMixin._initialize_default_page(self, pageno, permissions=permissions, **kwargs) - new_page.store( - 0, data, size=self.page_size, page_addr=pageno * self.page_size, endness="Iend_BE", memory=self, **kwargs - ) - return new_page - - class DictBackerMixin(PagedMemoryMixin): def __init__(self, dict_memory_backer=None, **kwargs): super().__init__(**kwargs) From dd1cefa64277a2700cfd62caa889a90da9408757 Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Tue, 21 Jul 2026 02:15:12 -0400 Subject: [PATCH 018/122] Handle case-insensitive P-code memory space names in AIL conversion (#6654) * Handle uppercase P-code memory spaces * Fix P-code regression test typing --- angr/ailment/converter_pcode.py | 4 ++-- tests/ailment/test_irsb.py | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/angr/ailment/converter_pcode.py b/angr/ailment/converter_pcode.py index 36ac5039e..dbc714299 100644 --- a/angr/ailment/converter_pcode.py +++ b/angr/ailment/converter_pcode.py @@ -329,7 +329,7 @@ class PCodeIRSBConverter(Converter): return Convert(self._manager.next_atom(), t.bits, size, False, t, ins_addr=self._manager.ins_addr) return Tmp(self._manager.next_atom(), offset, size) - if space_name in ["ram", "mem"]: + if space_name.lower() in ["ram", "mem"]: assert not is_write addr = Const(self._manager.next_atom(), varnode.offset, self._manager.arch.bits) # Note: Load takes bytes, not bits, for size @@ -359,7 +359,7 @@ class PCodeIRSBConverter(Converter): return Assignment( self._statement_idx, self._convert_varnode(varnode, True), value, ins_addr=self._manager.ins_addr ) - if space_name in ["ram", "mem"]: + if space_name.lower() in ["ram", "mem"]: addr = Const(self._manager.next_atom(), varnode.offset, self._manager.arch.bits) return Store( self._statement_idx, diff --git a/tests/ailment/test_irsb.py b/tests/ailment/test_irsb.py index d0b7d6c7c..714261402 100644 --- a/tests/ailment/test_irsb.py +++ b/tests/ailment/test_irsb.py @@ -6,6 +6,7 @@ import pickle import unittest import archinfo +import pypcode import pyvex from pyvex.enums import irop_enums_to_ints @@ -41,6 +42,32 @@ class TestIrsb(unittest.TestCase): ablock = ailment.IRSBConverter.convert(irsb, manager) assert ablock # TODO: test if this conversion is valid + def test_convert_pcode_uppercase_memory_space(self): + arch = archinfo.ArchPcode("6502:LE:16:default") + manager = ailment.Manager(arch=arch) # pyright: ignore[reportArgumentType] + translation = pypcode.Context(arch.name).translate(bytes.fromhex("ad34128d7856"), base_address=0) + load_varnode = translation.ops[1].inputs[0] + store_varnode = translation.ops[5].output + assert load_varnode is not None + assert store_varnode is not None + assert load_varnode.space.name == store_varnode.space.name == "RAM" + + converter = object.__new__(ailment.PCodeIRSBConverter) + converter._manager = manager + converter._statement_idx = 0 + + load = converter._get_value(load_varnode) + store = converter._set_value(store_varnode, ailment.Expr.Const(None, 0xAA, 8)) + + assert isinstance(load, ailment.Expr.Load) + assert isinstance(load.addr, ailment.Expr.Const) + assert load.addr.value == 0x1234 + assert load.size == 1 + assert isinstance(store, ailment.Stmt.Store) + assert isinstance(store.addr, ailment.Expr.Const) + assert store.addr.value == 0x5678 + assert store.size == 1 + def test_lift_path_matches_python_path(self): """The direct libVEX-lift fast path must produce the same AIL block as converting a cached pyvex Python IRSB.""" From 0fcf7c050b5aba4a028aa3907dd1e0005a182448 Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Tue, 21 Jul 2026 02:17:11 -0400 Subject: [PATCH 019/122] Copy stateful address concretization strategies on state fork (#6653) * Copy stateful address concretization strategies * Fix address strategy test typing --- .../address_concretization_mixin.py | 22 ++++++++++- tests/storage/test_address_concretization.py | 37 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 tests/storage/test_address_concretization.py diff --git a/angr/storage/memory_mixins/address_concretization_mixin.py b/angr/storage/memory_mixins/address_concretization_mixin.py index 2aaa05bd6..e14225174 100644 --- a/angr/storage/memory_mixins/address_concretization_mixin.py +++ b/angr/storage/memory_mixins/address_concretization_mixin.py @@ -66,8 +66,26 @@ class AddressConcretizationMixin(MemoryMixin): @MemoryMixin.memo def copy(self, memo): o = super().copy(memo) - o.read_strategies = list(self.read_strategies) - o.write_strategies = list(self.write_strategies) + + # Concretization strategies are usually stateless and their ``copy()`` method + # returns ``self``. Some strategies do carry per-state data, however (for + # example, the no-repeat strategies), and explicitly implement ``copy()`` so + # that forks do not mutate one another. Merely copying the two lists leaves the + # strategy instances shared and makes that API ineffective. + # + # A strategy may intentionally be installed for both reads and writes. Preserve + # that alias within the new memory while still separating it from the source + # memory by copying each distinct source strategy exactly once. + copied_strategies = {} + + def copy_strategy(strategy): + key = id(strategy) + if key not in copied_strategies: + copied_strategies[key] = strategy.copy() + return copied_strategies[key] + + o.read_strategies = [copy_strategy(s) for s in self.read_strategies] + o.write_strategies = [copy_strategy(s) for s in self.write_strategies] return o def merge(self, others, merge_conditions, common_ancestor=None) -> bool: diff --git a/tests/storage/test_address_concretization.py b/tests/storage/test_address_concretization.py new file mode 100644 index 000000000..dd31753a4 --- /dev/null +++ b/tests/storage/test_address_concretization.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import archinfo + +import angr + + +class _StatefulStrategy(angr.concretization_strategies.SimConcretizationStrategy): + def __init__(self, values=None): + super().__init__() + self.values = [] if values is None else values + + def _concretize(self, memory, addr, **kwargs): + return None + + def copy(self): + return _StatefulStrategy(list(self.values)) + + +def test_stateful_concretization_strategies_are_copied_with_memory(): + state = angr.SimState(arch=archinfo.ArchAMD64()) + strategy = _StatefulStrategy([1]) + state.memory.read_strategies = [strategy, strategy] + state.memory.write_strategies = [strategy] + + fork = state.copy() + assert fork.memory.read_strategies is not None + assert fork.memory.write_strategies is not None + fork_strategy = fork.memory.read_strategies[0] + + assert fork_strategy is not strategy + assert fork.memory.read_strategies[1] is fork_strategy + assert fork.memory.write_strategies[0] is fork_strategy + + fork_strategy.values.append(2) + assert strategy.values == [1] + assert fork_strategy.values == [1, 2] From a3c8d835c79d083b61489026d7643fcd8fdc382e Mon Sep 17 00:00:00 2001 From: Vedant Soni <83280635+tedanvosin@users.noreply.github.com> Date: Mon, 20 Jul 2026 23:17:44 -0700 Subject: [PATCH 020/122] RustStructuredCodeGenerator : align __init__ with CStructuredCodeGenerator.__init__ (#6650) * Align RustStructuredCodeGenerator.__init__ with CStructuredCodeGenerator * type fix --- .../decompiler/structured_codegen/rust.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/angr/analyses/decompiler/structured_codegen/rust.py b/angr/analyses/decompiler/structured_codegen/rust.py index 1a06ca187..12c45f304 100644 --- a/angr/analyses/decompiler/structured_codegen/rust.py +++ b/angr/analyses/decompiler/structured_codegen/rust.py @@ -2756,10 +2756,16 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): min_data_addr: int = 0x400_000, notes=None, display_notes: bool = True, - variable_map: VariableMap | None = None, + max_str_len: int | None = None, + prettify_thiscall: bool = False, + cstyle_void_param: bool = True, indent_size: int = INDENT_DELTA, + variable_map: VariableMap | None = None, ): - super().__init__(flavor=flavor, notes=notes) + super().__init__( + flavor=flavor, + notes=notes, + ) self._handlers = { CodeNode: self._handle_Code, @@ -2834,9 +2840,14 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): self.externs = externs or set() self.show_externs = show_externs self.show_demangled_name = show_demangled_name + self.show_disambiguated_name = show_disambiguated_name self.ail_graph = ail_graph self.simplify_else_scope = simplify_else_scope self.cstyle_ifs = cstyle_ifs + self.omit_func_header = omit_func_header + self.display_block_addrs = display_block_addrs + self.display_vvar_ids = display_vvar_ids + self.min_data_addr = min_data_addr self.text = None self.map_pos_to_node = None self.map_pos_to_addr = None @@ -2846,6 +2857,9 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): self.rust_func: RustFunction | None = None self.cexterns: set[RustVariable] | None = None self.display_notes = display_notes + self.max_str_len = max_str_len + self.prettify_thiscall = prettify_thiscall + self.cstyle_void_param = cstyle_void_param self.indent_delta = indent_size self._analyze() @@ -2872,6 +2886,8 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): self.simplify_else_scope = value elif option.param == "cstyle_ifs": self.cstyle_ifs = value + elif option.param == "cstyle_void_param": + self.cstyle_void_param = value elif option.param == "indent_size": self.indent_delta = value From 8ad0dc91aabec19c5cbe3a5c8350bf8b38f7f9bf Mon Sep 17 00:00:00 2001 From: Ati Priya Date: Mon, 20 Jul 2026 23:19:14 -0700 Subject: [PATCH 021/122] Decompiler: Recover CondNL (jge) over SUB and LOGIC in the amd64 ccall rewriter (#6657) --- .../ccall_rewriters/amd64_ccalls.py | 49 ++++++ .../decompiler/test_ccall_rewriting.py | 151 ++++++++++++++++++ 2 files changed, 200 insertions(+) diff --git a/angr/analyses/decompiler/ccall_rewriters/amd64_ccalls.py b/angr/analyses/decompiler/ccall_rewriters/amd64_ccalls.py index 5b003d1da..0cae1be51 100644 --- a/angr/analyses/decompiler/ccall_rewriters/amd64_ccalls.py +++ b/angr/analyses/decompiler/ccall_rewriters/amd64_ccalls.py @@ -319,6 +319,55 @@ class AMD64CCallRewriter(CCallRewriterBase): r = Expr.BinaryOp(ccall.idx, "CmpLT", (dep_1, zero), True, **ccall.tags) return Expr.Convert(self.ail_manager.next_atom(), r.bits, ccall.bits, False, r, **ccall.tags) + elif cond_v == AMD64_CondTypes["CondNL"]: + if op_v in { + AMD64_OpTypes["G_CC_OP_SUBB"], + AMD64_OpTypes["G_CC_OP_SUBW"], + AMD64_OpTypes["G_CC_OP_SUBL"], + AMD64_OpTypes["G_CC_OP_SUBQ"], + }: + # CondNL (jge) is SF == OF, i.e. dep_1 >=s dep_2 + + dep_1 = self._fix_size( + dep_1, + op_v, + AMD64_OpTypes["G_CC_OP_SUBB"], + AMD64_OpTypes["G_CC_OP_SUBW"], + AMD64_OpTypes["G_CC_OP_SUBL"], + ccall.tags, + ) + dep_2 = self._fix_size( + dep_2, + op_v, + AMD64_OpTypes["G_CC_OP_SUBB"], + AMD64_OpTypes["G_CC_OP_SUBW"], + AMD64_OpTypes["G_CC_OP_SUBL"], + ccall.tags, + ) + + r = Expr.BinaryOp(ccall.idx, "CmpGE", (dep_1, dep_2), True, **ccall.tags) + return Expr.Convert(self.ail_manager.next_atom(), r.bits, ccall.bits, False, r, **ccall.tags) + + if op_v in { + AMD64_OpTypes["G_CC_OP_LOGICB"], + AMD64_OpTypes["G_CC_OP_LOGICW"], + AMD64_OpTypes["G_CC_OP_LOGICL"], + AMD64_OpTypes["G_CC_OP_LOGICQ"], + }: + # and/or/xor clear OF, so CondNL = SF == 0, i.e. the result dep_1 >=s 0 + + dep_1 = self._fix_size( + dep_1, + op_v, + AMD64_OpTypes["G_CC_OP_LOGICB"], + AMD64_OpTypes["G_CC_OP_LOGICW"], + AMD64_OpTypes["G_CC_OP_LOGICL"], + ccall.tags, + ) + zero = Expr.Const(self.ail_manager.next_atom(), 0, dep_1.bits) + r = Expr.BinaryOp(ccall.idx, "CmpGE", (dep_1, zero), True, **ccall.tags) + return Expr.Convert(self.ail_manager.next_atom(), r.bits, ccall.bits, False, r, **ccall.tags) + elif cond_v == AMD64_CondTypes["CondNBE"]: if op_v in { AMD64_OpTypes["G_CC_OP_SUBB"], diff --git a/tests/analyses/decompiler/test_ccall_rewriting.py b/tests/analyses/decompiler/test_ccall_rewriting.py index b29be5295..5b682c8c5 100644 --- a/tests/analyses/decompiler/test_ccall_rewriting.py +++ b/tests/analyses/decompiler/test_ccall_rewriting.py @@ -4,14 +4,24 @@ from __future__ import annotations __package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin +import itertools import os import unittest +import claripy + import angr +from angr.ailment import Expr, Manager +from angr.analyses.decompiler.ccall_rewriters.amd64_ccalls import AMD64CCallRewriter +from angr.engines.vex.claripy.ccall import data, pc_calculate_condition from tests.common import bin_location, print_decompilation_result test_location = os.path.join(bin_location, "tests") +AMD64_CondTypes = data["AMD64"]["CondTypes"] +AMD64_OpTypes = data["AMD64"]["OpTypes"] +AMD64_CondBitMasks = data["AMD64"]["CondBitMasks"] + class TestCCallRewriting(unittest.TestCase): def test_NtGetCurrentPeb(self): @@ -32,5 +42,146 @@ class TestCCallRewriting(unittest.TestCase): assert "v0 = NtGetCurrentPeb();" in dec.codegen.text +def _make_ccall(cond, op, dep1=None, dep2=None, ndep=None, bits=64): + """Build a VEXCCallExpression for amd64g_calculate_condition.""" + if dep1 is None: + dep1 = Expr.Register(1, 16, 64) # rax + if dep2 is None: + dep2 = Expr.Register(2, 24, 64) # rcx + if ndep is None: + ndep = Expr.Const(3, 0, 64) + return Expr.VEXCCallExpression( + idx=0, + callee="amd64g_calculate_condition", + operands=(Expr.Const(0, cond, 64), Expr.Const(0, op, 64), dep1, dep2, ndep), + bits=bits, + ) + + +_PROJECT = angr.load_shellcode(b"\x90", arch="AMD64") + + +def _rewrite(ccall): + return AMD64CCallRewriter(ccall, _PROJECT, Manager(arch=_PROJECT.arch)).result + + +def _unwrap_convert(expr): + """Strip an outer Convert wrapper if present.""" + return expr.operand if isinstance(expr, Expr.Convert) else expr + + +def _mask(v, bits): + return v & ((1 << bits) - 1) + + +def _sext(v, bits): + v = _mask(v, bits) + return v - (1 << bits) if v >> (bits - 1) else v + + +def _eval(expr): + """Concretely evaluate a rewritten (constant-folded) AIL expression. Returns (value, bits).""" + if isinstance(expr, Expr.Const): + return _mask(expr.value_int, expr.bits), expr.bits + if isinstance(expr, Expr.Convert): + v, _ = _eval(expr.operand) + v = _mask(_sext(v, expr.from_bits) if expr.is_signed else v, expr.to_bits) + return v, expr.to_bits + if isinstance(expr, Expr.Call) and expr.target == "__CFADD__": + # carry-out of the addition at the operands' width + left, lbits = _eval(expr.args[0]) + right, rbits = _eval(expr.args[1]) + bits = max(lbits, rbits) + return int(_mask(left + right, bits) < left), expr.bits + if isinstance(expr, Expr.BinaryOp): + left, lbits = _eval(expr.operands[0]) + right, rbits = _eval(expr.operands[1]) + bits = max(lbits, rbits) + if expr.signed: + left, right = _sext(left, lbits), _sext(right, rbits) + cmps = { + "CmpEQ": lambda: (int(left == right), 1), + "CmpNE": lambda: (int(left != right), 1), + "CmpLT": lambda: (int(left < right), 1), + "CmpLE": lambda: (int(left <= right), 1), + "CmpGT": lambda: (int(left > right), 1), + "CmpGE": lambda: (int(left >= right), 1), + "And": lambda: (_mask(left & right, bits), bits), + "Add": lambda: (_mask(left + right, bits), bits), + } + if expr.op not in cmps: + raise NotImplementedError(expr.op) + return cmps[expr.op]() + raise NotImplementedError(type(expr)) + + +def _oracle(cond, op, dep1, dep2, ndep=0): + """Ground truth: ccall.py's executable amd64g_calculate_condition.""" + r = pc_calculate_condition( + None, + claripy.BVV(cond, 64), + claripy.BVV(op, 64), + claripy.BVV(dep1, 64), + claripy.BVV(dep2, 64), + claripy.BVV(ndep, 64), + platform="AMD64", + ) + return bool(claripy.backends.concrete.eval(r, 1)[0]) + + +def _rewritten_value(cond, op, dep1, dep2, ndep=0): + ccall = _make_ccall(cond, op, Expr.Const(1, dep1, 64), Expr.Const(2, dep2, 64), Expr.Const(3, ndep, 64)) + result = _rewrite(ccall) + assert result is not None + return bool(_eval(result)[0]) + + +class TestAMD64CCallRewriterCondNL(unittest.TestCase): + """CondNL (jge, SF == OF). Signed >= over SUB; sign-of-result >= 0 over LOGIC.""" + + def test_condnl_sub_is_signed_ge(self): + for op in ("G_CC_OP_SUBB", "G_CC_OP_SUBW", "G_CC_OP_SUBL", "G_CC_OP_SUBQ"): + cmp = _unwrap_convert(_rewrite(_make_ccall(AMD64_CondTypes["CondNL"], AMD64_OpTypes[op]))) + assert isinstance(cmp, Expr.BinaryOp), f"{op}: not rewritten" + assert cmp.op == "CmpGE", f"{op}: got {cmp.op}" + assert cmp.signed is True, f"{op}: expected signed" + + def test_condnl_logic_is_signed_ge_zero(self): + for op in ("G_CC_OP_LOGICB", "G_CC_OP_LOGICW", "G_CC_OP_LOGICL", "G_CC_OP_LOGICQ"): + cmp = _unwrap_convert(_rewrite(_make_ccall(AMD64_CondTypes["CondNL"], AMD64_OpTypes[op]))) + assert isinstance(cmp, Expr.BinaryOp), f"{op}: not rewritten" + assert cmp.op == "CmpGE", f"{op}: got {cmp.op}" + assert cmp.signed is True, f"{op}: expected signed" + assert cmp.operands[1].value_int == 0, f"{op}: expected comparison against 0" + + +_DEP2_SAMPLE = (0, 1, 2, 3, 0x7E, 0x7F, 0x80, 0x81, 0xFD, 0xFE, 0xFF, 0x55) + + +class TestAMD64CCallRewriterDifferential(unittest.TestCase): + """Differential-test 8-bit cells against ccall.py's executable semantics.""" + + # VEX only guarantees the low nbits of the deps; the rewriter must ignore anything above them + _DIRTY = 0xDEADBEEF_00000100 + + def _sweep(self, cond_name, op_name): + cond, op = AMD64_CondTypes[cond_name], AMD64_OpTypes[op_name] + for dep1, dep2 in itertools.product(range(256), _DEP2_SAMPLE): + got = _rewritten_value(cond, op, dep1, dep2) + want = _oracle(cond, op, dep1, dep2) + assert got == want, f"{cond_name} x {op_name} dep1={dep1:#x} dep2={dep2:#x}: {got} != {want}" + if dep1 % 8 == 0: + d1, d2 = dep1 | self._DIRTY, dep2 | self._DIRTY + got = _rewritten_value(cond, op, d1, d2) + want = _oracle(cond, op, d1, d2) + assert got == want, f"{cond_name} x {op_name} dep1={d1:#x} dep2={d2:#x}: {got} != {want}" + + def test_condnl_subb_differential(self): + self._sweep("CondNL", "G_CC_OP_SUBB") + + def test_condnl_logicb_differential(self): + self._sweep("CondNL", "G_CC_OP_LOGICB") + + if __name__ == "__main__": unittest.main() From 0c872298376c91dbc514de7725d270edf8eedd16 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Mon, 20 Jul 2026 23:22:14 -0700 Subject: [PATCH 022/122] Pass new ret_expr during construction rather than mutating (#6642) * Pass new ret_expr during construction rather than mutating * More improvements. --------- Co-authored-by: Fish --- .../optimization_passes/ret_expr_rewriter.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/angr/rust/optimization_passes/ret_expr_rewriter.py b/angr/rust/optimization_passes/ret_expr_rewriter.py index 7028d2bde..c2d7ca2f4 100644 --- a/angr/rust/optimization_passes/ret_expr_rewriter.py +++ b/angr/rust/optimization_passes/ret_expr_rewriter.py @@ -1,17 +1,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING - from angr.ailment import Const, Register -from angr.ailment.expression import ComboRegister +from angr.ailment.expression import Call, ComboRegister +from angr.ailment.statement import SideEffectStatement from angr.analyses.decompiler.optimization_passes.optimization_pass import OptimizationPass, OptimizationPassStage from angr.calling_conventions import SimFunctionArgument, SimRegArg, SimStructArg from .utils import SideEffectStatementRewriter -if TYPE_CHECKING: - from angr.ailment.statement import SideEffectStatement - class RetExprRewriter(OptimizationPass): """Rewrite return expressions for functions returning struct via multiple registers.""" @@ -39,9 +35,13 @@ class RetExprRewriter(OptimizationPass): def _analyze(self, cache=None): def callback(call_stmt: SideEffectStatement, _block, _stmt): - if isinstance(call_stmt.expr.target, Const) and call_stmt.expr.target.value in self.kb.functions: - func = self.kb.functions[call_stmt.expr.target.value] - if func.prototype and func.calling_convention and func.prototype.returnty: + if ( + isinstance(call_stmt.expr, Call) + and isinstance(call_stmt.expr.target, Const) + and self.kb.functions.contains_addr(call_stmt.expr.target.value_int) + ): + func = self.kb.functions.get_by_addr(call_stmt.expr.target.value_int, meta_only=True) + if func.prototype is not None and func.calling_convention and func.prototype.returnty: ret_val = func.calling_convention.return_val(func.prototype.returnty) ret_locs = self._flatten_locs(ret_val) # pyright: ignore[reportArgumentType] if ( @@ -61,9 +61,9 @@ class RetExprRewriter(OptimizationPass): ) regs.append(reg) ret_expr = ComboRegister(self.manager.next_atom(), regs) - new_call = call_stmt.copy() - new_call.ret_expr = ret_expr - return new_call + return SideEffectStatement( + call_stmt.idx, call_stmt.expr, ret_expr, call_stmt.fp_ret_expr, **call_stmt.tags + ) return call_stmt rewriter = SideEffectStatementRewriter(callback) From 26736ec0963909140ad25960ff6f9316c20af519 Mon Sep 17 00:00:00 2001 From: Ati Priya Date: Mon, 20 Jul 2026 23:33:10 -0700 Subject: [PATCH 023/122] decompiler: ExpressionNarrower over-narrows vvars used as Insert bases (#6649) EffectiveSizeExtractor skipped the base operand of an Insert, so a use as an Insert base contributed no width requirement in AILSimplifier's narrowing pass. A register vvar whose remaining uses were narrow (e.g. an ah-style byte Extract) was then narrowed below the Insert base width and zero-extended back at the use site, destroying every preserved byte of the base and misplacing the extracted byte: v1 = a0->field_10; // v1 narrowed to char a0->field_10 = _INSERT(v1, 1, v1 & 239); // upper 3 bytes zeroed, // wrong byte masked for what is really a full-width read-modify-write (field_10 &= 0xffffefff). Walk the Insert base in EffectiveSizeExtractor so it is recorded as a full-width use: every byte outside the inserted range is preserved into the result, so the base can never be narrowed below its own width. --- .../decompiler/expression_narrower.py | 4 ++- .../decompiler/test_narrowing_exprs.py | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/angr/analyses/decompiler/expression_narrower.py b/angr/analyses/decompiler/expression_narrower.py index 32eded63a..646defecf 100644 --- a/angr/analyses/decompiler/expression_narrower.py +++ b/angr/analyses/decompiler/expression_narrower.py @@ -122,7 +122,9 @@ class EffectiveSizeExtractor(AILBlockWalker[None, None, None]): super()._handle_expr(expr_idx, expr, stmt_idx, stmt, block) def _handle_Insert(self, expr_idx: int, expr, stmt_idx: int, stmt: Statement | None, block: Block | None): - # self._handle_expr(0, expr.base, stmt_idx, stmt, block) + # the base of an Insert is consumed at full width: every byte outside the inserted range is preserved + # into the result, so narrowing the base (and zero-extending it back) would destroy those bytes + self._handle_expr(0, expr.base, stmt_idx, stmt, block) if isinstance(expr.base, VirtualVariable): self.vvars_used_as_insert_base.add(expr.base.varid) self._handle_expr(1, expr.offset, stmt_idx, stmt, block) diff --git a/tests/analyses/decompiler/test_narrowing_exprs.py b/tests/analyses/decompiler/test_narrowing_exprs.py index 49d8bd73f..212185ca3 100644 --- a/tests/analyses/decompiler/test_narrowing_exprs.py +++ b/tests/analyses/decompiler/test_narrowing_exprs.py @@ -9,6 +9,9 @@ import os import unittest import angr +from angr.ailment.expression import BinaryOp, Const, Extract, Insert, VirtualVariable, VirtualVariableCategory +from angr.ailment.statement import Assignment +from angr.analyses.decompiler.expression_narrower import EffectiveSizeExtractor from tests.common import WORKER, bin_location, print_decompilation_result test_location = os.path.join(bin_location, "tests") @@ -17,6 +20,30 @@ l = logging.Logger(__name__) class TestNarrowingExpressions(unittest.TestCase): + def test_insert_base_is_a_full_width_use(self): + # the base of an Insert is consumed at full width: every byte outside the inserted range is + # preserved into the result. EffectiveSizeExtractor used to skip the base entirely, so a vvar + # whose only other uses were narrow (e.g. an ah-style Extract) was narrowed below the width + # of the Insert base and zero-extended back, destroying the preserved bytes: + # v1 = a0->field_10; a0->field_10 = _INSERT(v1, 1, v1 & 239); // v1 narrowed to char + # instead of the full-width read-modify-write of field_10. + base = VirtualVariable(1, 44, 64, VirtualVariableCategory.REGISTER, oident=16) + ah_vvar = VirtualVariable(4, 44, 64, VirtualVariableCategory.REGISTER, oident=16) + ah_read = Extract(3, 8, ah_vvar, Const(5, 1, 64), "Iend_LE") + value = BinaryOp(2, "And", [ah_read, Const(6, 239, 8)], False, bits=8) + dst = VirtualVariable(7, 48, 64, VirtualVariableCategory.REGISTER, oident=16) + stmt = Assignment(9, dst, Insert(8, base, Const(10, 1, 64), value, "Iend_LE")) + + walker = EffectiveSizeExtractor() + walker.walk_statement(stmt) + + occurrences = walker.vvar_effective_bits[44] + # the Insert-base occurrence must be recorded as a full-width use... + assert occurrences[base.idx] == (0, 64) + # ...while the byte-1 Extract occurrence stays narrow + assert occurrences[ah_vvar.idx] == (8, 16) + assert 44 in walker.vvars_used_as_insert_base + def test_narrowing_expressions_after_making_callsite_only(self): # narrowing expressions before making callsites may incorrectly remove some definitions that the calls use # in this test case, the definition of ecx at block 0x4066E5 will be replaced by cl, but ecx is actually used From 95fa7ea003eb470c70d22681b6f446c09f88e3f1 Mon Sep 17 00:00:00 2001 From: volodya Date: Wed, 22 Jul 2026 10:41:36 +0200 Subject: [PATCH 024/122] utils/graph: answer subgraph_between_nodes reachability in one pass (#6662) subgraph_between_nodes() copied the whole graph and then ran a fresh networkx.has_path() search for every (candidate successor, frontier node) pair, so its worst case was O(candidate_edges * frontier_nodes * (V + E)). Negative queries are the expensive ones: a successor that cannot reach any frontier node forces a full traversal of everything reachable from it, once per frontier node. RegionIdentifier._find_initial_loop_nodes() calls this for every loop it recovers, so a loop head whose successors lead into a large region that only returns to the head (which the function's own "remove all incoming edges of the source" step makes unable to reach any latch) makes loop recovery quadratic. On a 1.1k-node AArch64 CFG built to have that shape, decompilation spends 40.5s of 52.5s inside 608,847 has_path() calls. Replace the repeated searches with a single reverse multi-source BFS from the frontier that stops at the source; membership in the resulting set answers every reachability question the forward walk asks. Stopping the reverse walk at the source is exactly equivalent to deleting all incoming edges of the source, so the graph copy is no longer needed either. Also peel dead leaves with a degree worklist instead of rescanning all nodes after each removal. Results are unchanged, including node/edge insertion order and edge attributes: 6,000 randomized fixed-seed comparisons (3,000 graphs x both include_frontier modes) against the previous implementation are identical, as is the decompiler output on the CFG above (52.5s -> 12.5s end to end, 0 has_path() calls). One behaviour change is deliberate: frontier is now turned into a set before the "source not in graph or any(node not in graph for node in frontier)" check rather than after it. In the old order an iterator argument was consumed by that check, so the subsequent set(frontier) was empty and the function silently sliced with no frontier at all. Every in-tree caller passes a list or a set, so no in-tree behaviour changes. --- angr/utils/graph.py | 68 +++++++++++++++++++++++---------------- tests/utils/test_graph.py | 48 ++++++++++++++++++++++++++- 2 files changed, 87 insertions(+), 29 deletions(-) diff --git a/angr/utils/graph.py b/angr/utils/graph.py index e4473002d..2cb9bbdcc 100644 --- a/angr/utils/graph.py +++ b/angr/utils/graph.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from collections import OrderedDict, defaultdict +from collections import OrderedDict, defaultdict, deque from collections.abc import Iterable, Iterator from typing import TYPE_CHECKING, Any, cast @@ -148,51 +148,63 @@ def subgraph_between_nodes[T]( :rtype: networkx.DiGraph """ - graph = networkx.DiGraph(graph) # make a copy - for pred in list(graph.predecessors(source)): - # make sure we cannot go from any other node to the source node - graph.remove_edge(pred, source) - - g0 = networkx.DiGraph() + frontier = set(frontier) if source not in graph or any(node not in graph for node in frontier): raise KeyError("Source node or frontier nodes are not in the source graph.") + # Precompute, with a single reverse multi-source traversal, the set of nodes that can reach any frontier node. + # Not expanding the predecessors of the source is exactly equivalent to removing every incoming edge of the + # source before the search, which is what this function used to do on a full copy of the graph. + reaches_frontier = set(frontier) + reverse_queue = deque(frontier) + while reverse_queue: + node = reverse_queue.popleft() + if node == source: + continue + for pred in graph.predecessors(node): + if pred not in reaches_frontier: + reaches_frontier.add(pred) + reverse_queue.append(pred) + + g0 = networkx.DiGraph() + # BFS on graph and add new nodes to g0 - queue = [source] + queue = deque([source]) traversed = set() - frontier = set(frontier) - while queue: - node = queue.pop(0) + node = queue.popleft() traversed.add(node) for _, succ, data in graph.out_edges(node, data=True): - if g0.has_edge(node, succ): + if succ == source or g0.has_edge(node, succ): continue g0.add_edge(node, succ, **data) if succ in traversed or succ in frontier: continue - for frontier_node in frontier: - if networkx.has_path(graph, succ, frontier_node): - queue.append(succ) - break + if succ in reaches_frontier: + queue.append(succ) - # recursively remove all nodes that have less than two neighbors - to_remove = [ - n - for n in g0.nodes() - if n not in frontier and n is not source and (g0.out_degree[n] == 0 or g0.in_degree[n] == 0) - ] + # recursively remove all nodes that have less than two neighbors, using a degree worklist instead of rescanning + # the whole graph after every removal + def _removable(n) -> bool: + return n not in frontier and n is not source and (g0.out_degree[n] == 0 or g0.in_degree[n] == 0) + + to_remove = deque(n for n in g0 if _removable(n)) + queued = set(to_remove) while to_remove: - g0.remove_nodes_from(to_remove) - to_remove = [ - n - for n in g0.nodes() - if n not in frontier and n is not source and (g0.out_degree[n] == 0 or g0.in_degree[n] == 0) - ] + node = to_remove.popleft() + queued.discard(node) + if node not in g0 or not _removable(node): + continue + neighbors = [*g0.predecessors(node), *g0.successors(node)] + g0.remove_node(node) + for neighbor in neighbors: + if neighbor in g0 and neighbor not in queued and _removable(neighbor): + queued.add(neighbor) + to_remove.append(neighbor) if not include_frontier: # remove the frontier nodes diff --git a/tests/utils/test_graph.py b/tests/utils/test_graph.py index a11ef9845..1089e0b9b 100644 --- a/tests/utils/test_graph.py +++ b/tests/utils/test_graph.py @@ -3,11 +3,12 @@ from __future__ import annotations import unittest +import unittest.mock import networkx as nx from angr.ailment.block import Block -from angr.utils.graph import Dominators, GraphUtils, TemporaryNode +from angr.utils.graph import Dominators, GraphUtils, TemporaryNode, subgraph_between_nodes class TestGraph(unittest.TestCase): @@ -60,6 +61,51 @@ class TestGraph(unittest.TestCase): G_sorted = GraphUtils.quasi_topological_sort_nodes(G, panic_mode_threshold=num_nodes // 2) assert G_sorted == nodes + def test_subgraph_between_nodes_basic(self): + G = nx.DiGraph() + G.add_edge("head", "a", weight=1) + G.add_edge("a", "b") + G.add_edge("b", "latch") + G.add_edge("head", "dead_end") + G.add_edge("latch", "head") + + g0 = subgraph_between_nodes(G, "head", ["latch"]) + assert set(g0.nodes) == {"head", "a", "b"} + assert set(g0.edges) == {("head", "a"), ("a", "b")} + assert g0.edges["head", "a"]["weight"] == 1 + + g1 = subgraph_between_nodes(G, "head", ["latch"], include_frontier=True) + assert set(g1.nodes) == {"head", "a", "b", "latch"} + assert set(g1.edges) == {("head", "a"), ("a", "b"), ("b", "latch")} + + def test_subgraph_between_nodes_does_not_explore_unreachable_region(self): + # A loop head with a direct latch edge and a second edge into a large side region that only loops back to + # the head. Since all incoming edges of the source are ignored, the side region cannot reach the latch, and + # proving that must not cost a traversal of the side region per (candidate, frontier) pair. + side_nodes = 20000 + G = nx.DiGraph() + head, latch = side_nodes, side_nodes + 1 + G.add_edge(head, latch) + G.add_edge(head, 0) + G.add_edges_from((n, n + 1) for n in range(side_nodes - 1)) + G.add_edge(side_nodes - 1, 0) + G.add_edge(side_nodes - 1, head) + + has_path_calls = 0 + real_has_path = nx.has_path + + def counting_has_path(*args, **kwargs): + nonlocal has_path_calls + has_path_calls += 1 + return real_has_path(*args, **kwargs) + + with unittest.mock.patch.object(nx, "has_path", counting_has_path): + g0 = subgraph_between_nodes(G, head, [latch], include_frontier=True) + + assert list(g0.nodes) == [head, latch] + assert list(g0.edges) == [(head, latch)] + assert has_path_calls == 0 + if __name__ == "__main__": unittest.main() From 1102d5fc474ce5543761824cd87fae54b110a6f3 Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 22 Jul 2026 01:48:02 -0700 Subject: [PATCH 025/122] RustSimTypeInt: Include size in equality and hashing; fix copy() dropping size (#6659) Fixes #6625. --- angr/rust/sim_type.py | 6 ++++++ tests/analyses/test_rust_sim_type.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/angr/rust/sim_type.py b/angr/rust/sim_type.py index d6b1362ca..8cf3cbf0f 100644 --- a/angr/rust/sim_type.py +++ b/angr/rust/sim_type.py @@ -38,11 +38,17 @@ class RustSimType: class RustSimTypeInt(RustSimType, SimTypeInt): _ident = "rust_int" + # unlike SimTypeInt, the size is explicit and arch-independent, so it must participate in equality and hashing + _fields = (*SimTypeInt._fields, "_size") + _args = ("size", "signed", "label") def __init__(self, size=32, signed=True, label=None): super().__init__(signed, label) self._size = size + def copy(self): + return self.__class__(size=self._size, signed=self.signed, label=self.label).with_arch(self._arch) + def repr(self, name=None, full=0, memo=None, indent: int | None = 0): if name is None or len(name) == 0: return repr(self) diff --git a/tests/analyses/test_rust_sim_type.py b/tests/analyses/test_rust_sim_type.py index 60029aa84..40f5bdc9a 100644 --- a/tests/analyses/test_rust_sim_type.py +++ b/tests/analyses/test_rust_sim_type.py @@ -83,6 +83,25 @@ def test_rust_scalar_reference_and_array_repr_json_roundtrip(): assert fn.to_json()["variadic"] is True +def test_rust_int_equality_includes_size(): + # regression test for angr/angr#6625: ints of different sizes compared equal (and hashed equal), which made + # the declared type of unified variables depend on nondeterministic set iteration order + u8 = RustSimTypeInt(8, signed=False) + u32 = RustSimTypeInt(32, signed=False) + u64 = RustSimTypeInt(64, signed=False) + + assert u32 != u64 + assert u8 != u32 + assert hash(u32) != hash(u64) + assert u32 == RustSimTypeInt(32, signed=False) + assert hash(u32) == hash(RustSimTypeInt(32, signed=False)) + assert u32 != RustSimTypeInt(32, signed=True) + + # copy() must preserve the explicit size + assert u64.copy().size == 64 + assert u64.copy() == u64 + + def test_rust_struct_nested_field_lookup_and_json_roundtrip(): arch = archinfo.ArchAMD64() inner = RustSimStruct(OrderedDict({"value": RustSimTypeInt(16, signed=False)}), name="Inner", pack=True).with_arch( From 21cea3815f0a9f7a975fe98bf046bc3aac9b7819 Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Wed, 22 Jul 2026 01:50:12 -0700 Subject: [PATCH 026/122] Refine C++ symbol prototypes with machine ABI facts (#6652) * Refine C++ symbol prototypes with machine ABI facts * Preserve explicit C++ calling conventions * Use prebuilt C++ calling-convention fixture --- .../calling_convention/calling_convention.py | 60 ++++++++++++++++++- .../test_calling_convention_analysis.py | 14 +++++ 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/angr/analyses/calling_convention/calling_convention.py b/angr/analyses/calling_convention/calling_convention.py index c167588f2..d919bc889 100644 --- a/angr/analyses/calling_convention/calling_convention.py +++ b/angr/analyses/calling_convention/calling_convention.py @@ -45,6 +45,7 @@ from angr.sim_type import ( SimTypeInt128, SimTypeLongLong, SimTypePointer, + SimTypeReg, SimTypeShort, parse_cpp_file, ) @@ -183,13 +184,24 @@ class CallingConventionAnalysis(Analysis): assert self._function is not None + cpp_symbol_result: tuple[SimCC, SimTypeCppFunction, str | None] | None = None demangled_name = self._function.demangled_name if demangled_name != self._function.name: r_demangled = self._analyze_demangled_name(demangled_name) if r_demangled is not None: - self.cc, self.prototype, self.prototype_libname = r_demangled - self.proto_from_symbol = True - return + # Itanium names usually omit the return type, and a qualified name does + # not distinguish a namespace/static function from a non-static member. + # parse_cpp_file() consequently carries a possible-this placeholder. + # Do not let that incomplete declaration bypass callee/callsite analysis; + # refine it with machine facts below. Declarations that encode an explicit + # calling convention (such as Microsoft C++ symbols) remain authoritative. + demangled_cc, demangled_proto, demangled_libname = r_demangled + if isinstance(demangled_proto, SimTypeCppFunction) and demangled_proto.convention is None: + cpp_symbol_result = demangled_cc, demangled_proto, demangled_libname + else: + self.cc, self.prototype, self.prototype_libname = r_demangled + self.proto_from_symbol = True + return if self._function.is_simprocedure: hooker = self.project.hooked_by(self._function.addr) @@ -282,6 +294,9 @@ class CallingConventionAnalysis(Analysis): r = self._analyze_function() if r is None: l.warning("Cannot determine calling convention for %r.", self._function) + if cpp_symbol_result is not None: + self.cc, self.prototype, self.prototype_libname = cpp_symbol_result + self.proto_from_symbol = True else: # adjust prototype if needed cc, prototype = r @@ -296,9 +311,48 @@ class CallingConventionAnalysis(Analysis): else None ) + if cpp_symbol_result is not None and prototype is not None: + prototype = self._refine_cpp_symbol_prototype(prototype, cpp_symbol_result[1]) self.cc = cc self.prototype = prototype + @staticmethod + def _refine_cpp_symbol_prototype( + machine_proto: SimTypeFunction, symbol_proto: SimTypeCppFunction + ) -> SimTypeFunction: + """Merge encoded C++ types only where machine ABI arity disambiguates them. + + The parser's first pointer is a *possible* ``this``. If machine facts recover + one fewer arguments, the qualified name was a namespace/static function and the + placeholder is removed. If arity agrees it is retained. Any other disagreement + keeps the machine-derived arguments. A non-Bottom encoded template return may + refine the return type; ordinary Itanium names keep the machine-derived return. + """ + machine_args = tuple(machine_proto.args or ()) + symbol_args = tuple(symbol_proto.args or ()) + if len(symbol_args) == len(machine_args): + selected = symbol_args + elif len(symbol_args) == len(machine_args) + 1 and symbol_args and isinstance(symbol_args[0], SimTypePointer): + selected = symbol_args[1:] + else: + selected = machine_args + # Opaque C++ classes cannot be laid out by a calling convention. Preserve the + # machine-derived slot for those arguments; only scalar/reference or pointer + # types from the linkage name are safe refinements. + args = tuple( + sym if isinstance(sym, (SimTypeReg, SimTypePointer)) else machine + for machine, sym in zip(machine_args, selected) + ) + symbol_ret = symbol_proto.returnty + ret = ( + symbol_ret + if symbol_ret is not None + and not isinstance(symbol_ret, SimTypeBottom) + and isinstance(symbol_ret, (SimTypeReg, SimTypePointer)) + else machine_proto.returnty + ) + return SimTypeFunction(args, ret, variadic=machine_proto.variadic) + def _analyze_callsite_only(self): assert self.caller_func_addr is not None assert self.callsite_block_addr is not None diff --git a/tests/analyses/test_calling_convention_analysis.py b/tests/analyses/test_calling_convention_analysis.py index 176d330e0..5c4583f09 100755 --- a/tests/analyses/test_calling_convention_analysis.py +++ b/tests/analyses/test_calling_convention_analysis.py @@ -40,6 +40,20 @@ def cca_mode(modes: str): # pylint: disable=missing-class-docstring # pylint: disable=no-self-use class TestCallingConventionAnalysis(unittest.TestCase): + def test_itanium_qualified_free_function_does_not_gain_this(self): + """Machine facts must disambiguate namespace functions from members.""" + binary = os.path.join(test_location, "x86_64", "cpp_qualified_symbols.so") + project = angr.Project(binary, auto_load_libs=False) + cfg = project.analyses.CFGFast(normalize=True) + project.analyses.CompleteCallingConventions(recover_variables=True, cfg=cfg.model, analyze_callsites=True) + free = project.kb.functions["_ZN4demo10free_valueEv"] + assert free.prototype is not None + assert len(free.prototype.args) == 0 + assert isinstance(free.prototype.returnty, SimTypeInt) + member = project.kb.functions["_ZN4demo3Box3addEi"] + assert member.prototype is not None + assert len(member.prototype.args) == 2 + def _run_fauxware(self, arch, function_and_cc_list): binary_path = os.path.join(test_location, arch, "fauxware") fauxware = angr.Project(binary_path, auto_load_libs=False) From bcf5902d075b025c4df1c622c994eef8635a8b35 Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Wed, 22 Jul 2026 01:52:32 -0700 Subject: [PATCH 027/122] Keep entry jumps inside sized function symbols (#6651) * Keep entry jumps within sized function symbols * Narrow CFG regression symbol type * Use committed binary for CFG entry-jump regression --- angr/analyses/cfg/cfg_fast.py | 17 +++++++++++++++-- tests/analyses/cfg/test_cfgfast.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/angr/analyses/cfg/cfg_fast.py b/angr/analyses/cfg/cfg_fast.py index 19811c7fc..efa326fb8 100644 --- a/angr/analyses/cfg/cfg_fast.py +++ b/angr/analyses/cfg/cfg_fast.py @@ -3596,7 +3596,10 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): target_func_addr = node.function_address # case 2: if the source instruction is the first instruction of the current function, has only one branch # to the target address, and is a jump (Ijk_Boring, not a call), then the target address is likely the - # start of another function + # start of another function. A compiler may also begin a function with an + # unconditional jump to an internal loop guard (loop rotation). When the loader + # supplies a non-empty function symbol, its extent is stronger evidence than this + # tail-jump heuristic: keep a target inside that extent in the current function. if ( target_func_addr is None and len(src_node.instruction_addrs) == 1 @@ -3605,7 +3608,17 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): and all_successors is not None and len(all_successors) == 1 ): - target_func_addr = target_addr + current_symbol = self.project.loader.find_symbol(current_function_addr) + current_symbol_size = getattr(current_symbol, "size", 0) or 0 + target_is_inside_current_symbol = ( + current_symbol is not None + and current_symbol.is_function + and current_symbol.rebased_addr == current_function_addr + and current_symbol_size > 0 + and current_function_addr <= target_addr < current_function_addr + current_symbol_size + ) + if not target_is_inside_current_symbol: + target_func_addr = target_addr # last resort: the block probably belongs to the current function if target_func_addr is None: target_func_addr = current_function_addr diff --git a/tests/analyses/cfg/test_cfgfast.py b/tests/analyses/cfg/test_cfgfast.py index a0836b798..d9099d748 100755 --- a/tests/analyses/cfg/test_cfgfast.py +++ b/tests/analyses/cfg/test_cfgfast.py @@ -736,6 +736,21 @@ class TestCfgfast(unittest.TestCase): assert len(cfg.model.graph) == 2 + def test_entry_jump_within_function_symbol_is_not_tail_jump(self): + """Loop rotation may put an unconditional jump at a function's entry.""" + binary_path = os.path.join(test_location, "x86_64", "cfg_entry_jump_within_function") + proj = angr.Project(binary_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(normalize=True) + function_symbol = proj.loader.find_symbol("rotated_loop") + assert function_symbol is not None + function_addr = function_symbol.rebased_addr + entry = cfg.model.get_any_node(function_addr) + + assert entry is not None + assert len(entry.successors) == 1 + assert entry.successors[0].function_address == function_addr + assert entry.successors[0].addr in {node.addr for node in cfg.functions[function_addr].graph} + def test_starting_point_ordering(self): # project entry should always be first # so edge/path to unlabeled main function from _start From e8858b82cd6c307bc95bd6e8d6e8de864e135217 Mon Sep 17 00:00:00 2001 From: Ati Priya Date: Wed, 22 Jul 2026 02:28:25 -0700 Subject: [PATCH 028/122] typehoon: type pointer-to-array locals as pointers (#6620) * typehoon: type function-scope pointer-to-array locals as element pointers (T*) so they render as pointers, not arrays c_repr drops the "*" for pointer-to-array, so such locals were declared as arrays and assignments to them were invalid C. Flatten T (*)[N] to T * for function-scope variables; globals and plain arrays unchanged. * tests: accept element-pointer rendering in reverting-switch-lowering test Locals typed as element pointers render "ptr = p + 1;" instead of "ptr = &p[1];"; both are equivalent. Accept either form. --- angr/analyses/typehoon/typehoon.py | 17 ++++++++ angr/rust/typehoon/typehoon.py | 3 ++ tests/analyses/decompiler/test_decompiler.py | 4 +- tests/analyses/test_typehoon.py | 42 +++++++++++++++++++- 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/angr/analyses/typehoon/typehoon.py b/angr/analyses/typehoon/typehoon.py index eb3766aa0..5440598e7 100644 --- a/angr/analyses/typehoon/typehoon.py +++ b/angr/analyses/typehoon/typehoon.py @@ -132,10 +132,27 @@ class Typehoon(Analysis): else: the_type = type_candidates[0] + if func_addr != "global": + the_type = self._flatten_pointer_to_array(the_type, self.project.arch) + self.kb.variables[func_addr].set_variable_type( var, the_type, name=the_type.name if isinstance(the_type, SimStruct) else None ) + @staticmethod + def _flatten_pointer_to_array(ty: SimType, arch) -> SimType: + """ + Flatten a pointer-to-array type (``T(*)[N]``) into a pointer to the array's element type (``T*``). + + Function-scope variables hold pointer *values*; a pointer-to-array solution would be declared in C + with the pointer level dropped (see ``SimTypePointer.c_repr``), yielding an array declaration that is + not assignable. Only global variables, which denote the array objects themselves, may keep the + pointer-to-array form. + """ + while isinstance(ty, SimTypePointer) and isinstance(ty.pts_to, SimTypeArray): + ty = SimTypePointer(ty.pts_to.elem_type).with_arch(arch) + return ty + def pp_constraints(self) -> None: """ Pretty-print constraints between *variables* using the variable mapping. diff --git a/angr/rust/typehoon/typehoon.py b/angr/rust/typehoon/typehoon.py index af2c71d7c..70fc92802 100644 --- a/angr/rust/typehoon/typehoon.py +++ b/angr/rust/typehoon/typehoon.py @@ -96,6 +96,9 @@ class RustTypehoon(Typehoon): if isinstance(the_type, SimTypeBottom) and var.size is not None: the_type = RustSimTypeInt(signed=False, size=var.size * self.project.arch.byte_width) + if func_addr != "global": + the_type = self._flatten_pointer_to_array(the_type, self.project.arch) + self.kb.variables[func_addr].set_variable_type( var, the_type, name=the_type.name if isinstance(the_type, SimStruct) else None ) diff --git a/tests/analyses/decompiler/test_decompiler.py b/tests/analyses/decompiler/test_decompiler.py index 6f3d66727..034423ad3 100755 --- a/tests/analyses/decompiler/test_decompiler.py +++ b/tests/analyses/decompiler/test_decompiler.py @@ -2583,13 +2583,13 @@ class TestDecompiler(unittest.TestCase): assert len(following_logic) == 5, "Unexpected number of lines after switch-case" # expected: # v1 = p[1]; - # ptr = &p[1]; + # ptr = &p[1]; (or "ptr = p + 1;" when the local is typed as an element pointer) # if (!p[1]) # return; # } expected = [ r"[a-zA-Z0-9]+ = [a-zA-Z0-9\[\]]+;", - r"[a-zA-Z0-9]+ = &[a-zA-Z0-9\[\]]+;", + r"[a-zA-Z0-9]+ = (?:&[a-zA-Z0-9\[\]]+|[a-zA-Z0-9]+ \+ \d+);", r"if \(![a-zA-Z0-9\[\]]+\)", r"return;", r"}", diff --git a/tests/analyses/test_typehoon.py b/tests/analyses/test_typehoon.py index 547ec55ce..be4e24b8f 100755 --- a/tests/analyses/test_typehoon.py +++ b/tests/analyses/test_typehoon.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# pylint:disable=missing-class-docstring,no-self-use +# pylint:disable=missing-class-docstring,no-self-use,protected-access from __future__ import annotations __package__ = __package__ or "tests.analyses" # pylint:disable=redefined-builtin @@ -16,6 +16,7 @@ from angr.analyses.decompiler.clinic import Clinic from angr.analyses.typehoon.simple_solver import SimpleSolver from angr.analyses.typehoon.translator import TypeTranslator from angr.analyses.typehoon.typeconsts import Float32, Float64, Int32, Pointer64, Struct +from angr.analyses.typehoon.typehoon import Typehoon from angr.analyses.typehoon.typevars import ( DerivedTypeVariable, FuncIn, @@ -563,5 +564,44 @@ class TestFunctionArgTypeNormalization(unittest.TestCase): assert isinstance(joined.pts_to, SimTypeInt) +class TestLocalVariableTypeFlattening(unittest.TestCase): + """ + Tests for Typehoon._flatten_pointer_to_array, which flattens pointer-to-array solutions for function-scope + variables. A local variable (register or pointer-sized stack slot) holds a pointer value; keeping the + pointer-to-array type would make the C backend declare it as an array (``SimTypePointer.c_repr`` drops + the pointer level when the pointee is an array), and any later assignment to it would be illegal C + ("assignment to expression with array type"). + """ + + arch = archinfo.arch_from_id("amd64") + + def test_pointer_to_array_flattens_to_element_pointer(self): + # int[8] * -> int * + ty = SimTypePointer(SimTypeArray(SimTypeInt(), 8)).with_arch(self.arch) + flattened = Typehoon._flatten_pointer_to_array(ty, self.arch) + assert isinstance(flattened, SimTypePointer) + assert isinstance(flattened.pts_to, SimTypeInt) + # the resulting declaration must be a pointer, not an array + assert flattened.c_repr(name="v").endswith("*v") + + def test_pointer_to_nested_array_flattens_fully(self): + # int[4][8] * -> int *; single-level flattening would still be declared as an array + ty = SimTypePointer(SimTypeArray(SimTypeArray(SimTypeInt(), 4), 8)).with_arch(self.arch) + flattened = Typehoon._flatten_pointer_to_array(ty, self.arch) + assert isinstance(flattened, SimTypePointer) + assert isinstance(flattened.pts_to, SimTypeInt) + + def test_plain_array_unchanged(self): + # a genuine in-place local array (char v[22]) is typed as a plain array and must keep its type + ty = SimTypeArray(SimTypeChar(), 22).with_arch(self.arch) + flattened = Typehoon._flatten_pointer_to_array(ty, self.arch) + assert flattened is ty + + def test_plain_pointer_unchanged(self): + ty = SimTypePointer(SimTypeChar()).with_arch(self.arch) + flattened = Typehoon._flatten_pointer_to_array(ty, self.arch) + assert flattened is ty + + if __name__ == "__main__": unittest.main() From 02c374b5b492b8ed37c1ca74035650bd1ac4ac0a Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 22 Jul 2026 03:03:40 -0700 Subject: [PATCH 029/122] DecompilationCache: Serialization support. (#6624) Also, - Refactored variable_kb into kb.dec_variables. - Spill decompilation cache into RuntimeDb. - Save decompilation cache into angrDb. Decompilation results can be preserved across runs. - No longer check in _pb2.py files; they are generated during build. --- .gitignore | 3 +- MANIFEST.in | 2 + angr/analyses/decompiler/clinic.py | 378 ++++- .../decompiler/decompilation_cache.py | 273 +++- .../decompiler/decompilation_options.py | 4 + angr/analyses/decompiler/decompiler.py | 158 +- .../dephication/dephication_base.py | 3 - .../dephication/graph_dephication.py | 5 +- .../decompiler/dephication/graph_rewriting.py | 6 +- .../dephication/rewriting_engine.py | 10 +- .../dephication/seqnode_dephication.py | 8 +- angr/analyses/decompiler/notes/__init__.py | 5 + .../decompiler/notes/decompilation_note.py | 49 + .../decompiler/notes/deobfuscated_strings.py | 26 + .../decompiler/optimization_pass_registry.py | 31 + .../optimization_passes/expr_op_swapper.py | 26 +- .../optimization_passes/optimization_pass.py | 4 +- .../decompiler/structured_codegen/base.py | 25 +- .../decompiler/structured_codegen/c.py | 88 +- .../structured_codegen/c_serialize.py | 1319 +++++++++++++++++ .../decompiler/structured_codegen/rust.py | 10 +- angr/analyses/deobfuscator/api_obf_finder.py | 7 +- .../deobfuscator/api_obf_type2_finder.py | 6 +- .../s_reaching_definitions/__init__.py | 3 +- .../s_reaching_definitions/s_rda_model.py | 75 +- .../s_reaching_definitions.py | 66 +- angr/angrdb/models.py | 44 + angr/angrdb/serializers/kb.py | 6 + angr/angrdb/serializers/structured_code.py | 109 +- angr/angrdb/serializers/variables.py | 28 +- .../key_definitions/atoms.py | 3 + .../key_definitions/definition.py | 4 +- angr/knowledge_plugins/structured_code.py | 250 +++- angr/knowledge_plugins/variables/__init__.py | 3 +- .../variables/spilling_vardict.py | 182 +++ .../variables/variable_manager.py | 32 +- angr/protos/__init__.py | 9 +- angr/protos/ail_types.proto | 102 ++ angr/protos/cfg_pb2.py | 54 - angr/protos/clinic.proto | 126 ++ angr/protos/codegen.proto | 509 +++++++ angr/protos/decompilation_cache.proto | 97 ++ angr/protos/function_pb2.py | 41 - angr/protos/primitives_pb2.py | 63 - angr/protos/variables_pb2.py | 57 - angr/protos/xrefs_pb2.py | 36 - .../pattern_match_simplifier.py | 6 +- .../str_argument_simplifier.py | 4 +- angr/rustylib/ailment.pyi | 6 +- angr/utils/ail_serialization.py | 265 ++++ native/angr/src/ailment/block.rs | 56 +- pyproject.toml | 12 +- setup.py | 10 + tests/ailment/test_serialize.py | 30 + tests/analyses/decompiler/test_decompiler.py | 37 +- .../decompiler/test_decompiler_types.py | 8 +- tests/analyses/decompiler/test_outliner.py | 8 +- tests/analyses/test_typehoon.py | 5 +- tests/gui/test_decompilation_workflows.py | 12 +- .../test_variable_manager.py | 84 +- tests/llm/test_decompiler_llm.py | 44 +- tests/serialization/test_db.py | 160 +- .../test_decompilation_cache_serialization.py | 491 ++++++ 63 files changed, 4994 insertions(+), 589 deletions(-) create mode 100644 angr/analyses/decompiler/optimization_pass_registry.py create mode 100644 angr/analyses/decompiler/structured_codegen/c_serialize.py create mode 100644 angr/knowledge_plugins/variables/spilling_vardict.py create mode 100644 angr/protos/ail_types.proto delete mode 100644 angr/protos/cfg_pb2.py create mode 100644 angr/protos/clinic.proto create mode 100644 angr/protos/codegen.proto create mode 100644 angr/protos/decompilation_cache.proto delete mode 100644 angr/protos/function_pb2.py delete mode 100644 angr/protos/primitives_pb2.py delete mode 100644 angr/protos/variables_pb2.py delete mode 100644 angr/protos/xrefs_pb2.py create mode 100644 angr/utils/ail_serialization.py create mode 100644 tests/serialization/test_decompilation_cache_serialization.py diff --git a/.gitignore b/.gitignore index b19fa9a8a..269c7a9a4 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,5 @@ target/ .DS_Store *_angr_rtdb *_angr_rtdb_* -.claude/ \ No newline at end of file +.claude +angr/protos/*_pb2.py diff --git a/MANIFEST.in b/MANIFEST.in index 2b956ca16..c31d97c76 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,5 @@ include README.md include Cargo.toml include Cargo.lock graft native +recursive-include angr/protos *.proto +recursive-exclude angr/protos *_pb2.py diff --git a/angr/analyses/decompiler/clinic.py b/angr/analyses/decompiler/clinic.py index 6dc9ca08a..aa3a8c5d2 100644 --- a/angr/analyses/decompiler/clinic.py +++ b/angr/analyses/decompiler/clinic.py @@ -3,6 +3,7 @@ from __future__ import annotations import copy import enum +import importlib import logging from collections import defaultdict, namedtuple from collections.abc import Iterable @@ -15,12 +16,14 @@ import networkx from angr import ailment from angr.ailment import AILBlockRewriter, Assignment, Block, Statement from angr.ailment.block_walker import AILBlockViewer -from angr.ailment.expression import Array, FunctionLikeMacro, Let, RustEnum, Struct, VirtualVariable +from angr.ailment.expression import Array, Call, FunctionLikeMacro, Let, RustEnum, Struct, VirtualVariable from angr.analyses.analysis import Analysis, register_analysis from angr.analyses.cfg.cfg_base import CFGBase from angr.analyses.decompiler.callsite_maker import CallSiteMaker +from angr.analyses.decompiler.optimization_pass_registry import name_to_pass, pass_to_name from angr.analyses.s_liveness import SLivenessAnalysis from angr.analyses.s_reaching_definitions import SReachingDefinitionsAnalysis +from angr.analyses.s_reaching_definitions.s_rda_model import SRDAModel from angr.analyses.stack_pointer_tracker import OffsetVal, Register from angr.analyses.typehoon import Typehoon from angr.analyses.typehoon.simple_solver import SimpleSolver @@ -44,6 +47,8 @@ from angr.knowledge_plugins.key_definitions import atoms from angr.knowledge_plugins.variables.variable_manager import VariableManagerInternal from angr.procedures.stubs.UnresolvableCallTarget import UnresolvableCallTarget from angr.procedures.stubs.UnresolvableJumpTarget import UnresolvableJumpTarget +from angr.protos import clinic_pb2 +from angr.serializable import Serializable from angr.sim_type import ( SimCppClass, SimStruct, @@ -67,12 +72,22 @@ from angr.sim_variable import ( SimVariable, ) from angr.utils import timethis +from angr.utils.ail_serialization import ( + BlockPool, + pack_arg_vvars, + pack_graph, + parse_arg_vvars, + parse_graph, + simvar_from_bytes_polymorphic, + simvar_to_bytes_polymorphic, +) from angr.utils.graph import GraphUtils from angr.utils.ssa import is_phi_assignment from angr.utils.types import dereference_simtype_by_lib from .ail_simplifier import AILSimplifier from .ailgraph_walker import AILGraphWalker, RemoveNodeNotice +from .notes import DecompilationNote from .optimization_passes import ( CONDENSING_OPTS, DUPLICATING_OPTS, @@ -88,11 +103,9 @@ from .stackarg_offset_manager import StackArgOffsetManager from .variable_map import VariableMap if TYPE_CHECKING: - from angr.analyses.s_reaching_definitions import SRDAModel from angr.knowledge_plugins.cfg import CFGModel from .decompilation_cache import DecompilationCache - from .notes import DecompilationNote from .peephole_optimizations import PeepholeOptimizationExprBase, PeepholeOptimizationStmtBase l = logging.getLogger(name=__name__) @@ -193,9 +206,21 @@ class ComboRegReferenceWalker(AILBlockRewriter): return expr -class Clinic(Analysis): +class Clinic(Analysis, Serializable): """ - A Clinic deals with AILments. + A Clinic deals with AILments: it lifts a function to AIL and runs the decompiler's simplification pipeline on it. + + AIL graphs exposed after a DECOMPILE-mode run: + + - ``cc_graph``: the graph at the end of Clinic's own simplification, frozen as a copy before region + identification. Serialized; available on both live and cached-and-reloaded clinics. + - ``graph``: ``cc_graph`` further transformed by the decompiler's graph-simplification passes, region + identification, and region-simplification passes — the final graph. Serialized; available on both live and + cached-and-reloaded clinics. + - ``unoptimized_graph``: a copy taken before the first structure-altering optimization pass; use it for an + exact instruction-to-AIL mapping. Only built (and serialized) with ``Decompiler(save_unoptimized_graph=True)``; + otherwise it is None on both live and cached-and-reloaded clinics. + - ``_ail_graph`` / ``_init_ail_graph``: pipeline internals; never serialized. """ _ail_manager: ailment.Manager @@ -214,7 +239,6 @@ class Clinic(Analysis): peephole_optimizations: None | (Iterable[type[PeepholeOptimizationStmtBase] | type[PeepholeOptimizationExprBase]]) = None, # pylint:disable=line-too-long must_struct: set[str] | None = None, - variable_kb: KnowledgeBase | None = None, reset_variable_names=False, rewrite_ites_to_diamonds=True, rewrite_ites_to_diamond_max_cases: int = 15, @@ -246,6 +270,7 @@ class Clinic(Analysis): semvar_naming: bool = True, flavor: str = "pseudocode", variable_map: VariableMap | None = None, + save_unoptimized_graph: bool = False, ): if not func.normalized and mode == ClinicMode.DECOMPILE: raise ValueError("Decompilation must work on normalized function graphs.") @@ -260,7 +285,8 @@ class Clinic(Analysis): self.arg_vvars: dict[int, tuple[ailment.Expr.VirtualVariable, SimVariable]] | None = None self.func_args = None self.func_ret_var = SimVariable(0, "__retvar", "__retvar") - self.variable_kb = variable_kb + # True once _recover_and_link_variables has populated kb.dec_variables for this function this run + self._variables_recovered = False # VariableMap is a side container that holds variable/variable_offset/custom_string/reference_values and the # sibling reference_variable/reference_variable_offset for AIL atoms, keyed by their .idx. It supersedes # storing this information directly on AIL Statement/Expression objects. @@ -339,6 +365,7 @@ class Clinic(Analysis): self.copied_var_ids: set[int] = set() self._constrain_callee_prototypes = constrain_callee_prototypes + self._save_unoptimized_graph = save_unoptimized_graph self._new_block_addrs: set[int] = set() @@ -365,7 +392,7 @@ class Clinic(Analysis): self._analyze_for_decompiling() if ( self._end_stage >= ClinicStage.MAKE_CALLSITES - and self.variable_kb is not None + and self._variables_recovered and self._constrain_callee_prototypes ): self.constrain_callee_prototypes() @@ -498,11 +525,11 @@ class Clinic(Analysis): return self._apply_callsite_prototype_and_calling_convention(ail_graph) def _slice_variables(self, ail_graph: networkx.DiGraph[ailment.Block]) -> networkx.DiGraph[ailment.Block]: - assert self.variable_kb is not None and self._desired_variables is not None + assert self._variables_recovered and self._desired_variables is not None nodes_index = {(n.addr, n.idx): n for n in ail_graph.nodes()} - vfm = self.variable_kb.variables.function_managers[self.function.addr] + vfm = self.kb.dec_variables.function_managers[self.function.addr] for v_name in self._desired_variables: v = next(iter(vv for vv in vfm._unified_variables if vv.name == v_name)) for va in vfm.get_variable_accesses(v): @@ -515,7 +542,7 @@ class Clinic(Analysis): a = TagSlicer( self.function, graph=ail_graph, - variable_kb=self.variable_kb, + kb=self.kb, ) if a.out_graph: # use the new graph @@ -525,8 +552,10 @@ class Clinic(Analysis): def _inline_child_functions(self, ail_graph): for blk in ail_graph.nodes(): for idx, stmt in enumerate(blk.statements): - if isinstance(stmt, ailment.Stmt.SideEffectStatement) and isinstance( - stmt.expr.target, ailment.Expr.Const + if ( + isinstance(stmt, ailment.Stmt.SideEffectStatement) + and isinstance(stmt.expr, Call) + and isinstance(stmt.expr.target, ailment.Expr.Const) ): assert self.function._function_manager is not None callee = self.function._function_manager.function(stmt.expr.target.value) @@ -652,9 +681,11 @@ class Clinic(Analysis): self._ail_manager.next_atom(), reg_offset, reg_arg.bits, - ins_addr=caller_block.addr + caller_block.original_size, + ins_addr=caller_block.addr + + (caller_block.original_size if caller_block.original_size is not None else 0), ), - ins_addr=caller_block.addr + caller_block.original_size, + ins_addr=caller_block.addr + + (caller_block.original_size if caller_block.original_size is not None else 0), ) caller_block.statements.append(stmt) else: @@ -977,7 +1008,7 @@ class Clinic(Analysis): # Recover variables on AIL blocks self._update_progress(80.0, text="Recovering variables") - variable_kb = self._recover_and_link_variables( + self._recover_and_link_variables( self._ail_graph, self.arg_list, self.arg_vvars, self.vvar_to_vvar, self._type_hints ) @@ -989,14 +1020,13 @@ class Clinic(Analysis): self._ail_graph, stage=OptimizationPassStage.AFTER_VARIABLE_RECOVERY, avoid_vvar_ids=self.copied_var_ids, - variable_kb=variable_kb, ) # Make function prototype self._update_progress(90.0, text="Making function prototype") - self._make_function_prototype(self.arg_list, variable_kb) + self._make_function_prototype(self.arg_list) - self.variable_kb = variable_kb + self._variables_recovered = True def _stage_semantic_variable_naming(self) -> None: """ @@ -1005,8 +1035,8 @@ class Clinic(Analysis): This stage analyzes the AIL graph for semantic patterns and renames variables accordingly. """ - if self.variable_kb is None: - l.debug("variable_kb is None, skipping semantic variable naming") + if not self._variables_recovered: + l.debug("variables not recovered, skipping semantic variable naming") return if self.flavor == "rust": @@ -1016,7 +1046,7 @@ class Clinic(Analysis): self._update_progress(91.0, text="Applying semantic variable naming") # Get the variable manager for this function - var_manager = self.variable_kb.variables[self.function.addr] + var_manager = self.kb.dec_variables[self.function.addr] # Find the entry node entry_node: ailment.Block | None = None @@ -1035,7 +1065,7 @@ class Clinic(Analysis): l.debug("Semantic naming renamed %d variables", len(var_name_mapping)) def _stage_collect_externs(self) -> None: - self.externs = self._collect_externs(self._ail_graph, self.variable_kb, self.variable_map) + self.externs = self._collect_externs(self._ail_graph, self.kb, self.variable_map) def _analyze_for_data_refs(self): # Remove alignment blocks @@ -1115,7 +1145,7 @@ class Clinic(Analysis): self.graph = ail_graph self.arg_list = None - self.variable_kb = None + self._variables_recovered = False self.cc_graph = None self.externs = set() self.data_refs: dict[int, list[DataRefDesc]] = self._collect_data_refs(ail_graph) @@ -1979,7 +2009,6 @@ class Clinic(Analysis): self, ail_graph, stage: OptimizationPassStage = OptimizationPassStage.AFTER_GLOBAL_SIMPLIFICATION, - variable_kb=None, stack_items: dict[int, StackItem] | None = None, stack_pointer_tracker=None, **kwargs, @@ -1999,9 +2028,12 @@ class Clinic(Analysis): if stage != pass_.STAGE: continue - if pass_ in DUPLICATING_OPTS + CONDENSING_OPTS and self.unoptimized_graph is None: - # we should save a copy at the first time any optimization that could alter the structure - # of the graph is applied + if ( + self._save_unoptimized_graph + and pass_ in DUPLICATING_OPTS + CONDENSING_OPTS + and self.unoptimized_graph is None + ): + # save a copy the first time any optimization that could alter the structure of the graph is applied self.unoptimized_graph = self._copy_graph(ail_graph) pass_ = timethis(pass_) @@ -2011,7 +2043,7 @@ class Clinic(Analysis): blocks_by_addr=addr_to_blocks, blocks_by_addr_and_idx=addr_and_idx_to_blocks, graph=ail_graph, - variable_kb=variable_kb, + kb=self.kb, vvar_id_start=self.vvar_id_start, entry_node_addr=self.entry_node_addr, scratch=self.optimization_scratch, @@ -2324,7 +2356,7 @@ class Clinic(Analysis): return ail_graph @timethis - def _make_function_prototype(self, arg_list: list[SimVariable], variable_kb: KnowledgeBase): + def _make_function_prototype(self, arg_list: list[SimVariable]): if self.function.prototype is not None: if self.function.prototype_source.value >= PrototypeSource.CCA_DECOMPILER.value: # do not overwrite an existing function prototype @@ -2337,7 +2369,7 @@ class Clinic(Analysis): # FIXME: remove this branch once type inference supports floating point variables return - variables = variable_kb.variables[self.function.addr] + variables = self.kb.dec_variables[self.function.addr] func_args = [] for arg in arg_list: func_arg = None @@ -2380,8 +2412,11 @@ class Clinic(Analysis): type_hints: list[tuple[atoms.VirtualVariable | atoms.MemoryLocation, str]], ): # variable recovery - tmp_kb = KnowledgeBase(self.project) if self.variable_kb is None else self.variable_kb + # route recovery into kb.dec_variables: VariableRecoveryBase writes to the "variables" plugin of the KB + # it is given + tmp_kb = KnowledgeBase(self.project) tmp_kb.functions = self.kb.functions + tmp_kb.register_plugin("variables", self.kb.dec_variables) vr = self.project.analyses.VariableRecoveryFast( self.function, # pylint:disable=unused-variable fail_fast=self._fail_fast, # type: ignore @@ -3593,8 +3628,8 @@ class Clinic(Analysis): node.statements.insert(0, lbl) @staticmethod - def _collect_externs(ail_graph, variable_kb, variable_map: VariableMap): - global_vars = variable_kb.variables.global_manager.get_variables() + def _collect_externs(ail_graph, kb, variable_map: VariableMap): + global_vars = kb.dec_variables.global_manager.get_variables() walker = ailment.AILBlockRewriter() variables = set() @@ -4016,10 +4051,10 @@ class Clinic(Analysis): ail_graph.add_edge(new_node, succ) def _collect_callsite_prototypes(self) -> dict[int, list[tuple[list[SimType | None], SimType | None]]]: - if self.variable_kb is None: + if not self._variables_recovered: return {} - variables = self.variable_kb.variables[self.function.addr] + variables = self.kb.dec_variables[self.function.addr] func_proto_candidates: defaultdict[int, list[tuple[list[SimType | None], SimType | None]]] = defaultdict(list) # pylint:disable=unused-argument @@ -4186,5 +4221,276 @@ class Clinic(Analysis): .model ) + # ----------------------------------------------------------------------------------------------------------------- + # Protobuf serialization. Conventions: + # - Heavy sub-objects manage their own formats; AIL-typed slots use the typed messages from ail_types.proto. + # - Runtime back-references (project / kb / function / _cfg / _cache / typehoon / _spt) are not + # serialized; they are reattached at parse time from the caller's kwargs. + # - parse_from_cmessage uses __new__ to bypass __init__, which would run the full decompilation pipeline. + # ----------------------------------------------------------------------------------------------------------------- + + @classmethod + def _get_cmsg(cls): + return clinic_pb2.Clinic() # type: ignore # pylint:disable=no-member + + def serialize_to_cmessage(self): + msg = clinic_pb2.Clinic() # type: ignore # pylint:disable=no-member + + # Function and arch hints. + if self.function is not None and self.function.addr is not None: + msg.function_addr = self.function.addr + if self.flavor is not None: + msg.flavor = self.flavor + + # AIL-typed slots consumed by the decompiler's cache-reuse path and by post-decompilation consumers. The + # internal AIL graphs (_ail_graph, _init_ail_graph) and the remaining regenerable/runtime state are not + # serialized. unoptimized_graph is only serialized on request (Decompiler(save_unoptimized_graph=True)). + # All graphs share one block pool: most of their blocks are byte-identical and get stored once. + block_pool = BlockPool() + if self.cc_graph is not None: + msg.cc_graph.CopyFrom(pack_graph(self.cc_graph, pool=block_pool)) + if self.graph is not None: + msg.graph.CopyFrom(pack_graph(self.graph, pool=block_pool)) + if self._save_unoptimized_graph and self.unoptimized_graph is not None: + msg.unoptimized_graph.CopyFrom(pack_graph(self.unoptimized_graph, pool=block_pool)) + msg.block_pool.extend(block_pool.payloads) + if self.arg_vvars is not None: + msg.arg_vvars.CopyFrom(pack_arg_vvars(self.arg_vvars)) + + # Already-Serializable sub-objects. + if self.arg_list is not None: + for sv in self.arg_list: + msg.arg_list.add().payload = simvar_to_bytes_polymorphic(sv) + for sv in self.externs: + msg.externs.add().payload = simvar_to_bytes_polymorphic(sv) + + # CLEAN collections. + if self.vvar_to_vvar is not None: + for k, v in self.vvar_to_vvar.items(): + msg.vvar_to_vvar[k] = v + msg.secondary_stackvars.extend(sorted(self.secondary_stackvars)) + if self._removed_vvar_ids is not None: + msg._removed_vvar_ids_set = True + msg.removed_vvar_ids.extend(sorted(self._removed_vvar_ids)) + msg._preserve_vvar_ids.extend(sorted(self._preserve_vvar_ids)) + for k, v in self._inlined_counts.items(): + msg._inlined_counts[k] = v + msg._inlining_parents.extend(sorted(self._inlining_parents)) + if self._must_struct is not None: + msg._must_struct_set = True + msg._must_struct.extend(sorted(self._must_struct)) + if self._desired_variables is not None: + msg._desired_variables_set = True + msg._desired_variables.extend(sorted(self._desired_variables)) + for off, item in self.stack_items.items(): + msg.stack_items[off].offset = item.offset + msg.stack_items[off].size = item.size + msg.stack_items[off].name = item.name + msg.stack_items[off].item_type = item.item_type.value + for src, dst in self.edges_to_remove: + pair = msg.edges_to_remove.add() + pair.src.addr = src[0] + if src[1] is not None: + pair.src.idx = src[1] + pair.dst.addr = dst[0] + if dst[1] is not None: + pair.dst.idx = dst[1] + msg.copied_var_ids.extend(sorted(self.copied_var_ids)) + msg._new_block_addrs.extend(sorted(self._new_block_addrs)) + if self.entry_node_addr is not None: + msg.entry_node_addr.addr = self.entry_node_addr[0] + if self.entry_node_addr[1] is not None: + msg.entry_node_addr.idx = self.entry_node_addr[1] + + # CLEAN scalars. + msg.vvar_id_start = self.vvar_id_start + msg._max_stack_depth = self._max_stack_depth + msg._sp_shift = self._sp_shift + msg._max_type_constraints = self._max_type_constraints + msg._type_constraint_set_degradation_threshold = self._type_constraint_set_degradation_threshold + msg._fold_callexprs_into_conditions = self._fold_callexprs_into_conditions + msg._fold_expressions = self._fold_expressions + msg._insert_labels = self._insert_labels + msg._remove_dead_memdefs = self._remove_dead_memdefs + msg._exception_edges = self._exception_edges + msg._sp_tracker_track_memory = self._sp_tracker_track_memory + msg._reset_variable_names = self._reset_variable_names + msg._rewrite_ites_to_diamonds = self._rewrite_ites_to_diamonds + msg._flatten_args = self._flatten_args + msg._semvar_naming = self._semvar_naming + msg._force_loop_single_exit = self._force_loop_single_exit + msg._refine_loops_with_single_successor = self._refine_loops_with_single_successor + msg._register_save_areas_removed = self._register_save_areas_removed + msg._rewrite_ites_to_diamond_max_cases = self._rewrite_ites_to_diamond_max_cases + msg._expose_loop_head_backedges = self._expose_loop_head_backedges + msg._constrain_callee_prototypes = self._constrain_callee_prototypes + msg._save_unoptimized_graph = self._save_unoptimized_graph + + msg._mode = self._mode.value + msg._start_stage = self._start_stage.value + msg._end_stage = self._end_stage.value + msg._skip_stages.extend(s.value for s in self._skip_stages) + + # Pass class refs. peephole_optimizations=None means "use the default peephole set". + msg.peephole_optimizations_use_default = self.peephole_optimizations is None + if self.peephole_optimizations is not None: + msg.peephole_optimizations.extend(pass_to_name(cls_) for cls_ in self.peephole_optimizations) + if self._typehoon_cls is not None: + # _typehoon_cls is the Typehoon class itself (not a registered pass); store its fully-qualified name + msg._typehoon_cls = ( + f"{self._typehoon_cls.__module__}.{self._typehoon_cls.__qualname__}" + if self._typehoon_cls.__module__ != "builtins" + else "" + ) + + return msg + + @classmethod + def parse_from_cmessage( + cls, + cmsg, + *, + project=None, + kb=None, + function=None, + cfg=None, + **kwargs, + ): + """Bypasses :meth:`Clinic.__init__` (which runs the analysis) and reconstructs the instance directly. Runtime + back-references — project / kb / function / _cfg — come from kwargs. + + Only the state consumed by the decompiler's cache-reuse path and by post-decompilation consumers + (cc_graph, graph, optionally unoptimized_graph) is serialized; regenerable and runtime-only state is + restored to its default here. If ``function`` is None and ``kb`` is provided, the function is resolved by + address from the cmessage.""" + msg = cmsg + clinic = cls.__new__(cls) + + # Resolve function from address if not provided. + if function is None and kb is not None and msg.HasField("function_addr"): + function = kb.functions.function(msg.function_addr) + + # Initialize back-references and Analysis-base state. We bypass Analysis.__init__ so set the bare minimum. + clinic.project = project + clinic.kb = kb + clinic.function = function + clinic._cache = None + clinic._ail_manager = None + clinic._spt = None + clinic.typehoon = None + clinic._optimization_passes = [] + clinic.optimization_scratch = {} + + # AIL-typed slots consumed by the cache-reuse path and by post-decompilation consumers. + clinic.cc_graph = parse_graph(msg.cc_graph, msg.block_pool) if msg.HasField("cc_graph") else None + clinic.graph = parse_graph(msg.graph, msg.block_pool) if msg.HasField("graph") else None + clinic.unoptimized_graph = ( + parse_graph(msg.unoptimized_graph, msg.block_pool) if msg.HasField("unoptimized_graph") else None + ) + clinic.arg_vvars = parse_arg_vvars(msg.arg_vvars) if msg.HasField("arg_vvars") else None + + # Regenerable/runtime state that is not serialized; restored to its default. + clinic._ail_graph = None + clinic._init_ail_graph = None + clinic._init_arg_vvars = None + clinic._type_hints = [] + clinic._blocks_by_addr_and_size = None + clinic.func_args = None + clinic.func_ret_var = None + clinic.data_refs = {} + clinic.stack_items = { + off: StackItem(item.offset, item.size, item.name, StackItemType(item.item_type)) + for off, item in msg.stack_items.items() + } + clinic._inline_functions = set() + clinic.notes = {} + clinic._func_graph = None + clinic.reaching_definitions = None + + # Already-Serializable sub-objects. + clinic.arg_list = [simvar_from_bytes_polymorphic(e.payload) for e in msg.arg_list] if msg.arg_list else None + clinic.externs = {simvar_from_bytes_polymorphic(e.payload) for e in msg.externs} + + # cfg back-reference; decompilation variables live on kb.dec_variables, not on the clinic. + clinic._variables_recovered = False + clinic._cfg = cfg + + # Flavor. + clinic.flavor = msg.flavor if msg.HasField("flavor") else "pseudocode" + + # CLEAN collections. + clinic.vvar_to_vvar = dict(msg.vvar_to_vvar) if msg.vvar_to_vvar else None + clinic.secondary_stackvars = set(msg.secondary_stackvars) + clinic._stackarg_offset_manager = StackArgOffsetManager(project.arch.bits if project is not None else 64) + clinic._removed_vvar_ids = set(msg.removed_vvar_ids) if msg._removed_vvar_ids_set else None + clinic._preserve_vvar_ids = set(msg._preserve_vvar_ids) + clinic._inlined_counts = dict(msg._inlined_counts) + clinic._inlining_parents = set(msg._inlining_parents) + clinic._must_struct = set(msg._must_struct) if msg._must_struct_set else None + clinic._desired_variables = set(msg._desired_variables) if msg._desired_variables_set else None + clinic.edges_to_remove = [ + ( + (pair.src.addr, pair.src.idx if pair.src.HasField("idx") else None), + (pair.dst.addr, pair.dst.idx if pair.dst.HasField("idx") else None), + ) + for pair in msg.edges_to_remove + ] + clinic.copied_var_ids = set(msg.copied_var_ids) + clinic._new_block_addrs = set(msg._new_block_addrs) + clinic.entry_node_addr = ( + (msg.entry_node_addr.addr, msg.entry_node_addr.idx if msg.entry_node_addr.HasField("idx") else None) + if msg.HasField("entry_node_addr") + else None + ) + + # CLEAN scalars. + clinic.vvar_id_start = msg.vvar_id_start + clinic._max_stack_depth = msg._max_stack_depth + clinic._sp_shift = msg._sp_shift + clinic._max_type_constraints = msg._max_type_constraints + clinic._type_constraint_set_degradation_threshold = msg._type_constraint_set_degradation_threshold + clinic._fold_callexprs_into_conditions = msg._fold_callexprs_into_conditions + clinic._fold_expressions = msg._fold_expressions + clinic._insert_labels = msg._insert_labels + clinic._remove_dead_memdefs = msg._remove_dead_memdefs + clinic._exception_edges = msg._exception_edges + clinic._sp_tracker_track_memory = msg._sp_tracker_track_memory + clinic._reset_variable_names = msg._reset_variable_names + clinic._rewrite_ites_to_diamonds = msg._rewrite_ites_to_diamonds + clinic._flatten_args = msg._flatten_args + clinic._semvar_naming = msg._semvar_naming + clinic._force_loop_single_exit = msg._force_loop_single_exit + clinic._refine_loops_with_single_successor = msg._refine_loops_with_single_successor + clinic._register_save_areas_removed = msg._register_save_areas_removed + clinic._rewrite_ites_to_diamond_max_cases = msg._rewrite_ites_to_diamond_max_cases + clinic._expose_loop_head_backedges = msg._expose_loop_head_backedges + clinic._constrain_callee_prototypes = msg._constrain_callee_prototypes + clinic._save_unoptimized_graph = msg._save_unoptimized_graph + + clinic._mode = ClinicMode(msg._mode) if msg._mode in {m.value for m in ClinicMode} else ClinicMode.DECOMPILE + clinic._start_stage = ClinicStage(msg._start_stage) + clinic._end_stage = ClinicStage(msg._end_stage) + clinic._skip_stages = tuple(ClinicStage(s) for s in msg._skip_stages) + + # Pass class refs. peephole_optimizations=None means "use the default peephole set". + if msg.peephole_optimizations_use_default: + clinic.peephole_optimizations = None + else: + clinic.peephole_optimizations = [name_to_pass(n) for n in msg.peephole_optimizations] + if msg._typehoon_cls: + # _typehoon_cls is the Typehoon class itself (not a registered pass). Resolve directly by FQN. + module_name, _, cls_name = msg._typehoon_cls.rpartition(".") + clinic._typehoon_cls = getattr(importlib.import_module(module_name), cls_name) + else: + clinic._typehoon_cls = Typehoon + + # The remainder of the public Clinic surface that isn't part of the serialized state — set sensible defaults so + # attribute access doesn't crash. + clinic.static_vvars = {} + clinic.static_buffers = {} + clinic.variable_map = VariableMap() + + return clinic + register_analysis(Clinic, "Clinic") diff --git a/angr/analyses/decompiler/decompilation_cache.py b/angr/analyses/decompiler/decompilation_cache.py index 18e0fe57e..87b09d273 100644 --- a/angr/analyses/decompiler/decompilation_cache.py +++ b/angr/analyses/decompiler/decompilation_cache.py @@ -1,26 +1,170 @@ from __future__ import annotations +import json +import time from typing import TYPE_CHECKING, Any +from angr.protos import decompilation_cache_pb2 +from angr.serializable import Serializable +from angr.utils.ail_serialization import ( + pack_arg_vvars, + pack_ite_exprs, + pack_static_buffers, + pack_static_vvars, + parse_arg_vvars, + parse_ite_exprs, + parse_static_buffers, + parse_static_vvars, +) + from .clinic import Clinic if TYPE_CHECKING: + from angr import ailment from angr.analyses.decompiler.optimization_passes.expr_op_swapper import OpDescriptor from angr.analyses.typehoon.typevars import TypeConstraint, TypeVariable + from angr.knowledge_plugins.cfg import CFGModel + from .notes import DecompilationNote from .structured_codegen import BaseStructuredCodeGenerator from .variable_map import VariableMap -class DecompilationCache: +# --------------------------------------------------------------------------------------------------------------------- +# Serialization helpers. +# +# Conventions: +# - Heavy sub-objects (``clinic``, ``codegen``) are embedded as already-serialized bytes (each manages its own format). +# - AIL-typed top-level slots (``arg_vvars``, ``ite_exprs``) use the typed messages from ``ail_types.proto``. +# - ``cfg`` is intentionally not serialized — it comes from the parent Project. Decompilation variables live on +# kb.dec_variables. +# - The 4 typehoon-typed slots are skipped entirely (typehoon is out of scope for now). +# --------------------------------------------------------------------------------------------------------------------- + + +def _simvar_to_bytes(v) -> bytes: + return type(v).__name__.encode("ascii") + b"\0" + v.serialize() + + +def _simvar_from_bytes(b: bytes): + import angr.sim_variable as sv_mod # pylint:disable=import-outside-toplevel + + sep = b.index(b"\0") + return getattr(sv_mod, b[:sep].decode("ascii")).parse(b[sep + 1 :]) + + +def _serialize_binop_operators(binop_operators, out_msg, set_flag=None) -> None: + if binop_operators is None: + return + if set_flag is not None: + setattr(set_flag[0], set_flag[1], True) + for op_desc, value in binop_operators.items(): + entry = out_msg.add() + entry.key_json = op_desc.to_json() + entry.value = value + + +def _parse_binop_operators(entries): + from angr.analyses.decompiler.optimization_passes.expr_op_swapper import ( # pylint:disable=import-outside-toplevel + OpDescriptor, + ) + + return {OpDescriptor.from_json(e.key_json): e.value for e in entries} + + +def _serialize_parameters(params: dict, out_msg) -> None: + """Translate the 15-key parameters dict into a DecompilationParameters cmessage.""" + from angr.analyses.decompiler.optimization_pass_registry import ( # pylint:disable=import-outside-toplevel + pass_to_name, + ) + + if params.get("flavor") is not None: + out_msg.flavor = params["flavor"] + if "sp_tracker_track_memory" in params: + out_msg.sp_tracker_track_memory = bool(params["sp_tracker_track_memory"]) + # Collection-typed parameters are never None (the Decompiler normalizes them to empty collections), so each is + # written directly; an empty collection is left unset and parses back to empty. + out_msg.vars_must_struct.extend(sorted(params.get("vars_must_struct") or ())) + out_msg.desired_variables.extend(sorted(params.get("desired_variables") or ())) + out_msg.inline_functions.extend(sorted(params.get("inline_functions") or ())) + for option, value in params.get("options") or (): + entry = out_msg.options.add() + entry.param = option.param + try: + entry.value_json = json.dumps(value) + except (TypeError, ValueError): + entry.value_json = json.dumps(None) + for cls in params.get("optimization_passes") or (): + out_msg.optimization_passes.append(pass_to_name(cls)) + # peephole_optimizations is the one None-able collection: None means "use the default peephole set" + peepholes = params.get("peephole_optimizations") + out_msg.peephole_optimizations_use_default = peepholes is None + if peepholes is not None: + for cls in peepholes: + out_msg.peephole_optimizations.append(pass_to_name(cls)) + for k, v in (params.get("expr_comments") or {}).items(): + out_msg.expr_comments[k] = v + for k, v in (params.get("stmt_comments") or {}).items(): + out_msg.stmt_comments[k] = v + _serialize_binop_operators(params.get("binop_operators") or {}, out_msg.binop_operators) + if params.get("ite_exprs"): + out_msg.ite_exprs.CopyFrom(pack_ite_exprs(params["ite_exprs"])) + if params.get("static_vvars"): + out_msg.static_vvars.CopyFrom(pack_static_vvars(params["static_vvars"])) + if params.get("static_buffers"): + out_msg.static_buffers.CopyFrom(pack_static_buffers(params["static_buffers"])) + out_msg.save_unoptimized_graph = bool(params.get("save_unoptimized_graph")) + + +def _parse_parameters(msg) -> dict: + """Always populate every one of the 15 keys in the returned dict; scalar fields that were not set come back as + None and collection fields come back empty, except peephole_optimizations where None means "use the default + peephole set". This matches the decompiler's normalized _cache_parameters, which _can_use_decompilation_cache + compares key by key against the deserialized cache.""" + from angr.analyses.decompiler.decompilation_options import PARAM_TO_OPTION # pylint:disable=import-outside-toplevel + from angr.analyses.decompiler.optimization_pass_registry import ( # pylint:disable=import-outside-toplevel + name_to_pass, + ) + + # Collection-typed values come back as empty collections (never None) so they match the Decompiler's normalized + # _cache_parameters during cache-validity comparison. + return { + "flavor": msg.flavor if msg.HasField("flavor") else None, + "sp_tracker_track_memory": msg.sp_tracker_track_memory if msg.HasField("sp_tracker_track_memory") else None, + "vars_must_struct": set(msg.vars_must_struct), + "desired_variables": frozenset(msg.desired_variables), + "inline_functions": frozenset(msg.inline_functions), + "options": { + (PARAM_TO_OPTION[e.param], json.loads(e.value_json) if e.value_json else None) + for e in msg.options + if e.param in PARAM_TO_OPTION + }, + "optimization_passes": [name_to_pass(n) for n in msg.optimization_passes], + "peephole_optimizations": ( + None if msg.peephole_optimizations_use_default else [name_to_pass(n) for n in msg.peephole_optimizations] + ), + "expr_comments": dict(msg.expr_comments), + "stmt_comments": dict(msg.stmt_comments), + "binop_operators": _parse_binop_operators(msg.binop_operators), + "ite_exprs": parse_ite_exprs(msg.ite_exprs) if msg.HasField("ite_exprs") else set(), + "static_vvars": parse_static_vvars(msg.static_vvars) if msg.HasField("static_vvars") else {}, + "static_buffers": parse_static_buffers(msg.static_buffers) if msg.HasField("static_buffers") else {}, + "save_unoptimized_graph": msg.save_unoptimized_graph, + } + + +class DecompilationCache(Serializable): """ Caches key data structures that can be used later for refining decompilation results, such as retyping variables. """ + # ``cfg`` is a decompile-time input used only for cache-validity checks. It is not serialized; after + # deserialization it is None until the caller re-attaches it. __slots__ = ( "addr", "arg_vvars", "binop_operators", + "cfg", "clinic", "codegen", "errors", @@ -32,32 +176,139 @@ class DecompilationCache: "parameters", "stack_offset_typevars", "stackvar_max_sizes", + "timestamp", "type_constraints", "var_to_typevar", "variable_map", + "version", ) def __init__(self, addr): + import angr # pylint:disable=import-outside-toplevel,cyclic-import + self.parameters: dict[str, Any] = {} + # angr version and creation time of this decompilation + self.version: str = angr.__version__ + self.timestamp: int = int(time.time()) self.addr = addr - self.type_constraints: dict[TypeVariable, set[TypeConstraint]] | None = None - self.arg_vvars: dict | None = None + self.cfg: CFGModel | None = None + # Collection-typed fields default to empty containers rather than None, so serialization never has to + # distinguish None from empty. + self.type_constraints: dict[TypeVariable, set[TypeConstraint]] = {} + self.arg_vvars: dict = {} self.func_typevar: TypeVariable | None = None - self.var_to_typevar: dict | None = None - self.stackvar_max_sizes: dict | None = None - self.stack_offset_typevars: dict | None = None + self.var_to_typevar: dict = {} + self.stackvar_max_sizes: dict = {} + self.stack_offset_typevars: dict = {} self.codegen: BaseStructuredCodeGenerator | None = None self.clinic: Clinic | None = None self.variable_map: VariableMap | None = None - self.ite_exprs: set[tuple[int, Any]] | None = None - self.binop_operators: dict[OpDescriptor, str] | None = None + self.ite_exprs: set[tuple[int, ailment.Expression]] = set() + self.binop_operators: dict[OpDescriptor, str] = {} self.errors: list[str] = [] self.function_summary: str | None = None - self.notes: dict[str, str] = {} + self.notes: dict[str, DecompilationNote] = {} self.max_tv_id: int = 0 @property def local_types(self): - if self.clinic is None or self.clinic.variable_kb is None: + if self.clinic is None or self.clinic.kb is None or self.addr not in self.clinic.kb.dec_variables: return None - return self.clinic.variable_kb.variables[self.addr].types + return self.clinic.kb.dec_variables[self.addr].types + + # ----------------------------------------------------------------------------------------------------------------- + # Protobuf serialization. Heavy sub-objects (clinic, codegen) are embedded as already-serialized bytes; AIL-typed + # top-level fields (arg_vvars, ite_exprs) use the typed messages from ail_types.proto. The four typehoon-typed + # slots and the ``cfg`` input are not serialized and come back as None. + # ----------------------------------------------------------------------------------------------------------------- + + @classmethod + def _get_cmsg(cls): + return decompilation_cache_pb2.DecompilationCache() # pylint:disable=no-member + + def serialize_to_cmessage(self): + msg = decompilation_cache_pb2.DecompilationCache(addr=self.addr) # pylint:disable=no-member + + if self.clinic is not None: + msg.clinic = self.clinic.serialize() + if self.codegen is not None: + msg.codegen = self.codegen.serialize() + + msg.errors.extend(self.errors) + if self.function_summary is not None: + msg.function_summary = self.function_summary + # Collection fields are never None; an empty collection is simply left unset and parses back to empty. + if self.arg_vvars: + msg.arg_vvars.CopyFrom(pack_arg_vvars(self.arg_vvars)) + if self.ite_exprs: + msg.ite_exprs.CopyFrom(pack_ite_exprs(self.ite_exprs)) + _serialize_binop_operators(self.binop_operators, msg.binop_operators) + for simvar, size in self.stackvar_max_sizes.items(): + entry = msg.stackvar_max_sizes.add() + entry.simvar = _simvar_to_bytes(simvar) + entry.max_size = size + + msg.version = self.version + msg.timestamp = self.timestamp + + # An unset parameters message means "no recorded parameters"; cache-validity checks treat such a cache as + # always usable (matching runs with use_cache=False). + if self.parameters: + _serialize_parameters(self.parameters, msg.parameters) + + for k, note in self.notes.items(): + msg.notes_json[k] = note.to_json() + + return msg + + @classmethod + def parse_from_cmessage( + cls, + cmsg, + *, + project=None, + kb=None, + function=None, + cfg=None, + **_, + ): + """Parse a DecompilationCache from a cmessage. Runtime back-references (project, kb, function, cfg) are + passed through to the embedded Clinic / codegen parsers so the parsed cache is functional for cache-hit + validity checks. Decompilation variables live on kb.dec_variables.""" + from .notes import DecompilationNote # pylint:disable=import-outside-toplevel + from .structured_codegen.c import CStructuredCodeGenerator # pylint:disable=import-outside-toplevel + + cache = cls(cmsg.addr) + # cfg is not serialized; reattach from kwargs so cache-validity checks still work. + cache.cfg = cfg + + if cmsg.HasField("clinic"): + cache.clinic = Clinic.parse(cmsg.clinic, project=project, kb=kb, function=function, cfg=cfg) + if cmsg.HasField("codegen"): + cache.codegen = CStructuredCodeGenerator.parse(cmsg.codegen, project=project, kb=kb, func=function) + + cache.errors = list(cmsg.errors) + if cmsg.HasField("function_summary"): + cache.function_summary = cmsg.function_summary + # Collection fields default to empty (set in __init__); only assign when the message carries content. + if cmsg.HasField("arg_vvars"): + cache.arg_vvars = parse_arg_vvars(cmsg.arg_vvars) + if cmsg.HasField("ite_exprs"): + cache.ite_exprs = parse_ite_exprs(cmsg.ite_exprs) + cache.binop_operators = _parse_binop_operators(cmsg.binop_operators) + cache.stackvar_max_sizes = {_simvar_from_bytes(e.simvar): e.max_size for e in cmsg.stackvar_max_sizes} + + # legacy blobs carry the proto3 defaults ""/0, meaning "unknown"; do not re-stamp them with current values + cache.version = cmsg.version + cache.timestamp = cmsg.timestamp + if cache.codegen is not None: + # mirror the stamps onto the codegen (a fresh decompile does the same in Decompiler._decompile) + cache.codegen.version = cache.version + cache.codegen.timestamp = cache.timestamp + + if cmsg.HasField("parameters"): + cache.parameters = _parse_parameters(cmsg.parameters) + + cache.notes = {k: DecompilationNote.from_json(v) for k, v in cmsg.notes_json.items()} + + return cache diff --git a/angr/analyses/decompiler/decompilation_options.py b/angr/analyses/decompiler/decompilation_options.py index 3e19c61b2..39c43b45b 100644 --- a/angr/analyses/decompiler/decompilation_options.py +++ b/angr/analyses/decompiler/decompilation_options.py @@ -45,6 +45,10 @@ class DecompilationOption[T]: O = DecompilationOption +# Serialization contract for display options (cls="codegen"): to survive Codegen serialization, an option's param +# must have a matching optional scalar proto field, named identically, in the trailing display-option block of the +# Codegen message (protos/codegen.proto). Options without such a field are dropped on round-trip. + options = [ O( "Aggressively remove dead memdefs", diff --git a/angr/analyses/decompiler/decompiler.py b/angr/analyses/decompiler/decompiler.py index 33912530f..d16ef2593 100644 --- a/angr/analyses/decompiler/decompiler.py +++ b/angr/analyses/decompiler/decompiler.py @@ -16,7 +16,6 @@ from angr.analyses.s_propagator import sprop_cache_scope from angr.analyses.typehoon.typehoon import Typehoon from angr.analyses.typehoon.typevars import TypeVariableManager from angr.errors import AngrAIError -from angr.knowledge_base import KnowledgeBase from angr.knowledge_plugins.functions.function import Function from angr.rust.optimization_passes import get_rust_optimization_passes from angr.rust.typehoon.typehoon import RustTypehoon @@ -34,6 +33,7 @@ from .optimization_passes.optimization_pass import OptimizationPassStage from .presets import DECOMPILATION_PRESETS, DecompilationPreset from .region_identifier import RegionIdentifier from .sequence_walker import SequenceWalker +from .structured_codegen import DummyStructuredCodeGenerator from .structured_codegen.c import CStructuredCodeGenerator from .structured_codegen.rust import RustStructuredCodeGenerator from .structurer_nodes import SequenceNode @@ -62,6 +62,15 @@ class Decompiler(Analysis): Run this on a Function object for which a normalized CFG has been constructed. The fully processed output can be found in result.codegen.text + + AIL graphs exposed on the result (both on a fresh run and on a cache hit, including caches reloaded from + angrdb or the runtime-db spill): + + - ``ail_graph`` (= ``clinic.cc_graph``): the simplified graph before region identification. + - ``clinic.graph``: the final graph after region identification and region simplification. + - ``unoptimized_ail_graph`` (= ``clinic.unoptimized_graph``): a snapshot before the first structure-altering + optimization pass; use it for an exact instruction-to-AIL mapping. Only built when + ``save_unoptimized_graph=True`` is passed; otherwise this attribute is None on both fresh runs and cache hits. """ def __init__( @@ -72,7 +81,6 @@ class Decompiler(Analysis): preset: str | DecompilationPreset | None = None, optimization_passes=None, sp_tracker_track_memory=True, - variable_kb=None, peephole_optimizations: _PEEPHOLE_OPTIMIZATIONS_TYPE = None, vars_must_struct: set[str] | None = None, flavor="pseudocode", @@ -81,7 +89,7 @@ class Decompiler(Analysis): ite_exprs=None, binop_operators=None, decompile=True, - regen_clinic=True, + regen_clinic=False, inline_functions=None, desired_variables=None, update_memory_data: bool = True, @@ -98,6 +106,7 @@ class Decompiler(Analysis): static_vvars: dict | None = None, static_buffers: dict | None = None, codegen_cls=CStructuredCodeGenerator, + save_unoptimized_graph: bool = False, ): if not isinstance(func, Function): func = self.kb.functions[func] @@ -138,7 +147,6 @@ class Decompiler(Analysis): self._sp_tracker_track_memory = sp_tracker_track_memory self._peephole_optimizations = peephole_optimizations self._vars_must_struct = vars_must_struct - self._variable_kb = variable_kb self._expr_comments = expr_comments self._stmt_comments = stmt_comments self._ite_exprs = ite_exprs @@ -151,24 +159,29 @@ class Decompiler(Analysis): self._desired_variables = frozenset(desired_variables) if desired_variables else set() self._static_vvars = static_vvars if static_vvars is not None else {} self._static_buffers = static_buffers if static_buffers is not None else {} + self._save_unoptimized_graph = save_unoptimized_graph + # ``cfg`` is not in this dict: it is an input, not part of the decompilation result. Its identity is + # checked separately in :meth:`_can_use_decompilation_cache`. + # Collection-typed values are normalized to empty collections (never None) so the serialized cache does not + # need to distinguish None from empty. The exception is peephole_optimizations, where None means "use the + # default peephole set" and is distinct from an explicitly empty list. self._cache_parameters = ( { - "cfg": self._cfg, - "variable_kb": self._variable_kb, "options": {(o, v) for o, v in self._options if o.category != "Display" and v != o.default_value}, "optimization_passes": self._optimization_passes, "sp_tracker_track_memory": self._sp_tracker_track_memory, "peephole_optimizations": self._peephole_optimizations, - "vars_must_struct": self._vars_must_struct, + "vars_must_struct": self._vars_must_struct or set(), "flavor": self._flavor, - "expr_comments": self._expr_comments, - "stmt_comments": self._stmt_comments, - "ite_exprs": self._ite_exprs, - "binop_operators": self._binop_operators, + "expr_comments": self._expr_comments or {}, + "stmt_comments": self._stmt_comments or {}, + "ite_exprs": self._ite_exprs or set(), + "binop_operators": self._binop_operators or {}, "inline_functions": self._inline_functions, "desired_variables": self._desired_variables, "static_vvars": self._static_vvars, "static_buffers": self._static_buffers, + "save_unoptimized_graph": self._save_unoptimized_graph, } if use_cache else None @@ -238,9 +251,14 @@ class Decompiler(Analysis): def _can_use_decompilation_cache(self, cache: DecompilationCache) -> bool: if self._cache_parameters is None or cache.parameters is None: return False + # deserialized caches come back with cfg unset until the caller re-attaches it; unset is not a mismatch + if cache.cfg is not None and cache.cfg is not self._cfg: + return False a, b = self._cache_parameters, cache.parameters - id_checks = {"cfg", "variable_kb"} - return all(a[k] is b[k] if k in id_checks else a[k] == b[k] for k in self._cache_parameters) + if not b: + # AngrDB-loaded caches carry no recorded parameters; there is nothing to validate against + return True + return all(k in b and a[k] == b[k] for k in a) @staticmethod def _parse_options(options: list[tuple[DecompilationOption | str, Any]]) -> list[tuple[DecompilationOption, Any]]: @@ -260,6 +278,30 @@ class Decompiler(Analysis): with sprop_cache_scope(self._sprop_walker_cache): self._decompile() + def _reuse_cached_decompilation(self, cache, clinic, codegen) -> None: + """Full-reuse fast path: expose the cached clinic and codegen as this run's results without re-running the + pipeline. A live codegen's text is re-rendered to pick up in-place display edits; a freshly-deserialized + codegen (``_handlers is None``) keeps its stored text. The codegen inherits the cache's version and + timestamp.""" + codegen.version = cache.version + codegen.timestamp = cache.timestamp + if codegen._handlers is not None: + codegen.regenerate_text() + + self.cache = cache + self.clinic = clinic + self.codegen = codegen + self.seq_node = None + self.ail_graph = clinic.cc_graph + self.unoptimized_ail_graph = clinic.unoptimized_graph + self._variable_map = clinic.variable_map + self.vvar_id_start = clinic.vvar_id_start + self._copied_var_ids = clinic.copied_var_ids + + if self.update_cache: + self.kb.decompilations[(self.func.addr, self._flavor)] = cache + self._finish_progress() + @timethis def _decompile(self): if self.func.is_simprocedure: @@ -284,10 +326,28 @@ class Decompiler(Analysis): else: old_codegen = None old_clinic = None - ite_exprs = self._ite_exprs - binop_operators = self._binop_operators + # normalize to empty collections so the cache never stores None (passes treat None and empty the same) + ite_exprs = self._ite_exprs or set() + binop_operators = self._binop_operators or {} l.debug("Decompilation cache miss") + # Full-reuse fast path: with use_cache and without regen_clinic (the default), a valid cache short-circuits + # the entire pipeline and hands back the cached clinic and codegen. Requires an AST-carrying codegen (not + # DummyStructuredCodeGenerator) and this function's variables in kb.dec_variables; anything else falls + # through to a fresh decompilation. + if ( + self.use_cache + and not self._regen_clinic + and cache is not None + and old_clinic is not None + and old_codegen is not None + and not isinstance(old_codegen, DummyStructuredCodeGenerator) + and self.func.addr in self.kb.dec_variables + and self.func.prototype is not None + ): + self._reuse_cached_decompilation(cache, old_clinic, old_codegen) + return + self.options_by_class = defaultdict(list) if self._options: @@ -298,15 +358,7 @@ class Decompiler(Analysis): self._set_global_variables() self._update_progress(5.0, text="Converting to AIL") - variable_kb = self._variable_kb - # fall back to old codegen - if variable_kb is None and old_codegen is not None and isinstance(old_codegen, CStructuredCodeGenerator): - variable_kb = old_codegen._variable_kb - - if variable_kb is None: - reset_variable_names = True - else: - reset_variable_names = self.func.addr not in variable_kb.variables.function_managers + reset_variable_names = self.func.addr not in self.kb.dec_variables.function_managers # determine a few arguments according to the structuring algorithm fold_callexprs_into_conditions = False @@ -327,6 +379,7 @@ class Decompiler(Analysis): fold_callexprs_into_conditions = True cache = DecompilationCache(self.func.addr) + cache.cfg = self._cfg if self._cache_parameters is not None: cache.parameters = self._cache_parameters cache.ite_exprs = ite_exprs @@ -341,12 +394,17 @@ class Decompiler(Analysis): def progress_callback(p, **kwargs): return self._update_progress(p * (70 - 5) / 100.0 + 5, **kwargs) - if self._regen_clinic or old_clinic is None or self.func.prototype is None: + # a deserialized clinic whose function has no dec_variables cannot drive codegen; re-run Clinic instead + if ( + self._regen_clinic + or old_clinic is None + or self.func.prototype is None + or self.func.addr not in self.kb.dec_variables + ): clinic = self.project.analyses.Clinic( self.func, kb=self.kb, fail_fast=self._fail_fast, - variable_kb=variable_kb, reset_variable_names=reset_variable_names, optimization_passes=self._optimization_passes, sp_tracker_track_memory=self._sp_tracker_track_memory, @@ -371,6 +429,7 @@ class Decompiler(Analysis): notes=self.notes, static_vvars=self._static_vvars, static_buffers=self._static_buffers, + save_unoptimized_graph=self._save_unoptimized_graph, flavor=self._flavor, variable_map=variable_map, **self.options_to_params(self.options_by_class["clinic"]), @@ -380,13 +439,15 @@ class Decompiler(Analysis): # reuse the old, unaltered graph clinic.graph = clinic.cc_graph clinic.cc_graph = clinic.copy_graph() + # the SRDA model is tied to the previous run's graph; drop it so the simplification passes below + # regenerate it fresh for the reused graph + clinic.reaching_definitions = None self.clinic = clinic self.cache = cache # Make the VariableMap available on the cache regardless of whether Clinic re-linked variables (a partial # Clinic run, or the reuse-cached-Clinic path, may not repopulate cache.variable_map during linking). cache.variable_map = clinic.variable_map - self._variable_kb = clinic.variable_kb self._variable_map = clinic.variable_map self._update_progress(70.0, text="Identifying regions") self.vvar_id_start = clinic.vvar_id_start @@ -396,11 +457,13 @@ class Decompiler(Analysis): # the function is empty return - # expose a copy of the graph before any optimizations that may change the graph occur; - # use this graph if you need a reference of exact mapping of instructions to AIL statements - self.unoptimized_ail_graph = ( - clinic.unoptimized_graph if clinic.unoptimized_graph is not None else clinic.copy_graph() - ) + # expose a copy of the graph before any optimizations that may change the graph occur; use this graph if you + # need an exact instruction-to-AIL mapping. Only built when save_unoptimized_graph is set. clinic captured + # the snapshot iff a structure-altering pass ran; if none did, the current graph is itself unoptimized. + if self._save_unoptimized_graph: + self.unoptimized_ail_graph = ( + clinic.unoptimized_graph if clinic.unoptimized_graph is not None else clinic.copy_graph() + ) cond_proc = ConditionProcessor(self.project.arch, clinic._ail_manager) clinic.graph = self._run_graph_simplification_passes( @@ -458,8 +521,8 @@ class Decompiler(Analysis): # simplify it # Get variable manager for loop counter naming in RegionSimplifier variable_manager = None - if clinic.variable_kb is not None and self.func.addr in clinic.variable_kb.variables: - variable_manager = clinic.variable_kb.variables[self.func.addr] + if self.func.addr in self.kb.dec_variables: + variable_manager = self.kb.dec_variables[self.func.addr] region_simplifier_params = self.options_to_params(self.options_by_class["region_simplifier"]) # The Rust flavor forces if-else simplification off regardless of user options. region_simplifier_params.pop("simplify_ifelse", None) @@ -482,7 +545,7 @@ class Decompiler(Analysis): binop_operators=cache.binop_operators, goto_manager=s.goto_manager, graph=clinic.graph, - variable_kb=self._variable_kb, + kb=self.kb, ) # rewrite the sequence node to remove phi expressions @@ -501,7 +564,6 @@ class Decompiler(Analysis): ail_graph=clinic.graph, flavor=self._flavor, func_args=clinic.arg_list, - variable_kb=clinic.variable_kb, variable_map=clinic.variable_map, expr_comments=old_codegen.expr_comments if old_codegen is not None else None, stmt_comments=old_codegen.stmt_comments if old_codegen is not None else None, @@ -517,6 +579,10 @@ class Decompiler(Analysis): # save a copy of the AIL graph that is optimized but not modified by region identification self.ail_graph = clinic.cc_graph self.cache.codegen = codegen + if codegen is not None: + # copy the cache's version and timestamp onto the codegen + codegen.version = self.cache.version + codegen.timestamp = self.cache.timestamp self.cache.clinic = self.clinic # LLM refinement pass @@ -591,7 +657,7 @@ class Decompiler(Analysis): blocks_by_addr=addr_to_blocks, blocks_by_addr_and_idx=addr_and_idx_to_blocks, graph=ail_graph, - variable_kb=self._variable_kb, + kb=self.kb, reaching_definitions=reaching_definitions, entry_node_addr=self.clinic.entry_node_addr, scratch=self._optimization_scratch, @@ -656,7 +722,7 @@ class Decompiler(Analysis): blocks_by_addr=addr_to_blocks, blocks_by_addr_and_idx=addr_and_idx_to_blocks, graph=ail_graph, - variable_kb=self._variable_kb, + kb=self.kb, arg_vvars=arg_vvars, region_identifier=ri, reaching_definitions=reaching_definitions, @@ -747,13 +813,13 @@ class Decompiler(Analysis): # nothing to reflow; but this should not happen return None - var_kb = self._variable_kb if self._variable_kb is not None else KnowledgeBase(self.project) + var_kb = self.kb - if self.func.addr not in var_kb.variables: + if self.func.addr not in var_kb.dec_variables: # for some reason variables for the current function don't really exist... groundtruth = {} else: - var_manager = var_kb.variables[self.func.addr] + var_manager = var_kb.dec_variables[self.func.addr] # ground-truth types groundtruth = {} for variable in var_manager.variables_with_manual_types: @@ -816,7 +882,7 @@ class Decompiler(Analysis): and isinstance(codegen, CStructuredCodeGenerator) and codegen.cfunc is not None ): - var_manager = var_kb.variables[self.func.addr] + var_manager = var_kb.dec_variables[self.func.addr] for i, arg in enumerate(codegen.cfunc.arg_list): if i >= len(self.func.prototype.args): break @@ -885,12 +951,10 @@ class Decompiler(Analysis): :param ail_graph: The AIL graph to transform out of SSA form. :return: The translated AIL graph. """ - variable_kb = self._variable_kb dephication = self.project.analyses.GraphDephication( self.func, ail_graph, rewrite=True, - variable_kb=variable_kb, variable_map=self._variable_map, kb=self.kb, fail_fast=self._fail_fast, @@ -898,12 +962,10 @@ class Decompiler(Analysis): return dephication.output def transform_seqnode_from_ssa(self, seq_node: SequenceNode) -> SequenceNode: - variable_kb = self._variable_kb dephication = self.project.analyses.SeqNodeDephication( self.func, seq_node, rewrite=True, - variable_kb=variable_kb, variable_map=self._variable_map, kb=self.kb, fail_fast=self._fail_fast, @@ -964,7 +1026,7 @@ class Decompiler(Analysis): return False # collect unified variables - varman = self._variable_kb.variables[self.func.addr] + varman = self.kb.dec_variables[self.func.addr] unified_vars = varman.get_unified_variables(sort=None) # also collect argument variables @@ -1093,7 +1155,7 @@ class Decompiler(Analysis): if not code_text: return False - varman = self._variable_kb.variables[self.func.addr] + varman = self.kb.dec_variables[self.func.addr] unified_vars = varman.get_unified_variables(sort=None) if not unified_vars: diff --git a/angr/analyses/decompiler/dephication/dephication_base.py b/angr/analyses/decompiler/dephication/dephication_base.py index 0a8995339..2cdde8929 100644 --- a/angr/analyses/decompiler/dephication/dephication_base.py +++ b/angr/analyses/decompiler/dephication/dephication_base.py @@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any from angr.analyses.analysis import Analysis if TYPE_CHECKING: - from angr import KnowledgeBase from angr.analyses.decompiler.variable_map import VariableMap l = logging.getLogger(name=__name__) @@ -24,7 +23,6 @@ class DephicationBase(Analysis): func, vvar_to_vvar_mapping: dict[int, int] | None = None, rewrite: bool = False, - variable_kb: KnowledgeBase | None = None, variable_map: VariableMap | None = None, ): if isinstance(func, str): @@ -32,7 +30,6 @@ class DephicationBase(Analysis): else: self._function = func - self.variable_kb = variable_kb self.vvar_to_vvar_mapping = vvar_to_vvar_mapping if vvar_to_vvar_mapping is not None else None self.variable_map = variable_map self.rewrite = rewrite diff --git a/angr/analyses/decompiler/dephication/graph_dephication.py b/angr/analyses/decompiler/dephication/graph_dephication.py index 5f1d240c0..6086592fe 100644 --- a/angr/analyses/decompiler/dephication/graph_dephication.py +++ b/angr/analyses/decompiler/dephication/graph_dephication.py @@ -15,7 +15,6 @@ from .dephication_base import DephicationBase from .graph_rewriting import GraphRewritingAnalysis if TYPE_CHECKING: - from angr import KnowledgeBase from angr.analyses.decompiler.variable_map import VariableMap @@ -34,7 +33,6 @@ class GraphDephication(DephicationBase): # pylint:disable=abstract-method ail_graph, vvar_to_vvar_mapping: dict[int, int] | None = None, rewrite: bool = False, - variable_kb: KnowledgeBase | None = None, variable_map: VariableMap | None = None, ): """ @@ -48,7 +46,6 @@ class GraphDephication(DephicationBase): # pylint:disable=abstract-method func, vvar_to_vvar_mapping=vvar_to_vvar_mapping, rewrite=rewrite, - variable_kb=variable_kb, variable_map=variable_map, ) @@ -76,7 +73,7 @@ class GraphDephication(DephicationBase): # pylint:disable=abstract-method self._function, self._graph, self.vvar_to_vvar_mapping, - variable_kb=self.variable_kb, + kb=self.kb, variable_map=self.variable_map, ) return rewriter.out_graph diff --git a/angr/analyses/decompiler/dephication/graph_rewriting.py b/angr/analyses/decompiler/dephication/graph_rewriting.py index 05047e7f1..3ae44d4e1 100644 --- a/angr/analyses/decompiler/dephication/graph_rewriting.py +++ b/angr/analyses/decompiler/dephication/graph_rewriting.py @@ -26,13 +26,13 @@ class GraphRewritingAnalysis(ForwardAnalysis[None, NodeType, object, object, obj func, ail_graph, vvar_to_vvar: dict[int, int], - variable_kb=None, + kb=None, variable_map=None, ): self.project = project self._function = func self._graph_visitor = FunctionGraphVisitor(self._function, ail_graph) - self.variable_kb = variable_kb + self._dvars_kb = kb self.variable_map = variable_map ForwardAnalysis.__init__( @@ -44,7 +44,7 @@ class GraphRewritingAnalysis(ForwardAnalysis[None, NodeType, object, object, obj self.project, self._vvar_to_vvar, func_addr=self._function.addr, - variable_kb=self.variable_kb, + kb=self._dvars_kb, variable_map=self.variable_map, ) diff --git a/angr/analyses/decompiler/dephication/rewriting_engine.py b/angr/analyses/decompiler/dephication/rewriting_engine.py index 26ecd5956..f57de4f6c 100644 --- a/angr/analyses/decompiler/dephication/rewriting_engine.py +++ b/angr/analyses/decompiler/dephication/rewriting_engine.py @@ -56,7 +56,7 @@ class SimEngineDephiRewriting(SimEngineNostmtAIL[None, Expression | None, Statem project, vvar_to_vvar: dict[int, int], func_addr: int | None = None, - variable_kb: KnowledgeBase | None = None, + kb: KnowledgeBase | None = None, variable_map: VariableMap | None = None, ): super().__init__(project) @@ -64,7 +64,7 @@ class SimEngineDephiRewriting(SimEngineNostmtAIL[None, Expression | None, Statem self.vvar_to_vvar = vvar_to_vvar self.out_block = None self.func_addr = func_addr - self.variable_kb = variable_kb + self._dvars_kb = kb self.variable_map = variable_map self._stmt_handlers["IncompleteSwitchCaseHeadStatement"] = self._handle_stmt_IncompleteSwitchCaseHeadStatement @@ -121,13 +121,13 @@ class SimEngineDephiRewriting(SimEngineNostmtAIL[None, Expression | None, Statem return () if ( self.func_addr is not None - and self.variable_kb is not None - and self.func_addr in self.variable_kb.variables + and self._dvars_kb is not None + and self.func_addr in self._dvars_kb.dec_variables and self.variable_map is not None ): dst_var = self.variable_map.variable(dst) src_var = self.variable_map.variable(src) - var_manager = self.variable_kb.variables[self.func_addr] + var_manager = self._dvars_kb.dec_variables[self.func_addr] if ( dst_var is not None and src_var is not None diff --git a/angr/analyses/decompiler/dephication/seqnode_dephication.py b/angr/analyses/decompiler/dephication/seqnode_dephication.py index 35572400e..1b33f830f 100644 --- a/angr/analyses/decompiler/dephication/seqnode_dephication.py +++ b/angr/analyses/decompiler/dephication/seqnode_dephication.py @@ -63,8 +63,8 @@ class SeqNodeRewriter(SequenceWalker): seq_node: SequenceNode, vvar_to_vvar: dict[int, int], project: angr.Project, - variable_kb: KnowledgeBase | None = None, func_addr: int | None = None, + kb: KnowledgeBase | None = None, variable_map: VariableMap | None = None, ): super().__init__( @@ -81,7 +81,7 @@ class SeqNodeRewriter(SequenceWalker): self.vvar_to_vvar = vvar_to_vvar self.variable_map = variable_map self.engine = SimEngineDephiRewriting( - project, self.vvar_to_vvar, func_addr=func_addr, variable_kb=variable_kb, variable_map=self.variable_map + project, self.vvar_to_vvar, func_addr=func_addr, kb=kb, variable_map=self.variable_map ) self.output = self.walk(seq_node) @@ -134,14 +134,12 @@ class SeqNodeDephication(DephicationBase): seq_node, vvar_to_vvar_mapping: dict[int, int] | None = None, rewrite: bool = False, - variable_kb: KnowledgeBase | None = None, variable_map: VariableMap | None = None, ): super().__init__( func, vvar_to_vvar_mapping=vvar_to_vvar_mapping, rewrite=rewrite, - variable_kb=variable_kb, variable_map=variable_map, ) @@ -160,7 +158,7 @@ class SeqNodeDephication(DephicationBase): self.vvar_to_vvar_mapping, self.project, func_addr=self._function.addr, - variable_kb=self.variable_kb, + kb=self.kb, variable_map=self.variable_map, ) return rewriter.output diff --git a/angr/analyses/decompiler/notes/__init__.py b/angr/analyses/decompiler/notes/__init__.py index 433dc69fa..798bc6ce0 100644 --- a/angr/analyses/decompiler/notes/__init__.py +++ b/angr/analyses/decompiler/notes/__init__.py @@ -2,7 +2,12 @@ from __future__ import annotations from .decompilation_note import DecompilationNote, DecompilationNoteLevel +# importing the module registers the subclass for DecompilationNote.from_json dispatch +from .deobfuscated_strings import DeobfuscatedString, DeobfuscatedStringsNote + __all__ = ( "DecompilationNote", "DecompilationNoteLevel", + "DeobfuscatedString", + "DeobfuscatedStringsNote", ) diff --git a/angr/analyses/decompiler/notes/decompilation_note.py b/angr/analyses/decompiler/notes/decompilation_note.py index 7c842c788..72e594b38 100644 --- a/angr/analyses/decompiler/notes/decompilation_note.py +++ b/angr/analyses/decompiler/notes/decompilation_note.py @@ -1,8 +1,12 @@ from __future__ import annotations +import json +import logging from enum import Enum from typing import Any +l = logging.getLogger(name=__name__) + class DecompilationNoteLevel(Enum): """ @@ -28,6 +32,8 @@ class DecompilationNote: DecompilationNoteLevel.INFO, DecompilationNoteLevel.WARNING, and DecompilationNoteLevel.CRITICAL. """ + _subclasses: dict[str, type[DecompilationNote]] = {} + __slots__ = ( "content", "key", @@ -41,8 +47,51 @@ class DecompilationNote: self.content = content self.level = level + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + DecompilationNote._subclasses[cls.__name__] = cls + def __repr__(self): return f"" def __str__(self): return f"{self.name}: {self.content}" + + # + # JSON serialization + # + + def to_jsonable(self) -> dict[str, Any]: + try: + content = json.loads(json.dumps(self.content)) + except (TypeError, ValueError): + l.warning("Failed to serialize content of decompilation note %s to JSON", self.key) + content = None + return { + "class": type(self).__name__, + "key": self.key, + "name": self.name, + "content": content, + "level": int(self.level.value), + } + + def to_json(self) -> str: + return json.dumps(self.to_jsonable()) + + @classmethod + def from_jsonable(cls, d: dict[str, Any]) -> DecompilationNote: + klass = cls._subclasses.get(d.get("class", ""), DecompilationNote) + return klass._from_jsonable_impl(d) # pylint:disable=protected-access + + @classmethod + def _from_jsonable_impl(cls, d: dict[str, Any]) -> DecompilationNote: + return cls( + key=d["key"], + name=d["name"], + content=d.get("content"), + level=DecompilationNoteLevel(d.get("level", DecompilationNoteLevel.INFO.value)), + ) + + @classmethod + def from_json(cls, s: str) -> DecompilationNote: + return cls.from_jsonable(json.loads(s)) diff --git a/angr/analyses/decompiler/notes/deobfuscated_strings.py b/angr/analyses/decompiler/notes/deobfuscated_strings.py index 387fb58dc..254cfa0e1 100644 --- a/angr/analyses/decompiler/notes/deobfuscated_strings.py +++ b/angr/analyses/decompiler/notes/deobfuscated_strings.py @@ -1,5 +1,8 @@ from __future__ import annotations +import base64 +from typing import Any + from .decompilation_note import DecompilationNote @@ -54,3 +57,26 @@ class DeobfuscatedStringsNote(DecompilationNote): lines.append(f" Type {deobf_str.type} @ {deobf_str.ref_addr:#x}: {deobf_str.value!r}") return "\n".join(lines) + + # + # JSON serialization + # + + def to_jsonable(self) -> dict[str, Any]: + d = super().to_jsonable() + d["strings"] = [ + { + "ref_addr": s.ref_addr, + "type": s.type, + "value": base64.b64encode(s.value).decode("ascii"), + } + for _, s in sorted(self.strings.items()) + ] + return d + + @classmethod + def _from_jsonable_impl(cls, d: dict[str, Any]) -> DeobfuscatedStringsNote: + note = cls(key=d["key"], name=d["name"]) + for entry in d.get("strings", []): + note.add_string(entry["type"], base64.b64decode(entry["value"]), ref_addr=entry["ref_addr"]) + return note diff --git a/angr/analyses/decompiler/optimization_pass_registry.py b/angr/analyses/decompiler/optimization_pass_registry.py new file mode 100644 index 000000000..700181336 --- /dev/null +++ b/angr/analyses/decompiler/optimization_pass_registry.py @@ -0,0 +1,31 @@ +""" +Stable name resolution for decompiler optimization passes and peephole optimizations. +""" + +from __future__ import annotations + + +def _known_passes() -> dict[str, type]: + # Recomputed on each call to pick up classes added via ``register_optimization_pass`` after this module is imported. + + from .optimization_passes import ALL_OPTIMIZATION_PASSES # pylint:disable=import-outside-toplevel + from .peephole_optimizations import ALL_PEEPHOLE_OPTS # pylint:disable=import-outside-toplevel + + return {cls.__qualname__: cls for cls in (*ALL_OPTIMIZATION_PASSES, *ALL_PEEPHOLE_OPTS)} + + +def pass_to_name(cls: type) -> str: + """Return a stable string identifier (the class name) for an optimization pass or peephole class.""" + return cls.__qualname__ + + +def name_to_pass(name: str) -> type: + """Resolve a class name back to its registered pass class. + + :raises KeyError: if ``name`` does not refer to a class in + ``ALL_OPTIMIZATION_PASSES`` or ``ALL_PEEPHOLE_OPTS``. + """ + return _known_passes()[name] + + +__all__ = ("name_to_pass", "pass_to_name") diff --git a/angr/analyses/decompiler/optimization_passes/expr_op_swapper.py b/angr/analyses/decompiler/optimization_passes/expr_op_swapper.py index 8ef3461a7..b518892db 100644 --- a/angr/analyses/decompiler/optimization_passes/expr_op_swapper.py +++ b/angr/analyses/decompiler/optimization_passes/expr_op_swapper.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import logging from collections.abc import Callable from typing import TYPE_CHECKING, Any @@ -90,7 +91,7 @@ class ExpressionReplacer(AILBlockRewriter): class OpDescriptor: """ - Describes a specific operator. + Describes a specific operator. Serializes to JSON. """ def __init__(self, block_addr: int, stmt_idx: int, ins_addr: int, op: str): @@ -111,6 +112,29 @@ class OpDescriptor: and self.op == other.op ) + # + # JSON serialization + # + + def to_jsonable(self) -> dict[str, Any]: + return { + "block_addr": self.block_addr, + "stmt_idx": self.stmt_idx, + "ins_addr": self.ins_addr, + "op": self.op, + } + + def to_json(self) -> str: + return json.dumps(self.to_jsonable()) + + @classmethod + def from_jsonable(cls, d: dict[str, Any]) -> OpDescriptor: + return cls(d["block_addr"], d["stmt_idx"], d["ins_addr"], d["op"]) + + @classmethod + def from_json(cls, s: str) -> OpDescriptor: + return cls.from_jsonable(json.loads(s)) + class ExprOpSwapper(SequenceOptimizationPass): """ diff --git a/angr/analyses/decompiler/optimization_passes/optimization_pass.py b/angr/analyses/decompiler/optimization_passes/optimization_pass.py index 4a9371c49..1bac30917 100644 --- a/angr/analyses/decompiler/optimization_passes/optimization_pass.py +++ b/angr/analyses/decompiler/optimization_passes/optimization_pass.py @@ -133,7 +133,7 @@ class OptimizationPass(BaseOptimizationPass): graph, blocks_by_addr=None, blocks_by_addr_and_idx=None, - variable_kb=None, + kb=None, region_identifier=None, reaching_definitions=None, vvar_id_start: int = 0, @@ -155,7 +155,7 @@ class OptimizationPass(BaseOptimizationPass): self._blocks_by_addr: dict[int, set[ailment.Block]] = blocks_by_addr or {} self._blocks_by_addr_and_idx: dict[tuple[int, int | None], ailment.Block] = blocks_by_addr_and_idx or {} self._graph = graph - self._variable_kb = variable_kb + self._dvars_kb = kb self._ri = region_identifier self._rd = reaching_definitions self._scratch = scratch if scratch is not None else {} diff --git a/angr/analyses/decompiler/structured_codegen/base.py b/angr/analyses/decompiler/structured_codegen/base.py index 9b6232ba3..b66505a1b 100644 --- a/angr/analyses/decompiler/structured_codegen/base.py +++ b/angr/analyses/decompiler/structured_codegen/base.py @@ -143,7 +143,13 @@ class BaseStructuredCodeGenerator: self.expr_comments: dict[int, str] = expr_comments if expr_comments is not None else {} self.stmt_comments: dict[int, str] = stmt_comments if stmt_comments is not None else {} self.const_formats: dict[IdentType, dict[str, bool]] = const_formats if const_formats is not None else {} - self.idx_counters: dict[str, count] = {} + # angr version and decompile time copied from the owning DecompilationCache; "" / 0 = unknown + self.version: str = "" + self.timestamp: int = 0 + self.ident_counters: dict[str, count] = {} + # node idx allocator; 0 is reserved (serialization uses it as the "absent" sentinel), and the counter is + # never reset so idx values are unique across the lifetime of this codegen + self._next_node_idx: int = 1 @staticmethod def adjust_mapping_positions( @@ -185,10 +191,15 @@ class BaseStructuredCodeGenerator: def reload_variable_types(self) -> None: pass - def next_idx(self, key: str) -> str: - if key not in self.idx_counters: - self.idx_counters[key] = count() - return f"{key}_{next(self.idx_counters[key])}" + def next_ident(self, key: str) -> str: + if key not in self.ident_counters: + self.ident_counters[key] = count() + return f"{key}_{next(self.ident_counters[key])}" - def reset_idx_counters(self) -> None: - self.idx_counters = {} + def reset_ident_counters(self) -> None: + self.ident_counters = {} + + def next_node_idx(self) -> int: + v = self._next_node_idx + self._next_node_idx += 1 + return v diff --git a/angr/analyses/decompiler/structured_codegen/c.py b/angr/analyses/decompiler/structured_codegen/c.py index 4e1793414..c8203ec8d 100644 --- a/angr/analyses/decompiler/structured_codegen/c.py +++ b/angr/analyses/decompiler/structured_codegen/c.py @@ -1,4 +1,4 @@ -# pylint:disable=missing-class-docstring,too-many-boolean-expressions,unused-argument,no-self-use +# pylint:disable=missing-class-docstring,too-many-boolean-expressions,unused-argument,no-self-use,protected-access from __future__ import annotations import logging @@ -33,6 +33,7 @@ from angr.analyses.decompiler.variable_map import VariableMap from angr.errors import UnsupportedNodeTypeError from angr.knowledge_plugins.cfg.memory_data import MemoryData, MemoryDataSort from angr.knowledge_plugins.functions import Function +from angr.serializable import Serializable from angr.sim_type import ( SimCppClass, SimStruct, @@ -339,12 +340,16 @@ class CConstruct: Acts as the base class for all other representation constructions. """ - __slots__ = ("codegen", "idx", "tags") + __slots__ = ("codegen", "ident", "idx", "tags") def __init__(self, codegen, tags=None): + # a CConstruct cannot exist without its owning codegen: ``idx`` (the per-codegen unique node identity) and + # ``ident`` (a per-class-name display label; NOT unique) are both allocated from it + assert codegen is not None self.tags = tags or {} - self.codegen: CStructuredCodeGenerator = codegen # type: ignore[assignment] - self.idx = codegen.next_idx(self.__class__.__name__) + self.codegen: CStructuredCodeGenerator = codegen + self.ident: str = codegen.next_ident(self.__class__.__name__) + self.idx: int = codegen.next_node_idx() def c_repr(self, initial_pos=0, indent=0, pos_to_node=None, pos_to_addr=None, addr_to_pos=None): """ @@ -731,7 +736,7 @@ class CFunction(CConstruct): # pylint:disable=abstract-method if isinstance(field, SimStruct) and field not in extern_types: if field.name and not field.fields and field.name in defined_struct_names: continue - extern_types.append(field) + extern_types.append(field) # pylint:disable=modified-iterating-list # Emit in reverse order: nested structs first for ty in reversed(extern_types): @@ -853,7 +858,7 @@ class CStatement(CConstruct): # pylint:disable=abstract-method Represents a statement in C. """ - def __init__(self, tags=None, codegen=None): + def __init__(self, tags=None, *, codegen): super().__init__(codegen=codegen, tags=tags) @@ -864,7 +869,7 @@ class CExpression(CConstruct): __slots__ = ("_type", "collapsed") - def __init__(self, collapsed=False, tags=None, codegen=None): + def __init__(self, collapsed=False, tags=None, *, codegen): super().__init__(codegen=codegen, tags=tags) self._type = None self.collapsed = collapsed @@ -1511,7 +1516,8 @@ class CFunctionCall(CExpression): show_demangled_name=True, show_disambiguated_name: bool = True, tags=None, - codegen=None, + *, + codegen, **kwargs, ): super().__init__(tags=tags, codegen=codegen, **kwargs) @@ -2494,7 +2500,8 @@ class CConstant(CExpression): return yield hex(self.reference_values[self._type]), self return - elif isinstance(self._type, SimTypePointer) and isinstance(self._type.pts_to, SimTypeChar): + + if isinstance(self._type, SimTypePointer) and isinstance(self._type.pts_to, SimTypeChar): refval = self.reference_values[self._type] if isinstance(refval, MemoryData): v = refval.content.decode("utf-8") if refval.content else f"" @@ -2506,7 +2513,8 @@ class CConstant(CExpression): assert isinstance(v, str) yield CConstant.str_to_c_str(v, maxlen=self.codegen.max_str_len), self return - elif isinstance(self._type, SimTypePointer) and isinstance(self._type.pts_to, SimTypeWideChar): + + if isinstance(self._type, SimTypePointer) and isinstance(self._type.pts_to, SimTypeWideChar): refval = self.reference_values[self._type] if isinstance(refval, MemoryData): v = decode_utf16_string(refval.content) if refval.content else f"" @@ -2516,14 +2524,14 @@ class CConstant(CExpression): assert False, f"Unexpected reference value type {type(refval)} for wide char pointer" yield CConstant.str_to_c_str(v, prefix="L", maxlen=self.codegen.max_str_len), self return - else: - if isinstance(self.reference_values[self._type], int): - yield self.fmt_int(self.reference_values[self._type]), self - return - o = _default_output(self.reference_values[self.type]) - if o is not None: - yield o, self - return + + if isinstance(self.reference_values[self._type], int): + yield self.fmt_int(self.reference_values[self._type]), self + return + o = _default_output(self.reference_values[self.type]) + if o is not None: + yield o, self + return # default priority: string references -> variables -> other reference values for _ty, v in self.reference_values.items(): # pylint:disable=unused-variable @@ -2768,14 +2776,13 @@ class CStructFieldNameDef: self.name = name -class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): +class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis, Serializable): def __init__( self, func, sequence, indent=0, cfg=None, - variable_kb=None, func_args: list[SimVariable] | None = None, binop_depth_cutoff: int = 16, show_casts=True, @@ -2863,7 +2870,6 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): self._func_args = func_args self._cfg = cfg self._sequence = sequence - self._variable_kb = variable_kb if variable_kb is not None else self.kb self._variable_map: VariableMap = variable_map if variable_map is not None else VariableMap() self.binop_depth_cutoff = binop_depth_cutoff @@ -2942,7 +2948,7 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): arg_list = [self._variable(arg, None) for arg in self._func_args] if self._func_args else [] - self.reset_idx_counters() + self.reset_ident_counters() obj = self._handle(self._sequence) self.cnode2ailexpr = {v: k[0] for k, v in self.ailexpr2cnode.items()} @@ -2954,7 +2960,7 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): arg_list, obj, self._variables_in_use, - self._variable_kb.variables[self._func.addr], + self.kb.dec_variables[self._func.addr], demangled_name=self._func.demangled_name, show_demangled_name=self.show_demangled_name, codegen=self, @@ -3059,8 +3065,8 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): def _get_variable_type(self, var, is_global=False): if is_global: - return self._variable_kb.variables["global"].get_variable_type(var) - return self._variable_kb.variables[self._func.addr].get_variable_type(var) + return self.kb.dec_variables["global"].get_variable_type(var) + return self.kb.dec_variables[self._func.addr].get_variable_type(var) def _get_derefed_type(self, ty: SimType) -> SimType | None: if ty is None: @@ -3117,7 +3123,7 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): ) -> CVariable: # TODO: we need to fucking make sure that variable recovery and type inference actually generates a size # TODO: for each variable it links into the fucking ail. then we can remove fallback_type_size. - unified = self._variable_kb.variables[self._func.addr].unified_variable(variable) + unified = self.kb.dec_variables[self._func.addr].unified_variable(variable) variable_type = self._get_variable_type( variable, is_global=isinstance(variable, SimMemoryVariable) and not isinstance(variable, SimStackVariable) ) @@ -3692,7 +3698,7 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): return CAssignment(cdst, cdata, tags=stmt.tags, codegen=self) def variables_unify(self, v1: Expr.VirtualVariable, v2: Expr.VirtualVariable) -> bool: - vmi = self._variable_kb.variables[self._func.addr] + vmi = self.kb.dec_variables[self._func.addr] v1_var = self._variable_map.variable(v1) v2_var = self._variable_map.variable(v2) v1v = vmi.unified_variable(v1_var) if v1_var is not None else None @@ -4360,6 +4366,27 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): stack_base = CFakeVariable("stack_base", SimTypePointer(SimTypeBottom()), codegen=self) return CBinaryOp("Add", stack_base, CConstant(expr.offset, SimTypeInt(), codegen=self), codegen=self) + # + # Serialization + # + + @classmethod + def _get_cmsg(cls): + from angr.protos import codegen_pb2 # pylint:disable=import-outside-toplevel + + return codegen_pb2.Codegen() # pylint:disable=no-member + + def serialize_to_cmessage(self): + from . import c_serialize # pylint:disable=import-outside-toplevel + + return c_serialize.serialize_codegen(self) + + @classmethod + def parse_from_cmessage(cls, cmsg, *, project=None, kb=None, func=None, **kwargs): + from . import c_serialize # pylint:disable=import-outside-toplevel + + return c_serialize.parse_codegen(cmsg, project=project, kb=kb, func=func) + class CStructuredCodeWalker: def handle(self, obj): @@ -4607,3 +4634,10 @@ class PointerArithmeticFixer(CStructuredCodeWalker): # StructuredCodeGenerator = CStructuredCodeGenerator register_analysis(CStructuredCodeGenerator, "CStructuredCodeGenerator") + + +# Register protobuf serializer/parser pairs for every concrete CConstruct subclass. Imported after all classes are +# defined so that ``c_serialize.register_all`` can reference them by name. +from . import c_serialize as _c_serialize # noqa: E402 # pylint: disable=wrong-import-position + +_c_serialize.register_all() diff --git a/angr/analyses/decompiler/structured_codegen/c_serialize.py b/angr/analyses/decompiler/structured_codegen/c_serialize.py new file mode 100644 index 000000000..d7f316ee7 --- /dev/null +++ b/angr/analyses/decompiler/structured_codegen/c_serialize.py @@ -0,0 +1,1319 @@ +""" +Protobuf serialization helpers for the C AST defined in :mod:`c`. + +The AST is serialized as a flat indexed table of :class:`CConstructNode` cmessages: every :class:`CConstruct` instance +is assigned a non-zero ``uint32`` node_id at serialize time, and child CConstructs are referenced by id. This avoids +unbounded recursion in protobuf encoding and lets :class:`PositionMapping`, ``cexterns``, and ``map_addr_to_label`` +reference into the AST without duplicating subtrees. +""" +# pylint:disable=no-member,protected-access + +from __future__ import annotations + +import json +import zlib +from collections import defaultdict +from collections.abc import Callable +from typing import Any + +from angr import sim_variable +from angr.analyses.decompiler.notes import DecompilationNote +from angr.knowledge_plugins.cfg.memory_data import MemoryData +from angr.protos import codegen_pb2 +from angr.rustylib.ailment import Block as AilBlock +from angr.rustylib.ailment import Expression as AilExpression +from angr.rustylib.ailment import Statement as AilStatement +from angr.sim_type import SimType +from angr.sim_variable import SimVariable + +from .base import InstructionMapping, PositionMapping +from .c import ( + CITE, + CAILBlock, + CAssignment, + CBinaryOp, + CBreak, + CConstant, + CConstruct, + CContinue, + CDirtyExpression, + CDirtyStatement, + CDoWhileLoop, + CExpression, + CExpressionStatement, + CFakeVariable, + CForLoop, + CFunction, + CFunctionCall, + CGoto, + CIfBreak, + CIfElse, + CIncompleteSwitchCase, + CIndexedVariable, + CLabel, + CMultiStatementExpression, + CRegister, + CReturn, + CStatements, + CStructField, + CStructuredCodeGenerator, + CSwitchCase, + CTypeCast, + CUnaryOp, + CUnsupportedStatement, + CVariable, + CVariableField, + CVEXCCallExpression, + CWhileLoop, +) + +# --------------------------------------------------------------------------------------------------------------------- +# Tag sanitization +# --------------------------------------------------------------------------------------------------------------------- + + +# The four dominant tag keys get typed proto fields; (key, required type) pairs. +_TYPED_TAGS = ( + ("ins_addr", int), + ("vex_block_addr", int), + ("vex_stmt_idx", int), + ("is_prototype_guessed", bool), +) + + +def _sanitize_tags(tags: dict | None) -> tuple[tuple, codegen_pb2.CConstructTags | None]: + """Split a tags dict into a CConstructTags message (typed fields for the dominant keys, JSON for the rest; + entries we can't JSON-encode are dropped) plus a hashable canonical key for interning. Returns + ``(key, None)`` for empty tag dicts.""" + if not tags: + return (), None + out = codegen_pb2.CConstructTags() + key_parts = [] + any_set = False + for k, v in tags.items(): + if not isinstance(k, str): + continue + typed = next((t for name, t in _TYPED_TAGS if name == k), None) + if typed is not None and type(v) is typed: + try: + setattr(out, k, v) + except ValueError: + pass # out of range for the proto field; fall through to the JSON encoding + else: + key_parts.append((k, v)) + any_set = True + continue + try: + out.json_values[k] = json.dumps(v) + except (TypeError, ValueError): + continue # silently drop non-JSON-serializable values; they will be missing after round-trip + key_parts.append((k, out.json_values[k])) + any_set = True + if not any_set: + return (), None + if out.HasField("ins_addr") and out.HasField("vex_block_addr"): + # store ins_addr as a small delta from vex_block_addr instead of a second absolute address + out.ins_offset = out.ins_addr - out.vex_block_addr + out.ClearField("ins_addr") + return tuple(sorted(key_parts)), out + + +def _parse_tags(cmsg: codegen_pb2.CConstructTags) -> dict: + out = {k: json.loads(v) for k, v in cmsg.json_values.items()} + for name, _ in _TYPED_TAGS: + if cmsg.HasField(name): + out[name] = getattr(cmsg, name) + if cmsg.HasField("ins_offset"): + # ins_offset is only ever produced alongside vex_block_addr + out["ins_addr"] = cmsg.vex_block_addr + cmsg.ins_offset + return out + + +# --------------------------------------------------------------------------------------------------------------------- +# SimType / SimVariable plumbing +# --------------------------------------------------------------------------------------------------------------------- + + +def _simvar_to_bytes(v: SimVariable | None) -> bytes: + """Encode a SimVariable as ``b"\\0"`` so the dispatch type tag round-trips with the payload. + SimVariable subclasses each define their own _pb2 message; we need the type tag to know which Serializable.parse + classmethod to dispatch through.""" + if v is None: + return b"" + return type(v).__name__.encode("ascii") + b"\0" + v.serialize() + + +def _simvar_from_bytes(b: bytes) -> SimVariable | None: + if not b: + return None + sep = b.index(b"\0") + cls_name = b[:sep].decode("ascii") + payload = b[sep + 1 :] + cls = getattr(sim_variable, cls_name) + return cls.parse(payload) + + +# --------------------------------------------------------------------------------------------------------------------- +# Serialize / parse contexts +# --------------------------------------------------------------------------------------------------------------------- + + +class SerializeContext: + """ + Tracks which nodes have been serialized while walking the C AST. + """ + + __slots__ = ("_node_config", "_seen", "_simvar_pool", "_tag_pool", "_tag_pool_msgs", "_type_pool", "nodes") + + def __init__(self) -> None: + self._seen: set[int] = set() # stores CConstruct.idx for serialized nodes + self.nodes: list[codegen_pb2.CConstructNode] = [] + # SimType JSON interning: json string -> ref (index+1 into Codegen.type_pool; 0 means absent) + self._type_pool: dict[str, int] = {} + # tag-dict interning: canonical key -> ref (index+1 into Codegen.tag_pool; 0 means no tags) + self._tag_pool: dict[tuple, int] = {} + self._tag_pool_msgs: list[codegen_pb2.CConstructTags] = [] + # SimVariable payload interning: payload bytes -> ref (index+1 into Codegen.simvar_pool; 0 means absent) + self._simvar_pool: dict[bytes, int] = {} + # out-of-line per-node display config: only nodes deviating from the defaults are recorded here + self._node_config: list[codegen_pb2.NodeConfigEntry] = [] + + def intern_type(self, t: SimType | None) -> int: + """Intern a SimType's JSON encoding and return its ref (index+1 into the type pool); 0 for None.""" + if t is None: + return 0 + key = json.dumps(t.to_json()) + ref = self._type_pool.get(key) + if ref is None: + ref = len(self._type_pool) + 1 + self._type_pool[key] = ref + return ref + + @property + def type_pool(self) -> list[str]: + return list(self._type_pool) # insertion-ordered: pool[i] has ref i+1 + + def intern_tags(self, tags: dict | None) -> int: + """Intern a sanitized tags dict and return its ref (index+1 into the tag pool); 0 for no tags.""" + key, msg = _sanitize_tags(tags) + if msg is None: + return 0 + ref = self._tag_pool.get(key) + if ref is None: + ref = len(self._tag_pool) + 1 + self._tag_pool[key] = ref + self._tag_pool_msgs.append(msg) + return ref + + @property + def tag_pool(self) -> list[codegen_pb2.CConstructTags]: + return self._tag_pool_msgs + + def intern_simvar(self, v: SimVariable | None) -> int: + """Intern a SimVariable's polymorphic payload and return its ref (index+1 into the pool); 0 for None.""" + if v is None: + return 0 + payload = _simvar_to_bytes(v) + ref = self._simvar_pool.get(payload) + if ref is None: + ref = len(self._simvar_pool) + 1 + self._simvar_pool[payload] = ref + return ref + + @property + def simvar_pool(self) -> list[bytes]: + return list(self._simvar_pool) + + def add_cfuncall_config(self, node_id: int, show_demangled_name: bool, show_disambiguated_name: bool) -> None: + entry = codegen_pb2.NodeConfigEntry(node_id=node_id) + entry.cfuncall.show_demangled_name = show_demangled_name + entry.cfuncall.show_disambiguated_name = show_disambiguated_name + self._node_config.append(entry) + + @property + def node_config(self) -> list[codegen_pb2.NodeConfigEntry]: + return self._node_config + + def serialize(self, node: CConstruct | None) -> int: + """Serialize ``node`` (recursively) and return its node_id (== node.idx). 0 indicates absent.""" + if node is None: + return 0 + nid = node.idx + if nid in self._seen: + return nid + self._seen.add(nid) + + pb = codegen_pb2.CConstructNode() + pb.node_id = nid + pb.kind = _SERIALIZE_KIND_BY_CLASS[type(node)] + # ident is always "_" (allocated by BaseStructuredCodeGenerator.next_ident); the class name is + # recoverable from ``kind``, so only the numeric suffix is stored. + cls_name, _, ident_no = node.ident.rpartition("_") + if cls_name != type(node).__name__ or not ident_no.isdigit(): + raise TypeError(f"Cannot serialize non-canonical CConstruct.ident {node.ident!r} on {type(node).__name__}") + pb.ident_no = int(ident_no) + pb.tags_ref = self.intern_tags(getattr(node, "tags", None)) + if isinstance(node, CExpression): + pb.collapsed = bool(getattr(node, "collapsed", False)) + ty = getattr(node, "_type", None) + if ty is not None: + pb.expr_type_ref = self.intern_type(ty) + _SERIALIZERS[type(node)](node, pb, self) + self.nodes.append(pb) + return nid + + +class ParseContext: + """ + Tracks and resolves node.idx to the corresponding C AST. + """ + + __slots__ = ( + "_cfuncall_config", + "_msg_by_id", + "_parsed", + "_simvar_pool", + "_tag_pool", + "_type_json_cache", + "_type_pool", + "kb", + "project", + ) + + def __init__( + self, nodes_msg, project=None, kb=None, type_pool=(), tag_pool=(), simvar_pool=(), node_config=None + ) -> None: + self._msg_by_id: dict[int, codegen_pb2.CConstructNode] = {n.node_id: n for n in nodes_msg} + self._parsed: dict[int, Any] = {} + # Project / KB are used to resolve Function references and similar by-address pointers at parse time. + self.project = project + self.kb = kb + self._type_pool: list[str] = list(type_pool) + self._type_json_cache: dict[int, Any] = {} + self._tag_pool = list(tag_pool) + self._simvar_pool = list(simvar_pool) + # node_id -> (show_demangled_name, show_disambiguated_name); nodes not listed use the (True, True) default + self._cfuncall_config: dict[int, tuple[bool, bool]] = {} + for entry in node_config.config_entries if node_config is not None else (): + if entry.WhichOneof("config") == "cfuncall": + self._cfuncall_config[entry.node_id] = ( + entry.cfuncall.show_demangled_name, + entry.cfuncall.show_disambiguated_name, + ) + + def cfuncall_config(self, node_id: int) -> tuple[bool, bool]: + """(show_demangled_name, show_disambiguated_name) for a CFunctionCall node; defaults to (True, True).""" + return self._cfuncall_config.get(node_id, (True, True)) + + def resolve_type(self, ref: int) -> SimType | None: + """Resolve a type-pool ref (index+1; 0 means absent) into a fresh SimType instance, arch-bound so it can + report its size when re-rendered.""" + if ref == 0: + return None + loaded = self._type_json_cache.get(ref) + if loaded is None: + loaded = json.loads(self._type_pool[ref - 1]) + self._type_json_cache[ref] = loaded + ty = SimType.from_json(loaded) + if self.project is not None: + ty = ty.with_arch(self.project.arch) + return ty + + def resolve_simvar(self, ref: int) -> SimVariable | None: + """Resolve a simvar-pool ref (index+1; 0 means absent) into a fresh SimVariable instance.""" + if ref == 0: + return None + return _simvar_from_bytes(self._simvar_pool[ref - 1]) + + def resolve_tags(self, ref: int) -> dict: + """Resolve a tag-pool ref (index+1; 0 means no tags) into a fresh tags dict.""" + if ref == 0: + return {} + return _parse_tags(self._tag_pool[ref - 1]) + + def resolve(self, node_id: int): + if node_id == 0: + return None + if node_id in self._parsed: + return self._parsed[node_id] + pb = self._msg_by_id[node_id] + obj = _PARSERS[pb.kind](pb, self) + # CConstruct base state + obj.idx = pb.node_id + obj.ident = f"{_CLASS_BY_KIND[pb.kind].__name__}_{pb.ident_no}" + obj.tags = self.resolve_tags(pb.tags_ref) + obj.codegen = None # back-reference re-attached by set_codegen() + if isinstance(obj, CExpression): + obj.collapsed = bool(pb.collapsed) if pb.HasField("collapsed") else False + obj._type = self.resolve_type(pb.expr_type_ref) + self._parsed[node_id] = obj + return obj + + def set_codegen(self, codegen) -> None: + for node in self._parsed.values(): + node.codegen = codegen + + +# +# Dispatch tables (populated by register_subclass at the bottom of c.py / by this module) +# + +_SERIALIZE_KIND_BY_CLASS: dict[type, int] = {} +_CLASS_BY_KIND: dict[int, type] = {} +_SERIALIZERS: dict[type, Callable[[Any, codegen_pb2.CConstructNode, SerializeContext], None]] = {} +_PARSERS: dict[int, Callable[[codegen_pb2.CConstructNode, ParseContext], Any]] = {} + + +def _register(cls: type, kind: int, serializer: Callable, parser: Callable) -> None: + _SERIALIZE_KIND_BY_CLASS[cls] = kind + _CLASS_BY_KIND[kind] = cls + _SERIALIZERS[cls] = serializer + _PARSERS[kind] = parser + + +# +# Subtree parsing/serialization +# + + +def serialize_subtree(root: CConstruct) -> bytes: + """Serialize an AST subtree as a Codegen envelope carrying only the indexed table + root_id. For testing and any + caller that wants to round-trip a CConstruct without a full codegen object.""" + ctx = SerializeContext() + root_id = ctx.serialize(root) + msg = codegen_pb2.Codegen() + msg.root_id = root_id + msg.nodes.extend(ctx.nodes) + msg.type_pool.extend(ctx.type_pool) + msg.tag_pool.extend(ctx.tag_pool) + msg.simvar_pool.extend(ctx.simvar_pool) + msg.node_config.config_entries.extend(ctx.node_config) + return msg.SerializeToString() + + +def parse_subtree(data: bytes, project=None, kb=None) -> CConstruct | None: + msg = codegen_pb2.Codegen() + msg.ParseFromString(data) + ctx = ParseContext( + msg.nodes, + project=project, + kb=kb, + type_pool=msg.type_pool, + tag_pool=msg.tag_pool, + simvar_pool=msg.simvar_pool, + node_config=msg.node_config, + ) + return ctx.resolve(msg.root_id) + + +# +# Position mapping serialization +# + + +def _serialize_position_mappings(pos_to_node, pos_to_addr, ctx: SerializeContext, out_msg) -> None: + """Serialize map_pos_to_node and map_pos_to_addr into one merged entry table: the two maps mostly contain + identical (start, length, node) entries, so each merged entry carries membership flags instead.""" + merged: dict[tuple[int, int, int], list[bool]] = {} # (start, length, node_id) -> [in_node, in_addr] + for pm, slot in ((pos_to_node, 0), (pos_to_addr, 1)): + if pm is None: + continue + for _, elem in pm.items(): + obj = elem.obj + if obj is None or type(obj) not in _SERIALIZE_KIND_BY_CLASS: + continue + flags = merged.setdefault((elem.start, elem.length, ctx.serialize(obj)), [False, False]) + flags[slot] = True + for (start, length, node_id), (in_node, in_addr) in merged.items(): + entry = out_msg.entries.add() + entry.start = start + entry.length = length + entry.node_id = node_id + entry.in_pos_to_node = in_node + entry.in_pos_to_addr = in_addr + + +def _parse_position_mappings(pm_msg, ctx: ParseContext): + pos_to_node = PositionMapping() + pos_to_addr = PositionMapping() + for entry in pm_msg.entries: + obj = ctx.resolve(entry.node_id) if entry.node_id != 0 else None + if entry.in_pos_to_node: + pos_to_node.add_mapping(entry.start, entry.length, obj) + if entry.in_pos_to_addr: + pos_to_addr.add_mapping(entry.start, entry.length, obj) + return pos_to_node, pos_to_addr + + +def _serialize_instruction_mapping(im, out_msg) -> None: + if im is None: + return + for _, elem in im.items(): + entry = out_msg.entries.add() + entry.ins_addr = elem.ins_addr + entry.posmap_pos = elem.posmap_pos + + +def _parse_instruction_mapping(im_msg): + im = InstructionMapping() + for entry in im_msg.entries: + im.add_mapping(entry.ins_addr, entry.posmap_pos) + return im + + +# +# Codegen serialization +# + + +def _serialize_notes(notes: dict | None, out_msg) -> None: + """notes: dict[str, DecompilationNote]. Each DecompilationNote is serialized to JSON.""" + if not notes: + return + for k, note in notes.items(): + out_msg[k] = note.to_json() + + +def _parse_notes(notes_msg): + return {k: DecompilationNote.from_json(blob) for k, blob in notes_msg.items()} + + +def _serialize_const_formats(const_formats: dict | None, out_repeated) -> None: + """const_formats: dict[IdentType, dict[str, bool]]; IdentType = tuple[int, int, str].""" + if not const_formats: + return + for ident, fmt in const_formats.items(): + entry = out_repeated.add() + entry.ident_ins_addr = ident[0] + entry.ident_kind = ident[1] + entry.ident_value = ident[2] + for k, v in fmt.items(): + entry.fmt[k] = bool(v) + + +def _parse_const_formats(entries): + result = {} + for entry in entries: + key = (entry.ident_ins_addr, entry.ident_kind, entry.ident_value) + result[key] = dict(entry.fmt) + return result + + +# Display-option attribute names round-tripped on Codegen, derived from the descriptor so the proto stays the +# single source of truth. Display options are the trailing field block in the Codegen message (see codegen.proto): +# every field from ``indent`` onward is a display option. +_DISPLAY_OPTION_FIELD_FIRST = codegen_pb2.Codegen.DESCRIPTOR.fields_by_name["indent"].number +_DISPLAY_OPTION_ATTRS = tuple( + f.name for f in codegen_pb2.Codegen.DESCRIPTOR.fields if f.number >= _DISPLAY_OPTION_FIELD_FIRST +) + + +def serialize_codegen(codegen) -> codegen_pb2.Codegen: + """Build a Codegen cmessage from a live CStructuredCodeGenerator instance.""" + msg = codegen_pb2.Codegen() + ctx = SerializeContext() + + if codegen.cfunc is not None: + msg.root_id = ctx.serialize(codegen.cfunc) + + if codegen.text is not None: + msg.text_z = zlib.compress(codegen.text.encode("utf-8")) + if getattr(codegen, "flavor", None) is not None: + msg.flavor = codegen.flavor + + _serialize_position_mappings(codegen.map_pos_to_node, codegen.map_pos_to_addr, ctx, msg.pos_maps) + _serialize_instruction_mapping(codegen.map_addr_to_pos, msg.map_addr_to_pos) + # map_ast_to_pos is derivable from map_pos_to_node, so it is rebuilt after parse instead of serialized. + for (addr, idx), label in (codegen.map_addr_to_label or {}).items(): + entry = msg.map_addr_to_label.add() + entry.addr = addr + if idx is not None: + entry.idx = idx + entry.label_id = ctx.serialize(label) + if codegen.cexterns: + for v in codegen.cexterns: + msg.cexterns_ids.append(ctx.serialize(v)) + + if codegen.expr_comments: + for k, v in codegen.expr_comments.items(): + msg.expr_comments[k] = v + if codegen.stmt_comments: + for k, v in codegen.stmt_comments.items(): + msg.stmt_comments[k] = v + _serialize_notes(codegen.notes, msg.notes_json) + _serialize_const_formats(codegen.const_formats, msg.const_formats) + + for attr in _DISPLAY_OPTION_ATTRS: + # ``indent`` is stored on the codegen as ``_indent``; every other display option matches the proto field name. + cg_attr = "_indent" if attr == "indent" else attr + if not hasattr(codegen, cg_attr): + continue + value = getattr(codegen, cg_attr) + if value is None: + continue + setattr(msg, attr, value) + + # The flat AST node table is filled in by ctx.serialize during the recursive calls above. + msg.nodes.extend(ctx.nodes) + msg.type_pool.extend(ctx.type_pool) + msg.tag_pool.extend(ctx.tag_pool) + msg.simvar_pool.extend(ctx.simvar_pool) + msg.node_config.config_entries.extend(ctx.node_config) + return msg + + +def _rebuild_ast_to_pos(pos_to_node): + """Mirror of the logic in :meth:`CStructuredCodeGenerator.render_text` that builds ``map_ast_to_pos`` from + ``pos_to_node``. Used after parse to restore the cross-reference map.""" + ast_to_pos = defaultdict(set) + if pos_to_node is None: + return ast_to_pos + for elem, node in pos_to_node.items(): + obj = node.obj + if isinstance(obj, CConstant): + ast_to_pos[obj.value].add(elem) + elif isinstance(obj, CVariable): + ast_to_pos[obj.unified_variable if obj.unified_variable is not None else obj.variable].add(elem) + elif isinstance(obj, CFunctionCall): + key = obj.callee_func if obj.callee_func is not None else obj.callee_target + ast_to_pos[key].add(elem) + elif isinstance(obj, CStructField): + ast_to_pos[(obj.struct_type, obj.offset)].add(elem) + else: + ast_to_pos[obj].add(elem) + return ast_to_pos + + +def parse_codegen(msg, *, project=None, kb=None, func=None): + """Create a CStructuredCodeGenerator from a Codegen cmessage. Bypasses __init__ (which runs the full + decompilation pipeline) and populates the attributes directly. The parsed instance is suitable for display, + navigation, and cache-validity checks; re-rendering and re-running analyses require ``project`` / ``func`` / + ``kb`` to be reattached. Decompilation variables are read from ``kb.dec_variables``.""" + cg = CStructuredCodeGenerator.__new__(CStructuredCodeGenerator) + ctx = ParseContext( + msg.nodes, + project=project, + kb=kb, + type_pool=msg.type_pool, + tag_pool=msg.tag_pool, + simvar_pool=msg.simvar_pool, + node_config=msg.node_config, + ) + + # Create the AST. + cg.cfunc = ctx.resolve(msg.root_id) if msg.root_id != 0 else None + + # Base / display state. + cg.text = zlib.decompress(msg.text_z).decode("utf-8") if msg.HasField("text_z") else None + cg.flavor = msg.flavor if msg.HasField("flavor") else None + cg.notes = _parse_notes(msg.notes_json) + cg.expr_comments = dict(msg.expr_comments) + cg.stmt_comments = dict(msg.stmt_comments) + cg.const_formats = _parse_const_formats(msg.const_formats) + cg.ident_counters = {} + # resume idx allocation past every deserialized node so nodes created later stay unique + cg._next_node_idx = max((n.node_id for n in msg.nodes), default=0) + 1 + + cg.map_pos_to_node, cg.map_pos_to_addr = _parse_position_mappings(msg.pos_maps, ctx) + cg.map_addr_to_pos = _parse_instruction_mapping(msg.map_addr_to_pos) + # map_ast_to_pos is not serialized; it is rebuilt below from map_pos_to_node. + cg.map_ast_to_pos = _rebuild_ast_to_pos(cg.map_pos_to_node) + + cg.map_addr_to_label = {} + for entry in msg.map_addr_to_label: + idx = entry.idx if entry.HasField("idx") else None + cg.map_addr_to_label[(entry.addr, idx)] = ctx.resolve(entry.label_id) + + cg.cexterns = {ctx.resolve(i) for i in msg.cexterns_ids} if msg.cexterns_ids else None + + # Display options: only set those present in the cmessage. + for attr in _DISPLAY_OPTION_ATTRS: + if msg.HasField(attr): + setattr(cg, "_indent" if attr == "indent" else attr, getattr(msg, attr)) + + # Runtime / back-reference state — caller-provided. + cg._func = func + cg._func_args = None + cg._cfg = None + cg._sequence = None + cg.kb = kb + cg.externs = set() + # codegen._variables_in_use is the same {SimVariable: CVariable} dict the CFunction carries (and serializes). + cg._variables_in_use = cg.cfunc.variables_in_use if cg.cfunc is not None else None + cg._inlined_strings = set() + cg._function_pointers = set() + cg.ailexpr2cnode = None + cg.cnode2ailexpr = None + cg._handlers = None # callers that want to re-render should construct a fresh CStructuredCodeGenerator + # indent_delta (from the indent_size option) is not serialized; restore the default so re-rendering works. + from .c import INDENT_DELTA # pylint:disable=import-outside-toplevel + + cg.indent_delta = INDENT_DELTA + # Wire the CFunction's variable manager only when kb.dec_variables already has one; creating an empty manager + # here would fool the decompiler's fast-path gate. + if cg.cfunc is not None: + has_dvars = kb is not None and func is not None and func.addr in kb.dec_variables + cg.cfunc.variable_manager = kb.dec_variables[func.addr] if has_dvars else None + + # Re-attach codegen back-references on every AST node. + ctx.set_codegen(cg) + return cg + + +# --------------------------------------------------------------------------------------------------------------------- +# Per-class serializers / parsers (registered by register_all()). +# --------------------------------------------------------------------------------------------------------------------- + + +# ----------------------------------------------------------------------------------------------------------------- +# Trivial subclasses (no payload beyond base fields). +# ----------------------------------------------------------------------------------------------------------------- +def _ser_cbreak(_node, pb, _ctx): + pb.cbreak.SetInParent() + + +def _parse_cbreak(_pb, _ctx): + return CBreak.__new__(CBreak) + + +def _ser_ccontinue(_node, pb, _ctx): + pb.ccontinue.SetInParent() + + +def _parse_ccontinue(_pb, _ctx): + return CContinue.__new__(CContinue) + + +# ----------------------------------------------------------------------------------------------------------------- +# Plain-primitive subclasses. +# ----------------------------------------------------------------------------------------------------------------- +def _ser_clabel(node, pb, _ctx): + pb.clabel.name = node.name + + +def _parse_clabel(pb, _ctx): + obj = CLabel.__new__(CLabel) + obj.name = pb.clabel.name + return obj + + +def _ser_cregister(node, pb, _ctx): + pb.creg.reg = node.reg + + +def _parse_cregister(pb, _ctx): + obj = CRegister.__new__(CRegister) + obj.reg = pb.creg.reg + return obj + + +# ----------------------------------------------------------------------------------------------------------------- +# Simple statements (children-only). +# ----------------------------------------------------------------------------------------------------------------- +def _ser_cstatements(node, pb, ctx): + for stmt in node.statements: + pb.cstatements.statements_ids.append(ctx.serialize(stmt)) + if node.addr is not None: + pb.cstatements.addr = node.addr + + +def _parse_cstatements(pb, ctx): + obj = CStatements.__new__(CStatements) + obj.statements = [ctx.resolve(i) for i in pb.cstatements.statements_ids] + obj.addr = pb.cstatements.addr if pb.cstatements.HasField("addr") else None + return obj + + +def _ser_cassignment(node, pb, ctx): + pb.cassignment.lhs_id = ctx.serialize(node.lhs) + pb.cassignment.rhs_id = ctx.serialize(node.rhs) + + +def _parse_cassignment(pb, ctx): + obj = CAssignment.__new__(CAssignment) + obj.lhs = ctx.resolve(pb.cassignment.lhs_id) + obj.rhs = ctx.resolve(pb.cassignment.rhs_id) + return obj + + +def _ser_cexprstmt(node, pb, ctx): + pb.cexpression_stmt.expr_id = ctx.serialize(node.expr) + pb.cexpression_stmt.returning = node.returning + + +def _parse_cexprstmt(pb, ctx): + obj = CExpressionStatement.__new__(CExpressionStatement) + obj.expr = ctx.resolve(pb.cexpression_stmt.expr_id) + obj.returning = pb.cexpression_stmt.returning + return obj + + +def _ser_creturn(node, pb, ctx): + if node.retval is not None: + pb.creturn.retval_id = ctx.serialize(node.retval) + + +def _parse_creturn(pb, ctx): + obj = CReturn.__new__(CReturn) + obj.retval = ctx.resolve(pb.creturn.retval_id) if pb.creturn.HasField("retval_id") else None + return obj + + +def _ser_cifbreak(node, pb, ctx): + pb.cifbreak.condition_id = ctx.serialize(node.condition) + pb.cifbreak.cstyle_ifs = node.cstyle_ifs + + +def _parse_cifbreak(pb, ctx): + obj = CIfBreak.__new__(CIfBreak) + obj.condition = ctx.resolve(pb.cifbreak.condition_id) + obj.cstyle_ifs = pb.cifbreak.cstyle_ifs + return obj + + +def _ser_cdirtystmt(node, pb, ctx): + pb.cdirty_stmt.dirty_id = ctx.serialize(node.dirty) + + +def _parse_cdirtystmt(pb, ctx): + obj = CDirtyStatement.__new__(CDirtyStatement) + obj.dirty = ctx.resolve(pb.cdirty_stmt.dirty_id) + return obj + + +# ----------------------------------------------------------------------------------------------------------------- +# Loop family. +# ----------------------------------------------------------------------------------------------------------------- +def _ser_cwhile(node, pb, ctx): + if node.condition is not None: + pb.cwhile.condition_id = ctx.serialize(node.condition) + if node.body is not None: + pb.cwhile.body_id = ctx.serialize(node.body) + + +def _parse_cwhile(pb, ctx): + obj = CWhileLoop.__new__(CWhileLoop) + obj.condition = ctx.resolve(pb.cwhile.condition_id) if pb.cwhile.HasField("condition_id") else None + obj.body = ctx.resolve(pb.cwhile.body_id) if pb.cwhile.HasField("body_id") else None + return obj + + +def _ser_cdowhile(node, pb, ctx): + if node.condition is not None: + pb.cdowhile.condition_id = ctx.serialize(node.condition) + if node.body is not None: + pb.cdowhile.body_id = ctx.serialize(node.body) + + +def _parse_cdowhile(pb, ctx): + obj = CDoWhileLoop.__new__(CDoWhileLoop) + obj.condition = ctx.resolve(pb.cdowhile.condition_id) if pb.cdowhile.HasField("condition_id") else None + obj.body = ctx.resolve(pb.cdowhile.body_id) if pb.cdowhile.HasField("body_id") else None + return obj + + +def _ser_cfor(node, pb, ctx): + if node.initializer is not None: + pb.cfor.initializer_id = ctx.serialize(node.initializer) + if node.condition is not None: + pb.cfor.condition_id = ctx.serialize(node.condition) + if node.iterator is not None: + pb.cfor.iterator_id = ctx.serialize(node.iterator) + if node.body is not None: + pb.cfor.body_id = ctx.serialize(node.body) + + +def _parse_cfor(pb, ctx): + obj = CForLoop.__new__(CForLoop) + body = pb.cfor + obj.initializer = ctx.resolve(body.initializer_id) if body.HasField("initializer_id") else None + obj.condition = ctx.resolve(body.condition_id) if body.HasField("condition_id") else None + obj.iterator = ctx.resolve(body.iterator_id) if body.HasField("iterator_id") else None + obj.body = ctx.resolve(body.body_id) if body.HasField("body_id") else None + return obj + + +# ----------------------------------------------------------------------------------------------------------------- +# If / switch / goto / label. +# ----------------------------------------------------------------------------------------------------------------- +def _ser_cifelse(node, pb, ctx): + for cond, stmt in node.condition_and_nodes: + entry = pb.cifelse.condition_and_nodes.add() + entry.condition_id = ctx.serialize(cond) + if stmt is not None: + entry.statement_id = ctx.serialize(stmt) + if node.else_node is not None: + pb.cifelse.else_node_id = ctx.serialize(node.else_node) + pb.cifelse.simplify_else_scope = node.simplify_else_scope + pb.cifelse.cstyle_ifs = node.cstyle_ifs + + +def _parse_cifelse(pb, ctx): + obj = CIfElse.__new__(CIfElse) + body = pb.cifelse + obj.condition_and_nodes = [ + (ctx.resolve(e.condition_id), ctx.resolve(e.statement_id) if e.HasField("statement_id") else None) + for e in body.condition_and_nodes + ] + obj.else_node = ctx.resolve(body.else_node_id) if body.HasField("else_node_id") else None + obj.simplify_else_scope = body.simplify_else_scope + obj.cstyle_ifs = body.cstyle_ifs + return obj + + +def _ser_cswitch(node, pb, ctx): + pb.cswitch.switch_id = ctx.serialize(node.switch) + for case_ids, stmts in node.cases: + entry = pb.cswitch.cases.add() + if isinstance(case_ids, tuple): + entry.case_ids.extend(case_ids) + else: + entry.case_ids.append(case_ids) + entry.statements_id = ctx.serialize(stmts) + if node.default is not None: + pb.cswitch.default_id = ctx.serialize(node.default) + + +def _parse_cswitch(pb, ctx): + obj = CSwitchCase.__new__(CSwitchCase) + body = pb.cswitch + obj.switch = ctx.resolve(body.switch_id) + obj.cases = [ + (tuple(e.case_ids) if len(e.case_ids) > 1 else e.case_ids[0], ctx.resolve(e.statements_id)) for e in body.cases + ] + obj.default = ctx.resolve(body.default_id) if body.HasField("default_id") else None + return obj + + +def _ser_cincomplete_switch(node, pb, ctx): + pb.cincomplete_switch.head_id = ctx.serialize(node.head) + for case_addr, stmts in node.cases: + entry = pb.cincomplete_switch.cases.add() + entry.case_addr = case_addr + entry.statements_id = ctx.serialize(stmts) + + +def _parse_cincomplete_switch(pb, ctx): + obj = CIncompleteSwitchCase.__new__(CIncompleteSwitchCase) + body = pb.cincomplete_switch + obj.head = ctx.resolve(body.head_id) + obj.cases = [(e.case_addr, ctx.resolve(e.statements_id)) for e in body.cases] + return obj + + +def _ser_cgoto(node, pb, ctx): + if isinstance(node.target, int): + pb.cgoto.target_int = node.target + else: + pb.cgoto.target_expr_id = ctx.serialize(node.target) + if node.target_idx is not None: + pb.cgoto.target_idx = node.target_idx + + +def _parse_cgoto(pb, ctx): + obj = CGoto.__new__(CGoto) + which = pb.cgoto.WhichOneof("target") + if which == "target_int": + obj.target = pb.cgoto.target_int + elif which == "target_expr_id": + obj.target = ctx.resolve(pb.cgoto.target_expr_id) + else: + obj.target = None # should not happen but be defensive + obj.target_idx = pb.cgoto.target_idx if pb.cgoto.HasField("target_idx") else None + return obj + + +# ----------------------------------------------------------------------------------------------------------------- +# Expressions. +# ----------------------------------------------------------------------------------------------------------------- +def _ser_cunop(node, pb, ctx): + pb.cunop.op = node.op + pb.cunop.operand_id = ctx.serialize(node.operand) + + +def _parse_cunop(pb, ctx): + obj = CUnaryOp.__new__(CUnaryOp) + obj.op = pb.cunop.op + obj.operand = ctx.resolve(pb.cunop.operand_id) + return obj + + +def _ser_cbinop(node, pb, ctx): + pb.cbinop.op = node.op + pb.cbinop.lhs_id = ctx.serialize(node.lhs) + pb.cbinop.rhs_id = ctx.serialize(node.rhs) + pb.cbinop.common_type_ref = ctx.intern_type(node.common_type) + + +def _parse_cbinop(pb, ctx): + obj = CBinaryOp.__new__(CBinaryOp) + obj.op = pb.cbinop.op + obj.lhs = ctx.resolve(pb.cbinop.lhs_id) + obj.rhs = ctx.resolve(pb.cbinop.rhs_id) + obj.common_type = ctx.resolve_type(pb.cbinop.common_type_ref) + obj._cstyle_null_cmp = False # rebuilt from codegen flags after set_codegen; safe default + return obj + + +def _ser_ctypecast(node, pb, ctx): + pb.ctypecast.src_type_ref = ctx.intern_type(node.src_type) + pb.ctypecast.dst_type_ref = ctx.intern_type(node.dst_type) + pb.ctypecast.expr_id = ctx.serialize(node.expr) + + +def _parse_ctypecast(pb, ctx): + obj = CTypeCast.__new__(CTypeCast) + obj.src_type = ctx.resolve_type(pb.ctypecast.src_type_ref) + obj.dst_type = ctx.resolve_type(pb.ctypecast.dst_type_ref) + obj.expr = ctx.resolve(pb.ctypecast.expr_id) + return obj + + +def _ser_cite(node, pb, ctx): + pb.cite.cond_id = ctx.serialize(node.cond) + pb.cite.iftrue_id = ctx.serialize(node.iftrue) + pb.cite.iffalse_id = ctx.serialize(node.iffalse) + + +def _parse_cite(pb, ctx): + obj = CITE.__new__(CITE) + obj.cond = ctx.resolve(pb.cite.cond_id) + obj.iftrue = ctx.resolve(pb.cite.iftrue_id) + obj.iffalse = ctx.resolve(pb.cite.iffalse_id) + return obj + + +def _ser_cmulti(node, pb, ctx): + pb.cmulti_stmt_expr.stmts_id = ctx.serialize(node.stmts) + pb.cmulti_stmt_expr.expr_id = ctx.serialize(node.expr) + + +def _parse_cmulti(pb, ctx): + obj = CMultiStatementExpression.__new__(CMultiStatementExpression) + obj.stmts = ctx.resolve(pb.cmulti_stmt_expr.stmts_id) + obj.expr = ctx.resolve(pb.cmulti_stmt_expr.expr_id) + return obj + + +def _ser_cvex(node, pb, ctx): + pb.cvex_ccall.callee = node.callee + for op in node.operands: + pb.cvex_ccall.operands_ids.append(ctx.serialize(op)) + + +def _parse_cvex(pb, ctx): + obj = CVEXCCallExpression.__new__(CVEXCCallExpression) + obj.callee = pb.cvex_ccall.callee + obj.operands = [ctx.resolve(i) for i in pb.cvex_ccall.operands_ids] + return obj + + +# ----------------------------------------------------------------------------------------------------------------- +# Variables and structs. +# ----------------------------------------------------------------------------------------------------------------- +def _ser_cstructfield(node, pb, ctx): + pb.cstruct_field.struct_type_ref = ctx.intern_type(node.struct_type) + pb.cstruct_field.offset = node.offset + pb.cstruct_field.field = node.field + + +def _parse_cstructfield(pb, ctx): + obj = CStructField.__new__(CStructField) + obj.struct_type = ctx.resolve_type(pb.cstruct_field.struct_type_ref) + obj.offset = pb.cstruct_field.offset + obj.field = pb.cstruct_field.field + return obj + + +def _ser_cfakevar(node, pb, ctx): + pb.cfake_var.name = node.name + ty = getattr(node, "_type", None) + if ty is not None: + pb.cfake_var.type_ref = ctx.intern_type(ty) + + +def _parse_cfakevar(pb, ctx): + obj = CFakeVariable.__new__(CFakeVariable) + obj.name = pb.cfake_var.name + # _type is restored by ParseContext.resolve via expr_type_ref on the wrapper; CFakeVariable also has _type + # in __slots__, set there too if present in body. + if pb.cfake_var.type_ref: + obj._type = ctx.resolve_type(pb.cfake_var.type_ref) + return obj + + +def _ser_cvar(node, pb, ctx): + pb.cvar.variable_ref = ctx.intern_simvar(node.variable) + if node.unified_variable is not None: + pb.cvar.unified_variable_ref = ctx.intern_simvar(node.unified_variable) + if node.variable_type is not None: + pb.cvar.variable_type_ref = ctx.intern_type(node.variable_type) + if node.vvar_id is not None: + pb.cvar.vvar_id = node.vvar_id + + +def _parse_cvar(pb, ctx): + obj = CVariable.__new__(CVariable) + body = pb.cvar + obj.variable = ctx.resolve_simvar(body.variable_ref) + obj.unified_variable = ctx.resolve_simvar(body.unified_variable_ref) + obj.variable_type = ctx.resolve_type(body.variable_type_ref) + obj.vvar_id = body.vvar_id if body.HasField("vvar_id") else None + return obj + + +def _ser_cidxvar(node, pb, ctx): + pb.cindexed_var.variable_id = ctx.serialize(node.variable) + pb.cindexed_var.index_id = ctx.serialize(node.index) + ty = getattr(node, "_type", None) + if ty is not None: + pb.cindexed_var.type_ref = ctx.intern_type(ty) + + +def _parse_cidxvar(pb, ctx): + obj = CIndexedVariable.__new__(CIndexedVariable) + body = pb.cindexed_var + obj.variable = ctx.resolve(body.variable_id) + obj.index = ctx.resolve(body.index_id) + if body.type_ref: + obj._type = ctx.resolve_type(body.type_ref) + return obj + + +def _ser_cvarfield(node, pb, ctx): + pb.cvar_field.variable_id = ctx.serialize(node.variable) + pb.cvar_field.field_id = ctx.serialize(node.field) + pb.cvar_field.var_is_ptr = node.var_is_ptr + + +def _parse_cvarfield(pb, ctx): + obj = CVariableField.__new__(CVariableField) + body = pb.cvar_field + obj.variable = ctx.resolve(body.variable_id) + obj.field = ctx.resolve(body.field_id) + obj.var_is_ptr = body.var_is_ptr + return obj + + +# ----------------------------------------------------------------------------------------------------------------- +# CConstant (heterogeneous value + reference_values). +# ----------------------------------------------------------------------------------------------------------------- +def _ser_cconst(node, pb, ctx): + body = pb.cconst + if isinstance(node.value, bool): + body.int_value = int(node.value) + elif isinstance(node.value, int): + body.int_value = node.value + elif isinstance(node.value, float): + body.float_value = node.value + elif isinstance(node.value, str): + body.str_value = node.value + body.type_ref = ctx.intern_type(node._type) + if node.reference_values: + for ty, val in node.reference_values.items(): + entry = body.reference_values.add() + entry.type_ref = ctx.intern_type(ty) + if isinstance(val, bool): + entry.int_value = int(val) + elif isinstance(val, int): + entry.int_value = val + elif isinstance(val, bytes): + entry.raw_bytes = val + elif isinstance(val, str): + entry.str_value = val + elif isinstance(val, MemoryData): + entry.memory_data = val.serialize() + # other types intentionally dropped + + +def _parse_cconst(pb, ctx): + obj = CConstant.__new__(CConstant) + body = pb.cconst + which = body.WhichOneof("value") + if which == "int_value": + obj.value = body.int_value + elif which == "float_value": + obj.value = body.float_value + elif which == "str_value": + obj.value = body.str_value + else: + obj.value = None + obj._type = ctx.resolve_type(body.type_ref) + if body.reference_values: + refs = {} + for entry in body.reference_values: + key = ctx.resolve_type(entry.type_ref) + w = entry.WhichOneof("value") + if w == "int_value": + refs[key] = entry.int_value + elif w == "raw_bytes": + refs[key] = entry.raw_bytes + elif w == "str_value": + refs[key] = entry.str_value + elif w == "memory_data": + refs[key] = MemoryData.parse(entry.memory_data) + obj.reference_values = refs + else: + obj.reference_values = None + return obj + + +# ----------------------------------------------------------------------------------------------------------------- +# CFunctionCall (oneof callee_target + callee_func reference). +# ----------------------------------------------------------------------------------------------------------------- +def _ser_cfuncall(node, pb, ctx): + body = pb.cfuncall + target = node.callee_target + if isinstance(target, int): + body.callee_target_int = target + elif isinstance(target, str): + body.callee_target_str = target + else: + body.callee_target_expr_id = ctx.serialize(target) + if node.callee_func is not None: + body.callee_func_addr = node.callee_func.addr + for a in node.args: + body.args_ids.append(ctx.serialize(a)) + # show_demangled_name / show_disambiguated_name default to True; only record an override when either is False. + if not (node.show_demangled_name and node.show_disambiguated_name): + ctx.add_cfuncall_config(pb.node_id, node.show_demangled_name, node.show_disambiguated_name) + + +def _parse_cfuncall(pb, ctx): + obj = CFunctionCall.__new__(CFunctionCall) + body = pb.cfuncall + which = body.WhichOneof("callee_target") + if which == "callee_target_int": + obj.callee_target = body.callee_target_int + elif which == "callee_target_str": + obj.callee_target = body.callee_target_str + elif which == "callee_target_expr_id": + obj.callee_target = ctx.resolve(body.callee_target_expr_id) + else: + obj.callee_target = None + if body.HasField("callee_func_addr") and ctx.kb is not None: + obj.callee_func = ctx.kb.functions.function(body.callee_func_addr) + else: + obj.callee_func = None + obj.args = [ctx.resolve(i) for i in body.args_ids] + obj.show_demangled_name, obj.show_disambiguated_name = ctx.cfuncall_config(pb.node_id) + return obj + + +# ----------------------------------------------------------------------------------------------------------------- +# CFunction (the AST root). +# ----------------------------------------------------------------------------------------------------------------- +def _ser_cfunction(node, pb, ctx): + body = pb.cfunction + if node.addr is not None: + body.addr = node.addr + body.name = node.name + body.functy_ref = ctx.intern_type(node.functy) + for arg in node.arg_list: + body.arg_list_ids.append(ctx.serialize(arg)) + body.statements_id = ctx.serialize(node.statements) + for simvar, cvar in node.variables_in_use.items(): + entry = body.variables_in_use.add() + entry.simvariable_ref = ctx.intern_simvar(simvar) + entry.cvariable_id = ctx.serialize(cvar) + if node.demangled_name is not None: + body.demangled_name = node.demangled_name + body.show_demangled_name = node.show_demangled_name + body.omit_header = node.omit_header + + +def _parse_cfunction(pb, ctx): + obj = CFunction.__new__(CFunction) + body = pb.cfunction + obj.addr = body.addr if body.HasField("addr") else None + obj.name = body.name + obj.functy = ctx.resolve_type(body.functy_ref) + obj.arg_list = [ctx.resolve(i) for i in body.arg_list_ids] + obj.statements = ctx.resolve(body.statements_id) + obj.variables_in_use = { + ctx.resolve_simvar(e.simvariable_ref): ctx.resolve(e.cvariable_id) for e in body.variables_in_use + } + obj.demangled_name = body.demangled_name if body.HasField("demangled_name") else None + obj.show_demangled_name = body.show_demangled_name + obj.omit_header = body.omit_header + # variable_manager is attached by the codegen wrapper at parse time; unified_local_vars is recomputed via + # refresh() once codegen + variable_manager are wired up. + obj.variable_manager = None + obj.unified_local_vars = {} + return obj + + +# ----------------------------------------------------------------------------------------------------------------- +# Ailment-coupled subclasses (native AIL to_bytes payloads). +# ----------------------------------------------------------------------------------------------------------------- +def _ser_cailblock(node, pb, _ctx): + pb.cailblock.block = node.block.to_bytes() + + +def _parse_cailblock(pb, _ctx): + obj = CAILBlock.__new__(CAILBlock) + obj.block = AilBlock.from_bytes(pb.cailblock.block) + return obj + + +def _ser_cunsupported(node, pb, _ctx): + pb.cunsupported.stmt = node.stmt.to_bytes() + + +def _parse_cunsupported(pb, _ctx): + obj = CUnsupportedStatement.__new__(CUnsupportedStatement) + obj.stmt = AilStatement.from_bytes(pb.cunsupported.stmt) + return obj + + +def _ser_cdirtyexpr(node, pb, _ctx): + pb.cdirty_expr.dirty = node.dirty.to_bytes() + + +def _parse_cdirtyexpr(pb, _ctx): + obj = CDirtyExpression.__new__(CDirtyExpression) + obj.dirty = AilExpression.from_bytes(pb.cdirty_expr.dirty) + return obj + + +def register_all() -> None: + """Registers serializer/parser pairs for every concrete CConstruct subclass. Called from c.py at import time.""" + _register(CBreak, codegen_pb2.CCK_BREAK, _ser_cbreak, _parse_cbreak) + _register(CContinue, codegen_pb2.CCK_CONTINUE, _ser_ccontinue, _parse_ccontinue) + _register(CLabel, codegen_pb2.CCK_LABEL, _ser_clabel, _parse_clabel) + _register(CRegister, codegen_pb2.CCK_REGISTER, _ser_cregister, _parse_cregister) + _register(CStatements, codegen_pb2.CCK_STATEMENTS, _ser_cstatements, _parse_cstatements) + _register(CAssignment, codegen_pb2.CCK_ASSIGNMENT, _ser_cassignment, _parse_cassignment) + _register(CExpressionStatement, codegen_pb2.CCK_EXPRESSION_STATEMENT, _ser_cexprstmt, _parse_cexprstmt) + _register(CReturn, codegen_pb2.CCK_RETURN, _ser_creturn, _parse_creturn) + _register(CIfBreak, codegen_pb2.CCK_IF_BREAK, _ser_cifbreak, _parse_cifbreak) + _register(CDirtyStatement, codegen_pb2.CCK_DIRTY_STATEMENT, _ser_cdirtystmt, _parse_cdirtystmt) + _register(CWhileLoop, codegen_pb2.CCK_WHILE_LOOP, _ser_cwhile, _parse_cwhile) + _register(CDoWhileLoop, codegen_pb2.CCK_DO_WHILE_LOOP, _ser_cdowhile, _parse_cdowhile) + _register(CForLoop, codegen_pb2.CCK_FOR_LOOP, _ser_cfor, _parse_cfor) + _register(CIfElse, codegen_pb2.CCK_IF_ELSE, _ser_cifelse, _parse_cifelse) + _register(CSwitchCase, codegen_pb2.CCK_SWITCH_CASE, _ser_cswitch, _parse_cswitch) + _register( + CIncompleteSwitchCase, + codegen_pb2.CCK_INCOMPLETE_SWITCH_CASE, + _ser_cincomplete_switch, + _parse_cincomplete_switch, + ) + _register(CGoto, codegen_pb2.CCK_GOTO, _ser_cgoto, _parse_cgoto) + _register(CUnaryOp, codegen_pb2.CCK_UNARY_OP, _ser_cunop, _parse_cunop) + _register(CBinaryOp, codegen_pb2.CCK_BINARY_OP, _ser_cbinop, _parse_cbinop) + _register(CTypeCast, codegen_pb2.CCK_TYPE_CAST, _ser_ctypecast, _parse_ctypecast) + _register(CITE, codegen_pb2.CCK_ITE, _ser_cite, _parse_cite) + _register(CMultiStatementExpression, codegen_pb2.CCK_MULTI_STATEMENT_EXPRESSION, _ser_cmulti, _parse_cmulti) + _register(CVEXCCallExpression, codegen_pb2.CCK_VEX_CCALL_EXPRESSION, _ser_cvex, _parse_cvex) + _register(CStructField, codegen_pb2.CCK_STRUCT_FIELD, _ser_cstructfield, _parse_cstructfield) + _register(CFakeVariable, codegen_pb2.CCK_FAKE_VARIABLE, _ser_cfakevar, _parse_cfakevar) + _register(CVariable, codegen_pb2.CCK_VARIABLE, _ser_cvar, _parse_cvar) + _register(CIndexedVariable, codegen_pb2.CCK_INDEXED_VARIABLE, _ser_cidxvar, _parse_cidxvar) + _register(CVariableField, codegen_pb2.CCK_VARIABLE_FIELD, _ser_cvarfield, _parse_cvarfield) + _register(CConstant, codegen_pb2.CCK_CONSTANT, _ser_cconst, _parse_cconst) + _register(CFunctionCall, codegen_pb2.CCK_FUNCTION_CALL, _ser_cfuncall, _parse_cfuncall) + _register(CFunction, codegen_pb2.CCK_FUNCTION, _ser_cfunction, _parse_cfunction) + _register(CAILBlock, codegen_pb2.CCK_AIL_BLOCK, _ser_cailblock, _parse_cailblock) + _register(CUnsupportedStatement, codegen_pb2.CCK_UNSUPPORTED_STATEMENT, _ser_cunsupported, _parse_cunsupported) + _register(CDirtyExpression, codegen_pb2.CCK_DIRTY_EXPRESSION, _ser_cdirtyexpr, _parse_cdirtyexpr) diff --git a/angr/analyses/decompiler/structured_codegen/rust.py b/angr/analyses/decompiler/structured_codegen/rust.py index 12c45f304..7e4b45644 100644 --- a/angr/analyses/decompiler/structured_codegen/rust.py +++ b/angr/analyses/decompiler/structured_codegen/rust.py @@ -2730,7 +2730,6 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): sequence, indent=0, cfg=None, - variable_kb=None, func_args: list[SimVariable] | None = None, binop_depth_cutoff: int = 16, show_casts=True, @@ -2818,7 +2817,6 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): self._func_args = func_args self._cfg = cfg self._sequence = sequence - self._variable_kb = variable_kb if variable_kb is not None else self.kb self._variable_map: VariableMap = variable_map if variable_map is not None else VariableMap() self.binop_depth_cutoff = binop_depth_cutoff @@ -2927,7 +2925,7 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): arg_list, obj, self._variables_in_use, - self._variable_kb.variables[self._func.addr], + self.kb.dec_variables[self._func.addr], demangled_name=self._func.demangled_name, show_demangled_name=self.show_demangled_name, codegen=self, @@ -3029,8 +3027,8 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): def _get_variable_type(self, var, is_global=False): if is_global: - return self._variable_kb.variables["global"].get_variable_type(var) - return self._variable_kb.variables[self._func.addr].get_variable_type(var) + return self.kb.dec_variables["global"].get_variable_type(var) + return self.kb.dec_variables[self._func.addr].get_variable_type(var) def _get_derefed_type(self, ty: SimType) -> SimType | None: if ty is None: @@ -3075,7 +3073,7 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): # import ipdb # # ipdb.set_trace() - unified = self._variable_kb.variables[self._func.addr].unified_variable(variable) + unified = self.kb.dec_variables[self._func.addr].unified_variable(variable) variable_type = self._get_variable_type( variable, is_global=isinstance(variable, SimMemoryVariable) and not isinstance(variable, SimStackVariable) ) diff --git a/angr/analyses/deobfuscator/api_obf_finder.py b/angr/analyses/deobfuscator/api_obf_finder.py index e256d827f..9f5338d2f 100644 --- a/angr/analyses/deobfuscator/api_obf_finder.py +++ b/angr/analyses/deobfuscator/api_obf_finder.py @@ -29,7 +29,7 @@ from .api_obf_type2_finder import APIObfuscationType2Finder from .hash_lookup_api_deobfuscator import HashLookupAPIDeobfuscator if TYPE_CHECKING: - from angr.knowledge_base import KnowledgeBase + pass _l = logging.getLogger(name=__name__) @@ -100,9 +100,8 @@ class APIObfuscationFinder(Analysis): - Type 2: GetProcAddress(_, "api_name"). """ - def __init__(self, variable_kb: KnowledgeBase | None = None): + def __init__(self): self.type1_candidates = [] - self.variable_kb = variable_kb or self.project.kb self.analyze() @@ -114,7 +113,7 @@ class APIObfuscationFinder(Analysis): type1_deobfuscated = self._analyze_type1(desc.func_addr, desc) self.kb.obfuscations.type1_deobfuscated_apis.update(type1_deobfuscated) - APIObfuscationType2Finder(self.project, self.variable_kb).analyze() + APIObfuscationType2Finder(self.project, self.kb).analyze() self.project.analyses[HashLookupAPIDeobfuscator].prep(fail_fast=self._fail_fast)( self._hash_lookup_api_deobfuscator_lifter ) diff --git a/angr/analyses/deobfuscator/api_obf_type2_finder.py b/angr/analyses/deobfuscator/api_obf_type2_finder.py index 8cfeeacfb..334b95959 100644 --- a/angr/analyses/deobfuscator/api_obf_type2_finder.py +++ b/angr/analyses/deobfuscator/api_obf_type2_finder.py @@ -43,9 +43,9 @@ class APIObfuscationType2Finder: results: list[APIObfuscationType2] - def __init__(self, project: Project, variable_kb: KnowledgeBase | None = None): + def __init__(self, project: Project, kb: KnowledgeBase | None = None): self.project = project - self.variable_kb = variable_kb or self.project.kb + self._kb = kb or self.project.kb self.results = [] def analyze(self) -> list[APIObfuscationType2]: @@ -147,7 +147,7 @@ class APIObfuscationType2Finder: log.debug("...Created label %s for address %x", lbl, result.resolved_func_ptr.addr) # Create a variable - global_variables = self.variable_kb.variables["global"] + global_variables = self._kb.variables["global"] variables = global_variables.get_global_variables(result.resolved_func_ptr.addr) if not variables: ident = global_variables.next_variable_ident("global") diff --git a/angr/analyses/s_reaching_definitions/__init__.py b/angr/analyses/s_reaching_definitions/__init__.py index f3a669127..8cef939a2 100644 --- a/angr/analyses/s_reaching_definitions/__init__.py +++ b/angr/analyses/s_reaching_definitions/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -from .s_rda_model import SRDAModel +from .s_rda_model import SRDAModel, populate_model from .s_rda_view import SRDAView from .s_reaching_definitions import SReachingDefinitionsAnalysis @@ -8,4 +8,5 @@ __all__ = ( "SRDAModel", "SRDAView", "SReachingDefinitionsAnalysis", + "populate_model", ) diff --git a/angr/analyses/s_reaching_definitions/s_rda_model.py b/angr/analyses/s_reaching_definitions/s_rda_model.py index 1980c19eb..cc7130d80 100644 --- a/angr/analyses/s_reaching_definitions/s_rda_model.py +++ b/angr/analyses/s_reaching_definitions/s_rda_model.py @@ -5,9 +5,11 @@ from collections.abc import Iterator from typing import TYPE_CHECKING, Any, Literal, overload from angr.ailment import Address +from angr.ailment.block import Block from angr.ailment.expression import Tmp, VirtualVariable from angr.code_location import AILCodeLocation from angr.knowledge_plugins.key_definitions import Definition, atoms +from angr.utils.ssa import get_tmp_deflocs, get_tmp_uselocs, get_vvar_deflocs, get_vvar_uselocs if TYPE_CHECKING: from angr.knowledge_plugins.functions.function_manager import FunctionManager @@ -64,8 +66,6 @@ class SRDAModel: implicit call-site uses). Tmp tracking is not updated (AILSimplifier does not track tmps). """ - from angr.utils.ssa import get_vvar_deflocs, get_vvar_uselocs # pylint:disable=import-outside-toplevel - edited_blocks = list(edited_blocks) block_keys = {(b.addr, b.idx) for b in edited_blocks} @@ -279,3 +279,74 @@ class SRDAModel: if isinstance(def_.atom, atoms.VirtualVariable): return self.get_vvar_uses(def_.atom) return set() + + +def populate_model( + model: SRDAModel, + blocks: dict[tuple[int, int | None], Block], + func_args: set[VirtualVariable] | None, + *, + fix_undefined_vvars: bool = True, + track_tmps: bool = False, +) -> None: + """Populate the scan-derived part of an SRDAModel (vvar/tmp definitions and uses, phi bookkeeping) with a linear + scan over ``blocks``. An SRDAModel is never serialized; it is always rebuilt from an AIL graph through this + function (via :class:`SReachingDefinitionsAnalysis` or directly).""" + + phi_vvars: dict[int, set[int | None]] = {} + # find all vvar definitions + vvar_deflocs = get_vvar_deflocs(blocks.values(), phi_vvars=phi_vvars) + # find all explicit vvar uses + vvar_uselocs = get_vvar_uselocs(blocks.values()) + + # update vvar definitions using function arguments + if func_args: + for vvar in func_args: + if vvar.varid not in vvar_deflocs: + vvar_deflocs[vvar.varid] = vvar, AILCodeLocation.make_extern(vvar.varid) + model.func_args = func_args + + # update model + for vvar_id, (vvar, defloc) in vvar_deflocs.items(): + model.varid_to_vvar[vvar_id] = vvar + model.all_vvar_definitions[vvar_id] = defloc + if vvar_id in vvar_uselocs: + for useloc in vvar_uselocs[vvar_id]: + model.add_vvar_use(vvar_id, *useloc) + + model.phi_vvar_ids = set(phi_vvars) + model.phivarid_to_varids = {} + for vvar_id, src_vvars in phi_vvars.items(): + model.phivarid_to_varids_with_unknown[vvar_id] = src_vvars + model.phivarid_to_varids[vvar_id] = ( # type: ignore + {vvar_id for vvar_id in src_vvars if vvar_id is not None} if None in src_vvars else src_vvars + ) + + if fix_undefined_vvars: + # fix register definitions for arguments + defined_vvarids = set(vvar_deflocs) + undefined_vvarids = set(vvar_uselocs.keys()).difference(defined_vvarids) + for vvar_id in undefined_vvarids: + used_vvar = next(iter(vvar_uselocs[vvar_id]))[0] + model.varid_to_vvar[vvar_id] = used_vvar + model.all_vvar_definitions[vvar_id] = AILCodeLocation.make_extern(vvar_id) + if vvar_id in vvar_uselocs: + for vvar_useloc in vvar_uselocs[vvar_id]: + model.add_vvar_use(vvar_id, *vvar_useloc) + + if track_tmps: + # track tmps + tmp_deflocs = get_tmp_deflocs(blocks.values()) + # find all vvar uses + tmp_uselocs = get_tmp_uselocs(blocks.values()) + + # update model + for block_loc, d in tmp_deflocs.items(): + for tmp_atom, stmt_idx in d.items(): + model.all_tmp_definitions[block_loc][tmp_atom] = stmt_idx + + if tmp_atom in tmp_uselocs[block_loc]: + for tmp_at_use, use_stmt_idx in tmp_uselocs[block_loc][tmp_atom]: + if tmp_atom not in model.all_tmp_uses[block_loc]: + model.all_tmp_uses[block_loc][tmp_atom] = set() + model.all_tmp_uses[block_loc][tmp_atom].add((tmp_at_use, use_stmt_idx)) diff --git a/angr/analyses/s_reaching_definitions/s_reaching_definitions.py b/angr/analyses/s_reaching_definitions/s_reaching_definitions.py index 69646dd26..5db328357 100644 --- a/angr/analyses/s_reaching_definitions/s_reaching_definitions.py +++ b/angr/analyses/s_reaching_definitions/s_reaching_definitions.py @@ -11,9 +11,8 @@ from angr.calling_conventions import SimRegArg, default_cc from angr.code_location import AILCodeLocation from angr.knowledge_plugins.functions import Function from angr.knowledge_plugins.key_definitions.constants import ObservationPointType -from angr.utils.ssa import get_tmp_deflocs, get_tmp_uselocs, get_vvar_deflocs, get_vvar_uselocs -from .s_rda_model import SRDAModel +from .s_rda_model import SRDAModel, populate_model from .s_rda_view import SRDAView @@ -76,49 +75,17 @@ class SReachingDefinitionsAnalysis(Analysis): case _: raise NotImplementedError - phi_vvars: dict[int, set[int | None]] = {} - # find all vvar definitions - vvar_deflocs = get_vvar_deflocs(blocks.values(), phi_vvars=phi_vvars) - # find all explicit vvar uses - vvar_uselocs = get_vvar_uselocs(blocks.values()) - - # update vvar definitions using function arguments - if self.func_args: - for vvar in self.func_args: - if vvar.varid not in vvar_deflocs: - vvar_deflocs[vvar.varid] = vvar, AILCodeLocation.make_extern(vvar.varid) - self.model.func_args = self.func_args - - # update model - for vvar_id, (vvar, defloc) in vvar_deflocs.items(): - self.model.varid_to_vvar[vvar_id] = vvar - self.model.all_vvar_definitions[vvar_id] = defloc - if vvar_id in vvar_uselocs: - for useloc in vvar_uselocs[vvar_id]: - self.model.add_vvar_use(vvar_id, *useloc) - - self.model.phi_vvar_ids = set(phi_vvars) - self.model.phivarid_to_varids = {} - for vvar_id, src_vvars in phi_vvars.items(): - self.model.phivarid_to_varids_with_unknown[vvar_id] = src_vvars - self.model.phivarid_to_varids[vvar_id] = ( # type: ignore - {vvar_id for vvar_id in src_vvars if vvar_id is not None} if None in src_vvars else src_vvars - ) + populate_model( + self.model, + blocks, + self.func_args, + fix_undefined_vvars=self.mode == "function", + track_tmps=self._track_tmps, + ) if self.mode == "function": assert self.func is not None - # fix register definitions for arguments - defined_vvarids = set(vvar_deflocs) - undefined_vvarids = set(vvar_uselocs.keys()).difference(defined_vvarids) - for vvar_id in undefined_vvarids: - used_vvar = next(iter(vvar_uselocs[vvar_id]))[0] - self.model.varid_to_vvar[vvar_id] = used_vvar - self.model.all_vvar_definitions[vvar_id] = AILCodeLocation.make_extern(vvar_id) - if vvar_id in vvar_uselocs: - for vvar_useloc in vvar_uselocs[vvar_id]: - self.model.add_vvar_use(vvar_id, *vvar_useloc) - srda_view = SRDAView(self.model) # the function entry block, used by observe()'s dominance-based fast path to build the dominator tree assert self.func_addr is not None @@ -236,22 +203,5 @@ class SReachingDefinitionsAnalysis(Analysis): vvarid = reg_to_vvarids[reg_offset][max_vvar_size] self.model.add_vvar_use(vvarid, None, codeloc) - if self._track_tmps: - # track tmps - tmp_deflocs = get_tmp_deflocs(blocks.values()) - # find all vvar uses - tmp_uselocs = get_tmp_uselocs(blocks.values()) - - # update model - for block_loc, d in tmp_deflocs.items(): - for tmp_atom, stmt_idx in d.items(): - self.model.all_tmp_definitions[block_loc][tmp_atom] = stmt_idx - - if tmp_atom in tmp_uselocs[block_loc]: - for tmp_at_use, use_stmt_idx in tmp_uselocs[block_loc][tmp_atom]: - if tmp_atom not in self.model.all_tmp_uses[block_loc]: - self.model.all_tmp_uses[block_loc][tmp_atom] = set() - self.model.all_tmp_uses[block_loc][tmp_atom].add((tmp_at_use, use_stmt_idx)) - register_analysis(SReachingDefinitionsAnalysis, "SReachingDefinitions") diff --git a/angr/angrdb/models.py b/angr/angrdb/models.py index 6918563e3..8432d738b 100644 --- a/angr/angrdb/models.py +++ b/angr/angrdb/models.py @@ -50,7 +50,9 @@ class DbKnowledgeBase(Base): comments = relationship("DbComment", back_populates="kb") labels = relationship("DbLabel", back_populates="kb") var_collections = relationship("DbVariableCollection", back_populates="kb") + dec_var_collections = relationship("DbDecVariableCollection", back_populates="kb") structured_code = relationship("DbStructuredCode", back_populates="kb") + decompilation_caches = relationship("DbDecompilationCache", back_populates="kb") class DbCFGModel(Base): @@ -125,6 +127,27 @@ class DbVariableCollection(Base): blob = Column(BLOB) +class DbDecVariableCollection(Base): + """ + Models a VariableManagerInternal instance of the decompilation variable manager (kb.dec_variables). A separate + table from ``variables`` (the disassembly-level kb.variables) so databases created before it existed remain + loadable: ``create_all`` adds missing tables. + """ + + __tablename__ = "dec_variables" + + id = Column(Integer, primary_key=True) + kb_id = Column( + Integer, + ForeignKey("knowledgebases.id"), + nullable=False, + ) + kb = relationship("DbKnowledgeBase", uselist=False, back_populates="dec_var_collections") + func_addr = Column(Integer) + ident = Column(String, nullable=True) + blob = Column(BLOB) + + class DbStructuredCode(Base): """ Models a StructuredCode instance. @@ -149,6 +172,27 @@ class DbStructuredCode(Base): errors = Column(TEXT, nullable=True) +class DbDecompilationCache(Base): + """ + Models a fully serialized DecompilationCache instance (a protobuf DecompilationCache message). A separate table + from ``structured_code`` (which stores only codegen metadata) so databases created before it existed remain + loadable: ``create_all`` adds missing tables. + """ + + __tablename__ = "decompilation_caches" + + id = Column(Integer, primary_key=True) + kb_id = Column( + Integer, + ForeignKey("knowledgebases.id"), + nullable=False, + ) + kb = relationship("DbKnowledgeBase", uselist=False, back_populates="decompilation_caches") + func_addr = Column(Integer) + flavor = Column(String) + blob = Column(BLOB, nullable=True) + + class DbXRefs(Base): """ Models an XRefManager instance. diff --git a/angr/angrdb/serializers/kb.py b/angr/angrdb/serializers/kb.py index 2c0b880ae..6ceca7061 100644 --- a/angr/angrdb/serializers/kb.py +++ b/angr/angrdb/serializers/kb.py @@ -44,6 +44,7 @@ class KnowledgeBaseSerializer: CommentsSerializer.dump(session, db_kb, kb.comments) LabelsSerializer.dump(session, db_kb, kb.labels) VariableManagerSerializer.dump(session, db_kb, kb.variables) + VariableManagerSerializer.dump_dvars(session, db_kb, kb.dec_variables) StructuredCodeManagerSerializer.dump(session, db_kb, kb.decompilations) @staticmethod @@ -94,6 +95,11 @@ class KnowledgeBaseSerializer: if variables is not None: kb.variables = variables + # Load decompilation variables (kb.dec_variables) + dec_variables = VariableManagerSerializer.load_dvars(session, db_kb, kb) + if dec_variables is not None: + kb.dec_variables = dec_variables + # Load structured code structured_code = StructuredCodeManagerSerializer.load(session, db_kb, kb) if structured_code is not None: diff --git a/angr/angrdb/serializers/structured_code.py b/angr/angrdb/serializers/structured_code.py index b2b7568ca..27659a6c1 100644 --- a/angr/angrdb/serializers/structured_code.py +++ b/angr/angrdb/serializers/structured_code.py @@ -3,11 +3,14 @@ from __future__ import annotations import json from typing import TYPE_CHECKING, Any +from sqlalchemy import insert + from angr.analyses.decompiler.decompilation_cache import DecompilationCache from angr.analyses.decompiler.structured_codegen import DummyStructuredCodeGenerator from angr.analyses.decompiler.structured_codegen.base import CConstantType -from angr.angrdb.models import DbStructuredCode +from angr.angrdb.models import DbDecompilationCache, DbStructuredCode from angr.knowledge_plugins import StructuredCodeManager +from angr.knowledge_plugins.structured_code import CacheKey, SpillingDecompilationDict if TYPE_CHECKING: from angr.analyses.decompiler.structured_codegen.base import IdentType @@ -64,6 +67,9 @@ class StructuredCodeManagerSerializer: @staticmethod def dump(session, db_kb: DbKnowledgeBase, code_manager: StructuredCodeManager): """ + Store every decompilation cache as its fully serialized protobuf bytes (the ``decompilation_caches`` table). + Caches that cannot be serialized (e.g. dummy or rust-flavor codegens) fall back to the legacy + ``structured_code`` rows storing codegen metadata only. :param session: :param db_kb: @@ -73,38 +79,67 @@ class StructuredCodeManagerSerializer: # remove all existing stored structured code session.query(DbStructuredCode).filter_by(kb=db_kb).delete() + session.query(DbDecompilationCache).filter_by(kb=db_kb).delete() - for key, cache in code_manager.cached.items(): - func_addr, flavor = key + # make sure db_kb has a primary key so it can be used as a foreign key in the Core bulk insert below + session.flush() + assert db_kb.id is not None - # TODO: Cache types + backing = code_manager.cached + serialized: list[tuple[CacheKey, bytes]] + unserializable: dict[CacheKey, DecompilationCache] + if isinstance(backing, SpillingDecompilationDict): + # copy the serialized bytes of spilled caches directly out of the LMDB backing store instead of + # deserializing and re-serializing them + serialized, unserializable = backing.export_serialized() + else: + serialized = [] + unserializable = {} + for key, cache in backing.items(): + try: + serialized.append((key, cache.serialize())) + except Exception: # pylint:disable=broad-exception-caught + unserializable[key] = cache - expr_comments = None - if cache.codegen is not None and cache.codegen.expr_comments: - expr_comments = json.dumps(cache.codegen.expr_comments).encode("utf-8") + rows = [ + {"kb_id": db_kb.id, "func_addr": func_addr, "flavor": flavor, "blob": blob} + for (func_addr, flavor), blob in serialized + ] + # bulk-insert via Core to avoid the per-row ORM unit-of-work overhead + if rows: + session.execute(insert(DbDecompilationCache), rows) - stmt_comments = None - if cache.codegen is not None and cache.codegen.stmt_comments: - stmt_comments = json.dumps(cache.codegen.stmt_comments).encode("utf-8") + for key, cache in unserializable.items(): + StructuredCodeManagerSerializer._dump_legacy(session, db_kb, key, cache) - const_formats = None - if cache.codegen is not None and cache.codegen.const_formats: - const_formats = json.dumps(ConstFormatsSerializer.to_json(cache.codegen.const_formats)).encode("utf-8") + @staticmethod + def _dump_legacy(session, db_kb: DbKnowledgeBase, key: CacheKey, cache: DecompilationCache) -> None: + """Store codegen metadata (comments, constant formats, errors) of an unserializable cache.""" + func_addr, flavor = key - ite_exprs = None + expr_comments = None + if cache.codegen is not None and cache.codegen.expr_comments: + expr_comments = json.dumps(cache.codegen.expr_comments).encode("utf-8") - db_code = DbStructuredCode( - kb=db_kb, - func_addr=func_addr, - flavor=flavor, - expr_comments=expr_comments, - stmt_comments=stmt_comments, - const_formats=const_formats, - ite_exprs=ite_exprs, - errors="\n\n\n".join(cache.errors), - # configuration=configuration, - ) - session.add(db_code) + stmt_comments = None + if cache.codegen is not None and cache.codegen.stmt_comments: + stmt_comments = json.dumps(cache.codegen.stmt_comments).encode("utf-8") + + const_formats = None + if cache.codegen is not None and cache.codegen.const_formats: + const_formats = json.dumps(ConstFormatsSerializer.to_json(cache.codegen.const_formats)).encode("utf-8") + + db_code = DbStructuredCode( + kb=db_kb, + func_addr=func_addr, + flavor=flavor, + expr_comments=expr_comments, + stmt_comments=stmt_comments, + const_formats=const_formats, + ite_exprs=None, + errors="\n\n\n".join(cache.errors), + ) + session.add(db_code) @staticmethod def dict_strkey_to_intkey(d: dict[str, Any]) -> dict[int, Any]: @@ -121,6 +156,8 @@ class StructuredCodeManagerSerializer: @staticmethod def load(session, db_kb: DbKnowledgeBase, kb: KnowledgeBase) -> StructuredCodeManager: """ + Load decompilation caches: fully serialized caches from the ``decompilation_caches`` table first, then legacy + ``structured_code`` rows (from this database or from old databases) for any key not already loaded. :param session: :param db_kb: @@ -129,10 +166,30 @@ class StructuredCodeManagerSerializer: """ manager = StructuredCodeManager(kb) + backing = manager.cached + + db_caches = session.query(DbDecompilationCache).filter_by(kb=db_kb) + if isinstance(backing, SpillingDecompilationDict) and db_caches.count() > backing.cache_limit: + # move the serialized bytes directly into the LMDB backing store and register every cache as spilled, + # instead of deserializing every cache and thrashing the LRU cache + backing.bulk_import_serialized( + [((db_cache.func_addr, db_cache.flavor), db_cache.blob) for db_cache in db_caches] + ) + else: + for db_cache in db_caches: + cache = DecompilationCache.parse( + db_cache.blob, + project=kb._project, # pylint:disable=protected-access + kb=kb, + function=kb.functions.get(db_cache.func_addr), + ) + backing[(db_cache.func_addr, db_cache.flavor)] = cache db_code_collection = session.query(DbStructuredCode).filter_by(kb=db_kb) for db_code in db_code_collection: + if (db_code.func_addr, db_code.flavor) in backing: + continue if not db_code.expr_comments: expr_comments = None else: diff --git a/angr/angrdb/serializers/variables.py b/angr/angrdb/serializers/variables.py index ab5e37836..eff7bcca8 100644 --- a/angr/angrdb/serializers/variables.py +++ b/angr/angrdb/serializers/variables.py @@ -7,9 +7,9 @@ try: except ImportError: sqlalchemy = None -from angr.angrdb.models import DbVariableCollection +from angr.angrdb.models import DbDecVariableCollection, DbVariableCollection from angr.knowledge_plugins import VariableManager -from angr.knowledge_plugins.variables.variable_manager import VariableManagerInternal +from angr.knowledge_plugins.variables.variable_manager import DecompilationVariableManager, VariableManagerInternal if TYPE_CHECKING: from angr.angrdb.models import DbKnowledgeBase @@ -19,14 +19,18 @@ if TYPE_CHECKING: class VariableManagerSerializer: """ Serialize/unserialize a variable manager and its variables. + + The same machinery serializes the disassembly-level manager (``kb.variables`` into the ``variables`` table) and + the decompilation manager (``kb.dec_variables`` into the ``dec_variables`` table); the target table and manager class + are parameterized. """ @staticmethod - def dump(session, db_kb: DbKnowledgeBase, var_manager: VariableManager): + def dump(session, db_kb: DbKnowledgeBase, var_manager: VariableManager, table=DbVariableCollection): assert sqlalchemy is not None # Remove all existing variable collections - session.query(DbVariableCollection).filter_by(kb=db_kb).delete() + session.query(table).filter_by(kb=db_kb).delete() # make sure db_kb has a primary key so it can be used as a foreign key in the Core bulk insert below session.flush() @@ -45,7 +49,11 @@ class VariableManagerSerializer: # bulk-insert the variable collection rows for speed if rows: - session.execute(sqlalchemy.insert(DbVariableCollection), rows) + session.execute(sqlalchemy.insert(table), rows) + + @staticmethod + def dump_dvars(session, db_kb: DbKnowledgeBase, var_manager: VariableManager): + VariableManagerSerializer.dump(session, db_kb, var_manager, table=DbDecVariableCollection) @staticmethod def _internal_row( @@ -61,10 +69,10 @@ class VariableManagerSerializer: return {"kb_id": db_kb.id, "ident": ident or None, "func_addr": func_addr, "blob": blob} @staticmethod - def load(session, db_kb: DbKnowledgeBase, kb: KnowledgeBase, ident=None): - variable_manager = VariableManager(kb) + def load(session, db_kb: DbKnowledgeBase, kb: KnowledgeBase, ident=None, table=DbVariableCollection): + variable_manager = DecompilationVariableManager(kb) if table is DbDecVariableCollection else VariableManager(kb) - db_varcolls = session.query(DbVariableCollection).filter_by(kb=db_kb, ident=ident) + db_varcolls = session.query(table).filter_by(kb=db_kb, ident=ident) for db_varcoll in db_varcolls: if not db_varcoll.blob: # databases created by older versions of angr may contain empty variable managers; they decode to @@ -78,6 +86,10 @@ class VariableManagerSerializer: return variable_manager + @staticmethod + def load_dvars(session, db_kb: DbKnowledgeBase, kb: KnowledgeBase, ident=None): + return VariableManagerSerializer.load(session, db_kb, kb, ident=ident, table=DbDecVariableCollection) + @staticmethod def load_internal(db_varcoll, variable_manager: VariableManager) -> VariableManagerInternal: return VariableManagerInternal.parse( diff --git a/angr/knowledge_plugins/key_definitions/atoms.py b/angr/knowledge_plugins/key_definitions/atoms.py index cc67fa91c..8e3bff10b 100644 --- a/angr/knowledge_plugins/key_definitions/atoms.py +++ b/angr/knowledge_plugins/key_definitions/atoms.py @@ -170,6 +170,9 @@ class Atom: def __eq__(self, other): return type(self) is type(other) and self._identity() == other._identity() + # The atom is serialized as a wrapping ``Atom`` cmessage carrying the per-kind inner cmessage in a oneof field; + # ``parse_from_cmessage`` dispatches on ``WhichOneof("kind")`` to the right subclass. + class GuardUse(Atom): """ diff --git a/angr/knowledge_plugins/key_definitions/definition.py b/angr/knowledge_plugins/key_definitions/definition.py index 9eefea5ab..c5aea2137 100644 --- a/angr/knowledge_plugins/key_definitions/definition.py +++ b/angr/knowledge_plugins/key_definitions/definition.py @@ -4,7 +4,7 @@ import logging from dataclasses import dataclass from typing import TYPE_CHECKING, Literal, TypeVar -from angr.code_location import ExternalCodeLocation +from angr.code_location import AILCodeLocation, ExternalCodeLocation from angr.engines.light import SpOffset from angr.knowledge_plugins.variables.variable_manager import VariableManagerInternal from angr.misc.ux import once @@ -20,7 +20,7 @@ from .atoms import Atom, AtomKind, MemoryLocation, Register, Tmp, VirtualVariabl from .tag import Tag if TYPE_CHECKING: - from angr.code_location import AILCodeLocation, CodeLocation + from angr.code_location import CodeLocation log = logging.getLogger(__name__) diff --git a/angr/knowledge_plugins/structured_code.py b/angr/knowledge_plugins/structured_code.py index 6ff8202b6..648c4f344 100644 --- a/angr/knowledge_plugins/structured_code.py +++ b/angr/knowledge_plugins/structured_code.py @@ -1,23 +1,269 @@ # pylint:disable=import-outside-toplevel from __future__ import annotations +import collections.abc +import logging +import os +from collections import OrderedDict from typing import TYPE_CHECKING, Any +import lmdb + import angr from .plugin import KnowledgeBasePlugin if TYPE_CHECKING: + from collections.abc import Iterator, MutableMapping + from angr.analyses.decompiler.decompilation_cache import DecompilationCache from angr.analyses.decompiler.structured_codegen import BaseStructuredCodeGenerator + from angr.knowledge_base import KnowledgeBase + +l = logging.getLogger(name=__name__) + +# The default number of decompilation caches to keep in memory when spilling is enabled. +DECOMPILATION_CACHE_LIMIT = 128 +USE_SPILLING_CODE_CACHE = os.environ.get("USE_SPILLING_CODE_CACHE", "True").lower() not in ("0", "false", "no") + +# (function address, flavor) +CacheKey = tuple[int, str] + + +class SpillingDecompilationDict(collections.abc.MutableMapping): + """ + A dict of decompilation caches, keyed by (function address, flavor), that keeps only the most recently used + cache_limit entries in memory and spills the rest to an LMDB database managed by the RuntimeDb knowledge base + plugin. + + Evicted entries are always serialized and written out (caches are mutated in place, so there is no clean/dirty + distinction). Entries that cannot be serialized (e.g. DummyStructuredCodeGenerator or rust-flavor caches) are + parked in an unbounded in-memory dict. Spilled entries are deserialized on access with the owning knowledge + base's project/function attached. + """ + + def __init__(self, kb: KnowledgeBase, cache_limit: int = DECOMPILATION_CACHE_LIMIT): + self._kb = kb + self._cache_limit: int = cache_limit + self._cache: OrderedDict[CacheKey, DecompilationCache] = OrderedDict() # LRU order: oldest first + self._spilled: set[CacheKey] = set() + self._unspillable: dict[CacheKey, DecompilationCache] = {} + self._db: str | None = None + self._eviction_enabled: bool = True + self._warned_unspillable: bool = False + # serialized entries restored by __setstate__, imported into LMDB on first access (the owning knowledge + # base may still be mid-unpickle during __setstate__) + self._pending_import: dict[CacheKey, bytes] | None = None + + # + # LMDB management + # + + @property + def cache_limit(self) -> int: + return self._cache_limit + + def _init_lmdb(self) -> None: + if self._db is None: + self._db = self._kb.rtdb.open_db("decompilations") + + @staticmethod + def _lmdb_key(key: CacheKey) -> bytes: + addr, flavor = key + return f"{addr}:{flavor}".encode() + + def _bulk_put(self, items: list[tuple[CacheKey, bytes]]) -> None: + self._init_lmdb() + assert self._db is not None + while True: + try: + with self._kb.rtdb.begin_txn(self._db, write=True) as txn: + for key, blob in items: + txn.put(self._lmdb_key(key), blob) + break + except lmdb.MapFullError: + self._kb.rtdb.increase_lmdb_map_size() + + def _flush_pending(self) -> None: + if self._pending_import: + items = list(self._pending_import.items()) + self._pending_import = None + self._bulk_put(items) + else: + self._pending_import = None + + def _save_to_lmdb(self, key: CacheKey, blob: bytes) -> None: + self._flush_pending() + self._bulk_put([(key, blob)]) + + def _load_from_lmdb(self, key: CacheKey) -> DecompilationCache: + self._flush_pending() + from angr.analyses.decompiler.decompilation_cache import DecompilationCache + + assert self._db is not None + with self._kb.rtdb.begin_txn(self._db) as txn: + blob = txn.get(self._lmdb_key(key)) + if blob is None: + raise KeyError(key) + addr, _flavor = key + cache = DecompilationCache.parse( + blob, + project=self._kb._project, # pylint:disable=protected-access + kb=self._kb, + function=self._kb.functions.get(addr), + ) + self._spilled.discard(key) + self[key] = cache + return cache + + # + # Eviction + # + + def _evict_lru(self) -> None: + while self._eviction_enabled and len(self._cache) > self._cache_limit: + key, cache = self._cache.popitem(last=False) + try: + blob = cache.serialize() + except Exception: # pylint:disable=broad-exception-caught + if not self._warned_unspillable: + self._warned_unspillable = True + l.warning( + "Decompilation cache %r cannot be serialized and will be kept in memory. Further " + "occurrences will not be logged.", + key, + exc_info=True, + ) + self._unspillable[key] = cache + continue + self._save_to_lmdb(key, blob) + self._spilled.add(key) + + # + # MutableMapping interface + # + + def __getitem__(self, key: CacheKey) -> DecompilationCache: + if key in self._cache: + self._cache.move_to_end(key) + return self._cache[key] + if key in self._unspillable: + return self._unspillable[key] + if key in self._spilled: + return self._load_from_lmdb(key) + raise KeyError(key) + + def __setitem__(self, key: CacheKey, value: DecompilationCache) -> None: + self._spilled.discard(key) + self._unspillable.pop(key, None) + self._cache[key] = value + self._cache.move_to_end(key) + self._evict_lru() + + def __delitem__(self, key: CacheKey) -> None: + if key in self._cache: + del self._cache[key] + elif key in self._unspillable: + del self._unspillable[key] + elif key in self._spilled: + # don't bother deleting the LMDB record; the key is simply forgotten + self._spilled.discard(key) + else: + raise KeyError(key) + + def __contains__(self, key) -> bool: + return key in self._cache or key in self._unspillable or key in self._spilled + + def __len__(self) -> int: + return len(self._cache) + len(self._unspillable) + len(self._spilled) + + def __iter__(self) -> Iterator[CacheKey]: + yield from self._cache + yield from self._unspillable + yield from self._spilled + + # + # Bulk serialized access (for angr databases) + # + + def export_serialized(self) -> tuple[list[tuple[CacheKey, bytes]], dict[CacheKey, DecompilationCache]]: + """ + Export all serializable entries as (key, serialized bytes) pairs. Spilled entries are copied directly out of + the LMDB backing store without being deserialized and re-serialized. Entries that cannot be serialized are + returned separately in a dict. + """ + self._flush_pending() + serialized: list[tuple[CacheKey, bytes]] = [] + unserializable: dict[CacheKey, DecompilationCache] = dict(self._unspillable) + + if self._spilled: + assert self._db is not None + with self._kb.rtdb.begin_txn(self._db) as txn: + for key in self._spilled: + blob = txn.get(self._lmdb_key(key)) + if blob is not None: + serialized.append((key, blob)) + + for key, cache in self._cache.items(): + try: + serialized.append((key, cache.serialize())) + except Exception: # pylint:disable=broad-exception-caught + unserializable[key] = cache + + return serialized, unserializable + + def bulk_import_serialized(self, items: list[tuple[CacheKey, bytes]]) -> None: + """ + Move already-serialized decompilation caches directly into the LMDB backing store and register them as + spilled, without deserializing them. The bytes must be serialized DecompilationCache messages, i.e., the + exact format that eviction writes. + """ + if not items: + return + + self._flush_pending() + self._bulk_put(items) + + for key, _ in items: + # LMDB now holds the authoritative data; drop any stale in-memory copy + self._cache.pop(key, None) + self._unspillable.pop(key, None) + self._spilled.add(key) + + # + # Pickling + # + # Serializable entries are pickled as their protobuf bytes (live caches hold unpicklable analysis internals); + # unserializable entries are pickled as-is. + # + + def __getstate__(self) -> dict: + serialized, unspillable = self.export_serialized() + return { + "kb": self._kb, + "cache_limit": self._cache_limit, + "serialized": serialized, + "unspillable": unspillable, + } + + def __setstate__(self, state: dict) -> None: + self.__init__(state["kb"], cache_limit=state["cache_limit"]) # type: ignore[misc] + # defer the LMDB import to the first real access; the knowledge base may still be mid-unpickle here + self._pending_import = dict(state["serialized"]) + self._spilled = set(self._pending_import) + self._unspillable.update(state["unspillable"]) class StructuredCodeManager(KnowledgeBasePlugin): """A knowledge base plugin to store structured code generator results.""" - def __init__(self, kb): + def __init__(self, kb, cache_limit: int | None = None): super().__init__(kb=kb) - self.cached: dict[Any, DecompilationCache] = {} + if cache_limit is None and USE_SPILLING_CODE_CACHE: + cache_limit = DECOMPILATION_CACHE_LIMIT + self.cached: MutableMapping[Any, DecompilationCache] = ( + SpillingDecompilationDict(kb, cache_limit=cache_limit) if cache_limit is not None else {} + ) def _normalize_key(self, item): if type(item) is not tuple: diff --git a/angr/knowledge_plugins/variables/__init__.py b/angr/knowledge_plugins/variables/__init__.py index 4babc2ed2..04f191ddf 100644 --- a/angr/knowledge_plugins/variables/__init__.py +++ b/angr/knowledge_plugins/variables/__init__.py @@ -1,8 +1,9 @@ from __future__ import annotations -from .variable_manager import VariableManager, VariableType +from .variable_manager import DecompilationVariableManager, VariableManager, VariableType __all__ = ( + "DecompilationVariableManager", "VariableManager", "VariableType", ) diff --git a/angr/knowledge_plugins/variables/spilling_vardict.py b/angr/knowledge_plugins/variables/spilling_vardict.py new file mode 100644 index 000000000..5804352ea --- /dev/null +++ b/angr/knowledge_plugins/variables/spilling_vardict.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import collections.abc +import logging +import os +from collections import OrderedDict +from typing import TYPE_CHECKING + +import lmdb + +if TYPE_CHECKING: + from collections.abc import Iterator + + from .variable_manager import DecompilationVariableManager, VariableManagerInternal + +l = logging.getLogger(name=__name__) + +# The default number of per-function decompilation variable managers to keep in memory when spilling is enabled. +DECVARS_CACHE_LIMIT = 1000 +USE_SPILLING_DVARS = os.environ.get("USE_SPILLING_DVARS", "True").lower() not in ("0", "false", "no") + + +class SpillingVariableInternalDict(collections.abc.MutableMapping): + """ + A dict of per-function VariableManagerInternal instances (keyed by function address) that keeps only the most + recently used cache_limit entries in memory and spills the rest to an LMDB database managed by the RuntimeDb + knowledge base plugin. It is the backing store for ``DecompilationVariableManager.function_managers``. + + Evicted entries are serialized and written out; they are deserialized on access with the owning manager + reattached. Decompilation only mutates a function's internal manager while decompiling that function (when it is + the most-recently-used entry) and treats it as read-only afterwards, so spilling is safe. + + On pickle, the serialized entries travel inside the pickle and are re-imported into a fresh LMDB on first access + after unpickling. + """ + + def __init__(self, manager: DecompilationVariableManager, cache_limit: int = DECVARS_CACHE_LIMIT): + self._manager = manager + self._cache_limit: int = cache_limit + self._cache: OrderedDict[int, VariableManagerInternal] = OrderedDict() # LRU order: oldest first + self._spilled: set[int] = set() + self._db: str | None = None + self._eviction_enabled: bool = True + # serialized entries restored by __setstate__, imported into LMDB on first access (the owning knowledge + # base may still be mid-unpickle during __setstate__) + self._pending_import: dict[int, bytes] | None = None + + @property + def cache_limit(self) -> int: + return self._cache_limit + + # + # LMDB management + # + + @property + def _kb(self): + return self._manager._kb # pylint: disable=protected-access + + def _init_lmdb(self) -> None: + if self._db is None: + self._db = self._kb.rtdb.open_db("dvars") + + @staticmethod + def _lmdb_key(key: int) -> bytes: + return str(key).encode() + + def _bulk_put(self, items: list[tuple[int, bytes]]) -> None: + self._init_lmdb() + assert self._db is not None + while True: + try: + with self._kb.rtdb.begin_txn(self._db, write=True) as txn: + for key, blob in items: + txn.put(self._lmdb_key(key), blob) + break + except lmdb.MapFullError: + self._kb.rtdb.increase_lmdb_map_size() + + def _flush_pending(self) -> None: + if self._pending_import: + items = list(self._pending_import.items()) + self._pending_import = None + self._bulk_put(items) + else: + self._pending_import = None + + def _save_to_lmdb(self, key: int, blob: bytes) -> None: + self._flush_pending() + self._bulk_put([(key, blob)]) + + def _load_from_lmdb(self, key: int) -> VariableManagerInternal: + + from .variable_manager import VariableManagerInternal # pylint:disable=import-outside-toplevel + + self._flush_pending() + + assert self._db is not None + with self._kb.rtdb.begin_txn(self._db) as txn: + blob = txn.get(self._lmdb_key(key)) + if blob is None: + raise KeyError(key) + internal = VariableManagerInternal.parse(blob, variable_manager=self._manager, func_addr=key) + internal.set_manager(self._manager) + self._spilled.discard(key) + self[key] = internal + return internal + + # + # Eviction + # + + def _evict_lru(self) -> None: + while self._eviction_enabled and len(self._cache) > self._cache_limit: + key, internal = self._cache.popitem(last=False) + blob = internal.serialize() + self._save_to_lmdb(key, blob) + self._spilled.add(key) + + # + # MutableMapping interface + # + + def __getitem__(self, key: int) -> VariableManagerInternal: + if key in self._cache: + self._cache.move_to_end(key) + return self._cache[key] + if key in self._spilled: + return self._load_from_lmdb(key) + raise KeyError(key) + + def __setitem__(self, key: int, value: VariableManagerInternal) -> None: + self._spilled.discard(key) + self._cache[key] = value + self._cache.move_to_end(key) + self._evict_lru() + + def __delitem__(self, key: int) -> None: + if key in self._cache: + del self._cache[key] + elif key in self._spilled: + # don't bother deleting the LMDB record; the key is simply forgotten + self._spilled.discard(key) + else: + raise KeyError(key) + + def __contains__(self, key) -> bool: + return key in self._cache or key in self._spilled + + def __len__(self) -> int: + return len(self._cache) + len(self._spilled) + + def __iter__(self) -> Iterator[int]: + # snapshot the keys: consumers that call __getitem__ per key (e.g. items()/values()) mutate the live + # containers mid-iteration + yield from list(self._cache) + yield from list(self._spilled) + + # + # Pickling + # + # Live entries are serialized to their protobuf bytes and spilled entries are copied straight out of LMDB, so + # the pickle is self-contained and does not reference the (non-durable) RuntimeDb. + # + + def __getstate__(self) -> dict: + self._flush_pending() + serialized: dict[int, bytes] = {key: internal.serialize() for key, internal in self._cache.items()} + if self._spilled: + assert self._db is not None + with self._kb.rtdb.begin_txn(self._db) as txn: + for key in self._spilled: + blob = txn.get(self._lmdb_key(key)) + if blob is not None: + serialized[key] = blob + return {"manager": self._manager, "cache_limit": self._cache_limit, "serialized": serialized} + + def __setstate__(self, state: dict) -> None: + self.__init__(state["manager"], cache_limit=state["cache_limit"]) # type: ignore[misc] + # defer the LMDB import to the first real access; the knowledge base may still be mid-unpickle here + self._pending_import = dict(state["serialized"]) + self._spilled = set(self._pending_import) diff --git a/angr/knowledge_plugins/variables/variable_manager.py b/angr/knowledge_plugins/variables/variable_manager.py index 0c55992ca..c09adc45c 100644 --- a/angr/knowledge_plugins/variables/variable_manager.py +++ b/angr/knowledge_plugins/variables/variable_manager.py @@ -39,6 +39,7 @@ from angr.utils.ail import is_phi_assignment from angr.utils.orderedset import OrderedSet from angr.utils.types import replace_pointer_pts_to, unpack_pointer +from .spilling_vardict import USE_SPILLING_DVARS, SpillingVariableInternalDict from .variable_access import VariableAccess, VariableAccessSort if TYPE_CHECKING: @@ -1282,10 +1283,12 @@ class VariableManager(KnowledgeBasePlugin): Manage variables. """ + function_managers: dict[int, VariableManagerInternal] | SpillingVariableInternalDict + def __init__(self, kb): super().__init__(kb=kb) self.global_manager = VariableManagerInternal(self) - self.function_managers: dict[int, VariableManagerInternal] = {} + self.function_managers = {} def __contains__(self, key) -> bool: if key == "global": @@ -1394,4 +1397,31 @@ class VariableManager(KnowledgeBasePlugin): self.convert_variable_list(subp.local_variables, manager) +class DecompilationVariableManager(VariableManager): + """ + Holds variables discovered during decompilation, kept separate from the disassembly-level ``kb.variables``. + Exposed as ``kb.dec_variables``. Per-function managers are held in a :class:`SpillingVariableInternalDict`, which + spills least-recently-used entries to the RuntimeDb LMDB store (disable via ``USE_SPILLING_DVARS``). + """ + + def __init__(self, kb): + super().__init__(kb) + if USE_SPILLING_DVARS: + self.function_managers = SpillingVariableInternalDict(self) + + def copy(self) -> DecompilationVariableManager: + new = DecompilationVariableManager(self._kb) + new.global_manager = self._copy_internal(self.global_manager, new) + for addr, vmi in self.function_managers.items(): + new.function_managers[addr] = self._copy_internal(vmi, new) + return new + + @staticmethod + def _copy_internal(vmi: VariableManagerInternal, manager: VariableManager) -> VariableManagerInternal: + clone = VariableManagerInternal.parse(vmi.serialize(), variable_manager=manager, func_addr=vmi.func_addr) + clone.set_manager(manager) + return clone + + KnowledgeBasePlugin.register_default("variables", VariableManager) +KnowledgeBasePlugin.register_default("dec_variables", DecompilationVariableManager) diff --git a/angr/protos/__init__.py b/angr/protos/__init__.py index fa303bfc4..ddc3c9460 100644 --- a/angr/protos/__init__.py +++ b/angr/protos/__init__.py @@ -1,7 +1,10 @@ -# Generating proto files +# The *_pb2.py modules in this package are generated from the .proto sources at build/install time (see +# build_protos() in setup.py) and are not committed. After editing a .proto, regenerate manually with # -# $ cd angr # you would expect angr/protos to exist after this -# $ protoc -I=. --python_out=. angr/protos/*.proto +# $ cd angr # the repository root +# $ python -m grpc_tools.protoc -I. --python_out=. angr/protos/*.proto +# +# (grpcio-tools is a build dependency; installs with --no-build-isolation need it installed in the environment.) from __future__ import annotations from . import cfg_pb2, function_pb2, primitives_pb2, variables_pb2, xrefs_pb2 diff --git a/angr/protos/ail_types.proto b/angr/protos/ail_types.proto new file mode 100644 index 000000000..b7ee0c8fb --- /dev/null +++ b/angr/protos/ail_types.proto @@ -0,0 +1,102 @@ +syntax = "proto3"; + +package angr.protos; + +// +// networkx.DiGraph[ailment.Block] +// + +enum AilEdgeType { + AIL_EDGE_TYPE_UNSPECIFIED = 0; + AIL_EDGE_TRANSITION = 1; + AIL_EDGE_EXCEPTION = 2; + AIL_EDGE_FAKE_RETURN = 3; + AIL_EDGE_CALL = 4; + AIL_EDGE_SYSCALL = 5; + AIL_EDGE_RETURN = 6; +} + +message AilEdgeData { + optional AilEdgeType type = 1; + optional bool outside = 2; + optional uint64 ins_addr = 3; + optional sint64 stmt_idx = 4; // DEFAULT_STATEMENT == -2 + optional bool confirmed = 5; +} + +message AilEdge { + uint32 src = 1; // index into AilGraph.blocks + uint32 dst = 2; + optional AilEdgeData data = 3; + // When true, the edge's data is AilGraph.default_edge_data overlaid with ``data`` (which then carries only the + // fields that differ, e.g. a per-edge ins_addr). Lets the common default edge attributes be stored once. + bool has_default_data = 4; +} + +message AilGraph { + // Either ``blocks`` carries the payloads inline, or ``block_refs`` indexes into a caller-supplied shared pool + // (e.g. Clinic.block_pool) so that byte-identical blocks shared between graphs are stored once. + repeated bytes blocks = 1; // ailment.Block.to_bytes() payloads, in node-insertion order + repeated AilEdge edges = 2; + repeated uint32 block_refs = 3; // indices into the shared pool, in node-insertion order + // The modal edge-data value (excluding the per-edge ins_addr); edges with has_default_data reference it. + optional AilEdgeData default_edge_data = 4; +} + +// +// dict[int, tuple[ailment.Expr.VirtualVariable, SimVariable]] (Clinic.arg_vvars, DecompilationCache.arg_vvars) +// + +message ArgVVar { + bytes vvar = 1; // ailment.Expr.VirtualVariable via Expression.to_bytes() + bytes simvar = 2; // polymorphic "\0" SimVariable encoding +} + +message ArgVVars { + map entries = 1; +} + +// +// set[tuple[int, ailment.Expr.Expression]] (DecompilationCache.ite_exprs) +// + +message IteExpr { + int64 addr = 1; + bytes expr = 2; // Expression.to_bytes() +} + +message IteExprs { + repeated IteExpr entries = 1; +} + +// +// Static buffer parameters (optimization_passes.static_vvar_rewriter) +// + +message FixedBufferMsg { + string ident = 1; + int64 size = 2; + bytes content = 3; +} + +message FixedBufferPtrMsg { + string buffer_ident = 1; + int64 offset = 2; +} + +// dict[int, FixedBufferPtr | ailment.Expr.Const] (decompilation parameter "static_vvars") +message StaticVVar { + oneof v { + FixedBufferPtrMsg ptr = 1; + bytes const_expr = 2; // Expression.to_bytes() + } +} + +message StaticVVars { + map entries = 1; +} + +// dict[str, FixedBuffer] (decompilation parameter "static_buffers") +message StaticBuffers { + map entries = 1; +} diff --git a/angr/protos/cfg_pb2.py b/angr/protos/cfg_pb2.py deleted file mode 100644 index 4283f8c2d..000000000 --- a/angr/protos/cfg_pb2.py +++ /dev/null @@ -1,54 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# NO CHECKED-IN PROTOBUF GENCODE -# source: angr/protos/cfg.proto -# Protobuf Python Version: 6.33.0 -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import runtime_version as _runtime_version -from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -_runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 6, - 33, - 0, - '', - 'angr/protos/cfg.proto' -) -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from angr.protos import primitives_pb2 as angr_dot_protos_dot_primitives__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15\x61ngr/protos/cfg.proto\x12\x0b\x61ngr.protos\x1a\x1c\x61ngr/protos/primitives.proto\"1\n\rOptionalInt64\x12\x11\n\thas_value\x18\x01 \x01(\x08\x12\r\n\x05value\x18\x02 \x01(\x03\"d\n\x0c\x42lockIDProto\x12\x0c\n\x04\x61\x64\x64r\x18\x01 \x01(\x04\x12\x33\n\x0f\x63\x61llsite_tuples\x18\x02 \x03(\x0b\x32\x1a.angr.protos.OptionalInt64\x12\x11\n\tjump_type\x18\x03 \x01(\t\"_\n\x0b\x43\x46GEdgeData\x12,\n\x08jumpkind\x18\x01 \x01(\x0e\x32\x1a.angr.protos.Edge.JumpKind\x12\x10\n\x08ins_addr\x18\x02 \x01(\x04\x12\x10\n\x08stmt_idx\x18\x03 \x01(\x05\"\xd2\x02\n\x07\x43\x46GNode\x12\n\n\x02\x65\x61\x18\x01 \x01(\x04\x12\x0c\n\x04size\x18\x02 \x01(\r\x12\x10\n\x08\x62lock_id\x18\x03 \x03(\x04\x12\x1e\n\x11simprocedure_name\x18\x06 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06no_ret\x18\x07 \x01(\x08H\x01\x88\x01\x01\x12\x1d\n\x10\x66unction_address\x18\x08 \x01(\x04H\x02\x88\x01\x01\x12\r\n\x05thumb\x18\t \x01(\x08\x12\x18\n\x0b\x62yte_string\x18\n \x01(\x0cH\x03\x88\x01\x01\x12\x11\n\x04name\x18\x0b \x01(\tH\x04\x88\x01\x01\x12\x12\n\nis_syscall\x18\x0c \x01(\x08\x12\x15\n\rins_base_addr\x18\r \x01(\x04\x12\x11\n\tins_sizes\x18\x0e \x01(\x0c\x42\x14\n\x12_simprocedure_nameB\t\n\x07_no_retB\x13\n\x11_function_addressB\x0e\n\x0c_byte_stringB\x07\n\x05_name\"\xf5\x02\n\x08\x43\x46GENode\x12\"\n\x04\x62\x61se\x18\x01 \x01(\x0b\x32\x14.angr.protos.CFGNode\x12\x31\n\rcallstack_key\x18\x02 \x03(\x0b\x32\x1a.angr.protos.OptionalInt64\x12\x19\n\x0csyscall_name\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x15\n\rlooping_times\x18\x04 \x01(\x05\x12\x12\n\x05\x64\x65pth\x18\x05 \x01(\x05H\x01\x88\x01\x01\x12\x1a\n\rreturn_target\x18\x06 \x01(\x04H\x02\x88\x01\x01\x12\"\n\x15\x63reation_failure_info\x18\x07 \x01(\tH\x03\x88\x01\x01\x12\x34\n\x0c\x62lock_id_obj\x18\x08 \x01(\x0b\x32\x19.angr.protos.BlockIDProtoH\x04\x88\x01\x01\x42\x0f\n\r_syscall_nameB\x08\n\x06_depthB\x10\n\x0e_return_targetB\x18\n\x16_creation_failure_infoB\x0f\n\r_block_id_obj\"\xee\x01\n\x03\x43\x46G\x12\r\n\x05ident\x18\x01 \x01(\t\x12#\n\x05nodes\x18\x02 \x03(\x0b\x32\x14.angr.protos.CFGNode\x12 \n\x05\x65\x64ges\x18\x03 \x03(\x0b\x32\x11.angr.protos.Edge\x12,\n\x0bmemory_data\x18\x04 \x03(\x0b\x32\x17.angr.protos.MemoryData\x12\x12\n\nnormalized\x18\x05 \x01(\x08\x12.\n\x0bjump_tables\x18\x06 \x03(\x0b\x32\x19.angr.protos.IndirectJump\x12\x1f\n\x17\x62lock_addrs_with_return\x18\x07 \x03(\x04\"\xa4\x04\n\nMemoryData\x12\n\n\x02\x65\x61\x18\x01 \x01(\x04\x12\x11\n\x04size\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x34\n\x04type\x18\x03 \x01(\x0e\x32&.angr.protos.MemoryData.MemoryDataType\x12\x1b\n\x0ereference_size\x18\x04 \x01(\rH\x01\x88\x01\x01\"\x87\x03\n\x0eMemoryDataType\x12\x13\n\x0fUnknownDataType\x10\x00\x12\x0f\n\x0bUnspecified\x10\x01\x12\x0b\n\x07Integer\x10\x02\x12\x10\n\x0cPointerArray\x10\x03\x12\n\n\x06String\x10\x04\x12\x11\n\rUnicodeString\x10\x05\x12\x13\n\x0fSegmentBoundary\x10\x06\x12\x11\n\rCodeReference\x10\x07\x12\x0f\n\x0bGOTPLTEntry\x10\x08\x12\r\n\tELFHeader\x10\t\x12\x11\n\rFloatingPoint\x10\n\x12\r\n\tAlignment\x10\x0b\x12\x15\n\x11PEImportDirectory\x10\x0c\x12\x15\n\x11PEExportDirectory\x10\r\x12\x1a\n\x16PEDelayImportDirectory\x10\x0e\x12\x0e\n\nEHFuncInfo\x10\x0f\x12\x14\n\x10\x45HUnwindMapEntry\x10\x10\x12\x11\n\rEH4ScopeTable\x10\x11\x12\x11\n\rEHTryBlockMap\x10\x12\x12\x11\n\rEHHandlerType\x10\x13\x42\x07\n\x05_sizeB\x11\n\x0f_reference_size\"w\n\rJumptableInfo\x12\x11\n\x04\x61\x64\x64r\x18\x01 \x01(\x04H\x00\x88\x01\x01\x12\x0c\n\x04size\x18\x02 \x01(\x05\x12\x12\n\nentry_size\x18\x03 \x01(\x05\x12\x0f\n\x07\x65ntries\x18\x04 \x03(\x04\x12\x17\n\x0f\x65ntries_guessed\x18\x05 \x01(\x08\x42\x07\n\x05_addr\"\xd0\x01\n\x0cIndirectJump\x12\x0c\n\x04\x61\x64\x64r\x18\x01 \x01(\x04\x12\x10\n\x08ins_addr\x18\x02 \x01(\x04\x12\x11\n\tfunc_addr\x18\x03 \x01(\x04\x12\x10\n\x08jumpkind\x18\x04 \x01(\t\x12\x10\n\x08stmt_idx\x18\x05 \x01(\x05\x12\x18\n\x10resolved_targets\x18\x06 \x03(\x04\x12\x11\n\tjumptable\x18\x07 \x01(\x08\x12.\n\njumptables\x18\x08 \x03(\x0b\x32\x1a.angr.protos.JumptableInfo\x12\x0c\n\x04type\x18\t \x01(\x05\x62\x06proto3') - -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'angr.protos.cfg_pb2', _globals) -if not _descriptor._USE_C_DESCRIPTORS: - DESCRIPTOR._loaded_options = None - _globals['_OPTIONALINT64']._serialized_start=68 - _globals['_OPTIONALINT64']._serialized_end=117 - _globals['_BLOCKIDPROTO']._serialized_start=119 - _globals['_BLOCKIDPROTO']._serialized_end=219 - _globals['_CFGEDGEDATA']._serialized_start=221 - _globals['_CFGEDGEDATA']._serialized_end=316 - _globals['_CFGNODE']._serialized_start=319 - _globals['_CFGNODE']._serialized_end=657 - _globals['_CFGENODE']._serialized_start=660 - _globals['_CFGENODE']._serialized_end=1033 - _globals['_CFG']._serialized_start=1036 - _globals['_CFG']._serialized_end=1274 - _globals['_MEMORYDATA']._serialized_start=1277 - _globals['_MEMORYDATA']._serialized_end=1825 - _globals['_MEMORYDATA_MEMORYDATATYPE']._serialized_start=1406 - _globals['_MEMORYDATA_MEMORYDATATYPE']._serialized_end=1797 - _globals['_JUMPTABLEINFO']._serialized_start=1827 - _globals['_JUMPTABLEINFO']._serialized_end=1946 - _globals['_INDIRECTJUMP']._serialized_start=1949 - _globals['_INDIRECTJUMP']._serialized_end=2157 -# @@protoc_insertion_point(module_scope) diff --git a/angr/protos/clinic.proto b/angr/protos/clinic.proto new file mode 100644 index 000000000..f7e50c5e8 --- /dev/null +++ b/angr/protos/clinic.proto @@ -0,0 +1,126 @@ +syntax = "proto3"; + +import "angr/protos/ail_types.proto"; + +package angr.protos; + +// +// Mirrored enums +// +// ClinicMode and ClinicStage are stored as raw int32 values matching the Python ``.value`` of each enum member. + +// ===================================================================================================================== +// Small value types +// ===================================================================================================================== + +enum StackItemType { + SI_UNKNOWN = 0; + SI_SAVED_BP = 1; + SI_SAVED_REGS = 2; + SI_ARGUMENT = 3; + SI_RET_ADDR = 4; + SI_STACK_CANARY = 5; +} + +message StackItem { + int32 offset = 1; + int32 size = 2; + string name = 3; + StackItemType item_type = 4; +} + +message AddressTuple { + // ailment.Address = tuple[int, int | None] + int64 addr = 1; + optional int32 idx = 2; +} + +message AddressPair { + AddressTuple src = 1; + AddressTuple dst = 2; +} + +message ExternBytes { + // serialized SimMemoryVariable subclass via "\\0" so polymorphic dispatch works at parse. + bytes payload = 1; +} + +// ===================================================================================================================== +// Clinic +// ===================================================================================================================== +// Many ctor parameters double as instance attributes; we round-trip all of them so the cache-hit path sees the same +// configuration as the original Clinic. Runtime back-references (project / kb / function / cfg / +// _handlers / typehoon / _spt / _cache) are reattached at parse time from kwargs and are NOT serialized. + +message Clinic { + // ---- Function and arch hints (resolved from project at parse time) ----------------------------------------------- + optional uint64 function_addr = 1; + optional string flavor = 2; + + // ---- AIL-typed slots (typed messages from ail_types.proto) ------------------------------------------------------- + // Shared pool of ailment.Block.to_bytes() payloads; the graphs reference into it via AilGraph.block_refs so + // byte-identical blocks appearing in multiple graphs are stored once. + repeated bytes block_pool = 3; + optional AilGraph cc_graph = 4; + optional AilGraph graph = 5; // final graph after region simplification + optional AilGraph unoptimized_graph = 6; // only when Decompiler(save_unoptimized_graph=True) + optional ArgVVars arg_vvars = 7; // dict[int, tuple[VirtualVariable, SimVariable]] + + // ---- Already-Serializable sub-objects --------------------------------------------------------------------------- + // arg_list: list[SimVariable] | None + repeated ExternBytes arg_list = 8; + repeated ExternBytes externs = 9; // set[SimMemoryVariable] + // Only state consumed by the decompiler's cache-reuse path is serialized. + + // ---- CLEAN primitive collections -------------------------------------------------------------------------------- + map vvar_to_vvar = 10; // dict[int, int] + repeated uint32 secondary_stackvars = 11; // set[int] + repeated uint32 removed_vvar_ids = 12; // set[int] | None + optional bool _removed_vvar_ids_set = 13; + repeated uint32 _preserve_vvar_ids = 14; + map _inlined_counts = 15; // dict[int, int] + repeated uint64 _inlining_parents = 16; // set[int] + repeated string _must_struct = 17; // set[str] | None + optional bool _must_struct_set = 18; + repeated string _desired_variables = 19; + optional bool _desired_variables_set = 20; + map stack_items = 21; // dict[int, StackItem] + repeated AddressPair edges_to_remove = 22; + repeated uint32 copied_var_ids = 23; + repeated uint32 _new_block_addrs = 24; + optional AddressTuple entry_node_addr = 25; + + // ---- CLEAN scalars ---------------------------------------------------------------------------------------------- + int32 vvar_id_start = 26; + int32 _max_stack_depth = 27; + int32 _sp_shift = 28; + int32 _max_type_constraints = 29; + int32 _type_constraint_set_degradation_threshold = 30; + bool _fold_callexprs_into_conditions = 31; + bool _fold_expressions = 32; + bool _insert_labels = 33; + bool _remove_dead_memdefs = 34; + bool _exception_edges = 35; + bool _sp_tracker_track_memory = 36; + bool _reset_variable_names = 37; + bool _rewrite_ites_to_diamonds = 38; + bool _flatten_args = 39; + bool _semvar_naming = 40; + bool _force_loop_single_exit = 41; + bool _refine_loops_with_single_successor = 42; + bool _register_save_areas_removed = 43; + int32 _rewrite_ites_to_diamond_max_cases = 44; + bool _expose_loop_head_backedges = 45; + bool _constrain_callee_prototypes = 46; + bool _save_unoptimized_graph = 47; + + int32 _mode = 48; // ClinicMode.value + int32 _start_stage = 49; // ClinicStage.value + int32 _end_stage = 50; + repeated int32 _skip_stages = 51; + + // ---- Pass class references (round-tripped via the optimization-pass registry) ----------------------------------- + repeated string peephole_optimizations = 52; // FQNs from optimization_pass_registry + bool peephole_optimizations_use_default = 53; // True when peephole_optimizations was None (default set) + string _typehoon_cls = 54; // FQN; empty -> default Typehoon +} diff --git a/angr/protos/codegen.proto b/angr/protos/codegen.proto new file mode 100644 index 000000000..edcd659b2 --- /dev/null +++ b/angr/protos/codegen.proto @@ -0,0 +1,509 @@ +syntax = "proto3"; + + + +package angr.protos; + +// ===================================================================================================================== +// Indexed AST node table +// ===================================================================================================================== +// Every CConstruct instance is assigned a unique non-zero ``uint32 node_id`` at serialize time. Children that are +// CConstructs are referenced by their node_id rather than embedded inline. This lets PositionMapping / map_addr_to_label +// / cexterns reference into the AST without duplicating subtrees, and gives us a stable handle for resolving cross-tree +// references on parse. +// +// The codegen-level ``Codegen`` message carries: +// - ``root_id`` : node_id of the top-level CFunction. +// - ``nodes`` : flat list of every CConstruct in the AST. +// +// Reference semantics: a missing ``optional ChildId`` field means "no child" (e.g., CWhileLoop with no body). A 0 in a +// ``repeated`` child-id field is never produced (we never assign 0 as a node_id), but readers should treat 0 as absent +// just in case. +// ===================================================================================================================== + +// CConstruct.tags is constrained at serialize time to entries with string keys and primitive values (None / bool / +// int / float / str / list / dict of those, in any nesting). Non-conforming entries are dropped on serialize. +message CConstructTags { + // The four dominant tag keys get typed fields; everything else stays in the JSON map. + map json_values = 1; // map[key] -> json.dumps(value) + optional uint64 ins_addr = 2; + optional uint64 vex_block_addr = 3; + optional int32 vex_stmt_idx = 4; + optional bool is_prototype_guessed = 5; + // When both ins_addr and vex_block_addr are known, ins_addr is stored as this delta from vex_block_addr + // (typically a 1-2 byte varint) and the absolute ins_addr field is left unset. + optional sint64 ins_offset = 6; +} + +// Identifies which payload (oneof variant) a CConstructNode carries; drives the polymorphic dispatch table in +// Python code. +enum CConstructKind { + CCK_UNKNOWN = 0; + CCK_FUNCTION = 1; + CCK_STATEMENTS = 2; + CCK_AIL_BLOCK = 3; + CCK_WHILE_LOOP = 4; + CCK_DO_WHILE_LOOP = 5; + CCK_FOR_LOOP = 6; + CCK_IF_ELSE = 7; + CCK_IF_BREAK = 8; + CCK_BREAK = 9; + CCK_CONTINUE = 10; + CCK_SWITCH_CASE = 11; + CCK_INCOMPLETE_SWITCH_CASE = 12; + CCK_ASSIGNMENT = 13; + CCK_EXPRESSION_STATEMENT = 14; + CCK_RETURN = 15; + CCK_GOTO = 16; + CCK_UNSUPPORTED_STATEMENT = 17; + CCK_DIRTY_STATEMENT = 18; + CCK_LABEL = 19; + CCK_FUNCTION_CALL = 20; + CCK_STRUCT_FIELD = 21; + CCK_FAKE_VARIABLE = 22; + CCK_VARIABLE = 23; + CCK_INDEXED_VARIABLE = 24; + CCK_VARIABLE_FIELD = 25; + CCK_UNARY_OP = 26; + CCK_BINARY_OP = 27; + CCK_TYPE_CAST = 28; + CCK_CONSTANT = 29; + CCK_REGISTER = 30; + CCK_ITE = 31; + CCK_MULTI_STATEMENT_EXPRESSION = 32; + CCK_VEX_CCALL_EXPRESSION = 33; + CCK_DIRTY_EXPRESSION = 34; +} + +// ===================================================================================================================== +// Per-subclass payload messages +// ===================================================================================================================== + +// CFunction +message CFunctionMsg { + optional uint64 addr = 1; + string name = 2; + uint32 functy_ref = 3; // SimTypeFunction; index+1 into Codegen.type_pool, 0 = absent + repeated uint32 arg_list_ids = 4; // CVariable node_ids + uint32 statements_id = 5; // CStatements node_id + repeated VariableInUseEntry variables_in_use = 6; + optional string demangled_name = 7; + bool show_demangled_name = 8; + bool omit_header = 9; + // variable_manager is reattached at parse time from kwargs; unified_local_vars is recomputed via refresh(). +} + +message VariableInUseEntry { + uint32 simvariable_ref = 1; // index+1 into Codegen.simvar_pool; 0 = absent + uint32 cvariable_id = 2; // CVariable node_id +} + +// CStatements +message CStatementsMsg { + repeated uint32 statements_ids = 1; + optional uint64 addr = 2; +} + +// CAILBlock (ailment-coupled) +message CAILBlockMsg { + bytes block = 1; // ailment.Block.to_bytes() +} + +// CWhileLoop / CDoWhileLoop — same shape. +message CLoopMsg { + optional uint32 condition_id = 1; + optional uint32 body_id = 2; +} + +// CForLoop +message CForLoopMsg { + optional uint32 initializer_id = 1; + optional uint32 condition_id = 2; + optional uint32 iterator_id = 3; + optional uint32 body_id = 4; +} + +// CIfElse +message CIfElseBranch { + uint32 condition_id = 1; + optional uint32 statement_id = 2; +} +message CIfElseMsg { + repeated CIfElseBranch condition_and_nodes = 1; + optional uint32 else_node_id = 2; + bool simplify_else_scope = 3; + bool cstyle_ifs = 4; +} + +// CIfBreak +message CIfBreakMsg { + uint32 condition_id = 1; + bool cstyle_ifs = 2; +} + +// CBreak / CContinue +message CBreakMsg {} +message CContinueMsg {} + +// CSwitchCase +message CSwitchCaseEntry { + repeated int64 case_ids = 1; // a single int or a tuple of ints + uint32 statements_id = 2; // CStatements node_id +} +message CSwitchCaseMsg { + uint32 switch_id = 1; // CExpression node_id + repeated CSwitchCaseEntry cases = 2; + optional uint32 default_id = 3; +} + +// CIncompleteSwitchCase +message CIncompleteSwitchCaseEntry { + int64 case_addr = 1; + uint32 statements_id = 2; +} +message CIncompleteSwitchCaseMsg { + uint32 head_id = 1; + repeated CIncompleteSwitchCaseEntry cases = 2; +} + +// CAssignment +message CAssignmentMsg { + uint32 lhs_id = 1; + uint32 rhs_id = 2; +} + +// CExpressionStatement +message CExpressionStatementMsg { + uint32 expr_id = 1; + bool returning = 2; +} + +// CReturn +message CReturnMsg { + optional uint32 retval_id = 1; +} + +// CGoto. ``target`` may be an int or a CExpression — encode as oneof. +message CGotoMsg { + oneof target { + int64 target_int = 1; + uint32 target_expr_id = 2; + } + optional int32 target_idx = 3; +} + +// CUnsupportedStatement (ailment-coupled) +message CUnsupportedStatementMsg { + bytes stmt = 1; // ailment Statement.to_bytes() +} + +// CDirtyStatement (NOT ailment-coupled; wraps a CDirtyExpression) +message CDirtyStatementMsg { + uint32 dirty_id = 1; // CDirtyExpression node_id +} + +// CLabel +message CLabelMsg { + string name = 1; +} + +// CFunctionCall. ``callee_target`` is int | str | CExpression — oneof. +message CFunctionCallMsg { + oneof callee_target { + int64 callee_target_int = 1; + string callee_target_str = 2; + uint32 callee_target_expr_id = 3; + } + optional uint64 callee_func_addr = 4; // Function reference; resolved from KB at parse time + repeated uint32 args_ids = 5; + // show_demangled_name / show_disambiguated_name live in Codegen.node_config; only nodes that override the + // defaults store a NodeConfigEntry. +} + +// Per-node display configuration, stored out-of-line so the common default never occupies node bodies. +message CFunctionCallConfigEntry { + bool show_demangled_name = 1; + bool show_disambiguated_name = 2; +} +message NodeConfigEntry { + uint32 node_id = 1; + oneof config { + CFunctionCallConfigEntry cfuncall = 2; + } +} +message NodeConfigMsg { + repeated NodeConfigEntry config_entries = 1; +} + +// CStructField +message CStructFieldMsg { + uint32 struct_type_ref = 1; + int64 offset = 2; + string field = 3; +} + +// CFakeVariable +message CFakeVariableMsg { + string name = 1; + uint32 type_ref = 2; +} + +// CVariable +message CVariableMsg { + uint32 variable_ref = 1; // index+1 into Codegen.simvar_pool; 0 = absent + uint32 unified_variable_ref = 2; + uint32 variable_type_ref = 3; + optional uint32 vvar_id = 4; +} + +// CIndexedVariable +message CIndexedVariableMsg { + uint32 variable_id = 1; + uint32 index_id = 2; + uint32 type_ref = 3; +} + +// CVariableField +message CVariableFieldMsg { + uint32 variable_id = 1; + uint32 field_id = 2; // CStructField node_id + bool var_is_ptr = 3; +} + +// CUnaryOp +message CUnaryOpMsg { + string op = 1; + uint32 operand_id = 2; +} + +// CBinaryOp +message CBinaryOpMsg { + string op = 1; + uint32 lhs_id = 2; + uint32 rhs_id = 3; + uint32 common_type_ref = 4; +} + +// CTypeCast +message CTypeCastMsg { + uint32 src_type_ref = 1; + uint32 dst_type_ref = 2; + uint32 expr_id = 3; +} + +// CConstant. ``value`` is int | float | str. ``reference_values`` is dict[SimType, int|MemoryData|bytes|str]. +message CConstantReferenceValue { + uint32 type_ref = 1; + oneof value { + int64 int_value = 2; + bytes raw_bytes = 3; + string str_value = 4; + bytes memory_data = 5; // serialized MemoryData via its existing Serializable + } +} +message CConstantMsg { + oneof value { + int64 int_value = 1; + double float_value = 2; + string str_value = 3; + } + uint32 type_ref = 4; + repeated CConstantReferenceValue reference_values = 5; +} + +// CRegister +message CRegisterMsg { + string reg = 1; +} + +// CITE +message CITEMsg { + uint32 cond_id = 1; + uint32 iftrue_id = 2; + uint32 iffalse_id = 3; +} + +// CMultiStatementExpression +message CMultiStatementExpressionMsg { + uint32 stmts_id = 1; + uint32 expr_id = 2; +} + +// CVEXCCallExpression +message CVEXCCallExpressionMsg { + string callee = 1; + repeated uint32 operands_ids = 2; +} + +// CDirtyExpression (ailment-coupled) +message CDirtyExpressionMsg { + bytes dirty = 1; // ailment Expression.to_bytes() +} + +// ===================================================================================================================== +// Polymorphic CConstruct wrapper +// ===================================================================================================================== + +message CConstructNode { + uint32 node_id = 1; // == CConstruct.idx (unique across the owning codegen) + CConstructKind kind = 2; + uint32 tags_ref = 3; // index+1 into Codegen.tag_pool; 0 = no tags + uint32 ident_no = 4; // numeric suffix of CConstruct.ident ("_"); the class name is derived from kind + // For CExpression subclasses only: + optional bool collapsed = 5; + uint32 expr_type_ref = 6; // _type as index+1 into Codegen.type_pool; 0 = absent + + oneof body { + CFunctionMsg cfunction = 100; + CStatementsMsg cstatements = 101; + CAILBlockMsg cailblock = 102; + CLoopMsg cwhile = 103; + CLoopMsg cdowhile = 104; + CForLoopMsg cfor = 105; + CIfElseMsg cifelse = 106; + CIfBreakMsg cifbreak = 107; + CBreakMsg cbreak = 108; + CContinueMsg ccontinue = 109; + CSwitchCaseMsg cswitch = 110; + CIncompleteSwitchCaseMsg cincomplete_switch = 111; + CAssignmentMsg cassignment = 112; + CExpressionStatementMsg cexpression_stmt = 113; + CReturnMsg creturn = 114; + CGotoMsg cgoto = 115; + CUnsupportedStatementMsg cunsupported = 116; + CDirtyStatementMsg cdirty_stmt = 117; + CLabelMsg clabel = 118; + CFunctionCallMsg cfuncall = 119; + CStructFieldMsg cstruct_field = 120; + CFakeVariableMsg cfake_var = 121; + CVariableMsg cvar = 122; + CIndexedVariableMsg cindexed_var = 123; + CVariableFieldMsg cvar_field = 124; + CUnaryOpMsg cunop = 125; + CBinaryOpMsg cbinop = 126; + CTypeCastMsg ctypecast = 127; + CConstantMsg cconst = 128; + CRegisterMsg creg = 129; + CITEMsg cite = 130; + CMultiStatementExpressionMsg cmulti_stmt_expr = 131; + CVEXCCallExpressionMsg cvex_ccall = 132; + CDirtyExpressionMsg cdirty_expr = 133; + } +} + +// ===================================================================================================================== +// Position mappings +// ===================================================================================================================== +// PositionMappingElement holds (start, length, obj). For all three PositionMappings used by codegen (map_pos_to_node, +// map_pos_to_addr, and the per-SimVariable sets in map_ast_to_pos), ``obj`` is a CConstruct, so we store it as a +// node_id reference. The InstructionMapping (map_addr_to_pos) maps ins_addr -> posmap_pos which are both ints. + +message PositionMappingEntry { + int32 start = 1; + int32 length = 2; + uint32 node_id = 3; // 0 means "no node" — though in practice obj is always present + // Membership flags: map_pos_to_node and map_pos_to_addr mostly contain identical entries, so both are stored + // in one merged table and each entry marks which map(s) it belongs to. + bool in_pos_to_node = 4; + bool in_pos_to_addr = 5; +} + +message PositionMappingMsg { + repeated PositionMappingEntry entries = 1; +} + +message InstructionMappingEntry { + uint64 ins_addr = 1; + int32 posmap_pos = 2; +} + +message InstructionMappingMsg { + repeated InstructionMappingEntry entries = 1; +} + +// map_ast_to_pos: dict[SimVariable, set[PositionMappingElement]] +message AstToPosEntry { + bytes simvariable = 1; + repeated PositionMappingEntry elements = 2; +} + +// map_addr_to_label: dict[tuple[int, int | None], CLabel] +message AddrToLabelEntry { + int64 addr = 1; + optional int32 idx = 2; // tuple's second element (block_idx); absent means None + uint32 label_id = 3; // CLabel node_id +} + +// ===================================================================================================================== +// Top-level codegen wrapper +// ===================================================================================================================== + +message Codegen { + // AST table. + uint32 root_id = 1; // node_id of the top-level CFunction + repeated CConstructNode nodes = 2; + + // Rendered text. + optional bytes text_z = 3; // zlib-compressed UTF-8 rendered text + optional string flavor = 4; + + // Display-side state. + // Merged position maps (see PositionMappingEntry membership flags). + PositionMappingMsg pos_maps = 5; + InstructionMappingMsg map_addr_to_pos = 6; + repeated AstToPosEntry map_ast_to_pos = 7; + repeated AddrToLabelEntry map_addr_to_label = 8; + repeated uint32 cexterns_ids = 9; // CVariable node_ids + + // Comments and notes. + map expr_comments = 10; + map stmt_comments = 11; + map notes_json = 12; // DecompilationNote serialized to JSON; key = note key + // const_formats: dict[IdentType, dict[str, bool]]; IdentType = tuple[int, int, str] + repeated ConstFormatEntry const_formats = 13; + + // Interned SimType JSON strings (SimType.to_json output). All *_ref fields above and in the node bodies + // are index+1 into this pool; 0 means absent. + repeated string type_pool = 14; + + // Interned tag dicts; CConstructNode.tags_ref is index+1 into this pool (identical tag dicts are stored once). + repeated CConstructTags tag_pool = 15; + + // Interned SimVariable payloads ("\0"); *_ref fields are index+1, 0 = absent. + repeated bytes simvar_pool = 16; + + // Out-of-line per-node display config; a node only appears here if it deviates from the defaults. + NodeConfigMsg node_config = 17; + + // Display options (round-tripped but not strictly part of the result). This block must stay last: + // c_serialize treats every field from ``indent`` onward as a display option, so a new display option only + // needs an optional scalar field appended here whose name matches the attribute on CStructuredCodeGenerator. + optional int32 indent = 18; + optional bool show_casts = 19; + optional bool comment_gotos = 20; + optional bool braces_on_own_lines = 21; + optional bool use_compound_assignments = 22; + optional bool show_local_types = 23; + optional bool cstyle_null_cmp = 24; + optional bool show_externs = 25; + optional bool show_demangled_name = 26; + optional bool show_disambiguated_name = 27; + optional bool simplify_else_scope = 28; + optional bool cstyle_ifs = 29; + optional bool omit_func_header = 30; + optional bool display_block_addrs = 31; + optional bool display_vvar_ids = 32; + optional bool display_notes = 33; + optional bool prettify_thiscall = 34; + optional bool cstyle_void_param = 35; + optional int32 binop_depth_cutoff = 36; + optional uint64 min_data_addr = 37; + optional int32 max_str_len = 38; +} + +message ConstFormatEntry { + int64 ident_ins_addr = 1; + int32 ident_kind = 2; + string ident_value = 3; + map fmt = 4; +} diff --git a/angr/protos/decompilation_cache.proto b/angr/protos/decompilation_cache.proto new file mode 100644 index 000000000..df77f7eea --- /dev/null +++ b/angr/protos/decompilation_cache.proto @@ -0,0 +1,97 @@ +syntax = "proto3"; + +import "angr/protos/ail_types.proto"; + +package angr.protos; + +// ===================================================================================================================== +// Small value types +// ===================================================================================================================== + +message BinopOperatorEntry { + string key_json = 1; // OpDescriptor serialized to JSON + string value = 2; +} + +message StackvarMaxSizeEntry { + // SimStackVariable is polymorphic; we use the same "\0" wrapping as elsewhere. + bytes simvar = 1; + int32 max_size = 2; +} + +// ===================================================================================================================== +// DecompilationParameters: closed schema for the 15-key ``parameters`` dict. +// ===================================================================================================================== + +message OptionEntry { + // Identified by DecompilationOption.param (a unique key in PARAM_TO_OPTION). + string param = 1; + // JSON-encoded value. The original tuples (option_obj, value) are reconstructed by looking up the option from + // PARAM_TO_OPTION at parse time. + string value_json = 2; +} + +// Collection-typed parameters (except peephole_optimizations) are never None on the Python side (the Decompiler +// normalizes its inputs to empty collections), so an unset repeated/map field simply parses back to an empty +// collection. +message DecompilationParameters { + optional string flavor = 1; + optional bool sp_tracker_track_memory = 2; + + repeated string vars_must_struct = 3; // set[str] + repeated string desired_variables = 4; // frozenset[str] + repeated uint64 inline_functions = 5; // frozenset[int] + + repeated OptionEntry options = 6; + + repeated string optimization_passes = 7; // FQN strings + + // peephole_optimizations is the one None-able collection: None means "use the default peephole set", which is + // distinct from an explicitly empty list. + repeated string peephole_optimizations = 8; + bool peephole_optimizations_use_default = 9; // True when peephole_optimizations was None (default set in use) + + map expr_comments = 10; + map stmt_comments = 11; + repeated BinopOperatorEntry binop_operators = 12; + + optional IteExprs ite_exprs = 13; // set[tuple[int, ailment.Expression]]; unset -> empty set + optional StaticVVars static_vvars = 14; // unset -> empty dict + optional StaticBuffers static_buffers = 15; + + bool save_unoptimized_graph = 16; +} + +// ===================================================================================================================== +// DecompilationCache +// ===================================================================================================================== +// cfg is a decompile-time input, not part of the result; it is not serialized and comes back as None until the +// caller re-attaches it. Decompilation variables live on ``kb.dec_variables`` and are serialized separately. + +message DecompilationCache { + uint64 addr = 1; + + // Heavy sub-objects, embedded as already-serialized bytes from their own Serializable interfaces. + optional bytes clinic = 2; // Clinic.serialize() + optional bytes codegen = 3; // CStructuredCodeGenerator.serialize() + + // Top-level cache state. Collection-typed fields are never None on the Python side; unset fields parse back to + // empty collections. + repeated string errors = 4; + optional string function_summary = 5; + optional ArgVVars arg_vvars = 6; // dict[int, (VirtualVariable, SimVariable)]; unset -> empty dict + optional IteExprs ite_exprs = 7; // set[tuple[int, ailment.Expression]]; unset -> empty set + repeated BinopOperatorEntry binop_operators = 8; + repeated StackvarMaxSizeEntry stackvar_max_sizes = 9; + + // The angr version that produced this decompilation and the time (seconds since the epoch) when it happened. + // Legacy blobs carry the proto3 defaults ""/0, meaning "unknown". + string version = 10; + int64 timestamp = 11; + + // Unset means "no recorded parameters" (e.g. the run had use_cache=False); cache-validity checks treat such a + // cache as always usable. + optional DecompilationParameters parameters = 12; + + map notes_json = 13; // DecompilationNote serialized to JSON; key = note key +} diff --git a/angr/protos/function_pb2.py b/angr/protos/function_pb2.py deleted file mode 100644 index 00cc67674..000000000 --- a/angr/protos/function_pb2.py +++ /dev/null @@ -1,41 +0,0 @@ -# pylint:disable=wrong-import-position,unused-import,protected-access -# Generated by the protocol buffer compiler. DO NOT EDIT! -# NO CHECKED-IN PROTOBUF GENCODE -# source: angr/protos/function.proto -# Protobuf Python Version: 6.33.0 -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import runtime_version as _runtime_version -from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -_runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 6, - 33, - 0, - '', - 'angr/protos/function.proto' -) -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from angr.protos import primitives_pb2 as angr_dot_protos_dot_primitives__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1a\x61ngr/protos/function.proto\x12\x0b\x61ngr.protos\x1a\x1c\x61ngr/protos/primitives.proto\"b\n\x08\x43\x61llSite\x12\n\n\x02\x65\x61\x18\x01 \x01(\x04\x12\x16\n\ttarget_ea\x18\x02 \x01(\x04H\x00\x88\x01\x01\x12\x16\n\treturn_ea\x18\x03 \x01(\x04H\x01\x88\x01\x01\x42\x0c\n\n_target_eaB\x0c\n\n_return_ea\"\xe2\x05\n\x08\x46unction\x12\n\n\x02\x65\x61\x18\x01 \x01(\x04\x12\x15\n\ris_entrypoint\x18\x03 \x01(\x08\x12\"\n\x06\x62locks\x18\x02 \x03(\x0b\x32\x12.angr.protos.Block\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\x0e\n\x06is_plt\x18\x07 \x01(\x08\x12\x12\n\nis_syscall\x18\x08 \x01(\x08\x12\x17\n\x0fis_simprocedure\x18\t \x01(\x08\x12\x16\n\treturning\x18\n \x01(\x08H\x00\x88\x01\x01\x12\x13\n\x0b\x62inary_name\x18\x0b \x01(\t\x12&\n\x05graph\x18\x0c \x01(\x0b\x32\x17.angr.protos.BlockGraph\x12\x1a\n\x12\x65xternal_functions\x18\r \x03(\x04\x12\x11\n\talignment\x18\x0e \x01(\x08\x12\x12\n\nnormalized\x18\x0f \x01(\x08\x12;\n\x0cmatched_from\x18\x10 \x01(\x0e\x32%.angr.protos.Function.SignatureSource\x12\x11\n\tprototype\x18\x11 \x01(\x0c\x12\x1a\n\x12\x63\x61lling_convention\x18\x12 \x01(\x0c\x12\x19\n\x11prototype_libname\x18\x13 \x01(\t\x12\x0c\n\x04info\x18\x15 \x01(\x0c\x12(\n\tendpoints\x18\x16 \x03(\x0b\x32\x15.angr.protos.Endpoint\x12+\n\x0f\x65xternal_blocks\x18\x18 \x03(\x0b\x32\x12.angr.protos.Block\x12\x17\n\x0fis_default_name\x18\x19 \x01(\x08\x12\x0f\n\x07ran_cca\x18\x1a \x01(\x08\x12\x16\n\x0eprevious_names\x18\x1b \x03(\t\x12\x18\n\x10prototype_source\x18\x1c \x01(\r\x12)\n\ncall_sites\x18\x1d \x03(\x0b\x32\x15.angr.protos.CallSite\"+\n\x0fSignatureSource\x12\r\n\tUNMATCHED\x10\x00\x12\t\n\x05\x46LIRT\x10\x01\x42\x0c\n\n_returningb\x06proto3') - -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'angr.protos.function_pb2', _globals) -if not _descriptor._USE_C_DESCRIPTORS: - DESCRIPTOR._loaded_options = None - _globals['_CALLSITE']._serialized_start=73 - _globals['_CALLSITE']._serialized_end=171 - _globals['_FUNCTION']._serialized_start=174 - _globals['_FUNCTION']._serialized_end=912 - _globals['_FUNCTION_SIGNATURESOURCE']._serialized_start=855 - _globals['_FUNCTION_SIGNATURESOURCE']._serialized_end=898 -# @@protoc_insertion_point(module_scope) diff --git a/angr/protos/primitives_pb2.py b/angr/protos/primitives_pb2.py deleted file mode 100644 index 83f8624cb..000000000 --- a/angr/protos/primitives_pb2.py +++ /dev/null @@ -1,63 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# NO CHECKED-IN PROTOBUF GENCODE -# source: angr/protos/primitives.proto -# Protobuf Python Version: 6.33.0 -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import runtime_version as _runtime_version -from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -_runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 6, - 33, - 0, - '', - 'angr/protos/primitives.proto' -) -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x61ngr/protos/primitives.proto\x12\x0b\x61ngr.protos\"\x87\x05\n\rCodeReference\x12:\n\x0btarget_type\x18\x01 \x01(\x0e\x32%.angr.protos.CodeReference.TargetType\x12<\n\x0coperand_type\x18\x02 \x01(\x0e\x32&.angr.protos.CodeReference.OperandType\x12\x35\n\x08location\x18\x03 \x01(\x0e\x32#.angr.protos.CodeReference.Location\x12\n\n\x02\x65\x61\x18\x04 \x01(\x04\x12\x0c\n\x04mask\x18\x05 \x01(\x04\x12\x0c\n\x04name\x18\x06 \x01(\t\x12\x0f\n\x07\x64\x61ta_ea\x18\x07 \x01(\x04\x12\x10\n\x08\x62lock_ea\x18\x08 \x01(\x04\x12\x10\n\x08stmt_idx\x18\t \x01(\x05\x12\x13\n\x0boperand_idx\x18\n \x01(\x05\x12:\n\x08ref_type\x18\x0b \x01(\x0e\x32(.angr.protos.CodeReference.ReferenceType\"=\n\nTargetType\x12\x0e\n\nCodeTarget\x10\x00\x12\x0e\n\nDataTarget\x10\x01\x12\x0f\n\x0bStackTarget\x10\x02\"~\n\x0bOperandType\x12\x14\n\x10ImmediateOperand\x10\x00\x12\x11\n\rMemoryOperand\x10\x01\x12\x1d\n\x19MemoryDisplacementOperand\x10\x02\x12\x16\n\x12\x43ontrolFlowOperand\x10\x03\x12\x0f\n\x0bOffsetTable\x10\x04\"&\n\x08Location\x12\x0c\n\x08Internal\x10\x00\x12\x0c\n\x08\x45xternal\x10\x01\"0\n\rReferenceType\x12\n\n\x06offset\x10\x00\x12\x08\n\x04read\x10\x01\x12\t\n\x05write\x10\x02\"k\n\x0bInstruction\x12\n\n\x02\x65\x61\x18\x01 \x01(\x04\x12\r\n\x05\x62ytes\x18\x02 \x01(\x0c\x12)\n\x05xrefs\x18\x03 \x01(\x0b\x32\x1a.angr.protos.CodeReference\x12\x16\n\x0elocal_noreturn\x18\x04 \x01(\x08\"`\n\x05\x42lock\x12\n\n\x02\x65\x61\x18\x01 \x01(\x04\x12.\n\x0cinstructions\x18\x02 \x01(\x0b\x32\x18.angr.protos.Instruction\x12\x0c\n\x04size\x18\x04 \x01(\r\x12\r\n\x05\x62ytes\x18\x05 \x01(\x0c\"\x95\x02\n\x10\x45xternalFunction\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\n\n\x02\x65\x61\x18\x02 \x01(\x04\x12;\n\x02\x63\x63\x18\x03 \x01(\x0e\x32/.angr.protos.ExternalFunction.CallingConvention\x12\x12\n\nhas_return\x18\x04 \x01(\x08\x12\x11\n\tno_return\x18\x05 \x01(\x08\x12\x16\n\x0e\x61rgument_count\x18\x06 \x01(\x05\x12\x0f\n\x07is_weak\x18\x07 \x01(\x08\x12\x11\n\tprototype\x18\x08 \x01(\t\"G\n\x11\x43\x61llingConvention\x12\x11\n\rCallerCleanup\x10\x00\x12\x11\n\rCalleeCleanup\x10\x01\x12\x0c\n\x08\x46\x61stCall\x10\x02\"d\n\x10\x45xternalVariable\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\n\n\x02\x65\x61\x18\x02 \x01(\x04\x12\x0c\n\x04size\x18\x03 \x01(\r\x12\x0f\n\x07is_weak\x18\x04 \x01(\x08\x12\x17\n\x0fis_thread_local\x18\x05 \x01(\x08\"\x9a\x05\n\x04\x45\x64ge\x12\x0e\n\x06src_ea\x18\x01 \x01(\x04\x12\x0e\n\x06\x64st_ea\x18\x02 \x01(\x04\x12,\n\x08jumpkind\x18\x03 \x01(\x0e\x32\x1a.angr.protos.Edge.JumpKind\x12\x12\n\nis_outside\x18\x04 \x01(\x08\x12\x10\n\x08ins_addr\x18\x05 \x01(\x04\x12\x10\n\x08stmt_idx\x18\x06 \x01(\x03\x12\x11\n\tconfirmed\x18\x07 \x01(\r\"\xf8\x03\n\x08JumpKind\x12\x13\n\x0fUnknownJumpkind\x10\x00\x12\n\n\x06\x42oring\x10\x01\x12\x08\n\x04\x43\x61ll\x10\x02\x12\n\n\x06Return\x10\x03\x12\x0e\n\nFakeReturn\x10\x04\x12\x0b\n\x07Syscall\x10\x05\x12\x0f\n\x0bSys_syscall\x10\x06\x12\x0e\n\nSys_int128\x10\x07\x12\x0c\n\x08NoDecode\x10\x08\x12\n\n\x06\x45mWarn\x10\t\x12\x11\n\rSigFPE_IntDiv\x10\n\x12\x0b\n\x07SigTRAP\x10\x0b\x12\x0b\n\x07SigSEGV\x10\x0c\x12\x0b\n\x07MapFail\x10\r\x12\x0b\n\x07NoRedir\x10\x0e\x12\r\n\tClientReq\x10\x0f\x12\r\n\tException\x10\x10\x12\n\n\x06_8jzf8\x10\x11\x12\n\n\x06\x45mFail\x10\x12\x12\x0f\n\x0b\x46lushDCache\x10\x13\x12\x0f\n\x0bInvalICache\x10\x14\x12\x0e\n\nPrivileged\x10\x15\x12\n\n\x06SigBUS\x10\x16\x12\x11\n\rSigFPE_IntOvf\x10\x17\x12\n\n\x06SigILL\x10\x18\x12\x0e\n\nSys_int129\x10\x19\x12\x0e\n\nSys_int130\x10\x1a\x12\x0e\n\nSys_int145\x10\x1b\x12\x0e\n\nSys_int210\x10\x1c\x12\r\n\tSys_int32\x10\x1d\x12\x10\n\x0cSys_sysenter\x10\x1e\x12\t\n\x05Yield\x10\x1f\x12\n\n\x06SigFPE\x10 \x12\x0b\n\x07Sys_int\x10!\".\n\nBlockGraph\x12 \n\x05\x65\x64ges\x18\x01 \x03(\x0b\x32\x11.angr.protos.Edge\"M\n\x08\x45ndpoint\x12\'\n\x04type\x18\x01 \x01(\x0e\x32\x19.angr.protos.EndpointType\x12\n\n\x02\x65\x61\x18\x02 \x01(\x04\x12\x0c\n\x04size\x18\x03 \x01(\r*4\n\x0c\x45ndpointType\x12\x08\n\x04\x43\x41LL\x10\x00\x12\n\n\x06RETURN\x10\x01\x12\x0e\n\nTRANSITION\x10\x02\x62\x06proto3') - -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'angr.protos.primitives_pb2', _globals) -if not _descriptor._USE_C_DESCRIPTORS: - DESCRIPTOR._loaded_options = None - _globals['_ENDPOINTTYPE']._serialized_start=2080 - _globals['_ENDPOINTTYPE']._serialized_end=2132 - _globals['_CODEREFERENCE']._serialized_start=46 - _globals['_CODEREFERENCE']._serialized_end=693 - _globals['_CODEREFERENCE_TARGETTYPE']._serialized_start=414 - _globals['_CODEREFERENCE_TARGETTYPE']._serialized_end=475 - _globals['_CODEREFERENCE_OPERANDTYPE']._serialized_start=477 - _globals['_CODEREFERENCE_OPERANDTYPE']._serialized_end=603 - _globals['_CODEREFERENCE_LOCATION']._serialized_start=605 - _globals['_CODEREFERENCE_LOCATION']._serialized_end=643 - _globals['_CODEREFERENCE_REFERENCETYPE']._serialized_start=645 - _globals['_CODEREFERENCE_REFERENCETYPE']._serialized_end=693 - _globals['_INSTRUCTION']._serialized_start=695 - _globals['_INSTRUCTION']._serialized_end=802 - _globals['_BLOCK']._serialized_start=804 - _globals['_BLOCK']._serialized_end=900 - _globals['_EXTERNALFUNCTION']._serialized_start=903 - _globals['_EXTERNALFUNCTION']._serialized_end=1180 - _globals['_EXTERNALFUNCTION_CALLINGCONVENTION']._serialized_start=1109 - _globals['_EXTERNALFUNCTION_CALLINGCONVENTION']._serialized_end=1180 - _globals['_EXTERNALVARIABLE']._serialized_start=1182 - _globals['_EXTERNALVARIABLE']._serialized_end=1282 - _globals['_EDGE']._serialized_start=1285 - _globals['_EDGE']._serialized_end=1951 - _globals['_EDGE_JUMPKIND']._serialized_start=1447 - _globals['_EDGE_JUMPKIND']._serialized_end=1951 - _globals['_BLOCKGRAPH']._serialized_start=1953 - _globals['_BLOCKGRAPH']._serialized_end=1999 - _globals['_ENDPOINT']._serialized_start=2001 - _globals['_ENDPOINT']._serialized_end=2078 -# @@protoc_insertion_point(module_scope) diff --git a/angr/protos/variables_pb2.py b/angr/protos/variables_pb2.py deleted file mode 100644 index 7d7407416..000000000 --- a/angr/protos/variables_pb2.py +++ /dev/null @@ -1,57 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# NO CHECKED-IN PROTOBUF GENCODE -# source: angr/protos/variables.proto -# Protobuf Python Version: 6.33.0 -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import runtime_version as _runtime_version -from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -_runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 6, - 33, - 0, - '', - 'angr/protos/variables.proto' -) -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1b\x61ngr/protos/variables.proto\x12\x0b\x61ngr.protos\"\x90\x01\n\x0cVariableBase\x12\r\n\x05ident\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x13\n\x06region\x18\x03 \x01(\x04H\x00\x88\x01\x01\x12\x15\n\x08\x63\x61tegory\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x0f\n\x07renamed\x18\x05 \x01(\x08\x12\x0e\n\x06is_phi\x18\x06 \x01(\x08\x42\t\n\x07_regionB\x0b\n\t_category\"Z\n\x11TemporaryVariable\x12\'\n\x04\x62\x61se\x18\x01 \x01(\x0b\x32\x19.angr.protos.VariableBase\x12\x0e\n\x06tmp_id\x18\x02 \x01(\r\x12\x0c\n\x04size\x18\x03 \x01(\r\"\x80\x01\n\x10\x43onstantVariable\x12\'\n\x04\x62\x61se\x18\x01 \x01(\x0b\x32\x19.angr.protos.VariableBase\x12\x0c\n\x04size\x18\x02 \x01(\r\x12\r\n\x05value\x18\x03 \x01(\x04\x12\x17\n\nlong_value\x18\x04 \x01(\x0cH\x00\x88\x01\x01\x42\r\n\x0b_long_value\"V\n\x10RegisterVariable\x12\'\n\x04\x62\x61se\x18\x01 \x01(\x0b\x32\x19.angr.protos.VariableBase\x12\x0b\n\x03reg\x18\x02 \x01(\r\x12\x0c\n\x04size\x18\x03 \x01(\r\"U\n\x0eMemoryVariable\x12\'\n\x04\x62\x61se\x18\x01 \x01(\x0b\x32\x19.angr.protos.VariableBase\x12\x0c\n\x04\x61\x64\x64r\x18\x02 \x01(\x04\x12\x0c\n\x04size\x18\x03 \x01(\r\"u\n\rStackVariable\x12\'\n\x04\x62\x61se\x18\x01 \x01(\x0b\x32\x19.angr.protos.VariableBase\x12\x0c\n\x04\x61\x64\x64r\x18\x02 \x01(\x04\x12\x0c\n\x04size\x18\x03 \x01(\r\x12\x0f\n\x07sp_base\x18\x04 \x01(\x08\x12\x0e\n\x06offset\x18\x05 \x01(\x05\"\x9c\x02\n\x0eVariableAccess\x12\r\n\x05ident\x18\x01 \x01(\t\x12\x12\n\nblock_addr\x18\x02 \x01(\x04\x12\x10\n\x08stmt_idx\x18\x03 \x01(\x05\x12\x10\n\x08ins_addr\x18\x04 \x01(\x04\x12\x13\n\x06offset\x18\x05 \x01(\x03H\x00\x88\x01\x01\x12\x43\n\x0b\x61\x63\x63\x65ss_type\x18\x06 \x01(\x0e\x32..angr.protos.VariableAccess.VariableAccessSort\x12\x16\n\tatom_hash\x18\x07 \x01(\rH\x01\x88\x01\x01\"8\n\x12VariableAccessSort\x12\t\n\x05WRITE\x10\x00\x12\x08\n\x04READ\x10\x01\x12\r\n\tREFERENCE\x10\x02\x42\t\n\x07_offsetB\x0c\n\n_atom_hash\"/\n\x0cVariableType\x12\r\n\x05ident\x18\x01 \x01(\t\x12\x10\n\x08var_type\x18\x02 \x01(\t\";\n\x0bVar2Unified\x12\x11\n\tvar_ident\x18\x01 \x01(\t\x12\x19\n\x11unified_var_ident\x18\x02 \x01(\t\"/\n\x07Phi2Var\x12\x11\n\tphi_ident\x18\x01 \x01(\t\x12\x11\n\tvar_ident\x18\x02 \x01(\t\"\x98\x05\n\x17VariableManagerInternal\x12\x30\n\x08tempvars\x18\x01 \x03(\x0b\x32\x1e.angr.protos.TemporaryVariable\x12.\n\x07regvars\x18\x02 \x03(\x0b\x32\x1d.angr.protos.RegisterVariable\x12,\n\x07memvars\x18\x03 \x03(\x0b\x32\x1b.angr.protos.MemoryVariable\x12-\n\tstackvars\x18\x04 \x03(\x0b\x32\x1a.angr.protos.StackVariable\x12\x30\n\tconstvars\x18\r \x03(\x0b\x32\x1d.angr.protos.ConstantVariable\x12-\n\x08\x61\x63\x63\x65sses\x18\x05 \x03(\x0b\x32\x1b.angr.protos.VariableAccess\x12\x38\n\x10unified_tempvars\x18\x06 \x03(\x0b\x32\x1e.angr.protos.TemporaryVariable\x12\x36\n\x0funified_regvars\x18\x07 \x03(\x0b\x32\x1d.angr.protos.RegisterVariable\x12\x34\n\x0funified_memvars\x18\x08 \x03(\x0b\x32\x1b.angr.protos.MemoryVariable\x12\x35\n\x11unified_stackvars\x18\t \x03(\x0b\x32\x1a.angr.protos.StackVariable\x12-\n\x0bvar2unified\x18\n \x03(\x0b\x32\x18.angr.protos.Var2Unified\x12(\n\x05types\x18\x0b \x03(\x0b\x32\x19.angr.protos.VariableType\x12%\n\x07phi2var\x18\x0c \x03(\x0b\x32\x14.angr.protos.Phi2Varb\x06proto3') - -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'angr.protos.variables_pb2', _globals) -if not _descriptor._USE_C_DESCRIPTORS: - DESCRIPTOR._loaded_options = None - _globals['_VARIABLEBASE']._serialized_start=45 - _globals['_VARIABLEBASE']._serialized_end=189 - _globals['_TEMPORARYVARIABLE']._serialized_start=191 - _globals['_TEMPORARYVARIABLE']._serialized_end=281 - _globals['_CONSTANTVARIABLE']._serialized_start=284 - _globals['_CONSTANTVARIABLE']._serialized_end=412 - _globals['_REGISTERVARIABLE']._serialized_start=414 - _globals['_REGISTERVARIABLE']._serialized_end=500 - _globals['_MEMORYVARIABLE']._serialized_start=502 - _globals['_MEMORYVARIABLE']._serialized_end=587 - _globals['_STACKVARIABLE']._serialized_start=589 - _globals['_STACKVARIABLE']._serialized_end=706 - _globals['_VARIABLEACCESS']._serialized_start=709 - _globals['_VARIABLEACCESS']._serialized_end=993 - _globals['_VARIABLEACCESS_VARIABLEACCESSSORT']._serialized_start=912 - _globals['_VARIABLEACCESS_VARIABLEACCESSSORT']._serialized_end=968 - _globals['_VARIABLETYPE']._serialized_start=995 - _globals['_VARIABLETYPE']._serialized_end=1042 - _globals['_VAR2UNIFIED']._serialized_start=1044 - _globals['_VAR2UNIFIED']._serialized_end=1103 - _globals['_PHI2VAR']._serialized_start=1105 - _globals['_PHI2VAR']._serialized_end=1152 - _globals['_VARIABLEMANAGERINTERNAL']._serialized_start=1155 - _globals['_VARIABLEMANAGERINTERNAL']._serialized_end=1819 -# @@protoc_insertion_point(module_scope) diff --git a/angr/protos/xrefs_pb2.py b/angr/protos/xrefs_pb2.py deleted file mode 100644 index cdce846fe..000000000 --- a/angr/protos/xrefs_pb2.py +++ /dev/null @@ -1,36 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# NO CHECKED-IN PROTOBUF GENCODE -# source: angr/protos/xrefs.proto -# Protobuf Python Version: 6.33.0 -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import runtime_version as _runtime_version -from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -_runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 6, - 33, - 0, - '', - 'angr/protos/xrefs.proto' -) -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from angr.protos import primitives_pb2 as angr_dot_protos_dot_primitives__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17\x61ngr/protos/xrefs.proto\x12\x0b\x61ngr.protos\x1a\x1c\x61ngr/protos/primitives.proto\"2\n\x05XRefs\x12)\n\x05xrefs\x18\x01 \x03(\x0b\x32\x1a.angr.protos.CodeReferenceb\x06proto3') - -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'angr.protos.xrefs_pb2', _globals) -if not _descriptor._USE_C_DESCRIPTORS: - DESCRIPTOR._loaded_options = None - _globals['_XREFS']._serialized_start=70 - _globals['_XREFS']._serialized_end=120 -# @@protoc_insertion_point(module_scope) diff --git a/angr/rust/optimization_passes/pattern_match_simplifier.py b/angr/rust/optimization_passes/pattern_match_simplifier.py index 7c77b4f22..910009f9c 100644 --- a/angr/rust/optimization_passes/pattern_match_simplifier.py +++ b/angr/rust/optimization_passes/pattern_match_simplifier.py @@ -250,17 +250,17 @@ class PatternMatchSimplifier(SequenceOptimizationPass): def __init__(self, func, manager, **kwargs): super().__init__(func, manager, **kwargs) self._graph = kwargs.get("graph") - self._variable_kb = kwargs.get("variable_kb") + self._dvars_kb = kwargs.get("kb") self.analyze() def _check(self): return bool(self.seq is not None and self.seq.nodes), None def _analyze(self, cache=None): - if self._variable_kb is None: + if self._dvars_kb is None: return walker = PatternMatchWalker( - self._variable_kb.variables.get_function_manager(self._func.addr), + self._dvars_kb.dec_variables.get_function_manager(self._func.addr), self._graph, variable_map_of(self.manager), ) diff --git a/angr/rust/optimization_passes/str_argument_simplifier.py b/angr/rust/optimization_passes/str_argument_simplifier.py index 1ddfe47df..87375dc20 100644 --- a/angr/rust/optimization_passes/str_argument_simplifier.py +++ b/angr/rust/optimization_passes/str_argument_simplifier.py @@ -25,8 +25,8 @@ class StrArgumentSimplifier(OptimizationPass, SRDAMixin): super().__init__(func, manager, **kwargs) SRDAMixin.__init__(self, func, self._graph, self.project, variable_map_of(manager)) - if self._variable_kb is not None: - self._var_manager = self._variable_kb.variables.get_function_manager(self._func.addr) + if self._dvars_kb is not None: + self._var_manager = self._dvars_kb.dec_variables.get_function_manager(self._func.addr) else: self._var_manager = None diff --git a/angr/rustylib/ailment.pyi b/angr/rustylib/ailment.pyi index 6924ff788..8c45e5e69 100644 --- a/angr/rustylib/ailment.pyi +++ b/angr/rustylib/ailment.pyi @@ -513,6 +513,10 @@ class Block: def __str__(self) -> str: ... def copy(self, statements: list[Statement] | None = ...) -> Self: ... def dbg_repr(self, indent: int = ...) -> str: ... + # --- Serialization ------------------------------------------------- + def to_bytes(self) -> bytes: ... + @classmethod + def from_bytes(cls, data: bytes) -> Block: ... # --------------------------------------------------------------------------- # Manager + VEX -> AIL converter @@ -635,7 +639,7 @@ class VirtualVariable(Atom): def oident(self) -> Any: """VirtualVariable.oident""" @property - def reg_vvars(self) -> dict[int, Expression]: + def reg_vvars(self) -> list[VirtualVariable] | None: """VirtualVariable.reg_vvars Returns ``None`` for non-COMBO_REGISTER vvars, an empty list for COMBO_REGISTER vvars whose sub-registers haven't been populated yet, and a list of ``VirtualVariable`` Expression wrappers otherwise. Each call mints fresh wrappers around clones of the inner ``AilExpression`` nodes (same pattern as ``.operands``).""" @property def was_reg(self) -> bool: diff --git a/angr/utils/ail_serialization.py b/angr/utils/ail_serialization.py new file mode 100644 index 000000000..49e6e0bc8 --- /dev/null +++ b/angr/utils/ail_serialization.py @@ -0,0 +1,265 @@ +""" +Typed protobuf pack/parse helpers for AIL-typed containers. + +AIL leaves (Block / Statement / Expression) serialize through their native ``to_bytes()`` methods (postcard, +implemented in Rust); the helpers here only encode the Python container structure around them using the typed +messages in :mod:`angr.protos.ail_types_pb2`. There is no generic fallback: any value shape not covered by the +schema raises ``TypeError``. +""" +# pylint:disable=no-member + +from __future__ import annotations + +from collections import Counter +from typing import TYPE_CHECKING, Any + +import networkx + +from angr import sim_variable +from angr.analyses.decompiler.optimization_passes.static_vvar_rewriter import FixedBuffer, FixedBufferPtr +from angr.protos import ail_types_pb2 +from angr.rustylib.ailment import Block, Expression + +if TYPE_CHECKING: + from angr.sim_variable import SimVariable + + +# --------------------------------------------------------------------------------------------------------------------- +# SimVariable polymorphic encoding +# --------------------------------------------------------------------------------------------------------------------- + + +def simvar_to_bytes_polymorphic(v: SimVariable) -> bytes: + """Polymorphic SimVariable encoding: ``b"\\0"``.""" + return type(v).__name__.encode("ascii") + b"\0" + v.serialize() + + +def simvar_from_bytes_polymorphic(b: bytes) -> SimVariable: + sep = b.index(b"\0") + cls_name = b[:sep].decode("ascii") + return getattr(sim_variable, cls_name).parse(b[sep + 1 :]) + + +# --------------------------------------------------------------------------------------------------------------------- +# networkx.DiGraph[ailment.Block] +# --------------------------------------------------------------------------------------------------------------------- + +_EDGE_TYPE_TO_ENUM = { + "transition": ail_types_pb2.AIL_EDGE_TRANSITION, + "exception": ail_types_pb2.AIL_EDGE_EXCEPTION, + "fake_return": ail_types_pb2.AIL_EDGE_FAKE_RETURN, + "call": ail_types_pb2.AIL_EDGE_CALL, + "syscall": ail_types_pb2.AIL_EDGE_SYSCALL, + "return": ail_types_pb2.AIL_EDGE_RETURN, +} +_ENUM_TO_EDGE_TYPE = {v: k for k, v in _EDGE_TYPE_TO_ENUM.items()} + + +def _pack_edge_data(data: dict[str, Any], out: ail_types_pb2.AilEdgeData) -> bool: + """Fill an AilEdgeData message from a networkx edge-attribute dict. Returns True if any field was set. + + Attributes with value None are packed as unset (and come back as absent keys); unknown keys or value types raise + ``TypeError`` -- extend the AilEdgeData schema when a new edge attribute is introduced. + """ + any_set = False + for key, value in data.items(): + if value is None: + continue + if key == "type": + if value not in _EDGE_TYPE_TO_ENUM: + raise TypeError(f"Unsupported AIL graph edge type {value!r}; extend AilEdgeType in ail_types.proto") + out.type = _EDGE_TYPE_TO_ENUM[value] + elif key == "outside": + out.outside = bool(value) + elif key == "confirmed": + out.confirmed = bool(value) + elif key == "ins_addr": + out.ins_addr = value + elif key == "stmt_idx": + out.stmt_idx = value + else: + raise TypeError(f"Unsupported AIL graph edge attribute {key!r}; extend AilEdgeData in ail_types.proto") + any_set = True + return any_set + + +def _parse_edge_data(msg: ail_types_pb2.AilEdgeData) -> dict[str, Any]: + data: dict[str, Any] = {} + if msg.HasField("type"): + data["type"] = _ENUM_TO_EDGE_TYPE[msg.type] + if msg.HasField("outside"): + data["outside"] = msg.outside + if msg.HasField("ins_addr"): + data["ins_addr"] = msg.ins_addr + if msg.HasField("stmt_idx"): + data["stmt_idx"] = msg.stmt_idx + if msg.HasField("confirmed"): + data["confirmed"] = msg.confirmed + return data + + +class BlockPool: + """A shared pool of ``Block.to_bytes()`` payloads for deduplicating byte-identical blocks across graphs.""" + + __slots__ = ("_index_by_payload", "payloads") + + def __init__(self) -> None: + self.payloads: list[bytes] = [] + self._index_by_payload: dict[bytes, int] = {} + + def add(self, block) -> int: + payload = block.to_bytes() + idx = self._index_by_payload.get(payload) + if idx is None: + idx = len(self.payloads) + self._index_by_payload[payload] = idx + self.payloads.append(payload) + return idx + + +def _edge_data_key(data: dict[str, Any]) -> tuple: + """Hashable canonical form of an edge-data dict excluding ins_addr (which varies per edge). Skips None values to + match ``_pack_edge_data`` (which drops them), so it reflects exactly what round-trips.""" + return tuple(sorted((k, v) for k, v in data.items() if k != "ins_addr" and v is not None)) + + +def pack_graph(graph: networkx.DiGraph, pool: BlockPool | None = None) -> ail_types_pb2.AilGraph: + """Encode a DiGraph of ailment Blocks. Node identity is preserved through per-graph block indices. When ``pool`` + is given, block payloads are deduplicated into it and the message stores pool refs instead of inline payloads.""" + msg = ail_types_pb2.AilGraph() + node_to_idx: dict[Any, int] = {} + for i, node in enumerate(graph.nodes): + if not isinstance(node, Block): + raise TypeError(f"Unsupported AIL graph node type {type(node).__name__}; only ailment.Block is allowed") + node_to_idx[node] = i + if pool is None: + msg.blocks.append(node.to_bytes()) + else: + msg.block_refs.append(pool.add(node)) + + # Pick the modal non-ins_addr edge-data value as the default so common edge attributes are stored once. + edge_list = list(graph.edges(data=True)) + key_counts = Counter(_edge_data_key(data) for _, _, data in edge_list) + default_key, default_count = key_counts.most_common(1)[0] if key_counts else ((), 0) + default_dict = dict(default_key) + use_default = default_count >= 2 and bool(default_dict) + if use_default: + _pack_edge_data(default_dict, msg.default_edge_data) + + for src, dst, data in edge_list: + edge = msg.edges.add() + edge.src = node_to_idx[src] + edge.dst = node_to_idx[dst] + if use_default and _edge_data_key(data) == default_key: + # matches the graph default: keep only the per-edge ins_addr (if any) and let parse restore the rest + edge.has_default_data = True + ins_addr = data.get("ins_addr") + if ins_addr is not None: + edge.data.ins_addr = ins_addr + elif data: + edge_data = ail_types_pb2.AilEdgeData() + if _pack_edge_data(data, edge_data): + edge.data.CopyFrom(edge_data) + return msg + + +def parse_graph(msg: ail_types_pb2.AilGraph, pool_payloads=None) -> networkx.DiGraph: + graph = networkx.DiGraph() + if msg.block_refs: + # pool-backed encoding: a fresh Block per graph occurrence so graphs never share node objects + blocks = [Block.from_bytes(pool_payloads[i]) for i in msg.block_refs] + else: + blocks = [Block.from_bytes(b) for b in msg.blocks] + graph.add_nodes_from(blocks) + default_data = _parse_edge_data(msg.default_edge_data) if msg.HasField("default_edge_data") else {} + for edge in msg.edges: + if edge.has_default_data: + data = dict(default_data) + if edge.HasField("data") and edge.data.HasField("ins_addr"): + data["ins_addr"] = edge.data.ins_addr + else: + data = _parse_edge_data(edge.data) if edge.HasField("data") else {} + graph.add_edge(blocks[edge.src], blocks[edge.dst], **data) + return graph + + +# --------------------------------------------------------------------------------------------------------------------- +# dict[int, tuple[VirtualVariable, SimVariable]] +# --------------------------------------------------------------------------------------------------------------------- + + +def pack_arg_vvars(arg_vvars: dict[int, tuple[Any, Any]]) -> ail_types_pb2.ArgVVars: + msg = ail_types_pb2.ArgVVars() + for idx, (vvar, simvar) in arg_vvars.items(): + entry = msg.entries[idx] + entry.vvar = vvar.to_bytes() + entry.simvar = simvar_to_bytes_polymorphic(simvar) + return msg + + +def parse_arg_vvars(msg: ail_types_pb2.ArgVVars) -> dict[int, tuple[Any, Any]]: + return { + idx: (Expression.from_bytes(entry.vvar), simvar_from_bytes_polymorphic(entry.simvar)) + for idx, entry in msg.entries.items() + } + + +# --------------------------------------------------------------------------------------------------------------------- +# set[tuple[int, Expression]] +# --------------------------------------------------------------------------------------------------------------------- + + +def pack_ite_exprs(ite_exprs: set[tuple[int, Any]]) -> ail_types_pb2.IteExprs: + msg = ail_types_pb2.IteExprs() + for addr, expr in sorted(ite_exprs, key=lambda t: t[0]): + entry = msg.entries.add() + entry.addr = addr + entry.expr = expr.to_bytes() + return msg + + +def parse_ite_exprs(msg: ail_types_pb2.IteExprs) -> set[tuple[int, Any]]: + return {(entry.addr, Expression.from_bytes(entry.expr)) for entry in msg.entries} + + +# --------------------------------------------------------------------------------------------------------------------- +# Static buffer parameters (optimization_passes.static_vvar_rewriter) +# --------------------------------------------------------------------------------------------------------------------- + + +def pack_static_vvars(static_vvars: dict[int, Any]) -> ail_types_pb2.StaticVVars: + msg = ail_types_pb2.StaticVVars() + for varid, value in static_vvars.items(): + entry = msg.entries[varid] + if isinstance(value, FixedBufferPtr): + entry.ptr.buffer_ident = value.buffer_ident + entry.ptr.offset = value.offset + elif isinstance(value, Expression): + entry.const_expr = value.to_bytes() + else: + raise TypeError(f"Unsupported static_vvars value type {type(value).__name__}") + return msg + + +def parse_static_vvars(msg: ail_types_pb2.StaticVVars) -> dict[int, Any]: + result: dict[int, Any] = {} + for varid, entry in msg.entries.items(): + if entry.WhichOneof("v") == "ptr": + result[varid] = FixedBufferPtr(entry.ptr.buffer_ident, offset=entry.ptr.offset) + else: + result[varid] = Expression.from_bytes(entry.const_expr) + return result + + +def pack_static_buffers(static_buffers: dict[str, Any]) -> ail_types_pb2.StaticBuffers: + msg = ail_types_pb2.StaticBuffers() + for key, buf in static_buffers.items(): + entry = msg.entries[key] + entry.ident = buf.ident + entry.size = buf.size + entry.content = buf.content + return msg + + +def parse_static_buffers(msg: ail_types_pb2.StaticBuffers) -> dict[str, Any]: + return {key: FixedBuffer(entry.ident, entry.size, entry.content) for key, entry in msg.entries.items()} diff --git a/native/angr/src/ailment/block.rs b/native/angr/src/ailment/block.rs index 9a37ce803..78ff05c8a 100644 --- a/native/angr/src/ailment/block.rs +++ b/native/angr/src/ailment/block.rs @@ -4,10 +4,11 @@ use std::hash::{Hash, Hasher}; use std::sync::atomic::Ordering; use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; -use pyo3::types::{PyList, PyTuple}; +use pyo3::types::{PyBytes, PyList, PyTuple}; -use crate::ailment::ail_stmt::Statement; +use crate::ailment::ail_stmt::{AilStatement, Statement}; use crate::ailment::{CachedHash, hash_of}; #[pyclass( @@ -314,6 +315,57 @@ impl Block { Ok(slf.call_method0("copy")?.unbind()) } + // --- Byte serialization -------------------------------------------- + + /// Postcard-encode the full Block state -- ``(addr, original_size, + /// idx, statements)``, with statements embedded as their + /// [`AilStatement`] payloads. ``cached_hash`` is transient and is + /// recomputed after ``from_bytes``. Every statement must be an AIL + /// ``Statement``; anything else raises ``TypeError``. + fn to_bytes<'py>(&self, py: Python<'py>) -> PyResult> { + let stmt_list = self.statements.bind(py); + let mut stmts: Vec = Vec::with_capacity(stmt_list.len()); + for item in stmt_list.iter() { + let st = item.cast::().map_err(|_| { + PyTypeError::new_err(format!( + "Block.to_bytes: statements must all be AIL Statements, got {}", + item.get_type() + )) + })?; + stmts.push(st.borrow().stmt.clone()); + } + let payload = (self.addr, self.original_size, self.idx, stmts); + let bytes = postcard::to_stdvec(&payload) + .map_err(|e| PyTypeError::new_err(format!("serialize: {}", e)))?; + Ok(PyBytes::new(py, &bytes)) + } + + /// Inverse of ``to_bytes``. + #[classmethod] + fn from_bytes<'py>( + _cls: &Bound<'_, pyo3::types::PyType>, + py: Python<'py>, + data: &[u8], + ) -> PyResult> { + let (addr, original_size, idx, stmts): (i64, Option, Option, Vec) = + postcard::from_bytes(data) + .map_err(|e| PyTypeError::new_err(format!("deserialize: {}", e)))?; + let list = PyList::empty(py); + for st in stmts { + list.append(Py::new(py, Statement::wrap(st))?)?; + } + Py::new( + py, + Self { + addr, + original_size, + statements: list.unbind(), + idx, + cached_hash: CachedHash::new(), + }, + ) + } + fn __reduce__<'py>(slf: Bound<'py, Self>) -> PyResult> { let py = slf.py(); let cls = slf.get_type(); diff --git a/pyproject.toml b/pyproject.toml index 35df4db15..2b52e002c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,15 @@ [build-system] -requires = ["setuptools>=77.0.0", "setuptools-rust", "pyvex==9.3.1.dev0"] +# grpcio-tools generates angr/protos/*_pb2.py at build time. The generated code carries a protobuf gencode version +# stamp (6.31.1 for the pinned grpcio-tools) that the runtime protobuf validates at import: the runtime must be the +# same major version and no older than the stamp. Pinning grpcio-tools keeps the stamp deterministic and on the 6.x +# major, so the runtime "protobuf>=6.33.0" (and any 6.x/7.x resolution of it) can always load the generated modules. +requires = [ + "setuptools>=77.0.0", + "setuptools-rust", + "pyvex==9.3.1.dev0", + "grpcio-tools~=1.80.0", + "protobuf>=6.31.1,<7", +] build-backend = "setuptools.build_meta" [project] diff --git a/setup.py b/setup.py index a363a04a1..0ef7e29cd 100644 --- a/setup.py +++ b/setup.py @@ -64,6 +64,15 @@ def build_unicornlib(): shutil.copy(os.path.join("native/unicornlib", library_file), "angr") +def build_protos(): + proto_files = sorted(glob.glob("angr/protos/*.proto")) + cmd = [sys.executable, "-m", "grpc_tools.protoc", "-I.", "--python_out=.", *proto_files] + try: + subprocess.run(cmd, check=True) + except (FileNotFoundError, subprocess.CalledProcessError) as err: + raise LibError("Error while generating protobuf modules: " + str(err)) from err + + def clean_unicornlib(): oglob = glob.glob("native/*.o") oglob += glob.glob("native/*.obj") @@ -76,6 +85,7 @@ def clean_unicornlib(): class build(st_build): def run(self, *args): + self.execute(build_protos, (), msg="Generating protobuf modules") self.execute(build_unicornlib, (), msg="Building unicornlib") super().run(*args) diff --git a/tests/ailment/test_serialize.py b/tests/ailment/test_serialize.py index 35c0a14f5..e952e8fdf 100644 --- a/tests/ailment/test_serialize.py +++ b/tests/ailment/test_serialize.py @@ -484,6 +484,36 @@ class TestSerialize(unittest.TestCase): a2 = Statement.from_bytes(s) assert isinstance(a2, Assignment) + def test_block_to_from_bytes(self): + block = Block( + 0x400, + statements=[ + Label(0, "L1"), + Assignment(1, Tmp(0, 1, 32), Const(0, 42, 32)), + Jump(2, Const(0, 0x410, 64)), + ], + original_size=12, + ) + b2 = Block.from_bytes(block.to_bytes()) + assert b2 == block + assert b2.original_size == 12 and b2.idx is None + assert isinstance(b2.statements[1], Assignment) + + def test_block_to_from_bytes_with_idx(self): + block = Block(0x500, statements=[Return(0, [])], idx=3) + b2 = Block.from_bytes(block.to_bytes()) + assert b2 == block and b2.idx == 3 + + def test_empty_block_to_from_bytes(self): + b2 = Block.from_bytes(Block(0x600).to_bytes()) + assert b2.addr == 0x600 and b2.original_size is None and len(b2.statements) == 0 + + def test_block_to_bytes_rejects_non_statement(self): + block = Block(0x700) + block.statements.append("not a statement") + with self.assertRaises(TypeError): + block.to_bytes() + if __name__ == "__main__": unittest.main() diff --git a/tests/analyses/decompiler/test_decompiler.py b/tests/analyses/decompiler/test_decompiler.py index 034423ad3..783818f3f 100755 --- a/tests/analyses/decompiler/test_decompiler.py +++ b/tests/analyses/decompiler/test_decompiler.py @@ -1860,9 +1860,10 @@ class TestDecompiler(unittest.TestCase): """) d = proj.analyses.Decompiler(proj.kb.functions["main"], options=decompiler_options) - assert d.cache is not None and d.cache.clinic is not None and d.cache.clinic.variable_kb is not None + assert d.cache is not None and d.cache.clinic is not None + assert d.func.addr in proj.kb.dec_variables - vmi: VariableManagerInternal = d.cache.clinic.variable_kb.variables["main"] + vmi: VariableManagerInternal = proj.kb.dec_variables["main"] vmi.set_variable_type( next(iter(vmi.find_variables_by_stack_offset(-0x148))), SimTypePointer(typedefs[1]["struct C"]), @@ -1895,8 +1896,12 @@ class TestDecompiler(unittest.TestCase): assert unified is not None unified.name = "argc" unified.renamed = True + # variable types were edited on kb.dec_variables; force a fresh decompilation so the new + # types drive codegen (the default reuses the cached codegen AST as-is). d = proj.analyses.Decompiler( - proj.kb.functions["main"], variable_kb=d.cache.clinic.variable_kb, options=decompiler_options + proj.kb.functions["main"], + options=decompiler_options, + regen_clinic=True, ) assert d.codegen is not None and isinstance(d.codegen.text, str) @@ -3001,7 +3006,7 @@ class TestDecompiler(unittest.TestCase): # 1. Condition: (!a0) # 2. Has a scope ending in a return # 3. Has no else scope after the return - a0_name = d.clinic.variable_kb.variables[d.func.addr].unified_variable(d.clinic.arg_list[0]).name + a0_name = d.clinic.kb.dec_variables[d.func.addr].unified_variable(d.clinic.arg_list[0]).name good_if_pattern = r"if \(!" + a0_name + r"\)\s*\{[^}]*return 1;\s*\}(?!\s*else)" good_if = re.search(good_if_pattern, text) assert good_if is not None @@ -3053,7 +3058,7 @@ class TestDecompiler(unittest.TestCase): proj.analyses.CompleteCallingConventions(cfg=cfg) f = proj.kb.functions[0x404410] - d = proj.analyses[Decompiler](f, cfg=cfg.model, options=decompiler_options) + d = proj.analyses[Decompiler](f, cfg=cfg.model, options=decompiler_options, save_unoptimized_graph=True) print_decompilation_result(d) target_addrs = {0x4045D8, 0x404575} @@ -3405,7 +3410,7 @@ class TestDecompiler(unittest.TestCase): # the two function arguments that are passed through stack into prepare_padded_number must have been eliminated # at this point, leaving block 401f40 empty. - the_block = next(nn for nn in d.clinic.graph if nn.addr == 0x401F40) + the_block = next(nn for nn in d.ail_graph if nn.addr == 0x401F40) assert len(the_block.statements) == 1 # it has an unused label @for_all_structuring_algos @@ -3463,7 +3468,9 @@ class TestDecompiler(unittest.TestCase): for width in (2, 8): options = set_decompiler_option(list(decompiler_options or []), [("indent_size", width)]) - d = p.analyses[Decompiler].prep(fail_fast=True)(f, options=options) + # indent_size is a codegen display option, not a cache-validity parameter; force a fresh decompilation + # so the new width is applied instead of reusing the cached codegen. + d = p.analyses[Decompiler].prep(fail_fast=True)(f, options=options, regen_clinic=True) assert d.codegen is not None and d.codegen.text is not None levels = indent_levels(d.codegen.text) # every indentation level is a whole multiple of the configured width @@ -4054,7 +4061,7 @@ class TestDecompiler(unittest.TestCase): proj.analyses.CompleteCallingConventions(cfg=cfg) f = proj.kb.functions["recover_mode"] d = proj.analyses[Decompiler].prep(fail_fast=True)( - f, cfg=cfg.model, options=decompiler_options, generate_code=False + f, cfg=cfg.model, options=decompiler_options, generate_code=False, save_unoptimized_graph=True ) # we should have skipped generating code @@ -4155,7 +4162,7 @@ class TestDecompiler(unittest.TestCase): d = proj.analyses[Decompiler].prep(fail_fast=True)(f, cfg=cfg.model, options=decompiler_options) assert d.codegen is not None and d.clinic is not None - rd = proj.analyses.SReachingDefinitions(subject=f, func_graph=d.clinic.graph, func_args=set()).model + rd = proj.analyses.SReachingDefinitions(subject=f, func_graph=d.ail_graph, func_args=set()).model used_but_undefined_stack_vars = [ str(rd.varid_to_vvar[vid]) for vid, loc in rd.all_vvar_definitions.items() @@ -5678,21 +5685,21 @@ class TestDecompiler(unittest.TestCase): dec = proj.analyses.Decompiler(func, cfg=cfg, options=decompiler_options) assert dec.codegen is not None and dec.codegen.text is not None print_decompilation_result(dec) - a0 = dec.clinic.variable_kb.variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name + a0 = dec.clinic.kb.dec_variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name assert f"return test_tailcall_callee({a0} + 1);" in normalize_whitespace(dec.codegen.text) func = proj.kb.functions["test_noreturn_tailcall"] dec = proj.analyses.Decompiler(func, cfg=cfg, options=decompiler_options) assert dec.codegen is not None and dec.codegen.text is not None print_decompilation_result(dec) - a0 = dec.clinic.variable_kb.variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name + a0 = dec.clinic.kb.dec_variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name assert f"test_noreturn_tailcall_callee({a0} + 1); /* do not return */" in normalize_whitespace(dec.codegen.text) func = proj.kb.functions["test_cond_tailcall_jmp"] dec = proj.analyses.Decompiler(func, cfg=cfg, options=decompiler_options) assert dec.codegen is not None and dec.codegen.text is not None print_decompilation_result(dec) - a0 = dec.clinic.variable_kb.variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name + a0 = dec.clinic.kb.dec_variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name assert normalize_whitespace(f""" if ((int){a0}) return test_cond_tailcall_jmp_callee({a0}); @@ -5703,7 +5710,7 @@ class TestDecompiler(unittest.TestCase): dec = proj.analyses.Decompiler(func, cfg=cfg, options=decompiler_options) assert dec.codegen is not None and dec.codegen.text is not None print_decompilation_result(dec) - a0 = dec.clinic.variable_kb.variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name + a0 = dec.clinic.kb.dec_variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name assert normalize_whitespace(f""" if ({a0}) test_cond_noreturn_tailcall_jmp_callee(); /* do not return */ @@ -5714,7 +5721,7 @@ class TestDecompiler(unittest.TestCase): dec = proj.analyses.Decompiler(func, cfg=cfg, options=decompiler_options) assert dec.codegen is not None and dec.codegen.text is not None print_decompilation_result(dec) - a0 = dec.clinic.variable_kb.variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name + a0 = dec.clinic.kb.dec_variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name assert normalize_whitespace(f""" if ((int){a0}) return test_cond_tailcall_cjmp_callee({a0}); @@ -5725,7 +5732,7 @@ class TestDecompiler(unittest.TestCase): dec = proj.analyses.Decompiler(func, cfg=cfg, options=decompiler_options) assert dec.codegen is not None and dec.codegen.text is not None print_decompilation_result(dec) - a0 = dec.clinic.variable_kb.variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name + a0 = dec.clinic.kb.dec_variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name assert normalize_whitespace(f""" if ({a0}) test_cond_noreturn_tailcall_cjmp_callee(); /* do not return */ diff --git a/tests/analyses/decompiler/test_decompiler_types.py b/tests/analyses/decompiler/test_decompiler_types.py index 87b091a7e..aaac59c93 100644 --- a/tests/analyses/decompiler/test_decompiler_types.py +++ b/tests/analyses/decompiler/test_decompiler_types.py @@ -104,8 +104,8 @@ class TestDecompilerTypes(unittest.TestCase): print_decompilation_result(dec) # take the stack variable v2; it should be a char array of size 64 - assert dec._variable_kb is not None - varman = dec._variable_kb.variables.get_function_manager(func.addr) + assert dec.func.addr in dec.kb.dec_variables + varman = dec.kb.dec_variables.get_function_manager(func.addr) var2 = next(iter(varman.find_variables_by_stack_offset(-0x58))) assert var2 is not None var2 = varman.unified_variable(var2) @@ -117,7 +117,7 @@ class TestDecompilerTypes(unittest.TestCase): varman.set_variable_type(var2, SimTypeArray(SimTypeChar(), 0).with_arch(proj.arch), mark_manual=True) # decompile again; should not crash! - new_dec = proj.analyses.Decompiler(func, variable_kb=dec._variable_kb, fail_fast=True) + new_dec = proj.analyses.Decompiler(func, fail_fast=True, regen_clinic=True) assert dec.codegen is not None and dec.codegen.text is not None print_decompilation_result(new_dec) assert f"char {var2.name}[0];" in new_dec.codegen.text @@ -126,7 +126,7 @@ class TestDecompilerTypes(unittest.TestCase): varman.set_variable_type(var2, SimTypeInt().with_arch(proj.arch), mark_manual=True) # decompile again; should not crash! - new_dec = proj.analyses.Decompiler(func, variable_kb=dec._variable_kb, fail_fast=True) + new_dec = proj.analyses.Decompiler(func, fail_fast=True, regen_clinic=True) assert dec.codegen is not None and dec.codegen.text is not None print_decompilation_result(new_dec) assert f"int {var2.name};" in new_dec.codegen.text diff --git a/tests/analyses/decompiler/test_outliner.py b/tests/analyses/decompiler/test_outliner.py index e1920e584..f673c4979 100644 --- a/tests/analyses/decompiler/test_outliner.py +++ b/tests/analyses/decompiler/test_outliner.py @@ -28,7 +28,7 @@ class TestOutliner(TestCase): print("[+] Original function:") assert dec.codegen is not None assert dec.codegen.text is not None - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.clinic is not None print(dec.codegen.text) @@ -39,7 +39,7 @@ class TestOutliner(TestCase): ) # now we have two graphs; gotta decompile them individually - del dec._variable_kb.variables[func.addr] + del dec.kb.dec_variables.function_managers[func.addr] dec_outer = proj.analyses[Decompiler].prep( fail_fast=True, )( @@ -97,7 +97,7 @@ class TestOutliner(TestCase): dec = proj.analyses.Decompiler(func, cfg=cfg.model) assert dec.codegen is not None assert dec.codegen.text is not None - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.clinic is not None print(dec.codegen.text) @@ -172,7 +172,7 @@ class TestOutliner(TestCase): extracted = tt.extract(final_state, 0xC000_0000) print(extracted) - del dec._variable_kb.variables[func.addr] + del dec.kb.dec_variables.function_managers[func.addr] dec_outer = proj.analyses[Decompiler].prep( fail_fast=True, )( diff --git a/tests/analyses/test_typehoon.py b/tests/analyses/test_typehoon.py index be4e24b8f..09ee5df33 100755 --- a/tests/analyses/test_typehoon.py +++ b/tests/analyses/test_typehoon.py @@ -148,9 +148,10 @@ class TestTypehoon(unittest.TestCase): and func_read6numbers.prototype.args[1].pts_to.signed is True ) - # decompile phase_2 again, and we should see an unsigned int [6] on the stack + # decompile phase_2 again, and we should see an unsigned int [6] on the stack. regen_clinic=True forces a + # fresh run so the newly-inferred read_six_numbers prototype back-propagates instead of reusing the cache. dec_phase2 = proj.analyses.Decompiler( - func_phase2, fail_fast=True, options=[("constrain_callee_prototypes", True)] + func_phase2, fail_fast=True, options=[("constrain_callee_prototypes", True)], regen_clinic=True ) assert dec_phase2.codegen is not None and dec_phase2.codegen.text is not None print_decompilation_result(dec_phase2) diff --git a/tests/gui/test_decompilation_workflows.py b/tests/gui/test_decompilation_workflows.py index 8f49faa91..7253128c2 100644 --- a/tests/gui/test_decompilation_workflows.py +++ b/tests/gui/test_decompilation_workflows.py @@ -45,8 +45,8 @@ class TestDecompilationWorkflows(unittest.TestCase): assert dec.codegen is not None and dec.codegen.text is not None print_decompilation_result(dec) - assert dec._variable_kb is not None - types = dec._variable_kb.variables["main"].types + assert dec.func.addr in dec.kb.dec_variables + types = dec.kb.dec_variables["main"].types # let's rename a struct field new_type_name = "my_awesome_type" t = types["struct_0"] @@ -71,8 +71,8 @@ class TestDecompilationWorkflows(unittest.TestCase): assert dec.codegen is not None and dec.codegen.text is not None print_decompilation_result(dec) - assert dec._variable_kb is not None - types = dec._variable_kb.variables["main"].types + assert dec.func.addr in dec.kb.dec_variables + types = dec.kb.dec_variables["main"].types # let's rename a struct field t = types["struct_0"] assert isinstance(t, TypeRef) @@ -99,8 +99,8 @@ class TestDecompilationWorkflows(unittest.TestCase): print_decompilation_result(dec) assert "struct struct_1 *field_120;" in dec.codegen.text - assert dec._variable_kb is not None - types = dec._variable_kb.variables["main"].types + assert dec.func.addr in dec.kb.dec_variables + types = dec.kb.dec_variables["main"].types # let's type a struct field t = types["struct_0"] assert isinstance(t, TypeRef) diff --git a/tests/knowledge_plugins/test_variable_manager.py b/tests/knowledge_plugins/test_variable_manager.py index ccbc04790..e08d1bac0 100644 --- a/tests/knowledge_plugins/test_variable_manager.py +++ b/tests/knowledge_plugins/test_variable_manager.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# pylint: disable=missing-class-docstring,no-self-use,line-too-long +# pylint: disable=missing-class-docstring,no-self-use,line-too-long,protected-access from __future__ import annotations __package__ = __package__ or "tests.knowledge_plugins" # pylint:disable=redefined-builtin @@ -7,8 +7,11 @@ __package__ = __package__ or "tests.knowledge_plugins" # pylint:disable=redefin import os import pickle import unittest +from unittest import mock import angr +from angr.knowledge_plugins.variables import variable_manager as variable_manager_mod +from angr.knowledge_plugins.variables.spilling_vardict import SpillingVariableInternalDict from tests.common import bin_location test_location = os.path.join(bin_location, "tests") @@ -38,6 +41,85 @@ class TestVariableManager(unittest.TestCase): assert ident3 == "is_2" assert ident4 == "ir_1" + def test_dec_variables_spill_evict_and_reload(self): + # kb.dec_variables holds its per-function managers in a SpillingVariableInternalDict. Forcing a tiny cache + # limit spills the least-recently-used entries to the RuntimeDb LMDB store, and they reload with identical + # content on access. + p = angr.Project(os.path.join(test_location, "x86_64", "fauxware"), auto_load_libs=False) + cfg = p.analyses.CFGFast(normalize=True) + for name in ("main", "authenticate"): + p.analyses.Decompiler(name, cfg=cfg.model) + + dvm = p.kb.dec_variables + fm = dvm.function_managers + assert isinstance(fm, SpillingVariableInternalDict) + assert len(fm) >= 2 + + def content(internal): + return sorted(v.ident for v in internal._variables) + + pre = {addr: content(fm[addr]) for addr in list(fm)} + assert all(pre.values()), "each decompiled function should have variables" + + # force every entry out of the in-memory cache + fm._cache_limit = 0 + fm._evict_lru() + assert not fm._cache and len(fm._spilled) == len(pre) + + # accessing a spilled entry reloads it losslessly (with the manager reattached) + for addr, expected in pre.items(): + reloaded = fm[addr] + assert reloaded.manager is dvm + assert content(reloaded) == expected + + def test_dec_variables_spilling_pickle_roundtrip(self): + # A knowledge base whose dec_variables have been spilled pickles self-containedly (the non-durable RuntimeDb + # reference is dropped) and the per-function variables survive the round-trip. + p = angr.Project(os.path.join(test_location, "x86_64", "fauxware"), auto_load_libs=False) + cfg = p.analyses.CFGFast(normalize=True) + p.analyses.Decompiler("main", cfg=cfg.model) + + dvm = p.kb.dec_variables + addr = next(iter(dvm.function_managers)) + pre = sorted(v.ident for v in dvm.function_managers[addr]._variables) + # spill everything before pickling + dvm.function_managers._cache_limit = 0 + dvm.function_managers._evict_lru() + + kb2 = pickle.loads(pickle.dumps(p.kb)) + dvm2 = kb2.dec_variables + assert isinstance(dvm2.function_managers, SpillingVariableInternalDict) + assert list(dvm2.function_managers) == [addr] + post = sorted(v.ident for v in dvm2.function_managers[addr]._variables) + assert post == pre + + def test_dec_variables_decompile_under_tiny_spill_limit(self): + # Decompilation output is byte-identical whether dec_variables spill aggressively (cache limit 1, so every + # function's manager is evicted as soon as the next function is decompiled) or spilling is disabled. + binpath = os.path.join(test_location, "x86_64", "fauxware") + func_names = ("main", "authenticate", "accepted", "rejected") + + p = angr.Project(binpath, auto_load_libs=False) + cfg = p.analyses.CFGFast(normalize=True) + fm = p.kb.dec_variables.function_managers + assert isinstance(fm, SpillingVariableInternalDict) + fm._cache_limit = 1 + + texts = {} + for name in func_names: + dec = p.analyses.Decompiler(name, cfg=cfg.model) + assert dec.codegen is not None and dec.codegen.text is not None + texts[name] = dec.codegen.text + assert fm._spilled, "decompiling multiple functions under cache limit 1 must have spilled entries" + + with mock.patch.object(variable_manager_mod, "USE_SPILLING_DVARS", False): + p2 = angr.Project(binpath, auto_load_libs=False) + cfg2 = p2.analyses.CFGFast(normalize=True) + assert type(p2.kb.dec_variables.function_managers) is dict + for name in func_names: + dec2 = p2.analyses.Decompiler(name, cfg=cfg2.model) + assert dec2.codegen is not None and dec2.codegen.text == texts[name] + if __name__ == "__main__": unittest.main() diff --git a/tests/llm/test_decompiler_llm.py b/tests/llm/test_decompiler_llm.py index 9026ef858..c03512d8b 100644 --- a/tests/llm/test_decompiler_llm.py +++ b/tests/llm/test_decompiler_llm.py @@ -147,11 +147,11 @@ class TestDecompilerLLMSuggestVariableNames(TestDecompilerLLMRefineBase): def test_renames_variables(self): """Should rename variables when the LLM suggests new names.""" dec = self._decompile("main") - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.codegen is not None and dec.codegen.text is not None # collect current variable names - varman = dec._variable_kb.variables[dec.func.addr] + varman = dec.kb.dec_variables[dec.func.addr] unified_vars = varman.get_unified_variables(sort=None) assert len(unified_vars) > 0, "Expected at least one variable" @@ -183,10 +183,10 @@ class TestDecompilerLLMSuggestVariableNames(TestDecompilerLLMRefineBase): def test_skips_same_name_renames(self): """Should skip rename when old_name == new_name.""" dec = self._decompile("main") - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.codegen is not None and dec.codegen.text is not None - varman = dec._variable_kb.variables[dec.func.addr] + varman = dec.kb.dec_variables[dec.func.addr] unified_vars = varman.get_unified_variables(sort=None) target_var = unified_vars[0] old_name = target_var.name or str(target_var) @@ -217,10 +217,10 @@ class TestDecompilerLLMSuggestVariableNames(TestDecompilerLLMRefineBase): def test_skips_empty_new_name(self): """Should ignore renames with empty new_name.""" dec = self._decompile("main") - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.codegen is not None and dec.codegen.text is not None - varman = dec._variable_kb.variables[dec.func.addr] + varman = dec.kb.dec_variables[dec.func.addr] unified_vars = varman.get_unified_variables(sort=None) target_var = unified_vars[0] old_name = target_var.name or str(target_var) @@ -237,10 +237,10 @@ class TestDecompilerLLMSuggestVariableNames(TestDecompilerLLMRefineBase): def test_multiple_renames(self): """Should rename multiple variables at once.""" dec = self._decompile("main") - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.codegen is not None and dec.codegen.text is not None - varman = dec._variable_kb.variables[dec.func.addr] + varman = dec.kb.dec_variables[dec.func.addr] unified_vars = varman.get_unified_variables(sort=None) if len(unified_vars) < 2: self.skipTest("Need at least 2 variables for this test") @@ -375,10 +375,10 @@ class TestDecompilerLLMSuggestVariableTypes(TestDecompilerLLMRefineBase): def test_changes_variable_type(self): """Should change variable types when LLM suggests valid C types.""" dec = self._decompile("main") - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.codegen is not None and dec.codegen.text is not None - varman = dec._variable_kb.variables[dec.func.addr] + varman = dec.kb.dec_variables[dec.func.addr] unified_vars = varman.get_unified_variables(sort=None) assert len(unified_vars) > 0 @@ -401,10 +401,10 @@ class TestDecompilerLLMSuggestVariableTypes(TestDecompilerLLMRefineBase): def test_skips_unparseable_types(self): """Should skip variables with unparseable type strings.""" dec = self._decompile("main") - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.codegen is not None and dec.codegen.text is not None - varman = dec._variable_kb.variables[dec.func.addr] + varman = dec.kb.dec_variables[dec.func.addr] unified_vars = varman.get_unified_variables(sort=None) target_var = unified_vars[0] var_name = target_var.name or str(target_var) @@ -455,10 +455,10 @@ class TestDecompilerLLMSuggestVariableTypes(TestDecompilerLLMRefineBase): def test_pointer_type_change(self): """Should handle pointer type suggestions.""" dec = self._decompile("main") - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.codegen is not None and dec.codegen.text is not None - varman = dec._variable_kb.variables[dec.func.addr] + varman = dec.kb.dec_variables[dec.func.addr] unified_vars = varman.get_unified_variables(sort=None) assert len(unified_vars) > 0 @@ -476,10 +476,10 @@ class TestDecompilerLLMSuggestVariableTypes(TestDecompilerLLMRefineBase): def test_multiple_type_changes(self): """Should change types for multiple variables at once.""" dec = self._decompile("main") - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.codegen is not None and dec.codegen.text is not None - varman = dec._variable_kb.variables[dec.func.addr] + varman = dec.kb.dec_variables[dec.func.addr] unified_vars = varman.get_unified_variables(sort=None) if len(unified_vars) < 2: self.skipTest("Need at least 2 variables for this test") @@ -507,10 +507,10 @@ class TestDecompilerLLMSuggestVariableTypes(TestDecompilerLLMRefineBase): def test_partial_valid_types(self): """When some types parse and some don't, should apply the valid ones.""" dec = self._decompile("main") - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.codegen is not None and dec.codegen.text is not None - varman = dec._variable_kb.variables[dec.func.addr] + varman = dec.kb.dec_variables[dec.func.addr] unified_vars = varman.get_unified_variables(sort=None) assert len(unified_vars) > 0 @@ -584,7 +584,7 @@ class TestDecompilerLLMEndToEnd(TestDecompilerLLMRefineBase): def test_full_variable_rename_flow(self): """Full flow: decompile -> mock LLM suggests renames -> verify text changes.""" dec = self._decompile("main") - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.codegen is not None and dec.codegen.text is not None # get a variable to rename @@ -617,10 +617,10 @@ class TestDecompilerLLMEndToEnd(TestDecompilerLLMRefineBase): def test_full_type_change_flow(self): """Full flow: decompile -> mock LLM suggests types -> verify types applied.""" dec = self._decompile("main") - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.codegen is not None and dec.codegen.text is not None - varman = dec._variable_kb.variables[dec.func.addr] + varman = dec.kb.dec_variables[dec.func.addr] unified_vars = list(dec.codegen.cfunc.get_unified_local_vars()) assert len(unified_vars) > 0 @@ -654,7 +654,7 @@ class TestDecompilerLLMEndToEnd(TestDecompilerLLMRefineBase): def test_no_changes_flow(self): """Full flow: LLM returns empty results -> no changes, no regeneration.""" dec = self._decompile("main") - assert dec._variable_kb is not None + assert dec.func.addr in dec.kb.dec_variables assert dec.codegen is not None and dec.codegen.text is not None original_text = dec.codegen.text diff --git a/tests/serialization/test_db.py b/tests/serialization/test_db.py index 163568d52..8b9ecdf08 100755 --- a/tests/serialization/test_db.py +++ b/tests/serialization/test_db.py @@ -16,8 +16,11 @@ import archinfo import cle import angr +from angr.analyses.decompiler.decompilation_cache import DecompilationCache +from angr.analyses.decompiler.structured_codegen import DummyStructuredCodeGenerator from angr.analyses.decompiler.structured_codegen.c import CConstant from angr.angrdb import AngrDB +from angr.knowledge_plugins.structured_code import SpillingDecompilationDict from tests.common import bin_location, print_decompilation_result test_location = os.path.join(bin_location, "tests") @@ -374,9 +377,11 @@ class TestDb(unittest.TestCase): bin_path = os.path.join(test_location, "x86_64", "fauxware") proj = angr.Project(bin_path, auto_load_libs=False) - cfg = proj.analyses.CFGFast(normalize=True) - dec = proj.analyses.Decompiler("main", variable_kb=proj.kb, cfg=cfg.model) - assert dec.codegen is not None and dec.codegen.text is not None + proj.analyses.CFGFast(normalize=True) + # populate the disassembly-level variable manager (kb.variables), which is what the ``variables`` angrdb + # table serializes; decompilation variables live separately in kb.dec_variables + main = proj.kb.functions.function(name="main") + proj.analyses.VariableRecoveryFast(main) vm = proj.kb.variables # force-create empty variable managers for two functions that have none @@ -387,7 +392,7 @@ class TestDb(unittest.TestCase): assert not vm.function_managers[addr].serialize() nonempty_addrs = {addr for addr, internal in vm.function_managers.items() if internal.serialize()} - assert nonempty_addrs, "decompilation should have produced at least one non-empty variable manager" + assert nonempty_addrs, "variable recovery should have produced at least one non-empty variable manager" def content(internal): return ( @@ -424,6 +429,86 @@ class TestDb(unittest.TestCase): old_format_proj = AngrDB(nullpool=True).load(db_file) assert set(old_format_proj.kb.variables.function_managers) == nonempty_addrs + def test_angrdb_dec_variables_roundtrip(self): + # Decompilation variables (kb.dec_variables) are serialized into their own ``dec_variables`` table, isolated + # from the disassembly-level kb.variables, and round-trip through angrdb with their content intact. + bin_path = os.path.join(test_location, "x86_64", "fauxware") + + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(normalize=True) + main = proj.kb.functions.function(name="main") + dec = proj.analyses.Decompiler(main, cfg=cfg.model) + assert dec.codegen is not None and dec.codegen.text is not None + + dvm = proj.kb.dec_variables + nonempty_addrs = {addr for addr, internal in dvm.function_managers.items() if internal.serialize()} + assert nonempty_addrs, "decompilation should have produced at least one non-empty dec-variable manager" + # decompilation must not have populated the disassembly-level manager + assert not {addr for addr, internal in proj.kb.variables.function_managers.items() if internal.serialize()} + + def content(internal): + return ( + sorted(v.ident for v in internal._variables), + sorted(v.ident for v in internal._unified_variables), + sorted(v.ident for v in internal._phi_variables), + ) + + pre_content = {addr: content(dvm.function_managers[addr]) for addr in nonempty_addrs} + + dtemp = tempfile.mkdtemp() + db_file = os.path.join(dtemp, "fauxware.adb") + AngrDB(proj, nullpool=True).dump(db_file) + + # dec_variables rows land in their own table, not in variables + conn = sqlite3.connect(db_file) + dvar_rows = conn.execute("SELECT func_addr FROM dec_variables WHERE func_addr != -1").fetchall() + var_rows = conn.execute("SELECT func_addr FROM variables WHERE func_addr != -1").fetchall() + conn.close() + assert {func_addr for (func_addr,) in dvar_rows} == nonempty_addrs + assert not var_rows + + # dec_variables round-trip with identical content + new_proj = AngrDB(nullpool=True).load(db_file) + new_dvm = new_proj.kb.dec_variables + assert set(new_dvm.function_managers) == nonempty_addrs + for addr in nonempty_addrs: + assert content(new_dvm.function_managers[addr]) == pre_content[addr] + + def test_angrdb_dump_with_spilled_dec_variables(self): + # Dumping to angrdb while dec_variables entries are spilled to the RuntimeDb LMDB store faults them back in + # through the spilling dict's snapshot-safe iteration, and their content round-trips. + bin_path = os.path.join(test_location, "x86_64", "fauxware") + + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(normalize=True) + for name in ("main", "authenticate"): + dec = proj.analyses.Decompiler(name, cfg=cfg.model) + assert dec.codegen is not None and dec.codegen.text is not None + + dvm = proj.kb.dec_variables + fm = dvm.function_managers + + def content(internal): + return sorted(v.ident for v in internal._variables) + + pre_content = {addr: content(fm[addr]) for addr in list(fm)} + assert len(pre_content) >= 2 + + # spill every entry, then dump while nothing is in memory + fm._cache_limit = 0 + fm._evict_lru() + assert not fm._cache and set(fm._spilled) == set(pre_content) + + dtemp = tempfile.mkdtemp() + db_file = os.path.join(dtemp, "fauxware.adb") + AngrDB(proj, nullpool=True).dump(db_file) + + new_proj = AngrDB(nullpool=True).load(db_file) + new_fm = new_proj.kb.dec_variables.function_managers + assert set(new_fm) == set(pre_content) + for addr, expected in pre_content.items(): + assert content(new_fm[addr]) == expected + def test_angrdb_open_multiple_times(self): bin_path = os.path.join(test_location, "x86_64", "fauxware") @@ -645,6 +730,71 @@ class TestDb(unittest.TestCase): print_decompilation_result(dec_2) assert dec_2.codegen.text.count("0x8") == 1 + def test_angrdb_full_decompilation_cache_roundtrip(self): + # DecompilationCache objects in the structured code manager are fully serialized into the database and come + # back with real codegen (not DummyStructuredCodeGenerator) and their version and timestamp intact. + bin_path = os.path.join(test_location, "x86_64", "fauxware") + + with tempfile.TemporaryDirectory() as td: + db_file = os.path.join(td, "proj.adb") + + proj = angr.Project(bin_path, auto_load_libs=False) + proj.analyses.CFGFast(normalize=True) + func = proj.kb.functions.function(name="authenticate") + dec = proj.analyses.Decompiler(func) + assert dec.codegen is not None and dec.codegen.text is not None + + cache = proj.kb.decompilations[(func.addr, "pseudocode")] + assert cache.version == angr.__version__ + assert cache.timestamp > 0 + + # add an unserializable cache; it must round-trip through the legacy structured_code table + dummy_cache = DecompilationCache(0xDEAD) + dummy_cache.codegen = DummyStructuredCodeGenerator("pseudocode", stmt_comments={0x1000: "hi"}) + proj.kb.decompilations[(0xDEAD, "pseudocode")] = dummy_cache + + new_proj = self._roundtrip_angrdb(proj, db_file) + + new_cache = new_proj.kb.decompilations[(func.addr, "pseudocode")] + assert not isinstance(new_cache.codegen, DummyStructuredCodeGenerator) + assert new_cache.codegen.text == dec.codegen.text + assert new_cache.version == cache.version + assert new_cache.timestamp == cache.timestamp + + new_dummy = new_proj.kb.decompilations[(0xDEAD, "pseudocode")] + assert isinstance(new_dummy.codegen, DummyStructuredCodeGenerator) + assert new_dummy.codegen.stmt_comments == {0x1000: "hi"} + + def test_angrdb_fast_load_spilled_decompilation_caches(self): + # When the database contains more decompilation caches than the manager may keep in memory, the serialized + # bytes are moved directly into the LMDB backing store on load without being deserialized. + bin_path = os.path.join(test_location, "x86_64", "fauxware") + + with tempfile.TemporaryDirectory() as td: + db_file = os.path.join(td, "proj.adb") + + proj = angr.Project(bin_path, auto_load_libs=False) + proj.analyses.CFGFast(normalize=True) + auth_func = proj.kb.functions.function(name="authenticate") + main_func = proj.kb.functions.function(name="main") + auth_dec = proj.analyses.Decompiler(auth_func) + main_dec = proj.analyses.Decompiler(main_func) + assert auth_dec.codegen is not None and main_dec.codegen is not None + + AngrDB(proj, nullpool=True).dump(db_file) + + with mock.patch("angr.knowledge_plugins.structured_code.DECOMPILATION_CACHE_LIMIT", 1): + new_proj = AngrDB(nullpool=True).load(db_file) + backing = new_proj.kb.decompilations.cached + assert isinstance(backing, SpillingDecompilationDict) + # both caches were imported as bytes and registered as spilled, not deserialized + assert backing._spilled == {(auth_func.addr, "pseudocode"), (main_func.addr, "pseudocode")} + assert len(backing._cache) == 0 + + # accessing a spilled cache deserializes it lazily + new_cache = new_proj.kb.decompilations[(auth_func.addr, "pseudocode")] + assert new_cache.codegen.text == auth_dec.codegen.text + def test_angrdb_decompilation_load_variables(self): # https://github.com/angr/angr/issues/5990 @@ -659,7 +809,7 @@ class TestDb(unittest.TestCase): resolve_indirect_jumps=True, detect_tail_calls=True, ) - dec = proj.analyses.Decompiler("main", variable_kb=proj.kb, cfg=cfg.model, regen_clinic=False) + dec = proj.analyses.Decompiler("main", cfg=cfg.model, regen_clinic=False) assert dec.codegen is not None and dec.codegen.text is not None adb = AngrDB(proj, nullpool=True) diff --git a/tests/serialization/test_decompilation_cache_serialization.py b/tests/serialization/test_decompilation_cache_serialization.py new file mode 100644 index 000000000..9e193752c --- /dev/null +++ b/tests/serialization/test_decompilation_cache_serialization.py @@ -0,0 +1,491 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use,line-too-long,no-member,protected-access +from __future__ import annotations + +__package__ = __package__ or "tests.serialization" # pylint:disable=redefined-builtin + +import os +import pickle +import unittest + +import networkx + +import angr +from angr.ailment import Block as AilBlock +from angr.ailment import Expr +from angr.ailment.expression import Const +from angr.ailment.expression import Tmp as AilTmp +from angr.ailment.expression import VirtualVariable as AilVirtualVariable +from angr.ailment.statement import Assignment, Return +from angr.analyses.decompiler.decompilation_cache import DecompilationCache +from angr.analyses.decompiler.notes.decompilation_note import ( + DecompilationNote, + DecompilationNoteLevel, +) +from angr.analyses.decompiler.notes.deobfuscated_strings import DeobfuscatedStringsNote +from angr.analyses.decompiler.optimization_passes.expr_op_swapper import OpDescriptor +from angr.analyses.decompiler.optimization_passes.static_vvar_rewriter import FixedBuffer, FixedBufferPtr +from angr.analyses.decompiler.structured_codegen import DummyStructuredCodeGenerator +from angr.analyses.decompiler.structured_codegen.c import CConstruct +from angr.analyses.decompiler.structured_codegen.c_serialize import ( + _DISPLAY_OPTION_ATTRS, + _DISPLAY_OPTION_FIELD_FIRST, + _parse_tags, + _sanitize_tags, +) +from angr.knowledge_plugins.structured_code import SpillingDecompilationDict +from angr.protos import codegen_pb2 +from angr.sim_variable import SimRegisterVariable, SimStackVariable +from angr.utils.ail_serialization import ( + pack_arg_vvars, + pack_graph, + pack_ite_exprs, + pack_static_buffers, + pack_static_vvars, + parse_arg_vvars, + parse_graph, + parse_ite_exprs, + parse_static_buffers, + parse_static_vvars, +) +from tests.common import bin_location + +test_location = os.path.join(bin_location, "tests") + + +class TestSubObjectSerialization(unittest.TestCase): + def test_decompilation_note(self): + n = DecompilationNote( + key="warn1", name="Warning One", content={"foo": [1, 2]}, level=DecompilationNoteLevel.WARNING + ) + back = DecompilationNote.from_json(n.to_json()) + assert type(back) is DecompilationNote + assert back.key == n.key + assert back.name == n.name + assert back.level == n.level + assert back.content == n.content + + def test_decompilation_note_non_jsonable_content(self): + n = DecompilationNote(key="k", name="n", content=object()) + back = DecompilationNote.from_json(n.to_json()) + assert back.content is None + + def test_deobfuscated_strings_note_roundtrip(self): + n = DeobfuscatedStringsNote() + n.add_string("1", b"\x00binary\xffdata", ref_addr=0x400100) + n.add_string("2", b"hello", ref_addr=0x400200) + + back = DecompilationNote.from_json(n.to_json()) + assert isinstance(back, DeobfuscatedStringsNote) + assert back.key == n.key + assert back.name == n.name + assert set(back.strings) == {0x400100, 0x400200} + assert back.strings[0x400100].value == b"\x00binary\xffdata" + assert back.strings[0x400100].type == "1" + assert back.strings[0x400200].value == b"hello" + assert str(back) == str(n) + + def test_op_descriptor(self): + op = OpDescriptor(block_addr=0x400500, stmt_idx=3, ins_addr=0x400502, op="Sub") + back = OpDescriptor.from_json(op.to_json()) + assert back == op + assert hash(back) == hash(op) + + +class TestAilSerializationHelpers(unittest.TestCase): + def test_display_option_attrs_derived_from_proto(self): + # _DISPLAY_OPTION_ATTRS is generated from the Codegen descriptor's trailing display-option block; every entry + # must be an optional scalar (the serialize loop uses plain setattr, which cannot handle message fields). + assert {"indent", "show_casts", "max_str_len"} <= set(_DISPLAY_OPTION_ATTRS) + assert len(set(_DISPLAY_OPTION_ATTRS)) == len(_DISPLAY_OPTION_ATTRS) + for name in _DISPLAY_OPTION_ATTRS: + field = codegen_pb2.Codegen.DESCRIPTOR.fields_by_name[name] + assert field.number >= _DISPLAY_OPTION_FIELD_FIRST + # not a message and not repeated. Use the modern FieldDescriptor API: protobuf 7.x (the upb backend) + # removed the ``label``/``type`` attributes and the LABEL_*/TYPE_* constants. + assert field.message_type is None + assert not field.is_repeated + + def test_tags_roundtrip_with_ins_offset(self): + # both addresses known: ins_addr rides as a delta but round-trips to the absolute value + tags = {"ins_addr": 0x4010F0, "vex_block_addr": 0x401000, "vex_stmt_idx": 7, "custom": [1, 2]} + key, msg = _sanitize_tags(tags) + assert msg is not None and not msg.HasField("ins_addr") and msg.ins_offset == 0xF0 + assert _parse_tags(msg) == tags + # ins_addr alone stays absolute + key2, msg2 = _sanitize_tags({"ins_addr": 0x400123}) + assert msg2 is not None and msg2.HasField("ins_addr") and not msg2.HasField("ins_offset") + assert _parse_tags(msg2) == {"ins_addr": 0x400123} + assert key != key2 + + def _blocks(self): + b0 = AilBlock(0x1000, 4, statements=[Assignment(0, AilTmp(1, 2, 64), Const(2, 1, 64), ins_addr=0x1000)]) + b1 = AilBlock(0x1004, 4, statements=[Return(3, [], ins_addr=0x1004)]) + b2 = AilBlock(0x1008, 4, statements=[], idx=1) + return b0, b1, b2 + + def test_graph_roundtrip_with_edge_data(self): + b0, b1, b2 = self._blocks() + g = networkx.DiGraph() + g.add_edge(b0, b1, type="fake_return", outside=False, confirmed=True) + # ins_addr=None packs as unset and comes back as an absent key + g.add_edge(b0, b2, type="transition", ins_addr=None, stmt_idx=-2) + g.add_edge(b1, b2) # no edge data at all + + back = parse_graph(pack_graph(g)) + assert set(back.nodes) == set(g.nodes) + assert back[b0][b1] == {"type": "fake_return", "outside": False, "confirmed": True} + assert back[b0][b2] == {"type": "transition", "stmt_idx": -2} + assert back[b1][b2] == {} + + def test_graph_rejects_unknown_edge_attr(self): + b0, b1, _ = self._blocks() + g = networkx.DiGraph() + g.add_edge(b0, b1, color="red") + with self.assertRaises(TypeError): + pack_graph(g) + + def test_graph_rejects_unknown_edge_type_string(self): + b0, b1, _ = self._blocks() + g = networkx.DiGraph() + g.add_edge(b0, b1, type="teleport") + with self.assertRaises(TypeError): + pack_graph(g) + + def test_graph_rejects_non_block_node(self): + g = networkx.DiGraph() + g.add_node("not a block") + with self.assertRaises(TypeError): + pack_graph(g) + + def test_arg_vvars_roundtrip(self): + vvc = Expr.VirtualVariableCategory + d = { + 0: (AilVirtualVariable(0, 1, 64, vvc.REGISTER), SimRegisterVariable(16, 8, ident="arg_0")), + 1: (AilVirtualVariable(1, 2, 64, vvc.STACK), SimStackVariable(-8, 8, ident="arg_1")), + } + back = parse_arg_vvars(pack_arg_vvars(d)) + assert back == d + + def test_ite_exprs_roundtrip(self): + s = {(0x400123, Const(0, 5, 64)), (0x400456, Const(1, 7, 32))} + assert parse_ite_exprs(pack_ite_exprs(s)) == s + + def test_static_vvars_roundtrip_both_arms(self): + d = {3: FixedBufferPtr("buf0", offset=8), 4: Const(0, 0xDEAD, 64)} + back = parse_static_vvars(pack_static_vvars(d)) + assert set(back) == {3, 4} + assert isinstance(back[3], FixedBufferPtr) + assert back[3].buffer_ident == "buf0" and back[3].offset == 8 + assert back[4] == d[4] + + def test_static_buffers_roundtrip(self): + d = {"buf0": FixedBuffer("buf0", 16, b"\x00" * 16), "anon": FixedBuffer(None, 4, b"abcd")} + back = parse_static_buffers(pack_static_buffers(d)) + assert set(back) == {"buf0", "anon"} + assert back["buf0"].ident == "buf0" and back["buf0"].size == 16 and back["buf0"].content == b"\x00" * 16 + assert back["anon"].ident == "" # FixedBuffer normalizes a None ident at construction time + + +class TestDecompilationCacheEndToEnd(unittest.TestCase): + """End-to-end tests using a real fauxware decompilation.""" + + @classmethod + def setUpClass(cls): + cls.proj = angr.Project(os.path.join(test_location, "x86_64", "fauxware"), auto_load_libs=False) + cls.cfg = cls.proj.analyses.CFGFast(normalize=True) + cls.func = cls.proj.kb.functions.function(name="authenticate") + cls.decompiler = cls.proj.analyses.Decompiler(cls.func, cfg=cls.cfg.model, generate_code=True) + + def test_codegen_roundtrip(self): + codegen = self.decompiler.codegen + blob = codegen.serialize() + back = type(codegen).parse(blob, project=self.proj, kb=self.proj.kb) + assert back.text == codegen.text + assert back.cfunc is not None + assert back.cfunc.name == codegen.cfunc.name + assert back.cfunc.addr == codegen.cfunc.addr + assert back.flavor == codegen.flavor + # idx is the per-codegen unique node identity and doubles as the serialization node id + assert back.cfunc.idx == codegen.cfunc.idx + assert back.cfunc.ident == codegen.cfunc.ident + + live_nodes = { + id(elem.obj): elem.obj for _, elem in codegen.map_pos_to_node.items() if isinstance(elem.obj, CConstruct) + } + assert live_nodes + assert len({node.idx for node in live_nodes.values()}) == len(live_nodes) + + msg = codegen_pb2.Codegen() + msg.ParseFromString(blob) + node_ids = [n.node_id for n in msg.nodes] + assert len(set(node_ids)) == len(node_ids) + assert 0 not in node_ids + # nodes created after deserialization must not collide with deserialized ones + assert back._next_node_idx > max(node_ids) + + def test_clinic_roundtrip(self): + clinic = self.decompiler.clinic + back = type(clinic).parse( + clinic.serialize(), + project=self.proj, + kb=self.proj.kb, + function=clinic.function, + cfg=clinic._cfg, + ) + # the fields the decompiler's cache-reuse path consumes round-trip + assert back.cc_graph.number_of_nodes() == clinic.cc_graph.number_of_nodes() + assert back.graph.number_of_nodes() == clinic.graph.number_of_nodes() + assert back.graph.number_of_edges() == clinic.graph.number_of_edges() + # unoptimized_graph is only built (and serialized) with Decompiler(save_unoptimized_graph=True); this + # decompiler used the default, so it is absent on both the live and the deserialized clinic + assert clinic.unoptimized_graph is None and back.unoptimized_graph is None + assert back._save_unoptimized_graph is False + assert back.arg_vvars == clinic.arg_vvars + assert len(back.externs) == len(clinic.externs) + assert (back.arg_list is None) == (clinic.arg_list is None) + assert back.vvar_id_start == clinic.vvar_id_start + assert back.copied_var_ids == clinic.copied_var_ids + assert back.edges_to_remove == clinic.edges_to_remove + assert back.entry_node_addr == clinic.entry_node_addr + assert back._mode == clinic._mode + assert back._start_stage == clinic._start_stage + assert back._end_stage == clinic._end_stage + assert back._skip_stages == clinic._skip_stages + assert back.flavor == clinic.flavor + # regenerable / runtime-only state is not serialized, so the deserialized clinic comes back with the + # default (the caller's fast-path reuse regenerates whatever it needs) + for attr in ( + "_ail_graph", + "_init_ail_graph", + "_init_arg_vvars", + "func_args", + "func_ret_var", + "reaching_definitions", + "_blocks_by_addr_and_size", + "typehoon", + ): + assert getattr(back, attr) is None, attr + for attr in ("data_refs", "notes"): + assert getattr(back, attr) == {}, attr + # stack_items is primitive result data and is kept through downsize and serialization + assert {k: (v.offset, v.size, v.name, v.item_type) for k, v in back.stack_items.items()} == { + k: (v.offset, v.size, v.name, v.item_type) for k, v in clinic.stack_items.items() + } + assert clinic._inline_functions == set() and back._inline_functions == set() + + def test_clinic_roundtrip_with_save_unoptimized_graph(self): + # Decompiler(save_unoptimized_graph=True) opts the unoptimized graph into serialization. + dec = self.proj.analyses.Decompiler( + self.func, cfg=self.cfg.model, save_unoptimized_graph=True, regen_clinic=True + ) + clinic = dec.clinic + assert clinic is not None and clinic.unoptimized_graph is not None + back = type(clinic).parse( + clinic.serialize(), + project=self.proj, + kb=self.proj.kb, + function=clinic.function, + cfg=clinic._cfg, + ) + assert back._save_unoptimized_graph is True + assert back.unoptimized_graph is not None + assert back.unoptimized_graph.number_of_nodes() == clinic.unoptimized_graph.number_of_nodes() + assert back.unoptimized_graph.number_of_edges() == clinic.unoptimized_graph.number_of_edges() + + def test_decompilation_cache_roundtrip(self): + cache = self.decompiler.cache + blob = cache.serialize() + back = DecompilationCache.parse( + blob, + project=self.proj, + kb=self.proj.kb, + function=self.func, + cfg=self.cfg.model, + ) + assert back.addr == cache.addr + assert back.errors == cache.errors + assert back.function_summary == cache.function_summary + assert back.codegen.text == cache.codegen.text + # version and timestamp are set at decompile time and round-trip verbatim + assert cache.version == angr.__version__ + assert cache.timestamp > 0 + assert back.version == cache.version + assert back.timestamp == cache.timestamp + # parameters preserves the 15 keys + assert set(back.parameters.keys()) == set(cache.parameters.keys()) + assert len(back.parameters) == 15 + + def test_cache_hit_on_deserialized_cache(self): + cache = self.decompiler.cache + blob = cache.serialize() + parsed_cache = DecompilationCache.parse( + blob, + project=self.proj, + kb=self.proj.kb, + function=self.func, + cfg=self.cfg.model, + ) + + # Replace the live cache with the parsed one. + flavor = parsed_cache.parameters.get("flavor", "pseudocode") + self.proj.kb.decompilations[(self.func.addr, flavor)] = parsed_cache + + # Second decompile run with the same inputs should consume the deserialized cache. + d2 = self.proj.analyses.Decompiler(self.func, cfg=self.cfg.model, generate_code=True) + assert d2.codegen.text == self.decompiler.codegen.text + + def test_full_reuse_fast_path(self): + # With use_cache=True and regen_clinic=False (both defaults), a valid cache short-circuits the pipeline and + # returns the cached clinic + codegen objects, re-rendered. + proj = angr.Project(os.path.join(test_location, "x86_64", "fauxware"), auto_load_libs=False) + cfg = proj.analyses.CFGFast(normalize=True) + func = proj.kb.functions.function(name="authenticate") + d1 = proj.analyses.Decompiler(func, cfg=cfg.model) + cache = proj.kb.decompilations[(func.addr, "pseudocode")] + + d2 = proj.analyses.Decompiler(func, cfg=cfg.model) + assert d2.codegen is d1.codegen + assert d2.clinic is d1.clinic + assert d2.codegen.text == d1.codegen.text + assert d2.codegen.version == cache.version == angr.__version__ + assert d2.codegen.timestamp == cache.timestamp > 0 + + def test_regen_clinic_forces_fresh_decompilation(self): + proj = angr.Project(os.path.join(test_location, "x86_64", "fauxware"), auto_load_libs=False) + cfg = proj.analyses.CFGFast(normalize=True) + func = proj.kb.functions.function(name="authenticate") + d1 = proj.analyses.Decompiler(func, cfg=cfg.model) + d2 = proj.analyses.Decompiler(func, cfg=cfg.model, regen_clinic=True) + assert d2.codegen is not d1.codegen + assert d2.codegen.text == d1.codegen.text + + +class TestSpillingDecompilationDict(unittest.TestCase): + """Tests for the LRU + RtDb-spilling backing store of StructuredCodeManager.""" + + @classmethod + def setUpClass(cls): + cls.proj = angr.Project(os.path.join(test_location, "x86_64", "fauxware"), auto_load_libs=False) + cls.cfg = cls.proj.analyses.CFGFast(normalize=True) + cls.auth_func = cls.proj.kb.functions.function(name="authenticate") + cls.main_func = cls.proj.kb.functions.function(name="main") + cls.auth_dec = cls.proj.analyses.Decompiler(cls.auth_func, cfg=cls.cfg.model, generate_code=True) + cls.main_dec = cls.proj.analyses.Decompiler(cls.main_func, cfg=cls.cfg.model, generate_code=True) + + def test_default_backing_store_is_spilling(self): + assert isinstance(self.proj.kb.decompilations.cached, SpillingDecompilationDict) + + def test_eviction_and_reload(self): + d = SpillingDecompilationDict(self.proj.kb, cache_limit=1) + auth_key = (self.auth_func.addr, "pseudocode") + main_key = (self.main_func.addr, "pseudocode") + d[auth_key] = self.auth_dec.cache + d[main_key] = self.main_dec.cache + + # the LRU (authenticate) entry must have been spilled + assert auth_key in d._spilled + assert len(d) == 2 + assert auth_key in d + assert main_key in d + assert set(d) == {auth_key, main_key} + + # reloading the spilled entry deserializes it with full codegen and version/timestamp + back = d[auth_key] + assert back is not self.auth_dec.cache + assert back.codegen.text == self.auth_dec.cache.codegen.text + assert back.version == self.auth_dec.cache.version + assert back.timestamp == self.auth_dec.cache.timestamp + # ... and the reload evicted the other entry in turn + assert main_key in d._spilled + + def test_mutations_survive_respill(self): + d = SpillingDecompilationDict(self.proj.kb, cache_limit=1) + auth_key = (self.auth_func.addr, "pseudocode") + main_key = (self.main_func.addr, "pseudocode") + d[auth_key] = self.auth_dec.cache + d[main_key] = self.main_dec.cache + + # reload authenticate (spills main), mutate it in place, then spill it again by touching main + d[auth_key].errors.append("synthetic error") + _ = d[main_key] + assert auth_key in d._spilled + assert "synthetic error" in d[auth_key].errors + + def test_unserializable_cache_is_kept_in_memory(self): + d = SpillingDecompilationDict(self.proj.kb, cache_limit=1) + dummy_key = (0xDEAD, "pseudocode") + dummy_cache = DecompilationCache(0xDEAD) + dummy_cache.codegen = DummyStructuredCodeGenerator("pseudocode") + d[dummy_key] = dummy_cache + + # inserting another entry evicts the dummy cache, which cannot be serialized and must be parked in memory + main_key = (self.main_func.addr, "pseudocode") + d[main_key] = self.main_dec.cache + assert dummy_key in d._unspillable + assert d[dummy_key] is dummy_cache + assert len(d) == 2 + + def test_delete_and_discard(self): + d = SpillingDecompilationDict(self.proj.kb, cache_limit=1) + auth_key = (self.auth_func.addr, "pseudocode") + main_key = (self.main_func.addr, "pseudocode") + d[auth_key] = self.auth_dec.cache + d[main_key] = self.main_dec.cache + + del d[auth_key] # spilled entry + del d[main_key] # in-memory entry + assert len(d) == 0 + assert auth_key not in d + with self.assertRaises(KeyError): + _ = d[auth_key] + + def test_export_and_bulk_import_serialized(self): + d = SpillingDecompilationDict(self.proj.kb, cache_limit=1) + auth_key = (self.auth_func.addr, "pseudocode") + main_key = (self.main_func.addr, "pseudocode") + d[auth_key] = self.auth_dec.cache + d[main_key] = self.main_dec.cache + + serialized, unserializable = d.export_serialized() + assert not unserializable + assert {key for key, _ in serialized} == {auth_key, main_key} + + d2 = SpillingDecompilationDict(self.proj.kb, cache_limit=1) + d2.bulk_import_serialized(serialized) + assert set(d2) == {auth_key, main_key} + assert d2._spilled == {auth_key, main_key} + assert d2[auth_key].codegen.text == self.auth_dec.cache.codegen.text + + def test_pickle_roundtrip(self): + d = SpillingDecompilationDict(self.proj.kb, cache_limit=1) + auth_key = (self.auth_func.addr, "pseudocode") + main_key = (self.main_func.addr, "pseudocode") + d[auth_key] = self.auth_dec.cache + d[main_key] = self.main_dec.cache + assert auth_key in d._spilled + + # serializable entries pickle as protobuf bytes: live caches hold unpicklable analysis internals + back = pickle.loads(pickle.dumps(d, -1)) + assert set(back) == {auth_key, main_key} + assert back._spilled == {auth_key, main_key} + assert back[auth_key].codegen.text == self.auth_dec.cache.codegen.text + + def test_cache_hit_after_spill(self): + manager = self.proj.kb.decompilations + old_cached = manager.cached + try: + d = SpillingDecompilationDict(self.proj.kb, cache_limit=1) + manager.cached = d + manager[(self.auth_func.addr, "pseudocode")] = self.auth_dec.cache + manager[(self.main_func.addr, "pseudocode")] = self.main_dec.cache + assert (self.auth_func.addr, "pseudocode") in d._spilled + + d2 = self.proj.analyses.Decompiler(self.auth_func, cfg=self.cfg.model, generate_code=True) + assert d2.codegen.text == self.auth_dec.codegen.text + finally: + manager.cached = old_cached + + +if __name__ == "__main__": + unittest.main() From 2f8266bddae8ea94bcea7eb1a05eb2d534919a21 Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 22 Jul 2026 03:55:07 -0700 Subject: [PATCH 030/122] CFGFast: Constant-fold AMD64 PE IAT calls. (#6667) * CFGFast: constant-fold AMD64 PE IAT jumps/calls, bypassing the resolver * CFGFast: fold delay-load IAT calls too, matching MemoryLoadResolver * tests: update lwip xrefs for corrected RO-region ordering --- angr/analyses/cfg/cfg_fast.py | 85 +++++++++++++++++-- .../indirect_jump_resolvers/amd64_pe_iat.py | 4 +- tests/analyses/test_xrefs.py | 8 +- 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/angr/analyses/cfg/cfg_fast.py b/angr/analyses/cfg/cfg_fast.py index efa326fb8..555063755 100644 --- a/angr/analyses/cfg/cfg_fast.py +++ b/angr/analyses/cfg/cfg_fast.py @@ -48,6 +48,7 @@ from angr.knowledge_plugins.cfg.spilling_cfg import block_key_to_addr, block_key from angr.knowledge_plugins.xrefs import XRef, XRefType from angr.misc.ux import once from angr.rustylib import SegmentList +from angr.simos import SimWindows from angr.utils.constants import DEFAULT_STATEMENT from angr.utils.funcid import ( is_function_likely_security_init_cookie, @@ -869,6 +870,10 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): # mapping to all known thunks self._known_thunks = {} + # when True, jump/call targets loaded from registered read-only regions (e.g. PE IAT slots) are + # constant-folded at lift time and consumed in _create_jobs without invoking indirect jump resolvers + self._fold_ro_const_loads = False + self._initial_state = None self._next_addr: int | None = None @@ -3133,6 +3138,34 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): return entries + def _resolve_const_folded_next(self, irsb: pyvex.IRSB | None, jumpkind: str) -> int | None: + """ + Check whether the jump/call target of this block was constant-folded from a registered read-only region + at lift time (recorded in IRSB.const_vals), e.g. an import call through the IAT (or delay-load IAT) in a PE + binary. This reproduces the decision of the timeless load-based resolvers (AMD64PeIatResolver and + MemoryLoadResolver) without re-lifting the block or dispatching the resolvers: the loaded pointer is the + same value they would read, and it is accepted when it is a valid jump target (executable or hooked), which + is exactly MemoryLoadResolver's ``_is_target_valid`` criterion (a superset of AMD64PeIatResolver's hooked + check). Everything else falls back to the regular indirect jump resolution logic. + + :param irsb: The (possibly statement-less) IRSB of the block. + :param jumpkind: The jumpkind of the default exit. + :return: The resolved target, or None if unavailable. + """ + if not self._fold_ro_const_loads or irsb is None: + return None + if jumpkind not in ("Ijk_Call", "Ijk_Boring"): + return None + if not irsb.const_vals or not isinstance(irsb.next, pyvex.IRExpr.RdTmp): + return None + next_tmp = irsb.next.tmp + for cv in irsb.const_vals: + if cv.tmp == next_tmp: + if self._addr_in_exec_memory_regions(cv.value) or self.project.is_hooked(cv.value): + return cv.value + return None + return None + def _create_jobs( self, target: Any, @@ -3218,14 +3251,24 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): jumpkind in ("Ijk_Boring", "Ijk_Call", "Ijk_InvalICache") or jumpkind.startswith("Ijk_Sys") ): # This is an indirect jump. Try to resolve it. - # FIXME: in some cases, a statementless irsb will be missing its instr addresses - # and this next part will fail. Use the real IRSB instead - irsb = self._lift(cfg_node.addr, size=cfg_node.size).vex - assert irsb is not None - cfg_node.instruction_addrs = InsAddrList.from_addr_list(irsb.instruction_addresses) - resolved, resolved_targets, ij = self._indirect_jump_encountered( - addr, cfg_node, irsb, current_function_addr, stmt_idx - ) + # fast path: the target may have been constant-folded from a read-only region at lift time + # (e.g. an AMD64 PE IAT slot); consuming it here avoids re-lifting the block and running the + # indirect jump resolvers + folded_target = self._resolve_const_folded_next(irsb, jumpkind) + if folded_target is not None: + # the statement-less irsb already carries the instruction addresses that the resolver path + # would recompute from a re-lift + cfg_node.instruction_addrs = InsAddrList.from_addr_list(irsb.instruction_addresses) + resolved, resolved_targets, ij = True, {folded_target}, None + else: + # FIXME: in some cases, a statementless irsb will be missing its instr addresses + # and this next part will fail. Use the real IRSB instead + irsb = self._lift(cfg_node.addr, size=cfg_node.size).vex + assert irsb is not None + cfg_node.instruction_addrs = InsAddrList.from_addr_list(irsb.instruction_addresses) + resolved, resolved_targets, ij = self._indirect_jump_encountered( + addr, cfg_node, irsb, current_function_addr, stmt_idx + ) if resolved: for resolved_target in resolved_targets: if jumpkind == "Ijk_Call": @@ -5206,6 +5249,30 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): self._ro_region_cdata_cache.append(content_buf) pyvex.pvc.register_readonly_region(section.vaddr, section.memsize, content_buf) + elif self.project.arch.name in {"AMD64", "X86"} and isinstance(self.project.simos, SimWindows): + # register sections that hold jump/call targets so that rip-relative import calls and jumps + # (call/jmp qword ptr [rip+disp]) can be constant-folded at lift time and resolved without a re-lift + # and resolver dispatch: + # - non-writable sections (e.g. .rdata, the bound IAT), and + # - the delay-load import table (.didat): although writable, its slots are static during CFG recovery + # and point to the delay-load thunks, which MemoryLoadResolver already resolves by reading them. + # The folded value is only accepted when it is a valid jump target, so registering these regions cannot + # introduce edges the timeless load resolvers would not also produce. + self._ro_region_cdata_cache = [] + for section in self.project.loader.main_object.sections: + register = (section.is_readable and not section.is_writable and section.memsize >= 8) or ( + section.name == ".didat" and section.is_readable and section.memsize >= 8 + ) + if register: + try: + content = self.project.loader.memory.load(section.vaddr, section.memsize) + except KeyError: + continue + content_buf = pyvex.ffi.from_buffer(content) + self._ro_region_cdata_cache.append(content_buf) + pyvex.pvc.register_readonly_region(section.vaddr, section.memsize, content_buf) + self._fold_ro_const_loads = bool(self._ro_region_cdata_cache) + def _lifter_deregister_readonly_regions(self): pyvex.pvc.deregister_all_readonly_regions() self._ro_region_cdata_cache = None @@ -5469,6 +5536,7 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): collect_data_refs=True, strict_block_end=True, load_from_ro_regions=True, + const_prop=self._fold_ro_const_loads, initial_regs=initial_regs, ) irsb = lifted_block.vex_nostmt # may raise SimTranslationError @@ -5513,6 +5581,7 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): collect_data_refs=True, strict_block_end=True, load_from_ro_regions=True, + const_prop=self._fold_ro_const_loads, initial_regs=initial_regs, ) irsb = lifted_block.vex_nostmt diff --git a/angr/analyses/cfg/indirect_jump_resolvers/amd64_pe_iat.py b/angr/analyses/cfg/indirect_jump_resolvers/amd64_pe_iat.py index d07e64995..4fbfb9682 100644 --- a/angr/analyses/cfg/indirect_jump_resolvers/amd64_pe_iat.py +++ b/angr/analyses/cfg/indirect_jump_resolvers/amd64_pe_iat.py @@ -25,7 +25,7 @@ class AMD64PeIatResolver(IndirectJumpResolver): if jumpkind not in {"Ijk_Call", "Ijk_Boring"}: return False - insns = self.project.factory.block(addr).capstone.insns + insns = self.project.factory.block(addr, size=block.size).capstone.insns if not insns: return False if not insns[-1].insn.operands: @@ -36,7 +36,7 @@ class AMD64PeIatResolver(IndirectJumpResolver): return bool(opnd.type == X86_OP_MEM and opnd.mem.disp and opnd.mem.base == X86_REG_RIP and opnd.mem.index == 0) def resolve(self, cfg, addr, func_addr, block, jumpkind, func_graph_complete: bool = True, **kwargs): # pylint:disable=unused-argument - call_insn = self.project.factory.block(addr).capstone.insns[-1].insn + call_insn = self.project.factory.block(addr, size=block.size).capstone.insns[-1].insn addr = (call_insn.disp + call_insn.address + call_insn.size) & 0xFFFF_FFFF_FFFF_FFFF target = cfg._fast_memory_load_pointer(addr) if target is None: diff --git a/tests/analyses/test_xrefs.py b/tests/analyses/test_xrefs.py index 21758e6e5..55c8b2503 100755 --- a/tests/analyses/test_xrefs.py +++ b/tests/analyses/test_xrefs.py @@ -23,11 +23,17 @@ class TestXrefs(unittest.TestCase): func = cfg.functions[0x23C9] state = p.factory.blank_state() + # The two pointer-loads (ldr rX, [pc, #imm]) at 0x23C9 and 0x241D materialize 0x1FFF36F4, and pyvex's + # intra-block constant folding then follows each into its subsequent load (ldr rX, [rY]) at 0x23CB and + # 0x241F, recording the folded base as a data reference too. Without propagation context these are all + # reported as Offset references; the read/write typing is recovered by the Propagator+XRefs pass below. timenow_xrefs = p.kb.xrefs.get_xrefs_by_dst(0x1FFF36F4) # the value in .bss - assert len(timenow_xrefs) == 2 + assert len(timenow_xrefs) == 4 assert timenow_xrefs == { XRef(ins_addr=0x23C9, dst=0x1FFF36F4, xref_type=XRefType.Offset), + XRef(ins_addr=0x23CB, dst=0x1FFF36F4, xref_type=XRefType.Offset), XRef(ins_addr=0x241D, dst=0x1FFF36F4, xref_type=XRefType.Offset), + XRef(ins_addr=0x241F, dst=0x1FFF36F4, xref_type=XRefType.Offset), } # kill existing xrefs From 2eadec31c60ce5b48439da373b3aa619195eee54 Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 22 Jul 2026 03:56:06 -0700 Subject: [PATCH 031/122] Decompiler: Tolerate peephole optimizations not importable at parse time. (#6668) * Decompiler: Tolerate peephole optimizations not importable at parse time. * Lint code. --- angr/analyses/decompiler/clinic.py | 36 +++++++++++++++++-- .../decompiler/decompilation_cache.py | 8 +++-- angr/analyses/decompiler/decompiler.py | 3 ++ .../decompiler/optimization_pass_registry.py | 11 +++--- .../test_decompilation_cache_serialization.py | 33 +++++++++++++++++ 5 files changed, 80 insertions(+), 11 deletions(-) diff --git a/angr/analyses/decompiler/clinic.py b/angr/analyses/decompiler/clinic.py index aa3a8c5d2..31764c138 100644 --- a/angr/analyses/decompiler/clinic.py +++ b/angr/analyses/decompiler/clinic.py @@ -313,6 +313,9 @@ class Clinic(Analysis, Serializable): self._sp_tracker_track_memory = sp_tracker_track_memory self._cfg: CFGModel | None = cfg self.peephole_optimizations = peephole_optimizations + # peephole-optimization names that could not be resolved at parse time (their defining module was not + # imported); resolve_peephole_optimizations() retries them + self.unresolvable_peephole_optimizations: list[str] = [] self._must_struct = must_struct self._reset_variable_names = reset_variable_names self._rewrite_ites_to_diamonds = rewrite_ites_to_diamonds @@ -4221,6 +4224,22 @@ class Clinic(Analysis, Serializable): .model ) + def resolve_peephole_optimizations(self) -> None: + """Retry resolving peephole-optimization names that were unresolvable at parse time (their defining module + may have been imported since). Resolved classes move into ``peephole_optimizations``; names that still do not + resolve stay in ``unresolvable_peephole_optimizations``.""" + if not self.unresolvable_peephole_optimizations: + return + assert self.peephole_optimizations is not None + still_unresolvable = [] + for name in self.unresolvable_peephole_optimizations: + cls_ = name_to_pass(name) + if cls_ is None: + still_unresolvable.append(name) + else: + self.peephole_optimizations.append(cls_) + self.unresolvable_peephole_optimizations = still_unresolvable + # ----------------------------------------------------------------------------------------------------------------- # Protobuf serialization. Conventions: # - Heavy sub-objects manage their own formats; AIL-typed slots use the typed messages from ail_types.proto. @@ -4331,10 +4350,12 @@ class Clinic(Analysis, Serializable): msg._end_stage = self._end_stage.value msg._skip_stages.extend(s.value for s in self._skip_stages) - # Pass class refs. peephole_optimizations=None means "use the default peephole set". + # Pass class refs. peephole_optimizations=None means "use the default peephole set". Names that are still + # unresolvable are re-serialized as-is so they are not lost across round-trips. msg.peephole_optimizations_use_default = self.peephole_optimizations is None if self.peephole_optimizations is not None: msg.peephole_optimizations.extend(pass_to_name(cls_) for cls_ in self.peephole_optimizations) + msg.peephole_optimizations.extend(self.unresolvable_peephole_optimizations) if self._typehoon_cls is not None: # _typehoon_cls is the Typehoon class itself (not a registered pass); store its fully-qualified name msg._typehoon_cls = ( @@ -4472,11 +4493,20 @@ class Clinic(Analysis, Serializable): clinic._end_stage = ClinicStage(msg._end_stage) clinic._skip_stages = tuple(ClinicStage(s) for s in msg._skip_stages) - # Pass class refs. peephole_optimizations=None means "use the default peephole set". + # Pass class refs. peephole_optimizations=None means "use the default peephole set". Names that cannot be + # resolved (their defining module is not imported) are kept in unresolvable_peephole_optimizations; + # resolve_peephole_optimizations() retries them. + clinic.unresolvable_peephole_optimizations = [] if msg.peephole_optimizations_use_default: clinic.peephole_optimizations = None else: - clinic.peephole_optimizations = [name_to_pass(n) for n in msg.peephole_optimizations] + clinic.peephole_optimizations = [] + for n in msg.peephole_optimizations: + cls_ = name_to_pass(n) + if cls_ is None: + clinic.unresolvable_peephole_optimizations.append(n) + else: + clinic.peephole_optimizations.append(cls_) if msg._typehoon_cls: # _typehoon_cls is the Typehoon class itself (not a registered pass). Resolve directly by FQN. module_name, _, cls_name = msg._typehoon_cls.rpartition(".") diff --git a/angr/analyses/decompiler/decompilation_cache.py b/angr/analyses/decompiler/decompilation_cache.py index 87b09d273..7fe58bc2d 100644 --- a/angr/analyses/decompiler/decompilation_cache.py +++ b/angr/analyses/decompiler/decompilation_cache.py @@ -139,9 +139,13 @@ def _parse_parameters(msg) -> dict: for e in msg.options if e.param in PARAM_TO_OPTION }, - "optimization_passes": [name_to_pass(n) for n in msg.optimization_passes], + # unresolvable pass names (defining module not imported) drop out; the resulting shorter list will not match + # the live decompiler's parameters, so the cache falls through to a fresh decompilation + "optimization_passes": [cls for n in msg.optimization_passes if (cls := name_to_pass(n)) is not None], "peephole_optimizations": ( - None if msg.peephole_optimizations_use_default else [name_to_pass(n) for n in msg.peephole_optimizations] + None + if msg.peephole_optimizations_use_default + else [cls for n in msg.peephole_optimizations if (cls := name_to_pass(n)) is not None] ), "expr_comments": dict(msg.expr_comments), "stmt_comments": dict(msg.stmt_comments), diff --git a/angr/analyses/decompiler/decompiler.py b/angr/analyses/decompiler/decompiler.py index d16ef2593..001814e4f 100644 --- a/angr/analyses/decompiler/decompiler.py +++ b/angr/analyses/decompiler/decompiler.py @@ -436,6 +436,9 @@ class Decompiler(Analysis): ) else: clinic = old_clinic + # the deserialized clinic may carry peephole-optimization names that were unresolvable at parse time + # (their defining module was not imported then); retry resolving before its passes run again + clinic.resolve_peephole_optimizations() # reuse the old, unaltered graph clinic.graph = clinic.cc_graph clinic.cc_graph = clinic.copy_graph() diff --git a/angr/analyses/decompiler/optimization_pass_registry.py b/angr/analyses/decompiler/optimization_pass_registry.py index 700181336..0f9810214 100644 --- a/angr/analyses/decompiler/optimization_pass_registry.py +++ b/angr/analyses/decompiler/optimization_pass_registry.py @@ -19,13 +19,12 @@ def pass_to_name(cls: type) -> str: return cls.__qualname__ -def name_to_pass(name: str) -> type: - """Resolve a class name back to its registered pass class. - - :raises KeyError: if ``name`` does not refer to a class in - ``ALL_OPTIMIZATION_PASSES`` or ``ALL_PEEPHOLE_OPTS``. +def name_to_pass(name: str) -> type | None: """ - return _known_passes()[name] + Resolve a class name back to its registered pass class. Returns None for names that are not registered at the time + this method is called. Peephole passes might be defined by analyses or plugins that have not yet been imported. + """ + return _known_passes().get(name) __all__ = ("name_to_pass", "pass_to_name") diff --git a/tests/serialization/test_decompilation_cache_serialization.py b/tests/serialization/test_decompilation_cache_serialization.py index 9e193752c..269c83775 100644 --- a/tests/serialization/test_decompilation_cache_serialization.py +++ b/tests/serialization/test_decompilation_cache_serialization.py @@ -17,6 +17,7 @@ from angr.ailment.expression import Const from angr.ailment.expression import Tmp as AilTmp from angr.ailment.expression import VirtualVariable as AilVirtualVariable from angr.ailment.statement import Assignment, Return +from angr.analyses.decompiler import optimization_pass_registry from angr.analyses.decompiler.decompilation_cache import DecompilationCache from angr.analyses.decompiler.notes.decompilation_note import ( DecompilationNote, @@ -25,6 +26,7 @@ from angr.analyses.decompiler.notes.decompilation_note import ( from angr.analyses.decompiler.notes.deobfuscated_strings import DeobfuscatedStringsNote from angr.analyses.decompiler.optimization_passes.expr_op_swapper import OpDescriptor from angr.analyses.decompiler.optimization_passes.static_vvar_rewriter import FixedBuffer, FixedBufferPtr +from angr.analyses.decompiler.peephole_optimizations import EXPR_OPTS from angr.analyses.decompiler.structured_codegen import DummyStructuredCodeGenerator from angr.analyses.decompiler.structured_codegen.c import CConstruct from angr.analyses.decompiler.structured_codegen.c_serialize import ( @@ -293,6 +295,37 @@ class TestDecompilationCacheEndToEnd(unittest.TestCase): assert back.unoptimized_graph.number_of_nodes() == clinic.unoptimized_graph.number_of_nodes() assert back.unoptimized_graph.number_of_edges() == clinic.unoptimized_graph.number_of_edges() + def test_clinic_unresolvable_peephole_optimizations_roundtrip(self): + # name_to_pass returns None (not raising) for names that are not registered + assert optimization_pass_registry.name_to_pass("NotARegisteredPass") is None + + dec = self.proj.analyses.Decompiler( + self.func, cfg=self.cfg.model, peephole_optimizations=list(EXPR_OPTS[:2]), regen_clinic=True + ) + clinic = dec.clinic + # simulate a peephole pass defined by an analysis/plugin that is not imported + clinic.unresolvable_peephole_optimizations = ["PluginOnlyPeephole"] + + back = type(clinic).parse( + clinic.serialize(), project=self.proj, kb=self.proj.kb, function=clinic.function, cfg=clinic._cfg + ) + # the resolvable ones come back as classes; the unknown name is preserved, not dropped or crashed on + assert len(back.peephole_optimizations) == 2 + assert back.unresolvable_peephole_optimizations == ["PluginOnlyPeephole"] + + # once the defining module is imported (mocked here), a retry resolves it + class PluginOnlyPeephole: + __qualname__ = "PluginOnlyPeephole" + + original = optimization_pass_registry._known_passes + optimization_pass_registry._known_passes = lambda: {**original(), "PluginOnlyPeephole": PluginOnlyPeephole} + try: + back.resolve_peephole_optimizations() + finally: + optimization_pass_registry._known_passes = original + assert back.unresolvable_peephole_optimizations == [] + assert PluginOnlyPeephole in back.peephole_optimizations + def test_decompilation_cache_roundtrip(self): cache = self.decompiler.cache blob = cache.serialize() From 8608d976e3ef7ad9a1e93056fdef99462adbf0ee Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 22 Jul 2026 05:01:09 -0700 Subject: [PATCH 032/122] Decompiler: Fix re-rendering of decompilation caches reloaded from angrDb. (#6669) * Decompiler: Fix re-rendering of decompilation caches reloaded from angrdb. Re-rendering a deserialized codegen (as angr-management does on display/edit) dropped variable declarations and string constants and rendered slightly different C, because several pieces of state were not restored: - VariableManagerInternal never serialized variable_to_types / variables_with_ manual_types, so get_variable_type() returned None and all locals rendered as int. - parse_codegen did not attach the project, and left display options that serialize as None (e.g. max_str_len) unset. Attach project and initialize display options from the codegen constructor defaults. - CConstant string references lost MemoryData.content (not serialized); re-read it from the loader at parse time so strings render as strings, not raw addresses. - regenerate_text() now refreshes CFunction.unified_local_vars from the (restored or updated) variable manager so declarations reflect current types. - CBinaryOp._cstyle_null_cmp is rebuilt from the codegen flag in set_codegen, restoring !x vs x == 0. - Compound-assignment folding (x += 1) compared unified variables by identity; use == so it works across deserialized variables that are equal but not the same object. Adds an end-to-end test (1after909::doit) asserting a reloaded cache re-renders byte-identically. * Lint code. --- .../decompiler/structured_codegen/c.py | 7 +++- .../structured_codegen/c_serialize.py | 30 ++++++++++++-- .../variables/variable_manager.py | 40 +++++++++++++++++- angr/protos/variables.proto | 5 ++- tests/serialization/test_db.py | 41 +++++++++++++++++++ 5 files changed, 115 insertions(+), 8 deletions(-) diff --git a/angr/analyses/decompiler/structured_codegen/c.py b/angr/analyses/decompiler/structured_codegen/c.py index c8203ec8d..6d1fda7ba 100644 --- a/angr/analyses/decompiler/structured_codegen/c.py +++ b/angr/analyses/decompiler/structured_codegen/c.py @@ -1446,12 +1446,12 @@ class CAssignment(CStatement): and self.rhs.op in compound_assignment_ops and self.lhs.unified_variable is not None ): - if isinstance(self.rhs.lhs, CVariable) and self.lhs.unified_variable is self.rhs.lhs.unified_variable: + if isinstance(self.rhs.lhs, CVariable) and self.lhs.unified_variable == self.rhs.lhs.unified_variable: compound_expr_rhs = self.rhs.rhs elif ( self.rhs.op in commutative_ops and isinstance(self.rhs.rhs, CVariable) - and self.lhs.unified_variable is self.rhs.rhs.unified_variable + and self.lhs.unified_variable == self.rhs.rhs.unified_variable ): compound_expr_rhs = self.rhs.lhs @@ -2995,6 +2995,9 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis, Serializab """ if self.cfunc is None: return + # recompute the unified local variables and their types from the (possibly updated or freshly deserialized) + # variable manager, so re-rendering reflects the current variable types + self.cfunc.refresh() self.cleanup() ( self.text, diff --git a/angr/analyses/decompiler/structured_codegen/c_serialize.py b/angr/analyses/decompiler/structured_codegen/c_serialize.py index d7f316ee7..8fe853834 100644 --- a/angr/analyses/decompiler/structured_codegen/c_serialize.py +++ b/angr/analyses/decompiler/structured_codegen/c_serialize.py @@ -10,6 +10,7 @@ reference into the AST without duplicating subtrees. from __future__ import annotations +import inspect import json import zlib from collections import defaultdict @@ -352,6 +353,9 @@ class ParseContext: def set_codegen(self, codegen) -> None: for node in self._parsed.values(): node.codegen = codegen + # CBinaryOp caches this display flag per-instance at __init__ (bypassed by parse); rebuild it here + if isinstance(node, CBinaryOp): + node._cstyle_null_cmp = codegen.cstyle_null_cmp # @@ -506,6 +510,16 @@ _DISPLAY_OPTION_FIELD_FIRST = codegen_pb2.Codegen.DESCRIPTOR.fields_by_name["ind _DISPLAY_OPTION_ATTRS = tuple( f.name for f in codegen_pb2.Codegen.DESCRIPTOR.fields if f.number >= _DISPLAY_OPTION_FIELD_FIRST ) +# Constructor defaults for the display options, so a deserialized codegen has the same values a fresh one would for +# options that were not serialized (an option whose value is None, e.g. max_str_len, is skipped on serialize). +_CODEGEN_CTOR_DEFAULTS = { + name: param.default + for name, param in inspect.signature(CStructuredCodeGenerator.__init__).parameters.items() + if param.default is not inspect.Parameter.empty +} +_DISPLAY_OPTION_DEFAULTS = { + attr: _CODEGEN_CTOR_DEFAULTS[attr] for attr in _DISPLAY_OPTION_ATTRS if attr in _CODEGEN_CTOR_DEFAULTS +} def serialize_codegen(codegen) -> codegen_pb2.Codegen: @@ -626,12 +640,17 @@ def parse_codegen(msg, *, project=None, kb=None, func=None): cg.cexterns = {ctx.resolve(i) for i in msg.cexterns_ids} if msg.cexterns_ids else None - # Display options: only set those present in the cmessage. + # Display options: those present in the cmessage override the constructor defaults; options that were not + # serialized (a None value, e.g. max_str_len) fall back to the constructor default so the attribute exists. for attr in _DISPLAY_OPTION_ATTRS: + cg_attr = "_indent" if attr == "indent" else attr if msg.HasField(attr): - setattr(cg, "_indent" if attr == "indent" else attr, getattr(msg, attr)) + setattr(cg, cg_attr, getattr(msg, attr)) + elif attr in _DISPLAY_OPTION_DEFAULTS: + setattr(cg, cg_attr, _DISPLAY_OPTION_DEFAULTS[attr]) # Runtime / back-reference state — caller-provided. + cg.project = project cg._func = func cg._func_args = None cg._cfg = None @@ -1151,7 +1170,12 @@ def _parse_cconst(pb, ctx): elif w == "str_value": refs[key] = entry.str_value elif w == "memory_data": - refs[key] = MemoryData.parse(entry.memory_data) + md = MemoryData.parse(entry.memory_data) + # MemoryData.content (e.g. the string bytes) is not serialized; re-read it from the loader so string + # references render as strings rather than raw addresses + if md.content is None and ctx.project is not None: + md.fill_content(ctx.project.loader) + refs[key] = md obj.reference_values = refs else: obj.reference_values = None diff --git a/angr/knowledge_plugins/variables/variable_manager.py b/angr/knowledge_plugins/variables/variable_manager.py index c09adc45c..09c5b6541 100644 --- a/angr/knowledge_plugins/variables/variable_manager.py +++ b/angr/knowledge_plugins/variables/variable_manager.py @@ -1,5 +1,7 @@ +# pylint:disable=protected-access from __future__ import annotations +import json import logging from collections import defaultdict from collections.abc import Iterator @@ -287,7 +289,27 @@ class VariableManagerInternal(Serializable): phi_relations.append(relation) cmsg.phi2var.extend(phi_relations) - # TODO: Types + # Types: variable_to_types (SimVariable ident -> SimType), with the manual-type flag. SimType JSON strings + # are interned into a shared type pool since many variables share the same type. + type_pool: list[str] = [] + type_ref_by_json: dict[str, int] = {} + type_entries = [] + for var, var_type in self.variable_to_types.items(): + if var.ident is None: + continue + type_json = json.dumps(var_type.to_json()) + ref = type_ref_by_json.get(type_json) + if ref is None: + type_pool.append(type_json) + ref = len(type_pool) # index+1 + type_ref_by_json[type_json] = ref + entry = variables_pb2.VariableType() # type: ignore[reportAttributeAccessIssue] + entry.ident = var.ident + entry.type_ref = ref + entry.manual = var in self.variables_with_manual_types + type_entries.append(entry) + cmsg.types.extend(type_entries) + cmsg.type_pool.extend(type_pool) # TODO: vvarid_to_varialbes & variable_to_vvarids @@ -404,7 +426,21 @@ class VariableManagerInternal(Serializable): model._phi_variables[phi].add(var) model._variables_to_phivars[var].add(phi) - # TODO: Types + # Types: variable_to_types (keyed by both regular and unified variables) + variables_with_manual_types. + # Each pooled type JSON is parsed once and shared across the variables that reference it. + arch = model.manager._kb._project.arch if model.manager is not None else None + type_by_ref: dict[int, SimType] = {} + for ref, type_json in enumerate(cmsg.type_pool, start=1): + var_type = SimType.from_json(json.loads(type_json)) + type_by_ref[ref] = var_type.with_arch(arch) if arch is not None else var_type + for type_pb2 in cmsg.types: + var = variable_by_ident.get(type_pb2.ident) or unified_variable_by_ident.get(type_pb2.ident) + var_type = type_by_ref.get(type_pb2.type_ref) + if var is None or var_type is None: + continue + model.variable_to_types[var] = var_type + if type_pb2.manual: + model.variables_with_manual_types.add(var) for var in model._variables: if isinstance(var, SimStackVariable): diff --git a/angr/protos/variables.proto b/angr/protos/variables.proto index ff5a7cfa5..1bde11bd0 100644 --- a/angr/protos/variables.proto +++ b/angr/protos/variables.proto @@ -70,7 +70,8 @@ message VariableAccess { message VariableType { string ident = 1; - string var_type = 2; // FIXME: Use a better solution than a string! + uint32 type_ref = 2; // index+1 into VariableManagerInternal.type_pool + bool manual = 3; // whether the type was set manually (variables_with_manual_types) } message Var2Unified { @@ -101,6 +102,8 @@ message VariableManagerInternal { repeated Var2Unified var2unified = 10; // Types repeated VariableType types = 11; + // Interned SimType JSON strings (json.dumps(SimType.to_json())); VariableType.type_ref is index+1 into this pool + repeated string type_pool = 14; // Phi variables repeated Phi2Var phi2var = 12; } diff --git a/tests/serialization/test_db.py b/tests/serialization/test_db.py index 8b9ecdf08..46c3ede83 100755 --- a/tests/serialization/test_db.py +++ b/tests/serialization/test_db.py @@ -817,6 +817,47 @@ class TestDb(unittest.TestCase): _proj = AngrDB(nullpool=True).load(out_db) + def test_angrdb_reloaded_decompilation_rerenders_identically(self): + # Full workflow: load a binary, decompile a function (populating kb.dec_variables), spill the decompilation + # and dec_variables into angrdb, reload, then re-render the cached codegen. The re-rendered output must be + # byte-identical to the original — variable declarations (types) and string constants included. + bin_path = os.path.join(test_location, "x86_64", "1after909") + + with tempfile.TemporaryDirectory() as td: + db_file = os.path.join(td, "1after909.adb") + + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(normalize=True) + proj.analyses.CompleteCallingConventions(recover_variables=False) + func = proj.kb.functions.function(name="doit") + dec = proj.analyses.Decompiler(func, cfg=cfg.model) + assert dec.codegen is not None and dec.codegen.text is not None + original_text = dec.codegen.text + # sanity: the original has variable declarations and rendered strings + assert "int node;" in original_text + assert 'puts("1 AFTER 909:' in original_text or "1 AFTER 909" in original_text + + AngrDB(proj, nullpool=True).dump(db_file) + reloaded = AngrDB(nullpool=True).load(db_file) + func2 = reloaded.kb.functions.function(name="doit") + + # dec_variables (with types) round-tripped + assert func2.addr in reloaded.kb.dec_variables + assert reloaded.kb.dec_variables[func2.addr].variable_to_types + + # the cached codegen re-renders identically (what angr-management does on display/edit) + cached = reloaded.kb.decompilations[(func2.addr, "pseudocode")] + assert cached.codegen is not None + assert cached.codegen.text == original_text # stored text + cached.codegen.regenerate_text() + assert cached.codegen.text == original_text # re-rendered text + + # and going through the Decompiler again yields the same, re-renderable, result + dec2 = reloaded.analyses.Decompiler(func2, cfg=reloaded.kb.cfgs.get_most_accurate()) + assert dec2.codegen is not None + dec2.codegen.regenerate_text() + assert dec2.codegen.text == original_text + def test_angrdb_blob_loader_options_roundtrip(self): with tempfile.TemporaryDirectory() as td: blob_path = os.path.join(td, "sample.bin") From 8dc37727625197220d2bb9e35142c8a5b9bae2e6 Mon Sep 17 00:00:00 2001 From: Fish Date: Thu, 23 Jul 2026 14:16:54 -0700 Subject: [PATCH 033/122] tests: CFG resolves delay-load import calls to named imports. (#6666) --- tests/analyses/cfg/test_cfg_delay_import.py | 50 +++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/analyses/cfg/test_cfg_delay_import.py diff --git a/tests/analyses/cfg/test_cfg_delay_import.py b/tests/analyses/cfg/test_cfg_delay_import.py new file mode 100644 index 000000000..45243e18e --- /dev/null +++ b/tests/analyses/cfg/test_cfg_delay_import.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use + +"""Test that CFGFast resolves delay-load import calls to the named import. + +The fixture ``delay_import.exe`` delay-loads ``user32.dll!MessageBoxA``. CLE binds the delay-import slot to an extern +stub (see cle PE backend ``_handle_delay_imports``), so the ``call``/``jmp`` through the delay IAT should resolve in +the CFG to the ``MessageBoxA`` SimProcedure rather than being left unresolved or pointing at the delay-load thunk. +""" + +from __future__ import annotations + +import os +import unittest + +import angr +from tests.common import bin_location + +TEST_BINARY = os.path.join(bin_location, "tests", "x86_64", "windows", "delay_import.exe") + + +class TestCFGDelayImport(unittest.TestCase): + def test_delay_import_resolved_after_cfg(self): + proj = angr.Project(TEST_BINARY, auto_load_libs=False) + cfg = proj.analyses.CFGFast(normalize=True) + + # the delay-imported symbol is bound to a hooked extern stub + sym = proj.loader.find_symbol("MessageBoxA") + self.assertIsNotNone(sym) + self.assertTrue(proj.is_hooked(sym.rebased_addr)) + + # ... and recovered as a function during CFG recovery + self.assertTrue(proj.kb.functions.contains_addr(sym.rebased_addr)) + func = proj.kb.functions.get_by_addr(sym.rebased_addr) + self.assertEqual(func.name, "MessageBoxA") + self.assertTrue(func.is_simprocedure) + + # the delay-load call site resolved to it: MessageBoxA has at least one caller in the call graph, and its CFG + # node has predecessors (the indirect call through the delay IAT was resolved, not dropped as unresolvable) + callgraph = proj.kb.functions.callgraph + self.assertIn(sym.rebased_addr, callgraph) + self.assertGreater(callgraph.in_degree(sym.rebased_addr), 0) + + node = cfg.model.get_any_node(sym.rebased_addr) + self.assertIsNotNone(node) + self.assertGreater(len(cfg.model.get_predecessors(node)), 0) + + +if __name__ == "__main__": + unittest.main() From 0849ddb03c695b4bcead424664e61181bfbec912 Mon Sep 17 00:00:00 2001 From: Fish Date: Thu, 23 Jul 2026 14:43:32 -0700 Subject: [PATCH 034/122] CFGFast: Tolerate leading null bytes during string scanning. (#6670) * CFGFast: Tolerate a single leading null byte when scanning for strings. * CFGFast: Scan for mixed pointers in high-based images during complete scanning. * Tests: Add a regression test for data detection in a PE32 with data tables in .text. --- angr/analyses/cfg/cfg_fast.py | 41 +++++++++++++++------ tests/analyses/cfg/test_cfgfast_datarefs.py | 27 ++++++++++++++ 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/angr/analyses/cfg/cfg_fast.py b/angr/analyses/cfg/cfg_fast.py index 555063755..08c92c769 100644 --- a/angr/analyses/cfg/cfg_fast.py +++ b/angr/analyses/cfg/cfg_fast.py @@ -1348,11 +1348,19 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): ) start_addr += pointer_length - elif start_addr <= 0x100000: - # for high addresses, all pointers have been found in _scan_for_consecutive_pointers() because we - # set threshold there to 1 - threshold = 4 - pointer_count = self._scan_for_mixed_pointers(start_addr, threshold=threshold, window=6) + else: + if start_addr <= 0x100000: + # for low addresses, in-object values are common false positives, so + # _scan_for_consecutive_pointers() ran with a high threshold and may have missed + # non-consecutive pointers; require a high pointer density here + threshold, window = 4, 6 + else: + # for high addresses, all consecutive pointers have been found in + # _scan_for_consecutive_pointers() because we set threshold there to 1. what remains are + # interleaved tables (e.g., alternating value-pointer pairs), which have at most window // 2 + # pointers; use a wider window with the same evidence requirement + threshold, window = 4, 8 + pointer_count = self._scan_for_mixed_pointers(start_addr, threshold=threshold, window=window) pointer_length = pointer_count * self.project.arch.bytes if pointer_length: @@ -1364,24 +1372,33 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): start_addr += pointer_length if not matched_something: - # find strings + # find strings; tolerate a single leading null byte, which is usually the leftover of a multi-null + # string separator (string scans only consume one null terminator of the preceding string, and a + # single remaining null byte is not caught by the repeating-zero scan below) + leading_nulls = 1 if self._load_a_byte_as_int(start_addr) == 0 else 0 + str_addr = start_addr + leading_nulls is_widestring = False - string_length = self._scan_for_printable_strings(start_addr) + string_length = self._scan_for_printable_strings(str_addr) if string_length == 0: is_widestring = True - string_length = self._scan_for_printable_widestrings(start_addr) + string_length = self._scan_for_printable_widestrings(str_addr) if string_length: matched_something = True - self._seg_list.occupy(start_addr, string_length, "string") + if leading_nulls: + self._seg_list.occupy(start_addr, leading_nulls, "alignment") + self.model.memory_data[start_addr] = MemoryData( + start_addr, leading_nulls, MemoryDataSort.Alignment + ) + self._seg_list.occupy(str_addr, string_length, "string") md = MemoryData( - start_addr, + str_addr, string_length, MemoryDataSort.String if not is_widestring else MemoryDataSort.UnicodeString, ) md.fill_content(self.project.loader) - self.model.memory_data[start_addr] = md - start_addr += string_length + self.model.memory_data[str_addr] = md + start_addr = str_addr + string_length if not matched_something and self.project.arch.name in {"X86", "AMD64"}: cc_length = self._scan_for_repeating_bytes(start_addr, 0xCC, threshold=1) diff --git a/tests/analyses/cfg/test_cfgfast_datarefs.py b/tests/analyses/cfg/test_cfgfast_datarefs.py index 21e5f586e..91c10f0a2 100644 --- a/tests/analyses/cfg/test_cfgfast_datarefs.py +++ b/tests/analyses/cfg/test_cfgfast_datarefs.py @@ -178,6 +178,33 @@ class TestCfgfastDataReferences(unittest.TestCase): assert cfg_model.memory_data[0x10001004].size == 228 assert cfg_model.memory_data[0x10001004].sort == MemoryDataSort.PointerArray + def test_pe_32bit_string_tables_in_text_section(self): + # this binary carries read-only data (string tables, pointer tables, etc.) at the beginning of its .text + # section. its string tables separate strings with two null bytes; the string scans only consume one + # terminating null byte, and the single remaining null byte used to defeat all data heuristics in + # _next_code_addr_core(), causing entire string tables to be misidentified as code (and functions). + path = os.path.join( + test_location, "i386", "windows", "9888704382abfb694984c1c7a7707a45b4ebc406fc98d35622461077553aa797" + ) + proj = angr.Project(path, auto_load_libs=False) + + cfg = proj.analyses.CFGFast() + memory_data = cfg.model.memory_data + + # ASCII string tables: strings must be recovered and no functions created among them + assert memory_data[0x4019B0].sort == MemoryDataSort.String + assert memory_data[0x4019B0].content == b"address not available" + assert memory_data[0x402384].sort == MemoryDataSort.String + assert memory_data[0x402384].content == b"__vectorcall" + assert not [f for f in cfg.kb.functions if 0x4019A0 <= f < 0x402700] + + # UTF-16 string tables: same failure mode, with the leftover null byte also breaking the scan phase + assert cfg._seg_list.occupied_by_sort(0x405F12) == "unicode" + assert not [f for f in cfg.kb.functions if 0x405E00 <= f < 0x406100] + + # interleaved (value, pointer) tables are detected by the mixed-pointer scan + assert memory_data[0x401FBC].sort == MemoryDataSort.PointerArray + def test_long_printable_ascii_string_without_null_byte(self): # suboptimal logic in _scan_for_printable_strings was causing the CFG recovery of this binary to be extremely # slow; we were repeatedly trying (and failing) to build a super long ASCII string in this binary. From 85d77f0301340005f6f731b207fe4a6d849d18d9 Mon Sep 17 00:00:00 2001 From: Fish Date: Thu, 23 Jul 2026 14:43:45 -0700 Subject: [PATCH 035/122] RemoveRedundantShifts: Retain sign extensions. (#6671) `(x << N) >> N` was rewritten into a Convert-of-Convert pair whose outer Convert zero-extended for BOTH logical (Shr) and arithmetic (Sar) right shifts. For Sar this is unsound: the idiom sign-extends the low (M-N) bits, but the zero-extending Convert rendered as a bitmask, so e.g. `(int)(x << 20) >> 20` decompiled to `x & 0xfff` (and the 64-bit twin to `x & 0xffffffffff`), which drops the sign bit. --- .../remove_redundant_shifts.py | 11 ++++-- .../decompiler/structured_codegen/c.py | 28 +++++++++++++++ .../decompiler/test_peephole_optimizations.py | 34 +++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/angr/analyses/decompiler/peephole_optimizations/remove_redundant_shifts.py b/angr/analyses/decompiler/peephole_optimizations/remove_redundant_shifts.py index d4683e9f5..3bd889323 100644 --- a/angr/analyses/decompiler/peephole_optimizations/remove_redundant_shifts.py +++ b/angr/analyses/decompiler/peephole_optimizations/remove_redundant_shifts.py @@ -19,12 +19,19 @@ class RemoveRedundantShifts(PeepholeOptimizationExprBase): def optimize(self, expr: BinaryOp, **kwargs): # (expr << N) >> N ==> Convert((M-N)->M, Convert(M->(M-N), expr)) + # + # For a *logical* right shift (Shr) the outer conversion zero-extends, which the C backend renders as a + # bitmask (e.g. `& 0xfff`) that is correct for any width. For an *arithmetic* right shift (Sar) the outer + # conversion must sign-extend the low (M-N) bits; that is only rendered faithfully by the C backend when + # (M-N) is a standard integer width (8/16/32/64). For non-standard widths we leave the Sar/Shl pair intact + # (which renders as a signed `(expr << N) >> N`) rather than emit a bogus zero-extend mask that silently + # drops the sign bit. if expr.op in ("Shr", "Sar") and isinstance(expr.operands[1], Const): expr_a = expr.operands[0] n0 = expr.operands[1].value if isinstance(expr_a, BinaryOp) and expr_a.op in {"Shl", "Mul"} and isinstance(expr_a.operands[1], Const): n1 = get_expr_shift_left_amount(expr_a) - if n0 == n1: + if n0 == n1 and (expr.op == "Shr" or (expr_a.bits - n0) in (8, 16, 32, 64)): inner_expr = expr_a.operands[0] conv_inner_expr = Convert( self.manager.next_atom(), @@ -38,7 +45,7 @@ class RemoveRedundantShifts(PeepholeOptimizationExprBase): self.manager.next_atom(), expr_a.bits - n0, expr.bits, - False, + expr.op == "Sar", # sign-extend for arithmetic shift, zero-extend for logical shift conv_inner_expr, **expr.tags, ) diff --git a/angr/analyses/decompiler/structured_codegen/c.py b/angr/analyses/decompiler/structured_codegen/c.py index 6d1fda7ba..e83e87e47 100644 --- a/angr/analyses/decompiler/structured_codegen/c.py +++ b/angr/analyses/decompiler/structured_codegen/c.py @@ -2271,6 +2271,34 @@ class CBinaryOp(CExpression): yield from self._c_repr_chunks(" << ") def _c_repr_chunks_sar(self): + # Sar is an arithmetic (signed) right shift, but it renders as the C `>>` operator, which only performs an + # arithmetic shift when its left operand is signed. If the left operand renders as an unsigned integer, emit + # an explicit signed cast; otherwise `>>` would be a logical shift and silently drop the sign bit. The cast is + # emitted here at render time because the earlier typecast-collapsing passes treat same-size signed/unsigned + # integer casts as redundant and would strip a cast added during code generation. + lhs_ty = self.lhs.type + if ( + isinstance(lhs_ty, (SimTypeInt, SimTypeChar, SimTypeNum)) + and getattr(lhs_ty, "signed", None) is False + and lhs_ty.size is not None + ): + signed_ty = self.codegen.default_simtype_from_bits(lhs_ty.size, signed=True) + paren = CClosingObject("(") + yield "(", paren + yield f"{signed_ty.c_repr(name=None)}", signed_ty + yield ")", paren + yield "(", paren + yield from self._try_c_repr_chunks(self.lhs) + yield ")", paren + yield " >> ", self + if isinstance(self.rhs, CBinaryOp) and self.op_precedence > self.rhs.op_precedence: + paren2 = CClosingObject("(") + yield "(", paren2 + yield from self._try_c_repr_chunks(self.rhs) + yield ")", paren2 + else: + yield from self._try_c_repr_chunks(self.rhs) + return yield from self._c_repr_chunks(" >> ") def _c_repr_chunks_logicaland(self): diff --git a/tests/analyses/decompiler/test_peephole_optimizations.py b/tests/analyses/decompiler/test_peephole_optimizations.py index 33e86cce2..dd40eb80f 100755 --- a/tests/analyses/decompiler/test_peephole_optimizations.py +++ b/tests/analyses/decompiler/test_peephole_optimizations.py @@ -21,6 +21,7 @@ from angr.analyses.decompiler.peephole_optimizations import ( ConstantDereferences, EagerEvaluation, OptimizedDivisionSimplifier, + RemoveRedundantShifts, SimplifyBitwiseInserts, ) from angr.analyses.decompiler.utils import peephole_optimize_expr @@ -234,6 +235,39 @@ class TestPeepholeOptimizations(unittest.TestCase): divisor_operand = r.operand.operands[1] assert isinstance(divisor_operand, Const) and divisor_operand.value == divisor + def test_remove_redundant_shifts_preserves_sign_extension(self): + # (x << N) Sar N is a sign-extension of the low (bits - N) bits and must NOT be turned into a + # zero-extending bitmask. Regression test for the simplifier dropping the sign bit, decompiling + # `(int)(x << 20) >> 20` (sign-extend the low 12 bits) into the wrong `x & 0xfff` (zero-extend). + mgr = Manager(arch=archinfo.arch_from_id("AMD64")) + + # Arithmetic shift, standard resulting width (32 - 16 = 16): sign-extend via a *signed* outer Convert. + x = Register(mgr.next_atom(), 16, 32) + shl = BinaryOp(mgr.next_atom(), "Shl", [x, Const(mgr.next_atom(), 16, 8)], False, bits=32) + sar = BinaryOp(mgr.next_atom(), "Sar", [shl, Const(mgr.next_atom(), 16, 8)], True, bits=32) + r = RemoveRedundantShifts(None, None, mgr).optimize(sar) + assert isinstance(r, Convert) + assert r.from_bits == 16 and r.to_bits == 32 + assert r.is_signed is True # the outer conversion MUST sign-extend (not zero-extend) + assert isinstance(r.operand, Convert) and r.operand.from_bits == 32 and r.operand.to_bits == 16 + + # Arithmetic shift, non-standard resulting width (32 - 20 = 12): leave the Sar/Shl pair intact rather + # than emit a zero-extend mask that would silently drop the sign bit (12 bits has no clean C type). + x = Register(mgr.next_atom(), 16, 32) + shl = BinaryOp(mgr.next_atom(), "Shl", [x, Const(mgr.next_atom(), 20, 8)], False, bits=32) + sar = BinaryOp(mgr.next_atom(), "Sar", [shl, Const(mgr.next_atom(), 20, 8)], True, bits=32) + r = RemoveRedundantShifts(None, None, mgr).optimize(sar) + assert r is None + + # Logical shift (Shr): the low-bit *zero*-extension is correct for any width -> unsigned outer Convert. + x = Register(mgr.next_atom(), 16, 32) + shl = BinaryOp(mgr.next_atom(), "Shl", [x, Const(mgr.next_atom(), 20, 8)], False, bits=32) + shr = BinaryOp(mgr.next_atom(), "Shr", [shl, Const(mgr.next_atom(), 20, 8)], False, bits=32) + r = RemoveRedundantShifts(None, None, mgr).optimize(shr) + assert isinstance(r, Convert) + assert r.from_bits == 12 and r.to_bits == 32 + assert r.is_signed is False # zero-extend + def test_bswap32_intrinsic_name(self): proj = angr.load_shellcode(b"\x90", "AMD64") manager = Manager() From 75500dd271b4ad220037a6990079aad365994e53 Mon Sep 17 00:00:00 2001 From: Fish Date: Thu, 23 Jul 2026 15:57:26 -0700 Subject: [PATCH 036/122] VRA: Copy over integer signedness to type constraints during Convert. (#6672) * VRA: Copy over integer signedness to type constraints during Convert. * Adjust a test case. --- angr/analyses/variable_recovery/engine_ail.py | 10 ++++++ tests/analyses/decompiler/test_decompiler.py | 34 ++++++++++++++++--- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/angr/analyses/variable_recovery/engine_ail.py b/angr/analyses/variable_recovery/engine_ail.py index 561cbca40..50141099a 100644 --- a/angr/analyses/variable_recovery/engine_ail.py +++ b/angr/analyses/variable_recovery/engine_ail.py @@ -650,6 +650,16 @@ class SimEngineVRAIL( r.typevar, label=typevars.ConvertTo(expr.to_bits) ) + if ( + expr.from_bits != expr.to_bits + and r.typevar is not None + and not isinstance(r.typevar, typeconsts.TypeConstant) + ): + int_type_cls = typeconsts.signed_int_type if expr.is_signed else typeconsts.unsigned_int_type + tc = int_type_cls(expr.from_bits) + if tc is not None: + self.state.add_type_constraint(typevars.Subtype(r.typevar, tc)) + return RichR(self.state.top(expr.to_bits), typevar=typevar) def _handle_expr_Extract(self, expr: ailment.expression.Extract): diff --git a/tests/analyses/decompiler/test_decompiler.py b/tests/analyses/decompiler/test_decompiler.py index 783818f3f..d8b97b1ea 100755 --- a/tests/analyses/decompiler/test_decompiler.py +++ b/tests/analyses/decompiler/test_decompiler.py @@ -50,6 +50,7 @@ from angr.sim_type import ( SimTypeInt, SimTypeLongLong, SimTypePointer, + SimTypeShort, ) from angr.sim_variable import SimStackVariable from angr.utils.library import convert_cproto_to_py @@ -5672,7 +5673,7 @@ class TestDecompiler(unittest.TestCase): def test_tail_calls(self, decompiler_options=None): bin_path = os.path.join(test_location, "x86_64", "decompiler", "tail_calls.o") - proj = angr.Project(bin_path, auto_load_libs=False) + proj = angr.Project(bin_path) cfg = proj.analyses.CFG(normalize=True) proj.analyses.CompleteCallingConventions(analyze_callsites=False) @@ -5701,9 +5702,9 @@ class TestDecompiler(unittest.TestCase): print_decompilation_result(dec) a0 = dec.clinic.kb.dec_variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name assert normalize_whitespace(f""" - if ((int){a0}) + if ((unsigned int){a0}) return test_cond_tailcall_jmp_callee({a0}); - return (int){a0} - 1; + return (unsigned int){a0} - 1; """) in normalize_whitespace(dec.codegen.text) func = proj.kb.functions["test_cond_noreturn_tailcall_jmp"] @@ -5723,9 +5724,9 @@ class TestDecompiler(unittest.TestCase): print_decompilation_result(dec) a0 = dec.clinic.kb.dec_variables[dec.func.addr].unified_variable(dec.clinic.arg_list[0]).name assert normalize_whitespace(f""" - if ((int){a0}) + if ((unsigned int){a0}) return test_cond_tailcall_cjmp_callee({a0}); - return (int){a0} - 1; + return (unsigned int){a0} - 1; """) in normalize_whitespace(dec.codegen.text) func = proj.kb.functions["test_cond_noreturn_tailcall_cjmp"] @@ -5953,6 +5954,29 @@ class TestDecompiler(unittest.TestCase): assert dec.codegen is not None and dec.codegen.text is not None, f"Failed to decompile function {f!r}." print_decompilation_result(dec) + def test_widening_conversion_signedness(self, decompiler_options=None): + # A widening integer conversion carries the signedness of its source operand: a sign-extending Convert + # (e.g. movswl) implies a signed source, and a zero-extending Convert (e.g. movzwl) implies an unsigned + # source. Make sure a sign-extended 16-bit parameter is recovered as a signed short (not unsigned short). + bin_path = os.path.join(test_location, "x86_64", "sign_extend_widen") + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(normalize=True) + proj.analyses.CompleteCallingConventions(cfg=cfg.model) + + expected_signed = { + "w_s8": (SimTypeChar, True), + "w_u8": (SimTypeChar, False), + "w_s16": (SimTypeShort, True), + "w_u16": (SimTypeShort, False), + } + for name, (ty_cls, signed) in expected_signed.items(): + f = proj.kb.functions.function(name=name) + dec = proj.analyses[Decompiler].prep(fail_fast=True)(f, cfg=cfg.model, options=decompiler_options) + print_decompilation_result(dec) + arg0 = f.prototype.args[0] + assert isinstance(arg0, ty_cls), f"{name}: expected {ty_cls.__name__}, got {arg0!r}" + assert arg0.signed is signed, f"{name}: expected signed={signed}, got signed={arg0.signed}" + if __name__ == "__main__": unittest.main() From c13863214c31fd1d76cb4b97dbcd132b4939dc56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kowalczyk?= <5244565+mkow@users.noreply.github.com> Date: Fri, 24 Jul 2026 02:31:25 +0200 Subject: [PATCH 037/122] docs: Fix dangling links (#6533) --- docs/core-concepts/pathgroups.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/core-concepts/pathgroups.rst b/docs/core-concepts/pathgroups.rst index 953500fce..36147d67e 100644 --- a/docs/core-concepts/pathgroups.rst +++ b/docs/core-concepts/pathgroups.rst @@ -106,7 +106,7 @@ on. There are lots of fun tools that the simulation manager provides you for managing your stashes. We won't go into the rest of them for now, but you should -check out the API documentation. TODO: link +check out the `API documentation <../api/angr.sim_manager.html>`_. Stash types ----------- @@ -177,7 +177,7 @@ in the active stash before finding this many solutions, execution will stop anyway. Let's look at a simple crackme `example -<./examples.md#reverseme-modern-binary-exploitation---csci-4968>`: +`_: First, we load the binary. From 1808e7fadd0448d20a51bf4ddc04fe0835c98327 Mon Sep 17 00:00:00 2001 From: Vedant Soni <83280635+tedanvosin@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:32:00 -0700 Subject: [PATCH 038/122] Migrate rust demangling to pydemumble (#6663) * use pydemumble to demangle rust symbols * update tests * remove rust_demangle and unpin pydemumble from pyproject * pin pydemumble to latest version --- angr/rust/utils/demangler.py | 40 ++++++++++++++++--------- pyproject.toml | 3 +- tests/analyses/test_rust_utils.py | 49 ++++++++++++++++++++++++------- 3 files changed, 65 insertions(+), 27 deletions(-) diff --git a/angr/rust/utils/demangler.py b/angr/rust/utils/demangler.py index 33d4c3c17..5f1e092ee 100644 --- a/angr/rust/utils/demangler.py +++ b/angr/rust/utils/demangler.py @@ -3,35 +3,47 @@ from __future__ import annotations import re from functools import lru_cache -import rust_demangler -from rust_demangler.rust import TypeNotFoundError -from rust_demangler.rust_legacy import UnableToLegacyDemangle -from rust_demangler.rust_v0 import UnableTov0Demangle +import pydemumble def _is_rust_hash(s): return len(s) == 17 and s.startswith("h") and all(c in "0123456789abcdef" for c in s[1:]) +def _looks_like_rust(s): + # Rust manglings are either v0 ("_R...") or the legacy nested-Itanium form + # ("_ZN...E"). Strip one optional leading underscore (macOS). General C++ + # ("_Z3fooi"), MSVC ("?..."), and plain names are left untouched: pydemumble + # would happily demangle those, but this wrapper is Rust-only. + body = s[1:] if s.startswith("__") else s + return body.startswith(("_R", "_ZN")) + + GENERIC_TYPE_PATTERN = re.compile(r"(?:::)?<(?:(?!\sas\s)[^<])*?>") XXX_AS_YYY_PATTERN = re.compile(r"<(?!impl\s)([^<]+?)\sas\s([^<]+?)>") IMPL_XXX_AS_YYY_PATTERN = re.compile(r"") +# demumble renders trailing symbol suffixes (e.g. ".llvm.1234", ".0") as " (.suffix)" +TRAILING_SUFFIX_PATTERN = re.compile(r" \((\.[^)]+)\)$") + @lru_cache(maxsize=4096) def demangle(s): - try: - demangled = rust_demangler.demangle(s).split("::") - except (TypeNotFoundError, UnableTov0Demangle, UnableToLegacyDemangle): + if not _looks_like_rust(s): return s - except (IndexError, ValueError, RecursionError): - # work around bugs in rust_demangler. see angr issue #6598 + demangled = pydemumble.demangle(s) + if not demangled: return s - if len(demangled) >= 2 and _is_rust_hash(demangled[-1]): - demangled = "::".join(demangled[:-1]) - else: - demangled = "::".join(demangled) - return demangled + suffix = "" + match = TRAILING_SUFFIX_PATTERN.search(demangled) + if match is not None: + demangled = demangled[: match.start()] + if not match.group(1).startswith(".llvm."): + suffix = match.group(1) + parts = demangled.split("::") + if len(parts) >= 2 and _is_rust_hash(parts[-1]): + demangled = "::".join(parts[:-1]) + return demangled + suffix def normalize(name, monopolize=True, concise=False, use_trait_name=False): diff --git a/pyproject.toml b/pyproject.toml index 2b52e002c..bbe86f6ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,11 +43,10 @@ dependencies = [ "psutil", "pycparser~=3.0", "platformdirs", - "pydemumble==0.0.1", + "pydemumble~=0.1.3", "pypcode~=4.0", "pyvex==9.3.1.dev0", "rich>=13.1.0", - "rust-demangler==1.0", "sortedcontainers", "sympy", "typing-extensions", diff --git a/tests/analyses/test_rust_utils.py b/tests/analyses/test_rust_utils.py index a62448e43..1de5b2424 100644 --- a/tests/analyses/test_rust_utils.py +++ b/tests/analyses/test_rust_utils.py @@ -264,15 +264,15 @@ def test_demangle_falls_back_to_original_string_when_unrecognized(): def test_demangle_survives_malformed_rust_v0_symbol(): # Regression for angr#6598: a garbage name that merely *looks* like a Rust v0 symbol - # ("_R" prefix) makes the third-party rust_demangler walk past the end of its input and - # leak a raw IndexError. demangle() must swallow it and fall back to the original string. + # ("_R" prefix). pydemumble rejects it (returns an empty string) and demangle() must + # fall back to the original string. raw = "_RBOJyX" assert demangle(raw) == raw def test_demangle_survives_assorted_malformed_v0_symbols(): - # Various truncated/garbage "_R..." names that previously crashed the demangler with an - # IndexError. Every one must fall through to a non-empty string without raising. + # Various truncated/garbage "_R..." names that crashed the previously-used third-party + # demangler. Every one must fall through to a non-empty string without raising. for raw in ("_R", "_RNvC", "_RNvNtCs", "_RINvNtCs", "_RNvMC0", "_RNCNvC0"): result = demangle(raw) assert isinstance(result, str) @@ -317,18 +317,45 @@ def test_demangle_returns_input_for_legacy_symbol_with_no_hash_suffix(): # the demangler still parses it, but our hash-stripping branch must not fire. mangled = "_ZN4core3fmt9Formatter9write_strE" out = demangle(mangled) - # Whatever rust_demangler returns, our wrapper must not silently truncate the tail. + # The wrapper must not silently truncate the tail. assert out.endswith("write_str") def test_demangle_passes_through_non_rust_input(): - # Inputs that aren't recognized as Rust mangled names fall through unchanged. + # Inputs that aren't Rust mangled names must fall through unchanged. In particular, + # _Z3fooi (itanium C++) and MSVC names would demangle under pydemumble, but the + # wrapper is Rust-only and must not touch them. for raw in ("plain_c_symbol", "main", "?something@msvc@@", "_Z3fooi"): - # _Z3fooi is itanium C++ mangling; rust_demangler may or may not raise, - # but the wrapper must always return a non-None string for non-Rust input. - result = demangle(raw) - assert isinstance(result, str) - assert result # non-empty + assert demangle(raw) == raw + + +def test_demangle_decodes_legacy_dollar_escapes(): + mangled = "_ZN42_$LT$$RF$T$u20$as$u20$core..fmt..Debug$GT$3fmt17h517074eb1cb2b995E" + assert demangle(mangled) == "<&T as core::fmt::Debug>::fmt" + + +def test_demangle_decodes_legacy_closure_escapes(): + mangled = "_ZN4core3ptr42drop_in_place$LT$alloc..string..String$GT$17h5d180b0b0e91564fE" + assert demangle(mangled) == "core::ptr::drop_in_place" + + +def test_demangle_drops_llvm_suffix(): + # Compiler-generated ".llvm." suffixes are dropped, matching rustc-demangle. + mangled = ( + "_ZN3std2rt19lang_start_internal28_$u7b$$u7b$closure$u7d$$u7d$17hf421b6f6b8a4a2f4E.llvm.9325873435131735662" + ) + assert demangle(mangled) == "std::rt::lang_start_internal::{{closure}}" + + +def test_demangle_keeps_numeric_instantiation_suffix_attached(): + # A trailing "." (e.g. from symbol versioning/instantiation) stays attached to the + # demangled name, not rendered in demumble's " (.n)" style. + mangled = "_RNvNtNtCsjrHSEGnQ3l9_3std2io5stdio19OUTPUT_CAPTURE_USED.0" + assert demangle(mangled) == "std::io::stdio::OUTPUT_CAPTURE_USED.0" + + +def test_demangle_v0_symbol_with_generics(): + assert demangle("_RINvNtC3std3mem8align_ofjE") == "std::mem::align_of::" def test_normalize_strips_nested_generic_brackets(): From cf54c35b9b37d7ccb4db47577d44cec57b299cbc Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Thu, 23 Jul 2026 21:10:10 -0700 Subject: [PATCH 039/122] AIL: handle HAddV operations (#6680) * AIL: handle HAddV operations * Fix HAddV CI diagnostics --- .../dephication/rewriting_engine.py | 1 + .../optimization_passes/engine_base.py | 2 + ...nlined_string_transformation_simplifier.py | 1 + .../ssailification/rewriting_engine.py | 1 + .../ssailification/traversal_engine.py | 1 + angr/analyses/purity/engine.py | 1 + .../reaching_definitions/engine_ail.py | 1 + angr/analyses/variable_recovery/engine_ail.py | 4 ++ angr/engines/ail/engine_light.py | 13 ++++++ angr/engines/light/engine.py | 4 ++ native/angr/src/ailment/convert_vex.rs | 1 + tests/ailment/test_irsb.py | 31 ++++++++++++++ .../analyses/decompiler/test_vector_binops.py | 40 +++++++++++++++++++ tests/sim/test_ail_exec.py | 22 ++++++++++ 14 files changed, 123 insertions(+) create mode 100644 tests/analyses/decompiler/test_vector_binops.py diff --git a/angr/analyses/decompiler/dephication/rewriting_engine.py b/angr/analyses/decompiler/dephication/rewriting_engine.py index f57de4f6c..8bad7602d 100644 --- a/angr/analyses/decompiler/dephication/rewriting_engine.py +++ b/angr/analyses/decompiler/dephication/rewriting_engine.py @@ -549,6 +549,7 @@ class SimEngineDephiRewriting(SimEngineNostmtAIL[None, Expression | None, Statem _handle_binop_CmpLTV = _unreachable _handle_binop_MinV = _unreachable _handle_binop_MaxV = _unreachable + _handle_binop_HAddV = _unreachable _handle_binop_QAddV = _unreachable _handle_binop_QSubV = _unreachable _handle_binop_QNarrowBinV = _unreachable diff --git a/angr/analyses/decompiler/optimization_passes/engine_base.py b/angr/analyses/decompiler/optimization_passes/engine_base.py index e53e63694..644d2688e 100644 --- a/angr/analyses/decompiler/optimization_passes/engine_base.py +++ b/angr/analyses/decompiler/optimization_passes/engine_base.py @@ -509,6 +509,8 @@ class SimplifierAILEngine( _handle_binop_MaxV = _handle_binop_Default + _handle_binop_HAddV = _handle_binop_Default + _handle_binop_QAddV = _handle_binop_Default _handle_binop_QSubV = _handle_binop_Default diff --git a/angr/analyses/decompiler/optimization_passes/inlined_string_transformation_simplifier.py b/angr/analyses/decompiler/optimization_passes/inlined_string_transformation_simplifier.py index 3956c82eb..a5f8eaedc 100644 --- a/angr/analyses/decompiler/optimization_passes/inlined_string_transformation_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/inlined_string_transformation_simplifier.py @@ -502,6 +502,7 @@ class InlinedStringTransformationAILEngine( _handle_binop_SubV = _handle_binop_Default _handle_binop_MinV = _handle_binop_Default _handle_binop_MaxV = _handle_binop_Default + _handle_binop_HAddV = _handle_binop_Default _handle_binop_QAddV = _handle_binop_Default _handle_binop_QSubV = _handle_binop_Default _handle_binop_QNarrowBinV = _handle_binop_Default diff --git a/angr/analyses/decompiler/ssailification/rewriting_engine.py b/angr/analyses/decompiler/ssailification/rewriting_engine.py index bfea32fc8..d4331982c 100644 --- a/angr/analyses/decompiler/ssailification/rewriting_engine.py +++ b/angr/analyses/decompiler/ssailification/rewriting_engine.py @@ -877,6 +877,7 @@ class SimEngineSSARewriting( _handle_binop_CmpLTV = _unreachable _handle_binop_MinV = _unreachable _handle_binop_MaxV = _unreachable + _handle_binop_HAddV = _unreachable _handle_binop_QAddV = _unreachable _handle_binop_QSubV = _unreachable _handle_binop_QNarrowBinV = _unreachable diff --git a/angr/analyses/decompiler/ssailification/traversal_engine.py b/angr/analyses/decompiler/ssailification/traversal_engine.py index ab212fb6e..d4b3d75cb 100644 --- a/angr/analyses/decompiler/ssailification/traversal_engine.py +++ b/angr/analyses/decompiler/ssailification/traversal_engine.py @@ -871,6 +871,7 @@ class SimEngineSSATraversal(SimEngineLightAIL[TraversalState, Value, None, None] _handle_binop_CmpLTV = _unreachable _handle_binop_MinV = _unreachable _handle_binop_MaxV = _unreachable + _handle_binop_HAddV = _unreachable _handle_binop_QAddV = _unreachable _handle_binop_QSubV = _unreachable _handle_binop_QNarrowBinV = _unreachable diff --git a/angr/analyses/purity/engine.py b/angr/analyses/purity/engine.py index b477be081..b98dd5188 100644 --- a/angr/analyses/purity/engine.py +++ b/angr/analyses/purity/engine.py @@ -589,6 +589,7 @@ class PurityEngineAIL(SimEngineLightAIL[StateType, DataType_co, StmtDataType, Re _handle_binop_CmpLTV = _handle_binop_default _handle_binop_MinV = _handle_binop_default _handle_binop_MaxV = _handle_binop_default + _handle_binop_HAddV = _handle_binop_default _handle_binop_QAddV = _handle_binop_default _handle_binop_QSubV = _handle_binop_default _handle_binop_QNarrowBinV = _handle_binop_default diff --git a/angr/analyses/reaching_definitions/engine_ail.py b/angr/analyses/reaching_definitions/engine_ail.py index 924029bd8..a0e7dc5ac 100644 --- a/angr/analyses/reaching_definitions/engine_ail.py +++ b/angr/analyses/reaching_definitions/engine_ail.py @@ -788,6 +788,7 @@ class SimEngineRDAIL( _handle_binop_CmpLTV = _handle_binop_Default _handle_binop_MinV = _handle_binop_Default _handle_binop_MaxV = _handle_binop_Default + _handle_binop_HAddV = _handle_binop_Default _handle_binop_QAddV = _handle_binop_Default _handle_binop_QSubV = _handle_binop_Default _handle_binop_QNarrowBinV = _handle_binop_Default diff --git a/angr/analyses/variable_recovery/engine_ail.py b/angr/analyses/variable_recovery/engine_ail.py index 50141099a..f3cb8f3b6 100644 --- a/angr/analyses/variable_recovery/engine_ail.py +++ b/angr/analyses/variable_recovery/engine_ail.py @@ -1230,6 +1230,10 @@ class SimEngineVRAIL( _handle_binop_Set = _handle_binop_Default _handle_binop_MaxV = _handle_binop_Default _handle_binop_MinV = _handle_binop_Default + + def _handle_binop_HAddV(self, expr: ailment.expression.BinaryOp) -> RichR[claripy.ast.BV | claripy.ast.FP]: + return cast(RichR[claripy.ast.BV | claripy.ast.FP], self._handle_binop_Default(expr)) + _handle_binop_QAddV = _handle_binop_Default _handle_binop_QSubV = _handle_binop_Default _handle_binop_QNarrowBinV = _handle_binop_Default diff --git a/angr/engines/ail/engine_light.py b/angr/engines/ail/engine_light.py index 7dcc34ecd..34a19ff10 100644 --- a/angr/engines/ail/engine_light.py +++ b/angr/engines/ail/engine_light.py @@ -957,6 +957,19 @@ class SimEngineAILSimState(SimEngineLightAIL[StateType, DataType, bool, None]): def _handle_binop_MaxV(self, expr: ailment.expression.BinaryOp) -> DataType: raise NotImplementedError("Not sure of the semantics of this op") + def _handle_binop_HAddV(self, expr: ailment.expression.BinaryOp) -> DataType: + assert expr.vector_size is not None + extend = claripy.SignExt if expr.signed else claripy.ZeroExt + return claripy.Concat( + *( + (extend(expr.vector_size, a) + extend(expr.vector_size, b))[expr.vector_size : 1] + for a, b in zip( + self._expr_bv(expr.operands[0]).chop(expr.vector_size), + self._expr_bv(expr.operands[1]).chop(expr.vector_size), + ) + ) + ) + def _handle_binop_QAddV(self, expr: ailment.expression.BinaryOp) -> DataType: raise NotImplementedError("Not sure of the semantics of this op") diff --git a/angr/engines/light/engine.py b/angr/engines/light/engine.py index 0ebb0844b..347c884f2 100644 --- a/angr/engines/light/engine.py +++ b/angr/engines/light/engine.py @@ -637,6 +637,7 @@ class SimEngineLightAIL[StateType, DataType_co, StmtDataType, ResultType]( "CmpLTV": self._handle_binop_CmpLEV, "MinV": self._handle_binop_MinV, "MaxV": self._handle_binop_MaxV, + "HAddV": self._handle_binop_HAddV, "QAddV": self._handle_binop_QAddV, "QSubV": self._handle_binop_QSubV, "QNarrowBinV": self._handle_binop_QNarrowBinV, @@ -1046,6 +1047,9 @@ class SimEngineLightAIL[StateType, DataType_co, StmtDataType, ResultType]( @abstractmethod def _handle_binop_MaxV(self, expr: ailment.expression.BinaryOp) -> DataType_co: ... + @abstractmethod + def _handle_binop_HAddV(self, expr: ailment.expression.BinaryOp) -> DataType_co: ... + @abstractmethod def _handle_binop_QAddV(self, expr: ailment.expression.BinaryOp) -> DataType_co: ... diff --git a/native/angr/src/ailment/convert_vex.rs b/native/angr/src/ailment/convert_vex.rs index e7fc7ccb1..cbd0c0a74 100644 --- a/native/angr/src/ailment/convert_vex.rs +++ b/native/angr/src/ailment/convert_vex.rs @@ -597,6 +597,7 @@ impl<'py, 'r, R: IrReader> Conv<'py, 'r, R> { let mut vector_size: Option = None; if simop.vector_count.is_some() && simop.vector_size.is_some() { op_name = Some(format!("{}V", op_name.unwrap_or_default())); + signed = simop.is_signed(); vector_count = simop.vector_count.map(|v| v as i64); vector_size = simop.vector_size.map(|v| v as i64); } else if matches!( diff --git a/tests/ailment/test_irsb.py b/tests/ailment/test_irsb.py index 714261402..f39ad7a73 100644 --- a/tests/ailment/test_irsb.py +++ b/tests/ailment/test_irsb.py @@ -143,6 +143,37 @@ class TestNonConstRoundingMode(unittest.TestCase): assert isinstance(binop.rounding_mode, RoundingMode) +class TestVectorSignedness(unittest.TestCase): + @staticmethod + def _find_haddv(block): + return next( + stmt.src + for stmt in block.statements + if isinstance(getattr(stmt, "src", None), ailment.Expr.BinaryOp) and stmt.src.op == "HAddV" + ) + + def test_haddv_signedness(self): + arch = archinfo.arch_from_id("armel") + + for name, block_bytes, expected_signed in ( + ("sadd8", bytes.fromhex("920f11e6"), True), + ("uadd8", bytes.fromhex("920f51e6"), False), + ): + with self.subTest(instruction=name): + irsb = pyvex.IRSB(block_bytes, 0x1000, arch, opt_level=0) + from_py = VEXIRSBConverter.convert(irsb, ailment.Manager(arch=arch)) + from_lift = VEXIRSBConverter.convert_from_lift( + arch, 0x1000, block_bytes, ailment.Manager(arch=arch), opt_level=0 + ) + + assert from_py == from_lift + for block in (from_py, from_lift): + haddv = self._find_haddv(block) + assert haddv.signed is expected_signed + assert haddv.vector_count == 4 + assert haddv.vector_size == 8 + + class TestVexConverterAcrossArches(unittest.TestCase): """Convert real blocks from test binaries through both the Python-IRSB path and the libVEX-lift path, and assert the two agree.""" diff --git a/tests/analyses/decompiler/test_vector_binops.py b/tests/analyses/decompiler/test_vector_binops.py new file mode 100644 index 000000000..d1a801475 --- /dev/null +++ b/tests/analyses/decompiler/test_vector_binops.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import os +import unittest + +import angr +from tests.common import bin_location + +test_location = os.path.join(bin_location, "tests") + + +class TestVectorBinops(unittest.TestCase): + """Test vector operations throughout the decompiler pipeline.""" + + def test_haddv_survives_decompilation(self): + bin_path = os.path.join(test_location, "armel", "libc-2.31.so") + + for function_name in ("strlen", "strcmp"): + with self.subTest(function_name=function_name): + project = angr.Project(bin_path, auto_load_libs=False) + symbol = project.loader.find_symbol(function_name) + assert symbol is not None + + function_addr = symbol.rebased_addr + cfg = project.analyses.CFGFast( + function_starts=[function_addr], + regions=[(function_addr & ~1, (function_addr & ~1) + symbol.size)], + force_complete_scan=False, + normalize=True, + ) + function = cfg.functions[function_addr] + + decompilation = project.analyses.Decompiler(function, cfg=cfg.model) + assert decompilation.codegen is not None + assert decompilation.codegen.text is not None + assert "HAddV(" in decompilation.codegen.text + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/sim/test_ail_exec.py b/tests/sim/test_ail_exec.py index ca2b782dd..db420ad61 100644 --- a/tests/sim/test_ail_exec.py +++ b/tests/sim/test_ail_exec.py @@ -23,6 +23,28 @@ test_location = os.path.join(bin_location, "tests") class TestAILExec(unittest.TestCase): + def test_haddv_expression(self): + p = angr.load_shellcode(b"\x00", arch="ARMEL", load_address=0x400000) + state = p.factory.blank_state() + successors = SimSuccessors(state.addr, state) + engine = SimEngineAILSimState(p, successors) + + left = ailment.expression.Const(None, 0xFF008877, 32) + right = ailment.expression.Const(None, 0x11111111, 32) + for signed, expected in ((True, 0x0808CC44), (False, 0x88084C44)): + with self.subTest(signed=signed): + expr = ailment.expression.BinaryOp( + None, + "HAddV", + (left, right), + signed, + bits=32, + vector_count=4, + vector_size=8, + ) + result = engine._expr_bv(expr) # pylint: disable=protected-access + assert result.concrete and result.concrete_value == expected + def test_smoketest(self): p = angr.Project(os.path.join(test_location, "x86_64", "true"), auto_load_libs=False) cfg = p.analyses.CFGFast(normalize=True) From 2dd6cb393b7ae015db404d30ade8f8ab5fdffc9c Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Thu, 23 Jul 2026 21:15:18 -0700 Subject: [PATCH 040/122] Typehoon: preserve standard SimTypeNum types (#6678) --- angr/analyses/typehoon/translator.py | 9 +++ tests/analyses/test_typehoon.py | 89 +++++++++++++++++++++++++++- 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/angr/analyses/typehoon/translator.py b/angr/analyses/typehoon/translator.py index 4989df736..3c26b3af9 100644 --- a/angr/analyses/typehoon/translator.py +++ b/angr/analyses/typehoon/translator.py @@ -285,6 +285,14 @@ class TypeTranslator: # SimType handlers # + def _translate_SimTypeNum(self, st: sim_type.SimTypeNum) -> typeconsts.TypeConstant: + if st.size not in {8, 16, 32, 64}: + return typeconsts.BottomType() + + tc = typeconsts.signed_int_type(st.size) if st.signed else typeconsts.unsigned_int_type(st.size) + tc.name = st.label + return tc + def _translate_SimTypeInt128(self, st: sim_type.SimTypeChar) -> typeconsts.Int128: return typeconsts.Int128(name=st.label) @@ -441,6 +449,7 @@ TypeConstHandlers = { SimTypeHandlers = { sim_type.SimTypePointer: TypeTranslator._translate_SimTypePointer, + sim_type.SimTypeNum: TypeTranslator._translate_SimTypeNum, sim_type.SimTypeChar: TypeTranslator._translate_SimTypeChar, sim_type.SimTypeWideChar: TypeTranslator._translate_SimTypeWideChar, sim_type.SimTypeInt: TypeTranslator._translate_SimTypeInt, diff --git a/tests/analyses/test_typehoon.py b/tests/analyses/test_typehoon.py index 09ee5df33..d6ceff6d8 100755 --- a/tests/analyses/test_typehoon.py +++ b/tests/analyses/test_typehoon.py @@ -15,7 +15,20 @@ import angr from angr.analyses.decompiler.clinic import Clinic from angr.analyses.typehoon.simple_solver import SimpleSolver from angr.analyses.typehoon.translator import TypeTranslator -from angr.analyses.typehoon.typeconsts import Float32, Float64, Int32, Pointer64, Struct +from angr.analyses.typehoon.typeconsts import ( + Array, + BottomType, + Float32, + Float64, + Int1, + Int8, + Int32, + IntVar, + Pointer64, + SInt32, + SInt64, + Struct, +) from angr.analyses.typehoon.typehoon import Typehoon from angr.analyses.typehoon.typevars import ( DerivedTypeVariable, @@ -36,6 +49,7 @@ from angr.sim_type import ( SimTypeFloat, SimTypeFunction, SimTypeInt, + SimTypeNum, SimTypePointer, ) from tests.common import bin_location, print_decompilation_result @@ -442,6 +456,79 @@ class TestTypeTranslator(unittest.TestCase): tc = tx.simtype2tc(st) assert isinstance(tc, Float32) + def test_simtypenum_struct_round_trip(self): + arch = archinfo.arch_from_id("amd64") + st = SimTypePointer( + SimStruct( + OrderedDict( + { + "ut_addr_v6": SimTypeArray(SimTypeNum(32, signed=True, label="int32_t"), 4), + "tv_usec": SimTypeNum(64, signed=True, label="int64_t"), + } + ), + name="utmp", + ) + ).with_arch(arch) + tx = TypeTranslator(arch) + + tc = tx.simtype2tc(st) + assert isinstance(tc, Pointer64) + assert isinstance(tc.basetype, Struct) + assert isinstance(tc.basetype.fields[0], Array) + assert isinstance(tc.basetype.fields[0].element, SInt32) + assert tc.basetype.fields[0].element.name == "int32_t" + assert isinstance(tc.basetype.fields[16], SInt64) + assert tc.basetype.fields[16].name == "int64_t" + + restored, has_nonexistent_ref = tx.tc2simtype(tc) + assert not has_nonexistent_ref + assert isinstance(restored, SimTypePointer) + assert isinstance(restored.pts_to, SimStruct) + assert restored.pts_to.offsets == {"ut_addr_v6": 0, "tv_usec": 16} + assert restored.pts_to.size == 192 + restored_array = restored.pts_to.fields["ut_addr_v6"] + assert isinstance(restored_array, SimTypeArray) + assert restored_array.size == 128 + assert restored_array.elem_type.size == 32 + assert restored_array.elem_type.signed is True + + def test_unsupported_width_simtypenum(self): + arch = archinfo.arch_from_id("amd64") + tx = TypeTranslator(arch) + + for bits in (1, 9, 24, 128): + for signed in (False, True): + with self.subTest(bits=bits, signed=signed): + tc = tx.simtype2tc(SimTypeNum(bits, signed=signed, label=f"int{bits}_t").with_arch(arch)) + assert isinstance(tc, BottomType) + + # Int1 and IntVar cannot be converted to equally-sized SimTypes: their sizes use incompatible units. They must + # remain unsupported rather than creating zero-alignment or incorrectly laid-out fields in a SimStruct. + struct = Struct(fields={0: Int1(), 1: IntVar(9), 2: Int8()}) + restored, has_nonexistent_ref = tx.tc2simtype(struct) + assert not has_nonexistent_ref + assert isinstance(restored, SimStruct) + assert restored.offsets == {"field_0": 0, "field_1": 1, "field_2": 2} + + def test_standard_width_simtypenum_round_trip(self): + arch = archinfo.arch_from_id("amd64") + tx = TypeTranslator(arch) + + for bits in (8, 16, 32, 64): + for signed in (False, True): + with self.subTest(bits=bits, signed=signed): + label = f"{'u' if not signed else ''}int{bits}_t" + tc = tx.simtype2tc(SimTypeNum(bits, signed=signed, label=label).with_arch(arch)) + assert not isinstance(tc, IntVar) + assert tc.size * arch.byte_width == bits + assert tc.name == label + + restored, has_nonexistent_ref = tx.tc2simtype(tc) + assert not has_nonexistent_ref + assert restored.size == bits + assert restored.signed is signed + assert restored.label == label + def test_lift_recursive_struct(self): arch = archinfo.arch_from_id("amd64") fields = OrderedDict({"ptr": SimTypePointer(SimTypeBottom())}) From 8b98c5ad6c411ab8f826f907b9d5b123a000823d Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Thu, 23 Jul 2026 22:30:15 -0700 Subject: [PATCH 041/122] Keep eager evaluation integer-only (#6681) --- .../peephole_optimizations/eager_eval.py | 15 +++- .../decompiler/test_peephole_optimizations.py | 77 +++++++++++++++++++ 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/angr/analyses/decompiler/peephole_optimizations/eager_eval.py b/angr/analyses/decompiler/peephole_optimizations/eager_eval.py index 513b9c591..17aae80e1 100644 --- a/angr/analyses/decompiler/peephole_optimizations/eager_eval.py +++ b/angr/analyses/decompiler/peephole_optimizations/eager_eval.py @@ -29,6 +29,10 @@ class EagerEvaluation(PeepholeOptimizationExprBase): @staticmethod def _optimize_binaryop(expr: BinaryOp): + # The identities below assume integer arithmetic and are not generally valid under IEEE 754. + if expr.floating_point: + return None + op0, op1 = expr.operands if expr.op == "Add": if ( @@ -227,17 +231,20 @@ class EagerEvaluation(PeepholeOptimizationExprBase): elif ( expr.op == "Div" and isinstance(op1, Const) + and op1.is_int and isinstance(op0, BinaryOp) and op0.op == "Mul" + and not op0.floating_point and isinstance(op0.operands[1], Const) + and op0.operands[1].is_int ): expr0, const_0 = expr.operands const_1 = expr0.operands[1] - if const_0.value != 0 and const_1.value != 0: - gcd_ = gcd(const_0.value, const_1.value) + if const_0.value_int != 0 and const_1.value_int != 0: + gcd_ = gcd(const_0.value_int, const_1.value_int) if gcd_ != 1: - new_const_1 = Const(const_1.idx, const_1.value // gcd_, const_1.bits, **const_1.tags) - new_const_0 = Const(const_0.idx, const_0.value // gcd_, const_0.bits, **const_0.tags) + new_const_1 = Const(const_1.idx, const_1.value_int // gcd_, const_1.bits, **const_1.tags) + new_const_0 = Const(const_0.idx, const_0.value_int // gcd_, const_0.bits, **const_0.tags) mul = BinaryOp( expr0.idx, "Mul", diff --git a/tests/analyses/decompiler/test_peephole_optimizations.py b/tests/analyses/decompiler/test_peephole_optimizations.py index dd40eb80f..a57a5ea89 100755 --- a/tests/analyses/decompiler/test_peephole_optimizations.py +++ b/tests/analyses/decompiler/test_peephole_optimizations.py @@ -84,6 +84,83 @@ class TestPeepholeOptimizations(unittest.TestCase): expr_opt = opt.optimize(expr) assert expr_opt is None + def test_eager_eval_mul_div_cancellation_requires_integers(self): + proj = angr.load_shellcode(b"\x90", "AMD64") + manager = Manager() + opt = EagerEvaluation(proj, proj.kb, manager) + x = Register(manager.next_atom(), 0, 32) + + def mul_div(multiplier, divisor, *, mul_floating_point=False, div_floating_point=False): + mul = BinaryOp( + manager.next_atom(), + "Mul", + [x, Const(manager.next_atom(), multiplier, 32)], + False, + bits=32, + floating_point=mul_floating_point, + ) + return BinaryOp( + manager.next_atom(), + "Div", + [mul, Const(manager.next_atom(), divisor, 32)], + False, + bits=32, + floating_point=div_floating_point, + ) + + # (x * 6) / 8 -> (x * 3) / 4 + out = opt.optimize(mul_div(6, 8)) + assert isinstance(out, BinaryOp) and out.op == "Div" + assert isinstance(out.operands[0], BinaryOp) and out.operands[0].op == "Mul" + assert out.operands[0].operands[1].value == 3 + assert out.operands[1].value == 4 + + # AIL constants may contain floats. Integer cancellation does not apply to them. + for multiplier, divisor in ((6.0, 8), (6, 8.0), (6.0, 8.0)): + with self.subTest(multiplier=multiplier, divisor=divisor): + assert opt.optimize(mul_div(multiplier, divisor)) is None + + # Integer-valued constants do not make floating-point Mul or Div cancellable by an integer GCD. + for mul_floating_point, div_floating_point in ((True, False), (False, True), (True, True)): + with self.subTest( + mul_floating_point=mul_floating_point, + div_floating_point=div_floating_point, + ): + assert ( + opt.optimize( + mul_div( + 6, + 8, + mul_floating_point=mul_floating_point, + div_floating_point=div_floating_point, + ) + ) + is None + ) + + def test_eager_eval_skips_floating_point_binary_operations(self): + proj = angr.load_shellcode(b"\x90", "AMD64") + manager = Manager() + opt = EagerEvaluation(proj, proj.kb, manager) + x = Register(manager.next_atom(), 0, 32) + + for op, operands, bits in ( + ("Mul", (x, Const(manager.next_atom(), 1, 32)), 32), + ("Add", (x, Const(manager.next_atom(), 0, 32)), 32), + ("Add", (x, Const(manager.next_atom(), -1, 32)), 32), + ("CmpEQ", (x, x), 1), + ): + with self.subTest(op=op, operands=operands): + expr = BinaryOp( + manager.next_atom(), + op, + operands, + False, + bits=bits, + floating_point=True, + ) + assert opt.optimize(expr) is None + def test_cmp_masked_shift(self): proj = angr.load_shellcode(b"\x90", "AMD64") manager = Manager() From 208ec719a673e96de05b339101d295dc21a9ff89 Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Thu, 23 Jul 2026 22:33:21 -0700 Subject: [PATCH 042/122] Decompiler: disambiguate Extract condition placeholders (#6677) --- .../decompiler/condition_processor.py | 6 +++- .../decompiler/test_condition_processor.py | 29 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 tests/analyses/decompiler/test_condition_processor.py diff --git a/angr/analyses/decompiler/condition_processor.py b/angr/analyses/decompiler/condition_processor.py index 0ca32be18..6adb5ffe4 100644 --- a/angr/analyses/decompiler/condition_processor.py +++ b/angr/analyses/decompiler/condition_processor.py @@ -995,7 +995,11 @@ class ConditionProcessor: if not isinstance(condition.offset, ailment.expression.Const) else condition.offset.value ) - var = claripy.BVS(f"ailexpr_Extract({offset_expr}, {hash(var_)})", condition.bits, explicit_name=True) + var = claripy.BVS( + f"ailexpr_Extract({condition.bits}, {condition.endness}, {offset_expr}, {hash(var_)})", + condition.bits, + explicit_name=True, + ) self._condition_mapping[var.args[0]] = condition return var if isinstance(condition, ailment.expression.Insert): diff --git a/tests/analyses/decompiler/test_condition_processor.py b/tests/analyses/decompiler/test_condition_processor.py new file mode 100644 index 000000000..4def4b67f --- /dev/null +++ b/tests/analyses/decompiler/test_condition_processor.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import archinfo + +from angr import ailment +from angr.ailment.expression import Const, Extract, VirtualVariable, VirtualVariableCategory +from angr.analyses.decompiler.condition_processor import ConditionProcessor + + +def test_extract_placeholders_include_semantic_properties(): + arch = archinfo.ArchAMD64() + manager = ailment.Manager(arch=arch) + condition_processor = ConditionProcessor(arch, manager) + + base = VirtualVariable(0, 1, 64, VirtualVariableCategory.REGISTER, oident=arch.registers["rax"][0]) + offset = Const(1, 0, 64) + extract_byte = Extract(2, 8, base, offset, arch.memory_endness) + extract_word = Extract(3, 16, base, offset, arch.memory_endness) + extract_byte_be = Extract(4, 8, base, offset, archinfo.Endness.BE) + + byte_ast = condition_processor.claripy_ast_from_ail_condition(extract_byte) + word_ast = condition_processor.claripy_ast_from_ail_condition(extract_word) + byte_be_ast = condition_processor.claripy_ast_from_ail_condition(extract_byte_be) + + assert byte_ast.args[0] != word_ast.args[0] + assert byte_ast.args[0] != byte_be_ast.args[0] + assert condition_processor.convert_claripy_bool_ast(byte_ast) is extract_byte + assert condition_processor.convert_claripy_bool_ast(word_ast) is extract_word + assert condition_processor.convert_claripy_bool_ast(byte_be_ast) is extract_byte_be From fe434a049bdeeb134c0a1470c8d3343be667296c Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Fri, 24 Jul 2026 00:28:36 -0700 Subject: [PATCH 043/122] Decompiler: reject float constants in string simplifiers (#6682) * Decompiler: reject float constants in string simplifiers * Tests: allow private simplifier coverage --- .../inlined_strcpy_simplifier.py | 21 +- .../inlined_wcscpy_simplifier.py | 38 ++- .../test_inlined_string_simplifiers.py | 270 ++++++++++++++++++ 3 files changed, 308 insertions(+), 21 deletions(-) create mode 100644 tests/analyses/decompiler/test_inlined_string_simplifiers.py diff --git a/angr/analyses/decompiler/optimization_passes/inlined_strcpy_simplifier.py b/angr/analyses/decompiler/optimization_passes/inlined_strcpy_simplifier.py index 1dc69b079..dfe14c2eb 100644 --- a/angr/analyses/decompiler/optimization_passes/inlined_strcpy_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/inlined_strcpy_simplifier.py @@ -96,9 +96,11 @@ class InlinedStrcpySimplifier(OptimizationPass): elif ( isinstance(stmt.src, Insert) and isinstance(stmt.src.base, (Const, VirtualVariable)) + and (not isinstance(stmt.src.base, Const) or stmt.src.base.is_int) and isinstance(stmt.src.value, Const) and stmt.src.value.is_int and isinstance(stmt.src.offset, Const) + and stmt.src.offset.is_int ): inlined_strcpy_candidate = True src = stmt.src.value @@ -253,7 +255,7 @@ class InlinedStrcpySimplifier(OptimizationPass): delta = self._get_delta(addr_last, addr_curr) if delta is not None and delta == len(s_last): new_str = s_last + s_curr - elif isinstance(stmt, Store) and isinstance(stmt.data, Const): + elif isinstance(stmt, Store) and isinstance(stmt.data, Const) and stmt.data.is_int: addr_curr = stmt.addr delta = self._get_delta(addr_last, addr_curr) if delta is not None and delta == len(s_last): @@ -322,7 +324,7 @@ class InlinedStrcpySimplifier(OptimizationPass): and stmt.dst.was_stack and isinstance(stmt.dst.stack_offset, int) ): - if isinstance(stmt.src, Const): + if isinstance(stmt.src, Const) and stmt.src.is_int: r[stmt.dst.stack_offset] = idx, ail_const_to_be(stmt.src, self.project.arch.memory_endness) if ( isinstance(stmt.src, Insert) @@ -334,17 +336,20 @@ class InlinedStrcpySimplifier(OptimizationPass): and stmt.src.base.stack_offset == stmt.dst.stack_offset ) ) + and (not isinstance(stmt.src.base, Const) or stmt.src.base.is_int) and isinstance(stmt.src.offset, Const) + and stmt.src.offset.is_int and isinstance(stmt.src.value, Const) + and stmt.src.value.is_int ): - r[stmt.dst.stack_offset + stmt.src.offset.value] = ( + r[stmt.dst.stack_offset + stmt.src.offset.value_int] = ( idx, ail_const_to_be(stmt.src.value, self.project.arch.memory_endness), ) else: r[stmt.dst.stack_offset] = idx, None elif isinstance(stmt, Store) and isinstance(stmt.addr, StackBaseOffset): - if isinstance(stmt.data, Const): + if isinstance(stmt.data, Const) and stmt.data.is_int: r[stmt.addr.offset] = idx, ail_const_to_be(stmt.data, self.project.arch.memory_endness) else: r[stmt.addr.offset] = idx, None @@ -420,12 +425,12 @@ class InlinedStrcpySimplifier(OptimizationPass): ): return StackBaseOffset(-1, addr.bits, 0), addr.operand.stack_offset if isinstance(addr, BinaryOp): - if addr.op == "Add" and isinstance(addr.operands[1], Const): + if addr.op == "Add" and isinstance(addr.operands[1], Const) and addr.operands[1].is_int: base_0, offset_0 = InlinedStrcpySimplifier._parse_addr(addr.operands[0]) - return base_0, offset_0 + addr.operands[1].value - if addr.op == "Sub" and isinstance(addr.operands[1], Const): + return base_0, offset_0 + addr.operands[1].value_int + if addr.op == "Sub" and isinstance(addr.operands[1], Const) and addr.operands[1].is_int: base_0, offset_0 = InlinedStrcpySimplifier._parse_addr(addr.operands[0]) - return base_0, offset_0 - addr.operands[1].value + return base_0, offset_0 - addr.operands[1].value_int return addr, 0 @staticmethod diff --git a/angr/analyses/decompiler/optimization_passes/inlined_wcscpy_simplifier.py b/angr/analyses/decompiler/optimization_passes/inlined_wcscpy_simplifier.py index 0d5b6ed6e..76a30785e 100644 --- a/angr/analyses/decompiler/optimization_passes/inlined_wcscpy_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/inlined_wcscpy_simplifier.py @@ -176,8 +176,11 @@ class InlinedWcscpySimplifier(OptimizationPass): if isinstance(stmt, SideEffectStatement) and self.is_inlined_wcsncpy(stmt): assert stmt.expr.args is not None and len(stmt.expr.args) >= 3 base, off = self._parse_addr(stmt.expr.args[0]) - store_size = stmt.expr.args[2].value * 2 if isinstance(stmt.expr.args[2], Const) else None - if off is not None and store_size is not None: + count = stmt.expr.args[2] + if not isinstance(count, Const) or not count.is_int: + return None + store_size = count.value_int * 2 + if off is not None: candidates.append((i, base, off, store_size, stmt)) elif isinstance(stmt, Store) and isinstance(stmt.data, Const): base, off = self._parse_addr(stmt.addr) @@ -262,8 +265,10 @@ class InlinedWcscpySimplifier(OptimizationPass): merged_stmt = merged[0] new_base, new_off = self._parse_addr(merged_stmt.expr.args[0]) new_sz = ( - merged_stmt.expr.args[2].value * 2 - if len(merged_stmt.expr.args) >= 3 and isinstance(merged_stmt.expr.args[2], Const) + merged_stmt.expr.args[2].value_int * 2 + if len(merged_stmt.expr.args) >= 3 + and isinstance(merged_stmt.expr.args[2], Const) + and merged_stmt.expr.args[2].is_int else sz0 + sz1 ) new_item = idx0, new_base, new_off, new_sz, merged_stmt @@ -442,6 +447,7 @@ class InlinedWcscpySimplifier(OptimizationPass): and starting_stmt.addr.op == "Add" and isinstance(starting_stmt.addr.operands[0], VirtualVariable) and isinstance(starting_stmt.addr.operands[1], Const) + and starting_stmt.addr.operands[1].is_int ): expected_store_varid = starting_stmt.addr.operands[0].varid else: @@ -464,7 +470,9 @@ class InlinedWcscpySimplifier(OptimizationPass): ): offset = stmt.dst.stack_offset value = ( - ail_const_to_be(stmt.src, self.project.arch.memory_endness) if isinstance(stmt.src, Const) else None + ail_const_to_be(stmt.src, self.project.arch.memory_endness) + if isinstance(stmt.src, Const) and stmt.src.is_int + else None ) elif expected_type == "store" and isinstance(stmt, Store): if isinstance(stmt.addr, VirtualVariable) and stmt.addr.varid == expected_store_varid: @@ -474,14 +482,15 @@ class InlinedWcscpySimplifier(OptimizationPass): and stmt.addr.op == "Add" and isinstance(stmt.addr.operands[0], VirtualVariable) and isinstance(stmt.addr.operands[1], Const) + and stmt.addr.operands[1].is_int and stmt.addr.operands[0].varid == expected_store_varid ): - offset = stmt.addr.operands[1].value + offset = stmt.addr.operands[1].value_int else: offset = None value = ( ail_const_to_be(stmt.data, self.project.arch.memory_endness) - if isinstance(stmt.data, Const) + if isinstance(stmt.data, Const) and stmt.data.is_int else None ) else: @@ -508,16 +517,19 @@ class InlinedWcscpySimplifier(OptimizationPass): def even_offsets_are_zero(lst): if len(lst) >= 2 and lst[-1] == 0 and lst[-2] == 0: lst = lst[:-2] - return all((ch == 0 if i % 2 == 0 else ch != 0) for i, ch in enumerate(lst)) + return all(isinstance(ch, int) and (ch == 0 if i % 2 == 0 else ch != 0) for i, ch in enumerate(lst)) @staticmethod def odd_offsets_are_zero(lst): if len(lst) >= 2 and lst[-1] == 0 and lst[-2] == 0: lst = lst[:-2] - return all((ch == 0 if i % 2 == 1 else ch != 0) for i, ch in enumerate(lst)) + return all(isinstance(ch, int) and (ch == 0 if i % 2 == 1 else ch != 0) for i, ch in enumerate(lst)) @staticmethod def is_integer_likely_a_wide_string(v, size, endness, min_length=4): + if not isinstance(v, int) or not isinstance(size, int): + return False, None + chars = [] if endness == Endness.LE: while v != 0: @@ -579,12 +591,12 @@ class InlinedWcscpySimplifier(OptimizationPass): ): return StackBaseOffset(-1, 64, 0), addr.operand.stack_offset if isinstance(addr, BinaryOp): - if addr.op == "Add" and isinstance(addr.operands[1], Const) and isinstance(addr.operands[1].value, int): + if addr.op == "Add" and isinstance(addr.operands[1], Const) and addr.operands[1].is_int: base_0, offset_0 = InlinedWcscpySimplifier._parse_addr(addr.operands[0]) - return base_0, offset_0 + addr.operands[1].value - if addr.op == "Sub" and isinstance(addr.operands[1], Const) and isinstance(addr.operands[1].value, int): + return base_0, offset_0 + addr.operands[1].value_int + if addr.op == "Sub" and isinstance(addr.operands[1], Const) and addr.operands[1].is_int: base_0, offset_0 = InlinedWcscpySimplifier._parse_addr(addr.operands[0]) - return base_0, offset_0 - addr.operands[1].value + return base_0, offset_0 - addr.operands[1].value_int return addr, 0 @staticmethod diff --git a/tests/analyses/decompiler/test_inlined_string_simplifiers.py b/tests/analyses/decompiler/test_inlined_string_simplifiers.py new file mode 100644 index 000000000..e65ed5ef9 --- /dev/null +++ b/tests/analyses/decompiler/test_inlined_string_simplifiers.py @@ -0,0 +1,270 @@ +# pylint: disable=protected-access +from __future__ import annotations + +import angr +from angr.ailment.expression import ( + BinaryOp, + Call, + Const, + Insert, + StackBaseOffset, + VirtualVariable, + VirtualVariableCategory, +) +from angr.ailment.manager import Manager +from angr.ailment.statement import Assignment, SideEffectStatement, Store +from angr.analyses.decompiler.optimization_passes.inlined_strcpy_simplifier import InlinedStrcpySimplifier +from angr.analyses.decompiler.optimization_passes.inlined_wcscpy_simplifier import InlinedWcscpySimplifier +from angr.analyses.decompiler.variable_map import variable_map_of + + +def _simplifier(cls): + project = angr.load_shellcode(b"\x90", arch="AMD64") + func = project.kb.functions.function(addr=0, name="dummy", create=True) + simplifier = object.__new__(cls) + simplifier._func = func # pyright: ignore[reportAttributeAccessIssue] + simplifier.manager = Manager() + return simplifier + + +def _stack_vvar(varid: int, offset: int, bits: int = 32): + return VirtualVariable(varid, varid, bits, VirtualVariableCategory.STACK, oident=offset) + + +def _register_vvar(varid: int, bits: int = 64): + return VirtualVariable(varid, varid, bits, VirtualVariableCategory.REGISTER, oident=0) + + +def _float_const(idx: int, value: float = 1.0, bits: int = 32): + return Const(idx, value, bits) # pyright: ignore[reportArgumentType] + + +def _integer_stack_assignment(idx: int, offset: int): + return Assignment(idx, _stack_vvar(idx, offset), Const(idx, 0x41414141, 32)) + + +def _inlined_wcsncpy(simplifier, idx: int, offset: int, data: bytes, count=None): + string_id = simplifier.kb.custom_strings.allocate(data) + string_const = Const(idx, string_id, 64) + variable_map_of(simplifier.manager).set_custom_string(string_const) + if count is None: + count = Const(idx + 1, len(data) // 2, 64) + call = Call( + idx + 2, + "wcsncpy", + args=[StackBaseOffset(idx + 3, 64, offset), string_const, count], + ) + return SideEffectStatement(idx + 4, call) + + +def test_strcpy_collector_rejects_float_stack_assignment(): + simplifier = _simplifier(InlinedStrcpySimplifier) + statements = [ + _integer_stack_assignment(0, -8), + Assignment(1, _stack_vvar(1, -4), _float_const(1)), + ] + + collected = simplifier._collect_constant_stores(statements, 0) + + assert collected[-4][1] is None + + +def test_strcpy_collector_rejects_float_insert_value_and_offset(): + simplifier = _simplifier(InlinedStrcpySimplifier) + dst = _stack_vvar(1, -4) + statements = [ + _integer_stack_assignment(0, -8), + Assignment(1, dst, Insert(1, dst, Const(2, 0, 32), _float_const(3), "Iend_LE")), + ] + collected = simplifier._collect_constant_stores(statements, 0) + assert collected[-4][1] is None + + statements[1] = Assignment( + 1, + dst, + Insert(1, dst, _float_const(2, 0.0), Const(3, 0x41, 8), "Iend_LE"), + ) + collected = simplifier._collect_constant_stores(statements, 0) + assert collected[-4][1] is None + + +def test_strcpy_collector_rejects_float_insert_base(): + simplifier = _simplifier(InlinedStrcpySimplifier) + dst = _stack_vvar(1, -8) + statements = [ + Assignment( + 1, + dst, + Insert(1, _float_const(2), Const(3, 4, 32), Const(4, 0x44434241, 32), "Iend_LE"), + ) + ] + + collected = simplifier._collect_constant_stores(statements, 0) + + assert -4 not in collected + assert collected[-8][1] is None + + +def test_strcpy_single_statement_rejects_float_insert_offset(): + simplifier = _simplifier(InlinedStrcpySimplifier) + dst = _stack_vvar(0, -4) + stmt = Assignment( + 0, + dst, + Insert(0, dst, _float_const(1, 0.0), Const(2, 0x44434241, 32), "Iend_LE"), + ) + + assert simplifier._optimize_single_stmt(stmt, 0, [stmt]) is None + + +def test_strcpy_single_statement_rejects_float_insert_base(): + simplifier = _simplifier(InlinedStrcpySimplifier) + dst = _stack_vvar(0, -4) + stmt = Assignment( + 0, + dst, + Insert(0, _float_const(1), Const(2, 0, 32), Const(3, 0x44434241, 32), "Iend_LE"), + ) + + assert simplifier._optimize_single_stmt(stmt, 0, [stmt]) is None + + +def test_strcpy_collector_rejects_float_stack_store(): + simplifier = _simplifier(InlinedStrcpySimplifier) + statements = [ + _integer_stack_assignment(0, -8), + Store(1, StackBaseOffset(1, 64, -4), _float_const(1), 4, "Iend_LE"), + ] + + collected = simplifier._collect_constant_stores(statements, 0) + + assert collected[-4][1] is None + + +def test_strcpy_collector_keeps_integer_insert_and_stack_store(): + simplifier = _simplifier(InlinedStrcpySimplifier) + dst = _stack_vvar(0, -8) + statements = [ + Assignment(0, dst, Insert(0, dst, Const(1, 0, 32), Const(2, 0x44434241, 32), "Iend_LE")), + Store(1, StackBaseOffset(1, 64, -4), Const(3, 0x48474645, 32), 4, "Iend_LE"), + ] + + collected = simplifier._collect_constant_stores(statements, 0) + + assert collected[-8][1].is_int + assert collected[-4][1].is_int + + +def test_strcpy_consolidation_rejects_float_store(): + simplifier = _simplifier(InlinedStrcpySimplifier) + dst = StackBaseOffset(0, 64, -8) + string_id = simplifier.kb.custom_strings.allocate(b"abcd") + string_const = Const(1, string_id, 64) + variable_map_of(simplifier.manager).set_custom_string(string_const) + call = Call(2, "strncpy", args=[dst, string_const, Const(3, 4, 64)]) + inlined_strcpy = SideEffectStatement(4, call) + float_store = Store(5, StackBaseOffset(5, 64, -4), _float_const(6), 4, "Iend_LE") + + assert simplifier._consolidate_pair(inlined_strcpy, float_store) is None + + +def test_strcpy_address_parser_rejects_float_offset(): + base = _register_vvar(0) + addr = BinaryOp(1, "Add", [base, _float_const(2, 4.0, 64)]) + + assert InlinedStrcpySimplifier._get_delta(base, addr) is None + + +def test_wcscpy_collector_rejects_float_stack_assignment(): + simplifier = _simplifier(InlinedWcscpySimplifier) + statements = [ + _integer_stack_assignment(0, -8), + Assignment(1, _stack_vvar(1, -4), _float_const(1)), + ] + + collected = simplifier._collect_constant_stores(statements, 0) + + assert collected[-4][1] is None + + +def test_wcscpy_collector_rejects_float_store_and_offset(): + simplifier = _simplifier(InlinedWcscpySimplifier) + base = _register_vvar(0) + statements = [ + Store(0, base, Const(0, 0x41004200, 32), 4, "Iend_LE"), + Store( + 1, + BinaryOp(1, "Add", [base, Const(1, 4, 64)]), + _float_const(1), + 4, + "Iend_LE", + ), + ] + collected = simplifier._collect_constant_stores(statements, 0) + assert collected[4][1] is None + + statements[1] = Store( + 1, + BinaryOp(1, "Add", [base, _float_const(1, 4.0, 64)]), + Const(1, 0x43004400, 32), + 4, + "Iend_LE", + ) + collected = simplifier._collect_constant_stores(statements, 0) + assert 4 not in collected + + +def test_wcscpy_collector_keeps_integer_store_and_offset(): + simplifier = _simplifier(InlinedWcscpySimplifier) + base = _register_vvar(0) + statements = [ + Store(0, base, Const(0, 0x41004200, 32), 4, "Iend_LE"), + Store( + 1, + BinaryOp(1, "Add", [base, Const(1, 4, 64)]), + Const(2, 0x43004400, 32), + 4, + "Iend_LE", + ), + ] + + collected = simplifier._collect_constant_stores(statements, 0) + + assert collected[0][1].is_int + assert collected[4][1].is_int + + +def test_wcscpy_consolidation_preserves_float_store_as_overlap_barrier(): + simplifier = _simplifier(InlinedWcscpySimplifier) + call = _inlined_wcsncpy(simplifier, 0, 0, b"A\x00B\x00") + float_store = Store(5, StackBaseOffset(6, 64, 4), _float_const(7, bits=16), 2, "Iend_LE") + final_store = Store(8, StackBaseOffset(9, 64, 4), Const(10, 0, 16), 2, "Iend_LE") + + assert simplifier._consolidate_wcscpy_calls([call, final_store]) is not None + assert simplifier._consolidate_wcscpy_calls([call, float_store, final_store]) is None + + +def test_wcscpy_consolidation_preserves_float_assignment_as_overlap_barrier(): + simplifier = _simplifier(InlinedWcscpySimplifier) + call = _inlined_wcsncpy(simplifier, 0, 0, b"A\x00B\x00") + float_assignment = Assignment(5, _stack_vvar(6, 4, bits=16), _float_const(7, bits=16)) + final_assignment = Assignment(8, _stack_vvar(9, 4, bits=16), Const(10, 0x43, 16)) + + assert simplifier._consolidate_wcscpy_calls([call, final_assignment]) is not None + assert simplifier._consolidate_wcscpy_calls([call, float_assignment, final_assignment]) is None + + +def test_wcscpy_consolidation_aborts_on_noninteger_wcsncpy_count(): + simplifier = _simplifier(InlinedWcscpySimplifier) + invalid_call = _inlined_wcsncpy(simplifier, 0, 8, b"C\x00", count=_float_const(1, 1.0, 64)) + valid_call = _inlined_wcsncpy(simplifier, 10, 0, b"A\x00B\x00") + final_store = Store(20, StackBaseOffset(21, 64, 4), Const(22, 0, 16), 2, "Iend_LE") + + assert simplifier._consolidate_wcscpy_calls([valid_call, final_store]) is not None + assert simplifier._consolidate_wcscpy_calls([invalid_call, valid_call, final_store]) is None + + +def test_wcscpy_wide_string_predicates_reject_floats(): + assert not InlinedWcscpySimplifier.even_offsets_are_zero([0.0, 65.0]) + assert not InlinedWcscpySimplifier.odd_offsets_are_zero([65.0, 0.0]) + assert InlinedWcscpySimplifier.is_integer_likely_a_wide_string(1.0, 4, "Iend_LE") == (False, None) From 90062a9914be3e9ef683cb29fb5d32bf8d222883 Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Fri, 24 Jul 2026 00:43:03 -0700 Subject: [PATCH 044/122] Support native AIL Abs expressions in light engines (#6683) * Fix AIL Abs unary operation dispatch * Fix Abs regression test lint --- .../optimization_passes/engine_base.py | 1 + ...nlined_string_transformation_simplifier.py | 3 + angr/analyses/purity/engine.py | 1 + .../reaching_definitions/engine_ail.py | 4 + angr/analyses/variable_recovery/engine_ail.py | 7 ++ angr/engines/ail/engine_light.py | 6 ++ angr/engines/light/engine.py | 5 ++ tests/engines/light/test_light_engine.py | 81 ++++++++++++++++++- tests/sim/test_ail_exec.py | 26 ++++++ 9 files changed, 133 insertions(+), 1 deletion(-) diff --git a/angr/analyses/decompiler/optimization_passes/engine_base.py b/angr/analyses/decompiler/optimization_passes/engine_base.py index 644d2688e..967be3698 100644 --- a/angr/analyses/decompiler/optimization_passes/engine_base.py +++ b/angr/analyses/decompiler/optimization_passes/engine_base.py @@ -377,6 +377,7 @@ class SimplifierAILEngine( return ailment.expression.UnaryOp(expr.idx, expr.op, operand, **expr.tags) return expr + _handle_unop_Abs = _handle_unop_Default _handle_unop_Not = _handle_unop_Default _handle_unop_Neg = _handle_unop_Default _handle_unop_BitwiseNeg = _handle_unop_Default diff --git a/angr/analyses/decompiler/optimization_passes/inlined_string_transformation_simplifier.py b/angr/analyses/decompiler/optimization_passes/inlined_string_transformation_simplifier.py index a5f8eaedc..82c56f76a 100644 --- a/angr/analyses/decompiler/optimization_passes/inlined_string_transformation_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/inlined_string_transformation_simplifier.py @@ -347,6 +347,9 @@ class InlinedStringTransformationAILEngine( return ~v return None + def _handle_unop_Abs(self, expr: UnaryOp): + self._expr(expr.operand) + def _handle_unop_Default(self, expr: UnaryOp): return None diff --git a/angr/analyses/purity/engine.py b/angr/analyses/purity/engine.py index b98dd5188..35c596a80 100644 --- a/angr/analyses/purity/engine.py +++ b/angr/analyses/purity/engine.py @@ -420,6 +420,7 @@ class PurityEngineAIL(SimEngineLightAIL[StateType, DataType_co, StmtDataType, Re def _handle_unop_default(self, expr: ailment.Expression) -> DataType_co: return self._expr_noconst(expr.operand) + _handle_unop_Abs = _handle_unop_default _handle_unop_Clz = _handle_unop_default _handle_unop_Ctz = _handle_unop_default _handle_unop_GetMSBs = _handle_unop_default diff --git a/angr/analyses/reaching_definitions/engine_ail.py b/angr/analyses/reaching_definitions/engine_ail.py index a0e7dc5ac..a8fd40198 100644 --- a/angr/analyses/reaching_definitions/engine_ail.py +++ b/angr/analyses/reaching_definitions/engine_ail.py @@ -603,6 +603,10 @@ class SimEngineRDAIL( def _handle_unop_Default(self, expr): return self._top(expr.bits) + def _handle_unop_Abs(self, expr): + self._expr(expr.operand) + return self._top(expr.bits) + _handle_unop_Reference = _handle_unop_Default _handle_unop_Ctz = _handle_unop_Default _handle_unop_Dereference = _handle_unop_Default diff --git a/angr/analyses/variable_recovery/engine_ail.py b/angr/analyses/variable_recovery/engine_ail.py index f3cb8f3b6..344e85ac4 100644 --- a/angr/analyses/variable_recovery/engine_ail.py +++ b/angr/analyses/variable_recovery/engine_ail.py @@ -1294,6 +1294,13 @@ class SimEngineVRAIL( r = self.state.top(result_size) return RichR(r, typevar=expr.typevar) + def _handle_unop_Abs(self, expr): + operand = self._expr(expr.operand) + return cast( + RichR[claripy.ast.BV | claripy.ast.FP], + RichR(self.state.top(expr.bits), typevar=operand.typevar), + ) + def _handle_unop_Default(self, expr: ailment.expression.UnaryOp) -> RichR[claripy.ast.BV | claripy.ast.FP]: self._expr(expr.operands[0]) return cast(RichR[claripy.ast.BV | claripy.ast.FP], RichR(self.state.top(expr.bits))) diff --git a/angr/engines/ail/engine_light.py b/angr/engines/ail/engine_light.py index 34a19ff10..57ff0a32c 100644 --- a/angr/engines/ail/engine_light.py +++ b/angr/engines/ail/engine_light.py @@ -642,6 +642,12 @@ class SimEngineAILSimState(SimEngineLightAIL[StateType, DataType, bool, None]): v = self._expr_bv(expr.operand) return ~v + def _handle_unop_Abs(self, expr: ailment.UnaryOp) -> DataType: + value = self._expr(expr.operand) + if isinstance(value, claripy.ast.FP): + return claripy.fpAbs(value) + return self._top(expr.bits) + def _handle_unop_Reference(self, expr: ailment.expression.UnaryOp) -> DataType: match expr.operand: case ailment.expression.VirtualVariable(): diff --git a/angr/engines/light/engine.py b/angr/engines/light/engine.py index 347c884f2..4f42876cd 100644 --- a/angr/engines/light/engine.py +++ b/angr/engines/light/engine.py @@ -571,6 +571,7 @@ class SimEngineLightAIL[StateType, DataType_co, StmtDataType, ResultType]( "ComboRegister": self._handle_expr_ComboRegister, } self._unop_handlers: dict[str, Callable[[ailment.UnaryOp], DataType_co]] = { + "Abs": self._handle_unop_Abs, "Not": self._handle_unop_Not, "Neg": self._handle_unop_Neg, "BitwiseNeg": self._handle_unop_BitwiseNeg, @@ -852,6 +853,10 @@ class SimEngineLightAIL[StateType, DataType_co, StmtDataType, ResultType]( # UnOps # + def _handle_unop_Abs(self, expr: ailment.expression.UnaryOp) -> DataType_co: + self._expr(expr.operand) + return self._top(expr.bits) + @abstractmethod def _handle_unop_Not(self, expr: ailment.expression.UnaryOp) -> DataType_co: ... diff --git a/tests/engines/light/test_light_engine.py b/tests/engines/light/test_light_engine.py index 0a5fcd518..9e411fad4 100644 --- a/tests/engines/light/test_light_engine.py +++ b/tests/engines/light/test_light_engine.py @@ -1,10 +1,26 @@ #!/usr/bin/env python3 -# pylint: disable=missing-class-docstring,no-self-use +# pylint: disable=missing-class-docstring,no-self-use,protected-access from __future__ import annotations +from types import SimpleNamespace +from typing import Any from unittest import TestCase, main +import archinfo +import claripy + +from angr import ailment +from angr.analyses.decompiler.optimization_passes.engine_base import SimplifierAILEngine +from angr.analyses.decompiler.optimization_passes.inlined_string_transformation_simplifier import ( + InlinedStringTransformationAILEngine, +) +from angr.analyses.purity.engine import DataSource, PurityEngineAIL +from angr.analyses.reaching_definitions.engine_ail import SimEngineRDAIL +from angr.analyses.typehoon.typevars import TypeVariable +from angr.analyses.variable_recovery.engine_ail import SimEngineVRAIL +from angr.analyses.variable_recovery.engine_base import RichR from angr.engines.light.engine import longest_prefix_lookup +from angr.storage.memory_mixins.paged_memory.pages.multi_values import MultiValues class TestLightEngine(TestCase): @@ -21,6 +37,69 @@ class TestLightEngine(TestCase): assert longest_prefix_lookup("32to1_foo", mapping) is handle_unop_32to1 assert longest_prefix_lookup("32to", mapping) is None + def test_ail_abs_unop_dispatch(self): + arch = archinfo.ArchAMD64() + engine = SimplifierAILEngine(SimpleNamespace(arch=arch)) # pyright: ignore[reportArgumentType] + operand = ailment.Expr.Const(0, 1.0, 32) # pyright: ignore[reportArgumentType] + + abs_expr = ailment.Expr.UnaryOp(1, "Abs", operand) + assert engine._handle_expr_UnaryOp(abs_expr) is abs_expr + + def test_inlined_string_abs_visits_operand(self): + engine: Any = object.__new__(InlinedStringTransformationAILEngine) + seen = [] + engine._expr = seen.append + operand = ailment.Expr.Const(0, 1.0, 32) # pyright: ignore[reportArgumentType] + abs_expr = ailment.Expr.UnaryOp(1, "Abs", operand) + + engine._handle_unop_Abs(abs_expr) + + assert seen == [operand] + + def test_rda_abs_visits_operand(self): + engine: Any = object.__new__(SimEngineRDAIL) + seen = [] + engine._expr = lambda expr: (seen.append(expr), MultiValues(claripy.BVV(0, expr.bits)))[1] + engine.state = SimpleNamespace(top=lambda bits: claripy.BVS("rda_abs_top", bits)) + operand = ailment.Expr.Const(0, 0, 32) + abs_expr = ailment.Expr.UnaryOp(1, "Abs", operand) + + result = engine._handle_unop_Abs(abs_expr) + + assert seen == [operand] + assert isinstance(result, MultiValues) + + def test_variable_recovery_abs_preserves_typevar(self): + operand_typevar = TypeVariable(name="abs_operand") + engine: Any = object.__new__(SimEngineVRAIL) + seen = [] + engine._expr = lambda expr: ( + seen.append(expr), + RichR(claripy.BVV(0, expr.bits), typevar=operand_typevar), + )[1] + engine.state = SimpleNamespace(top=lambda bits: claripy.BVS("vr_abs_top", bits)) + operand = ailment.Expr.Const(0, 0, 32) + abs_expr = ailment.Expr.UnaryOp(1, "Abs", operand) + + result = engine._handle_unop_Abs(abs_expr) + + assert seen == [operand] + assert result.typevar is operand_typevar + assert len(result.data) == 32 + + def test_purity_abs_preserves_provenance(self): + provenance = frozenset((DataSource(function_arg=0),)) + engine: Any = object.__new__(PurityEngineAIL) + seen = [] + engine._expr_noconst = lambda expr: (seen.append(expr), provenance)[1] + operand = ailment.Expr.Const(0, 0, 32) + abs_expr = ailment.Expr.UnaryOp(1, "Abs", operand) + + result = engine._handle_unop_Abs(abs_expr) + + assert seen == [operand] + assert result is provenance + if __name__ == "__main__": main() diff --git a/tests/sim/test_ail_exec.py b/tests/sim/test_ail_exec.py index db420ad61..3786cf471 100644 --- a/tests/sim/test_ail_exec.py +++ b/tests/sim/test_ail_exec.py @@ -23,6 +23,32 @@ test_location = os.path.join(bin_location, "tests") class TestAILExec(unittest.TestCase): + def test_abs_expression_preserves_fp_sort(self): + class _Engine(SimEngineAILSimState): + value: claripy.ast.Bits + + def _expr(self, expr): # pylint: disable=unused-argument + return self.value + + engine = object.__new__(_Engine) + expr = SimpleNamespace(operand=object(), bits=32) + + engine.value = claripy.FPV(-1.5, claripy.FSORT_FLOAT) + fp_result = engine._handle_unop_Abs(expr) # pyright: ignore[reportArgumentType] # pylint: disable=protected-access + assert isinstance(fp_result, claripy.ast.FP) + assert fp_result.sort == claripy.FSORT_FLOAT + assert fp_result.concrete and fp_result.args[0] == 1.5 + + engine.value = claripy.BVV(0x80000000, 32) + bv_result = engine._handle_unop_Abs(expr) # pyright: ignore[reportArgumentType] # pylint: disable=protected-access + assert isinstance(bv_result, claripy.ast.BV) + assert len(bv_result) == 32 + assert ( + bv_result.op == "BVS" + and isinstance(bv_result.args[0], str) + and bv_result.args[0].startswith("ail_engine_top") + ) + def test_haddv_expression(self): p = angr.load_shellcode(b"\x00", arch="ARMEL", load_address=0x400000) state = p.factory.blank_state() From 91cc026062142cf8dbbbefbb6c5aacd7f09f54f7 Mon Sep 17 00:00:00 2001 From: Fish Date: Fri, 24 Jul 2026 01:43:20 -0700 Subject: [PATCH 045/122] AILVexLifter: Fix libVEX overread by padding in convert_from_lift. (#6686) --- native/angr/src/ailment/convert_vex.rs | 24 ++++++++++++++++++- tests/ailment/test_irsb.py | 33 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/native/angr/src/ailment/convert_vex.rs b/native/angr/src/ailment/convert_vex.rs index cbd0c0a74..183381662 100644 --- a/native/angr/src/ailment/convert_vex.rs +++ b/native/angr/src/ailment/convert_vex.rs @@ -2120,7 +2120,29 @@ impl VEXIRSBConverter { if (bytes_offset as usize) >= data_len { return Err(PyValueError::new_err("bytes_offset past end of data")); } - let insn_start = unsafe { data_ptr.add(bytes_offset as usize) }; + + // libVEX decoders may read the full length of an instruction that + // *starts* inside the [bytes_offset, bytes_offset + mb) window -- i.e. + // a few bytes past `mb`, and past the end of the caller's buffer when + // the window ends at (or near) it. pyvex guards its Python path by + // lifting from a copy padded with 8 NUL bytes; mirror that here, but + // only when the window actually ends within 8 bytes of the buffer end + // so the common case stays zero-copy. + let window_end = bytes_offset as usize + mb as usize; + // Held until the end of this function so `insn_start` stays valid. + let _padded: Option>; + let lift_base: *const u8 = if window_end + 8 <= data_len { + _padded = None; + data_ptr + } else { + let mut v = Vec::with_capacity(data_len + 8); + v.extend_from_slice(unsafe { std::slice::from_raw_parts(data_ptr, data_len) }); + v.extend_from_slice(&[0u8; 8]); + let p = v.as_ptr(); + _padded = Some(v); + p + }; + let insn_start = unsafe { lift_base.add(bytes_offset as usize) }; // SAFETY: GIL is held for the whole call; we read the result before any // other lift can run. libVEX's `_lift_r`/arena stay valid until then. diff --git a/tests/ailment/test_irsb.py b/tests/ailment/test_irsb.py index f39ad7a73..c3e846265 100644 --- a/tests/ailment/test_irsb.py +++ b/tests/ailment/test_irsb.py @@ -238,6 +238,39 @@ class TestVexConverterAcrossArches(unittest.TestCase): self._check_binary(os.path.normpath(os.path.join(base, rel))) +class TestLiftWindowOverread(unittest.TestCase): + """libVEX decoders may read the full length of an instruction that starts + inside the lift window, i.e. a few bytes past ``max_bytes`` -- and past the + end of the buffer when the window ends at it. The fast path must pad such + windows with NULs (mirroring pyvex) instead of lifting adjacent heap + garbage, which made the block content (jumpkind, temp numbering) + nondeterministic.""" + + # 38 bytes at 0x400b5a in s390x/fauxware: nopr padding + function prologue, + # ending with the first 4 bytes of a 6-byte `lg` at 0x400b7c. The last two + # bytes of the `lg` fall outside the window; whether it decodes depends + # entirely on out-of-window bytes. + window = bytes.fromhex("070707070707eb6ff0300024b904001fa7fbff60e310f0000024c0c000000a060707e340f110") + addr = 0x400B5A + + def test_lift_does_not_read_past_window(self): + arch = archinfo.arch_from_id("s390x") + from_py = VEXIRSBConverter.convert( + pyvex.IRSB(self.window, self.addr, arch, opt_level=1), ailment.Manager(arch=arch) + ) + # 0x04 completes the truncated `lg`: an unguarded overread decodes it + # and ends the block Ijk_Boring instead of Ijk_NoDecode. + backing = bytearray(self.window + b"\x04" * 8) + from_lift_mv = VEXIRSBConverter.convert_from_lift( + arch, self.addr, memoryview(backing)[: len(self.window)], ailment.Manager(arch=arch), opt_level=1 + ) + from_lift_bytes = VEXIRSBConverter.convert_from_lift( + arch, self.addr, self.window, ailment.Manager(arch=arch), opt_level=1 + ) + assert from_py == from_lift_mv + assert from_py == from_lift_bytes + + class TestVexOpParity(unittest.TestCase): """The Rust vexop classifier must match Python ``vexop_to_simop`` for every VEX op (guards against drift in the hand-ported claripy/irop name-sets).""" From db21fb0fee39bc223411df732cf66e70e6809372 Mon Sep 17 00:00:00 2001 From: Ati Priya Date: Fri, 24 Jul 2026 11:28:39 -0700 Subject: [PATCH 046/122] Rewrite the amd64 CondO/CondNO ccall (#6693) * Rewrite the amd64 CondO/CondNO ccall family amd64g_calculate_condition with cond CondO/CondNO had no rewrite arm at all, so every jo/jno/seto/cmovno site leaked into the decompilation as an uncompilable _ccall(0|1, cc_op, ...). Add arms for the cc_op families that define OF: LOGIC{B,W,L,Q} and/or/xor always clear OF -> constant 0 / 1 ADD{B,W,L,Q} -> __OFADD__(dep_1, dep_2) SUB{B,W,L,Q} -> __OFSUB__(dep_1, dep_2) UMUL{B,W,L,Q} -> __OFUMUL__(dep_1, dep_2) SMUL{B,W,L,Q} -> __OFSMUL__(dep_1, dep_2) INC{B,W,L,Q} result == signed minimum DEC{B,W,L,Q} result == signed maximum COPY test the stored OF bit The overflow helpers follow the existing __CFADD__ arm: a named usercall whose operands carry the operation width. CondNO reuses the same helper and compares it against zero. Unsigned multiply overflow is defined as "the high half of the full 2N-bit product is nonzero", i.e. the product does not fit in N unsigned bits. Note this is NOT the threshold used by the x86 rewriter, which compares the product against 1 << (N - 1) -- that is the signed threshold, half the correct unsigned one, and it reports overflow for every product in [2^(N-1), 2^N - 1] even though those fit. At 8 bits it misclassifies 820 of 65536 operand pairs, all false positives. pc_actions_UMUL in the VEX ccall helpers is itself wrong here: it multiplies two N-bit values without widening, so its `>> nbits` is always zero and its CF/OF do not agree with the hardware. The rewrite arm follows the hardware and pc_actions_SMUL's (correct) structure instead; fixing the helper is left alone. Every arm was checked exhaustively at 8 bits against pc_calculate_condition, and the ADD/SUB/UMUL/SMUL arms additionally against real setcc results. * Drop the synthetic CondO fixture test The real gzip and file fixtures already cover the CondO arms; a purpose built binary added nothing that the unit tests do not already check. * Cover the CondNO overflow path with a real binary tar's argp helper guards a multiply with 'mul %rbp; jno', exercising the CondNO side of UMULQ that gzip and file do not reach. Other cc_op families still leak a ccall in that function, so only the OF conditions are asserted. * Cover the CondO overflow arms with three more real binaries coreutils cat, grep and zlib's minigzip each carry the xalloc / __builtin_mul_overflow idiom, between them exercising CondO against ADDQ, SMULQ and UMULQ across three separate projects. Verified symbols and addresses are cited on each test. * Correct the tar overflow test to CondO The jno there is canonicalized into CondO with an inverted branch, so the ccall reaching the rewriter carries cond 0, not cond 1. The negation seen in the output is the structurer's, not the condition's. --- .../ccall_rewriters/amd64_ccalls.py | 205 +++++++++ .../decompiler/test_ccall_rewriting.py | 404 ++++++++++++------ 2 files changed, 475 insertions(+), 134 deletions(-) diff --git a/angr/analyses/decompiler/ccall_rewriters/amd64_ccalls.py b/angr/analyses/decompiler/ccall_rewriters/amd64_ccalls.py index 0cae1be51..dd2773051 100644 --- a/angr/analyses/decompiler/ccall_rewriters/amd64_ccalls.py +++ b/angr/analyses/decompiler/ccall_rewriters/amd64_ccalls.py @@ -545,6 +545,184 @@ class AMD64CCallRewriter(CCallRewriterBase): ) return Expr.Convert(self.ail_manager.next_atom(), r.bits, ccall.bits, False, r, **ccall.tags) + elif cond_v in {AMD64_CondTypes["CondO"], AMD64_CondTypes["CondNO"]}: + # overflow flag (jo / jno) + is_no = cond_v == AMD64_CondTypes["CondNO"] + + if op_v in { + AMD64_OpTypes["G_CC_OP_LOGICB"], + AMD64_OpTypes["G_CC_OP_LOGICW"], + AMD64_OpTypes["G_CC_OP_LOGICL"], + AMD64_OpTypes["G_CC_OP_LOGICQ"], + }: + # and/or/xor always clear OF: CondO -> 0, CondNO -> 1 + return Expr.Const(self.ail_manager.next_atom(), 1 if is_no else 0, ccall.bits, **ccall.tags) + + if op_v in { + AMD64_OpTypes["G_CC_OP_ADDB"], + AMD64_OpTypes["G_CC_OP_ADDW"], + AMD64_OpTypes["G_CC_OP_ADDL"], + AMD64_OpTypes["G_CC_OP_ADDQ"], + }: + # signed overflow of dep_1 + dep_2 + dep_1 = self._fix_size( + dep_1, + op_v, + AMD64_OpTypes["G_CC_OP_ADDB"], + AMD64_OpTypes["G_CC_OP_ADDW"], + AMD64_OpTypes["G_CC_OP_ADDL"], + ccall.tags, + ) + dep_2 = self._fix_size( + dep_2, + op_v, + AMD64_OpTypes["G_CC_OP_ADDB"], + AMD64_OpTypes["G_CC_OP_ADDW"], + AMD64_OpTypes["G_CC_OP_ADDL"], + ccall.tags, + ) + return self._overflow_helper(ccall, "__OFADD__", dep_1, dep_2, is_no) + + if op_v in { + AMD64_OpTypes["G_CC_OP_SUBB"], + AMD64_OpTypes["G_CC_OP_SUBW"], + AMD64_OpTypes["G_CC_OP_SUBL"], + AMD64_OpTypes["G_CC_OP_SUBQ"], + }: + # signed overflow of dep_1 - dep_2 + dep_1 = self._fix_size( + dep_1, + op_v, + AMD64_OpTypes["G_CC_OP_SUBB"], + AMD64_OpTypes["G_CC_OP_SUBW"], + AMD64_OpTypes["G_CC_OP_SUBL"], + ccall.tags, + ) + dep_2 = self._fix_size( + dep_2, + op_v, + AMD64_OpTypes["G_CC_OP_SUBB"], + AMD64_OpTypes["G_CC_OP_SUBW"], + AMD64_OpTypes["G_CC_OP_SUBL"], + ccall.tags, + ) + return self._overflow_helper(ccall, "__OFSUB__", dep_1, dep_2, is_no) + + if op_v in { + AMD64_OpTypes["G_CC_OP_UMULB"], + AMD64_OpTypes["G_CC_OP_UMULW"], + AMD64_OpTypes["G_CC_OP_UMULL"], + AMD64_OpTypes["G_CC_OP_UMULQ"], + }: + # unsigned multiply overflow: high half of the full product is nonzero + dep_1 = self._fix_size( + dep_1, + op_v, + AMD64_OpTypes["G_CC_OP_UMULB"], + AMD64_OpTypes["G_CC_OP_UMULW"], + AMD64_OpTypes["G_CC_OP_UMULL"], + ccall.tags, + ) + dep_2 = self._fix_size( + dep_2, + op_v, + AMD64_OpTypes["G_CC_OP_UMULB"], + AMD64_OpTypes["G_CC_OP_UMULW"], + AMD64_OpTypes["G_CC_OP_UMULL"], + ccall.tags, + ) + return self._overflow_helper(ccall, "__OFUMUL__", dep_1, dep_2, is_no) + + if op_v in { + AMD64_OpTypes["G_CC_OP_SMULB"], + AMD64_OpTypes["G_CC_OP_SMULW"], + AMD64_OpTypes["G_CC_OP_SMULL"], + AMD64_OpTypes["G_CC_OP_SMULQ"], + }: + # signed multiply overflow + dep_1 = self._fix_size( + dep_1, + op_v, + AMD64_OpTypes["G_CC_OP_SMULB"], + AMD64_OpTypes["G_CC_OP_SMULW"], + AMD64_OpTypes["G_CC_OP_SMULL"], + ccall.tags, + ) + dep_2 = self._fix_size( + dep_2, + op_v, + AMD64_OpTypes["G_CC_OP_SMULB"], + AMD64_OpTypes["G_CC_OP_SMULW"], + AMD64_OpTypes["G_CC_OP_SMULL"], + ccall.tags, + ) + return self._overflow_helper(ccall, "__OFSMUL__", dep_1, dep_2, is_no) + + if op_v in { + AMD64_OpTypes["G_CC_OP_INCB"], + AMD64_OpTypes["G_CC_OP_INCW"], + AMD64_OpTypes["G_CC_OP_INCL"], + AMD64_OpTypes["G_CC_OP_INCQ"], + }: + # inc overflows only when the result is the signed minimum + nbits = self._op_nbits( + op_v, + AMD64_OpTypes["G_CC_OP_INCB"], + AMD64_OpTypes["G_CC_OP_INCW"], + AMD64_OpTypes["G_CC_OP_INCL"], + ) + dep_1 = self._fix_size( + dep_1, + op_v, + AMD64_OpTypes["G_CC_OP_INCB"], + AMD64_OpTypes["G_CC_OP_INCW"], + AMD64_OpTypes["G_CC_OP_INCL"], + ccall.tags, + ) + signmin = Expr.Const(self.ail_manager.next_atom(), 1 << (nbits - 1), dep_1.bits) + expr_op = "CmpNE" if is_no else "CmpEQ" + r = Expr.BinaryOp(ccall.idx, expr_op, (dep_1, signmin), False, **ccall.tags) + return Expr.Convert(self.ail_manager.next_atom(), r.bits, ccall.bits, False, r, **ccall.tags) + + if op_v in { + AMD64_OpTypes["G_CC_OP_DECB"], + AMD64_OpTypes["G_CC_OP_DECW"], + AMD64_OpTypes["G_CC_OP_DECL"], + AMD64_OpTypes["G_CC_OP_DECQ"], + }: + # dec overflows only when the result is the signed maximum + nbits = self._op_nbits( + op_v, + AMD64_OpTypes["G_CC_OP_DECB"], + AMD64_OpTypes["G_CC_OP_DECW"], + AMD64_OpTypes["G_CC_OP_DECL"], + ) + dep_1 = self._fix_size( + dep_1, + op_v, + AMD64_OpTypes["G_CC_OP_DECB"], + AMD64_OpTypes["G_CC_OP_DECW"], + AMD64_OpTypes["G_CC_OP_DECL"], + ccall.tags, + ) + signmax = Expr.Const(self.ail_manager.next_atom(), (1 << (nbits - 1)) - 1, dep_1.bits) + expr_op = "CmpNE" if is_no else "CmpEQ" + r = Expr.BinaryOp(ccall.idx, expr_op, (dep_1, signmax), False, **ccall.tags) + return Expr.Convert(self.ail_manager.next_atom(), r.bits, ccall.bits, False, r, **ccall.tags) + + if op_v == AMD64_OpTypes["G_CC_OP_COPY"]: + # dep_1 holds the packed flags; test the stored OF bit + bitmask = AMD64_CondBitMasks["G_CC_MASK_O"] + assert isinstance(bitmask, int) + flag = Expr.Const(self.ail_manager.next_atom(), bitmask, dep_1.bits) + masked_dep = Expr.BinaryOp( + self.ail_manager.next_atom(), "And", [dep_1, flag], False, **ccall.tags + ) + zero = Expr.Const(self.ail_manager.next_atom(), 0, dep_1.bits) + expr_op = "CmpEQ" if is_no else "CmpNE" + r = Expr.BinaryOp(ccall.idx, expr_op, (masked_dep, zero), False, **ccall.tags) + return Expr.Convert(self.ail_manager.next_atom(), r.bits, ccall.bits, False, r, **ccall.tags) + elif ccall.callee == "amd64g_calculate_rflags_c": # calculate the carry flag op = ccall.operands[0] @@ -660,6 +838,33 @@ class AMD64CCallRewriter(CCallRewriterBase): return None + @staticmethod + def _op_nbits(op_v: int, type_8bit, type_16bit, type_32bit) -> int: + if op_v == type_8bit: + return 8 + if op_v == type_16bit: + return 16 + if op_v == type_32bit: + return 32 + return 64 + + def _overflow_helper(self, ccall, name: str, dep_1, dep_2, is_no: bool): + # Emit a named overflow-helper call (mirrors the __CFADD__ arm). The helper + # returns a 0/1 flag; for the negated condition (CondNO) compare it to 0. + call = Expr.Call( + ccall.idx, + name, + args=[dep_1, dep_2], + bits=ccall.bits, + **ccall.tags, + ) + variable_map_of(self.ail_manager).set_calling_convention(call, SimCCUsercall(self.project.arch, [], None)) + if not is_no: + return call + zero = Expr.Const(self.ail_manager.next_atom(), 0, ccall.bits) + r = Expr.BinaryOp(self.ail_manager.next_atom(), "CmpEQ", (call, zero), False, **ccall.tags) + return Expr.Convert(self.ail_manager.next_atom(), r.bits, ccall.bits, False, r, **ccall.tags) + def _fix_size(self, expr, op_v: int, type_8bit, type_16bit, type_32bit, tags): if op_v == type_8bit: bits = 8 diff --git a/tests/analyses/decompiler/test_ccall_rewriting.py b/tests/analyses/decompiler/test_ccall_rewriting.py index 5b682c8c5..6212a3354 100644 --- a/tests/analyses/decompiler/test_ccall_rewriting.py +++ b/tests/analyses/decompiler/test_ccall_rewriting.py @@ -2,27 +2,55 @@ # pylint: disable=missing-class-docstring,no-self-use,no-member from __future__ import annotations +from typing import Any, cast + __package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin -import itertools import os import unittest -import claripy - import angr from angr.ailment import Expr, Manager from angr.analyses.decompiler.ccall_rewriters.amd64_ccalls import AMD64CCallRewriter -from angr.engines.vex.claripy.ccall import data, pc_calculate_condition +from angr.engines.vex.claripy.ccall import data from tests.common import bin_location, print_decompilation_result test_location = os.path.join(bin_location, "tests") -AMD64_CondTypes = data["AMD64"]["CondTypes"] -AMD64_OpTypes = data["AMD64"]["OpTypes"] +AMD64_CondTypes = cast("dict[str, int]", data["AMD64"]["CondTypes"]) +AMD64_OpTypes = cast("dict[str, int]", data["AMD64"]["OpTypes"]) AMD64_CondBitMasks = data["AMD64"]["CondBitMasks"] +def _rewrite_amd64_cond(cond_v, op_v, dep_1=None, dep_2=None, bits=64) -> Any: + """Build an amd64g_calculate_condition ccall and run the AMD64 rewriter on it.""" + if dep_1 is None: + dep_1 = Expr.Register(1, 16, 64) + if dep_2 is None: + dep_2 = Expr.Register(2, 24, 64) + ccall = Expr.VEXCCallExpression( + idx=0, + callee="amd64g_calculate_condition", + operands=( + Expr.Const(0, cond_v, 64), + Expr.Const(0, op_v, 64), + dep_1, + dep_2, + Expr.Const(0, 0, 64), + ), + bits=bits, + ) + proj = angr.load_shellcode(b"\x90", arch="AMD64") + return AMD64CCallRewriter(ccall, proj, Manager()).result + + +def _unwrap_convert(expr): + """Strip an outer Convert wrapper if present.""" + if isinstance(expr, Expr.Convert): + return expr.operand + return expr + + class TestCCallRewriting(unittest.TestCase): def test_NtGetCurrentPeb(self): bin_path = os.path.join( @@ -42,145 +70,253 @@ class TestCCallRewriting(unittest.TestCase): assert "v0 = NtGetCurrentPeb();" in dec.codegen.text -def _make_ccall(cond, op, dep1=None, dep2=None, ndep=None, bits=64): - """Build a VEXCCallExpression for amd64g_calculate_condition.""" - if dep1 is None: - dep1 = Expr.Register(1, 16, 64) # rax - if dep2 is None: - dep2 = Expr.Register(2, 24, 64) # rcx - if ndep is None: - ndep = Expr.Const(3, 0, 64) - return Expr.VEXCCallExpression( - idx=0, - callee="amd64g_calculate_condition", - operands=(Expr.Const(0, cond, 64), Expr.Const(0, op, 64), dep1, dep2, ndep), - bits=bits, - ) +class TestAMD64CondOverflowRewriting(unittest.TestCase): + """Rewriting of the CondO / CondNO (jo / jno) family for amd64g_calculate_condition.""" + # ---- LOGIC: and/or/xor always clear OF ---- -_PROJECT = angr.load_shellcode(b"\x90", arch="AMD64") - - -def _rewrite(ccall): - return AMD64CCallRewriter(ccall, _PROJECT, Manager(arch=_PROJECT.arch)).result - - -def _unwrap_convert(expr): - """Strip an outer Convert wrapper if present.""" - return expr.operand if isinstance(expr, Expr.Convert) else expr - - -def _mask(v, bits): - return v & ((1 << bits) - 1) - - -def _sext(v, bits): - v = _mask(v, bits) - return v - (1 << bits) if v >> (bits - 1) else v - - -def _eval(expr): - """Concretely evaluate a rewritten (constant-folded) AIL expression. Returns (value, bits).""" - if isinstance(expr, Expr.Const): - return _mask(expr.value_int, expr.bits), expr.bits - if isinstance(expr, Expr.Convert): - v, _ = _eval(expr.operand) - v = _mask(_sext(v, expr.from_bits) if expr.is_signed else v, expr.to_bits) - return v, expr.to_bits - if isinstance(expr, Expr.Call) and expr.target == "__CFADD__": - # carry-out of the addition at the operands' width - left, lbits = _eval(expr.args[0]) - right, rbits = _eval(expr.args[1]) - bits = max(lbits, rbits) - return int(_mask(left + right, bits) < left), expr.bits - if isinstance(expr, Expr.BinaryOp): - left, lbits = _eval(expr.operands[0]) - right, rbits = _eval(expr.operands[1]) - bits = max(lbits, rbits) - if expr.signed: - left, right = _sext(left, lbits), _sext(right, rbits) - cmps = { - "CmpEQ": lambda: (int(left == right), 1), - "CmpNE": lambda: (int(left != right), 1), - "CmpLT": lambda: (int(left < right), 1), - "CmpLE": lambda: (int(left <= right), 1), - "CmpGT": lambda: (int(left > right), 1), - "CmpGE": lambda: (int(left >= right), 1), - "And": lambda: (_mask(left & right, bits), bits), - "Add": lambda: (_mask(left + right, bits), bits), - } - if expr.op not in cmps: - raise NotImplementedError(expr.op) - return cmps[expr.op]() - raise NotImplementedError(type(expr)) - - -def _oracle(cond, op, dep1, dep2, ndep=0): - """Ground truth: ccall.py's executable amd64g_calculate_condition.""" - r = pc_calculate_condition( - None, - claripy.BVV(cond, 64), - claripy.BVV(op, 64), - claripy.BVV(dep1, 64), - claripy.BVV(dep2, 64), - claripy.BVV(ndep, 64), - platform="AMD64", - ) - return bool(claripy.backends.concrete.eval(r, 1)[0]) - - -def _rewritten_value(cond, op, dep1, dep2, ndep=0): - ccall = _make_ccall(cond, op, Expr.Const(1, dep1, 64), Expr.Const(2, dep2, 64), Expr.Const(3, ndep, 64)) - result = _rewrite(ccall) - assert result is not None - return bool(_eval(result)[0]) - - -class TestAMD64CCallRewriterCondNL(unittest.TestCase): - """CondNL (jge, SF == OF). Signed >= over SUB; sign-of-result >= 0 over LOGIC.""" - - def test_condnl_sub_is_signed_ge(self): - for op in ("G_CC_OP_SUBB", "G_CC_OP_SUBW", "G_CC_OP_SUBL", "G_CC_OP_SUBQ"): - cmp = _unwrap_convert(_rewrite(_make_ccall(AMD64_CondTypes["CondNL"], AMD64_OpTypes[op]))) - assert isinstance(cmp, Expr.BinaryOp), f"{op}: not rewritten" - assert cmp.op == "CmpGE", f"{op}: got {cmp.op}" - assert cmp.signed is True, f"{op}: expected signed" - - def test_condnl_logic_is_signed_ge_zero(self): + def test_logic_o_is_false(self): for op in ("G_CC_OP_LOGICB", "G_CC_OP_LOGICW", "G_CC_OP_LOGICL", "G_CC_OP_LOGICQ"): - cmp = _unwrap_convert(_rewrite(_make_ccall(AMD64_CondTypes["CondNL"], AMD64_OpTypes[op]))) - assert isinstance(cmp, Expr.BinaryOp), f"{op}: not rewritten" - assert cmp.op == "CmpGE", f"{op}: got {cmp.op}" - assert cmp.signed is True, f"{op}: expected signed" - assert cmp.operands[1].value_int == 0, f"{op}: expected comparison against 0" + result = _rewrite_amd64_cond(AMD64_CondTypes["CondO"], AMD64_OpTypes[op]) + assert isinstance(result, Expr.Const), op + assert result.value_int == 0, op + assert result.bits == 64, op + + def test_logic_no_is_true(self): + for op in ("G_CC_OP_LOGICB", "G_CC_OP_LOGICW", "G_CC_OP_LOGICL", "G_CC_OP_LOGICQ"): + result = _rewrite_amd64_cond(AMD64_CondTypes["CondNO"], AMD64_OpTypes[op]) + assert isinstance(result, Expr.Const), op + assert result.value_int == 1, op + + # ---- ADD / SUB: signed overflow helpers ---- + + def test_add_o_emits_ofadd(self): + for op in ("G_CC_OP_ADDB", "G_CC_OP_ADDW", "G_CC_OP_ADDL", "G_CC_OP_ADDQ"): + result = _rewrite_amd64_cond(AMD64_CondTypes["CondO"], AMD64_OpTypes[op]) + assert isinstance(result, Expr.Call), op + assert result.target == "__OFADD__", op + assert len(result.args) == 2, op + assert result.bits == 64, op + + def test_sub_o_emits_ofsub(self): + for op in ("G_CC_OP_SUBB", "G_CC_OP_SUBW", "G_CC_OP_SUBL", "G_CC_OP_SUBQ"): + result = _rewrite_amd64_cond(AMD64_CondTypes["CondO"], AMD64_OpTypes[op]) + assert isinstance(result, Expr.Call), op + assert result.target == "__OFSUB__", op + + def test_add_no_negates_ofadd(self): + result = _rewrite_amd64_cond(AMD64_CondTypes["CondNO"], AMD64_OpTypes["G_CC_OP_ADDQ"]) + inner = _unwrap_convert(result) + assert isinstance(inner, Expr.BinaryOp) and inner.op == "CmpEQ" + assert isinstance(inner.operands[0], Expr.Call) and inner.operands[0].target == "__OFADD__" + assert isinstance(inner.operands[1], Expr.Const) and inner.operands[1].value_int == 0 + assert result.bits == 64 + + def test_sub_no_negates_ofsub(self): + result = _rewrite_amd64_cond(AMD64_CondTypes["CondNO"], AMD64_OpTypes["G_CC_OP_SUBQ"]) + inner = _unwrap_convert(result) + assert isinstance(inner, Expr.BinaryOp) and inner.op == "CmpEQ" + assert isinstance(inner.operands[0], Expr.Call) and inner.operands[0].target == "__OFSUB__" + + def test_add_operands_narrowed_to_op_width(self): + # the byte form must narrow both operands to 8 bits + result = _rewrite_amd64_cond(AMD64_CondTypes["CondO"], AMD64_OpTypes["G_CC_OP_ADDB"]) + assert all(arg.bits == 8 for arg in result.args) + + # ---- UMUL / SMUL: multiply overflow helpers ---- + + def test_umul_o_emits_ofumul(self): + for op in ("G_CC_OP_UMULB", "G_CC_OP_UMULW", "G_CC_OP_UMULL", "G_CC_OP_UMULQ"): + result = _rewrite_amd64_cond(AMD64_CondTypes["CondO"], AMD64_OpTypes[op]) + assert isinstance(result, Expr.Call), op + assert result.target == "__OFUMUL__", op + + def test_smul_o_emits_ofsmul(self): + for op in ("G_CC_OP_SMULB", "G_CC_OP_SMULW", "G_CC_OP_SMULL", "G_CC_OP_SMULQ"): + result = _rewrite_amd64_cond(AMD64_CondTypes["CondO"], AMD64_OpTypes[op]) + assert isinstance(result, Expr.Call), op + assert result.target == "__OFSMUL__", op + + def test_umul_no_negates_ofumul(self): + result = _rewrite_amd64_cond(AMD64_CondTypes["CondNO"], AMD64_OpTypes["G_CC_OP_UMULQ"]) + inner = _unwrap_convert(result) + assert isinstance(inner, Expr.BinaryOp) and inner.op == "CmpEQ" + assert isinstance(inner.operands[0], Expr.Call) and inner.operands[0].target == "__OFUMUL__" + + def test_umul_operands_narrowed_to_op_width(self): + result = _rewrite_amd64_cond(AMD64_CondTypes["CondO"], AMD64_OpTypes["G_CC_OP_UMULW"]) + assert all(arg.bits == 16 for arg in result.args) + + # ---- INC / DEC: overflow only at the signed extremes ---- + + def test_inc_o_compares_against_signed_min(self): + for op, nbits in ( + ("G_CC_OP_INCB", 8), + ("G_CC_OP_INCW", 16), + ("G_CC_OP_INCL", 32), + ("G_CC_OP_INCQ", 64), + ): + inner = _unwrap_convert(_rewrite_amd64_cond(AMD64_CondTypes["CondO"], AMD64_OpTypes[op])) + assert isinstance(inner, Expr.BinaryOp) and inner.op == "CmpEQ", op + assert inner.operands[1].value_int == 1 << (nbits - 1), op + + def test_inc_no_is_inverted(self): + inner = _unwrap_convert(_rewrite_amd64_cond(AMD64_CondTypes["CondNO"], AMD64_OpTypes["G_CC_OP_INCQ"])) + assert isinstance(inner, Expr.BinaryOp) and inner.op == "CmpNE" + + def test_dec_o_compares_against_signed_max(self): + for op, nbits in ( + ("G_CC_OP_DECB", 8), + ("G_CC_OP_DECW", 16), + ("G_CC_OP_DECL", 32), + ("G_CC_OP_DECQ", 64), + ): + inner = _unwrap_convert(_rewrite_amd64_cond(AMD64_CondTypes["CondO"], AMD64_OpTypes[op])) + assert isinstance(inner, Expr.BinaryOp) and inner.op == "CmpEQ", op + assert inner.operands[1].value_int == (1 << (nbits - 1)) - 1, op + + def test_dec_no_is_inverted(self): + inner = _unwrap_convert(_rewrite_amd64_cond(AMD64_CondTypes["CondNO"], AMD64_OpTypes["G_CC_OP_DECB"])) + assert isinstance(inner, Expr.BinaryOp) and inner.op == "CmpNE" + + # ---- COPY: test the stored OF bit ---- + + def test_copy_o_masks_of_bit(self): + inner = _unwrap_convert(_rewrite_amd64_cond(AMD64_CondTypes["CondO"], AMD64_OpTypes["G_CC_OP_COPY"])) + assert isinstance(inner, Expr.BinaryOp) and inner.op == "CmpNE" + masked = inner.operands[0] + assert isinstance(masked, Expr.BinaryOp) and masked.op == "And" + assert masked.operands[1].value_int == AMD64_CondBitMasks["G_CC_MASK_O"] + + def test_copy_no_masks_of_bit(self): + inner = _unwrap_convert(_rewrite_amd64_cond(AMD64_CondTypes["CondNO"], AMD64_OpTypes["G_CC_OP_COPY"])) + assert isinstance(inner, Expr.BinaryOp) and inner.op == "CmpEQ" + + # ---- guards ---- + + def test_symbolic_cond_returns_none(self): + ccall = Expr.VEXCCallExpression( + idx=0, + callee="amd64g_calculate_condition", + operands=( + Expr.Register(0, 0, 64), # non-constant cond + Expr.Const(0, AMD64_OpTypes["G_CC_OP_UMULQ"], 64), + Expr.Register(1, 16, 64), + Expr.Register(2, 24, 64), + Expr.Const(0, 0, 64), + ), + bits=64, + ) + proj = angr.load_shellcode(b"\x90", arch="AMD64") + assert AMD64CCallRewriter(ccall, proj, Manager()).result is None + + def test_symbolic_op_returns_none(self): + ccall = Expr.VEXCCallExpression( + idx=0, + callee="amd64g_calculate_condition", + operands=( + Expr.Const(0, AMD64_CondTypes["CondO"], 64), + Expr.Register(0, 0, 64), # non-constant cc_op + Expr.Register(1, 16, 64), + Expr.Register(2, 24, 64), + Expr.Const(0, 0, 64), + ), + bits=64, + ) + proj = angr.load_shellcode(b"\x90", arch="AMD64") + assert AMD64CCallRewriter(ccall, proj, Manager()).result is None + + def test_unhandled_op_returns_none(self): + # shifts do not have a CondO arm + result = _rewrite_amd64_cond(AMD64_CondTypes["CondO"], AMD64_OpTypes["G_CC_OP_SHLQ"]) + assert result is None -_DEP2_SAMPLE = (0, 1, 2, 3, 0x7E, 0x7F, 0x80, 0x81, 0xFD, 0xFE, 0xFF, 0x55) +class TestAMD64CondOverflowBinary(unittest.TestCase): + """Real-binary regression: no OF ccall may leak into the decompilation.""" + def test_gzip_overflow_checks_have_no_ccall(self): + # gzip has a size-computation helper guarded by jo on ADDQ and SMULQ + bin_path = os.path.join(test_location, "x86_64", "gzip_gcc13.3.0_O2") + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(fail_fast=True, normalize=True) + dec = proj.analyses.Decompiler(cfg.functions[0x40F1B0], cfg=cfg) + assert dec.codegen is not None and dec.codegen.text is not None + assert "_ccall" not in dec.codegen.text + assert "__OFADD__" in dec.codegen.text + assert "__OFSMUL__" in dec.codegen.text -class TestAMD64CCallRewriterDifferential(unittest.TestCase): - """Differential-test 8-bit cells against ccall.py's executable semantics.""" + def test_file_overflow_checks_have_no_ccall(self): + # file has several allocation helpers guarded by jo on UMULQ + bin_path = os.path.join(test_location, "x86_64", "file_gcc13.3.0_O2") + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(fail_fast=True, normalize=True) + for addr in (0x41E520, 0x41E5A0, 0x41EED0): + dec = proj.analyses.Decompiler(cfg.functions[addr], cfg=cfg) + assert dec.codegen is not None and dec.codegen.text is not None + assert "_ccall" not in dec.codegen.text, f"{addr:#x} still leaks a ccall" + assert "__OFUMUL__" in dec.codegen.text, f"{addr:#x} lost its overflow check" - # VEX only guarantees the low nbits of the deps; the rewriter must ignore anything above them - _DIRTY = 0xDEADBEEF_00000100 + def test_tar_umul_overflow_check_is_rewritten(self): + # tar guards a multiply with `mul %rbp` @ 0x53f77e / `jno` @ 0x53f784. + # The jno is canonicalized into CondO with an inverted branch, so the ccall + # reaching the rewriter is CondO x UMULQ (48), constant cc_op. + # This function also keeps ccalls from cc_op families outside this rewrite, + # so only the OF conditions are asserted -- the rewrite must be surgical. + bin_path = os.path.join(test_location, "x86_64", "tar_gcc17_O2") + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(fail_fast=True, normalize=True) + dec = proj.analyses.Decompiler(cfg.functions[0x53F6C0], cfg=cfg) + assert dec.codegen is not None and dec.codegen.text is not None + assert "__OFUMUL__" in dec.codegen.text + assert "_ccall(0, " not in dec.codegen.text + assert "_ccall(1, " not in dec.codegen.text - def _sweep(self, cond_name, op_name): - cond, op = AMD64_CondTypes[cond_name], AMD64_OpTypes[op_name] - for dep1, dep2 in itertools.product(range(256), _DEP2_SAMPLE): - got = _rewritten_value(cond, op, dep1, dep2) - want = _oracle(cond, op, dep1, dep2) - assert got == want, f"{cond_name} x {op_name} dep1={dep1:#x} dep2={dep2:#x}: {got} != {want}" - if dep1 % 8 == 0: - d1, d2 = dep1 | self._DIRTY, dep2 | self._DIRTY - got = _rewritten_value(cond, op, d1, d2) - want = _oracle(cond, op, d1, d2) - assert got == want, f"{cond_name} x {op_name} dep1={d1:#x} dep2={d2:#x}: {got} != {want}" + def test_coreutils_cat_overflow_checks_have_no_ccall(self): + # coreutils' xalloc idiom. Verified cc_op values, all constant: + # main @ 0x4023c0 -- CondO x SMULQ (52) and CondO x ADDQ (4) + # xpalloc @ 0x41f470 -- CondO x ADDQ (4) and CondO x SMULQ (52) + bin_path = os.path.join(test_location, "x86_64", "cat_gcc17.0.0_O2") + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(fail_fast=True, normalize=True) + for addr in (0x4023C0, 0x41F470): + dec = proj.analyses.Decompiler(cfg.functions[addr], cfg=cfg) + assert dec.codegen is not None and dec.codegen.text is not None + assert "_ccall" not in dec.codegen.text, f"{addr:#x} still leaks a ccall" + assert "__OFSMUL__" in dec.codegen.text, f"{addr:#x} lost its overflow check" + assert "__OFADD__" in dec.codegen.text, f"{addr:#x} lost its overflow check" - def test_condnl_subb_differential(self): - self._sweep("CondNL", "G_CC_OP_SUBB") + def test_grep_overflow_checks_have_no_ccall(self): + # Verified cc_op values, all constant: + # fillbuf @ 0x40cc10 -- CondO x ADDQ (4), 2 sites + # xstrtoimax @ 0x4888a0 -- CondO x SMULQ (52), 14 sites + bin_path = os.path.join(test_location, "x86_64", "grep_gcc17.0.0_O2") + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(fail_fast=True, normalize=True) - def test_condnl_logicb_differential(self): - self._sweep("CondNL", "G_CC_OP_LOGICB") + dec = proj.analyses.Decompiler(cfg.functions[0x40CC10], cfg=cfg) + assert dec.codegen is not None and dec.codegen.text is not None + assert "_ccall" not in dec.codegen.text + assert "__OFADD__" in dec.codegen.text + + dec = proj.analyses.Decompiler(cfg.functions[0x4888A0], cfg=cfg) + assert dec.codegen is not None and dec.codegen.text is not None + assert "_ccall" not in dec.codegen.text + assert "__OFSMUL__" in dec.codegen.text + + def test_zlib_minigzip_umul_overflow_has_no_ccall(self): + # zlib guards its gz buffer sizing with an unsigned multiply. Verified + # cc_op values, all constant: + # gzfread @ 0x40ea40 -- CondO x UMULQ (48) + # gzfwrite @ 0x413d50 -- CondO x UMULQ (48) + bin_path = os.path.join(test_location, "x86_64", "minigzip_gcc17.0.0_O2") + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(fail_fast=True, normalize=True) + for addr in (0x40EA40, 0x413D50): + dec = proj.analyses.Decompiler(cfg.functions[addr], cfg=cfg) + assert dec.codegen is not None and dec.codegen.text is not None + assert "_ccall" not in dec.codegen.text, f"{addr:#x} still leaks a ccall" + assert "__OFUMUL__" in dec.codegen.text, f"{addr:#x} lost its overflow check" if __name__ == "__main__": From 3efd1ec6dbec9eb299b0b5ebc2755e08592f91ba Mon Sep 17 00:00:00 2001 From: Fish Date: Fri, 24 Jul 2026 11:29:58 -0700 Subject: [PATCH 047/122] SimLibrary/SimSyscallLibrary: Treat None prototypes as absent. (#6673) --- angr/procedures/definitions/__init__.py | 12 ++++- .../procedures/test_syscall_none_prototype.py | 53 +++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 tests/procedures/test_syscall_none_prototype.py diff --git a/angr/procedures/definitions/__init__.py b/angr/procedures/definitions/__init__.py index a3aaaafa0..ac8483193 100644 --- a/angr/procedures/definitions/__init__.py +++ b/angr/procedures/definitions/__init__.py @@ -427,12 +427,16 @@ class SimLibrary: """ Check if a function has a prototype associated with it. + A prototype that is explicitly registered as ``None`` (i.e. the function is known but its signature is not) does + not count: this keeps ``has_prototype()`` consistent with ``get_prototype()``, which returns ``None`` for such + entries. + :param str func_name: The name of the function. :return: A bool indicating if a prototype of the function is available. :rtype: bool """ - return func_name in self.prototypes or func_name in self.prototypes_json + return self.prototypes.get(func_name) is not None or func_name in self.prototypes_json def is_returning(self, name: str) -> bool: """ @@ -808,13 +812,17 @@ class SimSyscallLibrary(SimLibrary): """ Check if a function has a prototype associated with it. Demangle the function name if it is a mangled C++ name. + Syscalls whose signature is unknown are registered with a ``None`` prototype (see, e.g., ``capset`` in + ``linux_kernel.py``). Those do not count as having a prototype: this keeps ``has_prototype()`` consistent with + ``get_prototype()``, which returns ``None`` for such entries. + :param abi: Name of the ABI. :param name: The syscall name. :return: bool """ if abi not in self.syscall_prototypes: return False - return name in self.syscall_prototypes[abi] + return self.syscall_prototypes[abi].get(name) is not None # diff --git a/tests/procedures/test_syscall_none_prototype.py b/tests/procedures/test_syscall_none_prototype.py new file mode 100644 index 000000000..06f2cea5f --- /dev/null +++ b/tests/procedures/test_syscall_none_prototype.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use +from __future__ import annotations + +__package__ = __package__ or "tests.procedures" # pylint:disable=redefined-builtin + +import unittest + +import archinfo + +from angr.procedures.definitions import SIM_LIBRARIES + + +class TestSyscallNonePrototype(unittest.TestCase): + """ + Regression test: syscalls registered with a ``None`` prototype (signature unknown, e.g. ``capset``) must not make + ``has_prototype()`` disagree with ``get_prototype()``. Otherwise ``SimSyscallLibrary.get()`` asserts in + ``_apply_numerical_metadata`` when the CFG or SimOS resolves such a syscall. + """ + + def _linux_syscall_library(self): + return SIM_LIBRARIES["linux"][0] + + def test_none_prototype_is_not_reported_as_present(self): + lib = self._linux_syscall_library() + none_names = [name for name, proto in lib.syscall_prototypes["amd64"].items() if proto is None] + assert none_names, "expected at least one None-prototype syscall in the linux definitions" + + for name in none_names: + # has_prototype() and get_prototype() must agree: no usable prototype. + assert not lib.has_prototype("amd64", name), name + assert lib.get_prototype("amd64", name, deref=True) is None, name + + def test_real_prototype_still_present(self): + lib = self._linux_syscall_library() + assert lib.has_prototype("amd64", "read") + assert lib.get_prototype("amd64", "read", deref=True) is not None + + def test_get_none_prototype_syscall_does_not_crash(self): + lib = self._linux_syscall_library() + arch = archinfo.arch_from_id("amd64") + none_names = [name for name, proto in lib.syscall_prototypes["amd64"].items() if proto is None] + name = none_names[0] + number = next(num for num, nm in lib.syscall_number_mapping["amd64"].items() if nm == name) + + # This used to raise `assert proto is not None` in _apply_numerical_metadata. + proc = lib.get(number, arch, ["amd64"]) + assert proc.is_syscall + assert proc.guessed_prototype + + +if __name__ == "__main__": + unittest.main() From be9c801b164a89d2a6152759c8f8395f96623da8 Mon Sep 17 00:00:00 2001 From: Max Ambaum Date: Fri, 24 Jul 2026 22:42:53 +0100 Subject: [PATCH 048/122] Add __sprintf_chk as a simprocedure (#6698) * Add __sprintf_chk as a simprocedure http://refspecs.linux-foundation.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/libc---sprintf-chk-1.html --- angr/procedures/libc/sprintf.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/angr/procedures/libc/sprintf.py b/angr/procedures/libc/sprintf.py index a79ba748e..a4e6fd23c 100644 --- a/angr/procedures/libc/sprintf.py +++ b/angr/procedures/libc/sprintf.py @@ -24,3 +24,22 @@ class sprintf(FormatParser): ) return out_str.size() // self.arch.byte_width + + +class __sprintf_chk(FormatParser): + # pylint:disable=arguments-differ + + def run(self, dst_ptr, flag, size, fmt): # pylint:disable=unused-argument + # See http://refspecs.linux-foundation.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/libc---sprintf-chk-1.html + # for argument layout + + fmt_str = self._parse(fmt) + out_str = fmt_str.replace(self.va_arg) + self.state.memory.store(dst_ptr, out_str) + + # place the terminating null byte + self.state.memory.store( + dst_ptr + (out_str.size() // self.arch.byte_width), claripy.BVV(0, self.arch.byte_width) + ) + + return out_str.size() // self.arch.byte_width From f74d1c5c1e8acb5d595c4c6f115333da059cb306 Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Fri, 24 Jul 2026 16:26:47 -0700 Subject: [PATCH 049/122] Typehoon: index subtype constraint components (#6696) * Typehoon: index subtype constraint components * Refactor the code to eliminate weird terminology. * Fix test cases. --------- Co-authored-by: Fish --- angr/analyses/typehoon/simple_solver.py | 83 ++++--- .../test_typehoon_constraint_subset.py | 204 ++++++++++++++++++ 2 files changed, 257 insertions(+), 30 deletions(-) create mode 100644 tests/analyses/test_typehoon_constraint_subset.py diff --git a/angr/analyses/typehoon/simple_solver.py b/angr/analyses/typehoon/simple_solver.py index 9c1e68596..c24b46d8e 100644 --- a/angr/analyses/typehoon/simple_solver.py +++ b/angr/analyses/typehoon/simple_solver.py @@ -3,7 +3,8 @@ from __future__ import annotations import enum import logging -from collections import defaultdict +from collections import defaultdict, deque +from collections.abc import Collection from contextlib import suppress from typing import TYPE_CHECKING @@ -709,18 +710,23 @@ class SimpleSolver: constrained_typevars.add(t) constraintset2tvs = defaultdict(set) + # build a mapping from type variable to constraints for faster lookups during constraint subset generation + tv_to_constraints = self._index_subtype_constraints(constraints) if constrained_typevars else {} tvs_seen = set() for idx, tv in enumerate(sorted(constrained_typevars, key=lambda x: x.idx)): _l.debug("Collecting constraints for type variable %r (%d/%d)", tv, idx + 1, len(constrained_typevars)) if tv in tvs_seen: continue # build a sub constraint set for the type variable - constraint_subset, related_tvs = self._generate_constraint_subset(constraints, {tv}) + constraint_subset, related_tvs = self._generate_constraint_subset( + constraints, {tv}, tv_to_constraints=tv_to_constraints + ) # drop all type vars outside constrained_typevars related_tvs = related_tvs.intersection(constrained_typevars) tvs_seen |= related_tvs frozen_constraint_subset = frozenset(constraint_subset) constraintset2tvs[frozen_constraint_subset] = related_tvs + del tv_to_constraints for idx, (constraint_subset, tvs) in enumerate(constraintset2tvs.items()): _l.debug( @@ -1579,39 +1585,56 @@ class SimpleSolver: # Constraint graph # + @staticmethod + def _index_subtype_constraints( + constraints: Collection[TypeConstraint], + ) -> dict[TypeVariable | TypeConstant, set[Subtype]]: + tv_to_constraints: dict[TypeVariable | TypeConstant, set[Subtype]] = defaultdict(set) + for constraint in constraints: + if not isinstance(constraint, Subtype): + continue + for type_ in (constraint.sub_type, constraint.super_type): + tv = ( + type_.type_var + if isinstance(type_, DerivedTypeVariable) + else type_ + if isinstance(type_, TypeVariable) + else None + ) + if tv is not None: + tv_to_constraints[tv].add(constraint) + return tv_to_constraints + @staticmethod def _generate_constraint_subset( - constraints: set[TypeConstraint], typevars: set[TypeVariable] - ) -> tuple[set[TypeConstraint], set[TypeVariable]]: - subset = set() + constraints: Collection[TypeConstraint], + typevars: Collection[TypeVariable | TypeConstant], + *, + tv_to_constraints: dict[TypeVariable | TypeConstant, set[Subtype]] | None = None, + ) -> tuple[set[TypeConstraint], set[TypeVariable | TypeConstant]]: + if tv_to_constraints is None: + tv_to_constraints = SimpleSolver._index_subtype_constraints(constraints) + + subset: set[TypeConstraint] = set() related_typevars = set(typevars) - while True: - new = set() - for constraint in constraints: + pending_typevars = deque(typevars) + while pending_typevars: + typevar = pending_typevars.pop() + for constraint in tv_to_constraints.get(typevar, ()): if constraint in subset: continue - if isinstance(constraint, Subtype): - if isinstance(constraint.sub_type, DerivedTypeVariable): - subt = constraint.sub_type.type_var - elif isinstance(constraint.sub_type, TypeVariable): - subt = constraint.sub_type - else: - subt = None - if isinstance(constraint.super_type, DerivedTypeVariable): - supert = constraint.super_type.type_var - elif isinstance(constraint.super_type, TypeVariable): - supert = constraint.super_type - else: - supert = None - if subt in related_typevars or supert in related_typevars: - new.add(constraint) - if subt is not None: - related_typevars.add(subt) - if supert is not None: - related_typevars.add(supert) - if not new: - break - subset |= new + subset.add(constraint) + for type_ in (constraint.sub_type, constraint.super_type): + tv = ( + type_.type_var + if isinstance(type_, DerivedTypeVariable) + else type_ + if isinstance(type_, TypeVariable) + else None + ) + if tv is not None and tv not in related_typevars: + related_typevars.add(tv) + pending_typevars.append(tv) return subset, related_typevars def _generate_constraint_graph( diff --git a/tests/analyses/test_typehoon_constraint_subset.py b/tests/analyses/test_typehoon_constraint_subset.py new file mode 100644 index 000000000..66c4d7940 --- /dev/null +++ b/tests/analyses/test_typehoon_constraint_subset.py @@ -0,0 +1,204 @@ +# pylint:disable=protected-access +from __future__ import annotations + +import random +from collections.abc import Collection, Iterator +from unittest.mock import patch + +from angr.analyses.typehoon.simple_solver import SimpleSolver +from angr.analyses.typehoon.typeconsts import Float32, Int8, Int32, TypeConstant +from angr.analyses.typehoon.typevars import ( + DerivedTypeVariable, + Existence, + Load, + Store, + Subtype, + TypeConstraint, + TypeVariable, +) + + +def _reference_generate_constraint_subset( + constraints: Collection[TypeConstraint], + typevars: Collection[TypeVariable | TypeConstant], +) -> tuple[set[TypeConstraint], set[TypeVariable | TypeConstant]]: + """The scan-to-fixpoint implementation that the indexed traversal replaces.""" + subset: set[TypeConstraint] = set() + related_typevars = set(typevars) + while True: + new: set[TypeConstraint] = set() + for constraint in constraints: + if constraint in subset or not isinstance(constraint, Subtype): + continue + if isinstance(constraint.sub_type, DerivedTypeVariable): + subtype = constraint.sub_type.type_var + elif isinstance(constraint.sub_type, TypeVariable): + subtype = constraint.sub_type + else: + subtype = None + if isinstance(constraint.super_type, DerivedTypeVariable): + supertype = constraint.super_type.type_var + elif isinstance(constraint.super_type, TypeVariable): + supertype = constraint.super_type + else: + supertype = None + if subtype in related_typevars or supertype in related_typevars: + new.add(constraint) + if subtype is not None: + related_typevars.add(subtype) + if supertype is not None: + related_typevars.add(supertype) + if not new: + break + subset |= new + return subset, related_typevars + + +class _IterationCountingSet(set[TypeConstraint]): + def __init__(self, values: Collection[TypeConstraint]): + super().__init__(values) + self.iterations = 0 + + def __iter__(self) -> Iterator[TypeConstraint]: + self.iterations += 1 + return super().__iter__() + + +def test_constraint_subset_preserves_tv_semantics(): + root = TypeVariable(name="root") + connected = TypeVariable(name="connected") + out_of_scope_bridge = TypeVariable(name="out_of_scope_bridge") + through_bridge = TypeVariable(name="through_bridge") + through_derived_constant = TypeVariable(name="through_derived_constant") + plain_constant_neighbor = TypeVariable(name="plain_constant_neighbor") + disconnected_a = TypeVariable(name="disconnected_a") + disconnected_b = TypeVariable(name="disconnected_b") + + derived_root = Subtype(DerivedTypeVariable(root, Load()), connected) + bridge_in = Subtype(connected, out_of_scope_bridge) + bridge_out = Subtype(DerivedTypeVariable(out_of_scope_bridge, Store()), through_bridge) + derived_constant_in = Subtype(through_bridge, DerivedTypeVariable(Int32(), Load())) + derived_constant_out = Subtype(DerivedTypeVariable(Int32(), Store()), through_derived_constant) + plain_constant_in = Subtype(through_derived_constant, Float32()) + plain_constant_out = Subtype(Float32(), plain_constant_neighbor) + constant_only = Subtype(Int8(), Float32()) + ignored_non_subtype = Existence(DerivedTypeVariable(root, Store())) + disconnected = Subtype(disconnected_a, disconnected_b) + constraints: set[TypeConstraint] = { + derived_root, + bridge_in, + bridge_out, + derived_constant_in, + derived_constant_out, + plain_constant_in, + plain_constant_out, + constant_only, + ignored_non_subtype, + disconnected, + } + + index = SimpleSolver._index_subtype_constraints(constraints) + subset, related = SimpleSolver._generate_constraint_subset(constraints, {root}, tv_to_constraints=index) + + assert subset == { + derived_root, + bridge_in, + bridge_out, + derived_constant_in, + derived_constant_out, + plain_constant_in, + } + assert related == { + root, + connected, + out_of_scope_bridge, + through_bridge, + Int32(), + through_derived_constant, + } + assert plain_constant_neighbor not in related + assert disconnected_a not in related + + +def test_constraint_subset_matches_reference_on_randomized_graphs(): + rng = random.Random(0x5EED) + + for trial in range(200): + typevars = [TypeVariable(idx=(trial, idx)) for idx in range(12)] + constants: list[TypeConstant] = [Int8(), Int32(), Float32()] + endpoints: list[TypeVariable | TypeConstant] = [*typevars, *constants] + endpoints += [ + DerivedTypeVariable(typevar, Load() if idx % 2 == 0 else Store()) + for idx, typevar in enumerate([*typevars, *constants]) + ] + + constraints: set[TypeConstraint] = {Subtype(rng.choice(endpoints), rng.choice(endpoints)) for _ in range(48)} + constraints.update(Existence(rng.choice(endpoints)) for _ in range(8)) + seeds = set(rng.sample(typevars, rng.randint(1, 4))) + + expected = _reference_generate_constraint_subset(constraints, seeds) + index = SimpleSolver._index_subtype_constraints(constraints) + indexed = SimpleSolver._generate_constraint_subset(constraints, seeds, tv_to_constraints=index) + on_demand = SimpleSolver._generate_constraint_subset(constraints, seeds) + + assert indexed == expected + assert on_demand == expected + + +def test_constraint_subset_unions_disconnected_seeds_and_preserves_orphans(): + left_a = TypeVariable(name="left_a") + left_b = TypeVariable(name="left_b") + right_a = TypeVariable(name="right_a") + right_b = TypeVariable(name="right_b") + orphan = TypeVariable(name="orphan") + left = Subtype(left_a, left_b) + right = Subtype(right_a, right_b) + constraints: set[TypeConstraint] = {left, right} + + subset, related = SimpleSolver._generate_constraint_subset(constraints, {left_a, right_a, orphan}) + + assert subset == constraints + assert related == {left_a, left_b, right_a, right_b, orphan} + + +def test_constraint_index_is_reused_across_many_components(): + typevars = [TypeVariable(idx=(0x400000, idx)) for idx in range(1024)] + constraints = _IterationCountingSet( + [Subtype(typevars[idx], typevars[idx + 1]) for idx in range(0, len(typevars), 2)] + ) + + index = SimpleSolver._index_subtype_constraints(constraints) + assert constraints.iterations == 1 + + for idx in range(0, len(typevars), 2): + subset, related = SimpleSolver._generate_constraint_subset( + constraints, {typevars[idx]}, tv_to_constraints=index + ) + assert subset == {Subtype(typevars[idx], typevars[idx + 1])} + assert related == {typevars[idx], typevars[idx + 1]} + + assert constraints.iterations == 1 + + +def test_simple_solver_builds_one_constraint_index_per_solve(): + function_typevar = TypeVariable(name="function") + lower = TypeVariable(name="lower") + upper = TypeVariable(name="upper") + constraints = {function_typevar: {Subtype(lower, upper), Subtype(upper, Int32())}} + typevars = {function_typevar: {lower, upper}} + + original_indexer = SimpleSolver._index_subtype_constraints + with patch.object(SimpleSolver, "_index_subtype_constraints", wraps=original_indexer) as indexer: + solver = SimpleSolver(64, constraints, typevars) + + assert indexer.call_count == 1 + assert solver.eqclass_constraints_count == [2] + assert solver.solution[lower] == Int32() + assert solver.solution[upper] == Int32() + + unconstrained_function = TypeVariable(name="unconstrained_function") + unconstrained = TypeVariable(name="unconstrained") + with patch.object(SimpleSolver, "_index_subtype_constraints", wraps=original_indexer) as indexer: + SimpleSolver(64, {unconstrained_function: set()}, {unconstrained_function: {unconstrained}}) + + assert indexer.call_count == 0 From 83d0061e12945abd722759a98d7cfb7f825ef980 Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Fri, 24 Jul 2026 16:34:21 -0700 Subject: [PATCH 050/122] Calling conventions: ignore stack canary comparisons as returns (#6699) * Calling conventions: ignore stack canary comparisons as returns --- .../calling_convention/fact_collector.py | 144 ++++++++++++++++++ tests/analyses/decompiler/test_decompiler.py | 3 +- tests/analyses/test_fact_collector.py | 91 +++++++++++ 3 files changed, 237 insertions(+), 1 deletion(-) diff --git a/angr/analyses/calling_convention/fact_collector.py b/angr/analyses/calling_convention/fact_collector.py index 0588723b9..2bc407e22 100644 --- a/angr/analyses/calling_convention/fact_collector.py +++ b/angr/analyses/calling_convention/fact_collector.py @@ -2,6 +2,7 @@ from __future__ import annotations from collections import defaultdict +from collections.abc import Container, Iterator from typing import TYPE_CHECKING import pyvex @@ -467,6 +468,141 @@ class FactCollector(Analysis): state.register_written(offset, self.project.arch.registers[reg_name][1]) state.simple_regs[offset] = None + @staticmethod + def _resolve_vex_tmp( + expr: pyvex.IRExpr.IRExpr, + tmp_definitions: dict[int, pyvex.IRExpr.IRExpr], + seen_tmps: frozenset[int] = frozenset(), + ) -> pyvex.IRExpr.IRExpr: + while isinstance(expr, pyvex.IRExpr.RdTmp) and expr.tmp not in seen_tmps: + definition = tmp_definitions.get(expr.tmp) + if definition is None: + break + seen_tmps |= {expr.tmp} + expr = definition + return expr + + @classmethod + def _walk_vex_expr( + cls, + expr: pyvex.IRExpr.IRExpr, + tmp_definitions: dict[int, pyvex.IRExpr.IRExpr], + seen_tmps: frozenset[int] = frozenset(), + ) -> Iterator[pyvex.IRExpr.IRExpr]: + if isinstance(expr, pyvex.IRExpr.RdTmp): + if expr.tmp in seen_tmps: + return + definition = tmp_definitions.get(expr.tmp) + if definition is not None: + yield from cls._walk_vex_expr(definition, tmp_definitions, seen_tmps | {expr.tmp}) + return + + yield expr + for child in expr.child_expressions: + yield from cls._walk_vex_expr(child, tmp_definitions, seen_tmps) + + def _stack_canary_tls_location(self) -> tuple[int, int] | None: + if self.project.arch.name == "AMD64": + reg_name, offset = "fs", 0x28 + elif self.project.arch.name == "X86": + reg_name, offset = "gs", 0x14 + else: + return None + return self.project.arch.registers[reg_name][0], offset + + @classmethod + def _is_tls_canary_load( + cls, + expr: pyvex.IRExpr.IRExpr, + tmp_definitions: dict[int, pyvex.IRExpr.IRExpr], + tls_reg_offset: int, + canary_offset: int, + ) -> bool: + expr = cls._resolve_vex_tmp(expr, tmp_definitions) + if not isinstance(expr, pyvex.IRExpr.Load): + return False + addr_nodes = tuple(cls._walk_vex_expr(expr.addr, tmp_definitions)) + return any(isinstance(node, pyvex.IRExpr.Get) and node.offset == tls_reg_offset for node in addr_nodes) and any( + isinstance(node, pyvex.IRExpr.Const) and node.con.value == canary_offset for node in addr_nodes + ) + + @classmethod + def _is_stack_load( + cls, + expr: pyvex.IRExpr.IRExpr, + tmp_definitions: dict[int, pyvex.IRExpr.IRExpr], + stack_reg_offsets: Container[int | None], + ) -> bool: + expr = cls._resolve_vex_tmp(expr, tmp_definitions) + if not isinstance(expr, pyvex.IRExpr.Load): + return False + return any( + isinstance(node, pyvex.IRExpr.Get) and node.offset in stack_reg_offsets + for node in cls._walk_vex_expr(expr.addr, tmp_definitions) + ) + + def _has_terminal_call_successor(self, node: BlockNode) -> bool: + func_graph = self.function.transition_graph + for _, succ, data in func_graph.out_edges(node, data=True): + if data.get("type") != "transition" or data.get("outside", False) or not isinstance(succ, BlockNode): + continue + succ_block = self.project.factory.block(succ.addr, size=succ.size) + if succ_block.vex.jumpkind != "Ijk_Call": + continue + if not any( + edge_data.get("type") == "fake_return" for _, _, edge_data in func_graph.out_edges(succ, data=True) + ): + return True + return False + + def _is_stack_canary_retval_write( + self, + node: BlockNode, + block: Block, + expr: pyvex.IRExpr.IRExpr, + tmp_definitions: dict[int, pyvex.IRExpr.IRExpr], + ) -> bool: + tls_location = self._stack_canary_tls_location() + if tls_location is None: + return False + + expr = self._resolve_vex_tmp(expr, tmp_definitions) + if not isinstance(expr, pyvex.IRExpr.Binop) or expr.op not in { + "Iop_Sub32", + "Iop_Sub64", + "Iop_Xor32", + "Iop_Xor64", + }: + return False + + tls_reg_offset, canary_offset = tls_location + stack_reg_offsets = {self.project.arch.sp_offset, self.project.arch.bp_offset} + op0, op1 = expr.args + if not ( + ( + self._is_tls_canary_load(op0, tmp_definitions, tls_reg_offset, canary_offset) + and self._is_stack_load(op1, tmp_definitions, stack_reg_offsets) + ) + or ( + self._is_tls_canary_load(op1, tmp_definitions, tls_reg_offset, canary_offset) + and self._is_stack_load(op0, tmp_definitions, stack_reg_offsets) + ) + ): + return False + + if not self._has_terminal_call_successor(node): + return False + + for stmt in block.vex.statements: + if not isinstance(stmt, pyvex.IRStmt.Exit): + continue + guard_nodes = tuple(self._walk_vex_expr(stmt.guard, tmp_definitions)) + if any( + self._is_tls_canary_load(node, tmp_definitions, tls_reg_offset, canary_offset) for node in guard_nodes + ) and any(self._is_stack_load(node, tmp_definitions, stack_reg_offsets) for node in guard_nodes): + return True + return False + def _analyze_endpoints_for_retval_size(self, end_states): """ Analyze all endpoints to determine the return value size. @@ -591,6 +727,7 @@ class FactCollector(Analysis): # to account for the common case where the shorter register (e.g., al) is extended to the full register # (e.g., rax) before returning. block_retval_size = None + stack_canary_barrier = False for stmt in reversed(block.vex.statements): if isinstance(stmt, pyvex.IRStmt.Put): assert block.vex.tyenv is not None @@ -610,6 +747,11 @@ class FactCollector(Analysis): size = 4 if stmt.offset == retreg_offset: + if isinstance(node, BlockNode) and self._is_stack_canary_retval_write( + node, block, stmt.data, tmp_definitions + ): + stack_canary_barrier = True + break block_retval_size = max(size, 1) if stmt.offset == overflow_retreg_offset: overflow_retval_sizes.append(max(size, 1)) @@ -617,6 +759,8 @@ class FactCollector(Analysis): if block_retval_size is not None: retval_sizes.append(block_retval_size) continue + if stack_canary_barrier: + continue for pred, _, data in func_graph.in_edges(node, data=True): edge_type = data.get("type") diff --git a/tests/analyses/decompiler/test_decompiler.py b/tests/analyses/decompiler/test_decompiler.py index d8b97b1ea..59bc59c9e 100755 --- a/tests/analyses/decompiler/test_decompiler.py +++ b/tests/analyses/decompiler/test_decompiler.py @@ -3639,7 +3639,8 @@ class TestDecompiler(unittest.TestCase): print_decompilation_result(d) text = d.codegen.text - good_if_return_pattern = r"if \(\!a2\)\s+return .*;" + # bridge_print_opt is void; do not accept a stack-canary comparison as its return value. + good_if_return_pattern = r"if \(\!a2\)\s+return;" good_if_return = re.search(good_if_return_pattern, text) assert good_if_return is not None diff --git a/tests/analyses/test_fact_collector.py b/tests/analyses/test_fact_collector.py index cb744bc35..0ea57f900 100644 --- a/tests/analyses/test_fact_collector.py +++ b/tests/analyses/test_fact_collector.py @@ -22,6 +22,97 @@ test_location = os.path.join(bin_location, "tests") # pylint: disable=missing-class-docstring # pylint: disable=no-self-use class TestFactCollector(unittest.TestCase): + @staticmethod + def _collect_shellcode_facts(code: bytes, arch: str = "amd64"): + base_addr = 0x400000 + project = angr.load_shellcode(code, arch=arch, load_address=base_addr) + cfg = project.analyses.CFGFast( + normalize=True, + regions=[(base_addr, base_addr + len(code))], + function_starts=[base_addr], + start_at_entry=False, + symbols=False, + force_smart_scan=False, + ) + return project.analyses.FunctionFactCollector(cfg.kb.functions[base_addr]) + + def test_stack_canary_comparison_is_not_a_return_value(self): + prefix = bytes.fromhex( + "4883ec18" # sub rsp, 0x18 + "64488b042528000000" # mov rax, qword ptr fs:[0x28] + "4889442408" # mov qword ptr [rsp + 8], rax + "31c0" # xor eax, eax + "488b442408" # mov rax, qword ptr [rsp + 8] + ) + void_tail = bytes.fromhex( + "7505" # jne stack_chk_fail + "4883c418" # add rsp, 0x18 + "c3" # ret + "e800000000" # call stack_chk_fail + ) + + for operation in ("64482b042528000000", "644833042528000000"): # sub/xor rax, qword ptr fs:[0x28] + with self.subTest(operation=operation): + facts = self._collect_shellcode_facts(prefix + bytes.fromhex(operation) + void_tail) + self.assertIsNone(facts.retval_size) + + def test_x86_stack_canary_comparison_is_not_a_return_value(self): + code = bytes.fromhex( + "83ec0c" # sub esp, 0xc + "65a114000000" # mov eax, dword ptr gs:[0x14] + "89442404" # mov dword ptr [esp + 4], eax + "31c0" # xor eax, eax + "8b442404" # mov eax, dword ptr [esp + 4] + "652b0514000000" # sub eax, dword ptr gs:[0x14] + "7504" # jne stack_chk_fail + "83c40c" # add esp, 0xc + "c3" # ret + "e800000000" # call stack_chk_fail + ) + + facts = self._collect_shellcode_facts(code, arch="x86") + + self.assertIsNone(facts.retval_size) + + def test_stack_canary_comparison_preserves_real_return_value(self): + prefix = bytes.fromhex( + "4883ec18" # sub rsp, 0x18 + "64488b042528000000" # mov rax, qword ptr fs:[0x28] + "4889442408" # mov qword ptr [rsp + 8], rax + "31c0" # xor eax, eax + "488b442408" # mov rax, qword ptr [rsp + 8] + "64482b042528000000" # sub rax, qword ptr fs:[0x28] + ) + epilogue = bytes.fromhex("4883c418c3e800000000") # add rsp, 0x18; ret; call stack_chk_fail + + cases = ( + ("750ab82a000000", 4), # jne +10; mov eax, 42 + ("750f48b82a00000078563412", 8), # jne +15; movabs rax, 0x123456780000002a + ) + for return_code, expected_size in cases: + with self.subTest(expected_size=expected_size): + facts = self._collect_shellcode_facts(prefix + bytes.fromhex(return_code) + epilogue) + self.assertEqual(facts.retval_size, expected_size) + + def test_tls_arithmetic_without_terminal_call_is_a_return_value(self): + code = bytes.fromhex( + "4883ec18" # sub rsp, 0x18 + "64488b042528000000" # mov rax, qword ptr fs:[0x28] + "4889442408" # mov qword ptr [rsp + 8], rax + "31c0" # xor eax, eax + "488b442408" # mov rax, qword ptr [rsp + 8] + "64482b042528000000" # sub rax, qword ptr fs:[0x28] + "7505" # jne alternate return + "4883c418" # add rsp, 0x18 + "c3" # ret + "4883c418" # add rsp, 0x18 + "c3" # ret + ) + + facts = self._collect_shellcode_facts(code) + + self.assertEqual(facts.retval_size, 8) + def _run_fauxware(self, arch, function_and_cc_list): binary_path = os.path.join(test_location, arch, "fauxware") fauxware = angr.Project(binary_path, auto_load_libs=False) From fc7ff8e62c5f26d73f8cb2d217f494c1041d46b5 Mon Sep 17 00:00:00 2001 From: Fish Date: Sun, 26 Jul 2026 00:35:43 -0700 Subject: [PATCH 051/122] CFGFast: Linear scan heuristics for monotonic byte ramps and floats. (#6701) --- angr/analyses/cfg/cfg_fast.py | 109 ++++++++++++++++++++ tests/analyses/cfg/test_cfgfast_datarefs.py | 32 ++++++ 2 files changed, 141 insertions(+) diff --git a/angr/analyses/cfg/cfg_fast.py b/angr/analyses/cfg/cfg_fast.py index 08c92c769..00204c60f 100644 --- a/angr/analyses/cfg/cfg_fast.py +++ b/angr/analyses/cfg/cfg_fast.py @@ -1238,6 +1238,86 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): return repeating_length return 0 + def _scan_for_fp_constants(self, start_addr: int, threshold: int = 4) -> int: + """ + Scan from a given address for a run of plausible floating-point constants. + + A double-precision value qualifies when its biased exponent falls within a band covering magnitudes + between 2 ** -64 and 2 ** 64, which is where constants in compiler- and libm-generated tables (polynomial + coefficients, logarithm and trigonometry tables, etc.) almost always live. Code bytes rarely produce + multiple consecutive qualifying values. + + Single-precision values are detected as well, but with a tighter magnitude band (2 ** -32 to 2 ** 32) and + twice the run-length requirement: an 8-bit exponent in a 4-byte value is a much weaker signal than an + 11-bit exponent in an 8-byte value, and anything looser starts matching real code. + + :param start_addr: The address to start scanning from. + :param threshold: The minimum number of consecutive qualifying double-precision values. + :return: The total size in bytes of the qualifying values, or 0 if not enough values are found. + """ + + for size, exp_shift, exp_mask, exp_lo, exp_hi, min_count in ( + (8, 52, 0x7FF, 959, 1087, threshold), # doubles: 1023 +/- 64 + (4, 23, 0xFF, 95, 159, threshold * 2), # floats: 127 +/- 32 + ): + addr = start_addr + fp_count = 0 + first_val = None + has_multiple_values = False + + uniform_mul = ((1 << (size * 8)) - 1) // 0xFF + + while self._inside_regions(addr): + val = self._fast_memory_load_pointer(addr, size=size) + if val is None: + break + if val == (val & 0xFF) * uniform_mul: + # all bytes are identical: this is filler (e.g., 0xCC padding or "????", whose bit patterns + # carry in-band exponents), not a constant + break + exponent = (val >> exp_shift) & exp_mask + if not exp_lo <= exponent <= exp_hi: + break + if first_val is None: + first_val = val + elif val != first_val: + has_multiple_values = True + fp_count += 1 + addr += size + + # a run of one repeated value carries no table evidence + if fp_count >= min_count and has_multiple_values: + return fp_count * size + return 0 + + def _scan_for_monotonic_byte_ramp(self, start_addr: int, threshold: int = 16) -> int: + """ + Scan from a given address for a run of monotonically increasing bytes, where each byte equals the previous + byte plus one, modulo 256. Character case-conversion and translation tables are laid out this way. + + :param start_addr: The address to start scanning from. + :param threshold: The minimum run length. + :return: The length of the run, or 0 if the run is shorter than threshold. + """ + + addr = start_addr + last_byte = None + ramp_length = 0 + + while self._inside_regions(addr): + val = self._load_a_byte_as_int(addr) + if val is None: + break + if last_byte is not None and val != (last_byte + 1) & 0xFF: + break + last_byte = val + ramp_length += 1 + addr += 1 + + if ramp_length >= threshold: + return ramp_length + return 0 + def _scan_for_consecutive_pointers(self, start_addr: int, threshold: int = 2) -> int: """ Scan from a given address and determine if there are at least `threshold` of pointers. @@ -1371,6 +1451,27 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): ) start_addr += pointer_length + if not matched_something: + # find floating-point constant tables; this must run before the string and repeating-zero scans + # because the low mantissa bytes of table entries are frequently zero or incidentally printable, + # which would misphase the table. since scanning misclassified code often dumps us in the middle + # of a table entry, probe the next 4- and 8-byte boundaries as well. + fp_addr_4 = start_addr + (-start_addr % 4) + fp_addr_8 = start_addr + (-start_addr % 8) + for fp_addr in (fp_addr_4,) if fp_addr_4 == fp_addr_8 else (fp_addr_4, fp_addr_8): + fp_length = self._scan_for_fp_constants(fp_addr) + if fp_length: + matched_something = True + if fp_addr > start_addr: + self._seg_list.occupy(start_addr, fp_addr - start_addr, "alignment") + self.model.memory_data[start_addr] = MemoryData( + start_addr, fp_addr - start_addr, MemoryDataSort.Alignment + ) + self._seg_list.occupy(fp_addr, fp_length, "fp") + self.model.memory_data[fp_addr] = MemoryData(fp_addr, fp_length, MemoryDataSort.FloatingPoint) + start_addr = fp_addr + fp_length + break + if not matched_something: # find strings; tolerate a single leading null byte, which is usually the leftover of a multi-null # string separator (string scans only consume one null terminator of the preceding string, and a @@ -1438,6 +1539,14 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): ) start_addr += repeating_byte_length + # a long run of monotonically increasing bytes is a character or translation table, not code + ramp_length = self._scan_for_monotonic_byte_ramp(start_addr, threshold=16) + if ramp_length: + matched_something = True + self._seg_list.occupy(start_addr, ramp_length, "nodecode") + self.model.memory_data[start_addr] = MemoryData(start_addr, ramp_length, MemoryDataSort.Unknown) + start_addr += ramp_length + if not matched_something: # umm now it's probably code break diff --git a/tests/analyses/cfg/test_cfgfast_datarefs.py b/tests/analyses/cfg/test_cfgfast_datarefs.py index 91c10f0a2..5546bc7bd 100644 --- a/tests/analyses/cfg/test_cfgfast_datarefs.py +++ b/tests/analyses/cfg/test_cfgfast_datarefs.py @@ -6,6 +6,7 @@ __package__ = __package__ or "tests.analyses.cfg" # pylint:disable=redefined-bu import logging import os +import struct import time import unittest @@ -205,6 +206,37 @@ class TestCfgfastDataReferences(unittest.TestCase): # interleaved (value, pointer) tables are detected by the mixed-pointer scan assert memory_data[0x401FBC].sort == MemoryDataSort.PointerArray + # character translation tables (runs of monotonically increasing bytes) are detected by the byte-ramp scan + # and must not become code + assert memory_data[0x403390].sort == MemoryDataSort.Unknown + assert memory_data[0x403390].size == 193 + assert not [f for f in cfg.kb.functions if 0x4033A0 <= f < 0x403690] + + # double-precision constant tables are detected by the floating-point constant scan + assert memory_data[0x406278].sort == MemoryDataSort.FloatingPoint + assert memory_data[0x406278].size == 2064 + assert not [f for f in cfg.kb.functions if 0x406270 <= f < 0x406B00] + + def test_complete_scan_single_precision_fp_table(self): + # a run of plausible single-precision floats must be classified as data, not code + code = b"\xc3\xcc\xcc\xcc" + struct.pack("<8f", 1.0, 0.5, 3.14, 100.0, 1e-5, 1e5, -2.5, 0.001) + proj = angr.load_shellcode(code, "x86") + cfg = proj.analyses.CFGFast() + + assert cfg.model.memory_data[4].sort == MemoryDataSort.FloatingPoint + assert cfg.model.memory_data[4].size == 32 + assert list(cfg.kb.functions) == [0] + + def test_complete_scan_cc_filler_is_not_fp(self): + # a run of one repeated in-band value (0xCC filler decodes as plausible negative floats) must not be + # classified as a floating-point constant table + code = b"\xc3" + b"\xcc" * 64 + proj = angr.load_shellcode(code, "x86") + cfg = proj.analyses.CFGFast() + + assert not [d for d in cfg.model.memory_data.values() if d.sort == MemoryDataSort.FloatingPoint] + assert list(cfg.kb.functions) == [0] + def test_long_printable_ascii_string_without_null_byte(self): # suboptimal logic in _scan_for_printable_strings was causing the CFG recovery of this binary to be extremely # slow; we were repeatedly trying (and failing) to build a super long ASCII string in this binary. From b37cca0101fde2959b023a0e757c41df4b79017d Mon Sep 17 00:00:00 2001 From: Fish Date: Sun, 26 Jul 2026 07:58:06 -0700 Subject: [PATCH 052/122] VRA: Register the Reference stack variable against its atom. (#6705) --- angr/analyses/variable_recovery/engine_ail.py | 2 +- .../decompiler/test_output_nondeterminism.py | 55 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/angr/analyses/variable_recovery/engine_ail.py b/angr/analyses/variable_recovery/engine_ail.py index 344e85ac4..8fdc2d140 100644 --- a/angr/analyses/variable_recovery/engine_ail.py +++ b/angr/analyses/variable_recovery/engine_ail.py @@ -729,7 +729,7 @@ class SimEngineVRAIL( def _handle_unop_Reference(self, expr: ailment.Expr.UnaryOp): if isinstance(expr.operand, ailment.Expr.VirtualVariable) and expr.operand.was_stack: if expr.tags.get("extra_def", False): - self._assign_to_vvar(expr.operand, self._top(expr.operand.bits)) + self._assign_to_vvar(expr.operand, self._top(expr.operand.bits), dst=expr.operand) refbase_typevar = None off = expr.operand.stack_offset diff --git a/tests/analyses/decompiler/test_output_nondeterminism.py b/tests/analyses/decompiler/test_output_nondeterminism.py index 3806ff872..6eb0528ee 100644 --- a/tests/analyses/decompiler/test_output_nondeterminism.py +++ b/tests/analyses/decompiler/test_output_nondeterminism.py @@ -44,6 +44,61 @@ class TestVariableNondeterminism(unittest.TestCase): print("=======") assert False, f"Output differs at iteration {i}" + def test_redecompilation_is_idempotent_for_referenced_stack_arrays(self): + """Regression: Re-decompiling a function must not rename its stack arrays. + + ``convert()`` in 1after909 has three ``char[2048]`` stack buffers that are passed by reference. The + ``Reference(vvar)`` handler in variable recovery used to create a brand-new ``SimStackVariable`` for each of + them on every run. + """ + + binary_path = os.path.join(bin_location, "tests", "x86_64", "1after909") + project = angr.Project(binary_path) + project.analyses.CFGFast(normalize=True) + + output = [] + for i in range(4): + # drop the decompilation cache so that the whole pipeline (including variable recovery) reruns + project.kb.decompilations.cached.clear() + dec = project.analyses.Decompiler("convert") + assert dec.codegen is not None and dec.codegen.text is not None + output.append(dec.codegen.text) + + if i > 0 and output[0] != output[i]: + diff = "".join( + difflib.unified_diff( + output[0].splitlines(keepends=True), + output[i].splitlines(keepends=True), + fromfile="output[0]", + tofile=f"output[{i}]", + ) + ) + assert False, f"Re-decompilation of convert() differs at iteration {i}:\n{diff}" + + # the stack arrays must carry generated names, not the default s_ fallback + assert "s_1818" not in output[0] + assert "s_1018" not in output[0] + assert "s_818" not in output[0] + + def test_redecompilation_does_not_duplicate_stack_variables(self): + """ + Regression: The number of recovered stack variables must not grow when a function is decompiled repeatedly. + """ + + binary_path = os.path.join(bin_location, "tests", "x86_64", "1after909") + project = angr.Project(binary_path) + project.analyses.CFGFast(normalize=True) + func = project.kb.functions["convert"] + + counts = [] + for _ in range(3): + project.kb.decompilations.cached.clear() + project.analyses.Decompiler(func) + varman = project.kb.dec_variables[func.addr] + counts.append(len(varman.get_variables("stack"))) + + assert counts[0] == counts[1] == counts[2], f"stack variables accumulate across decompilations: {counts}" + if __name__ == "__main__": unittest.main() From 787c2c7d8e81566d1fb89103da972e5003af0c35 Mon Sep 17 00:00:00 2001 From: Fish Date: Sun, 26 Jul 2026 17:17:09 -0700 Subject: [PATCH 053/122] SimpleSolver: Memoize least common ancestors on type lattices. (#6707) --- angr/analyses/typehoon/simple_solver.py | 205 +++++++++++-------- tests/analyses/decompiler/test_signedness.py | 70 +++---- 2 files changed, 157 insertions(+), 118 deletions(-) diff --git a/angr/analyses/typehoon/simple_solver.py b/angr/analyses/typehoon/simple_solver.py index c24b46d8e..f33bf35a8 100644 --- a/angr/analyses/typehoon/simple_solver.py +++ b/angr/analyses/typehoon/simple_solver.py @@ -6,7 +6,7 @@ import logging from collections import defaultdict, deque from collections.abc import Collection from contextlib import suppress -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import networkx from sortedcontainers import SortedDict @@ -143,85 +143,133 @@ PRIMITIVE_TYPES = { } +class TypeLattice: + """ + A lattice of type constants with a lowest-common-ancestor table for caching purposes. + + The lattice graph (`self.g`) should be treated as read-only, otherwise the cached LCA table will be invalidated + once the lattice graph changes. + """ + + __slots__ = ("_lca_table", "g") + + def __init__(self, g: networkx.DiGraph): + self.g: networkx.DiGraph = g + self._lca_table: dict[tuple, Any] | None = None + + def __contains__(self, node) -> bool: + return node in self.g + + def inverted(self) -> TypeLattice: + """ + Return a new lattice with every edge reversed, i.e. ordered from the most specific to the most general. + """ + + inverted_g = networkx.DiGraph() + for src, dst in self.g.edges: + inverted_g.add_edge(dst, src) + return TypeLattice(inverted_g) + + def lca(self, node_a, node_b): + """ + Return the lowest common ancestor of ``node_a`` and ``node_b``, or None if they have none. Initialize the LCA + table on the first query and memoize it for subsequent queries. This is critical for performance when the + lattice is relatively small (< 100 nodes) and the number of queries is large. + + Reconsider this implementation when the type lattice is large. + """ + + if self._lca_table is None: + table: dict[tuple, Any] = {} + for (a, b), ancestor in networkx.all_pairs_lowest_common_ancestor(self.g): + # LCA is symmetric; all_pairs_lowest_common_ancestor only yields one direction of each pair + table[(a, b)] = ancestor + table[(b, a)] = ancestor + self._lca_table = table + return self._lca_table.get((node_a, node_b)) + + # lattice for 64-bit binaries -BASE_LATTICE_64 = networkx.DiGraph() -BASE_LATTICE_64.add_edge(Top_, Int_) -BASE_LATTICE_64.add_edge(Int_, Int512_) -BASE_LATTICE_64.add_edge(Int_, Int256_) -BASE_LATTICE_64.add_edge(Int_, Int128_) -BASE_LATTICE_64.add_edge(Int_, Int64_) -BASE_LATTICE_64.add_edge(Int_, Int32_) -BASE_LATTICE_64.add_edge(Int_, Int16_) -BASE_LATTICE_64.add_edge(Int_, Int8_) -BASE_LATTICE_64.add_edge(Int512_, Bottom_) -BASE_LATTICE_64.add_edge(Int256_, Bottom_) -BASE_LATTICE_64.add_edge(Int128_, Bottom_) +BASE_LATTICE_64_g = networkx.DiGraph() +BASE_LATTICE_64_g.add_edge(Top_, Int_) +BASE_LATTICE_64_g.add_edge(Int_, Int512_) +BASE_LATTICE_64_g.add_edge(Int_, Int256_) +BASE_LATTICE_64_g.add_edge(Int_, Int128_) +BASE_LATTICE_64_g.add_edge(Int_, Int64_) +BASE_LATTICE_64_g.add_edge(Int_, Int32_) +BASE_LATTICE_64_g.add_edge(Int_, Int16_) +BASE_LATTICE_64_g.add_edge(Int_, Int8_) +BASE_LATTICE_64_g.add_edge(Int512_, Bottom_) +BASE_LATTICE_64_g.add_edge(Int256_, Bottom_) +BASE_LATTICE_64_g.add_edge(Int128_, Bottom_) # Int8: signed/unsigned children -BASE_LATTICE_64.add_edge(Int8_, SInt8_) -BASE_LATTICE_64.add_edge(Int8_, UInt8_) -BASE_LATTICE_64.add_edge(SInt8_, Bottom_) -BASE_LATTICE_64.add_edge(UInt8_, Bottom_) +BASE_LATTICE_64_g.add_edge(Int8_, SInt8_) +BASE_LATTICE_64_g.add_edge(Int8_, UInt8_) +BASE_LATTICE_64_g.add_edge(SInt8_, Bottom_) +BASE_LATTICE_64_g.add_edge(UInt8_, Bottom_) # Int16: signed/unsigned children -BASE_LATTICE_64.add_edge(Int16_, SInt16_) -BASE_LATTICE_64.add_edge(Int16_, UInt16_) -BASE_LATTICE_64.add_edge(SInt16_, Bottom_) -BASE_LATTICE_64.add_edge(UInt16_, Bottom_) +BASE_LATTICE_64_g.add_edge(Int16_, SInt16_) +BASE_LATTICE_64_g.add_edge(Int16_, UInt16_) +BASE_LATTICE_64_g.add_edge(SInt16_, Bottom_) +BASE_LATTICE_64_g.add_edge(UInt16_, Bottom_) # Int32: signed/unsigned children + Enum, Fd -BASE_LATTICE_64.add_edge(Int32_, SInt32_) -BASE_LATTICE_64.add_edge(Int32_, UInt32_) -BASE_LATTICE_64.add_edge(SInt32_, Bottom_) -BASE_LATTICE_64.add_edge(UInt32_, Bottom_) -BASE_LATTICE_64.add_edge(Int32_, Enum_) -BASE_LATTICE_64.add_edge(Enum_, Bottom_) -BASE_LATTICE_64.add_edge(Int32_, Fd_) -BASE_LATTICE_64.add_edge(Fd_, Bottom_) +BASE_LATTICE_64_g.add_edge(Int32_, SInt32_) +BASE_LATTICE_64_g.add_edge(Int32_, UInt32_) +BASE_LATTICE_64_g.add_edge(SInt32_, Bottom_) +BASE_LATTICE_64_g.add_edge(UInt32_, Bottom_) +BASE_LATTICE_64_g.add_edge(Int32_, Enum_) +BASE_LATTICE_64_g.add_edge(Enum_, Bottom_) +BASE_LATTICE_64_g.add_edge(Int32_, Fd_) +BASE_LATTICE_64_g.add_edge(Fd_, Bottom_) # Int64: signed/unsigned children + Pointer64 -BASE_LATTICE_64.add_edge(Int64_, SInt64_) -BASE_LATTICE_64.add_edge(Int64_, UInt64_) -BASE_LATTICE_64.add_edge(SInt64_, Bottom_) -BASE_LATTICE_64.add_edge(UInt64_, Bottom_) -BASE_LATTICE_64.add_edge(Int64_, Pointer64_) -BASE_LATTICE_64.add_edge(Pointer64_, Bottom_) +BASE_LATTICE_64_g.add_edge(Int64_, SInt64_) +BASE_LATTICE_64_g.add_edge(Int64_, UInt64_) +BASE_LATTICE_64_g.add_edge(SInt64_, Bottom_) +BASE_LATTICE_64_g.add_edge(UInt64_, Bottom_) +BASE_LATTICE_64_g.add_edge(Int64_, Pointer64_) +BASE_LATTICE_64_g.add_edge(Pointer64_, Bottom_) +BASE_LATTICE_64 = TypeLattice(BASE_LATTICE_64_g) # lattice for 32-bit binaries -BASE_LATTICE_32 = networkx.DiGraph() -BASE_LATTICE_32.add_edge(Top_, Int_) -BASE_LATTICE_32.add_edge(Int_, Int512_) -BASE_LATTICE_32.add_edge(Int_, Int256_) -BASE_LATTICE_32.add_edge(Int_, Int128_) -BASE_LATTICE_32.add_edge(Int_, Int64_) -BASE_LATTICE_32.add_edge(Int_, Int32_) -BASE_LATTICE_32.add_edge(Int_, Int16_) -BASE_LATTICE_32.add_edge(Int_, Int8_) -BASE_LATTICE_32.add_edge(Int512_, Bottom_) -BASE_LATTICE_32.add_edge(Int256_, Bottom_) -BASE_LATTICE_32.add_edge(Int128_, Bottom_) +BASE_LATTICE_32_g = networkx.DiGraph() +BASE_LATTICE_32_g.add_edge(Top_, Int_) +BASE_LATTICE_32_g.add_edge(Int_, Int512_) +BASE_LATTICE_32_g.add_edge(Int_, Int256_) +BASE_LATTICE_32_g.add_edge(Int_, Int128_) +BASE_LATTICE_32_g.add_edge(Int_, Int64_) +BASE_LATTICE_32_g.add_edge(Int_, Int32_) +BASE_LATTICE_32_g.add_edge(Int_, Int16_) +BASE_LATTICE_32_g.add_edge(Int_, Int8_) +BASE_LATTICE_32_g.add_edge(Int512_, Bottom_) +BASE_LATTICE_32_g.add_edge(Int256_, Bottom_) +BASE_LATTICE_32_g.add_edge(Int128_, Bottom_) # Int8: signed/unsigned children -BASE_LATTICE_32.add_edge(Int8_, SInt8_) -BASE_LATTICE_32.add_edge(Int8_, UInt8_) -BASE_LATTICE_32.add_edge(SInt8_, Bottom_) -BASE_LATTICE_32.add_edge(UInt8_, Bottom_) +BASE_LATTICE_32_g.add_edge(Int8_, SInt8_) +BASE_LATTICE_32_g.add_edge(Int8_, UInt8_) +BASE_LATTICE_32_g.add_edge(SInt8_, Bottom_) +BASE_LATTICE_32_g.add_edge(UInt8_, Bottom_) # Int16: signed/unsigned children -BASE_LATTICE_32.add_edge(Int16_, SInt16_) -BASE_LATTICE_32.add_edge(Int16_, UInt16_) -BASE_LATTICE_32.add_edge(SInt16_, Bottom_) -BASE_LATTICE_32.add_edge(UInt16_, Bottom_) +BASE_LATTICE_32_g.add_edge(Int16_, SInt16_) +BASE_LATTICE_32_g.add_edge(Int16_, UInt16_) +BASE_LATTICE_32_g.add_edge(SInt16_, Bottom_) +BASE_LATTICE_32_g.add_edge(UInt16_, Bottom_) # Int32: signed/unsigned children + Pointer32, Enum, Fd -BASE_LATTICE_32.add_edge(Int32_, SInt32_) -BASE_LATTICE_32.add_edge(Int32_, UInt32_) -BASE_LATTICE_32.add_edge(SInt32_, Bottom_) -BASE_LATTICE_32.add_edge(UInt32_, Bottom_) -BASE_LATTICE_32.add_edge(Int32_, Pointer32_) -BASE_LATTICE_32.add_edge(Pointer32_, Bottom_) -BASE_LATTICE_32.add_edge(Int32_, Enum_) -BASE_LATTICE_32.add_edge(Enum_, Bottom_) -BASE_LATTICE_32.add_edge(Int32_, Fd_) -BASE_LATTICE_32.add_edge(Fd_, Bottom_) +BASE_LATTICE_32_g.add_edge(Int32_, SInt32_) +BASE_LATTICE_32_g.add_edge(Int32_, UInt32_) +BASE_LATTICE_32_g.add_edge(SInt32_, Bottom_) +BASE_LATTICE_32_g.add_edge(UInt32_, Bottom_) +BASE_LATTICE_32_g.add_edge(Int32_, Pointer32_) +BASE_LATTICE_32_g.add_edge(Pointer32_, Bottom_) +BASE_LATTICE_32_g.add_edge(Int32_, Enum_) +BASE_LATTICE_32_g.add_edge(Enum_, Bottom_) +BASE_LATTICE_32_g.add_edge(Int32_, Fd_) +BASE_LATTICE_32_g.add_edge(Fd_, Bottom_) # Int64: signed/unsigned children -BASE_LATTICE_32.add_edge(Int64_, SInt64_) -BASE_LATTICE_32.add_edge(Int64_, UInt64_) -BASE_LATTICE_32.add_edge(SInt64_, Bottom_) -BASE_LATTICE_32.add_edge(UInt64_, Bottom_) +BASE_LATTICE_32_g.add_edge(Int64_, SInt64_) +BASE_LATTICE_32_g.add_edge(Int64_, UInt64_) +BASE_LATTICE_32_g.add_edge(SInt64_, Bottom_) +BASE_LATTICE_32_g.add_edge(UInt64_, Bottom_) +BASE_LATTICE_32 = TypeLattice(BASE_LATTICE_32_g) BASE_LATTICES = { 32: BASE_LATTICE_32, @@ -229,14 +277,7 @@ BASE_LATTICES = { } -def _invert_lattice(lattice: networkx.DiGraph) -> networkx.DiGraph: - inverted = networkx.DiGraph() - for src, dst in lattice.edges: - inverted.add_edge(dst, src) - return inverted - - -BASE_LATTICES_INVERTED = {bits: _invert_lattice(lattice) for bits, lattice in BASE_LATTICES.items()} +BASE_LATTICES_INVERTED = {bits: lattice.inverted() for bits, lattice in BASE_LATTICES.items()} # @@ -587,9 +628,7 @@ class SimpleSolver: self.stackvar_max_sizes = stackvar_max_sizes if stackvar_max_sizes is not None else {} self._constraint_set_degradation_threshold = constraint_set_degradation_threshold self._base_lattice = BASE_LATTICES[bits] - self._base_lattice_inverted = networkx.DiGraph() - for src, dst in self._base_lattice.edges: - self._base_lattice_inverted.add_edge(dst, src) + self._base_lattice_inverted = BASE_LATTICES_INVERTED[bits] # statistics self.processed_constraints_count: int = 0 @@ -1845,7 +1884,7 @@ class SimpleSolver: def _lattice_op( t1: TypeConstant, t2: TypeConstant, - lattice: networkx.DiGraph, + lattice: TypeLattice, unit: TypeConstant, ) -> TypeConstant: """ @@ -1860,7 +1899,7 @@ class SimpleSolver: abstract_t1 = SimpleSolver.abstract(t1) abstract_t2 = SimpleSolver.abstract(t2) if abstract_t1 in lattice and abstract_t2 in lattice: - ancestor = networkx.lowest_common_ancestor(lattice, abstract_t1, abstract_t2) + ancestor = lattice.lca(abstract_t1, abstract_t2) if ( isinstance(ancestor, Pointer) @@ -1920,7 +1959,7 @@ class SimpleSolver: @classmethod def _simtype_lattice_op( - cls, t1: SimType, t2: SimType, arch: archinfo.Arch, lattices: dict[int, networkx.DiGraph], unit: TypeConstant + cls, t1: SimType, t2: SimType, arch: archinfo.Arch, lattices: dict[int, TypeLattice], unit: TypeConstant ) -> SimType: if arch.bits not in lattices: raise ValueError(f"Pointer size {arch.bits} is not supported. Expect 32 or 64.") diff --git a/tests/analyses/decompiler/test_signedness.py b/tests/analyses/decompiler/test_signedness.py index db336cdc0..5e7e58b72 100644 --- a/tests/analyses/decompiler/test_signedness.py +++ b/tests/analyses/decompiler/test_signedness.py @@ -139,70 +139,70 @@ class TestLattice(unittest.TestCase): def test_lattice_64_edges_int32(self): """Int32 should have SInt32 and UInt32 as children in 64-bit lattice.""" - assert BASE_LATTICE_64.has_edge(Int32_, SInt32_) - assert BASE_LATTICE_64.has_edge(Int32_, UInt32_) - assert BASE_LATTICE_64.has_edge(SInt32_, Bottom_) - assert BASE_LATTICE_64.has_edge(UInt32_, Bottom_) + assert BASE_LATTICE_64.g.has_edge(Int32_, SInt32_) + assert BASE_LATTICE_64.g.has_edge(Int32_, UInt32_) + assert BASE_LATTICE_64.g.has_edge(SInt32_, Bottom_) + assert BASE_LATTICE_64.g.has_edge(UInt32_, Bottom_) def test_lattice_64_edges_int64(self): """Int64 should have SInt64 and UInt64 as children in 64-bit lattice.""" - assert BASE_LATTICE_64.has_edge(Int64_, SInt64_) - assert BASE_LATTICE_64.has_edge(Int64_, UInt64_) - assert BASE_LATTICE_64.has_edge(SInt64_, Bottom_) - assert BASE_LATTICE_64.has_edge(UInt64_, Bottom_) + assert BASE_LATTICE_64.g.has_edge(Int64_, SInt64_) + assert BASE_LATTICE_64.g.has_edge(Int64_, UInt64_) + assert BASE_LATTICE_64.g.has_edge(SInt64_, Bottom_) + assert BASE_LATTICE_64.g.has_edge(UInt64_, Bottom_) def test_lattice_64_edges_int8(self): """Int8 should have SInt8 and UInt8 as children in 64-bit lattice.""" - assert BASE_LATTICE_64.has_edge(Int8_, SInt8_) - assert BASE_LATTICE_64.has_edge(Int8_, UInt8_) - assert BASE_LATTICE_64.has_edge(SInt8_, Bottom_) - assert BASE_LATTICE_64.has_edge(UInt8_, Bottom_) + assert BASE_LATTICE_64.g.has_edge(Int8_, SInt8_) + assert BASE_LATTICE_64.g.has_edge(Int8_, UInt8_) + assert BASE_LATTICE_64.g.has_edge(SInt8_, Bottom_) + assert BASE_LATTICE_64.g.has_edge(UInt8_, Bottom_) def test_lattice_64_edges_int16(self): """Int16 should have SInt16 and UInt16 as children in 64-bit lattice.""" - assert BASE_LATTICE_64.has_edge(Int16_, SInt16_) - assert BASE_LATTICE_64.has_edge(Int16_, UInt16_) - assert BASE_LATTICE_64.has_edge(SInt16_, Bottom_) - assert BASE_LATTICE_64.has_edge(UInt16_, Bottom_) + assert BASE_LATTICE_64.g.has_edge(Int16_, SInt16_) + assert BASE_LATTICE_64.g.has_edge(Int16_, UInt16_) + assert BASE_LATTICE_64.g.has_edge(SInt16_, Bottom_) + assert BASE_LATTICE_64.g.has_edge(UInt16_, Bottom_) def test_lattice_32_edges(self): """32-bit lattice should also have signed/unsigned children.""" - assert BASE_LATTICE_32.has_edge(Int32_, SInt32_) - assert BASE_LATTICE_32.has_edge(Int32_, UInt32_) - assert BASE_LATTICE_32.has_edge(Int64_, SInt64_) - assert BASE_LATTICE_32.has_edge(Int64_, UInt64_) - assert BASE_LATTICE_32.has_edge(Int8_, SInt8_) - assert BASE_LATTICE_32.has_edge(Int8_, UInt8_) - assert BASE_LATTICE_32.has_edge(Int16_, SInt16_) - assert BASE_LATTICE_32.has_edge(Int16_, UInt16_) + assert BASE_LATTICE_32.g.has_edge(Int32_, SInt32_) + assert BASE_LATTICE_32.g.has_edge(Int32_, UInt32_) + assert BASE_LATTICE_32.g.has_edge(Int64_, SInt64_) + assert BASE_LATTICE_32.g.has_edge(Int64_, UInt64_) + assert BASE_LATTICE_32.g.has_edge(Int8_, SInt8_) + assert BASE_LATTICE_32.g.has_edge(Int8_, UInt8_) + assert BASE_LATTICE_32.g.has_edge(Int16_, SInt16_) + assert BASE_LATTICE_32.g.has_edge(Int16_, UInt16_) def test_lattice_no_direct_intN_to_bottom_64(self): """In 64-bit lattice, IntN should NOT have a direct edge to Bottom (goes through signed/unsigned).""" - assert not BASE_LATTICE_64.has_edge(Int8_, Bottom_) - assert not BASE_LATTICE_64.has_edge(Int16_, Bottom_) - assert not BASE_LATTICE_64.has_edge(Int32_, Bottom_) + assert not BASE_LATTICE_64.g.has_edge(Int8_, Bottom_) + assert not BASE_LATTICE_64.g.has_edge(Int16_, Bottom_) + assert not BASE_LATTICE_64.g.has_edge(Int32_, Bottom_) # Int64 still has edge to Pointer64 which goes to Bottom, but no direct edge - assert not BASE_LATTICE_64.has_edge(Int64_, Bottom_) + assert not BASE_LATTICE_64.g.has_edge(Int64_, Bottom_) def test_lattice_no_direct_intN_to_bottom_32(self): """In 32-bit lattice, IntN should NOT have a direct edge to Bottom.""" - assert not BASE_LATTICE_32.has_edge(Int8_, Bottom_) - assert not BASE_LATTICE_32.has_edge(Int16_, Bottom_) - assert not BASE_LATTICE_32.has_edge(Int64_, Bottom_) + assert not BASE_LATTICE_32.g.has_edge(Int8_, Bottom_) + assert not BASE_LATTICE_32.g.has_edge(Int16_, Bottom_) + assert not BASE_LATTICE_32.g.has_edge(Int64_, Bottom_) def test_lattice_join_signed_unsigned(self): """The LCA (join) of SInt32 and UInt32 should be Int32.""" # In the lattice, both SInt32 and UInt32 are children of Int32 # so their common ancestor is Int32 - ancestors_s = networkx.ancestors(BASE_LATTICE_64, SInt32_) | {SInt32_} - ancestors_u = networkx.ancestors(BASE_LATTICE_64, UInt32_) | {UInt32_} + ancestors_s = networkx.ancestors(BASE_LATTICE_64.g, SInt32_) | {SInt32_} + ancestors_u = networkx.ancestors(BASE_LATTICE_64.g, UInt32_) | {UInt32_} common = ancestors_s & ancestors_u assert Int32_ in common def test_lattice_meet_int_signed(self): """SInt32 is reachable from Int32 (meet of Int32 and SInt32 is SInt32).""" - assert networkx.has_path(BASE_LATTICE_64, Int32_, SInt32_) - assert networkx.has_path(BASE_LATTICE_64, Int32_, UInt32_) + assert networkx.has_path(BASE_LATTICE_64.g, Int32_, SInt32_) + assert networkx.has_path(BASE_LATTICE_64.g, Int32_, UInt32_) class TestTranslator(unittest.TestCase): From 83f364dc013438e5453093bdbcf727cae3589d9d Mon Sep 17 00:00:00 2001 From: Fish Date: Sun, 26 Jul 2026 21:33:20 -0700 Subject: [PATCH 054/122] Memoize C++ prototype parsing. (#6708) --- angr/sim_type.py | 34 ++++++++++++++++++++++------------ angr/utils/library.py | 35 ++++++++++++++++++++++------------- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/angr/sim_type.py b/angr/sim_type.py index db002c0d9..e1a6d3a2c 100644 --- a/angr/sim_type.py +++ b/angr/sim_type.py @@ -8,6 +8,7 @@ import logging import re from collections import ChainMap, OrderedDict, defaultdict from collections.abc import Iterable, MutableMapping +from functools import lru_cache from typing import TYPE_CHECKING, Any, Literal, cast, overload import claripy @@ -4425,6 +4426,25 @@ def normalize_cpp_function_name(name: str) -> str: return name.removesuffix(";") +@lru_cache(maxsize=32768) +def _parse_cpp_decl(s: str) -> cxxheaderparser.simple.ParsedData | None: + """ + Run cxxheaderparser on a normalized C++ declaration and return its parse tree, or None if it cannot be parsed. + + This method is cached to avoid re-parsing of the same declaration, which happens a lot during decompilation. + """ + try: + return cxxheaderparser.simple.parse_string(s) + except cxxheaderparser.errors.CxxParseError: + # GCC-mangled (and thus, demangled) function names do not have return types encoded; let's try to prefix s with + # "void" and try again + try: + return cxxheaderparser.simple.parse_string("void " + s) + except cxxheaderparser.errors.CxxParseError: + # if it still fails, we give up + return None + + def parse_cpp_file(cpp_decl, with_param_names: bool = False): # pylint: disable=unused-argument # # A series of hacks to make cxxheaderparser happy with whatever C++ function prototypes we feed in @@ -4439,19 +4459,9 @@ def parse_cpp_file(cpp_decl, with_param_names: bool = False): # pylint: disable # CppHeaderParser does not like missing function body s += "\n\n{}" - try: - h = cxxheaderparser.simple.parse_string(s) - except cxxheaderparser.errors.CxxParseError: - # GCC-mangled (and thus, demangled) function names do not have return types encoded; let's try to prefix s with - # "void" and try again - s = "void " + s - try: - h = cxxheaderparser.simple.parse_string(s) - except cxxheaderparser.errors.CxxParseError: - # if it still fails, we give up - return None, None + h = _parse_cpp_decl(s) - if not h.namespace: + if h is None or not h.namespace: return None, None func_decls: dict[str, SimTypeCppFunction | SimTypeFunction] = {} diff --git a/angr/utils/library.py b/angr/utils/library.py index 427ae64ec..f334bece5 100644 --- a/angr/utils/library.py +++ b/angr/utils/library.py @@ -1,5 +1,6 @@ from __future__ import annotations +from functools import lru_cache from typing import TYPE_CHECKING from angr.sim_type import ( @@ -194,6 +195,24 @@ def cprotos2py(cprotos: list[str], fd_spots=frozenset(), remove_sys_prefix=False return parsedcprotos2py(parsed_cprotos, fd_spots=fd_spots, remove_sys_prefix=remove_sys_prefix) +@lru_cache(maxsize=32768) +def _cpp_function_name_and_metadata(demangled_name: str) -> tuple[str, bool, bool]: + """ + The memoizable core of :func:`get_cpp_function_name_and_metadata`. + """ + func_decls, _ = parse_cpp_file(demangled_name) + if func_decls and len(func_decls) == 1: + key = next(iter(func_decls)) + decl = func_decls[key] + if isinstance(decl, SimTypeCppFunction): + if decl.ctor: + return key, True, False + if decl.dtor: + return key, False, True + return key, False, False + return normalize_cpp_function_name(demangled_name), False, False + + def get_cpp_function_name_and_metadata(demangled_name: str) -> tuple[str, dict[str, bool]]: """ Parse a demangled C++ declaration into a function name. @@ -206,19 +225,9 @@ def get_cpp_function_name_and_metadata(demangled_name: str) -> tuple[str, dict[s :return: The qualified function name, excluding return type and parameters, and a dictionary with keys indicating if the function is a constructor or a destructor. """ - demangled_name = demangled_name.strip() - func_decls, _ = parse_cpp_file(demangled_name) - d = {"ctor": False, "dtor": False} - if func_decls and len(func_decls) == 1: - key = next(iter(func_decls)) - decl = func_decls[key] - if isinstance(decl, SimTypeCppFunction): - if decl.ctor: - d["ctor"] = True - elif decl.dtor: - d["dtor"] = True - return key, d - return normalize_cpp_function_name(demangled_name), d + name, ctor, dtor = _cpp_function_name_and_metadata(demangled_name.strip()) + # always hand out a fresh dict: callers are free to mutate it + return name, {"ctor": ctor, "dtor": dtor} def get_cpp_function_name(demangled_name: str) -> str: From 3b41a92e29fd40af60114cb1f9044e65f03e3601 Mon Sep 17 00:00:00 2001 From: Fish Date: Sun, 26 Jul 2026 22:10:26 -0700 Subject: [PATCH 055/122] SimpleSolver: Hash memoization. (#6709) --- angr/analyses/typehoon/_typehash.py | 6 ----- angr/analyses/typehoon/simple_solver.py | 31 ++++++++++++++-------- angr/analyses/typehoon/typeconsts.py | 34 +++++++++++++++++-------- angr/analyses/typehoon/variance.py | 11 ++++---- 4 files changed, 50 insertions(+), 32 deletions(-) diff --git a/angr/analyses/typehoon/_typehash.py b/angr/analyses/typehoon/_typehash.py index d80c5047f..5a30ee214 100644 --- a/angr/analyses/typehoon/_typehash.py +++ b/angr/analyses/typehoon/_typehash.py @@ -8,12 +8,6 @@ _TAG_CACHE: dict[type, int] = {} def type_tag(cls: type) -> int: """ Return a stable, process-independent integer tag for ``cls``. - - The built-in ``hash(cls)`` is derived from the class object's identity (its memory address), which differs - from one process to the next. Embedding it in ``__hash__`` therefore makes the hashes of type-system - objects -- and the iteration order of every set, dict, or graph that contains them -- vary across runs, - which is a source of non-deterministic type inference. Hashing on a CRC of the qualified name instead is - stable across processes and independent of ``PYTHONHASHSEED``. """ tag = _TAG_CACHE.get(cls) if tag is None: diff --git a/angr/analyses/typehoon/simple_solver.py b/angr/analyses/typehoon/simple_solver.py index f33bf35a8..18756a65f 100644 --- a/angr/analyses/typehoon/simple_solver.py +++ b/angr/analyses/typehoon/simple_solver.py @@ -298,12 +298,13 @@ class SketchNode(SketchNodeBase): Represents a node in a sketch graph. """ - __slots__ = ("lower_bound", "typevar", "upper_bound") + __slots__ = ("_hash", "lower_bound", "typevar", "upper_bound") def __init__(self, typevar: TypeVariable | DerivedTypeVariable): self.typevar: TypeVariable | DerivedTypeVariable = typevar self.upper_bound: TypeConstant = TopType() self.lower_bound: TypeConstant = BottomType() + self._hash = hash((_SKETCH_NODE_TAG, typevar)) def __repr__(self): return f"{self.lower_bound} <: {self.typevar} <: {self.upper_bound}" @@ -312,7 +313,7 @@ class SketchNode(SketchNodeBase): return isinstance(other, SketchNode) and self.typevar == other.typevar def __hash__(self): - return hash((type_tag(SketchNode), self.typevar)) + return self._hash @property def size(self) -> int | None: @@ -333,6 +334,9 @@ class SketchNode(SketchNodeBase): return None +_SKETCH_NODE_TAG = type_tag(SketchNode) + + class RecursiveRefNode(SketchNodeBase): """ Represents a cycle in a sketch graph. @@ -484,24 +488,30 @@ class Sketch: class ConstraintGraphTag(enum.Enum): + def __init__(self, n): + self._hash = hash(("ConstraintGraphTag", n)) + + def __hash__(self): + return self._hash + LEFT = 0 RIGHT = 1 UNKNOWN = 2 - def __hash__(self): - return hash((type_tag(ConstraintGraphTag), self.value)) - class FORGOTTEN(enum.Enum): + def __init__(self, n): + self._hash = hash(("FORGOTTEN", n)) + + def __hash__(self): + return self._hash + PRE_FORGOTTEN = 0 POST_FORGOTTEN = 1 - def __hash__(self): - return hash((type_tag(FORGOTTEN), self.value)) - class ConstraintGraphNode: - __slots__ = ("forgotten", "tag", "typevar", "variance") + __slots__ = ("_hash", "forgotten", "tag", "typevar", "variance") def __init__( self, @@ -514,6 +524,7 @@ class ConstraintGraphNode: self.variance = variance self.tag = tag self.forgotten = forgotten + self._hash = hash(("ConstraintGraphNode", typevar, variance, tag, forgotten)) def __repr__(self): variance_str = "CO" if self.variance == Variance.COVARIANT else "CONTRA" @@ -540,7 +551,7 @@ class ConstraintGraphNode: ) def __hash__(self): - return hash((type_tag(ConstraintGraphNode), self.typevar, self.variance, self.tag, self.forgotten)) + return self._hash def forget_last_label(self) -> tuple[ConstraintGraphNode, BaseLabel] | None: if isinstance(self.typevar, DerivedTypeVariable) and self.typevar.labels: diff --git a/angr/analyses/typehoon/typeconsts.py b/angr/analyses/typehoon/typeconsts.py index 657211667..8cc19ef25 100644 --- a/angr/analyses/typehoon/typeconsts.py +++ b/angr/analyses/typehoon/typeconsts.py @@ -28,6 +28,12 @@ def memoize(f): class TypeConstant: SIZE = None + TYPE_HASH: int = 0 + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls.TYPE_HASH = type_tag(cls) + def __init__(self, name: str | None = None): self.name = name @@ -35,13 +41,15 @@ class TypeConstant: return repr(self) def _hash(self, visited: set[int]): # pylint:disable=unused-argument - return type_tag(type(self)) + return self.TYPE_HASH def __eq__(self, other): return type(self) is type(other) def __hash__(self): - return self._hash(set()) + # the hash of a plain type constant is fully determined by its class, so there is nothing to + # recompute here -- see TYPE_HASH above + return self.TYPE_HASH @property def size(self) -> int: @@ -60,6 +68,9 @@ class TypeConstant: return mapping.get(id(self), self) +TypeConstant.TYPE_HASH = type_tag(TypeConstant) + + class TopType(TypeConstant): SIZE = 1 @@ -221,8 +232,8 @@ class Pointer(TypeConstant): def _hash(self, visited: set[int]): if self.basetype is None: - return type_tag(type(self)) - return hash((type_tag(type(self)), self.basetype._hash(visited))) + return self.TYPE_HASH + return hash((self.TYPE_HASH, self.basetype._hash(visited))) def new(self, basetype, name: str | None = None): return self.__class__(basetype, name=name) @@ -302,7 +313,7 @@ class Array(TypeConstant): if id(self) in visited: return 0 visited.add(id(self)) - return hash((type_tag(type(self)), self.element, self.count)) + return hash((self.TYPE_HASH, self.element, self.count)) def __hash__(self): return self._hash(set()) @@ -333,7 +344,7 @@ class Struct(TypeConstant): if id(self) in visited: return 0 visited.add(id(self)) - return hash((type_tag(type(self)), self.idx, self._hash_fields(visited))) + return hash((self.TYPE_HASH, self.idx, self._hash_fields(visited))) def _hash_fields(self, visited: set[int]): keys = sorted(self.fields.keys()) @@ -402,7 +413,7 @@ class RustEnum(TypeConstant): if id(self) in visited: return 0 visited.add(id(self)) - return hash((type_tag(type(self)), self._hash_fields(visited))) + return hash((self.TYPE_HASH, self._hash_fields(visited))) def _hash_fields(self, visited: set[int]): # pylint:disable=unused-argument tpl = tuple(hash(variant) for variant in self.variants) @@ -466,6 +477,7 @@ class Enum(TypeConstant): self.members: dict[str, int] = members if members is not None else {} self.base_type = base_type self.idx = idx if idx != -1 else next(_ENUM_ID) + self._cached_hash = hash((self.TYPE_HASH, self.idx, tuple(sorted(self.members.items())))) @property def size(self) -> int: @@ -487,10 +499,10 @@ class Enum(TypeConstant): if id(self) in visited: return 0 visited.add(id(self)) - return hash((type_tag(type(self)), self.idx, tuple(sorted(self.members.items())))) + return self._cached_hash def __hash__(self): - return self._hash(set()) + return self._cached_hash def replace( self, @@ -527,7 +539,7 @@ class Function(TypeConstant): params_hash = tuple(param._hash(visited) for param in self.params) outputs_hash = tuple(out._hash(visited) for out in self.outputs) - return hash((Function, params_hash, outputs_hash)) + return hash((self.TYPE_HASH, params_hash, outputs_hash)) def __hash__(self): return self._hash(set()) @@ -571,7 +583,7 @@ class TypeVariableReference(TypeConstant): return type(other) is type(self) and self.typevar == other.typevar def __hash__(self): - return hash((type_tag(type(self)), self.typevar)) + return hash((self.TYPE_HASH, self.typevar)) # diff --git a/angr/analyses/typehoon/variance.py b/angr/analyses/typehoon/variance.py index e475a1964..61c5566f3 100644 --- a/angr/analyses/typehoon/variance.py +++ b/angr/analyses/typehoon/variance.py @@ -2,16 +2,17 @@ from __future__ import annotations import enum -from ._typehash import type_tag - class Variance(enum.Enum): """ Enum class describing the variance of type constraints. """ - COVARIANT = 0 - CONTRAVARIANT = 1 + def __init__(self, n): + self._hash = hash(("Variance", n)) def __hash__(self): - return hash((type_tag(Variance), self.value)) + return self._hash + + COVARIANT = 0 + CONTRAVARIANT = 1 From 6d5860d0ba4bdbc01c8955c2df2368f3a5b56c8e Mon Sep 17 00:00:00 2001 From: Fish Date: Sun, 26 Jul 2026 22:20:20 -0700 Subject: [PATCH 056/122] COWDict: Faster chain walks. (#6710) --- angr/utils/cowdict.py | 164 ++++++++++++++++++++++++++---------------- 1 file changed, 103 insertions(+), 61 deletions(-) diff --git a/angr/utils/cowdict.py b/angr/utils/cowdict.py index 9c50cfb5a..42cd258eb 100644 --- a/angr/utils/cowdict.py +++ b/angr/utils/cowdict.py @@ -40,10 +40,18 @@ class ChainMapCOW[K, V](ChainMap): Tracks logically deleted keys via a _deleted set so that pop() and del work correctly even when keys live in parent maps. + + Performance note: We deliberately re-implement the ChainMap walk for better performance. ``ChainMap.__getitem__`` + raises a ``KeyError`` for every layer that misses, and ``ChainMap.get`` walks the whole chain twice + (``__contains__`` and ``__getitem__``). We avoid both by inlining ``if key in mapping``. + + We may revisit the implementation in the future when the performance of ChainMap improves. """ + __slots__ = ("_deleted", "collapse_threshold", "dirty", "maps") + def __init__(self, *args, collapse_threshold=None): - super().__init__(*args) + self.maps = list(args) or [{}] self.dirty = False self.collapse_threshold = collapse_threshold self._deleted: set = set() @@ -55,19 +63,32 @@ class ChainMapCOW[K, V](ChainMap): def __getitem__(self, key: K) -> V: if key in self._deleted: raise KeyError(key) - return super().__getitem__(key) + for mapping in self.maps: + if key in mapping: + return mapping[key] + return self.__missing__(key) def __contains__(self, key) -> bool: if key in self._deleted: return False - return super().__contains__(key) + # an explicit loop, not any(), on purpose: this is hot and any() costs a generator frame per call + for mapping in self.maps: # noqa: SIM110 + if key in mapping: + return True + return False def __setitem__(self, key: K, value: V) -> None: - self._deleted.discard(key) - super().__setitem__(key, value) + if self._deleted: + self._deleted.discard(key) + self.maps[0][key] = value def __delitem__(self, key: K): - if key in self._deleted or not super().__contains__(key): + if key in self._deleted: + raise KeyError(key) + for mapping in self.maps: + if key in mapping: + break + else: raise KeyError(key) # Remove from maps[0] if present self.maps[0].pop(key, None) @@ -75,26 +96,26 @@ class ChainMapCOW[K, V](ChainMap): self._deleted.add(key) def pop(self, key: K, *args) -> V: # type: ignore[reportIncompatibleMethodOverride] - if key in self._deleted: - if args: - return args[0] - raise KeyError(key) - try: - value = super().__getitem__(key) - except KeyError: - if args: - return args[0] - raise - # Remove from maps[0] if present - self.maps[0].pop(key, None) - # Mark as deleted so parent maps don't expose it - self._deleted.add(key) - return value + if key not in self._deleted: + for mapping in self.maps: + if key in mapping: + value = mapping[key] + # Remove from maps[0] if present + self.maps[0].pop(key, None) + # Mark as deleted so parent maps don't expose it + self._deleted.add(key) + return value + if args: + return args[0] + raise KeyError(key) def get[TD](self, key: K, default: TD = None) -> V | TD: if key in self._deleted: return default - return super().get(key, default) + for mapping in self.maps: + if key in mapping: + return mapping[key] + return default def __iter__(self): seen = set(self._deleted) @@ -108,24 +129,35 @@ class ChainMapCOW[K, V](ChainMap): return len(set().union(*self.maps) - self._deleted) def new_child(self, m=None) -> ChainMapCOW[K, V]: - if m is None: - m = {} - return ChainMapCOW(m, *self.maps, collapse_threshold=self.collapse_threshold) + obj = ChainMapCOW.__new__(ChainMapCOW) + obj.maps = [{} if m is None else m, *self.maps] + obj.dirty = False + obj.collapse_threshold = self.collapse_threshold + obj._deleted = set() + return obj + + def _collapsed_dict(self) -> dict: + collapsed: dict = {} + for m in reversed(self.maps): + collapsed.update(m) + for k in self._deleted: + collapsed.pop(k, None) + return collapsed def clean(self) -> ChainMapCOW[K, V]: - if self.dirty: - # collapse? - if self.collapse_threshold is not None and len(self.maps) >= self.collapse_threshold: - collapsed = {} - for m in reversed(self.maps): - collapsed.update(m) - for k in self._deleted: - collapsed.pop(k, None) - return ChainMapCOW(collapsed, collapse_threshold=self.collapse_threshold) - ch = self.new_child() - ch._deleted = set(self._deleted) - return ch - return self + if not self.dirty: + return self + obj = ChainMapCOW.__new__(ChainMapCOW) + obj.dirty = False + obj.collapse_threshold = self.collapse_threshold + # collapse? + if self.collapse_threshold is not None and len(self.maps) >= self.collapse_threshold: + obj.maps = [self._collapsed_dict()] + obj._deleted = set() + else: + obj.maps = [{}, *self.maps] + obj._deleted = set(self._deleted) + return obj class DefaultChainMapCOW[K, V](ChainMapCOW): @@ -133,39 +165,49 @@ class DefaultChainMapCOW[K, V](ChainMapCOW): Implements a copy-on-write version of ChainMap with default values that supports auto-collapsing. """ + __slots__ = ("default_factory",) + def __init__(self, *args, default_factory: Callable, collapse_threshold=None): super().__init__(*args, collapse_threshold=collapse_threshold) self.default_factory = default_factory def __getitem__(self, key: K) -> V: - try: - return super().__getitem__(key) - except KeyError: - self.__setitem__(key, self.default_factory()) - return super().__getitem__(key) + deleted = self._deleted + if key not in deleted: + for mapping in self.maps: + if key in mapping: + return mapping[key] + else: + deleted.discard(key) + value = self.default_factory() + self.maps[0][key] = value + return value def new_child(self, m=None, **kwargs) -> DefaultChainMapCOW[K, V]: if m is None: m = kwargs elif kwargs: m.update(kwargs) - return DefaultChainMapCOW( - m, *self.maps, default_factory=self.default_factory, collapse_threshold=self.collapse_threshold - ) + obj = DefaultChainMapCOW.__new__(DefaultChainMapCOW) + obj.maps = [m, *self.maps] + obj.dirty = False + obj.collapse_threshold = self.collapse_threshold + obj.default_factory = self.default_factory + obj._deleted = set() + return obj def clean(self) -> DefaultChainMapCOW[K, V]: - if self.dirty: - # collapse? - if self.collapse_threshold is not None and len(self.maps) >= self.collapse_threshold: - collapsed = {} - for m in reversed(self.maps): - collapsed.update(m) - for k in self._deleted: - collapsed.pop(k, None) - return DefaultChainMapCOW( - collapsed, default_factory=self.default_factory, collapse_threshold=self.collapse_threshold - ) - r = self.new_child() - r._deleted = set(self._deleted) - return r - return self + if not self.dirty: + return self + obj = DefaultChainMapCOW.__new__(DefaultChainMapCOW) + obj.dirty = False + obj.collapse_threshold = self.collapse_threshold + obj.default_factory = self.default_factory + # collapse? + if self.collapse_threshold is not None and len(self.maps) >= self.collapse_threshold: + obj.maps = [self._collapsed_dict()] + obj._deleted = set() + else: + obj.maps = [{}, *self.maps] + obj._deleted = set(self._deleted) + return obj From e4ff240001db8fecc4ecac5f0215c773e13674ce Mon Sep 17 00:00:00 2001 From: Quintin Kong <71952215+DORA-B@users.noreply.github.com> Date: Mon, 27 Jul 2026 01:32:25 -0400 Subject: [PATCH 057/122] x86/amd64: fix UMUL CF/OF flags (widen before multiply) (#6703) Fixes #6067 pc_actions_UMUL computed the product at operand width, so the extracted "high half" was always zero and CF/OF (OF = CF) were constantly 0: lo = (cc_dep1 * cc_dep2)[nbits-1:0] # truncated to nbits hi = (lo >> nbits)[nbits-1:0] # lo is nbits wide -> always 0 The sibling pc_actions_SMUL is correct because it widens first (sign_extend). Mirror it with zero_extend: multiply the operands widened to 2*nbits and take the high half. For CC_OP_MUL{B,W,L,Q}, CF = OF = (high half != 0) per the Intel SDM. This is why `imul` reported CF/OF correctly while `mul` did not (issue #6067: `mul %ebx` left CF clear). --- angr/engines/vex/claripy/ccall.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/angr/engines/vex/claripy/ccall.py b/angr/engines/vex/claripy/ccall.py index cc78c6609..004fa76ba 100644 --- a/angr/engines/vex/claripy/ccall.py +++ b/angr/engines/vex/claripy/ccall.py @@ -551,9 +551,9 @@ def pc_actions_ROR(state, nbits, res, _, cc_ndep, platform=None): def pc_actions_UMUL(state, nbits, cc_dep1, cc_dep2, cc_ndep, platform=None): - lo = (cc_dep1 * cc_dep2)[nbits - 1 : 0] - rr = lo - hi = (rr >> nbits)[nbits - 1 : 0] + rr = cc_dep1.zero_extend(nbits) * cc_dep2.zero_extend(nbits) + hi = rr[2 * nbits - 1 : nbits] + lo = rr[nbits - 1 : 0] cf = claripy.If(hi != 0, claripy.BVV(1, 1), claripy.BVV(0, 1)) zf = calc_zerobit(lo) pf = calc_paritybit(lo) From 9c1fb2367efcd83807c6d6113a02ae56850f0757 Mon Sep 17 00:00:00 2001 From: Fish Date: Mon, 27 Jul 2026 00:03:32 -0700 Subject: [PATCH 058/122] StructuringOptimizationPass: Cache structurability across passes. (#6711) --- .../lowered_switch_simplifier.py | 6 ++ .../optimization_passes/optimization_pass.py | 102 +++++++++++++++--- 2 files changed, 96 insertions(+), 12 deletions(-) diff --git a/angr/analyses/decompiler/optimization_passes/lowered_switch_simplifier.py b/angr/analyses/decompiler/optimization_passes/lowered_switch_simplifier.py index 0c45a0677..84dd089ac 100644 --- a/angr/analyses/decompiler/optimization_passes/lowered_switch_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/lowered_switch_simplifier.py @@ -230,6 +230,7 @@ class LoweredSwitchSimplifier(StructuringOptimizationPass): graph_copy = networkx.DiGraph(self._graph) self.out_graph = graph_copy node_to_heads = defaultdict(set) + modified = False for _, caselists in variablehash_to_cases.items(): for cases, redundant_nodes in caselists: @@ -346,6 +347,7 @@ class LoweredSwitchSimplifier(StructuringOptimizationPass): new_head.statements[-1] = switch_stmt # update the block self._update_block(original_head, new_head) + modified = True # sanity check that no switch head points to either itself # or to any if-head that was merged into the new switch head; this @@ -412,6 +414,10 @@ class LoweredSwitchSimplifier(StructuringOptimizationPass): else: graph_copy.add_edge(node_copy, succ) + if not modified: + # the graph is not modified + self.out_graph = None + return False return True def _find_cascading_switch_variable_comparisons(self): diff --git a/angr/analyses/decompiler/optimization_passes/optimization_pass.py b/angr/analyses/decompiler/optimization_passes/optimization_pass.py index 1bac30917..02e8c85f5 100644 --- a/angr/analyses/decompiler/optimization_passes/optimization_pass.py +++ b/angr/analyses/decompiler/optimization_passes/optimization_pass.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging +import weakref from collections import namedtuple from collections.abc import Generator from enum import Enum @@ -20,6 +21,7 @@ from angr.analyses.decompiler.structuring import RecursiveStructurer, SAILRStruc from angr.analyses.decompiler.utils import add_labels, is_empty_node, remove_edges_in_ailgraph if TYPE_CHECKING: + from angr.analyses.decompiler.region_identifier import RegionIdentifier, RegionOverlay from angr.analyses.decompiler.stack_item import StackItem from angr.knowledge_plugins.functions import Function from angr.project import Project @@ -125,6 +127,9 @@ class OptimizationPass(BaseOptimizationPass): _graph: networkx.DiGraph + # self._scratch[_STRUCTURABILITY_CACHE_KEY] = weakref(graph), params for applying the structurability result + _STRUCTURABILITY_CACHE_KEY = "structurability_cache" + def __init__( self, func, @@ -178,6 +183,24 @@ class OptimizationPass(BaseOptimizationPass): self.out_graph: networkx.DiGraph | None = None self.stack_items: dict[int, StackItem] = {} + def analyze(self): + super().analyze() + self._invalidate_structurability_cache_if_modified() + + def _invalidate_structurability_cache_if_modified(self) -> None: + """ + Drop the shared structurability cache if this pass produced an output graph. + + ``out_graph`` is the single channel through which a pass reports a graph change, and it covers both flavors: + a freshly built graph object (which would miss the identity-keyed cache anyway) and an in-place mutation of + the graph the pass was handed (which would *wrongly hit* it). Invalidating on any non-None ``out_graph`` + keeps the cache sound without having to fingerprint AIL contents -- note that ``Block.__hash__`` is memoized + by the Rust backend and does not track in-place statement edits, so content fingerprints based on it would be + unsound. + """ + if self.out_graph is not None: + self._scratch.pop(self._STRUCTURABILITY_CACHE_KEY, None) + @property def blocks_by_addr(self) -> dict[int, set[ailment.Block]]: return self._blocks_by_addr @@ -500,6 +523,12 @@ class StructuringOptimizationPass(OptimizationPass): raise NotImplementedError def analyze(self): + try: + self._analyze_and_verify() + finally: + self._invalidate_structurability_cache_if_modified() + + def _analyze_and_verify(self): """ Wrapper for _analyze() that verifies the graph is structurable before and after the optimization. """ @@ -600,18 +629,55 @@ class StructuringOptimizationPass(OptimizationPass): if not had_any_changes: self.out_graph = None - def _graph_is_structurable(self, graph, readd_labels=False, initial=False) -> bool: + def _graph_is_structurable(self, graph, readd_labels: bool = False, initial: bool = False) -> bool: """ Checks weather the input graph is structurable under the Phoenix schema-matching structuring algorithm. As a side effect, this will also update the region identifier and goto manager of this optimization pass. - Consequently, a true return guarantees up-to-date goto information in the goto manager. + Consequently, a True return guarantees up-to-date goto information in the goto manager. + + We cache the structurability probe result in self._scratch. An optimization pass invalidates the cached + structurability result if it updates the graph. + """ + # Only the probe of an unmodified input graph is cacheable + cacheable = initial and not readd_labels + if self._edges_to_remove: + # remove_edges_in_ailgraph() below mutates the graph in place, so anything cached for it describes the + # graph as it was before those edges were dropped. + self._scratch.pop(self._STRUCTURABILITY_CACHE_KEY, None) + cacheable = False + + if cacheable: + # check cache + entry = self._scratch.get(self._STRUCTURABILITY_CACHE_KEY) + if entry is not None and entry[0]() is graph: + structurable, ri, goto_manager, region = entry[1] + self._apply_structurability_result(structurable, ri, goto_manager, region, initial=initial) + return structurable + + # the old route + structurable, ri, goto_manager, region = self._compute_structurability(graph, readd_labels) + if cacheable: + # cache the result + self._scratch[self._STRUCTURABILITY_CACHE_KEY] = ( + weakref.ref(graph), + (structurable, ri, goto_manager, region), + ) + self._apply_structurability_result(structurable, ri, goto_manager, region, initial=initial) + return structurable + + def _compute_structurability( + self, graph, readd_labels: bool + ) -> tuple[bool, RegionIdentifier | None, GotoManager | None, RegionOverlay | None]: + """ + Run region identification, structuring, and region simplification on ``graph`` to determine whether it is + structurable or not. """ if readd_labels: graph = add_labels(graph, self.manager) remove_edges_in_ailgraph(graph, self._edges_to_remove) - self._ri = self.project.analyses[angr.analyses.decompiler.RegionIdentifier].prep(kb=self.kb)( + ri = self.project.analyses[angr.analyses.decompiler.RegionIdentifier].prep(kb=self.kb)( self._func, graph=graph, ail_manager=self.manager, @@ -622,15 +688,15 @@ class StructuringOptimizationPass(OptimizationPass): expose_loop_head_backedges=True, entry_node_addr=self.entry_node_addr, ) - if self._ri is None: - return False + if ri is None: + return False, None, None, None # we should try-catch structuring here because we can often pass completely invalid graphs # that break the assumptions of the structuring algorithm try: rs = self.project.analyses[RecursiveStructurer].prep(kb=self.kb)( - self._ri.region, - cond_proc=self._ri.cond_proc, + ri.region, + cond_proc=ri.cond_proc, ail_manager=self.manager, func=self._func, structurer_cls=SAILRStructurer, @@ -641,17 +707,29 @@ class StructuringOptimizationPass(OptimizationPass): rs = None if not rs or not rs.result or is_empty_node(rs.result) or rs.result_incomplete: - return False + return False, ri, None, None rs = self.project.analyses.RegionSimplifier( self._func, rs.result, self.manager, arg_vvars=self._arg_vvars, kb=self.kb ) if not rs or rs.goto_manager is None or rs.result is None: - return False + return False, ri, None, None - self._analyze_simplified_region(rs.result, initial=initial) - self._goto_manager = rs.goto_manager - return True + return True, ri, rs.goto_manager, rs.result + + def _apply_structurability_result( + self, + structurable: bool, + ri: RegionIdentifier | None, + goto_manager: GotoManager | None, + region: RegionOverlay | None, + initial: bool = False, + ) -> None: + self._ri = ri + if structurable: + assert ri is not None and goto_manager is not None and region is not None + self._analyze_simplified_region(region, initial=initial) + self._goto_manager = goto_manager # pylint:disable=no-self-use def _analyze_simplified_region(self, region, initial=False): From 651d9cccc21dbe97b08e806fc97917f748757386 Mon Sep 17 00:00:00 2001 From: Fish Date: Mon, 27 Jul 2026 01:56:24 -0700 Subject: [PATCH 059/122] Tests: Speed up decompiler-related test cases (strike 1). (#6713) --- .../decompiler/test_ccall_rewriting.py | 8 +- tests/analyses/decompiler/test_decompiler.py | 58 +++++-- .../test_register_save_area_simplifier_adv.py | 20 +-- .../decompiler/test_rust_decompiler.py | 64 ++++++-- .../test_variable_nondeterminism.py | 24 ++- tests/common.py | 142 +++++++++++++++++- 6 files changed, 275 insertions(+), 41 deletions(-) diff --git a/tests/analyses/decompiler/test_ccall_rewriting.py b/tests/analyses/decompiler/test_ccall_rewriting.py index 6212a3354..8457dd384 100644 --- a/tests/analyses/decompiler/test_ccall_rewriting.py +++ b/tests/analyses/decompiler/test_ccall_rewriting.py @@ -13,7 +13,7 @@ import angr from angr.ailment import Expr, Manager from angr.analyses.decompiler.ccall_rewriters.amd64_ccalls import AMD64CCallRewriter from angr.engines.vex.claripy.ccall import data -from tests.common import bin_location, print_decompilation_result +from tests.common import bin_location, load_project_with_scoped_cfg, print_decompilation_result test_location = os.path.join(bin_location, "tests") @@ -264,8 +264,7 @@ class TestAMD64CondOverflowBinary(unittest.TestCase): # This function also keeps ccalls from cc_op families outside this rewrite, # so only the OF conditions are asserted -- the rewrite must be surgical. bin_path = os.path.join(test_location, "x86_64", "tar_gcc17_O2") - proj = angr.Project(bin_path, auto_load_libs=False) - cfg = proj.analyses.CFGFast(fail_fast=True, normalize=True) + proj, cfg = load_project_with_scoped_cfg(bin_path, 0x53F6C0, run_ccc=False) dec = proj.analyses.Decompiler(cfg.functions[0x53F6C0], cfg=cfg) assert dec.codegen is not None and dec.codegen.text is not None assert "__OFUMUL__" in dec.codegen.text @@ -291,8 +290,7 @@ class TestAMD64CondOverflowBinary(unittest.TestCase): # fillbuf @ 0x40cc10 -- CondO x ADDQ (4), 2 sites # xstrtoimax @ 0x4888a0 -- CondO x SMULQ (52), 14 sites bin_path = os.path.join(test_location, "x86_64", "grep_gcc17.0.0_O2") - proj = angr.Project(bin_path, auto_load_libs=False) - cfg = proj.analyses.CFGFast(fail_fast=True, normalize=True) + proj, cfg = load_project_with_scoped_cfg(bin_path, 0x40CC10, extra_func_addrs=[0x4888A0], run_ccc=False) dec = proj.analyses.Decompiler(cfg.functions[0x40CC10], cfg=cfg) assert dec.codegen is not None and dec.codegen.text is not None diff --git a/tests/analyses/decompiler/test_decompiler.py b/tests/analyses/decompiler/test_decompiler.py index 59bc59c9e..95b650db2 100755 --- a/tests/analyses/decompiler/test_decompiler.py +++ b/tests/analyses/decompiler/test_decompiler.py @@ -1351,15 +1351,25 @@ class TestDecompiler(unittest.TestCase): @for_all_structuring_algos def test_decompiling_newburry_main(self, decompiler_options=None): bin_path = os.path.join(test_location, "x86_64", "decompiler", "newbury") - p = angr.Project(bin_path, auto_load_libs=False) - - cfg = p.analyses[CFGFast].prep(show_progressbar=not WORKER)(data_references=True, normalize=True) + p, cfg = load_project_with_scoped_cfg( + bin_path, + 0x40F696, # main + extra_func_addrs=( + 0x4190E3, # log_failed_assert: noreturn, must be analyzed or the call is not marked as such + 0x4388B0, # plugins_call_handle_request_env: only referenced as a function pointer + ), + window=0x800, # main is 0x3ea bytes long + expand_call_tree=False, + include_plt=True, + run_ccc=False, + ) func = cfg.functions["main"] dec = p.analyses[Decompiler].prep(show_progressbar=not WORKER)(func, cfg=cfg.model, options=decompiler_options) assert dec.codegen is not None, f"Failed to decompile function {func!r}." print_decompilation_result(dec) + assert dec.codegen is not None and dec.codegen.text is not None code = dec.codegen.text # return statements should not be wrapped into a for statement @@ -3547,19 +3557,25 @@ class TestDecompiler(unittest.TestCase): dec = p.analyses[Decompiler].prep(fail_fast=True)( f, cfg=cfg.model, options=decompiler_options_2, optimization_passes=all_optimization_passes ) + assert dec.codegen is not None and dec.codegen.text is not None print_decompilation_result(dec) assert dec.codegen.text == saved @for_all_structuring_algos def test_function_pointer_identification(self, decompiler_options=None): bin_path = os.path.join(test_location, "x86_64", "rust_hello_world") - proj = angr.Project(bin_path, auto_load_libs=False) - cfg = proj.analyses.CFGFast(resolve_indirect_jumps=True, normalize=True) + proj, cfg = load_project_with_scoped_cfg( + bin_path, + 0x408A50, # main + window=0x400, + run_ccc=False, + ) f = proj.kb.functions["main"] d = proj.analyses[Decompiler](f, cfg=cfg.model, options=decompiler_options) print_decompilation_result(d) + assert d.codegen is not None and d.codegen.text is not None text = d.codegen.text assert "extern" not in text assert "std::rt::lang_start(rust_hello_world::main" in text @@ -5545,7 +5561,11 @@ class TestDecompiler(unittest.TestCase): def test_decompiling_rust_fmt_main(self, decompiler_options=None): bin_path = os.path.join(test_location, "x86_64", "decompiler", "fmt_rust") - proj = angr.Project(bin_path, auto_load_libs=False) + # turning off cache for better speed + proj = angr.Project( + bin_path, + cache_limits={"functions": None, "cfg_nodes": None, "cfg_edges": None}, + ) cfg = proj.analyses.CFG(normalize=True) func = proj.kb.functions[0x469200] decompiler_options = decompiler_options or [] @@ -5596,8 +5616,15 @@ class TestDecompiler(unittest.TestCase): def test_decompiling_rust_fmt_build_best_path_no_ref_using_args(self, decompiler_options=None): bin_path = os.path.join(test_location, "x86_64", "decompiler", "fmt_rust") - proj = angr.Project(bin_path, auto_load_libs=False) - cfg = proj.analyses.CFG(normalize=True) + # build_best_path is 0x75 bytes long inside a 9k-function Rust binary: a whole-binary CFG costs ~48 s + # while decompiling this function takes 0.2 s. + proj, cfg = load_project_with_scoped_cfg( + bin_path, + 0x4BC130, # uu_fmt::linebreak::build_best_path + extra_func_addrs=(0x4BAE60, 0x4BC1B0), # Iterator::reduce, build_best_path::{{closure}} + expand_call_tree=False, + run_ccc=False, + ) func = proj.kb.functions[0x4BC130] dec = proj.analyses.Decompiler(func, cfg=cfg, options=decompiler_options) assert dec.codegen is not None and dec.codegen.text is not None @@ -5939,6 +5966,9 @@ class TestDecompiler(unittest.TestCase): def test_x86_ccall_rewriter_condbe_sub_width(self, decompiler_options=None): # sub_74d6a1 uses unsigned byte comparisons (cmp cl, ...; jbe/jb ...) whose flags are recovered through # x86g_calculate_condition ccalls. This regression test covers a bug found in the x86 ccall rewriter. + # + # The ccalls under test are produced by sub_74d6a1's own instructions, so the call tree is deliberately + # not expanded. bin_path = os.path.join( test_location, "i386", "windows", "a71a3c3b922705cb5e2d8aa9c74f5c73c47fb27f10b1327eb2bb054d99a14397" ) @@ -5946,15 +5976,25 @@ class TestDecompiler(unittest.TestCase): proj, _ = load_project_with_scoped_cfg( bin_path, func_addr, - window=0x5000, + window=0x1000, + expand_call_tree=False, cfg_kwargs={"force_complete_scan": True}, ) f = proj.kb.functions[func_addr] + # guard against the scoped CFG window silently truncating the function under test + assert f.size >= 0x3F5, f"sub_74d6a1 was truncated by the scoped CFG: size {f.size:#x}." dec = proj.analyses[Decompiler].prep(fail_fast=True)(f, options=decompiler_options) assert dec.codegen is not None and dec.codegen.text is not None, f"Failed to decompile function {f!r}." print_decompilation_result(dec) + # Both SUBB-flavored condition ccalls must have been rewritten (and their operands narrowed to byte + # width): "cmp cl, 8; cmovb ..." at 0x74d7ed and "cmp al, byte ptr [esi + 0x142]; jbe" at 0x74d9b9. + # Before the fix, the rewriter returned a 1-bit expression for a 32-bit ccall and decompilation crashed. + text = dec.codegen.text + assert re.search(r"\bv\d+ < 8\b", text), "the CondB SUBB ccall was not rewritten into a byte comparison" + assert re.search(r"> \(char\)", text), "the CondBE SUBB ccall operands were not narrowed to byte width" + def test_widening_conversion_signedness(self, decompiler_options=None): # A widening integer conversion carries the signedness of its source operand: a sign-extending Convert # (e.g. movswl) implies a signed source, and a zero-extending Convert (e.g. movzwl) implies an unsigned diff --git a/tests/analyses/decompiler/test_register_save_area_simplifier_adv.py b/tests/analyses/decompiler/test_register_save_area_simplifier_adv.py index 2df992099..afd4822c2 100644 --- a/tests/analyses/decompiler/test_register_save_area_simplifier_adv.py +++ b/tests/analyses/decompiler/test_register_save_area_simplifier_adv.py @@ -7,10 +7,7 @@ __package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redef import os import unittest -import networkx - -import angr -from tests.common import WORKER, bin_location, print_decompilation_result +from tests.common import bin_location, load_project_with_scoped_cfg, print_decompilation_result test_location = os.path.join(bin_location, "tests") @@ -20,11 +17,16 @@ class TestRegisterSaveAreaSimplifierAdv(unittest.TestCase): bin_path = os.path.join( test_location, "x86_64", "windows", "131252a8059fdbb12d77cd4711e597c45bb48e6d4bc3ddc808697a5e0488ff2c" ) - proj = angr.Project(bin_path, auto_load_libs=False) - - cfg = proj.analyses.CFGFast(show_progressbar=not WORKER, fail_fast=True, normalize=True) - funcs = networkx.descendants(proj.kb.functions.callgraph, 0x46A6C0) | {0x46A6C0, 0x46AAE0} - proj.analyses.CompleteCallingConventions(fail_fast=True, prioritize_func_addrs=funcs, skip_other_funcs=True) + # The scoped CFG covers the function under test, sub_46aae0, the two functions that reach it, and one round of + # call-tree expansion: only the prototypes of the direct callees influence the output here. + proj, cfg = load_project_with_scoped_cfg( + bin_path, + 0x46A6C0, + extra_func_addrs=[0x46AAE0, 0x469F80, 0x46AA98], + call_tree_depth=1, + cfg_kwargs={"fail_fast": True}, + ccc_kwargs={"fail_fast": True}, + ) callee = cfg.functions[0x46AAE0] func = cfg.functions[0x46A6C0] diff --git a/tests/analyses/decompiler/test_rust_decompiler.py b/tests/analyses/decompiler/test_rust_decompiler.py index 26c1c96e6..e84bde09f 100644 --- a/tests/analyses/decompiler/test_rust_decompiler.py +++ b/tests/analyses/decompiler/test_rust_decompiler.py @@ -11,7 +11,7 @@ import networkx import angr from angr.rust.utils.rust_sigs import get_default_sig_dir -from tests.common import bin_location +from tests.common import bin_location, recover_call_tree_cfg RUST_BINARIES_BASE = os.path.join(bin_location, "tests", "x86_64", "rust", "coreutils") @@ -20,6 +20,15 @@ def rust_binary_path(configuration: str, binary: str) -> str: return os.path.join(RUST_BINARIES_BASE, configuration, binary) +def executable_regions(proj: angr.Project) -> list[tuple[int, int]]: + """Address ranges of the executable sections of the main object.""" + return [ + (sec.vaddr, sec.vaddr + sec.memsize) + for sec in proj.loader.main_object.sections + if sec.is_executable and sec.memsize > 0 + ] + + class TestRustcVersionIdentification(unittest.TestCase): """Test that RustcVersionIdentification correctly identifies the rustc version for coreutils binaries.""" @@ -29,6 +38,9 @@ class TestRustcVersionIdentification(unittest.TestCase): "nightly-2025-05-22-O0": "1.88.0", } + #: How many bytes at the end of each executable section the pre-built CFG covers. + CFG_TAIL_BYTES = 128 * 1024 + def test_default_sig_dir(self): sig_dir = get_default_sig_dir() self.assertTrue(sig_dir is not None, "get_default_sig_dir() returned None") @@ -38,11 +50,13 @@ class TestRustcVersionIdentification(unittest.TestCase): assert os.path.isfile(path) expected = self.EXPECTED_VERSIONS[configuration] p = angr.Project(path) - # Pre-build a sampled, region-limited CFG. - # FLIRT matching over the first quarter of each executable section is enough to discriminate rustc versions - # while being several times cheaper than a whole-binary CFG. + # Pre-build a sampled, region-limited CFG so this test does not pay for a whole-binary one. + # + # rustc ships std/core/alloc as prebuilt rlibs, and the linker emits the crate's own object files before those + # rlibs, so the library code the FLIRT signatures actually describe sits at the end of .text. Hence we limit + # the CFG to the last 128 KB of each executable section for better speed. regions = [ - (sec.vaddr, sec.vaddr + max(0x1000, int(sec.memsize * 0.25))) + (sec.vaddr + sec.memsize - min(self.CFG_TAIL_BYTES, sec.memsize), sec.vaddr + sec.memsize) for sec in p.loader.main_object.sections if sec.is_executable and sec.memsize > 0 ] @@ -80,6 +94,18 @@ class RustDecompilationTarget(unittest.TestCase): BINARY: str = "" FUNC_ADDRS: dict[str, dict[str, int]] = {} + #: When set, CFG recovery is scoped to the functions in ``FUNC_ADDRS`` plus this many levels of + #: callees instead of scanning the whole binary. + CALL_TREE_DEPTH: int | None = None + + #: Recover the CFG by recursive descent from the functions under test instead of scanning the whole binary. + CFG_FROM_FUNCS_UNDER_TEST: bool = False + + #: Limit the call depth of complete calling convention analysis. None refers to the entire call tree. + #: We manually tuned CCC_CALL_DEPTH for each test case to get equivalent decompilation output to a whole-binary + #: run; we may need a different CCC_CALL_DEPTH value in the future when angr decompiler changes substantially. + CCC_CALL_DEPTH: int | None = None + def decompile_functions(self): """Decompile every function in ``FUNC_ADDRS`` and return ``{label: {configuration: codegen_text}}``. @@ -93,14 +119,31 @@ class RustDecompilationTarget(unittest.TestCase): assert os.path.isfile(path), f"{path} not found" proj = angr.Project(path, auto_load_libs=False) assert proj.is_rust_binary, f"{path} is not identified as a rust binary." - proj.analyses.CFGFast(normalize=True) func_addrs = { addr for per_config_addrs in self.FUNC_ADDRS.values() if (addr := per_config_addrs.get(config)) } - call_tree = set(func_addrs) + if self.CALL_TREE_DEPTH is not None: + recover_call_tree_cfg(proj, func_addrs, depth=self.CALL_TREE_DEPTH) + elif self.CFG_FROM_FUNCS_UNDER_TEST: + proj.analyses.CFGFast( + normalize=True, + regions=executable_regions(proj), + start_at_entry=False, + function_starts=sorted(func_addrs), + force_smart_scan=False, + ) + else: + proj.analyses.CFGFast(normalize=True) + callgraph = proj.kb.functions.callgraph + ccc_funcs = set(func_addrs) for addr in func_addrs: - call_tree |= networkx.descendants(proj.kb.functions.callgraph, addr) - proj.analyses.CompleteCallingConventions(prioritize_func_addrs=call_tree, skip_other_funcs=True) + if self.CCC_CALL_DEPTH is None: + ccc_funcs |= networkx.descendants(callgraph, addr) + else: + ccc_funcs |= set( + networkx.single_source_shortest_path_length(callgraph, addr, cutoff=self.CCC_CALL_DEPTH) + ) + proj.analyses.CompleteCallingConventions(prioritize_func_addrs=ccc_funcs, skip_other_funcs=True) proj.rustc_version = TestRustcVersionIdentification.EXPECTED_VERSIONS[config] proj.analyses.RustSymbolRecovery() proj.analyses.TypeDBLoader() @@ -188,6 +231,8 @@ class TestFmtNightly20250522O3(_FmtTests): FUNC_ADDRS = { "uumain": {"nightly-2025-05-22-O3": 0x496920}, } + CFG_FROM_FUNCS_UNDER_TEST = True + CCC_CALL_DEPTH = 4 def test_uumain_2025052203(self): self._check_uumain() @@ -199,6 +244,7 @@ class TestFmtNightly20250522O0(_FmtTests): FUNC_ADDRS = { "uumain": {"nightly-2025-05-22-O0": 0x4D42F0}, } + CALL_TREE_DEPTH = 5 def test_uumain_2025052200(self): self._check_uumain() diff --git a/tests/analyses/decompiler/test_variable_nondeterminism.py b/tests/analyses/decompiler/test_variable_nondeterminism.py index 49940bee0..177bf9477 100644 --- a/tests/analyses/decompiler/test_variable_nondeterminism.py +++ b/tests/analyses/decompiler/test_variable_nondeterminism.py @@ -4,6 +4,7 @@ from __future__ import annotations __package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin +import concurrent.futures import difflib import os import subprocess @@ -17,10 +18,12 @@ import sys import angr project = angr.Project(sys.argv[1], auto_load_libs=False) -# scope CFG recovery to the region of sub_4012f5; a whole-binary CFG takes ~10s per seed +# Scope CFG recovery to sub_4012f5 itself (0x4012f5-0x4015ab is its exact extent). +# Callees are still recovered because CFGFast follows call targets out of the scanned region, so the decompilation +# output is identical to a wider scan. project.analyses.CFGFast( normalize=True, - regions=[(0x4012F5, 0x4022F5)], + regions=[(0x4012F5, 0x4015AB)], start_at_entry=False, function_starts=[0x4012F5], force_smart_scan=False, @@ -44,14 +47,23 @@ def _decompile_with_seed(seed: int, binary_path: str) -> str: capture_output=True, text=True, env=env, - timeout=10, + timeout=300, ) assert result.returncode == 0, f"Decompilation failed (seed={seed}):\n{result.stderr}" - if not is_testing: - print(result.stdout) return result.stdout +def _decompile_with_seeds(seeds: list[int], binary_path: str) -> dict[int, str]: + """Decompile once per seed.""" + with concurrent.futures.ThreadPoolExecutor(max_workers=len(seeds)) as pool: + futures = [pool.submit(_decompile_with_seed, seed, binary_path) for seed in seeds] + outputs = {seed: future.result() for seed, future in zip(seeds, futures)} + if not is_testing: + for seed in seeds: + print(outputs[seed]) + return outputs + + class TestVariableNondeterminism(unittest.TestCase): def test_stack_variable_naming_stability(self): """Regression test for https://github.com/angr/angr/issues/6246. @@ -68,7 +80,7 @@ class TestVariableNondeterminism(unittest.TestCase): "4a00ae5dacc4d7ab43d1da71db43c7df837053a4ed86846fd2d7fcf02a3f861c", ) seeds = list(range(10)) - outputs = {seed: _decompile_with_seed(seed, binary_path) for seed in seeds} + outputs = _decompile_with_seeds(seeds, binary_path) baseline = outputs[seeds[0]] for seed in seeds[1:]: if outputs[seed] != baseline: diff --git a/tests/common.py b/tests/common.py index c3a0356b4..c740f8c22 100644 --- a/tests/common.py +++ b/tests/common.py @@ -11,6 +11,7 @@ from tempfile import NamedTemporaryFile from unittest import SkipTest, skip, skipIf, skipUnless import networkx +from elftools.elf.elffile import ELFFile from rich.console import Console from rich.syntax import Syntax @@ -195,12 +196,27 @@ def _merged_regions(addrs: Iterable[int], window: int) -> list[tuple[int, int]]: return regions +PLT_SECTION_NAMES = (".plt", ".plt.got", ".plt.sec", ".plt.bnd", ".MIPS.stubs") + + +def _plt_regions(main_object) -> list[tuple[int, int]]: + regions = [] + sections_map = getattr(main_object, "sections_map", None) or {} + for name in PLT_SECTION_NAMES: + section = sections_map.get(name) + if section is not None and section.memsize: + regions.append((section.vaddr, section.vaddr + section.memsize)) + return regions + + def load_project_with_scoped_cfg( bin_path: str, func_addr: int, extra_func_addrs: Sequence[int] = (), window: int = 0x2000, expand_call_tree: bool = True, + include_plt: bool = False, + call_tree_depth: int = 8, project_kwargs: dict | None = None, cfg_kwargs: dict | None = None, run_ccc: bool = True, @@ -225,6 +241,8 @@ def load_project_with_scoped_cfg( :param window: Size in bytes of the region scanned after each function start; must cover the function's full extent. :param expand_call_tree: Also cover the transitive callees of the given functions. + :param include_plt: Also cover the PLT sections. Required for dynamically linked binaries. + :param call_tree_depth: Maximum number of call-tree discovery rounds. Each round adds one more level of callees. :param project_kwargs: Extra keyword arguments for angr.Project. :param cfg_kwargs: Overrides for the final CFGFast call. :param run_ccc: Run CompleteCallingConventions, scoped to the covered functions. @@ -235,13 +253,14 @@ def load_project_with_scoped_cfg( main_object = proj.loader.main_object roots = [func_addr, *extra_func_addrs] known: set[int] = set(roots) + extra_regions = _plt_regions(main_object) if include_plt else [] if expand_call_tree: - for _ in range(8): + for _ in range(call_tree_depth): tmp_kb = angr.KnowledgeBase(proj) proj.analyses[angr.analyses.CFGFast].prep(kb=tmp_kb)( normalize=True, - regions=_merged_regions(known, window), + regions=_merged_regions(known, window) + extra_regions, start_at_entry=False, function_starts=sorted(known), symbols=False, @@ -259,7 +278,7 @@ def load_project_with_scoped_cfg( final_cfg_kwargs = { "normalize": True, - "regions": _merged_regions(known, window), + "regions": _merged_regions(known, window) + extra_regions, "start_at_entry": False, "function_starts": roots, "symbols": True, @@ -274,3 +293,120 @@ def load_project_with_scoped_cfg( proj.analyses.CompleteCallingConventions(show_progressbar=not WORKER, **final_ccc_kwargs) return proj, cfg + + +def function_extents_from_eh_frame(proj: Project) -> dict[int, int]: + """ + Read exact function extents (``{addr: size}``) out of the ELF ``.eh_frame`` FDE table. Returns an empty dict if the + binary has no usable ``.eh_frame``. + """ + main_object = proj.loader.main_object + if proj.filename is None: + return {} + + extents: dict[int, int] = {} + try: + with open(proj.filename, "rb") as fp: + elf = ELFFile(fp) + if elf.get_section_by_name(".eh_frame") is None: + return {} + # pyelftools resolves pc-relative FDE pointers against the section's *linked* address, + # so initial_location lives in linked-address space and needs the load bias applied. + bias = main_object.mapped_base - main_object.linked_base + for entry in elf.get_dwarf_info().EH_CFI_entries(): + header = getattr(entry, "header", None) + if header is None or "initial_location" not in header or not header.address_range: + continue # a CIE, or a terminator/zero-length FDE + addr = header.initial_location + bias + if main_object.contains_addr(addr): + extents[addr] = header.address_range + except Exception: # pylint:disable=broad-exception-caught + return {} + return extents + + +def recover_call_tree_cfg( + proj: Project, + roots: Iterable[int], + depth: int, + window: int = 0x400, + cfg_kwargs: dict | None = None, +) -> angr.analyses.cfg.CFGFast: + """ + Build a CFG covering only ``roots`` and their callees up to ``depth`` call levels. + + CFG recovery in this method runs on temporary KnowledgeBase so partial results never leak into the global + KnowledgeBase. + """ + main_object = proj.loader.main_object + extents = function_extents_from_eh_frame(proj) + + def _regions(addrs: Iterable[int]) -> list[tuple[int, int]]: + return _merged_regions_with_sizes([(addr, extents.get(addr, window)) for addr in addrs]) + + graph = networkx.DiGraph() + roots = sorted(roots) + graph.add_nodes_from(roots) + scanned: set[int] = set() + pending: set[int] = set(roots) + known: set[int] = set(roots) + + while pending: + tmp_kb = angr.KnowledgeBase(proj) + proj.analyses[angr.analyses.CFGFast].prep(kb=tmp_kb)( + normalize=True, + regions=_regions(pending), + start_at_entry=False, + function_starts=sorted(pending), + symbols=False, + force_smart_scan=False, + ) + callgraph = tmp_kb.functions.callgraph + for addr in pending: + if addr in callgraph: + graph.add_edges_from( + (addr, callee) for callee in callgraph.successors(addr) if main_object.contains_addr(callee) + ) + scanned |= pending + # Functions at exactly ``depth`` are covered by the final CFG but never expanded, so the deepest level is never + # scanned. + known, pending = _bfs_levels(graph, roots, depth) + pending -= scanned + + final_cfg_kwargs = { + "normalize": True, + "regions": _regions(known), + "start_at_entry": False, + "function_starts": sorted(known), + "symbols": True, + "force_smart_scan": False, + } + final_cfg_kwargs.update(cfg_kwargs or {}) + return proj.analyses.CFGFast(show_progressbar=not WORKER, **final_cfg_kwargs) + + +def _bfs_levels(graph: networkx.DiGraph, roots: Sequence[int], depth: int) -> tuple[set[int], set[int]]: + """Return (nodes within ``depth`` hops of ``roots``, those strictly closer than ``depth``).""" + reached: set[int] = set(roots) + expandable: set[int] = set() + frontier: set[int] = set(roots) + for _ in range(depth): + expandable |= frontier + nxt: set[int] = set() + for node in frontier: + nxt |= set(graph.successors(node)) - reached + if not nxt: + break + reached |= nxt + frontier = nxt + return reached, expandable + + +def _merged_regions_with_sizes(addr_sizes: Iterable[tuple[int, int]]) -> list[tuple[int, int]]: + regions: list[tuple[int, int]] = [] + for addr, size in sorted(addr_sizes): + if regions and addr <= regions[-1][1]: + regions[-1] = regions[-1][0], max(regions[-1][1], addr + size) + else: + regions.append((addr, addr + size)) + return regions From bc72b9e1a6406ef84fe15d3f0206f5f4efec20f5 Mon Sep 17 00:00:00 2001 From: Fish Date: Mon, 27 Jul 2026 03:01:56 -0700 Subject: [PATCH 060/122] MCP: Protect stdio from forked workers. (#6717) --- angr/mcp/server.py | 66 +++++++++++++++++++ angr/utils/mp.py | 59 +++++++++++++++++ tests/utils/test_mp_stdio.py | 119 +++++++++++++++++++++++++++++++++++ 3 files changed, 244 insertions(+) create mode 100644 tests/utils/test_mp_stdio.py diff --git a/angr/mcp/server.py b/angr/mcp/server.py index e39a664e1..ea67aaa42 100644 --- a/angr/mcp/server.py +++ b/angr/mcp/server.py @@ -2,14 +2,18 @@ from __future__ import annotations +import contextlib import logging import re +import sys from typing import Any import networkx as nx from fastmcp import FastMCP +from fastmcp.server.middleware import CallNext, Middleware, MiddlewareContext from angr.knowledge_plugins.cfg.memory_data import MemoryDataSort +from angr.utils.mp import protect_stdio_from_forked_children from .errors import ( CFGNotBuiltError, @@ -29,8 +33,26 @@ from .session import ProjectSession, get_session_manager l = logging.getLogger(__name__) + +class StdoutGuardMiddleware(Middleware): + """ + Keep analysis output off of stdout. + + Under the stdio transport, stdout is the JSON-RPC channel. This middleware redirects stdout to + stderr to avoid messing with the stdio transport. + """ + + async def on_message(self, context: MiddlewareContext[Any], call_next: CallNext[Any, Any]) -> Any: + with contextlib.redirect_stdout(sys.stderr): + return await call_next(context) + + +# Worker processes should not inherit stdio. +protect_stdio_from_forked_children() + # Create the FastMCP server instance mcp = FastMCP("angr-mcp", instructions="Binary analysis server powered by angr") +mcp.add_middleware(StdoutGuardMiddleware()) def _get_session(project_id: str) -> ProjectSession: @@ -138,6 +160,50 @@ def get_cfg( } +@mcp.tool() +def recover_calling_conventions( + project_id: str, + workers: int = 0, + force: bool = False, +) -> dict[str, Any]: + """ + Recover calling conventions and prototypes for every function in the binary. + + Requires CFG to be built first via get_cfg. This improves the quality of decompilation output, especially the + arguments shown at call sites. + + Args: + project_id: The project ID + workers: Number of worker processes to use (0 = run in-process, the default) + force: Re-analyze functions that already have a calling convention or prototype + + Returns: + Number of functions for which a calling convention was recovered + """ + session = _get_session(project_id) + _require_cfg(session) + proj = session.project + + if workers < 0: + raise ValueError("workers must be >= 0") + + analysis = proj.analyses.CompleteCallingConventions( + cfg=session.cfg, + analyze_callsites=True, + workers=workers, + force=force, + ) + + recovered = sum(1 for func in proj.kb.functions.values() if func.calling_convention is not None) + return { + "project_id": project_id, + "workers": workers, + "functions_analyzed": len(proj.kb.functions), + "functions_with_calling_convention": recovered, + "prototype_libnames": sorted(analysis.prototype_libnames), + } + + @mcp.tool() def list_functions( project_id: str, diff --git a/angr/utils/mp.py b/angr/utils/mp.py index afdc5e35d..fcf088b08 100644 --- a/angr/utils/mp.py +++ b/angr/utils/mp.py @@ -1,7 +1,11 @@ from __future__ import annotations +import contextlib +import io import multiprocessing +import os import platform +import sys from collections.abc import Callable from typing import Any, NamedTuple @@ -66,3 +70,58 @@ def mp_context(): } spawn_method = spawn_methods.get(system, "fork") # default to fork on other platforms return multiprocessing.get_context(spawn_method) + + +_stdio_fork_hook_installed = False + + +def _detach_child_stdio() -> None: + """ + Give a freshly forked child its own, harmless stdin and stdout. + """ + try: + devnull = os.open(os.devnull, os.O_RDONLY) + try: + os.dup2(devnull, 0) + finally: + os.close(devnull) + except OSError: + pass + + # anything the child prints goes to stderr + with contextlib.suppress(OSError): + os.dup2(2, 1) + + try: + sys.stdin = io.TextIOWrapper( + io.BufferedReader(io.FileIO(0, "rb", closefd=False)), encoding="utf-8", errors="replace" + ) + except OSError: + sys.stdin = None # type: ignore[assignment] + + try: + sys.stdout = io.TextIOWrapper( + io.BufferedWriter(io.FileIO(1, "wb", closefd=False)), + encoding="utf-8", + errors="replace", + line_buffering=True, + ) + except OSError: + sys.stdout = None # type: ignore[assignment] + + +def protect_stdio_from_forked_children() -> None: + """ + Make forked worker processes safe to use from a process whose stdin/stdout are a protocol channel. + """ + global _stdio_fork_hook_installed # pylint:disable=global-statement + + if _stdio_fork_hook_installed: + return + if not hasattr(os, "register_at_fork"): + # Windows; the fork start method is not used there anyway + _stdio_fork_hook_installed = True + return + + os.register_at_fork(after_in_child=_detach_child_stdio) + _stdio_fork_hook_installed = True diff --git a/tests/utils/test_mp_stdio.py b/tests/utils/test_mp_stdio.py new file mode 100644 index 000000000..aea6de9f8 --- /dev/null +++ b/tests/utils/test_mp_stdio.py @@ -0,0 +1,119 @@ +# pylint:disable=no-self-use +""" +Regression tests for angr.utils.mp.protect_stdio_from_forked_children. + +An MCP server (or any other host) that speaks a protocol over stdin/stdout keeps a thread parked in a blocking read +on ``sys.stdin``. CPython's ``BufferedReader`` holds its internal lock across that blocking read, so a ``fork()`` +performed from a *different* thread produces a child that inherits the lock in a permanently locked state. The very +first thing ``multiprocessing.process.BaseProcess._bootstrap()`` does is ``util._close_stdin()``, which closes +``sys.stdin`` -- and hangs forever. Every angr analysis that takes a ``workers=N`` argument uses the ``fork`` start +method on Linux (see :func:`angr.utils.mp.mp_context`), so they all deadlock in that situation. + +These tests run in a subprocess because they need to own stdin/stdout and to install a process-global +``os.register_at_fork`` hook. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import textwrap + +import pytest + +# language=python +_CHILD_SCRIPT = textwrap.dedent( + """ + import io, multiprocessing, os, sys, threading, time + + if {install_fix}: + from angr.utils.mp import protect_stdio_from_forked_children + protect_stdio_from_forked_children() + + def stdin_reader(): + # this is what mcp.server.stdio.stdio_server() does + io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8", errors="replace").readline() + + threading.Thread(target=stdin_reader, daemon=True).start() + time.sleep(0.5) # let it settle into the blocking read, holding the buffer lock + + def child(): + # both of these would land on the protocol channel without the fix + print("CHILD_STDOUT") + os.write(1, b"CHILD_FD1\\n") + sys.stderr.write("CHILD_RAN\\n") + sys.stderr.flush() + + ctx = multiprocessing.get_context("fork") + procs = [ctx.Process(target=child, daemon=True) for _ in range(2)] + for p in procs: + p.start() + for p in procs: + p.join(timeout=5) + alive = [p.pid for p in procs if p.is_alive()] + for p in procs: + if p.is_alive(): + p.kill() + sys.stderr.write("ALIVE=%d\\n" % len(alive)) + sys.stderr.flush() + """ +) + + +def _run(install_fix: bool) -> subprocess.CompletedProcess[str]: + # The reader thread must still be *inside* the blocking read when fork() happens -- that is the steady state of + # a stdio JSON-RPC server between requests. So hand the child a pipe and keep the write end open here; + # subprocess.run(stdin=PIPE) would close it immediately, the read would return EOF, and the lock would be + # released before the fork. + read_fd, write_fd = os.pipe() + try: + return subprocess.run( + [sys.executable, "-c", _CHILD_SCRIPT.format(install_fix=install_fix)], + stdin=read_fd, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + finally: + os.close(read_fd) + os.close(write_fd) + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires fork()") +class TestProtectStdioFromForkedChildren: + """Tests for protect_stdio_from_forked_children().""" + + def test_forked_workers_deadlock_without_the_guard(self): + """Without the guard, fork()ed children wedge in multiprocessing's _close_stdin().""" + result = _run(install_fix=False) + assert "ALIVE=2" in result.stderr, f"expected both children to hang; stderr={result.stderr!r}" + assert "CHILD_RAN" not in result.stderr + # they never even get as far as running the target, so nothing reaches stdout either + assert result.stdout == "" + + def test_forked_workers_run_with_the_guard(self): + """With the guard installed, the children start normally and finish.""" + result = _run(install_fix=True) + assert "ALIVE=0" in result.stderr, f"children did not finish; stderr={result.stderr!r}" + assert result.stderr.count("CHILD_RAN") == 2 + + def test_child_output_is_kept_off_of_stdout(self): + """The parent's stdout is a protocol channel; nothing a child writes may reach it.""" + result = _run(install_fix=True) + assert result.stdout == "", f"child output leaked onto stdout: {result.stdout!r}" + assert "CHILD_STDOUT" in result.stderr + assert "CHILD_FD1" in result.stderr + + def test_is_idempotent(self): + """Calling it repeatedly must not stack up fork hooks.""" + from angr.utils.mp import protect_stdio_from_forked_children # pylint:disable=import-outside-toplevel + + protect_stdio_from_forked_children() + protect_stdio_from_forked_children() + protect_stdio_from_forked_children() + + +if __name__ == "__main__": + pytest.main([__file__]) From 59400706f8bb13287ffe1cecc7b0bd197b81f7d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:13:35 -0700 Subject: [PATCH 061/122] ci: bump astral-sh/setup-uv from 8.3.2 to 9.0.0 (#6720) Bumps [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) from 8.3.2 to 9.0.0. - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/11f9893b081a58869d3b5fccaea48c9e9e46f990...c771a70e6277c0a99b617c7a806ffedaca235ff9) --- updated-dependencies: - dependency-name: astral-sh/setup-uv dependency-version: 9.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- .github/workflows/coverage.yml | 2 +- .github/workflows/nightly-ci.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47d12c130..c9c27610c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 if: startsWith(runner.os, 'windows') - - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v6 + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v6 - name: Sync dependencies run: uv sync -p ${{ matrix.environment.python-version }} - name: Collect tests diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 60ebc0333..650bfd869 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 - - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v6 + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v6 - name: Restore test durations cache uses: actions/cache/restore@v6 with: diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 56d6dc45d..26b5cf2de 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -36,7 +36,7 @@ jobs: repository: angr/binaries path: binaries - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 - - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v6 + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v6 - name: Setup Java for pysoot uses: actions/setup-java@v5 with: @@ -61,7 +61,7 @@ jobs: with: repository: angr/binaries path: binaries - - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v6 + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v6 - name: Sync dependencies run: uv --directory angr sync -p 3.12 - name: Run pytest From 3e2e4b90fcd60c92072bda4cd93ff4f812a48ac0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:13:52 -0700 Subject: [PATCH 062/122] ci: bump taiki-e/install-action from 2.84.0 to 2.85.2 (#6719) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.84.0 to 2.85.2. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9...41049aa56687c35e0afa74eed4f09cec4f9afabf) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.85.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 650bfd869..e6d0bf6b4 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -99,7 +99,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1 - - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2 + - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2 with: tool: cargo-llvm-cov - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v5 From 2080c15a267020fb6229568f1814b80382673931 Mon Sep 17 00:00:00 2001 From: Fish Date: Mon, 27 Jul 2026 15:53:53 -0700 Subject: [PATCH 063/122] RemoveRedundantBitmasks: Fix an in-place expression update. (#6722) --- .../remove_redundant_bitmasks.py | 13 +++++++++++-- angr/analyses/decompiler/utils.py | 5 +---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/angr/analyses/decompiler/peephole_optimizations/remove_redundant_bitmasks.py b/angr/analyses/decompiler/peephole_optimizations/remove_redundant_bitmasks.py index 84820ee93..7132b70a0 100644 --- a/angr/analyses/decompiler/peephole_optimizations/remove_redundant_bitmasks.py +++ b/angr/analyses/decompiler/peephole_optimizations/remove_redundant_bitmasks.py @@ -139,8 +139,17 @@ class RemoveRedundantBitmasks(PeepholeOptimizationExprBase): new_op0 = op0.operands[0] replaced, new_operand_expr = operand_expr.replace(op0, new_op0) if replaced: - expr.operand = new_operand_expr - return expr + return Convert( + self.manager.next_atom(), + expr.from_bits, + expr.to_bits, + expr.is_signed, + new_operand_expr, + from_type=expr.from_type, + to_type=expr.to_type, + rounding_mode=expr.rounding_mode, + **expr.tags, + ) # Conv(64->32, (expr) - (expr) & 0xffffffff<64>))) # => Conv(64->32, (expr - expr)) elif ( diff --git a/angr/analyses/decompiler/utils.py b/angr/analyses/decompiler/utils.py index 9a5b92a18..add71d907 100644 --- a/angr/analyses/decompiler/utils.py +++ b/angr/analyses/decompiler/utils.py @@ -908,10 +908,7 @@ class _PeepholeExprsWalker(ailment.AILBlockRewriter): redo = True while redo: redo = False - kind = getattr(expr, "pykind", None) - if kind is None: - kind = type(expr).__name__ - expr_opts = self.expr_opts_by_kind.get(kind) + expr_opts = self.expr_opts_by_kind.get(expr.pykind) if not expr_opts: break for expr_opt in expr_opts: From 1a5eedf62237b9f459fecaf66d9cf2510dcca82a Mon Sep 17 00:00:00 2001 From: Fish Date: Mon, 27 Jul 2026 16:34:10 -0700 Subject: [PATCH 064/122] Decompiler: Speed up AIL block simplifications. (#6712) - Make SPropagator, SRDA, and BlockSimplifier normal classes instead of Analysis classes. - Share peephole optimizer instances across BlockSimplifiers. - BlockSimplifier: Skip unnecessary peephole passes; avoid block-level comparisons for fixpoint determination. - Add a runtime-only peephole_optimized flag to AIL statements so we skip running peephole optimizations on already optimized statements. --- angr/analyses/__init__.py | 6 +- angr/analyses/decompiler/ail_simplifier.py | 104 ++++--- angr/analyses/decompiler/block_simplifier.py | 277 +++++++++++++----- angr/analyses/decompiler/clinic.py | 140 ++++++--- .../dephication/graph_vvar_mapping.py | 10 +- .../condition_constprop.py | 3 +- .../optimization_passes/optimization_pass.py | 26 +- .../peephole_simplifier.py | 26 +- .../register_save_area_simplifier_adv.py | 24 +- .../decompiler/peephole_optimizations/base.py | 23 +- .../concat_simplifier.py | 2 + .../peephole_optimizations/rol_ror.py | 5 + .../sar_to_signed_div.py | 13 +- angr/analyses/decompiler/structurer_nodes.py | 6 +- angr/analyses/decompiler/utils.py | 58 +++- angr/analyses/outliner/outliner.py | 22 +- angr/analyses/s_propagator.py | 21 +- .../s_reaching_definitions/__init__.py | 3 +- .../s_reaching_definitions/s_rda_model.py | 2 +- .../s_reaching_definitions.py | 43 ++- angr/rust/mixins/srda_mixin.py | 4 +- angr/rustylib/ailment.pyi | 9 + native/angr/src/ailment/ail_stmt.rs | 25 ++ native/angr/src/ailment/block.rs | 30 +- tests/analyses/decompiler/test_decompiler.py | 3 +- .../decompiler/test_peephole_optimizations.py | 122 +++++++- tests/analyses/test_callsite_maker.py | 5 +- 27 files changed, 757 insertions(+), 255 deletions(-) diff --git a/angr/analyses/__init__.py b/angr/analyses/__init__.py index 183eb5987..1925a6d28 100644 --- a/angr/analyses/__init__.py +++ b/angr/analyses/__init__.py @@ -37,8 +37,8 @@ from .proximity_graph import ProximityGraphAnalysis from .reaching_definitions import ReachingDefinitionsAnalysis from .reassembler import Reassembler from .s_liveness import SLivenessAnalysis -from .s_propagator import SPropagatorAnalysis -from .s_reaching_definitions import SReachingDefinitionsAnalysis +from .s_propagator import SPropagator, SPropagatorAnalysis +from .s_reaching_definitions import SReachingDefinitions, SReachingDefinitionsAnalysis from .smc import SelfModifyingCodeAnalysis from .soot_class_hierarchy import SootClassHierarchy from .stack_pointer_tracker import StackPointerTracker @@ -97,7 +97,9 @@ __all__ = ( "ReachingDefinitionsAnalysis", "Reassembler", "SLivenessAnalysis", + "SPropagator", "SPropagatorAnalysis", + "SReachingDefinitions", "SReachingDefinitionsAnalysis", "SelfModifyingCodeAnalysis", "SootClassHierarchy", diff --git a/angr/analyses/decompiler/ail_simplifier.py b/angr/analyses/decompiler/ail_simplifier.py index ca32f09ec..2b30504d4 100644 --- a/angr/analyses/decompiler/ail_simplifier.py +++ b/angr/analyses/decompiler/ail_simplifier.py @@ -41,8 +41,8 @@ from angr.ailment.statement import ( WeakAssignment, ) from angr.analyses.analysis import AnalysesHub, Analysis -from angr.analyses.s_propagator import SPropagatorAnalysis -from angr.analyses.s_reaching_definitions import SRDAModel, SReachingDefinitionsAnalysis +from angr.analyses.s_propagator import SPropagator +from angr.analyses.s_reaching_definitions import SRDAModel, SReachingDefinitions from angr.code_location import AILCodeLocation from angr.errors import AngrRuntimeError from angr.knowledge_plugins.functions.function import Function @@ -189,7 +189,7 @@ class AILSimplifier(Analysis): self.func = func self.func_graph = func_graph self._reaching_definitions: SRDAModel | None = None - self._propagator: SPropagatorAnalysis | None = None + self._propagator: SPropagator | None = None self._remove_dead_memdefs = remove_dead_memdefs self._stackarg_offset_manager = stackarg_offset_manager @@ -341,27 +341,25 @@ class AILSimplifier(Analysis): if self._reaching_definitions is not None: return self._reaching_definitions func_args = {vvar for vvar, _ in self._arg_vvars.values()} if self._arg_vvars else set() - rd = ( - self.project.analyses[SReachingDefinitionsAnalysis] - .prep()( - subject=self.func, - func_graph=self.func_graph, - func_args=func_args, - use_callee_saved_regs_at_return=self._use_callee_saved_regs_at_return, - # track_tmps=True, - ) - .model - ) + rd = SReachingDefinitions( + self.project, + subject=self.func, + func_graph=self.func_graph, + func_args=func_args, + use_callee_saved_regs_at_return=self._use_callee_saved_regs_at_return, + # track_tmps=True, + ).model self._reaching_definitions = rd return rd @timethis - def _compute_propagation(self) -> SPropagatorAnalysis: + def _compute_propagation(self) -> SPropagator: # Propagate expressions or return the existing result if self._propagator is not None: return self._propagator func_args = {vvar for vvar, _ in self._arg_vvars.values()} if self._arg_vvars else set() - prop = self.project.analyses[SPropagatorAnalysis].prep(fail_fast=self._fail_fast)( + prop = SPropagator( + self.project, subject=self.func, func_graph=self.func_graph, func_args=func_args, @@ -527,8 +525,8 @@ class AILSimplifier(Analysis): vvar, simvar = self._arg_vvars[func_arg_idx] if vvar.varid == new_vvar.varid: simvar_new = simvar.copy() - simvar_new._hash = None simvar_new.size = new_vvar.size + simvar_new.clear_hash() self._arg_vvars[func_arg_idx] = new_vvar, simvar_new return narrowed @@ -933,7 +931,7 @@ class AILSimplifier(Analysis): } reps = filtered_reps - r, new_block = BlockSimplifier._replace_and_build( + r, new_block = BlockSimplifier.replace_and_build( block, reps, self._ail_manager, gp=self._gp, replace_loads=replace_loads ) replaced |= r @@ -1738,13 +1736,6 @@ class AILSimplifier(Analysis): if isinstance(eq.atom0, VirtualVariable): src = used_expr dst: Expression = call.copy() - - if isinstance(dst, SideEffectStatement): - dst_bits = dst.ret_expr.bits if dst.ret_expr is not None else dst.bits - # extract the Call expression from the SideEffectStatement - dst = dst.expr - dst.bits = dst_bits - if src.bits != dst.bits and not eq.is_weakassignment: dst = Convert( self._ail_manager.next_atom(), @@ -1891,16 +1882,13 @@ class AILSimplifier(Analysis): # rebuild on the current (NoOp-containing) graph. assert self._reaching_definitions is not None func_args = {vvar for vvar, _ in self._arg_vvars.values()} if self._arg_vvars else set() - reference = ( - self.project.analyses[SReachingDefinitionsAnalysis] - .prep()( - subject=self.func, - func_graph=self.func_graph, - func_args=func_args, - use_callee_saved_regs_at_return=self._use_callee_saved_regs_at_return, - ) - .model - ) + reference = SReachingDefinitions( + self.project, + subject=self.func, + func_graph=self.func_graph, + func_args=func_args, + use_callee_saved_regs_at_return=self._use_callee_saved_regs_at_return, + ).model if self._reaching_definitions.canonical_form() != reference.canonical_form(): raise AssertionError("Incremental SRDA update diverged from a full rebuild") @@ -1938,7 +1926,7 @@ class AILSimplifier(Analysis): if uses is None: vvar = rd.varid_to_vvar[vvar_id] - def_codeloc = rd.all_vvar_definitions[vvar_id] + def_codeloc = codeloc if def_codeloc.is_extern: def_stmt = None else: @@ -2052,12 +2040,22 @@ class AILSimplifier(Analysis): # this statement declares more than one variable. we should handle it surgically # case 1: stmt.ret_expr and stmt.fp_ret_expr are both set, but one of them is not used if isinstance(stmt.ret_expr, VirtualVariable) and stmt.ret_expr.varid in dead_vvar_ids: - stmt = stmt.copy() - stmt.ret_expr = None + stmt = SideEffectStatement( + self._ail_manager.next_atom(), + stmt.expr, + ret_expr=None, + fp_ret_expr=stmt.fp_ret_expr, + **stmt.tags, + ) simplified = True if isinstance(stmt.fp_ret_expr, VirtualVariable) and stmt.fp_ret_expr.varid in dead_vvar_ids: - stmt = stmt.copy() - stmt.fp_ret_expr = None + stmt = SideEffectStatement( + self._ail_manager.next_atom(), + stmt.expr, + ret_expr=stmt.ret_expr, + fp_ret_expr=None, + **stmt.tags, + ) simplified = True if idx in stmts_to_remove and idx not in stmts_to_keep and not isinstance(stmt, DirtyStatement): @@ -2118,9 +2116,9 @@ class AILSimplifier(Analysis): isinstance(stmt.ret_expr, VirtualVariable) and stmt.ret_expr.was_combo_reg ): # both the return expr and the fp_ret_expr are not used - stmt = stmt.copy() - stmt.ret_expr = None - stmt.fp_ret_expr = None + stmt = SideEffectStatement( + self._ail_manager.next_atom(), stmt.expr, ret_expr=None, fp_ret_expr=None, **stmt.tags + ) simplified = True else: # Should not happen! @@ -2258,7 +2256,14 @@ class AILSimplifier(Analysis): def _handle_VEXCCallExpression( expr_idx: int, expr: VEXCCallExpression, stmt_idx: int, stmt: Statement | None, block: Block | None ) -> Expression: - r_expr = AILBlockRewriter._handle_VEXCCallExpression(walker, expr_idx, expr, stmt_idx, stmt, block) + r_expr = AILBlockRewriter._handle_VEXCCallExpression( # pylint:disable=protected-access + walker, + expr_idx, + expr, + stmt_idx, + stmt, + block, + ) rewriter = rewriter_cls(r_expr, self.project, self._ail_manager, rename_ccalls=self._should_rename_ccalls) if rewriter.result is not None: _any_update.v = True @@ -2304,7 +2309,7 @@ class AILSimplifier(Analysis): rewriter = rewriter_cls(stmt, self.project.arch, self._ail_manager) if rewriter.result is not None: _any_update.v = True - if walker._update_block and block is not None: + if walker._update_block and block is not None: # pylint:disable=protected-access block.statements[stmt_idx] = rewriter.result # type: ignore assert isinstance(rewriter.result, Statement) return rewriter.result @@ -2313,7 +2318,14 @@ class AILSimplifier(Analysis): def _handle_DirtyExpression( expr_idx: int, expr: DirtyExpression, stmt_idx: int, stmt: Statement | None, block: Block | None ): - r_expr = AILBlockRewriter._handle_DirtyExpression(walker, expr_idx, expr, stmt_idx, stmt, block) + r_expr = AILBlockRewriter._handle_DirtyExpression( # pylint:disable=protected-access + walker, + expr_idx, + expr, + stmt_idx, + stmt, + block, + ) assert isinstance(r_expr, DirtyExpression) rewriter = rewriter_cls(r_expr, self.project.arch, self._ail_manager) if rewriter.result is not None: diff --git a/angr/analyses/decompiler/block_simplifier.py b/angr/analyses/decompiler/block_simplifier.py index 91c9a66be..d249d7055 100644 --- a/angr/analyses/decompiler/block_simplifier.py +++ b/angr/analyses/decompiler/block_simplifier.py @@ -8,9 +8,8 @@ from typing import TYPE_CHECKING from angr.ailment.expression import Call, Const, Convert, Expression, Load, Register, Tmp, VirtualVariable from angr.ailment.manager import Manager from angr.ailment.statement import Assignment, Jump, SideEffectStatement, Statement, Store -from angr.analyses.analysis import Analysis, register_analysis -from angr.analyses.s_propagator import SPropagatorAnalysis -from angr.analyses.s_reaching_definitions import SRDAModel, SReachingDefinitionsAnalysis +from angr.analyses.s_propagator import SPropagator +from angr.analyses.s_reaching_definitions import SRDAModel, SReachingDefinitions from angr.code_location import AILCodeLocation from angr.knowledge_plugins.key_definitions import atoms from angr.utils.ssa import has_reference_to_vvar @@ -34,6 +33,7 @@ from .utils import ( if TYPE_CHECKING: from angr.ailment.block import Block + from angr.project import Project _l = logging.getLogger(name=__name__) @@ -42,13 +42,98 @@ _l = logging.getLogger(name=__name__) _HAS_CALL_EXPR_WALKER = HasCallExprWalker() -class BlockSimplifier(Analysis): +class PeepholeOptimizationBundle: + """ + PeepholeOptimizationBundle describes a set of initialized peephole optimizer instances and the dispatch structures + derived from them. This bundle of peephole optimizations is reusable across `BlockSimplifier` invocations (so we + avoid rebuilding the same optimizer instances). + """ + + __slots__ = ( + "_params", + "expr_opts", + "expr_walker", + "multistmt_opts", + "stmt_opts", + "stmt_opts_by_kind", + ) + + def __init__( + self, + project, + kb, + ail_manager: Manager, + func_addr: int | None = None, + preserve_vvar_ids: set[int] | None = None, + type_hints: list[tuple[atoms.VirtualVariable | atoms.MemoryLocation, str]] | None = None, + peephole_optimizations: None + | ( + Iterable[ + type[PeepholeOptimizationStmtBase] + | type[PeepholeOptimizationExprBase] + | type[PeepholeOptimizationMultiStmtBase] + ] + ) = None, + ): + if peephole_optimizations is None: + expr_classes: Iterable = EXPR_OPTS + stmt_classes: Iterable = STMT_OPTS + multistmt_classes: Iterable = MULTI_STMT_OPTS + else: + peephole_optimizations = tuple(peephole_optimizations) + expr_classes = [cls for cls in peephole_optimizations if issubclass(cls, PeepholeOptimizationExprBase)] + stmt_classes = [cls for cls in peephole_optimizations if issubclass(cls, PeepholeOptimizationStmtBase)] + multistmt_classes = [ + cls for cls in peephole_optimizations if issubclass(cls, PeepholeOptimizationMultiStmtBase) + ] + + args = (project, kb, ail_manager, func_addr, preserve_vvar_ids, type_hints) + self.expr_opts = [cls(*args) for cls in expr_classes] + self.stmt_opts = [cls(*args) for cls in stmt_classes] + self.multistmt_opts = [cls(*args) for cls in multistmt_classes] + self.stmt_opts_by_kind = build_stmt_opts_by_kind(self.stmt_opts) + self.expr_walker = _PeepholeExprsWalker(expr_opts=self.expr_opts) + self._params = (project, ail_manager, func_addr, preserve_vvar_ids, type_hints, peephole_optimizations) + + def matches( + self, + project, + ail_manager: Manager, + func_addr: int | None, + preserve_vvar_ids: set[int] | None, + type_hints: list | None, + peephole_optimizations, + ) -> bool: + p_project, p_manager, p_func_addr, p_preserve, p_hints, p_opts = self._params + return ( + p_project is project + and p_manager is ail_manager + and p_func_addr == func_addr + and p_preserve is preserve_vvar_ids + and p_hints is type_hints + and ( + p_opts is peephole_optimizations + or ( + p_opts is not None + and peephole_optimizations is not None + and p_opts == tuple(peephole_optimizations) + ) + ) + ) + + +class BlockSimplifier: """ Simplify an AIL block. + + Deliberately not an :class:`Analysis`: it is instantiated once per block, hundreds of times per decompilation, + so it skips the analysis-factory ceremony. Instantiate it directly with the project as the first argument; + exceptions always propagate. """ def __init__( self, + project: Project, block: Block | None, ail_manager: Manager, func_addr: int | None = None, @@ -65,12 +150,18 @@ class BlockSimplifier(Analysis): type_hints: list[tuple[atoms.VirtualVariable | atoms.MemoryLocation, str]] | None = None, cached_reaching_definitions=None, cached_propagator=None, + peephole_bundle: PeepholeOptimizationBundle | None = None, ): """ :param block: The AIL block to simplify. Setting it to None to skip calling self._analyze(), which is useful in test cases. + :param peephole_bundle: A pre-built PeepholeOptimizationBundle to reuse. Its construction parameters must + match this BlockSimplifier's; callers that simplify many blocks should build one bundle and + pass it to every BlockSimplifier they create. """ + self.project = project + self.kb = project.kb self.block = block self.func_addr = func_addr @@ -79,42 +170,25 @@ class BlockSimplifier(Analysis): self._type_hints = type_hints self._ail_manager = ail_manager - if peephole_optimizations is None: - self._expr_peephole_opts = [ - cls(self.project, self.kb, ail_manager, self.func_addr, self._preserve_vvar_ids, self._type_hints) - for cls in EXPR_OPTS - ] - self._stmt_peephole_opts = [ - cls(self.project, self.kb, ail_manager, self.func_addr, self._preserve_vvar_ids, self._type_hints) - for cls in STMT_OPTS - ] - self._multistmt_peephole_opts = [ - cls(self.project, self.kb, ail_manager, self.func_addr, self._preserve_vvar_ids, self._type_hints) - for cls in MULTI_STMT_OPTS - ] - self._stmt_peephole_opts_by_kind = build_stmt_opts_by_kind(self._stmt_peephole_opts) - else: - self._expr_peephole_opts = [ - cls(self.project, self.kb, ail_manager, self.func_addr, self._preserve_vvar_ids, self._type_hints) - for cls in peephole_optimizations - if issubclass(cls, PeepholeOptimizationExprBase) - ] - self._stmt_peephole_opts = [ - cls(self.project, self.kb, ail_manager, self.func_addr, self._preserve_vvar_ids, self._type_hints) - for cls in peephole_optimizations - if issubclass(cls, PeepholeOptimizationStmtBase) - ] - self._multistmt_peephole_opts = [ - cls(self.project, self.kb, ail_manager, self.func_addr, self._preserve_vvar_ids, self._type_hints) - for cls in peephole_optimizations - if issubclass(cls, PeepholeOptimizationMultiStmtBase) - ] - self._stmt_peephole_opts_by_kind = build_stmt_opts_by_kind(self._stmt_peephole_opts) + if peephole_bundle is None: + peephole_bundle = PeepholeOptimizationBundle( + self.project, + self.kb, + ail_manager, + func_addr=self.func_addr, + preserve_vvar_ids=self._preserve_vvar_ids, + type_hints=self._type_hints, + peephole_optimizations=peephole_optimizations, + ) + self._expr_peephole_opts = peephole_bundle.expr_opts + self._stmt_peephole_opts = peephole_bundle.stmt_opts + self._multistmt_peephole_opts = peephole_bundle.multistmt_opts + self._stmt_peephole_opts_by_kind = peephole_bundle.stmt_opts_by_kind self.result_block = None # cached peephole expression walker - self._expr_peephole_walker = _PeepholeExprsWalker(expr_opts=self._expr_peephole_opts) + self._expr_peephole_walker = peephole_bundle.expr_walker # cached Propagator and ReachingDefinitions results. Clear them if the block is updated self._propagator = cached_propagator @@ -128,22 +202,29 @@ class BlockSimplifier(Analysis): ctr = 0 max_ctr = 30 - new_block = self._eliminate_self_assignments(block) + new_block, changed = self._eliminate_self_assignments(block) + # True once dead-assignment elimination is known to have nothing to do on the block the loop below starts + # from -- either because it just ran over it without a change, or because its gate is off for that block. + dead_assignments_clean = True if self._count_nonconstant_statements(new_block) >= 2 and self._has_propagatable_assignments(new_block): - new_block = self._eliminate_dead_assignments(new_block) - # Structural ``likes`` (idx-agnostic) instead of ``!=`` which always trips on fresh ``manager.next_atom()`` - # ids even when nothing changed structurally. - # TODO: Keep track of changes and skip .likes(); .likes() is expensive. - if not new_block.likes(block): + new_block, dead_changed = self._eliminate_dead_assignments(new_block) + changed |= dead_changed + dead_assignments_clean = not dead_changed + if changed: self._clear_cache() block = new_block while True: ctr += 1 - new_block = self._simplify_block_once(block) - # TODO: Keep track of changes and skip .likes(); .likes() is expensive. - if new_block.likes(block): + # the entry peephole pass is only useful on the first iteration: every later iteration receives the + # output of the previous iteration's exit peephole pass, so running peephole again on entry is redundant. + new_block, changed = self._simplify_block_once( + block, entry_peephole=ctr == 1, dead_assignments_clean=dead_assignments_clean + ) + if not changed: break + + assert block is not None self._clear_cache() block = new_block if ctr >= max_ctr: @@ -156,9 +237,10 @@ class BlockSimplifier(Analysis): self.result_block = block - def _compute_propagation(self, block) -> SPropagatorAnalysis: + def _compute_propagation(self, block) -> SPropagator: if self._propagator is None: - self._propagator = self.project.analyses[SPropagatorAnalysis].prep(fail_fast=self._fail_fast)( + self._propagator = SPropagator( + self.project, subject=block, func_addr=self.func_addr, stack_pointer_tracker=self._stack_pointer_tracker, @@ -168,15 +250,12 @@ class BlockSimplifier(Analysis): def _compute_reaching_definitions(self, block) -> SRDAModel: if self._reaching_definitions is None: - self._reaching_definitions = ( - self.project.analyses[SReachingDefinitionsAnalysis] - .prep(fail_fast=self._fail_fast)( - subject=block, - track_tmps=True, - func_addr=self.func_addr, - ) - .model - ) + self._reaching_definitions = SReachingDefinitions( + self.project, + subject=block, + track_tmps=True, + func_addr=self.func_addr, + ).model return self._reaching_definitions def _clear_cache(self): @@ -191,35 +270,56 @@ class BlockSimplifier(Analysis): def _count_nonconstant_statements(block) -> int: return sum(1 for stmt in block.statements if not (isinstance(stmt, Jump) and isinstance(stmt.target, Const))) - def _simplify_block_once(self, block): - block = self._peephole_optimize(block) + def _simplify_block_once( + self, block, entry_peephole: bool = True, dead_assignments_clean: bool = False + ) -> tuple[Block, bool]: + """ + Run one round of simplification. Returns the new block and if any step reported a change. + + :param dead_assignments_clean: True if dead-assignment elimination is known to have nothing to do on + ``block`` as passed in. Only meaningful together with ``entry_peephole``. + """ + changed = False + # True once we know ``block`` is untouched and already at the fixpoint of every pass that has run over it: + # re-running those passes on it cannot report a change. + clean = False + if entry_peephole: + block, peephole_changed, exprs_updated = self._peephole_optimize(block) + changed |= peephole_changed + clean = dead_assignments_clean and not peephole_changed and not exprs_updated nonconstant_stmts = self._count_nonconstant_statements(block) has_propagatable_assignments = self._has_propagatable_assignments(block) - # propagator + # only call propagation if something is potentially propagatable if nonconstant_stmts >= 2 and has_propagatable_assignments: propagator = self._compute_propagation(block) new_block = block if propagator.model is not None: replacements = propagator.model.replacements if replacements: - _, new_block = self._replace_and_build( + replaced, new_block = self.replace_and_build( block, replacements, self._ail_manager, replace_registers=True ) - new_block = self._eliminate_self_assignments(new_block) + changed |= replaced + new_block, self_assign_changed = self._eliminate_self_assignments(new_block) + changed |= self_assign_changed self._clear_cache() else: - # Skipped calling Propagator new_block = block - if nonconstant_stmts >= 2 and has_propagatable_assignments: - new_block = self._eliminate_dead_assignments(new_block) + if clean and new_block is block: + return block, False - return self._peephole_optimize(new_block) + if nonconstant_stmts >= 2 and has_propagatable_assignments: + new_block, dead_changed = self._eliminate_dead_assignments(new_block) + changed |= dead_changed + + new_block, peephole_changed, _ = self._peephole_optimize(new_block) + return new_block, changed | peephole_changed @staticmethod - def _replace_and_build( + def replace_and_build( block: Block, replacements: Mapping[AILCodeLocation, Mapping[Expression, Expression]], ail_manager: Manager, @@ -309,7 +409,7 @@ class BlockSimplifier(Analysis): return True, new_block @staticmethod - def _eliminate_self_assignments(block): + def _eliminate_self_assignments(block) -> tuple[Block, bool]: new_statements = [] for stmt in block.statements: @@ -329,9 +429,12 @@ class BlockSimplifier(Analysis): continue new_statements.append(stmt) - return block.copy(statements=new_statements) + if len(new_statements) == len(block.statements): + # nothing was eliminated; keep the original block + return block, False + return block.copy(statements=new_statements), True - def _eliminate_dead_assignments(self, block): + def _eliminate_dead_assignments(self, block) -> tuple[Block, bool]: def _statement_has_calls(stmt: Statement) -> bool: """ Check if a statement has any Call expressions. @@ -354,7 +457,7 @@ class BlockSimplifier(Analysis): new_statements = [] if not block.statements: - return block + return block, False rd = self._compute_reaching_definitions(block) block_loc = (block.addr, block.idx) @@ -382,6 +485,7 @@ class BlockSimplifier(Analysis): used_tmps.add(tmp.tmp_idx) # Remove dead assignments + changed = False for idx, stmt in enumerate(block.statements): if isinstance(stmt, Assignment): # tmps can't execute new code @@ -390,39 +494,50 @@ class BlockSimplifier(Analysis): # does .src involve any Call expressions? if so, we cannot remove it if not _expression_has_calls(stmt.src): + changed = True continue if isinstance(stmt.dst, Tmp) and isinstance(stmt.src, Call): # eliminate the assignment and replace it with the call stmt = SideEffectStatement(self._ail_manager.next_atom(), stmt.src, **stmt.tags) + changed = True if isinstance(stmt, Assignment) and stmt.src == stmt.dst: + changed = True continue new_statements.append(stmt) - return block.copy(statements=new_statements) + if not changed: + # nothing was eliminated; keep the original block + return block, False + return block.copy(statements=new_statements), True # # Peephole optimization # - def _peephole_optimize(self, block): - # expressions are updated in place - peephole_optimize_exprs(block, self._expr_peephole_opts, walker=self._expr_peephole_walker) + def _peephole_optimize(self, block) -> tuple[Block, bool, bool]: + """ + Run all three peephole optimization levels on the block. + + :return: (block, changed, exprs_updated), where ``changed`` is True if any optimization applied and + ``exprs_updated`` is True if the expression walker rewrote any expression. + """ + exprs_updated = peephole_optimize_exprs(block, self._expr_peephole_opts, walker=self._expr_peephole_walker) # run statement-level optimizations statements, stmts_updated = peephole_optimize_stmts( - block, self._stmt_peephole_opts, stmt_opts_by_kind=self._stmt_peephole_opts_by_kind + block, + self._stmt_peephole_opts, + stmt_opts_by_kind=self._stmt_peephole_opts_by_kind, + fixpoint_exprs=self._expr_peephole_walker.fixpoint_stmts, ) new_block = block.copy(statements=statements) if stmts_updated else block statements, multi_stmts_updated = peephole_optimize_multistmts(new_block, self._multistmt_peephole_opts) - if not multi_stmts_updated: - return new_block - return new_block.copy(statements=statements) - - -register_analysis(BlockSimplifier, "AILBlockSimplifier") + if multi_stmts_updated: + new_block = new_block.copy(statements=statements) + return new_block, stmts_updated or multi_stmts_updated, exprs_updated diff --git a/angr/analyses/decompiler/clinic.py b/angr/analyses/decompiler/clinic.py index 31764c138..816315dc3 100644 --- a/angr/analyses/decompiler/clinic.py +++ b/angr/analyses/decompiler/clinic.py @@ -19,10 +19,11 @@ from angr.ailment.block_walker import AILBlockViewer from angr.ailment.expression import Array, Call, FunctionLikeMacro, Let, RustEnum, Struct, VirtualVariable from angr.analyses.analysis import Analysis, register_analysis from angr.analyses.cfg.cfg_base import CFGBase +from angr.analyses.decompiler.block_simplifier import BlockSimplifier, PeepholeOptimizationBundle from angr.analyses.decompiler.callsite_maker import CallSiteMaker from angr.analyses.decompiler.optimization_pass_registry import name_to_pass, pass_to_name from angr.analyses.s_liveness import SLivenessAnalysis -from angr.analyses.s_reaching_definitions import SReachingDefinitionsAnalysis +from angr.analyses.s_reaching_definitions import SReachingDefinitions from angr.analyses.s_reaching_definitions.s_rda_model import SRDAModel from angr.analyses.stack_pointer_tracker import OffsetVal, Register from angr.analyses.typehoon import Typehoon @@ -179,6 +180,7 @@ class ComboRegReferenceWalker(AILBlockRewriter): self, expr_idx: int, expr: VirtualVariable, stmt_idx: int, stmt: Statement | None, block: Block | None ): if expr.was_combo_reg: + assert expr.reg_vvars is not None for reg_vvar in expr.reg_vvars: self.varid_to_combo_reg[reg_vvar.varid] = expr elif expr.was_reg and expr.varid in self.varid_to_combo_reg: @@ -312,7 +314,10 @@ class Clinic(Analysis, Serializable): self._exception_edges = exception_edges self._sp_tracker_track_memory = sp_tracker_track_memory self._cfg: CFGModel | None = cfg - self.peephole_optimizations = peephole_optimizations + self.peephole_optimizations = list(peephole_optimizations) if peephole_optimizations is not None else None + # cached PeepholeOptimizationBundle, shared by every BlockSimplifier this Clinic creates; rebuilt whenever + # the construction parameters change (see _get_peephole_bundle) + self._peephole_bundle: PeepholeOptimizationBundle | None = None # peephole-optimization names that could not be resolved at parse time (their defining module was not # imported); resolve_peephole_optimizations() retries them self.unresolvable_peephole_optimizations: list[str] = [] @@ -726,7 +731,7 @@ class Clinic(Analysis, Serializable): walker.walk(block) return ail_graph - def _rewrite_combo_reg_param_references(self, ail_graph): + def _rewrite_combo_reg_param_references(self, ail_graph) -> networkx.DiGraph: """ Rewrite reads of the constituent registers of combo-register arguments into loads from the arguments. @@ -736,6 +741,8 @@ class Clinic(Analysis, Serializable): linked to the recovered argument variables. """ + if self.arg_vvars is None: + return ail_graph combo_arg_vvars = [ arg_vvar for arg_vvar, _ in self.arg_vvars.values() @@ -746,6 +753,7 @@ class Clinic(Analysis, Serializable): walker = ComboRegReferenceWalker(self.project, self._ail_manager) for arg_vvar in combo_arg_vvars: + assert arg_vvar.reg_vvars is not None for reg_vvar in arg_vvar.reg_vvars: walker.varid_to_combo_reg[reg_vvar.varid] = arg_vvar for block in GraphUtils.quasi_topological_sort_nodes(ail_graph): @@ -1336,13 +1344,20 @@ class Clinic(Analysis, Serializable): and isinstance(cc.cc.RETURN_VAL, SimRegArg) ): reg_offset, reg_size = self.project.arch.registers[cc.cc.RETURN_VAL.reg_name] - last_stmt.ret_expr = ailment.Expr.Register( + ret_expr = ailment.Expr.Register( self._ail_manager.next_atom(), reg_offset, reg_size * 8, ins_addr=callsite_ins_addr, reg_name=cc.cc.RETURN_VAL.reg_name, ) + last_stmt = ailment.Stmt.SideEffectStatement( + self._ail_manager.next_atom(), + last_stmt.expr, + ret_expr=ret_expr, + fp_ret_expr=last_stmt.fp_ret_expr, + **last_stmt.tags, + ) # finally, recover the calling convention of the current function if ( @@ -1551,8 +1566,10 @@ class Clinic(Analysis, Serializable): if not block.statements: continue last_stmt = block.statements[-1] - if isinstance(last_stmt, ailment.Stmt.SideEffectStatement) and not isinstance( - last_stmt.expr.target, ailment.Expr.Const + if ( + isinstance(last_stmt, ailment.Stmt.SideEffectStatement) + and isinstance(last_stmt.expr, ailment.Expr.Call) + and not isinstance(last_stmt.expr.target, ailment.Expr.Const) ): # indirect call # consult CFG to see if this is a call with a single successor @@ -1571,9 +1588,8 @@ class Clinic(Analysis, Serializable): ): # found a single successor - replace the last statement. assert isinstance(last_stmt.expr.target, ailment.Expr.Expression) # not a string - new_last_stmt = last_stmt.copy() assert isinstance(successors[0].addr, int) - old_call = new_last_stmt.expr + old_call = last_stmt.expr old_cc = self.variable_map.calling_convention(old_call) old_proto = self.variable_map.prototype(old_call) new_call = ailment.Expr.Call( @@ -1583,11 +1599,17 @@ class Clinic(Analysis, Serializable): bits=old_call.bits, **old_call.tags, ) + new_last_stmt = ailment.Stmt.SideEffectStatement( + self._ail_manager.next_atom(), + new_call, + ret_expr=last_stmt.ret_expr, + fp_ret_expr=last_stmt.fp_ret_expr, + **last_stmt.tags, + ) if old_cc is not None: self.variable_map.set_calling_convention(new_call, old_cc) if old_proto is not None: self.variable_map.set_prototype(new_call, old_proto) - new_last_stmt.expr = new_call block.statements[-1] = new_last_stmt elif isinstance(last_stmt, ailment.Stmt.Jump) and not isinstance(last_stmt.target, ailment.Expr.Const): @@ -1691,8 +1713,10 @@ class Clinic(Analysis, Serializable): """ for block in list(ail_graph.nodes()): last_stmt = block.statements[-1] - if isinstance(last_stmt, ailment.Stmt.SideEffectStatement) and isinstance( - last_stmt.expr.target, ailment.Expr.Const + if ( + isinstance(last_stmt, ailment.Stmt.SideEffectStatement) + and isinstance(last_stmt.expr, ailment.Expr.Call) + and isinstance(last_stmt.expr.target, ailment.Expr.Const) ): target = last_stmt.expr.target.value else: @@ -1739,7 +1763,10 @@ class Clinic(Analysis, Serializable): continue last_stmt = block.statements[-1] - if not isinstance(last_stmt, ailment.Stmt.SideEffectStatement): + if not ( + isinstance(last_stmt, ailment.Stmt.SideEffectStatement) + and isinstance(last_stmt.expr, ailment.Expr.Call) + ): continue cc = self.variable_map.calling_convention(last_stmt.expr) @@ -1841,8 +1868,9 @@ class Clinic(Analysis, Serializable): preserve_vvar_ids=preserve_vvar_ids, type_hints=type_hints, ) - key = ail_block.addr, ail_block.idx - blocks_by_addr_and_idx[key] = simplified + if simplified is not None: + key = ail_block.addr, ail_block.idx + blocks_by_addr_and_idx[key] = simplified # update blocks_map to allow node_addr to node lookup def _replace_node_handler(node): @@ -1855,6 +1883,35 @@ class Clinic(Analysis, Serializable): return ail_graph + def _get_peephole_bundle( + self, + preserve_vvar_ids: set[int] | None, + type_hints: list[tuple[atoms.VirtualVariable | atoms.MemoryLocation, str]] | None, + ) -> PeepholeOptimizationBundle: + """ + Return the cached PeepholeOptimizationBundle, rebuilding if any construction parameter changed. + """ + bundle = self._peephole_bundle + if bundle is None or not bundle.matches( + self.project, + self._ail_manager, + self.function.addr, + preserve_vvar_ids, + type_hints, + self.peephole_optimizations, + ): + bundle = PeepholeOptimizationBundle( + self.project, + self.kb, + self._ail_manager, + func_addr=self.function.addr, + preserve_vvar_ids=preserve_vvar_ids, + type_hints=type_hints, + peephole_optimizations=self.peephole_optimizations, + ) + self._peephole_bundle = bundle + return bundle + def _simplify_block( self, ail_block, @@ -1862,7 +1919,7 @@ class Clinic(Analysis, Serializable): cache=None, preserve_vvar_ids: set[int] | None = None, type_hints: list[tuple[atoms.VirtualVariable | atoms.MemoryLocation, str]] | None = None, - ): + ) -> ailment.Block | None: """ Simplify a single AIL block. @@ -1881,17 +1938,17 @@ class Clinic(Analysis, Serializable): cached_rd = cache_item.rd cached_prop = cache_item.prop - simp = self.project.analyses.AILBlockSimplifier( + simp = BlockSimplifier( + self.project, ail_block, self._ail_manager, self.function.addr, - fail_fast=self._fail_fast, stack_pointer_tracker=stack_pointer_tracker, - peephole_optimizations=self.peephole_optimizations, cached_reaching_definitions=cached_rd, cached_propagator=cached_prop, preserve_vvar_ids=preserve_vvar_ids, type_hints=type_hints, + peephole_bundle=self._get_peephole_bundle(preserve_vvar_ids, type_hints), ) # update the cache if cache is not None: @@ -2327,14 +2384,14 @@ class Clinic(Analysis, Serializable): removed_vvar_ids |= csm.removed_vvar_ids if csm.result_block and csm.result_block != block: ail_block = csm.result_block - simp = self.project.analyses.AILBlockSimplifier( + simp = BlockSimplifier( + self.project, ail_block, self._ail_manager, self.function.addr, - fail_fast=self._fail_fast, stack_pointer_tracker=stack_pointer_tracker, - peephole_optimizations=self.peephole_optimizations, preserve_vvar_ids=preserve_vvar_ids, + peephole_bundle=self._get_peephole_bundle(preserve_vvar_ids, None), ) return simp.result_block return None @@ -2960,7 +3017,9 @@ class Clinic(Analysis, Serializable): continue assert block.addr is not None last_stmt = block.statements[-1] - if isinstance(last_stmt, ailment.Stmt.SideEffectStatement): + if isinstance(last_stmt, ailment.Stmt.SideEffectStatement) and isinstance( + last_stmt.expr, ailment.Expr.Call + ): # we can't examine the call target at this point because constant propagation hasn't run yet; we consult # the CFG instead callsite_node = self._cfg.get_any_node(block.addr, anyaddr=True) @@ -2974,7 +3033,7 @@ class Clinic(Analysis, Serializable): callee_func = self.kb.functions.get_by_addr(callee) if callee_func.info.get("jmp_rax", False) is True: call_stmt = last_stmt.copy() - old_call = call_stmt.expr + old_call = last_stmt.expr new_target = ailment.Expr.Register( self._ail_manager.next_atom(), self.project.arch.registers["rax"][0], @@ -3098,7 +3157,7 @@ class Clinic(Analysis, Serializable): break if ite_expr_stmt_idx is None: return None - assert ite_expr_stmt is not None + assert ite_expr_stmt is not None and isinstance(ite_expr_stmt.src, ailment.Expr.ITE) true_block_ail.statements[ite_expr_stmt_idx] = ailment.Stmt.Assignment( ite_expr_stmt.idx, ite_expr_stmt.dst, ite_expr_stmt.src.iftrue, **ite_expr_stmt.tags @@ -3118,7 +3177,7 @@ class Clinic(Analysis, Serializable): break if ite_expr_stmt_idx is None: return None - assert ite_expr_stmt is not None + assert ite_expr_stmt is not None and isinstance(ite_expr_stmt.src, ailment.Expr.ITE) false_block_ail.statements[ite_expr_stmt_idx] = ailment.Stmt.Assignment( ite_expr_stmt.idx, ite_expr_stmt.dst, ite_expr_stmt.src.iffalse, **ite_expr_stmt.tags @@ -3903,8 +3962,10 @@ class Clinic(Analysis, Serializable): if not node.statements or ail_graph.out_degree[node] != 1: continue last_stmt = node.statements[-1] - if isinstance(last_stmt, ailment.Stmt.SideEffectStatement) and isinstance( - last_stmt.expr.target, ailment.Expr.Const + if ( + isinstance(last_stmt, ailment.Stmt.SideEffectStatement) + and isinstance(last_stmt.expr, ailment.Expr.Call) + and isinstance(last_stmt.expr.target, ailment.Expr.Const) ): func = ( self.project.kb.functions.get_by_addr(last_stmt.expr.target.value) @@ -3926,6 +3987,7 @@ class Clinic(Analysis, Serializable): and last_stmt.data.value == succ.addr ) or ( isinstance(last_stmt, ailment.Stmt.Assignment) + and isinstance(last_stmt.dst, ailment.Expr.VirtualVariable) and last_stmt.dst.was_stack and last_stmt.dst.stack_offset < 0 and isinstance(last_stmt.src, ailment.Expr.Const) @@ -3944,8 +4006,10 @@ class Clinic(Analysis, Serializable): if not node.statements or ail_graph.out_degree[node] != 1: continue last_stmt = node.statements[-1] - if isinstance(last_stmt, ailment.Stmt.SideEffectStatement) and isinstance( - last_stmt.expr.target, ailment.Expr.Const + if ( + isinstance(last_stmt, ailment.Stmt.SideEffectStatement) + and isinstance(last_stmt.expr, ailment.Expr.Call) + and isinstance(last_stmt.expr.target, ailment.Expr.Const) ): func = ( self.project.kb.functions.get_by_addr(last_stmt.expr.target.value) @@ -3967,6 +4031,7 @@ class Clinic(Analysis, Serializable): and last_stmt.data.value == succ.addr ) or ( isinstance(last_stmt, ailment.Stmt.Assignment) + and isinstance(last_stmt.dst, ailment.Expr.VirtualVariable) and last_stmt.dst.was_stack and last_stmt.dst.stack_offset < 0 and isinstance(last_stmt.src, ailment.Expr.Const) @@ -4213,16 +4278,13 @@ class Clinic(Analysis, Serializable): def _compute_reaching_definitions(self, func_args=None) -> SRDAModel: # Computing reaching definitions # TODO: Refactor this into a method of the upcoming AILFunctionGraph class. - return ( - self.project.analyses[SReachingDefinitionsAnalysis] - .prep(fail_fast=self._fail_fast)( - subject=self.function, - func_graph=self._ail_graph, - func_args=func_args if func_args is not None else self.func_args, - use_callee_saved_regs_at_return=not self._register_save_areas_removed, - ) - .model - ) + return SReachingDefinitions( + self.project, + subject=self.function, + func_graph=self._ail_graph, + func_args=func_args if func_args is not None else self.func_args, + use_callee_saved_regs_at_return=not self._register_save_areas_removed, + ).model def resolve_peephole_optimizations(self) -> None: """Retry resolving peephole-optimization names that were unresolvable at parse time (their defining module diff --git a/angr/analyses/decompiler/dephication/graph_vvar_mapping.py b/angr/analyses/decompiler/dephication/graph_vvar_mapping.py index 6bbb34c78..998bd2109 100644 --- a/angr/analyses/decompiler/dephication/graph_vvar_mapping.py +++ b/angr/analyses/decompiler/dephication/graph_vvar_mapping.py @@ -9,7 +9,7 @@ from angr.ailment.block import Block from angr.ailment.expression import Phi, VirtualVariable from angr.ailment.statement import Assignment, ConditionalJump, Jump, Label from angr.analyses.analysis import Analysis, register_analysis -from angr.analyses.s_reaching_definitions import SRDAModel +from angr.analyses.s_reaching_definitions import SRDAModel, SReachingDefinitions from angr.knowledge_plugins.functions import Function from angr.utils.ssa import is_phi_assignment @@ -60,9 +60,7 @@ class GraphDephicationVVarMapping(Analysis): # pylint:disable=abstract-method self.vvar_to_vvar_mapping = None self.copied_vvar_ids: set[int] = set() - self._rd: SRDAModel = self.project.analyses.SReachingDefinitions( - subject=self._function, func_graph=self._graph - ).model + self._rd: SRDAModel = SReachingDefinitions(self.project, subject=self._function, func_graph=self._graph).model self._blocks: dict[tuple[int, int | None], Block] = {(block.addr, block.idx): block for block in self._graph} self._analyze() @@ -237,7 +235,9 @@ class GraphDephicationVVarMapping(Analysis): # pylint:disable=abstract-method if src not in stmt_appended_locs: # we have not yet appended a statement to this block the_block = self._blocks[src] - ins_addr = the_block.addr + the_block.original_size - 1 + ins_addr = ( + the_block.addr + (the_block.original_size if the_block.original_size is not None else 1) - 1 + ) new_category = phi_stmt.dst.category new_oident = phi_stmt.dst.oident new_vvar = VirtualVariable( diff --git a/angr/analyses/decompiler/optimization_passes/condition_constprop.py b/angr/analyses/decompiler/optimization_passes/condition_constprop.py index 1809297f9..fff80c6c6 100644 --- a/angr/analyses/decompiler/optimization_passes/condition_constprop.py +++ b/angr/analyses/decompiler/optimization_passes/condition_constprop.py @@ -9,6 +9,7 @@ from angr.ailment import AILBlockRewriter, Block, Expression from angr.ailment.expression import BinaryOp, Const, VirtualVariable from angr.ailment.statement import Assignment, ConditionalJump, Statement from angr.analyses.decompiler.utils import first_nonlabel_nonphi_statement +from angr.analyses.s_reaching_definitions import SReachingDefinitions from angr.utils.graph import dominates from angr.utils.timing import timethis @@ -140,7 +141,7 @@ class ConditionConstantPropagation(OptimizationPass): entry_node_addr, entry_node_idx = self.entry_node_addr entry_node = self._get_block(entry_node_addr, idx=entry_node_idx) idoms = networkx.algorithms.immediate_dominators(self._graph, entry_node) - rda: SRDAModel = self.project.analyses.SReachingDefinitions(self._func, func_graph=self._graph).model + rda: SRDAModel = SReachingDefinitions(self.project, self._func, func_graph=self._graph).model for src, cconds in cconds_by_src.items(): head_block = self._get_block(src[0], idx=src[1]) diff --git a/angr/analyses/decompiler/optimization_passes/optimization_pass.py b/angr/analyses/decompiler/optimization_passes/optimization_pass.py index 02e8c85f5..def8b58fc 100644 --- a/angr/analyses/decompiler/optimization_passes/optimization_pass.py +++ b/angr/analyses/decompiler/optimization_passes/optimization_pass.py @@ -14,6 +14,7 @@ import angr from angr import ailment from angr.ailment.manager import Manager from angr.analyses.decompiler.ailgraph_walker import AILGraphWalker +from angr.analyses.decompiler.block_simplifier import BlockSimplifier, PeepholeOptimizationBundle from angr.analyses.decompiler.condition_processor import ConditionProcessor from angr.analyses.decompiler.counters import ControlFlowStructureCounter from angr.analyses.decompiler.goto_manager import Goto, GotoManager @@ -365,8 +366,9 @@ class OptimizationPass(BaseOptimizationPass): ail_block, cache=cache, ) - key = ail_block.addr, ail_block.idx - blocks_by_addr_and_idx[key] = simplified + if simplified is not None: + key = ail_block.addr, ail_block.idx + blocks_by_addr_and_idx[key] = simplified # update blocks_map to allow node_addr to node lookup def _replace_node_handler(node): @@ -379,6 +381,21 @@ class OptimizationPass(BaseOptimizationPass): return ail_graph + def _get_peephole_bundle(self) -> PeepholeOptimizationBundle: + bundle: None | PeepholeOptimizationBundle = self._scratch.get("peephole_bundle") + if bundle is None or not bundle.matches( + self.project, self.manager, self._func.addr, None, None, self._peephole_optimizations + ): + bundle = PeepholeOptimizationBundle( + self.project, + self.kb, + self.manager, + func_addr=self._func.addr, + peephole_optimizations=self._peephole_optimizations, + ) + self._scratch["peephole_bundle"] = bundle + return bundle + def _simplify_block(self, ail_block, cache=None): """ Simplify a single AIL block. @@ -397,13 +414,14 @@ class OptimizationPass(BaseOptimizationPass): cached_rd = cache_item.rd cached_prop = cache_item.prop - simp = self.project.analyses.AILBlockSimplifier( + simp = BlockSimplifier( + self.project, ail_block, self.manager, self._func.addr, - peephole_optimizations=self._peephole_optimizations, cached_reaching_definitions=cached_rd, cached_propagator=cached_prop, + peephole_bundle=self._get_peephole_bundle(), ) # update the cache if cache is not None: diff --git a/angr/analyses/decompiler/optimization_passes/peephole_simplifier.py b/angr/analyses/decompiler/optimization_passes/peephole_simplifier.py index 35b7ebac6..e726b8df4 100644 --- a/angr/analyses/decompiler/optimization_passes/peephole_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/peephole_simplifier.py @@ -1,10 +1,7 @@ from __future__ import annotations from angr import ailment -from angr.analyses.decompiler.peephole_optimizations import ( - EXPR_OPTS, - PeepholeOptimizationExprBase, -) +from angr.analyses.decompiler.block_simplifier import BlockSimplifier, PeepholeOptimizationBundle from angr.analyses.decompiler.sequence_walker import SequenceWalker from angr.analyses.decompiler.utils import ( peephole_optimize_expr, @@ -40,11 +37,15 @@ class PostStructuringPeepholeOptimizationPass(SequenceOptimizationPass): def __init__(self, *args, peephole_optimizations=None, **kwargs): super().__init__(*args, **kwargs) self._peephole_optimizations = peephole_optimizations - self._expr_peephole_opts = [ - cls(self.project, self.kb, ail_manager=self.manager, func_addr=self._func.addr) - for cls in (self._peephole_optimizations or EXPR_OPTS) - if issubclass(cls, PeepholeOptimizationExprBase) - ] + # one bundle for all BlockSimplifier invocations of this pass + self._peephole_bundle = PeepholeOptimizationBundle( + self.project, + self.kb, + self.manager, + func_addr=self._func.addr, + peephole_optimizations=self._peephole_optimizations, + ) + self._expr_peephole_opts = self._peephole_bundle.expr_opts self.analyze() def _check(self): @@ -65,12 +66,13 @@ class PostStructuringPeepholeOptimizationPass(SequenceOptimizationPass): old_block, new_block = None, block while old_block != new_block: old_block = new_block - # Note: AILBlockSimplifier updates expressions in place - simp = self.project.analyses.AILBlockSimplifier( + # Note: BlockSimplifier updates expressions in place + simp = BlockSimplifier( + self.project, new_block, func_addr=self._func.addr, - peephole_optimizations=self._peephole_optimizations, ail_manager=self.manager, + peephole_bundle=self._peephole_bundle, ) assert simp.result_block is not None new_block = simp.result_block diff --git a/angr/analyses/decompiler/optimization_passes/register_save_area_simplifier_adv.py b/angr/analyses/decompiler/optimization_passes/register_save_area_simplifier_adv.py index 1d0110e1e..b70acbf1c 100644 --- a/angr/analyses/decompiler/optimization_passes/register_save_area_simplifier_adv.py +++ b/angr/analyses/decompiler/optimization_passes/register_save_area_simplifier_adv.py @@ -2,15 +2,20 @@ from __future__ import annotations import logging +from typing import TYPE_CHECKING from angr.ailment.expression import VirtualVariable from angr.ailment.statement import Assignment from angr.analyses.decompiler.stack_item import StackItem, StackItemType -from angr.code_location import CodeLocation +from angr.analyses.s_reaching_definitions import SReachingDefinitions from angr.utils.ail import is_phi_assignment from .optimization_pass import OptimizationPass, OptimizationPassStage +if TYPE_CHECKING: + from angr.analyses.s_reaching_definitions import SRDAModel + from angr.code_location import AILCodeLocation + _l = logging.getLogger(name=__name__) @@ -36,7 +41,8 @@ class RegisterSaveAreaSimplifierAdvanced(OptimizationPass): self.analyze() def _check(self): - self._srda = self.project.analyses.SReachingDefinitions( + self._srda = SReachingDefinitions( + self.project, subject=self._func, func_graph=self._graph, func_args={vvar for vvar, _ in arg_vvars.values()} if (arg_vvars := self._arg_vvars) is not None else set(), @@ -60,7 +66,7 @@ class RegisterSaveAreaSimplifierAdvanced(OptimizationPass): if cache is None: return - info: list[tuple[list[CodeLocation], int]] = cache["info"] + info: list[tuple[list[AILCodeLocation], int]] = cache["info"] updated_blocks = {} for locs, _ in info: @@ -84,13 +90,13 @@ class RegisterSaveAreaSimplifierAdvanced(OptimizationPass): stack_offset, self.project.arch.bytes, "regs", StackItemType.SAVED_REGS ) - def _find_reg_store_and_restore_locations(self) -> list[tuple[list[CodeLocation], int]]: - results: list[tuple[list[CodeLocation], int]] = [] + def _find_reg_store_and_restore_locations(self) -> list[tuple[list[AILCodeLocation], int]]: + results: list[tuple[list[AILCodeLocation], int]] = [] assert self._srda is not None - srda_model = self._srda.model + srda_model: SRDAModel = self._srda.model # find all registers that are defined externally and used exactly once - saved_vvars: set[tuple[int, CodeLocation]] = set() + saved_vvars: set[tuple[int, AILCodeLocation]] = set() for vvar_id, loc in srda_model.all_vvar_definitions.items(): # SReachingDefinitions records externally-defined (function live-in) vvars via # AILCodeLocation.make_extern(). These are AILCodeLocation instances (not ExternalCodeLocation), so the @@ -131,7 +137,7 @@ class RegisterSaveAreaSimplifierAdvanced(OptimizationPass): all_stack_vvar_uses = srda_model.all_vvar_uses.get(stack_vvar.varid, []) # partition the uses into phi uses and non-phi uses stack_vvar_uses = set() - phi_use_locs: list[CodeLocation] = [] + phi_use_locs: list[AILCodeLocation] = [] for vvar_, loc_ in all_stack_vvar_uses: use_block = self._get_block(loc_.block_addr, idx=loc_.block_idx) if use_block is None or loc_.stmt_idx is None: @@ -167,7 +173,7 @@ class RegisterSaveAreaSimplifierAdvanced(OptimizationPass): return results - def _phi_uses_are_dead(self, srda_model, phi_use_locs: list[CodeLocation]) -> bool: + def _phi_uses_are_dead(self, srda_model, phi_use_locs: list[AILCodeLocation]) -> bool: """Return True iff every phi statement at ``phi_use_locs`` defines a virtual variable that has no uses. Such a phi is dead and can be removed together with the store that feeds it.""" diff --git a/angr/analyses/decompiler/peephole_optimizations/base.py b/angr/analyses/decompiler/peephole_optimizations/base.py index 038a65517..7c7512ff4 100644 --- a/angr/analyses/decompiler/peephole_optimizations/base.py +++ b/angr/analyses/decompiler/peephole_optimizations/base.py @@ -16,9 +16,18 @@ if TYPE_CHECKING: class PeepholeOptimizationStmtBase: """ The base class for all peephole optimizations that are applied on AIL statements. + + ``fixpoint_reached`` is an output parameter; it tells the caller whether this optimizer should run on this + statement again (e.g., when statements prior to the current statement are optimized and changed). The caller sets + ``fixpoint_reached`` to True before every :meth:`optimize` call; the optimizer optionally sets it to False if it + wants to be invoked again on the same statement. + + So, ``fixpoint_reached`` should be set to (or kept) True if (a) the optimizer has optimized the statement and does + not expect to optimize it ever again, or (b) the optimizer cannot ever optimize the statement. """ __slots__ = ( + "fixpoint_reached", "func_addr", "kb", "manager", @@ -31,6 +40,7 @@ class PeepholeOptimizationStmtBase: func_addr: int | None preserve_vvar_ids: set[int] type_hints: list[tuple[atoms.VirtualVariable | atoms.MemoryLocation, str]] + fixpoint_reached: bool NAME = "Peephole Optimization - Statement" DESCRIPTION = "Peephole Optimization - Statement" @@ -51,6 +61,7 @@ class PeepholeOptimizationStmtBase: self.func_addr = func_addr self.preserve_vvar_ids = set() if preserve_vvar_ids is None else preserve_vvar_ids self.type_hints = [] if type_hints is None else type_hints + self.fixpoint_reached = False def optimize(self, stmt, stmt_idx: int | None = None, block=None, **kwargs): raise NotImplementedError("_optimize() is not implemented.") @@ -59,9 +70,13 @@ class PeepholeOptimizationStmtBase: class PeepholeOptimizationMultiStmtBase: """ The base class for all peephole optimizations that are applied on multiple AIL statements at once. + + ``fixpoint_reached`` exists for uniformity but is unused. Multi-statement optimizers always run regardless of + whether the statements have reached fixed points or not. """ __slots__ = ( + "fixpoint_reached", "func_addr", "kb", "manager", @@ -74,6 +89,7 @@ class PeepholeOptimizationMultiStmtBase: func_addr: int | None preserve_vvar_ids: set[int] type_hints: list[tuple[atoms.VirtualVariable | atoms.MemoryLocation, str]] + fixpoint_reached: bool NAME = "Peephole Optimization - Multi-statement" DESCRIPTION = "Peephole Optimization - Multi-statement" @@ -94,6 +110,7 @@ class PeepholeOptimizationMultiStmtBase: self.func_addr = func_addr self.preserve_vvar_ids = set() if preserve_vvar_ids is None else preserve_vvar_ids self.type_hints = [] if type_hints is None else type_hints + self.fixpoint_reached = False def optimize(self, stmts: list[Statement], stmt_idx: int | None = None, block=None, **kwargs): raise NotImplementedError("_optimize() is not implemented.") @@ -101,10 +118,12 @@ class PeepholeOptimizationMultiStmtBase: class PeepholeOptimizationExprBase: """ - The base class for all peephole optimizations that are applied on AIL expressions. + The base class for all peephole optimizations that are applied on AIL expressions. Please refer to + :class:`PeepholeOptimizationStmtBase` for the ``fixpoint_reached`` contract. """ __slots__ = ( + "fixpoint_reached", "func_addr", "kb", "manager", @@ -117,6 +136,7 @@ class PeepholeOptimizationExprBase: func_addr: int | None preserve_vvar_ids: set[int] type_hints: list[tuple[atoms.VirtualVariable | atoms.MemoryLocation, str]] + fixpoint_reached: bool NAME = "Peephole Optimization - Expression" DESCRIPTION = "Peephole Optimization - Expression" @@ -137,6 +157,7 @@ class PeepholeOptimizationExprBase: self.func_addr = func_addr self.preserve_vvar_ids = set() if preserve_vvar_ids is None else preserve_vvar_ids self.type_hints = [] if type_hints is None else type_hints + self.fixpoint_reached = False def optimize(self, expr, *, stmt_idx: int | None = None, block=None, **kwargs) -> Expression | None: raise NotImplementedError("_optimize() is not implemented.") diff --git a/angr/analyses/decompiler/peephole_optimizations/concat_simplifier.py b/angr/analyses/decompiler/peephole_optimizations/concat_simplifier.py index 24584efcf..ca084c242 100644 --- a/angr/analyses/decompiler/peephole_optimizations/concat_simplifier.py +++ b/angr/analyses/decompiler/peephole_optimizations/concat_simplifier.py @@ -106,6 +106,8 @@ class ConcatSimplifier(PeepholeOptimizationExprBase): low, **expr.tags, ) + # we may start matching once the block changes + self.fixpoint_reached = False return None diff --git a/angr/analyses/decompiler/peephole_optimizations/rol_ror.py b/angr/analyses/decompiler/peephole_optimizations/rol_ror.py index 9e48271c1..46a5eb27f 100644 --- a/angr/analyses/decompiler/peephole_optimizations/rol_ror.py +++ b/angr/analyses/decompiler/peephole_optimizations/rol_ror.py @@ -35,6 +35,9 @@ class RolRorRewriter(PeepholeOptimizationStmtBase): op0, op1 = stmt.src.operands if isinstance(op0, Tmp) and isinstance(op1, Tmp): + # matches against the two preceding statements, so it may start matching once the block changes + self.fixpoint_reached = False + if stmt_idx < 2: return None @@ -70,6 +73,7 @@ class RolRorRewriter(PeepholeOptimizationStmtBase): and shiftleft_amount + stmt2_op1.value == stmt.dst.bits ): rol_amount = Const(self.manager.next_atom(), shiftleft_amount, 8, **stmt1_op1.tags) + self.fixpoint_reached = True return Assignment( stmt.idx, stmt.dst, @@ -89,6 +93,7 @@ class RolRorRewriter(PeepholeOptimizationStmtBase): and (shiftleft_amount := get_expr_shift_left_amount(stmt_2.src)) is not None and stmt1_op1.value + shiftleft_amount == stmt.dst.bits ): + self.fixpoint_reached = True return Assignment( stmt.idx, stmt.dst, diff --git a/angr/analyses/decompiler/peephole_optimizations/sar_to_signed_div.py b/angr/analyses/decompiler/peephole_optimizations/sar_to_signed_div.py index c18c9130c..c82225d5c 100644 --- a/angr/analyses/decompiler/peephole_optimizations/sar_to_signed_div.py +++ b/angr/analyses/decompiler/peephole_optimizations/sar_to_signed_div.py @@ -19,10 +19,13 @@ class SarToSignedDiv(PeepholeOptimizationExprBase): if expr.op == "Sar" and isinstance(expr.operands[1], Const): op0, const = expr.operands - if isinstance(op0, VirtualVariable) and op0.was_reg and stmt_idx is not None and block is not None: - # look back by one statement to find its definition - op0 = self.find_definition(op0, stmt_idx, block) - # TODO: Ensure the new op0 does not have any expressions that overlap with the old op0 (a register) + if isinstance(op0, VirtualVariable) and op0.was_reg: + # depends on the preceding statement, so it may start matching once the block changes + self.fixpoint_reached = False + if stmt_idx is not None and block is not None: + # look back by one statement to find its definition + op0 = self.find_definition(op0, stmt_idx, block) + # TODO: Ensure the new op0 does not have any expressions that overlap with the old op0 (a register) const_value = const.value conv = None @@ -75,6 +78,8 @@ class SarToSignedDiv(PeepholeOptimizationExprBase): if conv is not None: # wrap it up with a Convert again r = Convert(conv.idx, conv.from_bits, conv.to_bits, conv.is_signed, r, **conv.tags) + # rewritten: the result no longer depends on the block context + self.fixpoint_reached = True return r return None diff --git a/angr/analyses/decompiler/structurer_nodes.py b/angr/analyses/decompiler/structurer_nodes.py index 000df8947..efa08e51b 100644 --- a/angr/analyses/decompiler/structurer_nodes.py +++ b/angr/analyses/decompiler/structurer_nodes.py @@ -433,7 +433,7 @@ class IncompleteSwitchCaseHeadStatement(_IncompleteSwitchCaseHeadStatementBase): Describes a switch-case head. This is only created by LoweredSwitchSimplifier. """ - __slots__ = ("_case_addrs_str", "addr", "case_addrs", "switch_variable") + __slots__ = ("_case_addrs_str", "addr", "case_addrs", "peephole_optimized", "switch_variable") # Mirror the rustlib ``Statement.kind`` slot so downstream code that # dispatches on ``stmt.kind`` doesn't have to fall back to @@ -441,8 +441,9 @@ class IncompleteSwitchCaseHeadStatement(_IncompleteSwitchCaseHeadStatementBase): # every rustlib variant so the kind-keyed dispatch sites land in # their default branch. kind = "IncompleteSwitchCaseHead" + pykind = "IncompleteSwitchCaseHead" - def __init__(self, idx, switch_variable, case_addrs, **kwargs): + def __init__(self, idx, switch_variable, case_addrs, peephole_optimized: bool = False, **kwargs): super().__init__(idx, **kwargs) self.switch_variable = switch_variable # original cmp node, case value | "default", address of the case node, idx of the case node, @@ -450,6 +451,7 @@ class IncompleteSwitchCaseHeadStatement(_IncompleteSwitchCaseHeadStatementBase): self.case_addrs: list[tuple[ailment.Block, int | str, int, int | None, int]] = case_addrs # a string representation of the addresses of all cases, used for hashing self._case_addrs_str = str(sorted([c[0].addr for c in self.case_addrs if c[0] is not None])) + self.peephole_optimized = peephole_optimized def __repr__(self): return f"SwitchCaseHead: switch {self.switch_variable} with {len(self.case_addrs)} cases" diff --git a/angr/analyses/decompiler/utils.py b/angr/analyses/decompiler/utils.py index add71d907..dc2fda2a3 100644 --- a/angr/analyses/decompiler/utils.py +++ b/angr/analyses/decompiler/utils.py @@ -540,7 +540,8 @@ def _merge_ail_nodes(graph, node_a: ailment.Block, node_b: ailment.Block) -> ail if new_node.statements and isinstance(new_node.statements[-1], ailment.Stmt.Jump): new_node.statements = new_node.statements[:-1] new_node.statements += old_node.statements - new_node.original_size += old_node.original_size + if new_node.original_size is not None and old_node.original_size is not None: + new_node.original_size += old_node.original_size graph.remove_node(node_a) graph.remove_node(node_b) @@ -864,7 +865,11 @@ def structured_node_is_simple_return_strict(node: BaseNode | SequenceNode | Mult def is_statement_terminating(stmt: ailment.statement.Statement, functions) -> bool: if isinstance(stmt, ailment.Stmt.Return): return True - if isinstance(stmt, ailment.Stmt.SideEffectStatement) and isinstance(stmt.expr.target, ailment.Expr.Const): + if ( + isinstance(stmt, ailment.Stmt.SideEffectStatement) + and isinstance(stmt.expr, ailment.Expr.Call) + and isinstance(stmt.expr.target, ailment.Expr.Const) + ): # is it calling a non-returning function? target_func_addr = stmt.expr.target.value try: @@ -883,6 +888,11 @@ class _PeepholeExprsWalker(ailment.AILBlockRewriter): def __init__(self, *args, expr_opts: list[PeepholeOptimizationExprBase], **kwargs): self.expr_opts = expr_opts self.any_update = False + # IDs of the statements this pass left untouched and whose expr optimizers all reported fixpoint_reached. + # Only valid until the next ``reset()``, and only for statements of the block just walked. + self.fixpoint_stmts: set[int] = set() + self._stmt_touched = False + self._stmt_fixpoint = True self.expr_opts_by_kind: dict[str, list[PeepholeOptimizationExprBase]] = {} for expr_opt in expr_opts: @@ -897,6 +907,22 @@ class _PeepholeExprsWalker(ailment.AILBlockRewriter): def reset(self) -> None: self.any_update = False + self.fixpoint_stmts.clear() + + def _handle_stmt(self, stmt_idx: int, stmt: ailment.Stmt.Statement, block) -> ailment.Stmt.Statement: + if stmt.peephole_optimized is True: + # a previous peephole pass ran this statement (and its expressions) to fixpoint + return stmt + # MultiStatementExpression re-enters _handle_stmt() mid-statement; losing the enclosing False here would + # wrongly mark the enclosing statement as being at its fixpoint. + outer_fixpoint = self._stmt_fixpoint + self._stmt_touched = False + self._stmt_fixpoint = True + new_stmt = super()._handle_stmt(stmt_idx, stmt, block) + if not self._stmt_touched and new_stmt is stmt and self._stmt_fixpoint: + self.fixpoint_stmts.add(id(stmt)) + self._stmt_fixpoint &= outer_fixpoint + return new_stmt def _handle_expr( self, expr_idx: int, expr: ailment.Expr.Expression, stmt_idx: int, stmt: ailment.Stmt.Statement | None, block @@ -912,7 +938,10 @@ class _PeepholeExprsWalker(ailment.AILBlockRewriter): if not expr_opts: break for expr_opt in expr_opts: + # context-insensitive optimizers never touch the flag, so default it to True here. + expr_opt.fixpoint_reached = True r = expr_opt.optimize(expr, stmt_idx=stmt_idx, block=block) + self._stmt_fixpoint &= expr_opt.fixpoint_reached if r is not None and r is not expr: if expr.bits != r.bits: # A few optimizers don't preserve bits; @@ -931,6 +960,7 @@ class _PeepholeExprsWalker(ailment.AILBlockRewriter): if expr is not old_expr: self.any_update = True + self._stmt_touched = True return expr @@ -1015,7 +1045,11 @@ def build_stmt_opts_by_kind(stmt_opts): return by_kind -def peephole_optimize_stmts(block, stmt_opts, *, stmt_opts_by_kind=None): +def peephole_optimize_stmts(block, stmt_opts, *, stmt_opts_by_kind=None, fixpoint_exprs=None): + """ + :param fixpoint_exprs: IDs of the statements the preceding expression pass left untouched *and* whose + expression optimizers all reported ``fixpoint_reached``. + """ any_update = False statements = [] if stmt_opts_by_kind is None: @@ -1027,17 +1061,23 @@ def peephole_optimize_stmts(block, stmt_opts, *, stmt_opts_by_kind=None): while stmt_idx < len(block.statements): stmt = block.statements[stmt_idx] old_stmt = stmt + if stmt.peephole_optimized is True: + # a previous peephole pass ran this statement to fixpoint + statements.append(stmt) + stmt_idx += 1 + continue + stmt_fixpoint = True redo = True while redo: redo = False - kind = getattr(stmt, "pykind", None) - if kind is None: - kind = type(stmt).__name__ - opts_for_kind = stmt_opts_by_kind.get(kind) + opts_for_kind = stmt_opts_by_kind.get(stmt.pykind) if not opts_for_kind: break for opt in opts_for_kind: + # context-insensitive optimizers never touch the flag, so default it to True here. + opt.fixpoint_reached = True r = opt.optimize(stmt, stmt_idx=stmt_idx, block=block) + stmt_fixpoint &= opt.fixpoint_reached if r is not None and r != stmt: stmt = r if r == (): @@ -1053,6 +1093,10 @@ def peephole_optimize_stmts(block, stmt_opts, *, stmt_opts_by_kind=None): any_update = True else: statements.append(old_stmt) + if stmt_fixpoint and fixpoint_exprs is not None and id(old_stmt) in fixpoint_exprs: + # nothing matched and nothing may start matching if the block changes: at the peephole + # fixpoint until this statement is rebuilt. + old_stmt.peephole_optimized = True stmt_idx += 1 return statements, any_update diff --git a/angr/analyses/outliner/outliner.py b/angr/analyses/outliner/outliner.py index e3fe939ee..f24c1250b 100644 --- a/angr/analyses/outliner/outliner.py +++ b/angr/analyses/outliner/outliner.py @@ -6,11 +6,11 @@ from collections import defaultdict import networkx from angr.ailment import Address, Block -from angr.ailment.expression import BinaryOp, Call, Const, VirtualVariable, VirtualVariableCategory +from angr.ailment.expression import BinaryOp, Call, Const, Phi, VirtualVariable, VirtualVariableCategory from angr.ailment.statement import Assignment, ConditionalJump, Jump, Return from angr.analyses.analysis import AnalysesHub, Analysis from angr.analyses.s_liveness import SLivenessAnalysis -from angr.analyses.s_reaching_definitions import SReachingDefinitionsAnalysis +from angr.analyses.s_reaching_definitions import SReachingDefinitions from angr.knowledge_plugins.functions import Function from angr.utils.graph import Dominators, compute_dominance_frontier, subgraph_between_nodes from angr.utils.ssa import is_phi_assignment @@ -84,7 +84,7 @@ class Outliner(Analysis): Remove all phi assignments whose all source variables are undefined in the graph. """ - srda = self.project.analyses[SReachingDefinitionsAnalysis].prep()(func, func_graph=g).model + srda = SReachingDefinitions(self.project, func, func_graph=g).model to_kill = defaultdict(set) for phi_var_id, src_var_ids in srda.phivarid_to_varids.items(): @@ -104,7 +104,7 @@ class Outliner(Analysis): Recover the interface from a function AIL graph. """ - srda = self.project.analyses[SReachingDefinitionsAnalysis].prep()(func, func_graph=g).model + srda = SReachingDefinitions(self.project, func, func_graph=g).model blocks: dict[tuple[int, int | None], Block] = {(node.addr, node.idx): node for node in g} @@ -178,17 +178,13 @@ class Outliner(Analysis): None, vvar_id, self.project.arch.bits, VirtualVariableCategory.REGISTER, oident=self.project.arch.ret_offset ) call_stmt = Assignment(None, switch_vvar, call_expr, ins_addr=src_node.addr) - new_src_node = Block(src_node.addr, src_node.original_size, [call_stmt], idx=src_node.idx) + new_src_node = Block(src_node.addr, src_node.original_size, statements=[call_stmt], idx=src_node.idx) for pred, _ in in_edges: self.parent_graph.add_edge(pred, new_src_node) # build the return statement if needed if self.frontier_vars: - srda = ( - self.project.analyses[SReachingDefinitionsAnalysis] - .prep()(self.parent_func, func_graph=self.parent_graph) - .model - ) + srda = SReachingDefinitions(self.project, self.parent_func, func_graph=self.parent_graph).model ret_exprs = [srda.varid_to_vvar[idx] for idx in self.frontier_vars] else: ret_exprs = [] @@ -216,13 +212,14 @@ class Outliner(Analysis): ret_node.statements.append(ret_stmt) else: # we will have to create a new node and act as the successor of ret_node - new_ret_node = Block(self._next_block_addr(), 0, [ret_stmt]) + new_ret_node = Block(self._next_block_addr(), 0, statements=[ret_stmt]) if ret_node.statements and isinstance(ret_node.statements[-1], ConditionalJump): cond_jump = ret_node.statements[-1] if isinstance(cond_jump.true_target, Const) and cond_jump.true_target.value == frontier_node.addr: _, cond_jump = cond_jump.replace( cond_jump.true_target, Const(None, new_ret_node.addr, self.project.arch.bits) ) + assert isinstance(cond_jump, ConditionalJump) if isinstance(cond_jump.false_target, Const) and cond_jump.false_target.value == frontier_node.addr: _, cond_jump = cond_jump.replace( cond_jump.false_target, Const(None, new_ret_node.addr, self.project.arch.bits) @@ -252,7 +249,7 @@ class Outliner(Analysis): false_target_idx=next_dispatcher_node_addr[1], ins_addr=dispatcher_node_addr[0], ) - dispatcher_node = Block(dispatcher_node_addr[0], 0, [stmt], dispatcher_node_addr[1]) + dispatcher_node = Block(dispatcher_node_addr[0], 0, statements=[stmt], idx=dispatcher_node_addr[1]) self.parent_graph.add_edge(parent, dispatcher_node) self.parent_graph.add_edge(dispatcher_node, node_dict[jump_target]) @@ -279,6 +276,7 @@ class Outliner(Analysis): src_addrs = [(src.addr, src.idx) for src in srcs] for stmt in block.statements: if is_phi_assignment(stmt): + assert isinstance(stmt, Assignment) and isinstance(stmt.src, Phi) all_stmt_srcs = [src for src, _ in stmt.src.src_and_vvars] new_addrs = set(src_addrs) - set(all_stmt_srcs) old_addrs = set(all_stmt_srcs) - set(src_addrs) diff --git a/angr/analyses/s_propagator.py b/angr/analyses/s_propagator.py index c83a12626..eeb54a9e3 100644 --- a/angr/analyses/s_propagator.py +++ b/angr/analyses/s_propagator.py @@ -4,6 +4,7 @@ import contextlib import threading from collections import defaultdict from collections.abc import Mapping +from typing import TYPE_CHECKING import networkx @@ -45,6 +46,9 @@ from angr.utils.ssa import ( is_vvar_propagatable, ) +if TYPE_CHECKING: + from angr.project import Project + # The cache of reusable AILBlockWalker instances, which are used by is_const_*(). The cache dict itself is # owned by the corresponding Decompiler instance (so it is released when the Decompiler is gone). This thread-local # points at the active Decompiler's walker cache for the duration of its run, which allows all SPropagator instances @@ -89,13 +93,16 @@ class SPropagatorModel: self.dead_vvar_ids: set[int] = set() -class SPropagatorAnalysis(Analysis): +class SPropagator: """ Constant and expression propagation that only supports SSA AIL graphs. + + Deliberately not an :class:`Analysis` for better speed. """ def __init__( # pylint: disable=too-many-positional-arguments self, + project: Project, subject: Block | Function, *, ail_manager: Manager, @@ -106,6 +113,9 @@ class SPropagatorAnalysis(Analysis): func_addr: int | None = None, stack_arg_offsets: set[int] | None = None, ): + self.project = project + self.kb = project.kb + if isinstance(subject, Block): self.block = subject self.func = None @@ -656,4 +666,13 @@ class SPropagatorAnalysis(Analysis): return result +class SPropagatorAnalysis(Analysis, SPropagator): + """ + A wrapper around SPropagator to make it an Analysis. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + register_analysis(SPropagatorAnalysis, "SPropagator") diff --git a/angr/analyses/s_reaching_definitions/__init__.py b/angr/analyses/s_reaching_definitions/__init__.py index 8cef939a2..4f2fce0a7 100644 --- a/angr/analyses/s_reaching_definitions/__init__.py +++ b/angr/analyses/s_reaching_definitions/__init__.py @@ -2,11 +2,12 @@ from __future__ import annotations from .s_rda_model import SRDAModel, populate_model from .s_rda_view import SRDAView -from .s_reaching_definitions import SReachingDefinitionsAnalysis +from .s_reaching_definitions import SReachingDefinitions, SReachingDefinitionsAnalysis __all__ = ( "SRDAModel", "SRDAView", + "SReachingDefinitions", "SReachingDefinitionsAnalysis", "populate_model", ) diff --git a/angr/analyses/s_reaching_definitions/s_rda_model.py b/angr/analyses/s_reaching_definitions/s_rda_model.py index cc7130d80..87da69937 100644 --- a/angr/analyses/s_reaching_definitions/s_rda_model.py +++ b/angr/analyses/s_reaching_definitions/s_rda_model.py @@ -291,7 +291,7 @@ def populate_model( ) -> None: """Populate the scan-derived part of an SRDAModel (vvar/tmp definitions and uses, phi bookkeeping) with a linear scan over ``blocks``. An SRDAModel is never serialized; it is always rebuilt from an AIL graph through this - function (via :class:`SReachingDefinitionsAnalysis` or directly).""" + function (via :class:`SReachingDefinitions` or directly).""" phi_vvars: dict[int, set[int | None]] = {} # find all vvar definitions diff --git a/angr/analyses/s_reaching_definitions/s_reaching_definitions.py b/angr/analyses/s_reaching_definitions/s_reaching_definitions.py index 5db328357..065e7fbc6 100644 --- a/angr/analyses/s_reaching_definitions/s_reaching_definitions.py +++ b/angr/analyses/s_reaching_definitions/s_reaching_definitions.py @@ -1,6 +1,8 @@ # pylint:disable=too-many-boolean-expressions from __future__ import annotations +from typing import TYPE_CHECKING + import networkx from angr.ailment.block import Block @@ -15,14 +17,22 @@ from angr.knowledge_plugins.key_definitions.constants import ObservationPointTyp from .s_rda_model import SRDAModel, populate_model from .s_rda_view import SRDAView +if TYPE_CHECKING: + from angr.project import Project -class SReachingDefinitionsAnalysis(Analysis): + +class SReachingDefinitions: """ Constant and expression propagation that only supports SSA AIL graphs. + + Deliberately not an :class:`Analysis`: it is instantiated hundreds of times per decompilation (often once per + block), so it skips the analysis-factory ceremony. Instantiate it directly with the project as the first + argument; exceptions always propagate. """ def __init__( # pylint: disable=too-many-positional-arguments self, + project: Project, subject, func_addr: int | None = None, func_graph: networkx.DiGraph[Block] | None = None, @@ -31,6 +41,9 @@ class SReachingDefinitionsAnalysis(Analysis): track_tmps: bool = False, variable_map=None, ): + self.project = project + self.kb = project.kb + if isinstance(subject, Block): self.block = subject self.func = None @@ -204,4 +217,32 @@ class SReachingDefinitionsAnalysis(Analysis): self.model.add_vvar_use(vvarid, None, codeloc) +class SReachingDefinitionsAnalysis(Analysis, SReachingDefinitions): + """ + A wrapper around SReachingDefinitions to make it usable as an :class:`Analysis` and registered in the + analysis hub. + """ + + def __init__( # pylint: disable=too-many-arguments,too-many-locals + self, + subject, + func_addr: int | None = None, + func_graph: networkx.DiGraph[Block] | None = None, + func_args: set[VirtualVariable] | None = None, + use_callee_saved_regs_at_return: bool = False, + track_tmps: bool = False, + variable_map=None, + ): + super().__init__( + self.project, + subject, + func_addr=func_addr, + func_graph=func_graph, + func_args=func_args, + use_callee_saved_regs_at_return=use_callee_saved_regs_at_return, + track_tmps=track_tmps, + variable_map=variable_map, + ) + + register_analysis(SReachingDefinitionsAnalysis, "SReachingDefinitions") diff --git a/angr/rust/mixins/srda_mixin.py b/angr/rust/mixins/srda_mixin.py index 581060b45..951bdbdfe 100644 --- a/angr/rust/mixins/srda_mixin.py +++ b/angr/rust/mixins/srda_mixin.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING from angr.ailment import Assignment, Expression from angr.ailment.expression import Call, FunctionLikeMacro, Phi, VirtualVariable -from angr.analyses.s_reaching_definitions import SRDAView +from angr.analyses.s_reaching_definitions import SRDAView, SReachingDefinitions from angr.knowledge_plugins.key_definitions.constants import OP_BEFORE from angr.rust.sim_type import RustSimType, RustSimTypeFunction @@ -17,7 +17,7 @@ class SRDAMixin: def __init__(self, subject, graph, project, variable_map: VariableMap): self._graph = graph - self.srda = project.analyses.SReachingDefinitions(subject=subject, func_graph=graph, variable_map=variable_map) + self.srda = SReachingDefinitions(project, subject=subject, func_graph=graph, variable_map=variable_map) self.srda_view = SRDAView(self.srda.model) self._gtv_cache = {} # varid -> terminal VirtualVariable diff --git a/angr/rustylib/ailment.pyi b/angr/rustylib/ailment.pyi index 8c45e5e69..2cf726262 100644 --- a/angr/rustylib/ailment.pyi +++ b/angr/rustylib/ailment.pyi @@ -391,6 +391,9 @@ class Statement: @property def depth(self) -> int: """Assignment/WeakAssignment/Store/CJump/SES/Return/CAS/Dirty depth""" + peephole_optimized: bool + """True once a peephole-optimization pass has run this statement to fixpoint, letting later passes skip it. + Runtime-only: never serialized, ignored by ``__eq__``/``__hash__``/``likes()``, and reset on clone.""" def clear_hash(self) -> None: ... # Utility query available on any statement (true only for Assignments # whose source is a ``Phi``); kept on the base. @@ -517,6 +520,12 @@ class Block: def to_bytes(self) -> bytes: ... @classmethod def from_bytes(cls, data: bytes) -> Block: ... + def likes(self, other: Block) -> bool: ... + """ + Check structural equality of two Blocks ignoring `idx` and `tags` on statements and expressions. + """ + + def deep_copy(self, manager: Manager) -> Block: ... # --------------------------------------------------------------------------- # Manager + VEX -> AIL converter diff --git a/native/angr/src/ailment/ail_stmt.rs b/native/angr/src/ailment/ail_stmt.rs index fd4654fa7..193050401 100644 --- a/native/angr/src/ailment/ail_stmt.rs +++ b/native/angr/src/ailment/ail_stmt.rs @@ -37,6 +37,11 @@ pub struct StmtHeader { pub idx: i64, pub tags: Tags, pub cached_hash: CachedHash, + /// Runtime-only marker: true once a peephole-optimization pass has run this statement to + /// fixpoint, letting later passes skip it. Never serialized, ignored by + /// `Hash`/`PartialEq`/`likes`, and conservatively reset on clone (shared references -- the + /// common case, since block copies share statement objects -- keep it). + pub peephole_optimized: std::sync::atomic::AtomicBool, } impl Clone for StmtHeader { @@ -45,6 +50,7 @@ impl Clone for StmtHeader { idx: self.idx, tags: self.tags.clone(), cached_hash: CachedHash::new(), + peephole_optimized: std::sync::atomic::AtomicBool::new(false), } } } @@ -55,6 +61,7 @@ impl StmtHeader { idx, tags, cached_hash: CachedHash::new(), + peephole_optimized: std::sync::atomic::AtomicBool::new(false), } } } @@ -1223,6 +1230,24 @@ impl Statement { self.stmt.header.cached_hash.clear(); } + /// True once a peephole-optimization pass has run this statement to fixpoint, letting later + /// passes skip it. Runtime-only: never serialized, ignored by ``__eq__``/``__hash__``/ + /// ``likes()``, and reset when the statement is cloned. + #[getter] + fn peephole_optimized(&self) -> bool { + self.stmt + .header + .peephole_optimized + .load(std::sync::atomic::Ordering::Relaxed) + } + #[setter] + fn set_peephole_optimized(&self, value: bool) { + self.stmt + .header + .peephole_optimized + .store(value, std::sync::atomic::Ordering::Relaxed); + } + // --- Per-variant accessors ---------------------------------------- /// True iff this is an SSA phi assignment: an ``Assignment`` whose diff --git a/native/angr/src/ailment/block.rs b/native/angr/src/ailment/block.rs index 78ff05c8a..7b1e95ee0 100644 --- a/native/angr/src/ailment/block.rs +++ b/native/angr/src/ailment/block.rs @@ -255,34 +255,24 @@ impl Block { s.statements.bind(py).as_any().eq(o.statements.bind(py)) } - fn likes(slf: Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult { + fn likes(slf: Bound<'_, Self>, other: &Bound<'_, Self>) -> PyResult { let py = slf.py(); - if !py.get_type::().is(other.get_type()) { - return Ok(false); - } let s = slf.borrow(); - let o = other.cast::()?.borrow(); + let o = other.borrow(); let sa = s.statements.bind(py); let ob = o.statements.bind(py); if sa.len() != ob.len() { return Ok(false); } for (xa, xb) in sa.iter().zip(ob.iter()) { - // Fast path: both elements are AIL Statements -- compare via the - // native `AilStatement::likes` instead of dispatching through the - // Python `likes` method. Falls back to Python dispatch for any - // non-Statement element so behavior stays identical. - match (xa.cast::(), xb.cast::()) { - (Ok(a), Ok(b)) => { - if !a.borrow().stmt.likes(&b.borrow().stmt) { - return Ok(false); - } - } - _ => { - if !xa.call_method1("likes", (&xb,))?.is_truthy()? { - return Ok(false); - } - } + // `Block.statements` only holds AIL Statements; compare via the native `AilStatement::likes` directly. + if !xa + .cast::()? + .borrow() + .stmt + .likes(&xb.cast::()?.borrow().stmt) + { + return Ok(false); } } Ok(true) diff --git a/tests/analyses/decompiler/test_decompiler.py b/tests/analyses/decompiler/test_decompiler.py index 95b650db2..46c1882c8 100755 --- a/tests/analyses/decompiler/test_decompiler.py +++ b/tests/analyses/decompiler/test_decompiler.py @@ -20,6 +20,7 @@ from angr.analyses import ( CFGFast, CompleteCallingConventionsAnalysis, Decompiler, + SReachingDefinitions, VariableRecoveryFast, ) from angr.analyses.complete_calling_conventions import CallingConventionAnalysisMode @@ -4180,7 +4181,7 @@ class TestDecompiler(unittest.TestCase): d = proj.analyses[Decompiler].prep(fail_fast=True)(f, cfg=cfg.model, options=decompiler_options) assert d.codegen is not None and d.clinic is not None - rd = proj.analyses.SReachingDefinitions(subject=f, func_graph=d.ail_graph, func_args=set()).model + rd = SReachingDefinitions(proj, subject=f, func_graph=d.ail_graph, func_args=set()).model used_but_undefined_stack_vars = [ str(rd.varid_to_vvar[vid]) for vid, loc in rd.all_vvar_definitions.items() diff --git a/tests/analyses/decompiler/test_peephole_optimizations.py b/tests/analyses/decompiler/test_peephole_optimizations.py index a57a5ea89..c3108a5fb 100755 --- a/tests/analyses/decompiler/test_peephole_optimizations.py +++ b/tests/analyses/decompiler/test_peephole_optimizations.py @@ -11,17 +11,33 @@ import archinfo import angr from angr import ailment -from angr.ailment.expression import BinaryOp, Call, Const, Convert, Extract, Insert, Register +from angr.ailment.block import Block +from angr.ailment.expression import ( + ITE, + BinaryOp, + Call, + Const, + Convert, + Extract, + Insert, + Register, + VirtualVariable, + VirtualVariableCategory, +) from angr.ailment.manager import Manager +from angr.ailment.statement import Assignment +from angr.analyses.decompiler.block_simplifier import BlockSimplifier from angr.analyses.decompiler.peephole_optimizations import ( EXPR_OPTS, Bswap, CmpMaskedShift, CmpSubConst, + ConcatSimplifier, ConstantDereferences, EagerEvaluation, OptimizedDivisionSimplifier, RemoveRedundantShifts, + SarToSignedDiv, SimplifyBitwiseInserts, ) from angr.analyses.decompiler.utils import peephole_optimize_expr @@ -506,5 +522,109 @@ class TestPeepholeOptimizations(unittest.TestCase): assert isinstance(out.operands[1], Const) and out.operands[1].value == 44570 and out.operands[1].bits == 64 +class TestPeepholeBlockContextFixpoint(unittest.TestCase): + """ + Statements that BlockSimplifier judged to be at peephole fixpoint get flagged (``Statement.peephole_optimized``) + and are skipped by later peephole passes. Expression optimizers that look at *other* statements of the block + (via ``PeepholeOptimizationExprBase.find_definition``) may start matching once their neighborhood changes, so + flagging a statement they did not match is only correct if they are re-run afterwards. + """ + + @staticmethod + def _vvar(varid: int, bits: int) -> VirtualVariable: + return VirtualVariable(varid, varid, bits, VirtualVariableCategory.REGISTER, oident=varid * 8) + + @staticmethod + def _unfolded_31(manager: Manager) -> BinaryOp: + # 0x10 + 0xf, which EagerEvaluation folds to 0x1f. The expression walker writes its rewrites back to the + # block only when the whole block has been walked, so the *consumer* statement still sees the unfolded + # definition during the first pass and only sees the folded one on the second pass. + return BinaryOp( + manager.next_atom(), + "Add", + [Const(manager.next_atom(), 0x10, 8), Const(manager.next_atom(), 0xF, 8)], + False, + ) + + def test_concat_simplifier_reruns_after_its_definition_changes(self): + # vvar_10 = vvar_5 >>s 0x1f + # vvar_11 = vvar_10 CONCAT vvar_5 => vvar_11 = Conv(32->s64, vvar_5) + manager = Manager() + v5 = self._vvar(5, 32) + v10 = self._vvar(10, 32) + v11 = self._vvar(11, 64) + stmt0 = Assignment( + manager.next_atom(), + v10, + BinaryOp(manager.next_atom(), "Sar", [v5, self._unfolded_31(manager)], False, bits=32), + ins_addr=0x400000, + ) + stmt1 = Assignment( + manager.next_atom(), + v11, + BinaryOp(manager.next_atom(), "Concat", [v10, v5], False, bits=64), + ins_addr=0x400004, + ) + block = Block(0x400000, 8, statements=[stmt0, stmt1]) + + proj = angr.load_shellcode(b"\x90", "AMD64") + simp = BlockSimplifier(proj, block, manager, peephole_optimizations=[ConcatSimplifier, EagerEvaluation]) + result = simp.result_block + assert result is not None + + assert isinstance(result.statements[1], Assignment) + src = result.statements[1].src + assert isinstance(src, Convert), f"Concat was not simplified:\n{result.dbg_repr()}" + assert src.from_bits == 32 + assert src.to_bits == 64 + assert src.is_signed is True + assert src.operand.likes(v5) + + def test_sar_to_signed_div_reruns_after_its_definition_changes(self): + # vvar_10 = ((vvar_5 >> 0x1f) == 1) ? (vvar_5 + 3) : vvar_5 + # vvar_11 = vvar_10 >>s 2 => vvar_11 = vvar_5 /s 4 + manager = Manager() + v5 = self._vvar(5, 32) + v10 = self._vvar(10, 32) + v11 = self._vvar(11, 32) + cond = BinaryOp( + manager.next_atom(), + "CmpEQ", + [ + BinaryOp(manager.next_atom(), "Shr", [v5, self._unfolded_31(manager)], False, bits=32), + Const(manager.next_atom(), 1, 32), + ], + False, + bits=1, + ) + ite = ITE( + manager.next_atom(), + cond, + v5, # iffalse + BinaryOp(manager.next_atom(), "Add", [v5, Const(manager.next_atom(), 3, 32)], False, bits=32), # iftrue + bits=32, + ) + stmt0 = Assignment(manager.next_atom(), v10, ite, ins_addr=0x400000) + stmt1 = Assignment( + manager.next_atom(), + v11, + BinaryOp(manager.next_atom(), "Sar", [v10, Const(manager.next_atom(), 2, 8)], False, bits=32), + ins_addr=0x400004, + ) + block = Block(0x400000, 8, statements=[stmt0, stmt1]) + + proj = angr.load_shellcode(b"\x90", "AMD64") + simp = BlockSimplifier(proj, block, manager, peephole_optimizations=[SarToSignedDiv, EagerEvaluation]) + result = simp.result_block + assert result is not None + + assert isinstance(result.statements[1], Assignment) + src = result.statements[1].src + assert isinstance(src, BinaryOp) and src.op == "Div", f"Sar was not rewritten:\n{result.dbg_repr()}" + assert src.signed is True + assert src.operands[0].likes(v5) + assert isinstance(src.operands[1], Const) and src.operands[1].value == 4 + + if __name__ == "__main__": unittest.main() diff --git a/tests/analyses/test_callsite_maker.py b/tests/analyses/test_callsite_maker.py index 8e4fee083..70db797b4 100755 --- a/tests/analyses/test_callsite_maker.py +++ b/tests/analyses/test_callsite_maker.py @@ -8,6 +8,7 @@ import unittest import angr import angr.ailment as ailment +from angr.analyses.decompiler.block_simplifier import BlockSimplifier from tests.common import bin_location test_location = os.path.join(bin_location, "tests") @@ -45,12 +46,12 @@ class TestCallsiteMaker(unittest.TestCase): for block in sorted(main_func.blocks, key=lambda x: x.addr): print(block.vex.pp()) ail_block = ailment.IRSBConverter.convert(block.vex, manager) - simp = project.analyses.AILBlockSimplifier(ail_block, manager, main_func.addr) + simp = BlockSimplifier(project, ail_block, manager, main_func.addr) csm = project.analyses.AILCallSiteMaker(simp.result_block, ail_manager=manager) if csm.result_block: ail_block = csm.result_block - simp = project.analyses.AILBlockSimplifier(ail_block, manager, main_func.addr) + simp = BlockSimplifier(project, ail_block, manager, main_func.addr) print(simp.result_block) From 55f059982b06af39cb7ea6ce8cd61d1c2ebe15d5 Mon Sep 17 00:00:00 2001 From: Fish Date: Mon, 27 Jul 2026 17:35:52 -0700 Subject: [PATCH 065/122] UltraPage: Make the symbolic map an actual bitmap. (#6714) --- angr/engines/icicle.py | 5 +- angr/engines/unicorn.py | 6 +- angr/state_plugins/unicorn_engine.py | 8 +- angr/storage/memory_mixins/memory_mixin.py | 9 + .../paged_memory/paged_memory_mixin.py | 42 ++- .../paged_memory/pages/symbolic_bitmap.py | 270 ++++++++++++++++++ .../paged_memory/pages/ultra_page.py | 158 +++++----- native/unicornlib/sim_unicorn.cpp | 113 ++++---- native/unicornlib/sim_unicorn.hpp | 59 +++- tests/storage/test_memory.py | 99 +++++-- 10 files changed, 587 insertions(+), 182 deletions(-) create mode 100644 angr/storage/memory_mixins/paged_memory/pages/symbolic_bitmap.py diff --git a/angr/engines/icicle.py b/angr/engines/icicle.py index 6b189d2ce..48678c7e8 100644 --- a/angr/engines/icicle.py +++ b/angr/engines/icicle.py @@ -177,8 +177,9 @@ class IcicleEngine(SuccessorsEngine): """ page_size = state.memory.page_size addr = page_num * page_size - memory, bitmap = state.memory.concrete_load(addr, page_size, with_bitmap=True) - if any(bitmap): + # concrete_load stops at the first symbolic byte, so a short result means the page is not fully concrete + memory = state.memory.concrete_load(addr, page_size) + if len(memory) != page_size: memory = state.solver.eval(state.memory.load(addr, page_size), cast_to=bytes) emu.mem_write(addr, memory) diff --git a/angr/engines/unicorn.py b/angr/engines/unicorn.py index 53f195b88..6d34ef17c 100644 --- a/angr/engines/unicorn.py +++ b/angr/engines/unicorn.py @@ -353,7 +353,7 @@ class SimEngineUnicorn(SuccessorsEngine): for offset in range(mem_read_len): page_num, page_off = state.memory._divide_addr(mem_read_addr + offset) page_obj = state.memory._get_page(page_num, writing=False) - saved_taints.append(page_obj.symbolic_bitmap[page_off]) + saved_taints.append(page_obj.symbolic_bitmap.get(page_off)) restore_taints = False if saved_taints != taint_map: @@ -363,7 +363,7 @@ class SimEngineUnicorn(SuccessorsEngine): if expected_taint != -1: page_num, page_off = state.memory._divide_addr(mem_read_addr + offset) page_obj = state.memory._get_page(page_num, writing=False) - page_obj.symbolic_bitmap[page_off] = expected_taint + page_obj.symbolic_bitmap.set(page_off, expected_taint) curr_value = state.memory.load( mem_read_addr, mem_read_len, endness=state.arch.memory_endness, inspect=False, disable_actions=True @@ -372,7 +372,7 @@ class SimEngineUnicorn(SuccessorsEngine): for offset, saved_taint in enumerate(saved_taints): page_num, page_off = state.memory._divide_addr(mem_read_addr + offset) page_obj = state.memory._get_page(page_num, writing=False) - page_obj.symbolic_bitmap[page_off] = saved_taint + page_obj.symbolic_bitmap.set(page_off, saved_taint) if taint_map.count(0) != 0: # Update concrete bytes using values reported by native interface diff --git a/angr/state_plugins/unicorn_engine.py b/angr/state_plugins/unicorn_engine.py index b9d455d65..b345956b4 100644 --- a/angr/state_plugins/unicorn_engine.py +++ b/angr/state_plugins/unicorn_engine.py @@ -1107,8 +1107,9 @@ class Unicorn(SimStatePlugin): else: perm = perm.args[0] - # this should return two memoryviews - # if they are writable they are direct references to the state backing store and can be mapped directly + # this should return two memoryviews: the page data and its symbolic-ness bitmap, one bit per byte + # if they are writable they are direct references to the state backing store and can be mapped directly -- + # native unicorn then writes taint straight back through the bitmap data, bitmap = self.state.memory.concrete_load(addr, 0x1000, with_bitmap=True, writing=(perm & 2) != 0) if not bitmap: @@ -1311,7 +1312,8 @@ class Unicorn(SimStatePlugin): # activate gdt page, which was written/mapped during set_regs if self.gdt is not None: - _UC_NATIVE.activate_page(self._uc_state, self.gdt.addr, bytes(0x1000), None) + # the taint bitmap holds one bit per page byte; the GDT page is entirely concrete + _UC_NATIVE.activate_page(self._uc_state, self.gdt.addr, bytes(0x1000 // 8), None) # Pass all concrete fd bytes to native interface so that it can handle relevant syscalls if fd_bytes is not None: diff --git a/angr/storage/memory_mixins/memory_mixin.py b/angr/storage/memory_mixins/memory_mixin.py index 84492bd0d..3bf8fe5d1 100644 --- a/angr/storage/memory_mixins/memory_mixin.py +++ b/angr/storage/memory_mixins/memory_mixin.py @@ -104,6 +104,15 @@ class MemoryMixin[InData, OutData, Addr](SimStatePlugin): """ raise NotImplementedError + def concrete_run_length(self, addr, size, **kwargs) -> int: + """ + Return the number of concrete bytes starting at ``addr``, capped at ``size``. + """ + _, bitmap = self.concrete_load(addr, size, with_bitmap=True, **kwargs) + # the bitmap is packed: one bit per byte, least-significant bit first + n = min(size, len(bitmap) * 8) + return next((i for i in range(n) if bitmap[i >> 3] >> (i & 7) & 1), n) + def erase(self, addr: Addr, size: int | None = None, **kwargs) -> None: """ Set [addr:addr+size) to uninitialized. In many cases this will be faster than overwriting those locations with diff --git a/angr/storage/memory_mixins/paged_memory/paged_memory_mixin.py b/angr/storage/memory_mixins/paged_memory/paged_memory_mixin.py index 070f57977..f0d477ff6 100644 --- a/angr/storage/memory_mixins/paged_memory/paged_memory_mixin.py +++ b/angr/storage/memory_mixins/paged_memory/paged_memory_mixin.py @@ -418,14 +418,19 @@ class PagedMemoryMixin[PageType: PageBase]( def _load_to_memoryview(self, addr, size, with_bitmap: Literal[False]) -> memoryview: ... def _load_to_memoryview(self, addr, size, with_bitmap): + bitmap_size = (size + 7) // 8 result = self.load(addr, size, endness="Iend_BE") if result.op == "BVV": if with_bitmap: - return memoryview(result.args[0].to_bytes(size, "big")), memoryview(bytes(size)) + return memoryview(result.args[0].to_bytes(size, "big")), memoryview(bytes(bitmap_size)) return memoryview(result.args[0].to_bytes(size, "big")) if result.op == "Concat": bytes_out = bytearray(size) - bitmap_out = bytearray(size) + bitmap_out = bytearray(bitmap_size) + + def mark_symbolic(i): + bitmap_out[i >> 3] |= 1 << (i & 7) + bit_idx = 0 byte_width = self.state.arch.byte_width for element in result.args: @@ -438,7 +443,7 @@ class PagedMemoryMixin[PageType: PageBase]( bit_idx += len(element) if not with_bitmap: return memoryview(bytes(bytes_out))[:byte_idx] - bitmap_out[byte_idx] = 1 + mark_symbolic(byte_idx) continue # if the current element has at least byte_width bits, the top `hi_chop` bits should be removed @@ -453,11 +458,11 @@ class PagedMemoryMixin[PageType: PageBase]( bit_idx += len(element) if not with_bitmap: return memoryview(bytes(bytes_out))[:byte_idx] - bitmap_out[byte_idx] = 1 + mark_symbolic(byte_idx) continue if hi_chop: - bitmap_out[byte_idx] = 1 + mark_symbolic(byte_idx) byte_idx += 1 if element.op == "BVV": @@ -473,18 +478,22 @@ class PagedMemoryMixin[PageType: PageBase]( if not with_bitmap: return memoryview(bytes(bytes_out))[:byte_idx] for byte_i in range(byte_idx, byte_idx + byte_size): - bitmap_out[byte_i] = 1 + mark_symbolic(byte_i) bit_idx += len(element) if bit_idx % byte_width != 0: if not with_bitmap: return memoryview(bytes(bytes_out))[: bit_idx // byte_width] - bitmap_out[bit_idx // byte_width] = 1 + mark_symbolic(bit_idx // byte_width) if with_bitmap: return memoryview(bytes(bytes_out)), memoryview(bytes(bitmap_out)) return memoryview(bytes(bytes_out)) if with_bitmap: - return memoryview(bytes(size)), memoryview(b"\x01" * size) + # every byte is symbolic; leave the bits past the end of the region clear + bitmap_out = bytearray(b"\xff" * bitmap_size) + if size & 7: + bitmap_out[-1] = (1 << (size & 7)) - 1 + return memoryview(bytes(size)), memoryview(bytes(bitmap_out)) return memoryview(b"") def concrete_load(self, addr, size, writing=False, *, with_bitmap: bool = False, **kwargs): @@ -503,14 +512,16 @@ class PagedMemoryMixin[PageType: PageBase]( return self._load_to_memoryview(addr, size, True) return self._load_to_memoryview(addr, size, False) - data, bitmap = page.concrete_load(offset, subsize, with_bitmap=True, **kwargs) if with_bitmap: - return data, bitmap + return page.concrete_load(offset, subsize, with_bitmap=True, **kwargs) # everything from here on out has exactly one goal: to maximize the amount of concrete data - # we can return (up to the limit!) - i = next((i for i, byte in enumerate(bitmap) if byte != 0), len(bitmap)) + # we can return (up to the limit!). + i = page.concrete_run_length(offset, subsize, **kwargs) + if i == 0: + return memoryview(b"") + data = page.concrete_load(offset, subsize, **kwargs) if i != subsize: return data[:i] @@ -526,11 +537,14 @@ class PagedMemoryMixin[PageType: PageBase]( try: page = self._get_page(pageno, writing, **kwargs) concrete_load = page.concrete_load + concrete_run_length = page.concrete_run_length except (SimMemoryError, AttributeError): break else: - newdata, bitmap = concrete_load(offset, subsize, with_bitmap=True, **kwargs) - i = next((i for i, byte in enumerate(bitmap) if byte != 0), len(bitmap)) + i = concrete_run_length(offset, subsize, **kwargs) + if i == 0: + break + newdata = concrete_load(offset, subsize, **kwargs) # magic: check if the memory regions are physically adjacent if physically_adjacent and ffi.cast(ffi.BVoidP, ffi.from_buffer(data)) + len(data) == ffi.cast( diff --git a/angr/storage/memory_mixins/paged_memory/pages/symbolic_bitmap.py b/angr/storage/memory_mixins/paged_memory/pages/symbolic_bitmap.py new file mode 100644 index 000000000..f32b7edce --- /dev/null +++ b/angr/storage/memory_mixins/paged_memory/pages/symbolic_bitmap.py @@ -0,0 +1,270 @@ +from __future__ import annotations + + +class SymbolicBitmap: + """ + Tracks, for every byte of an :class:`UltraPage`, whether that byte is symbolic or not. + + The map is stored as a bitmap where the i-th bit of the map lives in bit ``i & 7`` of byte ``i >> 3`` + (least-significant bit first). This makes ``int.from_bytes(..., "little")`` a direct view of the map as + a big integer and lets range scans be done with ``bit_length()`` instead of a Python-level loop. + + A page whose map is uniform (all bytes are symbolic or concrete) does not store any map. + + All ranges are half-open ``[start, stop)`` and are assumed to lie within ``[0, size]``. + """ + + __slots__ = ("_bits", "_pinned", "_uniform", "size") + + def __init__(self, size: int, value: int = 0): + self.size = size + self._bits: bytearray | None = None + self._uniform: int = 1 if value else 0 + # set once :meth:`view` hands out an aliasing buffer; the backing store may not be dropped afterwards. + self._pinned: bool = False + + # + # Introspection + # + + @property + def nbytes(self) -> int: + """ + The number of bytes of backing store this map currently occupies (0 while uniform). + """ + return 0 if self._bits is None else len(self._bits) + + @property + def uniform_value(self) -> int | None: + """ + Return 0/1 if every byte of the page has that symbolic-ness, or None if the map is mixed. + """ + return self._uniform if self._bits is None else None + + def _concretize(self) -> bytearray: + n = (self.size + 7) >> 3 + bits = bytearray(b"\xff" * n) if self._uniform else bytearray(n) + self._bits = bits + return bits + + # + # Scalar access + # + + def get(self, i: int) -> int: + """ + Return 1 if byte ``i`` is symbolic, 0 otherwise. + """ + bits = self._bits + if bits is None: + return self._uniform + return (bits[i >> 3] >> (i & 7)) & 1 + + def set(self, i: int, value: int) -> None: + """ + Mark byte ``i`` as symbolic (``value`` truthy) or concrete. + """ + bits = self._bits + if bits is None: + if bool(value) == bool(self._uniform): + return + bits = self._concretize() + if value: + bits[i >> 3] |= 1 << (i & 7) + else: + bits[i >> 3] &= ~(1 << (i & 7)) & 0xFF + + # + # Range writes + # + + def set_range(self, start: int, stop: int) -> None: + """ + Mark ``[start, stop)`` as symbolic. + """ + if start >= stop: + return + bits = self._bits + if start <= 0 and stop >= self.size: + # the whole page: collapse to the uniform representation and drop the backing store, unless someone is + # holding a pointer into it. + if self._pinned: + assert bits is not None + bits[:] = b"\xff" * len(bits) + else: + self._bits = None + self._uniform = 1 + return + if bits is None: + if self._uniform: + return + bits = self._concretize() + + fb, fo = start >> 3, start & 7 + lb, lo = stop >> 3, stop & 7 + if fb == lb: + bits[fb] |= (0xFF << fo) & (0xFF >> (8 - lo)) & 0xFF + return + if fo: + bits[fb] |= (0xFF << fo) & 0xFF + fb += 1 + if fb < lb: + bits[fb:lb] = b"\xff" * (lb - fb) + if lo: + bits[lb] |= 0xFF >> (8 - lo) + + def clear_range(self, start: int, stop: int) -> None: + """ + Mark ``[start, stop)`` as concrete. + """ + if start >= stop: + return + bits = self._bits + if start <= 0 and stop >= self.size: + if self._pinned: + assert bits is not None + bits[:] = bytes(len(bits)) + else: + self._bits = None + self._uniform = 0 + return + if bits is None: + if not self._uniform: + return + bits = self._concretize() + + fb, fo = start >> 3, start & 7 + lb, lo = stop >> 3, stop & 7 + if fb == lb: + bits[fb] &= ~((0xFF << fo) & (0xFF >> (8 - lo))) & 0xFF + return + if fo: + bits[fb] &= ~(0xFF << fo) & 0xFF + fb += 1 + if fb < lb: + bits[fb:lb] = bytes(lb - fb) + if lo: + bits[lb] &= ~(0xFF >> (8 - lo)) & 0xFF + + # + # Range scans + # + + def next_set(self, start: int, stop: int) -> int: + """ + Return the smallest ``i`` in ``[start, stop)`` whose byte is symbolic, or ``stop`` if there is none. + + A whole range is turned into one big integer and located with ``(w & -w).bit_length()`` rather than being + walked bit by bit. Long ranges are probed 64 bits at a time first. + """ + if start >= stop: + return stop + bits = self._bits + if bits is None: + return start if self._uniform else stop + pos = start + window = 64 + while pos < stop: + end = pos + window + end = min(end, stop) + n = end - pos + fb = pos >> 3 + off = pos & 7 + if fb == (end - 1) >> 3: + w = (bits[fb] >> off) & ((1 << n) - 1) + else: + w = (int.from_bytes(bits[fb : (end + 7) >> 3], "little") >> off) & ((1 << n) - 1) + if w: + return pos + (w & -w).bit_length() - 1 + pos = end + window = stop # after the initial probe, do the remainder in one shot + return stop + + def next_clear(self, start: int, stop: int) -> int: + """ + Return the smallest ``i`` in ``[start, stop)`` whose byte is concrete, or ``stop`` if there is none. + """ + if start >= stop: + return stop + bits = self._bits + if bits is None: + return stop if self._uniform else start + pos = start + window = 64 + while pos < stop: + end = pos + window + end = min(end, stop) + n = end - pos + fb = pos >> 3 + off = pos & 7 + if fb == (end - 1) >> 3: + w = (~bits[fb] >> off) & ((1 << n) - 1) + else: + w = (~int.from_bytes(bits[fb : (end + 7) >> 3], "little") >> off) & ((1 << n) - 1) + if w: + return pos + (w & -w).bit_length() - 1 + pos = end + window = stop + return stop + + def all_set(self, start: int, stop: int) -> bool: + """ + Return True if every byte in ``[start, stop)`` is symbolic. + """ + if start >= stop: + return True + if self._bits is None: + return bool(self._uniform) + return self.next_clear(start, stop) == stop + + def any_set(self, start: int, stop: int) -> bool: + """ + Return True if any byte in ``[start, stop)`` is symbolic. + """ + return self.next_set(start, stop) != stop + + # + # Buffer access + # + + def view(self, start: int, stop: int) -> memoryview: + """ + Return the packed bits covering ``[start, stop)``: bit ``i`` of the result is byte ``start + i`` of the page. + + When ``start`` is byte-aligned, the result is a writable view of this map's own backing store, so a native + consumer (e.g., unicorn engine) can update the page's symbolic-ness in place. Returning such a view pins the + backing store: it may no longer be dropped in favor of the uniform representation. + + An unaligned ``start`` cannot be expressed as a view of the backing store, so the bits are shifted into place + and returned read-only. Consumers like unicorn engine should not use this view for writing! + """ + if start & 7: + return memoryview(self._shifted(start, stop)) + bits = self._bits + if bits is None: + bits = self._concretize() + self._pinned = True + return memoryview(bits)[start >> 3 : (stop + 7) >> 3] + + def _shifted(self, start: int, stop: int) -> bytes: + n = stop - start + if n <= 0: + return b"" + bits = self._bits + if bits is None: + w = ((1 << n) - 1) if self._uniform else 0 + else: + w = (int.from_bytes(bits[start >> 3 : (stop + 7) >> 3], "little") >> (start & 7)) & ((1 << n) - 1) + return w.to_bytes((n + 7) >> 3, "little") + + def copy(self) -> SymbolicBitmap: + o = SymbolicBitmap.__new__(SymbolicBitmap) + o.size = self.size + bits = self._bits + o._bits = None if bits is None else bytearray(bits) # pylint:disable=protected-access + o._uniform = self._uniform # pylint:disable=protected-access + o._pinned = False # pylint:disable=protected-access + return o + + +__all__ = ("SymbolicBitmap",) diff --git a/angr/storage/memory_mixins/paged_memory/pages/ultra_page.py b/angr/storage/memory_mixins/paged_memory/pages/ultra_page.py index 2c8733a3b..5fbcf1b95 100644 --- a/angr/storage/memory_mixins/paged_memory/pages/ultra_page.py +++ b/angr/storage/memory_mixins/paged_memory/pages/ultra_page.py @@ -13,9 +13,12 @@ from angr.errors import SimMemoryError from .base import PageBase from .cooperation import MemoryObjectMixin, SimMemoryObject +from .symbolic_bitmap import SymbolicBitmap l = logging.getLogger(name=__name__) +DEFAULT_PAGE_SIZE = 4096 + class UltraPage(MemoryObjectMixin, PageBase): """ @@ -27,15 +30,10 @@ class UltraPage(MemoryObjectMixin, PageBase): def __init__(self, memory=None, init_zero=False, **kwargs): super().__init__(**kwargs) - if memory is not None: - self.concrete_data = bytearray(memory.page_size) - if init_zero: - self.symbolic_bitmap = bytearray(memory.page_size) - else: - self.symbolic_bitmap = bytearray(b"\1" * memory.page_size) - else: - self.concrete_data = None - self.symbolic_bitmap = None + self.concrete_data = None + self.symbolic_bitmap = SymbolicBitmap( + memory.page_size if memory is not None else DEFAULT_PAGE_SIZE, 0 if init_zero else 1 + ) self.symbolic_data = SortedDict() @@ -43,17 +41,27 @@ class UltraPage(MemoryObjectMixin, PageBase): def new_from_shared(cls, data, memory=None, **kwargs): o = cls(**kwargs) o.concrete_data = data - o.symbolic_bitmap = bytearray(memory.page_size) + o.symbolic_bitmap = SymbolicBitmap(memory.page_size if memory is not None else DEFAULT_PAGE_SIZE, 0) o.refcount = 2 # pylint: disable=attribute-defined-outside-init return o def copy(self, memo): o = super().copy(memo) - o.concrete_data = bytearray(self.concrete_data) - o.symbolic_bitmap = bytearray(self.symbolic_bitmap) + o.concrete_data = None if self.concrete_data is None else bytearray(self.concrete_data) + o.symbolic_bitmap = self.symbolic_bitmap.copy() o.symbolic_data = SortedDict(self.symbolic_data) return o + def _concrete(self): + """ + Return the concrete backing store, allocating it (zero-filled) on first use. + """ + data = self.concrete_data + if data is None: + data = bytearray(self.symbolic_bitmap.size) + self.concrete_data = data + return data + def load(self, addr, size=None, page_addr=None, endness=None, memory=None, cooperate=False, **kwargs): # pylint: disable=arguments-differ concrete_run = [] symbolic_run = ... @@ -86,9 +94,10 @@ class UltraPage(MemoryObjectMixin, PageBase): subaddr = addr end = addr + size + bitmap = self.symbolic_bitmap while subaddr < end: realaddr = subaddr + page_addr - if self.symbolic_bitmap[subaddr]: + if bitmap.get(subaddr): cur_val = self._get_object(subaddr, page_addr, memory=memory) # it must be a different object cycle(realaddr) @@ -101,25 +110,20 @@ class UltraPage(MemoryObjectMixin, PageBase): obj_end = subaddr + cur_val.length obj_end = min(end, obj_end) - # determine how many bytes come from this object - # loop until: end of object or not symbolic or until next object - next_place = None - while next_addr < obj_end and self.symbolic_bitmap[next_addr]: - if next_addr == subaddr + 1: # first loop - next_place = self._get_next_place(next_addr) - if next_place is not None and next_place <= next_addr: - break - next_addr += 1 + # determine how many bytes come from this object: scan forward until the end of the object, the first + # non-symbolic byte, or the start of the next object, whichever comes first + if next_addr < obj_end and bitmap.get(next_addr): + next_place = self._get_next_place(next_addr) + limit = obj_end if next_place is None or next_place > obj_end else next_place + next_addr = bitmap.next_clear(next_addr, limit) subaddr = next_addr last_run = symbolic_run = cur_val result.append((realaddr, cur_val)) else: - max_concrete_read = subaddr - while max_concrete_read < end and not self.symbolic_bitmap[max_concrete_read]: - max_concrete_read += 1 - cur_val = self.concrete_data[subaddr:max_concrete_read] + max_concrete_read = bitmap.next_set(subaddr, end) + cur_val = self._concrete()[subaddr:max_concrete_read] subaddr = max_concrete_read # we know the last run was not a concrete one cycle(realaddr) @@ -178,7 +182,7 @@ class UltraPage(MemoryObjectMixin, PageBase): if type(data) is int or (data.object.op == "BVV" and not data.object.annotations): # mark range as not symbolic - self.symbolic_bitmap[addr : addr + size] = b"\0" * size + self.symbolic_bitmap.clear_range(addr, addr + size) # store arange = range(addr, addr + size) @@ -188,12 +192,13 @@ class UltraPage(MemoryObjectMixin, PageBase): assert memory.state.arch.byte_width == 8 # TODO: Make UltraPage support architectures with greater byte_widths (but are still multiples of 8) + concrete_data = self._concrete() for subaddr in arange: - self.concrete_data[subaddr] = ival & 0xFF + concrete_data[subaddr] = ival & 0xFF ival >>= 8 else: # mark range as symbolic - self.symbolic_bitmap[addr : addr + size] = b"\1" * size + self.symbolic_bitmap.set_range(addr, addr + size) # set ending object try: @@ -220,6 +225,7 @@ class UltraPage(MemoryObjectMixin, PageBase): memory=None, changed_offsets: set[int] | None = None, ): + assert page_addr is not None all_pages = [self, *others] merged_to = None merged_objects = set() @@ -244,8 +250,8 @@ class UltraPage(MemoryObjectMixin, PageBase): # first get a list of all memory objects at that location, and # all memories that don't have those bytes for pg, fv in zip(all_pages, merge_conditions): - if pg.symbolic_bitmap[b]: - mo = pg._get_object(b, page_addr) + if pg.symbolic_bitmap.get(b): + mo = pg._get_object(b, page_addr) # pylint: disable=protected-access if mo is not None: l.debug("... MO present in %s", fv) memory_objects.append((mo, fv)) @@ -256,7 +262,7 @@ class UltraPage(MemoryObjectMixin, PageBase): unconstrained_in.append((pg, fv)) else: # concrete data - concretes.append((pg.concrete_data[b], fv)) + concretes.append((pg._concrete()[b], fv)) # pylint: disable=protected-access # fast path: no memory objects, no unconstrained positions, and only one concrete value if not memory_objects and not unconstrained_in and len({cv for cv, _ in concretes}) == 1: @@ -308,7 +314,7 @@ class UltraPage(MemoryObjectMixin, PageBase): min_size = min(mo.length - (page_addr + b - mo.base) for mo, _ in memory_objects) for um, _ in unconstrained_in: for i in range(min_size): - if um._contains(b + i, page_addr): + if um._contains(b + i, page_addr): # pylint: disable=protected-access min_size = i break merged_to = b + min_size @@ -342,32 +348,28 @@ class UltraPage(MemoryObjectMixin, PageBase): return merged_offsets - def concrete_load(self, addr, size, writing=False, with_bitmap=False, **kwargs): # pylint: disable=arguments-differ - assert self.concrete_data is not None + def concrete_run_length(self, addr, size, **kwargs) -> int: # pylint: disable=unused-argument + """ + Return the number of concrete bytes at ``addr``, capped at ``size``. + """ assert self.symbolic_bitmap is not None + return self.symbolic_bitmap.next_set(addr, addr + size) - addr + + def concrete_load(self, addr, size, writing=False, with_bitmap=False, **kwargs): # pylint: disable=arguments-differ + assert self.symbolic_bitmap is not None + concrete_data = self._concrete() mv_data = ( - self.concrete_data + concrete_data if isinstance( - self.concrete_data, + concrete_data, (memoryview, angr.storage.memory_mixins.paged_memory.page_backer_mixins.NotMemoryview), ) - else memoryview(self.concrete_data) + else memoryview(concrete_data) ) - mv_bitm = ( - self.symbolic_bitmap - if isinstance( - self.symbolic_bitmap, - (memoryview, angr.storage.memory_mixins.paged_memory.page_backer_mixins.NotMemoryview), - ) - else memoryview(self.symbolic_bitmap) - ) - result = ( - mv_data[addr : addr + size], - mv_bitm[addr : addr + size], - ) - if with_bitmap: - return result - return result[0] + data = mv_data[addr : addr + size] + if not with_bitmap: + return data + return data, self.symbolic_bitmap.view(addr, addr + size) def changed_bytes(self, other, page_addr=None) -> set[int]: changed_candidates = super().changed_bytes(other) @@ -375,12 +377,15 @@ class UltraPage(MemoryObjectMixin, PageBase): changed_candidates = self._ultra_changed_candidates(other) changes: set[int] = set() + self_bitmap = self.symbolic_bitmap + other_bitmap = other.symbolic_bitmap for addr in changed_candidates: - if self.symbolic_bitmap[addr] != other.symbolic_bitmap[addr]: + self_sym = self_bitmap.get(addr) + if bool(self_sym) != bool(other_bitmap.get(addr)): changes.add(addr) - elif self.symbolic_bitmap[addr] == 0: - if self.concrete_data[addr] != other.concrete_data[addr]: + elif not self_sym: + if self._concrete()[addr] != other._concrete()[addr]: # pylint: disable=protected-access changes.add(addr) else: try: @@ -439,16 +444,20 @@ class UltraPage(MemoryObjectMixin, PageBase): # calculate concrete offsets CHUNK_SIZE = 128 - FULLY_SYMBOLIC = b"\1" * CHUNK_SIZE - for i in range(0, len(self.symbolic_bitmap), CHUNK_SIZE): - chunk_self = self.symbolic_bitmap[i : i + CHUNK_SIZE] - chunk_other = other.symbolic_bitmap[i : i + CHUNK_SIZE] - if not chunk_self == chunk_other == FULLY_SYMBOLIC: - candidate_offsets |= set(range(i, i + CHUNK_SIZE)) + self_bitmap = self.symbolic_bitmap + other_bitmap = other.symbolic_bitmap + size = self_bitmap.size + if size % CHUNK_SIZE == 0 and self_bitmap.all_set(0, size) and other_bitmap.all_set(0, size): + # both pages are entirely symbolic; no concrete offsets are candidates + return candidate_offsets + for i in range(0, size, CHUNK_SIZE): + end = i + CHUNK_SIZE + if end > size or not (self_bitmap.all_set(i, end) and other_bitmap.all_set(i, end)): + candidate_offsets |= set(range(i, end)) return candidate_offsets def _contains(self, start: int, page_addr: int): - if not self.symbolic_bitmap[start]: + if not self.symbolic_bitmap.get(start): # concrete data return True # symbolic data or does not exist @@ -459,21 +468,19 @@ class UltraPage(MemoryObjectMixin, PageBase): place = next(self.symbolic_data.irange(maximum=start, reverse=True)) except StopIteration: return None - else: - obj = self.symbolic_data[place] - if obj.includes(start + page_addr) or ( - memory is not None and obj.includes(start + page_addr + (1 << memory.state.arch.bits)) - ): - return obj - return None + obj = self.symbolic_data[place] + if obj.includes(start + page_addr) or ( + memory is not None and obj.includes(start + page_addr + (1 << memory.state.arch.bits)) + ): + return obj + return None def _get_next_place(self, start): try: place = next(self.symbolic_data.irange(minimum=start, reverse=False)) except StopIteration: return None - else: - return place + return place def replace_all_with_offsets(self, offsets: Iterable[int], old: claripy.ast.BV, new: claripy.ast.BV, memory=None): memory_objects = set() @@ -521,7 +528,12 @@ class UltraPage(MemoryObjectMixin, PageBase): ) != new_content.size(): raise SimMemoryError("memory objects can only be replaced by the same length content") - new = SimMemoryObject(new_content, old.base, old.endness, byte_width=old._byte_width) + new = SimMemoryObject( + new_content, + old.base, + old.endness, + byte_width=old._byte_width, # pylint: disable=protected-access + ) for k in list(self.symbolic_data): if self.symbolic_data[k] is old: self.symbolic_data[k] = new diff --git a/native/unicornlib/sim_unicorn.cpp b/native/unicornlib/sim_unicorn.cpp index 79c71a23d..f33f29e64 100644 --- a/native/unicornlib/sim_unicorn.cpp +++ b/native/unicornlib/sim_unicorn.cpp @@ -407,12 +407,15 @@ void State::rollback() { break; } auto page = page_lookup(rit->address); - taint_t *bitmap = page.first; uint64_t start = rit->address & 0xFFF; int size = rit->size; for (auto i = 0; i < size; i++) { - bitmap[start + i] = rit->previous_taint[i]; + taint_t previous = rit->previous_taint[i]; + bitmap_set(page.symbolic, start + i, previous == TAINT_SYMBOLIC); + if (page.dirty != NULL) { + bitmap_set(page.dirty, start + i, previous == TAINT_DIRTY); + } } } mem_writes.clear(); @@ -425,14 +428,15 @@ void State::rollback() { } /* - * return the PageBitmap only if the page is remapped for writing, - * or initialized with symbolic variable, otherwise return NULL. + * return the taint bitmaps only if the page is remapped for writing, + * or initialized with symbolic variable, otherwise return NULLs. */ -std::pair State::page_lookup(address_t address) const { +page_taint_t State::page_lookup(address_t address) const { address &= ~0xFFFULL; auto it = active_pages.find(address); if (it == active_pages.end()) { - return std::pair(NULL, NULL); + page_taint_t missing = {NULL, NULL, NULL}; + return missing; } return it->second; } @@ -441,17 +445,21 @@ void State::page_activate(address_t address, uint8_t *taint, uint8_t *data) { address &= ~0xFFFULL; auto it = active_pages.find(address); if (it == active_pages.end()) { + page_taint_t page; + page.data = data; if (data == NULL) { - // We need to copy the taint bitmap - taint_t *bitmap = new PageBitmap; - memcpy(bitmap, taint, sizeof(PageBitmap)); - - active_pages.insert(std::pair>(address, std::pair(bitmap, NULL))); + // The page is copied into unicorn: we need our own copy of the symbolic bitmap, plus a dirty bitmap to + // remember which bytes have to be synced back to angr. + page.symbolic = new PageBitmap; + memcpy(page.symbolic, taint, sizeof(PageBitmap)); + page.dirty = new uint8_t[ANGR_PAGE_BITMAP_SIZE](); } else { - // We can directly use the passed taint and data - taint_t *bitmap = (taint_t*)taint; - active_pages.insert(std::pair>(address, std::pair(bitmap, data))); + // The page is direct-mapped: angr's symbolic bitmap is ours to update in place, and concrete writes go + // straight into angr's backing store, so nothing ever needs syncing. + page.symbolic = taint; + page.dirty = NULL; } + active_pages.insert(std::pair(address, page)); } else { // TODO: un-hardcode this address, or at least do this warning from python land if (address == 0x4000) { @@ -492,25 +500,23 @@ void State::page_activate(address_t address, uint8_t *taint, uint8_t *data) { mem_update_t *State::sync() { for (auto it = active_pages.begin(); it != active_pages.end(); it++) { - uint8_t *data = it->second.second; - if (data != NULL) { + uint8_t *dirty = it->second.dirty; + if (dirty == NULL) { // nothing to sync, direct mapped :) continue; } - taint_t *start = it->second.first; - taint_t *end = &it->second.first[0x1000]; - //LOG_D("found active page %#lx (%p)", it->first, start); - for (taint_t *i = start; i < end; i++) - if ((*i) == TAINT_DIRTY) { - taint_t *j = i; - while (j < end && (*j) == TAINT_DIRTY) j++; + //LOG_D("found active page %#lx (%p)", it->first, dirty); + for (uint64_t i = 0; i < ANGR_PAGE_SIZE; i++) + if (bitmap_get(dirty, i)) { + uint64_t j = i; + while (j < ANGR_PAGE_SIZE && bitmap_get(dirty, j)) j++; - char buf[0x1000]; - uc_mem_read(uc, it->first + (i - start), buf, j - i); - //LOG_D("sync [%#lx, %#lx] = %#lx", it->first + (i - start), it->first + (j - start), *(uint64_t *)buf); + char buf[ANGR_PAGE_SIZE]; + uc_mem_read(uc, it->first + i, buf, j - i); + //LOG_D("sync [%#lx, %#lx] = %#lx", it->first + i, it->first + j, *(uint64_t *)buf); mem_update_t *range = new mem_update_t; - range->address = it->first + (i - start); + range->address = it->first + i; range->length = j - i; range->next = mem_updates_head; mem_updates_head = range; @@ -632,7 +638,7 @@ bool State::in_cache(address_t address) const { // Finds tainted data in the provided range and returns the address. // Returns -1 if no tainted data is present. int64_t State::find_tainted(address_t address, int size) { - taint_t *bitmap = page_lookup(address).first; + uint8_t *bitmap = page_lookup(address).symbolic; int start = address & 0xFFF; int end = (address + size - 1) & 0xFFF; @@ -640,7 +646,7 @@ int64_t State::find_tainted(address_t address, int size) { if (end >= start) { if (bitmap) { for (auto i = start; i <= end; i++) { - if (bitmap[i] & TAINT_SYMBOLIC) { + if (bitmap_get(bitmap, i)) { return (address & ~0xFFF) + i; } } @@ -650,16 +656,16 @@ int64_t State::find_tainted(address_t address, int size) { // cross page boundary if (bitmap) { for (auto i = start; i <= 0xFFF; i++) { - if (bitmap[i] & TAINT_SYMBOLIC) { + if (bitmap_get(bitmap, i)) { return (address & ~0xFFF) + i; } } } - bitmap = page_lookup(address + size - 1).first; + bitmap = page_lookup(address + size - 1).symbolic; if (bitmap) { for (auto i = 0; i <= end; i++) { - if (bitmap[i] & TAINT_SYMBOLIC) { + if (bitmap_get(bitmap, i)) { return ((address + size - 1) & ~0xFFF) + i; } } @@ -697,9 +703,10 @@ void State::handle_write(address_t address, int size, bool is_interrupt = false, return; } - auto pair = page_lookup(address); - taint_t *bitmap = pair.first; - uint8_t *data = pair.second; + auto page = page_lookup(address); + uint8_t *bitmap = page.symbolic; + uint8_t *dirty = page.dirty; + uint8_t *data = page.data; int start = address & 0xFFF; int end = (address + size - 1) & 0xFFF; short clean; @@ -864,32 +871,28 @@ void State::handle_write(address_t address, int size, bool is_interrupt = false, } if (data == NULL) { for (auto i = start; i <= end; i++) { - record.previous_taint.push_back(bitmap[i]); - if (is_dst_symbolic) { - // Don't mark as TAINT_DIRTY since we don't want to sync it back to angr - // Also, no need to set clean: rollback will set it to TAINT_NONE which - // is fine for symbolic bytes and rollback is called when exiting unicorn - // due to an error encountered - bitmap[i] = TAINT_SYMBOLIC; + // a byte is never both symbolic and dirty, so the two bitmaps recover the previous taint between them + taint_t previous = TAINT_NONE; + if (bitmap_get(bitmap, i)) { + previous = TAINT_SYMBOLIC; } - else if (bitmap[i] != TAINT_DIRTY) { - bitmap[i] = TAINT_DIRTY; + else if (bitmap_get(dirty, i)) { + previous = TAINT_DIRTY; } + record.previous_taint.push_back(previous); + // Don't mark a symbolic write as TAINT_DIRTY since we don't want to sync it back to angr + // Also, no need to set clean: rollback will set it to TAINT_NONE which + // is fine for symbolic bytes and rollback is called when exiting unicorn + // due to an error encountered + bitmap_set(bitmap, i, is_dst_symbolic); + bitmap_set(dirty, i, !is_dst_symbolic); } } else { for (auto i = start; i <= end; i++) { - record.previous_taint.push_back(bitmap[i]); - if (is_dst_symbolic) { - // Don't mark as TAINT_DIRTY since we don't want to sync it back to angr - // Also, no need to set clean: rollback will set it to TAINT_NONE which - // is fine for symbolic bytes and rollback is called when exiting unicorn - // due to an error encountered - bitmap[i] = TAINT_SYMBOLIC; - } - else if (bitmap[i] != TAINT_NONE) { - bitmap[i] = TAINT_NONE; - } + // the page is direct-mapped, so concrete writes need no syncing and there is no dirty bitmap to update + record.previous_taint.push_back(bitmap_get(bitmap, i) ? TAINT_SYMBOLIC : TAINT_NONE); + bitmap_set(bitmap, i, is_dst_symbolic); } } mem_writes.push_back(record); diff --git a/native/unicornlib/sim_unicorn.hpp b/native/unicornlib/sim_unicorn.hpp index 636b1a0d1..955fd174f 100644 --- a/native/unicornlib/sim_unicorn.hpp +++ b/native/unicornlib/sim_unicorn.hpp @@ -28,6 +28,9 @@ static const uint8_t MAX_REGISTER_BYTE_SIZE = 32; static const uint16_t ANGR_PAGE_SIZE = 0x1000; static const uint8_t PAGE_SHIFT = 12; +// A page's taint maps are packed bitmaps: one bit per page byte, so a page needs ANGR_PAGE_SIZE / 8 bytes. +static const uint16_t ANGR_PAGE_BITMAP_SIZE = ANGR_PAGE_SIZE / 8; + typedef uint64_t address_t; typedef uint64_t unicorn_reg_id_t; typedef int64_t vex_reg_offset_t; @@ -41,7 +44,7 @@ enum simos_t: uint8_t { enum taint_t: uint8_t { TAINT_NONE = 0, - TAINT_SYMBOLIC = 1, // this should be 1 to match the UltraPage impl + TAINT_SYMBOLIC = 1, // this should be 1 to match the taint values angr passes in for file descriptor bytes TAINT_DIRTY = 2, }; @@ -490,7 +493,41 @@ struct caches_t { PageCache *page_cache; }; -typedef taint_t PageBitmap[ANGR_PAGE_SIZE]; +/* + * A packed bitmap over the bytes of a page: bit i of byte i >> 3, least significant bit first. This is the exact + * layout angr's UltraPage keeps its symbolic map in, so the map of a direct-mapped page can be aliased rather than + * expanded and copied. + */ +typedef uint8_t PageBitmap[ANGR_PAGE_BITMAP_SIZE]; + +static inline bool bitmap_get(const uint8_t *bitmap, uint64_t idx) { + return (bitmap[idx >> 3] >> (idx & 7)) & 1; +} + +static inline void bitmap_set(uint8_t *bitmap, uint64_t idx, bool value) { + if (value) { + bitmap[idx >> 3] |= (uint8_t)(1u << (idx & 7)); + } + else { + bitmap[idx >> 3] &= (uint8_t)~(1u << (idx & 7)); + } +} + +/* + * The taint state of an active page. Symbolic-ness and dirtiness are tracked in separate bitmaps because a byte can + * only be one or the other, and because the symbolic map has to match angr's own layout exactly when it is aliased. + */ +struct page_taint_t { + // Bytes holding symbolic values. Aliases the UltraPage's own map when the page is direct-mapped, in which case + // updates here are immediately visible to angr; otherwise it is a copy owned by us. + uint8_t *symbolic; + // Bytes unicorn wrote concretely that have to be synced back to angr. NULL for direct-mapped pages: their writes + // land in angr's backing store already, so there is nothing to sync. + uint8_t *dirty; + // The direct-mapped page data, or NULL if the page was copied into unicorn. + uint8_t *data; +}; + typedef std::unordered_map BlockTaintCache; extern std::map global_cache; @@ -583,9 +620,7 @@ class State { // List of instructions that should be executed symbolically std::vector blocks_with_symbolic_stmts; - // the latter part of the pair is a pointer to the page data if the page is direct-mapped, otherwise NULL - std::map> active_pages; - //std::map active_pages; + std::map active_pages; std::set stop_points; address_t trace_last_block_addr; @@ -641,7 +676,7 @@ class State { // Private functions - std::pair page_lookup(address_t address) const; + page_taint_t page_lookup(address_t address) const; void compute_slice_of_stmt(vex_stmt_details_t &instr); vex_stmt_details_t compute_vex_stmt_details(const vex_stmt_taint_entry_t &vex_stmt_taint_entry); @@ -833,11 +868,12 @@ class State { ~State() { for (auto it = active_pages.begin(); it != active_pages.end(); it++) { - // only delete if not direct-mapped - if (!it->second.second) { - // delete should use the bracket operator since PageBitmap is an array typedef - delete[] it->second.first; + // the symbolic map of a direct-mapped page belongs to angr; only a copied page's is ours to free + // delete should use the bracket operator since PageBitmap is an array typedef + if (!it->second.data) { + delete[] it->second.symbolic; } + delete[] it->second.dirty; } mem_update_t *next; for (mem_update_t *cur = mem_updates_head; cur; cur = next) { @@ -870,7 +906,8 @@ class State { void rollback(); /* - * allocate a new PageBitmap and put into active_pages. + * record the taint bitmaps of a page in active_pages. `taint` is a packed symbolic bitmap of + * ANGR_PAGE_BITMAP_SIZE bytes, aliased if `data` is non-NULL and copied otherwise. */ void page_activate(address_t address, uint8_t *taint, uint8_t *data); diff --git a/tests/storage/test_memory.py b/tests/storage/test_memory.py index faab3b76a..13394d537 100755 --- a/tests/storage/test_memory.py +++ b/tests/storage/test_memory.py @@ -23,6 +23,7 @@ from angr.storage.memory_mixins import ( UltraPagesMixin, ) from angr.storage.memory_mixins.paged_memory.pages.multi_values import MultiValues +from angr.storage.memory_mixins.paged_memory.pages.symbolic_bitmap import SymbolicBitmap class UltraPageMemory( @@ -816,25 +817,29 @@ class TestMemory(unittest.TestCase): ) def test_concrete_load(self): + # concrete_load's bitmap is packed: one bit per byte, least significant bit first + def is_symbolic(bitmap, i): + return bool(bitmap[i >> 3] >> (i & 7) & 1) + + def concrete_bytes(data, bitmap): + return bytes(0 if is_symbolic(bitmap, i) else d for i, d in enumerate(data)) + for memcls in [UltraPageMemory, ListPageMemory]: state = SimState(arch="AMD64", mode="symbolic", plugins={"memory": memcls()}) state.memory.store(0x20000, b"aaaabbbbccccdddd") data, bitmap = state.memory.concrete_load(0x20000, 4, with_bitmap=True) - data_bytes = bytes(d if b == 0 else 0 for d, b in zip(data, bitmap)) - assert data_bytes == b"aaaa" - assert bitmap.tobytes() == b"\x00\x00\x00\x00" + assert concrete_bytes(data, bitmap) == b"aaaa" + assert bitmap.tobytes() == b"\x00" data, bitmap = state.memory.concrete_load(0x20004, 8, with_bitmap=True) - data_bytes = bytes(d if b == 0 else 0 for d, b in zip(data, bitmap)) - assert data_bytes == b"bbbbcccc" - assert bitmap.tobytes() == b"\x00\x00\x00\x00\x00\x00\x00\x00" + assert concrete_bytes(data, bitmap) == b"bbbbcccc" + assert bitmap.tobytes() == b"\x00" state.memory.store(0x20001, claripy.BVS("flag", 8)) data, bitmap = state.memory.concrete_load(0x20000, 4, with_bitmap=True) - data_bytes = bytes(d if b == 0 else 0 for d, b in zip(data, bitmap)) - assert data_bytes == b"a\x00aa" - assert bitmap.tobytes() == b"\x00\x01\x00\x00" + assert concrete_bytes(data, bitmap) == b"a\x00aa" + assert bitmap.tobytes() == b"\x02" expr = claripy.Concat( claripy.BVS("flag_0", 1), @@ -845,9 +850,8 @@ class TestMemory(unittest.TestCase): ) state.memory.store(0x20001, expr) data, bitmap = state.memory.concrete_load(0x20000, 4, with_bitmap=True) - data_bytes = bytes(d if b == 0 else 0 for d, b in zip(data, bitmap)) - assert data_bytes == b"a\x00\x00a" - assert bitmap.tobytes() == b"\x00\x01\x01\x00" + assert concrete_bytes(data, bitmap) == b"a\x00\x00a" + assert bitmap.tobytes() == b"\x06" expr = claripy.Concat( claripy.BVS("flag_0", 1), @@ -858,9 +862,8 @@ class TestMemory(unittest.TestCase): ) state.memory.store(0x20005, expr) data, bitmap = state.memory.concrete_load(0x20004, 4, with_bitmap=True) - data_bytes = bytes(d if b == 0 else 0 for d, b in zip(data, bitmap)) - assert data_bytes == b"b\x00\x00b" - assert bitmap.tobytes() == b"\x00\x01\x01\x00" + assert concrete_bytes(data, bitmap) == b"b\x00\x00b" + assert bitmap.tobytes() == b"\x06" expr = claripy.Concat( claripy.BVV(7, 7), @@ -869,9 +872,8 @@ class TestMemory(unittest.TestCase): ) state.memory.store(0x20005, expr) data, bitmap = state.memory.concrete_load(0x20004, 4, with_bitmap=True) - data_bytes = bytes(d if b == 0 else 0 for d, b in zip(data, bitmap)) - assert data_bytes == b"b\x00\x00b" - assert bitmap.tobytes() == b"\x00\x01\x01\x00" + assert concrete_bytes(data, bitmap) == b"b\x00\x00b" + assert bitmap.tobytes() == b"\x06" expr = claripy.Concat( claripy.BVV(1, 1), @@ -880,9 +882,8 @@ class TestMemory(unittest.TestCase): ) state.memory.store(0x20005, expr) data, bitmap = state.memory.concrete_load(0x20004, 4, with_bitmap=True) - data_bytes = bytes(d if b == 0 else 0 for d, b in zip(data, bitmap)) - assert data_bytes == b"b\x00\x00b" - assert bitmap.tobytes() == b"\x00\x01\x01\x00" + assert concrete_bytes(data, bitmap) == b"b\x00\x00b" + assert bitmap.tobytes() == b"\x06" def test_multivalued_list_page(self): state = SimState(arch="AMD64", mode="symbolic", plugins={"memory": MultiValuedMemory()}) @@ -918,5 +919,61 @@ class TestMemory(unittest.TestCase): assert (val == 0x12345678).is_true() +class TestSymbolicBitmap(unittest.TestCase): + def test_view_is_packed(self): + bm = SymbolicBitmap(64) + bm.set_range(1, 3) + bm.set(9, 1) + assert bm.view(0, 64).tobytes() == b"\x06\x02" + bytes(6) + + def test_view_of_uniform_map(self): + assert SymbolicBitmap(64, 0).view(0, 64).tobytes() == bytes(8) + assert SymbolicBitmap(64, 1).view(0, 64).tobytes() == b"\xff" * 8 + + def test_unaligned_view_is_shifted_and_readonly(self): + bm = SymbolicBitmap(64) + bm.set_range(5, 8) + view = bm.view(4, 12) + assert view.readonly + # bytes 5..7 of the page are bits 1..3 of a view starting at byte 4 + assert view.tobytes() == b"\x0e" + + def test_aligned_view_aliases_the_map(self): + bm = SymbolicBitmap(64) + view = bm.view(0, 64) + assert not view.readonly + + # writes through the view are visible to the map... + view[0] = 0x05 + assert [bm.get(i) for i in range(4)] == [1, 0, 1, 0] + + # ...and writes to the map are visible through the view + bm.set(3, 1) + assert view[0] == 0x0D + + def test_whole_page_write_keeps_an_aliased_buffer(self): + # native unicorn holds a pointer into the map of a page it direct-mapped, so a write covering the whole page + # must fill the backing store in place rather than drop it for the uniform representation + bm = SymbolicBitmap(64) + view = bm.view(0, 64) + + bm.set_range(0, 64) + assert view.tobytes() == b"\xff" * 8 + assert all(bm.get(i) for i in range(64)) + + bm.clear_range(0, 64) + assert view.tobytes() == bytes(8) + assert not any(bm.get(i) for i in range(64)) + + def test_copy_is_not_aliased(self): + bm = SymbolicBitmap(64) + view = bm.view(0, 64) + other = bm.copy() + + other.set_range(0, 64) + assert view.tobytes() == bytes(8) + assert not any(bm.get(i) for i in range(64)) + + if __name__ == "__main__": unittest.main() From 2fffb71f8623e045be33f416e2cd9d9d0c4cb22b Mon Sep 17 00:00:00 2001 From: Fish Date: Mon, 27 Jul 2026 18:20:07 -0700 Subject: [PATCH 066/122] state_plugins: Map the heap region lazily and grow it on demand. (#6715) --- angr/sim_state.py | 2 + angr/state_plugins/gdb.py | 6 +- angr/state_plugins/heap/heap_base.py | 88 +++- angr/state_plugins/heap/heap_brk.py | 18 +- angr/state_plugins/heap/heap_ptmalloc.py | 26 ++ tests/state_plugins/test_heap_lazy_mapping.py | 441 ++++++++++++++++++ 6 files changed, 575 insertions(+), 6 deletions(-) create mode 100644 tests/state_plugins/test_heap_lazy_mapping.py diff --git a/angr/sim_state.py b/angr/sim_state.py index 424a89651..78a87597d 100644 --- a/angr/sim_state.py +++ b/angr/sim_state.py @@ -26,6 +26,7 @@ if TYPE_CHECKING: from angr.simos.javavm import SimJavaVM from .state_plugins.callstack import CallStack + from .state_plugins.heap.heap_base import SimHeapBase from .state_plugins.history import SimStateHistory from .state_plugins.inspect import SimInspector from .state_plugins.jni_references import SimStateJNIReferences @@ -90,6 +91,7 @@ class SimState[IPTypeConc, IPTypeSym](PluginHub[SimStatePlugin]): inspect: SimInspector jni_references: SimStateJNIReferences scratch: SimStateScratch + heap: SimHeapBase def __init__( self, diff --git a/angr/state_plugins/gdb.py b/angr/state_plugins/gdb.py index b1e1b1ad2..47a698969 100644 --- a/angr/state_plugins/gdb.py +++ b/angr/state_plugins/gdb.py @@ -66,7 +66,11 @@ class GDB(SimStatePlugin): """ # We set the heap at the same addresses as the gdb session to avoid pointer corruption. data = self._read_data(heap_dump) - self.state.heap.heap_location = heap_base + len(data) + if hasattr(self.state.heap, "heap_location"): + self.state.heap.heap_location = heap_base + len(data) # type: ignore[assignment] + # the heap region is mapped lazily; moving the break by hand has to grow it too. This is a no-op unless + # the dump was taken at the same base as the heap plugin is configured for. + self.state.heap._ensure_mapped(heap_base + len(data)) # pylint: disable=protected-access addr = heap_base l.info("Set heap from 0x%x to %#x", addr, addr + len(data)) # FIXME: we should probably make we don't overwrite other stuff loaded there diff --git a/angr/state_plugins/heap/heap_base.py b/angr/state_plugins/heap/heap_base.py index 198518a37..3ed752113 100644 --- a/angr/state_plugins/heap/heap_base.py +++ b/angr/state_plugins/heap/heap_base.py @@ -1,6 +1,8 @@ from __future__ import annotations +import contextlib import logging +from typing import Self import angr.sim_options as opts from angr.errors import SimMemoryError @@ -12,6 +14,12 @@ l = logging.getLogger("angr.state_plugins.heap.heap_base") DEFAULT_HEAP_LOCATION = 0xC0000000 DEFAULT_HEAP_SIZE = 0x00800000 +# The number of bytes the heap region ``init_state`` maps up front. +HEAP_INITIAL_MAPPED_SIZE = 0x2000 + +# The factor by which the mapped extent of the heap grows every time an allocation runs past its end. +HEAP_MAPPING_GROWTH_FACTOR = 2 + class SimHeapBase(SimStatePlugin): """ @@ -21,7 +29,8 @@ class SimHeapBase(SimStatePlugin): implementation, which is based on the original libc SimProcedure implementations. :ivar heap_base: the address of the base of the heap in memory - :ivar heap_size: the total size of the main memory region managed by the heap in memory + :ivar heap_size: the maximum size of the main memory region managed by the heap in memory. Only the part of it + that has actually been handed out is mapped; see ``_ensure_mapped``. :ivar mmap_base: the address of the region from which large mmap allocations will be made """ @@ -32,13 +41,84 @@ class SimHeapBase(SimStatePlugin): self.heap_size = heap_size if heap_size is not None else DEFAULT_HEAP_SIZE self.mmap_base = self.heap_base + self.heap_size * 2 - def copy(self, memo): - o = super().copy(memo) + # The exclusive end address of the part of the heap region that this plugin has mapped into the memory + # plugin. ``None`` means that this plugin does not manage the heap mapping. + self._mapped_end: int | None = None + + def copy(self, memo) -> Self: + o: SimHeapBase = super().copy(memo) o.heap_base = self.heap_base o.heap_size = self.heap_size o.mmap_base = self.mmap_base + o._mapped_end = self._mapped_end # pylint:disable=protected-access return o + def __setstate__(self, state): + if "_mapped_end" not in state: + # unpickling a state from before the heap mapping became lazy: back then init_state() mapped the + # entire region up front, so that is what the mapped extent has to be + state = dict(state) + state["_mapped_end"] = state["heap_base"] + state["heap_size"] + self.__dict__.update(state) + + def _combine_mapped_end(self, others): + """ + Reconcile the mapped extent of this heap with the heaps it is being merged with. + """ + ends = [self._mapped_end, *(o._mapped_end for o in others)] # pylint: disable=protected-access + self._mapped_end = None if any(e is None for e in ends) else min(ends) + + def _map_range(self, start, end): + """ + Map ``[start, end)`` as read/write, tolerating pages that happen to be mapped already. + + :param start: the first address to map (rounded down to a page boundary by the memory plugin) + :param end: the exclusive end of the range to map + """ + try: + self.state.memory.map_region(start, end - start, 3) + except SimMemoryError: + # something in this range is mapped already (an mmap, a gdb heap dump, a merge with a state that had + # grown further, ...). Redo it page by page and skip whatever exists. + page_size = getattr(self.state.memory, "page_size", 0x1000) + for page_addr in range(start - (start % page_size), end, page_size): + with contextlib.suppress(SimMemoryError): + self.state.memory.map_region(page_addr, page_size, 3) + + def _init_mapping(self): + """ + Map the initial slice of the heap region. Called by ``init_state`` once it has determined that this plugin + owns the mapping of the region. + """ + self._mapped_end = self.heap_base + self._ensure_mapped(self.heap_base + min(HEAP_INITIAL_MAPPED_SIZE, self.heap_size)) + + def _ensure_mapped(self, addr): + """ + Make sure every byte of the heap region below ``addr`` is backed by a mapped page, growing the mapped + extent geometrically. + + :param addr: the exclusive end of the heap range that is about to be used + """ + mapped_end = self._mapped_end + if mapped_end is None: + # this plugin does not manage the mapping of the heap region + return + + region_end = self.heap_base + self.heap_size + addr = min(addr, region_end) + if addr <= mapped_end: + return + + size = max(mapped_end - self.heap_base, HEAP_INITIAL_MAPPED_SIZE) + while self.heap_base + size < addr: + size *= HEAP_MAPPING_GROWTH_FACTOR + new_end = min(self.heap_base + size, region_end) + + l.debug("Growing the mapped heap region to %#x (%d bytes)", new_end, new_end - self.heap_base) + self._map_range(mapped_end, new_end) + self._mapped_end = new_end + def _conc_alloc_size(self, sim_size): """ Concretizes a size argument, if necessary, to something that makes sense when allocating space. Here we just @@ -124,4 +204,4 @@ class SimHeapBase(SimStatePlugin): self.state.memory.permissions(self.heap_base) except SimMemoryError: l.debug("Mapping base heap region") - self.state.memory.map_region(self.heap_base, self.heap_size, 3) + self._init_mapping() diff --git a/angr/state_plugins/heap/heap_brk.py b/angr/state_plugins/heap/heap_brk.py index 1046bdb5f..2cfd8114e 100644 --- a/angr/state_plugins/heap/heap_brk.py +++ b/angr/state_plugins/heap/heap_brk.py @@ -1,11 +1,13 @@ from __future__ import annotations import logging +from typing import Self import claripy from angr.errors import SimSolverError from angr.sim_state import SimState +from angr.state_plugins.libc import SimStateLibc from angr.state_plugins.plugin import SimStatePlugin from . import SimHeapBase @@ -36,7 +38,7 @@ class SimHeapBrk(SimHeapBase): self.heap_location = self.heap_base @SimStatePlugin.memo - def copy(self, memo): + def copy(self, memo) -> Self: o = super().copy(memo) o.heap_location = self.heap_location return o @@ -52,8 +54,12 @@ class SimHeapBrk(SimHeapBase): size = self._conc_alloc_size(sim_size) while size % 16 != 0: size += 1 + assert isinstance(self.state.heap, SimHeapBrk) addr = self.state.heap.heap_location self.state.heap.heap_location += size + # the heap region is mapped lazily, so bumping the break may run past the end of what is mapped. Grow the + # mapping before handing the space out; this is the only place in this heap where the break moves up. + self.state.heap._ensure_mapped(self.state.heap.heap_location) # pylint: disable=protected-access l.debug("Allocating %d bytes at address %#08x", size, addr) return addr @@ -62,6 +68,11 @@ class SimHeapBrk(SimHeapBase): The memory release primitive for this heap implementation. Decreases the position of the break to deallocate space. Guards against releasing beyond the initial heap base. + The mapped extent of the heap is deliberately not shrunk here. Unmapping the released pages would discard + their contents, so a release followed by an allocation would hand back blank memory instead of the old + bytes, which is a behavior change; and since the extent only ever grows to the high-water mark of the + break, keeping it costs nothing beyond what the program already asked for. + :param sim_size: a size specifying how much to decrease the break pointer by (may be symbolic or not) """ requested = self._conc_alloc_size(sim_size) @@ -78,6 +89,7 @@ class SimHeapBrk(SimHeapBase): def _calloc(self, sim_nmemb, sim_size): plugin = self.state.get_plugin("libc") + assert isinstance(plugin, SimStateLibc) if self.state.solver.symbolic(sim_nmemb): # TODO: find a better way @@ -98,6 +110,7 @@ class SimHeapBrk(SimHeapBase): ): final_size = plugin.max_variable_size + assert isinstance(self.state.heap, SimHeapBrk) addr = self.state.heap.allocate(final_size) v = claripy.BVV(0, final_size * 8) self.state.memory.store(addr, v) @@ -105,6 +118,7 @@ class SimHeapBrk(SimHeapBase): def _realloc(self, ptr, size): if size.symbolic: + assert isinstance(self.state.libc, SimStateLibc) try: size_int = self.state.solver.max(size, extra_constraints=(size < self.state.libc.max_variable_size,)) except SimSolverError: @@ -113,6 +127,7 @@ class SimHeapBrk(SimHeapBase): else: size_int = self.state.solver.eval(size) + assert isinstance(self.state.heap, SimHeapBrk) addr = self.state.heap.allocate(size_int) if self.state.solver.eval(ptr) != 0: @@ -122,6 +137,7 @@ class SimHeapBrk(SimHeapBase): return addr def _combine(self, others): + self._combine_mapped_end(others) new_heap_location = max(o.heap_location for o in others) if self.heap_location != new_heap_location: self.heap_location = new_heap_location diff --git a/angr/state_plugins/heap/heap_ptmalloc.py b/angr/state_plugins/heap/heap_ptmalloc.py index c35a6b8e2..3fbc9ba4b 100644 --- a/angr/state_plugins/heap/heap_ptmalloc.py +++ b/angr/state_plugins/heap/heap_ptmalloc.py @@ -367,6 +367,12 @@ class SimHeapPTMalloc(SimHeapFreelist): while chunk is None: free_size = free_chunk.get_size() free_size = concretize(free_size, self.state.solver, sym_free_size_handler) + if free_size >= size: + # This chunk is about to be carved up. The heap region is mapped lazily, so make sure the chunk + # itself, plus the metadata of whatever chunk ends up following it, is backed before anything + # writes to it. This is the only place where this heap reaches above the chunks it already + # handed out; free()/realloc() only ever touch chunks that malloc() has already mapped. + self._ensure_mapped(free_chunk.base + size + 2 * self._chunk_min_size) if free_size < size: # Chunk is too small to be used; move to the next or fail fwd = free_chunk.fwd_chunk() @@ -582,13 +588,33 @@ class SimHeapPTMalloc(SimHeapFreelist): if any(o._chunk_align_mask != self._chunk_align_mask for o in others): raise SimMergeError("Cannot merge heaps with different chunk alignments") + self._combine_mapped_end(others) + return False def merge(self, others, merge_conditions, common_ancestor=None): # pylint:disable=unused-argument return self._combine(others) + def _map_final_page(self): + """ + This heap keeps the usage information of the real final chunk in the last word of the heap region (see + ``_set_final_freeness``), which is written by ``init_state`` and read or written by every operation that + touches the last chunk. The region is mapped lazily and will not normally grow that far, so map the single + page that holds that word up front. + """ + if self._mapped_end is None: + return + page_size = getattr(self.state.memory, "page_size", 0x1000) + region_end = self.heap_base + self.heap_size + final_page = (region_end - 1) - ((region_end - 1) % page_size) + if final_page < self._mapped_end: + # already covered by the initial mapping + return + self._map_range(final_page, region_end) + def init_state(self): super().init_state() + self._map_final_page() self._chunk_size_t_size = self.state.arch.bytes self._chunk_min_size = 4 * self._chunk_size_t_size diff --git a/tests/state_plugins/test_heap_lazy_mapping.py b/tests/state_plugins/test_heap_lazy_mapping.py new file mode 100644 index 000000000..233a97f01 --- /dev/null +++ b/tests/state_plugins/test_heap_lazy_mapping.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use,protected-access +from __future__ import annotations + +__package__ = __package__ or "tests.state_plugins" # pylint:disable=redefined-builtin + +import os +import pickle +import unittest + +import claripy + +import angr +from angr import SimHeapBrk, SimHeapPTMalloc, SimState +from angr.state_plugins.heap.heap_base import HEAP_INITIAL_MAPPED_SIZE, HEAP_MAPPING_GROWTH_FACTOR +from tests.common import bin_location + +gdb_data_location = os.path.join(bin_location, "tests_data", "test_gdb_plugin") + +# unconstrained reads out of a fresh heap are noisy and (for SimHeapPTMalloc) slow; nothing here depends on the +# filler, so pin it to zero +ZERO_FILL = {angr.options.ZERO_FILL_UNCONSTRAINED_MEMORY, angr.options.ZERO_FILL_UNCONSTRAINED_REGISTERS} + + +def make_state(heap=None, add_options=None): + """ + A bare SimState is enough for everything here and keeps the tests fast: the heap plugin only ever talks to + ``state.memory``. + """ + plugins = {"heap": heap} if heap is not None else None + return SimState(arch="AMD64", plugins=plugins, add_options=ZERO_FILL | (add_options or set())) + + +def resident_heap_pages(state): + """ + The number of pages the memory plugin has actually materialized inside the heap region. + + This deliberately reaches into ``memory._pages``: the whole point of the lazy mapping is that page objects do + not exist, and only the memory object knows that. ``_mapped_end`` alone would not catch a regression that + re-introduces eager mapping somewhere else. + """ + page_size = state.memory.page_size + lo = state.heap.heap_base // page_size + hi = (state.heap.heap_base + state.heap.heap_size) // page_size + return sum(1 for pageno, page in state.memory._pages.items() if page is not None and lo <= pageno < hi) + + +def pattern(i, n): + """A deterministic n-byte pattern that differs for every i.""" + return bytes((i * 7 + j * 31 + 0x41) & 0xFF for j in range(n)) + + +def store_pattern(state, addr, i, n): + state.memory.store(addr, claripy.BVV(pattern(i, n), n * 8)) + + +def load_pattern(state, addr, n): + return state.solver.eval(state.memory.load(addr, n), cast_to=bytes) + + +class TestHeapLazyMapping(unittest.TestCase): + """ + The heap region is mapped lazily and grown on demand; ``heap_size`` is the maximum extent, not the amount that + is mapped. + """ + + def test_blank_state_maps_only_the_initial_extent(self): + # THE regression guard: before the heap mapping became lazy, init_state() mapped all of heap_size (2048 + # UltraPages, ~17 MB of Python objects) for every single state, including the throwaway states CFG + # recovery makes by the thousand. + state = make_state() + page_size = state.memory.page_size + + assert state.heap._mapped_end == state.heap.heap_base + HEAP_INITIAL_MAPPED_SIZE + assert resident_heap_pages(state) == HEAP_INITIAL_MAPPED_SIZE // page_size + # a bare SimState maps nothing but the heap, so the heap pages are all the pages there are + assert len(state.memory._pages) == HEAP_INITIAL_MAPPED_SIZE // page_size + + def test_project_blank_state_maps_only_the_initial_extent(self): + # same guard, but through the path everything else in angr actually uses + proj = angr.load_shellcode(b"\x90" * 16, "amd64") + state = proj.factory.blank_state() + + assert state.heap._mapped_end == state.heap.heap_base + HEAP_INITIAL_MAPPED_SIZE + assert resident_heap_pages(state) == HEAP_INITIAL_MAPPED_SIZE // state.memory.page_size + + def test_growth_is_geometric(self): + state = make_state() + base = state.heap.heap_base + page_size = state.memory.page_size + + extents = [] + for _ in range(12): + state.heap._malloc(0x1000) + extents.append(state.heap._mapped_end - base) + # the mapped extent must always cover everything that has been handed out + assert state.heap._mapped_end >= state.heap.heap_location + assert resident_heap_pages(state) == (state.heap._mapped_end - base) // page_size + + # 0x1000 handed out at a time against an 0x2000 initial extent that doubles every time it is overrun + expected = [0x2000, 0x2000, 0x4000, 0x4000, 0x8000, 0x8000, 0x8000, 0x8000, 0x10000, 0x10000, 0x10000, 0x10000] + assert extents == expected + # ...which is exactly HEAP_INITIAL_MAPPED_SIZE scaled by powers of the growth factor + for extent in extents: + size = HEAP_INITIAL_MAPPED_SIZE + while size < extent: + size *= HEAP_MAPPING_GROWTH_FACTOR + assert size == extent + + def test_growth_uses_a_logarithmic_number_of_mapping_operations(self): + # geometric growth is what keeps a program that really does use megabytes of heap from paying a mapping + # operation per allocation + state = make_state() + calls = [] + real_map_range = state.heap._map_range + state.heap._map_range = lambda start, end: (calls.append((start, end)), real_map_range(start, end))[1] + + for _ in range(4096): + state.heap._malloc(0x400) + + assert state.heap._mapped_end == state.heap.heap_base + 4096 * 0x400 + assert 1 <= len(calls) <= 16, f"{len(calls)} mapping operations for 4096 allocations is not logarithmic" + + def test_growth_never_maps_past_heap_size(self): + state = make_state(SimHeapBrk(heap_base=0xD0000000, heap_size=0x10000)) + region_end = state.heap.heap_base + state.heap.heap_size + + state.heap._malloc(0x100000) # sixteen times the whole region + assert state.heap._mapped_end == region_end + assert resident_heap_pages(state) == state.heap.heap_size // state.memory.page_size + + state.heap._malloc(0x100000) + assert state.heap._mapped_end == region_end + + def test_allocation_landing_exactly_on_a_growth_boundary(self): + state = make_state() + base = state.heap.heap_base + + # exactly fills the initial extent: nothing has run past the end yet, so nothing grows + first = state.heap._malloc(HEAP_INITIAL_MAPPED_SIZE) + assert first == base + assert state.heap._mapped_end == base + HEAP_INITIAL_MAPPED_SIZE + + # the last byte of the initial extent must be usable + store_pattern(state, base + HEAP_INITIAL_MAPPED_SIZE - 16, 1, 16) + assert load_pattern(state, base + HEAP_INITIAL_MAPPED_SIZE - 16, 16) == pattern(1, 16) + + # ...and the very next byte is the one that triggers the growth + second = state.heap._malloc(16) + assert second == base + HEAP_INITIAL_MAPPED_SIZE + assert state.heap._mapped_end == base + HEAP_INITIAL_MAPPED_SIZE * HEAP_MAPPING_GROWTH_FACTOR + store_pattern(state, second, 2, 16) + assert load_pattern(state, second, 16) == pattern(2, 16) + + def test_many_small_allocations_keep_their_data(self): + state = make_state() + addrs = [state.heap._malloc(1) for _ in range(20000)] + for i, addr in enumerate(addrs): + state.memory.store(addr, claripy.BVV(i & 0xFF, 8)) + + assert state.heap._mapped_end >= state.heap.heap_location + bad = [i for i, addr in enumerate(addrs) if state.solver.eval(state.memory.load(addr, 1)) != (i & 0xFF)] + assert not bad, f"{len(bad)} of {len(addrs)} one-byte allocations lost their data" + + def test_one_large_allocation_maps_everything_it_needs(self): + state = make_state() + size = 4 * 1024 * 1024 + addr = state.heap._malloc(size) + + assert state.heap._mapped_end == state.heap.heap_base + size + assert resident_heap_pages(state) == size // state.memory.page_size + + # every page of it has to be usable, not just the first one + for off in range(0, size, 0x1000): + state.memory.store(addr + off, claripy.BVV(off, 32)) + bad = [off for off in range(0, size, 0x1000) if state.solver.eval(state.memory.load(addr + off, 4)) != off] + assert not bad, f"{len(bad)} pages of the large allocation lost their data" + + def test_symbolic_size_allocation_keeps_its_data(self): + state = make_state() + state.libc.max_variable_size = 0x8000 # otherwise a symbolic size is clamped to 128 bytes and maps nothing + size = claripy.BVS("size", 64) + state.solver.add(size.UGE(0x1000), size.ULE(0x5000)) + + addr = state.heap._malloc(size) + # the symbolic size is maximized, so the whole 0x5000 has to be backed + assert state.heap._mapped_end >= addr + 0x5000 + for i, off in enumerate((0, 0x1000, 0x2FFF, 0x5000 - 64)): + store_pattern(state, addr + off, i, 64) + for i, off in enumerate((0, 0x1000, 0x2FFF, 0x5000 - 64)): + assert load_pattern(state, addr + off, 64) == pattern(i, 64) + + def test_release_does_not_shrink_and_does_not_lose_data(self): + # deliberate: release() does not unmap, because unmapping would discard the page contents and a release + # followed by an allocation would hand back blank memory instead of the old bytes + state = make_state() + first = state.heap._malloc(0x4000) + store_pattern(state, first + 0x3000, 5, 64) + extent_before = state.heap._mapped_end + + state.heap.release(0x4000) + assert state.heap.heap_location == state.heap.heap_base + assert state.heap._mapped_end == extent_before + assert resident_heap_pages(state) == (extent_before - state.heap.heap_base) // state.memory.page_size + + second = state.heap._malloc(0x4000) + assert second == first + assert load_pattern(state, second + 0x3000, 64) == pattern(5, 64) + + # over-releasing is clamped at the base and still does not unmap anything + state.heap.release(0x1000000) + assert state.heap.heap_location == state.heap.heap_base + assert state.heap._mapped_end == extent_before + + def test_copy_carries_the_mapped_extent(self): + state = make_state() + state.heap._malloc(0x9000) + copy = state.copy() + assert copy.heap._mapped_end == state.heap._mapped_end + assert resident_heap_pages(copy) == resident_heap_pages(state) + + def test_divergent_copies_do_not_interfere(self): + state = make_state() + common = [] + for i in range(64): + addr = state.heap._malloc(64) + store_pattern(state, addr, i, 64) + common.append((addr, i)) + + first, second = state.copy(), state.copy() + first_allocs, second_allocs = [], [] + for i in range(64): + addr = first.heap._malloc(0x200) + store_pattern(first, addr, i + 1000, 64) + first_allocs.append((addr, i + 1000)) + + addr = second.heap._malloc(0x2000) + store_pattern(second, addr, i + 5000, 64) + second_allocs.append((addr, i + 5000)) + + # the two copies really did grow to different extents, so this is exercising something + assert first.heap._mapped_end < second.heap._mapped_end + for copy, allocs in ((first, first_allocs), (second, second_allocs)): + for addr, i in common + allocs: + assert load_pattern(copy, addr, 64) == pattern(i, 64) + + def test_merge_takes_the_minimum_mapped_extent(self): + # understating the extent is safe (growing it skips pages that already exist); overstating it could leave + # a hole in the middle of the region that nothing would ever map + state = make_state() + state.heap._malloc(64) + first, second = state.copy(), state.copy() + first.heap._malloc(0x10000) + second.heap._malloc(0x40) + assert first.heap._mapped_end > second.heap._mapped_end + + merged, _, _ = first.merge(second) + assert merged.heap._mapped_end == min(first.heap._mapped_end, second.heap._mapped_end) + + # NOTE: SimHeapBrk._combine takes max() over `others` *excluding self*, so the merged break drops to the + # other state's value instead of the high-water mark. That is a pre-existing angr bug, unrelated to (and + # unchanged by) the lazy mapping; assert what actually happens rather than what should. + assert merged.heap.heap_location == second.heap.heap_location + + # whatever the extent ended up being, allocating out of the merged state still has to work + addr = merged.heap._malloc(0x20000) + assert addr >= merged.heap.heap_base + store_pattern(merged, addr, 7, 64) + store_pattern(merged, addr + 0x20000 - 64, 8, 64) + assert load_pattern(merged, addr, 64) == pattern(7, 64) + assert load_pattern(merged, addr + 0x20000 - 64, 64) == pattern(8, 64) + assert merged.heap._mapped_end >= merged.heap.heap_location + + def test_merge_with_an_unmanaged_extent_stays_unmanaged(self): + # bookkeeping-only path, so it is tested directly: if any side does not manage the mapping, neither does + # the result + first, second = SimHeapBrk(), SimHeapBrk() + first._mapped_end = first.heap_base + 0x4000 + second._mapped_end = None + first._combine_mapped_end([second]) + assert first._mapped_end is None + + def test_unpickling_a_state_from_before_lazy_mapping(self): + # states pickled before this change have no _mapped_end at all; back then init_state() mapped the whole + # region, so that is the extent they have to come back with + heap = SimHeapBrk() + old_style = dict(heap.__dict__) + del old_style["_mapped_end"] + + restored = SimHeapBrk.__new__(SimHeapBrk) + restored.__setstate__(old_style) + assert restored._mapped_end == restored.heap_base + restored.heap_size + + # and a state pickled *now* round-trips its actual extent + state = make_state() + state.heap._malloc(0x5000) + extent = state.heap._mapped_end + assert pickle.loads(pickle.dumps(state)).heap._mapped_end == extent + + def test_abstract_memory_does_not_manage_the_mapping(self): + state = SimState(arch="AMD64", add_options={angr.options.ABSTRACT_MEMORY}) + assert state.heap._mapped_end is None + # ...and _ensure_mapped stays a no-op forever after + state.heap._ensure_mapped(state.heap.heap_base + 0x100000) + assert state.heap._mapped_end is None + + def test_a_preexisting_mapping_is_left_alone(self): + state = make_state() + state.memory.map_region(0x50000000, 0x2000, 3) + pages_before = len(state.memory._pages) + + # somebody else owns this region, so the heap plugin must not map or grow anything in it + heap = SimHeapBrk(heap_base=0x50000000, heap_size=0x100000) + state.register_plugin("heap", heap) + assert heap._mapped_end is None + assert len(state.memory._pages) == pages_before + + heap._ensure_mapped(0x50000000 + 0x80000) + assert heap._mapped_end is None + assert len(state.memory._pages) == pages_before + + def test_gdb_set_heap_grows_the_extent(self): + state = make_state() + base = state.heap.heap_base + state.gdb.set_heap(os.path.join(gdb_data_location, "heap"), heap_base=base) + + dumped = os.path.getsize(os.path.join(gdb_data_location, "heap")) + assert state.heap.heap_location == base + dumped + assert state.heap._mapped_end >= base + dumped + assert resident_heap_pages(state) == (state.heap._mapped_end - base) // state.memory.page_size + + # the tail of the dump has to have landed in memory, not in a page that was never mapped + with open(os.path.join(gdb_data_location, "heap"), "rb") as f: + f.seek(dumped - 16) + tail = f.read(16) + assert load_pattern(state, base + dumped - 16, 16) == tail + + def test_strict_page_access_now_rejects_writes_past_the_allocations(self): + # DELIBERATE NARROWING: with the whole region mapped up front, a program could scribble anywhere in the + # 8 MB heap under STRICT_PAGE_ACCESS and get away with it. Now only what has been handed out (rounded up + # to the mapped extent) is backed, so writing past it faults. Pinned here so the change is visible. + state = make_state(add_options={angr.options.STRICT_PAGE_ACCESS}) + addr = state.heap._malloc(64) + beyond = state.heap.heap_base + HEAP_INITIAL_MAPPED_SIZE + assert state.heap._mapped_end == beyond + + # inside the allocation, and anywhere inside the mapped extent, is still fine + store_pattern(state, addr, 3, 64) + assert load_pattern(state, addr, 64) == pattern(3, 64) + state.memory.store(beyond - 1, claripy.BVV(0x5A, 8)) + + with self.assertRaises(angr.errors.SimSegfaultException): + state.memory.store(beyond, claripy.BVV(0x5A, 8)) + + def test_writes_past_the_extent_still_work_without_strict_page_access(self): + # the narrowing above is confined to STRICT_PAGE_ACCESS; the default memory still auto-maps + state = make_state() + beyond = state.heap.heap_base + HEAP_INITIAL_MAPPED_SIZE + state.memory.store(beyond, claripy.BVV(0x5A, 8)) + assert state.solver.eval(state.memory.load(beyond, 1)) == 0x5A + + +class TestHeapPTMallocLazyMapping(unittest.TestCase): + def test_final_word_page_is_mapped_at_init(self): + # this heap keeps the last chunk's usage flag in the final word of the region, which init_state writes, so + # that one page is mapped up front even though the extent stops far short of it + state = make_state(SimHeapPTMalloc()) + page_size = state.memory.page_size + region_end = state.heap.heap_base + state.heap.heap_size + final_pageno = (region_end - 1) // page_size + + assert state.heap._mapped_end == state.heap.heap_base + HEAP_INITIAL_MAPPED_SIZE + assert state.memory._pages.get(final_pageno) is not None + # the initial extent plus the single final page, and nothing else + assert resident_heap_pages(state) == HEAP_INITIAL_MAPPED_SIZE // page_size + 1 + + def test_malloc_free_and_reuse_across_a_growth_boundary(self): + state = make_state(SimHeapPTMalloc()) + base = state.heap.heap_base + + first = state.heap.malloc(0x1000) + second = state.heap.malloc(0x2000) # runs past the initial extent + third = state.heap.malloc(0x4000) # ...and past the next one too + assert first and second and third + assert state.heap._mapped_end > base + HEAP_INITIAL_MAPPED_SIZE + assert state.heap._mapped_end >= third + 0x4000 + + allocs = {first: 0x1000, second: 0x2000, third: 0x4000} + for i, (addr, size) in enumerate(allocs.items()): + store_pattern(state, addr, i, 64) + store_pattern(state, addr + size - 64, i + 100, 64) + chunks_before = [(c.base, c.is_free()) for c in state.heap.chunks()] + + # free the middle chunk and hand the same space back out + state.heap.free(second) + again = state.heap.malloc(0x2000) + assert again == second + store_pattern(state, again, 50, 64) + + # the untouched allocations on either side of the freed one kept their contents + for i, (addr, size) in enumerate(allocs.items()): + if addr == second: + continue + assert load_pattern(state, addr, 64) == pattern(i, 64) + assert load_pattern(state, addr + size - 64, 64) == pattern(i + 100, 64) + assert load_pattern(state, again, 64) == pattern(50, 64) + + # the chunk walk has to still terminate and describe the same heap + chunks_after = [(c.base, c.is_free()) for c in state.heap.chunks()] + assert chunks_before + assert [c[0] for c in chunks_after] == [c[0] for c in chunks_before] + + def test_large_allocation_maps_the_whole_region(self): + state = make_state(SimHeapPTMalloc()) + addr = state.heap.malloc(4 * 1024 * 1024) + assert addr + # the chunk plus the following chunk's metadata runs past half the region, so the extent doubles to cover + # all of it + assert state.heap._mapped_end == state.heap.heap_base + state.heap.heap_size + + store_pattern(state, addr, 11, 64) + store_pattern(state, addr + 4 * 1024 * 1024 - 64, 12, 64) + assert load_pattern(state, addr, 64) == pattern(11, 64) + assert load_pattern(state, addr + 4 * 1024 * 1024 - 64, 64) == pattern(12, 64) + + def test_many_small_allocations_keep_their_data(self): + state = make_state(SimHeapPTMalloc()) + allocs = [] + for i in range(200): + addr = state.heap.malloc(0x100) + assert addr + store_pattern(state, addr, i, 64) + allocs.append((addr, i)) + + assert state.heap._mapped_end > state.heap.heap_base + HEAP_INITIAL_MAPPED_SIZE + for addr, i in allocs: + assert load_pattern(state, addr, 64) == pattern(i, 64) + + +if __name__ == "__main__": + unittest.main() From 22613f4a0a23e85d58b9d0ee11116696afdf348e Mon Sep 17 00:00:00 2001 From: Fish Date: Mon, 27 Jul 2026 18:20:59 -0700 Subject: [PATCH 067/122] CompleteCallingConventions: fail loudly when all workers die. (#6718) --- angr/analyses/complete_calling_conventions.py | 34 ++++++++++++++++- .../test_calling_convention_analysis.py | 38 ++++++++++++++++++- .../cfg/test_spilling_cfg_graph.py | 38 +++++++++++++++++++ 3 files changed, 107 insertions(+), 3 deletions(-) diff --git a/angr/analyses/complete_calling_conventions.py b/angr/analyses/complete_calling_conventions.py index 03019fc37..099492046 100644 --- a/angr/analyses/complete_calling_conventions.py +++ b/angr/analyses/complete_calling_conventions.py @@ -17,6 +17,7 @@ from angr.analyses.analysis import Analysis, register_analysis from angr.analyses.calling_convention import CallingConventionAnalysis from angr.analyses.cfg import CFGFast from angr.analyses.variable_recovery import VariableRecoveryFast +from angr.errors import AngrRuntimeError from angr.knowledge_plugins.cfg import CFGModel from angr.knowledge_plugins.functions.function import PrototypeSource from angr.simos import SimWindows @@ -34,6 +35,10 @@ _l = logging.getLogger(name=__name__) _mp_context = mp_context() +# How long (in seconds) the result collector keeps polling the results queue after it observes that every worker +# process has exited. +DEAD_WORKER_GRACE_PERIOD = 5.0 + class CallingConventionAnalysisMode(Enum): """ @@ -295,6 +300,9 @@ class CompleteCallingConventionsAnalysis(Analysis): self._update_progress(0) idx = 0 assert self._results_lock is not None + # the timestamp when we first noticed that all workers had exited. + # None while at least one worker is still alive. + all_workers_dead_since: float | None = None while idx < total_funcs: try: with self._results_lock: @@ -302,9 +310,26 @@ class CompleteCallingConventionsAnalysis(Analysis): True, timeout=0.01 ) except queue.Empty: + # No result is available right now. + # If all workers have exited then no result will ever become available, and looping here would + # hang forever (angr #6529). Detect that and fail loudly. + if any(proc.is_alive() for proc in procs): + all_workers_dead_since = None + elif all_workers_dead_since is None: + all_workers_dead_since = time.time() + elif time.time() - all_workers_dead_since >= DEAD_WORKER_GRACE_PERIOD: + exitcodes = ", ".join(f"{proc.name}: {proc.exitcode}" for proc in procs) + raise AngrRuntimeError( + f"All {len(procs)} CompleteCallingConventions worker processes exited before the " + f"analysis finished; only {idx} of {total_funcs} functions were analyzed. Worker exit " + f"codes: {exitcodes}." + ) from None time.sleep(0.1) continue + # we made progress, so any previously observed all-dead state is no longer interesting + all_workers_dead_since = None + func = self.kb.functions.get_by_addr(func_addr) if cc is not None or proto is not None: func.calling_convention = cc @@ -376,8 +401,13 @@ class CompleteCallingConventionsAnalysis(Analysis): except Exception: # pylint:disable=broad-except _l.error("Worker %d: Exception occurred during _analyze_core().", worker_id, exc_info=True) cc, proto, proto_libname, proto_source, varman = None, None, None, None, None - with self._results_lock: - self._results.put((func_addr, cc, proto, proto_libname, proto_source, varman)) + try: + with self._results_lock: + self._results.put((func_addr, cc, proto, proto_libname, proto_source, varman)) + except Exception: # pylint:disable=broad-except + _l.error( + "Worker %d: Failed to report the result for function %#x.", worker_id, func_addr, exc_info=True + ) def _analyze_core( self, func_addr: int diff --git a/tests/analyses/test_calling_convention_analysis.py b/tests/analyses/test_calling_convention_analysis.py index 5c4583f09..2064bc408 100755 --- a/tests/analyses/test_calling_convention_analysis.py +++ b/tests/analyses/test_calling_convention_analysis.py @@ -5,19 +5,25 @@ __package__ = __package__ or "tests.analyses" # pylint:disable=redefined-builti import logging import os +import time import unittest from functools import wraps import archinfo import angr -from angr.analyses.complete_calling_conventions import CallingConventionAnalysisMode +from angr.analyses.complete_calling_conventions import ( + DEAD_WORKER_GRACE_PERIOD, + CallingConventionAnalysisMode, + CompleteCallingConventionsAnalysis, +) from angr.calling_conventions import ( SimCCCdecl, SimCCSystemVAMD64, SimRegArg, SimStackArg, ) +from angr.errors import AngrRuntimeError from angr.sim_type import SimTypeBottom, SimTypeFloat, SimTypeFunction, SimTypeInt, SimTypeLongLong from tests.common import bin_location, requires_binaries_private @@ -339,6 +345,36 @@ class TestCallingConventionAnalysis(unittest.TestCase): if not (func.is_alignment or func.is_simprocedure or func.is_plt) ) + def test_dead_workers_raise_instead_of_hanging(self): + """ + Regression test for angr issue #6529. + + If every worker process dies, the result-collection loop used to spin forever waiting for results that could + never arrive - the analysis "hung forever" with no error. It must now fail loudly instead. Workers can die + for reasons entirely outside this analysis (an unpicklable analysis under the "spawn" start method, an OOM + kill, a segfault in a native lifter), so we simply make the worker routine die on purpose. + """ + binary_path = os.path.join(test_location, "x86_64", "fauxware") + proj = angr.Project(binary_path, auto_load_libs=False, load_debug_info=False) + cfg = proj.analyses.CFG(normalize=True) + + def _dying_worker_routine(self, worker_id, initializer): # pylint:disable=unused-argument + raise RuntimeError(f"simulated worker {worker_id} crash") + + original = CompleteCallingConventionsAnalysis._worker_routine + CompleteCallingConventionsAnalysis._worker_routine = _dying_worker_routine + try: + start = time.time() + with self.assertRaises(AngrRuntimeError) as ctx: + proj.analyses.CompleteCallingConventions(cfg=cfg.model, workers=2) + elapsed = time.time() - start + finally: + CompleteCallingConventionsAnalysis._worker_routine = original + + assert "worker processes exited" in str(ctx.exception) + # it must give up promptly rather than hang; the grace period is the only intentional delay + assert elapsed < DEAD_WORKER_GRACE_PERIOD + 60, f"took {elapsed:.1f}s to notice that all workers had died" + @cca_mode("fast,variables") def test_tail_calls(self, *, mode): for opt_level in (1, 2): diff --git a/tests/knowledge_plugins/cfg/test_spilling_cfg_graph.py b/tests/knowledge_plugins/cfg/test_spilling_cfg_graph.py index fe908861f..fd2894b6d 100644 --- a/tests/knowledge_plugins/cfg/test_spilling_cfg_graph.py +++ b/tests/knowledge_plugins/cfg/test_spilling_cfg_graph.py @@ -7,6 +7,7 @@ from __future__ import annotations __package__ = __package__ or "tests.knowledge_plugins.cfg" # pylint:disable=redefined-builtin import os +import pickle import unittest import angr @@ -322,5 +323,42 @@ class TestCFGModelIntegration(unittest.TestCase): assert len(nodes_list) == len(cfg.model.graph), "Count should match" +class TestSpillingCFGPickling(unittest.TestCase): + """Regression tests for pickling a CFG whose nodes/edges spill to LMDB (angr issue #6529).""" + + @classmethod + def setUpClass(cls): + cls.bin_path = os.path.join(test_location, "x86_64", "fauxware") + + def test_pickle_roundtrip_with_eviction(self): + """ + Round-tripping a spilling CFG through pickle must not lose nodes or edges. + """ + proj = angr.Project(self.bin_path) + cfg = proj.analyses.CFGFast(normalize=True) + model = cfg.model + + total = len(model.graph) + if total <= 15: + self.skipTest("Binary too small to trigger eviction during unpickling") + + # shrink the limits so that re-inserting the nodes on the unpickle side triggers LRU eviction + model.graph.cache_limit = 5 + model.graph.db_batch_size = 10 + model.graph._graph._adj._cache_limit = 5 + model.graph._graph._adj._db_batch_size = 10 + + nodes_before = {(n.addr, n.size) for n in model.graph.nodes()} + edges_before = {(s.addr, d.addr) for s, d in model.graph.edges()} + + restored = pickle.loads(pickle.dumps(model)) + + nodes_after = {(n.addr, n.size) for n in restored.graph.nodes()} + edges_after = {(s.addr, d.addr) for s, d in restored.graph.edges()} + + assert nodes_after == nodes_before, f"Lost nodes across pickling: {sorted(nodes_before - nodes_after)[:10]}" + assert edges_after == edges_before, f"Lost edges across pickling: {sorted(edges_before - edges_after)[:10]}" + + if __name__ == "__main__": unittest.main() From 4462c849b9cd329a05c06fad263389d15c1433a0 Mon Sep 17 00:00:00 2001 From: Audrey Dutcher Date: Tue, 28 Jul 2026 19:52:00 -0700 Subject: [PATCH 068/122] RegionIdentifier: typecheck post-regionoverlay (#6664) --- angr/analyses/decompiler/region_identifier.py | 42 +-- angr/analyses/decompiler/region_overlay.py | 239 ++++++++++-------- 2 files changed, 157 insertions(+), 124 deletions(-) diff --git a/angr/analyses/decompiler/region_identifier.py b/angr/analyses/decompiler/region_identifier.py index b211110d1..01fcf5bb2 100644 --- a/angr/analyses/decompiler/region_identifier.py +++ b/angr/analyses/decompiler/region_identifier.py @@ -19,7 +19,7 @@ from angr.utils.doms import IncrementalDominators from angr.utils.graph import GraphUtils, dfs_back_edges, dominates, subgraph_between_nodes from .condition_processor import ConditionProcessor -from .region_overlay import OverlayManager, RegionOverlay +from .region_overlay import OverlayManager, RegionOverlay, Tx from .structurer_nodes import ConditionNode, IncompleteSwitchCaseHeadStatement, MultiNode from .utils import copy_graph, first_nonlabel_nonphi_statement, replace_last_statement @@ -29,7 +29,9 @@ l = logging.getLogger(name=__name__) # an ever-incrementing counter CONDITIONNODE_ADDR = count(0xFF000000) -type TNode = Block | RegionOverlay | MultiNode | ConditionNode +type TNode = Tx[Block | MultiNode | ConditionNode] +type TOverlay = RegionOverlay[Block | MultiNode | ConditionNode] +type TManager = OverlayManager[Block | MultiNode | ConditionNode] type TGraph = "networkx.DiGraph[TNode]" @@ -78,8 +80,8 @@ class RegionIdentifier(Analysis): # copy the graph so updates don't affect the original graph graph = copy_graph(graph) # type: ignore - self.region: RegionOverlay | None = None - self.overlay_manager: OverlayManager | None = None + self.region: TOverlay | None = None + self.overlay_manager: TManager | None = None self._start_node = None self._loop_headers: list | None = None self.regions_by_block_addrs = [] @@ -89,7 +91,7 @@ class RegionIdentifier(Analysis): self._expose_loop_head_backedges = expose_loop_head_backedges # we keep a dictionary of node and their traversal order in a quasi-topological traversal and update this # dictionary as we update the graph - self._node_order: dict[Any, tuple[int, int]] = {} + self._node_order: dict[TNode, tuple[int, int]] = {} self._graph = self._analyze(graph) @@ -182,11 +184,11 @@ class RegionIdentifier(Analysis): """ assert self.region is not None - work_list: list[RegionOverlay] = [self.region] + work_list: list[TOverlay] = [self.region] block_only_regions = [] seen_regions = set() while work_list: - children_regions: list[RegionOverlay] = [] + children_regions: list[TOverlay] = [] for region in work_list: children_blocks = [] for node in region.members: @@ -198,7 +200,7 @@ class RegionIdentifier(Analysis): if node not in seen_regions: children_regions.append(node) children_blocks.append( - (node.head.addr, node.head.idx if hasattr(node.head, "idx") else None) + (node.head.addr, node.head.idx if hasattr(node.head, "idx") else None) # type: ignore ) seen_regions.add(node) else: @@ -459,11 +461,11 @@ class RegionIdentifier(Analysis): return refined_loop_nodes, refined_exit_nodes - def _make_regions(self, graph: TGraph) -> RegionOverlay: + def _make_regions(self, graph: TGraph) -> TOverlay: assert self.overlay_manager is not None root = self.overlay_manager.root structured_loop_headers = set() - new_regions: list[RegionOverlay] = [] + new_regions: list[TOverlay] = [] # FIXME: _get_start_node() will fail if the graph is just a loop @@ -518,6 +520,7 @@ class RegionIdentifier(Analysis): # No more loops left. Structure acyclic regions. while new_regions: region = new_regions.pop(0) + assert region.head is not None head = region.head # collapse a working copy of the region body during acyclic region identification; for the root region, # the phase-1 working graph already matches its member-level view @@ -618,7 +621,7 @@ class RegionIdentifier(Analysis): return region - def _refine_loop_successors_to_guarded_successors(self, region: RegionOverlay, graph: TGraph): + def _refine_loop_successors_to_guarded_successors(self, region: TOverlay, graph: TGraph): """ If there are multiple successors of a loop, convert them into guarded successors. Eventually there should be only one loop successor. This is used in the DREAM structuring algorithm. @@ -747,7 +750,7 @@ class RegionIdentifier(Analysis): self, head: TNode, graph: TGraph, - parent_region: RegionOverlay, + parent_region: TOverlay, failed_region_attempts: set[tuple[TNode, TNode]], cyclic: bool, ): @@ -873,7 +876,7 @@ class RegionIdentifier(Analysis): return region_created @staticmethod - def _update_graph(graph: TGraph, new_region: RegionOverlay, replaced_nodes: set[TNode]) -> None: + def _update_graph(graph: TGraph, new_region: TOverlay, replaced_nodes: set[TNode]) -> None: region_in_edges = RegionIdentifier._region_in_edges(graph, new_region, data=True) region_out_edges = RegionIdentifier._region_out_edges(graph, new_region, data=True) for node in replaced_nodes: @@ -954,7 +957,7 @@ class RegionIdentifier(Analysis): @staticmethod def _abstract_acyclic_region( graph: TGraph, - region: RegionOverlay, + region: TOverlay, frontier: set[TNode], node_order: dict[TNode, tuple[int, int]], dummy_endnode: TNode | None = None, @@ -993,7 +996,7 @@ class RegionIdentifier(Analysis): normal_exit_node: TNode | None, abnormal_exit_nodes: set[TNode], node_order: dict[TNode, tuple[int, int]], - ) -> RegionOverlay: + ) -> TOverlay: loop_nodes = set(loop_nodes) region = self._parent_overlay_of(head).create_subregion(head, loop_nodes, cyclic=True) @@ -1033,7 +1036,7 @@ class RegionIdentifier(Analysis): return region - def _parent_overlay_of(self, node: TNode) -> RegionOverlay: + def _parent_overlay_of(self, node: TNode) -> TOverlay: """Find the overlay that the given working-graph node is currently a direct member of.""" if isinstance(node, RegionOverlay): assert node.parent is not None @@ -1046,19 +1049,19 @@ class RegionIdentifier(Analysis): @overload @staticmethod def _region_in_edges( - graph: TGraph, region: RegionOverlay, data: Literal[True] + graph: TGraph, region: TOverlay, data: Literal[True] ) -> list[tuple[TNode, TNode, dict[str, Any]]]: ... @overload @staticmethod - def _region_in_edges(graph: TGraph, region: RegionOverlay, data: Literal[False]) -> list[tuple[TNode, TNode]]: ... + def _region_in_edges(graph: TGraph, region: TOverlay, data: Literal[False]) -> list[tuple[TNode, TNode]]: ... @staticmethod def _region_in_edges(graph, region, data=False): return list(graph.in_edges(region.head, data=data)) @staticmethod - def _region_out_edges(graph, region: RegionOverlay, data=False): + def _region_out_edges(graph, region: TOverlay, data=False): out_edges = [] for node in region.members: out_ = graph.out_edges(node, data=data) @@ -1130,6 +1133,7 @@ class RegionIdentifier(Analysis): def _ensure_jump_at_loop_exit_ends(self, node: TNode) -> None: if isinstance(node, Block): + assert node.original_size is not None if not node.statements: node.statements.append( Jump( diff --git a/angr/analyses/decompiler/region_overlay.py b/angr/analyses/decompiler/region_overlay.py index 28eb8a711..034adddde 100644 --- a/angr/analyses/decompiler/region_overlay.py +++ b/angr/analyses/decompiler/region_overlay.py @@ -1,18 +1,30 @@ -# pylint:disable=protected-access +# pylint:disable=protected-access,invalid-sequence-index,unsubscriptable-object from __future__ import annotations import logging import os from collections import defaultdict -from collections.abc import Callable, Iterable, Iterator, Mapping -from typing import Any +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set +from typing import TYPE_CHECKING, Any, Protocol, cast import networkx l = logging.getLogger(name=__name__) -class OverlayManager: +class RegionBound(Protocol): + """ + The protocol indicating the minimum capabilities of graph nodes to be managed by an overlay region. + """ + + @property + def addr(self) -> int: ... + + +type Tx[U: RegionBound] = "U | RegionOverlay[U]" + + +class OverlayManager[T: RegionBound]: """ OverlayManager owns the single shared control-flow graph that all RegionOverlay objects are views of, plus the node-to-innermost-overlay ownership map. @@ -32,8 +44,8 @@ class OverlayManager: "root", ) - def __init__(self, graph: networkx.DiGraph, expose_loop_head_backedges: bool = False): - self.graph = graph + def __init__(self, graph: networkx.DiGraph[T], expose_loop_head_backedges: bool = False): + self.graph = cast("networkx.DiGraph[Tx[T]]", graph) self.expose_loop_head_backedges = expose_loop_head_backedges self._version: int = 0 # per-node topology version: bumped when an edge incident to a node changes (in the shared graph or in the @@ -42,26 +54,27 @@ class OverlayManager: # flip node ownership/representatives broadly (create_subregion/dissolve/finalize) and by rollback (whose # inverse closures mutate the graph outside the touch-instrumented primitives); a change forces a full # cache clear. - self._node_version: dict[Any, int] = {} + self._node_version: dict[Tx[T], int] = {} self._adj_epoch: int = 0 self._undo_log: list[Callable[[], None]] | None = None + # NOTE: this is the only callsite w/ head=None. If we can add a head param to this constructor we can clean up self.root = RegionOverlay(self, None, cyclic=False) self.root._under = set(graph) self.root._members = set(graph) - self._owner: dict[Any, RegionOverlay] = dict.fromkeys(graph, self.root) + self._owner: dict[Tx[T], RegionOverlay[T]] = dict.fromkeys(graph, self.root) @property def version(self) -> int: return self._version - def owner_of(self, node) -> RegionOverlay | None: + def owner_of(self, node: Tx[T]) -> RegionOverlay[T] | None: return self._owner.get(node) def _bump(self) -> None: self._version += 1 - def _touch(self, node) -> None: + def _touch(self, node: Tx[T]) -> None: """Bump a node's topology version (its cached view-adjacency must be rebuilt). Also bump every enclosing overlay: in an enclosing region's view the node is represented by the child overlay that contains it, so a change to the node's edges changes that representative's adjacency too. Monotonic; not undone on rollback, @@ -188,7 +201,7 @@ class OverlayManager: self._record(inverse) -class RegionOverlay: +class RegionOverlay[T: RegionBound]: """ A single-entry region marked over the shared graph held by an OverlayManager. The region tree is built out of RegionOverlay objects (RegionIdentifier emits them) and they are the only region type the decompiler uses. @@ -231,8 +244,8 @@ class RegionOverlay: def __init__( self, - mgr: OverlayManager, - head, + mgr: OverlayManager[T], + head: Tx[T] | None, cyclic: bool, cyclic_ancestor: bool = False, parent: RegionOverlay | None = None, @@ -243,16 +256,16 @@ class RegionOverlay: self.cyclic_ancestor = cyclic_ancestor self.parent = parent self.children: list[RegionOverlay] = [] - self._members: set = set() - self._under: set = set() + self._members: set[Tx[T]] = set() + self._under: set[T] = set() # edges (pairs of shared-graph nodes) hidden from this overlay's views only - self._hidden: set[tuple[Any, Any]] = set() + self._hidden: set[tuple[Tx[T], Tx[T]]] = set() # view-level edge pairs hidden from the with-successors view only - self._hidden_full: set[tuple[Any, Any]] = set() + self._hidden_full: set[tuple[Tx[T], Tx[T]]] = set() # view-level edge pairs injected into the with-successors view only (successor absorption) - self._extra_full_edges: set[tuple[Any, Any]] = set() + self._extra_full_edges: set[tuple[Tx[T], Tx[T]]] = set() # scratch edge marks (e.g., Phoenix's cyclic_refinement_outgoing), scoped to this overlay - self.edge_marks: defaultdict[str, set[tuple[Any, Any]]] = defaultdict(set) + self.edge_marks: defaultdict[str, set[tuple[Tx[T], Tx[T]]]] = defaultdict(set) # the node this overlay was finalized into, if any self.replacement = None @@ -267,6 +280,8 @@ class RegionOverlay: @property def addr(self): + if self.head is None: + raise TypeError("Region with no head has no address") return self.head.addr # @@ -274,11 +289,11 @@ class RegionOverlay: # @property - def manager(self) -> OverlayManager: + def manager(self) -> OverlayManager[T]: return self._mgr @property - def members(self) -> set: + def members(self) -> Set[Tx[T]]: return self._members def ancestors(self) -> set[RegionOverlay]: @@ -290,15 +305,17 @@ class RegionOverlay: node = node.parent return result - def underlying_nodes(self) -> set: + def underlying_nodes(self) -> Set[T]: """All shared-graph nodes inside this region (including nodes of nested regions).""" return self._under @staticmethod - def _underlying(x) -> set: + def _underlying(x) -> Set[T]: return x._under if isinstance(x, RegionOverlay) else {x} - def create_subregion(self, head, members: Iterable, cyclic: bool, cyclic_ancestor: bool = False) -> RegionOverlay: + def create_subregion( + self, head: Tx[T], members: Iterable, cyclic: bool, cyclic_ancestor: bool = False + ) -> RegionOverlay[T]: """ Carve a new child overlay out of this overlay. ``members`` must be a subset of this overlay's members (shared-graph nodes owned by this overlay and/or existing child overlays); ``head`` must be one of them. @@ -376,10 +393,10 @@ class RegionOverlay: # Derived views # - def _is_hidden(self, src, dst) -> bool: + def _is_hidden(self, src: Tx[T], dst: Tx[T]) -> bool: return (src, dst) in self._hidden - def _hidden_context_head_under(self) -> frozenset | set: + def _hidden_context_head_under(self) -> Set[Tx[T]]: """ Crossing edges that target the head of the region's processing context (the nearest cyclic ancestor, or the root region) were invisible during region identification: in-edges of the head are stripped before @@ -397,7 +414,7 @@ class RegionOverlay: return frozenset() return self._underlying(anc.head) - def _crossing_out_edges(self) -> Iterator[tuple[Any, Any, dict]]: + def _crossing_out_edges(self) -> Iterator[tuple[Tx[T], Tx[T], dict[str, Any]]]: """All shared-graph edges leaving this region, except hidden ones.""" graph = self._mgr.graph under = self._under @@ -413,7 +430,7 @@ class RegionOverlay: def _in_loop(self) -> bool: return self.cyclic or self.cyclic_ancestor - def successor_nodes(self) -> set: + def successor_nodes(self) -> Set[Tx[T]]: """ The derived successor set of this region: representatives of all shared-graph nodes targeted by edges leaving the region. @@ -429,14 +446,14 @@ class RegionOverlay: self._cache_succs = (self._mgr.version, succs) return succs - def _quotient_edges(self, with_successors: bool) -> Iterator[tuple[Any, Any, dict]]: + def _quotient_edges(self, with_successors: bool) -> Iterator[tuple[Tx[T], Tx[T], dict[str, Any]]]: """ Derive the edges of the region view (member -> member, and if requested member -> successor and successor -> successor) from the shared graph. """ graph = self._mgr.graph under = self._under - member_of: dict[Any, Any] = {} + member_of: dict[Tx[T], Tx[T]] = {} for m in self._members: for n in self._underlying(m): member_of[n] = m @@ -483,7 +500,7 @@ class RegionOverlay: def view_graph( self, full: bool = False, include_marked: bool = False, blacklisted_edges: frozenset = frozenset() - ) -> RegionOverlayGraph: + ) -> RegionOverlayGraph[T]: """A zero-copy, networkx-compatible view of this region (see RegionOverlayGraph).""" if blacklisted_edges: return RegionOverlayGraph( @@ -501,7 +518,7 @@ class RegionOverlay: # node so that RegionOverlayGraph can answer adjacency queries without materializing anything. # - def _iter_view_out_edges(self, n, full: bool) -> Iterator[tuple[Any, dict]]: + def _iter_view_out_edges(self, n, full: bool) -> Iterator[tuple[Tx[T], dict[str, Any]]]: """Visible out-edges of view node ``n`` (a member, or a successor in the full view), deduplicated.""" graph = self._mgr.graph under = self._under @@ -522,7 +539,7 @@ class RegionOverlay: if rep_v not in seen: seen.add(rep_v) yield rep_v, data - elif full and v not in hidden_head: + elif hidden_head is not None and v not in hidden_head: rep_v = self._representative_outside(v) if rep_v is not None and rep_v not in seen: seen.add(rep_v) @@ -560,7 +577,7 @@ class RegionOverlay: seen.add(v) yield v, {} - def _iter_view_in_edges(self, n, full: bool) -> Iterator[tuple[Any, dict]]: + def _iter_view_in_edges(self, n: Tx[T], full: bool) -> Iterator[tuple[Tx[T], dict[str, Any]]]: """Visible in-edges of view node ``n``, deduplicated. The transpose of _iter_view_out_edges.""" graph = self._mgr.graph under = self._under @@ -621,21 +638,21 @@ class RegionOverlay: seen.add(u) yield u, {} - def view(self) -> RegionOverlayGraph: + def view(self) -> RegionOverlayGraph[T]: """The region graph (members only): a zero-copy networkx-compatible view; treat it as read-only.""" return self.view_graph(full=False) - def view_with_successors(self) -> RegionOverlayGraph: + def view_with_successors(self) -> RegionOverlayGraph[T]: """The region graph including successor nodes: a zero-copy view; treat it as read-only.""" return self.view_graph(full=True) @property - def raw_graph(self) -> RegionOverlayGraph: + def raw_graph(self) -> RegionOverlayGraph[T]: """The member view including marked edges (the old graph with cyclic_refinement_outgoing attrs present).""" return self.view_graph(full=False, include_marked=True) @property - def raw_graph_with_successors(self) -> RegionOverlayGraph: + def raw_graph_with_successors(self) -> RegionOverlayGraph[T]: """The with-successors view including marked edges.""" return self.view_graph(full=True, include_marked=True) @@ -644,15 +661,15 @@ class RegionOverlay: # @property - def graph(self) -> networkx.DiGraph: + def graph(self) -> networkx.DiGraph[Tx[T]]: return self.view() @property - def graph_with_successors(self) -> networkx.DiGraph: + def graph_with_successors(self) -> networkx.DiGraph[Tx[T]]: return self.view_with_successors() @property - def successors(self) -> set: + def successors(self) -> Set[Tx[T]]: return self.successor_nodes() @property @@ -710,7 +727,7 @@ class RegionOverlay: self._on_node_added(node) self._invalidate() - def remove_node(self, node, absorbed_into=None, absorb_out_edges: bool = True) -> None: + def remove_node(self, node: Tx[T], absorbed_into: Tx[T] | None = None, absorb_out_edges: bool = True) -> None: """ Remove a node. If ``node`` is a member (or a member overlay's node), it is removed from the shared graph for real. If it is a successor of this region, the removal is interpreted as hiding all edges from this @@ -728,7 +745,9 @@ class RegionOverlay: self.hide_edge_to_successor(node) return external_in_edges = [] - rewire_out_edges = [] # (dst, data, hide): edges to rewire onto absorbed_into in the shared graph + rewire_out_edges: list[ + tuple[Tx[T], dict[str, Any], bool] + ] = [] # (dst, data, hide): edges to rewire onto absorbed_into in the shared graph if absorbed_into is not None: external_in_edges = [ (src, data) @@ -753,13 +772,14 @@ class RegionOverlay: for src, data in external_in_edges: self._mgr.graph_add_edge(src, absorbed_into, **data) hidden_added = [] - for dst, data, hide in rewire_out_edges: - self._mgr.graph_add_edge(absorbed_into, dst, **data) - if hide and (absorbed_into, dst) not in self._hidden: - self._hidden.add((absorbed_into, dst)) - hidden_added.append((absorbed_into, dst)) - if hidden_added: - self._mgr._record(lambda: self._hidden.difference_update(hidden_added)) + if absorbed_into is not None: + for dst, data, hide in rewire_out_edges: + self._mgr.graph_add_edge(absorbed_into, dst, **data) + if hide and (absorbed_into, dst) not in self._hidden: + self._hidden.add((absorbed_into, dst)) + hidden_added.append((absorbed_into, dst)) + if hidden_added: + self._mgr._record(lambda: self._hidden.difference_update(hidden_added)) self._invalidate() def hide_edge_to_successor(self, succ) -> None: @@ -782,7 +802,7 @@ class RegionOverlay: self._mgr._touch(rep) self._invalidate() - def underlying_edge_pairs(self, src, dst) -> list[tuple[Any, Any]]: + def underlying_edge_pairs(self, src: Tx[T], dst: Tx[T]) -> list[tuple[Tx[T], Tx[T]]]: graph = self._mgr.graph under_src = self._underlying(src) under_dst = self._underlying(dst) @@ -795,7 +815,7 @@ class RegionOverlay: pairs.append((u, v)) return pairs - def add_edge(self, src, dst, **data) -> None: + def add_edge(self, src: Tx[T], dst: Tx[T], **data) -> None: """ Add a real edge to the shared graph. Overlay endpoints are resolved to underlying nodes: the destination resolves to its entry (head chain); overlay sources are not supported. Endpoints that are not in the @@ -805,6 +825,7 @@ class RegionOverlay: dst_ = dst while isinstance(dst_, RegionOverlay): dst_ = dst_.head + assert dst_ is not None if src not in self._mgr.graph: self.add_node(src) if dst_ not in self._mgr.graph: @@ -815,7 +836,7 @@ class RegionOverlay: self._mgr._record(lambda: self._hidden.add((src, dst_))) self._invalidate() - def detach_edge(self, src, dst) -> None: + def detach_edge(self, src: Tx[T], dst: Tx[T]) -> None: """ Remove an edge from the shared graph for real (e.g., when the edge has been virtualized into a goto). Overlay endpoints remove all underlying edges between the two node sets. @@ -833,7 +854,7 @@ class RegionOverlay: self._mgr._touch(dst) self._invalidate() - def mark_edge(self, src, dst, **attrs) -> None: + def mark_edge(self, src: Tx[T], dst: Tx[T], **attrs) -> None: """ Mark a view-level edge (e.g. cyclic_refinement_outgoing) so RegionOverlayGraph hides it by default. Marks live in overlay state, never reach the shared graph, and are remapped/cleared with the region. @@ -847,13 +868,13 @@ class RegionOverlay: self._mgr._touch(dst) self._invalidate() - def absorb_successor_into(self, succ, new_node) -> None: + def absorb_successor_into(self, succ: Tx[T], new_node: Tx[T]) -> None: """ Absorb a successor node into a structured member node in this region's with-successors view only (the successor still belongs to an enclosing region): the successor's view out-edges are re-attached to the member node as view-only extra edges, then the successor disappears from this region's views. """ - added = [] + added: list[tuple[Tx[T], Tx[T]]] = [] for dst, _ in self.view_with_successors().overlay._iter_view_out_edges(succ, full=True): if dst is not new_node and (new_node, dst) not in self._extra_full_edges: self._extra_full_edges.add((new_node, dst)) @@ -865,18 +886,18 @@ class RegionOverlay: self._mgr._touch(dst) self.hide_edge_to_successor(succ) - def drop_edge_marks_from(self, node, key) -> None: + def drop_edge_marks_from(self, node: Tx[T], key: str) -> None: """Clear marks on all out-edges of a node (the new_node after a replace), undoably.""" removed = [(u, v) for (u, v) in self.edge_marks[key] if u is node] if removed: self.edge_marks[key].difference_update(removed) - self._mgr._record(lambda: self.edge_marks.update(removed)) + self._mgr._record(lambda: self.edge_marks[key].update(removed)) self._mgr._touch(node) for _, v in removed: self._mgr._touch(v) self._invalidate() - def remove_edge_with_successors_only(self, src, dst) -> None: + def remove_edge_with_successors_only(self, src: Tx[T], dst: Tx[T]) -> None: """ Hide an edge from the with-successors view only, leaving the member view and the shared graph alone (a rare asymmetric bookkeeping pattern in Phoenix's switch-case structuring). @@ -888,11 +909,11 @@ class RegionOverlay: self._mgr._touch(dst) self._invalidate() - def hide_edge(self, src, dst) -> None: + def hide_edge(self, src: Tx[T], dst: Tx[T]) -> None: """ Remove an edge from this overlay's views only. Enclosing regions still see the underlying edge(s). """ - added = [] + added: list[tuple[Tx[T], Tx[T]]] = [] for u, v in self.underlying_edge_pairs(src, dst): if (u, v) not in self._hidden: self._hidden.add((u, v)) @@ -903,7 +924,9 @@ class RegionOverlay: self._mgr._touch(dst) self._invalidate() - def replace_nodes(self, old_node_0, new_node, old_node_1=None, self_loop: bool = True) -> None: + def replace_nodes( + self, old_node_0: Tx[T], new_node: Tx[T], old_node_1: Tx[T] | None = None, self_loop: bool = True + ) -> None: """ Replace one or two member nodes with a new node, preserving and rewiring all underlying edges (including edges from/to nodes outside this region, which is how results become visible to enclosing regions). @@ -961,12 +984,12 @@ class RegionOverlay: self._mgr._record(lambda: setattr(self, "head", old_head)) self._invalidate() - def _remap_bookkeeping(self, old_nodes: set, new_node) -> None: + def _remap_bookkeeping(self, old_nodes: set[Tx[T]], new_node: Tx[T]) -> None: """Remap hidden edges and edge marks that reference replaced nodes, here and in all enclosing overlays.""" - anc: RegionOverlay | None = self + anc = self while anc is not None: for attr in ("_hidden", "_hidden_full", "_extra_full_edges"): - pairs: set[tuple[Any, Any]] = getattr(anc, attr) + pairs: set[tuple[Tx[T], Tx[T]]] = getattr(anc, attr) stale = [(u, v) for u, v in pairs if u in old_nodes or v in old_nodes] if stale: remapped = [ @@ -1002,7 +1025,7 @@ class RegionOverlay: # Region lifecycle # - def snapshot_successors(self) -> set: + def snapshot_successors(self) -> set[Tx[T]]: """ Capture this region's structural successors and how many member edges reach each, taken before the region is structured. finalize() uses it to re-establish the region-to-successor edges that structuring removes @@ -1011,12 +1034,12 @@ class RegionOverlay: return set(self.successor_nodes()) @staticmethod - def _resolve_entry(node): + def _resolve_entry(node) -> T | None: while isinstance(node, RegionOverlay): node = node.head return node - def finalize(self, result_node=None, succ_snapshot=None): + def finalize(self, result_node: Tx[T] | None = None, succ_snapshot: set[Tx[T]] | None = None): """ Collapse this fully-structured region into its parent: the region must consist of a single member node (the structuring result), which takes the region's place among the parent's members. Returns that node. @@ -1053,6 +1076,7 @@ class RegionOverlay: s_entry = self._resolve_entry(s) if ( s_entry is not result_node + and s_entry is not None and s_entry is not parent_loop_head and s_entry in graph and not graph.has_edge(result_node, s_entry) @@ -1097,7 +1121,8 @@ class RegionOverlay: self._invalidate() return result_node - def collapse_to(self, result_node): + # NOTE: this function seems unused + def collapse_to(self, result_node: RegionOverlay[T]): """ Collapse this region into its parent by replacing all of its member nodes with a single external result node (the structuring result). Used by structurers that compute their result without destructively @@ -1116,8 +1141,8 @@ class RegionOverlay: underset = self._under # capture crossing edges (one endpoint inside the region, the other outside) before removing the members - in_edges: list[tuple[Any, dict]] = [] - out_edges: list[tuple[Any, dict]] = [] + in_edges: list[tuple[Tx[T], dict[str, Any]]] = [] + out_edges: list[tuple[Tx[T], dict[str, Any]]] = [] seen_in: set = set() seen_out: set = set() for u in under: @@ -1231,12 +1256,12 @@ class RegionOverlay: _PARANOID_ADJ_CHECK = bool(os.environ.get("ANGR_PARANOID_ADJ")) -class _OverlayNodeAtlas(Mapping): +class _OverlayNodeAtlas[T: RegionBound](Mapping[Tx[T], dict[str, Any]]): """Lazy node mapping of a RegionOverlayGraph: the overlay's view nodes, attributes from the shared graph.""" __slots__ = ("_rog",) - def __init__(self, rog: RegionOverlayGraph): + def __init__(self, rog: RegionOverlayGraph[T]): self._rog = rog def __len__(self): @@ -1258,12 +1283,12 @@ class _OverlayNodeAtlas(Mapping): return shared.nodes[n] if n in shared else {} -class _OverlayAdjInner(Mapping): +class _OverlayAdjInner[T: RegionBound](Mapping[Tx[T], dict[str, Any]]): """Adjacency of one view node: target -> edge data, derived on construction from the overlay.""" __slots__ = ("_d",) - def __init__(self, rog: RegionOverlayGraph, n, pred: bool): + def __init__(self, rog: RegionOverlayGraph[T], n, pred: bool): overlay = rog.overlay it = overlay._iter_view_in_edges(n, rog.full) if pred else overlay._iter_view_out_edges(n, rog.full) if pred: @@ -1284,26 +1309,26 @@ class _OverlayAdjInner(Mapping): return self._d[n] -class _OverlayAdjAtlas(Mapping): +class _OverlayAdjAtlas[T: RegionBound](Mapping[Tx[T], _OverlayAdjInner[T]]): """Outer adjacency mapping of a RegionOverlayGraph: view node -> _OverlayAdjInner.""" __slots__ = ("_cache", "_epoch", "_pred", "_rog", "_succ_cache", "_succ_version") - def __init__(self, rog: RegionOverlayGraph, pred: bool): + def __init__(self, rog: RegionOverlayGraph[T], pred: bool): self._rog = rog self._pred = pred # Phoenix queries the same node's adjacency repeatedly (.successors/.predecessors/.in_degree/.out_degree/ # .has_edge all route through here); cache the derived _OverlayAdjInner per node, keyed by that node's # topology version so an unrelated mutation does not evict it. _epoch tracks the manager's coarse epoch # (bumped by lifecycle ops / rollback) and forces a full clear when it changes. - self._cache: dict[Any, tuple[int, _OverlayAdjInner]] = {} + self._cache: dict[Tx[T], tuple[int, _OverlayAdjInner]] = {} self._epoch: int | None = None # successor-node adjacency cache: a successor's view adjacency depends on the whole region's successor # set and view state, not just on that node, so it cannot be keyed by the node's own version. Key the # whole cache by the manager's global version instead (bumped by every mutation and view-state change, # the same invariant _node_set relies on): reads between two mutations -- dominator fixpoints, SAILR's # per-edge in_degree queries -- hit the cache, and any mutation drops it wholesale. - self._succ_cache: dict[Any, _OverlayAdjInner] = {} + self._succ_cache: dict[Tx[T], _OverlayAdjInner] = {} self._succ_version: int | None = None def __len__(self): @@ -1363,7 +1388,7 @@ class _OverlayAdjAtlas(Mapping): return entry[1] -class RegionOverlayGraph[T](networkx.DiGraph): +class RegionOverlayGraph[T: RegionBound](networkx.DiGraph[Tx[T]] if TYPE_CHECKING else networkx.DiGraph): """ A read-only, networkx-compatible view of a RegionOverlay that stores no copy of the region's subgraph: all queries traverse the original shared graph through the overlay's membership. Compatible with every networkx @@ -1378,10 +1403,10 @@ class RegionOverlayGraph[T](networkx.DiGraph): def __init__( self, - overlay: RegionOverlay, + overlay: RegionOverlay[T], full: bool = False, include_marked: bool = False, - blacklisted_edges: frozenset[tuple[Any, Any]] = frozenset(), + blacklisted_edges: frozenset[tuple[Tx[T], Tx[T]]] = frozenset(), ): super().__init__() self.overlay = overlay @@ -1399,7 +1424,7 @@ class RegionOverlayGraph[T](networkx.DiGraph): # internals # - def _node_set(self) -> frozenset: + def _node_set(self) -> frozenset[Tx[T]]: version = self.overlay.manager.version cached = self._ns_cache if cached is not None and cached[0] == version: @@ -1414,14 +1439,14 @@ class RegionOverlayGraph[T](networkx.DiGraph): self._ns_cache = (version, ns) return ns - def _pair_visible(self, src, dst) -> bool: + def _pair_visible(self, src: Tx[T], dst: Tx[T]) -> bool: if not self.include_marked and any((src, dst) in marks for marks in self.overlay.edge_marks.values()): return False if (src, dst) in self.blacklisted_edges: return False return not (self.full and (src, dst) in self.overlay._hidden_full) - def _variant(self, fullgraph, all_edges) -> RegionOverlayGraph: + def _variant(self, fullgraph: bool | None, all_edges: bool | None = None) -> RegionOverlayGraph[T]: full = self.full if fullgraph is None else fullgraph include_marked = self.include_marked if all_edges is None else all_edges if full == self.full and include_marked == self.include_marked: @@ -1435,24 +1460,24 @@ class RegionOverlayGraph[T](networkx.DiGraph): # @property - def full_view(self) -> RegionOverlayGraph: + def full_view(self) -> RegionOverlayGraph[T]: """The with-successors sibling of this view (zero-copy).""" return self._variant(True, None) @property - def member_view(self) -> RegionOverlayGraph: + def member_view(self) -> RegionOverlayGraph[T]: """The members-only sibling of this view (zero-copy).""" return self._variant(False, None) - def with_all_edges(self) -> RegionOverlayGraph: + def with_all_edges(self) -> RegionOverlayGraph[T]: """A sibling view that includes edges marked through RegionOverlay.mark_edge.""" return self._variant(None, True) - def filtered(self) -> RegionOverlayGraph: + def filtered(self) -> RegionOverlayGraph[T]: """A sibling view that hides edges marked through RegionOverlay.mark_edge (the default).""" return self._variant(None, False) - def to_acyclic_by_order(self, node_order) -> RegionOverlayGraph: + def to_acyclic_by_order(self, node_order: Mapping[Tx[T], int]) -> RegionOverlayGraph[T]: """ An acyclic view of this graph, obtained by blacklisting back edges (edges whose source is ordered at or after their destination in ``node_order``). Replaces utils.graph.to_acyclic_graph without a copy. @@ -1460,20 +1485,19 @@ class RegionOverlayGraph[T](networkx.DiGraph): back_edges = [(u, v) for u, v in self.edges if node_order[u] >= node_order[v]] return self.to_acyclic(back_edges) - def to_acyclic(self, blacklisted_edges) -> RegionOverlayGraph: + def to_acyclic(self, blacklisted_edges: Sequence[tuple[Tx[T], Tx[T]]]) -> RegionOverlayGraph[T]: """ A new view with the given (view-level) edges additionally blacklisted; used to traverse the region as an acyclic graph without copying it. """ - extra = frozenset((u, v) for u, v in blacklisted_edges) return RegionOverlayGraph( self.overlay, full=self.full, include_marked=self.include_marked, - blacklisted_edges=self.blacklisted_edges | extra, + blacklisted_edges=self.blacklisted_edges.union(blacklisted_edges), ) - def reverse_view(self) -> RegionOverlayGraph: + def reverse_view(self) -> RegionOverlayGraph[T]: """The reversed view of this graph (zero-copy).""" g = RegionOverlayGraph( self.overlay, @@ -1481,13 +1505,14 @@ class RegionOverlayGraph[T](networkx.DiGraph): include_marked=self.include_marked, blacklisted_edges=frozenset({(v, u) for u, v in self.blacklisted_edges}), ) - g._succ, g._pred = g._pred, g._succ # swap the adjacency mappings to reverse the graph + # swap the adjacency mappings to reverse the graph + g._succ, g._pred = g._pred, g._succ # type: ignore # g._adj is synced with _succ return g - def materialize(self) -> networkx.DiGraph: + def materialize(self) -> networkx.DiGraph[Tx[T]]: """An independent networkx.DiGraph copy of this view.""" - g: networkx.DiGraph = networkx.DiGraph() + g: networkx.DiGraph[Tx[T]] = networkx.DiGraph() g.add_nodes_from(self._node_set()) for u in self._node_set(): for v, data in self._adj[u].items(): @@ -1499,24 +1524,24 @@ class RegionOverlayGraph[T](networkx.DiGraph): # use .full_view / .member_view for those) # - def edge_marked(self, u, v, mark_name: str | None = None) -> bool: + def edge_marked(self, u: Tx[T], v: Tx[T], mark_name: str | None = None) -> bool: if mark_name is not None: return (u, v) in self.overlay.edge_marks.get(mark_name, set()) return any((u, v) in marks for marks in self.overlay.edge_marks.values()) - def successors(self, n, fullgraph: bool | None = None, all_edges: bool | None = None): + def successors(self, n: Tx[T], fullgraph: bool | None = None, all_edges: bool | None = None): g = self._variant(fullgraph, all_edges) if g is self: return super().successors(n) return g.successors(n) - def predecessors(self, n, fullgraph: bool | None = None, all_edges: bool | None = None): + def predecessors(self, n: Tx[T], fullgraph: bool | None = None, all_edges: bool | None = None): g = self._variant(fullgraph, all_edges) if g is self: return super().predecessors(n) return g.predecessors(n) - def has_edge(self, u, v, fullgraph: bool | None = None, all_edges: bool | None = None) -> bool: + def has_edge(self, u: Tx[T], v: Tx[T], fullgraph: bool | None = None, all_edges: bool | None = None) -> bool: g = self._variant(fullgraph, all_edges) if g is self: return super().has_edge(u, v) @@ -1526,17 +1551,20 @@ class RegionOverlayGraph[T](networkx.DiGraph): # overrides for inherited methods that would construct self.__class__() without arguments # - def copy(self, as_view: bool = False) -> networkx.DiGraph: + # NOTE: this does not respect the intended semantics of as_view + def copy(self, as_view: bool = False) -> networkx.DiGraph[Tx[T]]: return self.materialize() - def subgraph(self, nodes) -> networkx.DiGraph: + def subgraph(self, nodes) -> networkx.DiGraph[Tx[T]]: # Build only the induced subgraph (cost O(induced subgraph)) instead of materialize()+restrict (cost O(whole # region graph)): callers (e.g. quasi_topological_sort_nodes' SCC handling) want a small induced subgraph and # then mutate it. Returns an independent, mutable networkx.DiGraph with exactly the nodes/edges/data (and the # same node/edge iteration order) that self.materialize().subgraph(nodes) would have produced. + if not isinstance(nodes, Iterable): + raise TypeError("Please use the non-pathological versions of the NetworkX api") node_set = self._node_set() selset = {n for n in nodes if n in node_set} - g: networkx.DiGraph = networkx.DiGraph() + g: networkx.DiGraph[Tx[T]] = networkx.DiGraph() g.add_nodes_from(n for n in node_set if n in selset) for u in g: for v, data in self._adj[u].items(): @@ -1544,5 +1572,6 @@ class RegionOverlayGraph[T](networkx.DiGraph): g.add_edge(u, v, **data) return g - def to_directed(self, as_view: bool = False) -> networkx.DiGraph: + # NOTE: this does not respect the intended semantics of as_view + def to_directed(self, as_view: bool = False) -> networkx.DiGraph[Tx[T]]: return self.materialize() From 659f3d7f5da220e16bd9366c17f0082174fee279 Mon Sep 17 00:00:00 2001 From: Fish Date: Tue, 28 Jul 2026 22:29:12 -0700 Subject: [PATCH 069/122] CCodeGen: Fix the display of negative offsets. (#6730) * CCodeGen: Fix the display of negative offsets. * Fix a test case. --- angr/analyses/decompiler/block_simplifier.py | 3 ++- .../analyses/decompiler/structured_codegen/c.py | 7 +++++-- angr/rustylib/ailment.pyi | 10 ++++++---- tests/analyses/decompiler/test_decompiler.py | 17 ++++++++++++++++- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/angr/analyses/decompiler/block_simplifier.py b/angr/analyses/decompiler/block_simplifier.py index d249d7055..d15bb7dc6 100644 --- a/angr/analyses/decompiler/block_simplifier.py +++ b/angr/analyses/decompiler/block_simplifier.py @@ -199,6 +199,7 @@ class BlockSimplifier: def _analyze(self): block = self.block + assert block is not None ctr = 0 max_ctr = 30 @@ -224,7 +225,7 @@ class BlockSimplifier: if not changed: break - assert block is not None + assert new_block is not None self._clear_cache() block = new_block if ctr >= max_ctr: diff --git a/angr/analyses/decompiler/structured_codegen/c.py b/angr/analyses/decompiler/structured_codegen/c.py index e83e87e47..686eafc60 100644 --- a/angr/analyses/decompiler/structured_codegen/c.py +++ b/angr/analyses/decompiler/structured_codegen/c.py @@ -3399,7 +3399,10 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis, Serializab ) result = piece if result is None else CBinaryOp(op, result, piece, codegen=self) if o_constant != 0: - result = CBinaryOp("Add", CConstant(o_constant, SimTypeInt(), codegen=self), result, codegen=self) + if o_constant < 0: + result = CBinaryOp("Sub", result, CConstant(-o_constant, SimTypeInt(), codegen=self), codegen=self) + else: + result = CBinaryOp("Add", result, CConstant(o_constant, SimTypeInt(), codegen=self), codegen=self) return CUnaryOp( "Dereference", CTypeCast(result.type, SimTypePointer(data_type), result, codegen=self), codegen=self @@ -3410,7 +3413,7 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis, Serializab # also identify the "kernel", the root of the expression constant, terms = o_constant, list(o_terms) if constant < 0: - constant = -constant # TODO: This may not be correct. investigate later + return bail_out() i = 0 kernel = None diff --git a/angr/rustylib/ailment.pyi b/angr/rustylib/ailment.pyi index 2cf726262..d83876937 100644 --- a/angr/rustylib/ailment.pyi +++ b/angr/rustylib/ailment.pyi @@ -520,12 +520,14 @@ class Block: def to_bytes(self) -> bytes: ... @classmethod def from_bytes(cls, data: bytes) -> Block: ... - def likes(self, other: Block) -> bool: ... - """ - Check structural equality of two Blocks ignoring `idx` and `tags` on statements and expressions. - """ + def likes(self, other: Block) -> bool: + """ + Check structural equality of two Blocks ignoring `idx` and `tags` on statements and expressions. + """ def deep_copy(self, manager: Manager) -> Block: ... + def pp(self) -> None: + """Pretty-print the block as a string.""" # --------------------------------------------------------------------------- # Manager + VEX -> AIL converter diff --git a/tests/analyses/decompiler/test_decompiler.py b/tests/analyses/decompiler/test_decompiler.py index 46c1882c8..16b99db26 100755 --- a/tests/analyses/decompiler/test_decompiler.py +++ b/tests/analyses/decompiler/test_decompiler.py @@ -2979,6 +2979,21 @@ class TestDecompiler(unittest.TestCase): assert "case 51:" not in d.codegen.text assert "case 52:" not in d.codegen.text + @structuring_algo("sailr") + def test_codegen_accessing_negative_offsets(self, decompiler_options=None): + bin_path = os.path.join(test_location, "x86_64", "ls_ubuntu_2004") + proj = angr.Project(bin_path) + _ = proj.analyses.CFGFast(normalize=True, regions=[(0x410FC0, 0x410FC0 + 1000)]) + f = proj.kb.functions[0x410FC0] + d = proj.analyses[Decompiler].prep(fail_fast=True)(f, options=decompiler_options) + assert d.codegen is not None and d.codegen.text is not None + print_decompilation_result(d) + + # Find `iter = (char *)iter - 1;`` + m = re.search(r"([\S]+) = \(char \*\)([\S]+) \- 1;", d.codegen.text) + assert m is not None + assert m.group(1) == m.group(2) + @for_all_structuring_algos def test_df_add_uint_with_neg_flag_ite_expressions(self, decompiler_options=None): # properly handling cmovz and cmovnz in amd64 binaries @@ -5679,7 +5694,7 @@ class TestDecompiler(unittest.TestCase): # we expect two comparisons against v3[1] and 7 (== or != depending on structuring) var_ids = [] for line in lines: - m = re.search(r"v(\d+)\[1] [!=]= 7", line) + m = re.search(r"\*\(\(char \*\)v(\d+) - 1\) [!=]= 7", line) if m is not None: var_ids.append(m.group(1)) assert len(var_ids) == 2, f"Expected two comparisons with [1] and 7, found {len(var_ids)}: {var_ids}" From a7ae033c69d2bc496592c9512e5b4792a1b02fcb Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 29 Jul 2026 00:05:07 -0700 Subject: [PATCH 070/122] EagerEval: Fix broken expression type comparison. (#6734) This is a bug introduced by the Rusty AIL migration. --- angr/ailment/__init__.py | 3 +++ .../peephole_optimizations/eager_eval.py | 3 ++- .../decompiler/test_peephole_optimizations.py | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/angr/ailment/__init__.py b/angr/ailment/__init__.py index e074f7f24..4402d7a36 100644 --- a/angr/ailment/__init__.py +++ b/angr/ailment/__init__.py @@ -2,6 +2,8 @@ from __future__ import annotations import logging +from angr.rustylib.ailment import ExpressionKind + from . import expression, statement from .block import Block from .block_walker import AILBlockRewriter, AILBlockViewer, AILBlockWalker @@ -69,6 +71,7 @@ __all__ = [ "Const", "Expr", "Expression", + "ExpressionKind", "IRSBConverter", "Manager", "NoOp", diff --git a/angr/analyses/decompiler/peephole_optimizations/eager_eval.py b/angr/analyses/decompiler/peephole_optimizations/eager_eval.py index 17aae80e1..2558516f1 100644 --- a/angr/analyses/decompiler/peephole_optimizations/eager_eval.py +++ b/angr/analyses/decompiler/peephole_optimizations/eager_eval.py @@ -2,6 +2,7 @@ from __future__ import annotations from math import gcd +from angr.ailment import ExpressionKind from angr.ailment.expression import BinaryOp, Const, Convert, StackBaseOffset, UnaryOp from angr.utils.bits import sign_extend @@ -216,7 +217,7 @@ class EagerEvaluation(PeepholeOptimizationExprBase): # constant multiplication mask = (1 << expr.bits) - 1 return Const(expr.idx, (op0.value * op1.value) & mask, expr.bits, **expr.tags) - if {type(op0), type(op1)} == {BinaryOp, Const}: + if {op0.pykind, op1.pykind} == {ExpressionKind.BinaryOp, ExpressionKind.Const}: op0, op1 = expr.operands const_, x0 = (op0, op1) if isinstance(op0, Const) else (op1, op0) if x0.op == "Mul" and (isinstance(x0.operands[0], Const) or isinstance(x0.operands[1], Const)): diff --git a/tests/analyses/decompiler/test_peephole_optimizations.py b/tests/analyses/decompiler/test_peephole_optimizations.py index c3108a5fb..76a1ec4a1 100755 --- a/tests/analyses/decompiler/test_peephole_optimizations.py +++ b/tests/analyses/decompiler/test_peephole_optimizations.py @@ -177,6 +177,22 @@ class TestPeepholeOptimizations(unittest.TestCase): ) assert opt.optimize(expr) is None + def test_eager_eval_var_mul_a_mul_b(self): + bin_path = os.path.join(test_location, "x86_64", "ls_ubuntu_2004") + proj = angr.Project(bin_path) + _ = proj.analyses.CFGFast(normalize=True, regions=[(0x410FC0, 0x410FC0 + 1000)]) + f = proj.kb.functions[0x410FC0] + d = proj.analyses.Decompiler(f, fail_fast=True) + assert d.codegen is not None and d.codegen.text is not None + + # a * 2 * 5 ==> a * 10 + # we should see `a * 10`` or `a % 10` in the output + if "* 2" in d.codegen.text or "* 5" in d.codegen.text: + raise AssertionError( + "The decompiler output should not contain `* 2` or `* 5` after peephole optimization, but it does." + ) + assert "* 10" in d.codegen.text or "% 10" in d.codegen.text + def test_cmp_masked_shift(self): proj = angr.load_shellcode(b"\x90", "AMD64") manager = Manager() From 6b2637c446ebd81fedd2a3ce99cbe50c6a43017b Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 29 Jul 2026 00:42:21 -0700 Subject: [PATCH 071/122] Dephication: Consider the vvar used in block-end jumps during intersection. (#6733) * Dephication: Consider the vvar used in block-end jumps during intersection. This is a subtlety in the Sreedhar et. al. paper. * Fix test cases. --- .../dephication/graph_vvar_mapping.py | 13 ++- angr/analyses/s_liveness.py | 12 +- tests/analyses/decompiler/test_decompiler.py | 106 ++++++++++++++---- 3 files changed, 107 insertions(+), 24 deletions(-) diff --git a/angr/analyses/decompiler/dephication/graph_vvar_mapping.py b/angr/analyses/decompiler/dephication/graph_vvar_mapping.py index 998bd2109..8f1a83bfc 100644 --- a/angr/analyses/decompiler/dephication/graph_vvar_mapping.py +++ b/angr/analyses/decompiler/dephication/graph_vvar_mapping.py @@ -110,8 +110,17 @@ class GraphDephicationVVarMapping(Analysis): # pylint:disable=abstract-method continue if interference.has_edge(var1, var2): - intersection_1 = phi_congruence_class[var1].intersection(live_outs[src1]) - intersection_2 = phi_congruence_class[var2].intersection(live_outs[src2]) + # the intersection considers both liveouts and the vvars used in the last statement of the + # block if it is a jump or a conditional jump because we cannot insert a vvar copy statement + # after the jump. this is a special case that is not covered in Sreedhar et al.'s paper. It is + # documented in "Revisiting Out-of-SSA Translation for Correctness, Code Quality, and + # Efficiency" (Section II.A) by Boissinot et. al. + intersection_1 = phi_congruence_class[var1].intersection( + live_outs[src2] | liveness.model.block_end_vvars.get(src2, set()) + ) + intersection_2 = phi_congruence_class[var2].intersection( + live_outs[src1] | liveness.model.block_end_vvars.get(src1, set()) + ) if intersection_1 and not intersection_2: # case 1 candidate_vvar_set.add(var1) diff --git a/angr/analyses/s_liveness.py b/angr/analyses/s_liveness.py index 97755f75b..0f4bb1692 100644 --- a/angr/analyses/s_liveness.py +++ b/angr/analyses/s_liveness.py @@ -6,7 +6,7 @@ import networkx from angr.ailment import Address, Block from angr.ailment.expression import Phi, VirtualVariable -from angr.ailment.statement import Assignment, ConditionalJump, SideEffectStatement +from angr.ailment.statement import Assignment, ConditionalJump, Jump, SideEffectStatement from angr.analyses.analysis import Analysis, register_analysis from angr.knowledge_plugins.functions.function import Function from angr.utils.ail import is_head_controlled_loop_block, is_phi_assignment @@ -23,6 +23,9 @@ class SLivenessModel: def __init__(self): self.live_ins: dict[Address, set[int]] = {} self.live_outs: dict[Address, set[int]] = {} + # `block_end_vvars` stores for each Block the set of vvars that are used in the last statement of the Block if + # the statement is a jump or a conditional jump. + self.block_end_vvars: dict[Address, set[int]] = {} class SLivenessAnalysis(Analysis): @@ -113,7 +116,7 @@ class SLivenessAnalysis(Analysis): stmts = block.statements live_in_by_pred = {} - for stmt in reversed(stmts): + for i, stmt in enumerate(reversed(stmts)): # handle assignments: a defined vvar is not live before the assignment if isinstance(stmt, Assignment) and isinstance(stmt.dst, VirtualVariable): live.discard(stmt.dst.varid) @@ -150,6 +153,11 @@ class SLivenessAnalysis(Analysis): vvar_use_collector.walk_statement(stmt) live |= vvar_use_collector.vvars + if i == 0 and isinstance(stmt, (Jump, ConditionalJump, SideEffectStatement)): + # this is the last statement in the block and it is a jump or a conditional jump; we record the + # vvars used in this statement for later use + self.model.block_end_vvars[block_key] = vvar_use_collector.vvars.copy() + if live_ins[block_key] != live: live_ins[block_key] = live changed = True diff --git a/tests/analyses/decompiler/test_decompiler.py b/tests/analyses/decompiler/test_decompiler.py index 16b99db26..ba70044b6 100755 --- a/tests/analyses/decompiler/test_decompiler.py +++ b/tests/analyses/decompiler/test_decompiler.py @@ -1417,23 +1417,28 @@ class TestDecompiler(unittest.TestCase): stmts = dw.body.statements assert len(stmts) == 5 # Current decompilation output: - # do - # { - # v1 = v0 + 1; - # v3 = v2 + 1; - # *(v2) = *(v0); - # v0 = v1; - # v2 = v3; - # } while (*(v2)) - # We can improve it by re-arranging the first three statements; we leave it as future work - assert stmts[0].lhs.unified_variable == stmts[3].rhs.unified_variable + # do + # { + # v3 = v2; + # v1 = v0 + 1; + # v2 = v3 + 1; + # *(v3) = *(v0); + # v0 = v1; + # } while (*(v3)); + # We can improve it by re-arranging these statements or control what to propagate; we leave it as future work + assert stmts[0].lhs.unified_variable == stmts[2].rhs.lhs.unified_variable + assert stmts[0].rhs.unified_variable == stmts[2].lhs.unified_variable assert stmts[1].lhs.unified_variable == stmts[4].rhs.unified_variable - assert stmts[2].lhs.operand.variable == stmts[4].lhs.variable - assert stmts[2].rhs.operand.variable == stmts[3].lhs.variable + # *v3 = *v0; + assert stmts[3].lhs.operand.variable == stmts[2].rhs.lhs.variable + assert stmts[3].rhs.operand.variable == stmts[1].rhs.lhs.variable # v0 = v0; is incorrect - assert stmts[3].lhs.unified_variable != stmts[3].rhs.unified_variable, "Variable unification went wrong." + assert stmts[3].lhs.operand.unified_variable != stmts[3].rhs.operand.unified_variable, ( + "Variable unification went wrong." + ) assert stmts[4].lhs.unified_variable != stmts[4].rhs.unified_variable, "Variable unification went wrong." - assert dw.condition.lhs.operand.variable == stmts[2].lhs.operand.variable + # condition should be based on the value of *v3 + assert dw.condition.lhs.operand.variable == stmts[3].lhs.operand.variable @for_all_structuring_algos def test_decompiling_nl_i386_pie(self, decompiler_options=None): @@ -2994,6 +2999,69 @@ class TestDecompiler(unittest.TestCase): assert m is not None assert m.group(1) == m.group(2) + @structuring_algo("sailr") + def test_cssa_incorrect_loop_condition(self, decompiler_options=None): + bin_path = os.path.join(test_location, "x86_64", "ls_ubuntu_2004") + proj = angr.Project(bin_path) + _ = proj.analyses.CFGFast(normalize=True, regions=[(0x416C60, 0x416C60 + 1000)]) + f = proj.kb.functions["_obstack_memory_used"] + d = proj.analyses[Decompiler].prep(fail_fast=True)(f, options=decompiler_options) + assert d.codegen is not None and d.codegen.text is not None + print_decompilation_result(d) + + # there is a loop in this function. if the loop is discovered as `do { } while (cond);`, the condition must be + # v3->field_8 where v3 = v1. Incorrect CSSA translation may result in v3 being assigned to v1->field_8. + # + # Correct: + # + # v1 = a0->field_8; + # v2 = 0; + # if (a0->field_8) + # { + # do + # { + # v3 = v1; + # v4 = v3->field_8; + # v2 = v2 + v3->field_0 - (char *)v3; + # v1 = v4; + # } while (v3->field_8); + # } + # + # Incorrect: + # + # v1 = a0->field_8; + # v2 = 0; + # if (a0->field_8) + # { + # do + # { + # v3 = v1->field_8; + # v2 = v2 + v1->field_0 - (char *)v1; + # v1 = v3; + # } while (v1->field_8); + # } + # + # We may do better in the future by not propagating the memory read into the loop condition, in which case the + # loop body will be shorter. + + # find the loop + loop_match = re.search(r"do\s*\{([^}]*)\}\s*while\s*\(([^)]*)\);", d.codegen.text, re.MULTILINE) + assert loop_match is not None, "Cannot find the do-while loop in the decompilation output." + loop_body = loop_match.group(1) + loop_condition = loop_match.group(2) + # extract the variable used in the loop condition + condition_var_match = re.search(r"([\w\d_]+)->field_8", loop_condition) + assert condition_var_match is not None, "Cannot find the variable used in the loop condition." + condition_var = condition_var_match.group(1) + # find all assignments in the loop body + assignments = re.findall(r"([\w\d_]+)\s*=\s*([^;]+);", loop_body) + # find the assignment to the condition var + condition_var_assignment = next(i for i, (var, expr) in enumerate(assignments) if var == condition_var) + assert condition_var_assignment is not None, f"Cannot find the assignment to {condition_var} in the loop body." + assert condition_var_assignment == 0, ( + f"The assignment to {condition_var} should be the first assignment in the loop body." + ) + @for_all_structuring_algos def test_df_add_uint_with_neg_flag_ite_expressions(self, decompiler_options=None): # properly handling cmovz and cmovnz in amd64 binaries @@ -4702,12 +4770,10 @@ class TestDecompiler(unittest.TestCase): print_decompilation_result(d) lines = [line.strip(" ") for line in d.codegen.text.split("\n")] start_pos = lines.index("{") - assert lines[start_pos + 3 :][:6] == [ - "if (a1)", - "v1 = a1;", - "else", - "v1 = a0;", - "g_1234 = v1;", + assert lines[start_pos + 1 :][:4] == [ + "if (!a1)", + "a1 = a0;", + "g_1234 = a1;", "return 4660;", ] or lines[start_pos + 1 :][:2] == [ "*((int *)&g_1234) = (a1 ? a1 : a0);", From 508ac3a44c31409a5fdde30494984a5fe3da1de8 Mon Sep 17 00:00:00 2001 From: Quintin Kong <71952215+DORA-B@users.noreply.github.com> Date: Wed, 29 Jul 2026 03:57:43 -0400 Subject: [PATCH 072/122] aarch64: fix adcs/sbcs carry flag (select on cc_dep3, not cc_dep2) (#6702) arm64g_calculate_flag_c selected the ADC*/SBC* carry-in with `cc_dep2 != 0` (the second operand). The arm64 flag thunk layout puts the old carry in cc_dep3 (angr's own comment: "DEP3 = oldC (in LSB)", matching VEX's guest_arm64_helpers.c). So the C flag after adcs/sbcs was computed from an operand value instead of the incoming carry. Random operands usually mask it (cc_dep2 != 0 nearly always holds); the equal-operand case exposes it, e.g. `sbcs x,y,y` must give C = oldC but returned a value keyed on y. flag_n/z/v and the arm32 port (which correctly uses cc_dep3) were unaffected. --- angr/engines/vex/claripy/ccall.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/angr/engines/vex/claripy/ccall.py b/angr/engines/vex/claripy/ccall.py index 004fa76ba..c1b772e30 100644 --- a/angr/engines/vex/claripy/ccall.py +++ b/angr/engines/vex/claripy/ccall.py @@ -1902,11 +1902,11 @@ def arm64g_calculate_flag_c(state, cc_op, cc_dep1, cc_dep2, cc_dep3): elif concrete_op in (ARM64G_CC_OP_ADC32, ARM64G_CC_OP_ADC64): res = cc_dep1 + cc_dep2 + cc_dep3 flag = claripy.If( - cc_dep2 != 0, boolean_extend(claripy.ULE, res, cc_dep1, 64), boolean_extend(claripy.ULT, res, cc_dep1, 64) + cc_dep3 != 0, boolean_extend(claripy.ULE, res, cc_dep1, 64), boolean_extend(claripy.ULT, res, cc_dep1, 64) ) elif concrete_op in (ARM64G_CC_OP_SBC32, ARM64G_CC_OP_SBC64): flag = claripy.If( - cc_dep2 != 0, + cc_dep3 != 0, boolean_extend(claripy.UGE, cc_dep1, cc_dep2, 64), boolean_extend(claripy.UGT, cc_dep1, cc_dep2, 64), ) From 6933b065f0c41c96d7b8f66b4a867aa94828e3cb Mon Sep 17 00:00:00 2001 From: Ati Priya Date: Wed, 29 Jul 2026 01:11:09 -0700 Subject: [PATCH 073/122] Decompiler: rewrite CondBE and CondNB ccalls on amd64 (#6645) * Decompiler: rewrite CondBE and CondNB ccalls on amd64 * Decompiler: fix inverted CondZ/CondNZ over G_CC_OP_COPY on amd64 * tests: add binary-driven regressions for CondBE/CondNB ccall recovery --- .../ccall_rewriters/amd64_ccalls.py | 112 ++++++- .../decompiler/test_ccall_rewriting.py | 308 +++++++++++++++++- 2 files changed, 414 insertions(+), 6 deletions(-) diff --git a/angr/analyses/decompiler/ccall_rewriters/amd64_ccalls.py b/angr/analyses/decompiler/ccall_rewriters/amd64_ccalls.py index dd2773051..f9c7537f1 100644 --- a/angr/analyses/decompiler/ccall_rewriters/amd64_ccalls.py +++ b/angr/analyses/decompiler/ccall_rewriters/amd64_ccalls.py @@ -245,7 +245,8 @@ class AMD64CCallRewriter(CCallRewriterBase): self.ail_manager.next_atom(), "And", [dep_1, flag], False, **ccall.tags ) zero = Expr.Const(self.ail_manager.next_atom(), 0, dep_1.bits) - expr_op = "CmpEQ" if cond_v == AMD64_CondTypes["CondZ"] else "CmpNE" + # dep_1 holds the old flags: ZF is *set* iff the masked bit is non-zero + expr_op = "CmpNE" if cond_v == AMD64_CondTypes["CondZ"] else "CmpEQ" r = Expr.BinaryOp(ccall.idx, expr_op, (masked_dep, zero), False, **ccall.tags) return Expr.Convert(self.ail_manager.next_atom(), r.bits, ccall.bits, False, r, **ccall.tags) @@ -396,8 +397,10 @@ class AMD64CCallRewriter(CCallRewriterBase): r = Expr.BinaryOp(ccall.idx, "CmpGT", (dep_1, dep_2), False, **ccall.tags) return Expr.Convert(self.ail_manager.next_atom(), r.bits, ccall.bits, False, r, **ccall.tags) - elif cond_v == AMD64_CondTypes["CondB"]: - if op_v in { + elif cond_v in {AMD64_CondTypes["CondB"], AMD64_CondTypes["CondBE"]}: + # CondB tests CF; CondBE tests CF | ZF + is_be = cond_v == AMD64_CondTypes["CondBE"] + if not is_be and op_v in { AMD64_OpTypes["G_CC_OP_ADDB"], AMD64_OpTypes["G_CC_OP_ADDW"], AMD64_OpTypes["G_CC_OP_ADDL"], @@ -439,7 +442,8 @@ class AMD64CCallRewriter(CCallRewriterBase): AMD64_OpTypes["G_CC_OP_SUBL"], AMD64_OpTypes["G_CC_OP_SUBQ"], }: - # dep_1 =u a comparison would be wrong here: C integer + # promotion keeps sub-int additions from wrapping, making it a tautology at + # 8/16-bit widths. + + dep_1 = self._fix_size( + dep_1, + op_v, + AMD64_OpTypes["G_CC_OP_ADDB"], + AMD64_OpTypes["G_CC_OP_ADDW"], + AMD64_OpTypes["G_CC_OP_ADDL"], + ccall.tags, + ) + dep_2 = self._fix_size( + dep_2, + op_v, + AMD64_OpTypes["G_CC_OP_ADDB"], + AMD64_OpTypes["G_CC_OP_ADDW"], + AMD64_OpTypes["G_CC_OP_ADDL"], + ccall.tags, + ) + + cfadd_call = Expr.Call( + self.ail_manager.next_atom(), + "__CFADD__", + args=[dep_1, dep_2], + bits=ccall.bits, + **ccall.tags, + ) + variable_map_of(self.ail_manager).set_calling_convention( + cfadd_call, SimCCUsercall(self.project.arch, [], None) + ) + zero = Expr.Const(self.ail_manager.next_atom(), 0, cfadd_call.bits) + r = Expr.BinaryOp(ccall.idx, "CmpEQ", (cfadd_call, zero), False, **ccall.tags) + return Expr.Convert(self.ail_manager.next_atom(), r.bits, ccall.bits, False, r, **ccall.tags) + if op_v in { + AMD64_OpTypes["G_CC_OP_SUBB"], + AMD64_OpTypes["G_CC_OP_SUBW"], + AMD64_OpTypes["G_CC_OP_SUBL"], + AMD64_OpTypes["G_CC_OP_SUBQ"], + }: + # dep_1 >=u dep_2 + + dep_1 = self._fix_size( + dep_1, + op_v, + AMD64_OpTypes["G_CC_OP_SUBB"], + AMD64_OpTypes["G_CC_OP_SUBW"], + AMD64_OpTypes["G_CC_OP_SUBL"], + ccall.tags, + ) + dep_2 = self._fix_size( + dep_2, + op_v, + AMD64_OpTypes["G_CC_OP_SUBB"], + AMD64_OpTypes["G_CC_OP_SUBW"], + AMD64_OpTypes["G_CC_OP_SUBL"], + ccall.tags, + ) + + r = Expr.BinaryOp(ccall.idx, "CmpGE", (dep_1, dep_2), False, **ccall.tags) + return Expr.Convert(self.ail_manager.next_atom(), r.bits, ccall.bits, False, r, **ccall.tags) + if op_v in { + AMD64_OpTypes["G_CC_OP_LOGICB"], + AMD64_OpTypes["G_CC_OP_LOGICW"], + AMD64_OpTypes["G_CC_OP_LOGICL"], + AMD64_OpTypes["G_CC_OP_LOGICQ"], + }: + # and/or/xor always clear CF, so CondNB is always true + return Expr.Const(self.ail_manager.next_atom(), 1, ccall.bits, **ccall.tags) elif ( cond_v == AMD64_CondTypes["CondS"] and op_v diff --git a/tests/analyses/decompiler/test_ccall_rewriting.py b/tests/analyses/decompiler/test_ccall_rewriting.py index 8457dd384..70fbdd87c 100644 --- a/tests/analyses/decompiler/test_ccall_rewriting.py +++ b/tests/analyses/decompiler/test_ccall_rewriting.py @@ -6,13 +6,16 @@ from typing import Any, cast __package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin +import itertools import os import unittest +import claripy + import angr from angr.ailment import Expr, Manager from angr.analyses.decompiler.ccall_rewriters.amd64_ccalls import AMD64CCallRewriter -from angr.engines.vex.claripy.ccall import data +from angr.engines.vex.claripy.ccall import data, pc_calculate_condition from tests.common import bin_location, load_project_with_scoped_cfg, print_decompilation_result test_location = os.path.join(bin_location, "tests") @@ -317,5 +320,308 @@ class TestAMD64CondOverflowBinary(unittest.TestCase): assert "__OFUMUL__" in dec.codegen.text, f"{addr:#x} lost its overflow check" +# +# AMD64 rewriter unit tests: build amd64g_calculate_condition ccalls in memory, rewrite them, +# and differential-test the rewritten expressions against ccall.py's executable semantics. +# + + +def _make_ccall(cond, op, dep1=None, dep2=None, ndep=None, bits=64): + """Build a VEXCCallExpression for amd64g_calculate_condition.""" + if dep1 is None: + dep1 = Expr.Register(1, 16, 64) # rax + if dep2 is None: + dep2 = Expr.Register(2, 24, 64) # rcx + if ndep is None: + ndep = Expr.Const(3, 0, 64) + return Expr.VEXCCallExpression( + idx=0, + callee="amd64g_calculate_condition", + operands=(Expr.Const(0, cond, 64), Expr.Const(0, op, 64), dep1, dep2, ndep), + bits=bits, + ) + + +_PROJECT = angr.load_shellcode(b"\x90", arch="AMD64") + + +def _rewrite(ccall): + return AMD64CCallRewriter(ccall, _PROJECT, Manager(arch=_PROJECT.arch)).result + + +def _mask(v, bits): + return v & ((1 << bits) - 1) + + +def _sext(v, bits): + v = _mask(v, bits) + return v - (1 << bits) if v >> (bits - 1) else v + + +def _eval(expr): + """Concretely evaluate a rewritten (constant-folded) AIL expression. Returns (value, bits).""" + if isinstance(expr, Expr.Const): + return _mask(expr.value_int, expr.bits), expr.bits + if isinstance(expr, Expr.Convert): + v, _ = _eval(expr.operand) + v = _mask(_sext(v, expr.from_bits) if expr.is_signed else v, expr.to_bits) + return v, expr.to_bits + if isinstance(expr, Expr.Call) and expr.target == "__CFADD__": + # carry-out of the addition at the operands' width + left, lbits = _eval(expr.args[0]) + right, rbits = _eval(expr.args[1]) + bits = max(lbits, rbits) + return int(_mask(left + right, bits) < left), expr.bits + if isinstance(expr, Expr.BinaryOp): + left, lbits = _eval(expr.operands[0]) + right, rbits = _eval(expr.operands[1]) + bits = max(lbits, rbits) + if expr.signed: + left, right = _sext(left, lbits), _sext(right, rbits) + cmps = { + "CmpEQ": lambda: (int(left == right), 1), + "CmpNE": lambda: (int(left != right), 1), + "CmpLT": lambda: (int(left < right), 1), + "CmpLE": lambda: (int(left <= right), 1), + "CmpGT": lambda: (int(left > right), 1), + "CmpGE": lambda: (int(left >= right), 1), + "And": lambda: (_mask(left & right, bits), bits), + "Add": lambda: (_mask(left + right, bits), bits), + } + if expr.op not in cmps: + raise NotImplementedError(expr.op) + return cmps[expr.op]() + raise NotImplementedError(type(expr)) + + +def _oracle(cond, op, dep1, dep2, ndep=0): + """Ground truth: ccall.py's executable amd64g_calculate_condition.""" + r = pc_calculate_condition( + None, + claripy.BVV(cond, 64), + claripy.BVV(op, 64), + claripy.BVV(dep1, 64), + claripy.BVV(dep2, 64), + claripy.BVV(ndep, 64), + platform="AMD64", + ) + return bool(claripy.backends.concrete.eval(r, 1)[0]) + + +def _rewritten_value(cond, op, dep1, dep2, ndep=0): + ccall = _make_ccall(cond, op, Expr.Const(1, dep1, 64), Expr.Const(2, dep2, 64), Expr.Const(3, ndep, 64)) + result = _rewrite(ccall) + assert result is not None + return bool(_eval(result)[0]) + + +class TestAMD64CCallRewriterCondNL(unittest.TestCase): + """CondNL (jge, SF == OF). Signed >= over SUB; sign-of-result >= 0 over LOGIC.""" + + def test_condnl_sub_is_signed_ge(self): + for op in ("G_CC_OP_SUBB", "G_CC_OP_SUBW", "G_CC_OP_SUBL", "G_CC_OP_SUBQ"): + cmp = _unwrap_convert(_rewrite(_make_ccall(AMD64_CondTypes["CondNL"], AMD64_OpTypes[op]))) + assert isinstance(cmp, Expr.BinaryOp), f"{op}: not rewritten" + assert cmp.op == "CmpGE", f"{op}: got {cmp.op}" + assert cmp.signed is True, f"{op}: expected signed" + + def test_condnl_logic_is_signed_ge_zero(self): + for op in ("G_CC_OP_LOGICB", "G_CC_OP_LOGICW", "G_CC_OP_LOGICL", "G_CC_OP_LOGICQ"): + cmp = _unwrap_convert(_rewrite(_make_ccall(AMD64_CondTypes["CondNL"], AMD64_OpTypes[op]))) + assert isinstance(cmp, Expr.BinaryOp), f"{op}: not rewritten" + assert cmp.op == "CmpGE", f"{op}: got {cmp.op}" + assert cmp.signed is True, f"{op}: expected signed" + assert cmp.operands[1].value_int == 0, f"{op}: expected comparison against 0" + + +class TestAMD64CCallRewriterCondBE(unittest.TestCase): + """CondBE (jbe / unsigned <=) recovery. Previously unhandled, leaving a generic `_ccall`.""" + + def test_condbe_sub_is_unsigned_le(self): + for op in ("G_CC_OP_SUBB", "G_CC_OP_SUBW", "G_CC_OP_SUBL", "G_CC_OP_SUBQ"): + ccall = _make_ccall(AMD64_CondTypes["CondBE"], AMD64_OpTypes[op]) + result = _unwrap_convert(_rewrite(ccall)) + assert isinstance(result, Expr.BinaryOp), f"{op}: not rewritten" + assert result.op == "CmpLE", f"{op}: got {result.op}" + assert result.signed is False, f"{op}: expected unsigned" + + def test_condb_sub_still_unsigned_lt(self): + # control: CondB must be unchanged by the CondBE addition + for op in ("G_CC_OP_SUBB", "G_CC_OP_SUBW", "G_CC_OP_SUBL", "G_CC_OP_SUBQ"): + ccall = _make_ccall(AMD64_CondTypes["CondB"], AMD64_OpTypes[op]) + result = _unwrap_convert(_rewrite(ccall)) + assert isinstance(result, Expr.BinaryOp), f"{op}: not rewritten" + assert result.op == "CmpLT", f"{op}: got {result.op}" + assert result.signed is False, f"{op}: expected unsigned" + + def test_condb_add_still_uses_cfadd(self): + # control: CondB x ADD must keep emitting __CFADD__, not a comparison + ccall = _make_ccall(AMD64_CondTypes["CondB"], AMD64_OpTypes["G_CC_OP_ADDL"]) + result = _rewrite(ccall) + assert isinstance(result, Expr.Call) and result.target == "__CFADD__" + + def test_condbe_add_is_not_rewritten(self): + # CondBE x ADD needs CF|ZF, which the CondB __CFADD__ form does not express: leave it alone + ccall = _make_ccall(AMD64_CondTypes["CondBE"], AMD64_OpTypes["G_CC_OP_ADDL"]) + assert _rewrite(ccall) is None + + def test_condbe_logic_is_zero_test(self): + # and/or/xor clear CF, so BE == CF|ZF degenerates to ZF + ccall = _make_ccall(AMD64_CondTypes["CondBE"], AMD64_OpTypes["G_CC_OP_LOGICL"]) + result = _unwrap_convert(_rewrite(ccall)) + assert isinstance(result, Expr.BinaryOp) and result.op == "CmpEQ" + + def test_condb_logic_is_always_false(self): + ccall = _make_ccall(AMD64_CondTypes["CondB"], AMD64_OpTypes["G_CC_OP_LOGICL"]) + result = _rewrite(ccall) + assert isinstance(result, Expr.Const) and result.value_int == 0 + assert result.bits == ccall.bits + + def test_condnb_sub_is_unsigned_ge(self): + for op in ("G_CC_OP_SUBB", "G_CC_OP_SUBW", "G_CC_OP_SUBL", "G_CC_OP_SUBQ"): + ccall = _make_ccall(AMD64_CondTypes["CondNB"], AMD64_OpTypes[op]) + result = _unwrap_convert(_rewrite(ccall)) + assert isinstance(result, Expr.BinaryOp), f"{op}: not rewritten" + assert result.op == "CmpGE", f"{op}: got {result.op}" + assert result.signed is False, f"{op}: expected unsigned" + + def test_condnb_add_is_negated_cfadd(self): + # CondNB (jae) is !CF, the negation of the __CFADD__ carry test CondB emits. + # An inline (a + b) >= a comparison would be a C-promotion tautology at 8/16-bit widths. + for op in ("G_CC_OP_ADDB", "G_CC_OP_ADDW", "G_CC_OP_ADDL", "G_CC_OP_ADDQ"): + ccall = _make_ccall(AMD64_CondTypes["CondNB"], AMD64_OpTypes[op]) + result = _unwrap_convert(_rewrite(ccall)) + assert isinstance(result, Expr.BinaryOp), f"{op}: not rewritten" + assert result.op == "CmpEQ", f"{op}: got {result.op}" + lhs, rhs = result.operands + assert isinstance(lhs, Expr.Call) and lhs.target == "__CFADD__", f"{op}: expected a __CFADD__ call" + assert isinstance(rhs, Expr.Const) and rhs.value_int == 0, f"{op}: expected comparison against 0" + + def test_condnb_logic_is_always_true(self): + ccall = _make_ccall(AMD64_CondTypes["CondNB"], AMD64_OpTypes["G_CC_OP_LOGICL"]) + result = _rewrite(ccall) + assert isinstance(result, Expr.Const) and result.value_int == 1 + assert result.bits == ccall.bits + + +class TestAMD64CCallRewriterCondZCopy(unittest.TestCase): + """CondZ/CondNZ over G_CC_OP_COPY read ZF straight out of the saved flags.""" + + def test_condz_copy_is_true_when_zf_set(self): + zf = AMD64_CondBitMasks["G_CC_MASK_Z"] + copy = AMD64_OpTypes["G_CC_OP_COPY"] + assert _rewritten_value(AMD64_CondTypes["CondZ"], copy, zf, 0) is True + assert _rewritten_value(AMD64_CondTypes["CondZ"], copy, 0, 0) is False + assert _rewritten_value(AMD64_CondTypes["CondNZ"], copy, zf, 0) is False + assert _rewritten_value(AMD64_CondTypes["CondNZ"], copy, 0, 0) is True + + def test_condz_copy_matches_oracle(self): + copy = AMD64_OpTypes["G_CC_OP_COPY"] + for cond in ("CondZ", "CondNZ"): + for flags in range(256): + assert _rewritten_value(AMD64_CondTypes[cond], copy, flags, 0) == _oracle( + AMD64_CondTypes[cond], copy, flags, 0 + ), f"{cond} flags={flags:#x}" + + +# Boundary sweep: all 256 values of dep_1 against a fixed spread of dep_2 (zero, small values, the +# signed/unsigned transitions, the top of the range, and a bit pattern). These are ordering +# comparisons, so the boundary values cover every transition of the relation under test. +# A FULL 256x256 sweep (65,536 pairs) was run out-of-tree for every 8-bit cell below -- 0 +# mismatches against ccall.py's executable amd64g_calculate_condition, and independently against a +# native cmpb/setcc oracle on hardware (CondB x SUBB included as a control; ~200k further control +# cases on pre-existing rules also clean). The 16/32/64-bit widths are covered by ccall.py and the +# width-logic tests, not by the hardware oracle. ccall.py alone would be circular -- the rewriter +# and the oracle share a model. Hardware-harness note: for LOGIC* ops, VEX's cc_dep1 is the +# RESULT of the operation, not an operand. +_DEP2_SAMPLE = (0, 1, 2, 3, 0x7E, 0x7F, 0x80, 0x81, 0xFD, 0xFE, 0xFF, 0x55) + + +class TestAMD64CCallRewriterDifferential(unittest.TestCase): + """Differential-test 8-bit cells against ccall.py's executable semantics.""" + + # VEX only guarantees the low nbits of the deps; the rewriter must ignore anything above them + _DIRTY = 0xDEADBEEF_00000100 + + def _sweep(self, cond_name, op_name): + cond, op = AMD64_CondTypes[cond_name], AMD64_OpTypes[op_name] + for dep1, dep2 in itertools.product(range(256), _DEP2_SAMPLE): + got = _rewritten_value(cond, op, dep1, dep2) + want = _oracle(cond, op, dep1, dep2) + assert got == want, f"{cond_name} x {op_name} dep1={dep1:#x} dep2={dep2:#x}: {got} != {want}" + if dep1 % 8 == 0: + d1, d2 = dep1 | self._DIRTY, dep2 | self._DIRTY + got = _rewritten_value(cond, op, d1, d2) + want = _oracle(cond, op, d1, d2) + assert got == want, f"{cond_name} x {op_name} dep1={d1:#x} dep2={d2:#x}: {got} != {want}" + + def test_condnl_subb_differential(self): + self._sweep("CondNL", "G_CC_OP_SUBB") + + def test_condnl_logicb_differential(self): + self._sweep("CondNL", "G_CC_OP_LOGICB") + + def test_condbe_subb_differential(self): + self._sweep("CondBE", "G_CC_OP_SUBB") + + def test_condb_subb_differential(self): + self._sweep("CondB", "G_CC_OP_SUBB") + + def test_condnb_subb_differential(self): + self._sweep("CondNB", "G_CC_OP_SUBB") + + def test_condbe_logicb_differential(self): + self._sweep("CondBE", "G_CC_OP_LOGICB") + + def test_condb_logicb_differential(self): + self._sweep("CondB", "G_CC_OP_LOGICB") + + def test_condnb_logicb_differential(self): + self._sweep("CondNB", "G_CC_OP_LOGICB") + + def test_condnb_addb_differential(self): + self._sweep("CondNB", "G_CC_OP_ADDB") + + +class TestAMD64CCallRewriterRealBinaries(unittest.TestCase): + """Binary-driven regressions for the amd64 ccall rewriter. + + The synthetic tests above feed hand-built ccalls straight to the rewriter. They cannot + exercise the part of the pipeline that actually needed fixing: the propagator folding + cc_op/cc_dep across basic blocks so the ccall is even recognizable at rewrite time. When a + cell is unhandled the rewriter returns None, the callee survives as an undeclared `_ccall`, + and it reaches the C output. Each function below decompiled to a stray `_ccall(...)` before + these commits and is clean after; the assertion is simply that no `_ccall(` remains. + + The binaries are real gcc-13.3.0 -O2 objects (stripped) copied into the angr binaries repo. + A whole-binary CFGFast is required -- region/scoped CFGs do not reproduce these cross-block + ccall folds. Every target was confirmed stable across 3 repeated decompiles. + """ + + def _assert_no_ccall(self, bin_name, addrs): + bin_path = os.path.join(test_location, "x86_64", bin_name) + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(normalize=True) + proj.analyses.CompleteCallingConventions(cfg=cfg.model, recover_variables=True) + for addr in addrs: + func = cfg.functions.get_by_addr(addr) + dec = proj.analyses.Decompiler(func, cfg=cfg.model) + assert dec.codegen is not None and dec.codegen.text is not None, f"no codegen for {addr:#x}" + assert "_ccall(" not in dec.codegen.text, f"{addr:#x}: stray _ccall in decompilation" + + def test_file_ccalls_cleared(self): + # gcc-13.3.0 -O2 `file`. These carry CondBE/CondNB over SUB (unsigned <= / >=) folds; + # sub_409150 recovers the `v <= 0x200` selector of file_pstring_length_size. + self._assert_no_ccall( + "file_gcc13.3.0_O2", + [0x409150, 0x40C360, 0x4134D0, 0x413630, 0x418BD0, 0x41DD00], + ) + + def test_gzip_ccall_cleared(self): + # gcc-13.3.0 -O2 `gzip`. sub_409b60 had two ccalls; clearing them flips the function to + # fully compilable C. + self._assert_no_ccall("gzip_gcc13.3.0_O2", [0x409B60]) + + if __name__ == "__main__": unittest.main() From b9358da5da6b261a36fa69c3cc43d5504d40edd1 Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 29 Jul 2026 01:22:10 -0700 Subject: [PATCH 074/122] AIL: Fix __eq__. (#6728) * AIL: Merge likes/matches into a single mode-parameterized walk * AIL: Make __eq__ idx-aware at every node, not just the root * AIL: Stop hashing fields that equality does not compare * AIL: Regression-test the hash/eq contract * AIL: Compare bits in StringLiteral and Struct * AIL: Replace the CMP_* constants with a CmpMode enum * AIL: cargo fmt * AIL: Compare and hash rounding_mode on Convert and BinaryOp * Update comments. --- native/angr/src/ailment/ail_expr.rs | 586 +++++++++--------------- native/angr/src/ailment/ail_stmt.rs | 233 +++------- native/angr/src/ailment/const_value.rs | 29 +- native/angr/src/ailment/mod.rs | 68 +++ tests/ailment/test_hash_eq_contract.py | 264 +++++++++++ tests/ailment/test_statement_markers.py | 15 + 6 files changed, 649 insertions(+), 546 deletions(-) create mode 100644 tests/ailment/test_hash_eq_contract.py diff --git a/native/angr/src/ailment/ail_expr.rs b/native/angr/src/ailment/ail_expr.rs index 30e300869..7388174a2 100644 --- a/native/angr/src/ailment/ail_expr.rs +++ b/native/angr/src/ailment/ail_expr.rs @@ -28,7 +28,7 @@ use pyo3::types::{PyBytes, PyDict, PyList, PyString, PyTuple}; use crate::ailment::const_value::ConstValue; use crate::ailment::enums::{ConvertType, ExpressionKind, RoundingMode, VirtualVariableCategory}; use crate::ailment::tags::{Tags, TagsView}; -use crate::ailment::{CachedHash, hash_of}; +use crate::ailment::{CachedHash, CmpMode, hash_of}; use indexmap::IndexMap; use serde::de::{self, EnumAccess, SeqAccess, VariantAccess, Visitor}; use serde::ser::{SerializeStruct, SerializeTupleVariant}; @@ -70,15 +70,46 @@ impl ExprHeader { /// expression -- VEX sometimes carries the rounding mode in a tmp, which only /// becomes a constant later in the decompilation pipeline. /// -/// The expression form is a payload, not an operand: ``likes`` / ``matches`` -/// / ``__eq__`` ignore the rounding mode entirely (as they always have), and -/// the operand walks (``replace`` / recursive maps) do not descend into it. +/// The rounding mode is significant to every comparison relation -- +/// ``__eq__`` / ``likes`` / ``matches`` all observe it, as the legacy +/// Python AIL did (``Convert.likes`` and ``BinaryOp.likes`` both compared +/// ``self.rounding_mode``). Two float conversions that round differently +/// compute different values and are not the same expression. +/// +/// It remains a payload rather than an operand in one narrow sense: the +/// operand walks (``replace`` / recursive maps) still do not descend into +/// the expression form. #[derive(Clone, Debug, Serialize, Deserialize)] pub enum RoundingModeOrExpr { Mode(RoundingMode), Expr(Arc), } +impl RoundingModeOrExpr { + /// Compare two payloads under `MODE`. The expression form recurses + /// through [`AilExpression::cmp_ail`], so a rounding mode still held + /// in a tmp observes the same idx-awareness and SSA relaxations as + /// any other subtree. + pub fn cmp_ail(&self, other: &RoundingModeOrExpr) -> bool { + match (self, other) { + (RoundingModeOrExpr::Mode(a), RoundingModeOrExpr::Mode(b)) => a == b, + (RoundingModeOrExpr::Expr(a), RoundingModeOrExpr::Expr(b)) => a.cmp_ail::(b), + // A resolved mode and an unresolved tmp are not interchangeable + // even when the tmp would later resolve to that mode. + _ => false, + } + } + + /// [`Self::cmp_ail`] lifted over the ``Option`` the variants store. + pub fn opt_cmp_ail(a: &Option, b: &Option) -> bool { + match (a, b) { + (None, None) => true, + (Some(x), Some(y)) => x.cmp_ail::(y), + _ => false, + } + } +} + impl Hash for RoundingModeOrExpr { fn hash(&self, h: &mut H) { match self { @@ -434,8 +465,15 @@ impl CFGTarget { /// Structural-with-identity equality (idx-strict via inner /// ``AilExpression::likes``). pub fn likes(&self, other: &CFGTarget) -> bool { + self.cmp_ail::<{ CmpMode::Likes.as_u8() }>(other) + } + + /// Compare two targets under `MODE`, deferring to the inner + /// expression's [`AilExpression::cmp_ail`]. Symbol targets are leaves + /// and compare by name in every mode. + pub fn cmp_ail(&self, other: &CFGTarget) -> bool { match (self, other) { - (CFGTarget::Expr(a), CFGTarget::Expr(b)) => a.likes(b), + (CFGTarget::Expr(a), CFGTarget::Expr(b)) => a.cmp_ail::(b), (CFGTarget::Symbol(a), CFGTarget::Symbol(b)) => a == b, _ => false, } @@ -444,11 +482,7 @@ impl CFGTarget { /// Structural-only equality (idx-agnostic via inner /// ``AilExpression::matches``). pub fn matches(&self, other: &CFGTarget) -> bool { - match (self, other) { - (CFGTarget::Expr(a), CFGTarget::Expr(b)) => a.matches(b), - (CFGTarget::Symbol(a), CFGTarget::Symbol(b)) => a == b, - _ => false, - } + self.cmp_ail::<{ CmpMode::Matches.as_u8() }>(other) } /// Recursively substitute ``old`` with ``new`` inside an ``Expr`` @@ -796,7 +830,6 @@ impl Hash for AilExpression { from_type, to_type, rounding_mode, - .. } => { operand.cached_hash_or_compute().hash(h); from_bits.hash(h); @@ -826,6 +859,7 @@ impl Hash for AilExpression { operands, signed, floating_point, + rounding_mode, .. } => { op.hash(h); @@ -834,6 +868,7 @@ impl Hash for AilExpression { bits.hash(h); signed.hash(h); floating_point.hash(h); + rounding_mode.hash(h); } ExprInner::Load { addr, @@ -1077,14 +1112,22 @@ impl AilExpression { e } - /// ``__eq__`` semantics: same kind, same ``idx``, and structurally - /// ``likes``. This is what the Python ``Expression.__eq__`` computes - /// (idx-first short-circuit, then ``likes``). ``replace`` matches on - /// this, NOT on bare ``likes``: two distinct SSA occurrences of the - /// same value share a shape (``likes``) but have different ``idx``, - /// and a replace targeting one must not rewrite the other. + /// ``__eq__`` semantics: same kind and structurally ``likes``, with + /// ``idx`` equality required at **every** node -- not just the root. + /// This is what the Python ``Expression.__eq__`` computes. + /// ``replace`` matches on this, NOT on bare ``likes``: two distinct + /// SSA occurrences of the same value share a shape (``likes``) but + /// have different ``idx``, and a replace targeting one must not + /// rewrite the other. + /// + /// The idx-awareness has to reach the whole subtree because the + /// ``Hash`` impl folds every descendant's ``idx`` in (operands + /// contribute via ``cached_hash_or_compute``). Checking ``idx`` only + /// at the root -- as this did before -- left ``a == b`` true while + /// ``hash(a) != hash(b)`` for any node with children, so ``==`` + /// duplicates could coexist in a set and dict lookups missed. pub fn eq_ail(&self, other: &AilExpression) -> bool { - self.header.idx == other.header.idx && self.likes(other) + self.cmp_ail::<{ CmpMode::Eq.as_u8() }>(other) } /// Recursive ``replace`` -- walk the operand subtrees, substituting @@ -1920,30 +1963,37 @@ impl AilExpression { h } - /// Structural-with-identity equality. Two expressions ``likes`` each - /// other when they are the same variant carrying the same identifying - /// information AND their operands transitively ``likes`` each other. + /// The single comparison walk backing ``__eq__`` / ``likes`` / + /// ``matches``. See [`CmpMode`] for the mode hierarchy. /// - /// For SSA atoms ``VirtualVariable`` this means the ``varid`` must - /// agree -- ``likes`` will distinguish two structurally identical - /// reads of the same register that come from different definitions. - /// Contrast with ``matches``: ``matches`` is the structural-only - /// sibling that ignores ``varid`` (and other identifying fields) and - /// only requires the *shape* of the expression to be the same. + /// `MODE` is a const generic, so each relation monomorphizes into its + /// own specialized function and every ``mode ==`` test below folds + /// away at compile time -- one source of truth, no per-node branch in + /// a walk that runs millions of times per decompile. It is a ``u8`` + /// rather than a [`CmpMode`] only because stable Rust does not allow + /// enum const-generic parameters; ``mode`` below recovers the enum, + /// and being a compile-time constant it costs nothing. /// - /// Rule of thumb: - /// * ``likes`` = "is this the same value at the AIL level" -- used - /// by Python ``__eq__`` (after the idx-first short-circuit), by - /// rewriting passes that replace one node with an equivalent one, - /// and anywhere identity within the SSA-numbered IR matters. - /// * ``matches`` = "do these two expressions have the same shape" -- - /// used by deduplication / similarity passes that need to recognize - /// that the same source expression compiled into two different - /// SSA-numbered occurrences should be treated as identical. - pub fn likes(&self, other: &AilExpression) -> bool { + /// Children are compared under the *same* `MODE`. That is what makes + /// a relaxation propagate through every container variant instead of + /// only the ones someone remembered to override: the hand-maintained + /// ``matches`` override table this replaced silently fell back to + /// ``likes`` for ``Struct`` / ``Array`` / ``RustEnum`` / ``Let`` / + /// ``FunctionLikeMacro``, so the relaxation stopped dead at those + /// container boundaries. + pub fn cmp_ail(&self, other: &AilExpression) -> bool { + let mode = CmpMode::from_u8(MODE); if self.kind() != other.kind() { return false; } + // ``__eq__`` is ``likes`` plus ``idx`` equality at *every* node, + // enforced here -- once, uniformly, so a new variant cannot forget + // it. This is what keeps ``__eq__`` consistent with the ``Hash`` + // impl, which likewise folds ``header.idx`` in at every node: two + // expressions that compare equal must hash equal. + if mode == CmpMode::Eq && self.header.idx != other.header.idx { + return false; + } // Treat ``NaN`` as equal to ``NaN`` to mirror the legacy Python // ``Const.likes`` (which short-circuits via ``self.value is // other.value``). With the default IEEE 754 ``PartialEq`` on @@ -1972,7 +2022,7 @@ impl AilExpression { ( ExprInner::ComboRegister { registers: a, .. }, ExprInner::ComboRegister { registers: b, .. }, - ) => a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| x.likes(y)), + ) => a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| x.cmp_ail::(y)), ( ExprInner::Phi { src_and_vvars: a, .. @@ -1984,13 +2034,28 @@ impl AilExpression { if self.header.bits != other.header.bits || a.len() != b.len() { return false; } + if mode == CmpMode::Matches { + // Order-insensitive: every ``src`` (src_addr, src_idx) + // in ``a`` must appear in ``b``. The ``vvar`` payloads + // are intentionally ignored (mirrors master's + // ``Phi.matches``). + 'outer: for ea in a.iter() { + for eb in b.iter() { + if ea.src_addr == eb.src_addr && ea.src_idx == eb.src_idx { + continue 'outer; + } + } + return false; + } + return true; + } a.iter().zip(b.iter()).all(|(x, y)| { if x.src_addr != y.src_addr || x.src_idx != y.src_idx { return false; } match (&x.vvar, &y.vvar) { (None, None) => true, - (Some(xv), Some(yv)) => xv.likes(yv), + (Some(xv), Some(yv)) => xv.cmp_ail::(yv), _ => false, } }) @@ -2008,7 +2073,15 @@ impl AilExpression { oident: b_o, .. }, - ) => a_id == b_id && self.header.bits == other.header.bits && a_c == b_c && a_o == b_o, + ) => { + // ``matches`` drops ``varid`` -- the single most important + // relaxation. It lets the dedup passes recognize the same + // source-level read across two SSA branches. + (mode == CmpMode::Matches || a_id == b_id) + && self.header.bits == other.header.bits + && a_c == b_c + && a_o == b_o + } ( ExprInner::UnaryOp { op: a_op, @@ -2020,7 +2093,11 @@ impl AilExpression { operand: b_op_, .. }, - ) => a_op == b_op && self.header.bits == other.header.bits && a_op_.likes(b_op_), + ) => { + a_op == b_op + && self.header.bits == other.header.bits + && a_op_.cmp_ail::(b_op_) + } ( ExprInner::Convert { operand: a_o, @@ -2029,7 +2106,7 @@ impl AilExpression { is_signed: a_s, from_type: a_ft, to_type: a_tt, - .. + rounding_mode: a_rm, }, ExprInner::Convert { operand: b_o, @@ -2038,7 +2115,7 @@ impl AilExpression { is_signed: b_s, from_type: b_ft, to_type: b_tt, - .. + rounding_mode: b_rm, }, ) => { a_fb == b_fb @@ -2047,7 +2124,8 @@ impl AilExpression { && a_ft == b_ft && a_tt == b_tt && self.header.bits == other.header.bits - && a_o.likes(b_o) + && RoundingModeOrExpr::opt_cmp_ail::(a_rm, b_rm) + && a_o.cmp_ail::(b_o) } ( ExprInner::Reinterpret { @@ -2066,13 +2144,20 @@ impl AilExpression { to_type: b_tt, .. }, - ) => a_fb == b_fb && a_tb == b_tb && a_ft == b_ft && a_tt == b_tt && a_o.likes(b_o), + ) => { + a_fb == b_fb + && a_tb == b_tb + && a_ft == b_ft + && a_tt == b_tt + && a_o.cmp_ail::(b_o) + } ( ExprInner::BinaryOp { op: op_a, operands: ops_a, signed: s_a, floating_point: fp_a, + rounding_mode: rm_a, .. }, ExprInner::BinaryOp { @@ -2080,6 +2165,7 @@ impl AilExpression { operands: ops_b, signed: s_b, floating_point: fp_b, + rounding_mode: rm_b, .. }, ) => { @@ -2087,8 +2173,9 @@ impl AilExpression { && s_a == s_b && fp_a == fp_b && self.header.bits == other.header.bits - && ops_a[0].likes(&ops_b[0]) - && ops_a[1].likes(&ops_b[1]) + && RoundingModeOrExpr::opt_cmp_ail::(rm_a, rm_b) + && ops_a[0].cmp_ail::(&ops_b[0]) + && ops_a[1].cmp_ail::(&ops_b[1]) } ( ExprInner::Load { @@ -2103,7 +2190,7 @@ impl AilExpression { endness: b_end, .. }, - ) => a_size == b_size && a_end == b_end && a_addr.likes(b_addr), + ) => a_size == b_size && a_end == b_end && a_addr.cmp_ail::(b_addr), ( ExprInner::Struct { name: a_n, @@ -2118,14 +2205,18 @@ impl AilExpression { .. }, ) => { - if a_n != b_n || a_f.len() != b_f.len() || a_o != b_o { + if a_n != b_n + || a_f.len() != b_f.len() + || a_o != b_o + || self.header.bits != other.header.bits + { return false; } for (off, e) in a_f { let Some(other_e) = b_f.get(off) else { return false; }; - if !e.likes(other_e) { + if !e.cmp_ail::(other_e) { return false; } } @@ -2144,14 +2235,22 @@ impl AilExpression { a_n == b_n && self.header.bits == other.header.bits && a_f.len() == b_f.len() - && a_f.iter().zip(b_f.iter()).all(|(a, b)| a.likes(b)) + && a_f + .iter() + .zip(b_f.iter()) + .all(|(a, b)| a.cmp_ail::(b)) } (ExprInner::Array { elements: a_e }, ExprInner::Array { elements: b_e }) => { self.header.bits == other.header.bits && a_e.len() == b_e.len() - && a_e.iter().zip(b_e.iter()).all(|(a, b)| a.likes(b)) + && a_e + .iter() + .zip(b_e.iter()) + .all(|(a, b)| a.cmp_ail::(b)) + } + (ExprInner::Let { src: a_s, .. }, ExprInner::Let { src: b_s, .. }) => { + a_s.cmp_ail::(b_s) } - (ExprInner::Let { src: a_s, .. }, ExprInner::Let { src: b_s, .. }) => a_s.likes(b_s), ( ExprInner::Macro { name: a_n, @@ -2184,7 +2283,8 @@ impl AilExpression { match (a_a, b_a) { (None, None) => true, (Some(x), Some(y)) => { - x.len() == y.len() && x.iter().zip(y.iter()).all(|(a, b)| a.likes(b)) + x.len() == y.len() + && x.iter().zip(y.iter()).all(|(a, b)| a.cmp_ail::(b)) } _ => false, } @@ -2217,13 +2317,16 @@ impl AilExpression { let opt_likes = |a: &Option>, b: &Option>| match (a, b) { (None, None) => true, - (Some(x), Some(y)) => x.likes(y), + (Some(x), Some(y)) => x.cmp_ail::(y), _ => false, }; opt_likes(a_g, b_g) && opt_likes(a_ma, b_ma) && a_ops.len() == b_ops.len() - && a_ops.iter().zip(b_ops.iter()).all(|(x, y)| x.likes(y)) + && a_ops + .iter() + .zip(b_ops.iter()) + .all(|(x, y)| x.cmp_ail::(y)) } ( ExprInner::VEXCCallExpression { @@ -2238,7 +2341,10 @@ impl AilExpression { a_c == b_c && self.header.bits == other.header.bits && a_ops.len() == b_ops.len() - && a_ops.iter().zip(b_ops.iter()).all(|(x, y)| x.likes(y)) + && a_ops + .iter() + .zip(b_ops.iter()) + .all(|(x, y)| x.cmp_ail::(y)) } ( ExprInner::MultiStatementExpression { @@ -2251,8 +2357,11 @@ impl AilExpression { }, ) => { a_s.len() == b_s.len() - && a_s.iter().zip(b_s.iter()).all(|(x, y)| x.likes(y)) - && a_e.likes(b_e) + && a_s + .iter() + .zip(b_s.iter()) + .all(|(x, y)| x.cmp_ail::(y)) + && a_e.cmp_ail::(b_e) } ( ExprInner::Call { @@ -2268,13 +2377,14 @@ impl AilExpression { ) => { // ``CFGTarget::likes`` already dispatches structurally; // for the Expr arm it routes through ``AilExpression::likes``. - if !a_t.likes(b_t) { + if !a_t.cmp_ail::(b_t) { return false; } match (a_args, b_args) { (None, None) => true, (Some(a), Some(b)) => { - a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| x.likes(y)) + a.len() == b.len() + && a.iter().zip(b.iter()).all(|(x, y)| x.cmp_ail::(y)) } _ => false, } @@ -2294,9 +2404,9 @@ impl AilExpression { }, ) => { self.header.bits == other.header.bits - && ac.likes(bc) - && af.likes(bf) - && at.likes(bt) + && ac.cmp_ail::(bc) + && af.cmp_ail::(bf) + && at.cmp_ail::(bt) } ( ExprInner::Extract { @@ -2309,7 +2419,12 @@ impl AilExpression { offset: bo, endness: be, }, - ) => self.header.bits == other.header.bits && ae == be && ab.likes(bb) && ao.likes(bo), + ) => { + self.header.bits == other.header.bits + && ae == be + && ab.cmp_ail::(bb) + && ao.cmp_ail::(bo) + } ( ExprInner::Insert { base: ab, @@ -2323,8 +2438,15 @@ impl AilExpression { value: bv, endness: be, }, - ) => ae == be && ab.likes(bb) && ao.likes(bo) && av.likes(bv), - (ExprInner::StringLiteral { data: a }, ExprInner::StringLiteral { data: b }) => a == b, + ) => { + ae == be + && ab.cmp_ail::(bb) + && ao.cmp_ail::(bo) + && av.cmp_ail::(bv) + } + (ExprInner::StringLiteral { data: a }, ExprInner::StringLiteral { data: b }) => { + a == b && self.header.bits == other.header.bits + } ( ExprInner::BasePointerOffset { base: a_b, @@ -2345,6 +2467,30 @@ impl AilExpression { } } + /// Structural-with-identity equality. Two expressions ``likes`` each + /// other when they are the same variant carrying the same identifying + /// information AND their operands transitively ``likes`` each other. + /// + /// For SSA atoms ``VirtualVariable`` this means the ``varid`` must + /// agree -- ``likes`` will distinguish two structurally identical + /// reads of the same register that come from different definitions. + /// Contrast with ``matches``: ``matches`` is the structural-only + /// sibling that ignores ``varid`` (and other identifying fields) and + /// only requires the *shape* of the expression to be the same. + /// + /// Rule of thumb: + /// * ``likes`` = "is this the same value at the AIL level" -- used + /// by Python ``__eq__`` (after the idx-first short-circuit), by + /// rewriting passes that replace one node with an equivalent one, + /// and anywhere identity within the SSA-numbered IR matters. + /// * ``matches`` = "do these two expressions have the same shape" -- + /// used by deduplication / similarity passes that need to recognize + /// that the same source expression compiled into two different + /// SSA-numbered occurrences should be treated as identical. + pub fn likes(&self, other: &AilExpression) -> bool { + self.cmp_ail::<{ CmpMode::Likes.as_u8() }>(other) + } + /// Structural-only equality. Unlike ``likes``, ``matches`` ignores /// identifying fields that distinguish two structurally identical /// occurrences of the same source-level expression: @@ -2370,303 +2516,7 @@ impl AilExpression { /// duplicates even though SSA renumbering gave their values /// different ``varid``s. pub fn matches(&self, other: &AilExpression) -> bool { - if self.kind() != other.kind() { - return false; - } - match (&self.inner, &other.inner) { - // -- VirtualVariable: matches ignores ``varid``. This is the - // -- single most important relaxation -- it lets the dedup - // -- passes recognize the same source-level read across two - // -- SSA branches. - ( - ExprInner::VirtualVariable { - category: a_c, - oident: a_o, - .. - }, - ExprInner::VirtualVariable { - category: b_c, - oident: b_o, - .. - }, - ) => self.header.bits == other.header.bits && a_c == b_c && a_o == b_o, - // -- Phi: same shape, but per-source pairs only require the - // -- *source* to match; the vvar id is ignored (per master's - // -- ``Phi.matches``). The legacy contract walks the dicts - // -- and only verifies the keys (sources) line up. - ( - ExprInner::Phi { - src_and_vvars: a, .. - }, - ExprInner::Phi { - src_and_vvars: b, .. - }, - ) => { - if self.header.bits != other.header.bits || a.len() != b.len() { - return false; - } - // Order-insensitive: every ``src`` (src_addr, src_idx) - // in ``a`` must appear in ``b``. ``vvar_id`` payloads - // are intentionally ignored (mirrors master's - // ``Phi.matches``). - 'outer: for ea in a.iter() { - for eb in b.iter() { - if ea.src_addr == eb.src_addr && ea.src_idx == eb.src_idx { - continue 'outer; - } - } - return false; - } - true - } - // -- Recursive variants: descend via ``matches`` so the - // -- relaxation propagates. - ( - ExprInner::UnaryOp { - op: a_op, - operand: a_o, - .. - }, - ExprInner::UnaryOp { - op: b_op, - operand: b_o, - .. - }, - ) => a_op == b_op && self.header.bits == other.header.bits && a_o.matches(b_o), - ( - ExprInner::Convert { - operand: a_o, - from_bits: a_fb, - to_bits: a_tb, - is_signed: a_s, - from_type: a_ft, - to_type: a_tt, - .. - }, - ExprInner::Convert { - operand: b_o, - from_bits: b_fb, - to_bits: b_tb, - is_signed: b_s, - from_type: b_ft, - to_type: b_tt, - .. - }, - ) => { - a_fb == b_fb - && a_tb == b_tb - && a_s == b_s - && a_ft == b_ft - && a_tt == b_tt - && self.header.bits == other.header.bits - && a_o.matches(b_o) - } - ( - ExprInner::Reinterpret { - operand: a_o, - from_bits: a_fb, - from_type: a_ft, - to_bits: a_tb, - to_type: a_tt, - .. - }, - ExprInner::Reinterpret { - operand: b_o, - from_bits: b_fb, - from_type: b_ft, - to_bits: b_tb, - to_type: b_tt, - .. - }, - ) => a_fb == b_fb && a_tb == b_tb && a_ft == b_ft && a_tt == b_tt && a_o.matches(b_o), - ( - ExprInner::BinaryOp { - op: op_a, - operands: ops_a, - signed: s_a, - floating_point: fp_a, - .. - }, - ExprInner::BinaryOp { - op: op_b, - operands: ops_b, - signed: s_b, - floating_point: fp_b, - .. - }, - ) => { - op_a == op_b - && s_a == s_b - && fp_a == fp_b - && self.header.bits == other.header.bits - && ops_a[0].matches(&ops_b[0]) - && ops_a[1].matches(&ops_b[1]) - } - ( - ExprInner::Load { - addr: a_addr, - size: a_size, - endness: a_end, - .. - }, - ExprInner::Load { - addr: b_addr, - size: b_size, - endness: b_end, - .. - }, - ) => a_size == b_size && a_end == b_end && a_addr.matches(b_addr), - ( - ExprInner::ITE { - cond: ac, - iffalse: af, - iftrue: at, - .. - }, - ExprInner::ITE { - cond: bc, - iffalse: bf, - iftrue: bt, - .. - }, - ) => { - self.header.bits == other.header.bits - && ac.matches(bc) - && af.matches(bf) - && at.matches(bt) - } - ( - ExprInner::Extract { - base: ab, - offset: ao, - endness: ae, - }, - ExprInner::Extract { - base: bb, - offset: bo, - endness: be, - }, - ) => { - self.header.bits == other.header.bits - && ae == be - && ab.matches(bb) - && ao.matches(bo) - } - ( - ExprInner::Insert { - base: ab, - offset: ao, - value: av, - endness: ae, - }, - ExprInner::Insert { - base: bb, - offset: bo, - value: bv, - endness: be, - }, - ) => ae == be && ab.matches(bb) && ao.matches(bo) && av.matches(bv), - ( - ExprInner::Call { - target: a_t, - args: a_args, - .. - }, - ExprInner::Call { - target: b_t, - args: b_args, - .. - }, - ) => { - if !a_t.matches(b_t) { - return false; - } - match (a_args, b_args) { - (None, None) => true, - (Some(a), Some(b)) => { - a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| x.matches(y)) - } - _ => false, - } - } - ( - ExprInner::DirtyExpression { - callee: a_c, - operands: a_ops, - guard: a_g, - mfx: a_mfx, - maddr: a_ma, - msize: a_ms, - }, - ExprInner::DirtyExpression { - callee: b_c, - operands: b_ops, - guard: b_g, - mfx: b_mfx, - maddr: b_ma, - msize: b_ms, - }, - ) => { - if a_c != b_c - || a_mfx != b_mfx - || a_ms != b_ms - || self.header.bits != other.header.bits - { - return false; - } - let opt_matches = - |a: &Option>, b: &Option>| match (a, b) { - (None, None) => true, - (Some(x), Some(y)) => x.matches(y), - _ => false, - }; - opt_matches(a_g, b_g) - && opt_matches(a_ma, b_ma) - && a_ops.len() == b_ops.len() - && a_ops.iter().zip(b_ops.iter()).all(|(x, y)| x.matches(y)) - } - ( - ExprInner::VEXCCallExpression { - callee: a_c, - operands: a_ops, - }, - ExprInner::VEXCCallExpression { - callee: b_c, - operands: b_ops, - }, - ) => { - a_c == b_c - && self.header.bits == other.header.bits - && a_ops.len() == b_ops.len() - && a_ops.iter().zip(b_ops.iter()).all(|(x, y)| x.matches(y)) - } - ( - ExprInner::MultiStatementExpression { - stmts: a_s, - expr: a_e, - }, - ExprInner::MultiStatementExpression { - stmts: b_s, - expr: b_e, - }, - ) => { - a_s.len() == b_s.len() - && a_s.iter().zip(b_s.iter()).all(|(x, y)| x.matches(y)) - && a_e.matches(b_e) - } - // -- ComboRegister: recurses via matches but Python defines - // -- ``matches = likes`` for it. Since likes already recurses - // -- via ``likes`` and there's no varid in plain Register, the - // -- two are equivalent. Keep the recursion explicit for - // -- forward-consistency. - ( - ExprInner::ComboRegister { registers: a, .. }, - ExprInner::ComboRegister { registers: b, .. }, - ) => a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| x.matches(y)), - // -- All other variants: no identifying info distinguishes - // -- ``matches`` from ``likes``. Defer to ``likes``. - _ => self.likes(other), - } + self.cmp_ail::<{ CmpMode::Matches.as_u8() }>(other) } } @@ -5113,13 +4963,7 @@ impl Expression { }; let s = slf.borrow(); let o = o.borrow(); - if s.expr.kind() != o.expr.kind() { - return Ok(false); - } - if s.expr.header.idx != o.expr.header.idx { - return Ok(false); - } - Ok(s.expr.likes(&o.expr)) + Ok(s.expr.eq_ail(&o.expr)) } // --- Repr --------------------------------------------------------- diff --git a/native/angr/src/ailment/ail_stmt.rs b/native/angr/src/ailment/ail_stmt.rs index 193050401..c972d1a52 100644 --- a/native/angr/src/ailment/ail_stmt.rs +++ b/native/angr/src/ailment/ail_stmt.rs @@ -23,7 +23,7 @@ use pyo3::types::{PyBytes, PyDict, PyList}; use crate::ailment::ail_expr::{AilExpression, CFGTarget, Expression, VariantIdx, next}; use crate::ailment::enums::StatementKind; use crate::ailment::tags::{Tags, TagsView}; -use crate::ailment::{CachedHash, hash_of}; +use crate::ailment::{CachedHash, CmpMode, hash_of}; use serde::de::{self, EnumAccess, SeqAccess, VariantAccess, Visitor}; use serde::ser::{SerializeStruct, SerializeTupleVariant}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -622,23 +622,24 @@ impl AilStatement { } } - /// Structural-with-identity equality on statements. ``likes`` is the - /// statement-level analogue of ``AilExpression::likes``: two statements - /// like each other when they are the same variant, every sub-expression - /// like-matches, and every plain-Python slot (Jump targets, - /// SimCC/SimType payloads, etc.) compares equal via Python ``==``. - /// Sub-expressions are compared via ``AilExpression::likes``, so SSA - /// ``varid`` differences propagate up and cause two structurally - /// identical statements to not ``likes`` each other. + /// The single comparison walk backing ``__eq__`` / ``likes`` / + /// ``matches`` on statements -- the statement-level counterpart of + /// [`AilExpression::cmp_ail`]. See [`CmpMode`] for the mode hierarchy. /// - /// Backs Python ``Statement.__eq__`` (after an idx-first short-circuit) - /// and is used by rewriting passes that replace one statement with an - /// SSA-equivalent one. For the structural-only variant that dedup / - /// similarity passes want, see ``matches`` below. - pub fn likes(&self, other: &AilStatement) -> bool { + /// No arm here branches on `MODE`: statements carry no SSA + /// identifying info of their own, so the three relations differ only + /// in how their sub-expressions are compared, and `MODE` simply + /// passes through to [`AilExpression::cmp_ail`]. + pub fn cmp_ail(&self, other: &AilStatement) -> bool { + let mode = CmpMode::from_u8(MODE); if self.kind() != other.kind() { return false; } + // Same uniform idx guard as the expression side -- see + // ``AilExpression::cmp_ail``. + if mode == CmpMode::Eq && self.header.idx != other.header.idx { + return false; + } match (&self.inner, &other.inner) { ( StmtInner::Assignment { dst: a_d, src: a_s }, @@ -647,7 +648,7 @@ impl AilStatement { | ( StmtInner::WeakAssignment { dst: a_d, src: a_s }, StmtInner::WeakAssignment { dst: b_d, src: b_s }, - ) => a_d.likes(b_d) && a_s.likes(b_s), + ) => a_d.cmp_ail::(b_d) && a_s.cmp_ail::(b_s), (StmtInner::Label { name: a }, StmtInner::Label { name: b }) => a == b, ( StmtInner::Store { @@ -664,7 +665,7 @@ impl AilStatement { endness: b_e, .. }, - ) => a_s == b_s && a_e == b_e && a_a.likes(b_a) && a_d.likes(b_d), + ) => a_s == b_s && a_e == b_e && a_a.cmp_ail::(b_a) && a_d.cmp_ail::(b_d), ( StmtInner::Jump { target: a_t, @@ -674,7 +675,7 @@ impl AilStatement { target: b_t, target_idx: b_ti, }, - ) => a_ti == b_ti && a_t.likes(b_t), + ) => a_ti == b_ti && a_t.cmp_ail::(b_t), ( StmtInner::ConditionalJump { condition: a_c, @@ -689,15 +690,15 @@ impl AilStatement { .. }, ) => { - if !a_c.likes(b_c) { + if !a_c.cmp_ail::(b_c) { return false; } - let opt_likes = |a: &Option, b: &Option| match (a, b) { + let opt_cmp = |a: &Option, b: &Option| match (a, b) { (None, None) => true, - (Some(x), Some(y)) => x.likes(y), + (Some(x), Some(y)) => x.cmp_ail::(y), _ => false, }; - opt_likes(a_t, b_t) && opt_likes(a_f, b_f) + opt_cmp(a_t, b_t) && opt_cmp(a_f, b_f) } ( StmtInner::SideEffectStatement { @@ -711,16 +712,16 @@ impl AilStatement { fp_ret_expr: b_f, }, ) => { - let opt_likes = + let opt_cmp = |a: &Option>, b: &Option>| match (a, b) { (None, None) => true, - (Some(x), Some(y)) => x.likes(y), + (Some(x), Some(y)) => x.cmp_ail::(y), _ => false, }; - a_e.likes(b_e) && opt_likes(a_r, b_r) && opt_likes(a_f, b_f) + a_e.cmp_ail::(b_e) && opt_cmp(a_r, b_r) && opt_cmp(a_f, b_f) } (StmtInner::Return { ret_exprs: a }, StmtInner::Return { ret_exprs: b }) => { - a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| x.likes(y)) + a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| x.cmp_ail::(y)) } ( StmtInner::CAS { @@ -744,28 +745,45 @@ impl AilStatement { endness: b_e, }, ) => { - let opt_likes = + let opt_cmp = |a: &Option>, b: &Option>| match (a, b) { (None, None) => true, - (Some(x), Some(y)) => x.likes(y), + (Some(x), Some(y)) => x.cmp_ail::(y), _ => false, }; a_e == b_e - && a_a.likes(b_a) - && a_dl.likes(b_dl) - && opt_likes(a_dh, b_dh) - && a_el.likes(b_el) - && opt_likes(a_eh, b_eh) - && a_ol.likes(b_ol) - && opt_likes(a_oh, b_oh) + && a_a.cmp_ail::(b_a) + && a_dl.cmp_ail::(b_dl) + && opt_cmp(a_dh, b_dh) + && a_el.cmp_ail::(b_el) + && opt_cmp(a_eh, b_eh) + && a_ol.cmp_ail::(b_ol) + && opt_cmp(a_oh, b_oh) } (StmtInner::DirtyStatement { dirty: a }, StmtInner::DirtyStatement { dirty: b }) => { - a.likes(b) + a.cmp_ail::(b) } _ => false, } } + /// Structural-with-identity equality on statements. ``likes`` is the + /// statement-level analogue of ``AilExpression::likes``: two statements + /// like each other when they are the same variant, every sub-expression + /// like-matches, and every plain-Python slot (Jump targets, + /// SimCC/SimType payloads, etc.) compares equal via Python ``==``. + /// Sub-expressions are compared via ``AilExpression::likes``, so SSA + /// ``varid`` differences propagate up and cause two structurally + /// identical statements to not ``likes`` each other. + /// + /// Backs Python ``Statement.__eq__`` (after an idx-first short-circuit) + /// and is used by rewriting passes that replace one statement with an + /// SSA-equivalent one. For the structural-only variant that dedup / + /// similarity passes want, see ``matches`` below. + pub fn likes(&self, other: &AilStatement) -> bool { + self.cmp_ail::<{ CmpMode::Likes.as_u8() }>(other) + } + /// Structural-only equality on statements. The statement-level /// counterpart of ``AilExpression::matches``: sub-expressions are /// compared via ``AilExpression::matches`` rather than ``likes``, @@ -786,134 +804,15 @@ impl AilStatement { /// ``varid``s. Without this relaxation those passes never find /// merge candidates. pub fn matches(&self, other: &AilStatement) -> bool { - if self.kind() != other.kind() { - return false; - } - match (&self.inner, &other.inner) { - ( - StmtInner::Assignment { dst: a_d, src: a_s }, - StmtInner::Assignment { dst: b_d, src: b_s }, - ) - | ( - StmtInner::WeakAssignment { dst: a_d, src: a_s }, - StmtInner::WeakAssignment { dst: b_d, src: b_s }, - ) => a_d.matches(b_d) && a_s.matches(b_s), - (StmtInner::Label { name: a }, StmtInner::Label { name: b }) => a == b, - ( - StmtInner::Store { - addr: a_a, - data: a_d, - size: a_s, - endness: a_e, - .. - }, - StmtInner::Store { - addr: b_a, - data: b_d, - size: b_s, - endness: b_e, - .. - }, - ) => a_s == b_s && a_e == b_e && a_a.matches(b_a) && a_d.matches(b_d), - ( - StmtInner::Jump { - target: a_t, - target_idx: a_ti, - }, - StmtInner::Jump { - target: b_t, - target_idx: b_ti, - }, - ) => a_ti == b_ti && a_t.matches(b_t), - ( - StmtInner::ConditionalJump { - condition: a_c, - true_target: a_t, - false_target: a_f, - .. - }, - StmtInner::ConditionalJump { - condition: b_c, - true_target: b_t, - false_target: b_f, - .. - }, - ) => { - if !a_c.matches(b_c) { - return false; - } - let opt_matches = |a: &Option, b: &Option| match (a, b) { - (None, None) => true, - (Some(x), Some(y)) => x.matches(y), - _ => false, - }; - opt_matches(a_t, b_t) && opt_matches(a_f, b_f) - } - ( - StmtInner::SideEffectStatement { - expr: a_e, - ret_expr: a_r, - fp_ret_expr: a_f, - }, - StmtInner::SideEffectStatement { - expr: b_e, - ret_expr: b_r, - fp_ret_expr: b_f, - }, - ) => { - let opt_matches = - |a: &Option>, b: &Option>| match (a, b) { - (None, None) => true, - (Some(x), Some(y)) => x.matches(y), - _ => false, - }; - a_e.matches(b_e) && opt_matches(a_r, b_r) && opt_matches(a_f, b_f) - } - (StmtInner::Return { ret_exprs: a }, StmtInner::Return { ret_exprs: b }) => { - a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| x.matches(y)) - } - ( - StmtInner::CAS { - addr: a_a, - data_lo: a_dl, - data_hi: a_dh, - expd_lo: a_el, - expd_hi: a_eh, - old_lo: a_ol, - old_hi: a_oh, - endness: a_e, - }, - StmtInner::CAS { - addr: b_a, - data_lo: b_dl, - data_hi: b_dh, - expd_lo: b_el, - expd_hi: b_eh, - old_lo: b_ol, - old_hi: b_oh, - endness: b_e, - }, - ) => { - let opt_matches = - |a: &Option>, b: &Option>| match (a, b) { - (None, None) => true, - (Some(x), Some(y)) => x.matches(y), - _ => false, - }; - a_e == b_e - && a_a.matches(b_a) - && a_dl.matches(b_dl) - && opt_matches(a_dh, b_dh) - && a_el.matches(b_el) - && opt_matches(a_eh, b_eh) - && a_ol.matches(b_ol) - && opt_matches(a_oh, b_oh) - } - (StmtInner::DirtyStatement { dirty: a }, StmtInner::DirtyStatement { dirty: b }) => { - a.matches(b) - } - _ => false, - } + self.cmp_ail::<{ CmpMode::Matches.as_u8() }>(other) + } + + /// ``__eq__`` semantics: ``likes`` plus ``idx`` equality at every + /// node of the statement and of its operand subtrees. Backs Python + /// ``Statement.__eq__``. See [`AilExpression::eq_ail`] for why the + /// idx-awareness has to be recursive rather than root-only. + pub fn eq_ail(&self, other: &AilStatement) -> bool { + self.cmp_ail::<{ CmpMode::Eq.as_u8() }>(other) } } @@ -1877,13 +1776,7 @@ impl Statement { }; let s = slf.borrow(); let o = o.borrow(); - if s.stmt.kind() != o.stmt.kind() { - return Ok(false); - } - if s.stmt.header.idx != o.stmt.header.idx { - return Ok(false); - } - Ok(s.stmt.likes(&o.stmt)) + Ok(s.stmt.eq_ail(&o.stmt)) } // --- Repr --------------------------------------------------------- diff --git a/native/angr/src/ailment/const_value.rs b/native/angr/src/ailment/const_value.rs index f81338696..23d9b09ef 100644 --- a/native/angr/src/ailment/const_value.rs +++ b/native/angr/src/ailment/const_value.rs @@ -93,10 +93,21 @@ impl ConstValue { } } -/// Not derived only because of the ``f64`` payload, which is hashed by -/// bit pattern (consistent with the derived ``PartialEq``: values that -/// compare equal have equal bits, and ``NaN`` never compares equal so -/// its hash is irrelevant). +/// Not derived only because of the ``f64`` payload, which needs its bit +/// pattern canonicalized before hashing. +/// +/// The relevant equality is **not** the derived ``PartialEq`` on this +/// enum but ``AilExpression::cmp_ail``'s ``const_values_eq``, which is +/// what backs ``Const.__eq__``. Two float values compare equal there +/// while having different bits: +/// +/// * any two ``NaN``s -- ``const_values_eq`` deliberately treats +/// ``NaN == NaN`` (IEEE says otherwise), because fixed-point loops in +/// ``BlockSimplifier`` / ``DivSimplifier`` never converge without it; +/// * ``-0.0`` and ``+0.0`` -- IEEE itself calls these equal. +/// +/// Hashing raw ``to_bits()`` therefore broke ``a == b => hash(a) == +/// hash(b)`` for both, so each is folded to a single representative. impl Hash for ConstValue { fn hash(&self, h: &mut H) { match self { @@ -106,7 +117,15 @@ impl Hash for ConstValue { } Self::Float(v) => { 1u8.hash(h); - v.to_bits().hash(h); + let bits = if v.is_nan() { + f64::NAN.to_bits() + } else if *v == 0.0 { + // Catches -0.0; +0.0 already hashes to this. + 0f64.to_bits() + } else { + v.to_bits() + }; + bits.hash(h); } Self::BigInt(b) => { 2u8.hash(h); diff --git a/native/angr/src/ailment/mod.rs b/native/angr/src/ailment/mod.rs index 9ea3b587f..a2a357295 100644 --- a/native/angr/src/ailment/mod.rs +++ b/native/angr/src/ailment/mod.rs @@ -90,6 +90,74 @@ pub fn ailment(m: &Bound<'_, PyModule>) -> PyResult<()> { Ok(()) } +// --------------------------------------------------------------------------- +// Comparison modes +// --------------------------------------------------------------------------- + +/// The three AIL comparison relations, threaded through +/// ``AilExpression::cmp_ail`` / ``AilStatement::cmp_ail`` so each is +/// monomorphized into its own specialized function -- one source of +/// truth, zero per-node branching in the hot path. +/// +/// They form a strict hierarchy, from most to least discriminating: +/// +/// * [`CmpMode::Eq`] -- ``likes`` plus ``idx`` equality at **every** +/// node. Backs Python ``__eq__``. Must stay consistent with the +/// ``Hash`` impls, which fold ``idx`` in at every node: two values +/// that compare equal have to hash equal. +/// * [`CmpMode::Likes`] -- structural-with-identity. ``idx`` is ignored; +/// SSA identifying info (``VirtualVariable::varid``) is significant. +/// * [`CmpMode::Matches`] -- structural-only. Also drops the SSA +/// identifying info, so the same source expression compiled into two +/// different SSA numberings compares equal. +/// +/// Only two arms actually branch on the mode (``VirtualVariable`` and +/// ``Phi``); every other variant is mode-independent and simply passes +/// the mode down to its children, which is what makes the relaxation +/// propagate uniformly. +/// +/// # Why the const generic is a ``u8`` and not this enum +/// +/// Const-generic parameters may only be integers, ``bool`` or ``char`` +/// on stable Rust -- using an enum needs the unstable +/// ``adt_const_params`` feature, and this crate is pinned to a stable +/// toolchain. So ``cmp_ail`` is generic over ``const MODE: u8`` and +/// recovers the enum with [`CmpMode::from_u8`]. Because ``MODE`` is a +/// compile-time constant in every instantiation, that conversion and +/// the comparisons against it fold away entirely. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[repr(u8)] +pub enum CmpMode { + /// ``__eq__``: [`CmpMode::Likes`] plus ``idx`` equality at every node. + Eq = 0, + /// Structural-with-identity comparison. + Likes = 1, + /// Structural-only comparison. + Matches = 2, +} + +impl CmpMode { + /// Recover the mode from a ``cmp_ail`` ``MODE`` const parameter. + /// + /// Total over the three valid discriminants; anything else is a bug + /// at the call site, and since callers pass ``CmpMode::X as u8`` the + /// panic arm is unreachable and optimized out. + pub const fn from_u8(v: u8) -> Self { + match v { + 0 => Self::Eq, + 1 => Self::Likes, + 2 => Self::Matches, + _ => panic!("invalid CmpMode discriminant"), + } + } + + /// The ``MODE`` const-parameter value for this mode. Lets call sites + /// read ``cmp_ail::<{ CmpMode::Likes.as_u8() }>(..)``. + pub const fn as_u8(self) -> u8 { + self as u8 + } +} + // --------------------------------------------------------------------------- // Cross-module hashing utilities: the ``CachedHash`` slot used by the // ``ExprHeader`` / ``StmtHeader`` structs in ``ail_expr`` / ``ail_stmt``, diff --git a/tests/ailment/test_hash_eq_contract.py b/tests/ailment/test_hash_eq_contract.py new file mode 100644 index 000000000..378c53a23 --- /dev/null +++ b/tests/ailment/test_hash_eq_contract.py @@ -0,0 +1,264 @@ +# pylint: disable=missing-class-docstring,no-self-use +"""The ``a == b => hash(a) == hash(b)`` contract for AIL nodes. + +Python requires that objects comparing equal hash equal. The Rust version of AIL broke this in two ways, both +regression-tested here: + +* ``__eq__`` compared ``idx`` only at the root while the ``Hash`` impls fold ``idx`` in at every node, so any + composite whose children had different ``idx`` compared equal and hashed apart. +* A handful of variants hashed a field the comparison never looked at. Those split two ways once examined: + ``Const``'s ``NaN`` and signed-zero bit patterns were hash-side bugs (the values are equal, so they must hash + the same), while ``Convert.rounding_mode``, ``BinaryOp.rounding_mode``, ``StringLiteral.bits`` and + ``Struct.bits`` were comparison-side bugs (these fields were ignored in __eq__). +""" + +from __future__ import annotations + +import struct +import unittest + +import angr.ailment.expression as eu +import angr.ailment.statement as su +from angr.ailment.manager import Manager +from angr.rustylib.ailment import ( # pylint:disable=import-error,no-name-in-module + RoundingMode, + VirtualVariableCategory, +) + +ROOT = 0 # every pair below shares a root idx, so only the children vary + + +class _Kids: + """A fresh set of sub-expressions, each with a distinct ``idx``.""" + + def __init__(self, manager: Manager): + n = manager.next_atom + self.c1 = eu.Const(n(), 1, 32) + self.c2 = eu.Const(n(), 2, 32) + self.addr = eu.Const(n(), 0x1000, 64) + self.reg = eu.Register(n(), 16, 32) + self.vvar = eu.VirtualVariable(n(), 5, 32, VirtualVariableCategory.REGISTER, oident=16) + self.dirty = eu.DirtyExpression(n(), "h", [eu.Const(n(), 1, 32)], bits=32) + + +# name -> build one instance at idx ``i`` from the sub-expressions ``k``. +EXPR_FACTORIES = { + "Const": lambda i, k: eu.Const(i, 1, 32), + "Tmp": lambda i, k: eu.Tmp(i, 5, 64), + "Register": lambda i, k: eu.Register(i, 16, 64), + "ComboRegister": lambda i, k: eu.ComboRegister(i, [k.reg, k.reg]), + "VirtualVariable": lambda i, k: eu.VirtualVariable(i, 5, 64, VirtualVariableCategory.REGISTER, oident=16), + "Phi": lambda i, k: eu.Phi(i, 32, [((0x400000, None), k.vvar)]), + "UnaryOp": lambda i, k: eu.UnaryOp(i, "Neg", k.c1), + "Convert": lambda i, k: eu.Convert(i, 32, 64, False, k.c1), + "Reinterpret": lambda i, k: eu.Reinterpret(i, 32, "I", 32, "F", k.c1), + "BinaryOp": lambda i, k: eu.BinaryOp(i, "Add", (k.c1, k.c2)), + "Load": lambda i, k: eu.Load(i, k.addr, 8, "Iend_LE"), + "Call": lambda i, k: eu.Call(i, k.c1, args=(k.c2,), bits=64), + "ITE": lambda i, k: eu.ITE(i, k.c1, k.c2, k.c1), + "Extract": lambda i, k: eu.Extract(i, 8, k.c1, k.c2, "Iend_LE"), + "Insert": lambda i, k: eu.Insert(i, k.c1, k.c2, k.c1, "Iend_LE"), + "StringLiteral": lambda i, k: eu.StringLiteral(i, "x", 8), + "BasePointerOffset": lambda i, k: eu.BasePointerOffset(i, 64, "base", 0), + "StackBaseOffset": lambda i, k: eu.StackBaseOffset(i, 64, -32), + "DirtyExpression": lambda i, k: eu.DirtyExpression(i, "h", [k.c1], bits=32), + "VEXCCallExpression": lambda i, k: eu.VEXCCallExpression(i, "h", (k.c1,), 32), + "MultiStatementExpression": lambda i, k: eu.MultiStatementExpression(i, [], k.c1), + "Struct": lambda i, k: eu.Struct(i, "F", {0: k.c1}, {"a": 0}, 32), + "RustEnum": lambda i, k: eu.RustEnum(i, "E", [k.c1], 32), + "Array": lambda i, k: eu.Array(i, [k.c1], 32), + "Let": lambda i, k: eu.Let(i, [], k.c1), + "Macro": lambda i, k: eu.Macro(i, "m"), + "FunctionLikeMacro": lambda i, k: eu.FunctionLikeMacro(i, "m", [k.c1]), +} + +STMT_FACTORIES = { + "Assignment": lambda i, k: su.Assignment(i, k.reg, k.c1), + "WeakAssignment": lambda i, k: su.WeakAssignment(i, k.reg, k.c1), + "Label": lambda i, k: su.Label(i, "L"), + "Store": lambda i, k: su.Store(i, k.addr, k.c1, 4, "Iend_LE"), + "Jump": lambda i, k: su.Jump(i, k.addr), + "ConditionalJump": lambda i, k: su.ConditionalJump(i, k.c1, k.addr, k.addr), + "SideEffectStatement": lambda i, k: su.SideEffectStatement(i, k.c1), + "Return": lambda i, k: su.Return(i, [k.c1]), + "CAS": lambda i, k: su.CAS(i, k.addr, k.c1, None, k.c1, None, k.c1, None, "Iend_LE"), + "DirtyStatement": lambda i, k: su.DirtyStatement(i, k.dirty), +} + + +def _assert_contract(case, a, b): + """``a == b`` must imply ``hash(a) == hash(b)`` -- and the set agrees.""" + if a == b: + assert hash(a) == hash(b), f"{case}: a == b but hash(a) != hash(b)" + assert len({a, b}) == 1, f"{case}: a == b but the set kept both" + + +class TestHashEqContract(unittest.TestCase): + """Per-variant sweeps plus the individual node-local regressions.""" + + def _sweep(self, factories): + manager = Manager(arch=None) + for name, build in factories.items(): + with self.subTest(variant=name): + kids_a, kids_b = _Kids(manager), _Kids(manager) + a = build(ROOT, kids_a) + b = build(ROOT, kids_b) # same shape, freshly-built children + a_dup = build(ROOT, kids_a) # same shape AND the same children + + # Positive control: without it a sweep where nothing ever + # compares equal would pass while testing nothing. + assert a == a_dup, f"{name}: identical build did not compare equal" + assert hash(a) == hash(a_dup), f"{name}: identical build hashed differently" + + _assert_contract(name, a, b) + _assert_contract(name, a, a_dup) + + def test_expression_variants(self): + self._sweep(EXPR_FACTORIES) + + def test_statement_variants(self): + self._sweep(STMT_FACTORIES) + + def test_nested_statement_over_expression(self): + """The original report: a statement whose src subtree differs only in idx.""" + manager = Manager(arch=None) + n = manager.next_atom + + def src(): + return eu.BinaryOp(n(), "Add", [eu.Register(n(), 16, 32), eu.Const(n(), 1, 32)], False, bits=32) + + dst, sidx = eu.Register(n(), 24, 32), n() + a = su.Assignment(sidx, dst, src()) + b = su.Assignment(sidx, dst, src()) + assert a.likes(b), "the two should still be structurally alike" + _assert_contract("Assignment/nested", a, b) + + def test_const_nan(self): + """``likes`` calls any NaN equal to any NaN, so they must hash alike.""" + n1 = struct.unpack(" Date: Wed, 29 Jul 2026 01:53:02 -0700 Subject: [PATCH 075/122] Dephication: Correctly consider phi congruence classes. (#6735) --- .../dephication/graph_vvar_mapping.py | 140 ++++++++++++------ tests/analyses/decompiler/test_decompiler.py | 34 +++-- .../test_dephication_interference.py | 117 +++++++++++++++ 3 files changed, 234 insertions(+), 57 deletions(-) create mode 100644 tests/analyses/decompiler/test_dephication_interference.py diff --git a/angr/analyses/decompiler/dephication/graph_vvar_mapping.py b/angr/analyses/decompiler/dephication/graph_vvar_mapping.py index 8f1a83bfc..2298a8a07 100644 --- a/angr/analyses/decompiler/dephication/graph_vvar_mapping.py +++ b/angr/analyses/decompiler/dephication/graph_vvar_mapping.py @@ -72,14 +72,6 @@ class GraphDephicationVVarMapping(Analysis): # pylint:disable=abstract-method # collect phi assignments phi_to_srcvarid = self._collect_phi_assignments() - # initialize phi_congruence_class - phi_congruence_class: dict[int, set[int]] = {} - for phi_varid in phi_to_srcvarid: - phi_congruence_class[phi_varid] = {phi_varid} - for src_and_varids in phi_to_srcvarid.values(): - for _, varid in src_and_varids: - phi_congruence_class[varid] = {varid} - # compute liveness liveness = self.project.analyses.SLiveness( self._function, func_graph=self._graph, entry=self._entry, arg_vvars=self._arg_vvars @@ -89,13 +81,73 @@ class GraphDephicationVVarMapping(Analysis): # pylint:disable=abstract-method live_outs = liveness.model.live_outs interference = liveness.interference_graph() - unresolved_neighbor_map = defaultdict(set) + # A phi congruence class is the transitive closure over phi statements. This means two vvars that never appear + # together in a single phi statement can still land in the same phi congruence class through a chain of phi + # statements. + # + # The following code maintains phi congruence classes in a union-find structure. + # Each congruence class records its members and the set of vvars interfering with any of its members. + parent: dict[int, int] = {} + members: dict[int, set[int]] = {} + interferes_with: dict[int, set[int]] = {} - # check for interferences - candidate_vvar_set_to_phiid: defaultdict[frozenset[int], set[int]] = defaultdict(set) - for phi_id, src_and_varids in phi_to_srcvarid.items(): + def _make(v: int) -> None: + if v not in parent: + parent[v] = v + members[v] = {v} + nbrs = set(interference[v]) if interference.has_node(v) else set() + nbrs.discard(v) # a self-loop carries no information for coalescing + interferes_with[v] = nbrs + + def _find(v: int) -> int: + _make(v) + root = v + while parent[root] != root: + root = parent[root] + while parent[v] != root: + parent[v], v = root, parent[v] + return root + + def _classes_interfere(v0: int, v1: int) -> bool: + r0, r1 = _find(v0), _find(v1) + if r0 == r1: + return False + if len(members[r0]) > len(members[r1]): + r0, r1 = r1, r0 + return not interferes_with[r1].isdisjoint(members[r0]) + + def _union(v0: int, v1: int) -> None: + r0, r1 = _find(v0), _find(v1) + if r0 == r1: + return + if len(members[r0]) < len(members[r1]): + r0, r1 = r1, r0 + parent[r1] = r0 + members[r0] |= members[r1] + interferes_with[r0] |= interferes_with[r1] + del members[r1] + del interferes_with[r1] + + def _note_interference(v0: int, v1: int) -> None: + if v0 == v1: + return + interference.add_edge(v0, v1) + interferes_with[_find(v0)].add(v1) + interferes_with[_find(v1)].add(v0) + + def _loc_key(loc: tuple[int, int | None]) -> tuple[int, int]: + return loc[0], -1 if loc[1] is None else loc[1] + + # process phi statements in a deterministic order + phi_ids = sorted( + phi_to_srcvarid, + key=lambda vid: (_loc_key(self._vvar_defloc[vid][0]), self._vvar_defloc[vid][1], vid), + ) + + for phi_id in phi_ids: + src_and_varids = sorted(phi_to_srcvarid[phi_id], key=lambda t: (_loc_key(t[0]), t[1])) candidate_vvar_set: set[int] = set() - src_and_varids = list(src_and_varids) + unresolved_neighbor_map: defaultdict[int, set[int]] = defaultdict(set) for i in range(-1, len(src_and_varids)): for j in range(i + 1, len(src_and_varids)): @@ -109,16 +161,16 @@ class GraphDephicationVVarMapping(Analysis): # pylint:disable=abstract-method if var1 == var2: continue - if interference.has_edge(var1, var2): + if _classes_interfere(var1, var2): # the intersection considers both liveouts and the vvars used in the last statement of the # block if it is a jump or a conditional jump because we cannot insert a vvar copy statement # after the jump. this is a special case that is not covered in Sreedhar et al.'s paper. It is # documented in "Revisiting Out-of-SSA Translation for Correctness, Code Quality, and # Efficiency" (Section II.A) by Boissinot et. al. - intersection_1 = phi_congruence_class[var1].intersection( + intersection_1 = members[_find(var1)].intersection( live_outs[src2] | liveness.model.block_end_vvars.get(src2, set()) ) - intersection_2 = phi_congruence_class[var2].intersection( + intersection_2 = members[_find(var2)].intersection( live_outs[src1] | liveness.model.block_end_vvars.get(src1, set()) ) if intersection_1 and not intersection_2: @@ -138,7 +190,7 @@ class GraphDephicationVVarMapping(Analysis): # pylint:disable=abstract-method # process unresolved_neighbor_map in a decreasing order of the number of neighbors while unresolved_neighbor_map: - varid, neighbors = max(unresolved_neighbor_map.items(), key=lambda x: len(x[1])) + varid, neighbors = max(unresolved_neighbor_map.items(), key=lambda x: (len(x[1]), x[0])) del unresolved_neighbor_map[varid] candidate_vvar_set.add(varid) @@ -149,19 +201,16 @@ class GraphDephicationVVarMapping(Analysis): # pylint:disable=abstract-method if not unresolved_neighbor_map[neighbor]: del unresolved_neighbor_map[neighbor] - if candidate_vvar_set: - candidate_vvar_set_to_phiid[frozenset(candidate_vvar_set)].add(phi_id) - - for vvar_set, phi_ids in candidate_vvar_set_to_phiid.items(): # insert copies of variables as needed - for varid in vvar_set: - insertion_type, new_vvar_ids = self._insert_vvar_copy(varid, phi_ids) + for varid in sorted(candidate_vvar_set): + insertion_type, new_vvar_ids = self._insert_vvar_copy(varid, {phi_id}) if insertion_type == 0: - for src, old_vvar_id, new_vvar_id in new_vvar_ids: + for src, old_vvar_id, new_vvar_id in sorted( + new_vvar_ids, key=lambda t: (_loc_key(t[0]), t[1], t[2]) + ): self.copied_vvar_ids.add(new_vvar_id) - - phi_congruence_class[new_vvar_id] = {new_vvar_id} + _make(new_vvar_id) live_outs[src].add(new_vvar_id) src_block = self._blocks[(src[0], src[1])] @@ -174,47 +223,44 @@ class GraphDephicationVVarMapping(Analysis): # pylint:disable=abstract-method # update interference graph for vvar_id in live_outs[src]: - interference.add_edge(new_vvar_id, vvar_id) + _note_interference(new_vvar_id, vvar_id) - else: # insertion_type == 1, i.e. the set has only one element - for phi_block_loc, old_phi_varid, new_phi_varid in new_vvar_ids: + else: # insertion_type == 1, i.e. the copy replaces the phi destination + for phi_block_loc, old_phi_varid, new_phi_varid in sorted( + new_vvar_ids, key=lambda t: (_loc_key(t[0]), t[1], t[2]) + ): self.copied_vvar_ids.add(new_phi_varid) - - phi_congruence_class[new_phi_varid] = {new_phi_varid} + _make(new_phi_varid) live_ins[phi_block_loc].discard(old_phi_varid) live_ins[phi_block_loc].add(new_phi_varid) # update interference graph for vvar_id in live_ins[phi_block_loc]: - interference.add_edge(new_phi_varid, vvar_id) + _note_interference(new_phi_varid, vvar_id) - # update phi_congruence_class - for phi_id in phi_to_srcvarid: + # merge the congruence classes of the phi destination and its (possibly rewritten) sources (phidef_block_addr, phidef_block_idx), phidef_stmt_idx = self._vvar_defloc[phi_id] phi_block = self._blocks[(phidef_block_addr, phidef_block_idx)] # phi_stmt is the newly created phi statement with variables replaced phi_stmt = phi_block.statements[phidef_stmt_idx] - phi_src_vvar_ids = {src_vvar.varid for _, src_vvar in phi_stmt.src.src_and_vvars if src_vvar is not None} - new_class = phi_congruence_class[phi_stmt.dst.varid] - for src_vvar_id in phi_src_vvar_ids: - new_class |= phi_congruence_class[src_vvar_id] - phi_congruence_class[src_vvar_id] = new_class + for _, src_vvar in phi_stmt.src.src_and_vvars: + if src_vvar is not None: + _union(phi_stmt.dst.varid, src_vvar.varid) # append statements that were recorded for prepending for block, stmts in self._stmts_to_prepend.items(): for stmt in stmts: self._prepend_stmt(block, stmt) - # remove congruence classes with only one element - for phi_varid in list(phi_congruence_class): - if len(phi_congruence_class[phi_varid]) == 1: - del phi_congruence_class[phi_varid] - mapping: dict[int, int] = {} - for phi_varid, congruence_class in phi_congruence_class.items(): - for varid in congruence_class: - mapping[varid] = phi_varid + for class_members in members.values(): + if len(class_members) <= 1: + # congruence classes with only one element require no remapping + continue + rep = min(class_members) + for varid in class_members: + mapping[varid] = rep return mapping diff --git a/tests/analyses/decompiler/test_decompiler.py b/tests/analyses/decompiler/test_decompiler.py index ba70044b6..8b3136429 100755 --- a/tests/analyses/decompiler/test_decompiler.py +++ b/tests/analyses/decompiler/test_decompiler.py @@ -4770,15 +4770,27 @@ class TestDecompiler(unittest.TestCase): print_decompilation_result(d) lines = [line.strip(" ") for line in d.codegen.text.split("\n")] start_pos = lines.index("{") - assert lines[start_pos + 1 :][:4] == [ - "if (!a1)", - "a1 = a0;", - "g_1234 = a1;", - "return 4660;", - ] or lines[start_pos + 1 :][:2] == [ - "*((int *)&g_1234) = (a1 ? a1 : a0);", - "return 4660;", - ] + assert ( + lines[start_pos + 1 :][:4] + == [ + "if (!a1)", + "a1 = a0;", + "g_1234 = a1;", + "return 4660;", + ] + or lines[start_pos + 1 :][:4] + == [ + "if (a1)", + "a0 = a1;", + "g_1234 = a0;", + "return 4660;", + ] + or lines[start_pos + 1 :][:2] + == [ + "*((int *)&g_1234) = (a1 ? a1 : a0);", + "return 4660;", + ] + ) def test_decompiling_rust_binary_rust_probestack(self, decompiler_options=None): bin_path = os.path.join( @@ -5671,7 +5683,9 @@ class TestDecompiler(unittest.TestCase): assert from_matches_line_no is not None from_matches_line = lines[from_matches_line_no] v = from_matches_line[: from_matches_line.index(".from_matches(")] - for i in range(2): + # the assignments between from_matches() and the if are copies inserted to keep phi congruence classes + # conventional; how many there are depends on how the classes get split, so allow a little slack here + for i in range(3): if ( lines[from_matches_line_no + i + 1] == f"if ({v} != 9223372036854775809)" and lines[from_matches_line_no + i + 2] == "{" diff --git a/tests/analyses/decompiler/test_dephication_interference.py b/tests/analyses/decompiler/test_dephication_interference.py new file mode 100644 index 000000000..b561d8a59 --- /dev/null +++ b/tests/analyses/decompiler/test_dephication_interference.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use,no-member +from __future__ import annotations + +__package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin + +import os.path +import re +import unittest +from collections import defaultdict + +from angr.ailment.expression import Phi, VirtualVariable +from angr.ailment.statement import Assignment +from angr.analyses.decompiler.clinic import Clinic +from angr.analyses.s_liveness import SLivenessAnalysis +from tests.common import bin_location, load_project_with_scoped_cfg, print_decompilation_result + +test_location = os.path.join(bin_location, "tests") + + +class TestDephicationInterference(unittest.TestCase): + """ + The original implementation of Dephication did not properly check for all phi variables in a congruence class. + Instead, it only went the operands of each phi statement and checked for interference among them. + + As such, two vvars that never appear together in one phi statement can still end up in the same phi congruence + class through a chain, but my original implementation missed these. + """ + + @staticmethod + def _naive_congruence_classes(graph) -> dict[int, list[int]]: + """The transitive phi closure that SSA destruction performs, as a union-find.""" + parent: dict[int, int] = {} + + def find(x: int) -> int: + parent.setdefault(x, x) + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + + for block in graph: + for stmt in block.statements: + if isinstance(stmt, Assignment) and isinstance(stmt.dst, VirtualVariable) and isinstance(stmt.src, Phi): + for _, vvar in stmt.src.src_and_vvars: + if vvar is None: + continue + root_dst, root_src = find(stmt.dst.varid), find(vvar.varid) + if root_dst != root_src: + parent[max(root_dst, root_src)] = min(root_dst, root_src) + + classes = defaultdict(list) + for varid in parent: + classes[find(varid)].append(varid) + return classes + + def test_congruence_classes_are_conventional(self): + bin_path = os.path.join(test_location, "x86_64", "ALLSTAR_9base_awk") + proj, cfg = load_project_with_scoped_cfg(bin_path, 0x403010, run_ccc=False) # gettok + func = cfg.functions[0x403010] + + interfering: list[tuple[int, int]] = [] + original = Clinic._collect_dephi_vvar_mapping_and_rewrite_blocks + + def collect(clinic, ail_graph, arg_vvars): + mapping, copied = original(clinic, ail_graph, arg_vvars) + liveness = clinic.project.analyses[SLivenessAnalysis].prep()( + clinic.function, + func_graph=ail_graph, + entry=next(bb for bb in ail_graph if (bb.addr, bb.idx) == clinic.entry_node_addr), + arg_vvars=[vvar for vvar, _ in arg_vvars.values()], + ) + interference = liveness.interference_graph() + for members in self._naive_congruence_classes(ail_graph).values(): + for i, first in enumerate(members): + for second in members[i + 1 :]: + # first != second: a self-loop says nothing about whether two *different* vvars may share + # a variable, and SLivenessAnalysis puts one on nearly every node + if first != second and interference.has_edge(first, second): + interfering.append((first, second)) + return mapping, copied + + Clinic._collect_dephi_vvar_mapping_and_rewrite_blocks = collect + try: + dec = proj.analyses.Decompiler(func, cfg=cfg.model, fail_fast=True) + finally: + Clinic._collect_dephi_vvar_mapping_and_rewrite_blocks = original + + assert dec.codegen is not None and dec.codegen.text is not None + print_decompilation_result(dec) + assert not interfering, ( + "simultaneously live vvars share a phi congruence class, so SSA destruction will merge them into one " + f"variable: {interfering}" + ) + + def test_congruence_classes_causing_incorrect_branch_conditions(self): + bin_path = os.path.join(test_location, "x86_64", "cat") + proj, cfg = load_project_with_scoped_cfg(bin_path, 0x402FA0, run_ccc=False) # gettok + func = cfg.functions[0x402FA0] + dec = proj.analyses.Decompiler(func, fail_fast=True) + assert dec.codegen is not None and dec.codegen.text is not None + print_decompilation_result(dec) + + # check for incorrect branch conditions, which was a result of the bug in the original implementation of + # Dephication. + all_findings = [] + for m in re.finditer(r"!(v\d+) & (v\d+)", dec.codegen.text): + vvar1, vvar2 = m.group(1), m.group(2) + if vvar1 == vvar2: + assert False, f"Found an always-false branch condition (!{vvar1} & {vvar2}) in the decompiled code." + all_findings.append((vvar1, vvar2)) + if not all_findings: + assert False, "Did not find any expression of the form !vvar & vvar. Maybe the decompilation is incorrect?" + + +if __name__ == "__main__": + unittest.main() From a4b05a0ce92c3db820b70c21b49b4f184722b93f Mon Sep 17 00:00:00 2001 From: angr-bot Date: Wed, 29 Jul 2026 09:44:55 +0000 Subject: [PATCH 076/122] Update version to 9.3.2.dev0 [ci skip] --- angr/__init__.py | 2 +- pyproject.toml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/angr/__init__.py b/angr/__init__.py index 159465753..05a973086 100644 --- a/angr/__init__.py +++ b/angr/__init__.py @@ -1,7 +1,7 @@ # pylint: disable=wrong-import-position from __future__ import annotations -__version__ = "9.3.1.dev0" +__version__ = "9.3.2.dev0" if bytes is str: raise Exception(""" diff --git a/pyproject.toml b/pyproject.toml index bbe86f6ea..1e5d02f8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ requires = [ "setuptools>=77.0.0", "setuptools-rust", - "pyvex==9.3.1.dev0", + "pyvex==9.3.2.dev0", "grpcio-tools~=1.80.0", "protobuf>=6.31.1,<7", ] @@ -29,12 +29,12 @@ dependencies = [ "cxxheaderparser", "GitPython", "angr-data~=0.1.0", - "archinfo==9.3.1.dev0", + "archinfo==9.3.2.dev0", "cachetools", "capstone==5.0.6", "cffi>=1.14.0", - "claripy==9.3.1.dev0", - "cle==9.3.1.dev0", + "claripy==9.3.2.dev0", + "cle==9.3.2.dev0", "lmdb==2.1.1", "msgspec; implementation_name == 'cpython'", "mulpyplexer", @@ -45,7 +45,7 @@ dependencies = [ "platformdirs", "pydemumble~=0.1.3", "pypcode~=4.0", - "pyvex==9.3.1.dev0", + "pyvex==9.3.2.dev0", "rich>=13.1.0", "sortedcontainers", "sympy", From d38cc5a0196d8120ef5a21efc12c0a22435e7be5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:46:11 -0700 Subject: [PATCH 077/122] [pre-commit.ci] pre-commit autoupdate (#6721) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/astral-sh/ruff-pre-commit: v0.15.22 → v0.16.0](https://github.com/astral-sh/ruff-pre-commit/compare/v0.15.22...v0.16.0) * Apply fixes * Add values() --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Kevin Phoenix --- .pre-commit-config.yaml | 2 +- README.md | 2 + angr/analyses/backward_slice.py | 4 +- angr/analyses/binary_optimizer.py | 7 +- angr/analyses/bindiff.py | 3 +- angr/analyses/boyscout.py | 4 +- angr/analyses/cfg/cfg_emulated.py | 2 +- angr/analyses/cfg/cfg_fast_soot.py | 1 - angr/analyses/cfg/pe_msvc_eh_structs.py | 4 - angr/analyses/ddg.py | 8 +- angr/analyses/decompiler/block_simplifier.py | 24 ++--- .../ccall_rewriters/rewriter_base.py | 2 +- angr/analyses/decompiler/clinic.py | 4 +- .../counters/seq_cf_structure_counter.py | 2 +- angr/analyses/decompiler/goto_manager.py | 2 +- .../decompiler/jump_target_collector.py | 2 +- angr/analyses/decompiler/label_collector.py | 2 +- .../optimization_passes/__init__.py | 2 +- .../base_ptr_save_simplifier.py | 2 +- .../duplication_reverter/ail_merge_graph.py | 6 +- .../duplication_reverter.py | 8 +- .../duplication_reverter/utils.py | 2 +- .../optimization_passes/engine_base.py | 2 +- .../inlined_memset_simplifier.py | 2 +- .../lowered_switch_simplifier.py | 2 +- .../optimization_passes/optimization_pass.py | 2 +- .../register_save_area_simplifier.py | 2 +- .../ret_addr_save_simplifier.py | 2 +- .../return_duplicator_base.py | 2 +- .../stack_canary_simplifier.py | 6 +- .../x86_gcc_getpc_simplifier.py | 2 +- .../peephole_optimizations/bitwise_inserts.py | 2 +- .../extended_byte_and_mask.py | 1 - .../decompiler/redundant_label_remover.py | 2 +- angr/analyses/decompiler/region_overlay.py | 14 +-- .../region_simplifiers/cascading_ifs.py | 2 +- .../region_simplifiers/expr_folding.py | 4 - .../decompiler/region_simplifiers/goto.py | 2 +- .../decompiler/region_simplifiers/if_.py | 2 +- .../decompiler/region_simplifiers/loop.py | 2 +- .../region_simplifiers/node_address_finder.py | 2 +- .../region_simplifiers/region_simplifier.py | 2 - .../switch_cluster_simplifier.py | 5 +- .../switch_expr_simplifier.py | 2 +- .../semantic_naming/boolean_naming.py | 4 - .../semantic_naming/call_result_naming.py | 4 - .../decompiler/ssailification/traversal.py | 2 +- .../decompiler/structured_codegen/base.py | 6 ++ .../decompiler/structured_codegen/c.py | 4 +- .../structured_codegen/c_serialize.py | 4 +- .../decompiler/structured_codegen/rust.py | 6 +- angr/analyses/decompiler/structurer_nodes.py | 2 +- angr/analyses/decompiler/structuring/dream.py | 2 +- .../decompiler/structuring/phoenix.py | 4 +- .../decompiler/structuring/structurer_base.py | 5 +- angr/analyses/deobfuscator/api_obf_finder.py | 5 +- .../data_transformation_embedder.py | 2 +- angr/analyses/disassembly.py | 18 ++-- angr/analyses/identifier/functions/strcmp.py | 2 - angr/analyses/identifier/functions/strncmp.py | 1 - angr/analyses/identifier/runner.py | 10 +-- angr/analyses/propagator/engine_base.py | 2 +- angr/analyses/propagator/propagator.py | 3 +- angr/analyses/purity/engine.py | 2 +- .../reaching_definitions/dep_graph.py | 4 - .../reaching_definitions/engine_ail.py | 2 +- .../reaching_definitions/engine_vex.py | 2 +- .../reaching_definitions/function_handler.py | 2 +- .../analyses/reaching_definitions/rd_state.py | 2 +- .../reaching_definitions.py | 2 +- angr/analyses/reaching_definitions/subject.py | 2 +- angr/analyses/reassembler.py | 1 - angr/analyses/smc.py | 4 +- angr/analyses/soot_class_hierarchy.py | 6 +- angr/analyses/typehoon/simple_solver.py | 4 +- angr/analyses/variable_recovery/engine_ail.py | 2 +- .../analyses/variable_recovery/engine_base.py | 2 +- angr/analyses/variable_recovery/engine_vex.py | 5 +- .../variable_recovery_base.py | 2 +- angr/calling_conventions.py | 5 +- angr/concretization_strategies/any_named.py | 2 +- angr/concretization_strategies/logging.py | 6 +- angr/distributed/server.py | 2 +- angr/engines/light/data.py | 2 +- angr/engines/vex/claripy/ccall.py | 2 +- angr/engines/vex/claripy/datalayer.py | 4 +- angr/exploration_techniques/stochastic.py | 4 +- angr/exploration_techniques/tracer.py | 2 +- angr/keyed_region.py | 4 +- angr/knowledge_plugins/cfg/cfg_node.py | 4 +- angr/knowledge_plugins/functions/function.py | 4 +- .../functions/function_manager.py | 10 +-- .../functions/function_parser.py | 2 +- .../key_definitions/atoms.py | 2 +- .../key_definitions/definition.py | 2 +- .../key_definitions/live_definitions.py | 2 +- .../propagations/prop_value.py | 2 +- .../propagations/propagation_model.py | 2 +- angr/knowledge_plugins/propagations/states.py | 2 +- .../variables/variable_manager.py | 10 +-- .../procedures/definitions/parse_win32json.py | 25 +++--- angr/procedures/java_lang/stringbuilder.py | 1 - angr/procedures/java_util/list.py | 2 - angr/procedures/java_util/map.py | 2 - angr/procedures/libc/rewind.py | 2 - .../linux_kernel/arm_user_helpers.py | 2 - angr/procedures/stubs/CallReturn.py | 1 - angr/procedures/stubs/Redirect.py | 2 +- angr/project.py | 8 +- .../rust_calling_convention.py | 2 +- .../analyses/rustc_version_identification.py | 2 +- angr/rust/analyses/type_db_loader.py | 2 +- .../deref_coercion_simplifier.py | 2 +- angr/rust/sim_type.py | 4 +- angr/rustylib/__init__.pyi | 5 -- angr/rustylib/ailment.pyi | 6 -- angr/rustylib/automaton.pyi | 3 - angr/rustylib/fuzzer.pyi | 90 +++++-------------- angr/sim_procedure.py | 2 +- angr/sim_state.py | 8 +- angr/sim_type.py | 8 +- angr/simos/__init__.py | 2 +- angr/state_plugins/debug_variables.py | 4 +- angr/state_plugins/heap/heap_ptmalloc.py | 6 +- angr/state_plugins/history.py | 4 +- angr/state_plugins/inspect.py | 2 +- angr/state_plugins/solver.py | 6 +- angr/state_plugins/view.py | 2 +- angr/storage/file.py | 2 - .../paged_memory/page_backer_mixins.py | 2 +- .../paged_memory/pages/ispo_mixin.py | 6 ++ .../paged_memory/pages/multi_values.py | 2 +- .../regioned_memory/regioned_memory_mixin.py | 6 +- angr/utils/graph.py | 8 +- corpus_tests/README.md | 2 + pyproject.toml | 17 ++-- tests/ailment/test_expression.py | 2 +- .../test_baseptr_save_simplifier.py | 2 +- .../analyses/decompiler/test_cas_rewriting.py | 2 +- tests/analyses/decompiler/test_decompiler.py | 4 +- .../decompiler/test_expression_overfolding.py | 2 +- .../decompiler/test_head_controlled_loops.py | 2 +- .../decompiler/test_jumpkind_intrinsics.py | 2 +- .../decompiler/test_narrowing_exprs.py | 2 +- .../decompiler/test_partial_reg_reads.py | 2 +- .../decompiler/test_peephole_inline_memset.py | 2 +- .../test_peephole_redundant_bitshifts.py | 2 +- .../decompiler/test_peephole_wcscpy.py | 2 +- .../decompiler/test_unify_local_variables.py | 2 +- .../test_variable_nondeterminism.py | 1 + .../test_reachingdefinitions.py | 2 +- .../reaching_definitions/test_subject.py | 2 +- tests/analyses/test_callsite_maker.py | 2 +- tests/analyses/test_class_identifier.py | 3 +- tests/common.py | 2 +- tests/engines/test_java.py | 2 +- tests/serialization/test_db.py | 2 +- tests/serialization/test_pickle.py | 4 +- tests/utils/test_library.py | 56 ++++++------ 159 files changed, 303 insertions(+), 415 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3b926357..311fd5056 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,7 +62,7 @@ repos: args: [--py310-plus] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.22 + rev: v0.16.0 hooks: - id: ruff-check args: [--fix, --exit-non-zero-on-fix] diff --git a/README.md b/README.md index 207209332..2f74ac7ec 100644 --- a/README.md +++ b/README.md @@ -43,11 +43,13 @@ import angr project = angr.Project("angr-doc/examples/defcamp_r100/r100", auto_load_libs=False) + @project.hook(0x400844) def print_flag(state): print("FLAG SHOULD BE:", state.posix.dumps(0)) project.terminate_execution() + project.execute() ``` diff --git a/angr/analyses/backward_slice.py b/angr/analyses/backward_slice.py index 10bd266d8..6f12de056 100644 --- a/angr/analyses/backward_slice.py +++ b/angr/analyses/backward_slice.py @@ -323,9 +323,7 @@ class BackwardSlice(Analysis): if simrun not in cfg: l.error("SimRun instance %s is not in the CFG.", simrun) - stack = [] - for simrun in simruns: - stack.append(simrun) + stack = simruns.copy() self.runs_in_slice = networkx.DiGraph() self.cfg_nodes_in_slice = networkx.DiGraph() diff --git a/angr/analyses/binary_optimizer.py b/angr/analyses/binary_optimizer.py index 79c1081a7..1fe41e1f8 100644 --- a/angr/analyses/binary_optimizer.py +++ b/angr/analyses/binary_optimizer.py @@ -108,12 +108,9 @@ class BinaryOptimizer(Analysis): BLOCKS_THRESHOLD = 500 # do not optimize a function if it has more than this number of blocks - def __init__(self, cfg, techniques): + def __init__(self, cfg, techniques: set[str]): self.cfg = cfg - if techniques is None: - raise Exception("At least one optimization technique must be specified.") - supported_techniques = { "constant_propagation", "redundant_stack_variable_removal", @@ -122,7 +119,7 @@ class BinaryOptimizer(Analysis): } if techniques - supported_techniques: - raise Exception("At least one optimization technique specified is not supported.") + raise ValueError("At least one optimization technique specified is not supported.") self._techniques = techniques.copy() diff --git a/angr/analyses/bindiff.py b/angr/analyses/bindiff.py index cebbce8f6..4081680aa 100644 --- a/angr/analyses/bindiff.py +++ b/angr/analyses/bindiff.py @@ -747,8 +747,7 @@ class FunctionDiff: ordered_succ.append(x) # add the rest (sorting might be better than no order) - for s in sorted(succ - set(ordered_succ), key=lambda x: x.addr): - ordered_succ.append(s) + ordered_succ.extend(sorted(succ - set(ordered_succ), key=lambda x: x.addr)) return ordered_succ except (SimMemoryError, SimEngineError): return sorted(succ, key=lambda x: x.addr) diff --git a/angr/analyses/boyscout.py b/angr/analyses/boyscout.py index 0b7f4d780..c0e27f7bf 100644 --- a/angr/analyses/boyscout.py +++ b/angr/analyses/boyscout.py @@ -56,9 +56,7 @@ class BoyScout(Analysis): l.debug("%s %s hits %d times", arch.name, arch.memory_endness, votes[(arch.name, arch.memory_endness)]) - arch_name, endianness, hits = sorted( - [(k[0], k[1], v) for k, v in votes.items()], key=lambda x: x[2], reverse=True - )[0] + arch_name, endianness, hits = max([(k[0], k[1], v) for k, v in votes.items()], key=lambda x: x[2]) if hits < self.cookiesize * 2: # this cannot possibly be code diff --git a/angr/analyses/cfg/cfg_emulated.py b/angr/analyses/cfg/cfg_emulated.py index ab687f4b6..0476dbcc1 100644 --- a/angr/analyses/cfg/cfg_emulated.py +++ b/angr/analyses/cfg/cfg_emulated.py @@ -1139,7 +1139,7 @@ class CFGEmulated(ForwardAnalysis, CFGBase): # pylint: disable=abstract-method self._update_function_callsites(funcaddrs_do_not_return) # Create all pending edges - for _, edges in self._pending_edges.items(): + for edges in self._pending_edges.values(): for src_node, dst_node, data in edges: self._graph_add_edge(src_node, dst_node, **data) diff --git a/angr/analyses/cfg/cfg_fast_soot.py b/angr/analyses/cfg/cfg_fast_soot.py index 78e421424..dbec5ac86 100644 --- a/angr/analyses/cfg/cfg_fast_soot.py +++ b/angr/analyses/cfg/cfg_fast_soot.py @@ -341,7 +341,6 @@ class CFGFastSoot(CFGFast): addr = cfg_node.addr stmts_count = cfg_node.size else: - addr = addr stmts_count = size if addr is None: diff --git a/angr/analyses/cfg/pe_msvc_eh_structs.py b/angr/analyses/cfg/pe_msvc_eh_structs.py index 13664f50c..a160f3c37 100644 --- a/angr/analyses/cfg/pe_msvc_eh_structs.py +++ b/angr/analyses/cfg/pe_msvc_eh_structs.py @@ -14,10 +14,6 @@ from __future__ import annotations import logging import struct -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - pass log = logging.getLogger(__name__) diff --git a/angr/analyses/ddg.py b/angr/analyses/ddg.py index 77f5f7715..a7626f8b4 100644 --- a/angr/analyses/ddg.py +++ b/angr/analyses/ddg.py @@ -756,7 +756,7 @@ class DDG(Analysis): matched = True except (SimUnsatError, SimSolverModeError, ZeroDivisionError): # ignore - matched = matched + pass if not matched: break @@ -1017,7 +1017,7 @@ class DDG(Analysis): self._stmt_graph_annotate_edges(self._register_edges[reg_offset], subtype="mem_addr") reg_variable = SimRegisterVariable(reg_offset, self._get_register_size(reg_offset)) prev_defs = self._def_lookup(reg_variable) - for loc, _ in prev_defs.items(): + for loc in prev_defs: v = ProgramVariable(reg_variable, loc, arch=self.project.arch) self._data_graph_add_edge(v, prog_var, type="mem_addr") @@ -1039,7 +1039,7 @@ class DDG(Analysis): self._stmt_graph_annotate_edges(self._register_edges[reg_offset], subtype="mem_data") reg_variable = SimRegisterVariable(reg_offset, self._get_register_size(reg_offset)) prev_defs = self._def_lookup(reg_variable) - for loc, _ in prev_defs.items(): + for loc in prev_defs: v = ProgramVariable(reg_variable, loc, arch=self.project.arch) self._data_graph_add_edge(v, prog_var, type="mem_data") @@ -1497,7 +1497,7 @@ class DDG(Analysis): # Group all dependencies first block_addr_to_func = {} - for _, func in self.kb.functions.items(): + for func in self.kb.functions.values(): for block in func.blocks: block_addr_to_func[block.addr] = func diff --git a/angr/analyses/decompiler/block_simplifier.py b/angr/analyses/decompiler/block_simplifier.py index d15bb7dc6..ce540b5eb 100644 --- a/angr/analyses/decompiler/block_simplifier.py +++ b/angr/analyses/decompiler/block_simplifier.py @@ -66,14 +66,10 @@ class PeepholeOptimizationBundle: func_addr: int | None = None, preserve_vvar_ids: set[int] | None = None, type_hints: list[tuple[atoms.VirtualVariable | atoms.MemoryLocation, str]] | None = None, - peephole_optimizations: None - | ( - Iterable[ - type[PeepholeOptimizationStmtBase] - | type[PeepholeOptimizationExprBase] - | type[PeepholeOptimizationMultiStmtBase] - ] - ) = None, + peephole_optimizations: Iterable[ + type[PeepholeOptimizationStmtBase | PeepholeOptimizationExprBase | PeepholeOptimizationMultiStmtBase] + ] + | None = None, ): if peephole_optimizations is None: expr_classes: Iterable = EXPR_OPTS @@ -138,14 +134,10 @@ class BlockSimplifier: ail_manager: Manager, func_addr: int | None = None, stack_pointer_tracker=None, - peephole_optimizations: None - | ( - Iterable[ - type[PeepholeOptimizationStmtBase] - | type[PeepholeOptimizationExprBase] - | type[PeepholeOptimizationMultiStmtBase] - ] - ) = None, + peephole_optimizations: Iterable[ + type[PeepholeOptimizationStmtBase | PeepholeOptimizationExprBase | PeepholeOptimizationMultiStmtBase] + ] + | None = None, preserve_vvar_ids: set[int] | None = None, type_hints: list[tuple[atoms.VirtualVariable | atoms.MemoryLocation, str]] | None = None, cached_reaching_definitions=None, diff --git a/angr/analyses/decompiler/ccall_rewriters/rewriter_base.py b/angr/analyses/decompiler/ccall_rewriters/rewriter_base.py index ffacbe4f7..5952656a4 100644 --- a/angr/analyses/decompiler/ccall_rewriters/rewriter_base.py +++ b/angr/analyses/decompiler/ccall_rewriters/rewriter_base.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -import angr.ailment as ailment +from angr import ailment if TYPE_CHECKING: from angr.ailment.manager import Manager diff --git a/angr/analyses/decompiler/clinic.py b/angr/analyses/decompiler/clinic.py index 816315dc3..d057a39d5 100644 --- a/angr/analyses/decompiler/clinic.py +++ b/angr/analyses/decompiler/clinic.py @@ -238,8 +238,8 @@ class Clinic(Analysis, Serializable): insert_labels=True, optimization_passes=None, cfg=None, - peephole_optimizations: None - | (Iterable[type[PeepholeOptimizationStmtBase] | type[PeepholeOptimizationExprBase]]) = None, # pylint:disable=line-too-long + peephole_optimizations: Iterable[type[PeepholeOptimizationStmtBase | PeepholeOptimizationExprBase]] + | None = None, # pylint:disable=line-too-long must_struct: set[str] | None = None, reset_variable_names=False, rewrite_ites_to_diamonds=True, diff --git a/angr/analyses/decompiler/counters/seq_cf_structure_counter.py b/angr/analyses/decompiler/counters/seq_cf_structure_counter.py index 6632a37aa..395c75cae 100644 --- a/angr/analyses/decompiler/counters/seq_cf_structure_counter.py +++ b/angr/analyses/decompiler/counters/seq_cf_structure_counter.py @@ -2,7 +2,7 @@ from __future__ import annotations from collections import defaultdict -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.sequence_walker import SequenceWalker from angr.analyses.decompiler.structurer_nodes import LoopNode diff --git a/angr/analyses/decompiler/goto_manager.py b/angr/analyses/decompiler/goto_manager.py index 67991ff79..1a6563755 100644 --- a/angr/analyses/decompiler/goto_manager.py +++ b/angr/analyses/decompiler/goto_manager.py @@ -2,7 +2,7 @@ from __future__ import annotations import networkx -import angr.ailment as ailment +from angr import ailment from angr.ailment.block import Block from .utils import find_block_by_addr diff --git a/angr/analyses/decompiler/jump_target_collector.py b/angr/analyses/decompiler/jump_target_collector.py index 6a226e164..bc7c2ff74 100644 --- a/angr/analyses/decompiler/jump_target_collector.py +++ b/angr/analyses/decompiler/jump_target_collector.py @@ -1,7 +1,7 @@ # pylint:disable=unused-argument from __future__ import annotations -import angr.ailment as ailment +from angr import ailment from .sequence_walker import SequenceWalker diff --git a/angr/analyses/decompiler/label_collector.py b/angr/analyses/decompiler/label_collector.py index b5710a257..8b26b8095 100644 --- a/angr/analyses/decompiler/label_collector.py +++ b/angr/analyses/decompiler/label_collector.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections import defaultdict -import angr.ailment as ailment +from angr import ailment from .sequence_walker import SequenceWalker diff --git a/angr/analyses/decompiler/optimization_passes/__init__.py b/angr/analyses/decompiler/optimization_passes/__init__.py index 49b7fe74b..a2fab6494 100644 --- a/angr/analyses/decompiler/optimization_passes/__init__.py +++ b/angr/analyses/decompiler/optimization_passes/__init__.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING from archinfo import Arch -import angr.analyses.decompiler as decompiler +from angr.analyses import decompiler from .base_ptr_save_simplifier import BasePointerSaveSimplifier from .call_stmt_rewriter import CallStatementRewriter diff --git a/angr/analyses/decompiler/optimization_passes/base_ptr_save_simplifier.py b/angr/analyses/decompiler/optimization_passes/base_ptr_save_simplifier.py index 091ab7d4b..703cb959b 100644 --- a/angr/analyses/decompiler/optimization_passes/base_ptr_save_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/base_ptr_save_simplifier.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.stack_item import StackItem, StackItemType from .optimization_pass import OptimizationPass, OptimizationPassStage diff --git a/angr/analyses/decompiler/optimization_passes/duplication_reverter/ail_merge_graph.py b/angr/analyses/decompiler/optimization_passes/duplication_reverter/ail_merge_graph.py index e438b5424..74ae1fff6 100644 --- a/angr/analyses/decompiler/optimization_passes/duplication_reverter/ail_merge_graph.py +++ b/angr/analyses/decompiler/optimization_passes/duplication_reverter/ail_merge_graph.py @@ -343,7 +343,7 @@ class AILMergeGraph: self.original_ends = start_blocks # moved here - for _, pair in merge_to_end_pair.items(): + for pair in merge_to_end_pair.values(): for block in pair: other_block = pair[0] if pair[1] is block else pair[1] while True: @@ -390,7 +390,7 @@ class AILMergeGraph: # def _find_block_pair_in_originals(self, block: Block): - for _, originals in self.merge_blocks_to_originals.items(): + for originals in self.merge_blocks_to_originals.values(): # need at least 2 for a pair if len(originals) < 2: continue @@ -454,7 +454,7 @@ class AILMergeGraph: return None def _find_split_block_by_original(self, block: Block) -> AILBlockSplit | None: - for _, split_blocks in self.original_split_blocks.items(): + for split_blocks in self.original_split_blocks.values(): for split_block in split_blocks: if split_block.original == block: return split_block diff --git a/angr/analyses/decompiler/optimization_passes/duplication_reverter/duplication_reverter.py b/angr/analyses/decompiler/optimization_passes/duplication_reverter/duplication_reverter.py index 3de66d623..52b881b92 100644 --- a/angr/analyses/decompiler/optimization_passes/duplication_reverter/duplication_reverter.py +++ b/angr/analyses/decompiler/optimization_passes/duplication_reverter/duplication_reverter.py @@ -7,7 +7,7 @@ from itertools import combinations import networkx as nx -import angr.ailment as ailment +from angr import ailment from angr.ailment.block import Block from angr.ailment.expression import Const, Convert, Expression, Register, VirtualVariable from angr.ailment.statement import Assignment, ConditionalJump, Jump, Label, Return @@ -337,7 +337,7 @@ class DuplicationReverter(StructuringOptimizationPass): for node in graph.nodes: nodes_by_addr[node.addr].append(node) - for _, nodes in nodes_by_addr.items(): + for nodes in nodes_by_addr.values(): if len(nodes) == 1: continue @@ -681,9 +681,9 @@ class DuplicationReverter(StructuringOptimizationPass): # conditions yet. This means the graph is still missing the divergence of the two graphs. try: graph_lcs = longest_ail_graph_subseq(blocks, graph) - except SAILRSemanticError as e: + except SAILRSemanticError: self.candidate_blacklist.add(tuple(blocks)) - raise e + raise ail_merge_graph = AILMergeGraph(original_graph=graph) # some blocks in originals may update during this time (if-statements can change) diff --git a/angr/analyses/decompiler/optimization_passes/duplication_reverter/utils.py b/angr/analyses/decompiler/optimization_passes/duplication_reverter/utils.py index 6dc01787a..c196e4480 100644 --- a/angr/analyses/decompiler/optimization_passes/duplication_reverter/utils.py +++ b/angr/analyses/decompiler/optimization_passes/duplication_reverter/utils.py @@ -4,7 +4,7 @@ import logging import networkx as nx -import angr.ailment as ailment +from angr import ailment from angr.ailment import Const from angr.ailment.block import Block from angr.ailment.statement import ConditionalJump, Jump, Statement diff --git a/angr/analyses/decompiler/optimization_passes/engine_base.py b/angr/analyses/decompiler/optimization_passes/engine_base.py index 967be3698..adeff4c11 100644 --- a/angr/analyses/decompiler/optimization_passes/engine_base.py +++ b/angr/analyses/decompiler/optimization_passes/engine_base.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging -import angr.ailment as ailment +from angr import ailment from angr.engines.light import SimEngineLightAIL _l = logging.getLogger(name=__name__) diff --git a/angr/analyses/decompiler/optimization_passes/inlined_memset_simplifier.py b/angr/analyses/decompiler/optimization_passes/inlined_memset_simplifier.py index 433ec6f10..bc62a026c 100644 --- a/angr/analyses/decompiler/optimization_passes/inlined_memset_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/inlined_memset_simplifier.py @@ -104,7 +104,7 @@ class InlinedMemsetSimplifier(OptimizationPass): replaced_indices: set[int] = set() replacements: dict[int, SideEffectStatement] = {} - for _kind, lst in info_by_kind.items(): + for lst in info_by_kind.values(): if len(lst) <= 1: continue candidates = self._find_memset_candidates(lst) diff --git a/angr/analyses/decompiler/optimization_passes/lowered_switch_simplifier.py b/angr/analyses/decompiler/optimization_passes/lowered_switch_simplifier.py index 84dd089ac..2c32664a0 100644 --- a/angr/analyses/decompiler/optimization_passes/lowered_switch_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/lowered_switch_simplifier.py @@ -232,7 +232,7 @@ class LoweredSwitchSimplifier(StructuringOptimizationPass): node_to_heads = defaultdict(set) modified = False - for _, caselists in variablehash_to_cases.items(): + for caselists in variablehash_to_cases.values(): for cases, redundant_nodes in caselists: real_cases = [case for case in cases if case.value != "default"] max_continuous_cases = self._count_max_continuous_cases(real_cases) diff --git a/angr/analyses/decompiler/optimization_passes/optimization_pass.py b/angr/analyses/decompiler/optimization_passes/optimization_pass.py index def8b58fc..0103e2637 100644 --- a/angr/analyses/decompiler/optimization_passes/optimization_pass.py +++ b/angr/analyses/decompiler/optimization_passes/optimization_pass.py @@ -382,7 +382,7 @@ class OptimizationPass(BaseOptimizationPass): return ail_graph def _get_peephole_bundle(self) -> PeepholeOptimizationBundle: - bundle: None | PeepholeOptimizationBundle = self._scratch.get("peephole_bundle") + bundle: PeepholeOptimizationBundle | None = self._scratch.get("peephole_bundle") if bundle is None or not bundle.matches( self.project, self.manager, self._func.addr, None, None, self._peephole_optimizations ): diff --git a/angr/analyses/decompiler/optimization_passes/register_save_area_simplifier.py b/angr/analyses/decompiler/optimization_passes/register_save_area_simplifier.py index 68b4506eb..380212b7f 100644 --- a/angr/analyses/decompiler/optimization_passes/register_save_area_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/register_save_area_simplifier.py @@ -8,7 +8,7 @@ from itertools import chain import archinfo -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.stack_item import StackItem, StackItemType from angr.calling_conventions import SimRegArg from angr.code_location import CodeLocation diff --git a/angr/analyses/decompiler/optimization_passes/ret_addr_save_simplifier.py b/angr/analyses/decompiler/optimization_passes/ret_addr_save_simplifier.py index 9c88f9090..b64dc463f 100644 --- a/angr/analyses/decompiler/optimization_passes/ret_addr_save_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/ret_addr_save_simplifier.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging from typing import Any -import angr.ailment as ailment +from angr import ailment from angr.calling_conventions import DEFAULT_CC, SimRegArg, default_cc from .optimization_pass import OptimizationPass, OptimizationPassStage diff --git a/angr/analyses/decompiler/optimization_passes/return_duplicator_base.py b/angr/analyses/decompiler/optimization_passes/return_duplicator_base.py index 707992f83..78d26c6d8 100644 --- a/angr/analyses/decompiler/optimization_passes/return_duplicator_base.py +++ b/angr/analyses/decompiler/optimization_passes/return_duplicator_base.py @@ -5,7 +5,7 @@ from typing import Any import networkx -import angr.ailment as ailment +from angr import ailment from angr.ailment import AILBlockRewriter, Block from angr.ailment.expression import Const, Phi, VirtualVariable from angr.ailment.statement import Assignment, ConditionalJump, Jump, Label, Return, SideEffectStatement diff --git a/angr/analyses/decompiler/optimization_passes/stack_canary_simplifier.py b/angr/analyses/decompiler/optimization_passes/stack_canary_simplifier.py index 2d3838682..6983a58dd 100644 --- a/angr/analyses/decompiler/optimization_passes/stack_canary_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/stack_canary_simplifier.py @@ -4,7 +4,7 @@ from __future__ import annotations import logging from collections import defaultdict -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.stack_item import StackItem, StackItemType from angr.utils.bits import s2u @@ -78,9 +78,7 @@ class StackCanarySimplifier(OptimizationPass): pred_addr_to_endpoint_addrs[pred.addr].add(node_addr) found_endpoints = False - for pred_addr in pred_addr_to_endpoint_addrs: - endpoint_addrs = pred_addr_to_endpoint_addrs[pred_addr] - + for endpoint_addrs in pred_addr_to_endpoint_addrs.values(): if len(endpoint_addrs) != 2: # we expect there to be only two nodes: one for canary-check-success, and the other for # canary-check-failure. if not, we check the next predecessor diff --git a/angr/analyses/decompiler/optimization_passes/x86_gcc_getpc_simplifier.py b/angr/analyses/decompiler/optimization_passes/x86_gcc_getpc_simplifier.py index 8dd918327..6a36c9c5d 100644 --- a/angr/analyses/decompiler/optimization_passes/x86_gcc_getpc_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/x86_gcc_getpc_simplifier.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging from typing import Any -import angr.ailment as ailment +from angr import ailment from .optimization_pass import OptimizationPass, OptimizationPassStage diff --git a/angr/analyses/decompiler/peephole_optimizations/bitwise_inserts.py b/angr/analyses/decompiler/peephole_optimizations/bitwise_inserts.py index ce62be1a9..462075cbb 100644 --- a/angr/analyses/decompiler/peephole_optimizations/bitwise_inserts.py +++ b/angr/analyses/decompiler/peephole_optimizations/bitwise_inserts.py @@ -42,7 +42,7 @@ class SimplifyBitwiseInserts(PeepholeOptimizationExprBase): # I don't know if 0 is right here pb2l.append((0, potential_base.operand, other_base)) - for (pb1o, pb1), (pb2o, pb2, pb2x) in itertools.product(pb1l, pb2l): # noqa: B007 + for (pb1o, pb1), (pb2o, pb2, pb2x) in itertools.product(pb1l, pb2l): if pb1o == pb2o and pb1.bits == pb2.bits and pb1.likes(pb2): break else: diff --git a/angr/analyses/decompiler/peephole_optimizations/extended_byte_and_mask.py b/angr/analyses/decompiler/peephole_optimizations/extended_byte_and_mask.py index 3dacea41f..7bbc0a623 100644 --- a/angr/analyses/decompiler/peephole_optimizations/extended_byte_and_mask.py +++ b/angr/analyses/decompiler/peephole_optimizations/extended_byte_and_mask.py @@ -18,7 +18,6 @@ class ExtendedByteAndMask(PeepholeOptimizationExprBase): expr_classes = (BinaryOp,) # all expressions are allowed def optimize(self, expr: BinaryOp, **kwargs): - # if expr.op == "And" and isinstance(expr.operands[1], Const): mask = expr.operands[1].value to_bits = _MASK_TO_BITS.get(mask) diff --git a/angr/analyses/decompiler/redundant_label_remover.py b/angr/analyses/decompiler/redundant_label_remover.py index fcd5793aa..f3019d9f0 100644 --- a/angr/analyses/decompiler/redundant_label_remover.py +++ b/angr/analyses/decompiler/redundant_label_remover.py @@ -1,7 +1,7 @@ # pylint:disable=unused-argument from __future__ import annotations -import angr.ailment as ailment +from angr import ailment from .sequence_walker import SequenceWalker from .structurer_nodes import SequenceNode diff --git a/angr/analyses/decompiler/region_overlay.py b/angr/analyses/decompiler/region_overlay.py index 034adddde..3b8f54fc7 100644 --- a/angr/analyses/decompiler/region_overlay.py +++ b/angr/analyses/decompiler/region_overlay.py @@ -4,7 +4,7 @@ from __future__ import annotations import logging import os from collections import defaultdict -from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Protocol, cast import networkx @@ -293,7 +293,7 @@ class RegionOverlay[T: RegionBound]: return self._mgr @property - def members(self) -> Set[Tx[T]]: + def members(self) -> set[Tx[T]]: return self._members def ancestors(self) -> set[RegionOverlay]: @@ -305,12 +305,12 @@ class RegionOverlay[T: RegionBound]: node = node.parent return result - def underlying_nodes(self) -> Set[T]: + def underlying_nodes(self) -> set[T]: """All shared-graph nodes inside this region (including nodes of nested regions).""" return self._under @staticmethod - def _underlying(x) -> Set[T]: + def _underlying(x) -> set[T]: return x._under if isinstance(x, RegionOverlay) else {x} def create_subregion( @@ -396,7 +396,7 @@ class RegionOverlay[T: RegionBound]: def _is_hidden(self, src: Tx[T], dst: Tx[T]) -> bool: return (src, dst) in self._hidden - def _hidden_context_head_under(self) -> Set[Tx[T]]: + def _hidden_context_head_under(self) -> set[Tx[T]]: """ Crossing edges that target the head of the region's processing context (the nearest cyclic ancestor, or the root region) were invisible during region identification: in-edges of the head are stripped before @@ -430,7 +430,7 @@ class RegionOverlay[T: RegionBound]: def _in_loop(self) -> bool: return self.cyclic or self.cyclic_ancestor - def successor_nodes(self) -> Set[Tx[T]]: + def successor_nodes(self) -> set[Tx[T]]: """ The derived successor set of this region: representatives of all shared-graph nodes targeted by edges leaving the region. @@ -669,7 +669,7 @@ class RegionOverlay[T: RegionBound]: return self.view_with_successors() @property - def successors(self) -> Set[Tx[T]]: + def successors(self) -> set[Tx[T]]: return self.successor_nodes() @property diff --git a/angr/analyses/decompiler/region_simplifiers/cascading_ifs.py b/angr/analyses/decompiler/region_simplifiers/cascading_ifs.py index 7dfd04c50..c239c3c40 100644 --- a/angr/analyses/decompiler/region_simplifiers/cascading_ifs.py +++ b/angr/analyses/decompiler/region_simplifiers/cascading_ifs.py @@ -1,7 +1,7 @@ # pylint:disable=unused-argument,arguments-differ from __future__ import annotations -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.sequence_walker import SequenceWalker from angr.analyses.decompiler.structurer_nodes import ( CascadingConditionNode, diff --git a/angr/analyses/decompiler/region_simplifiers/expr_folding.py b/angr/analyses/decompiler/region_simplifiers/expr_folding.py index 2504687d9..e233f9312 100644 --- a/angr/analyses/decompiler/region_simplifiers/expr_folding.py +++ b/angr/analyses/decompiler/region_simplifiers/expr_folding.py @@ -174,7 +174,6 @@ class LoopNodeFinder(SequenceWalker): def _handle_Loop(self, node: LoopNode, **kwargs): super()._handle_Loop(node, **kwargs) self.loop_nodes.append(node) - return None class MultiStatementExpressionAssignmentFinder(AILBlockRewriter): @@ -387,8 +386,6 @@ class ExpressionCounter(SequenceWalker): super()._handle_Loop(node, **kwargs) self._outer_scope = outer_scope - return None - def _handle_SwitchCase(self, node: SwitchCaseNode, **kwargs): self._collect_uses(node.switch_expr, ConditionLocation(node.addr)) return super()._handle_SwitchCase(node, **kwargs) @@ -744,7 +741,6 @@ class ExpressionFolder(SequenceWalker): node.condition = r # again, do not replace into the loop body - return None def _handle_SwitchCase(self, node: SwitchCaseNode, **kwargs): replacer = ExpressionReplacer(self._assignments, self._uses, self._variable_map) diff --git a/angr/analyses/decompiler/region_simplifiers/goto.py b/angr/analyses/decompiler/region_simplifiers/goto.py index ad1fafde6..d94fcf25c 100644 --- a/angr/analyses/decompiler/region_simplifiers/goto.py +++ b/angr/analyses/decompiler/region_simplifiers/goto.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.goto_manager import Goto from angr.analyses.decompiler.sequence_walker import SequenceWalker from angr.analyses.decompiler.structurer_nodes import ( diff --git a/angr/analyses/decompiler/region_simplifiers/if_.py b/angr/analyses/decompiler/region_simplifiers/if_.py index 1e7de5456..976347f00 100644 --- a/angr/analyses/decompiler/region_simplifiers/if_.py +++ b/angr/analyses/decompiler/region_simplifiers/if_.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.sequence_walker import SequenceWalker from angr.analyses.decompiler.structurer_nodes import ( CascadingConditionNode, diff --git a/angr/analyses/decompiler/region_simplifiers/loop.py b/angr/analyses/decompiler/region_simplifiers/loop.py index 08d6dac6c..34f12ab52 100644 --- a/angr/analyses/decompiler/region_simplifiers/loop.py +++ b/angr/analyses/decompiler/region_simplifiers/loop.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections import defaultdict -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.condition_processor import ConditionProcessor, EmptyBlockNotice from angr.analyses.decompiler.sequence_walker import SequenceWalker from angr.analyses.decompiler.structurer_nodes import ( diff --git a/angr/analyses/decompiler/region_simplifiers/node_address_finder.py b/angr/analyses/decompiler/region_simplifiers/node_address_finder.py index 0d555666d..b3c002c5b 100644 --- a/angr/analyses/decompiler/region_simplifiers/node_address_finder.py +++ b/angr/analyses/decompiler/region_simplifiers/node_address_finder.py @@ -1,7 +1,7 @@ # pylint:disable=unused-argument,arguments-differ from __future__ import annotations -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.sequence_walker import SequenceWalker diff --git a/angr/analyses/decompiler/region_simplifiers/region_simplifier.py b/angr/analyses/decompiler/region_simplifiers/region_simplifier.py index 1d4f358ca..d7e18988c 100644 --- a/angr/analyses/decompiler/region_simplifiers/region_simplifier.py +++ b/angr/analyses/decompiler/region_simplifiers/region_simplifier.py @@ -101,9 +101,7 @@ class RegionSimplifier(Analysis): # Remove unnecessary else branches if the if branch will always return if self._should_simplify_ifelses: r = self._simplify_ifelses(r) - # r = self._simplify_cascading_ifs(r) - # r = self._simplify_loops(r) # Apply loop counter naming after loop simplification (when iterators are identified) if self._apply_loop_counter_naming and self._variable_manager is not None: diff --git a/angr/analyses/decompiler/region_simplifiers/switch_cluster_simplifier.py b/angr/analyses/decompiler/region_simplifiers/switch_cluster_simplifier.py index 6fe8c8740..e1657d729 100644 --- a/angr/analyses/decompiler/region_simplifiers/switch_cluster_simplifier.py +++ b/angr/analyses/decompiler/region_simplifiers/switch_cluster_simplifier.py @@ -5,7 +5,7 @@ import enum from collections import OrderedDict, defaultdict from typing import TYPE_CHECKING, Any -import angr.ailment as ailment +from angr import ailment from angr.ailment import UnaryOp from angr.ailment.expression import negate from angr.analyses.decompiler.condition_processor import ConditionProcessor, EmptyBlockNotice @@ -281,8 +281,7 @@ def simplify_switch_clusters( :return: None """ - for variable in var2switches: - switch_regions = var2switches[variable] + for variable, switch_regions in var2switches.items(): if len(switch_regions) <= 1: # nothing to simplify or merge if there is only one switch region continue diff --git a/angr/analyses/decompiler/region_simplifiers/switch_expr_simplifier.py b/angr/analyses/decompiler/region_simplifiers/switch_expr_simplifier.py index ae69b32dd..ff0ea624a 100644 --- a/angr/analyses/decompiler/region_simplifiers/switch_expr_simplifier.py +++ b/angr/analyses/decompiler/region_simplifiers/switch_expr_simplifier.py @@ -4,7 +4,7 @@ from __future__ import annotations from collections import OrderedDict from typing import TYPE_CHECKING -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.peephole_optimizations import RemoveNoopConversions from angr.analyses.decompiler.sequence_walker import SequenceWalker from angr.analyses.decompiler.structurer_nodes import SwitchCaseNode diff --git a/angr/analyses/decompiler/semantic_naming/boolean_naming.py b/angr/analyses/decompiler/semantic_naming/boolean_naming.py index 74f32d236..8c35d2309 100644 --- a/angr/analyses/decompiler/semantic_naming/boolean_naming.py +++ b/angr/analyses/decompiler/semantic_naming/boolean_naming.py @@ -10,7 +10,6 @@ from __future__ import annotations import logging from collections import defaultdict -from typing import TYPE_CHECKING from angr import ailment from angr.ailment.expression import BinaryOp, Const, UnaryOp @@ -19,9 +18,6 @@ from angr.sim_variable import SimVariable from .naming_base import ClinicNamingBase -if TYPE_CHECKING: - pass - l = logging.getLogger(name=__name__) # Names for boolean flag variables. diff --git a/angr/analyses/decompiler/semantic_naming/call_result_naming.py b/angr/analyses/decompiler/semantic_naming/call_result_naming.py index 430a672b1..90f2546de 100644 --- a/angr/analyses/decompiler/semantic_naming/call_result_naming.py +++ b/angr/analyses/decompiler/semantic_naming/call_result_naming.py @@ -9,7 +9,6 @@ based on the called function (e.g., malloc result -> ptr, strlen result -> len). from __future__ import annotations import logging -from typing import TYPE_CHECKING from angr import ailment from angr.ailment.expression import Call @@ -18,9 +17,6 @@ from angr.sim_variable import SimVariable from .naming_base import ClinicNamingBase -if TYPE_CHECKING: - pass - l = logging.getLogger(name=__name__) # Mapping of function names/patterns to suggested variable names diff --git a/angr/analyses/decompiler/ssailification/traversal.py b/angr/analyses/decompiler/ssailification/traversal.py index 7ed1d9e17..cdd5c12b0 100644 --- a/angr/analyses/decompiler/ssailification/traversal.py +++ b/angr/analyses/decompiler/ssailification/traversal.py @@ -4,7 +4,7 @@ import logging from collections.abc import Callable from typing import TYPE_CHECKING -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.ailgraph_walker import traverse_in_order from angr.utils.ssa import get_reg_offset_base_and_size diff --git a/angr/analyses/decompiler/structured_codegen/base.py b/angr/analyses/decompiler/structured_codegen/base.py index b66505a1b..b7911c552 100644 --- a/angr/analyses/decompiler/structured_codegen/base.py +++ b/angr/analyses/decompiler/structured_codegen/base.py @@ -52,6 +52,9 @@ class PositionMapping: def items(self): return self._posmap.items() + def values(self): + return self._posmap.values() + # # Public methods # @@ -109,6 +112,9 @@ class InstructionMapping: def items(self): return self._insmap.items() + def values(self): + return self._insmap.values() + def add_mapping(self, ins_addr, posmap_pos): if ins_addr in self._insmap: if posmap_pos <= self._insmap[ins_addr].posmap_pos: diff --git a/angr/analyses/decompiler/structured_codegen/c.py b/angr/analyses/decompiler/structured_codegen/c.py index 686eafc60..145e7c032 100644 --- a/angr/analyses/decompiler/structured_codegen/c.py +++ b/angr/analyses/decompiler/structured_codegen/c.py @@ -2562,7 +2562,7 @@ class CConstant(CExpression): return # default priority: string references -> variables -> other reference values - for _ty, v in self.reference_values.items(): # pylint:disable=unused-variable + for v in self.reference_values.values(): # pylint:disable=unused-variable o = _default_output(v) if o is not None: yield o, self @@ -4672,6 +4672,6 @@ register_analysis(CStructuredCodeGenerator, "CStructuredCodeGenerator") # Register protobuf serializer/parser pairs for every concrete CConstruct subclass. Imported after all classes are # defined so that ``c_serialize.register_all`` can reference them by name. -from . import c_serialize as _c_serialize # noqa: E402 # pylint: disable=wrong-import-position +from . import c_serialize as _c_serialize # pylint: disable=wrong-import-position _c_serialize.register_all() diff --git a/angr/analyses/decompiler/structured_codegen/c_serialize.py b/angr/analyses/decompiler/structured_codegen/c_serialize.py index 8fe853834..485e2158d 100644 --- a/angr/analyses/decompiler/structured_codegen/c_serialize.py +++ b/angr/analyses/decompiler/structured_codegen/c_serialize.py @@ -422,7 +422,7 @@ def _serialize_position_mappings(pos_to_node, pos_to_addr, ctx: SerializeContext for pm, slot in ((pos_to_node, 0), (pos_to_addr, 1)): if pm is None: continue - for _, elem in pm.items(): + for elem in pm.values(): obj = elem.obj if obj is None or type(obj) not in _SERIALIZE_KIND_BY_CLASS: continue @@ -452,7 +452,7 @@ def _parse_position_mappings(pm_msg, ctx: ParseContext): def _serialize_instruction_mapping(im, out_msg) -> None: if im is None: return - for _, elem in im.items(): + for elem in im.values(): entry = out_msg.entries.add() entry.ins_addr = elem.ins_addr entry.posmap_pos = elem.posmap_pos diff --git a/angr/analyses/decompiler/structured_codegen/rust.py b/angr/analyses/decompiler/structured_codegen/rust.py index 7e4b45644..a67ac16cc 100644 --- a/angr/analyses/decompiler/structured_codegen/rust.py +++ b/angr/analyses/decompiler/structured_codegen/rust.py @@ -78,8 +78,6 @@ from angr.utils.loader import is_in_readonly_section, is_in_readonly_segment from .base import BaseStructuredCodeGenerator, InstructionMapping, PositionMapping, PositionMappingElement if TYPE_CHECKING: - import archinfo - import angr from angr.knowledge_plugins.variables.variable_manager import VariableManagerInternal @@ -190,7 +188,7 @@ def is_machine_word_size_type(type_: SimType, arch: archinfo.Arch) -> bool: return isinstance(type_, SimTypeReg) and type_.size == arch.bits -def guess_value_type(value: int | float, project: angr.Project) -> SimType | None: +def guess_value_type(value: float, project: angr.Project) -> SimType | None: if not isinstance(value, int): return None if project.kb.functions.contains_addr(value): @@ -2480,7 +2478,7 @@ class RustConstant(RustExpression): # default priority: string references -> variables -> other reference values if self.reference_values is not None: - for _ty, v in self.reference_values.items(): # pylint:disable=unused-variable + for v in self.reference_values.values(): # pylint:disable=unused-variable if isinstance(v, MemoryData) and v.sort == MemoryDataSort.String and v.content is not None: yield RustConstant.str_to_rust_str(v.content.decode("utf-8")), self return diff --git a/angr/analyses/decompiler/structurer_nodes.py b/angr/analyses/decompiler/structurer_nodes.py index efa08e51b..6d5678628 100644 --- a/angr/analyses/decompiler/structurer_nodes.py +++ b/angr/analyses/decompiler/structurer_nodes.py @@ -8,8 +8,8 @@ from typing import TYPE_CHECKING, Any import claripy import angr -import angr.ailment as ailment import angr.ailment.utils +from angr import ailment from angr.ailment.block import Block INDENT_DELTA = 2 diff --git a/angr/analyses/decompiler/structuring/dream.py b/angr/analyses/decompiler/structuring/dream.py index 7af78553a..024056fbb 100644 --- a/angr/analyses/decompiler/structuring/dream.py +++ b/angr/analyses/decompiler/structuring/dream.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any import claripy import networkx -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.condition_processor import ConditionProcessor from angr.analyses.decompiler.empty_node_remover import EmptyNodeRemover from angr.analyses.decompiler.jumptable_entry_condition_rewriter import JumpTableEntryConditionRewriter diff --git a/angr/analyses/decompiler/structuring/phoenix.py b/angr/analyses/decompiler/structuring/phoenix.py index 5a09d31a2..84b0e35fb 100644 --- a/angr/analyses/decompiler/structuring/phoenix.py +++ b/angr/analyses/decompiler/structuring/phoenix.py @@ -2195,9 +2195,9 @@ class PhoenixStructurer(StructurerBase): elif o in full_graph: if o not in out_dst_succs_fullgraph: out_dst_succs_fullgraph.append(o) - out_dst_succ = sorted(out_dst_succs, key=lambda o: o.addr)[0] if out_dst_succs else None + out_dst_succ = min(out_dst_succs, key=lambda o: o.addr) if out_dst_succs else None out_dst_succ_fullgraph = ( - sorted(out_dst_succs_fullgraph, key=lambda o: o.addr)[0] if out_dst_succs_fullgraph else None + min(out_dst_succs_fullgraph, key=lambda o: o.addr) if out_dst_succs_fullgraph else None ) if len(out_dst_succs) > 1: if self.dowhile_known_tail_nodes: diff --git a/angr/analyses/decompiler/structuring/structurer_base.py b/angr/analyses/decompiler/structuring/structurer_base.py index deb205990..1515796ec 100644 --- a/angr/analyses/decompiler/structuring/structurer_base.py +++ b/angr/analyses/decompiler/structuring/structurer_base.py @@ -179,7 +179,7 @@ class StructurerBase(Analysis): top = {a: times for a, times in goto_addrs.items() if times == max_votes} top_in_region = {a: times for a, times in top.items() if a in region_node_addrs} goto_addrs = top_in_region or top - return sorted(goto_addrs.items(), key=lambda x: (-x[1], x[0]))[0][0] + return min(goto_addrs.items(), key=lambda x: (-x[1], x[0]))[0] def _switch_handle_gotos(self, cases: dict[int, BaseNode], default, switch_end_addr: int) -> None: """ @@ -193,8 +193,7 @@ class StructurerBase(Analysis): # ensure every case node ends with a control-flow transition statement # FIXME: The following logic only handles one case. are there other cases? - for case_addr in cases: - case_node = cases[case_addr] + for case_node in cases.values(): if ( isinstance(case_node, SequenceNode) and case_node.nodes diff --git a/angr/analyses/deobfuscator/api_obf_finder.py b/angr/analyses/deobfuscator/api_obf_finder.py index 9f5338d2f..c14ae43a4 100644 --- a/angr/analyses/deobfuscator/api_obf_finder.py +++ b/angr/analyses/deobfuscator/api_obf_finder.py @@ -4,7 +4,7 @@ from __future__ import annotations import logging import string from enum import IntEnum -from typing import TYPE_CHECKING, Any +from typing import Any import claripy import networkx @@ -28,9 +28,6 @@ from angr.sim_variable import SimMemoryVariable from .api_obf_type2_finder import APIObfuscationType2Finder from .hash_lookup_api_deobfuscator import HashLookupAPIDeobfuscator -if TYPE_CHECKING: - pass - _l = logging.getLogger(name=__name__) diff --git a/angr/analyses/deobfuscator/data_transformation_embedder.py b/angr/analyses/deobfuscator/data_transformation_embedder.py index 6c6c0707c..f3f2d93db 100644 --- a/angr/analyses/deobfuscator/data_transformation_embedder.py +++ b/angr/analyses/deobfuscator/data_transformation_embedder.py @@ -392,7 +392,7 @@ class DataTransformationEmbedder(Analysis): _l.debug("Running loop analysis on outlined function...") loop_analysis = self.project.analyses.LoopAnalysis(dec_outlined.codegen.cfunc) - for _loop_key, loop_meta in loop_analysis.result.items(): + for loop_meta in loop_analysis.result.values(): if loop_meta.get("fixed_iterations", False) and loop_meta.get("max_iterations", None) > 1: loop_blocks = {nodes_dict[(loop_block_addr, None)] for loop_block_addr in loop_meta["block_addrs"]} succs = set() diff --git a/angr/analyses/disassembly.py b/angr/analyses/disassembly.py index be6965f9c..026dbe3a6 100644 --- a/angr/analyses/disassembly.py +++ b/angr/analyses/disassembly.py @@ -744,10 +744,7 @@ class MemoryOperand(Operand): def _parse_memop_squarebracket(self): if self.children[0] != "[": - try: - square_bracket_pos = self.children.index("[") - except ValueError: # pylint: disable=try-except-raise - raise + square_bracket_pos = self.children.index("[") self.prefix = self.children[:square_bracket_pos] @@ -801,10 +798,7 @@ class MemoryOperand(Operand): self.values_style = "paren" if self.children[0] != "(": - try: - paren_pos = self.children.index("(") - except ValueError: # pylint: disable=try-except-raise - raise + paren_pos = self.children.index("(") if all(isinstance(item, str) for item in self.children[:paren_pos]): # parse prefix @@ -883,11 +877,11 @@ class MemoryOperand(Operand): # combine values and offsets according to self.offset_location if self.offset_location == "prefix": - value_str = "".join([offset_str, left_paren, value_str, right_paren]) + value_str = f"{offset_str}{left_paren}{value_str}{right_paren}" elif self.offset_location == "before_value": - value_str = "".join([left_paren, offset_str, value_str, right_paren]) + value_str = f"{left_paren}{offset_str}{value_str}{right_paren}" else: # after_value - value_str = "".join([left_paren, value_str, offset_str, right_paren]) + value_str = f"{left_paren}{value_str}{offset_str}{right_paren}" else: value_str = left_paren + value_str + right_paren @@ -943,7 +937,7 @@ class Value(OperandPiece): if labeloffset == 0: lbl = self.project.kb.labels[self.val] return [lbl] - s = "{}{}{:#+x}".format( + s = "{}{}+{:#x}".format( "+" if self.render_with_sign else "", self.project.kb.labels[self.val + labeloffset], labeloffset, diff --git a/angr/analyses/identifier/functions/strcmp.py b/angr/analyses/identifier/functions/strcmp.py index e015354de..dc99eafcf 100644 --- a/angr/analyses/identifier/functions/strcmp.py +++ b/angr/analyses/identifier/functions/strcmp.py @@ -22,8 +22,6 @@ class strcmp(Func): l = 5 rand_str(l, strcmp.non_null) # s - return - def can_call_other_funcs(self): return False diff --git a/angr/analyses/identifier/functions/strncmp.py b/angr/analyses/identifier/functions/strncmp.py index 453329af4..6f5a9d514 100644 --- a/angr/analyses/identifier/functions/strncmp.py +++ b/angr/analyses/identifier/functions/strncmp.py @@ -21,7 +21,6 @@ class strncmp(Func): def gen_input_output_pair(self): l = 5 rand_str(l, strncmp.non_null) # s - return def can_call_other_funcs(self): return False diff --git a/angr/analyses/identifier/runner.py b/angr/analyses/identifier/runner.py index ccc05fa3f..9f522e59e 100644 --- a/angr/analyses/identifier/runner.py +++ b/angr/analyses/identifier/runner.py @@ -192,7 +192,7 @@ class Runner: curr_buf_loc += max(len(i), 0x1000) else: if not isinstance(i, int): - raise Exception(f"Expected int/bytes got {type(i)}") + raise TypeError(f"Expected int/bytes got {type(i)}") mapped_input.append(i) cc = self.project.factory.cc() @@ -219,7 +219,7 @@ class Runner: curr_buf_loc += max(len(i), 0x1000) else: if not isinstance(i, int): - raise Exception(f"Expected int/str got {type(i)}") + raise TypeError(f"Expected int/str got {type(i)}") mapped_input.append(i) else: for i, off in zip(test_data.input_args, custom_offs): @@ -229,7 +229,7 @@ class Runner: curr_buf_loc += max(len(i), 0x1000) else: if not isinstance(i, int): - raise Exception(f"Expected int/str got {type(i)}") + raise TypeError(f"Expected int/str got {type(i)}") mapped_input.append(i) cc = self.project.factory.cc() @@ -325,7 +325,7 @@ class Runner: curr_buf_loc += max(len(i), 0x1000) else: if not isinstance(i, int): - raise Exception(f"Expected int/bytes got {type(i)}") + raise TypeError(f"Expected int/bytes got {type(i)}") mapped_input.append(i) else: @@ -336,7 +336,7 @@ class Runner: curr_buf_loc += max(len(i), 0x1000) else: if not isinstance(i, int): - raise Exception(f"Expected int/bytes got {type(i)}") + raise TypeError(f"Expected int/bytes got {type(i)}") mapped_input.append(i) cc = self.project.factory.cc() diff --git a/angr/analyses/propagator/engine_base.py b/angr/analyses/propagator/engine_base.py index ea9b26746..e60419139 100644 --- a/angr/analyses/propagator/engine_base.py +++ b/angr/analyses/propagator/engine_base.py @@ -54,7 +54,7 @@ class SimEnginePropagatorBaseMixin[StateType, DataType_co, BlockType: BlockProto result_state = super().process(state, block=block, **kwargs) except SimEngineError as ex: if kwargs.pop("fail_fast", is_testing) is True: - raise ex + raise l.error(ex, exc_info=True) result_state = state diff --git a/angr/analyses/propagator/propagator.py b/angr/analyses/propagator/propagator.py index 9a15c3937..90a4c7b8e 100644 --- a/angr/analyses/propagator/propagator.py +++ b/angr/analyses/propagator/propagator.py @@ -9,8 +9,7 @@ from typing import Any import claripy import pyvex -import angr.ailment as ailment -from angr import sim_options +from angr import ailment, sim_options from angr.analyses.analysis import Analysis, register_analysis from angr.analyses.forward_analysis import ForwardAnalysis, visitors from angr.code_location import CodeLocation diff --git a/angr/analyses/purity/engine.py b/angr/analyses/purity/engine.py index 35c596a80..76bc0320c 100644 --- a/angr/analyses/purity/engine.py +++ b/angr/analyses/purity/engine.py @@ -115,7 +115,7 @@ class ResultType: pure_functions is None or loc.callee_return_name not in pure_functions ): return False - for (_, _, _, func, _), _ in self.call_args.items(): + for _, _, _, func, _ in self.call_args: if pure_functions is None: return False if isinstance(func, Function) and func.name not in pure_functions: diff --git a/angr/analyses/reaching_definitions/dep_graph.py b/angr/analyses/reaching_definitions/dep_graph.py index 69591da04..d6075f5fd 100644 --- a/angr/analyses/reaching_definitions/dep_graph.py +++ b/angr/analyses/reaching_definitions/dep_graph.py @@ -2,7 +2,6 @@ from __future__ import annotations from collections.abc import Iterable, Iterator from typing import ( - TYPE_CHECKING, Any, Literal, overload, @@ -26,9 +25,6 @@ from angr.knowledge_plugins.key_definitions.atoms import ( from angr.knowledge_plugins.key_definitions.definition import A, Definition, DefinitionMatchPredicate from angr.knowledge_plugins.key_definitions.undefined import UNDEFINED -if TYPE_CHECKING: - pass - def _is_definition(node): return isinstance(node, Definition) diff --git a/angr/analyses/reaching_definitions/engine_ail.py b/angr/analyses/reaching_definitions/engine_ail.py index a8fd40198..1e175b005 100644 --- a/angr/analyses/reaching_definitions/engine_ail.py +++ b/angr/analyses/reaching_definitions/engine_ail.py @@ -11,7 +11,7 @@ import claripy from archinfo.types import RegisterOffset from claripy import FSORT_DOUBLE, FSORT_FLOAT -import angr.ailment as ailment +from angr import ailment from angr.calling_conventions import SimRegArg, SimTypeBottom, default_cc from angr.code_location import CodeLocation, ExternalCodeLocation from angr.engines.light import SpOffset diff --git a/angr/analyses/reaching_definitions/engine_vex.py b/angr/analyses/reaching_definitions/engine_vex.py index 395b1a9c6..6954a83d3 100644 --- a/angr/analyses/reaching_definitions/engine_vex.py +++ b/angr/analyses/reaching_definitions/engine_vex.py @@ -73,7 +73,7 @@ class SimEngineRDVEX( ) except SimEngineError as e: if fail_fast is True: - raise e + raise l.error(e) return self.state diff --git a/angr/analyses/reaching_definitions/function_handler.py b/angr/analyses/reaching_definitions/function_handler.py index 39cd004ee..dc2161e1c 100644 --- a/angr/analyses/reaching_definitions/function_handler.py +++ b/angr/analyses/reaching_definitions/function_handler.py @@ -306,7 +306,7 @@ class FunctionHandler: return self def make_function_codeloc( - self, target: None | int | MultiValues, callsite: CodeLocation, callsite_func_addr: int | None + self, target: int | MultiValues | None, callsite: CodeLocation, callsite_func_addr: int | None ): """ The RDA engine will call this function to transform a callsite CodeLocation into a callee CodeLocation. diff --git a/angr/analyses/reaching_definitions/rd_state.py b/angr/analyses/reaching_definitions/rd_state.py index ea073229c..1b97c2aea 100644 --- a/angr/analyses/reaching_definitions/rd_state.py +++ b/angr/analyses/reaching_definitions/rd_state.py @@ -520,7 +520,7 @@ class ReachingDefinitionsState: ) -> bytes | None: ... def get_concrete_value( - self, spec: Atom | Definition[Atom, CodeLoc] | Iterable[Atom], cast_to: type[int] | type[bytes] = int + self, spec: Atom | Definition[Atom, CodeLoc] | Iterable[Atom], cast_to: type[int | bytes] = int ) -> int | bytes | None: return self.live_definitions.get_concrete_value(spec, cast_to) diff --git a/angr/analyses/reaching_definitions/reaching_definitions.py b/angr/analyses/reaching_definitions/reaching_definitions.py index 64052317f..a6b0ef60f 100644 --- a/angr/analyses/reaching_definitions/reaching_definitions.py +++ b/angr/analyses/reaching_definitions/reaching_definitions.py @@ -8,7 +8,7 @@ from typing import Any import pyvex -import angr.ailment as ailment +from angr import ailment from angr.analyses.analysis import Analysis from angr.analyses.forward_analysis import ForwardAnalysis from angr.analyses.forward_analysis.visitors.graph import NodeType diff --git a/angr/analyses/reaching_definitions/subject.py b/angr/analyses/reaching_definitions/subject.py index 1ed51c384..bcbc33492 100644 --- a/angr/analyses/reaching_definitions/subject.py +++ b/angr/analyses/reaching_definitions/subject.py @@ -2,7 +2,7 @@ from __future__ import annotations from enum import Enum -import angr.ailment as ailment +from angr import ailment from angr.analyses.forward_analysis import FunctionGraphVisitor, SingleNodeGraphVisitor from angr.block import Block from angr.knowledge_plugins.functions.function_manager import Function diff --git a/angr/analyses/reassembler.py b/angr/analyses/reassembler.py index 3c8e6b096..58039f53d 100644 --- a/angr/analyses/reassembler.py +++ b/angr/analyses/reassembler.py @@ -2285,7 +2285,6 @@ class Reassembler(Analysis): "__dso_handle", "__init_array_start", "__init_array_end", - # "stdout", "stderr", "stdin", diff --git a/angr/analyses/smc.py b/angr/analyses/smc.py index e0f89ecbb..a32a7b960 100644 --- a/angr/analyses/smc.py +++ b/angr/analyses/smc.py @@ -99,7 +99,7 @@ class SelfModifyingCodeAnalysis(Analysis): result: bool regions: list[tuple[int, int]] - def __init__(self, subject: None | int | str | Function, max_bytes: int = 0, state: SimState | None = None): + def __init__(self, subject: int | str | Function | None, max_bytes: int = 0, state: SimState | None = None): """ :param subject: Subject of analysis :param max_bytes: Maximum number of bytes from subject address. 0 for no limit (default). @@ -118,7 +118,7 @@ class SelfModifyingCodeAnalysis(Analysis): elif isinstance(subject, int): addr = subject else: - raise ValueError("Not a supported subject") + raise TypeError("Not a supported subject") if state is None: init_state = self.project.factory.call_state(addr) diff --git a/angr/analyses/soot_class_hierarchy.py b/angr/analyses/soot_class_hierarchy.py index 73720e382..686bfe855 100644 --- a/angr/analyses/soot_class_hierarchy.py +++ b/angr/analyses/soot_class_hierarchy.py @@ -38,14 +38,14 @@ class SootClassHierarchy(Analysis): self.init_hierarchy() def init_hierarchy(self): - for _class_name, cls in self.project.loader.main_object.classes.items(): + for cls in self.project.loader.main_object.classes.values(): if "INTERFACE" in cls.attrs: self.interface_implementers[cls] = [] self.dir_sub_interfaces[cls] = [] else: self.dir_sub_classes[cls] = [] - for _class_name, cls in self.project.loader.main_object.classes.items(): + for cls in self.project.loader.main_object.classes.values(): if self.has_super_class(cls): if "INTERFACE" in cls.attrs: # TODO @@ -64,7 +64,7 @@ class SootClassHierarchy(Analysis): self.interface_implementers[i].append(cls) # fill direct implementers with subclasses - for _class_name, cls in self.project.loader.main_object.classes.items(): + for cls in self.project.loader.main_object.classes.values(): if "INTERFACE" in cls.attrs: implementers = self.interface_implementers[cls] s = set() diff --git a/angr/analyses/typehoon/simple_solver.py b/angr/analyses/typehoon/simple_solver.py index 18756a65f..79f4d3322 100644 --- a/angr/analyses/typehoon/simple_solver.py +++ b/angr/analyses/typehoon/simple_solver.py @@ -2097,7 +2097,7 @@ class SimpleSolver: elif isinstance(last_label, FuncOut): func_outputs[last_label.loc].add(succ) else: - raise RuntimeError("Unreachable") + raise TypeError("Unreachable") input_args = [] output_values = [] @@ -2330,7 +2330,7 @@ class SimpleSolver: return paths - def _pointer_class(self) -> type[Pointer32] | type[Pointer64]: + def _pointer_class(self) -> type[Pointer32 | Pointer64]: if self.bits == 32: return Pointer32 if self.bits == 64: diff --git a/angr/analyses/variable_recovery/engine_ail.py b/angr/analyses/variable_recovery/engine_ail.py index 8fdc2d140..ea75ecb5c 100644 --- a/angr/analyses/variable_recovery/engine_ail.py +++ b/angr/analyses/variable_recovery/engine_ail.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, cast import claripy -import angr.ailment as ailment +from angr import ailment from angr.ailment.constant import UNDETERMINED_SIZE from angr.ailment.expression import Array, FunctionLikeMacro, Let, RustEnum, StringLiteral, Struct from angr.analyses.typehoon import typeconsts, typevars diff --git a/angr/analyses/variable_recovery/engine_base.py b/angr/analyses/variable_recovery/engine_base.py index 0811f1e15..8619340f8 100644 --- a/angr/analyses/variable_recovery/engine_base.py +++ b/angr/analyses/variable_recovery/engine_base.py @@ -6,7 +6,7 @@ from typing import Any, cast import claripy -import angr.ailment as ailment +from angr import ailment from angr.analyses.typehoon import typeconsts, typevars from angr.analyses.typehoon.typevars import AddN, DerivedTypeVariable, Load, Store, SubN, TypeVariable from angr.analyses.variable_recovery.variable_recovery_base import VariableRecoveryStateBase diff --git a/angr/analyses/variable_recovery/engine_vex.py b/angr/analyses/variable_recovery/engine_vex.py index fab7c8699..c0a2da37b 100644 --- a/angr/analyses/variable_recovery/engine_vex.py +++ b/angr/analyses/variable_recovery/engine_vex.py @@ -1,7 +1,7 @@ # pylint:disable=unused-argument from __future__ import annotations -from typing import TYPE_CHECKING, cast +from typing import cast import claripy import pyvex @@ -20,9 +20,6 @@ from angr.storage.memory_mixins.paged_memory.pages.multi_values import MultiValu from .engine_base import RichR, SimEngineVRBase from .irsb_scanner import VEXIRSBScanner -if TYPE_CHECKING: - pass - binop_handler = SimEngineNostmtVEX[ "VariableRecoveryFastState", RichR[claripy.ast.BV | claripy.ast.FP], None ].binop_handler diff --git a/angr/analyses/variable_recovery/variable_recovery_base.py b/angr/analyses/variable_recovery/variable_recovery_base.py index c8ec8fc9f..d340afc3a 100644 --- a/angr/analyses/variable_recovery/variable_recovery_base.py +++ b/angr/analyses/variable_recovery/variable_recovery_base.py @@ -175,7 +175,7 @@ class VariableRecoveryBase(Analysis): stack_vars_by_offset = defaultdict(list) for sv in stack_vars: stack_vars_by_offset[sv.offset].append(sv) - for _offset, var_list in stack_vars_by_offset.items(): + for var_list in stack_vars_by_offset.values(): if len(var_list) < 2: continue single_byte_vars = [v for v in var_list if v.size == 1] diff --git a/angr/calling_conventions.py b/angr/calling_conventions.py index c80cb462a..35d50ef34 100644 --- a/angr/calling_conventions.py +++ b/angr/calling_conventions.py @@ -2270,10 +2270,7 @@ class SimCCRISCV64(SimCC): if isinstance(arg_type, (SimStruct, SimTypeFixedSizeArray)): flat_map = self._flatten(arg_type) if flat_map is not None: - all_types = [] - for tys in flat_map.values(): - for t in tys: - all_types.append(t) + all_types = [t for tys in flat_map.values() for t in tys] if 1 <= len(all_types) <= 2 and any(isinstance(t, (SimTypeFloat, SimTypeDouble)) for t in all_types): res = [] diff --git a/angr/concretization_strategies/any_named.py b/angr/concretization_strategies/any_named.py index 8107dd14f..9cef1678c 100644 --- a/angr/concretization_strategies/any_named.py +++ b/angr/concretization_strategies/any_named.py @@ -17,7 +17,7 @@ class SimConcretizationStrategyAnyNamed(SimConcretizationStrategy): mn, mx = self._range(memory, addr, **kwargs) if mn == mx: # Check if a variable already exists - for _, values in memory._name_mapping.items(): + for values in memory._name_mapping.values(): if mn in values: return [mn] # Get any solution diff --git a/angr/concretization_strategies/logging.py b/angr/concretization_strategies/logging.py index 1661c0f41..a33bfed2d 100644 --- a/angr/concretization_strategies/logging.py +++ b/angr/concretization_strategies/logging.py @@ -4,6 +4,8 @@ import logging from .base import SimConcretizationStrategy +log = logging.getLogger(name=__name__) + class SimConcretizationStrategyLogging(SimConcretizationStrategy): """ @@ -19,14 +21,14 @@ class SimConcretizationStrategyLogging(SimConcretizationStrategy): answers = self._strategy._concretize(memory, addr, **kwargs) if answers is not None: if self._is_read_strategy: - logging.debug( + log.debug( "Read strategy %s on %s gave [%s]", type(self._strategy).__name__, addr, ", ".join([hex(answer) for answer in answers]), ) else: - logging.debug( + log.debug( "Write strategy %s on %s gave [%s]", type(self._strategy).__name__, addr, diff --git a/angr/distributed/server.py b/angr/distributed/server.py index 7c5ce0bcb..26215bdd7 100644 --- a/angr/distributed/server.py +++ b/angr/distributed/server.py @@ -182,7 +182,7 @@ class Server: if self._worker_exit_callback and self._worker_exit_args: with self._worker_exit_args_lock: - for _, args in self._worker_exit_args.items(): + for args in self._worker_exit_args.values(): self._worker_exit_callback(*args) server_state["stopped"] = self.stopped diff --git a/angr/engines/light/data.py b/angr/engines/light/data.py index c5d994a04..8291c0262 100644 --- a/angr/engines/light/data.py +++ b/angr/engines/light/data.py @@ -1,6 +1,6 @@ from __future__ import annotations -import angr.ailment as ailment +from angr import ailment from angr.utils.constants import is_alignment_mask diff --git a/angr/engines/vex/claripy/ccall.py b/angr/engines/vex/claripy/ccall.py index c1b772e30..11e76dfa1 100644 --- a/angr/engines/vex/claripy/ccall.py +++ b/angr/engines/vex/claripy/ccall.py @@ -1765,7 +1765,7 @@ ARM64G_CC_OP_SBC32 = 7 # /* DEP1 = argL (Rn), DEP2 = arg2 (shifter_op), DEP3 = ARM64G_CC_OP_SBC64 = 8 # /* DEP1 = argL (Rn), DEP2 = arg2 (shifter_op), DEP3 = oldC (in LSB) */ ARM64G_CC_OP_LOGIC32 = 9 # /* DEP1 = result, DEP2 = 0, DEP3 = 0 */ ARM64G_CC_OP_LOGIC64 = 10 # /* DEP1 = result, DEP2 = 0, DEP3 = 0 */ -ARM64G_CC_OP_NUMBER = 11 # +ARM64G_CC_OP_NUMBER = 11 ARM64CondEQ = 0 # /* equal : Z=1 */ ARM64CondNE = 1 # /* not equal : Z=0 */ diff --git a/angr/engines/vex/claripy/datalayer.py b/angr/engines/vex/claripy/datalayer.py index cd96d2dd3..b3fe9e066 100644 --- a/angr/engines/vex/claripy/datalayer.py +++ b/angr/engines/vex/claripy/datalayer.py @@ -15,7 +15,7 @@ l = logging.getLogger(__name__) zero = claripy.BVV(0, 32) -def value(ty: str, val: int | float, size: int | None = None): +def value(ty: str, val: float, size: int | None = None): if ty == "Ity_F32": return claripy.FPV(float(val), claripy.FSORT_FLOAT) if ty == "Ity_F64": @@ -129,7 +129,7 @@ class ClaripyDataMixin(VEXMixin): return func(self.state, *args) except ccall.CCallMultivaluedException as e: cases, to_replace = e.args - for i, arg in enumerate(args): # noqa: B007 + for i, arg in enumerate(args): if arg is to_replace: break else: diff --git a/angr/exploration_techniques/stochastic.py b/angr/exploration_techniques/stochastic.py index d5a6528d2..69d38bd9c 100644 --- a/angr/exploration_techniques/stochastic.py +++ b/angr/exploration_techniques/stochastic.py @@ -44,10 +44,10 @@ class StochasticSearch(ExplorationTechnique): assert len(states) >= 2 total_weight = sum(self.affinity[s.addr] for s in states) selected = self._random.uniform(0, total_weight) - for i, state in enumerate(states): + for _, state in enumerate(states): weight = self.affinity[state.addr] if selected < weight: - return states[i] + return state selected -= weight return states[len(states) - 1] diff --git a/angr/exploration_techniques/tracer.py b/angr/exploration_techniques/tracer.py index cfeddb27e..2e5c3415a 100644 --- a/angr/exploration_techniques/tracer.py +++ b/angr/exploration_techniques/tracer.py @@ -679,7 +679,7 @@ class Tracer(ExplorationTechnique): def _translate_trace_addr(self, trace_addr, obj=None): if obj is None: - for obj, slide in self._aslr_slides.items(): # pylint: disable=redefined-argument-from-local + for obj, slide in self._aslr_slides.items(): # noqa: PLR1704 # pylint: disable=redefined-argument-from-local if obj.contains_addr(trace_addr - slide): break else: diff --git a/angr/keyed_region.py b/angr/keyed_region.py index f57174f82..0981456bd 100644 --- a/angr/keyed_region.py +++ b/angr/keyed_region.py @@ -191,7 +191,7 @@ class KeyedRegion: # TODO: is the current solution not optimal enough? _: RegionObject item: RegionObject - for _, item in other._storage.items(): + for item in other._storage.values(): so: StoredObject for so in item.stored_objects: if replacements and so.obj in replacements: @@ -212,7 +212,7 @@ class KeyedRegion: _: RegionObject item: RegionObject - for _, item in other._storage.items(): + for item in other._storage.values(): so: StoredObject for so in item.stored_objects: if replacements and so.obj in replacements: diff --git a/angr/knowledge_plugins/cfg/cfg_node.py b/angr/knowledge_plugins/cfg/cfg_node.py index 8a96faf33..7dccb69bf 100644 --- a/angr/knowledge_plugins/cfg/cfg_node.py +++ b/angr/knowledge_plugins/cfg/cfg_node.py @@ -416,7 +416,7 @@ class CFGNode(Serializable): def __eq__(self, other): if isinstance(other, SimSuccessors): - raise ValueError("You do not want to be comparing a SimSuccessors instance to a CFGNode.") + raise TypeError("You do not want to be comparing a SimSuccessors instance to a CFGNode.") if type(other) is not CFGNode: return False return self.addr == other.addr and self.size == other.size and self.simprocedure_name == other.simprocedure_name @@ -567,7 +567,7 @@ class CFGENode(CFGNode): def __eq__(self, other): if isinstance(other, SimSuccessors): - raise ValueError("You do not want to be comparing a SimSuccessors instance to a CFGNode.") + raise TypeError("You do not want to be comparing a SimSuccessors instance to a CFGNode.") if not isinstance(other, CFGENode): return False return ( diff --git a/angr/knowledge_plugins/functions/function.py b/angr/knowledge_plugins/functions/function.py index 46ba07682..48f935261 100644 --- a/angr/knowledge_plugins/functions/function.py +++ b/angr/knowledge_plugins/functions/function.py @@ -893,7 +893,7 @@ class Function(Serializable): return self.addr - self.binary.mapped_base @property - def symbol(self) -> None | Symbol: + def symbol(self) -> Symbol | None: """ :return: the function's Symbol, if any """ @@ -1611,7 +1611,7 @@ class Function(Serializable): """ Draw the graph and save it to a PNG file. """ - import matplotlib.pyplot as pyplot # pylint: disable=import-error,import-outside-toplevel,consider-using-from-import + from matplotlib import pyplot # pylint: disable=import-error,import-outside-toplevel,consider-using-from-import from networkx.drawing.nx_agraph import graphviz_layout # pylint: disable=import-error,import-outside-toplevel tmp_graph = networkx.classes.digraph.DiGraph() diff --git a/angr/knowledge_plugins/functions/function_manager.py b/angr/knowledge_plugins/functions/function_manager.py index 438a79ff5..23b3901f8 100644 --- a/angr/knowledge_plugins/functions/function_manager.py +++ b/angr/knowledge_plugins/functions/function_manager.py @@ -786,10 +786,10 @@ class FunctionManager[K: (int, SootMethodDescriptor)](KnowledgeBasePlugin, colle self._arg_registers = kb._project.arch.argument_registers # local PLT dictionary cache - self._rplt_cache_ranges: None | list[tuple[int, int]] = None - self._rplt_cache: None | set[int] = None + self._rplt_cache_ranges: list[tuple[int, int]] | None = None + self._rplt_cache: set[int] | None = None # local binary name cache: min_addr -> (max_addr, binary_name) - self._binname_cache: None | SortedDict[int, tuple[int, str | None]] = None + self._binname_cache: SortedDict[int, tuple[int, str | None]] | None = None # non-returning functions cache self._non_returning_func_addrs: set[K] = set() @@ -1181,7 +1181,7 @@ class FunctionManager[K: (int, SootMethodDescriptor)](KnowledgeBasePlugin, colle self._function_map[k] = v self._function_added(v) else: - raise ValueError("FunctionManager.__setitem__ keys must be an int") + raise TypeError("FunctionManager.__setitem__ keys must be an int") def __delitem__(self, k): if isinstance(k, self.function_address_types): @@ -1201,7 +1201,7 @@ class FunctionManager[K: (int, SootMethodDescriptor)](KnowledgeBasePlugin, colle for old_name in func_meta.previous_names: self._old_func_name_to_addrs.get(old_name, set()).discard(k) else: - raise ValueError( + raise TypeError( f"FunctionManager.__delitem__ only accepts the following address types: {self.function_address_types}" ) diff --git a/angr/knowledge_plugins/functions/function_parser.py b/angr/knowledge_plugins/functions/function_parser.py index 183897271..ce2033ede 100644 --- a/angr/knowledge_plugins/functions/function_parser.py +++ b/angr/knowledge_plugins/functions/function_parser.py @@ -116,7 +116,7 @@ class FunctionParser: # blocks blocks_list = [] - for _, b in function.code_nodes.items(): + for b in function.code_nodes.values(): block = primitives_pb2.Block() block.ea = b.addr block.size = b.size diff --git a/angr/knowledge_plugins/key_definitions/atoms.py b/angr/knowledge_plugins/key_definitions/atoms.py index 8e3bff10b..e8d2692a0 100644 --- a/angr/knowledge_plugins/key_definitions/atoms.py +++ b/angr/knowledge_plugins/key_definitions/atoms.py @@ -5,7 +5,7 @@ from enum import Enum, auto import claripy from archinfo import Arch, Endness, RegisterOffset -import angr.ailment as ailment +from angr import ailment from angr.calling_conventions import SimFunctionArgument, SimRegArg, SimStackArg from angr.engines.light import SpOffset diff --git a/angr/knowledge_plugins/key_definitions/definition.py b/angr/knowledge_plugins/key_definitions/definition.py index c5aea2137..f06d5a869 100644 --- a/angr/knowledge_plugins/key_definitions/definition.py +++ b/angr/knowledge_plugins/key_definitions/definition.py @@ -39,7 +39,7 @@ class DefinitionMatchPredicate: bbl_addr: int | None = None ins_addr: int | None = None variable: SimVariable | None = None - variable_manager: VariableManagerInternal | None | Literal[False] = None + variable_manager: VariableManagerInternal | Literal[False] | None = None stack_offset: int | None = None reg_name: str | int | None = None heap_offset: int | None = None diff --git a/angr/knowledge_plugins/key_definitions/live_definitions.py b/angr/knowledge_plugins/key_definitions/live_definitions.py index 13eeaaba3..55f217bf1 100644 --- a/angr/knowledge_plugins/key_definitions/live_definitions.py +++ b/angr/knowledge_plugins/key_definitions/live_definitions.py @@ -834,7 +834,7 @@ class LiveDefinitions: def get_concrete_value( self, spec: A | Definition[A] | Iterable[A] | Iterable[Definition[A]], - cast_to: type[int] | type[bytes] = int, + cast_to: type[int | bytes] = int, ) -> int | bytes | None: r = self.get_one_value(spec, strip_annotations=True) if r is None: diff --git a/angr/knowledge_plugins/propagations/prop_value.py b/angr/knowledge_plugins/propagations/prop_value.py index 2d2ccc28e..5a14618f1 100644 --- a/angr/knowledge_plugins/propagations/prop_value.py +++ b/angr/knowledge_plugins/propagations/prop_value.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any import claripy -import angr.ailment as ailment +from angr import ailment if TYPE_CHECKING: from angr.code_location import CodeLocation diff --git a/angr/knowledge_plugins/propagations/propagation_model.py b/angr/knowledge_plugins/propagations/propagation_model.py index b17554672..cb725e961 100644 --- a/angr/knowledge_plugins/propagations/propagation_model.py +++ b/angr/knowledge_plugins/propagations/propagation_model.py @@ -5,7 +5,7 @@ from typing import Any import claripy -import angr.ailment as ailment +from angr import ailment from angr.knowledge_plugins.functions.function import Function from angr.serializable import Serializable diff --git a/angr/knowledge_plugins/propagations/states.py b/angr/knowledge_plugins/propagations/states.py index 3a75ccac8..b88eeaae4 100644 --- a/angr/knowledge_plugins/propagations/states.py +++ b/angr/knowledge_plugins/propagations/states.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, Self import archinfo import claripy -import angr.ailment as ailment +from angr import ailment from angr.code_location import CodeLocation from angr.engines.light.engine import SimEngineLight from angr.errors import SimMemoryMissingError diff --git a/angr/knowledge_plugins/variables/variable_manager.py b/angr/knowledge_plugins/variables/variable_manager.py index 09c5b6541..68eaa5be3 100644 --- a/angr/knowledge_plugins/variables/variable_manager.py +++ b/angr/knowledge_plugins/variables/variable_manager.py @@ -455,7 +455,7 @@ class VariableManagerInternal(Serializable): elif isinstance(var, SimConstantVariable): continue else: - raise ValueError(f"Unsupported sort {type(var)} in parse_from_cmessage().") + raise TypeError(f"Unsupported sort {type(var)} in parse_from_cmessage().") region.add_variable(offset, var) @@ -667,7 +667,7 @@ class VariableManagerInternal(Serializable): return phi # allocate a new phi variable - repre = sorted(variables, key=lambda val: val.key)[0] + repre = min(variables, key=lambda val: val.key) repre_type = type(repre) repre_size = max(var.size for var in variables) if repre_type is SimRegisterVariable: @@ -821,9 +821,7 @@ class VariableManagerInternal(Serializable): @overload def get_variables(self, sort: Literal["reg"], collapse_same_ident: bool = False) -> list[SimRegisterVariable]: ... @overload - def get_variables( - self, sort: None = None, collapse_same_ident: bool = False - ) -> list[SimRegisterVariable | SimRegisterVariable]: ... + def get_variables(self, sort: None = None, collapse_same_ident: bool = False) -> list[SimRegisterVariable]: ... def get_variables(self, sort=None, collapse_same_ident=False): """ @@ -853,7 +851,7 @@ class VariableManagerInternal(Serializable): @overload def get_unified_variables(self, sort: Literal["reg"]) -> list[SimRegisterVariable]: ... @overload - def get_unified_variables(self, sort: None) -> list[SimRegisterVariable | SimRegisterVariable]: ... + def get_unified_variables(self, sort: None) -> list[SimRegisterVariable]: ... def get_unified_variables(self, sort=None): """ diff --git a/angr/procedures/definitions/parse_win32json.py b/angr/procedures/definitions/parse_win32json.py index fe4b7ec91..6e1766c23 100644 --- a/angr/procedures/definitions/parse_win32json.py +++ b/angr/procedures/definitions/parse_win32json.py @@ -16,6 +16,8 @@ from angr.errors import AngrMissingTypeError from angr.procedures.definitions import SimTypeCollection from angr.sim_type import PointerDisposition, SimTypeBottom, SimTypeFunction, SimTypeInt, SimTypeLong, SimTypePointer +log = logging.getLogger(__name__) + # The win32json marks some outparams as inout. fix this OVERRIDE_OUTPARAMS = { ("RtlInitUnicodeString", 0), @@ -225,11 +227,11 @@ def do_it(in_dir): files = p.glob("*.json") for file in files: - logging.info("Found file %s", file) + log.info("Found file %s", file) with open(file, encoding="utf-8-sig") as f: api_namespaces[file.stem] = json.load(f) - logging.info("Making a bunch of types...") + log.info("Making a bunch of types...") missing_types_last_round = set() while True: nosuchtype = 0 @@ -243,22 +245,22 @@ def do_it(in_dir): # skip this type for now nosuchtype += 1 missing_types.add(t["Name"]) - logging.info("... missing %d types", nosuchtype) + log.info("... missing %d types", nosuchtype) if nosuchtype == 0 or missing_types == missing_types_last_round: break missing_types_last_round = missing_types if missing_types_last_round: - logging.info("Missing types: %s", missing_types_last_round) + log.info("Missing types: %s", missing_types_last_round) else: - logging.info("All referenced types have been created") - logging.info("Alright, now let's do some functions") + log.info("All referenced types have been created") + log.info("Alright, now let's do some functions") i = 1 func_count = 0 parsed_cprotos = defaultdict(list) for namespace, metadata in api_namespaces.items(): - logging.debug("+++ %d/%d: Processing namespace %s", i, len(api_namespaces), namespace) + log.debug("+++ %d/%d: Processing namespace %s", i, len(api_namespaces), namespace) i += 1 funcs = metadata["Functions"] if namespace.startswith("Windows.Win32"): @@ -2519,7 +2521,7 @@ def do_it(in_dir): exists = True break if exists: - logging.warning("Declaration for function %s in %s.%s already exists. Skipping...", func, lib, suffix) + log.warning("Declaration for function %s in %s.%s already exists. Skipping...", func, lib, suffix) continue parsed_cprotos[(prefix, lib, suffix)].append((func, proto, "")) @@ -2534,7 +2536,7 @@ def do_it(in_dir): full_prefix = dump_root / prefix filename = libname.replace(".", "_") + ".json" os.makedirs(full_prefix, exist_ok=True) - logging.debug("Writing to file %s...", filename) + log.debug("Writing to file %s...", filename) non_returning = [] d = { "_t": "lib", @@ -2557,7 +2559,7 @@ def do_it(in_dir): # Dump the type collection to a JSON file with open(dump_root / "win32/_types_win32.json", "w", encoding="utf-8") as f: - logging.debug("Writing to file win32/win32_types.json...") + log.debug("Writing to file win32/win32_types.json...") f.write(json.dumps(typelib.to_json(types_as_string=True), indent="\t")) @@ -2567,10 +2569,9 @@ def main(): _args.add_argument("-v", action="count", help="Increase logging verbosity. Can specify multiple times.") args = _args.parse_args() if args.v is not None: - logging.root.setLevel(level=max(30 - (args.v * 10), 0)) + log.root.setLevel(level=max(30 - (args.v * 10), 0)) do_it(args.win32json_api_directory) if __name__ == "__main__": - logging.root.setLevel("DEBUG") main() diff --git a/angr/procedures/java_lang/stringbuilder.py b/angr/procedures/java_lang/stringbuilder.py index cf28b29c2..5d2f0c85b 100644 --- a/angr/procedures/java_lang/stringbuilder.py +++ b/angr/procedures/java_lang/stringbuilder.py @@ -18,7 +18,6 @@ class StringBuilderInit(JavaSimProcedure): str_ref = SimSootValue_StringRef.new_string(self.state, claripy.StringV("")) this_ref.store_field(self.state, "str", "java.lang.String", str_ref) - return class StringBuilderAppend(JavaSimProcedure): diff --git a/angr/procedures/java_util/list.py b/angr/procedures/java_util/list.py index 83c8b5a43..fa3eb1c7c 100644 --- a/angr/procedures/java_util/list.py +++ b/angr/procedures/java_util/list.py @@ -23,8 +23,6 @@ class ListInit(JavaSimProcedure): this_ref.store_field(self.state, ELEMS, "java.lang.Object[]", array_ref) this_ref.store_field(self.state, SIZE, "int", claripy.BVV(0, 32)) - return - class ListAdd(JavaSimProcedure): __provides__ = (("java.util.List", "add(java.lang.Object)"), ("java.util.LinkedList", "add(java.lang.Object)")) diff --git a/angr/procedures/java_util/map.py b/angr/procedures/java_util/map.py index 4183ee537..1c384fb41 100644 --- a/angr/procedures/java_util/map.py +++ b/angr/procedures/java_util/map.py @@ -36,8 +36,6 @@ class MapInit(JavaSimProcedure): array_ref = SimSootExpr_NewArray.new_array(self.state, "java.lang.Object", claripy.BVV(1000, 32)) this_ref.store_field(self.state, MAP_KEYS, "java.lang.Object[]", array_ref) - return - class MapPut(JavaSimProcedure): __provides__ = ( diff --git a/angr/procedures/libc/rewind.py b/angr/procedures/libc/rewind.py index c9f16dd84..38c1da9f1 100644 --- a/angr/procedures/libc/rewind.py +++ b/angr/procedures/libc/rewind.py @@ -9,5 +9,3 @@ class rewind(angr.SimProcedure): def run(self, file_ptr): fseek = angr.SIM_PROCEDURES["libc"]["fseek"] self.inline_call(fseek, file_ptr, 0, 0) - - return diff --git a/angr/procedures/linux_kernel/arm_user_helpers.py b/angr/procedures/linux_kernel/arm_user_helpers.py index 1b08aa98e..acd1d9965 100644 --- a/angr/procedures/linux_kernel/arm_user_helpers.py +++ b/angr/procedures/linux_kernel/arm_user_helpers.py @@ -14,7 +14,6 @@ class _kuser_helper_version(angr.SimProcedure): def run(self): # hardcoded version number extracted from QEMU self.state.regs.r0 = 0x884C0 - return class _kuser_helper_get_tls(angr.SimProcedure): @@ -23,7 +22,6 @@ class _kuser_helper_get_tls(angr.SimProcedure): def run(self): self.state.regs.r0 = self.project.loader.tls.threads[0].user_thread_pointer - return class _kuser_cmpxchg(angr.SimProcedure): diff --git a/angr/procedures/stubs/CallReturn.py b/angr/procedures/stubs/CallReturn.py index e3302621a..7da2592bd 100644 --- a/angr/procedures/stubs/CallReturn.py +++ b/angr/procedures/stubs/CallReturn.py @@ -12,4 +12,3 @@ class CallReturn(angr.SimProcedure): def run(self): l.info("A factory.call_state-created path returned!") - return diff --git a/angr/procedures/stubs/Redirect.py b/angr/procedures/stubs/Redirect.py index adb5aa9cf..2d7ca7eb2 100644 --- a/angr/procedures/stubs/Redirect.py +++ b/angr/procedures/stubs/Redirect.py @@ -12,7 +12,7 @@ class Redirect(angr.SimProcedure): def run(self, redirect_to=None): if redirect_to is None: - raise Exception("Please specify where you wanna jump to.") + raise ValueError("Please specify where you wanna jump to.") self._custom_name = f"Redirect to 0x{redirect_to:08x}" # There is definitely no refs diff --git a/angr/project.py b/angr/project.py index ce70fbc2c..3892a3a39 100644 --- a/angr/project.py +++ b/angr/project.py @@ -203,11 +203,11 @@ class Project: "Incompatible options selected for this project, please disable auto_load_libs if " "you want to use a concrete target." ) - raise Exception("Incompatible options for the project") + raise ValueError("Incompatible options for the project") if self.concrete_target and self.arch.name not in ["X86", "AMD64", "ARMHF", "ARMEL", "MIPS32"]: l.critical("Concrete execution does not support yet the selected architecture. Aborting.") - raise Exception("Incompatible options for the project") + raise ValueError("Incompatible options for the project") self._default_analysis_mode = default_analysis_mode self._exclude_sim_procedures_func = exclude_sim_procedures_func @@ -437,7 +437,7 @@ class Project: # we still want to try as hard as we can to figure out where it comes from # so we can get the calling convention as close to right as possible. elif reloc.resolvewith is not None and reloc.resolvewith in SIM_LIBRARIES: - sim_lib = sorted(SIM_LIBRARIES[reloc.resolvewith], key=lambda lib: lib.has_prototype(export.name))[-1] + sim_lib = max(SIM_LIBRARIES[reloc.resolvewith], key=lambda lib: lib.has_prototype(export.name)) if self._check_user_blacklists(export.name): if not func.is_weak: l.info("Using stub SimProcedure for unresolved %s from %s", func.name, sim_lib.name) @@ -667,7 +667,7 @@ class Project: for reloc in self.loader.find_relevant_relocations(symbol_name): assert reloc.symbol is not None if not reloc.symbol.is_weak: - raise Exception("Symbol is strong but we couldn't find its resolution? Report to @rhelmot.") + raise ValueError("Symbol is strong but we couldn't find its resolution? Report to @rhelmot.") if new_sym is None: new_sym = self.loader.extern_object.make_extern(symbol_name) reloc.resolve(new_sym) diff --git a/angr/rust/analyses/rust_calling_convention/rust_calling_convention.py b/angr/rust/analyses/rust_calling_convention/rust_calling_convention.py index a7bb8f114..206a7d608 100644 --- a/angr/rust/analyses/rust_calling_convention/rust_calling_convention.py +++ b/angr/rust/analyses/rust_calling_convention/rust_calling_convention.py @@ -342,7 +342,7 @@ class RustCallingConventionAnalysis(Analysis): ).with_arch(self.project.arch) candidates.append(struct_ty) - final_ty = sorted(candidates, key=lambda candidate: candidate.size, reverse=True)[0] if candidates else None + final_ty = max(candidates, key=lambda candidate: candidate.size) if candidates else None # Filter out register-size structs if final_ty and final_ty.size <= self.project.arch.bits: diff --git a/angr/rust/analyses/rustc_version_identification.py b/angr/rust/analyses/rustc_version_identification.py index b7841116e..17fbb1b5c 100644 --- a/angr/rust/analyses/rustc_version_identification.py +++ b/angr/rust/analyses/rustc_version_identification.py @@ -67,7 +67,7 @@ class RustcVersionIdentification(Analysis): fa = self.project.analyses.Flirt(sig_path, dry_run=True, match_named_functions=True) matched = fa.matched_suggestions["Temporary"][1] # {addr: name} count = 0 - for _addr, name in matched.items(): + for name in matched.values(): name = demangle(name) if not name.startswith(("core::", "std::", "alloc::")): continue diff --git a/angr/rust/analyses/type_db_loader.py b/angr/rust/analyses/type_db_loader.py index def7a70cb..31393d20d 100644 --- a/angr/rust/analyses/type_db_loader.py +++ b/angr/rust/analyses/type_db_loader.py @@ -296,7 +296,7 @@ class TypeDBLoader(Analysis): return type_db_json = json.loads(type_db_path.read_text(encoding="utf-8")) self._struct_db = {struct_data["name"]: struct_data for struct_data in type_db_json["structs"]} - for _, struct_data in self._struct_db.items(): + for struct_data in self._struct_db.values(): self._parse_type(struct_data) l.info("Loaded %d structs from type database.", len(self._structs)) diff --git a/angr/rust/optimization_passes/deref_coercion_simplifier.py b/angr/rust/optimization_passes/deref_coercion_simplifier.py index 5729b31b1..226f7436e 100644 --- a/angr/rust/optimization_passes/deref_coercion_simplifier.py +++ b/angr/rust/optimization_passes/deref_coercion_simplifier.py @@ -12,7 +12,7 @@ from angr.rust.mixins import CFAMixin, SRDAMixin from angr.rust.optimization_passes.utils import CallRewriter from angr.rust.sim_type import RustSimStruct -l = logging.getLogger(__file__) +l = logging.getLogger(__name__) STR_CMP_NE_FUNCTION = ">::ne" STR_CMP_EQ_FUNCTION = ">::eq" diff --git a/angr/rust/sim_type.py b/angr/rust/sim_type.py index 8cf3cbf0f..f1900afbe 100644 --- a/angr/rust/sim_type.py +++ b/angr/rust/sim_type.py @@ -353,7 +353,7 @@ class RustSimStruct(RustSimType, SimStruct): # Fixup the offsets to byte aligned addresses for all SimTypeNumOffset types offset_so_far = 0 - for _name, ty in out.fields.items(): + for ty in out.fields.values(): if isinstance(ty, RustSimTypeNumOffset): out._pack = True ty.offset = offset_so_far % arch.byte_width @@ -778,7 +778,7 @@ class RustSimEnum(RustSimType, SimType): return len(self.variants) def as_struct_ty(self): - largest = sorted(self.variants, key=lambda variant: variant.bits)[-1] + largest = max(self.variants, key=lambda variant: variant.bits) struct_ty = largest.as_struct_ty() struct_ty.name = self.name return struct_ty diff --git a/angr/rustylib/__init__.pyi b/angr/rustylib/__init__.pyi index 9013835fb..d53e45049 100644 --- a/angr/rustylib/__init__.pyi +++ b/angr/rustylib/__init__.pyi @@ -1,5 +1,3 @@ -from typing import override - from . import ailment, automaton, fuzzer, icicle class Segment: @@ -35,9 +33,6 @@ class Segment: :returns: Size of the Segment """ - @override - def __repr__(self) -> str: ... - class SegmentList: """ SegmentList describes a series of segmented memory blocks. You may query whether an address belongs to any of the diff --git a/angr/rustylib/ailment.pyi b/angr/rustylib/ailment.pyi index d83876937..241095668 100644 --- a/angr/rustylib/ailment.pyi +++ b/angr/rustylib/ailment.pyi @@ -346,8 +346,6 @@ class Expression: """Python ``copy.deepcopy`` protocol -- routes through ``deep_copy`` with a stand-in ``Manager`` from ``angr.ailment._deepcopy``.""" def __reduce__(self) -> tuple[Any, ...]: """Python ``pickle`` protocol. Routes through ``to_bytes`` / ``from_bytes`` to preserve the full ``AilExpression`` shape.""" - def __repr__(self) -> str: ... - def __str__(self) -> str: ... # --- Serialization ------------------------------------------------- def to_bytes(self) -> bytes: ... @@ -482,8 +480,6 @@ class Statement: """Python ``copy.deepcopy`` protocol -- routes through ``deep_copy`` with a stand-in ``Manager`` from ``angr.ailment._deepcopy``.""" def __reduce__(self) -> tuple[Any, ...]: """Python ``pickle`` protocol via ``to_bytes`` / ``from_bytes``. Same lossy-field caveat as ``Expression.__reduce__``.""" - def __repr__(self) -> str: ... - def __str__(self) -> str: ... # --- Serialization ------------------------------------------------- def to_bytes(self) -> bytes: ... @@ -512,8 +508,6 @@ class Block: def __eq__(self, other: object) -> bool: ... def __copy__(self) -> Self: ... def __deepcopy__(self, memo: Any) -> Self: ... - def __repr__(self) -> str: ... - def __str__(self) -> str: ... def copy(self, statements: list[Statement] | None = ...) -> Self: ... def dbg_repr(self, indent: int = ...) -> str: ... # --- Serialization ------------------------------------------------- diff --git a/angr/rustylib/automaton.pyi b/angr/rustylib/automaton.pyi index f6309073d..5a76e64de 100644 --- a/angr/rustylib/automaton.pyi +++ b/angr/rustylib/automaton.pyi @@ -22,7 +22,6 @@ class State: :returns: The wrapped value. """ - def __repr__(self) -> str: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... @@ -46,7 +45,6 @@ class Symbol: :returns: The wrapped value. """ - def __repr__(self) -> str: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... @@ -60,7 +58,6 @@ class Epsilon: Initialize an Epsilon marker. """ - def __repr__(self) -> str: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... diff --git a/angr/rustylib/fuzzer.pyi b/angr/rustylib/fuzzer.pyi index db5d32b52..ee6c4a6a1 100644 --- a/angr/rustylib/fuzzer.pyi +++ b/angr/rustylib/fuzzer.pyi @@ -1,93 +1,51 @@ from collections.abc import Callable from datetime import timedelta +from typing import Self from angr.sim_state import SimState class InMemoryCorpus: - def __new__(cls) -> InMemoryCorpus: - pass - + def __new__(cls) -> Self: ... @staticmethod - def from_list(inputs: list[bytes]) -> InMemoryCorpus: - pass - - def to_bytes_list(self) -> list[bytes]: - pass - - def __getitem__(self, idx: int) -> bytes: - pass - - def __len__(self) -> int: - pass + def from_list(inputs: list[bytes]) -> InMemoryCorpus: ... + def to_bytes_list(self) -> list[bytes]: ... + def __getitem__(self, idx: int) -> bytes: ... + def __len__(self) -> int: ... class OnDiskCorpus: - def __new__(cls, dir_path: str) -> OnDiskCorpus: - pass - - def add(self, input: bytes) -> int: - pass - - def to_bytes_list(self) -> list[bytes]: - pass - - def __getitem__(self, idx: int) -> bytes: - pass - - def __len__(self) -> int: - pass + def __new__(cls, dir_path: str) -> Self: ... + def add(self, input: bytes) -> int: ... + def to_bytes_list(self) -> list[bytes]: ... + def __getitem__(self, idx: int) -> bytes: ... + def __len__(self) -> int: ... class ClientStats: @property - def enabled(self) -> bool: - pass - + def enabled(self) -> bool: ... @property - def corpus_size(self) -> int: - pass - + def corpus_size(self) -> int: ... @property - def last_corpus_time(self) -> timedelta: - pass - + def last_corpus_time(self) -> timedelta: ... @property - def executions(self) -> int: - pass - + def executions(self) -> int: ... @property - def prev_state_executions(self) -> int: - pass - + def prev_state_executions(self) -> int: ... @property - def objective_size(self) -> int: - pass - + def objective_size(self) -> int: ... @property - def last_objective_time(self) -> timedelta: - pass - + def last_objective_time(self) -> timedelta: ... @property - def last_window_time(self) -> timedelta: - pass - + def last_window_time(self) -> timedelta: ... @property - def start_time(self) -> timedelta: - pass - + def start_time(self) -> timedelta: ... @property - def execs_per_sec(self) -> float: - pass - + def execs_per_sec(self) -> float: ... @property - def execs_per_sec_pretty(self) -> str: - pass - + def execs_per_sec_pretty(self) -> str: ... @property - def edges_hit(self) -> int | None: - pass - + def edges_hit(self) -> int | None: ... @property - def edges_total(self) -> int | None: - pass + def edges_total(self) -> int | None: ... class HavocMutator: def __init__(self, max_stack_pow: int | None = None): diff --git a/angr/sim_procedure.py b/angr/sim_procedure.py index 8d88ca3ec..a9fac401b 100644 --- a/angr/sim_procedure.py +++ b/angr/sim_procedure.py @@ -179,7 +179,7 @@ class SimProcedure: self.ret_expr = None self.call_ret_expr = None self.inhibit_autoret = None - self.arg_session: None | ArgSession | int = None + self.arg_session: ArgSession | int | None = None def __repr__(self): return "".format(*self._describe_me()) diff --git a/angr/sim_state.py b/angr/sim_state.py index 78a87597d..fefde8c18 100644 --- a/angr/sim_state.py +++ b/angr/sim_state.py @@ -297,12 +297,8 @@ class SimState[IPTypeConc, IPTypeSym](PluginHub[SimStatePlugin]): addr = self.addr if type(addr) is int: ip_str = f"{addr:#x}" - try: - scratch = self.scratch - if scratch.is_ail and scratch.ail_block_idx is not None: - ip_str = f"{addr:#x}.{scratch.ail_block_idx}" - except Exception: - pass + if self.scratch.is_ail and self.scratch.ail_block_idx is not None: + ip_str = f"{addr:#x}.{self.scratch.ail_block_idx}" else: ip_str = repr(addr) except (SimValueError, SimSolverModeError): diff --git a/angr/sim_type.py b/angr/sim_type.py index e1a6d3a2c..44bf76f27 100644 --- a/angr/sim_type.py +++ b/angr/sim_type.py @@ -1747,7 +1747,7 @@ class SimStruct(NamedTypeMixin, SimType): # Fixup the offsets to byte aligned addresses for all SimTypeNumOffset types offset_so_far = 0 - for _, ty in out.fields.items(): + for ty in out.fields.values(): if isinstance(ty, SimTypeNumOffset): out._pack = True ty.offset = offset_so_far % arch.byte_width @@ -2467,7 +2467,7 @@ class SimCppClass(SimStruct): # Fixup the offsets to byte aligned addresses for all SimTypeNumOffset types offset_so_far = 0 - for _, ty in out.members.items(): + for ty in out.members.values(): if isinstance(ty, SimTypeNumOffset): out._pack = True ty.offset = offset_so_far % arch.byte_width @@ -3820,7 +3820,7 @@ def parse_file( # pylint: disable=unexpected-keyword-arg node = pycparser.c_parser.CParser().parse(defn, scope_stack=_make_scope(predefined_types)) if not isinstance(node, c_ast.FileAST): - raise ValueError("Something went horribly wrong using pycparser") + raise TypeError("Something went horribly wrong using pycparser") out = {} out_types = {} extra_types = ChainMap(side_effect_types if side_effect_types is not None else out_types, predefined_types or {}) @@ -3838,7 +3838,7 @@ def parse_file( if isinstance(ty_real, (SimStruct, SimUnion)) and ty_real.name != "": if piece.name is None: out_types[("struct " if isinstance(ty, SimStruct) else "union ") + ty_real.name] = ty_real - for _, i in out_types.items(): + for i in out_types.values(): if isinstance(i, type(ty_real)) and i.name == ty_real.name: if isinstance(ty_real, SimStruct): assert isinstance(i, SimStruct) diff --git a/angr/simos/__init__.py b/angr/simos/__init__.py index 01facb522..0e9a84da4 100644 --- a/angr/simos/__init__.py +++ b/angr/simos/__init__.py @@ -25,7 +25,7 @@ def register_simos(name, cls): # Pulling in all EI_OSABI options supported by elftools -for _k, v in _DESCR_EI_OSABI.items(): +for v in _DESCR_EI_OSABI.values(): register_simos(v, SimLinux) register_simos("linux", SimLinux) diff --git a/angr/state_plugins/debug_variables.py b/angr/state_plugins/debug_variables.py index 75af577eb..0899144e2 100644 --- a/angr/state_plugins/debug_variables.py +++ b/angr/state_plugins/debug_variables.py @@ -38,7 +38,7 @@ class SimDebugVariable: @property def mem_untyped(self) -> SimMemView: if self.addr is None: - raise Exception("Cannot view a variable without an address") + raise ValueError("Cannot view a variable without an address") return self.state.mem[self.addr] @property @@ -114,7 +114,7 @@ class SimDebugVariable: addr = self.state.memory.load(self.addr, self.state.arch.bytes, endness=self.state.arch.memory_endness) el_type = self.type.referenced_type else: - raise Exception(f"{self.type} object cannot be dereferenced") + raise TypeError(f"{self.type} object cannot be dereferenced") if i == 0: new_addr = addr diff --git a/angr/state_plugins/heap/heap_ptmalloc.py b/angr/state_plugins/heap/heap_ptmalloc.py index 3fbc9ba4b..cc9d23153 100644 --- a/angr/state_plugins/heap/heap_ptmalloc.py +++ b/angr/state_plugins/heap/heap_ptmalloc.py @@ -455,8 +455,7 @@ class SimHeapPTMalloc(SimHeapFreelist): fwd.set_bck_chunk(chunk) chunk.set_bck_chunk(bck) chunk.set_fwd_chunk(fwd) - if chunk < self.free_head_chunk: - self.free_head_chunk = chunk + self.free_head_chunk = min(self.free_head_chunk, chunk) elif not p_in_use and not n_in_use: # If both the adjacent chunks are free, merging between all three will be needed p_ptr = chunk.prev_chunk() # The previous chunk will be the base of the overall new chunk @@ -480,8 +479,7 @@ class SimHeapPTMalloc(SimHeapFreelist): fwd = n_ptr.fwd_chunk() # In case the freed chunk preceded the free head - if base < self.free_head_chunk: - self.free_head_chunk = base + self.free_head_chunk = min(self.free_head_chunk, base) # In case the following chunk was the last free chunk, we can't use its links due to the merge if bck == fwd and bck == n_ptr: diff --git a/angr/state_plugins/history.py b/angr/state_plugins/history.py index 19898396e..f2d163db9 100644 --- a/angr/state_plugins/history.py +++ b/angr/state_plugins/history.py @@ -504,9 +504,9 @@ class TreeIter: def __getitem__(self, k): if isinstance(k, slice): - raise ValueError("Please use .hardcopy to use slices") + raise TypeError("Please use .hardcopy to use slices") if k >= 0: - raise ValueError("Please use .hardcopy to use nonnegative indexes") + raise TypeError("Please use .hardcopy to use nonnegative indexes") i = 0 for item in reversed(self): i -= 1 diff --git a/angr/state_plugins/inspect.py b/angr/state_plugins/inspect.py index dcddb22a5..8c9daa14f 100644 --- a/angr/state_plugins/inspect.py +++ b/angr/state_plugins/inspect.py @@ -166,7 +166,7 @@ def BP_IPDB(state: SimState) -> None: # pylint: disable=unused-argument def BP_IPYTHON(state: SimState) -> None: # pylint: disable=unused-argument import IPython - shell = IPython.terminal.embed.InteractiveShellEmbed() + shell = IPython.terminal.embed.InteractiveShellEmbed() # noqa: T100 shell.mainloop( display_banner="This is an ipython shell for you to happily debug your state!\n" + "The state can be accessed through the variable 'state'. You can\n" diff --git a/angr/state_plugins/solver.py b/angr/state_plugins/solver.py index b2bf3cb58..f6ff024cc 100644 --- a/angr/state_plugins/solver.py +++ b/angr/state_plugins/solver.py @@ -81,12 +81,12 @@ def disable_timing(): _timing_enabled = False -if os.environ.get("SOLVER_TIMING", False): +if os.environ.get("SOLVER_TIMING", None): enable_timing() else: disable_timing() -break_time = float(os.environ.get("SOLVER_BREAK_TIME", -1)) +break_time = float(os.environ.get("SOLVER_BREAK_TIME", "-1")) # # Various over-engineered crap @@ -752,7 +752,7 @@ class SimSolver(SimStatePlugin): @staticmethod def _cast_to( e: claripy.ast.Bool | claripy.ast.BV | claripy.ast.FP, - solution: bool | float | int, + solution: bool | float, cast_to: type[CastType] | None, ) -> CastType: """ diff --git a/angr/state_plugins/view.py b/angr/state_plugins/view.py index 4acccf6af..8e33a1b32 100644 --- a/angr/state_plugins/view.py +++ b/angr/state_plugins/view.py @@ -88,7 +88,7 @@ class SimRegNameView(SimStatePlugin): if isinstance(self.state.arch, ArchSoot): pass else: - raise AttributeError(k) from err + raise TypeError(k) from err def __dir__(self): if self.state.arch.name in ("X86", "AMD64"): diff --git a/angr/storage/file.py b/angr/storage/file.py index e35255540..df2416085 100644 --- a/angr/storage/file.py +++ b/angr/storage/file.py @@ -392,7 +392,6 @@ class SimFileStream(SimFile): def write(self, _, data, size=None, **kwargs): self.pos = super().write(self.pos, data, size, **kwargs) - return @SimStatePlugin.memo def copy(self, memo): @@ -684,7 +683,6 @@ class SimPacketsStream(SimPackets): def write(self, _, data, size=None, **kwargs): self.pos = super().write(self.pos, data, size, **kwargs) - return @SimStatePlugin.memo def copy(self, memo): diff --git a/angr/storage/memory_mixins/paged_memory/page_backer_mixins.py b/angr/storage/memory_mixins/paged_memory/page_backer_mixins.py index 345a7ff35..183e7c9e0 100644 --- a/angr/storage/memory_mixins/paged_memory/page_backer_mixins.py +++ b/angr/storage/memory_mixins/paged_memory/page_backer_mixins.py @@ -30,7 +30,7 @@ class NotMemoryview: class ClemoryBackerMixin(PagedMemoryMixin): - def __init__(self, cle_memory_backer: None | cle.Loader | cle.Clemory = None, **kwargs): + def __init__(self, cle_memory_backer: cle.Loader | cle.Clemory | None = None, **kwargs): super().__init__(**kwargs) if isinstance(cle_memory_backer, cle.Loader): diff --git a/angr/storage/memory_mixins/paged_memory/pages/ispo_mixin.py b/angr/storage/memory_mixins/paged_memory/pages/ispo_mixin.py index 92384f331..90cd39f2f 100644 --- a/angr/storage/memory_mixins/paged_memory/pages/ispo_mixin.py +++ b/angr/storage/memory_mixins/paged_memory/pages/ispo_mixin.py @@ -4,6 +4,12 @@ from __future__ import annotations from angr.storage.memory_mixins.memory_mixin import MemoryMixin +class MissingMemoryError(Exception): + """ + Raised when a memory kwarg is not passed to a stateless object. + """ + + class ISPOMixin(MemoryMixin): """ An implementation of the International Stateless Persons Organisation, a mixin which should be applied as a bottom diff --git a/angr/storage/memory_mixins/paged_memory/pages/multi_values.py b/angr/storage/memory_mixins/paged_memory/pages/multi_values.py index e0300a753..9059afcb2 100644 --- a/angr/storage/memory_mixins/paged_memory/pages/multi_values.py +++ b/angr/storage/memory_mixins/paged_memory/pages/multi_values.py @@ -29,7 +29,7 @@ class MultiValues[MVType: claripy.ast.BV | claripy.ast.FP]: def __init__( self, - v: MVType | MultiValues[MVType] | None | dict[int, set[MVType]] = None, + v: MVType | MultiValues[MVType] | dict[int, set[MVType]] | None = None, offset_to_values: dict[int, set[MVType]] | None = None, ): if v is not None and offset_to_values is not None: diff --git a/angr/storage/memory_mixins/regioned_memory/regioned_memory_mixin.py b/angr/storage/memory_mixins/regioned_memory/regioned_memory_mixin.py index 0b17ea3fc..7ed3dfc3c 100644 --- a/angr/storage/memory_mixins/regioned_memory/regioned_memory_mixin.py +++ b/angr/storage/memory_mixins/regioned_memory/regioned_memory_mixin.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging from collections.abc import Generator from itertools import count -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING import claripy from claripy.annotation import RegionAnnotation @@ -49,7 +49,7 @@ class RegionedMemoryMixin(MemoryMixin): stack_region_map: RegionMap | None = None, generic_region_map: RegionMap | None = None, stack_size: int = 65536, - cle_memory_backer: Optional | None = None, + cle_memory_backer: None = None, dict_memory_backer: dict | None = None, regioned_memory_cls: type | None = None, **kwargs, @@ -240,7 +240,7 @@ class RegionedMemoryMixin(MemoryMixin): state: SimState, related_function_addr: int, endness, - cle_memory_backer: Optional | None = None, + cle_memory_backer: None = None, dict_memory_backer: dict | None = None, ): """ diff --git a/angr/utils/graph.py b/angr/utils/graph.py index 2cb9bbdcc..a3fb1a6a6 100644 --- a/angr/utils/graph.py +++ b/angr/utils/graph.py @@ -130,7 +130,7 @@ def dfs_back_edges[T]( if visit_all_nodes: while len(visited) < len(graph): # If we need to visit all nodes, we can start from unvisited nodes - node = sorted(set(graph) - visited, key=GraphUtils.sort_node)[0] + node = min(set(graph) - visited, key=GraphUtils.sort_node) yield from dfs_back_edges(graph, node, visited=visited) @@ -359,7 +359,7 @@ class ContainerNode[T]: def obj(self) -> T: return self._obj - def __eq__(self, other: Any): + def __eq__(self, other: object): if isinstance(other, ContainerNode): return self._obj is other._obj return False @@ -431,7 +431,7 @@ class Dominators[T]: assert self._semi is not None bucket: dict[int, set[ContainerNode]] = defaultdict(set) - dom: list[None | ContainerNode] = [None] * (len(vertices)) + dom: list[ContainerNode | None] = [None] * (len(vertices)) self._ancestor = [None] * (len(vertices) + 1) # type: ignore for i in range(len(vertices) - 1, 0, -1): @@ -1006,7 +1006,7 @@ class GraphUtils: if loop_head is None: # pick the first one - loop_head = sorted(scc, key=GraphUtils.sort_node)[0] + loop_head = min(scc, key=GraphUtils.sort_node) subgraph: networkx.DiGraph = graph.subgraph(scc).copy() # type: ignore for src, _ in list(subgraph.in_edges(loop_head)): diff --git a/corpus_tests/README.md b/corpus_tests/README.md index 59fa164b1..426f1bc25 100644 --- a/corpus_tests/README.md +++ b/corpus_tests/README.md @@ -147,9 +147,11 @@ The pytest framework is also extended with a --binary option which allows the pa ### in conftest.py import pytest + def pytest_addoption(parser): parser.addoption("--binary", action="store", default="") + @pytest.fixture def binary(request): return request.config.getoption("--binary") diff --git a/pyproject.toml b/pyproject.toml index 1e5d02f8f..5993fbcd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,8 +134,6 @@ exclude = [ [tool.ruff.lint] extend-select = [ - "B", - "C4", "FURB", "I", "PIE", @@ -147,21 +145,30 @@ extend-select = [ "UP", ] ignore = [ - "B011", # Asserts + "BLE001", # Catching Exception + "DTZ", # timezones "E741", # Variable names + "EXE", + "RET501", # rewrites "return None" to "return" if it's the only return statement; pylint then removes the return "RUF012", # FIXME: Annotate class variables "RUF007", # Prefer `itertools.pairwise()` over `zip()` when iterating over successive pairs - "B905", # Zip calls without an explicit strict parameter - "RET501", # rewrites "return None" to "return" if it's the only return statement; pylint then removes the return + "S112", # Blind exception handlers + "TRY002", # Create your own Exceptions ] [tool.ruff.lint.per-file-ignores] "angr/__init__.py" = [ "E402", # Package root: bootstrap code is interleaved with imports and the import order is load-bearing ] +"angr/engines/soot/expressions/*" = [ + "N999", +] "angr/misc/bug_report.py" = [ "F821", # name not found ] +"angr/procedures/**" = [ + "N999", +] "angr/procedures/definitions/*" = [ # TODO: Move to exclude "F601", # TODO: BUG! This hides bugs! See: https://github.com/angr/angr/issues/3685 "E501", # No line length check diff --git a/tests/ailment/test_expression.py b/tests/ailment/test_expression.py index 0081a7e55..822e67600 100644 --- a/tests/ailment/test_expression.py +++ b/tests/ailment/test_expression.py @@ -7,7 +7,7 @@ from types import SimpleNamespace import pytest -import angr.ailment as ailment +from angr import ailment from angr.ailment.expression import ( Array, BasePointerOffset, diff --git a/tests/analyses/decompiler/test_baseptr_save_simplifier.py b/tests/analyses/decompiler/test_baseptr_save_simplifier.py index 321eb9fde..b88810a1b 100755 --- a/tests/analyses/decompiler/test_baseptr_save_simplifier.py +++ b/tests/analyses/decompiler/test_baseptr_save_simplifier.py @@ -7,7 +7,7 @@ import os.path import unittest import angr -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.optimization_passes.base_ptr_save_simplifier import ( BasePointerSaveSimplifier, ) diff --git a/tests/analyses/decompiler/test_cas_rewriting.py b/tests/analyses/decompiler/test_cas_rewriting.py index 125d7e66c..b9a799048 100644 --- a/tests/analyses/decompiler/test_cas_rewriting.py +++ b/tests/analyses/decompiler/test_cas_rewriting.py @@ -12,7 +12,7 @@ from tests.common import bin_location, load_project_with_scoped_cfg, print_decom test_location = os.path.join(bin_location, "tests") -l = logging.Logger(__name__) +l = logging.getLogger(__name__) class TestCASRewriting(unittest.TestCase): diff --git a/tests/analyses/decompiler/test_decompiler.py b/tests/analyses/decompiler/test_decompiler.py index 8b3136429..fb15400de 100755 --- a/tests/analyses/decompiler/test_decompiler.py +++ b/tests/analyses/decompiler/test_decompiler.py @@ -14,7 +14,7 @@ from functools import wraps import networkx import angr -import angr.ailment as ailment +from angr import ailment from angr.analyses import ( CallingConventionAnalysis, CFGFast, @@ -66,7 +66,7 @@ from tests.common import ( test_location = os.path.join(bin_location, "tests") -l = logging.Logger(__name__) +l = logging.getLogger(__name__) def normalize_whitespace(s: str) -> str: diff --git a/tests/analyses/decompiler/test_expression_overfolding.py b/tests/analyses/decompiler/test_expression_overfolding.py index 211cbccc3..321a6677b 100644 --- a/tests/analyses/decompiler/test_expression_overfolding.py +++ b/tests/analyses/decompiler/test_expression_overfolding.py @@ -13,7 +13,7 @@ from tests.common import WORKER, bin_location, print_decompilation_result test_location = os.path.join(bin_location, "tests") -l = logging.Logger(__name__) +l = logging.getLogger(__name__) class TestExpressionOverfolding(unittest.TestCase): diff --git a/tests/analyses/decompiler/test_head_controlled_loops.py b/tests/analyses/decompiler/test_head_controlled_loops.py index e1bcc4712..9ca351ea9 100644 --- a/tests/analyses/decompiler/test_head_controlled_loops.py +++ b/tests/analyses/decompiler/test_head_controlled_loops.py @@ -13,7 +13,7 @@ from tests.common import bin_location, load_project_with_scoped_cfg, print_decom test_location = os.path.join(bin_location, "tests") -l = logging.Logger(__name__) +l = logging.getLogger(__name__) class TestHeadControlledLoops(unittest.TestCase): diff --git a/tests/analyses/decompiler/test_jumpkind_intrinsics.py b/tests/analyses/decompiler/test_jumpkind_intrinsics.py index f38f60b07..7e961588c 100644 --- a/tests/analyses/decompiler/test_jumpkind_intrinsics.py +++ b/tests/analyses/decompiler/test_jumpkind_intrinsics.py @@ -10,7 +10,7 @@ from tests.common import bin_location test_location = os.path.join(bin_location, "tests") -l = logging.Logger(__name__) +l = logging.getLogger(__name__) class TestJumpkindIntrinsics(unittest.TestCase): diff --git a/tests/analyses/decompiler/test_narrowing_exprs.py b/tests/analyses/decompiler/test_narrowing_exprs.py index 212185ca3..c5699bd84 100644 --- a/tests/analyses/decompiler/test_narrowing_exprs.py +++ b/tests/analyses/decompiler/test_narrowing_exprs.py @@ -16,7 +16,7 @@ from tests.common import WORKER, bin_location, print_decompilation_result test_location = os.path.join(bin_location, "tests") -l = logging.Logger(__name__) +l = logging.getLogger(__name__) class TestNarrowingExpressions(unittest.TestCase): diff --git a/tests/analyses/decompiler/test_partial_reg_reads.py b/tests/analyses/decompiler/test_partial_reg_reads.py index 3425383b9..0bfe5c88e 100644 --- a/tests/analyses/decompiler/test_partial_reg_reads.py +++ b/tests/analyses/decompiler/test_partial_reg_reads.py @@ -16,7 +16,7 @@ from tests.common import WORKER, bin_location, print_decompilation_result test_location = os.path.join(bin_location, "tests") -l = logging.Logger(__name__) +l = logging.getLogger(__name__) class TestPartialRegReads(unittest.TestCase): diff --git a/tests/analyses/decompiler/test_peephole_inline_memset.py b/tests/analyses/decompiler/test_peephole_inline_memset.py index e05776d91..cc2753033 100644 --- a/tests/analyses/decompiler/test_peephole_inline_memset.py +++ b/tests/analyses/decompiler/test_peephole_inline_memset.py @@ -14,7 +14,7 @@ from tests.common import WORKER, bin_location, print_decompilation_result test_location = os.path.join(bin_location, "tests") -l = logging.Logger(__name__) +l = logging.getLogger(__name__) class TestPeepholeInlineMemset(unittest.TestCase): diff --git a/tests/analyses/decompiler/test_peephole_redundant_bitshifts.py b/tests/analyses/decompiler/test_peephole_redundant_bitshifts.py index a367479dd..9d7b6dea4 100644 --- a/tests/analyses/decompiler/test_peephole_redundant_bitshifts.py +++ b/tests/analyses/decompiler/test_peephole_redundant_bitshifts.py @@ -13,7 +13,7 @@ from tests.common import WORKER, bin_location, print_decompilation_result test_location = os.path.join(bin_location, "tests") -l = logging.Logger(__name__) +l = logging.getLogger(__name__) class TestPeepholeRedundantBitshifts(unittest.TestCase): diff --git a/tests/analyses/decompiler/test_peephole_wcscpy.py b/tests/analyses/decompiler/test_peephole_wcscpy.py index 44abcf5b8..fae913921 100644 --- a/tests/analyses/decompiler/test_peephole_wcscpy.py +++ b/tests/analyses/decompiler/test_peephole_wcscpy.py @@ -13,7 +13,7 @@ from tests.common import bin_location, load_project_with_scoped_cfg, print_decom test_location = os.path.join(bin_location, "tests") -l = logging.Logger(__name__) +l = logging.getLogger(__name__) class TestPeepholeWcscpy(unittest.TestCase): diff --git a/tests/analyses/decompiler/test_unify_local_variables.py b/tests/analyses/decompiler/test_unify_local_variables.py index 8b04bf338..e64b5f070 100644 --- a/tests/analyses/decompiler/test_unify_local_variables.py +++ b/tests/analyses/decompiler/test_unify_local_variables.py @@ -13,7 +13,7 @@ from tests.common import WORKER, bin_location, print_decompilation_result test_location = os.path.join(bin_location, "tests") -l = logging.Logger(__name__) +l = logging.getLogger(__name__) class TestUnifyLocalVariables(unittest.TestCase): diff --git a/tests/analyses/decompiler/test_variable_nondeterminism.py b/tests/analyses/decompiler/test_variable_nondeterminism.py index 177bf9477..f26cd6d5b 100644 --- a/tests/analyses/decompiler/test_variable_nondeterminism.py +++ b/tests/analyses/decompiler/test_variable_nondeterminism.py @@ -48,6 +48,7 @@ def _decompile_with_seed(seed: int, binary_path: str) -> str: text=True, env=env, timeout=300, + check=True, ) assert result.returncode == 0, f"Decompilation failed (seed={seed}):\n{result.stderr}" return result.stdout diff --git a/tests/analyses/reaching_definitions/test_reachingdefinitions.py b/tests/analyses/reaching_definitions/test_reachingdefinitions.py index f559abd6c..95fb83ca6 100755 --- a/tests/analyses/reaching_definitions/test_reachingdefinitions.py +++ b/tests/analyses/reaching_definitions/test_reachingdefinitions.py @@ -10,7 +10,7 @@ from unittest import TestCase, main import claripy import angr -import angr.ailment as ailment +from angr import ailment from angr.analyses import CFGFast, CompleteCallingConventionsAnalysis, ReachingDefinitionsAnalysis from angr.analyses.reaching_definitions.dep_graph import DepGraph from angr.analyses.reaching_definitions.function_handler_library import LibcHandlers diff --git a/tests/analyses/reaching_definitions/test_subject.py b/tests/analyses/reaching_definitions/test_subject.py index eeeeb8733..fe04ca431 100755 --- a/tests/analyses/reaching_definitions/test_subject.py +++ b/tests/analyses/reaching_definitions/test_subject.py @@ -7,7 +7,7 @@ from unittest import mock import networkx from archinfo.arch_x86 import ArchX86 -import angr.ailment as ailment +from angr import ailment from angr.analyses.forward_analysis.visitors import FunctionGraphVisitor from angr.analyses.reaching_definitions.subject import Subject, SubjectType from angr.block import Block diff --git a/tests/analyses/test_callsite_maker.py b/tests/analyses/test_callsite_maker.py index 70db797b4..511e12ff5 100755 --- a/tests/analyses/test_callsite_maker.py +++ b/tests/analyses/test_callsite_maker.py @@ -7,7 +7,7 @@ import os import unittest import angr -import angr.ailment as ailment +from angr import ailment from angr.analyses.decompiler.block_simplifier import BlockSimplifier from tests.common import bin_location diff --git a/tests/analyses/test_class_identifier.py b/tests/analyses/test_class_identifier.py index bfdd3c83e..1064f4538 100755 --- a/tests/analyses/test_class_identifier.py +++ b/tests/analyses/test_class_identifier.py @@ -22,8 +22,7 @@ class TestClassIdentifier(unittest.TestCase): class_labels = [] vtable_ptr_c = [0x403CB0, 0x403CD8] - for class_str in classes_found: - class_labels.append(class_str) + class_labels.extend(classes_found.keys()) assert "A" in class_labels assert "B" in class_labels diff --git a/tests/common.py b/tests/common.py index c740f8c22..53b986746 100644 --- a/tests/common.py +++ b/tests/common.py @@ -32,7 +32,7 @@ bin_location = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", " bin_priv_location = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "binaries-private") WORKER = is_testing or bool( - os.environ.get("WORKER", False) + os.environ.get("WORKER", None) ) # this variable controls whether we print the decompilation code or not if not os.path.isdir(bin_location) and not os.getenv("CI", "") == "true": diff --git a/tests/engines/test_java.py b/tests/engines/test_java.py index d369d529b..e536740bc 100755 --- a/tests/engines/test_java.py +++ b/tests/engines/test_java.py @@ -624,7 +624,7 @@ class TestJava(unittest.TestCase): assert val == assert_value if assertions: - for _, test in assertions.items(): + for test in assertions.values(): assert test(end_state) return end_state diff --git a/tests/serialization/test_db.py b/tests/serialization/test_db.py index 46c3ede83..a8821ceb4 100755 --- a/tests/serialization/test_db.py +++ b/tests/serialization/test_db.py @@ -687,7 +687,7 @@ class TestDb(unittest.TestCase): # find the CConstant whose value is 8 target_consts = [] - for _, elem in dec.codegen.map_pos_to_node.items(): + for elem in dec.codegen.map_pos_to_node.values(): if isinstance(elem.obj, CConstant) and elem.obj.value == 8: target_consts.append(elem.obj) diff --git a/tests/serialization/test_pickle.py b/tests/serialization/test_pickle.py index d8ba92950..85bd632c1 100755 --- a/tests/serialization/test_pickle.py +++ b/tests/serialization/test_pickle.py @@ -61,9 +61,9 @@ class TestPickle(unittest.TestCase): # If you do not have a state you cannot write _ = p.factory.entry_state(fs=fs) - for f in fs: # pylint:disable=consider-using-dict-items + for f, fo in fs.items(): mem = mem_bvv[f] - fs[f].write(0, mem, MEM_SIZE) + fo.write(0, mem, MEM_SIZE) with open("pickletest_bad", "wb") as f: pickle.dump(mem_bvv, f, -1) diff --git a/tests/utils/test_library.py b/tests/utils/test_library.py index bb8c032cd..5f1ae1660 100644 --- a/tests/utils/test_library.py +++ b/tests/utils/test_library.py @@ -24,32 +24,36 @@ class TestLibrary(unittest.TestCase): ("public: virtual __cdecl exception::~exception(void)", "exception::~exception"), ("public: virtual char const * __cdecl exception::what(void) const", "exception::what"), ( - "private: static long __cdecl wil::details_abi::ProcessLocalStorageData<" - "struct wil::details_abi::ProcessLocalData" - ">::MakeAndInitialize(" - "unsigned short const *, " - "class wil::unique_any_t<" - "class wil::mutex_t<" - "class wil::details::unique_storage<" - "struct wil::details::resource_policy<" - "void *, " - "void (__cdecl *)(void*) noexcept, " # cxxheaderparser has trouble supporting void (__cdecl *) - "&void __cdecl wil::details::CloseHandle(void *), " - "struct wistd::integral_constant, " - "void *, " - "void *, " - "0, " - "std::nullptr_t" - ">" - ">, " - "struct wil::err_returncode_policy" - ">" - "> " - "&&," - "class wil::details_abi::ProcessLocalStorageData **)", - "wil::details_abi::ProcessLocalStorageData<" - "struct wil::details_abi::ProcessLocalData" - ">::MakeAndInitialize", + ( + "private: static long __cdecl wil::details_abi::ProcessLocalStorageData<" + "struct wil::details_abi::ProcessLocalData" + ">::MakeAndInitialize(" + "unsigned short const *, " + "class wil::unique_any_t<" + "class wil::mutex_t<" + "class wil::details::unique_storage<" + "struct wil::details::resource_policy<" + "void *, " + "void (__cdecl *)(void*) noexcept, " # cxxheaderparser has trouble supporting void (__cdecl *) + "&void __cdecl wil::details::CloseHandle(void *), " + "struct wistd::integral_constant, " + "void *, " + "void *, " + "0, " + "std::nullptr_t" + ">" + ">, " + "struct wil::err_returncode_policy" + ">" + "> " + "&&," + "class wil::details_abi::ProcessLocalStorageData **)" + ), + ( + "wil::details_abi::ProcessLocalStorageData<" + "struct wil::details_abi::ProcessLocalData" + ">::MakeAndInitialize" + ), ), ( "void __cdecl UptimeTicksToFileTimeBasedULongLong(unsigned __int64, unsigned __int64 *)", From 61bac8ffd0338cdc43788276aa14eee6d50f995d Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 29 Jul 2026 13:58:20 -0700 Subject: [PATCH 078/122] SimConstantVariable: Fix overflows and value out of range errors. (#6738) * SimConstantVariable: Fix overflows and value out of range errors. * Fix usages of SimConstantVariable. * Mask the value. * Fix negative values. --- angr/analyses/ddg.py | 6 +++-- angr/protos/variables.proto | 3 ++- angr/sim_variable.py | 17 +++++++++---- tests/serialization/test_serialization.py | 30 ++++++++++++++++++++++- 4 files changed, 47 insertions(+), 9 deletions(-) diff --git a/angr/analyses/ddg.py b/angr/analyses/ddg.py index a7626f8b4..14f2f79f7 100644 --- a/angr/analyses/ddg.py +++ b/angr/analyses/ddg.py @@ -1156,9 +1156,11 @@ class DDG(Analysis): if not action.reg_deps and not action.tmp_deps: # moving a constant into the register # try to parse out the constant from statement - const_variable = SimConstantVariable(size=1) if statement is not None and isinstance(statement.data, pyvex.IRExpr.Const): const_variable = SimConstantVariable(value=statement.data.con.value, size=statement.data.con.size) + else: + # use a default value of 0 if we cannot find the constant + const_variable = SimConstantVariable(1, value=0) const_pv = ProgramVariable(const_variable, location, arch=self.project.arch) self._data_graph_add_edge(const_pv, pv) @@ -1229,7 +1231,7 @@ class DDG(Analysis): if not action.tmp_deps and not self._variables_per_statement and not ast: # read in a constant # try to parse out the constant from statement - const_variable = SimConstantVariable(size=1) + const_variable = SimConstantVariable(size=1, value=0) # default value if we can't find the constant if statement is not None: if isinstance(statement, pyvex.IRStmt.Dirty): l.warning("Dirty statements are not supported in DDG for now.") diff --git a/angr/protos/variables.proto b/angr/protos/variables.proto index 1bde11bd0..4bc0c54ec 100644 --- a/angr/protos/variables.proto +++ b/angr/protos/variables.proto @@ -23,8 +23,9 @@ message TemporaryVariable { message ConstantVariable { VariableBase base = 1; uint32 size = 2; - uint64 value = 3; + uint64 value = 3; // the absolute value optional bytes long_value = 4; // for large constants over 64 bits + bool is_negative = 5; } diff --git a/angr/sim_variable.py b/angr/sim_variable.py index f73bdade3..89fa3234b 100644 --- a/angr/sim_variable.py +++ b/angr/sim_variable.py @@ -116,9 +116,12 @@ class SimConstantVariable(SimVariable): __slots__ = ["value"] - def __init__(self, size: int, ident=None, value=None, region=None): + def __init__(self, size: int, *, value: int | float, region: int | None = None, ident=None): super().__init__(ident=ident, region=region, size=size) - self.value = value + is_negative = value < 0 + abs_value = -value if is_negative else value + abs_value = abs_value & ((1 << (size * 8)) - 1) if isinstance(abs_value, int) else abs_value + self.value = -abs_value if is_negative else abs_value def __repr__(self): return f"<{self.region}|const {self.value}>" @@ -148,7 +151,7 @@ class SimConstantVariable(SimVariable): @property def key(self) -> tuple[str | int | None, ...]: - return ("const", self.value, self.size, self.ident) + return "const", self.value, self.size, self.ident @classmethod def _get_cmsg(cls): @@ -158,17 +161,21 @@ class SimConstantVariable(SimVariable): obj = self._get_cmsg() self._set_base(obj) obj.size = self.size + abs_value = self.value if self.value >= 0 else -self.value if self.bits > 64: assert isinstance(self.value, int) # TODO: Handle float - obj.long_value = int.to_bytes(self.value, byteorder="little") + num_bytes = (self.bits + 7) // 8 + obj.long_value = self.value.to_bytes(num_bytes, byteorder="little") else: - obj.value = self.value + obj.value = abs_value + obj.is_negative = self.value is not None and self.value < 0 return obj @classmethod def parse_from_cmessage(cls, cmsg, **kwargs): value = int.from_bytes(cmsg.long_value, byteorder="little") if cmsg.size > 64 else cmsg.value + value = -value if cmsg.is_negative else value obj = cls(cmsg.size, value=value) obj._from_base(cmsg) return obj diff --git a/tests/serialization/test_serialization.py b/tests/serialization/test_serialization.py index bf3d45812..24afc9274 100755 --- a/tests/serialization/test_serialization.py +++ b/tests/serialization/test_serialization.py @@ -12,7 +12,7 @@ import tempfile import unittest import angr -from angr.sim_variable import SimStackVariable +from angr.sim_variable import SimConstantVariable, SimStackVariable from tests.common import bin_location test_location = os.path.join(bin_location, "tests") @@ -135,6 +135,34 @@ class TestSerialization(unittest.TestCase): cmsg = v1.serialize_to_cmessage() assert cmsg.offset == 0x7FFF_DEAD + def test_simconstantvariable_value_overflow(self): + v0 = SimConstantVariable(16, value=0x8000_0000_0000_0000_0000_0001, ident="c_0") + cmsg = v0.serialize_to_cmessage() + assert cmsg.size == 16 + assert cmsg.long_value == (0x8000_0000_0000_0000_0000_0001).to_bytes(16, byteorder="little") + assert cmsg.is_negative is False + + def test_simconstantvariable_value_out_of_range(self): + v1 = SimConstantVariable(8, value=-0x8000_0000_0000_0000, ident="c_1") + cmsg = v1.serialize_to_cmessage() + assert cmsg.size == 8 + assert cmsg.value == 0x8000_0000_0000_0000 + assert cmsg.is_negative is True + + def test_simconstantvariable_value_too_long(self): + v2 = SimConstantVariable(8, value=0x1_FFFF_FFFF_FFFF_FFF0, ident="c_2") + cmsg = v2.serialize_to_cmessage() + assert cmsg.size == 8 + assert cmsg.value == 0xFFFF_FFFF_FFFF_FFF0 + assert cmsg.is_negative is False + + def test_simconstantvariable_negative_value(self): + v1 = SimConstantVariable(8, value=-1, ident="c_1") + cmsg = v1.serialize_to_cmessage() + assert cmsg.size == 8 + assert cmsg.value == 1 + assert cmsg.is_negative is True + if __name__ == "__main__": unittest.main() From d46e56f891994a6c4cb06dca59c9db4c8f65bc04 Mon Sep 17 00:00:00 2001 From: Quintin Kong <71952215+DORA-B@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:23:05 -0400 Subject: [PATCH 079/122] Fix signed division and remainder in the pcode engine (#6739) * Fix signed division and remainder in the pcode engine OpBehaviorIntSdiv and OpBehaviorIntSrem used Claripy's `/` and `%`, which are unsigned bit-vector operations. For negative operands they therefore produced the same results as the unsigned INT_DIV and INT_REM behaviors. INT_SDIV now uses claripy.SDiv (truncation toward zero). INT_SREM is defined as in1 - SDiv(in1, in2) * in2, giving a remainder with the dividend's sign, which matches the p-code semantics documented in the class comments. For 64-bit -5 and 2, INT_SDIV now yields -2 (0xfffffffffffffffe) and INT_SREM yields -1 (0xffffffffffffffff) instead of large unsigned values. The arithmetic behavior test table enables both INT_SDIV and INT_SREM with the matching signed reference expressions, and a new concrete test checks mixed-sign combinations (-5/2, 5/-2, -5/-2, ...) that an unsigned implementation cannot satisfy. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Use claripy.SMod for INT_SREM Per review, INT_SREM uses claripy.SMod directly instead of the equivalent in1 - claripy.SDiv(in1, in2) * in2. Verified identical to a truncated-toward-zero reference over 100k random 64-bit pairs, including the INT_MIN / -1 corner. * Address pcode signed arithmetic review comments * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- angr/engines/pcode/behavior.py | 4 ++-- angr/sim_variable.py | 2 +- tests/engines/pcode/test_emulate.py | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/angr/engines/pcode/behavior.py b/angr/engines/pcode/behavior.py index f4a9dade9..4a32154ba 100644 --- a/angr/engines/pcode/behavior.py +++ b/angr/engines/pcode/behavior.py @@ -394,7 +394,7 @@ class OpBehaviorIntSdiv(OpBehavior): super().__init__(OpCode.INT_SDIV, False) def evaluate_binary(self, size_out: int, size_in: int, in1: BV, in2: BV) -> BV: - return in1 / in2 + return claripy.SDiv(in1, in2) # uintb OpBehaviorIntSdiv::evaluateBinary(int4 size_out,int4 size_in,uintb in1,uintb in2) const # @@ -432,7 +432,7 @@ class OpBehaviorIntSrem(OpBehavior): super().__init__(OpCode.INT_SREM, False) def evaluate_binary(self, size_out: int, size_in: int, in1: BV, in2: BV) -> BV: - return in1 % in2 + return claripy.SMod(in1, in2) # uintb OpBehaviorIntSrem::evaluateBinary(int4 size_out,int4 size_in,uintb in1,uintb in2) const # diff --git a/angr/sim_variable.py b/angr/sim_variable.py index 89fa3234b..0e965a615 100644 --- a/angr/sim_variable.py +++ b/angr/sim_variable.py @@ -116,7 +116,7 @@ class SimConstantVariable(SimVariable): __slots__ = ["value"] - def __init__(self, size: int, *, value: int | float, region: int | None = None, ident=None): + def __init__(self, size: int, *, value: float, region: int | None = None, ident=None): super().__init__(ident=ident, region=region, size=size) is_negative = value < 0 abs_value = -value if is_negative else value diff --git a/tests/engines/pcode/test_emulate.py b/tests/engines/pcode/test_emulate.py index 17a432f76..79535b41e 100644 --- a/tests/engines/pcode/test_emulate.py +++ b/tests/engines/pcode/test_emulate.py @@ -359,8 +359,10 @@ class TestPcodeEmulatorMixin(unittest.TestCase): OpCode.INT_OR: operator.or_, OpCode.INT_REM: operator.mod, OpCode.INT_RIGHT: claripy.LShR, + OpCode.INT_SDIV: claripy.SDiv, OpCode.INT_SLESS: claripy.SLT, OpCode.INT_SLESSEQUAL: claripy.SLE, + OpCode.INT_SREM: claripy.SMod, OpCode.INT_SRIGHT: operator.rshift, OpCode.INT_SUB: operator.sub, OpCode.INT_XOR: operator.xor, @@ -424,8 +426,7 @@ class TestPcodeEmulatorMixin(unittest.TestCase): expected_result = operation(x, y) solver = claripy.Solver() - maybe_true = solver.eval(result == expected_result, 1)[0] - assert solver.is_true(maybe_true) + assert solver.is_true(result == expected_result) def test_arith_binary_ops(self): for opcode in [ @@ -444,9 +445,10 @@ class TestPcodeEmulatorMixin(unittest.TestCase): OpCode.INT_OR, OpCode.INT_REM, OpCode.INT_RIGHT, - # OpCode.INT_SDIV, # FIXME + OpCode.INT_SDIV, OpCode.INT_SLESS, OpCode.INT_SLESSEQUAL, + OpCode.INT_SREM, OpCode.INT_SRIGHT, OpCode.INT_SUB, OpCode.INT_XOR, @@ -597,8 +599,6 @@ class TestPcodeEmulatorMixin(unittest.TestCase): # OpCode.INT_CARRY # OpCode.INT_SBORROW # OpCode.INT_SCARRY - # * OpCode.INT_SDIV - # * OpCode.INT_SREM # ! OpCode.NEW # OpCode.RETURN From 2f891d1d69b8bef78e6f59dc329ddfee3f6ce14e Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Fri, 31 Jul 2026 13:47:32 -0700 Subject: [PATCH 080/122] Update capstone to 5.0.9 (#6740) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5993fbcd6..2aa010864 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "angr-data~=0.1.0", "archinfo==9.3.2.dev0", "cachetools", - "capstone==5.0.6", + "capstone==5.0.9", "cffi>=1.14.0", "claripy==9.3.2.dev0", "cle==9.3.2.dev0", From 49434bc3fc80feca32c0ad77854803e6068e4409 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Fri, 31 Jul 2026 17:05:47 -0700 Subject: [PATCH 081/122] Render truncations to non-C widths as masks instead of casts (#6741) --- .../decompiler/structured_codegen/c.py | 45 +++++++++--------- .../decompiler/test_structured_codegen.py | 47 +++++++++++++++++++ 2 files changed, 70 insertions(+), 22 deletions(-) create mode 100644 tests/analyses/decompiler/test_structured_codegen.py diff --git a/angr/analyses/decompiler/structured_codegen/c.py b/angr/analyses/decompiler/structured_codegen/c.py index 145e7c032..b00b7499f 100644 --- a/angr/analyses/decompiler/structured_codegen/c.py +++ b/angr/analyses/decompiler/structured_codegen/c.py @@ -98,6 +98,16 @@ type RenderResult = tuple[str, PositionMapping, PositionMapping, InstructionMapp INDENT_DELTA = 4 +_CAST_TYPES_BY_BITS: dict[int, type[SimTypeInt | SimTypeChar]] = { + 8: SimTypeChar, + 16: SimTypeShort, + 32: SimTypeInt, + 64: SimTypeLongLong, + 128: SimTypeInt128, + 256: SimTypeInt256, + 512: SimTypeInt512, +} + def qualifies_for_simple_cast(ty1, ty2): # converting ty1 to ty2 - can this happen precisely? @@ -4223,29 +4233,20 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis, Serializab ) def _handle_Expr_Convert(self, expr: Expr.Convert, **kwargs): - # width of converted type is easy - dst_type: SimTypeInt | SimTypeChar - if 512 >= expr.to_bits > 256: - dst_type = SimTypeInt512() - elif 256 >= expr.to_bits > 128: - dst_type = SimTypeInt256() - elif 128 >= expr.to_bits > 64: - dst_type = SimTypeInt128() - elif 64 >= expr.to_bits > 32: - dst_type = SimTypeLongLong() - elif 32 >= expr.to_bits > 16: - dst_type = SimTypeInt() - elif 16 >= expr.to_bits > 8: - dst_type = SimTypeShort() - elif 8 >= expr.to_bits > 1: - dst_type = SimTypeChar() - elif expr.to_bits == 1: - dst_type = SimTypeChar() # FIXME: Add a SimTypeBit? - else: - raise UnsupportedNodeTypeError(f"Unsupported conversion bits {expr.to_bits}.") - - # convert child child = self._handle(expr.operand) + + # Use a mask to represent non-standard size conversions + if expr.to_bits < expr.from_bits and expr.to_bits not in _CAST_TYPES_BY_BITS: + const_type = child.type if child.type is not None else self.default_simtype_from_bits(expr.from_bits, False) + mask = CConstant((1 << expr.to_bits) - 1, const_type, codegen=self, tags=expr.tags) + return CBinaryOp("And", child, mask, codegen=self, tags=expr.tags) + + # Cast to the smallest size that can hold the new value + dst_type_cls = next((cls for bits, cls in _CAST_TYPES_BY_BITS.items() if bits >= expr.to_bits), None) + if dst_type_cls is None or expr.to_bits < 1: + raise UnsupportedNodeTypeError(f"Unsupported conversion bits {expr.to_bits}.") + dst_type: SimTypeInt | SimTypeChar = dst_type_cls() + orig_child_signed = getattr(child.type, "signed", False) # signedness of converted type is hard diff --git a/tests/analyses/decompiler/test_structured_codegen.py b/tests/analyses/decompiler/test_structured_codegen.py new file mode 100644 index 000000000..85f1353bc --- /dev/null +++ b/tests/analyses/decompiler/test_structured_codegen.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use,no-member,protected-access +from __future__ import annotations + +__package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin + +import unittest + +import angr +from angr.ailment import Expr + + +class TestConvertRendering(unittest.TestCase): + """How CStructuredCodeGenerator renders Convert expressions of assorted widths.""" + + @classmethod + def setUpClass(cls): + # any decompilation will do; all we need is a codegen instance to render expressions with + proj = angr.load_shellcode(b"\x31\xc0\xc3", arch="AMD64") # xor eax, eax; ret + cfg = proj.analyses.CFGFast(normalize=True) + cls.codegen = proj.analyses.Decompiler(cfg.functions[0], cfg=cfg).codegen + + def _render(self, from_bits: int, to_bits: int, value: int = 0x1234) -> str: + conv = Expr.Convert(0, from_bits, to_bits, False, Expr.Const(0, value, from_bits)) + return self.codegen._handle(conv).c_repr() + + def test_truncation_to_unrepresentable_width_is_masked(self): + # No C type is 5, 3 or 1 bits wide. A cast would round up to the next real type and keep + # bits the conversion discards, so these have to be spelled as a mask instead. + assert self._render(32, 5) == "4660 & 31" + assert self._render(32, 3) == "4660 & 7" + assert self._render(32, 1) == "4660 & 1" + + def test_truncation_to_representable_width_is_a_cast(self): + assert self._render(32, 8) == "(char)4660" + assert self._render(32, 16) == "(unsigned short)4660" + + def test_widening_is_always_a_cast(self): + # Rounding up only loses information when truncating, so widening keeps casting even to a + # width no C type has. + assert self._render(1, 5, value=1) == "(char)1" + assert self._render(8, 12, value=3) == "(unsigned short)3" + assert self._render(32, 64, value=3) == "(unsigned long long)3" + + +if __name__ == "__main__": + unittest.main() From 0f8e082f9a1b2763345c6937614108a62c93bdeb Mon Sep 17 00:00:00 2001 From: Yibo Liu <35004522+bluesadi@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:41:12 -0700 Subject: [PATCH 082/122] Clinic: recover variable-length arrays (VLAs) (#6634) --- angr/analyses/decompiler/clinic.py | 289 +++++++++++++++++- .../decompiler/structured_codegen/c.py | 25 +- .../structured_codegen/c_serialize.py | 37 ++- .../variables/variable_manager.py | 4 + angr/protos/codegen.proto | 17 +- tests/analyses/decompiler/test_decompiler.py | 30 ++ 6 files changed, 393 insertions(+), 9 deletions(-) diff --git a/angr/analyses/decompiler/clinic.py b/angr/analyses/decompiler/clinic.py index d057a39d5..ffa4703f8 100644 --- a/angr/analyses/decompiler/clinic.py +++ b/angr/analyses/decompiler/clinic.py @@ -8,7 +8,7 @@ import logging from collections import defaultdict, namedtuple from collections.abc import Iterable from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, NamedTuple +from typing import TYPE_CHECKING, Any, NamedTuple, TypeGuard import capstone import networkx @@ -61,6 +61,7 @@ from angr.sim_type import ( SimTypeFunction, SimTypeInt, SimTypeLongLong, + SimTypeNum, SimTypePointer, SimTypeShort, ) @@ -208,6 +209,27 @@ class ComboRegReferenceWalker(AILBlockRewriter): return expr +class _VLABufferBinder(AILBlockViewer): + """Map every virtual-variable use of a recovered VLA buffer to its unified array variable. + + AIL expressions are immutable, so the association is recorded in the clinic's :class:`VariableMap` + (keyed by the vvar's stable ``varid``) rather than on the expression itself. + """ + + def __init__(self, buffer_ids, variable, variable_map): + super().__init__() + self._buffer_ids = buffer_ids + self._variable = variable + self._variable_map = variable_map + + def _handle_VirtualVariable( + self, expr_idx: int, expr: VirtualVariable, stmt_idx: int, stmt: Statement | None, block: Block | None + ): + if expr.varid in self._buffer_ids: + self._variable_map.set_variable(expr, self._variable) + return super()._handle_VirtualVariable(expr_idx, expr, stmt_idx, stmt, block) + + class Clinic(Analysis, Serializable): """ A Clinic deals with AILments: it lifts a function to AIL and runs the decompiler's simplification pipeline on it. @@ -764,6 +786,8 @@ class Clinic(Analysis, Serializable): self.arg_vvars = self._init_arg_vvars if self._init_arg_vvars is not None else {} self.func_args = {arg_vvar for arg_vvar, _ in self.arg_vvars.values()} self._ail_graph = ail_graph + # recovered variable-length arrays: (buffer virtual-variable ids, size/dimension Load) per alloca + self._vla_allocas: list[tuple[frozenset[int], ailment.Expr.Load]] = [] stages = { ClinicStage.MAKE_RETURN_SITES: self._stage_make_return_sites, @@ -865,6 +889,8 @@ class Clinic(Analysis, Serializable): type_hints=self._type_hints, ) self._rewrite_alloca(self._ail_graph) + # recognize & excise GCC's variable-length-array probe idiom while it is still in its clean form + self._excise_vla(self._ail_graph) # Run simplification passes self._update_progress(40.0, text="Running simplifications 1") @@ -1025,6 +1051,10 @@ class Clinic(Analysis, Serializable): self._ail_graph = self._fix_combo_reg_references(self._ail_graph) + # Bind the excised variable-length arrays to named array variables before the dead-code passes run, + # so the leftover address arithmetic and page anchors get cleaned up. + self._bind_vla(self._ail_graph) + # Run simplification passes self._update_progress(85.0, text="Running simplifications 4") self._ail_graph = self._run_simplification_passes( @@ -1405,6 +1435,7 @@ class Clinic(Analysis, Serializable): ) regs |= self._find_regs_compared_against_sp(self._func_graph) + regs |= self._find_regs_saving_sp(self._func_graph) spt = self.project.analyses.StackPointerTracker( self.function, @@ -3957,6 +3988,37 @@ class Clinic(Analysis, Serializable): return extra_regs + def _find_regs_saving_sp(self, func_graph): + # Functions with a dynamic stack allocation (VLA / alloca) save the stack pointer into a + # callee-saved register on entry (``mov reg, rsp``) and restore it on exit (``mov rsp, reg``). + # Unless that register is tracked, the tracker cannot resolve sp after the restore and reports + # the whole function as inconsistent, leaking raw sp virtual variables into the output. Track + # any register that both receives a copy of sp and is later used to restore it. + # TODO: Implement this function for architectures beyond amd64 + if self.project.arch.name != "AMD64": + return set() + + saved_to = set() + restored_from = set() + for node in func_graph.nodes: + block = self.project.factory.block(node.addr, size=node.size).capstone + for insn in block.insns: + if insn.mnemonic != "mov" or len(insn.operands) != 2: + continue + dst, src = insn.operands + if dst.type != capstone.x86.X86_OP_REG or src.type != capstone.x86.X86_OP_REG: + continue + if src.reg == capstone.x86.X86_REG_RSP and dst.reg != capstone.x86.X86_REG_RBP: + saved_to.add(insn.reg_name(dst.reg)) + elif dst.reg == capstone.x86.X86_REG_RSP: + restored_from.add(insn.reg_name(src.reg)) + + return { + self.project.arch.registers[reg_name][0] + for reg_name in saved_to & restored_from + if reg_name in self.project.arch.registers + } + def _rewrite_rust_probestack_call(self, ail_graph): for node in ail_graph: if not node.statements or ail_graph.out_degree[node] != 1: @@ -4118,6 +4180,231 @@ class Clinic(Analysis, Serializable): for succ in succs: ail_graph.add_edge(new_node, succ) + # + # Variable-length array (VLA) recovery + # + + @staticmethod + def _vla_is_noop_touch(stmt) -> bool: + # a stack-probe page touch writes back exactly the byte it read: STORE(a, LOAD(a')) with a ~ a' + return ( + isinstance(stmt, ailment.Stmt.Store) + and isinstance(stmt.data, ailment.Expr.Load) + and stmt.addr.likes(stmt.data.addr) + ) + + @staticmethod + def _vla_is_stack_ref(expr) -> bool: + # ``&stack_slot`` (post variable-recovery) or a raw StackBaseOffset + if isinstance(expr, ailment.Expr.StackBaseOffset): + return True + return ( + isinstance(expr, ailment.Expr.UnaryOp) + and expr.op == "Reference" + and isinstance(expr.operand, ailment.Expr.VirtualVariable) + and expr.operand.was_stack + ) + + @staticmethod + def _vla_find_size_source(expr, defs): + # walk a rounded allocation-size expression back to the innermost memory Load — the source + # dimension (e.g. the ``e->bs`` field load feeding ``((bs + 15) >> 4) * 16``). Returns a tuple of + # (the id of the virtual variable whose definition yields that Load, the Load itself). AIL + # expressions are re-wrapped by intermediate passes, so we track the owning varid during the walk + # rather than matching the Load object by identity afterwards. + seen = set() + # (sub-expression, id of the vvar it was reached through) + stack: list[tuple[Any, int | None]] = [(expr, None)] + while stack: + e, owner = stack.pop() + if isinstance(e, ailment.Expr.Load): + return owner, e + if isinstance(e, ailment.Expr.VirtualVariable): + if e.varid in defs and e.varid not in seen: + seen.add(e.varid) + stack.append((defs[e.varid], e.varid)) + elif isinstance(e, (ailment.Expr.Convert, ailment.Expr.UnaryOp)): + stack.append((e.operand, owner)) + elif isinstance(e, ailment.Expr.BinaryOp): + for operand in e.operands: + stack.append((operand, owner)) + return None, None + + def _is_sp_vvar(self, vv) -> TypeGuard[ailment.Expr.VirtualVariable]: + return ( + isinstance(vv, ailment.Expr.VirtualVariable) and vv.was_reg and vv.reg_offset == self.project.arch.sp_offset + ) + + def _excise_vla(self, ail_graph) -> None: + """ + Detect GCC's inline stack-probe alloca idiom and excise the probe machinery. + + GCC lowers ``uint8_t blk[e->bs];`` to: align rsp (``and rsp, ~0xfff``), a page-touch probe loop, + and a remainder ``sub rsp, round16(bs)`` — the resulting stack-pointer value is the buffer. angr + would otherwise leak that value as raw stack-pointer virtual variables (``vvar_*{r48}``) and render + the page probe as ``*(p) = *(p)`` no-ops. + + This runs at ``_rewrite_alloca`` time (before SSA-level-1 / variable recovery), while the idiom is + still in its clean, un-simplified form: the probe self-loop still carries its page touch and the + buffer base is a plain ``StackBaseOffset - `` subtraction. We remove the probe loop and every + no-op touch, and record the buffer virtual-variable ids and the size value's id for ``_bind_vla`` to + turn into a named variable-length array once variable recovery has run. + """ + if self.project.arch.name != "AMD64": + return + + # 1. locate the probe self-loop: a self-edge block whose body contains a no-op page touch + probe_node = None + for node in ail_graph: + if node in ail_graph.successors(node) and any(self._vla_is_noop_touch(s) for s in node.statements): + probe_node = node + break + if probe_node is None: + return + + # 2. collect every register-SSA definition, then find the alloca base: an SP-category vvar + # defined as `` - `` (the aligned buffer pointer) + defs = { + stmt.dst.varid: stmt.src + for node in ail_graph + for stmt in node.statements + if isinstance(stmt, ailment.Stmt.Assignment) and isinstance(stmt.dst, ailment.Expr.VirtualVariable) + } + base_varid = None + size_load = None + for node in ail_graph: + for stmt in node.statements: + if ( + isinstance(stmt, ailment.Stmt.Assignment) + and self._is_sp_vvar(stmt.dst) + and isinstance(stmt.src, ailment.Expr.BinaryOp) + and stmt.src.op == "Sub" + and self._vla_is_stack_ref(stmt.src.operands[0]) + and not isinstance(stmt.src.operands[1], ailment.Expr.Const) + ): + base_varid = stmt.dst.varid + _, size_load = self._vla_find_size_source(stmt.src.operands[1], defs) + break + if base_varid is not None: + break + if base_varid is None or size_load is None: + return + + # 3. grow the buffer-pointer set: the base plus any SP vvar defined as ``buffer ± const`` + buffer_ids = {base_varid} + changed = True + while changed: + changed = False + for node in ail_graph: + for stmt in node.statements: + if not ( + isinstance(stmt, ailment.Stmt.Assignment) + and self._is_sp_vvar(stmt.dst) + and stmt.dst.varid not in buffer_ids + ): + continue + src = stmt.src + ref = None + if isinstance(src, ailment.Expr.VirtualVariable): + ref = src.varid + elif ( + isinstance(src, ailment.Expr.BinaryOp) + and src.op in ("Add", "Sub") + and isinstance(src.operands[0], ailment.Expr.VirtualVariable) + and isinstance(src.operands[1], ailment.Expr.Const) + ): + ref = src.operands[0].varid + if ref in buffer_ids: + buffer_ids.add(stmt.dst.varid) + changed = True + + # 4. remove every no-op page touch (the probe loop's and the remainder's) + for node in ail_graph: + node.statements = [s for s in node.statements if not self._vla_is_noop_touch(s)] + + # 5. remove the (now no-op) probe self-loop: drop its self-edge, empty it, and let + # remove_empty_nodes splice it out and repair the guard's conditional jump + probe_node.statements = [] + if ail_graph.has_edge(probe_node, probe_node): + ail_graph.remove_edge(probe_node, probe_node) + self.remove_empty_nodes(ail_graph) + + # splicing the loop out collapses the entry guard into ``if (c) goto X else goto X``; turn any such + # redundant conditional jump into a plain goto so structuring does not choke on it + for node in ail_graph: + if node.statements and isinstance(node.statements[-1], ailment.Stmt.ConditionalJump): + cj = node.statements[-1] + if ( + isinstance(cj.true_target, ailment.Expr.Const) + and isinstance(cj.false_target, ailment.Expr.Const) + and cj.true_target.value == cj.false_target.value + ): + node.statements[-1] = ailment.Stmt.Jump( + cj.idx, cj.true_target, target_idx=cj.true_target_idx, **cj.tags + ) + + # record for _bind_vla (post variable-recovery). The size Load is stored as-is: codegen resolves its + # inner pointer through the varid-keyed variable map, so the snapshot renders correctly (e.g. e->bs) + # regardless of the intervening simplifications. + self._vla_allocas.append((frozenset(buffer_ids), size_load)) + + def _bind_vla(self, ail_graph) -> None: + """ + Turn each excised alloca (recorded by :meth:`_excise_vla`) into a named variable-length array. + + Runs after variable recovery. For every recorded buffer, mint one array variable, delete the + leftover buffer-pointer address arithmetic, map every surviving use of the buffer virtual variables + to it, and attach its runtime dimension (the recorded size ``Load``, e.g. ``e->bs``) so codegen + renders ``uint8_t [e->bs]``. + """ + if not self._vla_allocas: + return + + varman = self.kb.dec_variables[self.function.addr] + reg_base = 0x100000 + + for buffer_ids, size_load in self._vla_allocas: + base_varid = min(buffer_ids) + # leave the buffer unnamed so it is auto-named like any other local (the binary carries no name + # for a stack VLA); the ident stays ``vvar_`` so the naming pass can parse it below + buf = SimRegisterVariable( + reg_base + base_varid, self.project.arch.bytes, ident=f"vvar_{reg_base + base_varid}" + ) + varman.add_variable("register", reg_base + base_varid, buf) + varman.set_unified_variable(buf, buf) + varman.set_variable_type( + buf, + SimTypeArray(SimTypeNum(8, signed=False), length=None).with_arch(self.project.arch), + mark_manual=True, + ) + varman.array_length_exprs[buf] = size_load + + # delete the leftover buffer-pointer address arithmetic (buf = ; buf -= 16; ...) + for node in ail_graph: + node.statements = [ + s + for s in node.statements + if not ( + isinstance(s, ailment.Stmt.Assignment) + and isinstance(s.dst, ailment.Expr.VirtualVariable) + and s.dst.varid in buffer_ids + ) + ] + + # map every surviving buffer-vvar use to the array variable so codegen renders it as the array + binder = _VLABufferBinder(buffer_ids, buf, self.variable_map) + for node in ail_graph: + binder.walk(node) + + # the default-naming pass already ran during variable recovery, before these buffers existed; re-run + # it (reset=False keeps every existing name) so the new buffers get the next ``v`` default names + varman.assign_unified_variable_names( + labels=self.kb.labels, + arg_names=list(self.function.prototype.arg_names) if self.function.prototype else None, + reset=False, + func_blocks=list(ail_graph), + ) + def _collect_callsite_prototypes(self) -> dict[int, list[tuple[list[SimType | None], SimType | None]]]: if not self._variables_recovered: return {} diff --git a/angr/analyses/decompiler/structured_codegen/c.py b/angr/analyses/decompiler/structured_codegen/c.py index b00b7499f..b70ced6d1 100644 --- a/angr/analyses/decompiler/structured_codegen/c.py +++ b/angr/analyses/decompiler/structured_codegen/c.py @@ -601,11 +601,24 @@ class CFunction(CConstruct): # pylint:disable=abstract-method key=lambda x, ct=count: (isinstance(x, (SimTypeChar, SimTypeInt, SimTypeFloat)), ct[x], repr(x)), ) + vla_dim = self.codegen._array_length_cexprs.get(variable) + for i, var_type in enumerate(vartypes): if i == 0: - yield from type_to_c_repr_chunks(var_type, name=name, name_type=cvariable) + if vla_dim is not None and isinstance(var_type, SimTypeArray) and var_type.length is None: + # variable-length array: render ``elem_type name[dim]`` with the runtime dimension + yield from type_to_c_repr_chunks(var_type.elem_type, name=name, name_type=cvariable) + yield "[", None + yield from vla_dim.c_repr_chunks() + yield "]", None + else: + yield from type_to_c_repr_chunks(var_type, name=name, name_type=cvariable) yield "; // ", None - yield variable.loc_repr(self.codegen.project.arch), None + if vla_dim is not None: + # the buffer lives at a synthesized register slot; show its origin instead + yield "alloca", None + else: + yield variable.loc_repr(self.codegen.project.arch), None # multiple types else: if i == 1: @@ -2942,6 +2955,7 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis, Serializab self.map_addr_to_label: dict[tuple[int, int | None], CLabel] = {} self.cfunc: CFunction | None = None self.cexterns: set[CVariable] | None = None + self._array_length_cexprs: dict[SimVariable, CExpression] = {} self.display_notes = display_notes self.max_str_len = max_str_len self.prettify_thiscall = prettify_thiscall @@ -2989,6 +3003,13 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis, Serializab self.reset_ident_counters() obj = self._handle(self._sequence) + # render the runtime dimension of every variable-length array (e.g. ``blk[e->bs]``) through the + # regular expression handler, so the field name/type match the rest of the output + self._array_length_cexprs = { + var: self._handle(dim_expr) + for var, dim_expr in self.kb.dec_variables[self._func.addr].array_length_exprs.items() + } + self.cnode2ailexpr = {v: k[0] for k, v in self.ailexpr2cnode.items()} self.cfunc = CFunction( diff --git a/angr/analyses/decompiler/structured_codegen/c_serialize.py b/angr/analyses/decompiler/structured_codegen/c_serialize.py index 485e2158d..d5e848524 100644 --- a/angr/analyses/decompiler/structured_codegen/c_serialize.py +++ b/angr/analyses/decompiler/structured_codegen/c_serialize.py @@ -505,10 +505,13 @@ def _parse_const_formats(entries): # Display-option attribute names round-tripped on Codegen, derived from the descriptor so the proto stays the # single source of truth. Display options are the trailing field block in the Codegen message (see codegen.proto): -# every field from ``indent`` onward is a display option. +# every optional scalar field from ``indent`` onward is a display option (repeated and message-typed fields with +# higher numbers, e.g. vla_dims, are data fields, not display options). _DISPLAY_OPTION_FIELD_FIRST = codegen_pb2.Codegen.DESCRIPTOR.fields_by_name["indent"].number _DISPLAY_OPTION_ATTRS = tuple( - f.name for f in codegen_pb2.Codegen.DESCRIPTOR.fields if f.number >= _DISPLAY_OPTION_FIELD_FIRST + f.name + for f in codegen_pb2.Codegen.DESCRIPTOR.fields + if f.number >= _DISPLAY_OPTION_FIELD_FIRST and not f.is_repeated and f.message_type is None ) # Constructor defaults for the display options, so a deserialized codegen has the same values a fresh one would for # options that were not serialized (an option whose value is None, e.g. max_str_len, is skipped on serialize). @@ -548,6 +551,12 @@ def serialize_codegen(codegen) -> codegen_pb2.Codegen: for v in codegen.cexterns: msg.cexterns_ids.append(ctx.serialize(v)) + # VLA runtime dimensions (SimVariable -> CExpression), so ``uint8_t [];`` re-renders after reload. + for var, dim_cexpr in (getattr(codegen, "_array_length_cexprs", None) or {}).items(): + entry = msg.vla_dims.add() + entry.simvar_ref = ctx.intern_simvar(var) + entry.node_id = ctx.serialize(dim_cexpr) + if codegen.expr_comments: for k, v in codegen.expr_comments.items(): msg.expr_comments[k] = v @@ -640,6 +649,13 @@ def parse_codegen(msg, *, project=None, kb=None, func=None): cg.cexterns = {ctx.resolve(i) for i in msg.cexterns_ids} if msg.cexterns_ids else None + # VLA runtime dimensions; keys are value-equal to the unified variables used at render time. + cg._array_length_cexprs = {} + for entry in msg.vla_dims: + var = ctx.resolve_simvar(entry.simvar_ref) + if var is not None: + cg._array_length_cexprs[var] = ctx.resolve(entry.node_id) + # Display options: those present in the cmessage override the constructor defaults; options that were not # serialized (a None value, e.g. max_str_len) fall back to the constructor default so the attribute exists. for attr in _DISPLAY_OPTION_ATTRS: @@ -1117,12 +1133,21 @@ def _parse_cvarfield(pb, ctx): # ----------------------------------------------------------------------------------------------------------------- # CConstant (heterogeneous value + reference_values). # ----------------------------------------------------------------------------------------------------------------- +def _set_const_int(body, v: int) -> None: + """Store an int into a message with ``int64 int_value`` / ``bytes big_int_value`` oneof arms. Values outside + the int64 range (e.g. the u64 page mask 0xfffffffffffff000) go into big_int_value as signed little-endian.""" + if -(1 << 63) <= v < (1 << 63): + body.int_value = v + else: + body.big_int_value = v.to_bytes((v.bit_length() + 8) // 8, "little", signed=True) + + def _ser_cconst(node, pb, ctx): body = pb.cconst if isinstance(node.value, bool): body.int_value = int(node.value) elif isinstance(node.value, int): - body.int_value = node.value + _set_const_int(body, node.value) elif isinstance(node.value, float): body.float_value = node.value elif isinstance(node.value, str): @@ -1135,7 +1160,7 @@ def _ser_cconst(node, pb, ctx): if isinstance(val, bool): entry.int_value = int(val) elif isinstance(val, int): - entry.int_value = val + _set_const_int(entry, val) elif isinstance(val, bytes): entry.raw_bytes = val elif isinstance(val, str): @@ -1151,6 +1176,8 @@ def _parse_cconst(pb, ctx): which = body.WhichOneof("value") if which == "int_value": obj.value = body.int_value + elif which == "big_int_value": + obj.value = int.from_bytes(body.big_int_value, "little", signed=True) elif which == "float_value": obj.value = body.float_value elif which == "str_value": @@ -1165,6 +1192,8 @@ def _parse_cconst(pb, ctx): w = entry.WhichOneof("value") if w == "int_value": refs[key] = entry.int_value + elif w == "big_int_value": + refs[key] = int.from_bytes(entry.big_int_value, "little", signed=True) elif w == "raw_bytes": refs[key] = entry.raw_bytes elif w == "str_value": diff --git a/angr/knowledge_plugins/variables/variable_manager.py b/angr/knowledge_plugins/variables/variable_manager.py index 68eaa5be3..3061f8c57 100644 --- a/angr/knowledge_plugins/variables/variable_manager.py +++ b/angr/knowledge_plugins/variables/variable_manager.py @@ -137,6 +137,10 @@ class VariableManagerInternal(Serializable): self.variable_to_types: dict[SimVariable, SimType] = {} self.variables_with_manual_types = set() + # dimension expressions for variable-length arrays (VLAs). Maps a (unified) variable typed as a + # length-less SimTypeArray to the ailment expression giving its runtime length, e.g. ``e->bs``. + self.array_length_exprs: dict[SimVariable, ailment.Expression] = {} + # optimization self._variables_without_writes = set() diff --git a/angr/protos/codegen.proto b/angr/protos/codegen.proto index edcd659b2..fb84954eb 100644 --- a/angr/protos/codegen.proto +++ b/angr/protos/codegen.proto @@ -299,6 +299,7 @@ message CConstantReferenceValue { bytes raw_bytes = 3; string str_value = 4; bytes memory_data = 5; // serialized MemoryData via its existing Serializable + bytes big_int_value = 6; // ints outside int64 range: two's-complement little-endian, signed } } message CConstantMsg { @@ -306,6 +307,7 @@ message CConstantMsg { int64 int_value = 1; double float_value = 2; string str_value = 3; + bytes big_int_value = 6; // ints outside int64 range: two's-complement little-endian, signed } uint32 type_ref = 4; repeated CConstantReferenceValue reference_values = 5; @@ -475,9 +477,15 @@ message Codegen { // Out-of-line per-node display config; a node only appears here if it deviates from the defaults. NodeConfigMsg node_config = 17; + // Runtime dimensions of variable-length arrays: SimVariable (simvar_pool ref) -> CExpression (node_id), + // rendered as ``uint8_t [];``. Field number 39 lives after the display-option block below, but + // repeated/message fields are excluded from the display-option scan, so this is safe. + repeated VlaDimEntry vla_dims = 39; + // Display options (round-tripped but not strictly part of the result). This block must stay last: - // c_serialize treats every field from ``indent`` onward as a display option, so a new display option only - // needs an optional scalar field appended here whose name matches the attribute on CStructuredCodeGenerator. + // c_serialize treats every optional scalar field from ``indent`` onward as a display option, so a new + // display option only needs an optional scalar field appended here whose name matches the attribute on + // CStructuredCodeGenerator. (Repeated and message-typed fields are not considered display options.) optional int32 indent = 18; optional bool show_casts = 19; optional bool comment_gotos = 20; @@ -501,6 +509,11 @@ message Codegen { optional int32 max_str_len = 38; } +message VlaDimEntry { + uint32 simvar_ref = 1; // index+1 into Codegen.simvar_pool (the VLA buffer variable) + uint32 node_id = 2; // node_id of the CExpression rendering the runtime dimension +} + message ConstFormatEntry { int64 ident_ins_addr = 1; int32 ident_kind = 2; diff --git a/tests/analyses/decompiler/test_decompiler.py b/tests/analyses/decompiler/test_decompiler.py index fb15400de..3b0d2ee97 100755 --- a/tests/analyses/decompiler/test_decompiler.py +++ b/tests/analyses/decompiler/test_decompiler.py @@ -6114,6 +6114,36 @@ class TestDecompiler(unittest.TestCase): assert isinstance(arg0, ty_cls), f"{name}: expected {ty_cls.__name__}, got {arg0!r}" assert arg0.signed is signed, f"{name}: expected signed={signed}, got signed={arg0.signed}" + def test_decompiling_vla_ffs(self, decompiler_options=None): + # sub_4051d0 declares a variable-length array `uint8_t blk[e->bs];`, which GCC lowers to an inline + # stack-probe alloca (align rsp, a page-touch probe loop, and a variable remainder subtraction). + # angr should recover it as an array variable with a runtime dimension instead of leaking raw + # stack-pointer virtual variables (vvar_*{r48}) and `*(p) = *(p)` no-op page touches. + bin_path = os.path.join(test_location, "x86_64", "decompiler", "ffs") + p = angr.Project(bin_path, auto_load_libs=False) + + cfg = p.analyses[CFGFast].prep()(normalize=True) + p.analyses[CompleteCallingConventionsAnalysis].prep()() + f = cfg.functions[0x4051D0] + dec = p.analyses[Decompiler].prep(fail_fast=True)(f, cfg=cfg.model, options=decompiler_options) + assert dec.codegen is not None, f"Failed to decompile function {f!r}." + print_decompilation_result(dec) + + text = dec.codegen.text + assert text is not None + # the VLA is recovered as an array declaration with a runtime dimension, tagged `// alloca`. + # the dimension may itself contain brackets or parens (e.g. `(int)v8[4]`), so match to end of line. + m = re.search(r"uint8_t (\w+)\[(.+)\];\s*// alloca", text) + assert m is not None, "expected a recovered VLA declaration `uint8_t []; // alloca`" + name = m.group(1) + assert m.group(2).strip(), "expected a non-empty runtime dimension" + # no raw stack-pointer virtual variables leak into the output + assert "{r48" not in text and "vvar_" not in text + # the page-probe loop and its `*(p) = *(p)` no-op touches are gone + assert re.search(r"do\s*\{\s*\}\s*while", text) is None + # the pread call addresses the recovered buffer by name (earlier args may contain parens) + assert re.search(rf"pread\(.*?\b{name}\b", text) is not None + if __name__ == "__main__": unittest.main() From 15757fb2daee31e0caf7e05c02e3b908483f3eb3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 09:37:54 -0700 Subject: [PATCH 083/122] ci: bump taiki-e/install-action from 2.85.2 to 2.85.5 (#6753) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.85.2 to 2.85.5. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/41049aa56687c35e0afa74eed4f09cec4f9afabf...6a1bd70eaac3c8bdf093356838d7ee09fda951cf) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.85.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index e6d0bf6b4..78a98bb8d 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -99,7 +99,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1 - - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2 + - uses: taiki-e/install-action@6a1bd70eaac3c8bdf093356838d7ee09fda951cf # v2 with: tool: cargo-llvm-cov - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v5 From f9c789ff6a535e94526b49a5a216abda866538d5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:15:42 -0700 Subject: [PATCH 084/122] [pre-commit.ci] pre-commit autoupdate (#6754) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.16.0 → v0.16.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.16.0...v0.16.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 311fd5056..554209485 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,7 +62,7 @@ repos: args: [--py310-plus] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.0 + rev: v0.16.1 hooks: - id: ruff-check args: [--fix, --exit-non-zero-on-fix] From f71b07cebf1876ffe1571a4f26f9867fffadb16c Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Mon, 3 Aug 2026 21:11:36 -0700 Subject: [PATCH 085/122] CodeGen: Skip parenthesis on binops when representing as an unop (#6755) --- angr/analyses/decompiler/structured_codegen/c.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/angr/analyses/decompiler/structured_codegen/c.py b/angr/analyses/decompiler/structured_codegen/c.py index b70ced6d1..d41ef8a10 100644 --- a/angr/analyses/decompiler/structured_codegen/c.py +++ b/angr/analyses/decompiler/structured_codegen/c.py @@ -2226,7 +2226,7 @@ class CBinaryOp(CExpression): elif self.op == "CmpNE": skip_op_and_rhs = True # lhs - if isinstance(self.lhs, CBinaryOp) and self.op_precedence > self.lhs.op_precedence: + if isinstance(self.lhs, CBinaryOp) and self.op_precedence > self.lhs.op_precedence and not skip_op_and_rhs: paren = CClosingObject("(") yield "(", paren yield from self._try_c_repr_chunks(self.lhs) From 551df09fc59488aaa34106823d49b6bd06a511d6 Mon Sep 17 00:00:00 2001 From: Fish Date: Mon, 3 Aug 2026 22:12:47 -0700 Subject: [PATCH 086/122] Decompiler: Fix duplicate struct defs for pre-defined structs. (#6756) --- .../decompiler/structured_codegen/c.py | 96 +++++++++++++++++-- angr/analyses/typehoon/translator.py | 28 +++++- .../decompiler/test_decompiler_types.py | 34 ++++++- tests/analyses/test_typehoon.py | 41 ++++++++ 4 files changed, 186 insertions(+), 13 deletions(-) diff --git a/angr/analyses/decompiler/structured_codegen/c.py b/angr/analyses/decompiler/structured_codegen/c.py index d41ef8a10..ccdf2b689 100644 --- a/angr/analyses/decompiler/structured_codegen/c.py +++ b/angr/analyses/decompiler/structured_codegen/c.py @@ -58,6 +58,7 @@ from angr.sim_type import ( SimTypeReg, SimTypeShort, SimTypeWideChar, + SimUnion, TypeRef, ) from angr.sim_variable import ( @@ -243,6 +244,53 @@ def cextern_sort_key(cextern) -> tuple: return (1, str(addr) if addr is not None else "") +def _iter_struct_union_member_types(ty): + """ + Yield the member types of a struct or a union, flattening nested unions. + """ + members = ty.members if isinstance(ty, SimUnion) else ty.fields + for member in members.values(): + member = unpack_typeref(member) + if isinstance(member, SimUnion): + yield from _iter_struct_union_member_types(member) + else: + yield member + + +def _is_anonymous_struct_or_union(ty) -> bool: + """ + Returns True if ``ty`` is an anonymous struct or union. + """ + if isinstance(ty, SimStruct): + return bool(ty.anonymous) or ty.name == "" + return isinstance(ty, SimUnion) and ty.name == "" + + +def _anonymous_struct_union_to_c_repr_chunks(ty, name, name_type, indent_str: str, indent_delta: int): + """ + Render an anonymous struct or union inline, as ``struct { ... } name``. + """ + yield indent_str, None + yield ("union {\n" if isinstance(ty, SimUnion) else "struct {\n"), None + + new_indent_str = (" " * indent_delta) + indent_str + members = ty.members if isinstance(ty, SimUnion) else ty.fields + for k, v in members.items(): + yield from type_to_c_repr_chunks( + v, + name=k, + name_type=CStructFieldNameDef(k), + full=False, + indent_str=new_indent_str, + indent_delta=indent_delta, + ) + yield ";\n", None + + yield indent_str, None + yield "} ", None + yield name, name_type + + def type_to_c_repr_chunks( ty: SimType, name=None, name_type=None, full=False, indent_str="", indent_delta: int = INDENT_DELTA ): @@ -251,7 +299,12 @@ def type_to_c_repr_chunks( :param indent_delta: Number of space characters used to indent each struct field one level deeper. """ - if isinstance(ty, SimStruct): + if not full and name is not None and _is_anonymous_struct_or_union(ty): + # anonymous structs and unions must be output inline + yield from _anonymous_struct_union_to_c_repr_chunks( + ty, name, name_type, indent_str=indent_str, indent_delta=indent_delta + ) + elif isinstance(ty, SimStruct): if full: # struct def preamble yield indent_str, None @@ -266,8 +319,14 @@ def type_to_c_repr_chunks( # fields should be indented new_indent_str = (" " * indent_delta) + indent_str for k, v in ty.fields.items(): - yield new_indent_str, None - yield from type_to_c_repr_chunks(v, name=k, name_type=CStructFieldNameDef(k), full=False, indent_str="") + yield from type_to_c_repr_chunks( + v, + name=k, + name_type=CStructFieldNameDef(k), + full=False, + indent_str=new_indent_str, + indent_delta=indent_delta, + ) yield ";\n", None # struct def postamble @@ -329,6 +388,9 @@ def _recursively_collect_referenced_structs(ty, out: dict[int, SimStruct], _seen out[id(ty)] = ty for ftype in ty.fields.values(): _recursively_collect_referenced_structs(ftype, out, _seen=_seen) + elif isinstance(ty, SimUnion): + for mtype in ty.members.values(): + _recursively_collect_referenced_structs(mtype, out, _seen=_seen) elif isinstance(ty, SimTypePointer): _recursively_collect_referenced_structs(ty.pts_to, out, _seen=_seen) elif isinstance(ty, (SimTypeArray, SimTypeFixedSizeArray)): @@ -691,7 +753,7 @@ class CFunction(CConstruct): # pylint:disable=abstract-method for ty in local_types: if isinstance(ty, SimStruct): name_to_structtypes[ty.name] = ty - for field in ty.fields.values(): + for field in _iter_struct_union_member_types(ty): if isinstance(field, SimTypePointer): if isinstance(field.pts_to, (SimTypeArray, SimTypeFixedSizeArray)): field = field.pts_to.elem_type @@ -719,12 +781,30 @@ class CFunction(CConstruct): # pylint:disable=abstract-method ) return (type_layout_key(ty), tiebreak) + emitted_struct_names: set[str] = set() for ty in sorted(local_types, key=_local_type_sort_key): - # drop unreferenced structs - if isinstance(ty, SimStruct) and ty.name in referenced_struct_names: - yield from type_to_c_repr_chunks( - ty, full=True, indent_str=indent_str, indent_delta=self.codegen.indent_delta + # drop unreferenced structs and anonymous ones + if ( + not isinstance(ty, SimStruct) + or _is_anonymous_struct_or_union(ty) + or ty.name not in referenced_struct_names + ): + continue + if ty.name in emitted_struct_names: + # multiple definitions share a name, which is probably because: + # - we incorrectly inferred types of fields of a struct with a library definition; + # - multiple types exist under the same name (from different libraries). + # we will fix them when encountering these cases. + l.warning( + "Multiple definitions of struct %s in function %s. Only the first one is emitted.", + ty.name, + self.name, ) + continue + emitted_struct_names.add(ty.name) + yield from type_to_c_repr_chunks( + ty, full=True, indent_str=indent_str, indent_delta=self.codegen.indent_delta + ) if self.codegen.show_externs and self.codegen.cexterns: # Emit struct definitions for types used by externs diff --git a/angr/analyses/typehoon/translator.py b/angr/analyses/typehoon/translator.py index 3c26b3af9..b6e1cf2d2 100644 --- a/angr/analyses/typehoon/translator.py +++ b/angr/analyses/typehoon/translator.py @@ -38,6 +38,7 @@ class TypeTranslator: "_struct_def_ctr", "_struct_sig_cache", "arch", + "known_structs", "memo", "named_struct_id_counter", "struct_name_to_idx", @@ -61,6 +62,8 @@ class TypeTranslator: self.memo = {} self.named_struct_id_counter = count(133337) self.struct_name_to_idx = {} + # definitions of known structs (library types or user-defined types), keyed by name + self.known_structs: dict[str, sim_type.SimStruct] = {} # will be updated every time .tc2simtype() is called self._has_nonexistent_ref = False @@ -138,6 +141,13 @@ class TypeTranslator: if tc in self.structs: return self.structs[tc] + if tc.name is not None: + known = self.known_structs.get(tc.name) + if known is not None: + # do not re-derive the fields of a known struct + self.structs[tc] = known + return known + name = tc.name or self.struct_name() if tc.is_cppclass: @@ -332,7 +342,8 @@ class TypeTranslator: def _translate_SimStruct(self, st: sim_type.SimStruct) -> typeconsts.Struct | typeconsts.BottomType: if st in self.memo: - return typeconsts.BottomType() + # a recursive reference: point back at the struct that is being translated + return self.memo[st] struct_idx = {} if st.name: @@ -342,16 +353,18 @@ class TypeTranslator: obj = typeconsts.Struct(fields={}, name=st.name, **struct_idx) self.memo[st] = obj + self._remember_known_struct(st) fields = {} field_names = {} offsets = st.offsets for field_name, simtype in st.fields.items(): if field_name not in offsets: + del self.memo[st] return typeconsts.BottomType() offset = offsets[field_name] fields[offset] = self._simtype2tc(simtype) - field_names[offsets[field_name]] = field_name + field_names[offset] = field_name obj.fields = fields obj.field_names = field_names del self.memo[st] @@ -360,7 +373,8 @@ class TypeTranslator: def _translate_SimCppClass(self, st: sim_type.SimCppClass) -> typeconsts.Struct | typeconsts.BottomType: if st in self.memo: - return typeconsts.BottomType() + # a recursive reference: point back at the class that is being translated + return self.memo[st] struct_idx = {} if st.name: @@ -370,22 +384,28 @@ class TypeTranslator: obj = typeconsts.Struct(fields={}, name=st.name, is_cppclass=True, **struct_idx) self.memo[st] = obj + self._remember_known_struct(st) fields = {} field_names = {} offsets = st.offsets for field_name, simtype in st.fields.items(): if field_name not in offsets: + del self.memo[st] return typeconsts.BottomType() offset = offsets[field_name] fields[offset] = self._simtype2tc(simtype) - field_names[offsets[field_name]] = field_name + field_names[offset] = field_name obj.fields = fields obj.field_names = field_names del self.memo[st] return obj + def _remember_known_struct(self, st: sim_type.SimStruct) -> None: + if st.name and st.name != "" and not st.anonymous: + self.known_structs.setdefault(st.name, st) + def _translate_SimTypeArray(self, st: sim_type.SimTypeArray) -> typeconsts.Array: elem_type = self._simtype2tc(st.elem_type) return typeconsts.Array(elem_type, count=st.length, name=st.label) diff --git a/tests/analyses/decompiler/test_decompiler_types.py b/tests/analyses/decompiler/test_decompiler_types.py index aaac59c93..3fe7ced28 100644 --- a/tests/analyses/decompiler/test_decompiler_types.py +++ b/tests/analyses/decompiler/test_decompiler_types.py @@ -6,9 +6,11 @@ __package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redef import os import re import unittest +from collections import OrderedDict import angr -from angr.sim_type import SimTypeArray, SimTypeChar, SimTypeInt +from angr import default_cc +from angr.sim_type import SimStruct, SimTypeArray, SimTypeChar, SimTypeFunction, SimTypeInt, SimTypePointer from tests.common import WORKER, bin_location, print_decompilation_result test_location = os.path.join(bin_location, "tests") @@ -131,6 +133,36 @@ class TestDecompilerTypes(unittest.TestCase): print_decompilation_result(new_dec) assert f"int {var2.name};" in new_dec.codegen.text + def test_mutually_recursive_known_structs_are_defined_once(self): + # a known struct is rendered as defined: one typedef per name, with the recursive references intact + bin_path = os.path.join(test_location, "x86_64", "fauxware") + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(show_progressbar=not WORKER, fail_fast=True, normalize=True) + proj.analyses.CompleteCallingConventions() + + alpha = SimStruct({}, name="Alpha") + beta = SimStruct({}, name="Beta") + alpha.fields = OrderedDict({"beta": SimTypePointer(beta), "x": SimTypeInt()}) + beta.fields = OrderedDict({"alpha": SimTypePointer(alpha), "y": SimTypeInt()}) + + callee = cfg.functions["authenticate"] + callee.calling_convention = default_cc(proj.arch.name, platform=proj.simos.name)(proj.arch) + callee.prototype = SimTypeFunction([SimTypePointer(alpha), SimTypePointer(beta)], SimTypeInt()).with_arch( + proj.arch + ) + + dec = proj.analyses.Decompiler(cfg.functions["main"], cfg=cfg, fail_fast=True) + assert dec.codegen is not None and dec.codegen.text is not None + print_decompilation_result(dec) + text = dec.codegen.text + + for name in ("Alpha", "Beta"): + assert text.count(f"typedef struct {name} {{") == 1 + assert "struct Beta *beta;" in text + assert "struct Alpha *alpha;" in text + assert "void* beta;" not in text + assert "void* alpha;" not in text + if __name__ == "__main__": unittest.main() diff --git a/tests/analyses/test_typehoon.py b/tests/analyses/test_typehoon.py index d6ceff6d8..368966dc8 100755 --- a/tests/analyses/test_typehoon.py +++ b/tests/analyses/test_typehoon.py @@ -543,6 +543,47 @@ class TestTypeTranslator(unittest.TestCase): assert isinstance(tc.fields[0], Pointer64) assert 0 in tc.field_names assert tc.field_names[0] == "ptr" + # the recursive reference points back at the struct being lifted instead of degrading into BOT + assert tc.fields[0].basetype is tc + + @staticmethod + def _mutually_recursive_structs(arch): + """ + struct Alpha { struct Beta *beta; int x; }; struct Beta { struct Alpha *alpha; int y; }; + The shape ``dereference_simtype`` produces for library types such as DRIVER_OBJECT/DEVICE_OBJECT. + """ + alpha = SimStruct({}, name="Alpha") + beta = SimStruct({}, name="Beta") + alpha.fields = OrderedDict({"beta": SimTypePointer(beta), "x": SimTypeInt()}) + beta.fields = OrderedDict({"alpha": SimTypePointer(alpha), "y": SimTypeInt()}) + return alpha.with_arch(arch), beta.with_arch(arch) + + def test_mutually_recursive_structs_keep_their_references(self): + arch = archinfo.arch_from_id("amd64") + alpha, _ = self._mutually_recursive_structs(arch) + tx = TypeTranslator(arch) + + tc = tx.simtype2tc(alpha) + assert isinstance(tc, Struct) + beta_tc = tc.fields[0].basetype + assert isinstance(beta_tc, Struct) + assert beta_tc.name == "Beta" + # ... and Beta points back at the same Alpha + assert beta_tc.fields[0].basetype is tc + + def test_known_struct_fields_are_not_re_inferred(self): + arch = archinfo.arch_from_id("amd64") + alpha, beta = self._mutually_recursive_structs(arch) + tx = TypeTranslator(arch) + + # translating back returns the known definition itself, whichever type is lifted first + for lifted in (alpha, beta): + restored, _ = tx.tc2simtype(tx.simtype2tc(lifted)) + assert restored is lifted + + # reaching Beta through Alpha yields the same Beta, not a re-derived copy + restored_alpha, _ = tx.tc2simtype(tx.simtype2tc(alpha)) + assert restored_alpha.fields["beta"].pts_to is beta class TestSimpleSolverLatticeOps(unittest.TestCase): From 233724f38e6030c4fc430c640b2e9073586e28d9 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Tue, 4 Aug 2026 09:54:45 -0700 Subject: [PATCH 087/122] icicle: Avoid repeatedly calling memory.permissions (#6758) --- angr/engines/icicle.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/angr/engines/icicle.py b/angr/engines/icicle.py index 48678c7e8..ca8e0f932 100644 --- a/angr/engines/icicle.py +++ b/angr/engines/icicle.py @@ -139,12 +139,11 @@ class IcicleEngine(SuccessorsEngine): is explicitly unmapped. """ metadata = {} - page_size = state.memory.page_size for page_num, page in state.memory._pages.items(): if page is None: metadata[page_num] = None else: - metadata[page_num] = state.memory.permissions(page_num * page_size).concrete_value + metadata[page_num] = page.permission_bits.concrete_value return metadata @staticmethod From 4a9c1454dcd823e00bb57dba816e9afc266aea44 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Tue, 4 Aug 2026 09:58:57 -0700 Subject: [PATCH 088/122] icicle: Remove double-underscore methods (#6759) --- angr/engines/icicle.py | 142 ++++++++++++++++++++--------------------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/angr/engines/icicle.py b/angr/engines/icicle.py index ca8e0f932..f7edf4e61 100644 --- a/angr/engines/icicle.py +++ b/angr/engines/icicle.py @@ -29,20 +29,6 @@ log = logging.getLogger(__name__) PROCESSORS_DIR = os.path.join(os.path.dirname(pypcode.__file__), "processors") -def _syscall_jumpkind(arch_name: str, emu) -> str: - """Map icicle's generic Syscall exception to the arch-specific VEX jumpkind.""" - if arch_name in ("AMD64", "X86"): - try: - insn = emu.mem_read(emu.pc, 2) - except RuntimeError: - insn = b"" - if insn == b"\xcd\x80": - return "Ijk_Sys_int128" - if insn == b"\x0f\x05": - return "Ijk_Sys_syscall" - return "Ijk_Sys_syscall" - - class IcicleEngine(SuccessorsEngine): """ An angr engine that uses Icicle to execute concrete states. The purpose of @@ -67,7 +53,7 @@ class IcicleEngine(SuccessorsEngine): """ @staticmethod - def __make_icicle_arch(arch: Arch) -> str | None: + def _make_icicle_arch(arch: Arch) -> str | None: """ Convert an angr architecture to an Icicle architecture. Not particularly accurate, just a set of heuristics to get the right architecture. When @@ -80,30 +66,44 @@ class IcicleEngine(SuccessorsEngine): return arch.linux_name @staticmethod - def __is_arm(icicle_arch: str) -> bool: + def _is_arm(icicle_arch: str) -> bool: """ Check if the architecture is arm based on the address. """ return icicle_arch.startswith(("arm", "thumb")) @staticmethod - def __is_cortex_m(angr_arch: Arch, icicle_arch: str) -> bool: + def _is_cortex_m(angr_arch: Arch, icicle_arch: str) -> bool: """ Check if the architecture is cortex-m based on the address. """ return isinstance(angr_arch, ArchARMCortexM) or icicle_arch == "armv7m" @staticmethod - def __is_thumb(angr_arch: Arch, icicle_arch: str, addr: int) -> bool: + def _is_thumb(angr_arch: Arch, icicle_arch: str, addr: int) -> bool: """ Check if the architecture is thumb based on the address. """ - return IcicleEngine.__is_cortex_m(angr_arch, icicle_arch) or ( - IcicleEngine.__is_arm(icicle_arch) and addr & 1 == 1 + return IcicleEngine._is_cortex_m(angr_arch, icicle_arch) or ( + IcicleEngine._is_arm(icicle_arch) and addr & 1 == 1 ) @staticmethod - def __get_pages(state: SimState[int, int]) -> set[int]: + def _syscall_jumpkind(arch_name: str, emu: Icicle) -> str: + """Map icicle's generic Syscall exception to the arch-specific VEX jumpkind.""" + if arch_name in ("AMD64", "X86"): + try: + insn = emu.mem_read(emu.pc, 2) + except RuntimeError: + insn = b"" + if insn == b"\xcd\x80": + return "Ijk_Sys_int128" + if insn == b"\x0f\x05": + return "Ijk_Sys_syscall" + return "Ijk_Sys_syscall" + + @staticmethod + def _get_pages(state: SimState[int, int]) -> set[int]: """ Unfortunately, the memory model doesn't have a way to get all pages. Instead, we can get all of the backers from the loader, then all of the @@ -132,7 +132,7 @@ class IcicleEngine(SuccessorsEngine): return pages @staticmethod - def __get_explicit_page_metadata(state: SimState[int, int]) -> dict[int, int | None]: + def _get_explicit_page_metadata(state: SimState[int, int]) -> dict[int, int | None]: """ Return explicit page overrides from the paged memory model. The key is page number. Value is permission bits, or None when the page @@ -147,7 +147,7 @@ class IcicleEngine(SuccessorsEngine): return metadata @staticmethod - def __sync_registers(emu: Icicle, state: SimState[int, int], register_names: Iterable[str]) -> set[str]: + def _sync_registers(emu: Icicle, state: SimState[int, int], register_names: Iterable[str]) -> set[str]: """Copy each named register from `state` into `emu`, plus the x86/AMD64 TLS segment base (which icicle exposes under a different name than angr). Returns the subset of `register_names` that succeeded — registers icicle @@ -170,7 +170,7 @@ class IcicleEngine(SuccessorsEngine): return copied @staticmethod - def __write_page(emu: Icicle, state: SimState[int, int], page_num: int) -> None: + def _write_page(emu: Icicle, state: SimState[int, int], page_num: int) -> None: """Copy `state`'s content at `page_num` into `emu`, resolving any symbolic bytes through the solver. """ @@ -183,7 +183,7 @@ class IcicleEngine(SuccessorsEngine): emu.mem_write(addr, memory) @staticmethod - def __sync_edge_hitmap(emu: Icicle, state: SimState[int, int]) -> None: + def _sync_edge_hitmap(emu: Icicle, state: SimState[int, int]) -> None: """Copy state's edge_hitmap into emu, if the plugin is present.""" if state.has_plugin("edge_hitmap"): hitmap_plugin = cast(SimStateEdgeHitmap, state.get_plugin("edge_hitmap")) @@ -191,20 +191,20 @@ class IcicleEngine(SuccessorsEngine): emu.edge_hitmap = hitmap_plugin.edge_hitmap @staticmethod - def __build_emu_for(state: SimState[int, int]) -> tuple[Icicle, IcicleStateTranslationData]: + def _build_emu_for(state: SimState[int, int]) -> tuple[Icicle, IcicleStateTranslationData]: """Construct a fresh `Icicle` VM and sync `state` onto it from scratch.""" - icicle_arch = IcicleEngine.__make_icicle_arch(state.arch) + icicle_arch = IcicleEngine._make_icicle_arch(state.arch) if icicle_arch is None: raise ValueError("Unsupported architecture") if state.project is None: raise ValueError("IcicleEngine requires a project to be set") emu = Icicle(icicle_arch, PROCESSORS_DIR, True, True) - translation_data = IcicleEngine.__sync_state_to_emu(emu, state, None, icicle_arch) + translation_data = IcicleEngine._sync_state_to_emu(emu, state, None, icicle_arch) return emu, translation_data @staticmethod - def __convert_icicle_state_to_angr( + def _convert_icicle_state_to_angr( emu: Icicle, translation_data: IcicleStateTranslationData, status: VmExit ) -> SimState[int, int]: state = translation_data.base_state.copy() @@ -213,7 +213,7 @@ class IcicleEngine(SuccessorsEngine): for register in translation_data.registers: state.registers.store(register, emu.reg_read(register)) - if IcicleEngine.__is_arm(emu.architecture): # Hack to work around us calling it r15t + if IcicleEngine._is_arm(emu.architecture): # Hack to work around us calling it r15t state.registers.store("pc", (emu.pc | 1) if emu.isa_mode == 1 else emu.pc) # Restore TLS base from FS/GS_OFFSET (register copy clobbers it). @@ -244,7 +244,7 @@ class IcicleEngine(SuccessorsEngine): ): state.history.jumpkind = "Ijk_SigSEGV" elif exc == ExceptionCode.Syscall: - state.history.jumpkind = _syscall_jumpkind(arch_name, emu) + state.history.jumpkind = IcicleEngine._syscall_jumpkind(arch_name, emu) # Icicle stops at the syscall instruction (unlike VEX # which computes the next IP during lifting), so we # advance IP using archinfo's instruction_alignment. @@ -277,7 +277,7 @@ class IcicleEngine(SuccessorsEngine): return state @staticmethod - def __sync_state_to_emu( + def _sync_state_to_emu( emu: Icicle, state: SimState[int, int], base: IcicleStateTranslationData | None, @@ -301,9 +301,9 @@ class IcicleEngine(SuccessorsEngine): icicle_arch = base.icicle_arch register_names = base.registers - copied_registers = IcicleEngine.__sync_registers(emu, state, register_names) + copied_registers = IcicleEngine._sync_registers(emu, state, register_names) - if IcicleEngine.__is_thumb(state.arch, icicle_arch, state.addr): + if IcicleEngine._is_thumb(state.arch, icicle_arch, state.addr): emu.pc = state.addr & ~1 emu.isa_mode = 1 elif "arm" in icicle_arch: # Hack to work around us calling it r15t @@ -311,10 +311,10 @@ class IcicleEngine(SuccessorsEngine): # Sync mapping/permission deltas. page_size = state.memory.page_size - explicit_page_metadata = IcicleEngine.__get_explicit_page_metadata(state) + explicit_page_metadata = IcicleEngine._get_explicit_page_metadata(state) if base is None: # Empty baseline: every mapped page is "newly mapped". - candidate_pages = IcicleEngine.__get_pages(state) + candidate_pages = IcicleEngine._get_pages(state) mapped_pages: set[int] = set() writable_pages: set[int] = set() base_state_pages: dict[int, typing.Any] = {} @@ -351,7 +351,7 @@ class IcicleEngine(SuccessorsEngine): if not perm_bits & 2: # R-only pages won't be visited by the writable-page loop # below, so this is the only place to seed their content. - IcicleEngine.__write_page(emu, state, page_num) + IcicleEngine._write_page(emu, state, page_num) elif old_mapped and new_mapped and base is not None: base_perm_bits = base.base_state.memory.permissions(addr).concrete_value if base_perm_bits != perm_bits: @@ -368,11 +368,11 @@ class IcicleEngine(SuccessorsEngine): for page_num in writable_pages: if state.memory._pages.get(page_num) is base_state_pages.get(page_num): continue - IcicleEngine.__write_page(emu, state, page_num) + IcicleEngine._write_page(emu, state, page_num) # restore_snapshot zeroes the hitmap; full init starts with no # hitmap. Either way we (re-)copy. - IcicleEngine.__sync_edge_hitmap(emu, state) + IcicleEngine._sync_edge_hitmap(emu, state) return IcicleStateTranslationData( base_state=state, @@ -423,7 +423,7 @@ class IcicleEngine(SuccessorsEngine): state.inspect.b("mem_write", when=BP_AFTER, action=_on_mem_write) @staticmethod - def __sync_continuation( + def _sync_continuation( emu: Icicle, state: SimState[int, int], translation_data: IcicleStateTranslationData, @@ -432,10 +432,10 @@ class IcicleEngine(SuccessorsEngine): """Sync only registers and changed pages to icicle (no snapshot restore).""" icicle_arch = translation_data.icicle_arch - IcicleEngine.__sync_registers(emu, state, translation_data.registers) + IcicleEngine._sync_registers(emu, state, translation_data.registers) # Explicitly set PC (the register copy may have written it to a sub-register). - if IcicleEngine.__is_thumb(state.arch, icicle_arch, state.addr): + if IcicleEngine._is_thumb(state.arch, icicle_arch, state.addr): emu.pc = state.addr & ~1 emu.isa_mode = 1 else: @@ -456,40 +456,18 @@ class IcicleEngine(SuccessorsEngine): mapped_pages.add(page_num) if perm_bits & 2: writable_pages.add(page_num) - IcicleEngine.__write_page(emu, state, page_num) + IcicleEngine._write_page(emu, state, page_num) return IcicleStateTranslationData( base_state=state, registers=translation_data.registers, mapped_pages=mapped_pages, writable_pages=writable_pages, - explicit_page_metadata=IcicleEngine.__get_explicit_page_metadata(state), + explicit_page_metadata=IcicleEngine._get_explicit_page_metadata(state), initial_cpu_icount=emu.cpu_icount, icicle_arch=icicle_arch, ) - @override - def process_successors(self, successors: SimSuccessors, *, num_inst: int | None = None, **kwargs: typing.Any): - extra_stop_points_arg = kwargs.pop("extra_stop_points", None) - extra_stop_points: set[int] | None = None - if extra_stop_points_arg is not None: - extra_stop_points = set(typing.cast(Iterable[int], extra_stop_points_arg)) - - if len(kwargs) > 0: - log.warning("IcicleEngine.process_successors received unknown kwargs: %s", kwargs) - - state = typing.cast(SimState[int, int], self.state) - - result = self._run_icicle(state, num_inst=num_inst, extra_stop_points=extra_stop_points) - successors.add_successor( - result, - result.ip, - claripy.true(), - result.history.jumpkind, - add_guard=False, - ) - successors.processed = True - def _run_icicle( self, state: SimState[int, int], @@ -502,7 +480,7 @@ class IcicleEngine(SuccessorsEngine): if icicle_plugin.vm_ref is None: # First run: build the VM and snapshot it for future branches. - emu, translation_data = self.__build_emu_for(state) + emu, translation_data = self._build_emu_for(state) emu.save_snapshot() icicle_plugin.vm_ref = IcicleVMRef(vm=emu) icicle_plugin.base_translation_data = translation_data @@ -518,7 +496,7 @@ class IcicleEngine(SuccessorsEngine): for page_num, page in state.memory._pages.items(): if page is not None and page_num not in icicle_plugin.translation_data.mapped_pages: pages_to_sync.add(page_num) - translation_data = self.__sync_continuation(emu, state, icicle_plugin.translation_data, list(pages_to_sync)) + translation_data = self._sync_continuation(emu, state, icicle_plugin.translation_data, list(pages_to_sync)) # Reset the path tracer so `emu.recent_blocks` reflects only # blocks executed during this run, not cumulative history. emu.clear_path_tracer() @@ -527,7 +505,7 @@ class IcicleEngine(SuccessorsEngine): assert icicle_plugin.base_translation_data is not None emu = icicle_plugin.vm_ref.vm emu.restore_snapshot() - translation_data = self.__sync_state_to_emu(emu, state, icicle_plugin.base_translation_data) + translation_data = self._sync_state_to_emu(emu, state, icicle_plugin.base_translation_data) # Sync simprocedure breakpoints. Simprocs can be registered # dynamically between runs (e.g. SimProcedure.call() makes a new @@ -540,7 +518,7 @@ class IcicleEngine(SuccessorsEngine): # Set extra stop points (cleaned up after the run). added_breakpoints = [] - is_arm = IcicleEngine.__is_arm(translation_data.icicle_arch) + is_arm = IcicleEngine._is_arm(translation_data.icicle_arch) if extra_stop_points is not None: for addr in extra_stop_points: if is_arm: @@ -569,7 +547,7 @@ class IcicleEngine(SuccessorsEngine): for addr in added_breakpoints: emu.remove_breakpoint(addr) - result = IcicleEngine.__convert_icicle_state_to_angr(emu, translation_data, status) + result = IcicleEngine._convert_icicle_state_to_angr(emu, translation_data, status) # Advance the VM's generation so any other plugin copies still pointing # at the prior generation falls into the snapshot-restore path on its @@ -589,6 +567,28 @@ class IcicleEngine(SuccessorsEngine): return result + @override + def process_successors(self, successors: SimSuccessors, *, num_inst: int | None = None, **kwargs: typing.Any): + extra_stop_points_arg = kwargs.pop("extra_stop_points", None) + extra_stop_points: set[int] | None = None + if extra_stop_points_arg is not None: + extra_stop_points = set(typing.cast(Iterable[int], extra_stop_points_arg)) + + if len(kwargs) > 0: + log.warning("IcicleEngine.process_successors received unknown kwargs: %s", kwargs) + + state = typing.cast(SimState[int, int], self.state) + + result = self._run_icicle(state, num_inst=num_inst, extra_stop_points=extra_stop_points) + successors.add_successor( + result, + result.ip, + claripy.true(), + result.history.jumpkind, + add_guard=False, + ) + successors.processed = True + class UberIcicleEngine(SimEngineFailure, SimEngineSyscall, HooksMixin, IcicleEngine): """ From f4b23d4444ceeb0301eb85eb95afa5de8c9632cd Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Tue, 4 Aug 2026 10:11:26 -0700 Subject: [PATCH 089/122] UltraPage: Write to concrete_data as a single block instead of loop (#6760) --- .../memory_mixins/paged_memory/pages/ultra_page.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/angr/storage/memory_mixins/paged_memory/pages/ultra_page.py b/angr/storage/memory_mixins/paged_memory/pages/ultra_page.py index 5fbcf1b95..fc2342cb3 100644 --- a/angr/storage/memory_mixins/paged_memory/pages/ultra_page.py +++ b/angr/storage/memory_mixins/paged_memory/pages/ultra_page.py @@ -185,17 +185,16 @@ class UltraPage(MemoryObjectMixin, PageBase): self.symbolic_bitmap.clear_range(addr, addr + size) # store - arange = range(addr, addr + size) ival = data if type(data) is int else data.object.args[0] - if endness == "Iend_BE": - arange = reversed(arange) assert memory.state.arch.byte_width == 8 # TODO: Make UltraPage support architectures with greater byte_widths (but are still multiples of 8) concrete_data = self._concrete() - for subaddr in arange: - concrete_data[subaddr] = ival & 0xFF - ival >>= 8 + + # Serialize in one step. + concrete_data[addr : addr + size] = (ival & ((1 << (size * 8)) - 1)).to_bytes( + size, "big" if endness == "Iend_BE" else "little" + ) else: # mark range as symbolic self.symbolic_bitmap.set_range(addr, addr + size) From fd235fcb905b7cf7aaaaabbc3c096cac33ee6ae5 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Tue, 4 Aug 2026 10:51:07 -0700 Subject: [PATCH 090/122] icicle: Disable inspect and actions during memory sync-back (#6761) --- angr/engines/icicle.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/angr/engines/icicle.py b/angr/engines/icicle.py index f7edf4e61..fc443cd66 100644 --- a/angr/engines/icicle.py +++ b/angr/engines/icicle.py @@ -229,7 +229,8 @@ class IcicleEngine(SuccessorsEngine): for page_num in translation_data.writable_pages: addr = page_num * page_size if addr in modified_addrs: - state.memory.store(addr, emu.mem_read(addr, page_size)) + # Disable inspect and actions during writeback. + state.memory.store(addr, emu.mem_read(addr, page_size), inspect=False, disable_actions=True) # 3. Set history # 3.1 history.jumpkind From 71bf42c9b894df8386ffef7ca0558ac9e17a5c46 Mon Sep 17 00:00:00 2001 From: Fish Date: Tue, 4 Aug 2026 18:27:09 -0700 Subject: [PATCH 091/122] Decompiler: Test common C conditions. (#6762) * Decompiler: Test common C conditions. * Improve c-style null compatison implementation * Make order-sensitive --------- Co-authored-by: Kevin Phoenix --- .../decompiler/structured_codegen/c.py | 56 ++++++++++--------- .../analyses/decompiler/test_c_conditions.py | 56 +++++++++++++++++++ 2 files changed, 85 insertions(+), 27 deletions(-) create mode 100644 tests/analyses/decompiler/test_c_conditions.py diff --git a/angr/analyses/decompiler/structured_codegen/c.py b/angr/analyses/decompiler/structured_codegen/c.py index ccdf2b689..8e4aa1089 100644 --- a/angr/analyses/decompiler/structured_codegen/c.py +++ b/angr/analyses/decompiler/structured_codegen/c.py @@ -2099,11 +2099,14 @@ class CUnaryOp(CExpression): # def _c_repr_chunks_not(self): - paren = CClosingObject("(") yield "!", self - yield "(", paren - yield from CExpression._try_c_repr_chunks(self.operand) - yield ")", paren + if isinstance(self.operand, CBinaryOp): + paren = CClosingObject("(") + yield "(", paren + yield from CExpression._try_c_repr_chunks(self.operand) + yield ")", paren + else: + yield from CExpression._try_c_repr_chunks(self.operand) def _c_repr_chunks_bitwiseneg(self): paren = CClosingObject("(") @@ -2298,15 +2301,8 @@ class CBinaryOp(CExpression): # def _c_repr_chunks(self, op): - skip_op_and_rhs = False - if self._cstyle_null_cmp and self._has_const_null_rhs(): - if self.op == "CmpEQ": - skip_op_and_rhs = True - yield "!", None - elif self.op == "CmpNE": - skip_op_and_rhs = True # lhs - if isinstance(self.lhs, CBinaryOp) and self.op_precedence > self.lhs.op_precedence and not skip_op_and_rhs: + if isinstance(self.lhs, CBinaryOp) and self.op_precedence > self.lhs.op_precedence: paren = CClosingObject("(") yield "(", paren yield from self._try_c_repr_chunks(self.lhs) @@ -2314,19 +2310,19 @@ class CBinaryOp(CExpression): else: yield from self._try_c_repr_chunks(self.lhs) - if not skip_op_and_rhs: - # operator - yield op, self - # rhs - if isinstance(self.rhs, CBinaryOp) and self.op_precedence > self.rhs.op_precedence - ( - 1 if self.op in ["Sub", "Div"] else 0 - ): - paren = CClosingObject("(") - yield "(", paren - yield from self._try_c_repr_chunks(self.rhs) - yield ")", paren - else: - yield from self._try_c_repr_chunks(self.rhs) + # operator + yield op, self + + # rhs + if isinstance(self.rhs, CBinaryOp) and self.op_precedence > self.rhs.op_precedence - ( + 1 if self.op in ["Sub", "Div"] else 0 + ): + paren = CClosingObject("(") + yield "(", paren + yield from self._try_c_repr_chunks(self.rhs) + yield ")", paren + else: + yield from self._try_c_repr_chunks(self.rhs) def _c_repr_chunks_opfirst(self, op): yield op, self @@ -2426,10 +2422,16 @@ class CBinaryOp(CExpression): yield from self._c_repr_chunks(" >= ") def _c_repr_chunks_cmpeq(self): - yield from self._c_repr_chunks(" == ") + if self._cstyle_null_cmp and self._has_const_null_rhs(): + yield from CUnaryOp("Not", self.lhs, codegen=self.codegen).c_repr_chunks() + else: + yield from self._c_repr_chunks(" == ") def _c_repr_chunks_cmpne(self): - yield from self._c_repr_chunks(" != ") + if self._cstyle_null_cmp and self._has_const_null_rhs(): + yield from self._try_c_repr_chunks(self.lhs) + else: + yield from self._c_repr_chunks(" != ") def _c_repr_chunks_concat(self): yield from self._c_repr_chunks(" CONCAT ") diff --git a/tests/analyses/decompiler/test_c_conditions.py b/tests/analyses/decompiler/test_c_conditions.py new file mode 100644 index 000000000..c7110ab8d --- /dev/null +++ b/tests/analyses/decompiler/test_c_conditions.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use,no-member +from __future__ import annotations + +__package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin + +import logging +import os +import re +import unittest + +import angr +from tests.common import WORKER, bin_location, print_decompilation_result + +test_location = os.path.join(bin_location, "tests") + +l = logging.getLogger(__name__) + + +class TestDecompilingCConditions(unittest.TestCase): + def test_decompiling_c_conditions_gcc_O0(self): + bin_path = os.path.join(test_location, "x86_64", "c_conditions_gcc_O0") + proj = angr.Project(bin_path, auto_load_libs=False) + + cfg = proj.analyses.CFGFast(show_progressbar=not WORKER, fail_fast=True, normalize=True) + func = cfg.functions["main"] + assert func is not None + dec = proj.analyses.Decompiler(func, cfg=cfg) + assert dec.codegen is not None and dec.codegen.text is not None + print_decompilation_result(dec) + + # dump all conditions + conditions = re.findall(r"if \((.*?)\)\n", dec.codegen.text) + if not WORKER: + print(conditions) + # adjust this list once the decompilation output changes, but make sure the output is correct! + assert conditions == [ + # !(a >> 8) & 1 + "!((char)(v0) >> 8)", + # !((a >> 8) & 1) + "!(v0 & 0x100)", + # !((a >> 8) & b) + "!(v1 & (char)(v0) >> 8)", + # (a >> 8) & 1 + "v0 & 0x100", + # (a >> 31) & !b + "!v1 & (char)(v0) >> 31", + # ~(a >> 31) & !b + "!v1 & ~((char)(v0) >> 31)", + # ~((a >> 31) & !b) + "(!v1 & (char)(v0) >> 31) != 0xffffffff", + ] + + +if __name__ == "__main__": + unittest.main() From f1601788371e574b748ea36754c39e606ef70d54 Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Wed, 5 Aug 2026 00:26:01 -0700 Subject: [PATCH 092/122] reaching_definitions: reject mismatched conversion widths (#6749) * reaching_definitions: reject mismatched conversion widths * Tests: satisfy RDA lint and type checks --- .../reaching_definitions/engine_ail.py | 2 + .../reaching_definitions/test_engine_ail.py | 58 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tests/analyses/reaching_definitions/test_engine_ail.py diff --git a/angr/analyses/reaching_definitions/engine_ail.py b/angr/analyses/reaching_definitions/engine_ail.py index 1e175b005..bc9428ee9 100644 --- a/angr/analyses/reaching_definitions/engine_ail.py +++ b/angr/analyses/reaching_definitions/engine_ail.py @@ -562,11 +562,13 @@ class SimEngineRDAIL( bits = expr.to_bits size = bits // self.arch.byte_width + # Reject inconsistent analysis values before performing Claripy slicing or extension. if ( to_conv.count() == 1 and 0 in to_conv and expr.from_type == ailment.Expr.Convert.TYPE_INT and expr.to_type == ailment.Expr.Convert.TYPE_INT + and all(len(value) == expr.from_bits for value in to_conv[0]) ): values = to_conv[0] else: diff --git a/tests/analyses/reaching_definitions/test_engine_ail.py b/tests/analyses/reaching_definitions/test_engine_ail.py new file mode 100644 index 000000000..e3d2e49d9 --- /dev/null +++ b/tests/analyses/reaching_definitions/test_engine_ail.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from typing import cast + +import claripy +from archinfo import Endness + +import angr +from angr import ailment +from angr.analyses.reaching_definitions.engine_ail import SimEngineRDAIL +from angr.analyses.reaching_definitions.function_handler import FunctionHandler +from angr.analyses.reaching_definitions.rd_state import ReachingDefinitionsState + + +class _ConstantState: + class _CodeLocation: + context = None + + codeloc = _CodeLocation() + + @staticmethod + def mark_const(_value: int, _size: int) -> None: + pass + + @staticmethod + def top(bits: int) -> claripy.ast.BV: + return claripy.BVS("TOP", bits) + + @staticmethod + def annotate_with_def(value: claripy.ast.BV, _definition) -> claripy.ast.BV: + return value + + @staticmethod + def add_memory_use_by_def(_definition, *, expr) -> None: + pass + + +def test_convert_handles_operand_width_mismatches(): + project = angr.load_shellcode(b"\xc3", arch="amd64") + engine = SimEngineRDAIL(project, FunctionHandler()) + engine.state = cast(ReachingDefinitionsState, _ConstantState()) + + base = ailment.Expr.Const(0, 0x0123456789ABCDEF, 64) + offset = ailment.Expr.Const(1, 0, 64) + + extracted = ailment.Expr.Extract(2, 32, base, offset, Endness.LE) + widened = ailment.Expr.Convert(3, 32, 64, False, extracted) + widened_result = engine._expr(widened) # pylint: disable=protected-access + assert len(widened_result) == widened.bits + widened_value = widened_result.one_value() + assert widened_value is not None and widened_value.symbolic + + inserted = ailment.Expr.Insert(4, base, offset, ailment.Expr.Const(5, 1, 8), Endness.LE) + narrowed = ailment.Expr.Convert(6, 64, 32, False, inserted) + narrowed_result = engine._expr(narrowed) # pylint: disable=protected-access + assert len(narrowed_result) == narrowed.bits + narrowed_value = narrowed_result.one_value() + assert narrowed_value is not None and narrowed_value.symbolic From 7c2e3501e6965dcbddce2ea487e0fda9ad203a3d Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 5 Aug 2026 01:12:55 -0700 Subject: [PATCH 093/122] SimStruct: Remove _arch_memo to fix the cache leak. (#6757) * SimStruct: Remove _arch_memo to fix the cache leak. * Fix the comment * Better typing. * Fix RustSimTypes. * More fixes. * Fix caching for anonymous structs. * SimUnion: Cache alignment. * Convert returnty to a arch-ed returnty. * RustSimType: Do not shadow SimType.with_arch. * Rust: Bind an arch to types before they reach the type solver. * RustSimType: Make it a subclass of SimType. --- .../decompiler/structured_codegen/rust.py | 8 +- angr/analyses/typehoon/translator.py | 70 +- .../rust_calling_convention.py | 10 +- .../struct_return_simplifier.py | 5 +- angr/rust/sim_type.py | 126 ++-- angr/rust/typehoon/translator.py | 2 +- angr/rust/typehoon/typehoon.py | 6 +- angr/sim_type.py | 151 +++-- angr/utils/types.py | 10 +- tests/analyses/test_rust_sim_type.py | 613 +++++++++--------- tests/analyses/test_rust_type_hints.py | 45 +- tests/analyses/test_typehoon.py | 11 +- tests/types/test_types.py | 23 + 13 files changed, 641 insertions(+), 439 deletions(-) diff --git a/angr/analyses/decompiler/structured_codegen/rust.py b/angr/analyses/decompiler/structured_codegen/rust.py index a67ac16cc..d49856721 100644 --- a/angr/analyses/decompiler/structured_codegen/rust.py +++ b/angr/analyses/decompiler/structured_codegen/rust.py @@ -210,7 +210,7 @@ def _with_arch(ty, arch): return ty -def type_to_rust_repr_chunks(ty: SimType | RustSimType, name=None, name_type=None, full=False, indent_str=""): +def type_to_rust_repr_chunks(ty: SimType, name=None, name_type=None, full=False, indent_str=""): """ Helper generator function to turn a SimType into generated tuples of (C-string, AST node). """ @@ -2340,8 +2340,8 @@ class RustTypeCast(RustExpression): def __init__( self, - src_type: RustSimType | SimType | None, - dst_type: RustSimType | SimType, + src_type: SimType | None, + dst_type: SimType, expr: RustExpression, tags=None, **kwargs, @@ -4385,7 +4385,7 @@ class RustStructuredCodeWalker: class MakeTypecastsImplicit(RustStructuredCodeWalker): @classmethod - def collapse(cls, dst_ty: SimType | RustSimType | None, child: RustExpression) -> RustExpression: + def collapse(cls, dst_ty: SimType | None, child: RustExpression) -> RustExpression: result = child if isinstance(child, RustTypeCast): intermediate_ty = child.dst_type diff --git a/angr/analyses/typehoon/translator.py b/angr/analyses/typehoon/translator.py index b6e1cf2d2..d0c3eb2aa 100644 --- a/angr/analyses/typehoon/translator.py +++ b/angr/analyses/typehoon/translator.py @@ -1,6 +1,7 @@ # pylint:disable=unused-argument,no-self-use from __future__ import annotations +import logging from itertools import count from typing import TYPE_CHECKING @@ -14,6 +15,9 @@ if TYPE_CHECKING: import archinfo +l = logging.getLogger(__name__) + + class SimTypeTempRef(sim_type.SimType): """ Represents a temporary reference to another type. TypeVariableReference is translated to SimTypeTempRef. @@ -59,7 +63,8 @@ class TypeTranslator: self._struct_ctr = count() # a name-independent, deterministic per-function ordering id stamped onto each translated struct self._struct_def_ctr = count() - self.memo = {} + # we encode SimStruct, SimCppClass, and SimTypeRef into strings so they can be used as unified keys for memo + self.memo: dict[str, typeconsts.Struct] = {} self.named_struct_id_counter = count(133337) self.struct_name_to_idx = {} # definitions of known structs (library types or user-defined types), keyed by name @@ -75,6 +80,21 @@ class TypeTranslator: def struct_name(self): return f"struct_{next(self._struct_ctr)}" + @staticmethod + def _simstruct_cppclass_to_memo_key( + st: sim_type.SimStruct | sim_type.SimCppClass | sim_type.SimTypeRef, + ) -> str | None: + if isinstance(st, sim_type.SimStruct): + return f"struct_{st.name}" + if isinstance(st, sim_type.SimCppClass): + return f"cppclass_{st.name}" + if isinstance(st, sim_type.SimTypeRef): + if st.original_type is sim_type.SimStruct: + return f"struct_{st.name}" + if st.original_type is sim_type.SimCppClass: + return f"cppclass_{st.name}" + return None + # # Type translation # @@ -340,10 +360,35 @@ class TypeTranslator: def _translate_SimTypeWideChar(self, st: sim_type.SimTypeWideChar) -> typeconsts.Int16: return typeconsts.Int16(name=st.label) + def _translate_SimTypeRef(self, st: sim_type.SimTypeRef) -> typeconsts.TypeConstant | typeconsts.BottomType: + # we really should not get SimTypeRef here, but if we do, we conduct a best-effort translation of SimTypeRef to + # a type constant. + l.error( + "TypeTranslator encountered an unexpected SimTypeRef. You probably forgot to call " + "dereference_simtype() to translate a SimTypeRef to a SimType!" + ) + type_key = self._simstruct_cppclass_to_memo_key(st) + if type_key is not None and type_key in self.memo: + return self.memo[type_key] + + if st.original_type is sim_type.SimStruct: + obj = typeconsts.Struct(fields={}, name=st.name) + if type_key is not None: + self.memo[type_key] = obj + return obj + if st.original_type is sim_type.SimCppClass: + obj = typeconsts.Struct(fields={}, name=st.name, is_cppclass=True) + if type_key is not None: + self.memo[type_key] = obj + return obj + return typeconsts.BottomType() + def _translate_SimStruct(self, st: sim_type.SimStruct) -> typeconsts.Struct | typeconsts.BottomType: - if st in self.memo: + type_key = self._simstruct_cppclass_to_memo_key(st) + assert type_key is not None + if type_key in self.memo: # a recursive reference: point back at the struct that is being translated - return self.memo[st] + return self.memo[type_key] struct_idx = {} if st.name: @@ -352,7 +397,7 @@ class TypeTranslator: struct_idx["idx"] = self.struct_name_to_idx[st.name] obj = typeconsts.Struct(fields={}, name=st.name, **struct_idx) - self.memo[st] = obj + self.memo[type_key] = obj self._remember_known_struct(st) fields = {} @@ -360,21 +405,23 @@ class TypeTranslator: offsets = st.offsets for field_name, simtype in st.fields.items(): if field_name not in offsets: - del self.memo[st] + del self.memo[type_key] return typeconsts.BottomType() offset = offsets[field_name] fields[offset] = self._simtype2tc(simtype) field_names[offset] = field_name obj.fields = fields obj.field_names = field_names - del self.memo[st] + del self.memo[type_key] return obj def _translate_SimCppClass(self, st: sim_type.SimCppClass) -> typeconsts.Struct | typeconsts.BottomType: - if st in self.memo: + type_key = self._simstruct_cppclass_to_memo_key(st) + assert type_key is not None + if type_key in self.memo: # a recursive reference: point back at the class that is being translated - return self.memo[st] + return self.memo[type_key] struct_idx = {} if st.name: @@ -383,7 +430,7 @@ class TypeTranslator: struct_idx["idx"] = self.struct_name_to_idx[st.name] obj = typeconsts.Struct(fields={}, name=st.name, is_cppclass=True, **struct_idx) - self.memo[st] = obj + self.memo[type_key] = obj self._remember_known_struct(st) fields = {} @@ -391,14 +438,14 @@ class TypeTranslator: offsets = st.offsets for field_name, simtype in st.fields.items(): if field_name not in offsets: - del self.memo[st] + del self.memo[type_key] return typeconsts.BottomType() offset = offsets[field_name] fields[offset] = self._simtype2tc(simtype) field_names[offset] = field_name obj.fields = fields obj.field_names = field_names - del self.memo[st] + del self.memo[type_key] return obj @@ -486,4 +533,5 @@ SimTypeHandlers = { sim_type.SimTypeEnum: TypeTranslator._translate_SimTypeEnum, sim_type.SimCppClass: TypeTranslator._translate_SimCppClass, sim_type.SimTypeFd: TypeTranslator._translate_SimTypeFd, + sim_type.SimTypeRef: TypeTranslator._translate_SimTypeRef, } diff --git a/angr/rust/analyses/rust_calling_convention/rust_calling_convention.py b/angr/rust/analyses/rust_calling_convention/rust_calling_convention.py index 206a7d608..35e7e136b 100644 --- a/angr/rust/analyses/rust_calling_convention/rust_calling_convention.py +++ b/angr/rust/analyses/rust_calling_convention/rust_calling_convention.py @@ -492,14 +492,14 @@ class RustCallingConventionAnalysis(Analysis): some_type, some_discriminant, some_discriminant_size, - ) + ).with_arch(self.project.arch) return RustSimTypeOption( none_discriminant, none_discriminant_size, some_type, some_discriminant, some_discriminant_size, - ) + ).with_arch(self.project.arch) if None not in discriminants: # Result with both discriminants known # Default: sort by discriminant value (smaller = Ok, larger = Err) @@ -526,7 +526,7 @@ class RustCallingConventionAnalysis(Analysis): err_type, err_discriminant.value, discriminant_size, - ) + ).with_arch(self.project.arch) if candidates_and_discriminants[1][1] is None: # Result with one discriminant missing (T is larger, discriminant omitted) (err_type, err_discriminant), (ok_type, _) = candidates_and_discriminants @@ -538,7 +538,7 @@ class RustCallingConventionAnalysis(Analysis): err_type, err_discriminant.value, discriminant_size, - ) + ).with_arch(self.project.arch) if len(candidates_and_discriminants) >= 2: structs_by_size = {} @@ -554,7 +554,7 @@ class RustCallingConventionAnalysis(Analysis): err_type=small_type, err_discriminant=None, err_discriminant_size=0, - ) + ).with_arch(self.project.arch) if ( len(structs_by_size) > 2 and min(structs_by_size) > 16 * self.project.arch.byte_width diff --git a/angr/rust/optimization_passes/struct_return_simplifier.py b/angr/rust/optimization_passes/struct_return_simplifier.py index 7c8f018a1..4dcd6cd1e 100644 --- a/angr/rust/optimization_passes/struct_return_simplifier.py +++ b/angr/rust/optimization_passes/struct_return_simplifier.py @@ -106,6 +106,7 @@ class StructReturnSimplifier(OptimizationPass, SRDAMixin, CFGTransformationMixin variant = None returnty = prototype.returnty if isinstance(returnty, (RustSimTypeResult, RustSimTypeOption)): + returnty = returnty.with_arch(self.project.arch) variant = returnty.get_variant(discriminant) if not variant and discriminant is not None: variant = returnty.get_variant(None) @@ -113,9 +114,7 @@ class StructReturnSimplifier(OptimizationPass, SRDAMixin, CFGTransformationMixin new_expr = self._remove_discriminant_from_struct(struct, variant) if len(new_expr.fields) == 1 and 0 in new_expr.fields: new_expr = new_expr.fields[0] - return RustEnum( - self.manager.next_atom(), variant.name, [new_expr], returnty.with_arch(self.project.arch).size - ) + return RustEnum(self.manager.next_atom(), variant.name, [new_expr], returnty.size) return struct def collect_ret_expr(self, path): diff --git a/angr/rust/sim_type.py b/angr/rust/sim_type.py index f1900afbe..702bafa76 100644 --- a/angr/rust/sim_type.py +++ b/angr/rust/sim_type.py @@ -1,7 +1,8 @@ +# pylint:disable=missing-class-docstring from __future__ import annotations -# pylint:disable=missing-class-docstring from collections import OrderedDict +from typing import Self, cast from angr.sim_type import ( IDENT_TO_CLS, @@ -21,10 +22,8 @@ def is_composite_type(ty): return isinstance(ty, (RustSimStruct, RustSimEnum)) -class RustSimType: - @property - def size(self) -> int: - raise NotImplementedError +class RustSimType(SimType): + _ident = "rust" def repr(self, name=None, full=0, memo=None, indent: int | None = 0): raise NotImplementedError @@ -82,6 +81,11 @@ class RustSimTypeInt(RustSimType, SimTypeInt): def from_json(d, type_collection=None, memo=None): return RustSimTypeInt(size=d.get("size", 32), signed=d.get("signed", True), label=d.get("label")) + def _with_arch(self, arch, *, memo: dict[str, SimType]): # pylint: disable=unused-argument + out = RustSimTypeInt(size=self._size, signed=self.signed, label=self.label) + out._arch = arch + return out + class RustSimTypeSize(RustSimTypeInt): _ident = "rust_size" @@ -113,6 +117,11 @@ class RustSimTypeSize(RustSimTypeInt): def from_json(d, type_collection=None, memo=None): return RustSimTypeSize(signed=d.get("signed", True), label=d.get("label")) + def _with_arch(self, arch, *, memo: dict[str, SimType]): # pylint: disable=unused-argument + out = RustSimTypeSize(signed=self.signed, label=self.label) + out._arch = arch + return out + class RustSimTypeFunction(RustSimType, SimTypeFunction): # pyright: ignore[reportIncompatibleMethodOverride] """ @@ -174,12 +183,17 @@ class RustSimTypeFunction(RustSimType, SimTypeFunction): # pyright: ignore[repo def size(self): return 4096 # ??????????? - def _with_arch(self, arch): + def repr(self, name=None, full=0, memo=None, indent: int | None = 0): + if not name: + return repr(self) + return f"{name}: {self!r}" + + def _with_arch(self, arch, *, memo: dict[str, SimType]): returnty = None if self.returnty is not None: - returnty = self.returnty.with_arch(arch) # pyright: ignore[reportAttributeAccessIssue] + returnty = cast(RustSimType, self.returnty.with_arch(arch, memo=memo)) out = RustSimTypeFunction( - [a.with_arch(arch) for a in self.args], # pyright: ignore[reportAttributeAccessIssue] + [cast(RustSimType, a.with_arch(arch, memo=memo)) for a in self.args], returnty, label=self.label, arg_names=self.arg_names, @@ -281,8 +295,8 @@ class RustSimTypeReference(RustSimType, SimTypePointer): raise ValueError("Can't tell my size without an arch!") return self._arch.bits - def _with_arch(self, arch): - out = RustSimTypeReference(self.pts_to.with_arch(arch), self.label, offset=self.offset) + def _with_arch(self, arch, *, memo: dict[str, SimType]): + out = RustSimTypeReference(self.pts_to.with_arch(arch, memo=memo), self.label, offset=self.offset) out._arch = arch return out @@ -316,8 +330,8 @@ class RustSimTypeArray(RustSimType, SimTypeArray): return f"{name}: {self!r}" - def _with_arch(self, arch): - out = RustSimTypeArray(self.elem_type.with_arch(arch), self.length, self.label) + def _with_arch(self, arch, *, memo: dict[str, SimType]): + out = RustSimTypeArray(self.elem_type.with_arch(arch, memo=memo), self.length, self.label) out._arch = arch return out @@ -341,13 +355,13 @@ class RustSimStruct(RustSimType, SimStruct): SimStruct.__init__(self, fields, name, pack, align) self._size = None - def _with_arch(self, arch): - if arch.name in self._arch_memo: - return self._arch_memo[arch.name] + def _with_arch(self, arch, *, memo: dict[str, SimType]): + if self.name in memo: + return cast(RustSimStruct, memo[self.name]) out = RustSimStruct(OrderedDict(), name=self.name, pack=self._pack, align=self._align) out._arch = arch - self._arch_memo[arch.name] = out + memo[self.name] = out out.fields = OrderedDict((k, v.with_arch(arch)) for k, v in self.fields.items()) @@ -476,6 +490,11 @@ class RustSimTypeNumOffset(RustSimType, SimTypeNumOffset): def repr(self, name=None, full=0, memo=None, indent: int | None = 0): super(SimTypeNumOffset, self).c_repr(name, full, memo, indent) + def _with_arch(self, arch, *, memo: dict[str, SimType]) -> RustSimTypeNumOffset: + out = RustSimTypeNumOffset(self.size, signed=self.signed, label=self.label, offset=self.offset) + out._arch = arch + return out + class RustSimTypeSlice(RustSimStruct, SimType): _ident = "rust_slice" @@ -492,14 +511,10 @@ class RustSimTypeSlice(RustSimStruct, SimType): ) SimType.__init__(self, label) - def _with_arch(self, arch): - if arch.name in self._arch_memo: - return self._arch_memo[arch.name] - - out = RustSimTypeSlice(self.element_type.with_arch(arch), label=self.label, arch=arch) + def _with_arch(self, arch, *, memo: dict[str, SimType]): + out = RustSimTypeSlice(self.element_type.with_arch(arch, memo=memo), label=self.label, arch=arch) out._arch = arch out._size = self._size - self._arch_memo[arch.name] = out return out @@ -563,14 +578,14 @@ class RustSimTypeVec(RustSimStruct, SimType): self.element_type = element_type self.order = order - def _with_arch(self, arch): - if arch.name in self._arch_memo: - return self._arch_memo[arch.name] + def _with_arch(self, arch, *, memo: dict[str, SimType]): + if self.name in memo: + return cast(RustSimTypeVec, memo[self.name]) out = RustSimTypeVec(self.element_type, self.order, label=self.label, arch=arch) out._arch = arch out._size = self._size - self._arch_memo[arch.name] = out + memo[self.name] = out return out @@ -614,6 +629,9 @@ class RustSimTypeBottom(RustSimType, SimTypeBottom): def repr(self, name=None, full=0, memo=None, indent: int | None = 0): return "BOT" + def _with_arch(self, arch, *, memo: dict[str, SimType]) -> Self: # pylint: disable=unused-argument + return self + class EnumVariant: def __init__(self, name, fields, discriminant, discriminant_size): @@ -682,8 +700,13 @@ class EnumVariant: return result.with_arch(self._arch) return result - def with_arch(self, arch): - fields = [(field_ty.with_arch(arch), name) for field_ty, name in self.fields] + def with_arch(self, arch, memo: dict[str, SimType] | None = None): + if memo is None: + memo = {} + return self._with_arch(arch, memo=memo) + + def _with_arch(self, arch, *, memo: dict[str, SimType]): + fields = [(field_ty.with_arch(arch, memo=memo), name) for field_ty, name in self.fields] result = EnumVariant(self.name, fields, self.discriminant, self.discriminant_size) result._arch = arch return result @@ -740,10 +763,15 @@ class RustSimEnum(RustSimType, SimType): out._size = self._size return out - def _with_arch(self, arch): - out = RustSimEnum(self.name, [variant.with_arch(arch) for variant in self.variants]) + def _with_arch(self, arch, *, memo: dict[str, SimType]): + if self.name in memo: + return cast(RustSimEnum, memo[self.name]) + + out = RustSimEnum(self.name, [variant.with_arch(arch, memo=memo) for variant in self.variants]) out._arch = arch out._size = self._size + memo[self.name] = out + return out def repr(self, name=None, full=0, memo=None, indent: int | None = 0): @@ -834,7 +862,10 @@ class RustSimTypeOption(RustSimEnum): out._size = self._size return out - def _with_arch(self, arch): + def _with_arch(self, arch, *, memo: dict[str, SimType]): + if self.name in memo: + return cast(RustSimTypeOption, memo[self.name]) + out = RustSimTypeOption( self.none_discriminant, self.none_discriminant_size, @@ -843,8 +874,10 @@ class RustSimTypeOption(RustSimEnum): self.some_discriminant_size, self.name, ) + memo[self.name] = out + out._arch = arch - out.variants = [variant.with_arch(arch) for variant in out.variants] + out.variants = [variant.with_arch(arch, memo=memo) for variant in out.variants] out._size = self._size return out @@ -925,18 +958,23 @@ class RustSimTypeResult(RustSimEnum): out._size = self._size return out - def _with_arch(self, arch): + def _with_arch(self, arch, *, memo: dict[str, SimType]): + if self.name in memo: + return cast(RustSimTypeResult, memo[self.name]) + out = RustSimTypeResult( - self.ok_type.with_arch(arch), + self.ok_type.with_arch(arch, memo=memo), self.ok_discriminant, self.ok_discriminant_size, - self.err_type.with_arch(arch), + self.err_type.with_arch(arch, memo=memo), self.err_discriminant, self.err_discriminant_size, self.name, ) + memo[self.name] = out + out._arch = arch - out.variants = [variant.with_arch(arch) for variant in out.variants] + out.variants = [variant.with_arch(arch, memo=memo) for variant in out.variants] out._size = self._size return out @@ -995,17 +1033,12 @@ class RustSimTypeUnit(RustSimStruct): out._size = self._size return out - def _with_arch(self, arch): - if arch.name in self._arch_memo: - return self._arch_memo[arch.name] - + def _with_arch(self, arch, *, memo: dict[str, SimType]): # pylint: disable=unused-argument out = RustSimTypeUnit() out._arch = arch out.fields = OrderedDict(()) out._size = self._size - self._arch_memo[arch.name] = out - return out @property @@ -1032,17 +1065,12 @@ class RustSimTypeStrRef(RustSimTypeSlice): out._size = self._size return out - def _with_arch(self, arch): - if arch.name in self._arch_memo: - return self._arch_memo[arch.name] - + def _with_arch(self, arch, *, memo: dict[str, SimType]): out = RustSimTypeStrRef() out._arch = arch - out.fields = OrderedDict((k, v.with_arch(arch)) for k, v in self.fields.items()) + out.fields = OrderedDict((k, v.with_arch(arch, memo=memo)) for k, v in self.fields.items()) out._size = self._size - self._arch_memo[arch.name] = out - return out def to_json(self, fields=None, memo=None): diff --git a/angr/rust/typehoon/translator.py b/angr/rust/typehoon/translator.py index 62457f9fe..9ebe4d2b8 100644 --- a/angr/rust/typehoon/translator.py +++ b/angr/rust/typehoon/translator.py @@ -241,7 +241,7 @@ class RustTypeTranslator(TypeTranslator): # Utility # ---------------------------------------------------------------- - def ctype2rust(self, simtype: sim_type.SimType | RustSimType): + def ctype2rust(self, simtype: sim_type.SimType): if isinstance(simtype, RustSimType): return simtype if isinstance(simtype, SimTypeNum): diff --git a/angr/rust/typehoon/typehoon.py b/angr/rust/typehoon/typehoon.py index 70fc92802..1a502cdfd 100644 --- a/angr/rust/typehoon/typehoon.py +++ b/angr/rust/typehoon/typehoon.py @@ -78,10 +78,10 @@ class RustTypehoon(Typehoon): # determine the best type - this logic can be made better! if not type_candidates: continue + type_candidates = [t.with_arch(self.project.arch) for t in type_candidates] if len(type_candidates) > 1: types_by_size: dict[int, list[SimType]] = defaultdict(list) for t in type_candidates: - t = t.with_arch(self.project.arch) if t.size is not None: types_by_size[t.size].append(t) if not types_by_size: @@ -94,7 +94,9 @@ class RustTypehoon(Typehoon): the_type = type_candidates[0] if isinstance(the_type, SimTypeBottom) and var.size is not None: - the_type = RustSimTypeInt(signed=False, size=var.size * self.project.arch.byte_width) + the_type = RustSimTypeInt(signed=False, size=var.size * self.project.arch.byte_width).with_arch( + self.project.arch + ) if func_addr != "global": the_type = self._flatten_pointer_to_array(the_type, self.project.arch) diff --git a/angr/sim_type.py b/angr/sim_type.py index 44bf76f27..979070bef 100644 --- a/angr/sim_type.py +++ b/angr/sim_type.py @@ -130,14 +130,15 @@ class SimType: return self._arch.bytes return self.size // self._arch.byte_width - def with_arch(self, arch: Arch | None): + def with_arch(self, arch: Arch | None, memo: dict[str, SimType] | None = None) -> SimType: if arch is None: return self if self._arch is not None and self._arch == arch: return self - return self._with_arch(arch) + memo = memo if memo is not None else {} + return self._with_arch(arch, memo=memo) - def _with_arch(self, arch): + def _with_arch(self, arch, *, memo: dict[str, SimType]): # pylint: disable=unused-argument cp = copy.copy(self) cp._arch = arch return cp @@ -307,8 +308,8 @@ class TypeRef(SimType): def alignment(self): return self.type.alignment - def with_arch(self, arch): - self.type = self.type.with_arch(arch) + def with_arch(self, arch, memo: dict[str, SimType] | None = None): + self.type = self.type.with_arch(arch, memo=memo) self._arch = arch return self @@ -969,9 +970,13 @@ class SimTypePointer(SimTypeReg): raise ValueError("Can't tell my size without an arch!") return self._arch.bits - def _with_arch(self, arch): + def _with_arch(self, arch, *, memo: dict[str, SimType]): out = SimTypePointer( - self.pts_to.with_arch(arch), self.label, self.offset, qualifier=self.qualifier, disposition=self.disposition + self.pts_to.with_arch(arch, memo=memo), + self.label, + self.offset, + qualifier=self.qualifier, + disposition=self.disposition, ) out._arch = arch return out @@ -1035,8 +1040,8 @@ class SimTypeReference(SimTypeReg): raise ValueError("Can't tell my size without an arch!") return self._arch.bits - def _with_arch(self, arch): - out = SimTypeReference(self.refs.with_arch(arch), label=self.label) + def _with_arch(self, arch, *, memo: dict[str, SimType]): + out = SimTypeReference(self.refs.with_arch(arch, memo=memo), label=self.label) out._arch = arch return out @@ -1111,8 +1116,8 @@ class SimTypeArray(SimType): def alignment(self): return self.elem_type.alignment - def _with_arch(self, arch): - out = SimTypeArray(self.elem_type.with_arch(arch), self.length, self.label) + def _with_arch(self, arch, *, memo: dict[str, SimType]): + out = SimTypeArray(self.elem_type.with_arch(arch, memo=memo), self.length, self.label, qualifier=self.qualifier) out._arch = arch return out @@ -1235,7 +1240,7 @@ class SimTypeString(NamedTypeMixin, SimType): def alignment(self): return 1 - def _with_arch(self, arch): + def _with_arch(self, arch, *, memo: dict[str, SimType]): # pylint: disable=unused-argument return self def copy(self): @@ -1320,7 +1325,7 @@ class SimTypeWString(NamedTypeMixin, SimType): def alignment(self): return 2 - def _with_arch(self, arch): + def _with_arch(self, arch, *, memo: dict[str, SimType]): # pylint: disable=unused-argument return self def copy(self): @@ -1399,10 +1404,10 @@ class SimTypeFunction(SimType): def size(self): return 4096 # ??????????? - def _with_arch(self, arch): + def _with_arch(self, arch, *, memo: dict[str, SimType]): out = SimTypeFunction( - [a.with_arch(arch) for a in self.args], - self.returnty.with_arch(arch) if self.returnty is not None else None, + [a.with_arch(arch, memo=memo) for a in self.args], + self.returnty.with_arch(arch, memo=memo) if self.returnty is not None else None, label=self.label, arg_names=self.arg_names, variadic=self.variadic, @@ -1475,10 +1480,10 @@ class SimTypeCppFunction(SimTypeFunction): ", variadic=True" if self.variadic else "", ) - def _with_arch(self, arch): + def _with_arch(self, arch, *, memo: dict[str, SimType]): out = SimTypeCppFunction( - [a.with_arch(arch) for a in self.args], - self.returnty.with_arch(arch) if self.returnty is not None else None, + [a.with_arch(arch, memo=memo) for a in self.args], + self.returnty.with_arch(arch, memo=memo) if self.returnty is not None else None, label=self.label, arg_names=self.arg_names, ctor=self.ctor, @@ -1643,10 +1648,14 @@ class SimStruct(NamedTypeMixin, SimType): if self.name == "_Anonymous_e__Struct": self.anonymous = True - self._arch_memo = {} # An optional, name-independent ordering id assigned by the type translator. self._def_order: int | None = def_order + # A memo that stores the ID of SimStruct and SimUnion instances that have been visited during size or offset + # calculation. this is used to avoid infinite recursion in pathological cases where a struct or a union + # contains itself not via pointers. + self._size_memo: set[int] | None = None + # # pack and align are for supporting SimType.from_json and SimType.to_json # @@ -1734,16 +1743,16 @@ class SimStruct(NamedTypeMixin, SimType): return SimStructValue(self, values=values) - def _with_arch(self, arch): - if arch.name in self._arch_memo: - return self._arch_memo[arch.name] + def _with_arch(self, arch, *, memo: dict[str, SimType]): + if self.name in memo: + return cast(SimStruct, memo[self.name]) out = SimStruct({}, name=self.name, pack=self._pack, align=self._align) out._arch = arch out._def_order = self._def_order - self._arch_memo[arch.name] = out + memo[self.name] = out - out.fields = OrderedDict((k, v.with_arch(arch)) for k, v in self.fields.items()) + out.fields = OrderedDict((k, v.with_arch(arch, memo=memo)) for k, v in self.fields.items()) # Fixup the offsets to byte aligned addresses for all SimTypeNumOffset types offset_so_far = 0 @@ -1781,24 +1790,47 @@ class SimStruct(NamedTypeMixin, SimType): def size(self): if not self.offsets: return 0 + if self._size_memo is None: + self._size_memo = set() + if id(self) in self._size_memo: + return 0 # bad bad bad + self._size_memo.add(id(self)) if self._arch is None: raise ValueError("Need an arch to compute size") last_name, last_off = list(self.offsets.items())[-1] last_type = self.fields[last_name] if isinstance(last_type, SimTypeNumOffset): + self._size_memo.remove(id(self)) + if not self._size_memo: + self._size_memo = None return last_off * self._arch.byte_width + (last_type.size + last_type.offset) if last_type.size is None: raise AngrTypeError("Cannot compute the size of a struct with elements with no size") + self._size_memo.remove(id(self)) + if not self._size_memo: + self._size_memo = None return last_off * self._arch.byte_width + last_type.size @property def alignment(self): if self._align is not None: return self._align + if self._size_memo is None: + self._size_memo = set() + if id(self) in self._size_memo: + return 1 # bad bad bad + self._size_memo.add(id(self)) if all(val.alignment is NotImplemented for val in self.fields.values()): + self._size_memo.remove(id(self)) + if not self._size_memo: + self._size_memo = None return NotImplemented - return max(val.alignment if val.alignment is not NotImplemented else 1 for val in self.fields.values()) + max_alignment = max(val.alignment if val.alignment is not NotImplemented else 1 for val in self.fields.values()) + self._size_memo.remove(id(self)) + if not self._size_memo: + self._size_memo = None + return max_alignment def _refine_dir(self): return list(self.fields.keys()) @@ -1944,22 +1976,57 @@ class SimUnion(NamedTypeMixin, SimType): if qualifier: self.qualifier = qualifier + # A memo that stores the ID of SimStruct and SimUnion instances that have been visited during size or offset + # calculation. this is used to avoid infinite recursion in pathological cases where a struct or a union + # contains itself not via pointers. + self._size_memo: set[int] | None = None + + # cached alignment + self._alignment: int | None = None + @property def size(self): if self._arch is None: raise ValueError("Can't tell my size without an arch!") + if self._size_memo is None: + self._size_memo = set() + if id(self) in self._size_memo: + return 0 # bad bad bad + self._size_memo.add(id(self)) all_member_sizes: list[int | None] = [ ty.size for ty in self.members.values() if not isinstance(ty, (SimTypeBottom, SimTypeRef)) ] member_sizes: list[int] = [s for s in all_member_sizes if s is not None] # fall back to word size in case all members are SimTypeBottom - return max(member_sizes) if member_sizes else self._arch.bytes + max_size = max(member_sizes) if member_sizes else self._arch.bytes + + self._size_memo.remove(id(self)) + if not self._size_memo: + self._size_memo = None + + return max_size @property def alignment(self): + if self._alignment is not None: + return self._alignment + + if self._size_memo is None: + self._size_memo = set() + if id(self) in self._size_memo: + return 1 # bad bad bad + self._size_memo.add(id(self)) if all(val.alignment is NotImplemented for val in self.members.values()): - return NotImplemented - return max(val.alignment if val.alignment is not NotImplemented else 1 for val in self.members.values()) + r = NotImplemented + else: + r = max(val.alignment if val.alignment is not NotImplemented else 1 for val in self.members.values()) + + self._size_memo.remove(id(self)) + if not self._size_memo: + self._size_memo = None + + self._alignment = r + return r def _refine_dir(self): return list(self.members.keys()) @@ -2018,8 +2085,8 @@ class SimUnion(NamedTypeMixin, SimType): def __str__(self): return f"union {self.name}" - def _with_arch(self, arch): - out = SimUnion({name: ty.with_arch(arch) for name, ty in self.members.items()}, self.label) + def _with_arch(self, arch, *, memo: dict[str, SimType]): + out = SimUnion({name: ty.with_arch(arch, memo=memo) for name, ty in self.members.items()}, self.label) out._arch = arch return out @@ -2115,10 +2182,10 @@ class SimTypeEnum(NamedTypeMixin, SimType): """ return self._reverse_members.get(value) - def _with_arch(self, arch): + def _with_arch(self, arch, *, memo: dict[str, SimType]): out = SimTypeEnum( members=self.members, - base_type=self._base_type.with_arch(arch), + base_type=self._base_type.with_arch(arch, memo=memo), name=self._name, qualifier=self.qualifier, ) @@ -2278,10 +2345,10 @@ class SimTypeBitfield(NamedTypeMixin, SimType): """ return not self.has_unknown_bits(value) - def _with_arch(self, arch): + def _with_arch(self, arch, *, memo: dict[str, SimType]): out = SimTypeBitfield( flags=self.flags, - base_type=self._base_type.with_arch(arch), + base_type=self._base_type.with_arch(arch, memo=memo), name=self._name, qualifier=self.qualifier, ) @@ -2440,9 +2507,9 @@ class SimCppClass(SimStruct): ty = self.fields[field] ty.store(state, addr + offset, value[field]) - def _with_arch(self, arch) -> SimCppClass: - if arch.name in self._arch_memo: - return self._arch_memo[arch.name] + def _with_arch(self, arch, *, memo: dict[str, SimType]) -> SimCppClass: + if self.name in memo: + return cast(SimCppClass, memo[self.name]) out = SimCppClass( unique_name=self.unique_name, @@ -2456,11 +2523,13 @@ class SimCppClass(SimStruct): ) out._arch = arch out._def_order = self._def_order - self._arch_memo[arch.name] = out + memo[self.name] = out - out.members = OrderedDict((k, v.with_arch(arch)) for k, v in self.members.items()) + out.members = OrderedDict((k, v.with_arch(arch, memo=memo)) for k, v in self.members.items()) out.function_members = ( - OrderedDict((k, v.with_arch(arch)) for k, v in self.function_members.items()) + OrderedDict( + (k, cast(SimTypeCppFunction, v.with_arch(arch, memo=memo))) for k, v in self.function_members.items() + ) if self.function_members is not None else None ) diff --git a/angr/utils/types.py b/angr/utils/types.py index 196d9670e..4a18a5769 100644 --- a/angr/utils/types.py +++ b/angr/utils/types.py @@ -78,7 +78,7 @@ def squash_array_reference(ty): def dereference_simtype( - t: SimType, type_collections: list[SimTypeCollection], memo: dict[str, SimType] | None = None + t: SimType, type_collections: list[SimTypeCollection], memo: dict[str | int, SimType] | None = None ) -> SimType: if memo is None: memo = {} @@ -104,12 +104,11 @@ def dereference_simtype( # the following code prepares a real_type SimType object that will be returned at the end of this method if isinstance(t, SimStruct): - if t.name in memo: - return memo[t.name] + if t.name in memo or (t.anonymous and id(t) in memo): + return memo[t.name if not t.anonymous else id(t)] real_type = t.copy() - if not t.anonymous: - memo[t.name] = real_type + memo[t.name if not t.anonymous else id(t)] = real_type fields = OrderedDict((k, dereference_simtype(v, type_collections, memo=memo)) for k, v in t.fields.items()) real_type.fields = fields elif isinstance(t, SimTypePointer): @@ -121,6 +120,7 @@ def dereference_simtype( real_type = t.copy() real_type.elem_type = real_elem_type elif isinstance(t, SimUnion): + memo[t.name] = t real_members = {k: dereference_simtype(v, type_collections, memo=memo) for k, v in t.members.items()} real_type = t.copy() real_type.members = real_members diff --git a/tests/analyses/test_rust_sim_type.py b/tests/analyses/test_rust_sim_type.py index 40f5bdc9a..2b061da18 100644 --- a/tests/analyses/test_rust_sim_type.py +++ b/tests/analyses/test_rust_sim_type.py @@ -1,5 +1,6 @@ from __future__ import annotations +import unittest from collections import OrderedDict import archinfo @@ -30,226 +31,6 @@ from angr.rust.typehoon.translator import RustTypeTranslator from angr.sim_type import SimTypeArray, SimTypeFunction, SimTypeLongLong -def test_rust_retbuf_function_normalization(): - arch = archinfo.ArchAMD64() - field_ty = RustSimTypeInt(64, signed=False).with_arch(arch) - ret_ty = RustSimStruct(OrderedDict({"field_0": field_ty}), name="Ret", pack=True).with_arch(arch) - prototype = RustSimTypeFunction( - [RustSimTypeReference(ret_ty).with_arch(arch), field_ty], - None, - is_arg0_retbuf=True, - ).with_arch(arch) - - normalized = prototype.normalize() - - assert is_composite_type(ret_ty) - assert normalized.returnty is ret_ty - assert tuple(normalized.args) == (field_ty,) - assert normalized.is_arg0_retbuf is False - assert prototype.normalize() is not prototype - - -def test_rust_scalar_reference_and_array_repr_json_roundtrip(): - arch = archinfo.ArchAMD64() - u32 = RustSimTypeInt(32, signed=False, label="len").with_arch(arch) - - assert repr(u32) == "u32" - assert u32.repr("n") == "n: u32" - assert RustSimTypeInt.from_json(u32.to_json()).label == "len" - - usize = RustSimTypeSize(signed=False).with_arch(arch) - assert usize.size == 64 - assert repr(usize) == "usize" - assert RustSimTypeSize.from_json(usize.to_json()).signed is False - assert usize.copy().size == 64 - - bottom_ref = RustSimTypeReference(RustSimTypeBottom()) - assert bottom_ref.repr("ptr") == "*u8 ptr" - - ref = RustSimTypeReference(u32, label="r", offset=4).with_arch(arch) - assert ref.size == 64 - assert ref.repr("arg") == "arg: &u32" - assert ref.copy().offset == 4 - - array = RustSimTypeArray(u32, length=3, label="arr").with_arch(arch) - assert repr(array) == "[u32; 3]" - assert array.repr("items") == "items: [u32; 3]" - assert array.copy().length == 3 - - fn = RustSimTypeFunction([ref, u32], u32, arg_names=["self", "n"], variadic=True).with_arch(arch) - assert "..." in repr(fn) - assert "..." in fn._repr("callee", full=1) - assert '"self"' in fn._arg_names_str() - assert fn.to_json()["variadic"] is True - - -def test_rust_int_equality_includes_size(): - # regression test for angr/angr#6625: ints of different sizes compared equal (and hashed equal), which made - # the declared type of unified variables depend on nondeterministic set iteration order - u8 = RustSimTypeInt(8, signed=False) - u32 = RustSimTypeInt(32, signed=False) - u64 = RustSimTypeInt(64, signed=False) - - assert u32 != u64 - assert u8 != u32 - assert hash(u32) != hash(u64) - assert u32 == RustSimTypeInt(32, signed=False) - assert hash(u32) == hash(RustSimTypeInt(32, signed=False)) - assert u32 != RustSimTypeInt(32, signed=True) - - # copy() must preserve the explicit size - assert u64.copy().size == 64 - assert u64.copy() == u64 - - -def test_rust_struct_nested_field_lookup_and_json_roundtrip(): - arch = archinfo.ArchAMD64() - inner = RustSimStruct(OrderedDict({"value": RustSimTypeInt(16, signed=False)}), name="Inner", pack=True).with_arch( - arch - ) - outer = RustSimStruct( - OrderedDict({"inner": inner, "tail": RustSimTypeInt(32, signed=False)}), name="Outer", pack=True - ).with_arch(arch) - - inner_value_ty = outer.get_field_ty("inner.value") - assert inner_value_ty is not None - assert inner_value_ty.size == 16 - assert outer.get_field_offset("inner.value") == 0 - assert outer.get_field_offset("missing", default=-1) == -1 - assert "struct Outer" in outer.repr(full=2) - - data = outer.to_json() - data["_size"] = outer.size - restored = RustSimStruct.from_json(data) - assert restored.name == "Outer" - assert restored._size == outer.size - - -def test_rust_enum_result_option_discriminants_and_json_roundtrip(): - arch = archinfo.ArchAMD64() - ok_ty = RustSimTypeInt(64, signed=False) - err_ty = RustSimTypeInt(16, signed=False) - - result_ty = RustSimTypeResult(ok_ty, 0, 0, err_ty, -(1 << 15), 2).with_arch(arch) - err_variant = result_ty.get_variant(1 << 15) - assert err_variant is not None - assert err_variant.name == "Err" - assert RustSimTypeResult.from_json(result_ty.to_json()).name.startswith("Result<") - - option_ty = RustSimTypeOption(0, 1, ok_ty, 1, 1).with_arch(arch) - none_variant = option_ty.get_variant(0) - assert none_variant is not None - assert none_variant.name == "None" - assert RustSimTypeOption.from_json(option_ty.to_json()).name.startswith("Option<") - - none = EnumVariant.from_no_data("None", 0, 1) - some = EnumVariant.from_single_field_ty("Some", ok_ty, 1, 1) - enum_ty = RustSimEnum("OptionLike", [none, some]).with_arch(arch) - some_variant = enum_ty.get_variant_by_name("Some") - assert some_variant is not None - assert some_variant.name == "Some" - assert enum_ty.num_variants() == 2 - assert RustSimEnum.from_json(enum_ty.to_json()).name == "OptionLike" - - some_with_arch = some.with_arch(arch) - assert some_with_arch.has_fields() - assert some_with_arch.first_field_offset >= 1 - assert some_with_arch.size == some_with_arch.bits // 8 - assert some_with_arch.as_struct_ty().fields["discriminant"].size == 8 - assert EnumVariant.from_json(some_with_arch.to_json()) == some - - -def test_rust_slice_layout_uses_two_machine_words(): - arch = archinfo.ArchAMD64() - slice_ty = RustSimTypeSlice(RustSimTypeInt(8, signed=False)).with_arch(arch) - - assert slice_ty.size == 128 - assert list(slice_ty.fields) == ["data_ptr", "length"] - assert slice_ty.repr("s") == "s: &[u8]" - - vec_ty = RustSimTypeVec(RustSimTypeInt(16, signed=False), order=("ptr", "len", "cap")).with_arch(arch) - assert repr(vec_ty) == "Vec" - assert list(vec_ty.fields) == ["ptr", "len", "cap"] - assert RustSimTypeVec.from_json(vec_ty.to_json()).order == ("ptr", "len", "cap") - - unit_ty = RustSimTypeUnit().with_arch(arch) - assert unit_ty.size == 0 - assert unit_ty.copy().name == "()" - assert RustSimTypeUnit.from_json(unit_ty.to_json()).name == "()" - - strref_ty = RustSimTypeStrRef().with_arch(arch) - assert repr(strref_ty) == "&str" - assert strref_ty.copy().name == "&str" - assert RustSimTypeStrRef.from_json(strref_ty.to_json()).name == "&str" - - -def test_rust_type_translator_handles_rust_simtypes_and_type_constants(): - arch = archinfo.ArchAMD64() - translator = RustTypeTranslator(arch) - - struct_tc = typeconsts.Struct( - fields={0: typeconsts.Int16(), 4: typeconsts.Pointer64(typeconsts.Int8())}, - field_names={0: "tag", 4: "ptr"}, - name="Pair", - ) - struct_ty, has_nonexistent_ref = translator.tc2simtype(struct_tc) - assert has_nonexistent_ref is False - assert isinstance(struct_ty, RustSimStruct) - assert struct_ty.name == "Pair" - assert list(struct_ty.fields) == ["tag", "ptr"] - assert isinstance(struct_ty.fields["tag"], RustSimTypeInt) - assert struct_ty.fields["tag"].size == 16 - assert isinstance(struct_ty.fields["ptr"], RustSimTypeReference) - - array_ty, has_nonexistent_ref = translator.tc2simtype(typeconsts.Array(typeconsts.Int32(), 2)) - assert has_nonexistent_ref is False - assert isinstance(array_ty, RustSimTypeArray) - assert array_ty.length == 2 - assert isinstance(array_ty.elem_type, RustSimTypeInt) - assert array_ty.elem_type.size == 32 - - result_tc = typeconsts.RustEnum( - "core::result::Result", - [ - typeconsts.EnumVariant("Ok", [(typeconsts.Int64(), "__0")], 0, 1, 8), - typeconsts.EnumVariant("Err", [(typeconsts.Int16(), "__0")], 1, 1, 2), - ], - ) - result_ty, has_nonexistent_ref = translator.tc2simtype(result_tc) - assert has_nonexistent_ref is False - assert isinstance(result_ty, RustSimTypeResult) - assert result_ty.get_variant(0) is not None - - option_tc = typeconsts.RustEnum( - "core::option::Option", - [ - typeconsts.EnumVariant("None", [], 0, 1, 0), - typeconsts.EnumVariant("Some", [(typeconsts.Int32(), "__0")], 1, 1, 4), - ], - ) - option_ty, has_nonexistent_ref = translator.tc2simtype(option_tc) - assert has_nonexistent_ref is False - assert isinstance(option_ty, RustSimTypeOption) - assert option_ty.get_variant_by_name("Some") is not None - - lifted_struct = translator.simtype2tc( - RustSimStruct(OrderedDict({"value": RustSimTypeInt(32, signed=False)}), name="Lifted", pack=True).with_arch( - arch - ) - ) - assert isinstance(lifted_struct, typeconsts.Struct) - assert lifted_struct.field_names == {0: "value"} - - lifted_enum = translator.simtype2tc( - RustSimEnum( - "EnumLike", - [EnumVariant.from_no_data("None", 0, 1), EnumVariant.from_single_field_ty("Some", RustSimTypeInt(8), 1, 1)], - ).with_arch(arch) - ) - assert isinstance(lifted_enum, typeconsts.RustEnum) - assert lifted_enum.get_variant("Some") is not None - - def _blank_type_db_loader() -> TypeDBLoader: project = angr.load_shellcode(b"\x90", arch="amd64") loader = object.__new__(TypeDBLoader) @@ -261,93 +42,335 @@ def _blank_type_db_loader() -> TypeDBLoader: return loader -def test_type_db_loader_parses_structs_slices_and_enums(): - loader = _blank_type_db_loader() +class TestRustSimType(unittest.TestCase): + def test_with_arch_never_strips_an_existing_arch(self): + # Function.prototype's setter copies arch-less prototypes, and copy() ends in with_arch(self._arch); + # rebuilding there instead of returning self strips the arch off every argument + arch = archinfo.ArchAMD64() + u64 = RustSimTypeInt(64, signed=False).with_arch(arch) + ok_ty = RustSimStruct(OrderedDict({"field_0": u64}), name="struct8", pack=True).with_arch(arch) + err_ty = RustSimStruct(OrderedDict({"field_0": u64}), name="struct16", pack=True).with_arch(arch) + ref = RustSimTypeReference(RustSimTypeResult(ok_ty, 0, 8, err_ty, 1, 8).with_arch(arch)).with_arch(arch) - bool_ty = loader._parse_type({"kind": "Primitive", "name": "bool", "size": 1}) - assert bool_ty is not None - assert bool_ty.size == 8 - assert loader._parse_type({"kind": "Primitive", "name": "f32", "size": 4}) is None + assert ref.with_arch(None) is ref + assert ref.with_arch(arch) is ref - str_data = { - "kind": "Struct", - "name": "&str", - "fields": { - "0": ["data_ptr", {"kind": "Pointer", "pts_to": {"kind": "Primitive", "name": "u8", "size": 1}}], - "8": ["length", {"kind": "Primitive", "name": "usize", "size": 8}], - }, - } - str_ty = loader._parse_type(str_data) - assert isinstance(str_ty, RustSimTypeStrRef) + prototype = RustSimTypeFunction([ref, u64], None, is_arg0_retbuf=True) + assert prototype._arch is None + assert prototype.copy().args[0]._arch == arch - vec_data = { - "kind": "Struct", - "name": "Vec2", - "fields": { - "0": ["items", {"kind": "Array", "ele_type": {"kind": "Primitive", "name": "u16", "size": 2}, "length": 2}] - }, - } - vec_ty = loader._parse_type(vec_data) - assert isinstance(vec_ty, RustSimStruct) - assert isinstance(vec_ty.fields["items"], RustSimTypeArray) + # every type entering the type solver has to carry an arch + RustTypeTranslator(arch).simtype2tc(prototype.copy().args[0]) - option_ty = loader._parse_type( - { - "kind": "Enumeration", - "name": "core::option::Option", - "discriminant_size": 1, - "variants": { - "None": [0, []], - "Some": [1, [["__0", {"kind": "Primitive", "name": "u32", "size": 4}]]], + def test_rust_retbuf_function_normalization(self): + arch = archinfo.ArchAMD64() + field_ty = RustSimTypeInt(64, signed=False).with_arch(arch) + ret_ty = RustSimStruct(OrderedDict({"field_0": field_ty}), name="Ret", pack=True).with_arch(arch) + prototype = RustSimTypeFunction( + [RustSimTypeReference(ret_ty).with_arch(arch), field_ty], + None, + is_arg0_retbuf=True, + ).with_arch(arch) + + normalized = prototype.normalize() + + assert is_composite_type(ret_ty) + assert normalized.returnty == ret_ty + assert tuple(normalized.args) == (field_ty,) + assert normalized.is_arg0_retbuf is False + assert prototype.normalize() is not prototype + + def test_rust_scalar_reference_and_array_repr_json_roundtrip(self): + arch = archinfo.ArchAMD64() + u32 = RustSimTypeInt(32, signed=False, label="len").with_arch(arch) + + assert repr(u32) == "u32" + assert u32.repr("n") == "n: u32" + assert RustSimTypeInt.from_json(u32.to_json()).label == "len" + + usize = RustSimTypeSize(signed=False).with_arch(arch) + assert usize.size == 64 + assert repr(usize) == "usize" + assert RustSimTypeSize.from_json(usize.to_json()).signed is False + assert usize.copy().size == 64 + + bottom_ref = RustSimTypeReference(RustSimTypeBottom()) + assert bottom_ref.repr("ptr") == "*u8 ptr" + + ref = RustSimTypeReference(u32, label="r", offset=4).with_arch(arch) + assert ref.size == 64 + assert ref.repr("arg") == "arg: &u32" + assert ref.copy().offset == 4 + + array = RustSimTypeArray(u32, length=3, label="arr").with_arch(arch) + assert repr(array) == "[u32; 3]" + assert array.repr("items") == "items: [u32; 3]" + assert array.copy().length == 3 + + fn = RustSimTypeFunction([ref, u32], u32, arg_names=["self", "n"], variadic=True).with_arch(arch) + assert "..." in repr(fn) + assert "..." in fn._repr("callee", full=1) + assert '"self"' in fn._arg_names_str() + assert fn.to_json()["variadic"] is True + + def test_rust_int_equality_includes_size(self): + # regression test for angr/angr#6625: ints of different sizes compared equal (and hashed equal), which made + # the declared type of unified variables depend on nondeterministic set iteration order + u8 = RustSimTypeInt(8, signed=False) + u32 = RustSimTypeInt(32, signed=False) + u64 = RustSimTypeInt(64, signed=False) + + assert u32 != u64 + assert u8 != u32 + assert hash(u32) != hash(u64) + assert u32 == RustSimTypeInt(32, signed=False) + assert hash(u32) == hash(RustSimTypeInt(32, signed=False)) + assert u32 != RustSimTypeInt(32, signed=True) + + # copy() must preserve the explicit size + assert u64.copy().size == 64 + assert u64.copy() == u64 + + def test_rust_struct_nested_field_lookup_and_json_roundtrip(self): + arch = archinfo.ArchAMD64() + inner = RustSimStruct( + OrderedDict({"value": RustSimTypeInt(16, signed=False)}), name="Inner", pack=True + ).with_arch(arch) + outer = RustSimStruct( + OrderedDict({"inner": inner, "tail": RustSimTypeInt(32, signed=False)}), name="Outer", pack=True + ).with_arch(arch) + + inner_value_ty = outer.get_field_ty("inner.value") + assert inner_value_ty is not None + assert inner_value_ty.size == 16 + assert outer.get_field_offset("inner.value") == 0 + assert outer.get_field_offset("missing", default=-1) == -1 + assert "struct Outer" in outer.repr(full=2) + + data = outer.to_json() + data["_size"] = outer.size + restored = RustSimStruct.from_json(data) + assert restored.name == "Outer" + assert restored._size == outer.size + + def test_rust_enum_result_option_discriminants_and_json_roundtrip(self): + arch = archinfo.ArchAMD64() + ok_ty = RustSimTypeInt(64, signed=False) + err_ty = RustSimTypeInt(16, signed=False) + + result_ty = RustSimTypeResult(ok_ty, 0, 0, err_ty, -(1 << 15), 2).with_arch(arch) + err_variant = result_ty.get_variant(1 << 15) + assert err_variant is not None + assert err_variant.name == "Err" + assert RustSimTypeResult.from_json(result_ty.to_json()).name.startswith("Result<") + + option_ty = RustSimTypeOption(0, 1, ok_ty, 1, 1).with_arch(arch) + none_variant = option_ty.get_variant(0) + assert none_variant is not None + assert none_variant.name == "None" + assert RustSimTypeOption.from_json(option_ty.to_json()).name.startswith("Option<") + + none = EnumVariant.from_no_data("None", 0, 1) + some = EnumVariant.from_single_field_ty("Some", ok_ty, 1, 1) + enum_ty = RustSimEnum("OptionLike", [none, some]).with_arch(arch) + some_variant = enum_ty.get_variant_by_name("Some") + assert some_variant is not None + assert some_variant.name == "Some" + assert enum_ty.num_variants() == 2 + assert RustSimEnum.from_json(enum_ty.to_json()).name == "OptionLike" + + some_with_arch = some.with_arch(arch) + assert some_with_arch.has_fields() + assert some_with_arch.first_field_offset >= 1 + assert some_with_arch.size == some_with_arch.bits // 8 + assert some_with_arch.as_struct_ty().fields["discriminant"].size == 8 + assert EnumVariant.from_json(some_with_arch.to_json()) == some + + def test_rust_slice_layout_uses_two_machine_words(self): + arch = archinfo.ArchAMD64() + slice_ty = RustSimTypeSlice(RustSimTypeInt(8, signed=False)).with_arch(arch) + + assert slice_ty.size == 128 + assert list(slice_ty.fields) == ["data_ptr", "length"] + assert slice_ty.repr("s") == "s: &[u8]" + + vec_ty = RustSimTypeVec(RustSimTypeInt(16, signed=False), order=("ptr", "len", "cap")).with_arch(arch) + assert repr(vec_ty) == "Vec" + assert list(vec_ty.fields) == ["ptr", "len", "cap"] + assert RustSimTypeVec.from_json(vec_ty.to_json()).order == ("ptr", "len", "cap") + + unit_ty = RustSimTypeUnit().with_arch(arch) + assert unit_ty.size == 0 + assert unit_ty.copy().name == "()" + assert RustSimTypeUnit.from_json(unit_ty.to_json()).name == "()" + + strref_ty = RustSimTypeStrRef().with_arch(arch) + assert repr(strref_ty) == "&str" + assert strref_ty.copy().name == "&str" + assert RustSimTypeStrRef.from_json(strref_ty.to_json()).name == "&str" + + def test_rust_type_translator_handles_rust_simtypes_and_type_constants(self): + arch = archinfo.ArchAMD64() + translator = RustTypeTranslator(arch) + + struct_tc = typeconsts.Struct( + fields={0: typeconsts.Int16(), 4: typeconsts.Pointer64(typeconsts.Int8())}, + field_names={0: "tag", 4: "ptr"}, + name="Pair", + ) + struct_ty, has_nonexistent_ref = translator.tc2simtype(struct_tc) + assert has_nonexistent_ref is False + assert isinstance(struct_ty, RustSimStruct) + assert struct_ty.name == "Pair" + assert list(struct_ty.fields) == ["tag", "ptr"] + assert isinstance(struct_ty.fields["tag"], RustSimTypeInt) + assert struct_ty.fields["tag"].size == 16 + assert isinstance(struct_ty.fields["ptr"], RustSimTypeReference) + + array_ty, has_nonexistent_ref = translator.tc2simtype(typeconsts.Array(typeconsts.Int32(), 2)) + assert has_nonexistent_ref is False + assert isinstance(array_ty, RustSimTypeArray) + assert array_ty.length == 2 + assert isinstance(array_ty.elem_type, RustSimTypeInt) + assert array_ty.elem_type.size == 32 + + result_tc = typeconsts.RustEnum( + "core::result::Result", + [ + typeconsts.EnumVariant("Ok", [(typeconsts.Int64(), "__0")], 0, 1, 8), + typeconsts.EnumVariant("Err", [(typeconsts.Int16(), "__0")], 1, 1, 2), + ], + ) + result_ty, has_nonexistent_ref = translator.tc2simtype(result_tc) + assert has_nonexistent_ref is False + assert isinstance(result_ty, RustSimTypeResult) + assert result_ty.get_variant(0) is not None + + option_tc = typeconsts.RustEnum( + "core::option::Option", + [ + typeconsts.EnumVariant("None", [], 0, 1, 0), + typeconsts.EnumVariant("Some", [(typeconsts.Int32(), "__0")], 1, 1, 4), + ], + ) + option_ty, has_nonexistent_ref = translator.tc2simtype(option_tc) + assert has_nonexistent_ref is False + assert isinstance(option_ty, RustSimTypeOption) + assert option_ty.get_variant_by_name("Some") is not None + + lifted_struct = translator.simtype2tc( + RustSimStruct(OrderedDict({"value": RustSimTypeInt(32, signed=False)}), name="Lifted", pack=True).with_arch( + arch + ) + ) + assert isinstance(lifted_struct, typeconsts.Struct) + assert lifted_struct.field_names == {0: "value"} + + lifted_enum = translator.simtype2tc( + RustSimEnum( + "EnumLike", + [ + EnumVariant.from_no_data("None", 0, 1), + EnumVariant.from_single_field_ty("Some", RustSimTypeInt(8), 1, 1), + ], + ).with_arch(arch) + ) + assert isinstance(lifted_enum, typeconsts.RustEnum) + assert lifted_enum.get_variant("Some") is not None + + def test_type_db_loader_parses_structs_slices_and_enums(self): + loader = _blank_type_db_loader() + + bool_ty = loader._parse_type({"kind": "Primitive", "name": "bool", "size": 1}) + assert bool_ty is not None + assert bool_ty.size == 8 + assert loader._parse_type({"kind": "Primitive", "name": "f32", "size": 4}) is None + + str_data = { + "kind": "Struct", + "name": "&str", + "fields": { + "0": ["data_ptr", {"kind": "Pointer", "pts_to": {"kind": "Primitive", "name": "u8", "size": 1}}], + "8": ["length", {"kind": "Primitive", "name": "usize", "size": 8}], }, } - ) - assert isinstance(option_ty, RustSimTypeOption) + str_ty = loader._parse_type(str_data) + assert isinstance(str_ty, RustSimTypeStrRef) - result_ty = loader._parse_type( - { - "kind": "Enumeration", - "name": "core::result::Result", - "discriminant_size": 1, - "variants": { - "Ok": [0, [["__0", {"kind": "Primitive", "name": "u64", "size": 8}]]], - "Err": [1, [["__0", {"kind": "Primitive", "name": "u16", "size": 2}]]], + vec_data = { + "kind": "Struct", + "name": "Vec2", + "fields": { + "0": [ + "items", + {"kind": "Array", "ele_type": {"kind": "Primitive", "name": "u16", "size": 2}, "length": 2}, + ] }, } - ) - assert isinstance(result_ty, RustSimTypeResult) + vec_ty = loader._parse_type(vec_data) + assert isinstance(vec_ty, RustSimStruct) + assert isinstance(vec_ty.fields["items"], RustSimTypeArray) - -def test_type_db_loader_fits_and_negotiates_large_abi_types(): - loader = _blank_type_db_loader() - large_struct = RustSimStruct( - OrderedDict( + option_ty = loader._parse_type( { - "a": RustSimTypeInt(64, signed=False), - "b": RustSimTypeInt(64, signed=False), - "c": RustSimTypeInt(64, signed=False), + "kind": "Enumeration", + "name": "core::option::Option", + "discriminant_size": 1, + "variants": { + "None": [0, []], + "Some": [1, [["__0", {"kind": "Primitive", "name": "u32", "size": 4}]]], + }, } - ), - name="Large", - pack=True, - ).with_arch(loader.project.arch) + ) + assert isinstance(option_ty, RustSimTypeOption) - direct_arg = loader._fit_abi(RustSimTypeFunction([large_struct], RustSimTypeInt(32, signed=False))).with_arch( - loader.project.arch - ) - assert isinstance(direct_arg.args[0], RustSimTypeReference) - assert direct_arg.returnty is not None + result_ty = loader._parse_type( + { + "kind": "Enumeration", + "name": "core::result::Result", + "discriminant_size": 1, + "variants": { + "Ok": [0, [["__0", {"kind": "Primitive", "name": "u64", "size": 8}]]], + "Err": [1, [["__0", {"kind": "Primitive", "name": "u16", "size": 2}]]], + }, + } + ) + assert isinstance(result_ty, RustSimTypeResult) - retbuf = loader._fit_abi(RustSimTypeFunction([], large_struct)).with_arch(loader.project.arch) - assert retbuf.returnty is None - assert retbuf.is_arg0_retbuf is True - assert isinstance(retbuf.args[0], RustSimTypeReference) + def test_type_db_loader_fits_and_negotiates_large_abi_types(self): + loader = _blank_type_db_loader() + large_struct = RustSimStruct( + OrderedDict( + { + "a": RustSimTypeInt(64, signed=False), + "b": RustSimTypeInt(64, signed=False), + "c": RustSimTypeInt(64, signed=False), + } + ), + name="Large", + pack=True, + ).with_arch(loader.project.arch) - two_word_struct = RustSimStruct( - OrderedDict({"a": RustSimTypeInt(64, signed=False), "b": RustSimTypeInt(64, signed=False)}), - name="Pair", - pack=True, - ).with_arch(loader.project.arch) - rust_proto = RustSimTypeFunction([], two_word_struct).with_arch(loader.project.arch) - old_direct = SimTypeFunction([], SimTypeArray(SimTypeLongLong(signed=False), 2)).with_arch(loader.project.arch) - assert loader._negotiate_prototype(rust_proto, old_direct) is rust_proto + direct_arg = loader._fit_abi(RustSimTypeFunction([large_struct], RustSimTypeInt(32, signed=False))).with_arch( + loader.project.arch + ) + assert isinstance(direct_arg.args[0], RustSimTypeReference) + assert direct_arg.returnty is not None + + retbuf = loader._fit_abi(RustSimTypeFunction([], large_struct)).with_arch(loader.project.arch) + assert retbuf.returnty is None + assert retbuf.is_arg0_retbuf is True + assert isinstance(retbuf.args[0], RustSimTypeReference) + + two_word_struct = RustSimStruct( + OrderedDict({"a": RustSimTypeInt(64, signed=False), "b": RustSimTypeInt(64, signed=False)}), + name="Pair", + pack=True, + ).with_arch(loader.project.arch) + rust_proto = RustSimTypeFunction([], two_word_struct).with_arch(loader.project.arch) + old_direct = SimTypeFunction([], SimTypeArray(SimTypeLongLong(signed=False), 2)).with_arch(loader.project.arch) + assert loader._negotiate_prototype(rust_proto, old_direct) is rust_proto + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/analyses/test_rust_type_hints.py b/tests/analyses/test_rust_type_hints.py index 7ddd4b3f6..77a505d79 100644 --- a/tests/analyses/test_rust_type_hints.py +++ b/tests/analyses/test_rust_type_hints.py @@ -1,31 +1,40 @@ from __future__ import annotations +import unittest from collections import OrderedDict -import pytest - import angr from angr.ailment.expression import VirtualVariable, VirtualVariableCategory from angr.rust.sim_type import RustSimStruct, RustSimTypeInt -def test_rust_type_hints_are_function_scoped(): - project = angr.load_shellcode(b"\xc3", arch="amd64") - vvar = VirtualVariable(0, 7, 64, VirtualVariableCategory.REGISTER, 0) +class TestRustTypeHints(unittest.TestCase): + def test_rust_type_hints_are_function_scoped(self): + project = angr.load_shellcode(b"\xc3", arch="amd64") + vvar = VirtualVariable(0, 7, 64, VirtualVariableCategory.REGISTER, 0) - ty_a = RustSimStruct(OrderedDict({"field_0": RustSimTypeInt(64, signed=False)}), name="TypeA", pack=True).with_arch( - project.arch - ) - ty_b = RustSimStruct(OrderedDict({"field_0": RustSimTypeInt(32, signed=False)}), name="TypeB", pack=True).with_arch( - project.arch - ) + ty_a = RustSimStruct( + OrderedDict({"field_0": RustSimTypeInt(64, signed=False)}), name="TypeA", pack=True + ).with_arch(project.arch) + ty_b = RustSimStruct( + OrderedDict({"field_0": RustSimTypeInt(32, signed=False)}), name="TypeB", pack=True + ).with_arch(project.arch) - project.kb.type_hints.add_type_hint(vvar, ty_a, 0x1000) - project.kb.type_hints.add_type_hint(vvar, ty_b, 0x2000) + project.kb.type_hints.add_type_hint(vvar, ty_a, 0x1000) + project.kb.type_hints.add_type_hint(vvar, ty_b, 0x2000) - with pytest.raises(TypeError): - project.kb.type_hints.add_type_hint(vvar, ty_a) + try: + project.kb.type_hints.add_type_hint(vvar, ty_a) + except TypeError: + # good + pass + else: + assert False, "Expected TypeError when adding a type hint without specifying an address" - assert project.kb.type_hints.get_type_hints(0x1000)[vvar.varid].name == "TypeA" - assert project.kb.type_hints.get_type_hints(0x2000)[vvar.varid].name == "TypeB" - assert vvar.varid not in project.kb.type_hints.get_type_hints(0x3000) + assert project.kb.type_hints.get_type_hints(0x1000)[vvar.varid].name == "TypeA" + assert project.kb.type_hints.get_type_hints(0x2000)[vvar.varid].name == "TypeB" + assert vvar.varid not in project.kb.type_hints.get_type_hints(0x3000) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/analyses/test_typehoon.py b/tests/analyses/test_typehoon.py index 368966dc8..ba2385289 100755 --- a/tests/analyses/test_typehoon.py +++ b/tests/analyses/test_typehoon.py @@ -8,6 +8,7 @@ import os import re import unittest from collections import OrderedDict +from typing import cast import archinfo @@ -552,11 +553,11 @@ class TestTypeTranslator(unittest.TestCase): struct Alpha { struct Beta *beta; int x; }; struct Beta { struct Alpha *alpha; int y; }; The shape ``dereference_simtype`` produces for library types such as DRIVER_OBJECT/DEVICE_OBJECT. """ - alpha = SimStruct({}, name="Alpha") - beta = SimStruct({}, name="Beta") - alpha.fields = OrderedDict({"beta": SimTypePointer(beta), "x": SimTypeInt()}) - beta.fields = OrderedDict({"alpha": SimTypePointer(alpha), "y": SimTypeInt()}) - return alpha.with_arch(arch), beta.with_arch(arch) + alpha = cast(SimStruct, SimStruct({}, name="Alpha").with_arch(arch)) + beta = cast(SimStruct, SimStruct({}, name="Beta").with_arch(arch)) + alpha.fields = OrderedDict({"beta": SimTypePointer(beta).with_arch(arch), "x": SimTypeInt().with_arch(arch)}) + beta.fields = OrderedDict({"alpha": SimTypePointer(alpha).with_arch(arch), "y": SimTypeInt().with_arch(arch)}) + return alpha, beta def test_mutually_recursive_structs_keep_their_references(self): arch = archinfo.arch_from_id("amd64") diff --git a/tests/types/test_types.py b/tests/types/test_types.py index 30382e690..b6a9c7eee 100644 --- a/tests/types/test_types.py +++ b/tests/types/test_types.py @@ -3,6 +3,7 @@ from __future__ import annotations import unittest +from typing import cast import archinfo import pydemumble @@ -432,6 +433,28 @@ class TestTypes(unittest.TestCase): assert t0 == t1 # should not raise RecursionError + def test_simstruct_arch_memo_cache_leak(self): + # regression test: this bug is causing angr management to fail to display complex win32 types during + # decompilation. + # + # during dereference_simtype(), SimStruct._arch_memo may leak to the parent SimStruct, causing with_arch() to + # return an incomplete archified SimStruct. + + angr.procedures.definitions.load_win32_type_collections() + st = SimStruct({}, name="foobar") + # the leak from _arch_memo happens when dereference_simtype() processes st.fields["a"].pts_to, which pollutes + # st._arch_memo + st.fields["a"] = SimTypePointer(st) + st.fields["b"] = SimTypeRef("UNICODE_STRING", SimStruct) + + st = st.with_arch(archinfo.ArchAMD64()) + + st_deref = cast(SimStruct, dereference_simtype(st, [angr.SIM_TYPE_COLLECTIONS["win32"]])) + assert "a" in st_deref.fields + assert "b" in st_deref.fields + assert "a" in st_deref.offsets + assert "b" in st_deref.offsets # this assertion fails because st_deref.fields["b"] is a SimTypeRef + if __name__ == "__main__": unittest.main() From 3333f39fff4dbd0757207840ce8f1b5c54103422 Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 5 Aug 2026 01:42:45 -0700 Subject: [PATCH 094/122] CFGFast: Make the smart scan nodecode ratio O(log n) (#6767) --- angr/analyses/cfg/cfg_fast.py | 23 +-- angr/rustylib/__init__.pyi | 18 ++ native/angr/src/segmentlist.rs | 291 +++++++++++++++++++++++++++++-- tests/utils/test_segment_list.py | 108 ++++++++++++ 4 files changed, 401 insertions(+), 39 deletions(-) diff --git a/angr/analyses/cfg/cfg_fast.py b/angr/analyses/cfg/cfg_fast.py index 00204c60f..13ecd5c74 100644 --- a/angr/analyses/cfg/cfg_fast.py +++ b/angr/analyses/cfg/cfg_fast.py @@ -1592,28 +1592,7 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): def _nodecode_bytes_ratio(self, cutoff_addr: int, window_size: int) -> float: if cutoff_addr - 1 < 0: return 0.0 - idx = self._seg_list.search(cutoff_addr - 1) - if idx is None or idx >= len(self._seg_list): - return 0.0 - segment = self._seg_list[idx] - if segment.sort != "nodecode": - return 0.0 - - total_bytes = 0 - nodecode_bytes = 0 - while idx >= 0: - segment = self._seg_list[idx] - if segment.sort == "nodecode": - nodecode_bytes += segment.size - total_bytes += segment.size - if total_bytes >= window_size: - break - idx -= 1 - - if total_bytes < window_size: - return 0.0 - - return nodecode_bytes / total_bytes + return self._seg_list.sort_ratio_backwards(cutoff_addr - 1, window_size, "nodecode") def _next_code_addr_smart(self) -> int | None: # in the smart scanning mode, if there are more than N consecutive no-decode cases, we skip an entire window of diff --git a/angr/rustylib/__init__.pyi b/angr/rustylib/__init__.pyi index d53e45049..5d420d54b 100644 --- a/angr/rustylib/__init__.pyi +++ b/angr/rustylib/__init__.pyi @@ -81,10 +81,28 @@ class SegmentList: Checks which segment that the address `addr` should belong to, and, returns the offset of that segment. Note that the address may not actually belong to the block. + This is O(n): an index into the segment list cannot be computed any faster. Prefer the point-query methods + (:meth:`occupied_by`, :meth:`occupied_by_sort`, ...) wherever an index is not strictly needed. + :arg addr: The address to search. :returns: The offset of the segment. """ + def sort_ratio_backwards(self, addr: int, window_size: int, sort: str | None) -> float: + """ + The fraction of bytes belonging to segments of sort `sort`, among the last `window_size` occupied bytes at or + before `addr`. + + The walk starts at the segment :meth:`search` would return for `addr` and moves backwards, skipping over gaps, + until `window_size` bytes have been covered. + + :arg addr: The address to start the backwards walk at. + :arg window_size: The number of occupied bytes to cover. + :arg sort: The sort to measure. + :returns: The ratio, or 0.0 if the starting segment has a different sort or fewer than `window_size` occupied + bytes are available. + """ + def next_free_pos(self, address: int) -> int: """ Returns the next free position with respect to an address, including that address itself. diff --git a/native/angr/src/segmentlist.rs b/native/angr/src/segmentlist.rs index fccdb4227..ff73f54b1 100644 --- a/native/angr/src/segmentlist.rs +++ b/native/angr/src/segmentlist.rs @@ -1,5 +1,6 @@ use std::cmp::{max, min}; use std::collections::HashSet; +use std::ops::Range; use pyo3::{exceptions::PyStopIteration, prelude::*, types::PyTuple}; use rangemap::RangeMap; @@ -67,6 +68,38 @@ impl SegmentList { .get_key_value(&address) .map(|(range, sort)| (range.start, range.end - range.start, sort.clone())) } + + /// The last segment that starts before `before`. + /// + /// `rangemap` offers no predecessor query, and its `Overlapping` iterator is unbounded above, + /// which makes iterating it backwards linear in the number of segments above `before`. Instead, + /// grow a search window exponentially until it reaches a segment, then bisect it down to the + /// smallest window that still does: that window can only contain the segment we are after. + /// Every step is a point query, so this is O(log n * log gap). + fn prev_segment(&self, before: u64) -> Option<(&Range, &Option)> { + if before == 0 { + return None; + } + let mut hi = 1u64; + while !self.map.overlaps(&(before.saturating_sub(hi)..before)) { + if before.saturating_sub(hi) == 0 { + return None; + } + hi = hi.saturating_mul(2); + } + let mut lo = hi / 2; // does not reach a segment (0 when hi == 1) + while hi - lo > 1 { + let mid = lo + (hi - lo) / 2; + if self.map.overlaps(&(before.saturating_sub(mid)..before)) { + hi = mid; + } else { + lo = mid; + } + } + self.map + .overlapping(before.saturating_sub(hi)..before) + .next() + } } #[pymethods] @@ -111,8 +144,8 @@ impl SegmentList { }) } - pub fn __iter__(self_: Py) -> SegmentListIter { - SegmentListIter::new(self_) + pub fn __iter__(&self) -> SegmentListIter { + SegmentListIter::new(self) } #[getter] @@ -128,6 +161,9 @@ impl SegmentList { /// Checks which segment that the address `addr` should belong to, /// and returns the offset of that segment. /// Note that the address may not actually belong to the block. + /// + /// This is O(n): an index into the map cannot be computed any faster. Prefer the point-query + /// methods (`occupied_by`, `occupied_by_sort`, ...) wherever an index is not strictly needed. pub fn search(&self, addr: u64) -> Option { self.map .iter() @@ -136,6 +172,45 @@ impl SegmentList { .map(|(index, _)| index) } + /// The fraction of bytes belonging to segments of sort `sort`, among the last `window_size` + /// occupied bytes at or before `addr`. + /// + /// The walk starts at the segment `search()` would return for `addr` and moves backwards, + /// skipping over gaps, until `window_size` bytes have been covered. Returns 0.0 when the + /// starting segment has a different sort, or when fewer than `window_size` occupied bytes + /// are available. + #[pyo3(signature = (addr, window_size, sort))] + pub fn sort_ratio_backwards(&self, addr: u64, window_size: u64, sort: Option) -> f64 { + // the first segment whose end is >= addr, i.e. what search(addr) points at + let Some((mut range, mut seg_sort)) = self + .map + .overlapping(addr.saturating_sub(1)..u64::MAX) + .next() + else { + return 0.0; + }; + if *seg_sort != sort { + return 0.0; + } + + let mut total: u64 = 0; + let mut matching: u64 = 0; + loop { + let size = range.end - range.start; + if *seg_sort == sort { + matching = matching.saturating_add(size); + } + total = total.saturating_add(size); + if total >= window_size { + return matching as f64 / total as f64; + } + match self.prev_segment(range.start) { + Some((r, s)) => (range, seg_sort) = (r, s), + None => return 0.0, + } + } + } + pub fn next_free_pos(&self, address: u64) -> Option { self.map .gaps(&(address..u64::MAX)) @@ -231,17 +306,21 @@ impl SegmentList { #[pyclass] pub struct SegmentListIter { - segmentlist: Py, - idx: u64, + // snapshot taken up front: walking the map by index would be quadratic over a full iteration + segments: std::vec::IntoIter, } #[pymethods] impl SegmentListIter { #[new] - fn new(segmentlist: Py) -> Self { + fn new(segmentlist: &SegmentList) -> Self { Self { - segmentlist, - idx: 0, + segments: segmentlist + .map + .iter() + .map(|(range, sort)| Segment::new(range.start, range.end, sort.clone())) + .collect::>() + .into_iter(), } } @@ -249,16 +328,10 @@ impl SegmentListIter { self_ } - fn __next__(&mut self, py: Python<'_>) -> PyResult { - let segmentlist_ref = self.segmentlist.bind(py).borrow(); - // Iterate by index: get the (range, sort) pair at position idx - // FIXME: This is linear time, should be no more than O(log n) - if let Some((range, sort)) = segmentlist_ref.map.iter().nth(self.idx as usize) { - self.idx += 1; - Ok(Segment::new(range.start, range.end, sort.clone())) - } else { - Err(PyErr::new::("")) - } + fn __next__(&mut self) -> PyResult { + self.segments + .next() + .ok_or_else(|| PyErr::new::("")) } } @@ -274,6 +347,190 @@ pub fn segmentlist(m: &Bound<'_, PyModule>) -> PyResult<()> { mod tests { use super::SegmentList; + /// The pre-existing O(n) algorithm: search() for the starting index, then walk backwards + /// through the map by index. Used to pin sort_ratio_backwards() to the old behavior. + fn ratio_reference(sl: &SegmentList, addr: u64, window_size: u64, sort: Option<&str>) -> f64 { + let sort = sort.map(str::to_string); + let segments: Vec<_> = sl + .map + .iter() + .map(|(r, s)| (r.end - r.start, s.clone())) + .collect(); + let Some(mut idx) = sl.search(addr) else { + return 0.0; + }; + if segments[idx].1 != sort { + return 0.0; + } + let (mut total, mut matching) = (0u64, 0u64); + loop { + let (size, seg_sort) = &segments[idx]; + if *seg_sort == sort { + matching += size; + } + total += size; + if total >= window_size { + break; + } + if idx == 0 { + return 0.0; + } + idx -= 1; + } + matching as f64 / total as f64 + } + + /// A deterministic xorshift, so the randomized cases stay reproducible. + struct Rng(u64); + + impl Rng { + fn next(&mut self, bound: u64) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 % bound + } + } + + #[test] + fn prev_segment_walks_backwards() { + let mut sl = SegmentList::new(); + sl.occupy(10, 5, Some("a".to_string())); + sl.occupy(1000, 5, Some("b".to_string())); + sl.occupy(1005, 5, Some("c".to_string())); + + assert_eq!(sl.prev_segment(0), None); + assert_eq!(sl.prev_segment(10), None); + assert_eq!(sl.prev_segment(1005).unwrap().0, &(1000..1005)); + assert_eq!(sl.prev_segment(1000).unwrap().0, &(10..15)); + // an address in the middle of a gap + assert_eq!(sl.prev_segment(500).unwrap().0, &(10..15)); + // past the end of every segment + assert_eq!(sl.prev_segment(u64::MAX).unwrap().0, &(1005..1010)); + } + + #[test] + fn prev_segment_matches_brute_force() { + let mut rng = Rng(0x2545F4914F6CDD1D); + for _ in 0..50 { + let mut sl = SegmentList::new(); + let mut addr = 0; + for i in 0..40 { + addr += rng.next(64); + let size = 1 + rng.next(16); + sl.occupy(addr, size, Some(format!("s{}", i % 3))); + addr += size; + } + let starts: Vec = sl.map.iter().map(|(r, _)| r.start).collect(); + for probe in 0..addr + 8 { + let expected = starts.iter().rev().find(|&&s| s < probe); + assert_eq!( + sl.prev_segment(probe).map(|(r, _)| r.start).as_ref(), + expected, + "probe {probe:#x}" + ); + } + } + } + + #[test] + fn sort_ratio_backwards_basics() { + let mut sl = SegmentList::new(); + sl.occupy(0, 100, Some("code".to_string())); + sl.occupy(100, 100, Some("nodecode".to_string())); + sl.occupy(200, 100, Some("code".to_string())); + sl.occupy(300, 100, Some("nodecode".to_string())); + + // the walk starts inside the last segment and covers the whole map + assert_eq!( + sl.sort_ratio_backwards(399, 400, Some("nodecode".into())), + 0.5 + ); + // ... and only the last two segments with a smaller window + assert_eq!( + sl.sort_ratio_backwards(399, 200, Some("nodecode".into())), + 0.5 + ); + assert_eq!( + sl.sort_ratio_backwards(399, 100, Some("nodecode".into())), + 1.0 + ); + // the starting segment has the wrong sort + assert_eq!( + sl.sort_ratio_backwards(250, 100, Some("nodecode".into())), + 0.0 + ); + // not enough occupied bytes + assert_eq!( + sl.sort_ratio_backwards(399, 500, Some("nodecode".into())), + 0.0 + ); + // beyond every segment + assert_eq!( + sl.sort_ratio_backwards(500, 100, Some("nodecode".into())), + 0.0 + ); + // empty list + assert_eq!( + SegmentList::new().sort_ratio_backwards(0, 100, Some("nodecode".into())), + 0.0 + ); + } + + #[test] + fn sort_ratio_backwards_skips_gaps() { + let mut sl = SegmentList::new(); + sl.occupy(0, 40, Some("nodecode".to_string())); + sl.occupy(60, 40, Some("code".to_string())); + // a 900-byte gap, which the walk steps over without counting it + sl.occupy(1000, 40, Some("nodecode".to_string())); + + assert_eq!( + sl.sort_ratio_backwards(1039, 40, Some("nodecode".into())), + 1.0 + ); + assert_eq!( + sl.sort_ratio_backwards(1039, 80, Some("nodecode".into())), + 0.5 + ); + // the gap contributes nothing, so all three segments fit in a 120-byte window + assert_eq!( + sl.sort_ratio_backwards(1039, 120, Some("nodecode".into())), + 2.0 / 3.0 + ); + // ... and there is nothing left to cover a larger one + assert_eq!( + sl.sort_ratio_backwards(1039, 121, Some("nodecode".into())), + 0.0 + ); + } + + #[test] + fn sort_ratio_backwards_matches_reference() { + let mut rng = Rng(0x9E3779B97F4A7C15); + for _ in 0..50 { + let mut sl = SegmentList::new(); + let mut addr = 0; + for _ in 0..60 { + // gaps and single-byte segments are both common in CFGFast's segment list + addr += rng.next(4); + let size = 1 + rng.next(8); + let sort = if rng.next(2) == 0 { "nodecode" } else { "code" }; + sl.occupy(addr, size, Some(sort.to_string())); + addr += size; + } + for probe in 0..addr + 4 { + for window in [1u64, 7, 32, 4096] { + assert_eq!( + sl.sort_ratio_backwards(probe, window, Some("nodecode".into())), + ratio_reference(&sl, probe, window, Some("nodecode")), + "probe {probe:#x} window {window}" + ); + } + } + } + } + #[test] fn empty_list() { let mut sl = SegmentList::new(); diff --git a/tests/utils/test_segment_list.py b/tests/utils/test_segment_list.py index 5360330d3..356561a8e 100644 --- a/tests/utils/test_segment_list.py +++ b/tests/utils/test_segment_list.py @@ -1,10 +1,38 @@ from __future__ import annotations +import time import unittest from angr.rustylib import SegmentList +def _ratio_reference(seg_list: SegmentList, addr: int, window_size: int, sort: str | None) -> float: + """ + The O(n) algorithm ``SegmentList.sort_ratio_backwards()`` replaced: search() for the starting index, then walk + backwards through the list by index. + """ + idx = seg_list.search(addr) + if idx is None or idx >= len(seg_list): + return 0.0 + if seg_list[idx].sort != sort: + return 0.0 + + total_bytes = 0 + matching_bytes = 0 + while idx >= 0: + segment = seg_list[idx] + if segment.sort == sort: + matching_bytes += segment.size + total_bytes += segment.size + if total_bytes >= window_size: + break + idx -= 1 + + if total_bytes < window_size: + return 0.0 + return matching_bytes / total_bytes + + class TestSegmentList(unittest.TestCase): """ Test the SegmentList class. @@ -112,6 +140,86 @@ class TestSegmentList(unittest.TestCase): assert seg_list[1].end == 30 assert seg_list[1].sort == "code" + def test_iteration(self): + seg_list = SegmentList() + seg_list.occupy(0, 10, "code") + seg_list.occupy(20, 5, "data") + + assert [(s.start, s.end, s.sort) for s in seg_list] == [(0, 10, "code"), (20, 25, "data")] + assert list(SegmentList()) == [] + + def test_sort_ratio_backwards(self): + seg_list = SegmentList() + seg_list.occupy(0, 100, "code") + seg_list.occupy(100, 100, "nodecode") + seg_list.occupy(200, 100, "code") + seg_list.occupy(300, 100, "nodecode") + + # the walk starts in the last segment and covers as many segments as the window needs + assert seg_list.sort_ratio_backwards(399, 100, "nodecode") == 1.0 + assert seg_list.sort_ratio_backwards(399, 200, "nodecode") == 0.5 + assert seg_list.sort_ratio_backwards(399, 400, "nodecode") == 0.5 + # the starting segment has the wrong sort + assert seg_list.sort_ratio_backwards(250, 100, "nodecode") == 0.0 + # fewer than window_size occupied bytes are available + assert seg_list.sort_ratio_backwards(399, 500, "nodecode") == 0.0 + # beyond every segment + assert seg_list.sort_ratio_backwards(500, 100, "nodecode") == 0.0 + assert SegmentList().sort_ratio_backwards(0, 100, "nodecode") == 0.0 + + def test_sort_ratio_backwards_skips_gaps(self): + seg_list = SegmentList() + seg_list.occupy(0, 40, "nodecode") + seg_list.occupy(60, 40, "code") + # a 900-byte gap, which the backwards walk steps over without counting it + seg_list.occupy(1000, 40, "nodecode") + + assert seg_list.sort_ratio_backwards(1039, 40, "nodecode") == 1.0 + assert seg_list.sort_ratio_backwards(1039, 80, "nodecode") == 0.5 + assert seg_list.sort_ratio_backwards(1039, 120, "nodecode") == 2 / 3 + assert seg_list.sort_ratio_backwards(1039, 121, "nodecode") == 0.0 + + def test_sort_ratio_backwards_matches_reference(self): + # single-byte segments and gaps are both common in CFGFast's segment list + seg_list = SegmentList() + addr = 0 + for i in range(80): + addr += i % 3 + size = 1 + (i * 7) % 9 + seg_list.occupy(addr, size, "nodecode" if i % 5 < 2 else "code") + addr += size + + for probe in range(addr + 4): + for window in (1, 7, 32, 4096): + assert seg_list.sort_ratio_backwards(probe, window, "nodecode") == _ratio_reference( + seg_list, probe, window, "nodecode" + ), f"probe {probe:#x}, window {window}" + + def test_sort_ratio_backwards_is_not_quadratic(self): + # regression test for angr#6765: CFGFast._nodecode_bytes_ratio used to search() and then walk backwards by + # index, both linear in the segment count, which made the smart scan quadratic in input size + seg_list = SegmentList() + for i in range(100_000): + seg_list.occupy(i * 2, 1, "nodecode" if i % 4 else "code") + assert len(seg_list) == 100_000 + + start = time.time() + for i in range(1000): + seg_list.sort_ratio_backwards(199_999 - i, 512, "nodecode") + elapsed = time.time() - start + # the old implementation needs minutes here; the current one needs milliseconds + assert elapsed < 5.0, f"1000 queries over 100k segments took {elapsed:.1f}s" + + def test_iteration_is_not_quadratic(self): + seg_list = SegmentList() + for i in range(100_000): + seg_list.occupy(i * 2, 1, "nodecode" if i % 4 else "code") + + start = time.time() + assert len(list(seg_list)) == 100_000 + elapsed = time.time() - start + assert elapsed < 5.0, f"iterating 100k segments took {elapsed:.1f}s" + if __name__ == "__main__": unittest.main() From f62f45363c956dabc921c58fc863e975433b43e3 Mon Sep 17 00:00:00 2001 From: angr-bot Date: Wed, 5 Aug 2026 09:02:54 +0000 Subject: [PATCH 095/122] Update version to 9.3.3.dev0 [ci skip] --- angr/__init__.py | 2 +- pyproject.toml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/angr/__init__.py b/angr/__init__.py index 05a973086..8db36f34b 100644 --- a/angr/__init__.py +++ b/angr/__init__.py @@ -1,7 +1,7 @@ # pylint: disable=wrong-import-position from __future__ import annotations -__version__ = "9.3.2.dev0" +__version__ = "9.3.3.dev0" if bytes is str: raise Exception(""" diff --git a/pyproject.toml b/pyproject.toml index 2aa010864..f25b3d997 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ requires = [ "setuptools>=77.0.0", "setuptools-rust", - "pyvex==9.3.2.dev0", + "pyvex==9.3.3.dev0", "grpcio-tools~=1.80.0", "protobuf>=6.31.1,<7", ] @@ -29,12 +29,12 @@ dependencies = [ "cxxheaderparser", "GitPython", "angr-data~=0.1.0", - "archinfo==9.3.2.dev0", + "archinfo==9.3.3.dev0", "cachetools", "capstone==5.0.9", "cffi>=1.14.0", - "claripy==9.3.2.dev0", - "cle==9.3.2.dev0", + "claripy==9.3.3.dev0", + "cle==9.3.3.dev0", "lmdb==2.1.1", "msgspec; implementation_name == 'cpython'", "mulpyplexer", @@ -45,7 +45,7 @@ dependencies = [ "platformdirs", "pydemumble~=0.1.3", "pypcode~=4.0", - "pyvex==9.3.2.dev0", + "pyvex==9.3.3.dev0", "rich>=13.1.0", "sortedcontainers", "sympy", From 1c99579f691a1da78b7ecdd3960697d280567e00 Mon Sep 17 00:00:00 2001 From: Md7 <108727157+dhammerg@users.noreply.github.com> Date: Wed, 5 Aug 2026 04:01:10 -0700 Subject: [PATCH 096/122] Fix #6744: avoid empty Or expression in memory.find (#6750) --- .../storage/memory_mixins/smart_find_mixin.py | 3 +- tests/storage/test_memory_find_6744.py | 28 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100755 tests/storage/test_memory_find_6744.py diff --git a/angr/storage/memory_mixins/smart_find_mixin.py b/angr/storage/memory_mixins/smart_find_mixin.py index 48233a512..d31ce80ec 100644 --- a/angr/storage/memory_mixins/smart_find_mixin.py +++ b/angr/storage/memory_mixins/smart_find_mixin.py @@ -63,7 +63,8 @@ class SmartFindMixin(MemoryMixin): else: # the loop terminated, meaning we exhausted some sort of limit instead of finding a concrete answer. if default is None: - constraints.append(claripy.Or(*(c for c, _ in cases))) + clauses = [c for c, _ in cases] + constraints.append(claripy.Or(*clauses) if clauses else claripy.BoolV(False)) except SimSegfaultException: if chunk_size > 1: return self.find( diff --git a/tests/storage/test_memory_find_6744.py b/tests/storage/test_memory_find_6744.py new file mode 100755 index 000000000..643e2a025 --- /dev/null +++ b/tests/storage/test_memory_find_6744.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import archinfo +import claripy + +import angr + + +def test_memory_find_empty_cases(): + state = angr.SimState(arch=archinfo.ArchAMD64()) + + addr = claripy.BVV(0x1204F0D, 64) + target = claripy.BVV(0, 8) + + fdata = b"POST /deviceService/queryDeviceInfoByNickName.do HTTP/1.1" + + state.memory.store(addr, fdata, len(fdata)) + + result = state.memory.find( + addr, + target, + 128, + max_symbolic_bytes=60, + chunk_size=None, + char_size=1, + ) + + assert result is not None From b948687876cba120bf1aca6aa53a27859f9ec18f Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 5 Aug 2026 17:42:30 -0700 Subject: [PATCH 097/122] AIL: Fix identity comparisons against re-read statement attributes. (#6771) --- angr/ailment/block_walker.py | 16 ++++++------- .../dephication/rewriting_engine.py | 24 ++++++++++++------- .../coalesce_same_cascading_ifs.py | 9 +++---- .../remove_empty_if_body.py | 21 ++++++++-------- .../region_simplifiers/expr_folding.py | 24 +++++++++++-------- .../ssailification/rewriting_engine.py | 24 ++++++++++++------- 6 files changed, 67 insertions(+), 51 deletions(-) diff --git a/angr/ailment/block_walker.py b/angr/ailment/block_walker.py index fff905b6b..6bdb63e29 100644 --- a/angr/ailment/block_walker.py +++ b/angr/ailment/block_walker.py @@ -935,11 +935,12 @@ class AILBlockRewriter(AILBlockWalker[Expression, Statement, Block]): return stmt def _handle_Return(self, stmt_idx: int, stmt: Return, block: Block | None) -> Statement: - if stmt.ret_exprs: + ret_exprs_in = stmt.ret_exprs + if ret_exprs_in: new_ret_exprs = [ - self._handle_expr(idx, expr, stmt_idx, stmt, block) for idx, expr in enumerate(stmt.ret_exprs) + self._handle_expr(idx, expr, stmt_idx, stmt, block) for idx, expr in enumerate(ret_exprs_in) ] - changed = any(old is not new for new, old in zip(new_ret_exprs, stmt.ret_exprs)) + changed = any(old != new for new, old in zip(new_ret_exprs, ret_exprs_in)) if changed: return Return(stmt.idx, new_ret_exprs, **stmt.tags) @@ -1114,13 +1115,12 @@ class AILBlockRewriter(AILBlockWalker[Expression, Statement, Block]): super()._handle_Phi(expr_idx, expr, stmt_idx, stmt, block) return expr - changed = False - + src_and_vvars_in = expr.src_and_vvars src_and_vvars = [ (src, self._handle_expr(idx, vvar, stmt_idx, stmt, block) if vvar is not None else None) - for idx, (src, vvar) in enumerate(expr.src_and_vvars) + for idx, (src, vvar) in enumerate(src_and_vvars_in) ] - changed = any(new is not old for (_, new), (_, old) in zip(src_and_vvars, expr.src_and_vvars)) + changed = any(new != old for (_, new), (_, old) in zip(src_and_vvars, src_and_vvars_in)) if changed: assert all(vvar is None or isinstance(vvar, VirtualVariable) for _, vvar in src_and_vvars) @@ -1137,7 +1137,7 @@ class AILBlockRewriter(AILBlockWalker[Expression, Statement, Block]): ) -> Expression: operands_in = expr.operands new_operands = [self._handle_expr(0, operand, stmt_idx, stmt, block) for operand in operands_in] - changed = any(new is not old for new, old in zip(new_operands, operands_in)) + changed = any(new != old for new, old in zip(new_operands, operands_in)) new_guard = None guard_in = expr.guard diff --git a/angr/analyses/decompiler/dephication/rewriting_engine.py b/angr/analyses/decompiler/dephication/rewriting_engine.py index 8bad7602d..bab0bb903 100644 --- a/angr/analyses/decompiler/dephication/rewriting_engine.py +++ b/angr/analyses/decompiler/dephication/rewriting_engine.py @@ -236,8 +236,9 @@ class SimEngineDephiRewriting(SimEngineNostmtAIL[None, Expression | None, Statem return None def _handle_stmt_DirtyStatement(self, stmt: DirtyStatement) -> DirtyStatement | None: - dirty = self._expr(stmt.dirty) - if dirty is None or dirty is stmt.dirty: + dirty_in = stmt.dirty + dirty = self._expr(dirty_in) + if dirty is None or dirty == dirty_in: return None assert isinstance(dirty, DirtyExpression) return DirtyStatement(stmt.idx, dirty, **stmt.tags) @@ -394,19 +395,24 @@ class SimEngineDephiRewriting(SimEngineNostmtAIL[None, Expression | None, Statem return None def _handle_expr_Extract(self, expr): - base = self._expr(expr.base) or expr.base - offset = self._expr(expr.offset) or expr.offset + base_in = expr.base + offset_in = expr.offset + base = self._expr(base_in) or base_in + offset = self._expr(offset_in) or offset_in - if base is not expr.base or offset is not expr.offset: + if base != base_in or offset != offset_in: return Extract(expr.idx, expr.bits, base, offset, expr.endness, **expr.tags) return None def _handle_expr_Insert(self, expr): - base = self._expr(expr.base) or expr.base - offset = self._expr(expr.offset) or expr.offset - value = self._expr(expr.value) or expr.value + base_in = expr.base + offset_in = expr.offset + value_in = expr.value + base = self._expr(base_in) or base_in + offset = self._expr(offset_in) or offset_in + value = self._expr(value_in) or value_in - if base is not expr.base or offset is not expr.offset or value is not expr.value: + if base != base_in or offset != offset_in or value != value_in: return Insert(expr.idx, base, offset, value, expr.endness, **expr.tags) return None diff --git a/angr/analyses/decompiler/peephole_optimizations/coalesce_same_cascading_ifs.py b/angr/analyses/decompiler/peephole_optimizations/coalesce_same_cascading_ifs.py index 36c6d9891..e79aa9712 100644 --- a/angr/analyses/decompiler/peephole_optimizations/coalesce_same_cascading_ifs.py +++ b/angr/analyses/decompiler/peephole_optimizations/coalesce_same_cascading_ifs.py @@ -14,14 +14,15 @@ class CoalesceSameCascadingIfs(PeepholeOptimizationStmtBase): def optimize(self, stmt: ConditionalJump, stmt_idx: int | None = None, block=None, **kwargs): cond = stmt.condition + true_target_in = stmt.true_target # if (cond) {ITE(cond, true_branch, false_branch)} else {} ==> if (cond) {true_branch} else {} - if isinstance(stmt.true_target, ITE) and cond == stmt.true_target.cond: - new_true_target = stmt.true_target.iftrue + if isinstance(true_target_in, ITE) and cond == true_target_in.cond: + new_true_target = true_target_in.iftrue else: - new_true_target = stmt.true_target + new_true_target = true_target_in - if cond is not stmt.condition or new_true_target is not stmt.true_target: + if new_true_target != true_target_in: # it's updated return ConditionalJump( stmt.idx, cond, new_true_target, stmt.false_target, false_target_idx=stmt.false_target_idx, **stmt.tags diff --git a/angr/analyses/decompiler/peephole_optimizations/remove_empty_if_body.py b/angr/analyses/decompiler/peephole_optimizations/remove_empty_if_body.py index 521dc36d1..e06835c01 100644 --- a/angr/analyses/decompiler/peephole_optimizations/remove_empty_if_body.py +++ b/angr/analyses/decompiler/peephole_optimizations/remove_empty_if_body.py @@ -13,26 +13,25 @@ class RemoveEmptyIfBody(PeepholeOptimizationStmtBase): stmt_classes = (ConditionalJump,) def optimize(self, stmt: ConditionalJump, stmt_idx: int | None = None, block=None, **kwargs): - cond = stmt.condition + cond_in = stmt.condition + true_target_in = stmt.true_target + false_target_in = stmt.false_target + cond = cond_in # if (!cond) {} else { ITE(cond, true_branch, false_branch } ==> if (cond) { ITE(...) } else {} - if isinstance(stmt.false_target, ITE) and isinstance(cond, UnaryOp) and cond.op == "Not": - new_true_target = stmt.false_target + if isinstance(false_target_in, ITE) and isinstance(cond, UnaryOp) and cond.op == "Not": + new_true_target = false_target_in new_true_idx = stmt.false_target_idx - new_false_target = stmt.true_target + new_false_target = true_target_in new_false_idx = stmt.true_target_idx cond = cond.operand else: - new_true_target = stmt.true_target + new_true_target = true_target_in new_true_idx = stmt.true_target_idx - new_false_target = stmt.false_target + new_false_target = false_target_in new_false_idx = stmt.false_target_idx - if ( - cond is not stmt.condition - or new_true_target is not stmt.true_target - or new_false_target is not stmt.false_target - ): + if cond != cond_in or new_true_target != true_target_in or new_false_target != false_target_in: # it's updated return ConditionalJump( stmt.idx, diff --git a/angr/analyses/decompiler/region_simplifiers/expr_folding.py b/angr/analyses/decompiler/region_simplifiers/expr_folding.py index e233f9312..c99f25c5b 100644 --- a/angr/analyses/decompiler/region_simplifiers/expr_folding.py +++ b/angr/analyses/decompiler/region_simplifiers/expr_folding.py @@ -599,11 +599,12 @@ class ExpressionReplacer(AILBlockRewriter): else: new_statements.append(stmt_) - new_expr = self._handle_expr(0, expr.expr, stmt_idx, stmt, block) - if new_expr is not None and new_expr is not expr.expr: + inner_in = expr.expr + new_expr = self._handle_expr(0, inner_in, stmt_idx, stmt, block) + if new_expr is not None and new_expr != inner_in: changed = True else: - new_expr = expr.expr + new_expr = inner_in if changed: if not new_statements: @@ -621,23 +622,26 @@ class ExpressionReplacer(AILBlockRewriter): if is_phi_assignment(stmt): return stmt - if isinstance(stmt.dst, VirtualVariable) and stmt.dst.varid in self._assignments: + dst_in = stmt.dst + src_in = stmt.src + + if isinstance(dst_in, VirtualVariable) and dst_in.varid in self._assignments: return stmt changed = False - dst = self._handle_expr(0, stmt.dst, stmt_idx, stmt, block) - if dst is not stmt.dst and not isinstance(dst, (Call, ITE)): + dst = self._handle_expr(0, dst_in, stmt_idx, stmt, block) + if dst != dst_in and not isinstance(dst, (Call, ITE)): changed = True else: - dst = stmt.dst + dst = dst_in assert isinstance(dst, Atom) - src = self._handle_expr(1, stmt.src, stmt_idx, stmt, block) - if src is not stmt.src: + src = self._handle_expr(1, src_in, stmt_idx, stmt, block) + if src != src_in: changed = True else: - src = stmt.src + src = src_in if changed: return Assignment(stmt.idx, dst, src, **stmt.tags) diff --git a/angr/analyses/decompiler/ssailification/rewriting_engine.py b/angr/analyses/decompiler/ssailification/rewriting_engine.py index d4331982c..6d660d500 100644 --- a/angr/analyses/decompiler/ssailification/rewriting_engine.py +++ b/angr/analyses/decompiler/ssailification/rewriting_engine.py @@ -331,8 +331,9 @@ class SimEngineSSARewriting( return new_stmt def _handle_stmt_DirtyStatement(self, stmt: DirtyStatement) -> DirtyStatement | None: - dirty = self._expr(stmt.dirty) - if dirty is None or dirty is stmt.dirty: + dirty_in = stmt.dirty + dirty = self._expr(dirty_in) + if dirty is None or dirty == dirty_in: return None assert isinstance(dirty, DirtyExpression) return DirtyStatement(stmt.idx, dirty, **stmt.tags) @@ -586,19 +587,24 @@ class SimEngineSSARewriting( ) def _handle_expr_Extract(self, expr: Extract): - base = self._expr(expr.base) or expr.base - offset = self._expr(expr.offset) or expr.offset + base_in = expr.base + offset_in = expr.offset + base = self._expr(base_in) or base_in + offset = self._expr(offset_in) or offset_in - if base is not expr.base or offset is not expr.offset: + if base != base_in or offset != offset_in: return Extract(expr.idx, expr.bits, base, offset, expr.endness, **expr.tags) return None def _handle_expr_Insert(self, expr: Insert): - base = self._expr(expr.base) or expr.base - offset = self._expr(expr.offset) or expr.offset - value = self._expr(expr.value) or expr.value + base_in = expr.base + offset_in = expr.offset + value_in = expr.value + base = self._expr(base_in) or base_in + offset = self._expr(offset_in) or offset_in + value = self._expr(value_in) or value_in - if base is not expr.base or offset is not expr.offset or value is not expr.value: + if base != base_in or offset != offset_in or value != value_in: return Insert(expr.idx, base, offset, value, expr.endness, **expr.tags) return None From 7d3c7c82cee6f06cf586166541fb99a833ab60e0 Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 5 Aug 2026 21:33:55 -0500 Subject: [PATCH 098/122] AILSimplifier: De-quadratify _unify_local_variables. (#6772) --- angr/analyses/decompiler/ail_simplifier.py | 66 +++++++------ .../s_reaching_definitions/s_rda_model.py | 93 +++++++++++++++++-- tests/analyses/test_srda_model.py | 70 ++++++++++++++ 3 files changed, 188 insertions(+), 41 deletions(-) create mode 100644 tests/analyses/test_srda_model.py diff --git a/angr/analyses/decompiler/ail_simplifier.py b/angr/analyses/decompiler/ail_simplifier.py index 2b30504d4..4d8b42e0d 100644 --- a/angr/analyses/decompiler/ail_simplifier.py +++ b/angr/analyses/decompiler/ail_simplifier.py @@ -4,7 +4,7 @@ from __future__ import annotations import logging import os from collections import defaultdict -from collections.abc import Container, Iterable +from collections.abc import Iterable from enum import Enum from typing import TYPE_CHECKING, Any @@ -1097,6 +1097,9 @@ class AILSimplifier(Analysis): # out-of-date updated_locs: set[AILCodeLocation] = set() + # built on-demand + stack_defs_by_offset: dict[int, list[Definition[atoms.VirtualVariable, AILCodeLocation]]] | None = None + for _, atom in sorted_loc_and_atoms: eqs = equivalences[atom] filtered_eqs: list[tuple[Equivalence, VirtualVariable, bool]] = [] @@ -1167,14 +1170,9 @@ class AILSimplifier(Analysis): rd = self._compute_reaching_definitions() the_def = None if to_replace_is_def: - # find defs - defs: Container[Definition[atoms.VirtualVariable, AILCodeLocation]] = [] - for def_ in rd.all_definitions: - if def_.atom.varid == to_replace.varid: - defs.append(def_) - if len(defs) != 1: + the_def = rd.get_definition_by_varid(to_replace.varid) + if the_def is None: continue - the_def = defs[0] else: # find uses defs = rd.get_uses_by_location(eq.codeloc) @@ -1200,38 +1198,42 @@ class AILSimplifier(Analysis): # (a) the on-stack or in-register copy of it has never been modified in this function # (b) the function argument register has never been updated. # TODO: we may loosen requirement (b) once we have real register versioning in AIL. - defs = [def_ for def_ in rd.all_definitions if def_.codeloc == eq.codeloc] + defs = rd.get_defs_by_location(eq.codeloc) all_uses_with_def = None replace_with = None remove_initial_assignment = None def_eq_rel = DefEqRelation.DEF_IS_FUNCARG - if defs and len(defs) == 1: - arg_copy_def = defs[0] + if len(defs) == 1: + arg_copy_def = next(iter(defs)) if (isinstance(arg_copy_def.atom, atoms.VirtualVariable) and arg_copy_def.atom.was_stack) or ( isinstance(arg_copy_def.atom, atoms.VirtualVariable) and arg_copy_def.atom.was_reg ): # found the copied definition (either a stack variable or a register variable) # Make sure there is no other write to this stack location if the copy is a stack variable - if ( - isinstance(arg_copy_def.atom, atoms.VirtualVariable) - and arg_copy_def.atom.was_stack - and any( - (def_ != arg_copy_def and def_.atom.stack_offset == arg_copy_def.atom.stack_offset) - for def_ in rd.all_definitions - if isinstance(def_.atom, atoms.VirtualVariable) and def_.atom.was_stack - ) - ): - continue + if isinstance(arg_copy_def.atom, atoms.VirtualVariable) and arg_copy_def.atom.was_stack: + if stack_defs_by_offset is None: + stack_defs_by_offset = defaultdict(list) + for def_ in rd.all_definitions: + if def_.atom.was_stack: + stack_defs_by_offset[def_.atom.stack_offset].append(def_) + if any( + def_ != arg_copy_def + for def_ in stack_defs_by_offset.get(arg_copy_def.atom.stack_offset, ()) + ): + continue - # Make sure the register is never updated across this function - if any( - (def_ != the_def and def_.atom == the_def.atom) - for def_ in rd.all_definitions - if isinstance(def_.atom, atoms.VirtualVariable) - and def_.atom.was_reg - and rd.get_vvar_uses(def_.atom) + # Make sure the register is never updated across this function. Only the definition of + # the_def's own vvar id can have an equal atom (atom equality is (varid, size)), and SSA gives + # that id exactly one definition, so this is a single lookup rather than a full scan. + other_def = rd.get_definition_by_varid(the_def.atom.varid) + if ( + other_def is not None + and other_def != the_def + and other_def.atom == the_def.atom + and other_def.atom.was_reg + and rd.get_vvar_uses(other_def.atom) ): continue @@ -1267,13 +1269,7 @@ class AILSimplifier(Analysis): def_eq_rel = DefEqRelation.DEF_EQ_SAME_BLOCK else: # the definition is in the predecessor block of the eq - eq_block = next( - iter( - bb - for bb in self.func_graph - if bb.addr == eq.codeloc.block_addr and bb.idx == eq.codeloc.block_idx - ) - ) + eq_block = addr_and_idx_to_block[(eq.codeloc.block_addr, eq.codeloc.block_idx)] eq_block_preds = set(self.func_graph.predecessors(eq_block)) if not any( pred.addr == the_def.codeloc.block_addr and pred.idx == the_def.codeloc.block_idx diff --git a/angr/analyses/s_reaching_definitions/s_rda_model.py b/angr/analyses/s_reaching_definitions/s_rda_model.py index 87da69937..3e6f67857 100644 --- a/angr/analyses/s_reaching_definitions/s_rda_model.py +++ b/angr/analyses/s_reaching_definitions/s_rda_model.py @@ -46,6 +46,9 @@ class SRDAModel: self.phivarid_to_varids_with_unknown: dict[int, set[int | None]] = {} self.phivarid_to_varids: dict[int, set[int]] = {} self.vvar_uses_by_loc: dict[AILCodeLocation, list[int]] = {} + # the inverse index of all_vvar_definitions; a bare int for the common case of a location defining a single + # vvar, a set when a statement defines more than one (e.g., a call defining both ret_expr and fp_ret_expr) + self.vvar_defs_by_loc: dict[AILCodeLocation, int | set[int]] = {} def add_vvar_use(self, vvar_id: int, expr: VirtualVariable | None, loc: AILCodeLocation) -> None: self.all_vvar_uses[vvar_id].append((expr, loc)) @@ -53,6 +56,47 @@ class SRDAModel: self.vvar_uses_by_loc[loc] = [] self.vvar_uses_by_loc[loc].append(vvar_id) + def add_vvar_def(self, vvar_id: int, loc: AILCodeLocation) -> None: + """ + Record the definition of a vvar id, keeping all_vvar_definitions and vvar_defs_by_loc in sync. + """ + old_loc = self.all_vvar_definitions.get(vvar_id) + self.all_vvar_definitions[vvar_id] = loc + if old_loc is not None: + if old_loc == loc: + return + self._unlink_vvar_def(vvar_id, old_loc) + existing = self.vvar_defs_by_loc.get(loc) + if existing is None: + self.vvar_defs_by_loc[loc] = vvar_id + elif isinstance(existing, int): + if existing != vvar_id: + self.vvar_defs_by_loc[loc] = {existing, vvar_id} + else: + existing.add(vvar_id) + + def remove_vvar_def(self, vvar_id: int) -> None: + """ + Drop the definition of a vvar id from both all_vvar_definitions and vvar_defs_by_loc. + """ + loc = self.all_vvar_definitions.pop(vvar_id, None) + if loc is not None: + self._unlink_vvar_def(vvar_id, loc) + + def _unlink_vvar_def(self, vvar_id: int, loc: AILCodeLocation) -> None: + existing = self.vvar_defs_by_loc.get(loc) + if existing is None: + return + if isinstance(existing, int): + if existing == vvar_id: + del self.vvar_defs_by_loc[loc] + else: + existing.discard(vvar_id) + if len(existing) == 1: + self.vvar_defs_by_loc[loc] = next(iter(existing)) + elif not existing: + del self.vvar_defs_by_loc[loc] + def update_after_block_edits(self, edited_blocks) -> None: """ Incrementally update the model after the statements of ``edited_blocks`` were edited *in place* (statement @@ -82,7 +126,7 @@ class SRDAModel: # this definition no longer exists; keep its uses for now (if the vvar is still used elsewhere it becomes # a used-but-undefined extern vvar, reconciled below) self.varid_to_vvar.pop(vid, None) - self.all_vvar_definitions.pop(vid, None) + self.remove_vvar_def(vid) self.phi_vvar_ids.discard(vid) self.phivarid_to_varids.pop(vid, None) self.phivarid_to_varids_with_unknown.pop(vid, None) @@ -90,7 +134,7 @@ class SRDAModel: # surviving defs keep their (index-stable) location; refresh vvar and phi info from the rescan for vid, (vvar, defloc) in new_deflocs.items(): self.varid_to_vvar[vid] = vvar - self.all_vvar_definitions[vid] = defloc + self.add_vvar_def(vid, defloc) if vid in new_phi: src = new_phi[vid] self.phi_vvar_ids.add(vid) @@ -139,14 +183,14 @@ class SRDAModel: for vid, expr in explicit_use_repr.items(): if vid not in self.all_vvar_definitions: self.varid_to_vvar[vid] = expr - self.all_vvar_definitions[vid] = AILCodeLocation.make_extern(vid) + self.add_vvar_def(vid, AILCodeLocation.make_extern(vid)) for vid in [ vid for vid, loc in self.all_vvar_definitions.items() if loc.is_extern and vid not in func_arg_ids and vid not in explicit_use_repr ]: self.varid_to_vvar.pop(vid, None) - self.all_vvar_definitions.pop(vid, None) + self.remove_vvar_def(vid) self.all_vvar_uses.pop(vid, None) self.phi_vvar_ids.discard(vid) self.phivarid_to_varids.pop(vid, None) @@ -169,6 +213,10 @@ class SRDAModel: {k: frozenset(v) for k, v in self.phivarid_to_varids.items()}, {k: frozenset(v) for k, v in self.phivarid_to_varids_with_unknown.items()}, {loc: Counter(vids) for loc, vids in self.vvar_uses_by_loc.items() if vids}, + { + loc: frozenset(vids) if isinstance(vids, set) else frozenset((vids,)) + for loc, vids in self.vvar_defs_by_loc.items() + }, ) @property @@ -177,6 +225,16 @@ class SRDAModel: vvar = self.varid_to_vvar[vvar_id] yield Definition(atoms.VirtualVariable(vvar_id, vvar.size, vvar.category, vvar.oident), defloc) + def get_definition_by_varid(self, varid: int) -> Definition[atoms.VirtualVariable, AILCodeLocation] | None: + """ + The definition of a vvar id, or None if it has none. + """ + defloc = self.all_vvar_definitions.get(varid) + if defloc is None: + return None + vvar = self.varid_to_vvar[varid] + return Definition(atoms.VirtualVariable(varid, vvar.size, vvar.category, vvar.oident), defloc) + def is_phi_vvar_id(self, idx: int) -> bool: return idx in self.phi_vvar_ids @@ -249,6 +307,29 @@ class SRDAModel: ) return defs + def get_defs_by_location(self, loc: AILCodeLocation) -> set[Definition[atoms.VirtualVariable, AILCodeLocation]]: + """ + Retrieve all vvar definitions at a given location. + + :param loc: The code location. + :return: A set of definitions that are defined at the given location. + """ + vvar_ids = self.vvar_defs_by_loc.get(loc) + if vvar_ids is None: + return set() + if isinstance(vvar_ids, int): + vvar_ids = (vvar_ids,) # type: ignore + defs: set[Definition[atoms.VirtualVariable, AILCodeLocation]] = set() + for vvar_id in vvar_ids: # type: ignore + vvar = self.varid_to_vvar[vvar_id] + defs.add( + Definition( + atoms.VirtualVariable(vvar_id, vvar.size, vvar.category, vvar.oident), + self.all_vvar_definitions[vvar_id], + ) + ) + return defs + def get_vvar_uses(self, obj: VirtualVariable | atoms.VirtualVariable) -> set[AILCodeLocation]: if obj.varid in self.all_vvar_uses: return {loc for _, loc in self.all_vvar_uses[obj.varid]} @@ -309,7 +390,7 @@ def populate_model( # update model for vvar_id, (vvar, defloc) in vvar_deflocs.items(): model.varid_to_vvar[vvar_id] = vvar - model.all_vvar_definitions[vvar_id] = defloc + model.add_vvar_def(vvar_id, defloc) if vvar_id in vvar_uselocs: for useloc in vvar_uselocs[vvar_id]: model.add_vvar_use(vvar_id, *useloc) @@ -329,7 +410,7 @@ def populate_model( for vvar_id in undefined_vvarids: used_vvar = next(iter(vvar_uselocs[vvar_id]))[0] model.varid_to_vvar[vvar_id] = used_vvar - model.all_vvar_definitions[vvar_id] = AILCodeLocation.make_extern(vvar_id) + model.add_vvar_def(vvar_id, AILCodeLocation.make_extern(vvar_id)) if vvar_id in vvar_uselocs: for vvar_useloc in vvar_uselocs[vvar_id]: model.add_vvar_use(vvar_id, *vvar_useloc) diff --git a/tests/analyses/test_srda_model.py b/tests/analyses/test_srda_model.py new file mode 100644 index 000000000..a9b888e1a --- /dev/null +++ b/tests/analyses/test_srda_model.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use +from __future__ import annotations + +__package__ = __package__ or "tests.analyses" # pylint:disable=redefined-builtin + +import unittest + +from angr.ailment.expression import VirtualVariable, VirtualVariableCategory +from angr.analyses.s_reaching_definitions.s_rda_model import SRDAModel +from angr.code_location import AILCodeLocation + + +class TestSRDAModelVVarDefs(unittest.TestCase): + @staticmethod + def _model() -> SRDAModel: + return SRDAModel(None, None, None) + + def test_single_def_is_stored_as_int(self): + model = self._model() + loc = AILCodeLocation(0x400000, None, 3) + model.add_vvar_def(1, loc) + assert model.all_vvar_definitions == {1: loc} + assert model.vvar_defs_by_loc == {loc: 1} + + def test_multiple_defs_at_one_location(self): + model = self._model() + loc = AILCodeLocation(0x400000, None, 3) + model.add_vvar_def(1, loc) + model.add_vvar_def(2, loc) + model.add_vvar_def(2, loc) # re-adding an existing def changes nothing + assert model.vvar_defs_by_loc == {loc: {1, 2}} + + # removals collapse the set back to an int and finally drop the key + model.remove_vvar_def(1) + assert model.vvar_defs_by_loc == {loc: 2} + model.remove_vvar_def(2) + assert model.vvar_defs_by_loc == {} + assert model.all_vvar_definitions == {} + + def test_moving_a_def_unlinks_the_old_location(self): + model = self._model() + loc0 = AILCodeLocation(0x400000, None, 3) + loc1 = AILCodeLocation(0x400000, None, 7) + model.add_vvar_def(1, loc0) + model.add_vvar_def(2, loc0) + model.add_vvar_def(2, loc1) + assert model.all_vvar_definitions == {1: loc0, 2: loc1} + assert model.vvar_defs_by_loc == {loc0: 1, loc1: 2} + + def test_removing_an_unknown_def_is_a_noop(self): + model = self._model() + model.remove_vvar_def(42) + assert model.vvar_defs_by_loc == {} + + def test_get_defs_by_location(self): + model = self._model() + loc = AILCodeLocation(0x400000, None, 3) + model.varid_to_vvar = { + vid: VirtualVariable(None, vid, 64, category=VirtualVariableCategory.REGISTER, oident=16) for vid in (1, 2) + } + model.add_vvar_def(1, loc) + assert {d.atom.varid for d in model.get_defs_by_location(loc)} == {1} + model.add_vvar_def(2, loc) + assert {d.atom.varid for d in model.get_defs_by_location(loc)} == {1, 2} + assert model.get_defs_by_location(AILCodeLocation(0x400000, None, 9)) == set() + + +if __name__ == "__main__": + unittest.main() From e76c4118a7e8a437981527eb3e3725bc204e2266 Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 5 Aug 2026 21:34:13 -0500 Subject: [PATCH 099/122] AILSimplifier: Only store replaced blocks into self.blocks. (#6773) _rebuild_func_graph marks all blocks in self.blocks dirty, so adding unchanged blocks to self.blocks led to redundant simplification runs after. --- angr/analyses/decompiler/ail_simplifier.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/angr/analyses/decompiler/ail_simplifier.py b/angr/analyses/decompiler/ail_simplifier.py index 4d8b42e0d..a9cf5fdd6 100644 --- a/angr/analyses/decompiler/ail_simplifier.py +++ b/angr/analyses/decompiler/ail_simplifier.py @@ -935,7 +935,8 @@ class AILSimplifier(Analysis): block, reps, self._ail_manager, gp=self._gp, replace_loads=replace_loads ) replaced |= r - self.blocks[block] = new_block + if r: + self.blocks[block] = new_block if replaced: # blocks have been rebuilt - expression propagation results are no longer reliable From 4018e2607673466af8abf4dbdfc752ad0a93a8d0 Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 5 Aug 2026 21:34:33 -0500 Subject: [PATCH 100/122] ExpressionNarrower: Compare rebuilt operands by value instead of identity. (#6774) --- .../decompiler/expression_narrower.py | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/angr/analyses/decompiler/expression_narrower.py b/angr/analyses/decompiler/expression_narrower.py index 646defecf..5675a7715 100644 --- a/angr/analyses/decompiler/expression_narrower.py +++ b/angr/analyses/decompiler/expression_narrower.py @@ -272,11 +272,14 @@ class ExpressionNarrower(AILBlockRewriter): return super().walk(block) def _handle_Assignment(self, stmt_idx: int, stmt: Assignment, block: Block | None) -> Assignment: - if isinstance(stmt.src, Phi): + src_in = stmt.src + dst_in = stmt.dst + + if isinstance(src_in, Phi): changed = False src_and_vvars = [] - for src, vvar in stmt.src.src_and_vvars: + for src, vvar in src_in.src_and_vvars: if vvar is None: src_and_vvars.append((src, None)) continue @@ -298,39 +301,39 @@ class ExpressionNarrower(AILBlockRewriter): src_and_vvars.append((src, new_var)) - new_src = Phi(stmt.src.idx, stmt.src.bits, src_and_vvars, **stmt.src.tags) + new_src = Phi(src_in.idx, src_in.bits, src_and_vvars, **src_in.tags) else: - new_src = self._handle_expr(1, stmt.src, stmt_idx, stmt, block) - changed = new_src is not stmt.src + new_src = self._handle_expr(1, src_in, stmt_idx, stmt, block) + changed = new_src != src_in - if isinstance(stmt.dst, VirtualVariable) and stmt.dst.varid in self.new_vvar_sizes: + if isinstance(dst_in, VirtualVariable) and dst_in.varid in self.new_vvar_sizes: changed = True new_dst = VirtualVariable( - stmt.dst.idx, - stmt.dst.varid, - self.new_vvar_sizes[stmt.dst.varid] * self.project.arch.byte_width, - category=stmt.dst.category, - oident=stmt.dst.oident, - **stmt.dst.tags, + dst_in.idx, + dst_in.varid, + self.new_vvar_sizes[dst_in.varid] * self.project.arch.byte_width, + category=dst_in.category, + oident=dst_in.oident, + **dst_in.tags, ) self.replacement_core_vvars[new_dst.varid].append(new_dst) if isinstance(new_src, Phi): - new_src.bits = self.new_vvar_sizes[stmt.dst.varid] * self.project.arch.byte_width + new_src.bits = self.new_vvar_sizes[dst_in.varid] * self.project.arch.byte_width else: new_src = Convert( self.manager.next_atom(), - stmt.src.bits, - self.new_vvar_sizes[stmt.dst.varid] * self.project.arch.byte_width, + src_in.bits, + self.new_vvar_sizes[dst_in.varid] * self.project.arch.byte_width, False, new_src, **new_src.tags, ) else: - new_dst = self._handle_expr(0, stmt.dst, stmt_idx, stmt, block) - changed |= new_dst is not stmt.dst + new_dst = self._handle_expr(0, dst_in, stmt_idx, stmt, block) + changed |= new_dst != dst_in if changed: self.narrowed_any = True From 04b40a1eec968e67fa74b196fc443ab7bd45f573 Mon Sep 17 00:00:00 2001 From: Fish Date: Wed, 5 Aug 2026 23:00:09 -0400 Subject: [PATCH 101/122] InlinedStringTransformationSimplifier: Pre-filter loops before symbolic execution. (#6775) --- ...nlined_string_transformation_simplifier.py | 72 ++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/angr/analyses/decompiler/optimization_passes/inlined_string_transformation_simplifier.py b/angr/analyses/decompiler/optimization_passes/inlined_string_transformation_simplifier.py index 82c56f76a..bd64be3ed 100644 --- a/angr/analyses/decompiler/optimization_passes/inlined_string_transformation_simplifier.py +++ b/angr/analyses/decompiler/optimization_passes/inlined_string_transformation_simplifier.py @@ -9,6 +9,7 @@ import archinfo import claripy from archinfo import Endness +from angr.ailment import AILBlockViewer from angr.ailment.expression import ( BinaryOp, Const, @@ -21,7 +22,7 @@ from angr.ailment.expression import ( UnaryOp, VirtualVariable, ) -from angr.ailment.statement import ConditionalJump, Jump, Store +from angr.ailment.statement import Assignment, ConditionalJump, Jump, Store from angr.code_location import CodeLocation from angr.engines.light import SimEngineNostmtAIL from angr.errors import SimMemoryMissingError @@ -513,6 +514,71 @@ class InlinedStringTransformationAILEngine( _handle_binop_Set = _handle_binop_Default +class _StackReadNotification(Exception): + """Abort the walk on the first potential stack read.""" + + +class _HasStackReadWalker(AILBlockViewer): + """ + Raises ``_StackReadNotification`` on the first expression that InlinedStringTransformationAILEngine could turn + into a "load" stack-access record: a Load, or a virtual variable that lives on the stack. + """ + + def _handle_Load(self, expr_idx, expr, stmt_idx, stmt, block): # pylint:disable=unused-argument + raise _StackReadNotification + + def _handle_VirtualVariable(self, expr_idx, expr, stmt_idx, stmt, block): # pylint:disable=unused-argument + if expr.was_stack: + raise _StackReadNotification + + +_HAS_STACK_READ_WALKER = _HasStackReadWalker() + + +def _addr_may_be_stack(addr: Expression) -> bool: + """ + Syntactic over-approximation of the addresses ``InlinedStringTransformationAILEngine._process_address`` can + resolve. Anything outside this shape never produces a stack-access record. + """ + if isinstance(addr, (Const, StackBaseOffset)): + return True + if isinstance(addr, UnaryOp) and addr.op == "Reference": + return True + return ( + isinstance(addr, BinaryOp) + and addr.op in {"Add", "Sub"} + and isinstance(addr.operands[0], (StackBaseOffset, UnaryOp, Const)) + ) + + +def _reads_stack(expr: Expression) -> bool: + try: + _HAS_STACK_READ_WALKER.walk_expression(expr) + except _StackReadNotification: + return True + return False + + +def _may_transform_stack_bytes(block) -> bool: + """ + A descriptor is only ever built when some statement records a "load" and a "store" stack access at the *same* + code location, i.e. a store to the stack whose value is derived from a stack read. Checking that syntactically is + far cheaper than symbolically executing the loop, and no statement outside this shape can produce that pair. + """ + for stmt in block.statements: + if isinstance(stmt, Store): + if _addr_may_be_stack(stmt.addr) and _reads_stack(stmt.data): + return True + elif ( + isinstance(stmt, Assignment) + and isinstance(stmt.dst, VirtualVariable) + and stmt.dst.was_stack + and _reads_stack(stmt.src) + ): + return True + return False + + class InlineStringTransformationDescriptor: """ Describes an instance of inline string transformation. @@ -634,6 +700,10 @@ class InlinedStringTransformationSimplifier(OptimizationPass): for loop_node in self_loops: pred = next(iter(nn for nn in self._graph.predecessors(loop_node) if nn is not loop_node)) succ = next(iter(nn for nn in self._graph.successors(loop_node) if nn is not loop_node)) + if not _may_transform_stack_bytes(loop_node) and not _may_transform_stack_bytes(pred): + # no statement here can produce the load-then-store-at-the-same-code-location pair a descriptor + # needs; skip the (expensive) symbolic execution entirely + continue engine = InlinedStringTransformationAILEngine( self.project, {pred.addr: pred, loop_node.addr: loop_node}, pred.addr, succ.addr, 1024 ) From 0f4be85db4229f0832b23428d9e940028407720a Mon Sep 17 00:00:00 2001 From: Fish Date: Thu, 6 Aug 2026 08:44:29 -0400 Subject: [PATCH 102/122] Make CallSiteMaker a normal class instead of an Analysis. (#6776) --- angr/analyses/decompiler/callsite_maker.py | 17 +++++++++++------ angr/analyses/decompiler/clinic.py | 5 ++--- tests/analyses/test_callsite_maker.py | 3 ++- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/angr/analyses/decompiler/callsite_maker.py b/angr/analyses/decompiler/callsite_maker.py index 924216596..53c4e7f20 100644 --- a/angr/analyses/decompiler/callsite_maker.py +++ b/angr/analyses/decompiler/callsite_maker.py @@ -8,7 +8,6 @@ import archinfo from angr.ailment import Const, Expr, Stmt from angr.ailment.manager import Manager -from angr.analyses.analysis import Analysis, register_analysis from angr.analyses.s_reaching_definitions import SRDAView from angr.calling_conventions import ( SimCC, @@ -38,19 +37,28 @@ if TYPE_CHECKING: from angr.analyses.s_reaching_definitions import SRDAModel from angr.knowledge_plugins.functions import Function from angr.knowledge_plugins.key_definitions.definition import Definition + from angr.project import Project l = logging.getLogger(name=__name__) -class CallSiteMaker(Analysis): +class CallSiteMaker: """ Add calling convention, declaration, and args to a call site. """ def __init__( - self, block, *, ail_manager: Manager, reaching_definitions: SRDAModel | None = None, stack_pointer_tracker=None + self, + project: Project, + block, + *, + ail_manager: Manager, + reaching_definitions: SRDAModel | None = None, + stack_pointer_tracker=None, ): + self.project = project + self.kb = project.kb self.block = block self._reaching_definitions = reaching_definitions @@ -652,6 +660,3 @@ class CallSiteMaker(Analysis): def _atom_idx(self) -> int: return self._ail_manager.next_atom() - - -register_analysis(CallSiteMaker, "AILCallSiteMaker") diff --git a/angr/analyses/decompiler/clinic.py b/angr/analyses/decompiler/clinic.py index ffa4703f8..c5306a70c 100644 --- a/angr/analyses/decompiler/clinic.py +++ b/angr/analyses/decompiler/clinic.py @@ -2402,9 +2402,8 @@ class Clinic(Analysis, Serializable): def _handler(block): nonlocal stackarg_offset_manager, removed_vvar_ids - csm = self.project.analyses[CallSiteMaker].prep( - fail_fast=self._fail_fast, - )( + csm = CallSiteMaker( + self.project, block, reaching_definitions=rd, stack_pointer_tracker=stack_pointer_tracker, diff --git a/tests/analyses/test_callsite_maker.py b/tests/analyses/test_callsite_maker.py index 511e12ff5..7a891089a 100755 --- a/tests/analyses/test_callsite_maker.py +++ b/tests/analyses/test_callsite_maker.py @@ -9,6 +9,7 @@ import unittest import angr from angr import ailment from angr.analyses.decompiler.block_simplifier import BlockSimplifier +from angr.analyses.decompiler.callsite_maker import CallSiteMaker from tests.common import bin_location test_location = os.path.join(bin_location, "tests") @@ -48,7 +49,7 @@ class TestCallsiteMaker(unittest.TestCase): ail_block = ailment.IRSBConverter.convert(block.vex, manager) simp = BlockSimplifier(project, ail_block, manager, main_func.addr) - csm = project.analyses.AILCallSiteMaker(simp.result_block, ail_manager=manager) + csm = CallSiteMaker(project, simp.result_block, ail_manager=manager) if csm.result_block: ail_block = csm.result_block simp = BlockSimplifier(project, ail_block, manager, main_func.addr) From 795eadfe84242d3145ad9febd3939d494c57124f Mon Sep 17 00:00:00 2001 From: Fish Date: Thu, 6 Aug 2026 21:12:43 -0400 Subject: [PATCH 103/122] Decompiler: Deterministically pick variables from sets. (#6778) * Decompiler: Deterministically pick variables from sets. * Add missing file. --- angr/ailment/utils.py | 63 ------------------- angr/analyses/decompiler/clinic.py | 34 ++++++++--- angr/analyses/decompiler/structurer_nodes.py | 5 +- angr/sim_variable.py | 14 +++-- angr/utils/hashing.py | 64 ++++++++++++++++++++ 5 files changed, 98 insertions(+), 82 deletions(-) create mode 100644 angr/utils/hashing.py diff --git a/angr/ailment/utils.py b/angr/ailment/utils.py index fb5f07ec3..157c8af65 100644 --- a/angr/ailment/utils.py +++ b/angr/ailment/utils.py @@ -1,7 +1,5 @@ from __future__ import annotations -import struct - import archinfo from angr import ailment @@ -11,11 +9,6 @@ try: except ImportError: from typing import Never as Bits -try: - import _md5 as md5lib # type: ignore # stdlib C module without stubs -except ImportError: - import hashlib as md5lib - type GetBitsTypeParams = "ailment.expression.Expression" @@ -27,62 +20,6 @@ def get_bits(expr: GetBitsTypeParams) -> int: raise TypeError(type(expr)) -md5_unpacker = struct.Struct("4I") - - -def stable_hash(t: tuple) -> int: - cnt = _dump_tuple(t) - hd = md5lib.md5(cnt).digest() - return md5_unpacker.unpack(hd)[0] # 32 bits - - -def _dump_tuple(t: tuple) -> bytes: - cnt = b"" - for item in t: - if item is not None: - type_ = type(item) - if type_ in _DUMP_BY_TYPE: - cnt += _DUMP_BY_TYPE[type_](item) - else: - # for TaggedObjects, hash(item) is stable - # other types of items may show up, such as pyvex.expr.CCall and Dirty. they will be removed some day. - cnt += struct.pack(" bytes: - return t.encode("utf-8") - - -def _dump_int(t: int) -> bytes: - prefix = b"" if t >= 0 else b"-" - t = abs(t) - if t <= 0xFFFF: - return prefix + struct.pack(" 0: - cnt += _dump_int(t & 0xFFFF_FFFF_FFFF_FFFF) - t >>= 64 - return prefix + cnt - - -def _dump_type(t: type) -> bytes: - return t.__name__.encode("ascii") - - -_DUMP_BY_TYPE = { - tuple: _dump_tuple, - str: _dump_str, - int: _dump_int, - type: _dump_type, -} - - def is_none_or_likeable(arg1, arg2, is_list=False): """ Returns whether two things are both None or can like each other diff --git a/angr/analyses/decompiler/clinic.py b/angr/analyses/decompiler/clinic.py index c5306a70c..eaea015e4 100644 --- a/angr/analyses/decompiler/clinic.py +++ b/angr/analyses/decompiler/clinic.py @@ -116,6 +116,20 @@ l = logging.getLogger(name=__name__) BlockCache = namedtuple("BlockCache", ("rd", "prop")) +def _pick_var(candidates: set[SimVariable]) -> SimVariable: + """ + Deterministically pick one SimVariable out of a candidate set. + """ + return min(candidates, key=lambda var: str(var.key)) + + +def _pick_var_and_offset(candidates: set[tuple[SimVariable, int | None]]) -> tuple[SimVariable, int | None]: + """ + Deterministically pick one (SimVariable, offset) pair out of a candidate set. + """ + return min(candidates, key=lambda var_and_offset: (str(var_and_offset[1]), str(var_and_offset[0].key))) + + class ClinicMode(enum.Enum): """ Analysis mode for Clinic. @@ -2697,7 +2711,7 @@ class Clinic(Analysis, Serializable): # global variable? variables = global_variables.get_global_variables(stmt.addr.value) if variables: - var = next(iter(variables)) + var = _pick_var(variables) self._set_store_variable(stmt, var, 0) else: self._link_variables_on_expr( @@ -2787,13 +2801,13 @@ class Clinic(Analysis, Serializable): else: final_reg_vars = reg_vars if len(final_reg_vars) >= 1: - reg_var, offset = next(iter(final_reg_vars)) + reg_var, offset = _pick_var_and_offset(final_reg_vars) self._set_expr_variable(expr, reg_var, offset) elif isinstance(expr, ailment.Expr.VirtualVariable): vars_ = variable_manager.find_variables_by_atom(block.addr, stmt_idx, expr, block_idx=block.idx) if len(vars_) >= 1: - var, offset = next(iter(vars_)) + var, offset = _pick_var_and_offset(vars_) self._set_expr_variable(expr, var, offset) if expr.was_combo_reg: @@ -2836,13 +2850,13 @@ class Clinic(Analysis, Serializable): l.error( "More than one variable are available for atom %s. Consider fixing it using phi nodes.", expr ) - var, offset = next(iter(variables)) + var, offset = _pick_var_and_offset(variables) self._set_expr_variable(expr, var, offset) elif isinstance(expr, ailment.Expr.BinaryOp): variables = variable_manager.find_variables_by_atom(block.addr, stmt_idx, expr, block_idx=block.idx) if len(variables) >= 1: - var, offset = next(iter(variables)) + var, offset = _pick_var_and_offset(variables) self._set_expr_variable(expr, var, offset) else: self._link_variables_on_expr( @@ -2855,7 +2869,7 @@ class Clinic(Analysis, Serializable): elif isinstance(expr, ailment.Expr.UnaryOp): variables = variable_manager.find_variables_by_atom(block.addr, stmt_idx, expr, block_idx=block.idx) if len(variables) >= 1: - var, offset = next(iter(variables)) + var, offset = _pick_var_and_offset(variables) self._set_expr_variable(expr, var, offset) else: self._link_variables_on_expr(variable_manager, global_variables, block, stmt_idx, stmt, expr.operand) @@ -2874,7 +2888,7 @@ class Clinic(Analysis, Serializable): elif isinstance(expr, ailment.Expr.ITE): variables = variable_manager.find_variables_by_atom(block.addr, stmt_idx, expr, block_idx=block.idx) if len(variables) >= 1: - var, offset = next(iter(variables)) + var, offset = _pick_var_and_offset(variables) self._set_expr_variable(expr, var, offset) else: self._link_variables_on_expr(variable_manager, global_variables, block, stmt_idx, stmt, expr.cond) @@ -2884,7 +2898,7 @@ class Clinic(Analysis, Serializable): elif isinstance(expr, ailment.Expr.BasePointerOffset): variables = variable_manager.find_variables_by_atom(block.addr, stmt_idx, expr, block_idx=block.idx) if len(variables) >= 1: - var, offset = next(iter(variables)) + var, offset = _pick_var_and_offset(variables) self._set_expr_variable(expr, var, offset) elif isinstance(expr, ailment.Expr.Const) and expr.is_int: @@ -2912,13 +2926,13 @@ class Clinic(Analysis, Serializable): global_variables.add_variable("global", global_var.addr, global_var) global_vars = {global_var} if global_vars: - global_var = next(iter(global_vars)) + global_var = _pick_var(global_vars) self._set_reference_variable(expr, global_var, 0) else: # is there a related constant variable? variables = variable_manager.find_variables_by_atom(block.addr, stmt_idx, expr, block_idx=block.idx) if len(variables) >= 1: - var, offset = next(iter(variables)) + var, offset = _pick_var_and_offset(variables) self._set_expr_variable(expr, var, offset) elif isinstance(expr, ailment.Expr.Call): diff --git a/angr/analyses/decompiler/structurer_nodes.py b/angr/analyses/decompiler/structurer_nodes.py index 6d5678628..8d1860a11 100644 --- a/angr/analyses/decompiler/structurer_nodes.py +++ b/angr/analyses/decompiler/structurer_nodes.py @@ -11,6 +11,7 @@ import angr import angr.ailment.utils from angr import ailment from angr.ailment.block import Block +from angr.utils.hashing import stable_hash INDENT_DELTA = 2 @@ -462,9 +463,7 @@ class IncompleteSwitchCaseHeadStatement(_IncompleteSwitchCaseHeadStatementBase): __hash__ = ailment.statement.TaggedObject.__hash__ def _hash_core(self): - return angr.ailment.utils.stable_hash( - (IncompleteSwitchCaseHeadStatement, self.idx, self.switch_variable, self._case_addrs_str) - ) + return stable_hash((IncompleteSwitchCaseHeadStatement, self.idx, self.switch_variable, self._case_addrs_str)) def replace(self, old_expr, new_expr): # pylint:disable=unused-argument return self diff --git a/angr/sim_variable.py b/angr/sim_variable.py index 0e965a615..d69e8c32f 100644 --- a/angr/sim_variable.py +++ b/angr/sim_variable.py @@ -5,6 +5,8 @@ from typing import TYPE_CHECKING, Self import claripy +from angr.utils.hashing import stable_hash + from .protos import variables_pb2 as pb2 from .serializable import Serializable @@ -141,7 +143,7 @@ class SimConstantVariable(SimVariable): def __hash__(self): if self._hash is None: - self._hash = hash(("const", self.value, self.ident, self.region, self.ident)) + self._hash = stable_hash(("const", self.value, self.ident, self.region, self.ident)) return self._hash def copy(self) -> SimConstantVariable: @@ -201,7 +203,7 @@ class SimTemporaryVariable(SimVariable): def __hash__(self): if self._hash is None: - self._hash = hash(f"tmp_{self.tmp_id}") + self._hash = stable_hash(("tmp", self.tmp_id)) return self._hash def __eq__(self, other): @@ -261,7 +263,7 @@ class SimRegisterVariable(SimVariable): def __hash__(self): if self._hash is None: - self._hash = hash(("reg", self.region, self.reg, self.size, self.ident)) + self._hash = stable_hash(("reg", self.region, self.reg, self.size, self.ident)) return self._hash def __eq__(self, other): @@ -334,7 +336,7 @@ class SimComboRegisterVariable(SimVariable): def __hash__(self): if self._hash is None: - self._hash = hash(("combo_reg", self.region, tuple(self.reg_offsets), self.size, self.ident)) + self._hash = stable_hash(("combo_reg", self.region, tuple(self.reg_offsets), self.size, self.ident)) return self._hash def __eq__(self, other): @@ -413,7 +415,7 @@ class SimMemoryVariable(SimVariable): if self._hash is not None: return self._hash - self._hash = hash((hash(self.addr), hash(self.size), self.ident)) + self._hash = stable_hash((self.addr, self.size, self.ident)) return self._hash def __eq__(self, other): @@ -517,7 +519,7 @@ class SimStackVariable(SimMemoryVariable): def __hash__(self): if self._hash is None: - self._hash = hash((self.ident, self.base, self.offset, self.size)) + self._hash = stable_hash((self.ident, self.base, self.offset, self.size)) return self._hash def copy(self) -> SimStackVariable: diff --git a/angr/utils/hashing.py b/angr/utils/hashing.py new file mode 100644 index 000000000..0ae10c10e --- /dev/null +++ b/angr/utils/hashing.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import struct + +try: + import _md5 as md5lib # type: ignore # stdlib C module without stubs +except ImportError: + import hashlib as md5lib + + +md5_unpacker = struct.Struct("4I") + + +def stable_hash(t: tuple) -> int: + cnt = _dump_tuple(t) + hd = md5lib.md5(cnt).digest() + return md5_unpacker.unpack(hd)[0] # 32 bits + + +def _dump_tuple(t: tuple) -> bytes: + cnt = b"" + for item in t: + if item is not None: + type_ = type(item) + if type_ in _DUMP_BY_TYPE: + cnt += _DUMP_BY_TYPE[type_](item) + else: + # for TaggedObjects, hash(item) is stable + # other types of items may show up, such as pyvex.expr.CCall and Dirty. they will be removed some day. + cnt += struct.pack(" bytes: + return t.encode("utf-8") + + +def _dump_int(t: int) -> bytes: + prefix = b"" if t >= 0 else b"-" + t = abs(t) + if t <= 0xFFFF: + return prefix + struct.pack(" 0: + cnt += _dump_int(t & 0xFFFF_FFFF_FFFF_FFFF) + t >>= 64 + return prefix + cnt + + +def _dump_type(t: type) -> bytes: + return t.__name__.encode("ascii") + + +_DUMP_BY_TYPE = { + tuple: _dump_tuple, + str: _dump_str, + int: _dump_int, + type: _dump_type, +} From c844a13eac6ff3e933d361b84c6d7a6758abb24e Mon Sep 17 00:00:00 2001 From: Fish Date: Fri, 7 Aug 2026 12:06:49 -0700 Subject: [PATCH 104/122] Clinic: Skip unnecessary simplification runs and recomputations. (#6780) * Clinic: Skip the second pre-SSA-level1 _simplify_function when it cannot help. * Clinic: Merge the two identical post-SSA-level1 _simplify_function calls. * AILSimplifier, SLiveness: Skip recomputation that cannot change anything. --- angr/analyses/decompiler/ail_simplifier.py | 137 +++++++++++++++++---- angr/analyses/decompiler/clinic.py | 83 +++++++------ angr/analyses/s_liveness.py | 7 ++ 3 files changed, 162 insertions(+), 65 deletions(-) diff --git a/angr/analyses/decompiler/ail_simplifier.py b/angr/analyses/decompiler/ail_simplifier.py index a9cf5fdd6..0813d29e6 100644 --- a/angr/analyses/decompiler/ail_simplifier.py +++ b/angr/analyses/decompiler/ail_simplifier.py @@ -80,6 +80,58 @@ _l = logging.getLogger(__name__) _VERIFY_INCREMENTAL_RD = os.environ.get("VERIFY_INCREMENTAL_RD", "").lower() not in {"", "0", "no", "false"} +def _strongly_connected_components(succs: dict[int, set[int]]): + """ + Iterative Tarjan SCC over a plain adjacency map. Yields sets of node IDs, like + networkx.strongly_connected_components(). + """ + index_of: dict[int, int] = {} + lowlink: dict[int, int] = {} + on_stack: set[int] = set() + stack: list[int] = [] + counter = 0 + + for root, root_succs in succs.items(): + if root in index_of: + continue + work = [(root, iter(root_succs))] + index_of[root] = lowlink[root] = counter + counter += 1 + stack.append(root) + on_stack.add(root) + + while work: + node, it = work[-1] + advanced = False + for succ in it: + if succ not in index_of: + index_of[succ] = lowlink[succ] = counter + counter += 1 + stack.append(succ) + on_stack.add(succ) + work.append((succ, iter(succs[succ]))) + advanced = True + break + if succ in on_stack and index_of[succ] < lowlink[node]: + lowlink[node] = index_of[succ] + if advanced: + continue + + work.pop() + if work: + parent = work[-1][0] + lowlink[parent] = min(lowlink[parent], lowlink[node]) + if lowlink[node] == index_of[node]: + scc = set() + while True: + member = stack.pop() + on_stack.discard(member) + scc.add(member) + if member == node: + break + yield scc + + class HasVVarNotification(Exception): """ Notifies the existence of a VirtualVariable. @@ -208,6 +260,11 @@ class AILSimplifier(Analysis): self._arg_vvars = arg_vvars self._avoid_vvar_ids = avoid_vvar_ids if avoid_vvar_ids is not None else set() self._propagator_dead_vvar_ids: set[int] = set() + # per-block cache of dirty/ccall-defined vvar IDs, keyed by block key, validated by block identity + self._dirty_vvar_scan_cache: dict[tuple[int, int | None], tuple[Block, set[int]]] = {} + # only set to True when any simplification pass has modified the graph or updated any blocks. + # skips _remove_dead_assignments if this flag is False. + self._should_eliminate_dead_assignments: bool = True self._calls_to_remove: set[AILCodeLocation] = set() self._assignments_to_remove: set[AILCodeLocation] = set() @@ -335,6 +392,7 @@ class AILSimplifier(Analysis): AILGraphWalker(self.func_graph, _handler, replace_nodes=True).walk() self.blocks = {} + self._should_eliminate_dead_assignments = True def _compute_reaching_definitions(self) -> SRDAModel: # Computing reaching definitions or return the cached one @@ -372,6 +430,7 @@ class AILSimplifier(Analysis): ) self._propagator = prop self._propagator_dead_vvar_ids = prop.dead_vvar_ids + self._should_eliminate_dead_assignments = True return prop @timethis @@ -1837,6 +1896,14 @@ class AILSimplifier(Analysis): @timethis def _iteratively_remove_dead_assignments(self) -> bool: + if ( + not self._should_eliminate_dead_assignments + and not self.blocks + and not self._calls_to_remove + and not self._assignments_to_remove + ): + # nothing that _remove_dead_assignments() reads has changed since it last reported nothing to remove + return False anything_removed = False while True: r, changed_block_keys = self._remove_dead_assignments() @@ -1857,6 +1924,8 @@ class AILSimplifier(Analysis): # propagation results are no longer reliable after removing statements self._propagator = None + self._should_eliminate_dead_assignments = False + # NoOp placeholders are left in the graph and the reaching-definitions cache is kept valid: subsequent # simplification steps reuse it instead of rebuilding from scratch. The placeholders are compacted away once, # at the end of _simplify(). @@ -2165,21 +2234,32 @@ class AILSimplifier(Analysis): def _find_cyclic_dependent_phis_and_dirty_vvars(self, rd: SRDAModel, dead_vvar_ids: set[int]) -> set[int]: blocks_dict: dict[tuple[int, int | None], Block] = {(bb.addr, bb.idx): bb for bb in self.func_graph} - # find dirty vvars and vexccall vvars dirty_vvar_ids = set() + # cache dirty or ccall vvar IDs per block to avoid re-scanning + # TODO: Move this cache to ailment.Block once per-block defs/uses cache lands on master. + cache = self._dirty_vvar_scan_cache for bb in self.func_graph: - for stmt in bb.statements: - # reg/tmp = ccall(...) - # we see tmps when it's used in a cycle; - # see binary ddc2b4cbf6ac841524375cdf82b93b9948f8ea09bbf6e8bf3410e6bc410a9d95 function 0x18001722c - # block 0x18001724c - if ( - isinstance(stmt, Assignment) - and isinstance(stmt.dst, VirtualVariable) - and (stmt.dst.was_reg or stmt.dst.was_tmp) - and isinstance(stmt.src, (DirtyExpression, VEXCCallExpression)) - ): - dirty_vvar_ids.add(stmt.dst.varid) + key = bb.addr, bb.idx + entry = cache.get(key) + if entry is not None and entry[0] is bb: + block_dirty_ids = entry[1] + else: + block_dirty_ids = set() + for stmt in bb.statements: + # reg/tmp = ccall(...) + # we see tmps when it's used in a cycle; + # see binary ddc2b4cbf6ac841524375cdf82b93b9948f8ea09bbf6e8bf3410e6bc410a9d95 function 0x18001722c + # block 0x18001724c + if ( + isinstance(stmt, Assignment) + and isinstance(stmt.dst, VirtualVariable) + and (stmt.dst.was_reg or stmt.dst.was_tmp) + and isinstance(stmt.src, (DirtyExpression, VEXCCallExpression)) + ): + block_dirty_ids.add(stmt.dst.varid) + cache[key] = bb, block_dirty_ids + if block_dirty_ids: + dirty_vvar_ids |= block_dirty_ids phi_and_dirty_vvar_ids = (rd.phi_vvar_ids | dirty_vvar_ids).difference(dead_vvar_ids) @@ -2197,18 +2277,25 @@ class AILSimplifier(Analysis): vvar_used_by[used_by_varid].add(var_id) # probably unnecessary vvar_used_by[var_id] |= self._get_vvar_used_by(var_id, rd, blocks_dict).difference(dead_vvar_ids) - g = networkx.DiGraph() + # build a plain adjacency map instead of a throwaway networkx DiGraph for better performance. the performance + # improvement is observable on notepad.exe:NPInit + # TODO: Investigate if switching to rustworkx eliminates the need for this optimization. dummy_vvar_id = -1 + succs_map: dict[int, set[int]] = {} for var_id, used_by_initial in vvar_used_by.items(): - for u in used_by_initial: - if u is None: - # we can't have None in networkx.DiGraph - g.add_edge(var_id, dummy_vvar_id) - else: - g.add_edge(var_id, u) + if not used_by_initial: + continue + targets = {dummy_vvar_id if u is None else u for u in used_by_initial} + if var_id in succs_map: + succs_map[var_id] |= targets + else: + succs_map[var_id] = set(targets) + for target in targets: + if target not in succs_map: + succs_map[target] = set() cyclic_dependent_phi_varids = set() - for scc in networkx.strongly_connected_components(g): + for scc in _strongly_connected_components(succs_map): if len(scc) == 1: continue @@ -2219,11 +2306,9 @@ class AILSimplifier(Analysis): if varid in vvar_used_by and None in vvar_used_by[varid]: bail = True break - if bail is False: - succs = list(g.successors(varid)) - if any(succ_varid not in scc for succ_varid in succs): - bail = True - break + if any(succ_varid not in scc for succ_varid in succs_map[varid]): + bail = True + break if bail: continue diff --git a/angr/analyses/decompiler/clinic.py b/angr/analyses/decompiler/clinic.py index eaea015e4..312fd059a 100644 --- a/angr/analyses/decompiler/clinic.py +++ b/angr/analyses/decompiler/clinic.py @@ -548,7 +548,7 @@ class Clinic(Analysis, Serializable): # Run simplification passes self._update_progress(22.0, text="Optimizing fresh ailment graph") - ail_graph = self._run_simplification_passes(ail_graph, OptimizationPassStage.AFTER_AIL_GRAPH_CREATION) + _, ail_graph = self._run_simplification_passes(ail_graph, OptimizationPassStage.AFTER_AIL_GRAPH_CREATION) # Fix "fake" indirect jumps and calls self._update_progress(25.0, text="Analyzing simple indirect jumps") @@ -836,7 +836,7 @@ class Clinic(Analysis, Serializable): self._update_progress(30.0, text="Making return sites") if self.function.prototype is None or not isinstance(self.function.prototype.returnty, SimTypeBottom): self._ail_graph = self._make_returns(self._ail_graph) - self._ail_graph = self._run_simplification_passes( + _, self._ail_graph = self._run_simplification_passes( self._ail_graph, stage=OptimizationPassStage.BEFORE_SSA_LEVEL0_TRANSFORMATION ) @@ -882,7 +882,7 @@ class Clinic(Analysis, Serializable): # Run simplification passes self._update_progress(49.0, text="Running simplifications 1.5") - self._ail_graph = self._run_simplification_passes( + _, self._ail_graph = self._run_simplification_passes( self._ail_graph, stage=OptimizationPassStage.AFTER_SSA_LEVEL1_TRANSFORMATION, arg_vvars=self.arg_vvars ) @@ -908,7 +908,7 @@ class Clinic(Analysis, Serializable): # Run simplification passes self._update_progress(40.0, text="Running simplifications 1") - self._ail_graph = self._run_simplification_passes( + _, self._ail_graph = self._run_simplification_passes( self._ail_graph, stack_pointer_tracker=self._spt, stack_items=self.stack_items, @@ -928,7 +928,7 @@ class Clinic(Analysis, Serializable): # Simplify the entire function for the first time self._update_progress(45.0, text="Simplifying function 1") - self._simplify_function( + converged = self._simplify_function( self._ail_graph, remove_dead_memdefs=False, unify_variables=False, @@ -939,19 +939,22 @@ class Clinic(Analysis, Serializable): # Run simplification passes self._update_progress(47.0, text="Running simplifications 2") - self._ail_graph = self._run_simplification_passes( + simplified, self._ail_graph = self._run_simplification_passes( self._ail_graph, stage=OptimizationPassStage.BEFORE_SSA_LEVEL1_TRANSFORMATION ) - self._update_progress(49.0, text="Simplifying blocks 1") - self._simplify_function( - self._ail_graph, - remove_dead_memdefs=False, - unify_variables=False, - narrow_expressions=False, - fold_callexprs_into_conditions=self._fold_callexprs_into_conditions, - arg_vvars=self.arg_vvars, - ) + # If the call above reached a fixed point and no optimization pass has touched the graph since, simplifying + # again can only re-prove that fixed point, so skip it. + if not converged or simplified: + self._update_progress(49.0, text="Simplifying blocks 1") + self._simplify_function( + self._ail_graph, + remove_dead_memdefs=False, + unify_variables=False, + narrow_expressions=False, + fold_callexprs_into_conditions=self._fold_callexprs_into_conditions, + arg_vvars=self.arg_vvars, + ) def _stage_make_function_callsites(self) -> None: # Make call-sites @@ -973,7 +976,7 @@ class Clinic(Analysis, Serializable): # Run simplification passes self._update_progress(53.0, text="Running simplifications 2.5") - self._ail_graph = self._run_simplification_passes( + _, self._ail_graph = self._run_simplification_passes( self._ail_graph, stage=OptimizationPassStage.AFTER_MAKING_CALLSITES, arg_vvars=self.arg_vvars ) @@ -993,14 +996,14 @@ class Clinic(Analysis, Serializable): # Run simplification passes self._update_progress(65.0, text="Running simplifications 3") - self._ail_graph = self._run_simplification_passes( + _, self._ail_graph = self._run_simplification_passes( self._ail_graph, stack_items=self.stack_items, stage=OptimizationPassStage.AFTER_GLOBAL_SIMPLIFICATION, arg_vvars=self.arg_vvars, ) - # Simplify the entire function for the third time + # Simplify the entire function for the third time. self._update_progress(70.0, text="Simplifying function 3") self._simplify_function( self._ail_graph, @@ -1008,19 +1011,7 @@ class Clinic(Analysis, Serializable): stackarg_offset_manager=self._stackarg_offset_manager, unify_variables=True, narrow_expressions=True, - fold_callexprs_into_conditions=self._fold_callexprs_into_conditions, - arg_vvars=self.arg_vvars, - preserve_vvar_ids=self._preserve_vvar_ids, - ) - - # Simplify the entire function for the fourth time - self._update_progress(78.0, text="Simplifying function 4") - self._simplify_function( - self._ail_graph, - remove_dead_memdefs=self._remove_dead_memdefs, - stackarg_offset_manager=self._stackarg_offset_manager, - unify_variables=True, - narrow_expressions=True, + narrow_rounds=None, fold_callexprs_into_conditions=self._fold_callexprs_into_conditions, arg_vvars=self.arg_vvars, preserve_vvar_ids=self._preserve_vvar_ids, @@ -1031,7 +1022,7 @@ class Clinic(Analysis, Serializable): self.copied_var_ids = set() self._update_progress(79.0, text="Running simplifications 4") - self._ail_graph = self._run_simplification_passes( + _, self._ail_graph = self._run_simplification_passes( self._ail_graph, stack_items=self.stack_items, stage=OptimizationPassStage.BEFORE_VARIABLE_RECOVERY, @@ -1071,7 +1062,7 @@ class Clinic(Analysis, Serializable): # Run simplification passes self._update_progress(85.0, text="Running simplifications 4") - self._ail_graph = self._run_simplification_passes( + _, self._ail_graph = self._run_simplification_passes( self._ail_graph, stage=OptimizationPassStage.AFTER_VARIABLE_RECOVERY, avoid_vvar_ids=self.copied_var_ids, @@ -2011,6 +2002,7 @@ class Clinic(Analysis, Serializable): unify_variables=False, max_iterations: int = 8, narrow_expressions=False, + narrow_rounds: int | None = 1, only_consts=False, fold_callexprs_into_conditions=False, rewrite_ccalls=True, @@ -2019,9 +2011,15 @@ class Clinic(Analysis, Serializable): arg_vvars: dict[int, tuple[ailment.Expr.VirtualVariable, SimVariable]] | None = None, preserve_vvar_ids: set[int] | None = None, simplify_blocks: bool = True, - ) -> None: + ) -> bool: """ Simplify the entire function until it reaches a fixed point. + + :param narrow_rounds: How many leading iterations run expression narrowing, or None to narrow on every + iteration. Narrowing is not part of the fixed point, so a caller that needs more than + one narrowing round asks for it here instead of calling this method twice. + :return: True if a fixed point was reached, False if the iteration limit was hit first (in + which case the graph may still be simplifiable). """ for idx in range(max_iterations): @@ -2030,8 +2028,7 @@ class Clinic(Analysis, Serializable): remove_dead_memdefs=remove_dead_memdefs, unify_variables=unify_variables, stackarg_offset_manager=stackarg_offset_manager, - # only narrow once - narrow_expressions=narrow_expressions and idx == 0, + narrow_expressions=narrow_expressions and (narrow_rounds is None or idx < narrow_rounds), only_consts=only_consts, fold_callexprs_into_conditions=fold_callexprs_into_conditions, rewrite_ccalls=rewrite_ccalls, @@ -2042,7 +2039,8 @@ class Clinic(Analysis, Serializable): simplify_blocks=simplify_blocks, ) if not simplified: - break + return True + return False @timethis def _simplify_function_once( @@ -2117,7 +2115,11 @@ class Clinic(Analysis, Serializable): stack_items: dict[int, StackItem] | None = None, stack_pointer_tracker=None, **kwargs, - ): + ) -> tuple[bool, networkx.DiGraph]: + """ + :return: A tuple of (any simplifications were made and the graph was changed, the resulting AIL graph). + """ + addr_and_idx_to_blocks: dict[ailment.Address, ailment.Block] = {} addr_to_blocks: dict[int, set[ailment.Block]] = defaultdict(set) @@ -2128,6 +2130,8 @@ class Clinic(Analysis, Serializable): AILGraphWalker(ail_graph, _updatedict_handler).walk() + simplified = False + # Run each pass for pass_ in self._optimization_passes: if stage != pass_.STAGE: @@ -2166,10 +2170,11 @@ class Clinic(Analysis, Serializable): # use the new graph ail_graph = a.out_graph self.vvar_id_start = a.vvar_id_start + simplified = True if stack_items is not None and a.stack_items: stack_items.update(a.stack_items) - return ail_graph + return simplified, ail_graph @timethis def _create_function_argument_vvars(self, arg_list) -> dict[int, tuple[ailment.Expr.VirtualVariable, SimVariable]]: diff --git a/angr/analyses/s_liveness.py b/angr/analyses/s_liveness.py index 0f4bb1692..b76113a82 100644 --- a/angr/analyses/s_liveness.py +++ b/angr/analyses/s_liveness.py @@ -71,6 +71,8 @@ class SLivenessAnalysis(Analysis): live_outs[block_key] = set() live_on_edges: dict[tuple[tuple[int, int | None], tuple[int, int | None]], set[int]] = {} + # blocks whose statements have been walked at least once + walked: set[tuple[int, int | None]] = set() worklist = deque(networkx.dfs_postorder_nodes(graph, source=entry)) worklist_set = set(worklist) @@ -104,6 +106,11 @@ class SLivenessAnalysis(Analysis): if live != live_outs[block_key]: changed = True live_outs[block_key] = live.copy() + elif block_key in walked: + # the live-out set is what it was the last time this block was walked, and the walk depends on + # nothing else, so live_ins and live_on_edges are already up to date for this block + continue + walked.add(block_key) if head_controlled_loop: # this is a head-controlled loop block; we start scanning from the first condition jump backwards From 08c7e48d5be62a3b96cfcab98d85a1ee2dcc34d6 Mon Sep 17 00:00:00 2001 From: Fish Date: Fri, 7 Aug 2026 14:16:21 -0700 Subject: [PATCH 105/122] CFGFast: Mark bad blocks found by complete scanning as nodecode. (#6781) --- angr/analyses/cfg/cfg_fast.py | 10 ++++++++++ tests/analyses/cfg/test_cfgfast.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/angr/analyses/cfg/cfg_fast.py b/angr/analyses/cfg/cfg_fast.py index 13ecd5c74..53a2caf35 100644 --- a/angr/analyses/cfg/cfg_fast.py +++ b/angr/analyses/cfg/cfg_fast.py @@ -5865,6 +5865,16 @@ class CFGFast(ForwardAnalysis[CFGNode, CFGNode, CFGJob, int, object], CFGBase): ) irsb_string = irsb_string[: irsb.size] + if ( + cfg_job.job_type == CFGJobType.COMPLETE_SCANNING + and addr == current_function_addr + and irsb.jumpkind == "Ijk_NoDecode" + ): + # linear sweep decided that this block is undecodable. because drop_bad_functions() will remove this + # function anyway, we bail out early and mark the whole block as nodecode for performance. + self._seg_list.occupy(real_addr, max(irsb.size, 1), "nodecode") + return None, None, None, None + # Occupy the block in segment list if irsb is not None and irsb.size > 0: self._seg_list.occupy(real_addr, irsb.size, "code") diff --git a/tests/analyses/cfg/test_cfgfast.py b/tests/analyses/cfg/test_cfgfast.py index d9099d748..6e13b78bf 100755 --- a/tests/analyses/cfg/test_cfgfast.py +++ b/tests/analyses/cfg/test_cfgfast.py @@ -4,8 +4,10 @@ from __future__ import annotations __package__ = __package__ or "tests.analyses.cfg" # pylint:disable=redefined-builtin +import io import logging import os +import random import unittest import archinfo @@ -1000,6 +1002,35 @@ class TestCfgfast(unittest.TestCase): for addr in not_separate_functions: assert addr not in cfg.kb.functions, f"{hex(addr)} should not be a separate function" + @staticmethod + def _blob_project(data: bytes) -> angr.Project: + return angr.Project( + io.BytesIO(data), + main_opts={"backend": "blob", "arch": "AMD64", "base_addr": 0, "entry_point": 0}, + auto_load_libs=False, + use_sim_procedures=False, + ) + + def test_smart_scan_marks_decode_error_blocks_as_nodecode(self): + # 0x0: xor eax, eax; ret - a real function + # 0x3: inc rax; - garbage that the linear sweep lands on and that dies on a decode error + proj = self._blob_project(b"\x31\xc0\xc3" + b"\x48\xff\xc0\x0f\x39" + b"\x00" * 16) + cfg = proj.analyses.CFGFast(force_smart_scan=True, data_references=True) + + assert 0 in cfg.kb.functions + assert 3 not in cfg.kb.functions + # the whole block is data, not a run of code followed by a single nodecode byte + assert cfg._seg_list.occupied_by_sort(3) == "nodecode" + + def test_smart_scan_does_not_explode_on_random_data(self): + # a blob of random bytes has no functions at all, but every address decodes into something, so the smart + # scan used to cover it with thousands of one-block functions that drop_bad_functions() threw away again + rng = random.Random(0xDEADBEEF) + proj = self._blob_project(bytes(rng.getrandbits(8) for _ in range(32768))) + cfg = proj.analyses.CFGFast(normalize=True) + + assert len(cfg.kb.functions) < 150, f"32 KB of random data produced {len(cfg.kb.functions)} functions" + if __name__ == "__main__": unittest.main() From c751678d556219927c7c8f3707567a7f93e731fd Mon Sep 17 00:00:00 2001 From: Fish Date: Fri, 7 Aug 2026 18:13:32 -0700 Subject: [PATCH 106/122] Decompiler: Handle compare-and-swap statements that CASIntrinsics leaves behind. (#6783) --- .../peephole_optimizations/cas_intrinsics.py | 50 ++++++++++++++----- .../decompiler/structured_codegen/c.py | 19 +++++++ 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/angr/analyses/decompiler/peephole_optimizations/cas_intrinsics.py b/angr/analyses/decompiler/peephole_optimizations/cas_intrinsics.py index 365378346..2b26fc60a 100644 --- a/angr/analyses/decompiler/peephole_optimizations/cas_intrinsics.py +++ b/angr/analyses/decompiler/peephole_optimizations/cas_intrinsics.py @@ -2,7 +2,7 @@ from __future__ import annotations from angr.ailment import Const -from angr.ailment.expression import BinaryOp, Call, Expression, Load, Tmp +from angr.ailment.expression import ITE, BinaryOp, Call, Expression, Load, Tmp from angr.ailment.statement import CAS, Assignment, ConditionalJump, Statement from .base import PeepholeOptimizationMultiStmtBase @@ -27,6 +27,16 @@ _INTRINSICS_NAMES = { } +def cas_intrinsic_name(mnemonic: str, os_name: str | None) -> str: + """ + Resolve the intrinsic name for a lock-prefixed instruction on the given OS, falling back to Linux naming. + """ + if mnemonic not in _INTRINSICS_NAMES: + return mnemonic + names = _INTRINSICS_NAMES[mnemonic] + return names[os_name] if os_name in names else names["Linux"] + + class CASIntrinsics(PeepholeOptimizationMultiStmtBase): """ Rewrite lock-prefixed instructions (or rather, their VEX/AIL forms) into intrinsic calls. @@ -135,7 +145,9 @@ class CASIntrinsics(PeepholeOptimizationMultiStmtBase): stmt = Assignment(cas_stmt.idx, assignment_dst, call_expr, **cas_stmt.tags) # type: ignore return [stmt] - if next_stmt.tags["ins_addr"] <= cas_stmt.tags["ins_addr"]: + if next_stmt.tags["ins_addr"] <= cas_stmt.tags["ins_addr"] and not self._is_cas_writeback_ite( + cas_stmt, next_stmt + ): # avoid matching against statements prematurely return None @@ -159,17 +171,31 @@ class CASIntrinsics(PeepholeOptimizationMultiStmtBase): return None + @staticmethod + def _is_cas_writeback_ite(cas_stmt: CAS, stmt: Statement) -> bool: + """ + Detect the ITE assignment that models "cmpxchg writes the memory value back into the accumulator". It belongs + to the same instruction as the CAS, so the same-instruction guard above would otherwise keep this shape from + ever being rewritten. Case 1 cannot apply to it either, because that requires the next statement to be a + CasCmpNE conditional jump. + + CAS(addr, expd_lo=X, data_lo=D, old_lo=OLD) + vvar = (X == OLD) ? X : OLD + """ + if cas_stmt.old_lo is None or not isinstance(stmt, Assignment) or not isinstance(stmt.src, ITE): + return False + ite = stmt.src + if not (isinstance(ite.cond, BinaryOp) and ite.cond.op == "CmpEQ"): + return False + expd, old = cas_stmt.expd_lo, cas_stmt.old_lo + cond_op0, cond_op1 = ite.cond.operands + return ((cond_op0.likes(expd) and cond_op1.likes(old)) or (cond_op0.likes(old) and cond_op1.likes(expd))) and ( + (ite.iftrue.likes(expd) and ite.iffalse.likes(old)) or (ite.iftrue.likes(old) and ite.iffalse.likes(expd)) + ) + def _get_instrincs_name(self, mnemonic: str) -> str: - if mnemonic in _INTRINSICS_NAMES: - os = ( - self.project.simos.name - if self.project is not None and self.project.simos is not None and self.project.simos.name is not None - else "Linux" - ) - if os not in _INTRINSICS_NAMES[mnemonic]: - os = "Linux" - return _INTRINSICS_NAMES[mnemonic][os] - return mnemonic + os_name = self.project.simos.name if self.project is not None and self.project.simos is not None else None + return cas_intrinsic_name(mnemonic, os_name) @staticmethod def _resolve_tmp_expr(expr: Expression, block) -> Expression: diff --git a/angr/analyses/decompiler/structured_codegen/c.py b/angr/analyses/decompiler/structured_codegen/c.py index 8e4aa1089..2d107654a 100644 --- a/angr/analyses/decompiler/structured_codegen/c.py +++ b/angr/analyses/decompiler/structured_codegen/c.py @@ -14,6 +14,7 @@ from angr.ailment.constant import UNDETERMINED_SIZE from angr.ailment.expression import BinaryOp, StackBaseOffset from angr.analyses.analysis import Analysis, register_analysis from angr.analyses.decompiler.notes.deobfuscated_strings import DeobfuscatedStringsNote +from angr.analyses.decompiler.peephole_optimizations.cas_intrinsics import cas_intrinsic_name from angr.analyses.decompiler.region_identifier import MultiNode from angr.analyses.decompiler.structurer_nodes import ( BreakNode, @@ -2979,6 +2980,7 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis, Serializab Stmt.Return: self._handle_Stmt_Return, Stmt.Label: self._handle_Stmt_Label, Stmt.DirtyStatement: self._handle_Stmt_Dirty, + Stmt.CAS: self._handle_Stmt_CAS, # AIL expressions Expr.Register: self._handle_Expr_Register, Expr.Load: self._handle_Expr_Load, @@ -4112,6 +4114,23 @@ class CStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis, Serializab dirty = self._handle(stmt.dirty) return CDirtyStatement(dirty, codegen=self) + def _handle_Stmt_CAS(self, stmt: Stmt.CAS, **kwargs): + # CASIntrinsics normally rewrites compare-and-swap statements into intrinsic calls before we get here, but it + # only recognizes a handful of statement shapes. Render whatever it left behind as the same intrinsic call + # instead of failing the whole function. + if stmt.old_hi is None: + os_name = self.project.simos.name if self.project.simos is not None else None + call = Expr.Call( + stmt.idx, + cas_intrinsic_name(f"cmpxchg{stmt.bits}", os_name), + args=[stmt.addr, stmt.data_lo, stmt.expd_lo], + bits=stmt.bits, + **stmt.tags, + ) + return self._handle(Stmt.Assignment(stmt.idx, stmt.old_lo, call, **stmt.tags), is_expr=False) + # a double-width CAS writes two destinations, which no single C expression captures + return CUnsupportedStatement(stmt, codegen=self) + # # AIL expression handlers # From 780a80de793e9593defbfede861b7e1e3ca681fb Mon Sep 17 00:00:00 2001 From: Fish Date: Fri, 7 Aug 2026 18:41:56 -0700 Subject: [PATCH 107/122] Phoenix: Correct Loop.continue_addr. (#6785) --- angr/analyses/decompiler/structurer_nodes.py | 1 + .../decompiler/structuring/phoenix.py | 11 +- .../decompiler/structuring/structurer_base.py | 11 +- .../decompiler/test_dowhile_latch_continue.py | 145 ++++++++++++++++++ 4 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 tests/analyses/decompiler/test_dowhile_latch_continue.py diff --git a/angr/analyses/decompiler/structurer_nodes.py b/angr/analyses/decompiler/structurer_nodes.py index 8d1860a11..4b5d21008 100644 --- a/angr/analyses/decompiler/structurer_nodes.py +++ b/angr/analyses/decompiler/structurer_nodes.py @@ -298,6 +298,7 @@ class LoopNode(BaseNode): self.condition, self.sequence_node, addr=self._addr, + continue_addr=self._continue_addr, initializer=self.initializer, iterator=self.iterator, ) diff --git a/angr/analyses/decompiler/structuring/phoenix.py b/angr/analyses/decompiler/structuring/phoenix.py index 84b0e35fb..a23d1fd59 100644 --- a/angr/analyses/decompiler/structuring/phoenix.py +++ b/angr/analyses/decompiler/structuring/phoenix.py @@ -589,7 +589,16 @@ class PhoenixStructurer(StructurerBase): drop_succ = True new_node = SequenceNode(node.addr, nodes=[node] if drop_succ else [node, succ]) - loop_node = LoopNode("do-while", edge_cond_succhead, new_node, addr=node.addr) + loop_node = LoopNode( + "do-while", + edge_cond_succhead, + new_node, + addr=node.addr, + # when the latch is folded into the loop condition it stops being a node of its own, + # so remember where a continue has to land; otherwise jumps to it become gotos to a + # label that no longer exists + continue_addr=succ.addr if drop_succ else None, + ) self.replace_nodes_both( node, loop_node, old_node_1=succ, self_loop=False, drop_refinement_marks=True diff --git a/angr/analyses/decompiler/structuring/structurer_base.py b/angr/analyses/decompiler/structuring/structurer_base.py index 1515796ec..21abf14b8 100644 --- a/angr/analyses/decompiler/structuring/structurer_base.py +++ b/angr/analyses/decompiler/structuring/structurer_base.py @@ -520,14 +520,9 @@ class StructurerBase(Analysis): walker.walk(loop_node) def _rewrite_jumps_to_continues(self, loop_seq: SequenceNode, loop_node: LoopNode | None = None): - continue_node_addr = loop_seq.addr - # exception: do-while with a multi-statement condition - if ( - loop_node is not None - and loop_node.sort == "do-while" - and isinstance(loop_node.condition, ailment.Expr.MultiStatementExpression) - ): - continue_node_addr = loop_node.condition.tags["ins_addr"] + # LoopNode.continue_addr is where a continue lands: the loop header, or -- for a do-while whose latch was + # folded into the loop condition -- the latch block that evaluates the condition. + continue_node_addr = loop_node.continue_addr if loop_node is not None else loop_seq.addr def _rewrite_jump_to_continue(node, *, parent, index: int, label=None, **kwargs): if not node.statements: diff --git a/tests/analyses/decompiler/test_dowhile_latch_continue.py b/tests/analyses/decompiler/test_dowhile_latch_continue.py new file mode 100644 index 000000000..5a877cedf --- /dev/null +++ b/tests/analyses/decompiler/test_dowhile_latch_continue.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use,no-member +from __future__ import annotations + +__package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin + +import os +import unittest + +import archinfo + +from angr.ailment import Manager +from angr.ailment.block import Block +from angr.ailment.expression import BinaryOp, Const, Register +from angr.ailment.statement import ConditionalJump, Jump +from angr.analyses.decompiler.condition_processor import ConditionProcessor +from angr.analyses.decompiler.structurer_nodes import ContinueNode, LoopNode, SequenceNode +from angr.analyses.decompiler.structuring.phoenix import PhoenixStructurer +from tests.common import bin_location, load_project_with_scoped_cfg, print_decompilation_result + +test_location = os.path.join(bin_location, "tests") + +LATCH_ADDR = 0x200 + + +def _loop_body_with(stmt, arch=None): + """A do-while whose latch at LATCH_ADDR was folded into its condition, with stmt somewhere in its body. + + The statement is kept off the body's tail on purpose: a continue at the very end of a do-while body is redundant + and gets dropped again, which would hide what we are testing. + """ + m = Manager(arch=arch) + block = Block(0x110, 1, statements=[stmt]) + inner = SequenceNode(0x110, nodes=[block]) + loop_seq = SequenceNode(0x100, nodes=[inner, Block(0x180, 1, statements=[])]) + loop_node = LoopNode("do-while", Const(m.next_atom(), 1, 8), loop_seq, addr=0x100, continue_addr=LATCH_ADDR) + return block, inner, loop_seq, loop_node + + +def _continue_nodes(node, out=None): + if out is None: + out = [] + if isinstance(node, ContinueNode): + out.append(node) + for attr in ("nodes", "node", "sequence_node", "true_node", "false_node"): + child = getattr(node, attr, None) + if isinstance(child, list): + for c in child: + _continue_nodes(c, out) + elif child is not None: + _continue_nodes(child, out) + return out + + +def _loop_nodes(node, out=None): + """Collect every LoopNode in a structured tree.""" + if out is None: + out = [] + if isinstance(node, LoopNode): + out.append(node) + for attr in ("nodes", "node", "sequence_node", "true_node", "false_node", "else_node", "default_node", "head"): + child = getattr(node, attr, None) + if isinstance(child, list): + for c in child: + _loop_nodes(c, out) + elif child is not None: + _loop_nodes(child, out) + cases = getattr(node, "cases", None) + if isinstance(cases, dict): + for c in cases.values(): + _loop_nodes(c, out) + elif isinstance(cases, list): + for c in cases: + _loop_nodes(c, out) + for _, c in getattr(node, "condition_and_nodes", None) or (): + _loop_nodes(c, out) + return out + + +class TestDoWhileLatchContinue(unittest.TestCase): + def test_jump_to_folded_latch_becomes_continue(self): + """ + A do-while's latch stops being a node of its own once Phoenix folds its condition into the loop condition, so + jumps to it have to become continues. Anything left behind is emitted as a goto to a label that no longer + exists anywhere in the function. + """ + m = Manager(arch=None) + jump = Jump(m.next_atom(), Const(m.next_atom(), LATCH_ADDR, 64), ins_addr=0x110) + block, inner, loop_seq, loop_node = _loop_body_with(jump) + + # this path of _rewrite_jumps_to_continues() reads no instance state, so a bare structurer can drive it + structurer = object.__new__(PhoenixStructurer) + structurer._rewrite_jumps_to_continues(loop_seq, loop_node=loop_node) + + continues = _continue_nodes(inner) + assert len(continues) == 1 + assert continues[0].target == LATCH_ADDR + # the jump must be consumed, not left behind next to the continue + assert not block.statements + + def test_conditional_jump_to_folded_latch_becomes_continue(self): + """ + The shape this actually broke on: one branch of a conditional jump goes to the folded latch. It has to split + into a condition guarding the other target plus a continue. + """ + arch = archinfo.arch_from_id("amd64") + m = Manager(arch=arch) + condition = BinaryOp( + m.next_atom(), "CmpEQ", [Register(m.next_atom(), 16, 64), Const(m.next_atom(), 0, 64)], False + ) + condjump = ConditionalJump( + m.next_atom(), + condition, + Const(m.next_atom(), LATCH_ADDR, 64), + Const(m.next_atom(), 0x300, 64), + ins_addr=0x110, + ) + block, inner, loop_seq, loop_node = _loop_body_with(condjump, arch=arch) + + structurer = object.__new__(PhoenixStructurer) + structurer.cond_proc = ConditionProcessor(arch, m) + structurer._rewrite_jumps_to_continues(loop_seq, loop_node=loop_node) + + continues = _continue_nodes(inner) + assert len(continues) == 1 + assert continues[0].target == LATCH_ADDR + assert not block.statements + + def test_folded_latch_address_is_recorded(self): + """ + The other half: Phoenix has to record the latch address on the loop node in the first place. sub_4012eb holds a + do-while at 0x40165c whose latch at 0x4016bb is folded into the loop condition. + """ + bin_path = os.path.join(test_location, "x86_64", "1after909") + proj, cfg = load_project_with_scoped_cfg(bin_path, 0x4012EB, expand_call_tree=False, run_ccc=False) + dec = proj.analyses.Decompiler(0x4012EB, cfg=cfg.model, fail_fast=True) + assert dec.codegen is not None and dec.codegen.text is not None + print_decompilation_result(dec) + + loops = {(loop.addr, loop.continue_addr) for loop in _loop_nodes(dec.seq_node) if loop.sort == "do-while"} + assert (0x40165C, 0x4016BB) in loops + + +if __name__ == "__main__": + unittest.main() From b0e3541dfd1ddbe17684f88dba441faccf245a70 Mon Sep 17 00:00:00 2001 From: Fish Date: Fri, 7 Aug 2026 19:13:10 -0700 Subject: [PATCH 108/122] SwitchClusterSimplifier: Do not merge different default nodes. (#6782) --- .../switch_cluster_simplifier.py | 8 ++ .../test_switch_cluster_simplifier.py | 99 +++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 tests/analyses/decompiler/test_switch_cluster_simplifier.py diff --git a/angr/analyses/decompiler/region_simplifiers/switch_cluster_simplifier.py b/angr/analyses/decompiler/region_simplifiers/switch_cluster_simplifier.py index e1657d729..cf34e991b 100644 --- a/angr/analyses/decompiler/region_simplifiers/switch_cluster_simplifier.py +++ b/angr/analyses/decompiler/region_simplifiers/switch_cluster_simplifier.py @@ -328,6 +328,14 @@ def simplify_switch_clusters( if len(default_node_addrs) > 1: continue + # the merged switch-case only keeps one default node. sharing an address is not enough: default-case nodes get + # duplicated, and structuring usually leaves the real continuation under one copy and a goto stub under the + # others. Dropping the wrong copy silently loses everything below it, so only merge when all switches literally + # share the same default node. + default_nodes = [r.node.default_node for r in switch_regions if r.node.default_node is not None] + if any(dn is not default_nodes[0] for dn in default_nodes[1:]): + continue + # ensure cases in each switch do not overlap case_ids = set() overlaps = False diff --git a/tests/analyses/decompiler/test_switch_cluster_simplifier.py b/tests/analyses/decompiler/test_switch_cluster_simplifier.py new file mode 100644 index 000000000..2387e1ca9 --- /dev/null +++ b/tests/analyses/decompiler/test_switch_cluster_simplifier.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use,no-member +from __future__ import annotations + +__package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin + +import os +import re +import unittest +from collections import OrderedDict + +from angr.ailment import Manager +from angr.ailment.block import Block +from angr.ailment.expression import Const +from angr.ailment.statement import Jump +from angr.analyses.decompiler.region_simplifiers.switch_cluster_simplifier import ( + CmpOp, + ConditionalRegion, + SwitchCaseRegion, + simplify_switch_clusters, +) +from angr.analyses.decompiler.structurer_nodes import ConditionNode, SequenceNode, SwitchCaseNode +from angr.analyses.decompiler.utils import sequence_to_blocks +from tests.common import bin_location, load_project_with_scoped_cfg, print_decompilation_result + +test_location = os.path.join(bin_location, "tests") + +# codegen falls back to this form when a goto target has no label in the output +_DANGLING_GOTO = re.compile(r"goto (LABEL_0x\w+);") + + +class TestSwitchClusterSimplifier(unittest.TestCase): + def test_merge_keeps_every_default_node_block(self): + """ + Two switches on the same variable, each guarded by a condition, whose default-case nodes are separate copies + sharing an address -- what SwitchDefaultCaseDuplicator plus structuring produce. The merged switch-case can + only carry one default node, so merging here would drop the blocks under the other copy. + """ + m = Manager(arch=None) + + def block(addr, target): + return Block(addr, 1, statements=[Jump(m.next_atom(), Const(m.next_atom(), target, 64))]) + + def switch(addr, case_ids, default_node): + cases = OrderedDict( + (case_id, SequenceNode(addr + 0x10 + i, nodes=[block(addr + 0x10 + i, 0x1000)])) + for i, case_id in enumerate(case_ids) + ) + return SwitchCaseNode(Const(m.next_atom(), 0, 32), cases, default_node, addr=addr) + + # both default nodes live at 0x500, but only one of them holds the code below the switch + stub_default = SequenceNode(0x500, nodes=[block(0x500, 0x1000)]) + real_default = SequenceNode(0x500, nodes=[block(0x500, 0x600), block(0x600, 0x1000)]) + + switch0, switch1 = switch(0x100, [0, 1], stub_default), switch(0x200, [2, 3], real_default) + cond0 = ConditionNode(0xF0, None, Const(m.next_atom(), 1, 8), switch0) + cond1 = ConditionNode(0xF8, None, Const(m.next_atom(), 1, 8), switch1) + region = SequenceNode(0xF0, nodes=[cond0, cond1]) + + var = "switch_variable" + cond_regions = [ + ConditionalRegion(var, CmpOp.LT, 2, cond0, region), + ConditionalRegion(var, CmpOp.GT, 1, cond1, region), + ] + switch_regions = [SwitchCaseRegion(var, switch0, cond0), SwitchCaseRegion(var, switch1, cond1)] + simplify_switch_clusters(region, {var: cond_regions}, {var: switch_regions}) + + # 0x600 only exists under real_default; losing it means the merge threw away a default node + assert 0x600 in {bb.addr for bb in sequence_to_blocks(region)} + + def test_bbbq_switch_cluster_with_duplicated_default_nodes(self): + """ + sub_410920 holds two switches on the same variable whose default-case nodes were duplicated: one copy heads + the structured continuation of the function, the other is a two-block goto stub. Merging the switches keeps a + single default node, so picking the stub used to drop everything below the other copy -- 247 of 332 blocks, + leaving gotos to labels that appeared nowhere in the output. + """ + bin_path = os.path.join(test_location, "x86_64", "bbbq") + proj, cfg = load_project_with_scoped_cfg(bin_path, 0x410920, expand_call_tree=False, run_ccc=False) + dec = proj.analyses.Decompiler(0x410920, cfg=cfg.model, fail_fast=True) + assert dec.codegen is not None and dec.codegen.text is not None + print_decompilation_result(dec) + text = dec.codegen.text + + # every goto must reach a label that exists + assert not _DANGLING_GOTO.findall(text) + + # the bulk of the function must be there: the switch cases, the loop below the default node, and the tail + assert text.count("switch (") >= 1 + assert len(text.splitlines()) > 1000 + + # piggybacking: a lock cmpxchg in this function is followed by the ITE that writes the memory value back into + # the accumulator. CASIntrinsics used to skip that shape, and the surviving CAS statement aborted codegen. + assert "atomic_compare_exchange" in text + assert "CAS(" not in text + + +if __name__ == "__main__": + unittest.main() From fdd66945ea0be6f89124c6874c13bb1d0273f808 Mon Sep 17 00:00:00 2001 From: Fish Date: Fri, 7 Aug 2026 23:46:34 -0700 Subject: [PATCH 109/122] CFGTransformationMixin: Rebuild conditional jumps when replacing branch targets. (#6786) --- angr/rust/mixins/cfg_transformation_mixin.py | 24 ++- .../test_rust_cfg_transformation.py | 137 ++++++++++++++++++ 2 files changed, 157 insertions(+), 4 deletions(-) create mode 100644 tests/analyses/decompiler/test_rust_cfg_transformation.py diff --git a/angr/rust/mixins/cfg_transformation_mixin.py b/angr/rust/mixins/cfg_transformation_mixin.py index 61d929517..010a4ee32 100644 --- a/angr/rust/mixins/cfg_transformation_mixin.py +++ b/angr/rust/mixins/cfg_transformation_mixin.py @@ -163,15 +163,31 @@ class CFGTransformationMixin: and last_stmt.true_target.value_int == old_target and last_stmt.true_target_idx == old_target_idx ): - last_stmt.true_target.value = new_target - last_stmt.true_target_idx = new_target_idx + old_tgt = last_stmt.true_target + block.statements[-1] = ConditionalJump( + last_stmt.idx, + last_stmt.condition, + Const(old_tgt.idx, new_target, old_tgt.bits, **old_tgt.tags), + last_stmt.false_target, + true_target_idx=new_target_idx, + false_target_idx=last_stmt.false_target_idx, + **last_stmt.tags, + ) elif ( isinstance(last_stmt.false_target, Const) and last_stmt.false_target.value_int == old_target and last_stmt.false_target_idx == old_target_idx ): - last_stmt.false_target.value = new_target - last_stmt.false_target_idx = new_target_idx + old_tgt = last_stmt.false_target + block.statements[-1] = ConditionalJump( + last_stmt.idx, + last_stmt.condition, + last_stmt.true_target, + Const(old_tgt.idx, new_target, old_tgt.bits, **old_tgt.tags), + true_target_idx=last_stmt.true_target_idx, + false_target_idx=new_target_idx, + **last_stmt.tags, + ) if old_target: try: diff --git a/tests/analyses/decompiler/test_rust_cfg_transformation.py b/tests/analyses/decompiler/test_rust_cfg_transformation.py new file mode 100644 index 000000000..51201a52b --- /dev/null +++ b/tests/analyses/decompiler/test_rust_cfg_transformation.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use,no-member +from __future__ import annotations + +__package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin + +import os +import re +import unittest + +import networkx + +import angr +from angr.ailment import Manager +from angr.ailment.block import Block +from angr.ailment.expression import BinaryOp, Const, Register +from angr.ailment.statement import ConditionalJump, Jump +from angr.rust.mixins.cfg_transformation_mixin import CFGTransformationMixin +from tests.common import bin_location, print_decompilation_result + +test_location = os.path.join(bin_location, "tests") + + +def _terminator_targets(block): + stmt = block.statements[-1] + if isinstance(stmt, Jump): + return {stmt.target.value_int} + return {t.value_int for t in (stmt.true_target, stmt.false_target) if isinstance(t, Const)} + + +def _dangling_terminators(graph): + """Terminators jumping to an address that is not a block of the graph.""" + addrs = {b.addr for b in graph.nodes} + out = [] + for b in graph.nodes: + if not b.statements or not isinstance(b.statements[-1], (Jump, ConditionalJump)): + continue + out += [(b.addr, b.idx, t) for t in _terminator_targets(b) - addrs] + return out + + +class TestRustCFGTransformation(unittest.TestCase): + """ + replace_jump_target() has to rewrite the terminator as well as the graph edge. AIL expressions are immutable, so + the branch that mutated stmt.true_target.value in place changed nothing, and the block was left jumping to a block + that the caller went on to delete. + """ + + @staticmethod + def _graph_with_condjump(): + m = Manager(arch=None) + cond = BinaryOp(m.next_atom(), "CmpEQ", [Register(m.next_atom(), 16, 64), Const(m.next_atom(), 0, 64)], False) + head = Block( + 0x100, + 1, + statements=[ + ConditionalJump( + m.next_atom(), + cond, + Const(m.next_atom(), 0x200, 64), + Const(m.next_atom(), 0x300, 64), + ins_addr=0x100, + ) + ], + ) + doomed = Block(0x200, 1, statements=[Jump(m.next_atom(), Const(m.next_atom(), 0x400, 64), ins_addr=0x200)]) + other = Block(0x300, 1, statements=[]) + successor = Block(0x400, 1, statements=[]) + graph = networkx.DiGraph() + graph.add_edge(head, doomed) + graph.add_edge(head, other) + graph.add_edge(doomed, successor) + return head, doomed, other, successor, graph + + def test_true_branch_target_is_rewritten(self): + head, doomed, _, successor, graph = self._graph_with_condjump() + transformer = CFGTransformationMixin(graph) + + transformer.replace_jump_target(head, 0x200, None, 0x400, None) + + assert _terminator_targets(head) == {0x400, 0x300} + assert graph.has_edge(head, successor) + assert not graph.has_edge(head, doomed) + + def test_false_branch_target_is_rewritten(self): + head, _, other, successor, graph = self._graph_with_condjump() + transformer = CFGTransformationMixin(graph) + + transformer.replace_jump_target(head, 0x300, None, 0x400, None) + + assert _terminator_targets(head) == {0x200, 0x400} + assert graph.has_edge(head, successor) + assert not graph.has_edge(head, other) + + def test_condjump_collapses_when_both_branches_converge(self): + # the other pre-existing path: replacing one target with the other one turns the branch into a plain jump + head, _, _, _, graph = self._graph_with_condjump() + transformer = CFGTransformationMixin(graph) + + transformer.replace_jump_target(head, 0x200, None, 0x300, None) + + assert isinstance(head.statements[-1], Jump) + assert _terminator_targets(head) == {0x300} + + def test_removing_a_block_leaves_no_dangling_terminator(self): + _, doomed, _, _, graph = self._graph_with_condjump() + transformer = CFGTransformationMixin(graph) + + assert transformer.remove_block(doomed) + + assert doomed not in graph + assert not _dangling_terminators(graph) + + def test_bbbq_rust_flavor_graph_has_no_dangling_terminators(self): + """ + The whole-binary CFG is what shows this: with a scoped CFG a handful of dangling terminators survive from + another source, so the invariant cannot be asserted outright. + """ + bin_path = os.path.join(test_location, "x86_64", "bbbq") + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(normalize=True, data_references=True, show_progressbar=False) + proj.analyses.CompleteCallingConventions() + proj.analyses.RustSymbolRecovery() + proj.analyses.TypeDBLoader() + dec = proj.analyses.Decompiler(0x410920, cfg=cfg.model, flavor="rust", fail_fast=True) + assert dec.codegen is not None and dec.codegen.text is not None + print_decompilation_result(dec) + + assert not _dangling_terminators(dec.ail_graph) + # and nothing in the output jumps to a label that was never emitted + text = dec.codegen.text + labels = set(re.findall(r"^\s*(LABEL_\w+):", text, re.MULTILINE)) + assert not set(re.findall(r"goto (LABEL_\w+);", text)) - labels + + +if __name__ == "__main__": + unittest.main() From 666fadabc5f12b735691f787e87601b799cdc242 Mon Sep 17 00:00:00 2001 From: Fish Date: Fri, 7 Aug 2026 23:50:03 -0700 Subject: [PATCH 110/122] Dephication: Keep the remapped assignment destination with source is not a vvar. (#6787) --- .../dephication/rewriting_engine.py | 1 + .../decompiler/test_dephication_rewriting.py | 115 ++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 tests/analyses/decompiler/test_dephication_rewriting.py diff --git a/angr/analyses/decompiler/dephication/rewriting_engine.py b/angr/analyses/decompiler/dephication/rewriting_engine.py index bab0bb903..0e27eb9b7 100644 --- a/angr/analyses/decompiler/dephication/rewriting_engine.py +++ b/angr/analyses/decompiler/dephication/rewriting_engine.py @@ -136,6 +136,7 @@ class SimEngineDephiRewriting(SimEngineNostmtAIL[None, Expression | None, Statem # skip it return () + if new_dst is not None or new_src is not None: return Assignment(stmt.idx, dst, src, **stmt.tags) return None diff --git a/tests/analyses/decompiler/test_dephication_rewriting.py b/tests/analyses/decompiler/test_dephication_rewriting.py new file mode 100644 index 000000000..1dc0b9142 --- /dev/null +++ b/tests/analyses/decompiler/test_dephication_rewriting.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use,no-member +from __future__ import annotations + +__package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin + +import os +import unittest + +import angr +from angr.ailment import Manager +from angr.ailment.expression import Const, UnaryOp, VirtualVariable, VirtualVariableCategory +from angr.ailment.statement import Assignment +from angr.analyses.decompiler.dephication.rewriting_engine import SimEngineDephiRewriting +from tests.common import bin_location, load_project_with_scoped_cfg, print_decompilation_result + +test_location = os.path.join(bin_location, "tests") + + +class TestDephicationRewriting(unittest.TestCase): + """ + Dephication remaps every vvar feeding a phi onto the phi's destination and then drops the phi. If a remapped + destination is not written back, the phi destination ends up with no definition and the phi sources with no uses, + which passes that reason about vvar uses read as dead code. + """ + + @staticmethod + def _engine(mapping): + # a project is only needed for the arch; no CFG or analyses required + proj = angr.Project(os.path.join(test_location, "x86_64", "fauxware"), auto_load_libs=False) + return proj, SimEngineDephiRewriting(proj, mapping) + + def test_remapped_dst_survives_non_vvar_src(self): + proj, engine = self._engine({2759: 1330}) + m = Manager(arch=proj.arch) + stmt = Assignment( + m.next_atom(), + VirtualVariable(m.next_atom(), 2759, 64, VirtualVariableCategory.REGISTER), + # a source that is not a bare vvar; this is what used to discard the remapped destination + UnaryOp( + m.next_atom(), + "Reference", + VirtualVariable(m.next_atom(), 4575, 128, VirtualVariableCategory.STACK), + bits=64, + ), + ins_addr=0x400100, + ) + + out = engine._handle_stmt_Assignment(stmt) + assert isinstance(out, Assignment) + assert out.dst.varid == 1330 + + def test_remapped_dst_survives_const_src(self): + proj, engine = self._engine({7: 3}) + m = Manager(arch=proj.arch) + stmt = Assignment( + m.next_atom(), + VirtualVariable(m.next_atom(), 7, 64, VirtualVariableCategory.REGISTER), + Const(m.next_atom(), 0x1234, 64), + ins_addr=0x400100, + ) + + out = engine._handle_stmt_Assignment(stmt) + assert isinstance(out, Assignment) + assert out.dst.varid == 3 + + def test_self_assignment_is_still_dropped(self): + # the reason the return used to sit behind the both-sides-are-vvars guard: once both sides map onto the same + # variable the statement is a no-op and has to go away + proj, engine = self._engine({11: 3, 12: 3}) + m = Manager(arch=proj.arch) + stmt = Assignment( + m.next_atom(), + VirtualVariable(m.next_atom(), 11, 64, VirtualVariableCategory.REGISTER), + VirtualVariable(m.next_atom(), 12, 64, VirtualVariableCategory.REGISTER), + ins_addr=0x400100, + ) + + assert engine._handle_stmt_Assignment(stmt) == () + + def test_unmapped_assignment_is_left_alone(self): + proj, engine = self._engine({2759: 1330}) + m = Manager(arch=proj.arch) + stmt = Assignment( + m.next_atom(), + VirtualVariable(m.next_atom(), 99, 64, VirtualVariableCategory.REGISTER), + Const(m.next_atom(), 1, 64), + ins_addr=0x400100, + ) + + # None means "unchanged" to the caller, which then keeps the original statement + assert engine._handle_stmt_Assignment(stmt) is None + + def test_bbbq_rust_flavor_keeps_string_constants(self): + """ + sub_410920 sets up a &str in a block whose only outward effect flows through a phi. With the remapped + destination dropped, RedundantBlockRemover deleted the block and the string with it. + """ + bin_path = os.path.join(test_location, "x86_64", "bbbq") + # the window has to reach past the function: with a tighter scope the callee prototypes differ enough that the + # block holding this string survives even with the bug, and the test stops testing anything + proj, cfg = load_project_with_scoped_cfg( + bin_path, 0x410920, window=0x4000, expand_call_tree=False, run_ccc=False + ) + proj.analyses.RustSymbolRecovery() + proj.analyses.TypeDBLoader() + dec = proj.analyses.Decompiler(0x410920, cfg=cfg.model, flavor="rust", fail_fast=True) + assert dec.codegen is not None and dec.codegen.text is not None + print_decompilation_result(dec) + + assert "seed_hex is not valid hex" in dec.codegen.text + + +if __name__ == "__main__": + unittest.main() From bdc13218454d78956ed1fec69bf433b6bcb01ee1 Mon Sep 17 00:00:00 2001 From: Fish Date: Sat, 8 Aug 2026 09:28:22 -0700 Subject: [PATCH 111/122] RustCodeGen: Add more handlers. (#6788) --- .../decompiler/structured_codegen/rust.py | 131 +++++++++++++ .../decompiler/test_rust_codegen_handlers.py | 182 ++++++++++++++++++ 2 files changed, 313 insertions(+) create mode 100644 tests/analyses/decompiler/test_rust_codegen_handlers.py diff --git a/angr/analyses/decompiler/structured_codegen/rust.py b/angr/analyses/decompiler/structured_codegen/rust.py index d49856721..fab4c5b2c 100644 --- a/angr/analyses/decompiler/structured_codegen/rust.py +++ b/angr/analyses/decompiler/structured_codegen/rust.py @@ -24,6 +24,7 @@ from angr.ailment.expression import ( RustEnum as AilRustEnum, ) from angr.analyses.analysis import Analysis, register_analysis +from angr.analyses.decompiler.peephole_optimizations.cas_intrinsics import cas_intrinsic_name from angr.analyses.decompiler.region_identifier import MultiNode from angr.analyses.decompiler.structurer_nodes import ( BreakNode, @@ -32,6 +33,8 @@ from angr.analyses.decompiler.structurer_nodes import ( ConditionalBreakNode, ConditionNode, ContinueNode, + IncompleteSwitchCaseHeadStatement, + IncompleteSwitchCaseNode, LoopNode, SequenceNode, SwitchCaseNode, @@ -1606,6 +1609,69 @@ class RustUnsupportedStatement(RustStatement): yield "\n", None +class RustIncompleteSwitchCase(RustStatement): + """ + An incomplete switch-case construct; only appears in the output when switch-case structuring failed. + """ + + __slots__ = ("cases", "head", "tags") + + def __init__(self, head, cases, tags=None, **kwargs): + super().__init__(**kwargs) + + self.head = head + self.cases: list[tuple[int, RustStatements]] = cases + self.tags = tags + + def c_repr_chunks(self, indent=0, asexpr=False): + indent_str = self.indent_str(indent=indent) + paren = RustClosingObject("(") + brace = RustClosingObject("{") + + yield from self.head.c_repr_chunks(indent=indent) + yield "\n", None + yield indent_str, None + yield "match ", self + yield "(", paren + yield "/* incomplete */", None + yield ")", paren + if self.codegen.braces_on_own_lines: + yield "\n", None + yield indent_str, None + else: + yield " ", None + yield "{", brace + yield "\n", None + + for case_addr, case in self.cases: + yield indent_str, None + yield f"{case_addr:#x} =>", self + yield "\n", None + yield from case.c_repr_chunks(indent=indent + self.codegen.indent_delta) + + yield indent_str, None + yield "}", brace + yield "\n", None + + +class RustDirtyStatement(RustStatement): + """ + A dirty expression used as a statement, i.e. only for its side effects. + """ + + __slots__ = ("dirty",) + + def __init__(self, dirty, **kwargs): + super().__init__(**kwargs) + self.dirty = dirty + + def c_repr_chunks(self, indent=0, asexpr=False): + yield self.indent_str(indent=indent), None + yield from RustExpression._try_c_repr_chunks(self.dirty) + yield ";", None + yield "\n", None + + class RustLabel(RustStatement): """ Represents a label in C code. @@ -2775,6 +2841,7 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): Block: self._handle_AILBlock, BreakNode: self._handle_Break, SwitchCaseNode: self._handle_SwitchCase, + IncompleteSwitchCaseNode: self._handle_IncompleteSwitchCase, ContinueNode: self._handle_Continue, PatternMatchNode: self._handle_PatternMatch, IfLetNode: self._handle_IfLet, @@ -2786,6 +2853,10 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): Stmt.ConditionalJump: self._handle_Stmt_ConditionalJump, Stmt.Return: self._handle_Stmt_Return, Stmt.Label: self._handle_Stmt_Label, + Stmt.WeakAssignment: self._handle_Stmt_Assignment, + Stmt.DirtyStatement: self._handle_Stmt_Dirty, + Stmt.CAS: self._handle_Stmt_CAS, + IncompleteSwitchCaseHeadStatement: self._handle_Stmt_IncompleteSwitchCaseHead, # AIL expressions Expr.Register: self._handle_Expr_Register, Expr.Load: self._handle_Expr_Load, @@ -2799,6 +2870,7 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): Expr.DirtyExpression: self._handle_Expr_Dirty, Expr.ITE: self._handle_Expr_ITE, Expr.Extract: self._handle_Expr_Extract, + Expr.Insert: self._handle_Expr_Insert, Expr.Reinterpret: self._handle_Reinterpret, Expr.MultiStatementExpression: self._handle_MultiStatementExpression, Expr.VirtualVariable: self._handle_VirtualVariable, @@ -3595,6 +3667,11 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): tags = {"ins_addr": node.addr} return RustSwitchCase(switch_expr, cases, default=default, tags=tags, codegen=self) + def _handle_IncompleteSwitchCase(self, node: IncompleteSwitchCaseNode, **kwargs): + head = self._handle(node.head, is_expr=False) + cases = [(case.addr, self._handle(case, is_expr=False)) for case in node.cases] + return RustIncompleteSwitchCase(head, cases, tags={"ins_addr": node.addr}, codegen=self) + def _get_bound_variable(self, move): expr = None var = None @@ -3808,6 +3885,49 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): self.map_addr_to_label[(ins_addr, block_idx)] = clabel return clabel + def _handle_Stmt_Dirty(self, stmt: Stmt.DirtyStatement, **kwargs): + return RustDirtyStatement(self._handle(stmt.dirty), codegen=self) + + def _handle_Stmt_IncompleteSwitchCaseHead(self, stmt: IncompleteSwitchCaseHeadStatement, **kwargs): + # render the dispatch as a cascade of if-gotos rather than dropping every target + switch_var = self._handle(stmt.switch_variable) + bits = getattr(stmt.switch_variable, "bits", None) or self.project.arch.bits + const_type = RustSimTypeInt(size=bits, signed=False).with_arch(self.project.arch) + condition_and_nodes = [] + default_goto = None + for _, case_value, target_addr, target_idx, _ in stmt.case_addrs: + goto = RustGoto(target_addr, target_idx, tags=stmt.tags, codegen=self) + if isinstance(case_value, str): + if case_value == "default": + default_goto = goto + continue + cond = RustBinaryOp( + "CmpEQ", + switch_var, + RustConstant(case_value, const_type, tags=stmt.tags, codegen=self), + codegen=self, + tags=stmt.tags, + ) + condition_and_nodes.append((cond, goto)) + if not condition_and_nodes: + return default_goto if default_goto is not None else RustUnsupportedStatement(stmt, codegen=self) + return RustIfElse(condition_and_nodes, else_node=default_goto, tags=stmt.tags, codegen=self) + + def _handle_Stmt_CAS(self, stmt: Stmt.CAS, **kwargs): + # render whatever CASIntrinsics did not rewrite as the same intrinsic call + if stmt.old_hi is None: + os_name = self.project.simos.name if self.project.simos is not None else None + call = Expr.Call( + stmt.idx, + cas_intrinsic_name(f"cmpxchg{stmt.bits}", os_name), + args=[stmt.addr, stmt.data_lo, stmt.expd_lo], + bits=stmt.bits, + **stmt.tags, + ) + return self._handle(Stmt.Assignment(stmt.idx, stmt.old_lo, call, **stmt.tags), is_expr=False) + # a double-width CAS writes two destinations, which no single expression captures + return RustUnsupportedStatement(stmt, codegen=self) + # # AIL expression handlers # @@ -4193,6 +4313,17 @@ class RustStructuredCodeGenerator(BaseStructuredCodeGenerator, Analysis): def _handle_Expr_Dirty(self, expr, **kwargs): return RustDirtyExpression(expr, codegen=self) + def _handle_Expr_Insert(self, expr: Expr.Insert, **kwargs): + # should never really be used - should be handled by Assignment + return RustFunctionCall( + "_INSERT", + None, + [self._handle(expr.base), self._handle(expr.offset), self._handle(expr.value)], + is_expr=True, + tags=expr.tags, + codegen=self, + ) + def _handle_Expr_ITE(self, expr: Expr.ITE, **kwargs): return RustITE( self._handle(expr.cond), self._handle(expr.iftrue), self._handle(expr.iffalse), tags=expr.tags, codegen=self diff --git a/tests/analyses/decompiler/test_rust_codegen_handlers.py b/tests/analyses/decompiler/test_rust_codegen_handlers.py new file mode 100644 index 000000000..33dbd32a4 --- /dev/null +++ b/tests/analyses/decompiler/test_rust_codegen_handlers.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use,no-member +from __future__ import annotations + +__package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin + +import os +import unittest + +import angr +from angr.ailment import Manager +from angr.ailment.block import Block +from angr.ailment.expression import ( + Const, + DirtyExpression, + Insert, + VirtualVariable, + VirtualVariableCategory, +) +from angr.ailment.statement import CAS, DirtyStatement, Jump, WeakAssignment +from angr.analyses.decompiler.structurer_nodes import ( + IncompleteSwitchCaseHeadStatement, + IncompleteSwitchCaseNode, + SequenceNode, +) +from tests.common import bin_location, load_project_with_scoped_cfg, print_decompilation_result + +test_location = os.path.join(bin_location, "tests") + +# what the Rust backend emits for a node it has no handler for +PLACEHOLDER = "unsupported instruction" + + +def _render(node): + return "".join(chunk for chunk, _ in node.c_repr_chunks()) + + +class TestRustCodegenHandlers(unittest.TestCase): + """ + _handle_AILBlock() substitutes a placeholder for any statement the backend has no handler for, so a missing + handler silently drops whatever the statement contained. + """ + + @classmethod + def setUpClass(cls): + # any binary will do: we only need a constructed Rust code generator to drive handlers with + proj = angr.Project(os.path.join(test_location, "x86_64", "fauxware"), auto_load_libs=False) + cfg = proj.analyses.CFGFast(normalize=True, show_progressbar=False) + dec = proj.analyses.Decompiler(proj.kb.functions["main"], cfg=cfg.model, flavor="rust", fail_fast=True) + assert dec.codegen is not None + cls.proj = proj + cls.codegen = dec.codegen + + def _manager(self): + return Manager(arch=self.proj.arch) + + @staticmethod + def _vvar(m, varid, bits=64, category=VirtualVariableCategory.REGISTER): + return VirtualVariable(m.next_atom(), varid, bits, category) + + def test_handler_table_covers_everything_the_c_backend_covers(self): + """A missing handler degrades silently, so guard the whole class rather than one node type at a time.""" + proj = self.proj + cfg = proj.kb.cfgs.get_most_accurate() + c_dec = proj.analyses.Decompiler(proj.kb.functions["main"], cfg=cfg, flavor="pseudocode", fail_fast=True) + assert c_dec.codegen is not None + + missing = set(c_dec.codegen._handlers) - set(self.codegen._handlers) + assert not missing, f"Rust backend has no handler for {sorted(str(k) for k in missing)}" + + def test_insert(self): + m = self._manager() + expr = Insert( + m.next_atom(), + self._vvar(m, 1), + Const(m.next_atom(), 0, 8), + self._vvar(m, 2, bits=32), + "Iend_LE", + ) + + out = _render(self.codegen._handle(expr)) + assert "_INSERT(" in out + assert PLACEHOLDER not in out + + def test_dirty_statement(self): + m = self._manager() + dirty = DirtyExpression(m.next_atom(), "amd64g_dirtyhelper_RDTSC", [], bits=64) + stmt = DirtyStatement(m.next_atom(), dirty, ins_addr=0x400100) + + out = _render(self.codegen._handle(stmt, is_expr=False)) + assert "amd64g_dirtyhelper_RDTSC" in out + assert PLACEHOLDER not in out + + def test_weak_assignment(self): + m = self._manager() + stmt = WeakAssignment(m.next_atom(), self._vvar(m, 1), self._vvar(m, 2), ins_addr=0x400100) + + out = _render(self.codegen._handle(stmt, is_expr=False)) + assert "=" in out + assert PLACEHOLDER not in out + + def test_cas(self): + m = self._manager() + stmt = CAS( + m.next_atom(), + Const(m.next_atom(), 0x1000, 64), + Const(m.next_atom(), 1, 32), + None, + Const(m.next_atom(), 0, 32), + None, + self._vvar(m, 2, bits=32), + None, + "Iend_LE", + ins_addr=0x400100, + ) + + out = _render(self.codegen._handle(stmt, is_expr=False)) + assert "atomic_compare_exchange" in out + assert PLACEHOLDER not in out + + def test_incomplete_switch_case_head_statement(self): + m = self._manager() + case_blocks = [Block(0x400200 + i, 1, statements=[]) for i in range(2)] + stmt = IncompleteSwitchCaseHeadStatement( + m.next_atom(), + self._vvar(m, 1), + [ + (case_blocks[0], 0, 0x400300, None, 0x400210), + (case_blocks[1], 1, 0x400400, None, 0x400220), + (None, "default", 0x400500, None, 0x400230), + ], + ins_addr=0x400100, + ) + + out = _render(self.codegen._handle(stmt, is_expr=False)) + # every case target has to survive, plus the default + assert "0x400300" in out + assert "0x400400" in out + assert "0x400500" in out + assert PLACEHOLDER not in out + + def test_incomplete_switch_case_node(self): + m = self._manager() + head = Block( + 0x400100, 1, statements=[Jump(m.next_atom(), Const(m.next_atom(), 0x400200, 64), ins_addr=0x400100)] + ) + cases = [ + SequenceNode( + 0x400200, + nodes=[ + Block( + 0x400200, + 1, + statements=[Jump(m.next_atom(), Const(m.next_atom(), 0x400300, 64), ins_addr=0x400200)], + ) + ], + ) + ] + node = IncompleteSwitchCaseNode(0x400100, head, cases) + + out = _render(self.codegen._handle(node, is_expr=False)) + assert "incomplete" in out + assert "0x400200" in out + assert PLACEHOLDER not in out + + def test_bbbq_rust_flavor_has_no_placeholders(self): + bin_path = os.path.join(test_location, "x86_64", "bbbq") + proj, cfg = load_project_with_scoped_cfg(bin_path, 0x410920, expand_call_tree=False, run_ccc=False) + proj.analyses.RustSymbolRecovery() + proj.analyses.TypeDBLoader() + dec = proj.analyses.Decompiler(0x410920, cfg=cfg.model, flavor="rust", fail_fast=True) + assert dec.codegen is not None and dec.codegen.text is not None + print_decompilation_result(dec) + + text = dec.codegen.text + assert PLACEHOLDER not in text + # the bit-insertions that used to be dropped are rendered now + assert "_INSERT(" in text + + +if __name__ == "__main__": + unittest.main() From ede5faf162ec847438196f8b281698da9959d221 Mon Sep 17 00:00:00 2001 From: Fish Date: Sun, 9 Aug 2026 01:48:07 -0700 Subject: [PATCH 112/122] Decompiler: Add edits layer; add more tools to MCP. (#6789) --- angr/analyses/decompiler/decompiler.py | 138 +++-- angr/analyses/decompiler/edits/__init__.py | 89 +++ angr/analyses/decompiler/edits/cache.py | 126 ++++ angr/analyses/decompiler/edits/errors.py | 56 ++ angr/analyses/decompiler/edits/hooks.py | 93 +++ angr/analyses/decompiler/edits/ops.py | 638 +++++++++++++++++++++ angr/analyses/decompiler/edits/resolve.py | 342 +++++++++++ angr/analyses/decompiler/edits/results.py | 46 ++ angr/mcp/__init__.py | 6 + angr/mcp/edit_tools.py | 546 ++++++++++++++++++ angr/mcp/errors.py | 15 +- angr/mcp/server.py | 147 +++-- angr/mcp/session.py | 46 +- tests/analyses/decompiler/test_edits.py | 376 ++++++++++++ tests/mcp/test_edit_tools.py | 217 +++++++ tests/mcp/test_session.py | 63 ++ 16 files changed, 2818 insertions(+), 126 deletions(-) create mode 100644 angr/analyses/decompiler/edits/__init__.py create mode 100644 angr/analyses/decompiler/edits/cache.py create mode 100644 angr/analyses/decompiler/edits/errors.py create mode 100644 angr/analyses/decompiler/edits/hooks.py create mode 100644 angr/analyses/decompiler/edits/ops.py create mode 100644 angr/analyses/decompiler/edits/resolve.py create mode 100644 angr/analyses/decompiler/edits/results.py create mode 100644 angr/mcp/edit_tools.py create mode 100644 tests/analyses/decompiler/test_edits.py create mode 100644 tests/mcp/test_edit_tools.py diff --git a/angr/analyses/decompiler/decompiler.py b/angr/analyses/decompiler/decompiler.py index 001814e4f..c5947b21f 100644 --- a/angr/analyses/decompiler/decompiler.py +++ b/angr/analyses/decompiler/decompiler.py @@ -19,7 +19,6 @@ from angr.errors import AngrAIError from angr.knowledge_plugins.functions.function import Function from angr.rust.optimization_passes import get_rust_optimization_passes from angr.rust.typehoon.typehoon import RustTypehoon -from angr.sim_type import parse_type from angr.sim_variable import SimMemoryVariable, SimRegisterVariable, SimStackVariable from angr.utils import timethis @@ -28,6 +27,15 @@ from .clinic import ClinicStage from .condition_processor import ConditionProcessor from .decompilation_cache import DecompilationCache from .decompilation_options import PARAM_TO_OPTION, DecompilationOption +from .edits import ( + DecompilationEditError, + list_variable_names, + reflow_types, + rename_function, + rename_variable, + resolve_variable, + set_variable_type, +) from .notes import DecompilationNote from .optimization_passes.optimization_pass import OptimizationPassStage from .presets import DECOMPILATION_PRESETS, DecompilationPreset @@ -1028,29 +1036,10 @@ class Decompiler(Analysis): if not code_text: return False - # collect unified variables - varman = self.kb.dec_variables[self.func.addr] - unified_vars = varman.get_unified_variables(sort=None) - - # also collect argument variables - arg_vars = [] - if ( - self.codegen - and isinstance(self.codegen, CStructuredCodeGenerator) - and self.codegen.cfunc - and self.codegen.cfunc.arg_list - ): - for cvar in self.codegen.cfunc.arg_list: - v = cvar.unified_variable if cvar.unified_variable is not None else cvar.variable - if v not in unified_vars: - arg_vars.append(v) - - all_vars = unified_vars + arg_vars - if not all_vars: + var_names = list_variable_names(self.codegen, self.kb, self.func.addr) + if not var_names: return False - var_names = [v.name or str(v) for v in all_vars] - prompt = ( "You are a reverse engineering assistant. Given the following decompiled C code, suggest better, " "more descriptive variable names. Only include variables that you want to rename. " @@ -1065,27 +1054,27 @@ class Decompiler(Analysis): if not result: return False - # build name-to-variable lookup - name_to_var = {} - for v in all_vars: - key = v.name or str(v) - name_to_var[key] = v - changed = False for rename in result.renames: - old_name = rename.old_name - new_name = rename.new_name - if not new_name: + old_name, new_name = rename.old_name, rename.new_name + if not new_name or old_name == new_name: continue - var = name_to_var.get(old_name) - if var is None: + try: + edit = rename_variable( + self.project, + self.func, + old_name, + new_name, + kb=self.kb, + flavor=self._flavor, + rerender=False, + ) + except DecompilationEditError as ex: + l.debug("LLM rename %s -> %s rejected: %s", old_name, new_name, ex) continue - if old_name == new_name: - continue - var.name = new_name - var.renamed = True - changed = True - l.info("LLM renamed variable %s -> %s", old_name, new_name) + if edit.changed: + changed = True + l.info("LLM renamed variable %s -> %s", old_name, new_name) return changed @@ -1129,13 +1118,15 @@ class Decompiler(Analysis): if not new_name or new_name == current_name: return False - l.info("LLM renamed function %s -> %s", current_name, new_name) - self.func.name = new_name - self.func.is_default_name = False - if self.codegen and isinstance(self.codegen, CStructuredCodeGenerator) and self.codegen.cfunc: - self.codegen.cfunc.name = new_name + try: + edit = rename_function(self.project, self.func, new_name, kb=self.kb, flavor=self._flavor, rerender=False) + except DecompilationEditError as ex: + l.debug("LLM function rename %s -> %s rejected: %s", current_name, new_name, ex) + return False - return True + if edit.changed: + l.info("LLM renamed function %s -> %s", current_name, new_name) + return edit.changed def llm_suggest_variable_types( self, llm_client=None, code_text: str | None = None, raise_exc: bool = False @@ -1159,18 +1150,19 @@ class Decompiler(Analysis): return False varman = self.kb.dec_variables[self.func.addr] - unified_vars = varman.get_unified_variables(sort=None) - if not unified_vars: - return False - - # build current type info var_type_info = {} - for v in unified_vars: - name = v.name or str(v) - current_type = varman.get_variable_type(v) + for name in list_variable_names(self.codegen, self.kb, self.func.addr): + try: + rv = resolve_variable(self.kb, self.func.addr, name, codegen=self.codegen, flavor=self._flavor) + except DecompilationEditError: + continue + current_type = varman.get_variable_type(rv.variable) var_type_info[name] = str(current_type) if current_type else "unknown" + if not var_type_info: + return False + prompt = ( "You are a reverse engineering assistant. Given the following decompiled C code and the current " "variable types, suggest better C types for the variables. Only include variables whose types " @@ -1185,33 +1177,33 @@ class Decompiler(Analysis): if not result: return False - # build name-to-variable lookup - name_to_var = {} - for v in unified_vars: - key = v.name or str(v) - name_to_var[key] = v - changed = False for type_change in result.type_changes: - var_name = type_change.variable_name - type_str = type_change.new_type + var_name, type_str = type_change.variable_name, type_change.new_type if not type_str: continue - var = name_to_var.get(var_name) - if var is None: - continue try: - new_type = parse_type(type_str, arch=self.project.arch) - except Exception: # pylint:disable=broad-exception-caught - l.debug("LLM suggested unparseable type '%s' for %s", type_str, var_name) + # reflow once at the end rather than per variable: Typehoon is expensive + edit = set_variable_type( + self.project, + self.func, + var_name, + type_str, + kb=self.kb, + flavor=self._flavor, + reflow=False, + ) + except DecompilationEditError as ex: + l.debug("LLM retype of %s to '%s' rejected: %s", var_name, type_str, ex) continue + if edit.changed: + changed = True + l.info("LLM changed type of %s to %s", var_name, type_str) - varman.set_variable_type(var, new_type, mark_manual=True, all_unified=True) - changed = True - l.info("LLM changed type of %s to %s", var_name, type_str) - - if changed and self.codegen: - self.codegen.reload_variable_types() + if changed: + new_codegen = reflow_types(self.project, self.func, kb=self.kb, flavor=self._flavor, rerender=False) + if new_codegen is not None: + self.codegen = new_codegen return changed diff --git a/angr/analyses/decompiler/edits/__init__.py b/angr/analyses/decompiler/edits/__init__.py new file mode 100644 index 000000000..8211e097a --- /dev/null +++ b/angr/analyses/decompiler/edits/__init__.py @@ -0,0 +1,89 @@ +""" +Knowledge-base edits for decompilation output: renaming, retyping, and commenting. + +This layer is deliberately free of any UI or transport dependency so both the headless MCP server +and angr-management drive the same code. It is *not* thread-safe: locking belongs to whoever owns +the knowledge base's lifetime. +""" + +from __future__ import annotations + +from .cache import ( + DEFAULT_FLAVOR, + get_cache, + invalidate, + require_cache, + restore_user_edits, + snapshot_user_edits, +) +from .errors import ( + AmbiguousFunctionError, + DecompilationEditError, + FunctionNotFoundError, + InvalidNameError, + NameCollisionError, + NotDecompiledError, + TypeParseError, + UnsupportedEditError, + VariableNotFoundError, +) +from .hooks import EditHooks, NullEditHooks +from .ops import ( + global_variable_at, + reflow_types, + rename_function, + rename_global, + rename_variable, + set_comment, + set_function_prototype, + set_global_type, + set_variable_type, +) +from .resolve import ( + ResolvedVariable, + concrete_variables, + list_variable_names, + parse_address, + resolve_function, + resolve_variable, + validate_name, +) +from .results import EditResult, Refresh + +__all__ = [ + "DEFAULT_FLAVOR", + "AmbiguousFunctionError", + "DecompilationEditError", + "EditHooks", + "EditResult", + "FunctionNotFoundError", + "InvalidNameError", + "NameCollisionError", + "NotDecompiledError", + "NullEditHooks", + "Refresh", + "ResolvedVariable", + "TypeParseError", + "UnsupportedEditError", + "VariableNotFoundError", + "concrete_variables", + "get_cache", + "global_variable_at", + "invalidate", + "list_variable_names", + "parse_address", + "reflow_types", + "rename_function", + "rename_global", + "rename_variable", + "require_cache", + "resolve_function", + "resolve_variable", + "restore_user_edits", + "set_comment", + "set_function_prototype", + "set_global_type", + "set_variable_type", + "snapshot_user_edits", + "validate_name", +] diff --git a/angr/analyses/decompiler/edits/cache.py b/angr/analyses/decompiler/edits/cache.py new file mode 100644 index 000000000..c5a2b645c --- /dev/null +++ b/angr/analyses/decompiler/edits/cache.py @@ -0,0 +1,126 @@ +""" +Decompilation-cache access for the edit layer. + +Both ``kb.decompilations`` and ``kb.dec_variables`` spill to LMDB under memory pressure and hand +back a freshly deserialized -- that is, *different* -- Python object on reload. Never hold a +``DecompilationCache`` or a ``VariableManagerInternal`` across anything that can decompile; always +re-fetch by key at the point of mutation. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from .errors import NotDecompiledError + +if TYPE_CHECKING: + from collections.abc import Iterable + + from angr.analyses.decompiler.decompilation_cache import DecompilationCache + from angr.knowledge_base import KnowledgeBase + from angr.sim_type import SimType + +DEFAULT_FLAVOR = "pseudocode" + + +def get_cache(kb: KnowledgeBase, func_addr: int, flavor: str = DEFAULT_FLAVOR) -> DecompilationCache | None: + """Return the cached decompilation for a function, or None if there is none.""" + return kb.decompilations.get((func_addr, flavor), None) + + +def require_cache(kb: KnowledgeBase, func_addr: int, flavor: str = DEFAULT_FLAVOR) -> DecompilationCache: + """Return the cached decompilation, raising NotDecompiledError if it is missing or empty.""" + cache = get_cache(kb, func_addr, flavor) + if cache is None or cache.codegen is None: + raise NotDecompiledError( + f"Function {func_addr:#x} has not been decompiled yet (flavor {flavor!r}). Decompile it first." + ) + return cache + + +def invalidate( + kb: KnowledgeBase, + func_addr: int, + *, + flavors: Iterable[str] | None = None, + drop_variables: bool = False, +) -> None: + """ + Drop cached decompilations for a function. + + :param flavors: Flavors to drop; None drops every flavor currently cached. A prototype + change must drop all of them, or a non-pseudocode flavor silently + retains the old signature. + :param drop_variables: Also drop ``kb.dec_variables[func_addr]``. Required for new argument + names to take effect, but it discards every rename and manual type for + the function -- see :func:`snapshot_user_edits`. + """ + if flavors is None: + flavors = list(kb.decompilations.available_flavors(func_addr)) + for flavor in flavors: + kb.decompilations.discard((func_addr, flavor)) + + if drop_variables and kb.dec_variables.has_function_manager(func_addr): + del kb.dec_variables[func_addr] + + +def snapshot_user_edits(kb: KnowledgeBase, func_addr: int) -> dict[str, tuple[str | None, SimType | None]]: + """ + Capture user renames and manual types for a function, keyed by ``SimVariable.ident``. + + Used to survive the ``dec_variables`` drop that a prototype change requires. + """ + if not kb.dec_variables.has_function_manager(func_addr): + return {} + + varman = kb.dec_variables[func_addr] + snapshot: dict[str, tuple[str | None, SimType | None]] = {} + for var in varman.get_unified_variables(sort=None): + if not var.ident: + continue + name = var.name if var.renamed else None + ty = varman.get_variable_type(var) if var in varman.variables_with_manual_types else None + if name is not None or ty is not None: + snapshot[var.ident] = (name, ty) + return snapshot + + +def restore_user_edits( + kb: KnowledgeBase, func_addr: int, snapshot: dict[str, tuple[str | None, SimType | None]] +) -> tuple[int, list[str]]: + """ + Re-apply a :func:`snapshot_user_edits` result after re-decompilation. + + Best-effort: idents can change across a re-decompile, so the unmatched ones are returned rather + than silently dropped. + """ + if not snapshot or not kb.dec_variables.has_function_manager(func_addr): + return 0, sorted(snapshot) + + varman = kb.dec_variables[func_addr] + by_ident = {var.ident: var for var in varman.get_unified_variables(sort=None) if var.ident} + + restored = 0 + missing: list[str] = [] + for ident, (name, ty) in snapshot.items(): + var = by_ident.get(ident) + if var is None: + missing.append(ident) + continue + if name is not None: + var.name = name + var.renamed = True + var.clear_hash() + if ty is not None: + varman.set_variable_type(var, ty, all_unified=True, mark_manual=True) + restored += 1 + + return restored, sorted(missing) + + +def function_summary(kb: KnowledgeBase, func_addr: int) -> dict[str, Any]: + """Small helper for edit results: what is currently cached for a function.""" + return { + "flavors": sorted(kb.decompilations.available_flavors(func_addr)), + "has_variables": kb.dec_variables.has_function_manager(func_addr), + } diff --git a/angr/analyses/decompiler/edits/errors.py b/angr/analyses/decompiler/edits/errors.py new file mode 100644 index 000000000..ff32ba24e --- /dev/null +++ b/angr/analyses/decompiler/edits/errors.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from angr.errors import AngrError + + +class DecompilationEditError(AngrError): + """Base class for every failure raised by the decompilation edit layer.""" + + +class FunctionNotFoundError(DecompilationEditError): + """No function matches the given address or name.""" + + +class AmbiguousFunctionError(DecompilationEditError): + """More than one function matches the given name. Renames make this reachable.""" + + def __init__(self, message: str, addresses: list[int] | None = None): + super().__init__(message) + self.addresses: list[int] = addresses if addresses is not None else [] + + +class VariableNotFoundError(DecompilationEditError): + """ + No variable in the decompilation matches the given display name. + + ``candidates`` carries the names that *are* available, so a caller that failed can correct + itself without a second round trip. + """ + + def __init__(self, message: str, candidates: list[str] | None = None): + super().__init__(message) + self.candidates: list[str] = candidates if candidates is not None else [] + + +class NotDecompiledError(DecompilationEditError): + """The function has no cached decompilation, so there is nothing to edit.""" + + +class InvalidNameError(DecompilationEditError): + """The requested name is not a usable identifier.""" + + +class NameCollisionError(DecompilationEditError): + """The requested name is already bound to a different function, variable, or label.""" + + def __init__(self, message: str, existing: int | str | None = None): + super().__init__(message) + self.existing = existing + + +class TypeParseError(DecompilationEditError): + """A C type declaration or function signature could not be parsed.""" + + +class UnsupportedEditError(DecompilationEditError): + """The requested edit is not supported for this kind of target.""" diff --git a/angr/analyses/decompiler/edits/hooks.py b/angr/analyses/decompiler/edits/hooks.py new file mode 100644 index 000000000..6b3e491b1 --- /dev/null +++ b/angr/analyses/decompiler/edits/hooks.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +if TYPE_CHECKING: + from angr.knowledge_plugins.functions import Function + from angr.sim_type import SimType, SimTypeFunction + from angr.sim_variable import SimVariable + + +@runtime_checkable +class EditHooks(Protocol): + """ + Notifications fired immediately *before* each mutation, while the old value is still readable + from the knowledge base. + + The method set mirrors angr-management's plugin hooks one-for-one so its adapter is a pure + forwarder. Firing before the mutation matches what the GUI's own edit dialogs do, and is what + lets a handler snapshot pre-edit state. + """ + + def before_function_renamed(self, func: Function, old_name: str, new_name: str) -> None: ... + + def before_stack_var_renamed(self, func: Function, offset: int, old_name: str, new_name: str) -> None: ... + + def before_func_arg_renamed(self, func: Function, arg_index: int, old_name: str, new_name: str) -> None: ... + + def before_global_var_renamed(self, addr: int, old_name: str, new_name: str) -> None: ... + + def before_stack_var_retyped( + self, func: Function, offset: int, old_type: SimType | None, new_type: SimType + ) -> None: ... + + def before_func_arg_retyped( + self, func: Function, arg_index: int, old_type: SimType | None, new_type: SimType + ) -> None: ... + + def before_global_var_retyped(self, addr: int, old_type: SimType | None, new_type: SimType) -> None: ... + + def before_other_var_retyped(self, var: SimVariable, old_type: SimType | None, new_type: SimType) -> None: ... + + def before_function_retyped( + self, func: Function, old_proto: SimTypeFunction | None, new_proto: SimTypeFunction + ) -> None: ... + + def before_comment_changed(self, addr: int, old: str, new: str, created: bool, decomp: bool) -> None: ... + + +class NullEditHooks: + """A concrete no-op implementation. Subclass it so an adapter only overrides what it needs.""" + + def before_function_renamed(self, func: Function, old_name: str, new_name: str) -> None: + pass + + def before_stack_var_renamed(self, func: Function, offset: int, old_name: str, new_name: str) -> None: + pass + + def before_func_arg_renamed(self, func: Function, arg_index: int, old_name: str, new_name: str) -> None: + pass + + def before_global_var_renamed(self, addr: int, old_name: str, new_name: str) -> None: + pass + + def before_stack_var_retyped( + self, func: Function, offset: int, old_type: SimType | None, new_type: SimType + ) -> None: + pass + + def before_func_arg_retyped( + self, func: Function, arg_index: int, old_type: SimType | None, new_type: SimType + ) -> None: + pass + + def before_global_var_retyped(self, addr: int, old_type: SimType | None, new_type: SimType) -> None: + pass + + def before_other_var_retyped(self, var: SimVariable, old_type: SimType | None, new_type: SimType) -> None: + pass + + def before_function_retyped( + self, func: Function, old_proto: SimTypeFunction | None, new_proto: SimTypeFunction + ) -> None: + pass + + def before_comment_changed(self, addr: int, old: str, new: str, created: bool, decomp: bool) -> None: + pass + + +NULL_HOOKS = NullEditHooks() + + +def coerce_hooks(hooks: EditHooks | None) -> EditHooks: + return NULL_HOOKS if hooks is None else hooks diff --git a/angr/analyses/decompiler/edits/ops.py b/angr/analyses/decompiler/edits/ops.py new file mode 100644 index 000000000..a25292c92 --- /dev/null +++ b/angr/analyses/decompiler/edits/ops.py @@ -0,0 +1,638 @@ +""" +The mutating operations of the edit layer. + +Every operation re-fetches the decompilation cache by key at the point of mutation rather than +holding a reference: see the note in :mod:`.cache`. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from angr.knowledge_plugins.functions.function import PrototypeSource +from angr.sim_type import parse_signature, parse_type +from angr.sim_variable import SimMemoryVariable, SimStackVariable + +from .cache import DEFAULT_FLAVOR, get_cache, invalidate, require_cache, restore_user_edits, snapshot_user_edits +from .errors import NameCollisionError, TypeParseError, UnsupportedEditError +from .hooks import coerce_hooks +from .resolve import concrete_variables, list_variable_names, resolve_variable, validate_name +from .results import EditResult, Refresh + +if TYPE_CHECKING: + from angr.knowledge_base import KnowledgeBase + from angr.knowledge_plugins.functions import Function + from angr.project import Project + from angr.sim_type import SimType, SimTypeFunction + + from .hooks import EditHooks + +l = logging.getLogger(name=__name__) + + +def _require_free_function_name(kb: KnowledgeBase, name: str, own_addr: int) -> None: + for other in kb.functions.get_by_name(name): + if other is not None and other.addr != own_addr: + raise NameCollisionError( + f"A function named {name!r} already exists at {other.addr:#x}.", existing=other.addr + ) + existing = kb.labels.lookup(name, None) + if existing is not None and existing != own_addr: + raise NameCollisionError(f"The label {name!r} is already bound to {existing:#x}.", existing=existing) + + +def _set_arg_name(codegen, arg_index: int, new_name: str) -> None: + """ + Rewrite an argument's name in the rendered signature. + + ``cfunc.functy`` *is* ``func.prototype``, so this single write also updates the stored + prototype -- do not write both. + """ + cfunc = getattr(codegen, "cfunc", None) + if cfunc is None or cfunc.functy is None or not cfunc.functy.arg_names: + return + arg_names = list(cfunc.functy.arg_names) + if 0 <= arg_index < len(arg_names): + arg_names[arg_index] = new_name + cfunc.functy.arg_names = tuple(arg_names) + + +def rename_function( + project: Project, + func: Function, + new_name: str, + *, + kb: KnowledgeBase | None = None, + hooks: EditHooks | None = None, + flavor: str = DEFAULT_FLAVOR, + allow_overwrite: bool = True, + strict_names: bool = True, + rerender: bool = True, +) -> EditResult: + """ + Rename a function. + + Nothing is invalidated: other functions' cached ASTs reference the same Function object, so only + their rendered text goes stale. That is reported through ``Refresh.text_stale_all`` for the + caller to act on lazily. + """ + kb = project.kb if kb is None else kb + hooks = coerce_hooks(hooks) + validate_name(new_name, strict=strict_names) + + old_name = func.name + if old_name == new_name: + return EditResult(changed=False, kind="function_name", func_addr=func.addr, old=old_name, new=new_name) + + if not allow_overwrite: + _require_free_function_name(kb, new_name, func.addr) + + hooks.before_function_renamed(func, old_name, new_name) + + kb.functions.get_by_addr(func.addr).name = new_name + # the name setter does not clear this, and leaving it set makes the default-naming machinery + # treat the function as still auto-named + func.is_default_name = False + + cache = get_cache(kb, func.addr, flavor) + if cache is not None and cache.codegen is not None and getattr(cache.codegen, "cfunc", None) is not None: + cache.codegen.cfunc.name = new_name + cache.codegen.cfunc.demangled_name = new_name + if rerender: + cache.codegen.regenerate_text() + + return EditResult( + changed=True, + kind="function_name", + func_addr=func.addr, + old=old_name, + new=new_name, + refresh=Refresh( + text_stale=frozenset({func.addr}), + text_stale_all=True, + function_list_dirty=True, + disassembly_dirty=True, + ), + ) + + +def rename_variable( + project: Project, + func: Function, + variable_name: str, + new_name: str, + *, + kb: KnowledgeBase | None = None, + hooks: EditHooks | None = None, + flavor: str = DEFAULT_FLAVOR, + allow_overwrite: bool = True, + strict_names: bool = True, + rerender: bool = True, + codegen=None, +) -> EditResult: + """ + Rename a local, an argument, or a global as it appears in a function's decompilation. + + Sets ``renamed`` on the variable, without which a later re-decompilation overwrites the name. + """ + kb = project.kb if kb is None else kb + hooks = coerce_hooks(hooks) + validate_name(new_name, strict=strict_names) + + cache = require_cache(kb, func.addr, flavor) + if codegen is None: + codegen = cache.codegen + + rv = resolve_variable(kb, func.addr, variable_name, codegen=codegen, flavor=flavor) + old_name = rv.name + + if old_name == new_name: + return EditResult( + changed=False, + kind="variable_name", + func_addr=func.addr, + old=old_name, + new=new_name, + detail=rv.detail(), + ) + + if not allow_overwrite and new_name in list_variable_names(codegen, kb, func.addr): + raise NameCollisionError(f"A variable named {new_name!r} already exists in {func.name}.", existing=new_name) + + # dispatch order matches the GUI's: a stack-backed argument fires the stack hook, not the + # argument hook + if rv.kind == "global": + addr = rv.global_addr + if addr is not None and addr in kb.functions: + raise UnsupportedEditError( + f"{variable_name!r} is the entry point of a function at {addr:#x}; " + "rename it with rename_function instead." + ) + hooks.before_global_var_renamed(addr, old_name, new_name) + target = rv.variable + elif rv.stack_offset is not None: + hooks.before_stack_var_renamed(func, rv.stack_offset, old_name, new_name) + target = rv.unified + elif rv.kind == "argument": + hooks.before_func_arg_renamed(func, rv.arg_index, old_name, new_name) + target = rv.unified + else: + target = rv.unified + + target.name = new_name + target.renamed = True + target.clear_hash() + + if rv.kind == "global" and rv.global_addr is not None: + kb.labels[rv.global_addr] = new_name + if rv.kind == "argument" and rv.arg_index is not None: + _set_arg_name(codegen, rv.arg_index, new_name) + + if rerender: + codegen.regenerate_text() + + return EditResult( + changed=True, + kind="variable_name", + func_addr=func.addr, + old=old_name, + new=new_name, + refresh=Refresh( + text_stale=frozenset({func.addr}), + disassembly_dirty=rv.kind == "global", + ), + detail=rv.detail(), + ) + + +def _parse_type(c_type: str | SimType, arch) -> SimType: + if not isinstance(c_type, str): + return c_type.with_arch(arch) + try: + return parse_type(c_type).with_arch(arch) + except Exception as ex: # pylint:disable=broad-exception-caught + raise TypeParseError(f"Could not parse C type {c_type!r}: {ex}") from ex + + +def _parse_prototype(prototype: str | SimTypeFunction, arch) -> SimTypeFunction: + if not isinstance(prototype, str): + return prototype.with_arch(arch) + try: + return parse_signature(prototype).with_arch(arch) + except Exception as ex: # pylint:disable=broad-exception-caught + raise TypeParseError(f"Could not parse C prototype {prototype!r}: {ex}") from ex + + +def reflow_types( + project: Project, + func: Function, + *, + kb: KnowledgeBase | None = None, + flavor: str = DEFAULT_FLAVOR, + rerender: bool = True, +): + """ + Re-run type inference over the cached constraints and refresh the rendered code. + + Separate from :func:`set_variable_type` so a batch can retype many variables and reflow once; + per-variable reflow would re-run Typehoon N times. + + This is not a re-decompilation: the AST is untouched, so a retype that should change how an + access renders (a struct field, an array index) needs a full re-decompilation instead. + """ + kb = project.kb if kb is None else kb + + dec = project.analyses.Decompiler(func, decompile=False, use_cache=True, flavor=flavor) + cache = require_cache(kb, func.addr, flavor) + new_codegen = dec.reflow_variable_types(cache) + if new_codegen is None: + return None + + cache.codegen = new_codegen + if rerender: + # reflow_variable_types ends at reload_variable_types(), which refreshes CVariable types but + # does not re-render; without this the text stays stale + new_codegen.regenerate_text() + return new_codegen + + +def _set_argument_type( + project: Project, + func: Function, + rv, + new_type: SimType, + *, + hooks: EditHooks, +) -> EditResult: + """Retyping an argument means rewriting the prototype; the variable's own type is not enough.""" + proto = func.prototype + if proto is None or rv.arg_index is None or rv.arg_index >= len(proto.args): + raise UnsupportedEditError( + f"Cannot retype argument {rv.name!r}: {func.name} has no prototype covering argument " + f"index {rv.arg_index}. Set the whole prototype instead." + ) + + old_type = proto.args[rv.arg_index] + hooks.before_func_arg_retyped(func, rv.arg_index, old_type, new_type) + + new_proto = proto.copy() + args = list(new_proto.args) + args[rv.arg_index] = new_type + new_proto.args = tuple(args) + func.prototype = new_proto.with_arch(project.arch) + func.prototype_source = PrototypeSource.USER + func.ran_cca = True + + return EditResult( + changed=True, + kind="variable_type", + func_addr=func.addr, + old=str(old_type), + new=str(new_type), + refresh=Refresh(redecompile=frozenset({func.addr})), + detail=rv.detail(), + ) + + +def set_variable_type( + project: Project, + func: Function, + variable_name: str, + c_type: str | SimType, + *, + kb: KnowledgeBase | None = None, + hooks: EditHooks | None = None, + flavor: str = DEFAULT_FLAVOR, + reflow: bool = True, + rerender: bool = True, + allow_prototype_change: bool = True, + codegen=None, +) -> EditResult: + """ + Change the type of a local, an argument, or a global. + + Arguments are retyped by rewriting the function prototype, which requires a re-decompilation -- + the returned Refresh says so. Pass ``allow_prototype_change=False`` to refuse instead. + """ + kb = project.kb if kb is None else kb + hooks = coerce_hooks(hooks) + new_type = _parse_type(c_type, project.arch) + + cache = require_cache(kb, func.addr, flavor) + if codegen is None: + codegen = cache.codegen + + rv = resolve_variable(kb, func.addr, variable_name, codegen=codegen, flavor=flavor) + + if rv.kind == "argument": + if not allow_prototype_change: + raise UnsupportedEditError( + f"{variable_name!r} is an argument of {func.name}; change it by setting the whole prototype." + ) + return _set_argument_type(project, func, rv, new_type, hooks=hooks) + + if rv.kind == "global": + varman = kb.dec_variables["global"] + old_type = varman.get_variable_type(rv.variable) + hooks.before_global_var_retyped(rv.global_addr, old_type, new_type) + varman.set_variable_type(rv.variable, new_type, all_unified=False, mark_manual=True) + else: + varman = kb.dec_variables[func.addr] + old_type = varman.get_variable_type(rv.variable) + if rv.stack_offset is not None: + hooks.before_stack_var_retyped(func, rv.stack_offset, old_type, new_type) + else: + hooks.before_other_var_retyped(rv.variable, old_type, new_type) + # mark_manual is what makes the type survive re-inference: reflow reads + # variables_with_manual_types as its ground truth + for var in concrete_variables(varman, rv): + varman.set_variable_type(var, new_type, all_unified=True, mark_manual=True) + + if reflow: + reflow_types(project, func, kb=kb, flavor=flavor, rerender=rerender) + + return EditResult( + changed=True, + kind="variable_type", + func_addr=func.addr, + old=None if old_type is None else str(old_type), + new=str(new_type), + refresh=Refresh(text_stale=frozenset({func.addr})), + detail=rv.detail(), + ) + + +def set_function_prototype( + project: Project, + func: Function, + prototype: str | SimTypeFunction, + *, + kb: KnowledgeBase | None = None, + hooks: EditHooks | None = None, + flavor: str = DEFAULT_FLAVOR, + invalidate_cache: bool = True, + preserve_user_edits: bool = True, + redecompile: bool = False, +) -> EditResult: + """ + Set a function's prototype. + + The function name inside the signature is ignored; use :func:`rename_function` to rename. + + Dropping kb.dec_variables is what makes new argument names take effect, but it also discards + every rename and manual type for the function. Those are snapshotted and, when this function + re-decompiles, restored. Otherwise the snapshot is returned in ``detail["user_edits"]`` so a + caller that decompiles asynchronously can restore it once its own job finishes. + """ + kb = project.kb if kb is None else kb + hooks = coerce_hooks(hooks) + new_proto = _parse_prototype(prototype, project.arch) + + old_proto = func.prototype + hooks.before_function_retyped(func, old_proto, new_proto) + + snapshot = snapshot_user_edits(kb, func.addr) if preserve_user_edits else {} + + func.prototype = new_proto + func.prototype_source = PrototypeSource.USER + # keep CompleteCallingConventions from overwriting a user-supplied prototype + func.ran_cca = True + + if invalidate_cache: + invalidate(kb, func.addr, flavors=None, drop_variables=True) + + detail: dict = {"user_edits": snapshot} + code = None + if redecompile: + dec = project.analyses.Decompiler(func, flavor=flavor) + if snapshot: + restored, missing = restore_user_edits(kb, func.addr, snapshot) + detail.update({"restored_user_edits": restored, "unrestored_user_edits": missing}) + detail["user_edits"] = {} + if restored and dec.codegen is not None: + dec.codegen.regenerate_text() + code = dec.codegen.text if dec.codegen is not None else None + detail["code"] = code + + return EditResult( + changed=True, + kind="prototype", + func_addr=func.addr, + old=None if old_proto is None else str(old_proto), + new=str(new_proto), + refresh=Refresh(redecompile=frozenset({func.addr}), function_list_dirty=True), + detail=detail, + ) + + +_ORPHAN_MARKER = "// Orphaned comments" + + +def _snap_comment_addr(codegen, addr: int) -> int: + """ + Snap to the nearest address the codegen tracks, at or below ``addr``. + + stmt_comments is keyed by ins_addr, so an address the codegen never emits cannot match. This + only maps an arbitrary in-function address onto a tracked one; whether the result actually + renders inline additionally depends on it being the *last* tracked address on its line, which + is not derivable from the position maps -- map_addr_to_pos records each address's first + position, not its last. :func:`_rendered_inline` reports the real outcome afterwards. + """ + insmap = getattr(codegen, "map_addr_to_pos", None) + if insmap is None: + return addr + rendered = [ins_addr for ins_addr, _ in insmap.items()] + if not rendered or addr in set(rendered): + return addr + below = [a for a in rendered if a < addr] + return max(below) if below else addr + + +def _rendered_inline(codegen, comment: str) -> bool: + """Whether a comment rendered next to a statement rather than in the orphaned block.""" + text = getattr(codegen, "text", None) or "" + marker = text.find(_ORPHAN_MARKER) + return marker == -1 or comment not in text[marker:] + + +def set_comment( + project: Project, + addr: int, + comment: str | None, + *, + kb: KnowledgeBase | None = None, + hooks: EditHooks | None = None, + flavor: str = DEFAULT_FLAVOR, + mirror_to_pseudocode: bool = True, + snap: bool = True, + rerender: bool = True, +) -> EditResult: + """ + Set the comment at an address, or clear it with an empty string or None. + + The comment goes into kb.comments, which the disassembly renders and which the decompiler reads + for the function header. Per-statement pseudocode comments live in codegen.stmt_comments + instead, so both are written -- except at the function entry, where kb.comments is already what + the header renders and mirroring would show the comment twice. + """ + kb = project.kb if kb is None else kb + hooks = coerce_hooks(hooks) + text = comment or "" + + old = kb.comments.get(addr, "") + existed = addr in kb.comments + + hooks.before_comment_changed(addr, old, text, not existed, False) + if text: + kb.comments[addr] = text + elif existed: + del kb.comments[addr] + + in_pseudocode = False + inline: bool | None = None + snapped_from = None + + func = kb.functions.floor_func(addr) if mirror_to_pseudocode else None + if func is not None and not (func.addr <= addr < func.addr + max(func.size, 1)): + func = None + + if func is not None: + cache = get_cache(kb, func.addr, flavor) + codegen = cache.codegen if cache is not None else None + if codegen is not None: + if addr == func.addr: + # rendered as the header comment straight from kb.comments; mirroring it into + # stmt_comments would show it twice + in_pseudocode = bool(text) + if rerender: + codegen.regenerate_text() + else: + target = _snap_comment_addr(codegen, addr) if snap else addr + if target != addr: + snapped_from = addr + cdict = codegen.stmt_comments + prev = cdict.get(target, "") + hooks.before_comment_changed(target, prev, text, target not in cdict, True) + if text: + cdict[target] = text + in_pseudocode = True + elif target in cdict: + del cdict[target] + if rerender: + codegen.regenerate_text() + if in_pseudocode: + inline = _rendered_inline(codegen, text) + + return EditResult( + changed=old != text, + kind="comment", + func_addr=None if func is None else func.addr, + old=old, + new=text, + refresh=Refresh( + text_stale=frozenset() if func is None else frozenset({func.addr}), + disassembly_dirty=True, + ), + detail={ + "address": hex(addr), + "shown_in_pseudocode": in_pseudocode, + # None when not re-rendered here, so the caller cannot yet know + "rendered_inline": inline, + "snapped_from": None if snapped_from is None else hex(snapped_from), + }, + ) + + +def global_variable_at(kb: KnowledgeBase, addr: int): + """The global SimVariable recorded at an address, if variable recovery produced one.""" + varman = kb.dec_variables["global"] + for var in varman.get_variables(sort=None): + if isinstance(var, SimMemoryVariable) and not isinstance(var, SimStackVariable) and var.addr == addr: + return var + return None + + +def _reject_function_entry(kb: KnowledgeBase, addr: int, what: str) -> None: + """kb.labels[addr] = name renames the function when addr is a function entry.""" + if addr in kb.functions: + raise UnsupportedEditError( + f"{addr:#x} is the entry point of a function; {what} it with the function-level operation instead." + ) + + +def rename_global( + project: Project, + addr: int, + new_name: str, + *, + kb: KnowledgeBase | None = None, + hooks: EditHooks | None = None, + allow_overwrite: bool = True, + strict_names: bool = True, +) -> EditResult: + """Rename a global by address, without needing a function whose decompilation shows it.""" + kb = project.kb if kb is None else kb + hooks = coerce_hooks(hooks) + validate_name(new_name, strict=strict_names) + _reject_function_entry(kb, addr, "rename") + + var = global_variable_at(kb, addr) + old_name = var.name if var is not None and var.name else kb.labels.get(addr, "") + if old_name == new_name: + return EditResult(changed=False, kind="global_name", old=old_name, new=new_name) + + if not allow_overwrite: + existing = kb.labels.lookup(new_name, None) + if existing is not None and existing != addr: + raise NameCollisionError(f"The label {new_name!r} is already bound to {existing:#x}.", existing=existing) + + hooks.before_global_var_renamed(addr, old_name, new_name) + kb.labels[addr] = new_name + if var is not None: + var.name = new_name + var.renamed = True + var.clear_hash() + + return EditResult( + changed=True, + kind="global_name", + old=old_name, + new=new_name, + refresh=Refresh(text_stale_all=True, disassembly_dirty=True), + detail={"global_address": hex(addr)}, + ) + + +def set_global_type( + project: Project, + addr: int, + c_type: str | SimType, + *, + kb: KnowledgeBase | None = None, + hooks: EditHooks | None = None, +) -> EditResult: + """Set the type of a global by address.""" + kb = project.kb if kb is None else kb + hooks = coerce_hooks(hooks) + new_type = _parse_type(c_type, project.arch) + + var = global_variable_at(kb, addr) + if var is None: + raise UnsupportedEditError( + f"No global variable is recorded at {addr:#x}. Decompile a function that references it first." + ) + + varman = kb.dec_variables["global"] + old_type = varman.get_variable_type(var) + hooks.before_global_var_retyped(addr, old_type, new_type) + varman.set_variable_type(var, new_type, all_unified=False, mark_manual=True) + + return EditResult( + changed=True, + kind="global_type", + old=None if old_type is None else str(old_type), + new=str(new_type), + refresh=Refresh(text_stale_all=True), + detail={"global_address": hex(addr)}, + ) diff --git a/angr/analyses/decompiler/edits/resolve.py b/angr/analyses/decompiler/edits/resolve.py new file mode 100644 index 000000000..1d4f176bc --- /dev/null +++ b/angr/analyses/decompiler/edits/resolve.py @@ -0,0 +1,342 @@ +""" +Resolution helpers shared by every edit operation: address-or-name to Function, and pseudocode +display name to the underlying SimVariable. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal + +from angr.sim_variable import SimMemoryVariable, SimStackVariable + +from .cache import DEFAULT_FLAVOR, require_cache +from .errors import ( + AmbiguousFunctionError, + FunctionNotFoundError, + InvalidNameError, + VariableNotFoundError, +) + +if TYPE_CHECKING: + from angr.analyses.decompiler.structured_codegen.c import CVariable + from angr.knowledge_base import KnowledgeBase + from angr.knowledge_plugins.functions import Function + from angr.sim_variable import SimVariable + +_C_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +VariableKind = Literal["argument", "local", "global"] + + +def parse_address(value: str | int) -> int: + """Parse an address given as an int or a string. Accepts 0x-prefixed hex and decimal.""" + if isinstance(value, int): + return value + try: + return int(value, 0) + except (TypeError, ValueError) as ex: + raise InvalidNameError(f'Invalid address {value!r}. Pass a hex string such as "0x401000".') from ex + + +def validate_name(name: str, *, strict: bool = True) -> None: + """ + Reject names that cannot be used as identifiers. + + ``strict`` requires a C identifier. Without it only whitespace-free names are required, which + still admits things like ``int;`` that render as uncompilable C -- so strict is the default. + """ + if not name: + raise InvalidNameError("Name must not be empty.") + if strict: + if _C_IDENTIFIER_RE.match(name) is None: + raise InvalidNameError( + f"Invalid name {name!r}: must be a C identifier (letters, digits and underscores, " + "not starting with a digit)." + ) + elif re.search(r"\s", name) is not None: + raise InvalidNameError(f"Invalid name {name!r}: names must not contain whitespace.") + + +def resolve_function( + kb: KnowledgeBase, + *, + address: str | int | None = None, + name: str | None = None, + containing: bool = True, +) -> Function: + """ + Find a function by address or by name. + + :param containing: If True, an address inside a function resolves to that function rather + than requiring the exact entry address. + """ + if address is None and name is None: + raise FunctionNotFoundError("Specify either an address or a name.") + + if address is not None: + addr = parse_address(address) + func = kb.functions.get(addr) + if func is None and containing: + func = kb.functions.floor_func(addr) + if func is not None and not (func.addr <= addr < func.addr + max(func.size, 1)): + func = None + if func is None: + raise FunctionNotFoundError(f"No function found at address {addr:#x}.") + return func + + matches = [f for f in kb.functions.get_by_name(name) if f is not None] + if not matches: + raise FunctionNotFoundError(f"No function named {name!r}.") + if len(matches) > 1: + addrs = sorted(f.addr for f in matches) + raise AmbiguousFunctionError( + f"{len(matches)} functions are named {name!r}: {', '.join(hex(a) for a in addrs)}. " + "Specify an address instead.", + addresses=addrs, + ) + return matches[0] + + +@dataclass(frozen=True) +class ResolvedVariable: + """ + A pseudocode display name resolved to the objects an edit needs. + + ``variable`` is the concrete/SSA variable that ``set_variable_type`` expects; ``unified`` is the + unified variable that a rename mutates (None for globals, which have no unified form). + """ + + kind: VariableKind + variable: SimVariable + unified: SimVariable | None + cvar: CVariable | None = None + arg_index: int | None = None + stack_offset: int | None = None + global_addr: int | None = None + ambiguous: bool = False + + @property + def name(self) -> str | None: + target = self.variable if self.unified is None else self.unified + return target.name + + @property + def storage(self) -> str: + if self.stack_offset is not None: + return "stack" + if isinstance(self.variable, SimMemoryVariable) and not isinstance(self.variable, SimStackVariable): + return "memory" + return "register" + + def detail(self) -> dict: + return { + "storage": self.storage, + "is_argument": self.kind == "argument", + "stack_offset": self.stack_offset, + "arg_index": self.arg_index, + "global_address": None if self.global_addr is None else hex(self.global_addr), + } + + +def _stack_offset(var: SimVariable) -> int | None: + return var.offset if isinstance(var, SimStackVariable) else None + + +def _iter_candidates(kb: KnowledgeBase, func_addr: int, codegen) -> list[ResolvedVariable]: + """ + Collect every addressable variable, in precedence order: arguments, then locals, then globals. + + Locals come from the codegen (which carries the concrete variable) unioned with the variable + manager's unified variables, so unified variables not currently referenced in the body are still + addressable. Globals come from cexterns first, then any non-stack memory variable in use, then a + walk of the position map for inline references that cexterns filters out. + """ + out: list[ResolvedVariable] = [] + seen: set[int] = set() + + cfunc = getattr(codegen, "cfunc", None) + + if cfunc is not None and cfunc.arg_list: + for idx, cvar in enumerate(cfunc.arg_list): + var = getattr(cvar, "variable", None) + if var is None: + continue + unified = getattr(cvar, "unified_variable", None) or var + out.append( + ResolvedVariable( + kind="argument", + variable=var, + unified=unified, + cvar=cvar, + arg_index=idx, + stack_offset=_stack_offset(var), + ) + ) + seen.add(id(unified)) + + if cfunc is not None and cfunc.variables_in_use: + for var, cvar in cfunc.variables_in_use.items(): + unified = getattr(cvar, "unified_variable", None) + if unified is None or id(unified) in seen: + continue + out.append( + ResolvedVariable( + kind="local", + variable=var, + unified=unified, + cvar=cvar, + stack_offset=_stack_offset(var), + ) + ) + seen.add(id(unified)) + + if kb.dec_variables.has_function_manager(func_addr): + varman = kb.dec_variables[func_addr] + concrete_by_unified: dict[int, SimVariable] = {} + for var in varman.get_variables(sort=None): + unified = varman.unified_variable(var) + if unified is not None: + concrete_by_unified.setdefault(id(unified), var) + for unified in varman.get_unified_variables(sort=None): + if id(unified) in seen: + continue + # prefer a concrete SSA variable: set_variable_type(all_unified=True) keys off the + # SSA-to-unified map, so passing the unified variable would not propagate + concrete = concrete_by_unified.get(id(unified), unified) + out.append( + ResolvedVariable( + kind="local", + variable=concrete, + unified=unified, + stack_offset=_stack_offset(concrete), + ) + ) + seen.add(id(unified)) + + for cvar in _iter_global_cvars(codegen, cfunc): + var = cvar.variable + if var is None or id(var) in seen: + continue + out.append( + ResolvedVariable( + kind="global", + variable=var, + unified=None, + cvar=cvar, + global_addr=getattr(var, "addr", None), + ) + ) + seen.add(id(var)) + + return out + + +def concrete_variables(varman, resolved: ResolvedVariable) -> list[SimVariable]: + """ + Every SSA variable sharing the resolved variable's unified form. + + Retyping applies to all of them. Computed here rather than relying on + ``set_variable_type(all_unified=True)``, which silently does nothing when the variable it is + handed is not a key in the SSA-to-unified map. + """ + if resolved.unified is None: + return [resolved.variable] + + out = [var for var in varman.get_variables(sort=None) if varman.unified_variable(var) is resolved.unified] + if not any(var is resolved.variable for var in out): + out.append(resolved.variable) + return out + + +def _iter_global_cvars(codegen, cfunc): + from angr.analyses.decompiler.structured_codegen.c import CVariable # pylint:disable=import-outside-toplevel + + if codegen.cexterns: + yield from codegen.cexterns + + if cfunc is not None and cfunc.variables_in_use: + for var, cvar in cfunc.variables_in_use.items(): + if ( + getattr(cvar, "unified_variable", None) is None + and isinstance(var, SimMemoryVariable) + and not isinstance(var, SimStackVariable) + ): + yield cvar + + # inline references (strings, function pointers) that cexterns filters out + if codegen.map_pos_to_node is not None: + for item in codegen.map_pos_to_node.values(): + obj = getattr(item, "obj", None) + if isinstance(obj, CVariable) and obj.unified_variable is None and obj.variable is not None: + yield obj + + +def list_variable_names(codegen, kb: KnowledgeBase | None = None, func_addr: int | None = None) -> list[str]: + """Every display name addressable in this decompilation, for error messages.""" + if kb is None or func_addr is None: + cfunc = getattr(codegen, "cfunc", None) + names = set() + if cfunc is not None: + for cvar in cfunc.arg_list or []: + if cvar.name: + names.add(cvar.name) + for cvar in (cfunc.variables_in_use or {}).values(): + if cvar.name: + names.add(cvar.name) + return sorted(names) + + return sorted({rv.name for rv in _iter_candidates(kb, func_addr, codegen) if rv.name}) + + +def resolve_variable( + kb: KnowledgeBase, + func_addr: int, + display_name: str, + *, + codegen=None, + flavor: str = DEFAULT_FLAVOR, +) -> ResolvedVariable: + """ + Resolve a name as it appears in the pseudocode to the underlying variable. + + Precedence is argument > local > global. A name matching more than one variable within the same + bucket resolves deterministically and sets ``ambiguous``, rather than failing -- a batch edit + should be able to report the collision and continue. + """ + if codegen is None: + codegen = require_cache(kb, func_addr, flavor).codegen + + candidates = _iter_candidates(kb, func_addr, codegen) + matches = [rv for rv in candidates if rv.name == display_name] + + if not matches: + raise VariableNotFoundError( + f"No variable named {display_name!r} in the decompilation of function {func_addr:#x}.", + candidates=sorted({rv.name for rv in candidates if rv.name}), + ) + + if len(matches) == 1: + return matches[0] + + order = {"argument": 0, "local": 1, "global": 2} + matches.sort( + key=lambda rv: ( + order[rv.kind], + rv.arg_index if rv.arg_index is not None else 0, + rv.variable.ident or "", + rv.global_addr if rv.global_addr is not None else 0, + ) + ) + best = matches[0] + return ResolvedVariable( + kind=best.kind, + variable=best.variable, + unified=best.unified, + cvar=best.cvar, + arg_index=best.arg_index, + stack_offset=best.stack_offset, + global_addr=best.global_addr, + ambiguous=True, + ) diff --git a/angr/analyses/decompiler/edits/results.py b/angr/analyses/decompiler/edits/results.py new file mode 100644 index 000000000..7c3ea1a2d --- /dev/null +++ b/angr/analyses/decompiler/edits/results.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class Refresh: + """ + What a caller has to redo after an edit. + + Lets a UI pick between re-rendering text, rebuilding the codegen AST, and a full + re-decompilation instead of guessing. ``text_stale_all`` means every cached decompilation's + rendered text is stale (a function rename changes call sites everywhere), but the caller should + re-render lazily: eagerly re-rendering the whole cache would thrash its LRU. + """ + + text_stale: frozenset[int] = frozenset() + text_stale_all: bool = False + reanalyze: frozenset[int] = frozenset() + redecompile: frozenset[int] = frozenset() + function_list_dirty: bool = False + disassembly_dirty: bool = False + + def merge(self, other: Refresh) -> Refresh: + return Refresh( + text_stale=self.text_stale | other.text_stale, + text_stale_all=self.text_stale_all or other.text_stale_all, + reanalyze=self.reanalyze | other.reanalyze, + redecompile=self.redecompile | other.redecompile, + function_list_dirty=self.function_list_dirty or other.function_list_dirty, + disassembly_dirty=self.disassembly_dirty or other.disassembly_dirty, + ) + + +@dataclass +class EditResult: + """The outcome of a single edit. ``changed`` is False when the edit was a no-op.""" + + changed: bool + kind: str + func_addr: int | None = None + old: Any = None + new: Any = None + refresh: Refresh = field(default_factory=Refresh) + detail: dict[str, Any] = field(default_factory=dict) diff --git a/angr/mcp/__init__.py b/angr/mcp/__init__.py index 588223507..426511cb2 100644 --- a/angr/mcp/__init__.py +++ b/angr/mcp/__init__.py @@ -22,6 +22,7 @@ Usage: Available Tools: - load_binary: Load a binary file for analysis - get_cfg: Build the control flow graph + - recover_calling_conventions: Recover calling conventions and prototypes - list_functions: List discovered functions - get_function_info: Get detailed function information - decompile_function: Decompile to pseudocode @@ -34,6 +35,11 @@ Available Tools: - find_functions_by_pattern: Search functions by name - list_projects: List active analysis sessions - close_project: Close a project session + +Editing Tools (batch; see angr.mcp.edit_tools): + - rename: Rename functions, decompilation variables, and globals + - set_type: Set variable types, globals, return types, and function prototypes + - set_comments: Set or clear comments at addresses """ from __future__ import annotations diff --git a/angr/mcp/edit_tools.py b/angr/mcp/edit_tools.py new file mode 100644 index 000000000..f94bef212 --- /dev/null +++ b/angr/mcp/edit_tools.py @@ -0,0 +1,546 @@ +""" +Batch edit tools: rename, set_type, and set_comments. + +Every tool takes a list of items and returns one result per item. Per-item failures are reported as +data rather than raised, so one bad item does not discard the work done by its siblings -- that is +the point of batching. Only whole-call problems (an unknown project, a malformed request) raise. +""" + +from __future__ import annotations + +import logging +from collections import Counter +from typing import TYPE_CHECKING, Any + +from angr.analyses.decompiler.edits import ( + DecompilationEditError, + get_cache, + parse_address, + reflow_types, + rename_function, + rename_global, + rename_variable, + resolve_function, + resolve_variable, + restore_user_edits, + set_comment, + set_function_prototype, + set_global_type, + set_variable_type, +) +from angr.analyses.decompiler.edits.ops import _parse_type + +from .errors import InvalidArgumentError +from .server import _as_tool_error, _get_session, _require_cfg, _serialized, mcp + +if TYPE_CHECKING: + from collections.abc import Callable + + from angr.knowledge_plugins.functions import Function + + from .session import ProjectSession + +l = logging.getLogger(__name__) + +RENAME_KINDS = ("function", "variable", "global") +TYPE_KINDS = ("function", "variable", "global", "return") +COMMENT_KINDS = ("address", "function") + + +def _require_items(items: Any) -> list[dict[str, Any]]: + if isinstance(items, dict): + items = [items] + if not isinstance(items, list) or not items: + raise InvalidArgumentError("'items' must be a non-empty list of objects.") + for index, item in enumerate(items): + if not isinstance(item, dict): + raise InvalidArgumentError(f"items[{index}] must be an object, got {type(item).__name__}.") + return items + + +def _kind_of(item: dict[str, Any], index: int, allowed: tuple[str, ...]) -> str: + kind = item.get("kind") + if kind not in allowed: + raise InvalidArgumentError( + f"items[{index}]: 'kind' is required and must be one of {', '.join(allowed)}; got {kind!r}." + ) + return kind + + +def _required(item: dict[str, Any], index: int, field: str) -> Any: + value = item.get(field) + if value is None or value == "": + raise InvalidArgumentError(f"items[{index}]: {field!r} is required.") + return value + + +def _function_of(session: ProjectSession, item: dict[str, Any], index: int, field: str = "function") -> Function: + target = _required(item, index, field) + kb = session.project.kb + if isinstance(target, int): + return resolve_function(kb, address=target) + text = str(target) + if text.startswith("0x") or text.isdigit(): + return resolve_function(kb, address=text) + return resolve_function(kb, name=text) + + +def _function_from(session: ProjectSession, item: dict[str, Any], index: int) -> Function: + """ + Identify a function-scoped item. + + "function" wins when present; otherwise the item's own "name"/"address" identify it. Variable + items must not use this: there "name" is the variable's name, not the function's. + """ + if item.get("function") is not None: + return _function_of(session, item, index) + kb = session.project.kb + if item.get("address") is not None: + return resolve_function(kb, address=item["address"]) + name = item.get("name") + if not name: + raise InvalidArgumentError(f"items[{index}]: needs a 'function', a 'name', or an 'address'.") + return resolve_function(kb, name=str(name)) + + +def _global_addr_of(session: ProjectSession, item: dict[str, Any], index: int) -> int: + kb = session.project.kb + if item.get("address") is not None: + return parse_address(item["address"]) + name = item.get("name") + if not name: + raise InvalidArgumentError(f"items[{index}]: kind 'global' needs an 'address' or a 'name'.") + addr = kb.labels.lookup(str(name), None) + if addr is None: + raise InvalidArgumentError(f"items[{index}]: no global named {name!r}.") + return addr + + +def _ensure_decompiled(session: ProjectSession, func: Function, auto_decompile: bool) -> None: + """Variable edits need a codegen. Decompiling on demand keeps callers from a two-step dance.""" + if get_cache(session.project.kb, func.addr) is not None: + return + if not auto_decompile: + raise InvalidArgumentError( + f"{func.name} has not been decompiled. Call decompile_function first, or pass auto_decompile=True." + ) + session.project.analyses.Decompiler(func) + + +def _entry(index: int, kind: str, **fields: Any) -> dict[str, Any]: + entry = { + "index": index, + "kind": kind, + "status": "ok", + "function_address": None, + "function_name": None, + "target": None, + "old": None, + "new": None, + "detail": {}, + "error": None, + "error_kind": None, + "candidates": None, + } + entry.update(fields) + return entry + + +def _error_entry(index: int, kind: str | None, target: Any, exc: Exception) -> dict[str, Any]: + return _entry( + index, + kind or "unknown", + status="error", + target=None if target is None else str(target), + error=str(exc), + error_kind=type(exc).__name__, + candidates=getattr(exc, "candidates", None) or None, + ) + + +def _run( + project_id: str, + items: Any, + allowed_kinds: tuple[str, ...], + apply_item: Callable[[ProjectSession, int, str, dict[str, Any]], dict[str, Any]], + *, + dry_run: bool, + stop_on_error: bool, + post: Callable[[ProjectSession, set[int]], None] | None = None, +) -> dict[str, Any]: + session = _get_session(project_id) + _require_cfg(session) + items = _require_items(items) + + results: list[dict[str, Any]] = [] + touched: set[int] = set() + halted = False + + for index, item in enumerate(items): + if halted: + results.append( + _entry(index, item.get("kind") or "unknown", status="skipped", error="stopped by an earlier failure") + ) + continue + kind = None + try: + kind = _kind_of(item, index, allowed_kinds) + entry = apply_item(session, index, kind, item) + except (DecompilationEditError, InvalidArgumentError, ValueError) as e: + entry = _error_entry(index, kind, item.get("name") or item.get("function"), e) + if stop_on_error: + halted = True + results.append(entry) + if entry["status"] == "ok" and entry["function_address"]: + touched.add(int(entry["function_address"], 16)) + + if post is not None and not dry_run: + post(session, touched) + + counts = Counter(entry["status"] for entry in results) + return { + "project_id": project_id, + "dry_run": dry_run, + "total": len(results), + "succeeded": counts.get("ok", 0), + "unchanged": counts.get("unchanged", 0), + "failed": counts.get("error", 0), + "skipped": counts.get("skipped", 0), + "results": results, + "functions_affected": sorted(hex(addr) for addr in touched), + } + + +@mcp.tool() +@_as_tool_error +@_serialized +def rename( + project_id: str, + items: list[dict[str, Any]], + dry_run: bool = False, + stop_on_error: bool = False, + allow_overwrite: bool = False, + auto_decompile: bool = True, +) -> dict[str, Any]: + """ + Rename functions, decompilation variables, and globals in one call. + + Each item needs a "kind" and a "new_name": + + - kind "function": identify it with "name" or "address". + - kind "variable": "function" (name or address) plus "name", the variable's current name as it + appears in the pseudocode. Covers locals, arguments, and stack slots -- angr routes them + through one path, so the item does not say which; the result reports the resolved storage. + - kind "global": "address", or "name" if the global already has a label. + + Renames persist across re-decompilation. Renaming a variable requires the function to have been + decompiled; with auto_decompile it is decompiled on demand. + + Args: + project_id: The project ID + items: The rename requests + dry_run: Resolve and validate without changing anything. Not side-effect free: resolving a + variable name may decompile the function. + stop_on_error: Stop at the first failure and mark the rest skipped. Edits already applied + are NOT rolled back; use dry_run as a pre-flight. + allow_overwrite: Permit a name already bound to something else (default: False) + auto_decompile: Decompile a function on demand when a variable rename needs it + + Returns: + A per-item result list plus counts of succeeded/unchanged/failed/skipped + """ + + def apply_item(session: ProjectSession, index: int, kind: str, item: dict[str, Any]) -> dict[str, Any]: + proj = session.project + new_name = _required(item, index, "new_name") + + if kind == "function": + func = _function_from(session, item, index) + if dry_run: + status = "unchanged" if func.name == new_name else "ok" + return _entry( + index, + kind, + status=status, + function_address=hex(func.addr), + function_name=func.name, + target=func.name, + old=func.name, + new=new_name, + ) + result = rename_function(proj, func, new_name, allow_overwrite=allow_overwrite) + return _entry( + index, + kind, + status="ok" if result.changed else "unchanged", + function_address=hex(func.addr), + function_name=func.name, + target=result.old, + old=result.old, + new=result.new, + ) + + if kind == "global": + addr = _global_addr_of(session, item, index) + if dry_run: + return _entry(index, kind, target=hex(addr), new=new_name, detail={"global_address": hex(addr)}) + result = rename_global(proj, addr, new_name, allow_overwrite=allow_overwrite) + return _entry( + index, + kind, + status="ok" if result.changed else "unchanged", + target=result.old, + old=result.old, + new=result.new, + detail=result.detail, + ) + + func = _function_of(session, item, index) + _ensure_decompiled(session, func, auto_decompile) + current = _required(item, index, "name") + if dry_run: + rv = resolve_variable(proj.kb, func.addr, str(current)) + return _entry( + index, + kind, + status="unchanged" if rv.name == new_name else "ok", + function_address=hex(func.addr), + function_name=func.name, + target=rv.name, + old=rv.name, + new=new_name, + detail=rv.detail(), + ) + result = rename_variable(proj, func, str(current), new_name, allow_overwrite=allow_overwrite) + return _entry( + index, + kind, + status="ok" if result.changed else "unchanged", + function_address=hex(func.addr), + function_name=func.name, + target=result.old, + old=result.old, + new=result.new, + detail=result.detail, + ) + + return _run(project_id, items, RENAME_KINDS, apply_item, dry_run=dry_run, stop_on_error=stop_on_error) + + +@mcp.tool() +@_as_tool_error +@_serialized +def set_type( + project_id: str, + items: list[dict[str, Any]], + dry_run: bool = False, + stop_on_error: bool = False, + redecompile: bool = False, + auto_decompile: bool = True, +) -> dict[str, Any]: + """ + Set types on decompilation variables, globals, and function prototypes in one call. + + Each item needs a "kind" and a "type": + + - kind "variable": "function" plus "name". Retyping an argument rewrites the function's + prototype, which forces that function to be re-decompiled. + - kind "function": identify it with "function", "name", or "address"; "type" is a full + signature, e.g. "int parse(char *buf, int len)". The name inside the signature is ignored -- + use rename for that. This discards the function's cached decompilation and variables; earlier + renames and manual types, including ones set by earlier items in the same batch, are restored + afterwards. + - kind "return": same identifiers, replaces only the return type and keeps the arguments. + - kind "global": "address" or "name", plus "type". + + Types are re-inferred through the cached constraints once per affected function rather than per + item. That reflow does NOT rebuild the syntax tree, so a retype that should change how an access + renders -- a struct field, an array index -- needs redecompile=True. + + Args: + project_id: The project ID + items: The type requests + dry_run: Validate and resolve without changing anything + stop_on_error: Stop at the first failure; already-applied edits are NOT rolled back + redecompile: Fully re-decompile each affected function instead of reflowing types + auto_decompile: Decompile a function on demand when an item needs it + + Returns: + A per-item result list plus counts of succeeded/unchanged/failed/skipped + """ + + # A prototype change drops the function's cached decompilation and its variables, which would + # discard the manual types set by earlier items in this same batch. set_function_prototype hands + # back a snapshot of them; the post-pass re-decompiles and restores it. + pending_restores: dict[int, dict] = {} + + def apply_item(session: ProjectSession, index: int, kind: str, item: dict[str, Any]) -> dict[str, Any]: + proj = session.project + type_text = _required(item, index, "type") + + if kind == "global": + addr = _global_addr_of(session, item, index) + if dry_run: + return _entry(index, kind, target=hex(addr), new=str(type_text)) + result = set_global_type(proj, addr, str(type_text)) + return _entry(index, kind, target=hex(addr), old=result.old, new=result.new, detail=result.detail) + + func = ( + _function_from(session, item, index) + if kind in ("function", "return") + else _function_of(session, item, index) + ) + + if kind in ("function", "return"): + signature = str(type_text) + if kind == "return" and func.prototype is None: + raise InvalidArgumentError( + f"items[{index}]: {func.name} has no prototype yet; " + "run recover_calling_conventions first, or set a full signature with kind 'function'." + ) + if dry_run: + return _entry( + index, + kind, + function_address=hex(func.addr), + function_name=func.name, + old=None if func.prototype is None else str(func.prototype), + new=signature, + ) + if kind == "return": + result = _set_return_type(proj, func, signature) + else: + result = set_function_prototype(proj, func, signature, redecompile=redecompile) + snapshot = result.detail.get("user_edits") + if snapshot: + pending_restores.setdefault(func.addr, {}).update(snapshot) + return _entry( + index, + kind, + function_address=hex(func.addr), + function_name=func.name, + old=result.old, + new=result.new, + detail={k: v for k, v in result.detail.items() if k not in ("user_edits", "code")}, + ) + + _ensure_decompiled(session, func, auto_decompile) + current = _required(item, index, "name") + if dry_run: + rv = resolve_variable(proj.kb, func.addr, str(current)) + return _entry( + index, + kind, + function_address=hex(func.addr), + function_name=func.name, + target=rv.name, + new=str(type_text), + detail=rv.detail(), + ) + # reflow once per function afterwards instead of per item: Typehoon is expensive + result = set_variable_type(proj, func, str(current), str(type_text), reflow=False) + return _entry( + index, + kind, + function_address=hex(func.addr), + function_name=func.name, + target=str(current), + old=result.old, + new=result.new, + detail=result.detail, + ) + + def post(session: ProjectSession, touched: set[int]) -> None: + proj, kb = session.project, session.project.kb + for addr in sorted(touched): + func = kb.functions.get(addr) + if func is None: + continue + + snapshot = pending_restores.get(addr) + if get_cache(kb, addr) is None: + # a prototype change discarded the cache: rebuild it, then put back the renames and + # manual types that came with it + proj.analyses.Decompiler(func) + if snapshot: + restored, _ = restore_user_edits(kb, addr, snapshot) + if restored: + reflow_types(proj, func) + continue + + if redecompile: + proj.analyses.Decompiler(func, regen_clinic=True) + else: + reflow_types(proj, func) + + return _run(project_id, items, TYPE_KINDS, apply_item, dry_run=dry_run, stop_on_error=stop_on_error, post=post) + + +def _set_return_type(proj, func: Function, type_text: str): + """Replace only the return type, keeping the argument list.""" + proto = func.prototype + if proto is None: + raise InvalidArgumentError(f"{func.name} has no prototype to modify.") + new_proto = proto.copy() + new_proto.returnty = _parse_type(type_text, proj.arch) + return set_function_prototype(proj, func, new_proto) + + +@mcp.tool() +@_as_tool_error +@_serialized +def set_comments( + project_id: str, + items: list[dict[str, Any]], + dry_run: bool = False, + stop_on_error: bool = False, +) -> dict[str, Any]: + """ + Set or clear comments at addresses in one call. + + Each item needs a "kind" and a "comment"; an empty comment clears it. + + - kind "address": "address" is any address in the binary. + - kind "function": identify it with "function", "name", or "address"; comments its header. + + A comment is written to the knowledge base -- where the disassembly and the function header read + it -- and mirrored next to the matching pseudocode statement. Because pseudocode comments are + keyed by instruction address, an address that is not a statement boundary is snapped down to the + nearest one; each result reports "snapped_from" and whether the comment actually + rendered inline ("rendered_inline") rather than in the orphaned-comments block. + + Args: + project_id: The project ID + items: The comment requests + dry_run: Resolve addresses without changing anything + stop_on_error: Stop at the first failure; already-applied edits are NOT rolled back + + Returns: + A per-item result list plus counts of succeeded/unchanged/failed/skipped + """ + + def apply_item(session: ProjectSession, index: int, kind: str, item: dict[str, Any]) -> dict[str, Any]: + proj = session.project + comment = item.get("comment") or "" + + if kind == "function": + addr = _function_from(session, item, index).addr + else: + addr = parse_address(_required(item, index, "address")) + + if dry_run: + return _entry(index, kind, target=hex(addr), new=comment) + + result = set_comment(proj, addr, comment) + return _entry( + index, + kind, + status="ok" if result.changed else "unchanged", + function_address=None if result.func_addr is None else hex(result.func_addr), + target=hex(addr), + old=result.old, + new=result.new, + detail=result.detail, + ) + + return _run(project_id, items, COMMENT_KINDS, apply_item, dry_run=dry_run, stop_on_error=stop_on_error) diff --git a/angr/mcp/errors.py b/angr/mcp/errors.py index 2dfc65914..bf44c7707 100644 --- a/angr/mcp/errors.py +++ b/angr/mcp/errors.py @@ -1,10 +1,17 @@ from __future__ import annotations +from fastmcp.exceptions import ToolError + from angr.errors import AngrError -class MCPAngrError(AngrError): - """Base exception for MCP angr server errors.""" +class MCPAngrError(AngrError, ToolError): + """ + Base exception for MCP angr server errors. + + Also a fastmcp ToolError: FastMCP masks the message of anything else, so a client would + otherwise see a generic failure instead of the reason. + """ class ProjectNotFoundError(MCPAngrError): @@ -21,3 +28,7 @@ class FunctionNotFoundError(MCPAngrError): class DecompilationError(MCPAngrError): """Raised when decompilation fails.""" + + +class InvalidArgumentError(MCPAngrError, ValueError): + """Raised when a tool is called with unusable arguments.""" diff --git a/angr/mcp/server.py b/angr/mcp/server.py index ea67aaa42..90b6ec3e6 100644 --- a/angr/mcp/server.py +++ b/angr/mcp/server.py @@ -3,6 +3,8 @@ from __future__ import annotations import contextlib +import functools +import inspect import logging import re import sys @@ -10,15 +12,19 @@ from typing import Any import networkx as nx from fastmcp import FastMCP +from fastmcp.exceptions import ToolError from fastmcp.server.middleware import CallNext, Middleware, MiddlewareContext +from angr.analyses.decompiler.edits import DecompilationEditError, parse_address, resolve_function from angr.knowledge_plugins.cfg.memory_data import MemoryDataSort +from angr.knowledge_plugins.functions import Function from angr.utils.mp import protect_stdio_from_forked_children from .errors import ( CFGNotBuiltError, DecompilationError, FunctionNotFoundError, + InvalidArgumentError, ProjectNotFoundError, ) from .serializers import ( @@ -70,10 +76,66 @@ def _require_cfg(session: ProjectSession) -> None: def _parse_address(address: str | int) -> int: - """Parse an address from hex string or int.""" - if isinstance(address, int): - return address - return int(address, 16) + """Parse an address given as an int or a string. Accepts 0x-prefixed hex and decimal.""" + return parse_address(address) + + +def _find_function(session: ProjectSession, address: str | int | None = None, name: str | None = None) -> Function: + """Find a function by address or name. Any address inside the function resolves to it.""" + if address is None and name is None: + raise InvalidArgumentError("Must specify either 'address' or 'name'") + try: + return resolve_function(session.project.kb, address=address, name=name) + except DecompilationEditError as e: + raise FunctionNotFoundError(str(e)) from e + + +def _as_tool_error(func): + """ + Convert this layer's exceptions into fastmcp ToolErrors. + + FastMCP masks the message of any exception that is not a ToolError, so without this a caller + sees a generic failure instead of "no function named 'foo'" or the list of valid variable names. + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except DecompilationEditError as ex: + raise ToolError(str(ex)) from ex + + return wrapper + + +def _serialized(func): + """ + Hold the project's lock for the duration of a tool call. + + FastMCP dispatches synchronous tools onto worker threads, so two calls can reach one Project at + the same time. Applied to every tool, including read-only ones: the decompilation and variable + caches spill to LMDB, and a read racing an eviction is the same hazard as a concurrent write. + """ + signature = inspect.signature(func) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + bound = signature.bind(*args, **kwargs) + except TypeError: + return func(*args, **kwargs) + project_id = bound.arguments.get("project_id") + if project_id is None: + return func(*args, **kwargs) + try: + session = get_session_manager().get_session(project_id) + except KeyError: + # let the tool itself raise the project-not-found error + return func(*args, **kwargs) + with session.exclusive(): + return func(*args, **kwargs) + + return wrapper # ============================================================================ @@ -82,6 +144,8 @@ def _parse_address(address: str | int) -> int: @mcp.tool() +@_as_tool_error +@_serialized def load_binary( binary_path: str, auto_load_libs: bool = False, @@ -120,6 +184,8 @@ def load_binary( @mcp.tool() +@_as_tool_error +@_serialized def get_cfg( project_id: str, normalize: bool = True, @@ -161,6 +227,8 @@ def get_cfg( @mcp.tool() +@_as_tool_error +@_serialized def recover_calling_conventions( project_id: str, workers: int = 0, @@ -185,7 +253,7 @@ def recover_calling_conventions( proj = session.project if workers < 0: - raise ValueError("workers must be >= 0") + raise InvalidArgumentError("workers must be >= 0") analysis = proj.analyses.CompleteCallingConventions( cfg=session.cfg, @@ -205,6 +273,8 @@ def recover_calling_conventions( @mcp.tool() +@_as_tool_error +@_serialized def list_functions( project_id: str, filter_plt: bool | None = None, @@ -261,6 +331,8 @@ def list_functions( @mcp.tool() +@_as_tool_error +@_serialized def get_function_info( project_id: str, address: str | None = None, @@ -284,22 +356,7 @@ def get_function_info( session = _get_session(project_id) _require_cfg(session) - if address is None and name is None: - raise ValueError("Must specify either 'address' or 'name'") - - func = None - if address: - addr = _parse_address(address) - func = session.project.kb.functions.get(addr) - elif name: - # Search by name - for f in session.project.kb.functions.values(): - if f.name == name: - func = f - break - - if func is None: - raise FunctionNotFoundError(f"Function not found: address={address}, name={name}") + func = _find_function(session, address=address, name=name) return { "project_id": project_id, @@ -308,6 +365,8 @@ def get_function_info( @mcp.tool() +@_as_tool_error +@_serialized def decompile_function( project_id: str, address: str | None = None, @@ -330,22 +389,7 @@ def decompile_function( _require_cfg(session) proj = session.project - if address is None and name is None: - raise ValueError("Must specify either 'address' or 'name'") - - # Find the function - func = None - if address: - addr = _parse_address(address) - func = proj.kb.functions.get(addr) - elif name: - for f in proj.kb.functions.values(): - if f.name == name: - func = f - break - - if func is None: - raise FunctionNotFoundError(f"Function not found: address={address}, name={name}") + func = _find_function(session, address=address, name=name) # Decompile try: @@ -367,6 +411,8 @@ def decompile_function( @mcp.tool() +@_as_tool_error +@_serialized def get_xrefs( project_id: str, address: str, @@ -394,7 +440,7 @@ def get_xrefs( elif direction == "from": xrefs = list(xref_manager.xrefs_by_ins_addr.get(addr, set())) else: - raise ValueError(f"Invalid direction: {direction}. Use 'to' or 'from'.") + raise InvalidArgumentError(f"Invalid direction: {direction}. Use 'to' or 'from'.") return { "project_id": project_id, @@ -411,6 +457,8 @@ def get_xrefs( @mcp.tool() +@_as_tool_error +@_serialized def get_strings( project_id: str, min_length: int = 4, @@ -470,6 +518,8 @@ def get_strings( @mcp.tool() +@_as_tool_error +@_serialized def get_imports(project_id: str) -> dict[str, Any]: """ List imported symbols (external functions/variables the binary depends on). @@ -506,6 +556,8 @@ def get_imports(project_id: str) -> dict[str, Any]: @mcp.tool() +@_as_tool_error +@_serialized def get_exports(project_id: str) -> dict[str, Any]: """ List exported symbols (functions/variables this binary provides). @@ -535,6 +587,8 @@ def get_exports(project_id: str) -> dict[str, Any]: @mcp.tool() +@_as_tool_error +@_serialized def get_basic_blocks( project_id: str, function_address: str, @@ -574,6 +628,8 @@ def get_basic_blocks( @mcp.tool() +@_as_tool_error +@_serialized def get_callgraph( project_id: str, max_depth: int | None = None, @@ -653,6 +709,8 @@ def get_callgraph( @mcp.tool() +@_as_tool_error +@_serialized def find_functions_by_pattern( project_id: str, pattern: str, @@ -689,9 +747,9 @@ def find_functions_by_pattern( try: match = bool(re.search(pattern, func.name, re.IGNORECASE)) except re.error as e: - raise ValueError(f"Invalid regex pattern: {pattern}") from e + raise InvalidArgumentError(f"Invalid regex pattern: {pattern}") from e else: - raise ValueError(f"Invalid search_type: {search_type}") + raise InvalidArgumentError(f"Invalid search_type: {search_type}") if match: matches.append(serialize_function_summary(func)) @@ -706,6 +764,8 @@ def find_functions_by_pattern( @mcp.tool() +@_as_tool_error +@_serialized def list_projects() -> dict[str, Any]: """ List all currently loaded projects/sessions. @@ -723,6 +783,8 @@ def list_projects() -> dict[str, Any]: @mcp.tool() +@_as_tool_error +@_serialized def close_project(project_id: str) -> dict[str, Any]: """ Close a project and free its resources. @@ -750,3 +812,8 @@ def close_project(project_id: str) -> dict[str, Any]: def create_server() -> FastMCP: """Create and return the configured MCP server instance.""" return mcp + + +# Imported last: edit_tools registers its tools on ``mcp`` and imports the helpers defined above, +# so it has to come after them. +from . import edit_tools # noqa: F401 pylint:disable=wrong-import-position,unused-import diff --git a/angr/mcp/session.py b/angr/mcp/session.py index 6fca0ece2..1f6af87b5 100644 --- a/angr/mcp/session.py +++ b/angr/mcp/session.py @@ -3,14 +3,18 @@ from __future__ import annotations import logging +import threading import uuid -from dataclasses import dataclass +from contextlib import contextmanager +from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any import angr if TYPE_CHECKING: + from collections.abc import Iterator + from angr.knowledge_plugins.cfg import CFGModel l = logging.getLogger(__name__) @@ -24,12 +28,26 @@ class ProjectSession: project: angr.Project binary_path: str cfg: CFGModel | None = None + lock: threading.RLock = field(default_factory=threading.RLock, repr=False, compare=False) @property def has_cfg(self) -> bool: """Check if CFG has been built for this project.""" return self.cfg is not None + @contextmanager + def exclusive(self) -> Iterator[ProjectSession]: + """ + Serialize access to this project. + + FastMCP runs synchronous tools on worker threads, so two calls can reach the same Project + and KnowledgeBase at once. Neither is thread-safe, and the decompilation and variable caches + spill to LMDB, where a concurrent eviction and reload can hand back a half-imported object. + Read-only tools are not exempt: a read racing an eviction is exactly that hazard. + """ + with self.lock: + yield self + class SessionManager: """ @@ -41,6 +59,7 @@ class SessionManager: def __init__(self) -> None: self._sessions: dict[str, ProjectSession] = {} + self._lock = threading.Lock() def create_session(self, binary_path: str, **kwargs: Any) -> ProjectSession: """ @@ -72,7 +91,8 @@ class SessionManager: binary_path=str(path.resolve()), ) - self._sessions[project_id] = session + with self._lock: + self._sessions[project_id] = session l.info("Created session %s for %s", project_id, binary_path) return session @@ -85,10 +105,11 @@ class SessionManager: :return: The ProjectSession :raises KeyError: If no project with that ID exists """ - if project_id not in self._sessions: - available = list(self._sessions.keys()) - raise KeyError(f"No project with ID '{project_id}' found. Available: {available}") - return self._sessions[project_id] + with self._lock: + if project_id not in self._sessions: + available = list(self._sessions.keys()) + raise KeyError(f"No project with ID '{project_id}' found. Available: {available}") + return self._sessions[project_id] def list_sessions(self) -> list[dict[str, Any]]: """ @@ -96,6 +117,8 @@ class SessionManager: :return: List of session metadata dictionaries """ + with self._lock: + sessions = list(self._sessions.values()) return [ { "project_id": s.project_id, @@ -103,7 +126,7 @@ class SessionManager: "has_cfg": s.has_cfg, "arch": s.project.arch.name, } - for s in self._sessions.values() + for s in sessions ] def close_session(self, project_id: str) -> bool: @@ -113,11 +136,12 @@ class SessionManager: :param project_id: The project ID to close :return: True if closed, False if not found """ - if project_id in self._sessions: + with self._lock: + if project_id not in self._sessions: + return False del self._sessions[project_id] - l.info("Closed session %s", project_id) - return True - return False + l.info("Closed session %s", project_id) + return True # Global session manager instance diff --git a/tests/analyses/decompiler/test_edits.py b/tests/analyses/decompiler/test_edits.py new file mode 100644 index 000000000..c4f2613d1 --- /dev/null +++ b/tests/analyses/decompiler/test_edits.py @@ -0,0 +1,376 @@ +# pylint: disable=missing-class-docstring,no-self-use,protected-access +from __future__ import annotations + +__package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin + +import os +import unittest + +import angr +from angr.analyses.decompiler.edits import ( + DecompilationEditError, + NameCollisionError, + NotDecompiledError, + NullEditHooks, + UnsupportedEditError, + VariableNotFoundError, + list_variable_names, + reflow_types, + rename_function, + rename_variable, + require_cache, + resolve_function, + resolve_variable, + set_comment, + set_function_prototype, + set_variable_type, +) +from angr.analyses.decompiler.edits.errors import AmbiguousFunctionError, InvalidNameError +from tests.common import bin_location + +test_location = os.path.join(bin_location, "tests") + +FAUXWARE = os.path.join(test_location, "x86_64", "fauxware") + + +def load(decompile: str | None = "authenticate"): + """Load fauxware and optionally decompile one function. Roughly 1s.""" + proj = angr.Project(FAUXWARE, auto_load_libs=False) + proj.analyses.CFGFast(normalize=True, data_references=True) + proj.analyses.CompleteCallingConventions(analyze_callsites=True) + func = proj.kb.functions[decompile] if decompile else None + if func is not None: + proj.analyses.Decompiler(func) + return proj, func + + +def text_of(proj, func): + return require_cache(proj.kb, func.addr).codegen.text + + +class RecordingHooks(NullEditHooks): + """Records hook calls so ordering and the values observed can be asserted.""" + + def __init__(self): + self.calls: list[tuple] = [] + + def before_function_renamed(self, func, old_name, new_name): + self.calls.append(("function_renamed", func.name, old_name, new_name)) + + def before_stack_var_renamed(self, func, offset, old_name, new_name): + self.calls.append(("stack_var_renamed", offset, old_name, new_name)) + + def before_func_arg_renamed(self, func, arg_index, old_name, new_name): + self.calls.append(("func_arg_renamed", arg_index, old_name, new_name)) + + def before_global_var_renamed(self, addr, old_name, new_name): + self.calls.append(("global_var_renamed", addr, old_name, new_name)) + + def before_stack_var_retyped(self, func, offset, old_type, new_type): + self.calls.append(("stack_var_retyped", offset, str(old_type), str(new_type))) + + def before_func_arg_retyped(self, func, arg_index, old_type, new_type): + self.calls.append(("func_arg_retyped", arg_index, str(old_type), str(new_type))) + + def before_global_var_retyped(self, addr, old_type, new_type): + self.calls.append(("global_var_retyped", addr, str(new_type))) + + def before_other_var_retyped(self, var, old_type, new_type): + self.calls.append(("other_var_retyped", str(new_type))) + + def before_function_retyped(self, func, old_proto, new_proto): + self.calls.append(("function_retyped", str(old_proto), str(new_proto))) + + def before_comment_changed(self, addr, old, new, created, decomp): + self.calls.append(("comment_changed", addr, old, new, created, decomp)) + + +class TestResolve(unittest.TestCase): + def test_resolve_function_by_name_and_address(self): + proj, func = load() + assert resolve_function(proj.kb, name="authenticate").addr == func.addr + assert resolve_function(proj.kb, address=func.addr).addr == func.addr + # any address inside the function resolves to it + assert resolve_function(proj.kb, address=func.addr + 4).addr == func.addr + with self.assertRaises(DecompilationEditError): + resolve_function(proj.kb, address=func.addr + 4, containing=False) + + def test_resolve_function_ambiguous_after_rename(self): + proj, _ = load(decompile=None) + rename_function(proj, proj.kb.functions["main"], "collide") + rename_function(proj, proj.kb.functions["authenticate"], "collide") + with self.assertRaises(AmbiguousFunctionError) as ctx: + resolve_function(proj.kb, name="collide") + assert len(ctx.exception.addresses) == 2 + + def test_resolve_variable_kinds(self): + proj, func = load() + codegen = require_cache(proj.kb, func.addr).codegen + + arg = resolve_variable(proj.kb, func.addr, "a0", codegen=codegen) + assert arg.kind == "argument" + assert arg.arg_index == 0 + + # fauxware's authenticate has a stack-allocated password buffer + local = next( + resolve_variable(proj.kb, func.addr, name, codegen=codegen) + for name in list_variable_names(codegen, proj.kb, func.addr) + if name.startswith("v") + ) + assert local.kind == "local" + + # `sneaky` is a global; neither llm_suggest_variable_names nor get_unified_variables sees it + glob = resolve_variable(proj.kb, func.addr, "sneaky", codegen=codegen) + assert glob.kind == "global" + assert glob.global_addr is not None + assert glob.unified is None + + def test_resolve_variable_unknown_lists_candidates(self): + proj, func = load() + with self.assertRaises(VariableNotFoundError) as ctx: + resolve_variable(proj.kb, func.addr, "no_such_variable") + assert "a0" in ctx.exception.candidates + + def test_rename_requires_decompilation(self): + proj, _ = load(decompile=None) + func = proj.kb.functions["authenticate"] + with self.assertRaises(NotDecompiledError): + rename_variable(proj, func, "a0", "user") + + +class TestRename(unittest.TestCase): + def test_rename_function_updates_kb_labels_and_codegen(self): + proj, func = load() + result = rename_function(proj, func, "check_login") + + assert result.changed + assert result.old == "authenticate" + assert func.name == "check_login" + assert proj.kb.labels.get(func.addr) == "check_login" + assert func.is_default_name is False + assert "check_login" in text_of(proj, func) + + def test_rename_function_rejects_collision(self): + proj, func = load() + with self.assertRaises(NameCollisionError): + rename_function(proj, func, "main", allow_overwrite=False) + # allowed by default + assert rename_function(proj, func, "main").changed + + def test_rename_function_same_name_is_a_noop(self): + proj, func = load() + assert rename_function(proj, func, "authenticate").changed is False + + def test_rename_rejects_non_identifier(self): + proj, func = load() + with self.assertRaises(InvalidNameError): + rename_variable(proj, func, "a0", "int;") + + def test_rename_variable_kinds_render(self): + proj, func = load() + rename_variable(proj, func, "a0", "user") + rename_variable(proj, func, "sneaky", "backdoor_pw") + + text = text_of(proj, func) + assert "user" in text + assert "backdoor_pw" in text + # a global rename is also a label rename + glob = resolve_variable(proj.kb, func.addr, "backdoor_pw") + assert proj.kb.labels.get(glob.global_addr) == "backdoor_pw" + + def test_rename_survives_redecompilation(self): + """Guards the `renamed` flag: without it re-unification overwrites the name.""" + proj, func = load() + rename_variable(proj, func, "a0", "user") + rename_variable(proj, func, "sneaky", "backdoor_pw") + rename_function(proj, func, "check_login") + + proj.analyses.Decompiler(func, regen_clinic=True) + text = text_of(proj, func) + assert "user" in text + assert "backdoor_pw" in text + assert "check_login" in text + + def test_rename_global_at_function_entry_is_refused(self): + """kb.labels[addr] = name silently renames a function when addr is its entry.""" + proj, func = load() + codegen = require_cache(proj.kb, func.addr).codegen + glob = resolve_variable(proj.kb, func.addr, "sneaky", codegen=codegen) + + # pretend the global lives at a function entry + proj.kb.functions._function_map[glob.global_addr] = proj.kb.functions["main"] + try: + with self.assertRaises(UnsupportedEditError): + rename_variable(proj, func, "sneaky", "not_a_function") + finally: + del proj.kb.functions._function_map[glob.global_addr] + + +class TestTypes(unittest.TestCase): + def test_set_variable_type_reflows_and_rerenders(self): + """reflow_variable_types does not re-render; the text must still be refreshed.""" + proj, func = load() + local = next(n for n in list_variable_names(require_cache(proj.kb, func.addr).codegen) if n.startswith("v")) + + set_variable_type(proj, func, local, "long long") + assert f"long long {local}" in text_of(proj, func) + + def test_manual_type_survives_reflow(self): + proj, func = load() + local = next(n for n in list_variable_names(require_cache(proj.kb, func.addr).codegen) if n.startswith("v")) + set_variable_type(proj, func, local, "long long") + + reflow_types(proj, func) + assert f"long long {local}" in text_of(proj, func) + + def test_set_argument_type_updates_prototype(self): + proj, func = load() + result = set_variable_type(proj, func, "a1", "long long *") + + assert func.prototype is not None + assert "long long" in str(func.prototype.args[1]) + assert func.addr in result.refresh.redecompile + + proj.analyses.Decompiler(func, regen_clinic=True) + assert "long long *a1" in text_of(proj, func) + + def test_set_argument_type_can_be_refused(self): + proj, func = load() + with self.assertRaises(UnsupportedEditError): + set_variable_type(proj, func, "a1", "long long *", allow_prototype_change=False) + + def test_unparseable_type_is_rejected(self): + proj, func = load() + with self.assertRaises(DecompilationEditError): + set_variable_type(proj, func, "a0", "not a real type !!") + + def test_set_function_prototype_invalidates_caches(self): + proj, func = load() + set_function_prototype(proj, func, "int authenticate(char *user, char *pw)") + + assert (func.addr, "pseudocode") not in proj.kb.decompilations + assert not proj.kb.dec_variables.has_function_manager(func.addr) + + def test_set_function_prototype_applies_new_argument_names(self): + proj, func = load() + result = set_function_prototype(proj, func, "int authenticate(char *user, char *pw)", redecompile=True) + + code = result.detail["code"] + assert "char *user" in code + assert "char *pw" in code + + def test_set_function_prototype_preserves_user_edits(self): + """Dropping dec_variables discards renames and manual types; they must be restored.""" + proj, func = load() + local = next(n for n in list_variable_names(require_cache(proj.kb, func.addr).codegen) if n.startswith("v")) + set_variable_type(proj, func, local, "long long") + rename_variable(proj, func, local, "counter") + + result = set_function_prototype(proj, func, "int authenticate(char *user, char *pw)", redecompile=True) + code = result.detail["code"] + assert "long long counter" in code + + +class TestComments(unittest.TestCase): + def test_header_comment_rendered_once(self): + proj, func = load() + result = set_comment(proj, func.addr, "the auth check") + + assert result.detail["shown_in_pseudocode"] is True + # the entry address renders from kb.comments; mirroring it would duplicate it + assert text_of(proj, func).count("the auth check") == 1 + + def test_comment_cleared_by_empty_string(self): + proj, func = load() + set_comment(proj, func.addr, "temporary") + set_comment(proj, func.addr, "") + + assert func.addr not in proj.kb.comments + assert "temporary" not in text_of(proj, func) + + def test_statement_comment_snaps_and_reports_placement(self): + proj, func = load() + codegen = require_cache(proj.kb, func.addr).codegen + first = min(a for a, _ in codegen.map_addr_to_pos.items()) + + result = set_comment(proj, first + 1, "loop over entries") + assert result.detail["snapped_from"] == hex(first + 1) + assert result.detail["rendered_inline"] is True + assert "loop over entries" in text_of(proj, func) + + +class TestHooks(unittest.TestCase): + def test_hooks_fire_before_mutation_in_order(self): + """ + The regression guard for angr-management's refactor: hooks must fire before each mutation, + with the old value still readable, and in the same order the GUI produces. + """ + proj, func = load() + hooks = RecordingHooks() + + rename_variable(proj, func, "a0", "user", hooks=hooks) + rename_variable(proj, func, "sneaky", "backdoor_pw", hooks=hooks) + rename_function(proj, func, "check_login", hooks=hooks) + set_comment(proj, func.addr, "the auth check", hooks=hooks) + + kinds = [c[0] for c in hooks.calls] + assert kinds == [ + "func_arg_renamed", + "global_var_renamed", + "function_renamed", + "comment_changed", + ] + + # each hook observed the OLD value, and the real argument index rather than a constant + assert hooks.calls[0][1:] == (0, "a0", "user") + assert hooks.calls[1][2:] == ("sneaky", "backdoor_pw") + assert hooks.calls[2][1] == "authenticate" + assert hooks.calls[3][2:] == ("", "the auth check", True, False) + + def test_retype_hooks(self): + proj, func = load() + hooks = RecordingHooks() + local = next(n for n in list_variable_names(require_cache(proj.kb, func.addr).codegen) if n.startswith("v")) + + set_variable_type(proj, func, local, "long long", hooks=hooks) + set_variable_type(proj, func, "a1", "long long *", hooks=hooks) + set_function_prototype(proj, func, "int authenticate(char *a, char *b)", hooks=hooks) + + assert [c[0] for c in hooks.calls] == [ + "stack_var_retyped", + "func_arg_retyped", + "function_retyped", + ] + + +class TestCacheSpill(unittest.TestCase): + def test_edits_survive_cache_spill(self): + """ + Guards the rule that operations re-fetch the cache by key. + + With a one-entry cache the DecompilationCache is evicted to LMDB and comes back as a + different object, so an edit applied through a held reference would be lost. + """ + proj = angr.Project(FAUXWARE, auto_load_libs=False) + proj.analyses.CFGFast(normalize=True, data_references=True) + proj.analyses.CompleteCallingConventions(analyze_callsites=True) + + cached = proj.kb.decompilations.cached + if not hasattr(cached, "_cache_limit"): + self.skipTest("decompilation cache spilling is disabled") + cached._cache_limit = 1 + + authenticate = proj.kb.functions["authenticate"] + main = proj.kb.functions["main"] + proj.analyses.Decompiler(authenticate) + proj.analyses.Decompiler(main) # evicts authenticate + + proj.analyses.Decompiler(authenticate) + rename_variable(proj, authenticate, "a0", "user") + proj.analyses.Decompiler(main) # evicts authenticate again + + assert "user" in text_of(proj, authenticate) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/mcp/test_edit_tools.py b/tests/mcp/test_edit_tools.py new file mode 100644 index 000000000..840bb84bd --- /dev/null +++ b/tests/mcp/test_edit_tools.py @@ -0,0 +1,217 @@ +# pylint:disable=redefined-outer-name,no-self-use,missing-class-docstring +from __future__ import annotations + +import os + +import pytest + +from angr.mcp.edit_tools import rename, set_comments, set_type +from angr.mcp.errors import InvalidArgumentError +from angr.mcp.server import decompile_function, get_cfg, load_binary +from angr.mcp.session import get_session_manager + +from .conftest import BIN_LOCATION + +FAUXWARE = os.path.join(BIN_LOCATION, "tests", "x86_64", "fauxware") + + +@pytest.fixture +def project_id(): + """A loaded fauxware project with a CFG and `authenticate` already decompiled.""" + if not os.path.exists(FAUXWARE): + pytest.skip(f"Test binary not found: {FAUXWARE}") + pid = load_binary(FAUXWARE)["project_id"] + get_cfg(pid) + decompile_function(pid, name="authenticate") + yield pid + get_session_manager().close_session(pid) + + +def code(project_id: str, name: str = "authenticate") -> str: + return decompile_function(project_id, name=name)["code"] + + +class TestRename: + def test_renames_variables_and_function(self, project_id): + result = rename( + project_id, + [ + {"kind": "variable", "function": "authenticate", "name": "a0", "new_name": "user"}, + {"kind": "variable", "function": "authenticate", "name": "sneaky", "new_name": "backdoor"}, + {"kind": "function", "name": "authenticate", "new_name": "check_login"}, + ], + ) + + assert result["succeeded"] == 3 + assert result["failed"] == 0 + text = code(project_id, "check_login") + assert "user" in text + assert "backdoor" in text + assert "check_login" in text + + def test_renames_a_global_by_address(self, project_id): + # resolve the global's address through a variable rename first + found = rename( + project_id, + [{"kind": "variable", "function": "authenticate", "name": "sneaky", "new_name": "backdoor"}], + ) + addr = found["results"][0]["detail"]["global_address"] + + result = rename(project_id, [{"kind": "global", "address": addr, "new_name": "secret_pw"}]) + assert result["succeeded"] == 1 + assert "secret_pw" in code(project_id) + + def test_one_bad_item_does_not_stop_the_others(self, project_id): + result = rename( + project_id, + [ + {"kind": "variable", "function": "authenticate", "name": "nope", "new_name": "x"}, + {"kind": "variable", "function": "authenticate", "name": "a0", "new_name": "user"}, + ], + ) + + assert result["failed"] == 1 + assert result["succeeded"] == 1 + assert result["results"][0]["status"] == "error" + # a failed lookup reports what the caller could have said instead + assert "a0" in result["results"][0]["candidates"] + + def test_stop_on_error_skips_the_rest(self, project_id): + result = rename( + project_id, + [ + {"kind": "variable", "function": "authenticate", "name": "nope", "new_name": "x"}, + {"kind": "variable", "function": "authenticate", "name": "a0", "new_name": "user"}, + ], + stop_on_error=True, + ) + + assert result["failed"] == 1 + assert result["skipped"] == 1 + assert result["results"][1]["status"] == "skipped" + assert "user" not in code(project_id) + + def test_dry_run_changes_nothing(self, project_id): + before = code(project_id) + result = rename( + project_id, + [{"kind": "variable", "function": "authenticate", "name": "a0", "new_name": "user"}], + dry_run=True, + ) + + assert result["dry_run"] is True + assert result["results"][0]["old"] == "a0" + assert code(project_id) == before + + def test_collision_is_rejected_by_default(self, project_id): + result = rename(project_id, [{"kind": "function", "name": "authenticate", "new_name": "main"}]) + assert result["failed"] == 1 + assert result["results"][0]["error_kind"] == "NameCollisionError" + + result = rename( + project_id, + [{"kind": "function", "name": "authenticate", "new_name": "main"}], + allow_overwrite=True, + ) + assert result["succeeded"] == 1 + + def test_renaming_to_the_same_name_is_unchanged(self, project_id): + result = rename(project_id, [{"kind": "function", "name": "authenticate", "new_name": "authenticate"}]) + assert result["unchanged"] == 1 + assert result["succeeded"] == 0 + + def test_missing_kind_is_an_item_error(self, project_id): + result = rename(project_id, [{"function": "authenticate", "name": "a0", "new_name": "user"}]) + assert result["failed"] == 1 + assert "kind" in result["results"][0]["error"] + + def test_malformed_request_raises(self, project_id): + with pytest.raises(InvalidArgumentError): + rename(project_id, []) + with pytest.raises(InvalidArgumentError): + rename(project_id, ["not an object"]) + + +class TestSetType: + def test_retypes_a_local(self, project_id): + result = set_type( + project_id, + [{"kind": "variable", "function": "authenticate", "name": "v2", "type": "long long"}], + ) + + assert result["succeeded"] == 1 + assert "long long v2" in code(project_id) + + def test_sets_a_prototype(self, project_id): + result = set_type( + project_id, + [{"kind": "function", "name": "authenticate", "type": "int authenticate(char *user, char *pw)"}], + ) + + assert result["succeeded"] == 1 + text = code(project_id) + assert "char *user" in text + assert "char *pw" in text + + def test_sets_only_the_return_type(self, project_id): + result = set_type(project_id, [{"kind": "return", "function": "authenticate", "type": "int"}]) + + assert result["succeeded"] == 1 + text = code(project_id) + assert text.startswith("extern") or "int authenticate" in text + assert "int authenticate(char *" in text + + def test_prototype_change_preserves_earlier_items(self, project_id): + """A prototype change drops the variable manager; edits from the same batch must survive.""" + result = set_type( + project_id, + [ + {"kind": "variable", "function": "authenticate", "name": "v2", "type": "long long"}, + {"kind": "return", "function": "authenticate", "type": "int"}, + ], + ) + + assert result["failed"] == 0 + text = code(project_id) + assert "long long v2" in text + assert "int authenticate" in text + + def test_unparseable_type_is_an_item_error(self, project_id): + result = set_type( + project_id, + [{"kind": "variable", "function": "authenticate", "name": "v2", "type": "not a type !!"}], + ) + assert result["failed"] == 1 + assert result["results"][0]["error_kind"] == "TypeParseError" + + +class TestSetComments: + def test_comments_a_function_header(self, project_id): + result = set_comments( + project_id, + [{"kind": "function", "function": "authenticate", "comment": "validates a password"}], + ) + + assert result["succeeded"] == 1 + # the header renders from the knowledge base; it must not also be mirrored into the + # statement comments, which would show it twice + assert code(project_id).count("validates a password") == 1 + + def test_comment_is_cleared_by_an_empty_string(self, project_id): + set_comments(project_id, [{"kind": "function", "function": "authenticate", "comment": "temporary"}]) + set_comments(project_id, [{"kind": "function", "function": "authenticate", "comment": ""}]) + + assert "temporary" not in code(project_id) + + def test_statement_comment_reports_placement(self, project_id): + entry = decompile_function(project_id, name="authenticate") + addr = int(entry["function_address"], 16) + + result = set_comments(project_id, [{"kind": "address", "address": hex(addr + 0x10), "comment": "here"}]) + detail = result["results"][0]["detail"] + assert "rendered_inline" in detail + assert "snapped_from" in detail + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/tests/mcp/test_session.py b/tests/mcp/test_session.py index 0f5fecbe0..e94e48429 100644 --- a/tests/mcp/test_session.py +++ b/tests/mcp/test_session.py @@ -1,8 +1,12 @@ # pylint:disable=no-self-use from __future__ import annotations +import threading +import time + import pytest +from angr.mcp import server from angr.mcp.session import ( get_session_manager, ) @@ -146,3 +150,62 @@ class TestGlobalSessionManager: # Cleanup manager.close_session(session.project_id) + + +class TestSessionLocking: + """Tests for the per-session lock that serializes concurrent tool calls.""" + + def test_exclusive_is_reentrant(self, loaded_session): + """Nested helpers must be able to re-acquire without deadlocking.""" + with loaded_session.exclusive(), loaded_session.exclusive(): + assert loaded_session.lock._is_owned() # pylint:disable=protected-access + + def test_exclusive_serializes_threads(self, loaded_session): + """Two threads must not be inside the critical section at the same time.""" + overlaps = [] + inside = [] + barrier = threading.Barrier(2) + + def worker(): + barrier.wait() + for _ in range(50): + with loaded_session.exclusive(): + inside.append(1) + if len(inside) > 1: + overlaps.append(1) + time.sleep(0) + inside.pop() + + threads = [threading.Thread(target=worker) for _ in range(2)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not overlaps + + def test_tool_call_holds_the_session_lock(self, global_session_manager, binary_path): + """Every tool runs under the lock, read-only ones included.""" + session = global_session_manager.create_session(binary_path) + observed = [] + + @server._serialized # pylint:disable=protected-access + def probe(project_id: str) -> None: + observed.append(session.lock._is_owned()) # pylint:disable=protected-access + + try: + probe(session.project_id) + finally: + global_session_manager.close_session(session.project_id) + assert observed == [True] + + def test_tool_call_without_a_known_project_still_runs(self): + """An unknown project must reach the tool so it can raise its own error.""" + ran = [] + + @server._serialized # pylint:disable=protected-access + def probe(project_id: str) -> None: + ran.append(project_id) + + probe("does-not-exist") + assert ran == ["does-not-exist"] From 0c293dc0dea6a97efd583b7fe9e56912f0e8e2c1 Mon Sep 17 00:00:00 2001 From: Fish Date: Sun, 9 Aug 2026 02:00:48 -0700 Subject: [PATCH 113/122] Phoenix: Fix incorrect virtualization of orphaned edges. (#6790) --- .../decompiler/structuring/phoenix.py | 4 + .../test_phoenix_last_resort_isolation.py | 160 ++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 tests/analyses/decompiler/test_phoenix_last_resort_isolation.py diff --git a/angr/analyses/decompiler/structuring/phoenix.py b/angr/analyses/decompiler/structuring/phoenix.py index a23d1fd59..05d9ed9bd 100644 --- a/angr/analyses/decompiler/structuring/phoenix.py +++ b/angr/analyses/decompiler/structuring/phoenix.py @@ -3067,6 +3067,10 @@ class PhoenixStructurer(StructurerBase): # this is a head of an incomplete switch-case construct (that we will definitely be structuring later), # so we do not want to remove any edges going out of this block continue + if dst in graph and graph.in_degree[dst] == 1 and dst is not head: + # dst would be left with no way in, and no schema can reattach an isolated node + other_edges.append((src, dst)) + continue src_dominates_dst = dominates_by_intervals(dominance_intervals, src, dst) if not src_dominates_dst and not dominates_by_intervals(dominance_intervals, dst, src): if (src.addr, dst.addr) not in self.whitelist_edges: diff --git a/tests/analyses/decompiler/test_phoenix_last_resort_isolation.py b/tests/analyses/decompiler/test_phoenix_last_resort_isolation.py new file mode 100644 index 000000000..cdb9be007 --- /dev/null +++ b/tests/analyses/decompiler/test_phoenix_last_resort_isolation.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use,no-member,protected-access +from __future__ import annotations + +__package__ = __package__ or "tests.analyses.decompiler" # pylint:disable=redefined-builtin + +import logging +import os +import unittest + +import networkx + +import angr +from angr.ailment import Manager +from angr.ailment.block import Block +from angr.ailment.expression import Const +from angr.ailment.statement import Jump +from angr.analyses.decompiler.structuring.phoenix import PhoenixStructurer +from angr.analyses.decompiler.utils import sequence_to_blocks +from tests.common import bin_location, print_decompilation_result + +test_location = os.path.join(bin_location, "tests") + + +class _Overlay: + """Stands in for a region overlay graph: _last_resort_refinement() only calls filtered() on these.""" + + def __init__(self, graph): + self._graph = graph + + def filtered(self): + return self._graph + + +class _Region: + """Only the type-4 cycle fallback reads these, and that branch needs a cyclic graph.""" + + parent = None + cyclic = False + + +class TestPhoenixLastResortIsolation(unittest.TestCase): + """ + Last-resort refinement buckets candidate edges by dominance, and immediate_dominators() only covers nodes + reachable from the region head. For unreachable debris both dominance queries answer False, which is the + first-choice bucket -- so an edge that is a node's only way in gets virtualized and the node is orphaned. No + schema can reattach an isolated node, so the region never reduces. + """ + + @staticmethod + def _block(m, addr): + return Block(addr, 1, statements=[Jump(m.next_atom(), Const(m.next_atom(), addr + 1, 64), ins_addr=addr)]) + + @staticmethod + def _refine(graph, head): + """Run the candidate selection, recording the edge it would virtualize instead of performing it.""" + structurer = object.__new__(PhoenixStructurer) + structurer._improve_algorithm = False + structurer._edge_virtualization_hints = [] + structurer.whitelist_edges = set() + structurer._region = _Region() + chosen = [] + + def _virtualize_edge(src, dst): + chosen.append((src, dst)) + return True + + structurer._virtualize_edge = _virtualize_edge + overlay = _Overlay(graph) + progressed = PhoenixStructurer._last_resort_refinement(structurer, head, overlay, overlay) + return progressed, chosen + + def test_edge_that_would_orphan_its_destination_is_not_picked(self): + m = Manager(arch=None) + head = self._block(m, 0x100) + # x -> y hangs off the region unreachable from head, and it is y's only way in + x, y = self._block(m, 0x200), self._block(m, 0x300) + graph = networkx.DiGraph() + graph.add_node(head) + graph.add_edge(x, y) + + progressed, chosen = self._refine(graph, head) + + # nothing left to virtualize; reporting no progress dissolves the region instead of orphaning y + assert not progressed + assert not chosen + + def test_a_safe_edge_is_still_picked(self): + m = Manager(arch=None) + head = self._block(m, 0x100) + a, b = self._block(m, 0x200), self._block(m, 0x300) + # b keeps a second way in, so cutting a -> b orphans nothing + graph = networkx.DiGraph() + graph.add_edge(head, a) + graph.add_edge(head, b) + graph.add_edge(a, b) + + progressed, chosen = self._refine(graph, head) + + assert progressed + assert chosen == [(a, b)] + + def test_orphaning_edge_is_skipped_in_favour_of_a_safe_one(self): + m = Manager(arch=None) + head = self._block(m, 0x100) + a, b = self._block(m, 0x200), self._block(m, 0x300) + x, y = self._block(m, 0x400), self._block(m, 0x500) + graph = networkx.DiGraph() + graph.add_edge(head, a) + graph.add_edge(head, b) + graph.add_edge(a, b) + graph.add_edge(x, y) + + progressed, chosen = self._refine(graph, head) + + assert progressed + assert chosen == [(a, b)] + + def test_bbbq_rust_root_region_structures_completely(self): + """ + sub_410920 is where this showed up: three edges were cut that each orphaned their destination, the region + dissolved, and the root ended up as three disconnected components. _pick_incomplete_result_from_region() + keeps only the one at the function address, so the other two were dropped from the output. + + The whole-binary CFG is required -- under a scoped CFG the region never reaches that state. + """ + bin_path = os.path.join(test_location, "x86_64", "bbbq") + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFGFast(normalize=True, data_references=True, show_progressbar=False) + proj.analyses.CompleteCallingConventions() + proj.analyses.RustSymbolRecovery() + proj.analyses.TypeDBLoader() + + incomplete = [] + + class _Watch(logging.Handler): + def emit(self, record): + if "Structuring failed to complete" in record.getMessage(): + incomplete.append(record) + + logger = logging.getLogger("angr.analyses.decompiler.structuring.recursive_structurer") + watch = _Watch() + logger.addHandler(watch) + try: + dec = proj.analyses.Decompiler(0x410920, cfg=cfg.model, flavor="rust", fail_fast=True) + finally: + logger.removeHandler(watch) + assert dec.codegen is not None and dec.codegen.text is not None + print_decompilation_result(dec) + + assert not incomplete + + # the blocks that used to be dropped along with the discarded components + structured = {b.addr for b in sequence_to_blocks(dec.seq_node)} + for addr in (0x4115BA, 0x4115CC, 0x4115CF, 0x411435, 0x411458): + assert addr in structured, f"{addr:#x} missing from the structured output" + + +if __name__ == "__main__": + unittest.main() From 2bfaa7e8f77b8946168def3e1c14d03d0bf6e952 Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Tue, 11 Aug 2026 13:27:06 -0700 Subject: [PATCH 114/122] CFGModel: Stop aborting when tidying a data reference at an unmapped address (#6811) Fix #6770. --- angr/knowledge_plugins/cfg/cfg_model.py | 2 +- tests/analyses/cfg/test_cfg_model.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/angr/knowledge_plugins/cfg/cfg_model.py b/angr/knowledge_plugins/cfg/cfg_model.py index 070e5169d..2ea4fa27d 100644 --- a/angr/knowledge_plugins/cfg/cfg_model.py +++ b/angr/knowledge_plugins/cfg/cfg_model.py @@ -850,7 +850,7 @@ class CFGModel(Serializable): else: boundary = min(last_addr, next_data_addr) - if next_sec_addr is not None: + if boundary is not None and next_sec_addr is not None: boundary = min(boundary, next_sec_addr) if boundary is not None: diff --git a/tests/analyses/cfg/test_cfg_model.py b/tests/analyses/cfg/test_cfg_model.py index 8b25f4a99..e446b288b 100755 --- a/tests/analyses/cfg/test_cfg_model.py +++ b/tests/analyses/cfg/test_cfg_model.py @@ -16,6 +16,7 @@ log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) FAUXWARE_PATH = os.path.join(bin_location, "tests", "x86_64", "fauxware") REFLOW_FAUXWARE_PATH = os.path.join(bin_location, "tests", "x86_64", "fauxware_reflow") +LWIP_TCPECHO_PATH = os.path.join(bin_location, "tests", "armel", "lwip_tcpecho_bm.elf") class TestCfgModel(unittest.TestCase): @@ -90,6 +91,24 @@ class TestCfgModel(unittest.TestCase): func = cfg.model.find_function_for_reflow_into_addr(addr) assert func is None + def test_tidy_data_references_unmapped_data_addr_before_a_section(self): + """Test CFGModel::tidy_data_references on a data address that no section or segment covers.""" + proj = angr.Project(LWIP_TCPECHO_PATH, auto_load_libs=False) + cfg = proj.analyses[CFGFast].prep()() + + # this firmware image leaves a wide hole between its two loaded regions, and the pointer-array pass keeps + # every pointer that lands in the main object's address range, the hole included + unmapped_addr = 0x50020 + assert unmapped_addr in cfg.model.memory_data + assert proj.loader.find_section_containing(unmapped_addr) is None + assert proj.loader.find_segment_containing(unmapped_addr) is None + assert proj.loader.find_section_next_to(unmapped_addr) is not None + + # tidying that address on its own, the way CFGFast tidies each batch of newly found data, leaves the next + # section as the only bound in hand for it + cfg.model.tidy_data_references(memory_data_addrs=[unmapped_addr]) + assert cfg.model.memory_data[unmapped_addr].max_size == 0 + if __name__ == "__main__": unittest.main() From 41a7db2f1717d66f0f99e2e61a42e87416ce9d0f Mon Sep 17 00:00:00 2001 From: Fish Date: Thu, 13 Aug 2026 12:38:23 -0700 Subject: [PATCH 115/122] SLiveness: propagate to predecessors instead of re-walking the graph (#6830) --- angr/analyses/s_liveness.py | 33 ++++---- tests/analyses/test_s_liveness.py | 120 ++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 17 deletions(-) create mode 100644 tests/analyses/test_s_liveness.py diff --git a/angr/analyses/s_liveness.py b/angr/analyses/s_liveness.py index b76113a82..a4c2eb6d2 100644 --- a/angr/analyses/s_liveness.py +++ b/angr/analyses/s_liveness.py @@ -74,13 +74,12 @@ class SLivenessAnalysis(Analysis): # blocks whose statements have been walked at least once walked: set[tuple[int, int | None]] = set() - worklist = deque(networkx.dfs_postorder_nodes(graph, source=entry)) - worklist_set = set(worklist) - single_exit_single_entry_nodes = { - node - for node in graph - if graph.in_degree[node] == 1 and graph.out_degree[node] == 1 and not graph.has_edge(node, node) - } + successors = {block: list(graph.successors(block)) for block in graph} + predecessors = {block: list(graph.predecessors(block)) for block in graph} + + order: list[Block] = list(networkx.dfs_postorder_nodes(graph, source=entry)) + worklist = deque(order) + worklist_set = set(order) while worklist: block = worklist.popleft() @@ -92,16 +91,17 @@ class SLivenessAnalysis(Analysis): head_controlled_loop = is_head_controlled_loop_block(block) live = set() - for succ in graph.successors(block): - if head_controlled_loop and (block.addr, block.idx) == (succ.addr, succ.idx): + for succ in successors[block]: + succ_key = succ.addr, succ.idx + if head_controlled_loop and block_key == succ_key: # this is a head-controlled loop block; we ignore the self-loop edge because all variables defined # in the block after the conditional jump will be dead after leaving the current block continue - edge = (block.addr, block.idx), (succ.addr, succ.idx) + edge = block_key, succ_key if edge in live_on_edges: live |= live_on_edges[edge] else: - live |= live_ins[(succ.addr, succ.idx)] + live |= live_ins[succ_key] if live != live_outs[block_key]: changed = True @@ -175,12 +175,11 @@ class SLivenessAnalysis(Analysis): live_on_edges[key] = live changed = True - if changed and block not in single_exit_single_entry_nodes: - new_nodes = [ - node for node in networkx.dfs_postorder_nodes(graph, source=block) if node not in worklist_set - ] - worklist.extend(new_nodes) - worklist_set |= set(new_nodes) + if changed: + for pred in predecessors[block]: + if pred not in worklist_set: + worklist.append(pred) + worklist_set.add(pred) # set the model accordingly self.model.live_ins = live_ins diff --git a/tests/analyses/test_s_liveness.py b/tests/analyses/test_s_liveness.py new file mode 100644 index 000000000..0496c9568 --- /dev/null +++ b/tests/analyses/test_s_liveness.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# pylint: disable=missing-class-docstring,no-self-use +from __future__ import annotations + +__package__ = __package__ or "tests.analyses" # pylint:disable=redefined-builtin + +import os.path +import unittest +from unittest import mock + +import networkx + +import angr +from angr.ailment.expression import VirtualVariable +from angr.ailment.statement import Assignment, SideEffectStatement +from angr.analyses.s_liveness import SLivenessAnalysis +from angr.utils.ail import is_phi_assignment +from angr.utils.ssa import VVarUsesCollector +from tests.common import bin_location + +test_location = os.path.join(bin_location, "tests") + + +class TestSLiveness(unittest.TestCase): + @classmethod + def setUpClass(cls): + proj = angr.Project(os.path.join(test_location, "x86_64", "1after909"), auto_load_libs=False) + cfg = proj.analyses.CFG(normalize=True) + proj.analyses.CompleteCallingConventions() + cls.proj = proj + cls.cfg = cfg + cls.graphs = {} + for name in ("main", "verify_password", "doit", "read_str"): + func = proj.kb.functions[name] + dec = proj.analyses.Decompiler(func, cfg=cfg.model) + assert dec.ail_graph is not None + entry = next(b for b in dec.ail_graph if b.addr == func.addr and b.idx is None) + cls.graphs[name] = (func, dec.ail_graph, entry) + + def _liveness(self, name): + func, graph, entry = self.graphs[name] + return graph, self.proj.analyses[SLivenessAnalysis].prep()(func, func_graph=graph, entry=entry, arg_vvars=[]) + + def test_live_ins_propagate_to_predecessor_live_outs(self): + """The worklist must run to a fixpoint, not stop early.""" + for name in self.graphs: + graph, liveness = self._liveness(name) + for block in graph: + for succ in graph.successors(block): + if any(is_phi_assignment(stmt) for stmt in succ.statements): + # a phi block's live-in is per-predecessor, tracked on the edge + continue + if block is succ: + continue + live_in = liveness.model.live_ins[succ.addr, succ.idx] + live_out = liveness.model.live_outs[block.addr, block.idx] + assert live_in <= live_out, ( + f"{name}: {succ.addr:#x} live-in leaks past {block.addr:#x} live-out: " + f"{sorted(live_in - live_out)}" + ) + + def test_vars_used_before_definition_are_live_in(self): + """The defining property of liveness, checked block by block.""" + collector = VVarUsesCollector() + for name in self.graphs: + graph, liveness = self._liveness(name) + for block in graph: + defined: set[int] = set() + live_in = liveness.model.live_ins[block.addr, block.idx] + for stmt in block.statements: + if is_phi_assignment(stmt): + # phi operands come from specific predecessors, not from live-in + assert isinstance(stmt, Assignment) + defined.add(stmt.dst.varid) + continue + collector.reset() + collector.walk_statement(stmt) + for varid in collector.vvars - defined: + assert varid in live_in, ( + f"{name}: vvar {varid} is used at {block.addr:#x} before any definition " + f"but is not live-in there" + ) + if isinstance(stmt, Assignment) and isinstance(stmt.dst, VirtualVariable): + defined.add(stmt.dst.varid) + elif isinstance(stmt, SideEffectStatement) and isinstance(stmt.ret_expr, VirtualVariable): + defined.add(stmt.ret_expr.varid) + + def test_worklist_does_not_traverse_the_graph_per_update(self): + """One ordering DFS for the whole analysis, not one per changed block. + + Liveness is backward, so a block that changes only needs its predecessors + re-queued. Re-deriving the reachable set every time made this quadratic: + on a 2215-block function it visited 1.4M nodes, 639x the graph. + """ + func, graph, entry = self.graphs["main"] + real = networkx.dfs_postorder_nodes + calls = [] + + def counting(G, source=None, depth_limit=None): + calls.append(source) + return real(G, source=source, depth_limit=depth_limit) + + with mock.patch.object(networkx, "dfs_postorder_nodes", counting): + self.proj.analyses[SLivenessAnalysis].prep()(func, func_graph=graph, entry=entry, arg_vvars=[]) + + assert len(calls) == 1, f"expected a single ordering traversal, got {len(calls)}" + assert calls[0] is entry + + def test_result_is_independent_of_run(self): + """The worklist order must not leak into the result.""" + for name in self.graphs: + _, first = self._liveness(name) + _, second = self._liveness(name) + assert first.model.live_ins == second.model.live_ins + assert first.model.live_outs == second.model.live_outs + assert first.model.block_end_vvars == second.model.block_end_vvars + + +if __name__ == "__main__": + unittest.main() From b80f4cf2afac3fb52bfa03844e0cba5f55ad9d12 Mon Sep 17 00:00:00 2001 From: Fish Date: Thu, 13 Aug 2026 13:51:21 -0700 Subject: [PATCH 116/122] Outliner: Rebuild phi statements; update output phis; return only region-defined vvars. (#6831) * Rebuild phi statements instead of mutating them in place * Update the phis of every dispatcher target * Return only the variables the outlined region defines --- angr/analyses/outliner/outliner.py | 76 +++++++++---- tests/analyses/decompiler/test_outliner.py | 122 ++++++++++++++++++++- 2 files changed, 173 insertions(+), 25 deletions(-) diff --git a/angr/analyses/outliner/outliner.py b/angr/analyses/outliner/outliner.py index f24c1250b..b2d88541c 100644 --- a/angr/analyses/outliner/outliner.py +++ b/angr/analyses/outliner/outliner.py @@ -183,9 +183,16 @@ class Outliner(Analysis): self.parent_graph.add_edge(pred, new_src_node) # build the return statement if needed - if self.frontier_vars: + vvars_defined_in_region = { + stmt.dst.varid + for node in subgraph + for stmt in node.statements + if isinstance(stmt, Assignment) and isinstance(stmt.dst, VirtualVariable) + } + owned_frontier_vars = self.frontier_vars & vvars_defined_in_region + if owned_frontier_vars: srda = SReachingDefinitions(self.project, self.parent_func, func_graph=self.parent_graph).model - ret_exprs = [srda.varid_to_vvar[idx] for idx in self.frontier_vars] + ret_exprs = [srda.varid_to_vvar[idx] for idx in owned_frontier_vars] else: ret_exprs = [] @@ -253,16 +260,29 @@ class Outliner(Analysis): self.parent_graph.add_edge(parent, dispatcher_node) self.parent_graph.add_edge(dispatcher_node, node_dict[jump_target]) - self._update_phi_stmts(node_dict[jump_target]) + self._update_phi_stmts( + node_dict[jump_target], + collapsed_loc=(dispatcher_node.addr, dispatcher_node.idx), + ret_vvar=ret_exprs[0] if ret_exprs else None, + ) parent = dispatcher_node self.parent_graph.add_edge(parent, node_dict[last_retval_to_target_item[1]]) + self._update_phi_stmts( + node_dict[last_retval_to_target_item[1]], + collapsed_loc=(parent.addr, parent.idx), + ret_vvar=ret_exprs[0] if ret_exprs else None, + ) else: (frontier_node,) = frontier self.parent_graph.add_edge(new_src_node, frontier_node) - self._update_phi_stmts(frontier_node) + self._update_phi_stmts( + frontier_node, + collapsed_loc=(new_src_node.addr, new_src_node.idx), + ret_vvar=ret_exprs[0] if ret_exprs else None, + ) if ret_exprs: if len(ret_exprs) > 1: @@ -271,26 +291,38 @@ class Outliner(Analysis): return callee_func, subgraph, callee_arg_vvars - def _update_phi_stmts(self, block: Block): + def _update_phi_stmts(self, block: Block, collapsed_loc=None, ret_vvar=None): srcs = list(self.parent_graph.predecessors(block)) src_addrs = [(src.addr, src.idx) for src in srcs] - for stmt in block.statements: - if is_phi_assignment(stmt): - assert isinstance(stmt, Assignment) and isinstance(stmt.src, Phi) - all_stmt_srcs = [src for src, _ in stmt.src.src_and_vvars] - new_addrs = set(src_addrs) - set(all_stmt_srcs) - old_addrs = set(all_stmt_srcs) - set(src_addrs) - if len(old_addrs) == 1 and len(new_addrs) == 1: - # only source block is replaced by a new one - old_addr = next(iter(old_addrs)) - new_addr = next(iter(new_addrs)) - for idx, _ in enumerate(stmt.src.src_and_vvars): - src, vvars = stmt.src.src_and_vvars[idx] - if src == old_addr: - stmt.src.src_and_vvars[idx] = new_addr, vvars - else: - # multiple source blocks have been replaced... it's bad - continue + for i, stmt in enumerate(block.statements): + if not is_phi_assignment(stmt): + continue + assert isinstance(stmt, Assignment) and isinstance(stmt.src, Phi) + pairs = list(stmt.src.src_and_vvars) + all_stmt_srcs = [src for src, _ in pairs] + new_addrs = set(src_addrs) - set(all_stmt_srcs) + old_addrs = set(all_stmt_srcs) - set(src_addrs) + new_src_and_vvars = None + if len(old_addrs) == 1 and len(new_addrs) == 1: + old_addr = next(iter(old_addrs)) + new_addr = next(iter(new_addrs)) + new_src_and_vvars = [((new_addr if src == old_addr else src), vvar) for src, vvar in pairs] + elif ( + old_addrs + and collapsed_loc is not None + and set(src_addrs) == {collapsed_loc} + and set(all_stmt_srcs) <= (old_addrs | {collapsed_loc}) + ): + # an outlined region collapsed into a call, and every phi source vvar arrives through that one block. + vvars = [vvar for _, vvar in pairs] + distinct = {v.varid for v in vvars if v is not None} + value = vvars[0] if len(distinct) == 1 else ret_vvar + if value is not None: + new_src_and_vvars = [(collapsed_loc, value)] + # else: multiple source blocks have been replaced... it's bad + if new_src_and_vvars is not None: + new_phi = Phi(stmt.src.idx, stmt.src.bits, new_src_and_vvars, **stmt.src.tags) + block.statements[i] = Assignment(stmt.idx, stmt.dst, new_phi, **stmt.tags) @staticmethod def _node_addr_to_str(addr: tuple[int, int | None]) -> str: diff --git a/tests/analyses/decompiler/test_outliner.py b/tests/analyses/decompiler/test_outliner.py index f673c4979..150c8489e 100644 --- a/tests/analyses/decompiler/test_outliner.py +++ b/tests/analyses/decompiler/test_outliner.py @@ -1,14 +1,15 @@ +# pylint: disable=missing-class-docstring,no-self-use from __future__ import annotations -# pylint: disable=missing-class-docstring,no-self-use import logging import os.path import unittest from unittest import TestCase import angr -from angr.ailment.expression import VirtualVariableCategory -from angr.analyses.decompiler.clinic import ClinicStage +from angr.ailment.expression import Call, Phi, VirtualVariable, VirtualVariableCategory +from angr.ailment.statement import Assignment +from angr.analyses.decompiler.clinic import Clinic, ClinicStage from angr.analyses.decompiler.decompiler import Decompiler from angr.analyses.outliner import Outliner from angr.sim_type import SimStruct, SimTypeArray, SimTypeChar, SimTypeWideChar @@ -207,3 +208,118 @@ if __name__ == "__main__": logging.getLogger("angr.analyses.outliner").setLevel(logging.DEBUG) # TestOutliner().test_outlining_authenticate() TestOutliner().test_outlining_notepad_npinit() + + +class TestOutlinerSSAInvariants(TestCase): + """Outlining rewrites the parent graph; the result must still be valid SSA.""" + + @classmethod + def setUpClass(cls): + bin_path = os.path.join(bin_location, "tests", "x86_64", "1after909") + proj = angr.Project(bin_path, auto_load_libs=False) + cfg = proj.analyses.CFG(normalize=True) + proj.analyses.CompleteCallingConventions() + cls.proj = proj + cls.cfg = cfg + cls.graphs = {} + for name in ("convert", "read_str"): + func = proj.kb.functions[name] + dec = proj.analyses.Decompiler(func, cfg=cfg.model) + assert dec.ail_graph is not None + cls.graphs[name] = (func, dec.ail_graph) + + @staticmethod + def _ssa_problems(graph) -> set: + """Phis naming a block that is not a predecessor, and vvars defined twice.""" + locs = {(b.addr, b.idx) for b in graph} + problems = set() + definitions = {} + for block in graph: + preds = {(p.addr, p.idx) for p in graph.predecessors(block)} + for i, stmt in enumerate(block.statements): + if isinstance(stmt, Assignment) and isinstance(stmt.dst, VirtualVariable): + definitions.setdefault(stmt.dst.varid, []).append((block.addr, block.idx, i)) + if not isinstance(stmt, Assignment) or not isinstance(stmt.src, Phi): + continue + for src, _ in stmt.src.src_and_vvars: + if src not in locs: + problems.add(("phi-sources-removed-block", block.addr, block.idx, i, src)) + elif src not in preds: + problems.add(("phi-sources-non-predecessor", block.addr, block.idx, i, src)) + for varid, locations in definitions.items(): + if len(locations) > 1: + problems.add(("vvar-defined-more-than-once", varid, None, None, None)) + return problems + + def test_outlining_never_introduces_ssa_problems(self): + """Outline at every viable location and check nothing new breaks.""" + for name, (func, base) in self.graphs.items(): + baseline = self._ssa_problems(base) + for block in sorted(base, key=lambda b: (b.addr, -1 if b.idx is None else b.idx)): + if base.in_degree[block] == 0 or base.out_degree[block] == 0: + continue + graph = Clinic._copy_graph(base) + try: + self.proj.analyses[Outliner](func, graph, src_loc=(block.addr, block.idx), min_step=2) + except Exception: # pylint:disable=broad-except + continue + introduced = self._ssa_problems(graph) - baseline + assert not introduced, ( + f"outlining {name} at {block.addr:#x}.{block.idx} introduced " + f"{len(introduced)} SSA problems, e.g. {min(introduced)}" + ) + + def test_call_returns_only_variables_the_region_defines(self): + """The synthesized call must not become a second definition of a variable + whose value reaches the frontier from outside the outlined region. + + Needs an explicit frontier: with a derived one ``frontier_vars`` is empty + and the call only ever returns its own dispatcher variable. + """ + checked = 0 + for name, (func, base) in self.graphs.items(): + for block in sorted(base, key=lambda b: (b.addr, -1 if b.idx is None else b.idx)): + if base.in_degree[block] == 0 or base.out_degree[block] == 0: + continue + # let the Outliner pick a frontier, then feed it back in explicitly + probe_graph = Clinic._copy_graph(base) + try: + probe = self.proj.analyses[Outliner](func, probe_graph, src_loc=(block.addr, block.idx), min_step=2) + except Exception: # pylint:disable=broad-except + continue + if not probe.frontier_locs: + continue + graph = Clinic._copy_graph(base) + synthetic = 0x100000 # vvars the Outliner mints for itself start here + try: + outliner = self.proj.analyses[Outliner]( + func, + graph, + src_loc=(block.addr, block.idx), + frontier=set(probe.frontier_locs), + vvar_id_start=synthetic, + min_step=2, + ) + except Exception: # pylint:disable=broad-except + continue + child_defs = { + stmt.dst.varid + for node in outliner.child_graph + for stmt in node.statements + if isinstance(stmt, Assignment) and isinstance(stmt.dst, VirtualVariable) + } + call_block = next((b for b in graph if (b.addr, b.idx) == (block.addr, block.idx)), None) + if call_block is None: + continue + for stmt in call_block.statements: + if not isinstance(stmt, Assignment) or not isinstance(stmt.src, Call): + continue + if not isinstance(stmt.dst, VirtualVariable) or stmt.dst.varid >= synthetic: + # the call's own dispatcher variable, not a returned value + continue + checked += 1 + assert stmt.dst.varid in child_defs, ( + f"outlining {name} at {block.addr:#x}: the call returns vvar " + f"{stmt.dst.varid}, which the outlined region never defines" + ) + assert checked, "no explicit-frontier outline produced a call with a return value" From 2c95eb92e6b15b005788f5f3903bd4e97e584c9c Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Thu, 13 Aug 2026 15:26:22 -0700 Subject: [PATCH 117/122] CFG: Warn when the regions to analyze cover no bytes. (#6825) --- angr/analyses/cfg/cfg_base.py | 21 +++++++++++++++++++ .../cfg/test_cfg_no_executable_regions.py | 20 ++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/angr/analyses/cfg/cfg_base.py b/angr/analyses/cfg/cfg_base.py index ad39c6c54..66f23879e 100644 --- a/angr/analyses/cfg/cfg_base.py +++ b/angr/analyses/cfg/cfg_base.py @@ -255,6 +255,7 @@ class CFGBase(Analysis): '"auto_load_libs" disabled, or specify "regions" to limit the scope of CFG recovery.' ) + regions_derived_from_objects = regions is None if regions is None: regions = self._exec_mem_regions if not self._skip_unmapped_addrs and not regions: @@ -284,6 +285,14 @@ class CFGBase(Analysis): for start, end in self._regions.items(): l.debug("... %#x - %#x", start, end) + if regions_derived_from_objects and not self._regions_size: + l.warning( + "CFG recovery has nothing to scan: the regions to analyze cover 0 bytes. If %s does contain code, " + 'pass the address ranges that hold it in "regions", or set "force_segment" to derive regions from ' + "segments instead of sections.", + self._binary, + ) + def __contains__(self, cfg_node): return cfg_node in self.graph @@ -325,6 +334,15 @@ class CFGBase(Analysis): """ return self.kb.functions + @property + def regions(self) -> list[tuple[int, int]]: + """ + The memory regions that this analysis covers. An empty list means it had nothing to scan. + + :return: A sorted list of (start address, end address) tuples. + """ + return list(self._regions.items()) + # # Methods # @@ -891,6 +909,9 @@ class CFGBase(Analysis): if not memory_regions and not has_executable: memory_regions = [(start, start + len(backer)) for start, backer in self.project.loader.memory.backers()] + # A section or segment that maps no bytes, such as the empty .text of a data-only relocatable, is not a region. + memory_regions = [(start, end) for start, end in memory_regions if end > start] + return sorted(memory_regions, key=lambda x: x[0]) def _addr_in_exec_memory_regions(self, addr): diff --git a/tests/analyses/cfg/test_cfg_no_executable_regions.py b/tests/analyses/cfg/test_cfg_no_executable_regions.py index 3bacfa770..8c9df0bc6 100644 --- a/tests/analyses/cfg/test_cfg_no_executable_regions.py +++ b/tests/analyses/cfg/test_cfg_no_executable_regions.py @@ -4,6 +4,7 @@ from __future__ import annotations __package__ = __package__ or "tests.analyses.cfg" # pylint:disable=redefined-builtin +import logging import os import unittest @@ -19,14 +20,29 @@ class TestCfgNoExeutableRegions(unittest.TestCase): test_location, "x86_64", "windows", "65e25ea21a2f873affee8034e2c3381df48ff4129d447fa288fbd92307647582" ) p = angr.Project(bin_path) - cfg = p.analyses.CFG() + with self.assertLogs("angr.analyses.cfg.cfg_base", level=logging.WARNING) as logs: + cfg = p.analyses.CFG() assert len(cfg.kb.functions) == 0 + assert cfg.regions == [] + assert any("nothing to scan" in record for record in logs.output) + + def test_cfg_empty_executable_section_is_not_a_region(self): + # riscv-reloc-64-pic.o only carries data: its .text is SHF_EXECINSTR with sh_size 0, so it maps no bytes. + bin_path = os.path.join(test_location, "riscv64", "riscv-reloc-64-pic.o") + p = angr.Project(bin_path, auto_load_libs=False) + with self.assertLogs("angr.analyses.cfg.cfg_base", level=logging.WARNING) as logs: + cfg = p.analyses.CFGFast() + assert len(cfg.kb.functions) == 0 + assert cfg.regions == [] + assert any("nothing to scan" in record for record in logs.output) def test_cfg_elf_no_section_headers(self): # Regression test for #6409: stripped ELFs with no section headers fall back to segments. bin_path = os.path.join(test_location, "armel", "dbus-cleanup-sockets_stripped") p = angr.Project(bin_path) - cfg = p.analyses.CFG() + # Regions come from segments here, so there are bytes to scan. + with self.assertNoLogs("angr.analyses.cfg.cfg_base", level=logging.WARNING): + cfg = p.analyses.CFG() assert len(cfg.kb.functions) > 0 From 503b1be0663e65de225194eeb8e2cee6fa20d8d3 Mon Sep 17 00:00:00 2001 From: Fish Date: Fri, 14 Aug 2026 02:13:58 -0500 Subject: [PATCH 118/122] SLiveness: Drop the redundant successors/predecessors caches. (#6843) --- angr/analyses/s_liveness.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/angr/analyses/s_liveness.py b/angr/analyses/s_liveness.py index a4c2eb6d2..214a196af 100644 --- a/angr/analyses/s_liveness.py +++ b/angr/analyses/s_liveness.py @@ -74,9 +74,6 @@ class SLivenessAnalysis(Analysis): # blocks whose statements have been walked at least once walked: set[tuple[int, int | None]] = set() - successors = {block: list(graph.successors(block)) for block in graph} - predecessors = {block: list(graph.predecessors(block)) for block in graph} - order: list[Block] = list(networkx.dfs_postorder_nodes(graph, source=entry)) worklist = deque(order) worklist_set = set(order) @@ -91,7 +88,7 @@ class SLivenessAnalysis(Analysis): head_controlled_loop = is_head_controlled_loop_block(block) live = set() - for succ in successors[block]: + for succ in graph.successors(block): succ_key = succ.addr, succ.idx if head_controlled_loop and block_key == succ_key: # this is a head-controlled loop block; we ignore the self-loop edge because all variables defined @@ -176,7 +173,7 @@ class SLivenessAnalysis(Analysis): changed = True if changed: - for pred in predecessors[block]: + for pred in graph.predecessors(block): if pred not in worklist_set: worklist.append(pred) worklist_set.add(pred) From 55530509cebeafb633ade69149ad0b394a31a1e3 Mon Sep 17 00:00:00 2001 From: Yan Shoshitaishvili Date: Mon, 17 Aug 2026 05:55:58 -0700 Subject: [PATCH 119/122] SimLinux: Stop pre-growing the stack past address zero. (#6806) SimLinux.state_blank pre-grows the stack by a fixed 0x20 pages without checking that 0x20 pages exist beneath the stack pointer. When they do not, the allocation loop wraps past address 0 and hands out the remainder at the top of the address space. On x86-64 that is silent: blank_state(stack_end=0x10000) maps sixteen stack pages from 0x0 up and sixteen more from 0xfffffffffffff000 down. Where the wrap reaches a page the same call already handed out, the state fails with SimSegfaultException("stack collided with heap") instead, and where it reaches the loaded image it replaces it with blank pages. Skip the pre-grow when that much space does not exist. Clamping it to the space that does exist is not an option: the pre-allocated pages are not backed by the loader, so a stack that reaches down to an image beneath it hides that image. The pages that are skipped are still faulted in on demand. Also reject an allocation that does not fit beneath the top of the stack in allocate_stack_pages() itself, so a caller that asks for one gets an error rather than pages at the top of the address space. Co-authored-by: Claude Opus 5 --- angr/simos/linux.py | 4 +- .../paged_memory/stack_allocation_mixin.py | 3 + tests/simos/test_linux.py | 60 +++++++++++++++++++ tests/storage/test_memory.py | 16 +++++ 4 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 tests/simos/test_linux.py diff --git a/angr/simos/linux.py b/angr/simos/linux.py index b1ee22544..efb539908 100644 --- a/angr/simos/linux.py +++ b/angr/simos/linux.py @@ -216,7 +216,9 @@ class SimLinux(SimUserland): # pre-grow the stack by 0x20 pages. unsure if this is strictly required or just a hack around a compiler bug if not self._is_core and hasattr(state.memory, "allocate_stack_pages"): - state.memory.allocate_stack_pages(state.solver.eval(state.regs.sp) - 1, 0x20 * 0x1000) + sp = state.solver.eval(state.regs.sp) + if sp >= 0x20 * 0x1000: # ...but only when the stack has that much room beneath it + state.memory.allocate_stack_pages(sp - 1, 0x20 * 0x1000) if self.project.loader.tls.threads: tls_obj = self.project.loader.tls.threads[thread_idx if thread_idx is not None else 0] diff --git a/angr/storage/memory_mixins/paged_memory/stack_allocation_mixin.py b/angr/storage/memory_mixins/paged_memory/stack_allocation_mixin.py index 01f1542c4..d9332a4e3 100644 --- a/angr/storage/memory_mixins/paged_memory/stack_allocation_mixin.py +++ b/angr/storage/memory_mixins/paged_memory/stack_allocation_mixin.py @@ -44,6 +44,9 @@ class StackAllocationMixin(PagedMemoryMixin): pageno = addr // self.page_size if pageno != self._red_pageno: raise SimMemoryError("Trying to allocate stack space in a place that isn't the top of the stack") + if size > addr + 1: + # the loop below would wrap past address 0 and hand out pages at the top of memory instead + raise SimMemoryError("Trying to allocate more stack space than exists below the top of the stack") num = pageno - ((addr - size + 1) // self.page_size) + 1 result = [] diff --git a/tests/simos/test_linux.py b/tests/simos/test_linux.py new file mode 100644 index 000000000..f6493db13 --- /dev/null +++ b/tests/simos/test_linux.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# pylint: disable=protected-access +from __future__ import annotations + +__package__ = __package__ or "tests.simos" # pylint:disable=redefined-builtin + +import os +import unittest + +import archinfo + +import angr +from angr.errors import SimMemoryError +from tests.common import bin_location + +test_location = os.path.join(bin_location, "tests") + + +class TestSimLinuxStateBlank(unittest.TestCase): + """ + Tests for the stack that SimLinux.state_blank() pre-grows. + """ + + binary = os.path.join(test_location, "x86_64", "fauxware") + + def test_stack_is_pre_grown(self): + state = angr.Project(self.binary, auto_load_libs=False).factory.blank_state() + sp = state.solver.eval(state.regs.sp) + + assert set(state.memory._pages) == set(range((sp - 0x20 * 0x1000) // 0x1000, sp // 0x1000)) + + def test_stack_is_not_pre_grown_past_address_zero(self): + project = angr.Project(self.binary, auto_load_libs=False) + + # 0x20 pages is more room than this stack has beneath it + state = project.factory.blank_state(stack_end=0x10000) + + # none of them are pre-allocated, and in particular the excess does not wrap to the top of the address space + assert not state.memory._pages + with self.assertRaises(SimMemoryError): + state.memory.permissions(0xFFFFFFFFFFFFF000) + + def test_pre_grown_stack_does_not_cover_the_loaded_image(self): + # the same stack, with the image beneath it rather than above it + project = angr.Project( + self.binary, + main_opts={"backend": "blob", "arch": "AMD64", "base_addr": 0x1000, "entry_point": 0x1000}, + auto_load_libs=False, + simos="linux", + ) + + state = project.factory.blank_state(stack_end=0x10000) + + # pre-allocated stack pages are not backed by the loader, so clamping the pre-grow to the room that does + # exist would hide the image behind blank pages + assert state.solver.eval(state.memory.load(0x1000, 4, endness=archinfo.Endness.BE)) == 0x7F454C46 + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/storage/test_memory.py b/tests/storage/test_memory.py index 13394d537..ef81c329c 100755 --- a/tests/storage/test_memory.py +++ b/tests/storage/test_memory.py @@ -6,10 +6,12 @@ import time import unittest import claripy +from archinfo import ArchAMD64 from claripy.annotation import UninitializedAnnotation from angr import SIM_PROCEDURES, SimState from angr import options as o +from angr.errors import SimMemoryError from angr.state_plugins import SimLightRegisters, SimSystemPosix from angr.storage.file import SimFile from angr.storage.memory_mixins import ( @@ -752,6 +754,20 @@ class TestMemory(unittest.TestCase): state.memory.store(0xFFFFFFFF, symbol) assert state.memory.load(0, 1) is symbol[64 - 8 - 1 : 64 - 16] + def test_allocate_stack_pages_stops_at_address_zero(self): + state = SimState(arch=ArchAMD64(), stack_end=0x1000) + + # only one page exists below the top of this stack, so two of them do not fit under it + with self.assertRaises(SimMemoryError): + state.memory.allocate_stack_pages(0xFFF, 0x2000) + # in particular the second one is not handed out at the top of the address space + with self.assertRaises(SimMemoryError): + state.memory.permissions(0xFFFFFFFFFFFFF000) + + # the one page that does fit is still handed out + assert len(state.memory.allocate_stack_pages(0xFFF, 0x1000)) == 1 + assert state.memory.permissions(0) is not None + def test_underconstrained(self): state = SimState(arch="AMD64", add_options={o.UNDER_CONSTRAINED_SYMEXEC}) From 09e59e420417c72f66db9c898de83b838ca938a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 05:56:43 -0700 Subject: [PATCH 120/122] ci: bump taiki-e/install-action from 2.85.5 to 2.85.10 (#6799) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.85.5 to 2.85.10. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/6a1bd70eaac3c8bdf093356838d7ee09fda951cf...6c6fd71fe4fb72c3697d269963d0e15df8adedad) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.85.10 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 78a98bb8d..60d60edd9 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -99,7 +99,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1 - - uses: taiki-e/install-action@6a1bd70eaac3c8bdf093356838d7ee09fda951cf # v2 + - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2 with: tool: cargo-llvm-cov - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v5 From 2165b8e2a584fdabf50378094ce0ad0d70b4231e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 05:56:49 -0700 Subject: [PATCH 121/122] ci: bump Swatinem/rust-cache from 2.9.1 to 2.9.2 (#6801) Bumps [Swatinem/rust-cache](https://github.com/swatinem/rust-cache) from 2.9.1 to 2.9.2. - [Release notes](https://github.com/swatinem/rust-cache/releases) - [Changelog](https://github.com/Swatinem/rust-cache/blob/master/CHANGELOG.md) - [Commits](https://github.com/swatinem/rust-cache/compare/c19371144df3bb44fab255c43d04cbc2ab54d1c4...6323deb102c322ba6fcbdcafc7e3dddab59af2b6) --- updated-dependencies: - dependency-name: Swatinem/rust-cache dependency-version: 2.9.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/coverage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 60d60edd9..fbc006956 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v6 - name: Restore test durations cache uses: actions/cache/restore@v6 @@ -97,7 +97,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1 - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2 with: From a8a5cabdfaff3962e375bbc35d03996346d675f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 06:22:01 -0700 Subject: [PATCH 122/122] rust: bump pyo3 from 0.29.0 to 0.29.2 (#6800) Bumps [pyo3](https://github.com/pyo3/pyo3) from 0.29.0 to 0.29.2. - [Release notes](https://github.com/pyo3/pyo3/releases) - [Changelog](https://github.com/PyO3/pyo3/blob/main/CHANGELOG.md) - [Commits](https://github.com/pyo3/pyo3/compare/v0.29.0...v0.29.2) --- updated-dependencies: - dependency-name: pyo3 dependency-version: 0.29.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c61d6df5e..84fd2d76b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1257,9 +1257,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ "inventory", "libc", @@ -1274,18 +1274,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" dependencies = [ "libc", "pyo3-build-config", @@ -1293,9 +1293,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1305,9 +1305,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" dependencies = [ "heck", "proc-macro2",