Parse GNU C++ mangled function names and extract types. (#2215)

* Parses GNU C++ mangled function names and extract types.

* Lint the code.
This commit is contained in:
Fish 2020-06-24 10:37:32 -07:00 committed by GitHub
parent 5952fb5163
commit c14bc39aac
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 433 additions and 20 deletions

View file

@ -1430,8 +1430,7 @@ class Function(Serializable):
if not library.has_prototype(self.name):
return
proto = library.prototypes[self.name]
proto = library.get_prototype(self.name)
self.prototype = proto
if self.calling_convention is not None:
self.calling_convention.args = None

View file

@ -4,10 +4,12 @@ import archinfo
from collections import defaultdict
import logging
import inspect
from typing import Optional, Dict
import itanium_demangler
from ...calling_conventions import DEFAULT_CC
from ...sim_type import parse_cpp_file, SimTypeFunction
from ...calling_conventions import SimCC, DEFAULT_CC
from ...misc import autoimport
from ...sim_type import parse_file
from ..stubs.ReturnUnconstrained import ReturnUnconstrained
@ -32,7 +34,7 @@ class SimLibrary:
def __init__(self):
self.procedures = {}
self.non_returning = set()
self.prototypes = {}
self.prototypes: Dict[str,SimTypeFunction] = {}
self.default_ccs = {}
self.names = []
self.fallback_cc = dict(DEFAULT_CC)
@ -172,12 +174,12 @@ class SimLibrary:
def _apply_metadata(self, proc, arch):
if proc.cc is None and arch.name in self.default_ccs:
proc.cc = self.default_ccs[arch.name](arch)
if proc.cc.func_ty is not None:
if proc.cc.func_ty is not None and proc.cc.func_ty.arg_names:
# Use inspect to extract the parameters from the run python function
proc.cc.func_ty.arg_names = inspect.getfullargspec(proc.run).args[1:]
if proc.cc is None and arch.name in self.fallback_cc:
proc.cc = self.fallback_cc[arch.name](arch)
if proc.display_name in self.prototypes:
if proc.cc is None:
proc.cc = self.fallback_cc[arch.name](arch)
proc.cc.func_ty = self.prototypes[proc.display_name].with_arch(arch)
# Use inspect to extract the parameters from the run python function
proc.cc.func_ty.arg_names = inspect.getfullargspec(proc.run).args[1:]
@ -221,6 +223,21 @@ class SimLibrary:
self._apply_metadata(proc, arch)
return proc
def get_prototype(self, name: str, arch=None) -> Optional[SimTypeFunction]:
"""
Get a prototype of the given function name, optionally specialize the prototype to a given architecture.
:param name: Name of the function.
:param arch: The architecture to specialize to.
:return: Prototype of the function, or None if the prototype does not exist.
"""
proto = self.prototypes.get(name, None)
if proto is None:
return None
if arch is not None:
return proto.with_arch(arch)
return proto
def has_metadata(self, name):
"""
Check if a function has either an implementation or any metadata associated with it
@ -267,6 +284,25 @@ class SimCppLibrary(SimLibrary):
return str(ast)
return name
@staticmethod
def _proto_from_demangled_name(name: str) -> Optional[SimCC]:
"""
Attempt to extract arguments and calling convention information for a C++ function whose name was mangled
according to the Itanium C++ ABI symbol mangling language.
:param name: The demangled function name.
:return: A calling convention or None if a calling convention cannot be found.
"""
try:
parsed, _ = parse_cpp_file(name, with_param_names=False)
except ValueError:
return None
if not parsed:
return None
_, func_proto = next(iter(parsed.items()))
return func_proto
def get(self, name, arch):
"""
Get an implementation of the given function specialized for the given arch, or a stub procedure if none exists.
@ -276,8 +312,10 @@ class SimCppLibrary(SimLibrary):
:param arch: The architecure to use, as either a string or an archinfo.Arch instance
:return: A SimProcedure instance representing the function as found in the library
"""
name = self._try_demangle(name)
return super().get(name, arch)
demangled_name = self._try_demangle(name)
if demangled_name not in self.procedures:
return self.get_stub(name, arch) # get_stub() might use the mangled name to derive the function prototype
return super().get(demangled_name, arch)
def get_stub(self, name, arch):
"""
@ -289,8 +327,28 @@ class SimCppLibrary(SimLibrary):
:param arch: The architecture to use, as either a string or an archinfo.Arch instance
:return: A SimProcedure instance representing a plausable stub as could be found in the library.
"""
name = self._try_demangle(name)
return super().get_stub(name, arch)
demangled_name = self._try_demangle(name)
stub = super().get_stub(demangled_name, arch)
# try to determine a prototype from the function name if possible
if demangled_name != name:
# itanium-mangled function name
stub.cc.func_ty = self._proto_from_demangled_name(demangled_name)
if stub.cc.func_ty is not None and not stub.ARGS_MISMATCH:
stub.cc.num_args = len(stub.cc.func_ty.args)
stub.num_args = len(stub.cc.func_ty.args)
return stub
def get_prototype(self, name: str, arch=None) -> Optional[SimTypeFunction]:
"""
Get a prototype of the given function name, optionally specialize the prototype to a given architecture. The
function name will be demangled first.
:param name: Name of the function.
:param arch: The architecture to specialize to.
:return: Prototype of the function, or None if the prototype does not exist.
"""
demangled_name = self._try_demangle(name)
return super().get_prototype(demangled_name, arch=arch)
def has_metadata(self, name):
"""

View file

@ -1,4 +1,5 @@
from ...sim_type import SimTypeFunction, SimTypePointer, SimTypeChar, SimTypeBottom
from .. import SIM_PROCEDURES as P
from . import SimCppLibrary
@ -7,3 +8,14 @@ libstdcpp = SimCppLibrary()
libstdcpp.set_library_names('libstdc++.so', 'libstdc++.so.6')
libstdcpp.add_all_from_dict(P["libstdcpp"])
_decls = {
"std::__throw_logic_error(char const*)": SimTypeFunction([SimTypePointer(SimTypeChar())], SimTypeBottom(label="void"), arg_names=("error",)),
"std::__throw_length_error(char const*)": SimTypeFunction([SimTypePointer(SimTypeChar())], SimTypeBottom(label="void"), arg_names=("error",)),
}
for name, proto in _decls.items():
if proto is not None:
libstdcpp.set_prototype(name, proto)

View file

@ -10,6 +10,6 @@ class std____throw_logic_error(angr.SimProcedure): #pylint:disable=redefined-bui
NO_RET = True
ALT_NAMES = ('std::__throw_length_error(char const*)', )
def run(self):
def run(self, error): # pylint:disable=unused-argument
# FIXME: we need the concept of C++ exceptions to implement this right
self.exit(1)

View file

@ -10,6 +10,6 @@ class std____throw_logic_error(angr.SimProcedure): #pylint:disable=redefined-bui
NO_RET = True
ALT_NAMES = ('std::__throw_logic_error(char const*)', )
def run(self):
def run(self, error): # pylint:disable=unused-argument
# FIXME: we need the concept of C++ exceptions to implement this right
self.exit(1)

View file

@ -5,6 +5,9 @@ import angr
######################################
class ReturnUnconstrained(angr.SimProcedure):
ARGS_MISMATCH = True
def run(self, *args, **kwargs): #pylint:disable=arguments-differ
#pylint:disable=attribute-defined-outside-init

View file

@ -3,7 +3,7 @@ from .misc.ux import deprecated
import copy
import re
import logging
from typing import Optional
from typing import Optional, Dict, Any, Tuple, List
import claripy
@ -18,6 +18,11 @@ try:
except ImportError:
pycparser = None
try:
import CppHeaderParser
except ImportError:
CppHeaderParser = None
class SimType:
"""
@ -461,6 +466,44 @@ class SimTypePointer(SimTypeReg):
)
class SimTypeReference(SimTypeReg):
"""
SimTypeReference is a type that specifies a reference to some other type.
"""
def __init__(self, refs, label=None):
super().__init__(None, label=label)
self.refs: SimType = refs
def __repr__(self):
return "{}&".format(self.refs)
def c_repr(self):
return "{}&".format(self.refs.c_repr())
def make(self, refs):
new = type(self)(refs)
new._arch = self._arch
return new
@property
def size(self):
if self._arch is None:
raise ValueError("Can't tell my size without an arch!")
return self._arch.bits
def _with_arch(self, arch):
out = SimTypeReference(self.refs.with_arch(arch), label=self.label)
out._arch = arch
return out
def _init_str(self):
return "%s(%s%s)" % (
self.__class__.__name__,
self.refs._init_str(),
(', label="%s"' % self.label) if self.label is not None else "",
)
class SimTypeFixedSizeArray(SimType):
"""
SimTypeFixedSizeArray is a literal (i.e. not a pointer) fixed-size array.
@ -682,7 +725,7 @@ class SimTypeFunction(SimType):
super(SimTypeFunction, self).__init__(label=label)
self.args = args
self.returnty: Optional[SimType] = returnty
self.arg_names = arg_names if arg_names else []
self.arg_names = arg_names if arg_names else ()
self.variadic = variadic
def __repr__(self):
@ -724,6 +767,39 @@ class SimTypeFunction(SimType):
)
class SimTypeCppFunction(SimTypeFunction):
"""
SimTypeCppFunction is a type that specifies an actual C++-style function with information about arguments, return
value, and more C++-specific properties.
:ivar ctor: Whether the function is a constructor or not.
:ivar dtor: Whether the function is a destructor or not.
"""
def __init__(self, args, returnty, label=None, arg_names: Tuple[str]=None, ctor: bool=False, dtor: bool=False):
super().__init__(args, returnty, label=label, arg_names=arg_names, variadic=False)
self.ctor = ctor
self.dtor = dtor
def __repr__(self):
argstrs = [str(a) for a in self.args]
if self.variadic:
argstrs.append('...')
return '({}) -> {}'.format(', '.join(argstrs), self.returnty)
def c_repr(self):
return '({}) -> {}'.format(', '.join(str(a) for a in self.args), self.returnty)
def _init_str(self):
return "%s([%s], %s%s%s%s)" % (
self.__class__.__name__,
", ".join([arg._init_str() for arg in self.args]),
self.returnty,
(", label=%s" % self.label) if self.label else "",
(", arg_names=[%s]" % self._arg_names_str(show_variadic=False)) if self.arg_names else "",
", variadic=True" if self.variadic else "",
)
class SimTypeLength(SimTypeLong):
"""
SimTypeLength is a type that specifies the length of some buffer in memory.
@ -1003,6 +1079,40 @@ class SimUnion(SimType):
return out
class SimCppClass(SimStruct):
def __init__(self, members: Dict[str,SimStruct], name: Optional[str]=None, pack: bool=False, align=None):
super().__init__(members, name=name, pack=pack, align=align)
class SimCppNamespaced(SimType):
"""
Describes a type within a namespace.
"""
def __init__(self, namespace: str, type_: SimType, label: Optional[str]=None):
super().__init__(label=label)
self.namespace = namespace
self.type = type_
def __repr__(self):
return "{}::{}".format(self.namespace, self.type)
def c_repr(self):
return "{}::{}".format(self.namespace, self.type.c_repr())
def _with_arch(self, arch):
out = SimCppNamespaced(self.namespace, self.type.with_arch(arch), label=self.label)
out._arch = arch
return out
def _init_str(self):
return "%s(%s, %s%s)" % (
self.__class__.__name__,
self.namespace,
self.type._init_str(),
(', label="%s"' % self.label) if self.label is not None else "",
)
BASIC_TYPES = {
'char': SimTypeChar(),
'signed char': SimTypeChar(),
@ -1065,7 +1175,11 @@ ALL_TYPES = {
'string': SimTypeString(),
'wstring': SimTypeWString(),
'va_list': SimStruct({}, name='va_list')
'va_list': SimStruct({}, name='va_list'),
# C++-specific
'basic_string': SimTypeString(),
'CharT': SimTypeChar(),
}
@ -1398,6 +1512,141 @@ def _parse_const(c):
else:
raise ValueError(c)
def _cpp_decl_to_type(decl: Any, extra_types: Dict[str,SimType], opaque_classes=True):
if isinstance(decl, CppHeaderParser.CppMethod):
the_func = decl
func_name = the_func['name']
if "__deleting_dtor__" in func_name:
the_func['destructor'] = True
elif "__base_dtor__" in func_name:
the_func['destructor'] = True
elif "__dtor__" in func_name:
the_func['destructor'] = True
# translate parameters
args = [ ]
arg_names: List[str] = [ ]
for param in the_func['parameters']:
arg_type = param['type']
args.append(_cpp_decl_to_type(arg_type, extra_types, opaque_classes=opaque_classes))
arg_name = param['name']
arg_names.append(arg_name)
args = tuple(args)
arg_names: Tuple[str] = tuple(arg_names)
# returns
if not the_func['returns'].strip():
returnty = SimTypeBottom()
else:
returnty = _cpp_decl_to_type(the_func['returns'].strip(), extra_types, opaque_classes=opaque_classes)
# other properties
ctor = the_func['constructor']
dtor = the_func['destructor']
func = SimTypeCppFunction(args, returnty, arg_names=arg_names, ctor=ctor, dtor=dtor)
return func
elif isinstance(decl, str):
# a string that represents type
if decl.endswith("&"):
# reference
subdecl = decl.rstrip("&").strip()
subt = _cpp_decl_to_type(subdecl, extra_types, opaque_classes=opaque_classes)
t = SimTypeReference(subt)
return t
if decl.endswith(" const"):
# drop const
return _cpp_decl_to_type(decl[:-6].strip(), extra_types, opaque_classes=opaque_classes)
# namespaced?
if "::" in decl:
splitted = decl.split("::")
subt = _cpp_decl_to_type(splitted[-1], extra_types, opaque_classes=opaque_classes)
if len(splitted) > 1:
# namespaced!
for namespace in reversed(splitted[:-1]):
subt = SimCppNamespaced(namespace, subt)
return subt
key = decl
if key in extra_types:
return extra_types[key]
elif key in ALL_TYPES:
return ALL_TYPES[key]
elif opaque_classes is True:
# create a class without knowing the internal members
return SimCppClass({}, name=decl)
else:
raise TypeError("Unknown type '%s'" % ' '.join(key))
raise NotImplementedError()
def parse_cpp_file(cpp_decl, with_param_names: bool=False):
#
# A series of hacks to make CppHeaderParser happy with whatever C++ function prototypes we feed in
#
if CppHeaderParser is None:
raise ImportError("Please install CppHeaderParser to parse C++ definitions")
# CppHeaderParser does not support specialization
_s = cpp_decl
s = None
while s != _s:
_s = s if s is not None else _s
s = re.sub(r"<[^<>]+>", "", _s)
m = re.search(r"{([a-z\s]+)}", s)
if m is not None:
s = s[:m.start()] + "__" + m.group(1).replace(" ", "_") + "__" + s[m.end():]
# CppHeaderParser does not like missing parameter names
# FIXME: The following logic is only dealing with *one* C++ function declaration. Support multiple declarations
# FIXME: when needed in the future.
if not with_param_names:
last_pos = 0
i = 0
while True:
idx = s.find(",", last_pos)
if idx == -1:
break
arg_name = "a%d" % i
i += 1
s = s[:idx] + " " + arg_name + s[idx:]
last_pos = idx + len(arg_name) + 1 + 1
# the last parameter
idx = s.find(")", last_pos)
if idx != -1:
# TODO: consider the case where there are one or multiple spaces between ( and )
if s[idx - 1] != "(":
arg_name = "a%d" % i
s = s[:idx] + " " + arg_name + s[idx:]
# CppHeaderParser does not like missing function body
s += "\n\n{}"
h = CppHeaderParser.CppHeader(s, argType="string")
if not h.functions:
return None, None
func_decls: Dict[str,SimTypeCppFunction] = { }
for the_func in h.functions:
# FIXME: We always assume that there is a "this" pointer but it is not the case for static methods.
proto: Optional[SimTypeCppFunction] = _cpp_decl_to_type(the_func, {}, opaque_classes=True)
if proto is not None and the_func['class']:
func_name = the_func['class'] + "::" + the_func['name']
proto.args = (SimTypePointer(pts_to=SimTypeBottom(label="void")),) + proto.args # pylint:disable=attribute-defined-outside-init
proto.arg_names = ("this",) + proto.arg_names # pylint:disable=attribute-defined-outside-init
else:
func_name = the_func['name']
func_decls[func_name] = proto
return func_decls, { }
if pycparser is not None:
_accepts_scope_stack()

View file

@ -1,5 +1,6 @@
from typing import Tuple, Optional
from ..sim_type import parse_file
from ..sim_type import parse_file, parse_cpp_file, SimTypeCppFunction
def get_function_name(s):
@ -73,6 +74,41 @@ def convert_cproto_to_py(c_decl):
return func_name, func_proto, "\n".join(s)
def convert_cppproto_to_py(cpp_decl: str,
with_param_names: bool=False) -> Tuple[Optional[str],Optional[SimTypeCppFunction],Optional[str]]:
"""
Pre-process a C++-style function declaration string to its corresponding SimTypes-based Python representation.
:param cpp_decl: The C++-style function declaration string.
:return: A tuple of the function name, the prototype, and a string representing the SimType-based Python
representation.
"""
s = [ ]
try:
s.append("# %s" % cpp_decl)
parsed = parse_cpp_file(cpp_decl, with_param_names=with_param_names)
parsed_decl = parsed[0]
if not parsed_decl:
raise ValueError("Cannot parse the function prototype.")
func_name, func_proto = next(iter(parsed_decl.items()))
s.append('"%s": %s,' % (func_name, func_proto._init_str())) # The real Python string
except Exception: # pylint:disable=broad-except
try:
func_name = get_function_name(cpp_decl)
func_proto = None
s.append('"%s": None,' % func_name)
except ValueError:
# Failed to extract the function name. Is it a function declaration?
func_name, func_proto = None, None
return func_name, func_proto, "\n".join(s)
def cprotos2py(cprotos):
"""
Parse a list of C function declarations and output to Python code that can be embedded into

View file

@ -158,6 +158,7 @@ setup(
'psutil',
'pycparser>=2.18',
'itanium_demangler',
'CppHeaderParser',
'protobuf',
],
setup_requires=[_UNICORN, 'pyvex'],

View file

@ -1,10 +1,13 @@
# pylint:disable=unused-variable
import nose
import claripy
import angr
from angr.sim_type import SimTypeFunction, SimTypeInt, SimTypePointer, SimTypeChar, SimStruct, SimTypeFloat, SimUnion, SimTypeDouble, SimTypeLongLong, SimTypeLong, SimTypeNum
from angr.utils.library import convert_cproto_to_py
from angr.sim_type import (SimTypeFunction, SimTypeInt, SimTypePointer, SimTypeChar, SimStruct, SimTypeFloat, SimUnion,
SimTypeDouble, SimTypeLongLong, SimTypeLong, SimTypeNum, SimTypeReference, SimTypeBottom,
SimTypeString, SimCppNamespaced)
from angr.utils.library import convert_cproto_to_py, convert_cppproto_to_py
def test_type_annotation():
@ -49,6 +52,57 @@ def test_cproto_conversion():
nose.tools.assert_equal(pyproto_name, "foo")
def test_cppproto_conversion():
# a demangled class constructor prototype, without parameter names
proto_0 = "std::basic_ifstream<char, std::char_traits<char>>::{ctor}(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&, std::_Ios_Openmode)"
name, proto, s = convert_cppproto_to_py(proto_0, with_param_names=False)
assert proto.ctor is True
assert name == "std::basic_ifstream::__ctor__"
assert len(proto.args) == 3
assert isinstance(proto.args[0], SimTypePointer) # this
assert isinstance(proto.args[1], SimTypeReference)
assert isinstance(proto.args[1].refs, SimCppNamespaced)
assert proto.args[1].refs.namespace == "std"
assert isinstance(proto.args[1].refs.type, SimCppNamespaced)
assert proto.args[1].refs.type.namespace == "__cxx11"
assert isinstance(proto.args[1].refs.type.type, SimTypeString)
proto_1 = "void std::basic_string<CharT,Traits,Allocator>::push_back(CharT ch)"
name, proto, s = convert_cppproto_to_py(proto_1, with_param_names=True)
assert name == "std::basic_string::push_back"
assert isinstance(proto.returnty, SimTypeBottom)
assert isinstance(proto.args[0], SimTypePointer) # this
assert isinstance(proto.args[1], SimTypeChar)
proto_2 = "void std::basic_string<CharT,Traits,Allocator>::swap(basic_string& other)"
name, proto, s = convert_cppproto_to_py(proto_2, with_param_names=True)
assert name == "std::basic_string::swap"
assert isinstance(proto.returnty, SimTypeBottom)
assert isinstance(proto.args[0], SimTypePointer) # this
assert isinstance(proto.args[1], SimTypeReference)
assert isinstance(proto.args[1].refs, SimTypeString)
proto_3 = "std::ios_base::{base dtor}()"
name, proto, s = convert_cppproto_to_py(proto_3, with_param_names=True)
assert name == "std::ios_base::__base_dtor__"
assert proto.dtor is True
assert isinstance(proto.returnty, SimTypeBottom)
proto_4 = "std::ios_base::{base dtor}()"
name, proto, s = convert_cppproto_to_py(proto_4, with_param_names=True)
assert name == "std::ios_base::__base_dtor__"
proto_5 = "void foo(int & bar);"
name, proto, s = convert_cppproto_to_py(proto_5, with_param_names=True)
assert name == "foo"
# note that there is no "this" pointer
assert isinstance(proto.args[0], SimTypeReference)
assert isinstance(proto.args[0].refs, SimTypeInt)
assert isinstance(proto.returnty, SimTypeBottom)
def test_struct_deduplication():
angr.types.register_types(angr.types.parse_type('struct ahdr { int a ;}'))
angr.types.register_types(angr.types.parse_type('struct bhdr { int b ;}'))
@ -176,7 +230,7 @@ def test_arg_names():
fdef = angr.types.parse_defns("int f();") # type: Dict[str, SimTypeFunction]
sig = fdef['f']
nose.tools.assert_equal(sig.arg_names, [])
nose.tools.assert_equal(sig.arg_names, ())
def test_varargs():
fdef = angr.types.parse_defns("int printf(const char *fmt, ...);")
@ -192,6 +246,7 @@ def test_varargs():
if __name__ == '__main__':
test_type_annotation()
test_cproto_conversion()
test_cppproto_conversion()
test_struct_deduplication()
test_parse_type()
test_parse_type_no_basic_types()