denuvo-ref/VTIL-Core
1
0
Fork 0
mirror of https://github.com/vtil-project/VTIL-Core synced 2026-08-17 08:23:03 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implemented O(1) tree matching approx with compressed signatures.

Browse source
This commit is contained in:
Can Bölük 2020-07-25 06:37:34 +02:00
parent 58afba5c86
commit d472f2f8eb
10 changed files with 252 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

2
VTIL-SymEx/VTIL-SymEx.vcxproj
View file

@ -113,6 +113,7 @@
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="directives\directive.cpp" />
<ClCompile Include="directives\expression_signature.cpp" />
<ClCompile Include="directives\transformer.cpp" />
<ClCompile Include="expressions\expression.cpp" />
<ClCompile Include="expressions\unique_identifier.cpp" />
@ -121,6 +122,7 @@
</ItemGroup>
<ItemGroup>
<ClInclude Include="directives\directive.hpp" />
<ClInclude Include="directives\expression_signature.hpp" />
<ClInclude Include="directives\fast_matcher.hpp" />
<ClInclude Include="directives\transformer.hpp" />
<ClInclude Include="expressions\expression.hpp" />

6
VTIL-SymEx/VTIL-SymEx.vcxproj.filters
View file

@ -19,6 +19,9 @@
<ClCompile Include="simplifier\boolean_directives.cpp">
<Filter>Simplifier</Filter>
</ClCompile>
<ClCompile Include="directives\expression_signature.cpp">
<Filter>Directives</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="directives\directive.hpp">
@ -48,6 +51,9 @@
<ClInclude Include="simplifier\boolean_directives.hpp">
<Filter>Simplifier</Filter>
</ClInclude>
<ClInclude Include="directives\expression_signature.hpp">
<Filter>Directives</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Filter Include="Directives">

10
VTIL-SymEx/directives/directive.cpp
View file

@ -29,16 +29,6 @@
namespace vtil::symbolic::directive
{
// Constructor for directive representing the result of an unary operator.
//
instance::instance( math::operator_id op, const instance& e1 )
: rhs( e1 ), op( op ) {}
// Constructor for directive representing the result of a binary operator.
//
instance::instance( const instance& e1, math::operator_id op, const instance& e2 )
: lhs( e1 ), rhs( e2 ), op( op ) {}
// Enumerates each unique variable.
//
void instance::enum_variables( const std::function<void( const instance& )>& fn, std::unordered_set<const char*>* s ) const

27
VTIL-SymEx/directives/directive.hpp
View file

@ -30,6 +30,7 @@
#include <vtil/utility>
#include <type_traits>
#include <unordered_set>
#include "expression_signature.hpp"
namespace vtil::symbolic::directive
{
@ -197,6 +198,10 @@ namespace vtil::symbolic::directive
reference lhs = {};
reference rhs = {};
// Signature of the directive for each possible size.
//
std::array<expression_signature, 64> signatures = {};
// Default/copy/move constructors.
//
instance() {};
@ -208,17 +213,31 @@ namespace vtil::symbolic::directive
// Variable constructor.
//
template<typename T = uint64_t, std::enable_if_t<std::is_integral_v<T>, int> = 0>
instance( T value ) : operable( int64_t( value ) ) {}
instance( T v ) : operable( int64_t( v ) )
{
for ( auto [out, idx] : zip( signatures, iindices ) )
out = { make_copy( value ).resize( math::narrow_cast<bitcnt_t>( idx + 1 ) ) };
}
instance( const char* id, int lookup_index, matching_type mtype = match_any ) :
id( id ), lookup_index( lookup_index ), mtype( mtype ) { }
id( id ), lookup_index( lookup_index ), mtype( mtype ) {}
// Constructor for directive representing the result of an unary operator.
//
instance( math::operator_id _op, const instance& e1 );
instance( math::operator_id op, const instance& e1 ) :
rhs( e1 ), op( op )
{
for ( auto [out, rhs] : zip( signatures, e1.signatures ) )
out = { op, rhs };
}
// Constructor for directive representing the result of a binary operator.
//
instance( const instance& e1, math::operator_id _op, const instance& e2 );
instance( const instance& e1, math::operator_id op, const instance& e2 ) :
lhs( e1 ), rhs( e2 ), op( op )
{
for ( auto [lhs, out, rhs] : zip( e1.signatures, signatures, e2.signatures ) )
out = { lhs, op, rhs };
}
// Enumerates each unique variable.
//

124
VTIL-SymEx/directives/expression_signature.cpp Normal file
View file

@ -0,0 +1,124 @@
// Copyright (c) 2020 Can Boluk and contributors of the VTIL Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// 3. Neither the name of VTIL Project nor the names of its contributors
// may be used to endorse or promote products derived from this software
// without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//
#include "expression_signature.hpp"
namespace vtil::symbolic
{
// Declare number of bits used to save operator id.
// -- Intellisense has a hard time bitscanning apparently (kills it across entire project) so yeah...
//
#ifdef __INTELLISENSE__
static constexpr bitcnt_t num_operator_bits = 6;
#else
static constexpr bitcnt_t num_operator_bits = math::msb( ( uint64_t ) math::operator_id::max );
#endif
// Declare shrinking factor, determines how many bits are conserved from grand-child nodes.
//
static constexpr bitcnt_t shrink_to = ( 64 - num_operator_bits ) / 2;
// Extend from N bits into 64 bits.
//
template<bitcnt_t N>
static constexpr uint64_t extend( uint64_t i )
{
constexpr bitcnt_t middle_original = 64 / 2;
constexpr bitcnt_t middle_new = N / 2;
constexpr bitcnt_t shl_n = middle_original - middle_new;
return i << shl_n;
}
static constexpr uint64_t extend( math::operator_id o )
{
return extend<num_operator_bits>( ( uint64_t ) o );
}
// Shrink from 64 bits into N bits.
//
template<bitcnt_t N>
static constexpr uint64_t shrink( uint64_t i )
{
constexpr bitcnt_t middle_original = 64 / 2;
constexpr bitcnt_t middle_new = N / 2;
constexpr bitcnt_t shr_n = middle_original - middle_new;
constexpr bitcnt_t shl_n = 64 - shr_n;
constexpr bitcnt_t mask = math::fill( N );
i |= i >> shl_n;
i |= i << shl_n;
return ( ( i >> shr_n ) | ( i << shl_n ) ) & mask;
}
template<bitcnt_t N>
static constexpr uint64_t shrink( const expression_signature& sig )
{
return shrink<N>( sig.signature[ 0 ] ) | sig.signature[ 1 ] | ( shrink<N>( sig.signature[ 2 ] ) << ( 64 - N ) );
}
// Rebalance I64 so that middle is LSB.
//
static constexpr uint64_t rebalance( uint64_t i ) { return ( i >> 32 ) | ( i << 32 ); }
// Declare constructors.
//
expression_signature::expression_signature( const math::bit_vector& value )
{
// Write rebalanced integer.
//
signature[ 0 ] = 0;
signature[ 1 ] = rebalance( value.known_one() );
signature[ 2 ] = 0;
}
expression_signature::expression_signature( math::operator_id op, const expression_signature& rhs )
{
// Write [rhs, op, rhs].
//
signature[ 0 ] = shrink<shrink_to>( rhs );
signature[ 1 ] = extend( op );
signature[ 2 ] = signature[ 0 ];
}
expression_signature::expression_signature( const expression_signature& lhs, math::operator_id op, const expression_signature& rhs )
{
// Skip if invalid operator.
//
if ( op >= math::operator_id::max )
{
signature.fill( 0 );
return;
}
// Write [lhs, op, rhs].
//
signature[ 0 ] = shrink<shrink_to>( lhs );
signature[ 1 ] = extend( op );
signature[ 2 ] = shrink<shrink_to>( rhs );
// Or both sides with each other if commutative.
//
if ( math::descriptor_of( op ).is_commutative )
signature[ 2 ] = ( signature[ 0 ] |= signature[ 2 ] );
}
};

73
VTIL-SymEx/directives/expression_signature.hpp Normal file
View file

@ -0,0 +1,73 @@
// Copyright (c) 2020 Can Boluk and contributors of the VTIL Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// 3. Neither the name of VTIL Project nor the names of its contributors
// may be used to endorse or promote products derived from this software
// without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//
#pragma once
#include <vtil/math>
#include <vtil/utility>
#include <vtil/io>
#include <array>
namespace vtil::symbolic
{
// This class allows O(1) approximation of tree-matching by storing a
// compressed signature.
//
struct expression_signature : reducable<expression_signature>
{
// Signature itself.
//
std::array<uint64_t, 3> signature;
// Declare constructors.
//
expression_signature() {}
expression_signature( const math::bit_vector& value );
expression_signature( math::operator_id op, const expression_signature& rhs );
expression_signature( const expression_signature& lhs, math::operator_id op, const expression_signature& rhs );
// Default copy/move.
//
expression_signature( expression_signature&& ) = default;
expression_signature( const expression_signature& ) = default;
expression_signature& operator=( expression_signature&& ) = default;
expression_signature& operator=( const expression_signature& ) = default;
// Checks if RHS can match into LHS.
//
bool can_match( const expression_signature& o ) const
{
for ( auto [a, b] : zip( signature, o.signature ) )
if ( ( a & b ) != b )
return false;
return true;
}
// Declare reduction.
//
REDUCE_TO( signature );
};
};

8
VTIL-SymEx/directives/transformer.hpp
View file

@ -54,10 +54,16 @@ namespace vtil::symbolic
{
using namespace logger;
// Fast path: check if signature matches.
//
if ( !exp->signature.can_match( from->signatures[ exp->size() - 1 ] ) )
return {};
// Match the expresison.
//
stack_vector<directive::symbol_table_t, 8> results;
if ( !directive::fast_match( &results, from, exp ) ) return {};
if ( !directive::fast_match( &results, from, exp ) )
return {};
// If a filter is provided:
//

15
VTIL-SymEx/expressions/expression.cpp
View file

@ -453,6 +453,10 @@ namespace vtil::symbolic
out = ( hash_value ^ key ) & value.value_mask();
}
// Set the signature.
//
signature = { value };
// Set simplification state.
//
simplify_hint = true;
@ -668,11 +672,14 @@ namespace vtil::symbolic
for ( auto [out, idx] : zip( xvalues, iindices ) )
{
if ( lhs )
out = math::evaluate( op, lhs->size(), lhs->xvalues[ idx ], rhs->size(), rhs->xvalues[ idx ] & rhs_mask ).first;
else
out = math::evaluate( op, 0, 0, rhs->size(), rhs->xvalues[ idx ] & rhs_mask ).first;
if ( lhs ) out = math::evaluate( op, lhs->size(), lhs->xvalues[ idx ], rhs->size(), rhs->xvalues[ idx ] & rhs_mask ).first;
else out = math::evaluate( op, 0, 0, rhs->size(), rhs->xvalues[ idx ] & rhs_mask ).first;
}
// Set the signature.
//
if ( lhs ) signature = { lhs->signature, op, rhs->signature };
else signature = { op, rhs->signature };
// If auto simplification is relevant, invoke it.
//

6
VTIL-SymEx/expressions/expression.hpp
View file

@ -30,7 +30,7 @@
#include <vtil/utility>
#include <set>
#include "unique_identifier.hpp"
#include "../directives/expression_signature.hpp"
// [Configuration]
// Determine the number of x value keys we use to estimate values.
@ -211,6 +211,10 @@ namespace vtil::symbolic
//
hash_t hash_value = {};
// Signature of the expression.
//
expression_signature signature = {};
// Whether expression passed the simplifier already or not, note that this is a hint and there may
// be cases where it already has passed it and this flag was not set. Albeit those cases will most
// likely not cause performance issues due to the caching system.

1
VTIL-SymEx/includes/vtil/symex
View file

@ -4,5 +4,6 @@
#include "../../simplifier/simplifier.hpp"
#include "../../simplifier/directives.hpp"
#include "../../directives/directive.hpp"
#include "../../directives/expression_signature.hpp"
#include "../../directives/transformer.hpp"
#include "../../directives/fast_matcher.hpp"