mirror of
https://github.com/schnaader/precomp-cpp
synced 2026-08-23 00:23:04 -04:00
Update preflate to v0.2.1
This commit is contained in:
Deus Libri
parent
48b114b1ef
commit
0cdefd8561
52 changed files with 991 additions and 5235 deletions
|
|
@ -85,13 +85,17 @@ if (MSVC)
|
|||
add_definitions(-D_UNICODE -DUNICODE)
|
||||
endif (MSVC)
|
||||
|
||||
if ("${CMAKE_SIZEOF_VOID_P}" EQUAL "8")
|
||||
add_definitions(-DBIT64)
|
||||
endif ("${CMAKE_SIZEOF_VOID_P}" EQUAL "8")
|
||||
|
||||
|
||||
set(PREFLATE_SRC "")
|
||||
add_stem2file(PREFLATE_SRC "${SRCDIR}/contrib/preflate/preflate_%STEM%.cpp"
|
||||
"block_decoder;block_reencoder;block_trees;\
|
||||
complevel_estimator;constants;decoder;hash_chain;info;\
|
||||
parameter_estimator;parser_config;predictor_state;reencoder;\
|
||||
statistical_codec;statistical_model;token;\
|
||||
seq_chain;statistical_codec;statistical_model;token;\
|
||||
token_predictor;tree_predictor")
|
||||
add_stem2file(PREFLATE_SRC "${SRCDIR}/contrib/preflate/support/%STEM%.cpp"
|
||||
"arithmetic_coder;array_helper;bit_helper;bitstream;const_division;\
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
31
contrib/preflate/packARI/.gitignore
vendored
31
contrib/preflate/packARI/.gitignore
vendored
|
|
@ -1,31 +0,0 @@
|
|||
# Compiled Object files
|
||||
*.slo
|
||||
*.lo
|
||||
*.o
|
||||
*.obj
|
||||
|
||||
# Precompiled Headers
|
||||
*.gch
|
||||
*.pch
|
||||
|
||||
# Compiled Dynamic libraries
|
||||
*.so
|
||||
*.dylib
|
||||
*.dll
|
||||
|
||||
# Fortran module files
|
||||
*.mod
|
||||
|
||||
# Compiled Static libraries
|
||||
*.lai
|
||||
*.la
|
||||
*.a
|
||||
*.lib
|
||||
|
||||
# Executables
|
||||
*.exe
|
||||
*.out
|
||||
*.app
|
||||
|
||||
# Folders to ignore
|
||||
/zzz_backup/
|
||||
|
|
@ -1,165 +0,0 @@
|
|||
GNU LESSER GENERAL PUBLIC LICENSE
|
||||
Version 3, 29 June 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
|
||||
This version of the GNU Lesser General Public License incorporates
|
||||
the terms and conditions of version 3 of the GNU General Public
|
||||
License, supplemented by the additional permissions listed below.
|
||||
|
||||
0. Additional Definitions.
|
||||
|
||||
As used herein, "this License" refers to version 3 of the GNU Lesser
|
||||
General Public License, and the "GNU GPL" refers to version 3 of the GNU
|
||||
General Public License.
|
||||
|
||||
"The Library" refers to a covered work governed by this License,
|
||||
other than an Application or a Combined Work as defined below.
|
||||
|
||||
An "Application" is any work that makes use of an interface provided
|
||||
by the Library, but which is not otherwise based on the Library.
|
||||
Defining a subclass of a class defined by the Library is deemed a mode
|
||||
of using an interface provided by the Library.
|
||||
|
||||
A "Combined Work" is a work produced by combining or linking an
|
||||
Application with the Library. The particular version of the Library
|
||||
with which the Combined Work was made is also called the "Linked
|
||||
Version".
|
||||
|
||||
The "Minimal Corresponding Source" for a Combined Work means the
|
||||
Corresponding Source for the Combined Work, excluding any source code
|
||||
for portions of the Combined Work that, considered in isolation, are
|
||||
based on the Application, and not on the Linked Version.
|
||||
|
||||
The "Corresponding Application Code" for a Combined Work means the
|
||||
object code and/or source code for the Application, including any data
|
||||
and utility programs needed for reproducing the Combined Work from the
|
||||
Application, but excluding the System Libraries of the Combined Work.
|
||||
|
||||
1. Exception to Section 3 of the GNU GPL.
|
||||
|
||||
You may convey a covered work under sections 3 and 4 of this License
|
||||
without being bound by section 3 of the GNU GPL.
|
||||
|
||||
2. Conveying Modified Versions.
|
||||
|
||||
If you modify a copy of the Library, and, in your modifications, a
|
||||
facility refers to a function or data to be supplied by an Application
|
||||
that uses the facility (other than as an argument passed when the
|
||||
facility is invoked), then you may convey a copy of the modified
|
||||
version:
|
||||
|
||||
a) under this License, provided that you make a good faith effort to
|
||||
ensure that, in the event an Application does not supply the
|
||||
function or data, the facility still operates, and performs
|
||||
whatever part of its purpose remains meaningful, or
|
||||
|
||||
b) under the GNU GPL, with none of the additional permissions of
|
||||
this License applicable to that copy.
|
||||
|
||||
3. Object Code Incorporating Material from Library Header Files.
|
||||
|
||||
The object code form of an Application may incorporate material from
|
||||
a header file that is part of the Library. You may convey such object
|
||||
code under terms of your choice, provided that, if the incorporated
|
||||
material is not limited to numerical parameters, data structure
|
||||
layouts and accessors, or small macros, inline functions and templates
|
||||
(ten or fewer lines in length), you do both of the following:
|
||||
|
||||
a) Give prominent notice with each copy of the object code that the
|
||||
Library is used in it and that the Library and its use are
|
||||
covered by this License.
|
||||
|
||||
b) Accompany the object code with a copy of the GNU GPL and this license
|
||||
document.
|
||||
|
||||
4. Combined Works.
|
||||
|
||||
You may convey a Combined Work under terms of your choice that,
|
||||
taken together, effectively do not restrict modification of the
|
||||
portions of the Library contained in the Combined Work and reverse
|
||||
engineering for debugging such modifications, if you also do each of
|
||||
the following:
|
||||
|
||||
a) Give prominent notice with each copy of the Combined Work that
|
||||
the Library is used in it and that the Library and its use are
|
||||
covered by this License.
|
||||
|
||||
b) Accompany the Combined Work with a copy of the GNU GPL and this license
|
||||
document.
|
||||
|
||||
c) For a Combined Work that displays copyright notices during
|
||||
execution, include the copyright notice for the Library among
|
||||
these notices, as well as a reference directing the user to the
|
||||
copies of the GNU GPL and this license document.
|
||||
|
||||
d) Do one of the following:
|
||||
|
||||
0) Convey the Minimal Corresponding Source under the terms of this
|
||||
License, and the Corresponding Application Code in a form
|
||||
suitable for, and under terms that permit, the user to
|
||||
recombine or relink the Application with a modified version of
|
||||
the Linked Version to produce a modified Combined Work, in the
|
||||
manner specified by section 6 of the GNU GPL for conveying
|
||||
Corresponding Source.
|
||||
|
||||
1) Use a suitable shared library mechanism for linking with the
|
||||
Library. A suitable mechanism is one that (a) uses at run time
|
||||
a copy of the Library already present on the user's computer
|
||||
system, and (b) will operate properly with a modified version
|
||||
of the Library that is interface-compatible with the Linked
|
||||
Version.
|
||||
|
||||
e) Provide Installation Information, but only if you would otherwise
|
||||
be required to provide such information under section 6 of the
|
||||
GNU GPL, and only to the extent that such information is
|
||||
necessary to install and execute a modified version of the
|
||||
Combined Work produced by recombining or relinking the
|
||||
Application with a modified version of the Linked Version. (If
|
||||
you use option 4d0, the Installation Information must accompany
|
||||
the Minimal Corresponding Source and Corresponding Application
|
||||
Code. If you use option 4d1, you must provide the Installation
|
||||
Information in the manner specified by section 6 of the GNU GPL
|
||||
for conveying Corresponding Source.)
|
||||
|
||||
5. Combined Libraries.
|
||||
|
||||
You may place library facilities that are a work based on the
|
||||
Library side by side in a single library together with other library
|
||||
facilities that are not Applications and are not covered by this
|
||||
License, and convey such a combined library under terms of your
|
||||
choice, if you do both of the following:
|
||||
|
||||
a) Accompany the combined library with a copy of the same work based
|
||||
on the Library, uncombined with any other library facilities,
|
||||
conveyed under the terms of this License.
|
||||
|
||||
b) Give prominent notice with the combined library that part of it
|
||||
is a work based on the Library, and explaining where to find the
|
||||
accompanying uncombined form of the same work.
|
||||
|
||||
6. Revised Versions of the GNU Lesser General Public License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions
|
||||
of the GNU Lesser General Public License from time to time. Such new
|
||||
versions will be similar in spirit to the present version, but may
|
||||
differ in detail to address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Library as you received it specifies that a certain numbered version
|
||||
of the GNU Lesser General Public License "or any later version"
|
||||
applies to it, you have the option of following the terms and
|
||||
conditions either of that published version or of any later version
|
||||
published by the Free Software Foundation. If the Library as you
|
||||
received it does not specify a version number of the GNU Lesser
|
||||
General Public License, you may choose any version of the GNU Lesser
|
||||
General Public License ever published by the Free Software Foundation.
|
||||
|
||||
If the Library as you received it specifies that a proxy can decide
|
||||
whether future versions of the GNU Lesser General Public License shall
|
||||
apply, that proxy's public statement of acceptance of any version is
|
||||
permanent authorization for you to choose that version for the
|
||||
Library.
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
# packARI
|
||||
A very basic compression tool for any file, mainly intended as base for bigger projects
|
||||
|
||||
packARI is a compression tool based solely on the core entropy encoding component used in all my other compression softwares (that means packJPG, packMP3, packPNM, …). On it’s own it might not achieve good results for some file types, but it makes for a great starting point when developing compression tools of your own.
|
||||
|
||||
The entropy coding in packARI has been tested since the beginning in all my other compression softwares, so it might be the most stable of them all.
|
||||
|
||||
You may freely use packARI and it’s source code under the terms of the LGPL.
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
# packXXX Makefile, based on UniMake: Universal Makefile
|
||||
# Created by Matthias Stirner, 01/2016
|
||||
|
||||
TARGET = packARI
|
||||
CC = gcc
|
||||
CPP = g++
|
||||
RC = windres -O coff
|
||||
CPPFLAGS = -I. -O3 -Wall -pedantic -funroll-loops -ffast-math -fsched-spec-load -fomit-frame-pointer
|
||||
LDFLAGS = -static -static-libgcc -static-libstdc++
|
||||
CSRC = $(wildcard *.c)
|
||||
CPPSRC = $(wildcard *.cpp)
|
||||
DEPS = $(wildcard *.h) Makefile
|
||||
OBJ = $(patsubst %.c,%.o,$(CSRC)) $(patsubst %.cpp,%.o,$(CPPSRC))
|
||||
|
||||
# conditional stuff
|
||||
ifeq ($(OS),Windows_NT)
|
||||
LDFLAGS += -lpthread -L libwinpthread-1.dll
|
||||
RES = icons.res
|
||||
UPX := -upx --best --lzma $(TARGET).exe
|
||||
else
|
||||
CPPFLAGS += -DUNIX
|
||||
RC =
|
||||
RES =
|
||||
UPX =
|
||||
endif
|
||||
|
||||
%.o: %.cpp $(DEPS)
|
||||
$(CPP) -c -o $@ $< $(CPPFLAGS)
|
||||
|
||||
%.res: %.rc
|
||||
@-$(RC) $< $@
|
||||
|
||||
$(TARGET): $(OBJ) $(RES)
|
||||
$(CPP) -o $@ $^ -s $(LDFLAGS)
|
||||
$(UPX)
|
||||
|
||||
.PHONY: all dev lib dll
|
||||
|
||||
all: $(TARGET)
|
||||
|
||||
dev: CPPFLAGS += -DDEV_BUILD
|
||||
dev: $(TARGET)
|
||||
|
||||
lib: CPPFLAGS += -DBUILD_LIB
|
||||
lib: $(OBJ)
|
||||
ar r $(TARGET)lib.a $(OBJ)
|
||||
ranlib $(TARGET)lib.a
|
||||
|
||||
dll: CPPFLAGS += -DBUILD_DLL
|
||||
dll: LDFLAGS += -Wl,--out-implib,libpackJPG.a -fvisibility=hidden
|
||||
dll: $(OBJ)
|
||||
$(CPP) -shared -o $(TARGET).dll $^ $(LDFLAGS)
|
||||
|
||||
clean:
|
||||
@echo clean...
|
||||
@-rm *.o *.a $(TARGET) $(TARGET).exe $(TARGET).dll
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 3.2 KiB |
File diff suppressed because it is too large
Load diff
|
|
@ -1,268 +0,0 @@
|
|||
// defines for coder
|
||||
#define CODER_USE_BITS 31 // must never be above 31
|
||||
#define CODER_LIMIT100 ( (unsigned int) ( 1 << CODER_USE_BITS ) )
|
||||
#define CODER_LIMIT025 ( ( CODER_LIMIT100 / 4 ) * 1 )
|
||||
#define CODER_LIMIT050 ( ( CODER_LIMIT100 / 4 ) * 2 )
|
||||
#define CODER_LIMIT075 ( ( CODER_LIMIT100 / 4 ) * 3 )
|
||||
#define CODER_MAXSCALE CODER_LIMIT025 - 1
|
||||
#define ESCAPE_SYMBOL CODER_LIMIT025
|
||||
|
||||
|
||||
// symbol struct, used in arithmetic coding
|
||||
struct symbol {
|
||||
unsigned int low_count;
|
||||
unsigned int high_count;
|
||||
unsigned int scale;
|
||||
};
|
||||
|
||||
// table struct, used in in statistical models,
|
||||
// holding all info needed for one context
|
||||
struct table {
|
||||
// counts for each symbol contained in the table
|
||||
unsigned short* counts;
|
||||
// links to higher order contexts
|
||||
struct table** links;
|
||||
// link to lower order context
|
||||
struct table* lesser;
|
||||
// accumulated counts
|
||||
unsigned int scale;
|
||||
};
|
||||
|
||||
// special table struct, used in in model_s,
|
||||
// holding additional info for a speedier 'totalize_table'
|
||||
struct table_s {
|
||||
// counts for each symbol contained in the table
|
||||
unsigned short* counts;
|
||||
// links to higher order contexts
|
||||
struct table_s** links;
|
||||
// link to lower order context
|
||||
struct table_s* lesser;
|
||||
// speedup info
|
||||
unsigned short max_count;
|
||||
unsigned short max_symbol;
|
||||
// unsigned short esc_prob;
|
||||
};
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
class for arithmetic coding of data to/from iostream
|
||||
----------------------------------------------- */
|
||||
|
||||
class aricoder
|
||||
{
|
||||
public:
|
||||
aricoder( iostream* stream, int iomode );
|
||||
~aricoder( void );
|
||||
void encode( symbol* s );
|
||||
unsigned int decode_count( symbol* s );
|
||||
void decode( symbol* s );
|
||||
|
||||
private:
|
||||
// bitwise operations
|
||||
void write_bit( unsigned char bit );
|
||||
unsigned char read_bit( void );
|
||||
|
||||
// i/o variables
|
||||
iostream* sptr;
|
||||
int mode;
|
||||
unsigned char bbyte;
|
||||
unsigned char cbit;
|
||||
|
||||
// arithmetic coding variables
|
||||
unsigned int ccode;
|
||||
unsigned int clow;
|
||||
unsigned int chigh;
|
||||
unsigned int cstep;
|
||||
unsigned int nrbits;
|
||||
};
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
universal statistical model for arithmetic coding
|
||||
----------------------------------------------- */
|
||||
|
||||
class model_s
|
||||
{
|
||||
public:
|
||||
|
||||
model_s( int max_s, int max_c, int max_o, int c_lim );
|
||||
~model_s( void );
|
||||
|
||||
void update_model( int symbol );
|
||||
void shift_context( int c );
|
||||
void flush_model( int scale_factor );
|
||||
void exclude_symbols( char rule, int c );
|
||||
|
||||
int convert_int_to_symbol( int c, symbol *s );
|
||||
void get_symbol_scale( symbol *s );
|
||||
int convert_symbol_to_int( int count, symbol *s );
|
||||
|
||||
bool error;
|
||||
|
||||
|
||||
private:
|
||||
|
||||
// unsigned short* totals;
|
||||
unsigned int* totals;
|
||||
char* scoreboard;
|
||||
int sb0_count;
|
||||
table_s **contexts;
|
||||
table_s **storage;
|
||||
|
||||
int max_symbol;
|
||||
int max_context;
|
||||
int current_order;
|
||||
int max_order;
|
||||
int max_count;
|
||||
|
||||
inline void totalize_table(table_s* context );
|
||||
inline void rescale_table(table_s* context, int scale_factor );
|
||||
inline void recursive_flush(table_s* context, int scale_factor );
|
||||
inline void recursive_cleanup(table_s* context );
|
||||
};
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
binary statistical model for arithmetic coding
|
||||
----------------------------------------------- */
|
||||
|
||||
class model_b
|
||||
{
|
||||
public:
|
||||
|
||||
model_b( int max_c, int max_o, int c_lim );
|
||||
~model_b( void );
|
||||
|
||||
void update_model( int symbol );
|
||||
void shift_context( int c );
|
||||
void flush_model( int scale_factor );
|
||||
|
||||
int convert_int_to_symbol( int c, symbol *s );
|
||||
void get_symbol_scale( symbol *s );
|
||||
int convert_symbol_to_int( int count, symbol *s );
|
||||
|
||||
bool error;
|
||||
|
||||
|
||||
private:
|
||||
|
||||
table **contexts;
|
||||
table **storage;
|
||||
|
||||
int max_context;
|
||||
int max_order;
|
||||
int max_count;
|
||||
|
||||
inline void check_counts( table *context );
|
||||
inline void rescale_table( table* context, int scale_factor );
|
||||
inline void recursive_flush( table* context, int scale_factor );
|
||||
inline void recursive_cleanup( table *context );
|
||||
};
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
shift context x2 model_s function
|
||||
----------------------------------------------- */
|
||||
static inline void shift_model( model_s* model, int ctx1, int ctx2 )
|
||||
{
|
||||
model->shift_context( ctx1 );
|
||||
model->shift_context( ctx2 );
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
shift context x3 model_s function
|
||||
----------------------------------------------- */
|
||||
static inline void shift_model( model_s* model, int ctx1, int ctx2, int ctx3 )
|
||||
{
|
||||
model->shift_context( ctx1 );
|
||||
model->shift_context( ctx2 );
|
||||
model->shift_context( ctx3 );
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
shift context x2 model_b function
|
||||
----------------------------------------------- */
|
||||
static inline void shift_model( model_b* model, int ctx1, int ctx2 )
|
||||
{
|
||||
model->shift_context( ctx1 );
|
||||
model->shift_context( ctx2 );
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
shift context x3 model_b function
|
||||
----------------------------------------------- */
|
||||
static inline void shift_model( model_b* model, int ctx1, int ctx2, int ctx3 )
|
||||
{
|
||||
model->shift_context( ctx1 );
|
||||
model->shift_context( ctx2 );
|
||||
model->shift_context( ctx3 );
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
generic model_s encoder function
|
||||
----------------------------------------------- */
|
||||
static inline void encode_ari( aricoder* encoder, model_s* model, int c )
|
||||
{
|
||||
static symbol s;
|
||||
static int esc;
|
||||
|
||||
do {
|
||||
esc = model->convert_int_to_symbol( c, &s );
|
||||
encoder->encode( &s );
|
||||
} while ( esc );
|
||||
model->update_model( c );
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
generic model_s decoder function
|
||||
----------------------------------------------- */
|
||||
static inline int decode_ari( aricoder* decoder, model_s* model )
|
||||
{
|
||||
static symbol s;
|
||||
static unsigned int count;
|
||||
static int c;
|
||||
|
||||
do{
|
||||
model->get_symbol_scale( &s );
|
||||
count = decoder->decode_count( &s );
|
||||
c = model->convert_symbol_to_int( count, &s );
|
||||
decoder->decode( &s );
|
||||
} while ( c == ESCAPE_SYMBOL );
|
||||
model->update_model( c );
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
generic model_b encoder function
|
||||
----------------------------------------------- */
|
||||
static inline void encode_ari( aricoder* encoder, model_b* model, int c )
|
||||
{
|
||||
static symbol s;
|
||||
|
||||
model->convert_int_to_symbol( c, &s );
|
||||
encoder->encode( &s );
|
||||
model->update_model( c );
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
generic model_b decoder function
|
||||
----------------------------------------------- */
|
||||
static inline int decode_ari( aricoder* decoder, model_b* model )
|
||||
{
|
||||
static symbol s;
|
||||
static unsigned int count;
|
||||
static int c;
|
||||
|
||||
model->get_symbol_scale( &s );
|
||||
count = decoder->decode_count( &s );
|
||||
c = model->convert_symbol_to_int( count, &s );
|
||||
decoder->decode( &s );
|
||||
model->update_model( c );
|
||||
|
||||
return c;
|
||||
}
|
||||
|
|
@ -1,926 +0,0 @@
|
|||
/*
|
||||
This file contains special classes for bitwise
|
||||
reading and writing of arrays
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <fcntl.h>
|
||||
#include "bitops.h"
|
||||
|
||||
#ifdef _MSC_VER // Changed by schnaader to enable compilation by MSVC
|
||||
#include <io.h>
|
||||
#define setmode _setmode
|
||||
#endif
|
||||
|
||||
#define BUFFER_SIZE 1024 * 1024
|
||||
|
||||
|
||||
// special realloc with guaranteed free() of previous memory
|
||||
static inline void* frealloc( void* ptr, size_t size ) {
|
||||
void* n_ptr = realloc( ptr, (size) ? size : 1 );
|
||||
if ( n_ptr == NULL ) free( ptr );
|
||||
return n_ptr;
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
constructor for abitreader class
|
||||
----------------------------------------------- */
|
||||
|
||||
abitreader::abitreader( unsigned char* array, int size )
|
||||
{
|
||||
cbyte = 0;
|
||||
cbit = 8;
|
||||
peof = 0;
|
||||
eof = false;
|
||||
|
||||
data = array;
|
||||
lbyte = size;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
destructor for abitreader class
|
||||
----------------------------------------------- */
|
||||
|
||||
abitreader::~abitreader( void )
|
||||
{
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
reads n bits from abitreader
|
||||
----------------------------------------------- */
|
||||
|
||||
unsigned int abitreader::read( int nbits )
|
||||
{
|
||||
unsigned int retval = 0;
|
||||
|
||||
// safety check for eof
|
||||
if ( eof ) {
|
||||
peof += nbits;
|
||||
return 0;
|
||||
}
|
||||
|
||||
while ( nbits >= cbit ) {
|
||||
nbits -= cbit;
|
||||
retval |= ( RBITS( data[cbyte], cbit ) << nbits );
|
||||
cbit = 8;
|
||||
if ( ++cbyte >= lbyte ) {
|
||||
peof = nbits;
|
||||
eof = true;
|
||||
return retval;
|
||||
}
|
||||
}
|
||||
|
||||
if ( nbits > 0 ) {
|
||||
retval |= ( MBITS( data[cbyte], cbit, (cbit-nbits) ) );
|
||||
cbit -= nbits;
|
||||
}
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
reads one bit from abitreader
|
||||
----------------------------------------------- */
|
||||
|
||||
unsigned char abitreader::read_bit( void )
|
||||
{
|
||||
unsigned char bit;
|
||||
|
||||
// safety check for eof
|
||||
if (eof) {
|
||||
peof++;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// read one bit
|
||||
bit = BITN( data[cbyte], --cbit );
|
||||
if ( cbit == 0 ) {
|
||||
if ( ++cbyte == lbyte ) eof = true;
|
||||
cbit = 8;
|
||||
}
|
||||
|
||||
return bit;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
to skip padding from current byte
|
||||
----------------------------------------------- */
|
||||
|
||||
unsigned char abitreader::unpad( unsigned char fillbit )
|
||||
{
|
||||
if ( ( cbit == 8 ) || eof ) return fillbit;
|
||||
else {
|
||||
fillbit = read( 1 );
|
||||
while ( cbit != 8 ) read( 1 );
|
||||
}
|
||||
|
||||
return fillbit;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
get current position in array
|
||||
----------------------------------------------- */
|
||||
|
||||
int abitreader::getpos( void )
|
||||
{
|
||||
return cbyte;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
get current bit position
|
||||
----------------------------------------------- */
|
||||
|
||||
int abitreader::getbitp( void )
|
||||
{
|
||||
return cbit;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
set byte and bit position
|
||||
----------------------------------------------- */
|
||||
|
||||
void abitreader::setpos( int pbyte, int pbit )
|
||||
{
|
||||
if ( pbyte < lbyte ) {
|
||||
// reset eof
|
||||
eof = false;
|
||||
// set positions
|
||||
cbyte = pbyte;
|
||||
cbit = pbit;
|
||||
} else {
|
||||
// set eof
|
||||
eof = true;
|
||||
// set positions
|
||||
cbyte = lbyte;
|
||||
cbit = 8;
|
||||
peof = ( ( pbyte - lbyte ) * 8 ) + 8 - pbit;
|
||||
}
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
rewind n bits
|
||||
----------------------------------------------- */
|
||||
|
||||
void abitreader::rewind_bits( int nbits )
|
||||
{
|
||||
if ( eof ) {
|
||||
if ( nbits > peof ) nbits -= peof;
|
||||
else {
|
||||
peof -= nbits;
|
||||
return;
|
||||
}
|
||||
eof = false;
|
||||
}
|
||||
for ( cbit += nbits; cbit > 8; cbyte--, cbit -= 8 );
|
||||
if ( cbyte < 0 ) {
|
||||
cbyte = 0;
|
||||
cbit = 8;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
constructor for abitwriter class
|
||||
----------------------------------------------- */
|
||||
|
||||
abitwriter::abitwriter( int size )
|
||||
{
|
||||
fillbit = 1;
|
||||
adds = 65536;
|
||||
cbyte = 0;
|
||||
cbit = 8;
|
||||
|
||||
error = false;
|
||||
fmem = true;
|
||||
|
||||
dsize = ( size > 0 ) ? size : adds;
|
||||
data = ( unsigned char* ) malloc ( dsize );
|
||||
if ( data == NULL ) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
|
||||
// fill buffer with zeroes
|
||||
memset( data, 0, dsize * sizeof( char ) );
|
||||
// for ( int i = 0; i < dsize; i++ ) data[i] = 0;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
destructor for abitwriter class
|
||||
----------------------------------------------- */
|
||||
|
||||
abitwriter::~abitwriter( void )
|
||||
{
|
||||
// free memory if pointer was not given out
|
||||
if ( fmem ) free( data );
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
writes n bits to abitwriter
|
||||
----------------------------------------------- */
|
||||
|
||||
void abitwriter::write( unsigned int val, int nbits )
|
||||
{
|
||||
// safety check for error
|
||||
if ( error ) return;
|
||||
|
||||
// test if pointer beyond flush treshold
|
||||
if ( cbyte > ( dsize - 5 ) ) {
|
||||
dsize += adds;
|
||||
data = (unsigned char*) frealloc( data, dsize );
|
||||
if ( data == NULL ) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
memset( ( data + cbyte + 1 ), 0, ( dsize - ( cbyte + 1 ) ) * sizeof( char ) );
|
||||
}
|
||||
|
||||
// write data
|
||||
while ( nbits >= cbit ) {
|
||||
data[cbyte] |= ( MBITS32(val, nbits, (nbits-cbit)) );
|
||||
nbits -= cbit;
|
||||
cbyte++;
|
||||
cbit = 8;
|
||||
}
|
||||
|
||||
if ( nbits > 0 ) {
|
||||
data[cbyte] |= ( (RBITS32(val, nbits)) << (cbit - nbits) );
|
||||
cbit -= nbits;
|
||||
}
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
writes one bit to abitwriter
|
||||
----------------------------------------------- */
|
||||
|
||||
void abitwriter::write_bit( unsigned char bit )
|
||||
{
|
||||
// safety check for error
|
||||
if ( error ) return;
|
||||
|
||||
// write data
|
||||
if ( bit ) data[cbyte] |= 0x1 << (--cbit);
|
||||
else --cbit;
|
||||
if ( cbit == 0 ) {
|
||||
// test if pointer beyond flush treshold
|
||||
if ( ++cbyte > ( dsize - 5 ) ) {
|
||||
dsize += adds;
|
||||
data = (unsigned char*) frealloc( data, dsize );
|
||||
if ( data == NULL ) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
memset( ( data + cbyte + 1 ), 0, ( dsize - ( cbyte + 1 ) ) * sizeof( char ) );
|
||||
}
|
||||
cbit = 8;
|
||||
}
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
pads data using fillbit
|
||||
----------------------------------------------- */
|
||||
|
||||
void abitwriter::pad( unsigned char fillbit )
|
||||
{
|
||||
while ( cbit < 8 )
|
||||
write( fillbit, 1 );
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
gets data array from abitwriter
|
||||
----------------------------------------------- */
|
||||
|
||||
unsigned char* abitwriter::getptr( void )
|
||||
{
|
||||
// data is padded here
|
||||
pad( fillbit );
|
||||
// forbid freeing memory
|
||||
fmem = false;
|
||||
// realloc data
|
||||
data = (unsigned char*) frealloc( data, cbyte );
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
gets size of data array from abitwriter
|
||||
----------------------------------------------- */
|
||||
|
||||
int abitwriter::getpos( void )
|
||||
{
|
||||
return cbyte;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
get current bit position
|
||||
----------------------------------------------- */
|
||||
|
||||
int abitwriter::getbitp( void )
|
||||
{
|
||||
return cbit;
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
constructor for abytewriter class
|
||||
----------------------------------------------- */
|
||||
|
||||
abytereader::abytereader( unsigned char* array, int size )
|
||||
{
|
||||
cbyte = 0;
|
||||
eof = false;
|
||||
|
||||
data = array;
|
||||
lbyte = size;
|
||||
|
||||
if ( ( data == NULL ) || ( lbyte == 0 ) )
|
||||
eof = true;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
destructor for abytewriter class
|
||||
----------------------------------------------- */
|
||||
|
||||
abytereader::~abytereader( void )
|
||||
{
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
reads 1 byte from abytereader
|
||||
----------------------------------------------- */
|
||||
|
||||
int abytereader::read( unsigned char* byte )
|
||||
{
|
||||
if ( cbyte >= lbyte ) {
|
||||
cbyte = lbyte;
|
||||
eof = true;
|
||||
return 0;
|
||||
}
|
||||
else {
|
||||
*byte = data[ cbyte++ ];
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
reads n bytes from abytereader
|
||||
----------------------------------------------- */
|
||||
|
||||
int abytereader::read_n( unsigned char* byte, int n )
|
||||
{
|
||||
int nl = lbyte - cbyte;
|
||||
int i;
|
||||
|
||||
if ( nl < n ) {
|
||||
for ( i = 0; i < nl; i++ )
|
||||
byte[ i ] = data[ cbyte + i ];
|
||||
cbyte = lbyte;
|
||||
eof = true;
|
||||
return nl;
|
||||
}
|
||||
else {
|
||||
for ( i = 0; i < n; i++ )
|
||||
byte[ i ] = data[ cbyte + i ];
|
||||
cbyte += n;
|
||||
return n;
|
||||
}
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
go to position in data
|
||||
----------------------------------------------- */
|
||||
|
||||
void abytereader::seek( int pos )
|
||||
{
|
||||
if ( pos >= lbyte ) {
|
||||
cbyte = lbyte;
|
||||
eof = true;
|
||||
}
|
||||
else {
|
||||
cbyte = pos;
|
||||
eof = false;
|
||||
}
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
gets size of current data
|
||||
----------------------------------------------- */
|
||||
|
||||
int abytereader::getsize( void )
|
||||
{
|
||||
return lbyte;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
gets current position from abytereader
|
||||
----------------------------------------------- */
|
||||
|
||||
int abytereader::getpos( void )
|
||||
{
|
||||
return cbyte;
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
constructor for abytewriter class
|
||||
----------------------------------------------- */
|
||||
|
||||
abytewriter::abytewriter( int size )
|
||||
{
|
||||
adds = 65536;
|
||||
cbyte = 0;
|
||||
|
||||
error = false;
|
||||
fmem = true;
|
||||
|
||||
dsize = ( size > 0 ) ? size : adds;
|
||||
data = (unsigned char*) malloc( dsize );
|
||||
if ( data == NULL ) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
destructor for abytewriter class
|
||||
----------------------------------------------- */
|
||||
|
||||
abytewriter::~abytewriter( void )
|
||||
{
|
||||
// free data if pointer is not read
|
||||
if ( fmem ) free( data );
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
writes 1 byte to abytewriter
|
||||
----------------------------------------------- */
|
||||
|
||||
void abytewriter::write( unsigned char byte )
|
||||
{
|
||||
// safety check for error
|
||||
if ( error ) return;
|
||||
|
||||
// test if pointer beyond flush threshold
|
||||
if ( cbyte >= ( dsize - 2 ) ) {
|
||||
dsize += adds;
|
||||
data = (unsigned char*) frealloc( data, dsize );
|
||||
if ( data == NULL ) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// write data
|
||||
data[ cbyte++ ] = byte;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
writes n byte to abytewriter
|
||||
----------------------------------------------- */
|
||||
|
||||
void abytewriter::write_n( unsigned char* byte, int n )
|
||||
{
|
||||
// safety check for error
|
||||
if ( error ) return;
|
||||
|
||||
// make sure that pointer doesn't get beyond flush threshold
|
||||
while ( ( cbyte + n ) >= ( dsize - 2 ) ) {
|
||||
dsize += adds;
|
||||
data = (unsigned char*) frealloc( data, dsize );
|
||||
if ( data == NULL ) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// copy data from array
|
||||
while ( n-- > 0 )
|
||||
data[ cbyte++ ] = *(byte++);
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
gets data array from abytewriter
|
||||
----------------------------------------------- */
|
||||
|
||||
unsigned char* abytewriter::getptr( void )
|
||||
{
|
||||
// safety check for error
|
||||
if ( error ) return NULL;
|
||||
// forbid freeing memory
|
||||
fmem = false;
|
||||
// realloc data
|
||||
data = (unsigned char*) frealloc( data, cbyte );
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
peeks into data array from abytewriter
|
||||
----------------------------------------------- */
|
||||
|
||||
unsigned char* abytewriter::peekptr( void )
|
||||
{
|
||||
return data;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
gets size of data array from abytewriter
|
||||
----------------------------------------------- */
|
||||
|
||||
int abytewriter::getpos( void )
|
||||
{
|
||||
return cbyte;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
reset without realloc
|
||||
----------------------------------------------- */
|
||||
|
||||
void abytewriter::reset( void )
|
||||
{
|
||||
// set position of current byte
|
||||
cbyte = 0;
|
||||
}
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
constructor for iostream class
|
||||
----------------------------------------------- */
|
||||
|
||||
iostream::iostream( void* src, int srctype, int srcsize, int iomode )
|
||||
{
|
||||
// locally copy source, source type # and io mode #
|
||||
source = src;
|
||||
srct = srctype;
|
||||
srcs = srcsize;
|
||||
mode = iomode;
|
||||
|
||||
// don't free memory when reading - this will be useful if switching occurs
|
||||
free_mem_sw = false;
|
||||
|
||||
// set binary mode for streams
|
||||
#if defined( _WIN32 )
|
||||
setmode( fileno( stdin ), O_BINARY );
|
||||
setmode( fileno( stdout ), O_BINARY );
|
||||
#endif
|
||||
|
||||
// open file/mem/stream
|
||||
switch ( srct )
|
||||
{
|
||||
case 0:
|
||||
open_file();
|
||||
break;
|
||||
|
||||
case 1:
|
||||
open_mem();
|
||||
break;
|
||||
|
||||
case 2:
|
||||
open_stream();
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
destructor for iostream class
|
||||
----------------------------------------------- */
|
||||
|
||||
iostream::~iostream( void )
|
||||
{
|
||||
// if needed, write memory to stream or free memory from buffered stream
|
||||
if ( srct == 2 ) {
|
||||
if ( mode == 1 ) {
|
||||
if ( !(mwrt->error) ) {
|
||||
srcs = mwrt->getpos();
|
||||
source = mwrt->getptr();
|
||||
fwrite( source, sizeof( char ), srcs, stdout );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// free all buffers
|
||||
if ( srct == 0 ) {
|
||||
if ( fptr != NULL ) {
|
||||
if ( mode == 1 ) fflush( fptr );
|
||||
fclose( fptr );
|
||||
}
|
||||
}
|
||||
else if ( mode == 0 ) {
|
||||
if ( free_mem_sw )
|
||||
free( source );
|
||||
delete( mrdr );
|
||||
}
|
||||
else
|
||||
delete( mwrt );
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
switches mode from reading to writing and vice versa
|
||||
----------------------------------------------- */
|
||||
|
||||
void iostream::switch_mode( void )
|
||||
{
|
||||
// return immediately if there's an error
|
||||
if ( chkerr() ) return;
|
||||
|
||||
|
||||
if ( mode == 0 ) {
|
||||
// WARNING: when switching from reading to writing, information might be lost forever
|
||||
switch ( srct ) {
|
||||
case 0:
|
||||
fclose( fptr );
|
||||
fptr = fopen( ( char* ) source, "wb" );
|
||||
break;
|
||||
case 1:
|
||||
case 2:
|
||||
delete( mrdr );
|
||||
if ( free_mem_sw )
|
||||
free( source ); // see? I've told you so :-)
|
||||
mwrt = new abytewriter( srcs );
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
mode = 1;
|
||||
}
|
||||
else {
|
||||
// switching from writing to reading is a bit more complicated
|
||||
switch ( srct ) {
|
||||
case 0:
|
||||
fflush( fptr );
|
||||
fclose( fptr );
|
||||
fptr = fopen( ( char* ) source, "rb" );
|
||||
break;
|
||||
case 1:
|
||||
case 2:
|
||||
source = mwrt->getptr();
|
||||
srcs = mwrt->getpos();
|
||||
delete( mwrt );
|
||||
mrdr = new abytereader( ( unsigned char* ) source, srcs );
|
||||
free_mem_sw = true;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
mode = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
generic read function
|
||||
----------------------------------------------- */
|
||||
|
||||
int iostream::read( void* to, int tpsize, int dtsize )
|
||||
{
|
||||
return ( srct == 0 ) ? read_file( to, tpsize, dtsize ) : read_mem( to, tpsize, dtsize );
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
generic write function
|
||||
----------------------------------------------- */
|
||||
|
||||
int iostream::write( void* from, int tpsize, int dtsize )
|
||||
{
|
||||
return ( srct == 0 ) ? write_file( from, tpsize, dtsize ) : write_mem( from, tpsize, dtsize );
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
flush function
|
||||
----------------------------------------------- */
|
||||
|
||||
int iostream::flush( void )
|
||||
{
|
||||
if ( srct == 0 )
|
||||
fflush( fptr );
|
||||
|
||||
return getpos();
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
rewind to beginning of stream
|
||||
----------------------------------------------- */
|
||||
|
||||
int iostream::rewind( void )
|
||||
{
|
||||
// WARNING: when writing, rewind might lose all your data
|
||||
if ( srct == 0 )
|
||||
fseek( fptr, 0, SEEK_SET );
|
||||
else if ( mode == 0 )
|
||||
mrdr->seek( 0 );
|
||||
else
|
||||
mwrt->reset();
|
||||
|
||||
return getpos();
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
get current position in stream
|
||||
----------------------------------------------- */
|
||||
|
||||
int iostream::getpos( void )
|
||||
{
|
||||
int pos;
|
||||
|
||||
if ( srct == 0 )
|
||||
pos = ftell( fptr );
|
||||
else if ( mode == 0 )
|
||||
pos = mrdr->getpos();
|
||||
else
|
||||
pos = mwrt->getpos();
|
||||
|
||||
return pos;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
get size of file
|
||||
----------------------------------------------- */
|
||||
|
||||
int iostream::getsize( void )
|
||||
{
|
||||
int pos;
|
||||
int siz;
|
||||
|
||||
if ( mode == 0 ) {
|
||||
if ( srct == 0 ) {
|
||||
pos = ftell( fptr );
|
||||
fseek( fptr, 0, SEEK_END );
|
||||
siz = ftell( fptr );
|
||||
fseek( fptr, pos, SEEK_SET );
|
||||
}
|
||||
else {
|
||||
siz = mrdr->getsize();
|
||||
}
|
||||
}
|
||||
else {
|
||||
siz = getpos();
|
||||
}
|
||||
|
||||
return siz;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
get data pointer (for mem io only)
|
||||
----------------------------------------------- */
|
||||
|
||||
unsigned char* iostream::getptr( void )
|
||||
{
|
||||
if ( srct == 1 )
|
||||
return ( mode == 0 ) ? ( unsigned char* ) source : mwrt->getptr();
|
||||
else
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
check for errors
|
||||
----------------------------------------------- */
|
||||
|
||||
bool iostream::chkerr( void )
|
||||
{
|
||||
bool error = false;
|
||||
|
||||
// check for user input errors
|
||||
if ( ( mode != 0 ) && ( mode != 1 ) )
|
||||
error = true;
|
||||
if ( ( srct != 0 ) && ( srct != 1 ) && ( srct != 2 ) )
|
||||
error = true;
|
||||
|
||||
// check for io errors
|
||||
if ( srct == 0 ) {
|
||||
if ( fptr == NULL )
|
||||
error = true;
|
||||
else if ( ferror( fptr ) )
|
||||
error = true;
|
||||
}
|
||||
else if ( mode == 0 ) {
|
||||
if ( mrdr == NULL )
|
||||
error = true;
|
||||
}
|
||||
else {
|
||||
if ( mwrt == NULL )
|
||||
error = true;
|
||||
else if ( mwrt->error )
|
||||
error = true;
|
||||
}
|
||||
|
||||
return error;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
check for eof (read only)
|
||||
----------------------------------------------- */
|
||||
|
||||
bool iostream::chkeof( void )
|
||||
{
|
||||
if ( mode == 0 )
|
||||
return ( srct == 0 ) ? feof( fptr ) : mrdr->eof;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
open function for files
|
||||
----------------------------------------------- */
|
||||
|
||||
void iostream::open_file( void )
|
||||
{
|
||||
char* fn = (char*) source;
|
||||
|
||||
// open file for reading / writing
|
||||
fptr = fopen( fn, ( mode == 0 ) ? "rb" : "wb" );
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
open function for memory
|
||||
----------------------------------------------- */
|
||||
|
||||
void iostream::open_mem( void )
|
||||
{
|
||||
if ( mode == 0 )
|
||||
mrdr = new abytereader( ( unsigned char* ) source, srcs );
|
||||
else
|
||||
mwrt = new abytewriter( srcs );
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
open function for streams
|
||||
----------------------------------------------- */
|
||||
|
||||
void iostream::open_stream( void )
|
||||
{
|
||||
abytewriter* strwrt;
|
||||
unsigned char* buffer;
|
||||
int i;
|
||||
|
||||
if ( mode == 0 ) {
|
||||
// read whole stream into memory buffer
|
||||
strwrt = new abytewriter( 0 );
|
||||
buffer = ( unsigned char* ) calloc( BUFFER_SIZE, sizeof( char ) );
|
||||
if ( buffer != NULL ) {
|
||||
while ( ( i = fread( buffer, sizeof( char ), BUFFER_SIZE, stdin ) ) > 0 )
|
||||
strwrt->write_n( buffer, i );
|
||||
}
|
||||
if ( strwrt->error ) {
|
||||
source = NULL;
|
||||
srcs = 0;
|
||||
}
|
||||
else {
|
||||
source = strwrt->getptr();
|
||||
srcs = strwrt->getpos();
|
||||
}
|
||||
delete ( strwrt );
|
||||
free( buffer );
|
||||
// free memory after done
|
||||
free_mem_sw = true;
|
||||
}
|
||||
|
||||
// for writing: simply open new stream in mem writer
|
||||
// writing to stream will be done later
|
||||
open_mem();
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
write function for files
|
||||
----------------------------------------------- */
|
||||
|
||||
int iostream::write_file( void* from, int tpsize, int dtsize )
|
||||
{
|
||||
return fwrite( from, tpsize, dtsize, fptr );
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
read function for files
|
||||
----------------------------------------------- */
|
||||
|
||||
int iostream::read_file( void* to, int tpsize, int dtsize )
|
||||
{
|
||||
return fread( to, tpsize, dtsize, fptr );
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
write function for memory
|
||||
----------------------------------------------- */
|
||||
|
||||
int iostream::write_mem( void* from, int tpsize, int dtsize )
|
||||
{
|
||||
int n = tpsize * dtsize;
|
||||
|
||||
mwrt->write_n( ( unsigned char* ) from, n );
|
||||
|
||||
return ( mwrt->error ) ? 0 : n;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------
|
||||
read function for memory
|
||||
----------------------------------------------- */
|
||||
|
||||
int iostream::read_mem( void* to, int tpsize, int dtsize )
|
||||
{
|
||||
int n = tpsize * dtsize;
|
||||
|
||||
return ( mrdr->read_n( ( unsigned char* ) to, n ) ) / tpsize;
|
||||
}
|
||||
|
|
@ -1,168 +0,0 @@
|
|||
#define RBITS( c, n ) ( c & ( 0xFF >> (8 - n) ) )
|
||||
#define LBITS( c, n ) ( c >> (8 - n) )
|
||||
#define MBITS( c, l, r ) ( RBITS( c,l ) >> r )
|
||||
#define RBITS16( c, n ) ( c & ( 0xFFFFFFFF >> (16 - n) ) )
|
||||
#define LBITS16( c, n ) ( c >> (16 - n) )
|
||||
#define MBITS16( c, l, r ) ( RBITS16( c,l ) >> r )
|
||||
#define RBITS32( c, n ) ( c & ( 0xFFFFFFFF >> (32 - n) ) )
|
||||
#define LBITS32( c, n ) ( c >> (32 - n) )
|
||||
#define MBITS32( c, l, r ) ( RBITS32( c,l ) >> r )
|
||||
#define BITN( c, n ) ( (c >> n) & 0x1 )
|
||||
#define BITLEN( l, v ) for ( l = 0; ( v >> l ) > 0; l++ )
|
||||
#define FDIV2( v, p ) ( ( v < 0 ) ? -( (-v) >> p ) : ( v >> p ) )
|
||||
|
||||
#define TYPE_FILE 0
|
||||
#define TYPE_MEMORY 1
|
||||
#define TYPE_STREAM 2
|
||||
#define MODE_READ 0
|
||||
#define MODE_WRITE 1
|
||||
|
||||
#define BTST_BUFF 1024 * 1024
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
class to read arrays bitwise
|
||||
----------------------------------------------- */
|
||||
|
||||
class abitreader
|
||||
{
|
||||
public:
|
||||
abitreader( unsigned char* array, int size );
|
||||
~abitreader( void );
|
||||
unsigned int read( int nbits );
|
||||
unsigned char read_bit( void );
|
||||
unsigned char unpad( unsigned char fillbit );
|
||||
int getpos( void );
|
||||
int getbitp( void );
|
||||
void setpos( int pbyte, int pbit );
|
||||
void rewind_bits( int nbits );
|
||||
bool eof;
|
||||
int peof;
|
||||
|
||||
private:
|
||||
unsigned char* data;
|
||||
int lbyte;
|
||||
int cbyte;
|
||||
int cbit;
|
||||
};
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
class to write arrays bitwise
|
||||
----------------------------------------------- */
|
||||
|
||||
class abitwriter
|
||||
{
|
||||
public:
|
||||
abitwriter( int size );
|
||||
~abitwriter( void );
|
||||
void write( unsigned int val, int nbits );
|
||||
void write_bit( unsigned char bit );
|
||||
void pad ( unsigned char fillbit );
|
||||
unsigned char* getptr( void );
|
||||
int getpos( void );
|
||||
int getbitp( void );
|
||||
bool error;
|
||||
unsigned char fillbit;
|
||||
|
||||
private:
|
||||
unsigned char* data;
|
||||
int dsize;
|
||||
int adds;
|
||||
int cbyte;
|
||||
int cbit;
|
||||
bool fmem;
|
||||
};
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
class to read arrays bytewise
|
||||
----------------------------------------------- */
|
||||
|
||||
class abytereader
|
||||
{
|
||||
public:
|
||||
abytereader( unsigned char* array, int size );
|
||||
~abytereader( void );
|
||||
int read( unsigned char* byte );
|
||||
int read_n( unsigned char* byte, int n );
|
||||
void seek( int pos );
|
||||
int getsize( void );
|
||||
int getpos( void );
|
||||
bool eof;
|
||||
|
||||
private:
|
||||
unsigned char* data;
|
||||
int lbyte;
|
||||
int cbyte;
|
||||
};
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
class to write arrays bytewise
|
||||
----------------------------------------------- */
|
||||
|
||||
class abytewriter
|
||||
{
|
||||
public:
|
||||
abytewriter( int size );
|
||||
~abytewriter( void );
|
||||
void write( unsigned char byte );
|
||||
void write_n( unsigned char* byte, int n );
|
||||
unsigned char* getptr( void );
|
||||
unsigned char* peekptr( void );
|
||||
int getpos( void );
|
||||
void reset( void );
|
||||
bool error;
|
||||
|
||||
private:
|
||||
unsigned char* data;
|
||||
int dsize;
|
||||
int adds;
|
||||
int cbyte;
|
||||
bool fmem;
|
||||
};
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
class for input and output from file or memory
|
||||
----------------------------------------------- */
|
||||
|
||||
class iostream
|
||||
{
|
||||
public:
|
||||
iostream( void* src, int srctype, int srcsize, int iomode );
|
||||
~iostream( void );
|
||||
void switch_mode( void );
|
||||
int read( void* to, int tpsize, int dtsize );
|
||||
int write( void* from, int tpsize, int dtsize );
|
||||
int flush( void );
|
||||
int rewind( void );
|
||||
int getpos( void );
|
||||
int getsize( void );
|
||||
unsigned char* getptr( void );
|
||||
bool chkerr( void );
|
||||
bool chkeof( void );
|
||||
|
||||
private:
|
||||
void open_file( void );
|
||||
void open_mem( void );
|
||||
void open_stream( void );
|
||||
|
||||
int write_file( void* from, int tpsize, int dtsize );
|
||||
int read_file( void* to, int tpsize, int dtsize );
|
||||
int write_mem( void* from, int tpsize, int dtsize );
|
||||
int read_mem( void* to, int tpsize, int dtsize );
|
||||
|
||||
FILE* fptr;
|
||||
abytewriter* mwrt;
|
||||
abytereader* mrdr;
|
||||
|
||||
bool free_mem_sw;
|
||||
void* source;
|
||||
int mode;
|
||||
int srct;
|
||||
int srcs;
|
||||
};
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 3.2 KiB |
Binary file not shown.
File diff suppressed because it is too large
Load diff
|
|
@ -1,37 +0,0 @@
|
|||
// packARIdll.h - function import declarations for the packARI DLL
|
||||
#define IMPORT __declspec( dllimport )
|
||||
|
||||
/* -----------------------------------------------
|
||||
function declarations: library only functions
|
||||
----------------------------------------------- */
|
||||
|
||||
IMPORT bool parlib_convert_stream2stream( char* msg );
|
||||
IMPORT bool parlib_convert_file2file( char* in, char* out, char* msg );
|
||||
IMPORT bool parlib_convert_stream2mem( unsigned char** out_file, unsigned int* out_size, char* msg );
|
||||
IMPORT void parlib_init_streams( void* in_src, int in_type, int in_size, void* out_dest, int out_type );
|
||||
IMPORT void parlib_force_encoding( bool setting );
|
||||
IMPORT const char* parlib_version_info( void );
|
||||
IMPORT const char* parlib_short_name( void );
|
||||
|
||||
/* a short reminder about input/output stream types
|
||||
for the parlib_init_streams() function
|
||||
|
||||
if input is file
|
||||
----------------
|
||||
in_scr -> name of input file
|
||||
in_type -> 0
|
||||
in_size -> ignore
|
||||
|
||||
if input is memory
|
||||
------------------
|
||||
in_scr -> array containg data
|
||||
in_type -> 1
|
||||
in_size -> size of data array
|
||||
|
||||
if input is *FILE (f.e. stdin)
|
||||
------------------------------
|
||||
in_src -> stream pointer
|
||||
in_type -> 2
|
||||
in_size -> ignore
|
||||
|
||||
vice versa for output streams! */
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
// packARIlib.h - function declarations for the packARI library
|
||||
#if defined BUILD_DLL
|
||||
#define EXPORT __declspec( dllexport )
|
||||
#else
|
||||
#define EXPORT extern
|
||||
#endif
|
||||
|
||||
/* -----------------------------------------------
|
||||
function declarations: library only functions
|
||||
----------------------------------------------- */
|
||||
|
||||
EXPORT bool parlib_convert_stream2stream( char* msg );
|
||||
EXPORT bool parlib_convert_file2file( char* in, char* out, char* msg );
|
||||
EXPORT bool parlib_convert_stream2mem( unsigned char** out_file, unsigned int* out_size, char* msg );
|
||||
EXPORT void parlib_init_streams( void* in_src, int in_type, int in_size, void* out_dest, int out_type );
|
||||
EXPORT void parlib_force_encoding( bool setting );
|
||||
EXPORT const char* parlib_version_info( void );
|
||||
EXPORT const char* parlib_short_name( void );
|
||||
|
||||
/* a short reminder about input/output stream types
|
||||
for the parlib_init_streams() function
|
||||
|
||||
if input is file
|
||||
----------------
|
||||
in_scr -> name of input file
|
||||
in_type -> 0
|
||||
in_size -> ignore
|
||||
|
||||
if input is memory
|
||||
------------------
|
||||
in_scr -> array containg data
|
||||
in_type -> 1
|
||||
in_size -> size of data array
|
||||
|
||||
if input is *FILE (f.e. stdin)
|
||||
------------------------------
|
||||
in_src -> stream pointer
|
||||
in_type -> 2
|
||||
in_size -> ignore
|
||||
|
||||
vice versa for output streams! */
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
/* -----------------------------------------------
|
||||
setting for packARI
|
||||
----------------------------------------------- */
|
||||
|
||||
// !EDIT WITH CAUTION!
|
||||
// each change may lead to incompatibilities with previous version
|
||||
|
||||
// action defines
|
||||
#define A_COMPRESS 1
|
||||
|
||||
// file type defines
|
||||
#define F_UNK 1
|
||||
#define F_PAR 2
|
||||
|
||||
// compression parameters (tweaking possible)
|
||||
#define MAX_ORDER 3
|
||||
#define CHECK_BYTES ( 1 << 14 )
|
||||
|
||||
// other setting
|
||||
#define ANALYSIS_CSV "par_analysis.csv"
|
||||
|
||||
|
||||
/* -----------------------------------------------
|
||||
pearson hashing table
|
||||
----------------------------------------------- */
|
||||
|
||||
// copied from Table 1 in:
|
||||
// Pearson, Peter K. (June 1990), "Fast Hashing of Variable-Length Text Strings",
|
||||
// Communications of the ACM 33 (6): 677, doi:10.1145/78973.78978
|
||||
static int pearson_hash[ 256 ] =
|
||||
{
|
||||
1, 87, 49, 12, 176, 178, 102, 166, 121, 193, 6, 84, 249, 230, 44, 163,
|
||||
14, 197, 213, 181, 161, 85, 218, 80, 64, 239, 24, 226, 236, 142, 38, 200,
|
||||
110, 177, 104, 103, 141, 253, 255, 50, 77, 101, 81, 18, 45, 96, 31, 222,
|
||||
25, 107, 190, 70, 86, 237, 240, 34, 72, 242, 20, 214, 244, 227, 149, 235,
|
||||
97, 234, 57, 22, 60, 250, 82, 175, 208, 5, 127, 199, 111, 62, 135, 248,
|
||||
174, 169, 211, 58, 66, 154, 106, 195, 245, 171, 17, 187, 182, 179, 0, 243,
|
||||
132, 56, 148, 75, 128, 133, 158, 100, 130, 126, 91, 13, 153, 246, 216, 219,
|
||||
119, 68, 223, 78, 83, 88, 201, 99, 122, 11, 92, 32, 136, 114, 52, 10,
|
||||
138, 30, 48, 183, 156, 35, 61, 26, 143, 74, 251, 94, 129, 162, 63, 152,
|
||||
170, 7, 115, 167, 241, 206, 3, 150, 55, 59, 151, 220, 90, 53, 23, 131,
|
||||
125, 173, 15, 238, 79, 95, 89, 16, 105, 137, 225, 224, 217, 160, 37, 123,
|
||||
118, 73, 2, 157, 46, 116, 9, 145, 134, 228, 207, 212, 202, 215, 69, 229,
|
||||
27, 188, 67, 124, 168, 252, 42, 4, 29, 108, 21, 247, 19, 205, 39, 203,
|
||||
233, 40, 186, 147, 198, 192, 155, 33, 164, 191, 98, 204, 165, 180, 117, 76,
|
||||
140, 36, 210, 172, 41, 54, 159, 8, 185, 232, 113, 196, 231, 47, 146, 120,
|
||||
51, 65, 28, 144, 254, 221, 93, 189, 194, 139, 112, 43, 71, 109, 184, 209
|
||||
};
|
||||
|
|
@ -83,16 +83,17 @@ bool PreflateBlockDecoder::readBlock(PreflateTokenBlock &block, bool &last) {
|
|||
block.uncompressedLen = _output.cacheEndPos() - block.uncompressedStartPos;
|
||||
block.contextLen = -earliest_reference;
|
||||
return true;
|
||||
} else if (litLen <= PreflateConstants::L_CODES) {
|
||||
unsigned lcode = litLen - PreflateConstants::LITERALS - 1;
|
||||
} else {
|
||||
unsigned lcode = litLen - PreflateConstants::NONLEN_CODE_COUNT;
|
||||
if (lcode >= PreflateConstants::LEN_CODE_COUNT) {
|
||||
return false;
|
||||
}
|
||||
unsigned len = PreflateConstants::MIN_MATCH
|
||||
+ PreflateConstants::lengthBaseTable[lcode]
|
||||
+ _readBits(PreflateConstants::lengthExtraTable[lcode]);
|
||||
if (len == 258 && lcode != PreflateConstants::L_CODES - PreflateConstants::LITERALS - 2) {
|
||||
len |= 512;
|
||||
}
|
||||
bool irregular258 = len == 258 && lcode != PreflateConstants::LEN_CODE_COUNT - 1;
|
||||
unsigned dcode = _distDecoder->decode(_input);
|
||||
if (dcode > PreflateConstants::D_CODES) {
|
||||
if (dcode >= PreflateConstants::DIST_CODE_COUNT) {
|
||||
return false;
|
||||
}
|
||||
unsigned dist = 1
|
||||
|
|
@ -102,11 +103,9 @@ bool PreflateBlockDecoder::readBlock(PreflateTokenBlock &block, bool &last) {
|
|||
return false;
|
||||
}
|
||||
_writeReference(dist, len);
|
||||
block.tokens.push_back(PreflateToken(PreflateToken::REFERENCE, len, dist));
|
||||
block.tokens.push_back(PreflateToken(PreflateToken::REFERENCE, len, dist, irregular258));
|
||||
earliest_reference = std::min(earliest_reference, curPos - (int32_t)dist);
|
||||
curPos += len;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -118,17 +117,17 @@ void PreflateBlockDecoder::_setupStaticTables() {
|
|||
}
|
||||
|
||||
bool PreflateBlockDecoder::_readDynamicTables(PreflateTokenBlock& block) {
|
||||
block.nlen = PreflateConstants::LITERALS + 1 + _readBits(5);
|
||||
block.nlen = PreflateConstants::NONLEN_CODE_COUNT + _readBits(5);
|
||||
block.ndist = 1 + _readBits(5);
|
||||
block.ncode = 4 + _readBits(4);
|
||||
if (block.nlen > PreflateConstants::L_CODES || block.ndist > PreflateConstants::D_CODES) {
|
||||
if (block.nlen > PreflateConstants::LITLEN_CODE_COUNT || block.ndist > PreflateConstants::DIST_CODE_COUNT) {
|
||||
return false;
|
||||
}
|
||||
block.treecodes.clear();
|
||||
block.treecodes.reserve(block.nlen + block.ndist + block.ncode);
|
||||
|
||||
unsigned char tcBitLengths[PreflateConstants::BL_CODES];
|
||||
unsigned char ldBitLengths[PreflateConstants::LD_CODES];
|
||||
unsigned char tcBitLengths[PreflateConstants::CODETREE_CODE_COUNT];
|
||||
unsigned char ldBitLengths[PreflateConstants::LITLENDIST_CODE_COUNT];
|
||||
memset(tcBitLengths, 0, sizeof(tcBitLengths));
|
||||
memset(ldBitLengths, 0, sizeof(ldBitLengths));
|
||||
for (unsigned i = 0, n = block.ncode; i < n; ++i) {
|
||||
|
|
@ -136,7 +135,7 @@ bool PreflateBlockDecoder::_readDynamicTables(PreflateTokenBlock& block) {
|
|||
block.treecodes.push_back(tc);
|
||||
tcBitLengths[PreflateConstants::treeCodeOrderTable[i]] = tc;
|
||||
}
|
||||
HuffmanDecoder tcTree(tcBitLengths, PreflateConstants::BL_CODES, true, 7);
|
||||
HuffmanDecoder tcTree(tcBitLengths, PreflateConstants::CODETREE_CODE_COUNT, true, 7);
|
||||
if (tcTree.error()) {
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,15 +40,15 @@ void PreflateBlockReencoder::_setupStaticTables() {
|
|||
}
|
||||
|
||||
bool PreflateBlockReencoder::_buildAndWriteDynamicTables(const PreflateTokenBlock& block) {
|
||||
if (block.ncode < 4 || block.ncode > PreflateConstants::BL_CODES
|
||||
if (block.ncode < 4 || block.ncode > PreflateConstants::CODETREE_CODE_COUNT
|
||||
|| block.treecodes.size() < (size_t)block.ncode
|
||||
|| block.nlen < PreflateConstants::LITERALS + 1
|
||||
|| block.nlen > PreflateConstants::L_CODES
|
||||
|| block.ndist < 1 || block.ndist > PreflateConstants::D_CODES) {
|
||||
|| block.nlen < PreflateConstants::NONLEN_CODE_COUNT
|
||||
|| block.nlen > PreflateConstants::LITLEN_CODE_COUNT
|
||||
|| block.ndist < 1 || block.ndist > PreflateConstants::DIST_CODE_COUNT) {
|
||||
return _error(TREE_OUT_OF_RANGE);
|
||||
}
|
||||
unsigned char tcBitLengths[PreflateConstants::BL_CODES];
|
||||
unsigned char ldBitLengths[PreflateConstants::LD_CODES];
|
||||
unsigned char tcBitLengths[PreflateConstants::CODETREE_CODE_COUNT];
|
||||
unsigned char ldBitLengths[PreflateConstants::LITLENDIST_CODE_COUNT];
|
||||
memset(tcBitLengths, 0, sizeof(tcBitLengths));
|
||||
memset(ldBitLengths, 0, sizeof(ldBitLengths));
|
||||
|
||||
|
|
@ -57,7 +57,7 @@ bool PreflateBlockReencoder::_buildAndWriteDynamicTables(const PreflateTokenBloc
|
|||
_output.put(tc, 3);
|
||||
tcBitLengths[PreflateConstants::treeCodeOrderTable[i]] = tc;
|
||||
}
|
||||
HuffmanEncoder tcTree(tcBitLengths, PreflateConstants::BL_CODES, true);
|
||||
HuffmanEncoder tcTree(tcBitLengths, PreflateConstants::CODETREE_CODE_COUNT, true);
|
||||
if (tcTree.error()) {
|
||||
return _error(BAD_CODE_TREE);
|
||||
}
|
||||
|
|
@ -124,14 +124,12 @@ bool PreflateBlockReencoder::_writeTokens(const std::vector<PreflateToken>& toke
|
|||
_litLenEncoder->encode(_output, literal);
|
||||
} else {
|
||||
// handle irregular length of 258
|
||||
if (token.len == 258 + 512) {
|
||||
unsigned lencode = PreflateConstants::LCode(token.len);
|
||||
_litLenEncoder->encode(_output, PreflateConstants::L_CODES - PreflateConstants::LITERALS - 3);
|
||||
if (token.irregular258) {
|
||||
_litLenEncoder->encode(_output, PreflateConstants::LITLEN_CODE_COUNT - 2);
|
||||
_output.put(31, 5);
|
||||
token.len -= 512;
|
||||
} else {
|
||||
unsigned lencode = PreflateConstants::LCode(token.len);
|
||||
_litLenEncoder->encode(_output, PreflateConstants::LITERALS + 1 + lencode);
|
||||
_litLenEncoder->encode(_output, PreflateConstants::NONLEN_CODE_COUNT + lencode);
|
||||
unsigned lenextra = PreflateConstants::lengthExtraTable[lencode];
|
||||
if (lenextra) {
|
||||
_output.put(token.len - PreflateConstants::MIN_MATCH - PreflateConstants::lengthBaseTable[lencode], lenextra);
|
||||
|
|
@ -146,7 +144,7 @@ bool PreflateBlockReencoder::_writeTokens(const std::vector<PreflateToken>& toke
|
|||
_uncompressedDataPos += token.len;
|
||||
}
|
||||
}
|
||||
_litLenEncoder->encode(_output, PreflateConstants::LITERALS);
|
||||
_litLenEncoder->encode(_output, PreflateConstants::LITERAL_COUNT); // EOB
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -158,7 +156,7 @@ bool PreflateBlockReencoder::writeBlock(const PreflateTokenBlock& block, bool la
|
|||
switch (block.type) {
|
||||
case PreflateTokenBlock::DYNAMIC_HUFF:
|
||||
_output.put(2, 2); //
|
||||
_output.put(block.nlen - PreflateConstants::LITERALS - 1, 5);
|
||||
_output.put(block.nlen - PreflateConstants::NONLEN_CODE_COUNT, 5);
|
||||
_output.put(block.ndist - 1, 5);
|
||||
_output.put(block.ncode - 4, 4);
|
||||
if (!_buildAndWriteDynamicTables(block)) {
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@
|
|||
#include "preflate_statistical_model.h"
|
||||
#include "preflate_token_predictor.h"
|
||||
#include "preflate_tree_predictor.h"
|
||||
#include "preflate_unpack.h"
|
||||
#include "support/bitstream.h"
|
||||
#include "support/memstream.h"
|
||||
#include "support/outputcachestream.h"
|
||||
|
|
@ -31,13 +30,6 @@
|
|||
bool preflate_checker(const std::vector<unsigned char>& deflate_raw) {
|
||||
printf("Checking raw deflate file of size %d\n", (int)deflate_raw.size());
|
||||
|
||||
std::vector<unsigned char> unpacked_output;
|
||||
std::vector<PreflateTokenBlock> blocksRef;
|
||||
if (!preflate_unpack(unpacked_output, blocksRef, deflate_raw)) {
|
||||
printf("inflating error (modified zlib)\n");
|
||||
return false;
|
||||
}
|
||||
printf("Unpacked data has size %d\n", (int)unpacked_output.size());
|
||||
MemStream decIn(deflate_raw);
|
||||
MemStream decUnc;
|
||||
BitInputStream decInBits(decIn);
|
||||
|
|
@ -56,25 +48,14 @@ bool preflate_checker(const std::vector<unsigned char>& deflate_raw) {
|
|||
printf("inflating error (preflate)\n");
|
||||
return false;
|
||||
}
|
||||
if ((last && i + 1 != blocksRef.size())
|
||||
|| (!last && i + 1 == blocksRef.size())) {
|
||||
return false;
|
||||
}
|
||||
if (!isEqual(newBlock, blocksRef[i])) {
|
||||
return false;
|
||||
}
|
||||
blocks.push_back(newBlock);
|
||||
++i;
|
||||
} while (!last);
|
||||
uint8_t remaining_bit_count = (8 - decInBits.bitPos()) & 7;
|
||||
uint8_t remaining_bits = decInBits.get(remaining_bit_count);
|
||||
decOutCache.flush();
|
||||
if (decUnc.data() != unpacked_output) {
|
||||
for (unsigned i = 0, n = std::min(decUnc.data().size(), unpacked_output.size()); i < n; ++i) {
|
||||
if (decUnc.data()[i] != unpacked_output[i]) {
|
||||
printf("xxx %d\n", i);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
std::vector<unsigned char> unpacked_output = decUnc.extractData();
|
||||
printf("Unpacked data has size %d\n", (int)unpacked_output.size());
|
||||
|
||||
// Encode
|
||||
PreflateParameters paramsE = estimatePreflateParameters(unpacked_output, blocks);
|
||||
|
|
@ -102,6 +83,7 @@ bool preflate_checker(const std::vector<unsigned char>& deflate_raw) {
|
|||
tokenPredictorE.updateCounters(&counterE, i);
|
||||
treePredictorE.updateCounters(&counterE, i);
|
||||
}
|
||||
counterE.block.incNonZeroPadding(remaining_bits != 0);
|
||||
|
||||
counterE.print();
|
||||
|
||||
|
|
@ -127,6 +109,14 @@ bool preflate_checker(const std::vector<unsigned char>& deflate_raw) {
|
|||
}
|
||||
tokenPredictorE.encodeEOF(&pcodecE, i, i + 1 == blocks.size());
|
||||
}
|
||||
pcodecE.encodeNonZeroPadding(remaining_bits != 0);
|
||||
if (remaining_bits != 0) {
|
||||
unsigned bitsToSave = bitLength(remaining_bits);
|
||||
pcodecE.encodeValue(bitsToSave, 3);
|
||||
if (bitsToSave > 1) {
|
||||
pcodecE.encodeValue(remaining_bits & ((1 << (bitsToSave - 1)) - 1), bitsToSave - 1);
|
||||
}
|
||||
}
|
||||
if (!codecE.endMetaBlock(pcodecE, unpacked_output.size())) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -244,6 +234,15 @@ bool preflate_checker(const std::vector<unsigned char>& deflate_raw) {
|
|||
deflater.writeBlock(block, eof);
|
||||
++blockno;
|
||||
} while (!eof);
|
||||
bool non_zero_bits = pcodecD.decodeNonZeroPadding();
|
||||
if (non_zero_bits) {
|
||||
unsigned bitsToLoad = pcodecD.decodeValue(3);
|
||||
unsigned padding = 0;
|
||||
if (bitsToLoad > 0) {
|
||||
padding = (1 << (bitsToLoad - 1)) + pcodecD.decodeValue(bitsToLoad - 1);
|
||||
}
|
||||
bos.put(padding, bitsToLoad);
|
||||
}
|
||||
if (!codecD.endMetaBlock(pcodecD)) {
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,12 +22,12 @@ PreflateCompLevelEstimatorState::PreflateCompLevelEstimatorState(
|
|||
const int mbits,
|
||||
const std::vector<unsigned char>& unpacked_output_,
|
||||
const std::vector<PreflateTokenBlock>& blocks_)
|
||||
: predictor(slowHash, slowPreflateParserSettings[5], wbits, mbits)
|
||||
, slowHash(unpacked_output_, mbits)
|
||||
: slowHash(unpacked_output_, mbits)
|
||||
, fastL1Hash(unpacked_output_, mbits)
|
||||
, fastL2Hash(unpacked_output_, mbits)
|
||||
, fastL3Hash(unpacked_output_, mbits)
|
||||
, blocks(blocks_)
|
||||
, wsize(1 << wbits)
|
||||
{
|
||||
memset(&info, 0, sizeof(info));
|
||||
info.possibleCompressionLevels = (1 << 10) - (1 << 1);
|
||||
|
|
@ -68,12 +68,30 @@ void PreflateCompLevelEstimatorState::updateOrSkipHash(const unsigned len) {
|
|||
}
|
||||
slowHash.updateHash(len);
|
||||
}
|
||||
|
||||
unsigned short PreflateCompLevelEstimatorState::matchDepth(
|
||||
const unsigned hashHead,
|
||||
const PreflateToken& targetReference,
|
||||
const PreflateHashChainExt& hash) {
|
||||
unsigned curPos = hash.input().pos();
|
||||
unsigned curMaxDist = std::min(curPos, windowSize());
|
||||
|
||||
unsigned startDepth = hash.getNodeDepth(hashHead);
|
||||
PreflateHashIterator chainIt = hash.iterateFromPos(curPos - targetReference.dist, curPos, curMaxDist);
|
||||
if (!chainIt.curPos || targetReference.dist > curMaxDist) {
|
||||
return 0xffffu;
|
||||
}
|
||||
unsigned endDepth = chainIt.depth();
|
||||
return std::min(startDepth - endDepth, 0xffffu);
|
||||
}
|
||||
|
||||
|
||||
bool PreflateCompLevelEstimatorState::checkMatchSingleFastHash(
|
||||
const PreflateToken& token,
|
||||
const PreflateHashChainExt& hash,
|
||||
const PreflateParserConfig& config,
|
||||
const unsigned hashHead) {
|
||||
unsigned mdepth = predictor.matchDepth(hash.getHead(hashHead), token, hash);
|
||||
unsigned mdepth = matchDepth(hash.getHead(hashHead), token, hash);
|
||||
if (mdepth > config.max_chain) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -99,13 +117,13 @@ void PreflateCompLevelEstimatorState::checkMatch(const PreflateToken& token) {
|
|||
|
||||
info.referenceCount++;
|
||||
|
||||
unsigned short mdepth = predictor.matchDepth(slowHash.getHead(hashHead), token, slowHash);
|
||||
unsigned short mdepth = matchDepth(slowHash.getHead(hashHead), token, slowHash);
|
||||
if (mdepth >= 0x8001) {
|
||||
info.unfoundReferences++;
|
||||
} else {
|
||||
info.maxChainDepth = std::max(info.maxChainDepth, mdepth);
|
||||
}
|
||||
if (token.dist == predictor.currentInputPos()) {
|
||||
if (token.dist == slowHash.input().pos()) {
|
||||
info.matchToStart = true;
|
||||
}
|
||||
if (mdepth == 0) {
|
||||
|
|
@ -152,8 +170,8 @@ void PreflateCompLevelEstimatorState::checkDump(bool early_out) {
|
|||
}
|
||||
void PreflateCompLevelEstimatorState::recommend() {
|
||||
info.recommendedCompressionLevel = 9;
|
||||
info.veryFarMatches = !(info.longestDistAtHop0 <= predictor.windowSize() - PreflateConstants::MIN_LOOKAHEAD
|
||||
&& info.longestDistAtHop1Plus < predictor.windowSize() - PreflateConstants::MIN_LOOKAHEAD);
|
||||
info.veryFarMatches = !(info.longestDistAtHop0 <= windowSize() - PreflateConstants::MIN_LOOKAHEAD
|
||||
&& info.longestDistAtHop1Plus < windowSize() - PreflateConstants::MIN_LOOKAHEAD);
|
||||
info.farLen3Matches = info.longestLen3Dist > 4096;
|
||||
|
||||
info.zlibCompatible = info.possibleCompressionLevels > 1
|
||||
|
|
|
|||
|
|
@ -35,13 +35,13 @@ struct PreflateCompLevelInfo {
|
|||
};
|
||||
|
||||
struct PreflateCompLevelEstimatorState {
|
||||
PreflatePredictorState predictor;
|
||||
PreflateHashChainExt slowHash;
|
||||
PreflateHashChainExt fastL1Hash;
|
||||
PreflateHashChainExt fastL2Hash;
|
||||
PreflateHashChainExt fastL3Hash;
|
||||
const std::vector<PreflateTokenBlock>& blocks;
|
||||
PreflateCompLevelInfo info;
|
||||
uint16_t wsize;
|
||||
|
||||
PreflateCompLevelEstimatorState(const int wbits, const int mbits,
|
||||
const std::vector<unsigned char>& unpacked_output,
|
||||
|
|
@ -56,6 +56,11 @@ private:
|
|||
void updateOrSkipSingleFastHash(PreflateHashChainExt&, const unsigned len, const PreflateParserConfig&);
|
||||
bool checkMatchSingleFastHash(const PreflateToken& token, const PreflateHashChainExt&, const PreflateParserConfig&,
|
||||
const unsigned hashHead);
|
||||
uint16_t matchDepth(const unsigned hashHead, const PreflateToken& targetReference,
|
||||
const PreflateHashChainExt& hash);
|
||||
unsigned windowSize() const {
|
||||
return wsize;
|
||||
}
|
||||
};
|
||||
|
||||
PreflateCompLevelInfo estimatePreflateCompLevel(
|
||||
|
|
|
|||
|
|
@ -63,25 +63,25 @@ const unsigned char PreflateConstants::lengthCodeTable[MAX_MATCH - MIN_MATCH + 1
|
|||
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 28
|
||||
};
|
||||
|
||||
const unsigned char PreflateConstants::lengthBaseTable[LENGTH_CODES] = {
|
||||
const unsigned char PreflateConstants::lengthBaseTable[LEN_CODE_COUNT] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32, 40, 48, 56,
|
||||
64, 80, 96, 112, 128, 160, 192, 224, 255
|
||||
};
|
||||
|
||||
const unsigned short PreflateConstants::distBaseTable[D_CODES] = {
|
||||
const unsigned short PreflateConstants::distBaseTable[DIST_CODE_COUNT] = {
|
||||
0, 1, 2, 3, 4, 6, 8, 12, 16, 24,
|
||||
32, 48, 64, 96, 128, 192, 256, 384, 512, 768,
|
||||
1024, 1536, 2048, 3072, 4096, 6144, 8192, 12288, 16384, 24576
|
||||
};
|
||||
|
||||
const unsigned char PreflateConstants::lengthExtraTable[LENGTH_CODES] = { /* extra bits for each length code */
|
||||
const unsigned char PreflateConstants::lengthExtraTable[LEN_CODE_COUNT] = { /* extra bits for each length code */
|
||||
0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0
|
||||
};
|
||||
|
||||
const unsigned char PreflateConstants::distExtraTable[D_CODES] = { /* extra bits for each distance code */
|
||||
const unsigned char PreflateConstants::distExtraTable[DIST_CODE_COUNT] = { /* extra bits for each distance code */
|
||||
0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13
|
||||
};
|
||||
|
||||
const unsigned char PreflateConstants::treeCodeOrderTable[BL_CODES] = {
|
||||
const unsigned char PreflateConstants::treeCodeOrderTable[CODETREE_CODE_COUNT] = {
|
||||
16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15
|
||||
};
|
||||
|
|
|
|||
|
|
@ -17,12 +17,13 @@
|
|||
|
||||
struct PreflateConstants {
|
||||
enum {
|
||||
LENGTH_CODES = 29,
|
||||
LITERALS = 256,
|
||||
L_CODES = LITERALS + 1 /* eob */ + LENGTH_CODES,
|
||||
D_CODES = 30,
|
||||
LD_CODES = L_CODES + D_CODES,
|
||||
BL_CODES = 19,
|
||||
LITERAL_COUNT = 256,
|
||||
NONLEN_CODE_COUNT = LITERAL_COUNT + 1, // EOB
|
||||
LEN_CODE_COUNT = 29,
|
||||
LITLEN_CODE_COUNT = NONLEN_CODE_COUNT + LEN_CODE_COUNT,
|
||||
DIST_CODE_COUNT = 30,
|
||||
LITLENDIST_CODE_COUNT = LITLEN_CODE_COUNT + DIST_CODE_COUNT,
|
||||
CODETREE_CODE_COUNT = 19,
|
||||
|
||||
MIN_MATCH = 3,
|
||||
MAX_MATCH = 258,
|
||||
|
|
@ -34,12 +35,12 @@ struct PreflateConstants {
|
|||
|
||||
static const unsigned char distCodeTable[512];
|
||||
static const unsigned char lengthCodeTable[MAX_MATCH - MIN_MATCH + 1];
|
||||
static const unsigned char lengthBaseTable[LENGTH_CODES];
|
||||
static const unsigned short distBaseTable[D_CODES];
|
||||
static const unsigned char lengthBaseTable[LEN_CODE_COUNT];
|
||||
static const unsigned short distBaseTable[DIST_CODE_COUNT];
|
||||
|
||||
static const unsigned char lengthExtraTable[LENGTH_CODES];
|
||||
static const unsigned char distExtraTable[D_CODES];
|
||||
static const unsigned char treeCodeOrderTable[BL_CODES];
|
||||
static const unsigned char lengthExtraTable[LEN_CODE_COUNT];
|
||||
static const unsigned char distExtraTable[DIST_CODE_COUNT];
|
||||
static const unsigned char treeCodeOrderTable[CODETREE_CODE_COUNT];
|
||||
|
||||
static inline unsigned DCode(const unsigned dist) {
|
||||
return distCodeTable[dist <= 256 ? dist - 1 : 256 + ((dist - 1) >> 7)];
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@
|
|||
#include "preflate_statistical_model.h"
|
||||
#include "preflate_token_predictor.h"
|
||||
#include "preflate_tree_predictor.h"
|
||||
#include "preflate_unpack.h"
|
||||
#include "support/bitstream.h"
|
||||
#include "support/memstream.h"
|
||||
#include "support/outputcachestream.h"
|
||||
|
|
@ -29,7 +28,8 @@ bool preflate_decode(std::vector<unsigned char>& unpacked_output,
|
|||
std::vector<unsigned char>& preflate_diff,
|
||||
uint64_t& deflate_size,
|
||||
InputStream& deflate_raw,
|
||||
std::function<void(void)> block_callback) {
|
||||
std::function<void(void)> block_callback,
|
||||
const size_t min_deflate_size) {
|
||||
deflate_size = 0;
|
||||
uint64_t deflate_bits = 0;
|
||||
size_t prevBitPos = 0;
|
||||
|
|
@ -62,6 +62,9 @@ bool preflate_decode(std::vector<unsigned char>& unpacked_output,
|
|||
decOutCache.flush();
|
||||
unpacked_output = decUnc.extractData();
|
||||
deflate_size = (deflate_bits + 7) >> 3;
|
||||
if (deflate_size < min_deflate_size) {
|
||||
return false;
|
||||
}
|
||||
uint8_t remaining_bit_count = (8 - deflate_bits) & 7;
|
||||
uint8_t remaining_bits = decInBits.get(remaining_bit_count);
|
||||
|
||||
|
|
@ -97,10 +100,10 @@ bool preflate_decode(std::vector<unsigned char>& unpacked_output,
|
|||
}
|
||||
pcodec.encodeNonZeroPadding(remaining_bits != 0);
|
||||
if (remaining_bits != 0) {
|
||||
unsigned bitsToSave = bitLength(remaining_bit_count);
|
||||
unsigned bitsToSave = bitLength(remaining_bits);
|
||||
pcodec.encodeValue(bitsToSave, 3);
|
||||
if (bitsToSave > 1) {
|
||||
pcodec.encodeValue(remaining_bits, bitsToSave - 1);
|
||||
pcodec.encodeValue(remaining_bits & ((1 << (bitsToSave - 1)) - 1), bitsToSave - 1);
|
||||
}
|
||||
}
|
||||
if (!encoder.endMetaBlock(pcodec, unpacked_output.size())) {
|
||||
|
|
@ -116,5 +119,5 @@ bool preflate_decode(std::vector<unsigned char>& unpacked_output,
|
|||
MemStream mem(deflate_raw);
|
||||
uint64_t raw_size;
|
||||
return preflate_decode(unpacked_output, preflate_diff,
|
||||
raw_size, mem, [] {}) && raw_size == deflate_raw.size();
|
||||
raw_size, mem, [] {}, 0) && raw_size == deflate_raw.size();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ bool preflate_decode(std::vector<unsigned char>& unpacked_output,
|
|||
std::vector<unsigned char>& preflate_diff,
|
||||
uint64_t& deflate_size,
|
||||
InputStream& deflate_raw,
|
||||
std::function<void (void)> block_callback);
|
||||
std::function<void (void)> block_callback,
|
||||
const size_t min_deflate_size);
|
||||
|
||||
#endif /* PREFLATE_DECODER_H */
|
||||
|
|
|
|||
|
|
@ -1,145 +0,0 @@
|
|||
/* Copyright 2018 Dirk Steinke
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "zlib1.2.11.dec/zlib.h"
|
||||
#include "zlib1.2.11.dec/prefdump.h"
|
||||
#include "preflate_dumper.h"
|
||||
|
||||
PreflateDumper::PreflateDumper()
|
||||
: error(false)
|
||||
, hasOpenBlock(false)
|
||||
, wbits(0)
|
||||
, hdr2(0)
|
||||
, dictid(0) {
|
||||
}
|
||||
|
||||
PreflateDumper::~PreflateDumper() {
|
||||
}
|
||||
|
||||
void PreflateDumper::setStreamHeader(int wlen, int zhdr2) {
|
||||
if (hasOpenBlock) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
wbits = wlen;
|
||||
hdr2 = zhdr2;
|
||||
}
|
||||
void PreflateDumper::setDictId(unsigned id) {
|
||||
if (hasOpenBlock) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
dictid = id;
|
||||
}
|
||||
void PreflateDumper::addStoredBlock(int len) {
|
||||
if (hasOpenBlock) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
blocks.push_back(PreflateTokenBlock(PreflateTokenBlock::STORED_X, len));
|
||||
}
|
||||
void PreflateDumper::addHuffBlock(bool dynamic) {
|
||||
if (hasOpenBlock) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
openBlock.type = dynamic ? PreflateTokenBlock::DYNAMIC_HUFF : PreflateTokenBlock::STATIC_HUFF;
|
||||
hasOpenBlock = true;
|
||||
}
|
||||
void PreflateDumper::addDynamicHuffLengths(int nlen, int ndist, int ncode) {
|
||||
if (!hasOpenBlock || openBlock.type != PreflateTokenBlock::DYNAMIC_HUFF) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
openBlock.setHuffLengths(nlen, ndist, ncode);
|
||||
}
|
||||
void PreflateDumper::addDynamicHuffTreeCode(int code) {
|
||||
if (!hasOpenBlock || openBlock.type != PreflateTokenBlock::DYNAMIC_HUFF) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
openBlock.addTreeCode(code);
|
||||
}
|
||||
void PreflateDumper::addLiteral() {
|
||||
if (!hasOpenBlock) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
openBlock.tokens.push_back(PreflateToken(PreflateToken::LITERAL));
|
||||
}
|
||||
void PreflateDumper::addReference(int dist, int len) {
|
||||
if (!hasOpenBlock) {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
openBlock.tokens.push_back(PreflateToken(PreflateToken::REFERENCE, len, dist));
|
||||
}
|
||||
void PreflateDumper::addEOB() {
|
||||
if (hasOpenBlock) {
|
||||
blocks.push_back(std::move(openBlock));
|
||||
hasOpenBlock = false;
|
||||
}
|
||||
}
|
||||
|
||||
void PreflateDumper::addUncompressedData(const unsigned char* data, const unsigned len) {
|
||||
uncompressed.insert(uncompressed.end(), data, data + len);
|
||||
}
|
||||
|
||||
// ------------------------------------
|
||||
|
||||
void prefdump_stream_header(void* dumper, int wlen, int zhdr2) {
|
||||
if (dumper) {
|
||||
((PreflateDumper*)dumper)->setStreamHeader(wlen, zhdr2);
|
||||
}
|
||||
}
|
||||
void prefdump_dictid(void* dumper, uLong dictid) {
|
||||
if (dumper) {
|
||||
((PreflateDumper*)dumper)->setDictId(dictid);
|
||||
}
|
||||
}
|
||||
void prefdump_new_stored_block(void* dumper, int len) {
|
||||
if (dumper) {
|
||||
((PreflateDumper*)dumper)->addStoredBlock(len);
|
||||
}
|
||||
}
|
||||
void prefdump_new_huff_block(void* dumper, int dynamic) {
|
||||
if (dumper) {
|
||||
((PreflateDumper*)dumper)->addHuffBlock(dynamic != 0);
|
||||
}
|
||||
}
|
||||
void prefdump_dyn_huff_lengths(void* dumper, int nlen, int ndist, int ncode) {
|
||||
if (dumper) {
|
||||
((PreflateDumper*)dumper)->addDynamicHuffLengths(nlen, ndist, ncode);
|
||||
}
|
||||
}
|
||||
void prefdump_dyn_huff_tcode(void* dumper, int code) {
|
||||
if (dumper) {
|
||||
((PreflateDumper*)dumper)->addDynamicHuffTreeCode(code);
|
||||
}
|
||||
}
|
||||
void prefdump_literal(void* dumper) {
|
||||
if (dumper) {
|
||||
((PreflateDumper*)dumper)->addLiteral();
|
||||
}
|
||||
}
|
||||
void prefdump_reference(void* dumper, int dist, int len) {
|
||||
if (dumper) {
|
||||
((PreflateDumper*)dumper)->addReference(dist, len);
|
||||
}
|
||||
}
|
||||
void prefdump_eob(void* dumper) {
|
||||
if (dumper) {
|
||||
((PreflateDumper*)dumper)->addEOB();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
/* Copyright 2018 Dirk Steinke
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#ifndef PREFLATE_DUMPER_H
|
||||
#define PREFLATE_DUMPER_H
|
||||
|
||||
#include <vector>
|
||||
#include "preflate_token.h"
|
||||
|
||||
class PreflateDumper {
|
||||
public:
|
||||
PreflateDumper();
|
||||
~PreflateDumper();
|
||||
|
||||
void setStreamHeader(int wlen, int zhdr2);
|
||||
void setDictId(unsigned id);
|
||||
void addStoredBlock(int len);
|
||||
void addHuffBlock(bool dynamic);
|
||||
void addDynamicHuffLengths(int nlen, int ndist, int ncode);
|
||||
void addDynamicHuffTreeCode(int code);
|
||||
void addLiteral();
|
||||
void addReference(int dist, int len);
|
||||
void addEOB();
|
||||
void addUncompressedData(const unsigned char*, const unsigned);
|
||||
|
||||
const std::vector<PreflateTokenBlock>& getBlocks() const {
|
||||
return blocks;
|
||||
}
|
||||
bool hadErrors() const {
|
||||
return error;
|
||||
}
|
||||
const std::vector<unsigned char>& uncompressedData() const {
|
||||
return uncompressed;
|
||||
}
|
||||
|
||||
//private:
|
||||
bool error;
|
||||
bool hasOpenBlock;
|
||||
PreflateTokenBlock openBlock;
|
||||
unsigned char wbits, hdr2;
|
||||
unsigned dictid;
|
||||
std::vector<PreflateTokenBlock> blocks;
|
||||
std::vector<unsigned char> uncompressed;
|
||||
};
|
||||
|
||||
#endif // PREFLATE_DUMPER_H
|
||||
|
|
@ -92,6 +92,9 @@ struct PreflateHashChainExt {
|
|||
unsigned getNodeDepth(const unsigned node) const {
|
||||
return chainDepth[node];
|
||||
}
|
||||
unsigned getRelPosDepth(const unsigned refPos, const unsigned head) const {
|
||||
return chainDepth[head] - chainDepth[refPos - totalShift];
|
||||
}
|
||||
|
||||
PreflateHashIterator iterateFromHead(const unsigned hash, const unsigned refPos, const unsigned maxDist) const {
|
||||
return PreflateHashIterator(prev, chainDepth, refPos - totalShift, maxDist, head[hash & hashMask]);
|
||||
|
|
|
|||
|
|
@ -26,6 +26,10 @@ public:
|
|||
return _pos;
|
||||
}
|
||||
|
||||
const unsigned size() const {
|
||||
return _size;
|
||||
}
|
||||
|
||||
const unsigned char* curChars(int offset = 0) const {
|
||||
return _data + _pos + offset;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,10 +18,12 @@
|
|||
|
||||
PreflatePredictorState::PreflatePredictorState(
|
||||
const PreflateHashChainExt& hash_,
|
||||
const PreflateSeqChain& seq_,
|
||||
const PreflateParserConfig& config_,
|
||||
const int wbits,
|
||||
const int mbits)
|
||||
: hash(hash_)
|
||||
, seq(seq_)
|
||||
, windowBytes(1 << wbits)
|
||||
, maxTokenCount((1 << (6 + mbits)) - 1)
|
||||
, config(config_) {
|
||||
|
|
@ -90,6 +92,56 @@ unsigned PreflatePredictorState::prefixCompare(
|
|||
return scan - s2;
|
||||
}
|
||||
|
||||
unsigned PreflatePredictorState::suffixCompare(
|
||||
const unsigned char* s1,
|
||||
const unsigned char* s2,
|
||||
const unsigned bestLen,
|
||||
const unsigned maxLen) {
|
||||
if (s1[bestLen] != s2[bestLen]) {
|
||||
return 0;
|
||||
}
|
||||
unsigned len = 0;
|
||||
while (s1[len] == s2[len] && ++len < maxLen) {
|
||||
}
|
||||
return len;
|
||||
}
|
||||
|
||||
bool PreflatePredictorState::createMatchHelper(
|
||||
MatchHelper& helper,
|
||||
const unsigned prevLen,
|
||||
const unsigned startPos,
|
||||
const bool veryFarMatches,
|
||||
const bool matchesToStart,
|
||||
const unsigned maxDepth) {
|
||||
helper.maxLen = std::min(totalInputSize() - startPos, (unsigned)PreflateConstants::MAX_MATCH);
|
||||
if (helper.maxLen < std::max<uint32_t>(prevLen + 1, PreflateConstants::MIN_MATCH)) {
|
||||
return false;
|
||||
}
|
||||
helper.startPos = startPos;
|
||||
unsigned maxDistToStart = startPos - (matchesToStart ? 0 : 1);
|
||||
if (veryFarMatches) {
|
||||
helper.curMaxDistHop1Plus
|
||||
= helper.curMaxDistHop0
|
||||
= std::min(maxDistToStart, windowSize());
|
||||
} else {
|
||||
unsigned maxDist = windowSize() - PreflateConstants::MIN_LOOKAHEAD;
|
||||
helper.curMaxDistHop0 = std::min(maxDistToStart, maxDist);
|
||||
helper.curMaxDistHop1Plus = std::min(maxDistToStart, maxDist - 1);
|
||||
}
|
||||
if (maxDepth > 0) {
|
||||
helper.maxChain = maxDepth;
|
||||
helper.niceLen = helper.maxLen;
|
||||
} else {
|
||||
helper.maxChain = maxChainLength();/* max hash chain length */
|
||||
helper.niceLen = std::min(niceMatchLength(), helper.maxLen);
|
||||
|
||||
if (prevLen >= goodMatchLength()) {
|
||||
helper.maxChain >>= 2;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
PreflateToken PreflatePredictorState::match(
|
||||
const unsigned hashHead,
|
||||
const unsigned prevLen,
|
||||
|
|
@ -98,66 +150,139 @@ PreflateToken PreflatePredictorState::match(
|
|||
const bool matchesToStart,
|
||||
const unsigned maxDepth) {
|
||||
PreflateToken bestMatch(PreflateToken::NONE);
|
||||
unsigned maxLen = std::min(availableInputSize() - offset, (unsigned)PreflateConstants::MAX_MATCH);
|
||||
if (maxLen < std::max(prevLen + 1, (unsigned)PreflateConstants::MIN_MATCH)) {
|
||||
MatchHelper h;
|
||||
if (!createMatchHelper(h, prevLen, currentInputPos() + offset,
|
||||
veryFarMatches, matchesToStart, maxDepth)) {
|
||||
return bestMatch;
|
||||
}
|
||||
|
||||
unsigned maxDistHop0 = windowSize() - (veryFarMatches ? 0 : PreflateConstants::MIN_LOOKAHEAD);
|
||||
unsigned maxDistHop1Plus = windowSize() - (veryFarMatches ? 0 : PreflateConstants::MIN_LOOKAHEAD + 1);
|
||||
unsigned curPos = currentInputPos() + offset;
|
||||
unsigned maxDistToStart = curPos - (matchesToStart ? 0 : 1);
|
||||
unsigned curMaxDistHop0 = std::min(maxDistToStart, maxDistHop0);
|
||||
unsigned curMaxDistHop1Plus = std::min(maxDistToStart, maxDistHop1Plus);
|
||||
|
||||
PreflateHashIterator chainIt = iterateFromNode(hashHead, curPos, curMaxDistHop1Plus);
|
||||
if (chainIt.dist() > curMaxDistHop0) {
|
||||
PreflateHashIterator chainIt = iterateFromNode(hashHead, h.startPos, h.curMaxDistHop1Plus);
|
||||
// Handle ZLIB quirk: the very first entry in the hash chain can have a larger
|
||||
// distance than all following entries
|
||||
if (chainIt.dist() > h.curMaxDistHop0) {
|
||||
return bestMatch;
|
||||
}
|
||||
|
||||
const unsigned char* input = inputCursor() + offset;
|
||||
|
||||
unsigned maxChain = maxChainLength();/* max hash chain length */
|
||||
unsigned niceLen = maxDepth > 0 ? maxLen : std::min(niceMatchLength(), maxLen);
|
||||
|
||||
if (prevLen >= goodMatchLength()) {
|
||||
maxChain >>= 2;
|
||||
}
|
||||
if (maxDepth > 0) {
|
||||
maxChain = maxDepth;
|
||||
}
|
||||
|
||||
unsigned bestLen = prevLen;
|
||||
|
||||
do {
|
||||
const unsigned char* match = input - chainIt.dist();
|
||||
|
||||
unsigned matchLength = prefixCompare(match, input, bestLen, maxLen);
|
||||
unsigned matchLength = prefixCompare(match, input, bestLen, h.maxLen);
|
||||
if (matchLength > bestLen) {
|
||||
bestLen = matchLength;
|
||||
bestMatch = PreflateToken(PreflateToken::REFERENCE, matchLength, chainIt.dist());
|
||||
if (bestLen >= niceLen) {
|
||||
if (bestLen >= h.niceLen) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} while (chainIt.next() && maxChain-- > 1);
|
||||
} while (chainIt.next() && h.maxChain-- > 1);
|
||||
return bestMatch;
|
||||
}
|
||||
|
||||
unsigned short PreflatePredictorState::matchDepth(
|
||||
const unsigned hashHead,
|
||||
const PreflateToken& targetReference,
|
||||
const PreflateHashChainExt& hash) {
|
||||
unsigned curPos = currentInputPos();
|
||||
unsigned curMaxDist = std::min(curPos, windowSize());
|
||||
|
||||
unsigned startDepth = hash.getNodeDepth(hashHead);
|
||||
PreflateHashIterator chainIt = hash.iterateFromPos(curPos - targetReference.dist, curPos, curMaxDist);
|
||||
if (!chainIt.curPos || targetReference.dist > curMaxDist) {
|
||||
return 0xffffu;
|
||||
PreflateToken PreflatePredictorState::seqMatch(
|
||||
const unsigned startPos,
|
||||
const unsigned hashHead,
|
||||
const unsigned prevLen,
|
||||
const bool veryFarMatches,
|
||||
const bool matchesToStart,
|
||||
const unsigned maxDepth) {
|
||||
PreflateToken bestMatch(PreflateToken::NONE);
|
||||
MatchHelper h;
|
||||
if (!createMatchHelper(h, prevLen, startPos,
|
||||
veryFarMatches, matchesToStart, maxDepth)) {
|
||||
return bestMatch;
|
||||
}
|
||||
unsigned endDepth = chainIt.depth();
|
||||
return std::min(startDepth - endDepth, 0xffffu);
|
||||
|
||||
PreflateSeqIterator chainIt = seq.iterateFromPos(startPos);
|
||||
if (!chainIt) {
|
||||
return bestMatch;
|
||||
}
|
||||
unsigned curSeqLen = std::min<uint32_t>(seq.len(startPos), h.maxLen);
|
||||
unsigned curMaxDist = h.curMaxDistHop1Plus;
|
||||
unsigned bestLen = prevLen;
|
||||
if (curSeqLen < PreflateConstants::MIN_MATCH) {
|
||||
// startPos is part of a bigger sequence,
|
||||
// and the ZLIB quirk does not apply, yeah!
|
||||
curSeqLen = std::min(chainIt.len() - chainIt.dist(), h.maxLen);
|
||||
if (curSeqLen > prevLen && 1 <= h.curMaxDistHop0) {
|
||||
bestLen = curSeqLen;
|
||||
bestMatch = PreflateToken(PreflateToken::REFERENCE, curSeqLen, 1);
|
||||
}
|
||||
if (bestLen >= h.niceLen || !chainIt.next()) {
|
||||
return bestMatch;
|
||||
}
|
||||
unsigned minDistOff = chainIt.len() - PreflateConstants::MIN_MATCH;
|
||||
if (chainIt.dist() > h.curMaxDistHop1Plus + chainIt.len() - PreflateConstants::MIN_MATCH) {
|
||||
return bestMatch;
|
||||
}
|
||||
} else {
|
||||
unsigned minDistOff = chainIt.len() - PreflateConstants::MIN_MATCH;
|
||||
if (chainIt.dist() > h.curMaxDistHop1Plus + minDistOff) {
|
||||
if (chainIt.dist() > h.curMaxDistHop0 + minDistOff) {
|
||||
return bestMatch;
|
||||
}
|
||||
// Handle ZLIB quirk: the very first entry in the hash chain can have a larger
|
||||
// distance than all following entries
|
||||
unsigned latestPos = h.startPos - chainIt.dist() + minDistOff;
|
||||
unsigned depth = hash.getRelPosDepth(latestPos, hashHead);
|
||||
if (depth == 0) {
|
||||
curMaxDist = h.curMaxDistHop0;
|
||||
}
|
||||
}
|
||||
}
|
||||
const unsigned char* input = inputCursor() + startPos - currentInputPos();
|
||||
unsigned bestSeqLen = std::min(curSeqLen, bestLen);
|
||||
|
||||
do {
|
||||
if (chainIt.len() < bestSeqLen) {
|
||||
// If we do not even meet the already matched number of sequence bytes,
|
||||
// we can just skip this
|
||||
continue;
|
||||
}
|
||||
|
||||
unsigned oldBestSeqLen = bestSeqLen;
|
||||
bestSeqLen = std::min<uint32_t>(std::min<uint32_t>(curSeqLen, chainIt.len()), h.niceLen);
|
||||
unsigned bestDist = chainIt.dist() - chainIt.len() + bestSeqLen;
|
||||
unsigned error = 0;
|
||||
if (bestDist > curMaxDist) {
|
||||
// best subsequence is already beyond the search range
|
||||
error = bestDist - curMaxDist;
|
||||
if (error > chainIt.len() - PreflateConstants::MIN_MATCH) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
unsigned bestChainDepth = hash.getRelPosDepth(h.startPos - bestDist + error, hashHead);
|
||||
if (bestChainDepth >= h.maxChain) {
|
||||
// best subsequence is already beyond the search range
|
||||
error += bestChainDepth - h.maxChain + 1;
|
||||
if (error > chainIt.len() - PreflateConstants::MIN_MATCH) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (error) {
|
||||
if (bestSeqLen > std::max<uint32_t>(oldBestSeqLen, PreflateConstants::MIN_MATCH - 1) + error) {
|
||||
bestMatch = PreflateToken(PreflateToken::REFERENCE, bestSeqLen - error, bestDist - error);
|
||||
}
|
||||
// Since we had to correct the length down, we know that
|
||||
// the comparer cannot find a better match
|
||||
break;
|
||||
}
|
||||
if (bestSeqLen == h.maxLen) {
|
||||
bestMatch = PreflateToken(PreflateToken::REFERENCE, bestSeqLen, bestDist);
|
||||
break;
|
||||
} else {
|
||||
const unsigned char* match = input - bestDist;
|
||||
|
||||
unsigned matchLength = bestSeqLen + suffixCompare(match + bestSeqLen, input + bestSeqLen, std::max(bestLen, bestSeqLen) - bestSeqLen, h.maxLen - bestSeqLen);
|
||||
if (matchLength > bestLen) {
|
||||
bestLen = matchLength;
|
||||
bestMatch = PreflateToken(PreflateToken::REFERENCE, matchLength, bestDist);
|
||||
if (bestLen >= h.niceLen) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
curMaxDist = h.curMaxDistHop1Plus;
|
||||
} while (chainIt.next());
|
||||
return bestMatch;
|
||||
}
|
||||
|
||||
PreflateNextMatchInfo PreflatePredictorState::nextMatchInfo(
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
#include "preflate_input.h"
|
||||
#include "preflate_hash_chain.h"
|
||||
#include "preflate_parser_config.h"
|
||||
#include "preflate_seq_chain.h"
|
||||
#include "preflate_token.h"
|
||||
|
||||
struct PreflatePreviousMatchInfo {
|
||||
|
|
@ -41,11 +42,13 @@ struct PreflateRematchInfo {
|
|||
|
||||
struct PreflatePredictorState {
|
||||
const PreflateHashChainExt& hash;
|
||||
const PreflateSeqChain& seq;
|
||||
unsigned short windowBytes;
|
||||
unsigned maxTokenCount;
|
||||
const PreflateParserConfig& config;
|
||||
|
||||
PreflatePredictorState(const PreflateHashChainExt&,
|
||||
const PreflateSeqChain&,
|
||||
const PreflateParserConfig&,
|
||||
const int wbits,
|
||||
const int mbits);
|
||||
|
|
@ -59,6 +62,9 @@ struct PreflatePredictorState {
|
|||
unsigned windowSize() const {
|
||||
return windowBytes;
|
||||
}
|
||||
unsigned totalInputSize() const {
|
||||
return hash.input().size();
|
||||
}
|
||||
unsigned availableInputSize() const {
|
||||
return hash.input().remaining();
|
||||
}
|
||||
|
|
@ -100,6 +106,23 @@ struct PreflatePredictorState {
|
|||
const unsigned bestLen,
|
||||
const unsigned maxLen);
|
||||
|
||||
static unsigned suffixCompare(
|
||||
const unsigned char* s1,
|
||||
const unsigned char* s2,
|
||||
const unsigned bestLen,
|
||||
const unsigned maxLen);
|
||||
|
||||
bool betterMatchPossible(
|
||||
const unsigned prevLen,
|
||||
const unsigned startPos);
|
||||
|
||||
PreflateToken matchHop0MaxDist(
|
||||
const unsigned hashHead,
|
||||
const unsigned prevLen,
|
||||
const unsigned offset,
|
||||
const bool veryFarMatches,
|
||||
const bool matchesToStart);
|
||||
|
||||
PreflateToken match(
|
||||
const unsigned hashHead,
|
||||
const unsigned prevLen,
|
||||
|
|
@ -107,13 +130,47 @@ struct PreflatePredictorState {
|
|||
const bool veryFarMatches,
|
||||
const bool matchesToStart,
|
||||
const unsigned maxDepth);
|
||||
|
||||
PreflateToken seqMatch(
|
||||
const unsigned startPos,
|
||||
const unsigned hashHead,
|
||||
const unsigned prevLen,
|
||||
const bool veryFarMatches,
|
||||
const bool matchesToStart,
|
||||
const unsigned maxDepth);
|
||||
|
||||
/*
|
||||
unsigned short matchDepth(const unsigned hashHead, const PreflateToken& targetReference,
|
||||
const PreflateHashChainExt&);
|
||||
const PreflateHashChainExt&);*/
|
||||
PreflateNextMatchInfo nextMatchInfo(const unsigned hashHead, const PreflateToken& targetReference,
|
||||
const PreflateHashChainExt&);
|
||||
PreflateRematchInfo rematchInfo(const unsigned hashHead, const PreflateToken& targetReference);
|
||||
unsigned firstMatch(const unsigned len);
|
||||
unsigned hopMatch(const PreflateToken& token, const unsigned hops);
|
||||
|
||||
private:
|
||||
struct MatchHelper {
|
||||
unsigned startPos;
|
||||
unsigned maxLen;
|
||||
unsigned curMaxDistHop0;
|
||||
unsigned curMaxDistHop1Plus;
|
||||
unsigned maxChain;
|
||||
unsigned niceLen;
|
||||
|
||||
bool validHop0Dist(const unsigned d) const {
|
||||
return d <= curMaxDistHop0;
|
||||
}
|
||||
bool validHop1PlusDist(const unsigned d) const {
|
||||
return d <= curMaxDistHop1Plus;
|
||||
}
|
||||
};
|
||||
bool createMatchHelper(
|
||||
MatchHelper& helper,
|
||||
const unsigned prevLen,
|
||||
const unsigned startPos,
|
||||
const bool veryFarMatches,
|
||||
const bool matchesToStart,
|
||||
const unsigned maxDepth);
|
||||
};
|
||||
|
||||
#endif /* PREFLATE_PREDICTOR_STATE_H */
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ bool preflate_reencode(OutputStream& os,
|
|||
deflater.writeBlock(block, eof);
|
||||
block_callback();
|
||||
} while (!eof);
|
||||
bool non_zero_bits = pcodec.decodeValue(1) != 0;
|
||||
bool non_zero_bits = pcodec.decodeNonZeroPadding();
|
||||
if (non_zero_bits) {
|
||||
unsigned bitsToLoad = pcodec.decodeValue(3);
|
||||
unsigned padding = 0;
|
||||
|
|
|
|||
139
contrib/preflate/preflate_seq_chain.cpp
Normal file
139
contrib/preflate/preflate_seq_chain.cpp
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
/* Copyright 2018 Dirk Steinke
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include <algorithm>
|
||||
#include <string.h>
|
||||
#include "preflate_constants.h"
|
||||
#include "preflate_seq_chain.h"
|
||||
|
||||
PreflateSeqChain::PreflateSeqChain(
|
||||
const std::vector<unsigned char>& input_)
|
||||
: _input(input_)
|
||||
, totalShift(-8)
|
||||
, curPos(0) {
|
||||
prev = new SeqChainEntry[1 << 16];
|
||||
memset(heads, 0x00, sizeof(heads));
|
||||
_build(8, std::min<uint32_t>((1 << 16) - 8, _input.remaining()));
|
||||
}
|
||||
PreflateSeqChain::~PreflateSeqChain() {
|
||||
delete[] prev;
|
||||
}
|
||||
|
||||
void PreflateSeqChain::_reshift() {
|
||||
const unsigned short delta = 0x7e00;
|
||||
unsigned remaining = (1 << 16) - (delta + 8);
|
||||
// If the head of large sequence is shifted out,
|
||||
// but the tail remains in the cache,
|
||||
// we need to adapt the head and all pointers to it,
|
||||
// that is all members, the next non-member pointing to it
|
||||
// or heads
|
||||
if (prev[delta + 8].distToNext != 0xffff && prev[delta + 8].length < PreflateConstants::MIN_MATCH) {
|
||||
unsigned d = prev[delta + 8].distToNext;
|
||||
prev[delta + 8].distToNext = 0xffff;
|
||||
prev[delta + 8].length = prev[delta + 8 - d].length - d;
|
||||
for (unsigned i = 3; i < prev[delta + 8].length; ++i) {
|
||||
prev[delta + 8 + i - 2].distToNext -= d;
|
||||
}
|
||||
uint8_t c = *_input.curChars(-(int)remaining);
|
||||
if (heads[c] == delta + 8 - d) {
|
||||
heads[c] += d;
|
||||
} else {
|
||||
for (unsigned i = prev[delta + 8].length; i < remaining; ++i) {
|
||||
if (prev[delta + 8 + i].distToNext == i + d) {
|
||||
prev[delta + 8 + i].distToNext -= d;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (unsigned i = 0; i < 256; ++i) {
|
||||
heads[i] = std::max(heads[i], delta) - delta;
|
||||
}
|
||||
memmove(prev + 8, prev + (delta + 8), sizeof(SeqChainEntry) * remaining);
|
||||
totalShift += delta;
|
||||
_build(8 + remaining, std::min<uint32_t>(delta, _input.remaining()));
|
||||
}
|
||||
void PreflateSeqChain::_build(const unsigned off0, const unsigned size) {
|
||||
if (!size) {
|
||||
return;
|
||||
}
|
||||
const unsigned char* b = _input.curChars();
|
||||
uint8_t curChar = b[0];
|
||||
SeqChainEntry startOfSeq = {0xffff, 0x0}, *ptrToFirstOfSeq;
|
||||
unsigned startOff = off0;
|
||||
prev[off0] = startOfSeq;
|
||||
if (off0 > 8 && curChar == b[-1]) {
|
||||
--startOff;
|
||||
// new block continues the old
|
||||
if (curChar == b[-2]) {
|
||||
--startOff;
|
||||
// this is definitely a sequence
|
||||
if (curChar == b[-3]) {
|
||||
// This was already a sequence in the previous block,
|
||||
// just append
|
||||
startOff = heads[curChar];
|
||||
prev[off0 - 2].distToNext = off0 - startOff - 2;
|
||||
prev[off0 - 1].distToNext = off0 - startOff - 1;
|
||||
prev[off0].distToNext = off0 - startOff;
|
||||
prev[off0].length = 1;
|
||||
} else {
|
||||
// Otherwise enter the sequence in the books
|
||||
prev[startOff].distToNext = startOff - heads[curChar];
|
||||
prev[startOff + 1].distToNext = 1;
|
||||
prev[startOff + 2].distToNext = 2;
|
||||
prev[startOff + 2].length = 1;
|
||||
heads[curChar] = startOff;
|
||||
}
|
||||
} else {
|
||||
prev[startOff + 1].distToNext = 1;
|
||||
prev[startOff + 1].length = 1;
|
||||
}
|
||||
}
|
||||
ptrToFirstOfSeq = &prev[startOff];
|
||||
++ptrToFirstOfSeq->length;
|
||||
|
||||
uint8_t prevChar = curChar;
|
||||
for (unsigned i = 1; i < size; ++i) {
|
||||
curChar = b[i];
|
||||
if (prevChar == curChar) {
|
||||
if (++ptrToFirstOfSeq->length == 3) {
|
||||
prev[startOff].distToNext = startOff - heads[prevChar];
|
||||
heads[prevChar] = startOff;
|
||||
}
|
||||
prev[off0 + i].distToNext = off0 + i - startOff;
|
||||
prev[off0 + i].length = 1;
|
||||
} else {
|
||||
// Last two of a sequence are not a sequence themselves
|
||||
if (ptrToFirstOfSeq->length >= 2) {
|
||||
if (ptrToFirstOfSeq->length >= 3) {
|
||||
prev[off0 + i - 2].distToNext = 0xffff;
|
||||
}
|
||||
prev[off0 + i - 1].distToNext = 0xffff;
|
||||
}
|
||||
prev[off0 + i] = startOfSeq;
|
||||
startOff = off0 + i;
|
||||
ptrToFirstOfSeq = &prev[startOff];
|
||||
++ptrToFirstOfSeq->length;
|
||||
}
|
||||
prevChar = curChar;
|
||||
}
|
||||
// Last two of a sequence are not a sequence themselves
|
||||
if (ptrToFirstOfSeq->length >= 2) {
|
||||
if (ptrToFirstOfSeq->length >= 3) {
|
||||
prev[off0 + size - 2].distToNext = 0xffff;
|
||||
}
|
||||
prev[off0 + size - 1].distToNext = 0xffff;
|
||||
}
|
||||
_input.advance(size);
|
||||
}
|
||||
88
contrib/preflate/preflate_seq_chain.h
Normal file
88
contrib/preflate/preflate_seq_chain.h
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
/* Copyright 2018 Dirk Steinke
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#ifndef PREFLATE_SEQ_CHAIN_H
|
||||
#define PREFLATE_SEQ_CHAIN_H
|
||||
|
||||
#include <algorithm>
|
||||
#include "preflate_input.h"
|
||||
|
||||
struct SeqChainEntry {
|
||||
uint16_t distToNext;
|
||||
uint16_t length;
|
||||
};
|
||||
|
||||
struct PreflateSeqIterator {
|
||||
const SeqChainEntry* chain;
|
||||
const unsigned refPos;
|
||||
unsigned curDist;
|
||||
|
||||
PreflateSeqIterator(
|
||||
const SeqChainEntry* chain_,
|
||||
const unsigned refPos_)
|
||||
: chain(chain_)
|
||||
, refPos(refPos_)
|
||||
, curDist(chain_[refPos_].distToNext) {
|
||||
}
|
||||
|
||||
inline bool valid() const {
|
||||
return curDist <= refPos - 8;
|
||||
}
|
||||
inline bool operator !() const {
|
||||
return !valid();
|
||||
}
|
||||
inline unsigned dist() const {
|
||||
return curDist;
|
||||
}
|
||||
inline uint16_t len() const {
|
||||
return chain[refPos - curDist].length;
|
||||
}
|
||||
inline bool next() {
|
||||
curDist += chain[refPos - curDist].distToNext;
|
||||
return valid();
|
||||
}
|
||||
};
|
||||
|
||||
struct PreflateSeqChain {
|
||||
PreflateInput _input;
|
||||
SeqChainEntry* prev;
|
||||
unsigned totalShift;
|
||||
unsigned curPos;
|
||||
uint16_t heads[256];
|
||||
|
||||
PreflateSeqChain(const std::vector<unsigned char>& input_);
|
||||
~PreflateSeqChain();
|
||||
|
||||
bool valid(const unsigned refPos) const {
|
||||
return prev[refPos - totalShift].distToNext != 0xffff;
|
||||
}
|
||||
uint16_t len(const unsigned refPos) const {
|
||||
return prev[refPos - totalShift].length;
|
||||
}
|
||||
PreflateSeqIterator iterateFromPos(const unsigned refPos) const {
|
||||
return PreflateSeqIterator(prev, refPos - totalShift);
|
||||
}
|
||||
void updateSeq(const unsigned l) {
|
||||
curPos += l;
|
||||
while (curPos - totalShift >= 0xfe08) {
|
||||
_reshift();
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
void _reshift();
|
||||
void _build(const unsigned off0, const unsigned size);
|
||||
};
|
||||
|
||||
#endif /* PREFLATE_SEQ_CHAIN_H */
|
||||
|
|
@ -67,6 +67,8 @@ void PreflateSubModel<N>::build_impl(const unsigned* arr, const unsigned defval,
|
|||
bounds[N] = 1 << 16;
|
||||
}
|
||||
isDefault = N == 0 || bounds[N] == 0 || (bounds[N - 1] == 0 && ids[N - 1] == defval);
|
||||
|
||||
build_scale_down();
|
||||
}
|
||||
template <unsigned N>
|
||||
void PreflateSubModel<N>::buildDefault(const unsigned defval) {
|
||||
|
|
@ -75,46 +77,67 @@ void PreflateSubModel<N>::buildDefault(const unsigned defval) {
|
|||
return;
|
||||
}
|
||||
memset(bounds, 0, N * sizeof(unsigned));
|
||||
memset(scaledDownBounds, 0, N * sizeof(unsigned));
|
||||
bounds[N] = 0x10000;
|
||||
ids[N - 1] = defval;
|
||||
rids[defval] = N - 1;
|
||||
isDefault = true;
|
||||
build_scale_down();
|
||||
}
|
||||
template <unsigned N>
|
||||
void PreflateSubModel<N>::build_scale_down() {
|
||||
unsigned boundBits = 0;
|
||||
for (unsigned i = 0; i <= N; ++i) {
|
||||
boundBits |= bounds[i];
|
||||
}
|
||||
unsigned zeroJunk = bitTrailingZeroes(boundBits);
|
||||
scaleDownBits = (16 - zeroJunk);
|
||||
for (unsigned i = 0; i <= N; ++i) {
|
||||
scaledDownBounds[i] = bounds[i] >> zeroJunk;
|
||||
}
|
||||
|
||||
isFixed = bounds[N - 1] == 0;
|
||||
|
||||
/* for (unsigned i = 0; i <= N; ++i) {
|
||||
scaledDownBounds[i] = bounds[i];
|
||||
}
|
||||
scaleDownBits = 16;*/
|
||||
}
|
||||
|
||||
static void encodeProb(aricoder* codec, const unsigned val) {
|
||||
|
||||
static void encodeProb(ArithmeticEncoder& codec, const unsigned val) {
|
||||
unsigned bits = bitLength(val);
|
||||
// encode shift
|
||||
PreflateBaseModel::encodeValue(codec, bits - 1, 4);
|
||||
codec.encodeBits(bits - 1, 4);
|
||||
// and precision
|
||||
if (bits >= 5) {
|
||||
PreflateBaseModel::encodeValue(codec, (val >> (bits - 5)) & 0xf, 4);
|
||||
codec.encodeBits((val >> (bits - 5)) & 0xf, 4);
|
||||
} else {
|
||||
PreflateBaseModel::encodeValue(codec, val & ~(1 << (bits - 1)), bits - 1);
|
||||
codec.encodeBits(val & ~(1 << (bits - 1)), bits - 1);
|
||||
}
|
||||
}
|
||||
static void encodeId(aricoder* codec,
|
||||
static void encodeId(ArithmeticEncoder& codec,
|
||||
const unsigned id, const unsigned count) {
|
||||
unsigned bits = bitLength(count - 1);
|
||||
PreflateBaseModel::encodeValue(codec, id, bits);
|
||||
codec.encodeBits(id, bits);
|
||||
}
|
||||
static unsigned decodeProb(aricoder* codec) {
|
||||
static unsigned decodeProb(ArithmeticDecoder& codec) {
|
||||
// encode shift
|
||||
unsigned bits = PreflateBaseModel::decodeValue(codec, 4) + 1;
|
||||
unsigned bits = codec.decodeBits(4) + 1;
|
||||
// and precision
|
||||
if (bits >= 5) {
|
||||
return (PreflateBaseModel::decodeValue(codec, 4) | 0x10) << (bits - 5);
|
||||
return (codec.decodeBits(4) | 0x10) << (bits - 5);
|
||||
} else {
|
||||
return PreflateBaseModel::decodeValue(codec, bits - 1) | (1 << (bits - 1));
|
||||
return codec.decodeBits(bits - 1) | (1 << (bits - 1));
|
||||
}
|
||||
}
|
||||
static unsigned decodeId(aricoder* codec, const unsigned count) {
|
||||
static unsigned decodeId(ArithmeticDecoder& codec, const unsigned count) {
|
||||
unsigned bits = bitLength(count - 1);
|
||||
return PreflateBaseModel::decodeValue(codec, bits);
|
||||
return codec.decodeBits(bits);
|
||||
}
|
||||
|
||||
|
||||
template <unsigned N>
|
||||
void PreflateSubModel<N>::write(aricoder* codec, const uint8_t) const {
|
||||
void PreflateSubModel<N>::write(ArithmeticEncoder& codec, const uint8_t) const {
|
||||
unsigned zeros = 0;
|
||||
for (unsigned i = 1; i < N; ++i) {
|
||||
if (!bounds[i]) {
|
||||
|
|
@ -123,7 +146,7 @@ void PreflateSubModel<N>::write(aricoder* codec, const uint8_t) const {
|
|||
break;
|
||||
}
|
||||
}
|
||||
PreflateBaseModel::encodeValue(codec, zeros, bitLength(N - 1));
|
||||
codec.encodeBits(zeros, bitLength(N - 1));
|
||||
// Transmit values
|
||||
for (unsigned i = 1 + zeros; i < N; ++i) {
|
||||
encodeProb(codec, bounds[i] - bounds[i - 1]);
|
||||
|
|
@ -134,8 +157,8 @@ void PreflateSubModel<N>::write(aricoder* codec, const uint8_t) const {
|
|||
}
|
||||
}
|
||||
template <unsigned N>
|
||||
void PreflateSubModel<N>::read(aricoder* codec, const uint8_t) {
|
||||
unsigned zeros = PreflateBaseModel::decodeValue(codec, bitLength(N - 1));
|
||||
void PreflateSubModel<N>::read(ArithmeticDecoder& codec, const uint8_t) {
|
||||
unsigned zeros = codec.decodeBits(bitLength(N - 1));
|
||||
memset(bounds, 0, sizeof(bounds));
|
||||
// Transmit values
|
||||
for (unsigned i = 1 + zeros; i < N; ++i) {
|
||||
|
|
@ -147,6 +170,7 @@ void PreflateSubModel<N>::read(aricoder* codec, const uint8_t) {
|
|||
ids[i] = decodeId(codec, N);
|
||||
rids[ids[i]] = i;
|
||||
}
|
||||
build_scale_down();
|
||||
}
|
||||
|
||||
template <unsigned NEG, unsigned POS>
|
||||
|
|
@ -174,7 +198,7 @@ void PreflateCorrectionSubModel<NEG, POS>::buildDefault(const unsigned defval) {
|
|||
isDefault = sign.isDefault && pos.isDefault && neg.isDefault;
|
||||
}
|
||||
template <unsigned NEG, unsigned POS>
|
||||
void PreflateCorrectionSubModel<NEG, POS>::write(aricoder* codec, const uint8_t prec) const {
|
||||
void PreflateCorrectionSubModel<NEG, POS>::write(ArithmeticEncoder& codec, const uint8_t prec) const {
|
||||
sign.write(codec, prec);
|
||||
if (POS > 0) {
|
||||
pos.write(codec, prec);
|
||||
|
|
@ -184,7 +208,7 @@ void PreflateCorrectionSubModel<NEG, POS>::write(aricoder* codec, const uint8_t
|
|||
}
|
||||
}
|
||||
template <unsigned NEG, unsigned POS>
|
||||
void PreflateCorrectionSubModel<NEG, POS>::read(aricoder* codec, const uint8_t prec) {
|
||||
void PreflateCorrectionSubModel<NEG, POS>::read(ArithmeticDecoder& codec, const uint8_t prec) {
|
||||
sign.read(codec, prec);
|
||||
if (POS > 0) {
|
||||
pos.read(codec, prec);
|
||||
|
|
@ -197,29 +221,32 @@ void PreflateCorrectionSubModel<NEG, POS>::read(aricoder* codec, const uint8_t p
|
|||
// -------------------------------------
|
||||
|
||||
PreflateBaseModel::PreflateBaseModel()
|
||||
: codec(nullptr) {}
|
||||
: encoder(nullptr), decoder(nullptr) {}
|
||||
|
||||
void PreflateBaseModel::setStream(aricoder* codec_) {
|
||||
codec = codec_;
|
||||
void PreflateBaseModel::setEncoderStream(ArithmeticEncoder* codec_) {
|
||||
encoder = codec_;
|
||||
}
|
||||
void PreflateBaseModel::setDecoderStream(ArithmeticDecoder* codec_) {
|
||||
decoder = codec_;
|
||||
}
|
||||
|
||||
template <unsigned N>
|
||||
void PreflateBaseModel::readSubModel(PreflateSubModel<N>& sm, const bool isFullDef, const PreflateModelCodec& cc,
|
||||
const unsigned defVal, const uint8_t prec) {
|
||||
if (isFullDef || cc.nonDefaultValue.decode(codec) == 0) {
|
||||
if (isFullDef || cc.nonDefaultValue.decode(*decoder) == 0) {
|
||||
sm.buildDefault(defVal);
|
||||
} else {
|
||||
sm.read(codec, prec);
|
||||
sm.read(*decoder, prec);
|
||||
}
|
||||
}
|
||||
|
||||
template <unsigned N, unsigned M>
|
||||
void PreflateBaseModel::readSubModel(PreflateCorrectionSubModel<N, M>& sm, const bool isFullDef, const PreflateModelCodec& cc,
|
||||
const unsigned defVal, const uint8_t prec) {
|
||||
if (isFullDef || cc.nonDefaultValue.decode(codec) == 0) {
|
||||
if (isFullDef || cc.nonDefaultValue.decode(*decoder) == 0) {
|
||||
sm.buildDefault(defVal);
|
||||
} else {
|
||||
sm.read(codec, prec);
|
||||
sm.read(*decoder, prec);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -230,9 +257,9 @@ void PreflateBaseModel::writeSubModel(const PreflateSubModel<N>& sm, const bool
|
|||
return;
|
||||
}
|
||||
bool ndef = !sm.isDefault;
|
||||
cc.nonDefaultValue.encode(codec, ndef);
|
||||
cc.nonDefaultValue.encode(*encoder, ndef);
|
||||
if (ndef) {
|
||||
sm.write(codec, prec);
|
||||
sm.write(*encoder, prec);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -243,9 +270,9 @@ void PreflateBaseModel::writeSubModel(const PreflateCorrectionSubModel<N, M>& sm
|
|||
return;
|
||||
}
|
||||
bool ndef = !sm.isDefault;
|
||||
cc.nonDefaultValue.encode(codec, ndef);
|
||||
cc.nonDefaultValue.encode(*encoder, ndef);
|
||||
if (ndef) {
|
||||
sm.write(codec, prec);
|
||||
sm.write(*encoder, prec);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -363,24 +390,23 @@ void PreflateTokenPredictionModel::writeToStream(const PreflateModelCodec& cc) {
|
|||
}
|
||||
|
||||
|
||||
PreflatePredictionModel::PreflatePredictionModel()
|
||||
: codec(nullptr)
|
||||
, data(nullptr) {
|
||||
}
|
||||
PreflatePredictionModel::~PreflatePredictionModel() {
|
||||
delete codec;
|
||||
delete data;
|
||||
}
|
||||
PreflatePredictionModel::PreflatePredictionModel() {}
|
||||
PreflatePredictionModel::~PreflatePredictionModel() {}
|
||||
|
||||
void PreflatePredictionModel::read(const PreflateStatisticsCounter& model, const PreflateModelCodec& cc) {
|
||||
block.read(model.block, cc);
|
||||
treecode.read(model.treecode, cc);
|
||||
token.read(model.token, cc);
|
||||
}
|
||||
void PreflatePredictionModel::setStream(aricoder* codec) {
|
||||
block.setStream(codec);
|
||||
treecode.setStream(codec);
|
||||
token.setStream(codec);
|
||||
void PreflatePredictionModel::setEncoderStream(ArithmeticEncoder* codec) {
|
||||
block.setEncoderStream(codec);
|
||||
treecode.setEncoderStream(codec);
|
||||
token.setEncoderStream(codec);
|
||||
}
|
||||
void PreflatePredictionModel::setDecoderStream(ArithmeticDecoder* codec) {
|
||||
block.setDecoderStream(codec);
|
||||
treecode.setDecoderStream(codec);
|
||||
token.setDecoderStream(codec);
|
||||
}
|
||||
void PreflatePredictionModel::readFromStream(const PreflateModelCodec& cc) {
|
||||
block.readFromStream(cc);
|
||||
|
|
@ -445,10 +471,10 @@ void PreflateModelCodec::read(const PreflateStatisticsCounter& m) {
|
|||
MBprecisionP1 = 16;
|
||||
}
|
||||
|
||||
void PreflateModelCodec::readFromStream(aricoder* codec) {
|
||||
blockFullDefault = PreflateBaseModel::decodeValue(codec, 1);
|
||||
treecodeFullDefault = PreflateBaseModel::decodeValue(codec, 1);
|
||||
tokenFullDefault = PreflateBaseModel::decodeValue(codec, 1);
|
||||
void PreflateModelCodec::readFromStream(ArithmeticDecoder& codec) {
|
||||
blockFullDefault = codec.decodeBits(1);
|
||||
treecodeFullDefault = codec.decodeBits(1);
|
||||
tokenFullDefault = codec.decodeBits(1);
|
||||
totalModels = 0;
|
||||
if (!blockFullDefault) {
|
||||
totalModels += PreflateStatisticsCounter::BlockPrediction::totalModels();
|
||||
|
|
@ -468,50 +494,67 @@ void PreflateModelCodec::readFromStream(aricoder* codec) {
|
|||
MBprecision = 16;
|
||||
MBprecisionP1 = 16;
|
||||
}
|
||||
void PreflateModelCodec::writeToStream(aricoder* codec) {
|
||||
PreflateBaseModel::encodeValue(codec, blockFullDefault, 1);
|
||||
PreflateBaseModel::encodeValue(codec, treecodeFullDefault, 1);
|
||||
PreflateBaseModel::encodeValue(codec, tokenFullDefault, 1);
|
||||
PreflateBaseModel::encodeValue(codec, defaultingModels, bitLength(totalModels));
|
||||
void PreflateModelCodec::writeToStream(ArithmeticEncoder& codec) {
|
||||
codec.encodeBits(blockFullDefault, 1);
|
||||
codec.encodeBits(treecodeFullDefault, 1);
|
||||
codec.encodeBits(tokenFullDefault, 1);
|
||||
codec.encodeBits(defaultingModels, bitLength(totalModels));
|
||||
}
|
||||
|
||||
// ------------------------------------
|
||||
|
||||
PreflatePredictionEncoder::PreflatePredictionEncoder()
|
||||
: storage(nullptr)
|
||||
, bos(nullptr)
|
||||
, encoder(nullptr)
|
||||
{}
|
||||
|
||||
void PreflatePredictionEncoder::start(const PreflatePredictionModel& model_, const PreflateParameters& params_,
|
||||
const unsigned modelId_) {
|
||||
PreflatePredictionModel::operator =(model_);
|
||||
params = params_;
|
||||
data = new iostream(nullptr, TYPE_MEMORY, 0, MODE_WRITE);
|
||||
codec = new aricoder(data, MODE_WRITE);
|
||||
modelid = modelId_;
|
||||
|
||||
setStream(codec);
|
||||
storage = new MemStream;
|
||||
bos = new BitOutputStream(*storage);
|
||||
encoder = new ArithmeticEncoder(*bos);
|
||||
setEncoderStream(encoder);
|
||||
}
|
||||
std::vector<uint8_t> PreflatePredictionEncoder::end() {
|
||||
setStream(nullptr);
|
||||
setEncoderStream(nullptr);
|
||||
encoder->flush();
|
||||
delete encoder;
|
||||
|
||||
delete codec;
|
||||
codec = nullptr;
|
||||
std::vector<unsigned char> result(data->getptr(), data->getptr() + data->getsize());
|
||||
delete data;
|
||||
data = nullptr;
|
||||
bos->flush();
|
||||
delete bos;
|
||||
|
||||
std::vector<unsigned char> result = storage->extractData();
|
||||
delete storage;
|
||||
return result;
|
||||
}
|
||||
|
||||
PreflatePredictionDecoder::PreflatePredictionDecoder()
|
||||
: storage(nullptr)
|
||||
, bis(nullptr)
|
||||
, decoder(nullptr) {}
|
||||
|
||||
void PreflatePredictionDecoder::start(const PreflatePredictionModel& model_, const PreflateParameters& params_,
|
||||
const std::vector<uint8_t>& storage, size_t off0, size_t size) {
|
||||
const std::vector<uint8_t>& storage_, size_t off0, size_t size) {
|
||||
PreflatePredictionModel::operator =(model_);
|
||||
params = params_;
|
||||
data = new iostream(const_cast<uint8_t*>(storage.data() + off0), TYPE_MEMORY, (int)size, MODE_READ);
|
||||
codec = new aricoder(data, MODE_READ);
|
||||
setStream(codec);
|
||||
storage = new MemStream(storage_, off0, size);
|
||||
bis = new BitInputStream(*storage);
|
||||
decoder = new ArithmeticDecoder(*bis);
|
||||
setDecoderStream(decoder);
|
||||
}
|
||||
void PreflatePredictionDecoder::end() {
|
||||
setStream(nullptr);
|
||||
delete codec;
|
||||
codec = nullptr;
|
||||
delete data;
|
||||
data = nullptr;
|
||||
setDecoderStream(nullptr);
|
||||
delete decoder;
|
||||
delete bis;
|
||||
delete storage;
|
||||
decoder = nullptr;
|
||||
bis = nullptr;
|
||||
storage = nullptr;
|
||||
}
|
||||
|
||||
// ------------------------------------
|
||||
|
|
@ -592,15 +635,18 @@ std::vector<unsigned char> PreflateMetaEncoder::finish() {
|
|||
bos.put(mt.params.matchesToStartDetected, 1);
|
||||
}
|
||||
bos.put(mt.params.log2OfMaxChainDepthM1, 4);
|
||||
iostream tmp_mem(nullptr, TYPE_MEMORY, 0, MODE_WRITE);
|
||||
MemStream tmp_data;
|
||||
{
|
||||
aricoder tmp_codec(&tmp_mem, MODE_WRITE);
|
||||
mt.mcodec.writeToStream(&tmp_codec);
|
||||
mt.model.setStream(&tmp_codec);
|
||||
BitOutputStream tmp_bos(tmp_data);
|
||||
ArithmeticEncoder tmp_codec(tmp_bos);
|
||||
mt.mcodec.writeToStream(tmp_codec);
|
||||
mt.model.setEncoderStream(&tmp_codec);
|
||||
mt.model.writeToStream(mt.mcodec);
|
||||
mt.model.setStream(nullptr);
|
||||
mt.model.setEncoderStream(nullptr);
|
||||
tmp_codec.flush();
|
||||
tmp_bos.flush();
|
||||
}
|
||||
std::vector<uint8_t> tmp_res(tmp_mem.getptr(), tmp_mem.getptr() + tmp_mem.getsize());
|
||||
std::vector<uint8_t> tmp_res = tmp_data.extractData();
|
||||
// write length (vli) and model data
|
||||
bos.putVLI(tmp_res.size());
|
||||
bos.putBytes(tmp_res.data(), tmp_res.size());
|
||||
|
|
@ -686,19 +732,17 @@ PreflateMetaDecoder::PreflateMetaDecoder(const std::vector<uint8_t>& reconData_,
|
|||
mt.params.log2OfMaxChainDepthM1 = bis.get(4);
|
||||
// read length (vli) and model data
|
||||
size_t res_size = bis.getVLI();
|
||||
std::vector<uint8_t> modeldata(res_size);
|
||||
if (bis.getBytes(modeldata.data(), modeldata.size()) != res_size) {
|
||||
inError = true;
|
||||
return;
|
||||
}
|
||||
// interpret model data
|
||||
iostream tmp_mem(modeldata.data(), TYPE_MEMORY, modeldata.size(), MODE_READ);
|
||||
{
|
||||
aricoder tmp_codec(&tmp_mem, MODE_READ);
|
||||
mt.mcodec.readFromStream(&tmp_codec);
|
||||
mt.model.setStream(&tmp_codec);
|
||||
MemStream tmp_mem;
|
||||
bis.copyBytesTo(tmp_mem, res_size);
|
||||
tmp_mem.seek(0);
|
||||
BitInputStream tmp_bis(tmp_mem);
|
||||
ArithmeticDecoder tmp_codec(tmp_bis);
|
||||
mt.mcodec.readFromStream(tmp_codec);
|
||||
mt.model.setDecoderStream(&tmp_codec);
|
||||
mt.model.readFromStream(mt.mcodec);
|
||||
mt.model.setStream(nullptr);
|
||||
mt.model.setDecoderStream(nullptr);
|
||||
}
|
||||
}
|
||||
mb.modelId = modelList.size();
|
||||
|
|
|
|||
|
|
@ -16,8 +16,7 @@
|
|||
#define PREFLATE_STATISTICAL_CODEC_H
|
||||
|
||||
#include <vector>
|
||||
#include "packARI/source/bitops.h"
|
||||
#include "packARI/source/aricoder.h"
|
||||
#include "support/arithmetic_coder.h"
|
||||
#include "support/bit_helper.h"
|
||||
#include "support/bitstream.h"
|
||||
#include "support/memstream.h"
|
||||
|
|
@ -38,37 +37,32 @@ struct PreflateSubModel {
|
|||
arr[i] = bounds[rids[i] + 1] - bounds[rids[i]];
|
||||
}
|
||||
}
|
||||
void read(aricoder*, const uint8_t);
|
||||
void write(aricoder*, const uint8_t) const;
|
||||
void encode(aricoder* codec, const unsigned item) const {
|
||||
symbol s;
|
||||
s.scale = 1 << 16;
|
||||
s.low_count = bounds[rids[item]];
|
||||
s.high_count = bounds[rids[item] + 1];
|
||||
codec->encode(&s);
|
||||
}
|
||||
unsigned decode(aricoder* codec) const {
|
||||
symbol s;
|
||||
s.scale = 1 << 16;
|
||||
unsigned cnt = codec->decode_count(&s);
|
||||
for (unsigned i = 0; i < N; ++i) {
|
||||
if (cnt < bounds[i + 1]) {
|
||||
s.low_count = bounds[i];
|
||||
s.high_count = bounds[i + 1];
|
||||
codec->decode(&s);
|
||||
return ids[i];
|
||||
}
|
||||
void read(ArithmeticDecoder&, const uint8_t);
|
||||
void write(ArithmeticEncoder&, const uint8_t) const;
|
||||
void encode(ArithmeticEncoder& codec, const unsigned item) const {
|
||||
if (!isFixed) {
|
||||
size_t idx = rids[item];
|
||||
codec.encodeShiftScale(scaleDownBits, scaledDownBounds[idx], scaledDownBounds[idx + 1]);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
unsigned decode(ArithmeticDecoder& codec) const {
|
||||
if (isFixed) {
|
||||
return ids[N - 1];
|
||||
}
|
||||
unsigned val = codec.decodeShiftScale(scaleDownBits, scaledDownBounds, N);
|
||||
return ids[val];
|
||||
}
|
||||
bool isEqualTo(const PreflateSubModel<N>& m) const;
|
||||
|
||||
unsigned bounds[N + 1];
|
||||
unsigned scaledDownBounds[N + 1];
|
||||
unsigned short ids[N + 1], rids[N + 1];
|
||||
bool isDefault;
|
||||
uint8_t scaleDownBits;
|
||||
bool isDefault, isFixed;
|
||||
|
||||
private:
|
||||
void build_impl(const unsigned* arr, const unsigned defval, const uint8_t prec);
|
||||
void build_scale_down();
|
||||
template <unsigned NEG, unsigned POS>
|
||||
friend struct PreflateCorrectionSubModel;
|
||||
};
|
||||
|
|
@ -82,9 +76,9 @@ struct PreflateCorrectionSubModel {
|
|||
build_impl(arr, defval, prec);
|
||||
}
|
||||
void buildDefault(const unsigned defval);
|
||||
void read(aricoder*, const uint8_t);
|
||||
void write(aricoder*, const uint8_t) const;
|
||||
void encode(aricoder* codec, const unsigned actvalue,
|
||||
void read(ArithmeticDecoder&, const uint8_t);
|
||||
void write(ArithmeticEncoder&, const uint8_t) const;
|
||||
void encode(ArithmeticEncoder& codec, const unsigned actvalue,
|
||||
const unsigned refvalue,
|
||||
const unsigned minvalue,
|
||||
const unsigned maxvalue) {
|
||||
|
|
@ -97,7 +91,7 @@ struct PreflateCorrectionSubModel {
|
|||
sign.encode(codec, 1);
|
||||
if (diff >= (int)POS) {
|
||||
pos.encode(codec, POS - 1);
|
||||
PreflateBaseModel::encodeValue(codec, diff - POS, bitLength(maxvalue - POS - refvalue));
|
||||
codec.encodeBits(diff - POS, bitLength(maxvalue - POS - refvalue));
|
||||
} else {
|
||||
pos.encode(codec, diff - 1);
|
||||
}
|
||||
|
|
@ -105,13 +99,13 @@ struct PreflateCorrectionSubModel {
|
|||
sign.encode(codec, 2);
|
||||
if (-diff >= (int)NEG) {
|
||||
neg.encode(codec, NEG - 1);
|
||||
PreflateBaseModel::encodeValue(codec, -diff - NEG, bitLength(refvalue - NEG - minvalue));
|
||||
codec.encodeBits(-diff - NEG, bitLength(refvalue - NEG - minvalue));
|
||||
} else {
|
||||
neg.encode(codec, -diff - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
unsigned decode(aricoder* codec,
|
||||
unsigned decode(ArithmeticDecoder& codec,
|
||||
const unsigned refvalue,
|
||||
const unsigned minvalue,
|
||||
const unsigned maxvalue) {
|
||||
|
|
@ -122,14 +116,14 @@ struct PreflateCorrectionSubModel {
|
|||
if (s == 1) {
|
||||
int diff = pos.decode(codec);
|
||||
if (diff >= (int)(POS - 1)) {
|
||||
return refvalue + PreflateBaseModel::decodeValue(codec, bitLength(maxvalue - POS - refvalue)) + POS;
|
||||
return refvalue + codec.decodeBits(bitLength(maxvalue - POS - refvalue)) + POS;
|
||||
} else {
|
||||
return refvalue + diff + 1;
|
||||
}
|
||||
} else {
|
||||
int diff = neg.decode(codec);
|
||||
if (diff >= (int)(NEG - 1)) {
|
||||
return refvalue - PreflateBaseModel::decodeValue(codec, bitLength(refvalue - NEG - minvalue)) - NEG;
|
||||
return refvalue - codec.decodeBits(bitLength(refvalue - NEG - minvalue)) - NEG;
|
||||
} else {
|
||||
return refvalue - diff - 1;
|
||||
}
|
||||
|
|
@ -158,36 +152,30 @@ struct PreflateModelCodec {
|
|||
PreflateModelCodec();
|
||||
void initDefault();
|
||||
void read(const PreflateStatisticsCounter&);
|
||||
void readFromStream(aricoder*);
|
||||
void writeToStream(aricoder*);
|
||||
void readFromStream(ArithmeticDecoder&);
|
||||
void writeToStream(ArithmeticEncoder&);
|
||||
};
|
||||
|
||||
struct PreflateBaseModel {
|
||||
public:
|
||||
PreflateBaseModel();
|
||||
void setStream(aricoder*);
|
||||
void setEncoderStream(ArithmeticEncoder*);
|
||||
void setDecoderStream(ArithmeticDecoder*);
|
||||
|
||||
static void encodeValue(aricoder* codec, const unsigned value, const unsigned maxBits) {
|
||||
symbol s;
|
||||
s.scale = 1 << maxBits;
|
||||
s.low_count = value;
|
||||
s.high_count = value + 1;
|
||||
codec->encode(&s);
|
||||
static void encodeValue(ArithmeticEncoder& codec, const unsigned value, const unsigned maxBits) {
|
||||
#ifdef _DEBUG
|
||||
_ASSERT(value < (1 << maxBits));
|
||||
#endif
|
||||
return codec.encodeBits(value, maxBits);
|
||||
}
|
||||
void encodeValue(const unsigned value, const unsigned maxBits) {
|
||||
encodeValue(codec, value, maxBits);
|
||||
encodeValue(*encoder, value, maxBits);
|
||||
}
|
||||
static unsigned decodeValue(aricoder* codec, const unsigned maxBits) {
|
||||
symbol s;
|
||||
s.scale = 1 << maxBits;
|
||||
unsigned cnt = codec->decode_count(&s);
|
||||
s.low_count = cnt;
|
||||
s.high_count = cnt + 1;
|
||||
codec->decode(&s);
|
||||
return cnt;
|
||||
static unsigned decodeValue(ArithmeticDecoder& codec, const unsigned maxBits) {
|
||||
return codec.decodeBits(maxBits);
|
||||
}
|
||||
unsigned decodeValue(const unsigned maxBits) {
|
||||
return decodeValue(codec, maxBits);
|
||||
return decodeValue(*decoder, maxBits);
|
||||
}
|
||||
|
||||
protected:
|
||||
|
|
@ -207,7 +195,8 @@ protected:
|
|||
void writeSubModel(const PreflateCorrectionSubModel<N, M>& sm, const bool isFullDef, const PreflateModelCodec& cc,
|
||||
const unsigned defVal, const uint8_t prec = 16);
|
||||
|
||||
aricoder* codec;
|
||||
ArithmeticEncoder* encoder;
|
||||
ArithmeticDecoder* decoder;
|
||||
};
|
||||
|
||||
struct PreflateBlockPredictionModel : public PreflateBaseModel {
|
||||
|
|
@ -217,23 +206,23 @@ public:
|
|||
void writeToStream(const PreflateModelCodec&);
|
||||
|
||||
unsigned decodeBlockType() {
|
||||
return blockType.decode(codec);
|
||||
return blockType.decode(*decoder);
|
||||
}
|
||||
bool decodeEOBMisprediction() {
|
||||
return EOBMisprediction.decode(codec);
|
||||
return EOBMisprediction.decode(*decoder);
|
||||
}
|
||||
bool decodeNonZeroPadding() {
|
||||
return nonZeroPadding.decode(codec);
|
||||
return nonZeroPadding.decode(*decoder);
|
||||
}
|
||||
|
||||
void encodeBlockType(const unsigned type) {
|
||||
blockType.encode(codec, type);
|
||||
blockType.encode(*encoder, type);
|
||||
}
|
||||
void encodeEOBMisprediction(const bool misprediction) {
|
||||
EOBMisprediction.encode(codec, misprediction);
|
||||
EOBMisprediction.encode(*encoder, misprediction);
|
||||
}
|
||||
void encodeNonZeroPadding(const bool nonzeropadding) {
|
||||
nonZeroPadding.encode(codec, nonzeropadding);
|
||||
nonZeroPadding.encode(*encoder, nonzeropadding);
|
||||
}
|
||||
|
||||
bool isEqualTo(const PreflateBlockPredictionModel& m) const;
|
||||
|
|
@ -252,51 +241,51 @@ public:
|
|||
void writeToStream(const PreflateModelCodec& cc);
|
||||
|
||||
bool decodeTreeCodeCountMisprediction() {
|
||||
return TCCountMisprediction.decode(codec);
|
||||
return TCCountMisprediction.decode(*decoder);
|
||||
}
|
||||
bool decodeLiteralCountMisprediction() {
|
||||
return LCountMisprediction.decode(codec);
|
||||
return LCountMisprediction.decode(*decoder);
|
||||
}
|
||||
bool decodeDistanceCountMisprediction() {
|
||||
return DCountMisprediction.decode(codec);
|
||||
return DCountMisprediction.decode(*decoder);
|
||||
}
|
||||
int decodeTreeCodeBitLengthCorrection(unsigned predval) {
|
||||
return TCBitlengthCorrection.decode(codec, predval, 0, 7);
|
||||
return TCBitlengthCorrection.decode(*decoder, predval, 0, 7);
|
||||
}
|
||||
unsigned decodeLDTypeCorrection(unsigned predtype) {
|
||||
return DerivedLDTypeReplacement[predtype].decode(codec);
|
||||
return DerivedLDTypeReplacement[predtype].decode(*decoder);
|
||||
}
|
||||
unsigned decodeRepeatCountCorrection(const unsigned predval, const unsigned ldtype) {
|
||||
static const uint8_t minVal[4] = {0, 3, 3, 11};
|
||||
static const uint8_t lenVal[4] = {0, 3, 7, 127};
|
||||
return LDRepeatCountCorrection.decode(codec, predval, minVal[ldtype], minVal[ldtype] + lenVal[ldtype]);
|
||||
return LDRepeatCountCorrection.decode(*decoder, predval, minVal[ldtype], minVal[ldtype] + lenVal[ldtype]);
|
||||
}
|
||||
int decodeLDBitLengthCorrection(unsigned predval) {
|
||||
return LDBitlengthCorrection.decode(codec, predval, 0, 15);
|
||||
return LDBitlengthCorrection.decode(*decoder, predval, 0, 15);
|
||||
}
|
||||
|
||||
void encodeTreeCodeCountMisprediction(const bool misprediction) {
|
||||
TCCountMisprediction.encode(codec, misprediction);
|
||||
TCCountMisprediction.encode(*encoder, misprediction);
|
||||
}
|
||||
void encodeLiteralCountMisprediction(const bool misprediction) {
|
||||
LCountMisprediction.encode(codec, misprediction);
|
||||
LCountMisprediction.encode(*encoder, misprediction);
|
||||
}
|
||||
void encodeDistanceCountMisprediction(const bool misprediction) {
|
||||
DCountMisprediction.encode(codec, misprediction);
|
||||
DCountMisprediction.encode(*encoder, misprediction);
|
||||
}
|
||||
void encodeTreeCodeBitLengthCorrection(const unsigned predval, const unsigned actval) {
|
||||
TCBitlengthCorrection.encode(codec, actval, predval, 0, 7);
|
||||
TCBitlengthCorrection.encode(*encoder, actval, predval, 0, 7);
|
||||
}
|
||||
void encodeLDTypeCorrection(const unsigned predval, const unsigned actval) {
|
||||
DerivedLDTypeReplacement[predval].encode(codec, actval);
|
||||
DerivedLDTypeReplacement[predval].encode(*encoder, actval);
|
||||
}
|
||||
void encodeRepeatCountCorrection(const unsigned predval, const unsigned actval, unsigned ldtype) {
|
||||
static const uint8_t minVal[4] = {0, 3, 3, 11};
|
||||
static const uint8_t lenVal[4] = {0, 3, 7, 127};
|
||||
LDRepeatCountCorrection.encode(codec, actval, predval, minVal[ldtype], minVal[ldtype] + lenVal[ldtype]);
|
||||
LDRepeatCountCorrection.encode(*encoder, actval, predval, minVal[ldtype], minVal[ldtype] + lenVal[ldtype]);
|
||||
}
|
||||
void encodeLDBitLengthCorrection(const unsigned predval, const unsigned actval) {
|
||||
LDBitlengthCorrection.encode(codec, actval, predval, 0, 15);
|
||||
LDBitlengthCorrection.encode(*encoder, actval, predval, 0, 15);
|
||||
}
|
||||
|
||||
bool isEqualTo(const PreflateTreeCodePredictionModel& m) const;
|
||||
|
|
@ -322,41 +311,41 @@ public:
|
|||
void writeToStream(const PreflateModelCodec& cc);
|
||||
|
||||
bool decodeLiteralPredictionWrong() {
|
||||
return LITMisprediction.decode(codec);
|
||||
return LITMisprediction.decode(*decoder);
|
||||
}
|
||||
bool decodeReferencePredictionWrong() {
|
||||
return REFMisprediction.decode(codec);
|
||||
return REFMisprediction.decode(*decoder);
|
||||
}
|
||||
int decodeLenCorrection(const unsigned predval) {
|
||||
return LENCorrection.decode(codec, predval, 3, 258);
|
||||
return LENCorrection.decode(*decoder, predval, 3, 258);
|
||||
}
|
||||
unsigned decodeDistOnlyCorrection() {
|
||||
return DISTOnlyCorrection.decode(codec, 0, 0, 32767);
|
||||
return DISTOnlyCorrection.decode(*decoder, 0, 0, 32767);
|
||||
}
|
||||
unsigned decodeDistAfterLenCorrection() {
|
||||
return DISTAfterLenCorrection.decode(codec, 0, 0, 32767);
|
||||
return DISTAfterLenCorrection.decode(*decoder, 0, 0, 32767);
|
||||
}
|
||||
bool decodeIrregularLen258() {
|
||||
return IrregularLen258Encoding.decode(codec);
|
||||
return IrregularLen258Encoding.decode(*decoder);
|
||||
}
|
||||
|
||||
void encodeLiteralPredictionWrong(const bool misprediction) {
|
||||
LITMisprediction.encode(codec, misprediction);
|
||||
LITMisprediction.encode(*encoder, misprediction);
|
||||
}
|
||||
void encodeReferencePredictionWrong(const bool misprediction) {
|
||||
REFMisprediction.encode(codec, misprediction);
|
||||
REFMisprediction.encode(*encoder, misprediction);
|
||||
}
|
||||
void encodeLenCorrection(const unsigned predval, const unsigned actval) {
|
||||
LENCorrection.encode(codec, actval, predval, 3, 258);
|
||||
LENCorrection.encode(*encoder, actval, predval, 3, 258);
|
||||
}
|
||||
void encodeDistOnlyCorrection(const unsigned hops) {
|
||||
DISTOnlyCorrection.encode(codec, hops, 0, 0, 32767);
|
||||
DISTOnlyCorrection.encode(*encoder, hops, 0, 0, 32767);
|
||||
}
|
||||
void encodeDistAfterLenCorrection(const unsigned hops) {
|
||||
DISTAfterLenCorrection.encode(codec, hops, 0, 0, 32767);
|
||||
DISTAfterLenCorrection.encode(*encoder, hops, 0, 0, 32767);
|
||||
}
|
||||
void encodeIrregularLen258(const bool irregular) {
|
||||
IrregularLen258Encoding.encode(codec, irregular);
|
||||
IrregularLen258Encoding.encode(*encoder, irregular);
|
||||
}
|
||||
|
||||
bool isEqualTo(const PreflateTokenPredictionModel& m) const;
|
||||
|
|
@ -375,7 +364,8 @@ struct PreflatePredictionModel {
|
|||
~PreflatePredictionModel();
|
||||
|
||||
void read(const PreflateStatisticsCounter& model, const PreflateModelCodec& cc);
|
||||
void setStream(aricoder* codec);
|
||||
void setEncoderStream(ArithmeticEncoder* codec);
|
||||
void setDecoderStream(ArithmeticDecoder* codec);
|
||||
void readFromStream(const PreflateModelCodec& cc);
|
||||
void writeToStream(const PreflateModelCodec& cc);
|
||||
|
||||
|
|
@ -388,16 +378,16 @@ protected:
|
|||
PreflateTreeCodePredictionModel treecode;
|
||||
// Tokens
|
||||
PreflateTokenPredictionModel token;
|
||||
iostream* data;
|
||||
aricoder* codec;
|
||||
};
|
||||
|
||||
struct PreflatePredictionEncoder : public PreflatePredictionModel {
|
||||
PreflatePredictionEncoder();
|
||||
|
||||
void start(const PreflatePredictionModel&, const PreflateParameters&, const unsigned modelId);
|
||||
std::vector<uint8_t> end();
|
||||
|
||||
void encodeValue(const unsigned value, const unsigned maxBits) {
|
||||
PreflateBaseModel::encodeValue(codec, value, maxBits);
|
||||
encoder->encodeBits(value, maxBits);
|
||||
}
|
||||
|
||||
// Block
|
||||
|
|
@ -463,15 +453,19 @@ struct PreflatePredictionEncoder : public PreflatePredictionModel {
|
|||
private:
|
||||
PreflateParameters params;
|
||||
unsigned modelid;
|
||||
MemStream* storage;
|
||||
BitOutputStream* bos;
|
||||
ArithmeticEncoder* encoder;
|
||||
};
|
||||
|
||||
struct PreflatePredictionDecoder : public PreflatePredictionModel {
|
||||
PreflatePredictionDecoder();
|
||||
void start(const PreflatePredictionModel&, const PreflateParameters&,
|
||||
const std::vector<uint8_t>&, size_t off0, size_t size);
|
||||
void end();
|
||||
|
||||
unsigned decodeValue(const unsigned maxBits) {
|
||||
return PreflateBaseModel::decodeValue(codec, maxBits);
|
||||
return decoder->decodeBits(maxBits);
|
||||
}
|
||||
// Block
|
||||
unsigned decodeBlockType() {
|
||||
|
|
@ -526,8 +520,10 @@ struct PreflatePredictionDecoder : public PreflatePredictionModel {
|
|||
}
|
||||
|
||||
private:
|
||||
iostream* data;
|
||||
PreflateParameters params;
|
||||
MemStream* storage;
|
||||
BitInputStream* bis;
|
||||
ArithmeticDecoder* decoder;
|
||||
};
|
||||
|
||||
struct PreflateMetaEncoder {
|
||||
|
|
|
|||
|
|
@ -29,12 +29,14 @@ struct PreflateToken {
|
|||
enum typeNon {
|
||||
NONE
|
||||
};
|
||||
unsigned short len;
|
||||
unsigned short len : 9;
|
||||
unsigned short irregular258 : 1;
|
||||
unsigned short dist;
|
||||
|
||||
PreflateToken(typeNon n) : len(0), dist(0) {}
|
||||
PreflateToken(typeLit l) : len(1), dist(0) {}
|
||||
PreflateToken(typeRef r, unsigned short l, unsigned short d) : len(l), dist(d) {}
|
||||
PreflateToken(typeNon n) : len(0), irregular258(0), dist(0) {}
|
||||
PreflateToken(typeLit l) : len(1), irregular258(0), dist(0) {}
|
||||
PreflateToken(typeRef r, unsigned short l, unsigned short d, bool irregular258_ = false)
|
||||
: len(l), irregular258(irregular258_), dist(d) {}
|
||||
};
|
||||
|
||||
struct PreflateTokenBlock {
|
||||
|
|
|
|||
|
|
@ -21,8 +21,9 @@
|
|||
PreflateTokenPredictor::PreflateTokenPredictor(
|
||||
const PreflateParameters& params_,
|
||||
const std::vector<unsigned char>& dump)
|
||||
: state(hash, params_.config(), params_.windowBits, params_.memLevel)
|
||||
: state(hash, seq, params_.config(), params_.windowBits, params_.memLevel)
|
||||
, hash(dump, params_.memLevel)
|
||||
, seq(dump)
|
||||
, params(params_)
|
||||
, predictionFailure(false)
|
||||
, fast(params_.isFastCompressor())
|
||||
|
|
@ -33,6 +34,7 @@ PreflateTokenPredictor::PreflateTokenPredictor(
|
|||
if (state.availableInputSize() >= 2) {
|
||||
hash.updateRunningHash(state.inputCursor()[0]);
|
||||
hash.updateRunningHash(state.inputCursor()[1]);
|
||||
seq.updateSeq(2);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -40,11 +42,12 @@ bool PreflateTokenPredictor::predictEOB() {
|
|||
return state.availableInputSize() == 0 || currentTokenCount == state.maxTokenCount;
|
||||
}
|
||||
void PreflateTokenPredictor::commitToken(const PreflateToken& t) {
|
||||
if (fast && (t.len & 511) > state.lazyMatchLength()) {
|
||||
hash.skipHash(t.len & 511);
|
||||
if (fast && t.len > state.lazyMatchLength()) {
|
||||
hash.skipHash(t.len);
|
||||
} else {
|
||||
hash.updateHash(t.len & 511);
|
||||
hash.updateHash(t.len);
|
||||
}
|
||||
seq.updateSeq(t.len);
|
||||
}
|
||||
# define TOO_FAR 4096
|
||||
/* Matches of length 3 are discarded if their distance exceeds TOO_FAR */
|
||||
|
|
@ -53,17 +56,23 @@ PreflateToken PreflateTokenPredictor::predictToken() {
|
|||
if (state.currentInputPos() == 0 || state.availableInputSize() < PreflateConstants::MIN_MATCH) {
|
||||
return PreflateToken(PreflateToken::LITERAL);
|
||||
}
|
||||
unsigned hash = state.calculateHash();
|
||||
unsigned head = state.getCurrentHashHead(hash);
|
||||
PreflateToken match(PreflateToken::NONE);
|
||||
|
||||
unsigned hash = state.calculateHash();
|
||||
if (pendingToken.len > 1) {
|
||||
match = pendingToken;
|
||||
} else {
|
||||
match = state.match(head, prevLen, 0,
|
||||
params.veryFarMatchesDetected,
|
||||
params.matchesToStartDetected,
|
||||
params.zlibCompatible ? 0 : (1 << params.log2OfMaxChainDepthM1));
|
||||
unsigned head = state.getCurrentHashHead(hash);
|
||||
if (!fast && seq.valid(state.currentInputPos())) {
|
||||
match = state.seqMatch(state.currentInputPos(), head, prevLen,
|
||||
params.veryFarMatchesDetected,
|
||||
params.matchesToStartDetected,
|
||||
params.zlibCompatible ? 0 : (1 << params.log2OfMaxChainDepthM1));
|
||||
} else {
|
||||
match = state.match(head, prevLen, 0,
|
||||
params.veryFarMatchesDetected,
|
||||
params.matchesToStartDetected,
|
||||
params.zlibCompatible ? 0 : (1 << params.log2OfMaxChainDepthM1));
|
||||
}
|
||||
}
|
||||
prevLen = 0;
|
||||
pendingToken = PreflateToken(PreflateToken::NONE);
|
||||
|
|
@ -78,23 +87,32 @@ PreflateToken PreflateTokenPredictor::predictToken() {
|
|||
}
|
||||
|
||||
if (match.len < state.lazyMatchLength() && state.availableInputSize() >= (unsigned)match.len + 2) {
|
||||
PreflateToken matchNext(PreflateToken::NONE);
|
||||
unsigned hashNext = state.calculateHashNext();
|
||||
unsigned headNext = state.getCurrentHashHead(hashNext);
|
||||
PreflateToken matchNext = state.match(headNext, match.len, 1,
|
||||
params.veryFarMatchesDetected,
|
||||
params.matchesToStartDetected,
|
||||
params.zlibCompatible ? 0 : (2 << params.log2OfMaxChainDepthM1));
|
||||
if (((hashNext ^ hash) & this->hash.hashMask) == 0) {
|
||||
unsigned maxSize = std::min(state.availableInputSize() - 1, (unsigned)PreflateConstants::MAX_MATCH);
|
||||
unsigned rle = 1;
|
||||
const unsigned char *c = state.inputCursor();
|
||||
unsigned char b = c[0];
|
||||
while (rle < maxSize && c[1 + rle] == b) {
|
||||
++rle;
|
||||
}
|
||||
if (rle >= matchNext.len) {
|
||||
matchNext.len = rle;
|
||||
matchNext.dist = 1;
|
||||
if (!fast && seq.valid(state.currentInputPos() + 1)) {
|
||||
matchNext = state.seqMatch(state.currentInputPos() + 1, headNext, match.len,
|
||||
params.veryFarMatchesDetected,
|
||||
params.matchesToStartDetected,
|
||||
params.zlibCompatible ? 0 : (2 << params.log2OfMaxChainDepthM1));
|
||||
} else {
|
||||
matchNext = state.match(headNext, match.len, 1,
|
||||
params.veryFarMatchesDetected,
|
||||
params.matchesToStartDetected,
|
||||
params.zlibCompatible ? 0 : (2 << params.log2OfMaxChainDepthM1));
|
||||
|
||||
if (((hashNext ^ hash) & this->hash.hashMask) == 0) {
|
||||
unsigned maxSize = std::min(state.availableInputSize() - 1, (unsigned)PreflateConstants::MAX_MATCH);
|
||||
unsigned rle = 1;
|
||||
const unsigned char *c = state.inputCursor();
|
||||
unsigned char b = c[0];
|
||||
while (rle < maxSize && c[1 + rle] == b) {
|
||||
++rle;
|
||||
}
|
||||
if (rle > match.len && rle >= matchNext.len) {
|
||||
matchNext.len = rle;
|
||||
matchNext.dist = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (matchNext.len > match.len) {
|
||||
|
|
@ -159,6 +177,7 @@ void PreflateTokenPredictor::analyzeBlock(
|
|||
if (analysis.type == PreflateTokenBlock::STORED) {
|
||||
analysis.tokenCount = block.uncompressedLen;
|
||||
hash.updateHash(block.uncompressedLen);
|
||||
seq.updateSeq(block.uncompressedLen);
|
||||
analysis.inputEOF = state.availableInputSize() == 0;
|
||||
analysis.paddingBits = block.paddingBits;
|
||||
analysis.paddingCounts = block.paddingBitCount;
|
||||
|
|
@ -167,8 +186,6 @@ void PreflateTokenPredictor::analyzeBlock(
|
|||
|
||||
for (unsigned i = 0, n = block.tokens.size(); i < n; ++i) {
|
||||
PreflateToken targetToken = block.tokens[i];
|
||||
unsigned orgTargetTokenLen = targetToken.len;
|
||||
targetToken.len &= 511;
|
||||
if (predictEOB()) {
|
||||
analysis.blockSizePredicted = false;
|
||||
}
|
||||
|
|
@ -216,9 +233,9 @@ void PreflateTokenPredictor::analyzeBlock(
|
|||
}
|
||||
}
|
||||
}
|
||||
if ((orgTargetTokenLen & 511) == 258) {
|
||||
if (targetToken.len == 258) {
|
||||
analysis.tokenInfo[currentTokenCount] += 16;
|
||||
if (orgTargetTokenLen & 512) {
|
||||
if (targetToken.irregular258) {
|
||||
analysis.tokenInfo[currentTokenCount] += 32;
|
||||
}
|
||||
}
|
||||
|
|
@ -246,7 +263,7 @@ void PreflateTokenPredictor::encodeBlock(
|
|||
unsigned bitsToSave = bitLength(analysis.paddingBits);
|
||||
codec->encodeValue(bitsToSave, 3);
|
||||
if (bitsToSave > 1) {
|
||||
codec->encodeValue(analysis.paddingBits, bitsToSave - 1);
|
||||
codec->encodeValue(analysis.paddingBits & ((1 << (bitsToSave - 1)) - 1), bitsToSave - 1);
|
||||
}
|
||||
}
|
||||
return;
|
||||
|
|
@ -390,6 +407,7 @@ PreflateTokenBlock PreflateTokenPredictor::decodeBlock(
|
|||
}
|
||||
}
|
||||
hash.updateHash(block.uncompressedLen);
|
||||
seq.updateSeq(block.uncompressedLen);
|
||||
return block;
|
||||
case PreflateTokenBlock::STATIC_HUFF:
|
||||
block.type = PreflateTokenBlock::STATIC_HUFF;
|
||||
|
|
@ -462,9 +480,7 @@ PreflateTokenBlock PreflateTokenPredictor::decodeBlock(
|
|||
}
|
||||
}
|
||||
if (predictedToken.len == 258) {
|
||||
if (codec->decodeIrregularLen258()) {
|
||||
predictedToken.len |= 512;
|
||||
}
|
||||
predictedToken.irregular258 = codec->decodeIrregularLen258();
|
||||
}
|
||||
block.tokens.push_back(predictedToken);
|
||||
commitToken(predictedToken);
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ struct PreflateStatisticalCodec;
|
|||
struct PreflateTokenPredictor {
|
||||
PreflatePredictorState state;
|
||||
PreflateHashChainExt hash;
|
||||
PreflateSeqChain seq;
|
||||
PreflateParameters params;
|
||||
bool predictionFailure;
|
||||
bool fast;
|
||||
|
|
|
|||
|
|
@ -90,10 +90,10 @@ unsigned PreflateTreePredictor::calcBitLengths(
|
|||
const unsigned symCount,
|
||||
const unsigned maxBits,
|
||||
const unsigned minMaxCode) {
|
||||
FreqIdxPair toSort[PreflateConstants::L_CODES];
|
||||
TreeNode nodes[PreflateConstants::L_CODES * 2 + 1];
|
||||
unsigned char nodeBitLen[PreflateConstants::L_CODES * 2 + 1];
|
||||
unsigned char nodeDepth[PreflateConstants::L_CODES * 2 + 1];
|
||||
FreqIdxPair toSort[PreflateConstants::LITLEN_CODE_COUNT];
|
||||
TreeNode nodes[PreflateConstants::LITLEN_CODE_COUNT * 2 + 1];
|
||||
unsigned char nodeBitLen[PreflateConstants::LITLEN_CODE_COUNT * 2 + 1];
|
||||
unsigned char nodeDepth[PreflateConstants::LITLEN_CODE_COUNT * 2 + 1];
|
||||
memset(nodeBitLen, 0, sizeof(nodeBitLen));
|
||||
memset(nodeDepth, 0, sizeof(nodeDepth));
|
||||
unsigned maxCode = 0, len = 0, nodeCount = 0, nodeId = symCount;
|
||||
|
|
@ -240,7 +240,7 @@ void PreflateTreePredictor::predictLDTrees(
|
|||
const unsigned symDCount,
|
||||
const unsigned char* targetCodes,
|
||||
const unsigned targetCodeSize) {
|
||||
memset(frequencies, 0, sizeof(unsigned) * PreflateConstants::BL_CODES);
|
||||
memset(frequencies, 0, sizeof(unsigned) * PreflateConstants::CODETREE_CODE_COUNT);
|
||||
const unsigned char* ptr = symBitLen;
|
||||
const unsigned char* code = targetCodes;
|
||||
unsigned codeSize = targetCodeSize;
|
||||
|
|
@ -315,8 +315,8 @@ void PreflateTreePredictor::collectTokenStatistics(
|
|||
unsigned& Lcount,
|
||||
unsigned& Dcount,
|
||||
const PreflateTokenBlock& block) {
|
||||
memset(Lcodes, 0, sizeof(unsigned) * PreflateConstants::L_CODES);
|
||||
memset(Dcodes, 0, sizeof(unsigned) * PreflateConstants::D_CODES);
|
||||
memset(Lcodes, 0, sizeof(unsigned) * PreflateConstants::LITLEN_CODE_COUNT);
|
||||
memset(Dcodes, 0, sizeof(unsigned) * PreflateConstants::DIST_CODE_COUNT);
|
||||
Lcount = 0;
|
||||
Dcount = 0;
|
||||
for (unsigned i = 0, n = block.tokens.size(); i < n; ++i) {
|
||||
|
|
@ -326,7 +326,7 @@ void PreflateTreePredictor::collectTokenStatistics(
|
|||
Lcount++;
|
||||
input.advance(1);
|
||||
} else {
|
||||
Lcodes[PreflateConstants::LITERALS + 1 + PreflateConstants::LCode(targetToken.len)]++;
|
||||
Lcodes[PreflateConstants::NONLEN_CODE_COUNT + PreflateConstants::LCode(targetToken.len)]++;
|
||||
Lcount++;
|
||||
Dcodes[PreflateConstants::DCode(targetToken.dist)]++;
|
||||
Dcount++;
|
||||
|
|
@ -338,19 +338,19 @@ void PreflateTreePredictor::collectTokenStatistics(
|
|||
unsigned PreflateTreePredictor::buildLBitlenghs(
|
||||
unsigned char bitLengths[],
|
||||
unsigned Lcodes[]) {
|
||||
return calcBitLengths(bitLengths, Lcodes, PreflateConstants::L_CODES, 15, PreflateConstants::LITERALS + 1);
|
||||
return calcBitLengths(bitLengths, Lcodes, PreflateConstants::LITLEN_CODE_COUNT, 15, PreflateConstants::NONLEN_CODE_COUNT);
|
||||
}
|
||||
unsigned PreflateTreePredictor::buildDBitlenghs(
|
||||
unsigned char bitLengths[],
|
||||
unsigned Dcodes[]) {
|
||||
return calcBitLengths(bitLengths, Dcodes, PreflateConstants::D_CODES, 15, 0);
|
||||
return calcBitLengths(bitLengths, Dcodes, PreflateConstants::DIST_CODE_COUNT, 15, 0);
|
||||
}
|
||||
unsigned PreflateTreePredictor::buildTCBitlengths(
|
||||
unsigned char (&simpleCodeTree)[PreflateConstants::BL_CODES],
|
||||
unsigned (&BLfreqs)[PreflateConstants::BL_CODES]) {
|
||||
unsigned char (&simpleCodeTree)[PreflateConstants::CODETREE_CODE_COUNT],
|
||||
unsigned (&BLfreqs)[PreflateConstants::CODETREE_CODE_COUNT]) {
|
||||
memset(simpleCodeTree, 0, sizeof(simpleCodeTree));
|
||||
calcBitLengths(simpleCodeTree, BLfreqs, PreflateConstants::BL_CODES, 7, 0);
|
||||
unsigned predictedCTreeSize = PreflateConstants::BL_CODES;
|
||||
calcBitLengths(simpleCodeTree, BLfreqs, PreflateConstants::CODETREE_CODE_COUNT, 7, 0);
|
||||
unsigned predictedCTreeSize = PreflateConstants::CODETREE_CODE_COUNT;
|
||||
while (predictedCTreeSize > 4
|
||||
&& simpleCodeTree[PreflateConstants::treeCodeOrderTable[predictedCTreeSize - 1]] == 0) {
|
||||
--predictedCTreeSize;
|
||||
|
|
@ -371,11 +371,11 @@ void PreflateTreePredictor::analyzeBlock(
|
|||
return;
|
||||
}
|
||||
|
||||
unsigned Lcodes[PreflateConstants::L_CODES], Dcodes[PreflateConstants::D_CODES];
|
||||
unsigned Lcodes[PreflateConstants::LITLEN_CODE_COUNT], Dcodes[PreflateConstants::DIST_CODE_COUNT];
|
||||
unsigned Lcount = 0, Dcount = 0;
|
||||
collectTokenStatistics(Lcodes, Dcodes, Lcount, Dcount, block);
|
||||
|
||||
unsigned char bitLengths[PreflateConstants::LD_CODES];
|
||||
unsigned char bitLengths[PreflateConstants::LITLENDIST_CODE_COUNT];
|
||||
memset(bitLengths, 0, sizeof(bitLengths));
|
||||
unsigned predictedLTreeSize = buildLBitlenghs(bitLengths, Lcodes);
|
||||
analysis.tokenInfo.push_back(predictedLTreeSize != block.nlen);
|
||||
|
|
@ -391,12 +391,12 @@ void PreflateTreePredictor::analyzeBlock(
|
|||
}
|
||||
predictedDTreeSize = block.ndist;
|
||||
|
||||
unsigned BLfreqs[PreflateConstants::BL_CODES];
|
||||
unsigned BLfreqs[PreflateConstants::CODETREE_CODE_COUNT];
|
||||
const unsigned char* targetCodes = &block.treecodes[0];
|
||||
unsigned targetCodeSize = block.treecodes.size();
|
||||
predictLDTrees(analysis, BLfreqs, bitLengths, predictedLTreeSize, predictedDTreeSize, targetCodes + block.ncode, targetCodeSize - block.ncode);
|
||||
|
||||
unsigned char simpleCodeTree[PreflateConstants::BL_CODES];
|
||||
unsigned char simpleCodeTree[PreflateConstants::CODETREE_CODE_COUNT];
|
||||
unsigned predictedCTreeSize = buildTCBitlengths(simpleCodeTree, BLfreqs);
|
||||
analysis.tokenInfo.push_back(block.ncode);
|
||||
analysis.tokenInfo.push_back(predictedCTreeSize != block.ncode);
|
||||
|
|
@ -419,7 +419,7 @@ void PreflateTreePredictor::encodeBlock(
|
|||
unsigned char info = analysis.tokenInfo[infoPos++];
|
||||
codec->encodeLiteralCountMisprediction(info);
|
||||
if (info) {
|
||||
codec->encodeValue(analysis.correctives[correctivePos++] - PreflateConstants::LITERALS - 1, 5);
|
||||
codec->encodeValue(analysis.correctives[correctivePos++] - PreflateConstants::NONLEN_CODE_COUNT, 5);
|
||||
}
|
||||
info = analysis.tokenInfo[infoPos++];
|
||||
codec->encodeDistanceCountMisprediction(info);
|
||||
|
|
@ -524,7 +524,7 @@ unsigned PreflateTreePredictor::reconstructLDTrees(
|
|||
const unsigned char* symBitLen,
|
||||
const unsigned symLCount,
|
||||
const unsigned symDCount) {
|
||||
memset(frequencies, 0, sizeof(unsigned) * PreflateConstants::BL_CODES);
|
||||
memset(frequencies, 0, sizeof(unsigned) * PreflateConstants::CODETREE_CODE_COUNT);
|
||||
const unsigned char* ptr = symBitLen;
|
||||
unsigned osize = 0;
|
||||
unsigned count1 = symLCount;
|
||||
|
|
@ -601,15 +601,15 @@ bool PreflateTreePredictor::decodeBlock(
|
|||
return true;
|
||||
}
|
||||
|
||||
unsigned Lcodes[PreflateConstants::L_CODES], Dcodes[PreflateConstants::D_CODES];
|
||||
unsigned Lcodes[PreflateConstants::LITLEN_CODE_COUNT], Dcodes[PreflateConstants::DIST_CODE_COUNT];
|
||||
unsigned Lcount = 0, Dcount = 0;
|
||||
collectTokenStatistics(Lcodes, Dcodes, Lcount, Dcount, block);
|
||||
|
||||
unsigned char bitLengths[PreflateConstants::LD_CODES];
|
||||
unsigned char bitLengths[PreflateConstants::LITLENDIST_CODE_COUNT];
|
||||
memset(bitLengths, 0, sizeof(bitLengths));
|
||||
unsigned predictedLTreeSize = buildLBitlenghs(bitLengths, Lcodes);
|
||||
if (codec->decodeLiteralCountMisprediction()) {
|
||||
predictedLTreeSize = codec->decodeValue(5) + PreflateConstants::LITERALS + 1;
|
||||
predictedLTreeSize = codec->decodeValue(5) + PreflateConstants::NONLEN_CODE_COUNT;
|
||||
}
|
||||
block.nlen = predictedLTreeSize;
|
||||
|
||||
|
|
@ -619,21 +619,21 @@ bool PreflateTreePredictor::decodeBlock(
|
|||
}
|
||||
block.ndist = predictedDTreeSize;
|
||||
|
||||
unsigned BLfreqs[PreflateConstants::BL_CODES];
|
||||
unsigned char compressedLDtrees[PreflateConstants::LD_CODES];
|
||||
unsigned targetCodeSize = reconstructLDTrees(codec, BLfreqs, compressedLDtrees, PreflateConstants::LD_CODES,
|
||||
unsigned BLfreqs[PreflateConstants::CODETREE_CODE_COUNT];
|
||||
unsigned char compressedLDtrees[PreflateConstants::LITLENDIST_CODE_COUNT];
|
||||
unsigned targetCodeSize = reconstructLDTrees(codec, BLfreqs, compressedLDtrees, PreflateConstants::LITLENDIST_CODE_COUNT,
|
||||
bitLengths, predictedLTreeSize, predictedDTreeSize);
|
||||
if (predictionFailure) {
|
||||
return false;
|
||||
}
|
||||
|
||||
unsigned char simpleCodeTree[PreflateConstants::BL_CODES];
|
||||
unsigned char simpleCodeTree[PreflateConstants::CODETREE_CODE_COUNT];
|
||||
unsigned predictedCTreeSize = buildTCBitlengths(simpleCodeTree, BLfreqs);
|
||||
if (codec->decodeTreeCodeCountMisprediction()) {
|
||||
predictedCTreeSize = codec->decodeValue(4) + 4;
|
||||
}
|
||||
block.ncode = predictedCTreeSize;
|
||||
unsigned char shuffledCodeTree[PreflateConstants::BL_CODES];
|
||||
unsigned char shuffledCodeTree[PreflateConstants::CODETREE_CODE_COUNT];
|
||||
for (unsigned i = 0; i < predictedCTreeSize; ++i) {
|
||||
unsigned predictedBL = simpleCodeTree[PreflateConstants::treeCodeOrderTable[i]];
|
||||
shuffledCodeTree[i] = codec->decodeTreeCodeBitLengthCorrection(predictedBL);
|
||||
|
|
|
|||
|
|
@ -54,8 +54,8 @@ struct PreflateTreePredictor {
|
|||
unsigned char bitLengths[],
|
||||
unsigned Dcodes[]);
|
||||
unsigned buildTCBitlengths(
|
||||
unsigned char (&bitLengths)[PreflateConstants::BL_CODES],
|
||||
unsigned (&BLfreqs)[PreflateConstants::BL_CODES]);
|
||||
unsigned char (&bitLengths)[PreflateConstants::CODETREE_CODE_COUNT],
|
||||
unsigned (&BLfreqs)[PreflateConstants::CODETREE_CODE_COUNT]);
|
||||
|
||||
|
||||
unsigned calcBitLengths(unsigned char* symBitLen,
|
||||
|
|
|
|||
|
|
@ -1,69 +0,0 @@
|
|||
/* Copyright 2018 Dirk Steinke
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include <stdlib.h>
|
||||
#include "preflate_dumper.h"
|
||||
#include "preflate_unpack.h"
|
||||
#include "zlib1.2.11.dec/zlib.h"
|
||||
|
||||
void* zalloc(void*, uInt items, uInt size) {
|
||||
return malloc(items * size);
|
||||
}
|
||||
void zfree(void*, void* addr) {
|
||||
free(addr);
|
||||
}
|
||||
|
||||
bool preflate_unpack(std::vector<unsigned char>& unpacked_output,
|
||||
std::vector<PreflateTokenBlock>& blocks,
|
||||
const std::vector<unsigned char>& deflate_raw) {
|
||||
z_stream inf; /* zlib deflate and inflate states */
|
||||
inf.zalloc = zalloc;
|
||||
inf.zfree = zfree;
|
||||
inf.opaque = Z_NULL;
|
||||
inf.avail_in = deflate_raw.size();
|
||||
inf.next_in = const_cast<unsigned char*>(deflate_raw.data());
|
||||
|
||||
PreflateDumper dumper;
|
||||
inf.dumper = &dumper;
|
||||
int ret = inflateInit2(&inf, -15);
|
||||
if (ret != Z_OK) {
|
||||
return false;
|
||||
}
|
||||
std::vector<unsigned char> tmp(64 * 1024);
|
||||
inf.avail_out = tmp.size();
|
||||
inf.next_out = tmp.data();
|
||||
bool failure = false;
|
||||
|
||||
do {
|
||||
/* decompress */
|
||||
inf.avail_out = tmp.size();
|
||||
inf.next_out = tmp.data();
|
||||
ret = inflate(&inf, Z_NO_FLUSH);
|
||||
if (ret == Z_STREAM_ERROR || ret == Z_DATA_ERROR || ret == Z_NEED_DICT || ret == Z_MEM_ERROR
|
||||
|| ret == Z_BUF_ERROR) {
|
||||
return false;
|
||||
}
|
||||
dumper.uncompressed.insert(dumper.uncompressed.end(), tmp.begin(), tmp.end() - inf.avail_out);
|
||||
} while (ret != Z_STREAM_END);
|
||||
|
||||
inflateEnd(&inf);
|
||||
|
||||
if (failure || dumper.hadErrors()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
unpacked_output = std::move(dumper.uncompressed);
|
||||
blocks = std::move(dumper.blocks);
|
||||
return true;
|
||||
}
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
/* Copyright 2018 Dirk Steinke
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#ifndef PREFLATE_UNPACK_H
|
||||
#define PREFLATE_UNPACK_H
|
||||
|
||||
#include <vector>
|
||||
#include "preflate_token.h"
|
||||
|
||||
bool preflate_unpack(std::vector<unsigned char>& unpacked_output,
|
||||
std::vector<PreflateTokenBlock>& blocks,
|
||||
const std::vector<unsigned char>& deflate_raw);
|
||||
|
||||
#endif /* PREFLATE_UNPACK_H */
|
||||
|
|
@ -17,10 +17,17 @@
|
|||
#include "array_helper.h"
|
||||
#include "bit_helper.h"
|
||||
|
||||
const uint8_t ArithmeticCodecBase::_normCheckLUT[8] = {
|
||||
0x33, 0x77, 0xff, 0xff, 0x33, 0x77, 0xff, 0xff
|
||||
};
|
||||
|
||||
ArithmeticCodecBase::ArithmeticCodecBase()
|
||||
: _low(0)
|
||||
, _high(0x7fffffff) {}
|
||||
|
||||
|
||||
ArithmeticEncoder::ArithmeticEncoder(BitOutputStream& bos)
|
||||
: _bos(bos)
|
||||
, _low(0)
|
||||
, _high(0x7fffffff)
|
||||
, _e3cnt(0) {}
|
||||
|
||||
void ArithmeticEncoder::_writeE3(const unsigned w) {
|
||||
|
|
@ -38,9 +45,14 @@ void ArithmeticEncoder::flush() {
|
|||
} else {
|
||||
_bos.put(1, 1);
|
||||
}
|
||||
_low = 0;
|
||||
_high = 0x7fffffff;
|
||||
}
|
||||
|
||||
void ArithmeticEncoder::_normalize() {
|
||||
#ifdef _DEBUG
|
||||
_ASSERT(_low <= _high && _high < 0x80000000);
|
||||
#endif
|
||||
// write determinated bits
|
||||
// this is the case if _low features 1 bits
|
||||
// or _high features 0 bits
|
||||
|
|
@ -75,17 +87,21 @@ void ArithmeticEncoder::_normalize() {
|
|||
_high = ((((_high + 1) << l) - 1) & 0x3fffffff)
|
||||
| 0x40000000;
|
||||
}
|
||||
#ifdef _DEBUG
|
||||
_ASSERT(_low <= _high && _high < 0x80000000);
|
||||
#endif
|
||||
}
|
||||
|
||||
ArithmeticDecoder::ArithmeticDecoder(BitInputStream& bis)
|
||||
: _bis(bis)
|
||||
, _value(0)
|
||||
, _low(0)
|
||||
, _high(0x7fffffff) {
|
||||
, _value(0) {
|
||||
_value = _bis.getReverse(16) << 15;
|
||||
_value |= _bis.getReverse(15);
|
||||
}
|
||||
void ArithmeticDecoder::_normalize() {
|
||||
#ifdef _DEBUG
|
||||
_ASSERT(_low <= _value && _value <= _high && _high < 0x80000000);
|
||||
#endif
|
||||
// skip determinated bits
|
||||
// this is the case if _low features 1 bits
|
||||
// or _high features 0 bits
|
||||
|
|
@ -118,12 +134,15 @@ void ArithmeticDecoder::_normalize() {
|
|||
_high = ((((_high + 1) << l) - 1) & 0x3fffffff)
|
||||
| 0x40000000;
|
||||
if (l <= 16) {
|
||||
_value = ((_value << l) + _bis.getReverse(l)) & 0x7fffffff;
|
||||
_value = (((_value << l) + _bis.getReverse(l)) -0x40000000) & 0x7fffffff;
|
||||
} else {
|
||||
_value = ((_value << 16) + _bis.getReverse(16)) & 0x7fffffff;
|
||||
_value = ((_value << (l - 16)) + _bis.getReverse(l - 16)) & 0x7fffffff;
|
||||
_value = ((_value << 16) + _bis.getReverse(16));
|
||||
_value = (((_value << (l - 16)) + _bis.getReverse(l - 16)) - 0x40000000) & 0x7fffffff;
|
||||
}
|
||||
}
|
||||
#ifdef _DEBUG
|
||||
_ASSERT(_low <= _value && _value <= _high && _high < 0x80000000);
|
||||
#endif
|
||||
}
|
||||
|
||||
bool modelCheckFixed(unsigned bounds[], unsigned short ids[], unsigned short rids[],
|
||||
|
|
|
|||
|
|
@ -20,7 +20,21 @@
|
|||
#include "bitstream.h"
|
||||
#include "const_division.h"
|
||||
|
||||
class ArithmeticEncoder {
|
||||
class ArithmeticCodecBase {
|
||||
public:
|
||||
ArithmeticCodecBase();
|
||||
|
||||
// array for fast check if normalization is required
|
||||
static const uint8_t _normCheckLUT[8];
|
||||
bool _needsNormalization() const {
|
||||
return (_normCheckLUT[_low >> 29] & (1 << (_high >> 29))) != 0;
|
||||
}
|
||||
// arithmetic coding variables
|
||||
uint32_t _low;
|
||||
uint32_t _high;
|
||||
};
|
||||
|
||||
class ArithmeticEncoder : public ArithmeticCodecBase {
|
||||
public:
|
||||
ArithmeticEncoder(BitOutputStream& bos);
|
||||
void flush();
|
||||
|
|
@ -29,36 +43,45 @@ public:
|
|||
uint32_t step = ((_high - _low) + 1) / scale;
|
||||
_high = _low + step * high - 1;
|
||||
_low += step * low;
|
||||
_normalize();
|
||||
_checkNormalize();
|
||||
}
|
||||
void encodeShiftScale(const uint32_t shift, const uint32_t low, const uint32_t high) {
|
||||
// update steps, low count, high count
|
||||
uint32_t step = ((_high - _low) + 1) >> shift;
|
||||
_high = _low + step * high - 1;
|
||||
_low += step * low;
|
||||
_normalize();
|
||||
_checkNormalize();
|
||||
}
|
||||
void encode(const udivider_t<32>& scale, const uint32_t low, const uint32_t high) {
|
||||
// update steps, low count, high count
|
||||
uint32_t step = divide((_high - _low) + 1, scale);
|
||||
_high = _low + step * high - 1;
|
||||
_low += step * low;
|
||||
_checkNormalize();
|
||||
}
|
||||
void encodeBits(const uint32_t value, const uint32_t bits) {
|
||||
uint32_t step = ((_high - _low) + 1) >> bits;
|
||||
_low += step * value;
|
||||
_high = _low + step - 1;
|
||||
_normalize();
|
||||
}
|
||||
|
||||
private:
|
||||
void _checkNormalize() {
|
||||
if (_needsNormalization()) {
|
||||
_normalize();
|
||||
}
|
||||
}
|
||||
void _normalize();
|
||||
void _writeE3(const unsigned w);
|
||||
|
||||
BitOutputStream& _bos;
|
||||
|
||||
// arithmetic coding variables
|
||||
uint32_t _low;
|
||||
uint32_t _high;
|
||||
uint32_t _e3cnt;
|
||||
};
|
||||
|
||||
class ArithmeticDecoder {
|
||||
class ArithmeticDecoder : public ArithmeticCodecBase {
|
||||
public:
|
||||
ArithmeticDecoder(BitInputStream& bis);
|
||||
unsigned decode(const uint32_t scale, const unsigned bounds[], const unsigned N) {
|
||||
|
|
@ -87,40 +110,53 @@ public:
|
|||
return _decodeBinary(step, bounds);
|
||||
}
|
||||
|
||||
unsigned decodeBits(const uint32_t bits) {
|
||||
uint32_t step = ((_high - _low) + 1) >> bits;
|
||||
unsigned result = (_value - _low) / step;
|
||||
_low += step * result;
|
||||
_high = _low + step - 1;
|
||||
_normalize();
|
||||
return result;
|
||||
}
|
||||
|
||||
private:
|
||||
unsigned _findIndex(const unsigned bounds[],
|
||||
const unsigned N,
|
||||
const unsigned val) {
|
||||
for (unsigned i = 0; i < N; ++i) {
|
||||
if (val < bounds[i + 1]) {
|
||||
return i;
|
||||
for (unsigned i = N; i > 1; --i) {
|
||||
if (val >= bounds[i - 1]) {
|
||||
return i - 1;
|
||||
}
|
||||
}
|
||||
return N - 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
unsigned _decode(const uint32_t step, const unsigned bounds[], const unsigned N) {
|
||||
uint32_t val = (_value - _low) / step;
|
||||
unsigned result = _findIndex(bounds, N, val);
|
||||
_high = _low + step * bounds[result + 1] - 1;
|
||||
_low += step * bounds[result];
|
||||
_normalize();
|
||||
_checkNormalize();
|
||||
return result;
|
||||
}
|
||||
unsigned _decodeBinary(const uint32_t step, const unsigned bounds[]) {
|
||||
unsigned result = (_value >= _low + bounds[1] * step);
|
||||
_high = _low + step * bounds[result + 1] - 1;
|
||||
_low += step * bounds[result];
|
||||
_normalize();
|
||||
_checkNormalize();
|
||||
return result;
|
||||
}
|
||||
void _checkNormalize() {
|
||||
if (_needsNormalization()) {
|
||||
_normalize();
|
||||
}
|
||||
}
|
||||
void _normalize();
|
||||
|
||||
BitInputStream& _bis;
|
||||
|
||||
// arithmetic coding variables
|
||||
uint32_t _value;
|
||||
uint32_t _low;
|
||||
uint32_t _high;
|
||||
};
|
||||
|
||||
bool modelCheckFixed(unsigned bounds[], unsigned short ids[], unsigned short rids[],
|
||||
|
|
|
|||
|
|
@ -56,4 +56,17 @@ unsigned bitLeadingZeroes(const unsigned value_) {
|
|||
}
|
||||
return result + leading4[value >> 28];
|
||||
}
|
||||
static unsigned char trailing4[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0};
|
||||
unsigned bitTrailingZeroes(const unsigned value_) {
|
||||
if (value_ == 0) {
|
||||
return 32;
|
||||
}
|
||||
unsigned value = value_;
|
||||
unsigned result = 0;
|
||||
while ((value & 0xf) == 0) {
|
||||
value >>= 4;
|
||||
result += 4;
|
||||
}
|
||||
return result + trailing4[value & 0xf];
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,5 +18,6 @@
|
|||
unsigned bitLength(unsigned value);
|
||||
unsigned bitReverse(const unsigned value, const unsigned bits);
|
||||
unsigned bitLeadingZeroes(const unsigned value);
|
||||
unsigned bitTrailingZeroes(const unsigned value);
|
||||
|
||||
#endif /* BIT_HELPER_H */
|
||||
|
|
|
|||
|
|
@ -20,6 +20,11 @@ MemStream::MemStream() : _pos(0) {}
|
|||
MemStream::MemStream(const std::vector<uint8_t>& content)
|
||||
: _data(content)
|
||||
, _pos(0) {}
|
||||
MemStream::MemStream(const std::vector<uint8_t>& content, const size_t off, const size_t sz)
|
||||
: _data(std::max(std::min(content.size(), off + sz), off) - off)
|
||||
, _pos(0) {
|
||||
memcpy(_data.data(), content.data() + off, _data.size());
|
||||
}
|
||||
|
||||
bool MemStream::eof() const {
|
||||
return _pos == _data.size();
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ class MemStream : public SeekableInputOutputStream {
|
|||
public:
|
||||
MemStream();
|
||||
MemStream(const std::vector<uint8_t>& content);
|
||||
MemStream(const std::vector<uint8_t>& content, const size_t off, const size_t sz);
|
||||
|
||||
virtual bool eof() const;
|
||||
virtual size_t read(unsigned char* buffer, const size_t size);
|
||||
|
|
@ -32,6 +33,10 @@ public:
|
|||
virtual uint64_t tell() const;
|
||||
virtual uint64_t seek(const uint64_t newPos);
|
||||
|
||||
void replaceData(const std::vector<uint8_t>& content) {
|
||||
_data = content;
|
||||
}
|
||||
|
||||
const std::vector<uint8_t>& data() const {
|
||||
return _data;
|
||||
}
|
||||
|
|
|
|||
23
precomp.cpp
23
precomp.cpp
|
|
@ -19,7 +19,7 @@
|
|||
// version information
|
||||
#define V_MAJOR 0
|
||||
#define V_MINOR 4
|
||||
#define V_MINOR2 129
|
||||
#define V_MINOR2 131
|
||||
//#define V_STATE "ALPHA"
|
||||
#define V_STATE "EXPERIMENTAL (w/ preflate support)"
|
||||
#define V_MSG "USE FOR TESTING ONLY"
|
||||
|
|
@ -537,7 +537,7 @@ int init(int argc, char* argv[]) {
|
|||
}
|
||||
printf(" - %s\n",V_MSG);
|
||||
printf("Free for non-commercial use - Copyright 2006-2018 by Christian Schneider\n");
|
||||
printf("- experimental preflate support - Copyright 2018 by Dirk Steinke\n\n");
|
||||
printf("- experimental preflate v0.2.1 support - Copyright 2018 by Dirk Steinke\n\n");
|
||||
|
||||
// init compression and memory level count
|
||||
bool use_zlib_level[81];
|
||||
|
|
@ -3117,7 +3117,8 @@ recompress_deflate_result try_recompression_deflate(FILE* file) {
|
|||
std::vector<unsigned char> unpacked_output;
|
||||
uint64_t compressed_stream_size = 0;
|
||||
result.accepted = preflate_decode(unpacked_output, result.recon_data,
|
||||
compressed_stream_size, is, []() { print_work_sign(true); });
|
||||
compressed_stream_size, is, []() { print_work_sign(true); },
|
||||
0); // you can set a minimum deflate stream size here
|
||||
result.compressed_stream_size = compressed_stream_size;
|
||||
result.uncompressed_stream_size = unpacked_output.size();
|
||||
{
|
||||
|
|
@ -3451,14 +3452,26 @@ void try_decompression_deflate_type(unsigned& dcounter, unsigned& rcounter,
|
|||
|
||||
// end uncompressed data
|
||||
|
||||
debug_pos();
|
||||
|
||||
compressed_data_found = true;
|
||||
end_uncompressed_data();
|
||||
|
||||
debug_pos();
|
||||
|
||||
// check recursion
|
||||
recursion_result r = recursion_write_file_and_compress(rdres);
|
||||
|
||||
#if 0
|
||||
// Do we really want to allow uncompressed streams that are smaller than the compressed
|
||||
// ones? (It makes sense if the uncompressed stream contains a JPEG, or something similar.
|
||||
if (rdres.uncompressed_stream_size <= rdres.compressed_stream_size && !r.success) {
|
||||
recompressed_streams_count--;
|
||||
compressed_data_found = false;
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
debug_pos();
|
||||
|
||||
// write compressed data header without first bytes
|
||||
fout_fput_deflate_rec(type, rdres, hdr, hdr_length, inc_last, r);
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue