mirror of
https://github.com/schnaader/precomp-cpp
synced 2026-08-23 00:23:04 -04:00
Merge branch 'mpais_mtf_heuristic'
This commit is contained in:
Christian Schneider
commit
d92cc2d779
2 changed files with 86 additions and 103 deletions
155
precomp.cpp
155
precomp.cpp
|
|
@ -193,7 +193,7 @@ int old_lzma_progress_text_length = -1;
|
|||
int lzma_mib_total = 0, lzma_mib_written = 0;
|
||||
|
||||
int comp_mem_level_count[81];
|
||||
int levels_sorted[81];
|
||||
zLibMTF MTF;
|
||||
bool zlib_level_was_used[81];
|
||||
bool anything_was_used;
|
||||
bool level_switch_used = false;
|
||||
|
|
@ -389,8 +389,6 @@ DLL bool precompress_file(char* in_file, char* out_file, char* msg, Switches swi
|
|||
output_file_name = new char[strlen(out_file)+1];
|
||||
strcpy(output_file_name, out_file);
|
||||
|
||||
sort_comp_mem_levels();
|
||||
|
||||
start_time = get_time_ms();
|
||||
|
||||
penalty_bytes = new char[MAX_PENALTY_BYTES];
|
||||
|
|
@ -437,8 +435,6 @@ DLL bool recompress_file(char* in_file, char* out_file, char* msg, Switches swit
|
|||
output_file_name = new char[strlen(out_file)+1];
|
||||
strcpy(output_file_name, out_file);
|
||||
|
||||
sort_comp_mem_levels();
|
||||
|
||||
start_time = get_time_ms();
|
||||
|
||||
penalty_bytes = new char[MAX_PENALTY_BYTES];
|
||||
|
|
@ -1149,8 +1145,6 @@ int init(int argc, char* argv[]) {
|
|||
|
||||
}
|
||||
|
||||
sort_comp_mem_levels();
|
||||
|
||||
packjpg_mp3_dll_msg();
|
||||
|
||||
return operation;
|
||||
|
|
@ -2049,8 +2043,6 @@ int init_comfort(int argc, char* argv[]) {
|
|||
|
||||
}
|
||||
|
||||
sort_comp_mem_levels();
|
||||
|
||||
packjpg_mp3_dll_msg();
|
||||
|
||||
return operation;
|
||||
|
|
@ -2786,12 +2778,16 @@ bool check_inf_result(int cb_pos, int windowbits, bool use_brute_parameters = fa
|
|||
// if a byte is present 8 or more times, it's most likely not a deflate stream
|
||||
// and could slow down the process (e.g. repeated patterns of "0xEBE1F1" or "0xEBEBEBFF"
|
||||
// did this before)
|
||||
for (int i = 0; i < 256; i++)
|
||||
histogram[i] = 0;
|
||||
|
||||
for (int i = 0; i < 64; i++) {
|
||||
histogram[in_buf[cb_pos + i]]++;
|
||||
if (histogram[in_buf[cb_pos + i]] == 8) return false;
|
||||
memset(&histogram[0], 0, sizeof(histogram));
|
||||
int maximum=0, used=0, offset=cb_pos;
|
||||
for (int i=0;i<4;i++,offset+=64){
|
||||
for (int j=0;j<64;j++){
|
||||
int* freq = &histogram[in_buf[offset+j]];
|
||||
used+=((*freq)==0);
|
||||
maximum+=(++(*freq))>maximum;
|
||||
}
|
||||
if (maximum>=((12+i)<<i) || used*(7-(i+(i/2)))<(i+1)*64)
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3189,15 +3185,14 @@ void try_decompression_pdf(int windowbits, int pdf_header_length, int img_width,
|
|||
printf("Compressed size: %i\n", compressed_stream_size);
|
||||
printf ("Can be decompressed to %i bytes\n", retval);
|
||||
}
|
||||
|
||||
for (int index = 0; index < 81; index++) {
|
||||
if (levels_sorted[index] == -1) break;
|
||||
int comp_level = (levels_sorted[index] % 9) + 1;
|
||||
int mem_level = (levels_sorted[index] / 9) + 1;
|
||||
for (int index = MTF.First(); index>=0; index=MTF.Next()){
|
||||
if (comp_mem_level_count[index] == -1) break;
|
||||
int comp_level = (index % 9) + 1;
|
||||
int mem_level = (index / 9) + 1;
|
||||
|
||||
try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory);
|
||||
|
||||
if (final_compression_found) break;
|
||||
if (final_compression_found){ MTF.Update(); break; }
|
||||
}
|
||||
|
||||
if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) {
|
||||
|
|
@ -3243,12 +3238,10 @@ void try_decompression_pdf(int windowbits, int pdf_header_length, int img_width,
|
|||
}
|
||||
}
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
} else {
|
||||
comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++;
|
||||
zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true;
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3444,14 +3437,14 @@ void try_decompression_zip(int zip_header_length) {
|
|||
}
|
||||
|
||||
for (windowbits = -15; windowbits < -7; windowbits++) {
|
||||
for (int index = 0; index < 81; index++) {
|
||||
if (levels_sorted[index] == -1) break;
|
||||
int comp_level = (levels_sorted[index] % 9) + 1;
|
||||
int mem_level = (levels_sorted[index] / 9) + 1;
|
||||
for (int index = MTF.First(); index>=0; index=MTF.Next()){
|
||||
if (comp_mem_level_count[index] == -1) break;
|
||||
int comp_level = (index % 9) + 1;
|
||||
int mem_level = (index / 9) + 1;
|
||||
|
||||
try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory);
|
||||
|
||||
if (final_compression_found) break;
|
||||
if (final_compression_found){ MTF.Update(); break; }
|
||||
}
|
||||
if (final_compression_found) break;
|
||||
}
|
||||
|
|
@ -3475,12 +3468,10 @@ void try_decompression_zip(int zip_header_length) {
|
|||
}
|
||||
}
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
} else {
|
||||
comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++;
|
||||
zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true;
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3553,34 +3544,6 @@ void try_decompression_zip(int zip_header_length) {
|
|||
}
|
||||
}
|
||||
|
||||
void sort_comp_mem_levels() {
|
||||
int i,j;
|
||||
int comp_mem_level_count_copy[81];
|
||||
|
||||
for (i = 0; i < 81; i++) {
|
||||
comp_mem_level_count_copy[i] = comp_mem_level_count[i];
|
||||
}
|
||||
|
||||
int rekcount, reki = -1;
|
||||
|
||||
for (j = 0; j < 81; j++) {
|
||||
rekcount = -2;
|
||||
for (i = 0; i < 81; i++) {
|
||||
if (comp_mem_level_count_copy[i] > rekcount) {
|
||||
rekcount = comp_mem_level_count_copy[i];
|
||||
reki = i;
|
||||
}
|
||||
}
|
||||
if (rekcount == -1) {
|
||||
levels_sorted[j] = -1;
|
||||
} else {
|
||||
levels_sorted[j] = reki;
|
||||
}
|
||||
comp_mem_level_count_copy[reki] = -1;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void show_used_levels() {
|
||||
if (!anything_was_used) {
|
||||
if (!non_zlib_was_used) {
|
||||
|
|
@ -6743,14 +6706,14 @@ void try_decompression_gzip(int gzip_header_length) {
|
|||
}
|
||||
|
||||
for (windowbits = -15; windowbits < -7; windowbits++) {
|
||||
for (int index = 0; index < 81; index++) {
|
||||
if (levels_sorted[index] == -1) break;
|
||||
int comp_level = (levels_sorted[index] % 9) + 1;
|
||||
int mem_level = (levels_sorted[index] / 9) + 1;
|
||||
for (int index = MTF.First(); index>=0; index=MTF.Next()){
|
||||
if (comp_mem_level_count[index] == -1) break;
|
||||
int comp_level = (index % 9) + 1;
|
||||
int mem_level = (index / 9) + 1;
|
||||
|
||||
try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory);
|
||||
|
||||
if (final_compression_found) break;
|
||||
if (final_compression_found){ MTF.Update(); break; }
|
||||
}
|
||||
if (final_compression_found) break;
|
||||
}
|
||||
|
|
@ -6774,12 +6737,10 @@ void try_decompression_gzip(int gzip_header_length) {
|
|||
}
|
||||
}
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
} else {
|
||||
comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++;
|
||||
zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true;
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -6878,14 +6839,14 @@ void try_decompression_png (int windowbits) {
|
|||
printf ("Can be decompressed to %i bytes\n", retval);
|
||||
}
|
||||
|
||||
for (int index = 0; index < 81; index++) {
|
||||
if (levels_sorted[index] == -1) break;
|
||||
int comp_level = (levels_sorted[index] % 9) + 1;
|
||||
int mem_level = (levels_sorted[index] / 9) + 1;
|
||||
for (int index = MTF.First(); index>=0; index=MTF.Next()){
|
||||
if (comp_mem_level_count[index] == -1) break;
|
||||
int comp_level = (index % 9) + 1;
|
||||
int mem_level = (index / 9) + 1;
|
||||
|
||||
try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory);
|
||||
|
||||
if (final_compression_found) break;
|
||||
if (final_compression_found){ MTF.Update(); break; }
|
||||
}
|
||||
|
||||
if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) {
|
||||
|
|
@ -6906,12 +6867,10 @@ void try_decompression_png (int windowbits) {
|
|||
}
|
||||
}
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
} else {
|
||||
comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++;
|
||||
zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true;
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -6995,14 +6954,14 @@ void try_decompression_png_multi(int windowbits) {
|
|||
printf ("Can be decompressed to %i bytes\n", retval);
|
||||
}
|
||||
|
||||
for (int index = 0; index < 81; index++) {
|
||||
if (levels_sorted[index] == -1) break;
|
||||
int comp_level = (levels_sorted[index] % 9) + 1;
|
||||
int mem_level = (levels_sorted[index] / 9) + 1;
|
||||
for (int index = MTF.First(); index>=0; index=MTF.Next()){
|
||||
if (comp_mem_level_count[index] == -1) break;
|
||||
int comp_level = (index % 9) + 1;
|
||||
int mem_level = (index / 9) + 1;
|
||||
|
||||
try_recompress(fpng, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory);
|
||||
try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory);
|
||||
|
||||
if (final_compression_found) break;
|
||||
if (final_compression_found){ MTF.Update(); break; }
|
||||
}
|
||||
|
||||
if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) {
|
||||
|
|
@ -7023,12 +6982,10 @@ void try_decompression_png_multi(int windowbits) {
|
|||
}
|
||||
}
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
} else {
|
||||
comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++;
|
||||
zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true;
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -8141,14 +8098,14 @@ void try_decompression_zlib(int windowbits) {
|
|||
printf ("Can be decompressed to %i bytes\n", retval);
|
||||
}
|
||||
|
||||
for (int index = 0; index < 81; index++) {
|
||||
if (levels_sorted[index] == -1) break;
|
||||
int comp_level = (levels_sorted[index] % 9) + 1;
|
||||
int mem_level = (levels_sorted[index] / 9) + 1;
|
||||
for (int index = MTF.First(); index>=0; index=MTF.Next()){
|
||||
if (comp_mem_level_count[index] == -1) break;
|
||||
int comp_level = (index % 9) + 1;
|
||||
int mem_level = (index / 9) + 1;
|
||||
|
||||
try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory);
|
||||
|
||||
if (final_compression_found) break;
|
||||
if (final_compression_found){ MTF.Update(); break; }
|
||||
}
|
||||
|
||||
if ((best_identical_bytes > min_ident_size_intense_brute_mode) && (best_identical_bytes < best_identical_bytes_decomp)) {
|
||||
|
|
@ -8169,12 +8126,10 @@ void try_decompression_zlib(int windowbits) {
|
|||
}
|
||||
}
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
} else {
|
||||
comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++;
|
||||
zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true;
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -8274,16 +8229,16 @@ void try_decompression_brute() {
|
|||
printf("Compressed size: %i\n", compressed_stream_size);
|
||||
printf ("Can be decompressed to %i bytes\n", retval);
|
||||
}
|
||||
|
||||
|
||||
for (windowbits = -15; windowbits < -7; windowbits++) {
|
||||
for (int index = 0; index < 81; index++) {
|
||||
if (levels_sorted[index] == -1) break;
|
||||
int comp_level = (levels_sorted[index] % 9) + 1;
|
||||
int mem_level = (levels_sorted[index] / 9) + 1;
|
||||
for (int index = MTF.First(); index>=0; index=MTF.Next()){
|
||||
if (comp_mem_level_count[index] == -1) break;
|
||||
int comp_level = (index % 9) + 1;
|
||||
int mem_level = (index / 9) + 1;
|
||||
|
||||
try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory);
|
||||
|
||||
if (final_compression_found) break;
|
||||
if (final_compression_found){ MTF.Update(); break; }
|
||||
}
|
||||
if (final_compression_found) break;
|
||||
}
|
||||
|
|
@ -8307,12 +8262,10 @@ void try_decompression_brute() {
|
|||
}
|
||||
}
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
} else {
|
||||
comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++;
|
||||
zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true;
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -8402,14 +8355,14 @@ void try_decompression_swf(int windowbits) {
|
|||
printf ("Can be decompressed to %i bytes\n", retval);
|
||||
}
|
||||
|
||||
for (int index = 0; index < 81; index++) {
|
||||
if (levels_sorted[index] == -1) break;
|
||||
int comp_level = (levels_sorted[index] % 9) + 1;
|
||||
int mem_level = (levels_sorted[index] / 9) + 1;
|
||||
for (int index = MTF.First(); index>=0; index=MTF.Next()){
|
||||
if (comp_mem_level_count[index] == -1) break;
|
||||
int comp_level = (index % 9) + 1;
|
||||
int mem_level = (index / 9) + 1;
|
||||
|
||||
try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory);
|
||||
|
||||
if (final_compression_found) break;
|
||||
if (final_compression_found){ MTF.Update(); break; }
|
||||
}
|
||||
|
||||
if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) {
|
||||
|
|
@ -8430,12 +8383,10 @@ void try_decompression_swf(int windowbits) {
|
|||
}
|
||||
}
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
} else {
|
||||
comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++;
|
||||
zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true;
|
||||
anything_was_used = true;
|
||||
sort_comp_mem_levels();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
34
precomp.h
34
precomp.h
|
|
@ -98,6 +98,38 @@ struct recursion_result {
|
|||
FILE* frecurse;
|
||||
};
|
||||
|
||||
class zLibMTF{
|
||||
struct MTFItem{
|
||||
int Next, Previous;
|
||||
};
|
||||
alignas(16) MTFItem List[81];
|
||||
int Root, Index;
|
||||
public:
|
||||
zLibMTF(): Root(0), Index(0) {
|
||||
for (int i=0;i<81;i++){
|
||||
List[i].Next = i+1;
|
||||
List[i].Previous = i-1;
|
||||
}
|
||||
List[80].Next = -1;
|
||||
}
|
||||
inline int First(){
|
||||
return Index=Root;
|
||||
}
|
||||
inline int Next(){
|
||||
return (Index>=0)?Index=List[Index].Next:Index;
|
||||
}
|
||||
inline void Update(){
|
||||
if (Index==Root) return;
|
||||
|
||||
List[ List[Index].Previous ].Next = List[Index].Next;
|
||||
if (List[Index].Next>=0)
|
||||
List[ List[Index].Next ].Previous = List[Index].Previous;
|
||||
List[Root].Previous = Index;
|
||||
List[Index].Next = Root;
|
||||
List[Root=Index].Previous = -1;
|
||||
}
|
||||
};
|
||||
|
||||
void write_ftempout_if_not_present(int byte_count, bool in_memory, bool leave_open = false);
|
||||
recursion_result recursion_compress(int compressed_bytes, int decompressed_bytes);
|
||||
recursion_result recursion_decompress(long long recursion_data_length);
|
||||
|
|
@ -119,4 +151,4 @@ void denit_compress_otf();
|
|||
void init_decompress_otf();
|
||||
void denit_decompress_otf();
|
||||
int auto_detected_thread_count();
|
||||
int lzma_max_memory_default();
|
||||
int lzma_max_memory_default();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue