From d585b545bea76c0419596afb4414c251d7e4f0e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Pais?= Date: Tue, 31 Oct 2017 23:04:40 +0100 Subject: [PATCH 1/7] MTF List class to speed up zlib parameter search --- precomp.h | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/precomp.h b/precomp.h index ce501f1..abbfcf0 100644 --- a/precomp.h +++ b/precomp.h @@ -98,6 +98,37 @@ struct recursion_result { FILE* frecurse; }; +class zLibMTF{ + struct MTFItem{ + int Next, Previous; + }; + alignas(16) MTFItem List[81]; + int Root, Index; +public: + zLibMTF(): Root(0), Index(0) { + for (int i=0;i<81;i++){ + List[i].Next = i+1; + List[i].Previous = i-1; + } + List[80].Next = -1; + } + inline int First(){ + return Index=Root; + } + inline int Next(){ + return (Index>=0)?Index=List[Index].Next:Index; + } + inline void Update(){ + if (Index==Root) return; + + List[ List[Index].Previous ].Next = List[Index].Next; + List[ List[Index].Next ].Previous = List[Index].Previous; + List[Root].Previous = Index; + List[Index].Next = Root; + List[Root=Index].Previous = -1; + } +}; + void write_ftempout_if_not_present(int byte_count, bool in_memory, bool leave_open = false); recursion_result recursion_compress(int compressed_bytes, int decompressed_bytes); recursion_result recursion_decompress(long long recursion_data_length); @@ -119,4 +150,4 @@ void denit_compress_otf(); void init_decompress_otf(); void denit_decompress_otf(); int auto_detected_thread_count(); -int lzma_max_memory_default(); \ No newline at end of file +int lzma_max_memory_default(); From 74a431341bc2ad93d368cf772922495b57851a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Pais?= Date: Tue, 31 Oct 2017 23:08:01 +0100 Subject: [PATCH 2/7] MTF list for zlib parameters Use an MTF list to keep the possible zlib parameter combinations indexed by recency --- precomp.cpp | 136 ++++++++++++++++------------------------------------ 1 file changed, 42 insertions(+), 94 deletions(-) diff --git a/precomp.cpp b/precomp.cpp index 070463d..e2a4da0 100644 --- a/precomp.cpp +++ b/precomp.cpp @@ -193,7 +193,7 @@ int old_lzma_progress_text_length = -1; int lzma_mib_total = 0, lzma_mib_written = 0; int comp_mem_level_count[81]; -int levels_sorted[81]; +zLibMTF MTF; bool zlib_level_was_used[81]; bool anything_was_used; bool level_switch_used = false; @@ -389,8 +389,6 @@ DLL bool precompress_file(char* in_file, char* out_file, char* msg, Switches swi output_file_name = new char[strlen(out_file)+1]; strcpy(output_file_name, out_file); - sort_comp_mem_levels(); - start_time = get_time_ms(); penalty_bytes = new char[MAX_PENALTY_BYTES]; @@ -437,8 +435,6 @@ DLL bool recompress_file(char* in_file, char* out_file, char* msg, Switches swit output_file_name = new char[strlen(out_file)+1]; strcpy(output_file_name, out_file); - sort_comp_mem_levels(); - start_time = get_time_ms(); penalty_bytes = new char[MAX_PENALTY_BYTES]; @@ -1149,8 +1145,6 @@ int init(int argc, char* argv[]) { } - sort_comp_mem_levels(); - packjpg_mp3_dll_msg(); return operation; @@ -2049,8 +2043,6 @@ int init_comfort(int argc, char* argv[]) { } - sort_comp_mem_levels(); - packjpg_mp3_dll_msg(); return operation; @@ -3190,14 +3182,14 @@ void try_decompression_pdf(int windowbits, int pdf_header_length, int img_width, printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -3243,12 +3235,10 @@ void try_decompression_pdf(int windowbits, int pdf_header_length, int img_width, } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -3444,14 +3434,14 @@ void try_decompression_zip(int zip_header_length) { } for (windowbits = -15; windowbits < -7; windowbits++) { - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if (final_compression_found) break; } @@ -3475,12 +3465,10 @@ void try_decompression_zip(int zip_header_length) { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -3553,34 +3541,6 @@ void try_decompression_zip(int zip_header_length) { } } -void sort_comp_mem_levels() { - int i,j; - int comp_mem_level_count_copy[81]; - - for (i = 0; i < 81; i++) { - comp_mem_level_count_copy[i] = comp_mem_level_count[i]; - } - - int rekcount, reki = -1; - - for (j = 0; j < 81; j++) { - rekcount = -2; - for (i = 0; i < 81; i++) { - if (comp_mem_level_count_copy[i] > rekcount) { - rekcount = comp_mem_level_count_copy[i]; - reki = i; - } - } - if (rekcount == -1) { - levels_sorted[j] = -1; - } else { - levels_sorted[j] = reki; - } - comp_mem_level_count_copy[reki] = -1; - } - -} - void show_used_levels() { if (!anything_was_used) { if (!non_zlib_was_used) { @@ -6743,14 +6703,14 @@ void try_decompression_gzip(int gzip_header_length) { } for (windowbits = -15; windowbits < -7; windowbits++) { - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if (final_compression_found) break; } @@ -6774,12 +6734,10 @@ void try_decompression_gzip(int gzip_header_length) { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -6878,14 +6836,14 @@ void try_decompression_png (int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -6906,12 +6864,10 @@ void try_decompression_png (int windowbits) { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -6995,14 +6951,14 @@ void try_decompression_png_multi(int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; - try_recompress(fpng, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); + try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -7023,12 +6979,10 @@ void try_decompression_png_multi(int windowbits) { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -8141,14 +8095,14 @@ void try_decompression_zlib(int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size_intense_brute_mode) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -8169,12 +8123,10 @@ void try_decompression_zlib(int windowbits) { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -8276,14 +8228,14 @@ void try_decompression_brute() { } for (windowbits = -15; windowbits < -7; windowbits++) { - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if (final_compression_found) break; } @@ -8307,12 +8259,10 @@ void try_decompression_brute() { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -8402,14 +8352,14 @@ void try_decompression_swf(int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -8430,12 +8380,10 @@ void try_decompression_swf(int windowbits) { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } From ec0a5ae8b3e831c4f21356b84294f43b65fe9ec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Pais?= Date: Sat, 4 Nov 2017 20:49:45 +0100 Subject: [PATCH 3/7] New brute mode heuristic, context-dependent MTF lists --- precomp.cpp | 59 ++++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/precomp.cpp b/precomp.cpp index e2a4da0..fb22b68 100644 --- a/precomp.cpp +++ b/precomp.cpp @@ -193,7 +193,7 @@ int old_lzma_progress_text_length = -1; int lzma_mib_total = 0, lzma_mib_written = 0; int comp_mem_level_count[81]; -zLibMTF MTF; +zLibMTF SWF_MTF, PNG_MTF, mPNG_MTF, PDF_MTF, ZIP_MTF[4], GZIP_MTF[4], MTF[4]; bool zlib_level_was_used[81]; bool anything_was_used; bool level_switch_used = false; @@ -2778,12 +2778,16 @@ bool check_inf_result(int cb_pos, int windowbits, bool use_brute_parameters = fa // if a byte is present 8 or more times, it's most likely not a deflate stream // and could slow down the process (e.g. repeated patterns of "0xEBE1F1" or "0xEBEBEBFF" // did this before) - for (int i = 0; i < 256; i++) - histogram[i] = 0; - - for (int i = 0; i < 64; i++) { - histogram[in_buf[cb_pos + i]]++; - if (histogram[in_buf[cb_pos + i]] == 8) return false; + memset(&histogram[0], 0, sizeof(histogram)); + int maximum=0, used=0, offset=cb_pos; + for (int i=0;i<8;i++,offset+=64){ + for (int j=0;j<64;j++){ + int* freq = &histogram[in_buf[offset+j]]; + used+=((*freq)==0); + maximum+=(++(*freq))>maximum; + } + if (maximum>=8+i || used*4<(i+1)*64) + return false; } } @@ -3181,15 +3185,14 @@ void try_decompression_pdf(int windowbits, int pdf_header_length, int img_width, printf("Compressed size: %i\n", compressed_stream_size); printf ("Can be decompressed to %i bytes\n", retval); } - - for (int index = MTF.First(); index>=0; index=MTF.Next()){ + for (int index = PDF_MTF.First(); index>=0; index=PDF_MTF.Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ MTF.Update(); break; } + if (final_compression_found){ PDF_MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -3432,16 +3435,16 @@ void try_decompression_zip(int zip_header_length) { printf("Compressed size: %i\n", compressed_stream_size); printf ("Can be decompressed to %i bytes\n", retval); } - + int k = min(3, recursion_depth); for (windowbits = -15; windowbits < -7; windowbits++) { - for (int index = MTF.First(); index>=0; index=MTF.Next()){ + for (int index = ZIP_MTF[k].First(); index>=0; index=ZIP_MTF[k].Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ MTF.Update(); break; } + if (final_compression_found){ ZIP_MTF[k].Update(); break; } } if (final_compression_found) break; } @@ -6701,16 +6704,16 @@ void try_decompression_gzip(int gzip_header_length) { printf("Compressed size: %i\n", compressed_stream_size); printf ("Can be decompressed to %i bytes\n", retval); } - + int k = min(3, recursion_depth); for (windowbits = -15; windowbits < -7; windowbits++) { - for (int index = MTF.First(); index>=0; index=MTF.Next()){ + for (int index = GZIP_MTF[k].First(); index>=0; index=GZIP_MTF[k].Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ MTF.Update(); break; } + if (final_compression_found){ GZIP_MTF[k].Update(); break; } } if (final_compression_found) break; } @@ -6836,14 +6839,14 @@ void try_decompression_png (int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = MTF.First(); index>=0; index=MTF.Next()){ + for (int index = PNG_MTF.First(); index>=0; index=PNG_MTF.Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ MTF.Update(); break; } + if (final_compression_found){ PNG_MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -6951,14 +6954,14 @@ void try_decompression_png_multi(int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = MTF.First(); index>=0; index=MTF.Next()){ + for (int index = mPNG_MTF.First(); index>=0; index=mPNG_MTF.Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ MTF.Update(); break; } + if (final_compression_found){ mPNG_MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -8094,15 +8097,15 @@ void try_decompression_zlib(int windowbits) { printf("Compressed size: %i\n", compressed_stream_size); printf ("Can be decompressed to %i bytes\n", retval); } - - for (int index = MTF.First(); index>=0; index=MTF.Next()){ + int k = min(3, recursion_depth); + for (int index = MTF[k].First(); index>=0; index=MTF[k].Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ MTF.Update(); break; } + if (final_compression_found){ MTF[k].Update(); break; } } if ((best_identical_bytes > min_ident_size_intense_brute_mode) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -8226,16 +8229,16 @@ void try_decompression_brute() { printf("Compressed size: %i\n", compressed_stream_size); printf ("Can be decompressed to %i bytes\n", retval); } - + int k = min(3, recursion_depth); for (windowbits = -15; windowbits < -7; windowbits++) { - for (int index = MTF.First(); index>=0; index=MTF.Next()){ + for (int index = MTF[k].First(); index>=0; index=MTF[k].Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ MTF.Update(); break; } + if (final_compression_found){ MTF[k].Update(); break; } } if (final_compression_found) break; } @@ -8352,14 +8355,14 @@ void try_decompression_swf(int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = MTF.First(); index>=0; index=MTF.Next()){ + for (int index = SWF_MTF.First(); index>=0; index=SWF_MTF.Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ MTF.Update(); break; } + if (final_compression_found){ SWF_MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { From 61a6c5ed9edcb671bcf47796366af3f8d8ad2bca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Pais?= Date: Sun, 5 Nov 2017 20:28:15 +0100 Subject: [PATCH 4/7] Tweaked brute mode heuristic --- precomp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/precomp.cpp b/precomp.cpp index fb22b68..b5755b6 100644 --- a/precomp.cpp +++ b/precomp.cpp @@ -2786,7 +2786,7 @@ bool check_inf_result(int cb_pos, int windowbits, bool use_brute_parameters = fa used+=((*freq)==0); maximum+=(++(*freq))>maximum; } - if (maximum>=8+i || used*4<(i+1)*64) + if (maximum>=((12+i)< Date: Sun, 5 Nov 2017 20:47:32 +0100 Subject: [PATCH 5/7] Update precomp.cpp --- precomp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/precomp.cpp b/precomp.cpp index b5755b6..2ab2fbe 100644 --- a/precomp.cpp +++ b/precomp.cpp @@ -2780,7 +2780,7 @@ bool check_inf_result(int cb_pos, int windowbits, bool use_brute_parameters = fa // did this before) memset(&histogram[0], 0, sizeof(histogram)); int maximum=0, used=0, offset=cb_pos; - for (int i=0;i<8;i++,offset+=64){ + for (int i=0;i<4;i++,offset+=64){ for (int j=0;j<64;j++){ int* freq = &histogram[in_buf[offset+j]]; used+=((*freq)==0); From 848c801f230e7bb4ad2ceee8992e18672d459ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Pais?= Date: Sat, 23 Dec 2017 02:38:12 +0100 Subject: [PATCH 6/7] Bug fix in zLib MTF --- precomp.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/precomp.h b/precomp.h index abbfcf0..c56bc88 100644 --- a/precomp.h +++ b/precomp.h @@ -122,7 +122,8 @@ public: if (Index==Root) return; List[ List[Index].Previous ].Next = List[Index].Next; - List[ List[Index].Next ].Previous = List[Index].Previous; + if (List[Index].Next>=0) + List[ List[Index].Next ].Previous = List[Index].Previous; List[Root].Previous = Index; List[Index].Next = Root; List[Root=Index].Previous = -1; From e63e4898abd674143b6c11e6a56937f0b22efefb Mon Sep 17 00:00:00 2001 From: Christian Schneider Date: Mon, 25 Dec 2017 11:14:05 +0100 Subject: [PATCH 7/7] Undo context-dependent MTF --- precomp.cpp | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/precomp.cpp b/precomp.cpp index 2ab2fbe..6df108a 100644 --- a/precomp.cpp +++ b/precomp.cpp @@ -193,7 +193,7 @@ int old_lzma_progress_text_length = -1; int lzma_mib_total = 0, lzma_mib_written = 0; int comp_mem_level_count[81]; -zLibMTF SWF_MTF, PNG_MTF, mPNG_MTF, PDF_MTF, ZIP_MTF[4], GZIP_MTF[4], MTF[4]; +zLibMTF MTF; bool zlib_level_was_used[81]; bool anything_was_used; bool level_switch_used = false; @@ -3185,14 +3185,14 @@ void try_decompression_pdf(int windowbits, int pdf_header_length, int img_width, printf("Compressed size: %i\n", compressed_stream_size); printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = PDF_MTF.First(); index>=0; index=PDF_MTF.Next()){ + for (int index = MTF.First(); index>=0; index=MTF.Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ PDF_MTF.Update(); break; } + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -3435,16 +3435,16 @@ void try_decompression_zip(int zip_header_length) { printf("Compressed size: %i\n", compressed_stream_size); printf ("Can be decompressed to %i bytes\n", retval); } - int k = min(3, recursion_depth); + for (windowbits = -15; windowbits < -7; windowbits++) { - for (int index = ZIP_MTF[k].First(); index>=0; index=ZIP_MTF[k].Next()){ + for (int index = MTF.First(); index>=0; index=MTF.Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ ZIP_MTF[k].Update(); break; } + if (final_compression_found){ MTF.Update(); break; } } if (final_compression_found) break; } @@ -6704,16 +6704,16 @@ void try_decompression_gzip(int gzip_header_length) { printf("Compressed size: %i\n", compressed_stream_size); printf ("Can be decompressed to %i bytes\n", retval); } - int k = min(3, recursion_depth); + for (windowbits = -15; windowbits < -7; windowbits++) { - for (int index = GZIP_MTF[k].First(); index>=0; index=GZIP_MTF[k].Next()){ + for (int index = MTF.First(); index>=0; index=MTF.Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ GZIP_MTF[k].Update(); break; } + if (final_compression_found){ MTF.Update(); break; } } if (final_compression_found) break; } @@ -6839,14 +6839,14 @@ void try_decompression_png (int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = PNG_MTF.First(); index>=0; index=PNG_MTF.Next()){ + for (int index = MTF.First(); index>=0; index=MTF.Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ PNG_MTF.Update(); break; } + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -6954,14 +6954,14 @@ void try_decompression_png_multi(int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = mPNG_MTF.First(); index>=0; index=mPNG_MTF.Next()){ + for (int index = MTF.First(); index>=0; index=MTF.Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ mPNG_MTF.Update(); break; } + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -8097,15 +8097,15 @@ void try_decompression_zlib(int windowbits) { printf("Compressed size: %i\n", compressed_stream_size); printf ("Can be decompressed to %i bytes\n", retval); } - int k = min(3, recursion_depth); - for (int index = MTF[k].First(); index>=0; index=MTF[k].Next()){ + + for (int index = MTF.First(); index>=0; index=MTF.Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ MTF[k].Update(); break; } + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size_intense_brute_mode) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -8229,16 +8229,16 @@ void try_decompression_brute() { printf("Compressed size: %i\n", compressed_stream_size); printf ("Can be decompressed to %i bytes\n", retval); } - int k = min(3, recursion_depth); + for (windowbits = -15; windowbits < -7; windowbits++) { - for (int index = MTF[k].First(); index>=0; index=MTF[k].Next()){ + for (int index = MTF.First(); index>=0; index=MTF.Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ MTF[k].Update(); break; } + if (final_compression_found){ MTF.Update(); break; } } if (final_compression_found) break; } @@ -8355,14 +8355,14 @@ void try_decompression_swf(int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = SWF_MTF.First(); index>=0; index=SWF_MTF.Next()){ + for (int index = MTF.First(); index>=0; index=MTF.Next()){ if (comp_mem_level_count[index] == -1) break; int comp_level = (index % 9) + 1; int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found){ SWF_MTF.Update(); break; } + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) {