diff --git a/precomp.cpp b/precomp.cpp index 070463d..6df108a 100644 --- a/precomp.cpp +++ b/precomp.cpp @@ -193,7 +193,7 @@ int old_lzma_progress_text_length = -1; int lzma_mib_total = 0, lzma_mib_written = 0; int comp_mem_level_count[81]; -int levels_sorted[81]; +zLibMTF MTF; bool zlib_level_was_used[81]; bool anything_was_used; bool level_switch_used = false; @@ -389,8 +389,6 @@ DLL bool precompress_file(char* in_file, char* out_file, char* msg, Switches swi output_file_name = new char[strlen(out_file)+1]; strcpy(output_file_name, out_file); - sort_comp_mem_levels(); - start_time = get_time_ms(); penalty_bytes = new char[MAX_PENALTY_BYTES]; @@ -437,8 +435,6 @@ DLL bool recompress_file(char* in_file, char* out_file, char* msg, Switches swit output_file_name = new char[strlen(out_file)+1]; strcpy(output_file_name, out_file); - sort_comp_mem_levels(); - start_time = get_time_ms(); penalty_bytes = new char[MAX_PENALTY_BYTES]; @@ -1149,8 +1145,6 @@ int init(int argc, char* argv[]) { } - sort_comp_mem_levels(); - packjpg_mp3_dll_msg(); return operation; @@ -2049,8 +2043,6 @@ int init_comfort(int argc, char* argv[]) { } - sort_comp_mem_levels(); - packjpg_mp3_dll_msg(); return operation; @@ -2786,12 +2778,16 @@ bool check_inf_result(int cb_pos, int windowbits, bool use_brute_parameters = fa // if a byte is present 8 or more times, it's most likely not a deflate stream // and could slow down the process (e.g. repeated patterns of "0xEBE1F1" or "0xEBEBEBFF" // did this before) - for (int i = 0; i < 256; i++) - histogram[i] = 0; - - for (int i = 0; i < 64; i++) { - histogram[in_buf[cb_pos + i]]++; - if (histogram[in_buf[cb_pos + i]] == 8) return false; + memset(&histogram[0], 0, sizeof(histogram)); + int maximum=0, used=0, offset=cb_pos; + for (int i=0;i<4;i++,offset+=64){ + for (int j=0;j<64;j++){ + int* freq = &histogram[in_buf[offset+j]]; + used+=((*freq)==0); + maximum+=(++(*freq))>maximum; + } + if (maximum>=((12+i)<=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -3243,12 +3238,10 @@ void try_decompression_pdf(int windowbits, int pdf_header_length, int img_width, } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -3444,14 +3437,14 @@ void try_decompression_zip(int zip_header_length) { } for (windowbits = -15; windowbits < -7; windowbits++) { - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if (final_compression_found) break; } @@ -3475,12 +3468,10 @@ void try_decompression_zip(int zip_header_length) { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -3553,34 +3544,6 @@ void try_decompression_zip(int zip_header_length) { } } -void sort_comp_mem_levels() { - int i,j; - int comp_mem_level_count_copy[81]; - - for (i = 0; i < 81; i++) { - comp_mem_level_count_copy[i] = comp_mem_level_count[i]; - } - - int rekcount, reki = -1; - - for (j = 0; j < 81; j++) { - rekcount = -2; - for (i = 0; i < 81; i++) { - if (comp_mem_level_count_copy[i] > rekcount) { - rekcount = comp_mem_level_count_copy[i]; - reki = i; - } - } - if (rekcount == -1) { - levels_sorted[j] = -1; - } else { - levels_sorted[j] = reki; - } - comp_mem_level_count_copy[reki] = -1; - } - -} - void show_used_levels() { if (!anything_was_used) { if (!non_zlib_was_used) { @@ -6743,14 +6706,14 @@ void try_decompression_gzip(int gzip_header_length) { } for (windowbits = -15; windowbits < -7; windowbits++) { - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if (final_compression_found) break; } @@ -6774,12 +6737,10 @@ void try_decompression_gzip(int gzip_header_length) { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -6878,14 +6839,14 @@ void try_decompression_png (int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -6906,12 +6867,10 @@ void try_decompression_png (int windowbits) { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -6995,14 +6954,14 @@ void try_decompression_png_multi(int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; - try_recompress(fpng, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); + try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -7023,12 +6982,10 @@ void try_decompression_png_multi(int windowbits) { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -8141,14 +8098,14 @@ void try_decompression_zlib(int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size_intense_brute_mode) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -8169,12 +8126,10 @@ void try_decompression_zlib(int windowbits) { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -8274,16 +8229,16 @@ void try_decompression_brute() { printf("Compressed size: %i\n", compressed_stream_size); printf ("Can be decompressed to %i bytes\n", retval); } - + for (windowbits = -15; windowbits < -7; windowbits++) { - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if (final_compression_found) break; } @@ -8307,12 +8262,10 @@ void try_decompression_brute() { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } @@ -8402,14 +8355,14 @@ void try_decompression_swf(int windowbits) { printf ("Can be decompressed to %i bytes\n", retval); } - for (int index = 0; index < 81; index++) { - if (levels_sorted[index] == -1) break; - int comp_level = (levels_sorted[index] % 9) + 1; - int mem_level = (levels_sorted[index] / 9) + 1; + for (int index = MTF.First(); index>=0; index=MTF.Next()){ + if (comp_mem_level_count[index] == -1) break; + int comp_level = (index % 9) + 1; + int mem_level = (index / 9) + 1; try_recompress(fin, comp_level, mem_level, windowbits, compressed_stream_size, retval, in_memory); - if (final_compression_found) break; + if (final_compression_found){ MTF.Update(); break; } } if ((best_identical_bytes > min_ident_size) && (best_identical_bytes < best_identical_bytes_decomp)) { @@ -8430,12 +8383,10 @@ void try_decompression_swf(int windowbits) { } } anything_was_used = true; - sort_comp_mem_levels(); } else { comp_mem_level_count[(best_compression - 1) + (best_mem_level - 1) * 9]++; zlib_level_was_used[(best_compression - 1) + (best_mem_level - 1) * 9] = true; anything_was_used = true; - sort_comp_mem_levels(); } } diff --git a/precomp.h b/precomp.h index ce501f1..c56bc88 100644 --- a/precomp.h +++ b/precomp.h @@ -98,6 +98,38 @@ struct recursion_result { FILE* frecurse; }; +class zLibMTF{ + struct MTFItem{ + int Next, Previous; + }; + alignas(16) MTFItem List[81]; + int Root, Index; +public: + zLibMTF(): Root(0), Index(0) { + for (int i=0;i<81;i++){ + List[i].Next = i+1; + List[i].Previous = i-1; + } + List[80].Next = -1; + } + inline int First(){ + return Index=Root; + } + inline int Next(){ + return (Index>=0)?Index=List[Index].Next:Index; + } + inline void Update(){ + if (Index==Root) return; + + List[ List[Index].Previous ].Next = List[Index].Next; + if (List[Index].Next>=0) + List[ List[Index].Next ].Previous = List[Index].Previous; + List[Root].Previous = Index; + List[Index].Next = Root; + List[Root=Index].Previous = -1; + } +}; + void write_ftempout_if_not_present(int byte_count, bool in_memory, bool leave_open = false); recursion_result recursion_compress(int compressed_bytes, int decompressed_bytes); recursion_result recursion_decompress(long long recursion_data_length); @@ -119,4 +151,4 @@ void denit_compress_otf(); void init_decompress_otf(); void denit_decompress_otf(); int auto_detected_thread_count(); -int lzma_max_memory_default(); \ No newline at end of file +int lzma_max_memory_default();