diff --git a/README.md b/README.md index 23822e1..df3aac5 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,6 @@ Fork note --------- This is an experimental fork of precomp, that uses the preflate library for deflate stream recompression. -The bitstream format output by the preflate library is not stable yet, -so do not use this experimental tool for archiving. - -Also, the build system isn't up to date yet. I'll provide a CMake script -in the near future. Also, importing the preflate sources in a GUI, e.g. MSVC, -is pretty straight forward. - -For the time being, there are executables for testing. What is Precomp? diff --git a/bin/precomp_w_preflate_msvc17_aka_vc140_x64.exe b/bin/precomp_w_preflate_msvc17_aka_vc140_x64.exe index 210e755..aefc3b0 100644 Binary files a/bin/precomp_w_preflate_msvc17_aka_vc140_x64.exe and b/bin/precomp_w_preflate_msvc17_aka_vc140_x64.exe differ diff --git a/bin/precomp_w_preflate_msvc17_aka_vc140_x86.exe b/bin/precomp_w_preflate_msvc17_aka_vc140_x86.exe index 77c175f..263bf36 100644 Binary files a/bin/precomp_w_preflate_msvc17_aka_vc140_x86.exe and b/bin/precomp_w_preflate_msvc17_aka_vc140_x86.exe differ diff --git a/contrib/preflate/main.cpp b/contrib/preflate/main.cpp index 4aa2cd9..7190d22 100644 --- a/contrib/preflate/main.cpp +++ b/contrib/preflate/main.cpp @@ -125,7 +125,10 @@ int combine(const char* const * const fns, const unsigned fncnt, const std::stri return ok ? 0 : -1; } + +#include "preflate_seq_chain.h" int main(int argc, const char * const * const argv) { + puts("preflate v0.3.2"); if (!support_self_tests()) { return -1; } diff --git a/contrib/preflate/main2.cpp b/contrib/preflate/main2.cpp new file mode 100644 index 0000000..ce4ccf6 --- /dev/null +++ b/contrib/preflate/main2.cpp @@ -0,0 +1,179 @@ +/* Copyright 2018 Dirk Steinke + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include +#include +#include +#include +#include +#include + +#include "preflate_checker.h" +#include "preflate_decoder.h" +#include "preflate_info.h" +#include "preflate_reencoder.h" +#include "support/support_tests.h" + +bool loadfile( + std::vector& content, + const std::string& fn) { + FILE* f = fopen(fn.c_str(), "rb"); + if (!f) { + return false; + } + fseek(f, 0, SEEK_END); + long length = ftell(f); + fseek(f, 0, SEEK_SET); + content.resize(length); + long read = fread(content.data(), 1, content.size(), f); + fclose(f); + return read == length; +} +bool savefile( + const std::vector& content, + const std::string& fn) { + FILE* f = fopen(fn.c_str(), "wb"); + if (!f) { + return false; + } + size_t written = fwrite(content.data(), 1, content.size(), f); + fclose(f); + return written == content.size(); +} + +int test(const char* const * const fns, const unsigned fncnt) { + bool ok = true; + for (unsigned i = 0; i < fncnt; ++i) { + std::vector content; + if (!loadfile(content, fns[i])) { + printf("loading of %s failed\n", fns[i]); + ok = false; + } else { + std::vector unpacked; + std::vector recon; + bool check_ok = preflate_decode(unpacked, recon, content, 1 << 18); + if (check_ok) { + std::vector rebuilt_content; + check_ok = preflate_reencode(rebuilt_content, recon, unpacked); + if (check_ok) { + check_ok = content == rebuilt_content; + if (check_ok) { + printf("splitting & recombining %s successful (%d -> %d + %d)\n", + fns[i], (int)content.size(), (int)unpacked.size(), (int)recon.size()); + } else { + printf("splitting & recombining %s failed: output not bitexact (%d -> %d + %d)\n", + fns[i], (int)content.size(), (int)unpacked.size(), (int)recon.size()); + } + } else { + printf("recombining %s failed (%d -> %d + %d)\n", + fns[i], (int)content.size(), (int)unpacked.size(), (int)recon.size()); + } + } else { + printf("splitting %s failed\n", fns[i]); + } + ok = ok && check_ok; + } + } + if (ok) { + printf("All checks ok\n"); + } + return ok ? 0 : -1; +} +int split(const char* const * const fns, const unsigned fncnt) { + bool ok = true; + for (unsigned i = 0; i < fncnt; ++i) { + std::vector content; + if (!loadfile(content, fns[i])) { + printf("loading of %s failed\n", fns[i]); + ok = false; + } else { + std::vector unpacked; + std::vector recon; + bool check_ok = preflate_decode(unpacked, recon, content); + if (check_ok) { + savefile(unpacked, std::string(fns[i]) + ".u"); + savefile(recon, std::string(fns[i]) + ".r"); + printf("splitting %s successful (%d -> %d + %d)\n", + fns[i], (int)content.size(), (int)unpacked.size(), (int)recon.size()); + } else { + printf("splitting %s failed\n", fns[i]); + } + ok = ok && check_ok; + } + } + if (ok) { + printf("All ok\n"); + } + return ok ? 0 : -1; +} +int combine(const char* const * const fns, const unsigned fncnt, const std::string& ext) { + bool ok = true; + for (unsigned i = 0; i < fncnt; ++i) { + std::vector unpacked; + std::vector recon; + if (!loadfile(unpacked, std::string(fns[i]) + ".u") + || !loadfile(recon, std::string(fns[i]) + ".r")) { + printf("loading of %s.u/.r failed\n", fns[i]); + ok = false; + } else { + std::vector content; + bool check_ok = preflate_reencode(content, recon, unpacked); + if (check_ok) { + savefile(content, std::string(fns[i]) + ext); + printf("recombining %s%s successful (%d + %d -> %d)\n", + fns[i], ext.c_str(), (int)unpacked.size(), (int)recon.size(), (int)content.size()); + } else { + printf("recombining %s%s failed\n", fns[i], ext.c_str()); + } + ok = ok && check_ok; + } + } + if (ok) { + printf("All ok\n"); + } + return ok ? 0 : -1; +} + + +#include "preflate_seq_chain.h" +int main(int argc, const char * const * const argv) { + puts("preflate v0.3.2"); + if (argc >= 3) { + if (!strcmp(argv[1], "-t")) { + const char* const * fns = argv + 2; + size_t fncnt = argc - 2; + return test(fns, fncnt); + } + if (!strcmp(argv[1], "-s")) { + return split(argv + 2, argc - 2); + } + if (!strcmp(argv[1], "-r")) { + return combine(argv + 2, argc - 2, ""); + } + if (!strcmp(argv[1], "-x")) { + return combine(argv + 2, argc - 2, ".x"); + } + } + printf("usage: %s -t FILE [FILE ... FILE]\n", argv[0]); + printf(" test uncompression and recompression\n"); + printf(" %s -s FILE [FILE ... FILE]\n", argv[0]); + printf(" split deflate stream (FILE) into uncompressed part\n"); + printf(" (FILE.u) and reconstruction info (FILE.r)\n"); + printf(" %s -r FILE [FILE ... FILE]\n", argv[0]); + printf(" recombine uncompressed part (FILE.u) and reconstruction\n"); + printf(" info (FILE.r) into deflate stream (FILE)\n"); + printf(" %s -x FILE [FILE ... FILE]\n", argv[0]); + printf(" recombines into FILE.x instead of FILE\n"); + return -1; +} diff --git a/contrib/preflate/preflate_checker.cpp b/contrib/preflate/preflate_checker.cpp index 51f0680..73d7061 100644 --- a/contrib/preflate/preflate_checker.cpp +++ b/contrib/preflate/preflate_checker.cpp @@ -26,6 +26,7 @@ #include "support/outputcachestream.h" #include +#include bool preflate_checker(const std::vector& deflate_raw) { printf("Checking raw deflate file of size %d\n", (int)deflate_raw.size()); @@ -35,6 +36,8 @@ bool preflate_checker(const std::vector& deflate_raw) { BitInputStream decInBits(decIn); OutputCacheStream decOutCache(decUnc); std::vector blocks; + + auto ts_start = std::chrono::steady_clock::now(); PreflateBlockDecoder bdec(decInBits, decOutCache); if (bdec.status() != PreflateBlockDecoder::OK) { return false; @@ -55,7 +58,9 @@ bool preflate_checker(const std::vector& deflate_raw) { uint8_t remaining_bits = decInBits.get(remaining_bit_count); decOutCache.flush(); std::vector unpacked_output = decUnc.extractData(); + auto ts_end = std::chrono::steady_clock::now(); printf("Unpacked data has size %d\n", (int)unpacked_output.size()); + printf("Unpacking took %g seconds\n", std::chrono::duration(ts_end - ts_start).count()); // Encode PreflateParameters paramsE = estimatePreflateParameters(unpacked_output, 0, blocks); @@ -65,6 +70,8 @@ bool preflate_checker(const std::vector& deflate_raw) { paramsE.veryFarMatchesDetected, paramsE.matchesToStartDetected, paramsE.log2OfMaxChainDepthM1); + + ts_start = std::chrono::steady_clock::now(); PreflateStatisticsCounter counterE; memset(&counterE, 0, sizeof(counterE)); PreflateTokenPredictor tokenPredictorE(paramsE, unpacked_output, 0); @@ -84,9 +91,12 @@ bool preflate_checker(const std::vector& deflate_raw) { treePredictorE.updateCounters(&counterE, i); } counterE.block.incNonZeroPadding(remaining_bits != 0); + ts_end = std::chrono::steady_clock::now(); + printf("Prediction took %g seconds\n", std::chrono::duration(ts_end - ts_start).count()); counterE.print(); + ts_start = std::chrono::steady_clock::now(); PreflateMetaEncoder codecE; if (codecE.error()) { return false; @@ -121,18 +131,53 @@ bool preflate_checker(const std::vector& deflate_raw) { return false; } std::vector preflate_diff = codecE.finish(); + ts_end = std::chrono::steady_clock::now(); printf("Prediction diff has size %d\n", (int)preflate_diff.size()); + printf("Encoding diff took %g seconds\n", std::chrono::duration(ts_end - ts_start).count()); // Decode + ts_start = std::chrono::steady_clock::now(); PreflateMetaDecoder codecD(preflate_diff, unpacked_output.size()); PreflatePredictionDecoder pcodecD; PreflateParameters paramsD; + if (codecD.error() || codecD.metaBlockCount() != 1) { return false; } if (!codecD.beginMetaBlock(pcodecD, paramsD, 0)) { return false; } + + PreflateTokenPredictor tokenPredictorD(paramsD, unpacked_output, 0); + PreflateTreePredictor treePredictorD(unpacked_output, 0); + + MemStream mem; + BitOutputStream bos(mem); + + std::vector dblocks; + unsigned blockno = 0; + bool eof = true; + do { + PreflateTokenBlock block = tokenPredictorD.decodeBlock(&pcodecD); + if (tokenPredictorD.predictionFailure) { + printf("block %d: token uncompress failed\n", blockno); + return false; + } + if (!treePredictorD.decodeBlock(block, &pcodecD)) { + printf("block %d: tree uncompress failed\n", blockno); + return false; + } + if (treePredictorD.predictionFailure) { + printf("block %d: tree uncompress failed\n", blockno); + return false; + } + eof = tokenPredictorD.decodeEOF(&pcodecD); + dblocks.push_back(block); + ++blockno; + } while (!eof); + ts_end = std::chrono::steady_clock::now(); + printf("Decoding diff and reprediction took %g seconds\n", std::chrono::duration(ts_end - ts_start).count()); + if (paramsD.windowBits != paramsE.windowBits) { printf("parameter decoding failed: windowBits mismatch\n"); return false; @@ -150,10 +195,10 @@ bool preflate_checker(const std::vector& deflate_raw) { return false; } if (!paramsD.zlibCompatible && (0 -// || paramsD.farLen3MatchesDetected != paramsE.farLen3MatchesDetected - || paramsD.veryFarMatchesDetected != paramsE.veryFarMatchesDetected - || paramsD.matchesToStartDetected != paramsE.matchesToStartDetected - || paramsD.log2OfMaxChainDepthM1 != paramsE.log2OfMaxChainDepthM1)) { + // || paramsD.farLen3MatchesDetected != paramsE.farLen3MatchesDetected + || paramsD.veryFarMatchesDetected != paramsE.veryFarMatchesDetected + || paramsD.matchesToStartDetected != paramsE.matchesToStartDetected + || paramsD.log2OfMaxChainDepthM1 != paramsE.log2OfMaxChainDepthM1)) { printf("parameter decoding failed: non-zlib flag mismatch\n"); return false; } @@ -163,77 +208,53 @@ bool preflate_checker(const std::vector& deflate_raw) { return false; } - PreflateTokenPredictor tokenPredictorD(paramsD, unpacked_output, 0); - PreflateTreePredictor treePredictorD(unpacked_output, 0); - - MemStream mem; - BitOutputStream bos(mem); - - PreflateBlockReencoder deflater(bos, unpacked_output, 0); - unsigned blockno = 0; - bool eof = true; - do { - if (blockno >= blocks.size()) { - printf("block number too big: org %d, new %d\n", (int)blocks.size(), blockno); + for (size_t blockno = 0, n = std::min(blocks.size(), dblocks.size()); blockno < n; ++blockno) { + if (dblocks[blockno].type != blocks[blockno].type) { + printf("block %d: type differs: org %d, new %d\n", blockno, blocks[blockno].type, dblocks[blockno].type); return false; } - PreflateTokenBlock block = tokenPredictorD.decodeBlock(&pcodecD); - if (tokenPredictorD.predictionFailure) { - printf("block %d: token uncompress failed\n", blockno); - return false; - } - if (block.type != blocks[blockno].type) { - printf("block %d: type differs: org %d, new %d\n", blockno, blocks[blockno].type, block.type); - return false; - } - for (unsigned i = 0, n = std::min(block.tokens.size(), blocks[blockno].tokens.size()); i < n; ++i) { + for (unsigned i = 0, n = std::min(dblocks[blockno].tokens.size(), blocks[blockno].tokens.size()); i < n; ++i) { PreflateToken orgToken = blocks[blockno].tokens[i]; - PreflateToken newToken = block.tokens[i]; + PreflateToken newToken = dblocks[blockno].tokens[i]; if (newToken.len != orgToken.len || newToken.dist != orgToken.dist) { printf("block %d: generated token %d differs: org(%d,%d), new(%d,%d)\n", blockno, i, orgToken.len, orgToken.dist, newToken.len, newToken.dist); return false; } } - if (block.tokens.size() != blocks[blockno].tokens.size()) { + if (dblocks[blockno].tokens.size() != blocks[blockno].tokens.size()) { printf("block %d: differing token count: org %d, new %d\n", - blockno, (int)blocks[blockno].tokens.size(), (int)block.tokens.size()); + blockno, (int)blocks[blockno].tokens.size(), (int)dblocks[blockno].tokens.size()); return false; } - - if (!treePredictorD.decodeBlock(block, &pcodecD)) { - printf("block %d: tree uncompress failed\n", blockno); - return false; - } - if (treePredictorD.predictionFailure) { - printf("block %d: tree uncompress failed\n", blockno); - return false; - } - if (block.type == PreflateTokenBlock::DYNAMIC_HUFF) { - if (block.nlen != blocks[blockno].nlen) { + if (dblocks[blockno].type == PreflateTokenBlock::DYNAMIC_HUFF) { + if (dblocks[blockno].nlen != blocks[blockno].nlen) { printf("block %d: literal/len count differs: org %d, new %d\n", - blockno, blocks[blockno].nlen, block.nlen); + blockno, blocks[blockno].nlen, dblocks[blockno].nlen); return false; } - if (block.ndist != blocks[blockno].ndist) { + if (dblocks[blockno].ndist != blocks[blockno].ndist) { printf("block %d: dist count differs: org %d, new %d\n", - blockno, blocks[blockno].ndist, block.ndist); + blockno, blocks[blockno].ndist, dblocks[blockno].ndist); return false; } - if (block.ncode != blocks[blockno].ncode) { + if (dblocks[blockno].ncode != blocks[blockno].ncode) { printf("block %d: tree code count differs: org %d, new %d\n", - blockno, blocks[blockno].ncode, block.ncode); + blockno, blocks[blockno].ncode, dblocks[blockno].ncode); return false; } - if (block.treecodes != blocks[blockno].treecodes) { + if (dblocks[blockno].treecodes != blocks[blockno].treecodes) { printf("block %d: generated tree codes differs\n", blockno); return false; } } - eof = tokenPredictorD.decodeEOF(&pcodecD); - deflater.writeBlock(block, eof); - ++blockno; - } while (!eof); + } + + ts_start = std::chrono::steady_clock::now(); + PreflateBlockReencoder deflater(bos, unpacked_output, 0); + for (size_t i = 0; i < dblocks.size(); ++i) { + deflater.writeBlock(dblocks[i], i + 1 == dblocks.size()); + } bool non_zero_bits = pcodecD.decodeNonZeroPadding(); if (non_zero_bits) { unsigned bitsToLoad = pcodecD.decodeValue(3); @@ -247,8 +268,10 @@ bool preflate_checker(const std::vector& deflate_raw) { return false; } deflater.flush(); - std::vector deflate_raw_out = mem.extractData(); + ts_end = std::chrono::steady_clock::now(); + printf("Reencoding deflate stream took %g seconds\n", std::chrono::duration(ts_end - ts_start).count()); + for (unsigned i = 0, n = std::min(deflate_raw.size(), deflate_raw_out.size()); i < n; ++i) { if (deflate_raw[i] != deflate_raw_out[i]) { printf("created deflate stream differs at offset %d\n", i); diff --git a/contrib/preflate/preflate_token_predictor.cpp b/contrib/preflate/preflate_token_predictor.cpp index 5f7506e..bff2b0e 100644 --- a/contrib/preflate/preflate_token_predictor.cpp +++ b/contrib/preflate/preflate_token_predictor.cpp @@ -106,7 +106,7 @@ PreflateToken PreflateTokenPredictor::predictToken() { if (((hashNext ^ hash) & this->hash.hashMask) == 0) { unsigned maxSize = std::min(state.availableInputSize() - 1, (unsigned)PreflateConstants::MAX_MATCH); - unsigned rle = 1; + unsigned rle = 0; const unsigned char *c = state.inputCursor(); unsigned char b = c[0]; while (rle < maxSize && c[1 + rle] == b) { @@ -189,11 +189,14 @@ void PreflateTokenPredictor::analyzeBlock( for (unsigned i = 0, n = block.tokens.size(); i < n; ++i) { PreflateToken targetToken = block.tokens[i]; + if (predictEOB()) { analysis.blockSizePredicted = false; } PreflateToken predictedToken = predictToken(); -// printf("T(%d,%d) -> P(%d,%d)\n", targetToken.len, targetToken.dist, predictedToken.len, predictedToken.dist); +#ifdef _DEBUG + printf("B%dT%d: TGT(%d,%d) -> PRD(%d,%d)\n", blockno, i, targetToken.len, targetToken.dist, predictedToken.len, predictedToken.dist); +#endif if (targetToken.len == 1) { if (predictedToken.len > 1) { diff --git a/precomp.cpp b/precomp.cpp index ab57bad..5b3d5c9 100644 --- a/precomp.cpp +++ b/precomp.cpp @@ -19,7 +19,7 @@ // version information #define V_MAJOR 0 #define V_MINOR 4 -#define V_MINOR2 132 +#define V_MINOR2 133 //#define V_STATE "ALPHA" #define V_STATE "EXPERIMENTAL (w/ preflate support)" #define V_MSG "USE FOR TESTING ONLY" @@ -209,7 +209,8 @@ bool level_switch_used = false; bool non_zlib_was_used; // preflate config -size_t meta_block_size = 1 << 21; // 2 MB blocks by default +size_t preflate_meta_block_size = 1 << 21; // 2 MB blocks by default +bool preflate_verify = false; // statistics unsigned int recompressed_streams_count = 0; @@ -543,7 +544,7 @@ bool parseSwitch(bool& val, const char* c, const char* ref) { val = true; return true; } else if (c[l] == '-' && !c[l + 1]) { - val = true; + val = false; return true; } printf("ERROR: Only + or - for this switch (%s) allowed\n", c); @@ -622,7 +623,7 @@ int init(int argc, char* argv[]) { } printf(" - %s\n",V_MSG); printf("Free for non-commercial use - Copyright 2006-2018 by Christian Schneider\n"); - printf("- experimental preflate v0.3.1 support - Copyright 2018 by Dirk Steinke\n\n"); + printf("- experimental preflate v0.3.2 support - Copyright 2018 by Dirk Steinke\n\n"); // init compression and memory level count bool use_zlib_level[81]; @@ -857,14 +858,15 @@ int init(int argc, char* argv[]) { case 'P': { if (!parseSwitch(pdf_bmp_mode, argv[i] + 1, "pdfbmp") - && !parseSwitch(prog_only, argv[i] + 1, "progonly")) { + && !parseSwitch(prog_only, argv[i] + 1, "progonly") + && !parseSwitch(preflate_verify, argv[i] + 1, "pfverify")) { if (parsePrefixText(argv[i] + 1, "pfmeta")) { int mbsize = parseIntUntilEnd(argv[i] + 7, "preflate meta block size"); if (mbsize >= INT_MAX / 1024) { printf("preflate meta block size set too big\n"); exit(1); } - meta_block_size = mbsize * 1024; + preflate_meta_block_size = mbsize * 1024; } else { printf("ERROR: Unknown switch \"%s\"\n", argv[i]); exit(1); @@ -1192,6 +1194,7 @@ int init(int argc, char* argv[]) { //printf(" zl[1..9][1..9] zLib levels to try for compression (comma separated) \n"); if (long_help) { printf(" pfmeta[amount] Split deflate streams into meta blocks of this size in KiB <2048>\n"); + printf(" pfverify Force preflate to verify its generated reconstruction data\n"); } printf(" intense Detect raw zLib headers, too. Slower and more sensitive \n"); if (long_help) { @@ -3112,9 +3115,45 @@ recompress_deflate_result try_recompression_deflate(FILE* file) { result.accepted = preflate_decode(uos, result.recon_data, compressed_stream_size, is, []() { print_work_sign(true); }, 0, - meta_block_size); // you can set a minimum deflate stream size here + preflate_meta_block_size); // you can set a minimum deflate stream size here result.compressed_stream_size = compressed_stream_size; result.uncompressed_stream_size = uos.written(); + + if (preflate_verify && result.accepted) { + if (file == fin) { + seek_64(file, input_file_pos); + } else { + seek_64(file, 0); + } + OwnFileInputStream is2(file); + std::vector orgdata(result.compressed_stream_size); + is2.read(orgdata.data(), orgdata.size()); + + MemStream reencoded_deflate; + MemStream uncompressed_mem(result.uncompressed_in_memory ? std::vector(decomp_io_buf, decomp_io_buf + result.uncompressed_stream_size) : std::vector()); + OwnFileInputStream uncompressed_file(result.uncompressed_in_memory ? NULL : ftempout); + if (!preflate_reencode(reencoded_deflate, result.recon_data, + result.uncompressed_in_memory ? (InputStream&)uncompressed_mem : (InputStream&)uncompressed_file, + result.uncompressed_stream_size, + [] {}) + || orgdata != reencoded_deflate.data()) { + result.accepted = false; + static size_t counter = 0; + char namebuf[50]; + while (true) { + snprintf(namebuf, 49, "preflate_error_%04d.raw", counter++); + FILE* f = fopen(namebuf, "rb"); + if (f) { + fclose(f); + continue; + } + f = fopen(namebuf, "wb"); + fwrite(orgdata.data(), 1, orgdata.size(), f); + fclose(f); + break; + } + } + } } return std::move(result); }