From 7be858bd417ee2e258bf721f0e271f2f6bed1c87 Mon Sep 17 00:00:00 2001 From: Christian Schneider Date: Sun, 30 Sep 2018 10:08:45 +0200 Subject: [PATCH] Merge preflate 0.3.5 changes - fixes #88 and #89 --- contrib/preflate/preflate_checker.cpp | 14 +++--- contrib/preflate/preflate_decoder.cpp | 12 +++-- contrib/preflate/preflate_decoder.h | 5 +- contrib/preflate/preflate_hash_chain.cpp | 47 +++++++++++-------- contrib/preflate/preflate_predictor_state.cpp | 12 ++--- contrib/preflate/preflate_reencoder.h | 5 +- .../preflate/preflate_statistical_codec.cpp | 2 +- contrib/preflate/preflate_statistical_codec.h | 24 ++++++++-- .../preflate/preflate_statistical_debug.cpp | 4 +- contrib/preflate/support/bitstream.h | 2 +- contrib/preflate/support/outputcachestream.h | 2 +- contrib/preflate/support/stream.h | 6 +++ precomp.cpp | 2 +- 13 files changed, 87 insertions(+), 50 deletions(-) diff --git a/contrib/preflate/preflate_checker.cpp b/contrib/preflate/preflate_checker.cpp index 73d7061..bbf569d 100644 --- a/contrib/preflate/preflate_checker.cpp +++ b/contrib/preflate/preflate_checker.cpp @@ -210,41 +210,41 @@ bool preflate_checker(const std::vector& deflate_raw) { for (size_t blockno = 0, n = std::min(blocks.size(), dblocks.size()); blockno < n; ++blockno) { if (dblocks[blockno].type != blocks[blockno].type) { - printf("block %d: type differs: org %d, new %d\n", blockno, blocks[blockno].type, dblocks[blockno].type); + printf("block %zu: type differs: org %d, new %d\n", blockno, blocks[blockno].type, dblocks[blockno].type); return false; } for (unsigned i = 0, n = std::min(dblocks[blockno].tokens.size(), blocks[blockno].tokens.size()); i < n; ++i) { PreflateToken orgToken = blocks[blockno].tokens[i]; PreflateToken newToken = dblocks[blockno].tokens[i]; if (newToken.len != orgToken.len || newToken.dist != orgToken.dist) { - printf("block %d: generated token %d differs: org(%d,%d), new(%d,%d)\n", + printf("block %zu: generated token %d differs: org(%d,%d), new(%d,%d)\n", blockno, i, orgToken.len, orgToken.dist, newToken.len, newToken.dist); return false; } } if (dblocks[blockno].tokens.size() != blocks[blockno].tokens.size()) { - printf("block %d: differing token count: org %d, new %d\n", + printf("block %zu: differing token count: org %d, new %d\n", blockno, (int)blocks[blockno].tokens.size(), (int)dblocks[blockno].tokens.size()); return false; } if (dblocks[blockno].type == PreflateTokenBlock::DYNAMIC_HUFF) { if (dblocks[blockno].nlen != blocks[blockno].nlen) { - printf("block %d: literal/len count differs: org %d, new %d\n", + printf("block %zu: literal/len count differs: org %d, new %d\n", blockno, blocks[blockno].nlen, dblocks[blockno].nlen); return false; } if (dblocks[blockno].ndist != blocks[blockno].ndist) { - printf("block %d: dist count differs: org %d, new %d\n", + printf("block %zu: dist count differs: org %d, new %d\n", blockno, blocks[blockno].ndist, dblocks[blockno].ndist); return false; } if (dblocks[blockno].ncode != blocks[blockno].ncode) { - printf("block %d: tree code count differs: org %d, new %d\n", + printf("block %zu: tree code count differs: org %d, new %d\n", blockno, blocks[blockno].ncode, dblocks[blockno].ncode); return false; } if (dblocks[blockno].treecodes != blocks[blockno].treecodes) { - printf("block %d: generated tree codes differs\n", blockno); + printf("block %zu: generated tree codes differs\n", blockno); return false; } } diff --git a/contrib/preflate/preflate_decoder.cpp b/contrib/preflate/preflate_decoder.cpp index c8ce283..e17eb2d 100644 --- a/contrib/preflate/preflate_decoder.cpp +++ b/contrib/preflate/preflate_decoder.cpp @@ -144,7 +144,7 @@ bool preflate_decode(OutputStream& unpacked_output, uint64_t sumBlockSizes = 0; uint64_t lastEndPos = 0; uint64_t uncompressedMetaStart = 0; - size_t MBSize = std::min(std::max(metaBlockSize, 1 << 18), (1 << 31) - 1); + size_t MBSize = std::min(std::max(metaBlockSize, 1u << 18), (1u << 31) - 1); size_t MBThreshold = (MBSize * 3) >> 1; PreflateDecoderHandler encoder(block_callback); size_t MBcount = 0; @@ -158,14 +158,16 @@ bool preflate_decode(OutputStream& unpacked_output, bool ok = bdec.readBlock(newBlock, last); if (!ok) { - return false; + fail = true; + break; } uint64_t blockSize = decOutCache.cacheEndPos() - lastEndPos; lastEndPos = decOutCache.cacheEndPos(); if (blockSize >= (1 << 31)) { // No mega blocks - return false; + fail = true; + break; } blocks.push_back(newBlock); @@ -223,7 +225,8 @@ bool preflate_decode(OutputStream& unpacked_output, uncompressedOffset, last, paddingBits); if (!task.analyze() || !task.encode()) { - return false; + fail = true; + break; } } else { if (futureQueue.size() >= queueLimit) { @@ -232,6 +235,7 @@ bool preflate_decode(OutputStream& unpacked_output, std::shared_ptr data = first.get(); if (!data || !data->encode()) { fail = true; + break; } } std::shared_ptr ptask; diff --git a/contrib/preflate/preflate_decoder.h b/contrib/preflate/preflate_decoder.h index 5abe317..d682219 100644 --- a/contrib/preflate/preflate_decoder.h +++ b/contrib/preflate/preflate_decoder.h @@ -23,13 +23,14 @@ #include "support/stream.h" #include "support/task_pool.h" -class PreflateTokenPredictor; -class PreflateTreePredictor; +struct PreflateTokenPredictor; +struct PreflateTreePredictor; class PreflateDecoderTask { public: class Handler { public: + virtual ~Handler() {} virtual uint32_t setModel(const PreflateStatisticsCounter&, const PreflateParameters&) = 0; virtual bool beginEncoding(const uint32_t metaBlockId, PreflatePredictionEncoder&, const uint32_t modelId) = 0; virtual bool endEncoding(const uint32_t metaBlockId, PreflatePredictionEncoder&, const size_t uncompressedSize) = 0; diff --git a/contrib/preflate/preflate_hash_chain.cpp b/contrib/preflate/preflate_hash_chain.cpp index 1c92518..7776fbd 100644 --- a/contrib/preflate/preflate_hash_chain.cpp +++ b/contrib/preflate/preflate_hash_chain.cpp @@ -59,10 +59,10 @@ void PreflateHashChainExt::updateHash(const unsigned l) { if (pos - totalShift >= 0xfe08) { reshift(); } - for (unsigned i = 0; i < l; ++i) { - updateRunningHash(b[2 + i]); + for (unsigned i = 2; i < std::min(l + 2, _input.remaining()); ++i) { + updateRunningHash(b[i]); unsigned h = runningHash & hashMask; - unsigned p = (pos + i) - totalShift; + unsigned p = (pos + i - 2) - totalShift; chainDepth[p] = chainDepth[head[h]] + 1; prev[p] = head[h]; head[h] = p; @@ -75,24 +75,31 @@ void PreflateHashChainExt::skipHash(const unsigned l) { if (pos - totalShift >= 0xfe08) { reshift(); } - updateRunningHash(b[2]); - unsigned h = runningHash & hashMask; - unsigned p = (pos) - totalShift; - chainDepth[p] = chainDepth[head[h]] + 1; - prev[p] = head[h]; - head[h] = p; + unsigned remaining = _input.remaining(); + if (remaining > 2) { + updateRunningHash(b[2]); + unsigned h = runningHash & hashMask; + unsigned p = (pos) - totalShift; + chainDepth[p] = chainDepth[head[h]] + 1; + prev[p] = head[h]; + head[h] = p; - // Skipped data is not inserted into the hash chain, - // but we must still update the chainDepth, to avoid - // bad analysis results - // -------------------- - for (unsigned i = 1; i < l; ++i) { - unsigned p = (pos + i)-totalShift; - chainDepth[p] = 0xffff8000; + // Skipped data is not inserted into the hash chain, + // but we must still update the chainDepth, to avoid + // bad analysis results + // -------------------- + for (unsigned i = 1; i < l; ++i) { + unsigned p = (pos + i) - totalShift; + chainDepth[p] = 0xffff8000; + } + // l must be at least 3 + if (remaining > l) { + updateRunningHash(b[l]); + if (remaining > l + 1) { + updateRunningHash(b[l + 1]); + } + } } - // l must be at least 3 - updateRunningHash(b[l]); - updateRunningHash(b[l + 1]); _input.advance(l); } void PreflateHashChainExt::reshift() { @@ -103,6 +110,6 @@ void PreflateHashChainExt::reshift() { for (unsigned i = delta + 8, n = 1 << 16; i < n; ++i) { prev[i - delta] = std::max(prev[i], delta) - delta; } - memmove(chainDepth + 8, chainDepth + 8 + delta, (0x10000 - delta) * sizeof(chainDepth[0])); + memmove(chainDepth + 8, chainDepth + 8 + delta, (0x10000 - 8 - delta) * sizeof(chainDepth[0])); totalShift += delta; } diff --git a/contrib/preflate/preflate_predictor_state.cpp b/contrib/preflate/preflate_predictor_state.cpp index 0e404b5..d34b604 100644 --- a/contrib/preflate/preflate_predictor_state.cpp +++ b/contrib/preflate/preflate_predictor_state.cpp @@ -74,9 +74,9 @@ unsigned PreflatePredictorState::prefixCompare( return 0; } - const unsigned char* scan = s2 + 2; - const unsigned char* match = s1 + 2; - const unsigned char* scanend = s2 + maxLen - 8; + const unsigned char* scan = s2 + 3; + const unsigned char* match = s1 + 3; + const unsigned char* scanend = s2 + maxLen; /* while (scan < scanend && *++scan == *++match && *++scan == *++match @@ -84,9 +84,10 @@ unsigned PreflatePredictorState::prefixCompare( && *++scan == *++match && *++scan == *++match && *++scan == *++match && *++scan == *++match) { }*/ - scanend = s2 + maxLen; while (scan < scanend - && *++scan == *++match) { + && *scan == *match) { + ++scan; + ++match; } return scan - s2; @@ -209,7 +210,6 @@ PreflateToken PreflatePredictorState::seqMatch( if (bestLen >= h.niceLen || !chainIt.next()) { return bestMatch; } - unsigned minDistOff = chainIt.len() - PreflateConstants::MIN_MATCH; if (chainIt.dist() > h.curMaxDistHop1Plus + chainIt.len() - PreflateConstants::MIN_MATCH) { return bestMatch; } diff --git a/contrib/preflate/preflate_reencoder.h b/contrib/preflate/preflate_reencoder.h index c3e5d47..51f8ba4 100644 --- a/contrib/preflate/preflate_reencoder.h +++ b/contrib/preflate/preflate_reencoder.h @@ -24,6 +24,7 @@ class PreflateReencoderTask { public: class Handler { public: + virtual ~Handler() {} virtual bool beginDecoding(const uint32_t metaBlockId, PreflatePredictionDecoder&, PreflateParameters&) = 0; virtual bool endDecoding(const uint32_t metaBlockId, PreflatePredictionDecoder&, @@ -41,8 +42,8 @@ public: const size_t uncompressedOffset, const bool lastMetaBlock); - virtual bool decodeAndRepredict(); - virtual bool reencode(); + bool decodeAndRepredict(); + bool reencode(); uint32_t id() { return metaBlockId; diff --git a/contrib/preflate/preflate_statistical_codec.cpp b/contrib/preflate/preflate_statistical_codec.cpp index 6b23807..097d2c9 100644 --- a/contrib/preflate/preflate_statistical_codec.cpp +++ b/contrib/preflate/preflate_statistical_codec.cpp @@ -86,7 +86,7 @@ void PreflateSubModel::buildDefault(const unsigned defval) { } template void PreflateSubModel::build_scale_down() { - unsigned boundBits = 0; + unsigned boundBits = ~0xFFFFu; // Make sure that upper bits are all set, to limit the range of zeroJunk for (unsigned i = 0; i <= N; ++i) { boundBits |= bounds[i]; } diff --git a/contrib/preflate/preflate_statistical_codec.h b/contrib/preflate/preflate_statistical_codec.h index 56b3e2f..9c883f3 100644 --- a/contrib/preflate/preflate_statistical_codec.h +++ b/contrib/preflate/preflate_statistical_codec.h @@ -67,6 +67,27 @@ private: friend struct PreflateCorrectionSubModel; }; +template <> +struct PreflateSubModel<0u> { + static const unsigned L = 0u; + PreflateSubModel() {} + +// void build(const unsigned(&arr)[1], const unsigned defval, const uint8_t prec = 16) {} + void buildDefault(const unsigned defval) {} + void read(ArithmeticDecoder&, const uint8_t) {} + void write(ArithmeticEncoder&, const uint8_t) const {} + void encode(ArithmeticEncoder& codec, const unsigned item) const {} + unsigned decode(ArithmeticDecoder& codec) const { return 0; } + bool isEqualTo(const PreflateSubModel<0u>& m) const { return true; } + + enum { isDefault = 1, isFixed = 1 }; + +private: + void build_impl(const unsigned *arr, const unsigned defval, const uint8_t prec) {} + template + friend struct PreflateCorrectionSubModel; +}; + template struct PreflateCorrectionSubModel { static const unsigned LNEG = NEG; @@ -554,7 +575,6 @@ private: }; bool inError; - bool inBlock; std::vector modelList; std::vector blockList; std::vector reconData; @@ -596,9 +616,7 @@ private: }; bool inError; - bool inBlock; - size_t currentMetaBlockId; const std::vector& reconData; const uint64_t uncompressedSize; std::vector modelList; diff --git a/contrib/preflate/preflate_statistical_debug.cpp b/contrib/preflate/preflate_statistical_debug.cpp index 8b1ae32..3249a76 100644 --- a/contrib/preflate/preflate_statistical_debug.cpp +++ b/contrib/preflate/preflate_statistical_debug.cpp @@ -92,7 +92,7 @@ void printFlagStatistics(const char *txt, unsigned(&flag)[2]) { } } void printCorrectionStatistics(const char *txt, - unsigned data[], unsigned size, unsigned sum, int offset) { + unsigned data[], unsigned size, unsigned sum, unsigned offset) { if (data[offset] == sum) { return; } @@ -106,7 +106,7 @@ void printCorrectionStatistics(const char *txt, if (i != offset && (i == 0 || i + 1 == size)) { printf(" %sx %g%%", i == 0 ? "-" : "+", data[i] * 100.0 / sum); } else { - printf(" %s%d %g%%", i == offset ? "" : (i < offset ? "-" : "+"), (int)labs(i - offset), data[i] * 100.0 / sum); + printf(" %s%d %g%%", i == offset ? "" : (i < offset ? "-" : "+"), (int)labs((int)(i - offset)), data[i] * 100.0 / sum); } } } diff --git a/contrib/preflate/support/bitstream.h b/contrib/preflate/support/bitstream.h index da243c4..eb848aa 100644 --- a/contrib/preflate/support/bitstream.h +++ b/contrib/preflate/support/bitstream.h @@ -94,7 +94,7 @@ public: BitOutputStream(OutputStream&); void put(const size_t value, const unsigned n) { - if (_bitPos + n > BITS) { + if (_bitPos + n >= BITS) { _flush(); } _bits |= (value & ((1 << n) - 1)) << _bitPos; diff --git a/contrib/preflate/support/outputcachestream.h b/contrib/preflate/support/outputcachestream.h index f8193fc..5656c47 100644 --- a/contrib/preflate/support/outputcachestream.h +++ b/contrib/preflate/support/outputcachestream.h @@ -22,7 +22,7 @@ class OutputCacheStream : public OutputStream { public: OutputCacheStream(OutputStream& os); - ~OutputCacheStream(); + virtual ~OutputCacheStream(); size_t write(const unsigned char* buffer, const size_t size) { /* if (size == 1) { diff --git a/contrib/preflate/support/stream.h b/contrib/preflate/support/stream.h index f41f0c6..2d57a06 100644 --- a/contrib/preflate/support/stream.h +++ b/contrib/preflate/support/stream.h @@ -19,17 +19,23 @@ class InputStream { public: + virtual ~InputStream() {} + virtual bool eof() const = 0; virtual size_t read(unsigned char* buffer, const size_t size) = 0; }; class OutputStream { public: + virtual ~OutputStream() {} + virtual size_t write(const unsigned char* buffer, const size_t size) = 0; }; class SeekableStream { public: + virtual ~SeekableStream() {} + virtual uint64_t tell() const = 0; virtual uint64_t seek(const uint64_t newPos) = 0; }; diff --git a/precomp.cpp b/precomp.cpp index 5a7a161..46826d5 100644 --- a/precomp.cpp +++ b/precomp.cpp @@ -624,7 +624,7 @@ int init(int argc, char* argv[]) { } printf(" - %s\n",V_MSG); printf("Free for non-commercial use - Copyright 2006-2018 by Christian Schneider\n"); - printf(" preflate v0.3.4 support - Copyright 2018 by Dirk Steinke\n\n"); + printf(" preflate v0.3.5 support - Copyright 2018 by Dirk Steinke\n\n"); // init compression and memory level count bool use_zlib_level[81];