denuvo-ref/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd synced 2026-08-22 22:32:07 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve compression performance by conditionally skipping lazy evaluation

Browse source 
Optional ZSTD_LAZY_SKIP_LONG_MATCHES flag provided to speed up compression
  by restricting lazy evaluation to short matches.
  Applied to levels 6 and 7.

  This helps improve lazy compression strategy.
  Skip determinism test when enabled.

  Optimization disabled by default.
This commit is contained in:
Jangra, Ravi 2025-11-18 16:40:25 +05:30 • committed by sraut
parent d462f691ba
commit 58cebf8552
9 changed files with 110 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

5
lib/README.md
View file

@ -212,6 +212,11 @@ The file structure is designed to make this selection manually achievable for an
For this scenario, it can be set as `ZDICT_QSORT=ZDICT_QSORT_C90`.
Other selectable suffixes are `_GNU`, `_APPLE`, `_MSVC` and `_C11`.
- The build macro `ZSTD_LAZY_SKIP_LONG_MATCHES` can be defined for fast lazy
evaluation from compression level 6 to 7. This improves compression performance with
little to no tradeoff for compressibility and decompression performance.
This option is designed for applications with demanding compression speed requirements.
#### Windows : using MinGW+MSYS to create DLL
DLL can be created using MinGW+MSYS with the `make libzstd` command.

26
lib/compress/clevels.h
View file

@ -129,6 +129,32 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
},
};
#ifdef ZSTD_LAZY_SKIP_LONG_MATCHES
static const ZSTD_compressionParameters ZSTD_CParametersFastCompress[4][2] = {
{ /* "default" - for any srcSize > 256 KB */
/* W, C, H, S, L, TL, strat */
/* Larger windows and settings from next levels to improve ratio.
* Accounts for ratio drop from selective lazy evaluation */
{ 23, 19, 20, 4, 5, 8, ZSTD_lazy }, /* level 6 */
{ 23, 19, 20, 4, 5, 16, ZSTD_lazy }, /* level 7 */
},
{ /* for srcSize <= 256 KB */
/* W, C, H, S, L, T, strat */
{ 20, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 6 */
{ 20, 18, 19, 4, 5, 8, ZSTD_lazy }, /* level 7 */
},
{ /* for srcSize <= 128 KB */
/* W, C, H, S, L, T, strat */
{ 19, 16, 17, 3, 4, 8, ZSTD_lazy }, /* level 6 */
{ 19, 16, 17, 4, 5, 16, ZSTD_lazy }, /* level 7 */
},
{ /* for srcSize <= 16 KB */
/* W, C, H, S, L, T, strat */
{ 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */
{ 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */
},
};
#endif
#endif /* ZSTD_CLEVELS_H */

14
lib/compress/zstd_compress.c
View file

@ -5260,6 +5260,13 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
ZSTD_buffered_policy_e zbuff)
{
size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize;
#ifdef ZSTD_LAZY_SKIP_LONG_MATCHES
int compressionLevel = (params->compressionLevel == 0) ? cctx->requestedParams.compressionLevel
: params->compressionLevel;
compressionLevel = MIN(MAX(0, compressionLevel), ZSTD_maxCLevel());
assert(compressionLevel >= 0 && compressionLevel <= ZSTD_MAX_CLEVEL);
cctx->seqStore.lazyLimit = ZSTD_compressFastLazyLimit[compressionLevel];
#endif
#if ZSTD_TRACE
cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0;
#endif
@ -8288,7 +8295,14 @@ static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel,
else if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL;
else row = compressionLevel;
#ifdef ZSTD_LAZY_SKIP_LONG_MATCHES
{ ZSTD_compressionParameters cp =
((unsigned)(compressionLevel - 6) <= 1) /* compressionLevel == 6 || compressionLevel == 7 */
? ZSTD_CParametersFastCompress[tableID][compressionLevel - 6]
: ZSTD_defaultCParameters[tableID][row];
#else
{ ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row];
#endif
DEBUGLOG(5, "ZSTD_getCParams_internal selected tableID: %u row: %u strat: %u", tableID, row, (U32)cp.strategy);
/* acceleration factor */
if (compressionLevel < 0) {

35
lib/compress/zstd_compress_internal.h
View file

@ -39,6 +39,38 @@
The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table reuse with a different strategy.
This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */
#ifdef ZSTD_LAZY_SKIP_LONG_MATCHES
/* Lazy evaluation is performed only if the first match length
* is <= ZSTD_compressFastLazyLimit */
#define ZSTD_COMPRESS_FAST_BASE_LAZY_LIMIT 5
#define ZSTD_MAX_CLEVEL 22
static const size_t ZSTD_compressFastLazyLimit[ZSTD_MAX_CLEVEL + 1] = {
ZSTD_COMPRESS_FAST_BASE_LAZY_LIMIT, /* base for negative levels */
0, /* level 1 */
0, /* level 2 */
0, /* level 3 */
0, /* level 4 */
0, /* level 5 */
ZSTD_COMPRESS_FAST_BASE_LAZY_LIMIT, /* level 6 */
ZSTD_COMPRESS_FAST_BASE_LAZY_LIMIT + 1, /* level 7 */
0, /* level 8.*/
0, /* level 9.*/
0, /* level 10.*/
0, /* level 11.*/
0, /* level 12.*/
0, /* level 13 */
0, /* level 14 */
0, /* level 15 */
0, /* level 16 */
0, /* level 17 */
0, /* level 18 */
0, /* level 19 */
0, /* level 20 */
0, /* level 21 */
0, /* level 22 */
};
#endif
/*-*************************************
* Context memory management
@ -105,6 +137,9 @@ typedef struct {
BYTE* ofCode;
size_t maxNbSeq;
size_t maxNbLit;
#ifdef ZSTD_LAZY_SKIP_LONG_MATCHES
size_t lazyLimit; /* Match length limit to allow lazy evaluation */
#endif
/* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
* in the seqStore that has a value larger than U16 (if it exists). To do so, we increment

7
lib/compress/zstd_lazy.c
View file

@ -1590,6 +1590,9 @@ size_t ZSTD_compressBlock_lazy_generic(
prefixLowestIndex - (U32)(dictEnd - dictBase) :
0;
const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest));
#ifdef ZSTD_LAZY_SKIP_LONG_MATCHES
size_t lazyLimit = seqStore->lazyLimit;
#endif
DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u) (searchFunc=%u)", (U32)dictMode, (U32)searchMethod);
ip += (dictAndPrefixLength == 0);
@ -1669,7 +1672,11 @@ size_t ZSTD_compressBlock_lazy_generic(
}
/* let's try to find a better solution */
#ifdef ZSTD_LAZY_SKIP_LONG_MATCHES
if ((lazyLimit == 0 || matchLength <= lazyLimit) && depth >= 1) /* restrict lazy eval to short matches only */
#else
if (depth>=1)
#endif
while (ip<ilimit) {
DEBUGLOG(7, "search depth 1");
ip ++;

8
lib/libzstd.mk
View file

@ -53,6 +53,10 @@ else
ZSTD_STRIP_ERROR_STRINGS ?= 0
endif
# Lazy match skip optimization (levels 6-7): skip lazy evaluation when
# match length exceeds threshold, trading minimal compression for speed.
ZSTD_LAZY_SKIP_LONG_MATCHES ?= 0
# Assembly support
ZSTD_NO_ASM ?= 0
@ -155,6 +159,10 @@ ZSTD_LEGACY_FILES :=
ZSTD_DECOMPRESS_AMD64_ASM_FILES := $(sort $(wildcard $(LIB_SRCDIR)/decompress/*_amd64.S))
ifneq ($(ZSTD_LAZY_SKIP_LONG_MATCHES), 0)
CFLAGS += -DZSTD_LAZY_SKIP_LONG_MATCHES
endif
ifneq ($(ZSTD_NO_ASM), 0)
CPPFLAGS += -DZSTD_DISABLE_ASM
else

2
tests/cli-tests/common/platform.sh
View file

@ -40,7 +40,7 @@ else
hasMT="true"
fi
if zstd -vv --version | grep -q 'non-deterministic'; then
if zstd -vv --version | grep -q 'non-deterministic' || [ "$ZSTD_LAZY_SKIP_LONG_MATCHES" = "1" ]; then
NON_DETERMINISTIC="true"
else
NON_DETERMINISTIC=""

1
tests/cli-tests/run.py
View file

@ -709,6 +709,7 @@ if __name__ == "__main__":
env["COMMON"] = os.path.abspath(os.path.join(args.test_dir, "common"))
env["PATH"] = bin_dir + ":" + os.getenv("PATH", "")
env["LC_ALL"] = "C"
env["ZSTD_LAZY_SKIP_LONG_MATCHES"] = os.environ.get('ZSTD_LAZY_SKIP_LONG_MATCHES', '0')
opts = Options(
env=env,

13
tests/fuzzer.c
View file

@ -2866,6 +2866,18 @@ static int basicUnitTests(U32 const seed, double compressibility)
const void* const dict = (const char*)CNBuffer;
const void* const contentStart = (const char*)dict + flatdictSize;
/* These upper bounds are generally within a few bytes of the compressed size */
#ifdef ZSTD_LAZY_SKIP_LONG_MATCHES
size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770,
3770, 3772, 3772, 3750, 3750,
3742, 3675, 3674, 3665, 3664,
3663, 3662, 3661, 3660, 3660,
3660, 3660, 3660 };
size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2840, 2950,
2950, 2952, 2927, 2900, 2892,
2910, 2910, 2910, 2780, 2775,
2765, 2760, 2755, 2754, 2753,
2753, 2753, 2753 };
#else
size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770,
3770, 3770, 3770, 3750, 3750,
3742, 3675, 3674, 3665, 3664,
@ -2876,6 +2888,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
2910, 2910, 2910, 2780, 2775,
2765, 2760, 2755, 2754, 2753,
2753, 2753, 2753 };
#endif
int l = 1;
int const maxLevel = ZSTD_maxCLevel();
/* clevels with strategies that support rowhash on small inputs */