mirror of
https://github.com/facebook/zstd
synced 2026-08-22 22:32:07 -04:00
Merge 2131c667af into 82d322c497
This commit is contained in:
Rohith Matam
GitHub
commit
f1c93c6180
2 changed files with 78 additions and 31 deletions
|
|
@ -156,10 +156,8 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
|
|||
|
||||
assert(optPtr->symbolCosts != NULL);
|
||||
if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) {
|
||||
|
||||
/* huffman stats covering the full value set : table presumed generated by dictionary */
|
||||
optPtr->priceType = zop_dynamic;
|
||||
|
||||
if (compressedLiterals) {
|
||||
/* generate literals statistics from huffman table */
|
||||
unsigned lit;
|
||||
|
|
@ -172,9 +170,20 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
|
|||
optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
|
||||
optPtr->litSum += optPtr->litFreq[lit];
|
||||
} }
|
||||
} else {
|
||||
assert(optPtr->litFreq != NULL);
|
||||
if (compressedLiterals) {
|
||||
/* base initial cost of literals on direct frequency within src */
|
||||
unsigned lit = MaxLit;
|
||||
HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */
|
||||
optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible);
|
||||
}
|
||||
}
|
||||
|
||||
if (optPtr->symbolCosts->fse.litlength_repeatMode == FSE_repeat_valid) {
|
||||
{ unsigned ll;
|
||||
FSE_CState_t llstate;
|
||||
optPtr->priceType = zop_dynamic;
|
||||
FSE_initCState(&llstate, optPtr->symbolCosts->fse.litlengthCTable);
|
||||
optPtr->litLengthSum = 0;
|
||||
for (ll=0; ll<=MaxLL; ll++) {
|
||||
|
|
@ -184,9 +193,22 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
|
|||
optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
|
||||
optPtr->litLengthSum += optPtr->litLengthFreq[ll];
|
||||
} }
|
||||
} else {
|
||||
unsigned const baseLLfreqs[MaxLL+1] = {
|
||||
4, 2, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1
|
||||
};
|
||||
ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs));
|
||||
optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1);
|
||||
}
|
||||
|
||||
if (optPtr->symbolCosts->fse.matchlength_repeatMode == FSE_repeat_valid) {
|
||||
{ unsigned ml;
|
||||
FSE_CState_t mlstate;
|
||||
optPtr->priceType = zop_dynamic;
|
||||
FSE_initCState(&mlstate, optPtr->symbolCosts->fse.matchlengthCTable);
|
||||
optPtr->matchLengthSum = 0;
|
||||
for (ml=0; ml<=MaxML; ml++) {
|
||||
|
|
@ -196,9 +218,17 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
|
|||
optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
|
||||
optPtr->matchLengthSum += optPtr->matchLengthFreq[ml];
|
||||
} }
|
||||
} else {
|
||||
unsigned ml;
|
||||
for (ml=0; ml<=MaxML; ml++)
|
||||
optPtr->matchLengthFreq[ml] = 1;
|
||||
optPtr->matchLengthSum = MaxML+1;
|
||||
}
|
||||
|
||||
if (optPtr->symbolCosts->fse.offcode_repeatMode == FSE_repeat_valid) {
|
||||
{ unsigned of;
|
||||
FSE_CState_t ofstate;
|
||||
optPtr->priceType = zop_dynamic;
|
||||
FSE_initCState(&ofstate, optPtr->symbolCosts->fse.offcodeCTable);
|
||||
optPtr->offCodeSum = 0;
|
||||
for (of=0; of<=MaxOff; of++) {
|
||||
|
|
@ -208,34 +238,7 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
|
|||
optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
|
||||
optPtr->offCodeSum += optPtr->offCodeFreq[of];
|
||||
} }
|
||||
|
||||
} else { /* first block, no dictionary */
|
||||
|
||||
assert(optPtr->litFreq != NULL);
|
||||
if (compressedLiterals) {
|
||||
/* base initial cost of literals on direct frequency within src */
|
||||
unsigned lit = MaxLit;
|
||||
HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */
|
||||
optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible);
|
||||
}
|
||||
|
||||
{ unsigned const baseLLfreqs[MaxLL+1] = {
|
||||
4, 2, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1
|
||||
};
|
||||
ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs));
|
||||
optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1);
|
||||
}
|
||||
|
||||
{ unsigned ml;
|
||||
for (ml=0; ml<=MaxML; ml++)
|
||||
optPtr->matchLengthFreq[ml] = 1;
|
||||
}
|
||||
optPtr->matchLengthSum = MaxML+1;
|
||||
|
||||
} else {
|
||||
{ unsigned const baseOFCfreqs[MaxOff+1] = {
|
||||
6, 2, 1, 1, 2, 3, 4, 4,
|
||||
4, 3, 2, 1, 1, 1, 1, 1,
|
||||
|
|
@ -245,7 +248,6 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
|
|||
ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs));
|
||||
optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
} else { /* new block : scale down accumulated statistics */
|
||||
|
|
|
|||
|
|
@ -3327,6 +3327,51 @@ static int basicUnitTests(U32 const seed, double compressibility)
|
|||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : optimal parser with CDict containing partial FSE tables : ", testNb++);
|
||||
{ static const BYTE tinyDictInput[446] = {
|
||||
0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f, 0x0f, 0x00, 0x28, 0xe5,
|
||||
0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24,
|
||||
0x40, 0x80, 0x00, 0x01, 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08,
|
||||
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1,
|
||||
0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18, 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18,
|
||||
0x63, 0x8c, 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00,
|
||||
0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61, 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f,
|
||||
0x72, 0x74, 0x6f, 0x72, 0x20, 0x65, 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41,
|
||||
0x6c, 0x69, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x20, 0x6d, 0x61, 0x74,
|
||||
0x63, 0x68, 0x20, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x64, 0x69, 0x63, 0x74, 0x69,
|
||||
0x6f, 0x6e, 0x61, 0x72, 0x79, 0x20, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x20, 0x61, 0x61, 0x61, 0x61,
|
||||
0x61, 0x61, 0x61, 0x61, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x20, 0x6d,
|
||||
0x61, 0x74, 0x63, 0x68, 0x20, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x64, 0x69, 0x63,
|
||||
0x74, 0x61, 0x72, 0x79, 0x20, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x20, 0x61, 0x61, 0x61, 0x61, 0x61,
|
||||
0x61, 0x61, 0x61, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x20, 0x6d, 0x61,
|
||||
0x74, 0x63, 0x68, 0x20, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x64, 0x69, 0x63, 0x74,
|
||||
0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
|
||||
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
|
||||
0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18, 0x86, 0x61, 0x18, 0xc6, 0x18,
|
||||
0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x72, 0x79, 0x20, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x20, 0x61,
|
||||
0x61, 0x61, 0x61, 0x61, 0x61, 0x79, 0x20, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x20, 0x61, 0x61, 0x61,
|
||||
0x61, 0x61, 0x61, 0x61, 0x61, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x20,
|
||||
0x6d, 0x61, 0x74, 0x63, 0x68, 0x20, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x64, 0x69,
|
||||
0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x20, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x20, 0x61,
|
||||
0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x64, 0x69, 0x63, 0x74, 0x69, 0x30, 0x31, 0x32, 0x33,
|
||||
0x34, 0x35, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x30,
|
||||
0x31, 0x32, 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x10, 0x00, 0x02,
|
||||
0x6f, 0x08, 0x80, 0x04, 0x04, 0x0a, 0x0a, 0x06, 0x01, 0x00, 0x00, 0x90, 0x00, 0x05
|
||||
};
|
||||
size_t const tinyDictSize = 0x90;
|
||||
size_t const tinySrcSize = 0x100;
|
||||
ZSTD_compressionParameters const cParams = ZSTD_getCParams(22, tinySrcSize, tinyDictSize);
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(tinyDictInput, tinyDictSize,
|
||||
ZSTD_dlm_byRef, ZSTD_dct_fullDict,
|
||||
cParams, ZSTD_defaultCMem);
|
||||
if (cdict==NULL) goto _output_error;
|
||||
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
|
||||
tinyDictInput + tinyDictSize, tinySrcSize, cdict);
|
||||
ZSTD_freeCDict(cdict);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
|
||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue