diff --git a/programs/fileio.c b/programs/fileio.c index 78a906d53..08a6a7b29 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -573,6 +573,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->allowBlockDevices = 0; ret->asyncIO = AIO_supported(); ret->passThrough = -1; + ret->inputBufferSize = 0; /* 0 = use default */ return ret; } @@ -682,6 +683,10 @@ void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) { prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint); } +void FIO_setInputBufferSize(FIO_prefs_t* const prefs, size_t inputBufferSize) { + prefs->inputBufferSize = inputBufferSize; +} + void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) { prefs->testMode = (testMode!=0); } @@ -2615,7 +2620,8 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi } ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_DStreamOutSize()); - ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_DStreamInSize()); + ress.readCtx = AIO_ReadPool_create(prefs, + prefs->inputBufferSize > 0 ? prefs->inputBufferSize : ZSTD_DStreamInSize()); return ress; } diff --git a/programs/fileio.h b/programs/fileio.h index 5d7334ef5..154042997 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -87,6 +87,7 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable); void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize); void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize); void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint); +void FIO_setInputBufferSize(FIO_prefs_t* const prefs, size_t inputBufferSize); void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode); void FIO_setLiteralCompressionMode( FIO_prefs_t* const prefs, diff --git a/programs/fileio_types.h b/programs/fileio_types.h index 9bbb51549..3f827c3b9 100644 --- a/programs/fileio_types.h +++ b/programs/fileio_types.h @@ -59,6 +59,7 @@ typedef struct FIO_prefs_s { int removeSrcFile; int overwrite; int asyncIO; + size_t inputBufferSize; /* size of the input buffer for decompression (0 = default) */ /* Computation resources preferences */ unsigned memLimit; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 31b06d67b..186c7f19b 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -247,6 +247,7 @@ static void usageAdvanced(const char* programName) DISPLAYOUT(" --exclude-compressed Only compress files that are not already compressed.\n\n"); DISPLAYOUT(" --stream-size=# Specify size of streaming input from STDIN.\n"); + DISPLAYOUT(" --ibuf-size=# Specify input buffer size for decompression. Helps with large files on slow disks.\n"); DISPLAYOUT(" --size-hint=# Optimize compression parameters for streaming input of approximately size #.\n"); DISPLAYOUT(" --target-compressed-block-size=#\n"); DISPLAYOUT(" Generate compressed blocks of approximately # size.\n\n"); @@ -928,6 +929,7 @@ int main(int argCount, const char* argv[]) size_t streamSrcSize = 0; size_t targetCBlockSize = 0; size_t srcSizeHint = 0; + size_t inputBufferSize = 0; size_t nbInputFileNames = 0; int dictCLevel = g_defaultDictCLevel; unsigned dictSelect = g_defaultSelectivityLevel; @@ -1112,6 +1114,7 @@ int main(int argCount, const char* argv[]) if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; } if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badUsage(programName, originalArgument); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; } if (longCommandWArg(&argument, "--stream-size")) { NEXT_TSIZE(streamSrcSize); continue; } + if (longCommandWArg(&argument, "--ibuf-size")) { NEXT_TSIZE(inputBufferSize); continue; } if (longCommandWArg(&argument, "--target-compressed-block-size")) { NEXT_TSIZE(targetCBlockSize); continue; } if (longCommandWArg(&argument, "--size-hint")) { NEXT_TSIZE(srcSizeHint); continue; } if (longCommandWArg(&argument, "--output-dir-flat")) { @@ -1636,6 +1639,7 @@ int main(int argCount, const char* argv[]) FIO_setStreamSrcSize(prefs, streamSrcSize); FIO_setTargetCBlockSize(prefs, targetCBlockSize); FIO_setSrcSizeHint(prefs, srcSizeHint); + FIO_setInputBufferSize(prefs, inputBufferSize); FIO_setLiteralCompressionMode(prefs, literalCompressionMode); FIO_setSparseWrite(prefs, 0); if (adaptMin > cLevel) cLevel = adaptMin;