denuvo-ref/precomp-cpp
1
0
Fork 0
mirror of https://github.com/schnaader/precomp-cpp synced 2026-08-23 00:23:04 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Autodetect number of threads for LZMA

Browse source 
- Print memory usage
- Minor changes in compress_easy_mt.cpp (indentation, comments, removing
some unnecessary code)
This commit is contained in:
Christian Schneider 2016-12-28 11:14:41 +01:00
parent 89d6bf423f
commit 769bdc3a74
4 changed files with 36 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

98
contrib/liblzma/compress_easy_mt.cpp
View file

@ -1,20 +1,6 @@
///////////////////////////////////////////////////////////////////////////////
//
/// \file 04_compress_easy_mt.c
/// \brief Compress in multi-call mode using LZMA2 in multi-threaded mode
///
/// Usage: ./04_compress_easy_mt < INFILE > OUTFILE
///
/// Example: ./04_compress_easy_mt < foo > foo.xz
//
// Author: Lasse Collin
//
// This file has been put into the public domain.
// You can do whatever you want with this file.
//
///////////////////////////////////////////////////////////////////////////////
// LZMA compression routines, based on the public domain file 04_compress_easy_mt.c from XZ Utils examples
// Original author: Lasse Collin
//#include <stdbool.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
@ -92,79 +78,41 @@ bool init_decoder(lzma_stream *strm)
return check(ret);
}
//static
bool init_encoder_mt(lzma_stream *strm)
bool init_encoder_mt(lzma_stream *strm, int threads, uint64_t &memory_usage)
{
// The threaded encoder takes the options as pointer to
// a lzma_mt structure.
lzma_mt mt;// = {
// No flags are needed.
lzma_mt mt;
// No flags are needed.
mt .flags = 0;
// Let liblzma determine a sane block size.
mt .block_size = 0;
// Let liblzma determine a sane block size.
mt .block_size = 16 * 1024 * 1024;
// Use no timeout for lzma_code() calls by setting timeout
// to zero. That is, sometimes lzma_code() might block for
// a long time (from several seconds to even minutes).
// If this is not OK, for example due to progress indicator
// needing updates, specify a timeout in milliseconds here.
// See the documentation of lzma_mt in lzma/container.h for
// information how to choose a reasonable timeout.
// Use no timeout for lzma_code() calls by setting timeout
// to zero. That is, sometimes lzma_code() might block for
// a long time (from several seconds to even minutes).
// If this is not OK, for example due to progress indicator
// needing updates, specify a timeout in milliseconds here.
// See the documentation of lzma_mt in lzma/container.h for
// information how to choose a reasonable timeout.
mt .timeout = 110;
// Use the default preset (6) for LZMA2.
// To use a preset, filters must be set to NULL.
// Use the default preset (6) for LZMA2.
// To use a preset, filters must be set to NULL.
mt .preset = //LZMA_PRESET_DEFAULT; //UINT32_C(8);
UINT32_C(9);// | LZMA_PRESET_EXTREME; // 9 OR 9e
#ifdef __BIG_DICT_REQUIRES_MUCH_MORE_MEM // > 8 GB @ _WIN64 || __amd64__
lzma_options_lzma opt_lzma;
if (lzma_lzma_preset(&opt_lzma, mt.preset)) { // LZMA_PRESET_DEFAULT
fprintf(stderr, "Unsupported preset, possibly a bug\n");
return false;
}
opt_lzma.dict_size *= 2;
lzma_filter filters[] = {
{ LZMA_FILTER_LZMA2, &opt_lzma },
{ LZMA_VLI_UNKNOWN, NULL },
};
mt.filters = filters;
#else
UINT32_C(6);// | LZMA_PRESET_EXTREME; // 9 OR 9e
mt.filters = NULL;
#endif
// Use CRC64 for integrity checking. See also
// 01_compress_easy.c about choosing the integrity check.
mt.check = LZMA_CHECK_CRC32;
//};
// Detect how many threads the CPU supports.
mt.threads = dict;//lzma_cputhreads();
// If the number of CPU cores/threads cannot be detected,
// use one thread. Note that this isn't the same as the normal
// single-threaded mode as this will still split the data into
// blocks and use more RAM than the normal single-threaded mode.
// You may want to consider using lzma_easy_encoder() or
// lzma_stream_encoder() instead of lzma_stream_encoder_mt() if
// lzma_cputhreads() returns 0 or 1.
if (mt.threads == 0)
mt.threads = 1;
// If the number of CPU cores/threads exceeds threads_max,
// limit the number of threads to keep memory usage lower.
// The number 8 is arbitrarily chosen and may be too low or
// high depending on the compression preset and the computer
// being used.
//
// FIXME: A better way could be to check the amount of RAM
// (or available RAM) and use lzma_stream_encoder_mt_memusage()
// to determine if the number of threads should be reduced.
const uint32_t threads_max = 8;
if (mt.threads > threads_max)
mt.threads = threads_max;
mt.threads = threads;
// Initialize the threaded encoder.
lzma_ret ret = lzma_stream_encoder_mt(strm, &mt);
memory_usage = lzma_stream_encoder_mt_memusage(&mt);
return check(ret);
}
}

2
contrib/liblzma/precomp_xz.h
View file

@ -5,7 +5,7 @@
bool init_lzma1(lzma_stream *strm);
bool init_lzma2(lzma_stream *strm);
bool init_encoder_mt(lzma_stream *strm);
bool init_encoder_mt(lzma_stream *strm, int threads, uint64_t &memory_usage);
bool init_decoder(lzma_stream *strm);
#endif /* ifndef PRECOMP_XZ_H */

2
make.bat
View file

@ -114,7 +114,7 @@ move /Y *.o ..\..\ > nul
popd
:nocontrib
echo Building precomp...
%GPP% %DCOMFORT% %MPARAM% -static -static-libgcc -static-libstdc++ -lpthread -Wall precomp.cpp %JPG_O% %MP3_O% %GIF_O% %BZIP_O% %ZLIB_O% %LIBLZMA_CPP% %LIBLZMA_O% -O2 -fomit-frame-pointer -s -o%EXE1%%EXE2%.exe
%GPP% %DCOMFORT% %MPARAM% -static -static-libgcc -static-libstdc++ -DMINGW -lpthread -Wall precomp.cpp %JPG_O% %MP3_O% %GIF_O% %BZIP_O% %ZLIB_O% %LIBLZMA_CPP% %LIBLZMA_O% -O2 -fomit-frame-pointer -s -o%EXE1%%EXE2%.exe
if not %ERRORLEVEL% == 0 echo ERROR!!!
if %ERRORLEVEL% == 0 echo.
if %ERRORLEVEL% == 0 echo Build successful.

12
precomp.cpp
View file

@ -49,6 +49,10 @@
#include <sstream>
#include <string>
#include <signal.h>
#include <thread>
#ifdef MINGW
#include "contrib\mingw_std_threads\mingw.thread.h"
#endif
#ifdef _MSC_VER
#include <io.h>
#define ftruncate _chsize
@ -9175,10 +9179,16 @@ void init_compress_otf() {
break;
}
case OTF_XZ_MT: {
if (!init_encoder_mt(&otf_xz_stream_c)) {
uint64_t memory_usage = 0;
int threads = std::thread::hardware_concurrency();
if (threads == 0) threads = 2;
if (!init_encoder_mt(&otf_xz_stream_c, threads, memory_usage)) {
printf("ERROR: xz Multi-Threaded init failed\n");
exit(1);
}
printf("Using LZMA for compression, %i threads, memory usage: ", threads);
print64(memory_usage / 1000000);
printf(" MB\n");
break;
}
}