diff --git a/contrib/liblzma/compress_easy_mt.cpp b/contrib/liblzma/compress_easy_mt.cpp index a0e3e5d..092450e 100644 --- a/contrib/liblzma/compress_easy_mt.cpp +++ b/contrib/liblzma/compress_easy_mt.cpp @@ -1,20 +1,6 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file 04_compress_easy_mt.c -/// \brief Compress in multi-call mode using LZMA2 in multi-threaded mode -/// -/// Usage: ./04_compress_easy_mt < INFILE > OUTFILE -/// -/// Example: ./04_compress_easy_mt < foo > foo.xz -// -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. -// -/////////////////////////////////////////////////////////////////////////////// +// LZMA compression routines, based on the public domain file 04_compress_easy_mt.c from XZ Utils examples +// Original author: Lasse Collin -//#include #include #include #include @@ -92,79 +78,41 @@ bool init_decoder(lzma_stream *strm) return check(ret); } -//static -bool init_encoder_mt(lzma_stream *strm) +bool init_encoder_mt(lzma_stream *strm, int threads, uint64_t &memory_usage) { // The threaded encoder takes the options as pointer to // a lzma_mt structure. - lzma_mt mt;// = { - // No flags are needed. + lzma_mt mt; + + // No flags are needed. mt .flags = 0; - // Let liblzma determine a sane block size. - mt .block_size = 0; + // Let liblzma determine a sane block size. + mt .block_size = 16 * 1024 * 1024; - // Use no timeout for lzma_code() calls by setting timeout - // to zero. That is, sometimes lzma_code() might block for - // a long time (from several seconds to even minutes). - // If this is not OK, for example due to progress indicator - // needing updates, specify a timeout in milliseconds here. - // See the documentation of lzma_mt in lzma/container.h for - // information how to choose a reasonable timeout. + // Use no timeout for lzma_code() calls by setting timeout + // to zero. That is, sometimes lzma_code() might block for + // a long time (from several seconds to even minutes). + // If this is not OK, for example due to progress indicator + // needing updates, specify a timeout in milliseconds here. + // See the documentation of lzma_mt in lzma/container.h for + // information how to choose a reasonable timeout. mt .timeout = 110; - // Use the default preset (6) for LZMA2. - // To use a preset, filters must be set to NULL. + // Use the default preset (6) for LZMA2. + // To use a preset, filters must be set to NULL. mt .preset = //LZMA_PRESET_DEFAULT; //UINT32_C(8); - UINT32_C(9);// | LZMA_PRESET_EXTREME; // 9 OR 9e -#ifdef __BIG_DICT_REQUIRES_MUCH_MORE_MEM // > 8 GB @ _WIN64 || __amd64__ - lzma_options_lzma opt_lzma; - if (lzma_lzma_preset(&opt_lzma, mt.preset)) { // LZMA_PRESET_DEFAULT - fprintf(stderr, "Unsupported preset, possibly a bug\n"); - return false; - } - opt_lzma.dict_size *= 2; - lzma_filter filters[] = { - { LZMA_FILTER_LZMA2, &opt_lzma }, - { LZMA_VLI_UNKNOWN, NULL }, - }; - mt.filters = filters; -#else + UINT32_C(6);// | LZMA_PRESET_EXTREME; // 9 OR 9e mt.filters = NULL; -#endif - // Use CRC64 for integrity checking. See also - // 01_compress_easy.c about choosing the integrity check. mt.check = LZMA_CHECK_CRC32; - //}; - // Detect how many threads the CPU supports. - mt.threads = dict;//lzma_cputhreads(); - - // If the number of CPU cores/threads cannot be detected, - // use one thread. Note that this isn't the same as the normal - // single-threaded mode as this will still split the data into - // blocks and use more RAM than the normal single-threaded mode. - // You may want to consider using lzma_easy_encoder() or - // lzma_stream_encoder() instead of lzma_stream_encoder_mt() if - // lzma_cputhreads() returns 0 or 1. - if (mt.threads == 0) - mt.threads = 1; - - // If the number of CPU cores/threads exceeds threads_max, - // limit the number of threads to keep memory usage lower. - // The number 8 is arbitrarily chosen and may be too low or - // high depending on the compression preset and the computer - // being used. - // - // FIXME: A better way could be to check the amount of RAM - // (or available RAM) and use lzma_stream_encoder_mt_memusage() - // to determine if the number of threads should be reduced. - const uint32_t threads_max = 8; - if (mt.threads > threads_max) - mt.threads = threads_max; + mt.threads = threads; // Initialize the threaded encoder. lzma_ret ret = lzma_stream_encoder_mt(strm, &mt); + + memory_usage = lzma_stream_encoder_mt_memusage(&mt); + return check(ret); -} +} \ No newline at end of file diff --git a/contrib/liblzma/precomp_xz.h b/contrib/liblzma/precomp_xz.h index 2ca6441..029e897 100644 --- a/contrib/liblzma/precomp_xz.h +++ b/contrib/liblzma/precomp_xz.h @@ -5,7 +5,7 @@ bool init_lzma1(lzma_stream *strm); bool init_lzma2(lzma_stream *strm); -bool init_encoder_mt(lzma_stream *strm); +bool init_encoder_mt(lzma_stream *strm, int threads, uint64_t &memory_usage); bool init_decoder(lzma_stream *strm); #endif /* ifndef PRECOMP_XZ_H */ diff --git a/make.bat b/make.bat index 4703441..b0ee4fa 100644 --- a/make.bat +++ b/make.bat @@ -114,7 +114,7 @@ move /Y *.o ..\..\ > nul popd :nocontrib echo Building precomp... -%GPP% %DCOMFORT% %MPARAM% -static -static-libgcc -static-libstdc++ -lpthread -Wall precomp.cpp %JPG_O% %MP3_O% %GIF_O% %BZIP_O% %ZLIB_O% %LIBLZMA_CPP% %LIBLZMA_O% -O2 -fomit-frame-pointer -s -o%EXE1%%EXE2%.exe +%GPP% %DCOMFORT% %MPARAM% -static -static-libgcc -static-libstdc++ -DMINGW -lpthread -Wall precomp.cpp %JPG_O% %MP3_O% %GIF_O% %BZIP_O% %ZLIB_O% %LIBLZMA_CPP% %LIBLZMA_O% -O2 -fomit-frame-pointer -s -o%EXE1%%EXE2%.exe if not %ERRORLEVEL% == 0 echo ERROR!!! if %ERRORLEVEL% == 0 echo. if %ERRORLEVEL% == 0 echo Build successful. diff --git a/precomp.cpp b/precomp.cpp index 7ad7852..4aff0f1 100644 --- a/precomp.cpp +++ b/precomp.cpp @@ -49,6 +49,10 @@ #include #include #include +#include +#ifdef MINGW +#include "contrib\mingw_std_threads\mingw.thread.h" +#endif #ifdef _MSC_VER #include #define ftruncate _chsize @@ -9175,10 +9179,16 @@ void init_compress_otf() { break; } case OTF_XZ_MT: { - if (!init_encoder_mt(&otf_xz_stream_c)) { + uint64_t memory_usage = 0; + int threads = std::thread::hardware_concurrency(); + if (threads == 0) threads = 2; + if (!init_encoder_mt(&otf_xz_stream_c, threads, memory_usage)) { printf("ERROR: xz Multi-Threaded init failed\n"); exit(1); } + printf("Using LZMA for compression, %i threads, memory usage: ", threads); + print64(memory_usage / 1000000); + printf(" MB\n"); break; } }