Merge pull request #4721 from facebook/dependabot/github_actions/actions/setup-java-5.6.0

Bump actions/setup-java from 5.2.0 to 5.6.0

.circleci/config.yml

@@ -1,123 +0,0 @@
-version: 2
-
-jobs:
-  # the first half of the jobs are in this test
-  short-tests-0:
-    # TODO: Create a small custom docker image with all the dependencies we need
-    #       preinstalled to reduce installation time.
-    docker:
-      - image: fbopensource/zstd-circleci-primary:0.0.1
-    steps:
-      - checkout
-      - run:
-          name: Test
-          command: |
-            ./tests/test-license.py
-            cc -v
-            CFLAGS="-O0 -Werror -pedantic" make allmost; make clean
-            make c99build; make clean
-            make c11build; make clean
-            make -j regressiontest; make clean
-            make shortest; make clean
-            make cxxtest; make clean
-  # the second half of the jobs are in this test
-  short-tests-1:
-    docker:
-      - image: fbopensource/zstd-circleci-primary:0.0.1
-    steps:
-      - checkout
-      - run:
-          name: Test
-          command: |
-            make gnu90build; make clean
-            make gnu99build; make clean
-            make ppc64build V=1; make clean
-            make ppcbuild   V=1; make clean
-            make armbuild   V=1; make clean
-            make aarch64build V=1; make clean
-            make -C tests test-legacy test-longmatch; make clean
-            make -C lib libzstd-nomt; make clean
-  # This step should only be run in a cron job
-  regression-test:
-    docker:
-      - image: fbopensource/zstd-circleci-primary:0.0.1
-    environment:
-      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
-    steps:
-      - checkout
-      # Restore the cached resources.
-      - restore_cache:
-          # We try our best to bust the cache when the data changes by hashing
-          # data.c. If that doesn't work, simply update the version number here
-          # and below. If we fail to bust the cache, the regression testing will
-          # still work, since it has its own stamp, but will need to redownload
-          # everything.
-          keys:
-            - regression-cache-{{ checksum "tests/regression/data.c" }}-v0
-      - run:
-          name: Regression Test
-          command: |
-            make -C programs zstd
-            make -C tests/regression test
-            mkdir -p $CIRCLE_ARTIFACTS
-            ./tests/regression/test                     \
-                --cache  tests/regression/cache         \
-                --output $CIRCLE_ARTIFACTS/results.csv  \
-                --zstd   programs/zstd
-            echo "NOTE: The new results.csv is uploaded as an artifact to this job"
-            echo "      If this fails, go to the Artifacts pane in CircleCI, "
-            echo "      download /tmp/circleci-artifacts/results.csv, and if they "
-            echo "      are still good, copy it into the repo and commit it."
-            echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
-            diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
-      # Only save the cache on success (default), since if the failure happened
-      # before we stamp the data cache, we will have a bad cache for this key.
-      - save_cache:
-          key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
-          paths:
-            - tests/regression/cache
-      - store_artifacts:
-          path: /tmp/circleci-artifacts
-
-
-workflows:
-  version: 2
-  commit:
-    jobs:
-      # Run the tests in parallel
-      - short-tests-0
-      - short-tests-1
-      - regression-test
-
-  nightly:
-    triggers:
-      - schedule:
-          cron: "0 0 * * *"
-          filters:
-            branches:
-              only:
-                - release
-                - dev
-                - master
-    jobs:
-      # Run daily regression tests
-      - regression-test
-
-
-
-  # Longer tests
-    #- make -C tests test-zstd-nolegacy && make clean
-    #- pyenv global 3.4.4; make -C tests versionsTest && make clean
-    #- make zlibwrapper         && make clean
-    #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
-    #- make uasan               && make clean
-    #- make asan32              && make clean
-    #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
-  # Valgrind tests
-    #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
-    #- make -C tests valgrindTest && make clean
-  # ARM, AArch64, PowerPC, PowerPC64 tests
-    #- make ppctest             && make clean
-    #- make ppc64test           && make clean
-    #- make armtest             && make clean
-    #- make aarch64test         && make clean

.circleci/images/primary/Dockerfile

@@ -1,9 +0,0 @@
-FROM circleci/buildpack-deps@sha256:f6f10c11b7b8ccfd4f4a5b830c3256803604ce61292b60cb22e26b12f62b0e8c
-
-RUN sudo dpkg --add-architecture i386
-RUN sudo apt-get -y -qq update
-RUN sudo apt-get -y install \
-    gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
-    libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
-    libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
-    libcurl4-openssl-dev

.cirrus.yml

@@ -1,10 +1,9 @@
 task:
-  name: FreeBSD (shortest)
+  name: FreeBSD (make check)
   freebsd_instance:
     matrix:
-      image_family: freebsd-13-0
-      image_family: freebsd-12-2
+      image_family: freebsd-15-0-amd64-zfs
   install_script: pkg install -y gmake coreutils
   script: |
     MOREFLAGS="-Werror" gmake -j all
-    gmake shortest
+    gmake check

.github/dependabot.yml

@@ -3,4 +3,4 @@ updates:
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"

.github/workflows/android-ndk-build.yml

@@ -0,0 +1,50 @@
+name: Android NDK Build
+
+on:
+  pull_request:
+    branches: [ dev, release, actionsTest ]
+  push:
+    branches: [ actionsTest, '*ndk*' ]
+
+permissions: read-all
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+
+    - name: Set up JDK 17
+      uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
+      with:
+        java-version: '17'
+        distribution: 'temurin'
+
+    - name: Setup Android SDK
+      uses: android-actions/setup-android@651bceb6f9ca583f16b8d75b62c36ded2ae6fc9c # v4.0.0
+
+    - name: Install Android NDK
+      run: |
+        sdkmanager --install "ndk;27.0.12077973"
+        echo "ANDROID_NDK_HOME=$ANDROID_SDK_ROOT/ndk/27.0.12077973" >> $GITHUB_ENV
+
+    - name: Build with NDK
+      run: |
+        export PATH=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH
+        make CC=aarch64-linux-android21-clang \
+             AR=llvm-ar \
+             RANLIB=llvm-ranlib \
+             STRIP=llvm-strip
+
+    - name: Build with CMake and NDK
+      run: |
+        mkdir -p build-android
+        cd build-android
+        cmake --version
+        cmake ../build/cmake \
+          -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake \
+          -DANDROID_ABI=arm64-v8a \
+          -DANDROID_PLATFORM=android-21 \
+          -DCMAKE_BUILD_TYPE=Release
+        cmake --build . --parallel

.github/workflows/cmake-tests.yml

@@ -0,0 +1,168 @@
+name: cmake-tests
+# CMake-specific build and test workflows
+# This workflow validates zstd builds across different CMake configurations,
+# platforms, and edge cases to ensure broad compatibility.
+
+concurrency:
+  group: cmake-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    branches: [ dev, release, actionsTest ]
+
+permissions: read-all
+
+env:
+  # Centralized test timeouts for consistency
+  QUICK_TEST_TIME: "30s"
+  STANDARD_TEST_TIME: "1mn"
+  # Common CMake flags
+  COMMON_CMAKE_FLAGS: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
+
+jobs:
+  # Basic cmake build using the root CMakeLists.txt
+  # Provides a lightweight sanity check that the top-level project config builds
+  # with the default Unix Makefiles generator driven purely through cmake commands
+  cmake-root-basic:
+    name: "CMake Root Build"
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: Configure (Root)
+      run: |
+        cmake -S . -B cmake-build -DCMAKE_BUILD_TYPE=Release ${{ env.COMMON_CMAKE_FLAGS }}
+    - name: Build (Root)
+      run: |
+        cmake --build cmake-build --config Release
+
+  # Ubuntu-based cmake build using make wrapper
+  # This test uses the make-driven cmake build to ensure compatibility
+  # with the existing build system integration
+  cmake-ubuntu-basic:
+    name: "CMake build using make wrapper"
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: Install dependencies
+      run: |
+        sudo apt install liblzma-dev  # Required for compression algorithms
+    - name: CMake build and test via make
+      run: |
+        # Use make wrapper for cmake build with quick test timeouts
+        FUZZERTEST=-T${{ env.STANDARD_TEST_TIME }} ZSTREAM_TESTTIME=-T${{ env.STANDARD_TEST_TIME }} make cmakebuild V=1
+
+  # Cross-platform cmake build with edge case: source paths containing spaces
+  # This test ensures cmake handles filesystem paths with spaces correctly
+  # across different operating systems and build generators
+  cmake-cross-platform-spaces:
+    name: "CMake Cross-Platform (Spaces in Path)"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            generator: "Unix Makefiles"
+            name: "Linux"
+          - os: windows-latest
+            generator: "NMake Makefiles"
+            name: "Windows NMake"
+          - os: macos-latest
+            generator: "Unix Makefiles"
+            name: "macOS"
+    env:
+      SRC_DIR: "source directory with spaces"
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+      with:
+        path: "${{ env.SRC_DIR }}"
+    - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
+      if: ${{ matrix.generator == 'NMake Makefiles' }}
+    - name: "CMake build and install (${{ matrix.name }})"
+      run: |
+        # Test Release build with installation to verify packaging
+        cmake -S "${{ env.SRC_DIR }}/build/cmake" -B build -DBUILD_TESTING=ON -G "${{ matrix.generator }}" -DCMAKE_BUILD_TYPE=Release --install-prefix "${{ runner.temp }}/install"
+        cmake --build build --config Release
+        cmake --install build --config Release
+
+  # Windows-specific cmake testing with Visual Studio 2022
+  # Tests multiple generators and toolchains to ensure broad Windows compatibility
+  # including MSVC (x64, Win32, ARM64), MinGW, and Clang-CL with various architectures and optimizations
+  cmake-windows-comprehensive:
+    name: "CMake Windows VS2022 (${{ matrix.name }})"
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - generator: "Visual Studio 17 2022"
+            flags: "-A x64"
+            name: "MSVC x64"
+            runner: "windows-2022"
+            cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
+          - generator: "Visual Studio 17 2022"
+            flags: "-A Win32"
+            name: "MSVC Win32"
+            runner: "windows-2022"
+            cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
+          - generator: "Visual Studio 17 2022"
+            flags: "-A x64"
+            name: "MSVC x64 (No ZSTD_BUILD_TESTS)"
+            runner: "windows-2022"
+            # Intentionally omit ZSTD_BUILD_TESTS to reproduce the CXX language configuration bug
+            cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON"
+          - generator: "Visual Studio 17 2022"
+            flags: "-A ARM64"
+            name: "MSVC ARM64"
+            runner: "windows-11-arm"  # githuh runner for WOA instance
+          - generator: "MinGW Makefiles"
+            flags: ""
+            name: "MinGW"
+            runner: "windows-2022"
+            cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
+          - generator: "Visual Studio 17 2022"
+            flags: "-T ClangCL"
+            name: "Clang-CL"
+            runner: "windows-2022"
+            cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
+          - generator: "Visual Studio 17 2022"
+            flags: "-T ClangCL -A x64 -DCMAKE_C_FLAGS=/arch:AVX2"
+            name: "Clang-CL AVX2"
+            runner: "windows-2022"
+            cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: Add MSBuild to PATH
+      uses: microsoft/setup-msbuild@30375c66a4eea26614e0d39710365f22f8b0af57 # tag=v3.0.0
+    - name: "Configure CMake (${{ matrix.name }})"
+      run: |
+        cd build\cmake
+        mkdir build
+        cd build
+        cmake.exe -G "${{matrix.generator}}" ${{matrix.flags}} -DCMAKE_BUILD_TYPE=Debug ${{ matrix.cmake_extra_flags }} -DZSTD_ZSTREAM_FLAGS=-T${{ env.QUICK_TEST_TIME }} -DZSTD_FUZZER_FLAGS=-T${{ env.QUICK_TEST_TIME }} -DZSTD_FULLBENCH_FLAGS=-i0 ..
+    - name: "Build (${{ matrix.name }})"
+      run: |
+        cd build\cmake\build
+        cmake.exe --build .
+    - name: "Test (${{ matrix.name }})"
+      run: |
+        cd build\cmake\build
+        ctest.exe -V -C Debug
+
+  # macOS ARM64 (Apple Silicon) specific cmake testing
+  # Validates zstd builds and runs correctly on Apple Silicon architecture
+  # Uses native ARM64 hardware for optimal performance and compatibility testing
+  cmake-macos-arm64:
+    name: "CMake macOS ARM64 (Apple Silicon)"
+    runs-on: macos-14  # ARM64 runner
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: "CMake build and test (ARM64)"
+      run: |
+        # Configure and build with ARM64-specific optimizations
+        cd build/cmake
+        mkdir build
+        cd build
+        cmake -DCMAKE_BUILD_TYPE=Release ${{ env.COMMON_CMAKE_FLAGS }} -DZSTD_ZSTREAM_FLAGS=-T${{ env.QUICK_TEST_TIME }} -DZSTD_FUZZER_FLAGS=-T${{ env.QUICK_TEST_TIME }} -DZSTD_FULLBENCH_FLAGS=-i1 ..
+        make -j$(sysctl -n hw.ncpu)
+        ctest -V

.github/workflows/commit.yml

@@ -0,0 +1,104 @@
+name: facebook/zstd/commit
+on:
+  push:
+    branches:
+    - dev
+  pull_request:
+    branches:
+    - dev
+
+permissions: read-all
+
+jobs:
+  short-tests-0:
+    runs-on: ubuntu-latest
+    services:
+      docker:
+        image: fbopensource/zstd-circleci-primary:0.0.1
+        options: --entrypoint /bin/bash
+    steps:
+    - uses: actions/checkout@v6.0.2
+    - name: Install Dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install libcurl4-gnutls-dev
+    - name: Test
+      run: |
+        ./tests/test-license.py
+        cc -v
+        CFLAGS="-O0 -Werror -pedantic" make allmost; make clean
+        make c99build; make clean
+        make c11build; make clean
+        make -j regressiontest; make clean
+        make check; make clean
+        make cxxtest; make clean
+
+  short-tests-1:
+    runs-on: ubuntu-latest
+    services:
+      docker:
+        image: fbopensource/zstd-circleci-primary:0.0.1
+        options: --entrypoint /bin/bash
+    steps:
+    - uses: actions/checkout@v6.0.2
+    - name: Install Dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu libc6-dev-ppc64-powerpc-cross libcurl4-gnutls-dev lib64gcc-13-dev-powerpc-cross
+    - name: gnu90 build
+      run: make gnu90build && make clean
+    - name: gnu99 build
+      run: make gnu99build && make clean
+    - name: ppc64 build
+      run: make ppc64build V=1 && make clean
+    - name: ppc build
+      run: make ppcbuild V=1 && make clean
+    - name: arm build
+      run: make armbuild V=1 && make clean
+    - name: aarch64 build
+      run: make aarch64build V=1 && make clean
+    - name: test-legacy
+      run: make -C tests test-legacy V=1 && make clean
+    - name: test-longmatch
+      run: make -C tests test-longmatch V=1 && make clean
+    - name: libzstd-nomt build
+      run: make -C lib libzstd-nomt V=1 && make clean
+
+  regression-test:
+    runs-on: ubuntu-latest
+    services:
+      docker:
+        image: fbopensource/zstd-circleci-primary:0.0.1
+        options: --entrypoint /bin/bash
+    env:
+      CIRCLE_ARTIFACTS: "/tmp/circleci-artifacts"
+    steps:
+    - uses: actions/checkout@v6.0.2
+    - name: restore_cache
+      uses: actions/cache@v6
+      with:
+        key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
+        path: tests/regression/cache
+        restore-keys: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
+    - name: Install Dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install libcurl4-gnutls-dev
+    - name: Regression Test
+      run: |
+        make -C programs zstd
+        make -C tests/regression test
+        mkdir -p $CIRCLE_ARTIFACTS
+        ./tests/regression/test                     \
+            --cache  tests/regression/cache         \
+            --output $CIRCLE_ARTIFACTS/results.csv  \
+            --zstd   programs/zstd
+        echo "NOTE: The new results.csv is uploaded as an artifact to this job"
+        echo "      If this fails, go to the Artifacts pane in CircleCI, "
+        echo "      download /tmp/circleci-artifacts/results.csv, and if they "
+        echo "      are still good, copy it into the repo and commit it."
+        echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
+        diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
+    - uses: actions/upload-artifact@v7.0.1
+      with:
+        path: "/tmp/circleci-artifacts"

.github/workflows/dev-long-tests.yml

@@ -1,5 +1,5 @@
 name: dev-long-tests
-# Tests longer than 10mn
+# Tests generally longer than 10mn
 
 concurrency:
   group: long-${{ github.ref }}
@@ -12,91 +12,106 @@ on:
 permissions: read-all
 
 jobs:
+  # lasts ~7mn
   make-all:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: make all
       run: make all
 
-  # lasts ~24mn
+  # lasts ~19mn
   make-test:
     runs-on: ubuntu-latest
     env:
       DEVNULLRIGHTS: 1
       READFROMBLOCKDEVICE: 1
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: make test
-      run: make test
+      run: |
+        make test
+        make -j zstd
+        ./tests/test_process_substitution.bash ./zstd
 
-  # lasts ~26mn
-  make-test-osx:
+  # lasts ~16mn
+  make-test-macos:
     runs-on: macos-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - name: OS-X test
-      run: make test # make -c lib all doesn't work because of the fact that it's not a tty
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: make test on macos
+      run: make test
 
-  # lasts ~24mn
+  # lasts ~10mn
   make-test-32bit:
     runs-on: ubuntu-latest
     env:
       DEVNULLRIGHTS: 1
       READFROMBLOCKDEVICE: 1
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - name: make test
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: make test # note: `make -j test success` seems to require a clean state
       run: |
         sudo apt-get -qqq update
         make libc6install
-        CFLAGS="-m32" make test
+        make clean
+        CFLAGS="-m32 -O2" make -j test V=1
 
+  # lasts ~7mn
+  test-largeDictionary:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: largeDictionary
+      run: |
+        CFLAGS="-Werror -O3" make -j -C tests test-largeDictionary
+
+  # lasts ~9mn
   no-intrinsics-fuzztest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: no intrinsics fuzztest
       run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest
 
+  # lasts ~8mn
   tsan-zstreamtest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: thread sanitizer zstreamtest
       run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
 
-  ubsan-zstreamtest:
+  uasan-zstreamtest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - name: undefined behavior sanitizer zstreamtest
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: ub + address sanitizer on zstreamtest
       run: CC=clang make uasan-test-zstream
 
-  # lasts ~15mn
+  # lasts ~11mn
   tsan-fuzztest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: thread sanitizer fuzztest
       run: CC=clang make tsan-fuzztest
 
-
   big-tests-zstreamtest32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: zstream tests in 32bit mode, with big tests
       run: |
         sudo apt-get -qqq update
         make libc6install
         CC=clang make -C tests test-zstream32 FUZZER_FLAGS="--big-tests"
 
-  # lasts ~23mn
+  # lasts ~13mn
   gcc-8-asan-ubsan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: gcc-8 + ASan + UBSan + Test Zstd
       # See https://askubuntu.com/a/1428822
       run: |
@@ -108,14 +123,14 @@ jobs:
   clang-asan-ubsan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: clang + ASan + UBSan + Test Zstd
       run: CC=clang make -j uasan-test-zstd </dev/null V=1
 
   gcc-asan-ubsan-testzstd-32bit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: ASan + UBSan + Test Zstd, 32bit mode
       run: |
         sudo apt-get -qqq update
@@ -129,7 +144,7 @@ jobs:
   gcc-8-asan-ubsan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: gcc-8 + ASan + UBSan + Fuzz Test
       # See https://askubuntu.com/a/1428822
       run: |
@@ -141,55 +156,70 @@ jobs:
   clang-asan-ubsan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: clang + ASan + UBSan + Fuzz Test
       run: CC=clang FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
 
   gcc-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: ASan + UBSan + Fuzz Test 32bit
       run: |
         sudo apt-get -qqq update
         make libc6install
         CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
 
-  clang-asan-ubsan-fuzz32:
+  clang-asan-fuzz32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - name: clang + ASan + UBSan + Fuzz Test 32bit
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: clang + ASan + Fuzz Test 32bit
       run: |
         sudo apt-get -qqq update
         make libc6install
-        CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
+        CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make asan-fuzztest
+
+# The following test seems to have issues on github CI specifically,
+# it does not provide the `__mulodi4` instruction emulation
+# required for signed 64-bit multiplication.
+# Replaced by asan-only test (above)
+#
+#  clang-asan-ubsan-fuzz32:
+#    runs-on: ubuntu-20.04
+#    steps:
+#    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+#    - name: clang + ASan + UBSan + Fuzz Test 32bit
+#      run: |
+#        sudo apt-get -qqq update
+#        make libc6install
+#        CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
 
   asan-ubsan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: ASan + UBSan + Regression Test
       run: make -j uasanregressiontest
 
-  clang-ubsan-regression:
+  clang-asan-ubsan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: clang + ASan + UBSan + Regression Test
       run: CC=clang make -j uasanregressiontest
 
   msan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: MSan + Regression Test
       run: make -j msanregressiontest
 
   clang-msan-fuzz-unoptimized:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
       - name: clang + MSan + Fuzz Test
         run: |
           sudo apt-get -qqq update
@@ -199,28 +229,25 @@ jobs:
   clang-msan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: clang + MSan + Fuzz Test
       run: |
         sudo apt-get -qqq update
         sudo apt-get install clang
         CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest
 
-  # lasts ~24mn
   clang-msan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: clang + MSan + Test Zstd
       run: |
-        sudo apt-get update
-        sudo apt-get install clang
-        CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
+        CC=clang make -j msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
 
   armfuzz:
       runs-on: ubuntu-latest
       steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
       - name: Qemu ARM emulation + Fuzz Test
         run: |
           sudo apt-get -qqq update
@@ -230,7 +257,7 @@ jobs:
   valgrind-fuzz-test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: valgrind + fuzz test stack mode    # ~ 7mn
       shell: 'script -q -e -c "bash {0}"'
       run: |
@@ -246,8 +273,8 @@ jobs:
       run:
         shell: msys2 {0}
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - uses: msys2/setup-msys2@5beef6d11f48bba68b9eb503e3adc60b23c0cc36 # tag=v2
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # tag=v2.32.0
       with:
         msystem: MINGW64
         install: make
@@ -290,7 +317,7 @@ jobs:
         dry-run: false
         sanitizer: ${{ matrix.sanitizer }}
     - name: Upload Crash
-      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # tag=v3.1.2
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # tag=v7.0.1
       if: failure() && steps.build.outcome == 'success'
       with:
         name: ${{ matrix.sanitizer }}-artifacts

.github/workflows/dev-short-tests.yml

@@ -16,60 +16,49 @@ jobs:
   linux-kernel:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: linux kernel, library + build + test
       run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"
 
   benchmarking:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: make benchmarking
       run: make benchmarking
 
   check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: make check on 32-bit
       run: |
         sudo apt update
         APT_PACKAGES="gcc-multilib" make apt-install
         CFLAGS="-m32 -O1 -fstack-protector" make check V=1
-
-  check-x32:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - name: make check on x32 ABI # https://en.wikipedia.org/wiki/X32_ABI
-      env:
-        CHECK_CONSTRAINED_MEM: true
-      run: |
-        sudo apt update
-        APT_PACKAGES="gcc-multilib" make apt-install
-        CFLAGS="-mx32 -O1 -fstack-protector" make check V=1
+        CFLAGS="-m32 -O1 -fstack-protector" make V=1 -C tests test-cli-tests
 
   build-c89:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - name: ensure zstd can be build with c89/c90 compilers (+ long long support + variadic macros)
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: ensure zstd can be built with c89/c90 compilers (+ long long support + variadic macros)
       run: |
         make c89build V=1
 
   build-zstd-dll:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: build zstd bin against a dynamic lib (debuglevel for more dependencies)
       run: |
-        make -C lib lib-mt-release 
+        make -C lib lib-mt-release
         DEBUGLEVEL=2 make -C programs zstd-dll
 
   gcc-7-libzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: gcc-7 + libzstdmt compilation
       # See https://askubuntu.com/a/1428822
       run: |
@@ -80,24 +69,13 @@ jobs:
         make clean
         LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
 
-    # candidate test (to check) : underlink test
+    # candidate test (for discussion) : underlink test
     # LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
 
-  cmake-build-and-test-check:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - name: cmake build and test check
-      run: |
-        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
-        cp -r ./ "../zstd source"
-        cd "../zstd source"
-        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
-
   cpp-gnu90-c99-compatibility:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: C++, gnu90 and c99 compatibility
       run: |
         make cxxtest
@@ -111,7 +89,7 @@ jobs:
   mingw-cross-compilation:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: mingw cross-compilation
       run: |
         # sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
@@ -122,7 +100,7 @@ jobs:
   armbuild:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: ARM Build Test
       run: |
         sudo apt-get -qqq update
@@ -132,7 +110,7 @@ jobs:
   bourne-shell:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: Bourne shell compatibility (shellcheck)
       run: |
         wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
@@ -142,21 +120,22 @@ jobs:
   zlib-wrapper:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - name: zlib wrapper test
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: install valgrind
       run: |
         sudo apt-get -qqq update
-        make valgrindinstall
-        make -C zlibWrapper test
-        make -C zlibWrapper test-valgrind
+        make valgrindinstall V=1
+    - name: zlib wrapper test
+      run: make -C zlibWrapper test V=1
+    - name: zlib wrapper test under valgrind
+      run: make -C zlibWrapper test-valgrind V=1
 
   lz4-threadpool-libs:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: LZ4, thread pool, and libs build testslib wrapper test
       run: |
-        make lz4install
         make -C tests test-lz4
         make check < /dev/null | tee   # mess with lz4 console detection
         make clean
@@ -164,20 +143,41 @@ jobs:
         make clean
         bash tests/libzstd_builds.sh
 
-  gcc-make-tests-32bit:
+  gcc-make-all-avx2:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: Make all, with AVX2
+      run: |
+        sudo apt-get -qqq update
+        make libc6install
+        CFLAGS="-Werror -mavx2" make -j all
+
+  gcc-make-all-32bit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: Make all, 32bit mode
       run: |
         sudo apt-get -qqq update
         make libc6install
         CFLAGS="-Werror -m32" make -j all32
 
+  gcc-make-all-32bit-avx2:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: Make all, 32bit + AVX2 mode
+      run: |
+        sudo apt-get -qqq update
+        make libc6install
+        CPPFLAGS="-DSTATIC_BMI2=1" CFLAGS="-Werror -m32 -mavx2 -mbmi2" make -j all32
+
+
   gcc-8-make:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
       - name: gcc-8 build
         # See https://askubuntu.com/a/1428822
         run: |
@@ -200,15 +200,17 @@ jobs:
             flags: "HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=1"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
       - name: Build with ${{matrix.name}}
-        run: ${{matrix.flags}} make zstd
+        run: |
+          sudo apt install liblzma-dev
+          ${{matrix.flags}} make zstd
 
 
   implicit-fall-through:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
       - name: -Wimplicit-fallthrough build
         run: |
           make clean
@@ -219,11 +221,11 @@ jobs:
   meson-linux:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
       - name: Install packages
         run: |
           sudo apt-get update
-          sudo apt-get -y install build-essential python3-pip ninja-build liblz4-dev
+          sudo apt-get -y install build-essential python3-pip ninja-build liblz4-dev liblzma-dev
           pip install --pre meson
       - name: Build with Meson
         run: |
@@ -235,25 +237,67 @@ jobs:
             -Dbin_tests=true \
             -Dbin_contrib=true \
             -Ddefault_library=both \
+            build/meson mesonBuild
+          ninja -C mesonBuild/
+          meson test -C mesonBuild/ --print-errorlogs
+          meson install -C mesonBuild --destdir staging/
+
+  meson-mingw-cross-compilation:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+      - name: Install packages
+        run: |
+          sudo apt-get -qqq update
+          sudo apt-get -y install build-essential python3-pip ninja-build {gcc,g++}-mingw-w64-x86-64
+          pip install --pre meson
+      - name: Build with Meson
+        run: |
+          cat > cross.ini <<EOF
+          [binaries]
+          ar = 'x86_64-w64-mingw32-ar'
+          c = 'x86_64-w64-mingw32-gcc'
+          cpp = 'x86_64-w64-mingw32-g++'
+          ld = 'x86_64-w64-mingw32-ld'
+          objcopy = 'x86_64-w64-mingw32-objcopy'
+          objdump = 'x86_64-w64-mingw32-objdump'
+          strip = 'x86_64-w64-mingw32-strip'
+          windres = 'x86_64-w64-mingw32-windres'
+
+          [host_machine]
+          system = 'windows'
+          endian = 'little'
+          cpu_family = 'x86_64'
+          cpu = 'x86_64'
+          EOF
+
+          # pzstd doesn't build; skip -Dbin_contrib=true
+          meson setup \
+            --buildtype=debugoptimized \
+            --cross-file=cross.ini \
+            -Db_lundef=false \
+            -Dbin_programs=true \
+            -Dbin_tests=true \
+            -Ddefault_library=both \
             build/meson builddir
           ninja -C builddir/
-          meson test -C builddir/ --print-errorlogs
-          meson install -C builddir --destdir staging/
+          if grep -- -pthread builddir/meson-private/libzstd.pc; then
+            echo "Error: found stray pthread dependency"
+            exit 1
+          fi
 
   meson-windows:
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
       - name: Install packages
         run: pip install --pre meson
-      - name: Initialize the MSVC dev command prompt
-        uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89
       - name: Configure with Meson
         run: |
-          meson setup build/meson/ builddir -Dbin_tests=true -Dbin_programs=true -Dbin_contrib=true
+          meson setup --vsenv build/meson/ builddir -Dbin_tests=true -Dbin_programs=true -Dbin_contrib=true
       - name: Build with Meson
         run: |
-          ninja -C builddir/
+          meson compile -C builddir/
       - name: Test with Meson
         run: |
           meson test -C builddir/ --print-errorlogs
@@ -261,31 +305,6 @@ jobs:
         run: |
           meson install -C builddir --destdir staging/
 
-  cmake-visual-2022:
-    strategy:
-      matrix:
-        include:
-          - generator: "Visual Studio 17 2022"
-            flags: "-A x64"
-          - generator: "Visual Studio 17 2022"
-            flags: "-A Win32"
-          - generator: "MinGW Makefiles"
-          - generator: "Visual Studio 17 2022"
-            flags: "-T ClangCL"
-    runs-on: windows-2022
-    steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@1ff57057b5cfdc39105cd07a01d78e9b0ea0c14c # tag=v1.3
-    - name: Build
-      working-directory: ${{env.GITHUB_WORKSPACE}}
-      run: |
-        cd build\cmake
-        mkdir build
-        cd build
-        cmake.exe -G "${{matrix.generator}}" ${{matrix.flags}} ..
-        cmake.exe --build .
-
   msbuild-visual-studio:
     strategy:
       fail-fast: false  # 'false' means Don't stop matrix workflows even if some matrix failed.
@@ -295,15 +314,14 @@ jobs:
           { name: "VS 2022 Win32 Debug", platform: Win32, configuration: Debug, toolset: v143, runner: "windows-2022", arch: "" },
           { name: "VS 2022 x64 Release", platform: x64, configuration: Release, toolset: v143, runner: "windows-2022", arch: ""},
           { name: "VS 2022 Win32 Release", platform: Win32, configuration: Release, toolset: v143, runner: "windows-2022", arch: ""},
-          { name: "VS 2019 x64 Release", platform: Win32, configuration: Release, toolset: v142, runner: "windows-2019", arch: ""},
-          { name: "VS 2019 Win32 Release", platform: x64, configuration: Release, toolset: v142, runner: "windows-2019", arch: ""},
+          { name: "VS 2025 x64 Debug", platform: x64, configuration: Debug, toolset: v143, runner: "windows-2025", arch: ""},
           { name: "VS 2022 x64 Release AVX2", platform: x64, configuration: Release, toolset: v143, runner: "windows-2022", arch: "AdvancedVectorExtensions2" },
         ]
     runs-on: ${{matrix.runner}}
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@1ff57057b5cfdc39105cd07a01d78e9b0ea0c14c # tag=v1.3
+      uses: microsoft/setup-msbuild@30375c66a4eea26614e0d39710365f22f8b0af57 # tag=v3.0.0
     - name: Build ${{matrix.name}}
       working-directory: ${{env.GITHUB_WORKSPACE}}
       # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
@@ -326,7 +344,7 @@ jobs:
   libzstd-size:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: libzstd size test
       run: |
         make clean && make -j -C lib libzstd && ./tests/check_size.py lib/libzstd.so 1100000
@@ -337,7 +355,7 @@ jobs:
   minimal-decompressor-macros:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: minimal decompressor macros
       run: |
         make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
@@ -348,11 +366,13 @@ jobs:
         make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
         make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
         make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
+        make clean && make check ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP=1 MOREFLAGS="-Werror"
+        make clean && make check ZSTD_LIB_EXCLUDE_COMPRESSORS_GREEDY_AND_UP=1 MOREFLAGS="-Werror"
 
   dynamic-bmi2:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: dynamic bmi2 tests
       run: |
         make clean && make -j check MOREFLAGS="-O0 -Werror -mbmi2"
@@ -364,7 +384,7 @@ jobs:
   test-variants:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: make all variants & validate
       run: |
         make -j -C programs allVariants MOREFLAGS=-O0
@@ -372,24 +392,26 @@ jobs:
 
   qemu-consistency:
     name: QEMU ${{ matrix.name }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false  # 'false' means Don't stop matrix workflows even if some matrix failed.
       matrix:
         include: [
           { name: ARM,      xcc_pkg: gcc-arm-linux-gnueabi,     xcc: arm-linux-gnueabi-gcc,     xemu_pkg: qemu-system-arm,    xemu: qemu-arm-static     },
-          { name: ARM64,    xcc_pkg: gcc-aarch64-linux-gnu,     xcc: aarch64-linux-gnu-gcc,     xemu_pkg: qemu-system-arm,    xemu: qemu-aarch64-static },
+          { name: ARM64,    xcc_pkg: gcc-aarch64-linux-gnu,     xcc: aarch64-linux-gnu-gcc,     xemu_pkg: qemu-system-aarch64,xemu: qemu-aarch64-static },
           { name: PPC,      xcc_pkg: gcc-powerpc-linux-gnu,     xcc: powerpc-linux-gnu-gcc,     xemu_pkg: qemu-system-ppc,    xemu: qemu-ppc-static     },
           { name: PPC64LE,  xcc_pkg: gcc-powerpc64le-linux-gnu, xcc: powerpc64le-linux-gnu-gcc, xemu_pkg: qemu-system-ppc,    xemu: qemu-ppc64le-static },
           { name: S390X,    xcc_pkg: gcc-s390x-linux-gnu,       xcc: s390x-linux-gnu-gcc,       xemu_pkg: qemu-system-s390x,  xemu: qemu-s390x-static   },
           { name: MIPS,     xcc_pkg: gcc-mips-linux-gnu,        xcc: mips-linux-gnu-gcc,        xemu_pkg: qemu-system-mips,   xemu: qemu-mips-static    },
+          { name: RISC-V,   xcc_pkg: gcc-14-riscv64-linux-gnu,  xcc: riscv64-linux-gnu-gcc-14,  xemu_pkg: qemu-system-riscv64,xemu: qemu-riscv64-static },
           { name: M68K,     xcc_pkg: gcc-m68k-linux-gnu,        xcc: m68k-linux-gnu-gcc,        xemu_pkg: qemu-system-m68k,   xemu: qemu-m68k-static    },
+          { name: SPARC,    xcc_pkg: gcc-sparc64-linux-gnu,     xcc: sparc64-linux-gnu-gcc,     xemu_pkg: qemu-system-sparc,  xemu: qemu-sparc64-static },
         ]
     env:                        # Set environment variables
       XCC: ${{ matrix.xcc }}
       XEMU: ${{ matrix.xemu }}
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: apt update & install
       run: |
         sudo apt-get update
@@ -409,7 +431,15 @@ jobs:
     - name: ARM64
       if: ${{ matrix.name == 'ARM64' }}
       run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
+        make clean
+        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j check
+        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j -C tests test-cli-tests
+        CFLAGS="-O3 -march=armv8.2-a+sve2" LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j check
+        CFLAGS="-O3 -march=armv8.2-a+sve2" LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j -C tests test-cli-tests
+# This test is only compatible with standard libraries that support BTI (Branch Target Identification).
+# Unfortunately, the standard library provided on Ubuntu 24.04 does not have this feature enabled.
+#        make clean
+#        LDFLAGS="-static -z force-bti" MOREFLAGS="-mbranch-protection=standard" CC=$XCC QEMU_SYS=$XEMU make check V=1
     - name: PPC
       if: ${{ matrix.name == 'PPC' }}
       run: |
@@ -426,10 +456,21 @@ jobs:
       if: ${{ matrix.name == 'MIPS' }}
       run: |
         LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
+    - name: RISC-V
+      if: ${{ matrix.name == 'RISC-V' }}
+      run: |
+        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
+        CFLAGS="-march=rv64gcv -O3" LDFLAGS="-static -DMEM_FORCE_MEMORY_ACCESS=0" CC=$XCC QEMU_SYS="$XEMU -cpu rv64,v=true,vlen=128" make clean check
+        CFLAGS="-march=rv64gcv -O3" LDFLAGS="-static -DMEM_FORCE_MEMORY_ACCESS=0" CC=$XCC QEMU_SYS="$XEMU -cpu rv64,v=true,vlen=256" make clean check
+        CFLAGS="-march=rv64gcv -O3" LDFLAGS="-static -DMEM_FORCE_MEMORY_ACCESS=0" CC=$XCC QEMU_SYS="$XEMU -cpu rv64,v=true,vlen=512" make clean check
     - name: M68K
       if: ${{ matrix.name == 'M68K' }}
       run: |
         LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
+    - name: SPARC
+      if: ${{ matrix.name == 'SPARC' }}
+      run: |
+        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
 
   mingw-short-test:
     runs-on: windows-latest
@@ -445,8 +486,8 @@ jobs:
       run:
         shell: msys2 {0}
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - uses: msys2/setup-msys2@5beef6d11f48bba68b9eb503e3adc60b23c0cc36 # tag=v2
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # tag=v2.32.0
       with:
         msystem: ${{ matrix.msystem }}
         install: make diffutils
@@ -481,9 +522,9 @@ jobs:
         platform: [x64, Win32]
         configuration: [Release]
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@1ff57057b5cfdc39105cd07a01d78e9b0ea0c14c # tag=v1.3
+      uses: microsoft/setup-msbuild@30375c66a4eea26614e0d39710365f22f8b0af57 # tag=v3.0.0
     - name: Build and run tests
       working-directory: ${{env.GITHUB_WORKSPACE}}
       env:
@@ -502,8 +543,8 @@ jobs:
     runs-on: windows-latest
     steps:
     - run: git config --global core.autocrlf input
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - uses: cygwin/cygwin-install-action@f5e0f048310c425e84bc789f493a828c6dc80a25 # tag=master
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - uses: cygwin/cygwin-install-action@3f0a3f9f988f7e96b8c18098ae05eaec175f5b52 # tag=v6
       with:
         platform: x86_64
         packages: >-
@@ -523,32 +564,18 @@ jobs:
         make -j allzstd &&
         make -C tests fuzzer &&
         ./tests/fuzzer.exe -v -T1m
-
-  intel-cet-compatibility:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - name: Build Zstd
-      run: |
-        make -j zstd V=1
-        readelf -n zstd
-    - name: Get Intel SDE
-      run: |
-        curl -LO https://downloadmirror.intel.com/684899/sde-external-9.0.0-2021-11-07-lin.tar.xz
-        tar xJvf sde-external-9.0.0-2021-11-07-lin.tar.xz
-    - name: Configure Permissions
-      run: |
-        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
-    - name: Run Under SDE
-      run: |
-        sde-external-9.0.0-2021-11-07-lin/sde -cet -cet-raise 0 -cet-endbr-exe -cet-stderr -cet-abort -- ./zstd -b3
+    - name: cygwin install test
+      shell: C:\cygwin\bin\bash.exe --noprofile --norc -eo pipefail '{0}'
+      run: >-
+        make -j &&
+        make install
 
   pkg-config:
     runs-on: ubuntu-latest
     container:
       image: debian:testing
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
       - name: Install dependencies
         run: |
           apt -y update
@@ -563,7 +590,7 @@ jobs:
   versions-compatibility:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: Versions Compatibility Test
       run: |
         make -C tests versionsTest
@@ -571,27 +598,15 @@ jobs:
   clangbuild:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: make clangbuild
       run: |
         make clangbuild
 
-  clang-pgo:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - name: Build PGO Zstd with Clang
-      env:
-        CC: clang-14
-        LLVM_PROFDATA: llvm-profdata-14
-      run: |
-        make -C programs zstd-pgo
-        ./programs/zstd -b
-
   gcc-pgo:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
     - name: Build PGO Zstd with GCC
       env:
         CC: gcc
@@ -599,26 +614,67 @@ jobs:
         make -C programs zstd-pgo
         ./programs/zstd -b
 
+  clang-pgo:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: Build PGO Zstd with Clang
+      env:
+        CC: clang
+      run: |
+        sudo apt install -y llvm
+        llvm-profdata --version
+        make -C programs zstd-pgo
+        ./programs/zstd -b
 
-# For reference : icc tests
-# icc tests are currently failing on Github Actions, likely to issues during installation stage
-# To be fixed later
-#
-#  icc:
-#    name: icc-check
-#    runs-on: ubuntu-latest
-#    steps:
-#    - name: install icc
-#      run: |
-#        export DEBIAN_FRONTEND=noninteractive
-#        sudo apt-get -qqq update
-#        sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
-#        sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-#        sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-#        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
-#        sudo apt-get update
-#        sudo apt-get install -y intel-basekit intel-hpckit
-#    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-#    - name: make check
-#      run: |
-#        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
+  musl-build:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+      - name: Install musl-tools
+        run: |
+          sudo apt install -y musl-tools
+      - name: Compile with musl-gcc and test-zstd
+        run: |
+          CC=musl-gcc CFLAGS="-Werror -O3" CPPFLAGS=-DZDICT_QSORT=ZDICT_QSORT_C90 make -j -C tests test-zstd V=1
+
+  intel-cet-compatibility:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: Build Zstd
+      run: |
+        make -j zstd V=1
+        readelf -n zstd
+    - name: Get Intel SDE
+      run: |
+        curl -LO https://downloadmirror.intel.com/813591/sde-external-9.33.0-2024-01-07-lin.tar.xz
+        tar xJvf sde-external-9.33.0-2024-01-07-lin.tar.xz
+    - name: Configure Permissions
+      run: |
+        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+    - name: Run Under SDE
+      run: |
+        sde-external-9.33.0-2024-01-07-lin/sde -cet -cet-raise 0 -cet-endbr-exe -cet-stderr -cet-abort -- ./zstd -b3
+
+  icx:
+    # install instructions: https://www.intel.com/content/www/us/en/docs/oneapi/installation-guide-linux/2025-0/apt-005.html
+    name: icx-check
+    runs-on: ubuntu-latest
+    steps:
+    - name: install icx
+      run: |
+        # download the key to system keyring
+        wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
+        | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
+
+        # add signed entry to apt sources and configure the APT client to use Intel repository:
+        echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
+        sudo apt-get update
+        sudo apt-get install -y intel-basekit intel-hpckit
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: make check
+      run: |
+        source /opt/intel/oneapi/setvars.sh
+        make CC=icx check
+        make CC=icx -C tests test-cli-tests

.github/workflows/nightly.yml

@@ -0,0 +1,38 @@
+name: facebook/zstd/nightly
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  push:
+    branches:
+      - release
+      - dev
+      - '*nightly*'
+permissions: read-all
+jobs:
+  regression-test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6.0.2
+    - name: Install dependencies
+      run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+    - name: Regression Test
+      run: |
+        make -C programs zstd
+        make -C tests/regression test
+
+# Longer tests
+  #- make -C tests test-zstd-nolegacy && make clean
+  #- pyenv global 3.4.4; make -C tests versionsTest && make clean
+  #- make zlibwrapper         && make clean
+  #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
+  #- make uasan               && make clean
+  #- make asan32              && make clean
+  #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
+# Valgrind tests
+  #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
+  #- make -C tests valgrindTest && make clean
+# ARM, AArch64, PowerPC, PowerPC64 tests
+  #- make ppctest             && make clean
+  #- make ppc64test           && make clean
+  #- make armtest             && make clean
+  #- make aarch64test         && make clean

.github/workflows/publish-release-artifacts.yml

@@ -17,7 +17,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
 
       - name: Archive
         env:

.github/workflows/release_check.yml

@@ -0,0 +1,64 @@
+name: release_checks
+
+on:
+  push:
+    branches:
+      - release
+  pull_request:
+    branches:
+      - release
+
+permissions: read-all
+
+jobs:
+  verify-manual:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6.0.2
+
+      - name: Save current manual
+        run: mv doc/zstd_manual.html doc/zstd_manual_saved.html
+
+      - name: Generate new manual
+        run: make manual
+
+      - name: Compare manuals
+        run: |
+          if ! cmp -s doc/zstd_manual.html doc/zstd_manual_saved.html; then
+            echo "The API manual was not updated before release !"
+            exit 1
+          fi
+
+  verify-man-pages:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6.0.2
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ruby ruby-dev
+          sudo gem install ronn
+
+      - name: Display ronn version
+        run: ronn --version
+
+      - name: Save current man pages
+        run: |
+          mv programs/zstd.1 programs/zstd.1.saved
+          mv programs/zstdgrep.1 programs/zstdgrep.1.saved
+          mv programs/zstdless.1 programs/zstdless.1.saved
+
+      - name: Generate new manual pages
+        run: make -C programs man
+
+      - name: Compare man pages
+        run: |
+          for file in zstd.1 zstdgrep.1 zstdless.1; do
+            if ! cmp -s programs/$file programs/$file.saved; then
+              echo "The man page $file should have been updated."
+              exit 1
+            fi
+          done

.github/workflows/scorecards.yml

@@ -27,12 +27,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86 # tag=v2.1.2
+        uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # tag=v2.4.1
         with:
           results_file: results.sarif
           results_format: sarif
@@ -51,7 +51,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # tag=v3.1.2
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # tag=v7.0.1
         with:
           name: SARIF file
           path: results.sarif
@@ -59,6 +59,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@67a35a08586135a9573f4327e904ecbf517a882d # tag=v2.2.8
+        uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # tag=v4.36.2
         with:
           sarif_file: results.sarif

.github/workflows/windows-artifacts.yml

@@ -2,7 +2,7 @@ name: windows-artifacts
 
 on:
   push:
-    branches: [ test_artifacts, win_artifacts ]
+    branches: [ test_artifacts, win_artifacts, release ]
   release:
     types:
       - published
@@ -10,42 +10,110 @@ on:
 permissions: read-all
 
 jobs:
-  windows-64-artifacts:
-    # see https://ariya.io/2020/07/on-github-actions-with-msys2
-    runs-on: windows-latest
+  windows-artifacts:
+    permissions:
+      contents: write # to fetch code and upload artifacts
+    # For msys2, see https://ariya.io/2020/07/on-github-actions-with-msys2
+    runs-on: ${{ matrix.shell == 'cmake' && 'windows-11-arm' || 'windows-latest' }}
+    strategy:
+      # For msys2, see https://github.com/msys2/setup-msys2
+      matrix:
+        include:
+          - { msystem: mingw64, env: x86_64, ziparch: win64, shell: msys2 }
+          - { msystem: mingw32, env: i686, ziparch: win32, shell: msys2 }
+          - { msystem: null, env: arm64, ziparch: win-arm64, shell: cmake }
+
     defaults:
       run:
-        shell: msys2 {0}
+        shell: ${{ matrix.shell == 'cmake' && 'pwsh' || 'msys2 {0}' }}
+
     steps:
-    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
-    - uses: msys2/setup-msys2@5beef6d11f48bba68b9eb503e3adc60b23c0cc36 # tag=v2
-      with:
-        msystem: MINGW64
-        install: make zlib git p7zip mingw-w64-x86_64-gcc
-        update: true
-    - name: display versions
-      run: |
-        make -v
-        cc -v
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
 
-    - name: Building zlib to static link
-      run: |
-        git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib
-        make -C zlib -f win32/Makefile.gcc libz.a
+      # MSYS2 setup
+      - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # tag=v2.32.0
+        if: matrix.shell == 'msys2'
+        with:
+          msystem: ${{ matrix.msystem }}
+          install: make p7zip git mingw-w64-${{matrix.env}}-gcc
+          update: true
 
-    - name: Building zstd programs in 64-bit mode
-      run: |
-        CPPFLAGS=-I../zlib LDFLAGS=../zlib/libz.a make -j allzstd MOREFLAGS=-static V=1
+      - name: display versions (MSYS2)
+        if: matrix.shell == 'msys2'
+        run: |
+          make -v
+          cc -v
 
-    - name: Create artifacts
-      run: |
-        ./lib/dll/example/build_package.bat
-        mv bin/ zstd-${{ github.ref_name }}-win64/
-        7z a -tzip -mx9 zstd-${{ github.ref_name }}-win64.zip zstd-${{ github.ref_name }}-win64/
-        cd ..
+      - name: display versions (CMake)
+        if: matrix.shell == 'cmake'
+        run: |
+          cmake --version
 
-    - name: Publish zstd-$VERSION-win64.zip
-      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # tag=v3
-      with:
-        path: ${{ github.workspace }}/zstd-${{ github.ref_name }}-win64.zip
-        name: zstd-${{ github.ref_name }}-win64.zip
+      # Build dependencies (MSYS2 only)
+      - name: Building zlib to static link
+        if: matrix.shell == 'msys2'
+        run: |
+          git clone --depth 1 --branch v1.3.1 https://github.com/madler/zlib
+          make -C zlib -f win32/Makefile.gcc libz.a
+
+      - name: Building lz4 to static link
+        if: matrix.shell == 'msys2'
+        run: |
+          git clone --depth 1 --branch v1.10.0 https://github.com/lz4/lz4
+          # ensure both libraries use the same version of libxxhash
+          cp lib/common/xxhash.* lz4/lib
+          CPPFLAGS=-DXXH_NAMESPACE=LZ4_ make -C lz4/lib liblz4.a V=1
+
+      # Build zstd
+      - name: Building zstd programs
+        if: matrix.shell == 'msys2'
+        run: |
+          CPPFLAGS="-I../zlib -I../lz4/lib" LDFLAGS=-static make -j allzstd V=1 HAVE_ZLIB=1 HAVE_LZ4=1 HAVE_LZMA=0 LDLIBS="../zlib/libz.a ../lz4/lib/liblz4.a"
+
+      - name: Build zstd (CMake ARM64)
+        if: matrix.shell == 'cmake'
+        run: |
+          cd build\cmake
+          mkdir build
+          cd build
+          cmake.exe -G "Visual Studio 17 2022" -A ARM64 -DCMAKE_BUILD_TYPE=Release -DZSTD_BUILD_PROGRAMS=ON -DZSTD_BUILD_SHARED=ON -DZSTD_BUILD_STATIC=ON ..
+          cmake.exe --build . --config Release --parallel
+
+      - name: Create artifacts (MSYS2)
+        if: matrix.shell == 'msys2'
+        run: |
+          ./lib/dll/example/build_package.bat || exit 1
+          mv bin/ zstd-${{ github.ref_name }}-${{matrix.ziparch}}/
+
+      - name: Create artifacts (CMake)
+        if: matrix.shell == 'cmake'
+        run: |
+          .\lib\dll\example\build_package.bat
+          if ($LASTEXITCODE -ne 0) { exit 1 }
+          mv bin/ zstd-${{ github.ref_name }}-${{matrix.ziparch}}/
+
+      - name: Publish zstd-$VERSION-${{matrix.ziparch}}.zip for manual inspection
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # tag=v7.0.1
+        with:
+          compression-level: 9  # maximum compression
+          if-no-files-found: error # defaults to `warn`
+          path: ${{ github.workspace }}/zstd-${{ github.ref_name }}-${{matrix.ziparch}}/
+          name: zstd-${{ github.ref_name }}-${{matrix.ziparch}}
+
+      - name: Package artifact for upload (MSYS2)
+        if: matrix.shell == 'msys2'
+        run: |
+          7z a -tzip -mx9 "$(cygpath -u '${{ github.workspace }}/zstd-${{ github.ref_name }}-${{ matrix.ziparch }}.zip')" "$(cygpath -u '${{ github.workspace }}/zstd-${{ github.ref_name }}-${{ matrix.ziparch }}')"
+
+      - name: Package artifact for upload (CMake)
+        if: matrix.shell == 'cmake'
+        run: |
+          Compress-Archive -Path "zstd-${{ github.ref_name }}-${{ matrix.ziparch }}" -DestinationPath "zstd-${{ github.ref_name }}-${{ matrix.ziparch }}.zip" -CompressionLevel Optimal
+
+      - name: Upload release asset
+        if: github.event_name == 'release'
+        shell: pwsh
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release upload "${{ github.ref_name }}" "$env:GITHUB_WORKSPACE/zstd-${{ github.ref_name }}-${{ matrix.ziparch }}.zip" --clobber

.gitignore

@@ -12,6 +12,8 @@
 *.so
 *.so.*
 *.dylib
+*.framework
+*.xcframework
 
 # Executables
 /zstd
@@ -20,14 +22,6 @@ zstdmt
 *.out
 *.app
 
-# Test artefacts
-tmp*
-*.zst
-*.zstd
-dictionary.
-dictionary
-NUL
-
 # Build artefacts
 contrib/linux-kernel/linux/
 projects/
@@ -36,12 +30,27 @@ bin/
 buck-out/
 build-*
 *.gcda
+cmakebuild/
+cmake-build/
+
+# Test artefacts
+tmp*
+*.zst
+*.zstd
+dictionary.
+dictionary
+NUL
+install/
+
+# IDE
+.clang_complete
+compile_flags.txt
+.clang-format
 
 # Other files
 .directory
 _codelite/
 _zstdbench/
-.clang_complete
 *.idea
 *.swp
 .DS_Store

.travis.yml

@@ -1,128 +0,0 @@
-# Travis CI is used to test platforms that github-actions currently doesn't support
-# without either self-hosting or some finnicky work-around. Also, some tests
-# are troublesome to migrate since GH Actions runs tests not in a tty.
-language: c
-
-git:
-  depth: 1
-
-branches:
-  only:
-  - dev
-  - release
-  - master
-  - travisTest
-
-addons:
-  apt:
-    update: true
-
-env:
-  global:
-    - FUZZERTEST=-T1mn
-      ZSTREAM_TESTTIME=-T1mn
-      DECODECORPUS_TESTTIME=-T1mn
-
-matrix:
-  fast_finish: true
-  include:
-    - name: S390X (big endian) + Fuzz test
-      dist: trusty
-      arch: s390x
-      script:
-        - FUZZER_FLAGS=--no-big-tests make -C tests fuzztest
-
-    - name: S390X (big endian) + Fuzz test + no intrinsics
-      dist: trusty
-      arch: s390x
-      script:
-        - MOREFLAGS="-DZSTD_NO_INTRINSICS" FUZZER_FLAGS=--no-big-tests make -C tests fuzztest
-
-    - name: arm64    # ~2.5 mn
-      os: linux
-      arch: arm64
-      script:
-        - make check
-
-    - name: arm64fuzz
-      os: linux
-      arch: arm64
-      script:
-        - make -C tests fuzztest
-
-    # TODO: migrate to GH Actions once newest clang staticanalyze warnings are fixed
-    - name: static analyzer scanbuild    # ~8mn
-      dist: trusty  # note : it's important to pin down a version of static analyzer, since different versions report different false positives
-      script:
-        - make staticAnalyze
-
-    # GH actions can't run this command on OS-X, non-tty issues
-    - name: OS-X make all lib
-      os: osx
-      script:
-        - make -C lib all
-
-    # Introduced to check compat with old toolchains, to prevent e.g. #1872
-    - name: ARM Build Test (on Trusty)
-      dist: trusty
-      script:
-        - make arminstall
-        - make armbuild
-
-    # check release number (release/new tag only)
-    - name: Tag-Specific Test
-      if: tag =~ ^v[0-9]\.[0-9]
-      script:
-        - make -C tests checkTag
-        - tests/checkTag "$TRAVIS_BRANCH"
-
-    - name: PPC64LE + Fuzz test  # ~13mn
-      arch: ppc64le
-      env:
-        - FUZZER_FLAGS=--no-big-tests
-        - MOREFLAGS="-static"
-      script:
-        - cat /proc/cpuinfo
-        - make -C tests fuzztest
-
-    # This test currently fails on GA specifically, for no obvious reason
-    # (it works fine on travisCI, and on local test platforms).
-    - name: Versions Compatibility Test   # ~6mn
-      script:
-        - make -C tests versionsTest
-
-    # meson dedicated test
-    - name: Focal (Meson + clang)    # ~15mn
-      dist: focal
-      language: cpp
-      compiler: clang
-      install:
-        - sudo apt-get install -qq liblz4-dev valgrind tree
-        - |
-          travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' &&
-          unzip ~/ninja.zip -d ~/.local/bin
-        - |
-          travis_retry curl -o ~/get-pip.py -L 'https://bootstrap.pypa.io/pip/3.6/get-pip.py' &&
-          python3 ~/get-pip.py --user &&
-          pip3 install --user meson
-      script:
-        - |
-          meson setup \
-            --buildtype=debugoptimized \
-            -Db_lundef=false \
-            -Dauto_features=enabled \
-            -Dbin_programs=true \
-            -Dbin_tests=true \
-            -Dbin_contrib=true \
-            -Ddefault_library=both \
-             build/meson builddir
-        - pushd builddir
-        - ninja
-        - meson test --verbose --no-rebuild
-        - DESTDIR=./staging ninja install
-        - tree ./staging
-      after_failure:
-        - cat "$TRAVIS_BUILD_DIR"/builddir/meson-logs/testlog.txt
-
-  allow_failures:
-    - env: ALLOW_FAILURES=true

CHANGELOG

@@ -1,3 +1,70 @@
+v1.6.0 (Dec 2025)
+api: legacy format support is now disabled by default
+build: `ZSTD_LEGACY_SUPPORT` defaults to `0` in Makefile and CMake
+
+V1.5.7 (Feb 2025)
+fix: compression bug in 32-bit mode associated with long-lasting sessions
+api: new method `ZSTD_compressSequencesAndLiterals()` (#4217, #4232)
+api: `ZSTD_getFrameHeader()` works on skippable frames (#4228)
+perf: substantial compression speed improvements (up to +30%) on small data, by @TocarIP (#4144) and @cyan4973 (#4165)
+perf: improved compression speed (~+5%) for dictionary compression at low levels (#4170)
+perf: much faster speed for `--patch-from` at high compression levels (#4276)
+perf: higher `--patch-from` compression ratios, notably at high levels (#4288)
+perf: better speed for binaries on Windows (@pps83) and when compiled with Visual Studio (@MessyHack)
+perf: slight compression ratio improvement thanks to better block boundaries (#4136, #4176, #4178)
+perf: slight compression ratio improvement for `dfast`, aka levels 3 and 4 (#4171)
+perf: runtime bmi2 detection enabled on x86 32-bit mode (#4251)
+cli: multi-threading as default CLI setting, by @daniellerozenblit
+cli: new `--max` command (#4290)
+build: improve `msbuild` version autodetection, support VS2022, by @ManuelBlanc
+build: fix `meson` build by @artem and @Victor-C-Zhang, and on Windows by @bgilbert
+build: compatibility with Apple Framework, by @Treata11
+build: improve icc/icx compatibility, by @josepho0918 and @luau-project
+build: improve compatibility with Android NDK, by Adenilson Cavalcanti
+portability: linux kernel branch, with improved support for Sequence producers (@embg, @gcabiddu, @cyan4973)
+portability: improved qnx compatibility, suggested by @rainbowball
+portability: improved install script for FreeBSD, by @sunpoet
+portability: fixed test suite compatibility with gnu hurd, by @diegonc
+doc: clarify specification, by @elasota
+misc: improved tests/decodecorpus validation tool (#4102), by antmicro
+
+V1.5.6 (Mar 2024)
+api: Promote `ZSTD_c_targetCBlockSize` to Stable API by @felixhandte
+api: new `ZSTD_d_maxBlockSize` experimental parameter, to reduce streaming decompression memory, by @terrelln
+perf: improve performance of param `ZSTD_c_targetCBlockSize`, by @Cyan4973
+perf: improved compression of arrays of integers at high compression, by @Cyan4973
+lib: reduce binary size with selective build-time exclusion, by @felixhandte
+lib: improved huffman speed on small data and linux kernel, by @terrelln
+lib: accept dictionaries with partial literal tables, by @terrelln
+lib: fix CCtx size estimation with external sequence producer, by @embg
+lib: fix corner case decoder behaviors, by @Cyan4973 and @aimuz
+lib: fix zdict prototype mismatch in static_only mode, by @ldv-alt
+lib: fix several bugs in magicless-format decoding, by @embg
+cli: add common compressed file types to `--exclude-compressed`` by @daniellerozenblit
+cli: fix mixing `-c` and `-o` commands with `--rm`, by @Cyan4973
+cli: fix erroneous exclusion of hidden files with `--output-dir-mirror` by @felixhandte
+cli: improved time accuracy on BSD, by @felixhandte
+cli: better errors on argument parsing, by @KapJI
+tests: better compatibility with older versions of `grep`, by @Cyan4973
+tests: lorem ipsum generator as default backup content, by @Cyan4973
+build: cmake improvements by @terrelln, @sighingnow, @gjasny, @JohanMabille, @Saverio976, @gruenich, @teo-tsirpanis
+build: bazel support, by @jondo2010
+build: fix cross-compiling for AArch64 with lld by @jcelerier
+build: fix Apple platform compatibility, by @nidhijaju
+build: fix Visual 2012 and lower compatibility, by @Cyan4973
+build: improve win32 support, by @DimitriPapadopoulos
+build: better C90 compliance for zlibWrapper, by @emaste
+port: make: fat binaries on macos, by @mredig
+port: ARM64EC compatibility for Windows, by @dunhor
+port: QNX support by @klausholstjacobsen
+port: MSYS2 and Cygwin makefile installation and test support, by @QBos07
+port: risc-v support validation in CI, by @Cyan4973
+port: sparc64 support validation in CI, by @Cyan4973
+port: AIX compatibility, by @likema
+port: HP-UX compatibility, by @likema
+doc: Improved specification accuracy, by @elasota
+bug: Fix and deprecate ZSTD_generateSequences (#3981)
+
 v1.5.5 (Apr 2023)
 fix: fix rare corruption bug affecting the high compression mode, reported by @danlark1 (#3517, @terrelln)
 perf: improve mid-level compression speed (#3529, #3533, #3543, @yoniko and #3552, @terrelln)
@@ -98,7 +165,7 @@ build: support for m68k (Motorola 68000's), by @cyan4973
 build: improved AIX support, by @Helflym
 build: improved meson unofficial build, by @eli-schwartz
 cli : custom memory limit when training dictionary (#2925), by @embg
-cli : report advanced parameters information when compressing in very verbose mode (``-vv`), by @Svetlitski-FB
+cli : report advanced parameters information when compressing in very verbose mode (`-vv`), by @Svetlitski-FB
 
 v1.5.0  (May 11, 2021)
 api: Various functions promoted from experimental to stable API: (#2579-2581, @senhuang42)
@@ -165,7 +232,7 @@ api: Add Function to Generate Skippable Frame (#2439, @senhuang42)
 perf: New Algorithms for the Long Distance Matcher (#2483, @mpu)
 perf: Performance Improvements for Long Distance Matcher (#2464, @mpu)
 perf: Don't Shrink Window Log when Streaming with a Dictionary (#2451, @terrelln)
-cli: Fix `--output-dir-mirror`'s Rejection of `..`-Containing Paths (#2512, @felixhandte)
+cli: Fix `--output-dir-mirror` rejection of `..` -containing paths (#2512, @felixhandte)
 cli: Allow Input From Console When `-f`/`--force` is Passed (#2466, @felixhandte)
 cli: Improve Help Message (#2500, @senhuang42)
 tests: Remove Flaky Tests (#2455, #2486, #2445, @Cyan4973)
@@ -452,7 +519,7 @@ misc: added /contrib/docker script by @gyscos
 
 v1.3.3  (Dec 21, 2017)
 perf: faster zstd_opt strategy (levels 16-19)
-fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
+fix : bug #944 : multithreading with shared dictionary and large data, reported by @gsliepen
 cli : fix : content size written in header by default
 cli : fix : improved LZ4 format support, by @felixhandte
 cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file

CMakeLists.txt

@@ -0,0 +1,11 @@
+cmake_minimum_required(VERSION 3.10)
+
+# Thin wrapper so `cmake -S .` behaves like `cmake -S build/cmake`.
+# Policy lives in build/cmake; keep parent project language-less.
+project(zstd-superbuild LANGUAGES NONE)
+
+if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR)
+    message(FATAL_ERROR "In-source builds are not supported. Specify -B <build-dir>.")
+endif()
+
+add_subdirectory(build/cmake)

CONTRIBUTING.md

@@ -60,7 +60,7 @@ Our contribution process works in three main stages:
         * Note: run local tests to ensure that your changes didn't break existing functionality
             * Quick check
             ```
-            make shortest
+            make check
             ```
             * Longer check
             ```
@@ -171,8 +171,8 @@ who want earlier signal.
 | Cirrus CI | Used for testing on FreeBSD                                                                                | https://github.com/marketplace/cirrus-ci/                                                                                                              | `.cirrus.yml`          |
 | Circle CI | Historically was used to provide faster signal,<br/> but we may be able to migrate these to Github Actions | https://circleci.com/docs/2.0/getting-started/#setting-up-circleci <br> https://youtu.be/Js3hMUsSZ2c <br> https://circleci.com/docs/2.0/enable-checks/ | `.circleci/config.yml` |
 
-Note: the instructions linked above mostly cover how to set up a repository with CI from scratch. 
-The general idea should be the same for setting up CI on your fork of zstd, but you may have to 
+Note: the instructions linked above mostly cover how to set up a repository with CI from scratch.
+The general idea should be the same for setting up CI on your fork of zstd, but you may have to
 follow slightly different steps. In particular, please ignore any instructions related to setting up
 config files (since zstd already has configs for each of these services).
 
@@ -216,7 +216,7 @@ will typically not be stable enough to obtain reliable benchmark results. If you
 hands on a desktop, this is usually a better scenario.
 
 Of course, benchmarking can be done on non-hyper-stable machines as well. You will just have to
-do a little more work to ensure that you are in fact measuring the changes you've made not and
+do a little more work to ensure that you are in fact measuring the changes you've made and not
 noise. Here are some things you can do to make your benchmarks more stable:
 
 1. The most simple thing you can do to drastically improve the stability of your benchmark is
@@ -373,7 +373,12 @@ counter `L1-dcache-load-misses`
 
 #### Visual Studio
 
-TODO
+Build Zstd with symbols first (for example `cmake -B build -S build/cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo && ninja -C build zstd`) so the profiler resolves call stacks.
+
+* Launch Visual Studio’s Performance Profiler (`Alt+F2`), enable CPU Usage (optionally Instrumentation), and point it at the `programs/zstd` benchmark you want to run.
+* If you prefer to start the benchmark from a terminal, use “Attach to running process” to latch onto it mid-run; keep frame pointers (`-fno-omit-frame-pointer`) for clean stacks.
+* When you stop the capture, review the call tree, hot path, and annotated source panes
+* Microsoft’s [Performance Profiling docs](https://learn.microsoft.com/en-us/visualstudio/profiling/?view=vs-2022) cover deeper sampling, ETW, and collection options if required.
 
 ## Issues
 We use GitHub issues to track public bugs. Please ensure your description is

Makefile

@@ -85,14 +85,10 @@ test:
 	$(MAKE) -C $(TESTDIR) $@
 	ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@
 
-## shortest: same as `make check`
-.PHONY: shortest
-shortest:
-	$(Q)$(MAKE) -C $(TESTDIR) $@
-
 ## check: run basic tests for `zstd` cli
 .PHONY: check
-check: shortest
+check:
+	$(Q)$(MAKE) -C $(TESTDIR) $@
 
 .PHONY: automated_benchmarking
 automated_benchmarking:
@@ -145,13 +141,16 @@ clean:
 	$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
 	$(Q)$(MAKE) -C contrib/externalSequenceProducer $@ > $(VOID)
 	$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
-	$(Q)$(RM) -r lz4
+	$(Q)$(RM) -r lz4 cmakebuild mesonbuild install
 	@echo Cleaning completed
 
+LIBZSTD_MK_DIR = $(ZSTDDIR)
+include $(LIBZSTD_MK_DIR)/install_oses.mk # UNAME, INSTALL_OS_LIST
+
 #------------------------------------------------------------------------------
 # make install is validated only for Linux, macOS, Hurd and some BSD targets
 #------------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku AIX))
+ifneq (,$(filter $(INSTALL_OS_LIST),$(UNAME)))
 
 HOST_OS = POSIX
 
@@ -197,6 +196,15 @@ uninstall:
 travis-install:
 	$(MAKE) install PREFIX=~/install_test_dir
 
+.PHONY: clangbuild-darwin-fat
+clangbuild-darwin-fat: clean
+	clang -v
+	CXX=clang++ CC=clang CFLAGS+="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch arm64" $(MAKE) zstd-release
+	mv programs/zstd programs/zstd_arm64
+	CXX=clang++ CC=clang CFLAGS+="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch x86_64" $(MAKE) zstd-release
+	mv programs/zstd programs/zstd_x64
+	lipo -create programs/zstd_x64 programs/zstd_arm64 -output programs/zstd
+
 .PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build
 gcc5build: clean
 	gcc-5 -v
@@ -284,16 +292,16 @@ regressiontest:
 	$(MAKE) -C $(FUZZDIR) regressiontest
 
 uasanregressiontest:
-	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined" CXXFLAGS="-O3 -fsanitize=address,undefined"
+	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined -Werror" CXXFLAGS="-O3 -fsanitize=address,undefined -Werror"
 
 msanregressiontest:
-	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
+	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory -Werror" CXXFLAGS="-O3 -fsanitize=memory -Werror"
 
 update_regressionResults : REGRESS_RESULTS_DIR := /tmp/regress_results_dir/
 update_regressionResults:
-	$(MAKE) -C programs zstd
-	$(MAKE) -C tests/regression test
-	$(RM) -rf $(REGRESS_RESULTS_DIR)
+	$(MAKE) -j -C programs zstd
+	$(MAKE) -j -C tests/regression test
+	$(RM) -r $(REGRESS_RESULTS_DIR)
 	$(MKDIR) $(REGRESS_RESULTS_DIR)
 	./tests/regression/test                         \
         --cache  tests/regression/cache             \
@@ -308,7 +316,7 @@ update_regressionResults:
 # run UBsan with -fsanitize-recover=pointer-overflow
 # this only works with recent compilers such as gcc 8+
 usan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror $(MOREFLAGS)"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=undefined -Werror $(MOREFLAGS)"
 
 asan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror $(MOREFLAGS)"
@@ -319,17 +327,18 @@ asan-%: clean
 msan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" HAVE_LZMA=0   # datagen.c fails this test for no obvious reason
 
-msan-%: clean
-	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $*
+msan-%:
+	$(MAKE) clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -j -C $(TESTDIR) HAVE_LZMA=0 $*
 
 asan32: clean
 	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address $(MOREFLAGS)"
 
 uasan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror $(MOREFLAGS)"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address,undefined -Werror $(MOREFLAGS)"
 
 uasan-%: clean
-	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $*
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address,undefined -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $*
 
 tsan-%: clean
 	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
@@ -345,7 +354,7 @@ apt-add-repo:
 	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
 	sudo apt-get update -y -qq
 
-.PHONY: ppcinstall arminstall valgrindinstall libc6install gcc6install gcc7install gcc8install gpp6install clang38install lz4install
+.PHONY: ppcinstall arminstall valgrindinstall libc6install gcc6install gcc7install gcc8install gpp6install clang38install
 ppcinstall:
 	APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install
 
@@ -373,35 +382,53 @@ gpp6install: apt-add-repo
 clang38install:
 	APT_PACKAGES="clang-3.8" $(MAKE) apt-install
 
-# Ubuntu 14.04 ships a too-old lz4
-lz4install:
-	[ -e lz4 ] || git clone https://github.com/lz4/lz4 && sudo $(MAKE) -C lz4 install
-
 endif
 
 
-CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
-
-ifneq (,$(filter MSYS%,$(shell uname)))
+ifneq (,$(filter MSYS%,$(shell sh -c 'MSYSTEM="MSYS" uname') ))
 HOST_OS = MSYS
-CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
 endif
 
 #------------------------------------------------------------------------
 # target specific tests
 #------------------------------------------------------------------------
-ifneq (,$(filter $(HOST_OS),MSYS POSIX))
-.PHONY: cmakebuild c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze
-cmakebuild:
-	cmake --version
-	$(RM) -r $(BUILDIR)/cmake/build
-	$(MKDIR) $(BUILDIR)/cmake/build
-	cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) ..
-	$(MAKE) -C $(BUILDIR)/cmake/build -j4;
-	$(MAKE) -C $(BUILDIR)/cmake/build install;
-	$(MAKE) -C $(BUILDIR)/cmake/build uninstall;
-	cd $(BUILDIR)/cmake/build; ctest -V -L Medium
+ifneq (,$(filter MSYS POSIX,$(HOST_OS)))
 
+CMAKE ?= cmake
+CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
+
+ifneq (,$(filter MSYS%,$(shell sh -c 'MSYSTEM="MSYS" uname')))
+CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
+endif
+
+.PHONY: cmakebuild
+cmakebuild:
+	$(CMAKE) --version
+	$(RM) -r cmakebuild install
+	$(MKDIR) cmakebuild install
+	cd cmakebuild; $(CMAKE) -Wdev -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS="-Werror -O0" -DCMAKE_INSTALL_PREFIX=install $(CMAKE_PARAMS) ../build/cmake
+	$(CMAKE) --build cmakebuild --target install -- -j V=1
+	cd cmakebuild; ctest -V -L Medium
+
+MESON ?= meson
+NINJA ?= ninja
+
+.PHONY: mesonbuild
+mesonbuild:
+	$(MESON) setup \
+		--buildtype=debugoptimized \
+		-Db_lundef=false \
+		-Dauto_features=enabled \
+		-Dbin_programs=true \
+		-Dbin_tests=true \
+		-Dbin_contrib=true \
+		-Ddefault_library=both \
+		build/meson mesonbuild
+	$(NINJA) -C mesonbuild/
+	$(MESON) test -C mesonbuild/ --print-errorlogs
+	$(MESON) install -C mesonbuild --destdir staging/
+
+.PHONY: c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze
 c89build: clean
 	$(CC) -v
 	CFLAGS="-std=c89 -Werror -Wno-attributes -Wpedantic -Wno-long-long -Wno-variadic-macros -O0" $(MAKE) lib zstd

README.md

@@ -5,7 +5,7 @@ targeting real-time compression scenarios at zlib-level and better compression r
 It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy).
 
 Zstandard's format is stable and documented in [RFC8878](https://datatracker.ietf.org/doc/html/rfc8878). Multiple independent implementations are already available.
-This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
+This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) OR [GPLv2](COPYING) licensed **C** library,
 and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
 Should your project require another programming language,
 a list of known ports and bindings is provided on [Zstandard homepage](https://facebook.github.io/zstd/#other-languages).
@@ -29,10 +29,10 @@ a list of known ports and bindings is provided on [Zstandard homepage](https://f
 ## Benchmarks
 
 For reference, several fast compression algorithms were tested and compared
-on a desktop running Ubuntu 20.04 (`Linux 5.11.0-41-generic`),
-with a Core i7-9700K CPU @ 4.9GHz,
+on a desktop featuring a Core i7-9700K CPU @ 4.9GHz
+and running Ubuntu 24.04 (`Linux 6.8.0-53-generic`),
 using [lzbench], an open-source in-memory benchmark by @inikep
-compiled with [gcc] 9.3.0,
+compiled with [gcc] 14.2.0,
 on the [Silesia compression corpus].
 
 [lzbench]: https://github.com/inikep/lzbench
@@ -41,24 +41,23 @@ on the [Silesia compression corpus].
 
 | Compressor name         | Ratio | Compression| Decompress.|
 | ---------------         | ------| -----------| ---------- |
-| **zstd 1.5.1 -1**       | 2.887 |   530 MB/s |  1700 MB/s |
-| [zlib] 1.2.11 -1        | 2.743 |    95 MB/s |   400 MB/s |
-| brotli 1.0.9 -0         | 2.702 |   395 MB/s |   450 MB/s |
-| **zstd 1.5.1 --fast=1** | 2.437 |   600 MB/s |  2150 MB/s |
-| **zstd 1.5.1 --fast=3** | 2.239 |   670 MB/s |  2250 MB/s |
-| quicklz 1.5.0 -1        | 2.238 |   540 MB/s |   760 MB/s |
-| **zstd 1.5.1 --fast=4** | 2.148 |   710 MB/s |  2300 MB/s |
-| lzo1x 2.10 -1           | 2.106 |   660 MB/s |   845 MB/s |
-| [lz4] 1.9.3             | 2.101 |   740 MB/s |  4500 MB/s |
-| lzf 3.6 -1              | 2.077 |   410 MB/s |   830 MB/s |
-| snappy 1.1.9            | 2.073 |   550 MB/s |  1750 MB/s |
+| **zstd 1.5.7 -1**       | 2.896 |   510 MB/s |  1550 MB/s |
+| brotli 1.1.0 -1         | 2.883 |   290 MB/s |   425 MB/s |
+| [zlib] 1.3.1 -1         | 2.743 |   105 MB/s |   390 MB/s |
+| **zstd 1.5.7 --fast=1** | 2.439 |   545 MB/s |  1850 MB/s |
+| quicklz 1.5.0 -1        | 2.238 |   520 MB/s |   750 MB/s |
+| **zstd 1.5.7 --fast=4** | 2.146 |   665 MB/s |  2050 MB/s |
+| lzo1x 2.10 -1           | 2.106 |   650 MB/s |   780 MB/s |
+| [lz4] 1.10.0            | 2.101 |   675 MB/s |  3850 MB/s |
+| snappy 1.2.1            | 2.089 |   520 MB/s |  1500 MB/s |
+| lzf 3.6 -1              | 2.077 |   410 MB/s |   820 MB/s |
 
 [zlib]: https://www.zlib.net/
 [lz4]: https://lz4.github.io/lz4/
 
 The negative compression levels, specified with `--fast=#`,
 offer faster compression and decompression speed
-at the cost of compression ratio (compared to level 1).
+at the cost of compression ratio.
 
 Zstd can also offer stronger compression ratios at the cost of compression speed.
 Speed vs Compression trade-off is configurable by small increments.
@@ -121,33 +120,40 @@ Dictionary gains are mostly effective in the first few KB. Then, the compression
 
 ## Build instructions
 
-`make` is the officially maintained build system of this project.
-All other build systems are "compatible" and 3rd-party maintained,
-they may feature small differences in advanced options.
-When your system allows it, prefer using `make` to build `zstd` and `libzstd`.
+`make` is the main build system of this project.
+It is the reference, and other build systems are periodically updated to stay compatible.
+However, small drifts and feature differences can be present, since perfect synchronization is difficult.
+For this reason, when your build system allows it, prefer employing `make`.
 
 ### Makefile
 
-If your system is compatible with standard `make` (or `gmake`),
-invoking `make` in root directory will generate `zstd` cli in root directory.
-It will also create `libzstd` into `lib/`.
+Assuming your system supports standard `make` (or `gmake`),
+just invoking `make` in root directory generates `zstd` cli at root,
+and also generates `libzstd` into `lib/`.
 
-Other available options include:
-- `make install` : create and install zstd cli, library and man pages
-- `make check` : create and run `zstd`, test its behavior on local platform
+Other standard targets include:
+- `make install` : install zstd cli, library and man pages
+- `make check` : run `zstd`, test its essential behavior on local platform
 
 The `Makefile` follows the [GNU Standard Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html),
-allowing staged install, standard flags, directory variables and command variables.
+allowing staged install, standard compilation flags, directory variables and command variables.
 
-For advanced use cases, specialized compilation flags which control binary generation
-are documented in [`lib/README.md`](lib/README.md#modular-build) for the `libzstd` library
+For advanced use cases, specialized flags which control binary generation and installation paths are documented
+in [`lib/README.md`](lib/README.md#modular-build) for the `libzstd` library
 and in [`programs/README.md`](programs/README.md#compilation-variables) for the `zstd` CLI.
 
 ### cmake
 
-A `cmake` project generator is provided within `build/cmake`.
-It can generate Makefiles or other build scripts
-to create `zstd` binary, and `libzstd` dynamic and static libraries.
+A `cmake` project generator is available for generating Makefiles or other build scripts
+to create the `zstd` binary as well as `libzstd` dynamic and static libraries.
+The repository root now contains a minimal `CMakeLists.txt` that forwards to `build/cmake`,
+so you can configure the project with a standard `cmake -S .` invocation,
+while the historical `cmake -S build/cmake` entry point remains fully supported.
+
+```bash
+cmake -S . -B build-cmake
+cmake --build build-cmake
+```
 
 By default, `CMAKE_BUILD_TYPE` is set to `Release`.
 
@@ -157,7 +163,7 @@ By default, `CMAKE_BUILD_TYPE` is set to `Release`.
 To perform a Fat/Universal2 build and install use the following commands:
 
 ```bash
-cmake -B build-cmake-debug -S build/cmake -G Ninja -DCMAKE_OSX_ARCHITECTURES="x86_64;x86_64h;arm64"
+cmake -S . -B build-cmake-debug -G Ninja -DCMAKE_OSX_ARCHITECTURES="x86_64;x86_64h;arm64"
 cd build-cmake-debug
 ninja
 sudo ninja install
@@ -185,19 +191,35 @@ You can build and install zstd [vcpkg](https://github.com/Microsoft/vcpkg/) depe
 The zstd port in vcpkg is kept up to date by Microsoft team members and community contributors.
 If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
 
+### Conan
+
+You can install pre-built binaries for zstd or build it from source using [Conan](https://conan.io/). Use the following command:
+
+```bash
+conan install --requires="zstd/[*]" --build=missing
+```
+
+The zstd Conan recipe is kept up to date by Conan maintainers and community contributors.
+If the version is out of date, please [create an issue or pull request](https://github.com/conan-io/conan-center-index) on the ConanCenterIndex repository.
+
 ### Visual Studio (Windows)
 
 Going into `build` directory, you will find additional possibilities:
-- Projects for Visual Studio 2005, 2008 and 2010.
+- Projects for Visual Studio 2008 and 2010.
   + VS2010 project is compatible with VS2012, VS2013, VS2015 and VS2017.
 - Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,
   which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
+- It is now recommended to generate Visual Studio solutions from `cmake`
 
 ### Buck
 
 You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
 The output binary will be in `buck-out/gen/programs/`.
 
+### Bazel
+
+You can integrate zstd into your Bazel project by using the module hosted on the [Bazel Central Repository](https://registry.bazel.build/modules/zstd).
+
 ## Testing
 
 You can run quick local smoke tests by running `make check`.
@@ -207,17 +229,16 @@ For information on CI testing, please refer to `TESTING.md`.
 
 ## Status
 
-Zstandard is currently deployed within Facebook and many other large cloud infrastructures.
-It is run continuously to compress large amounts of data in multiple formats and use cases.
-Zstandard is considered safe for production environments.
+Zstandard is deployed within Meta and many other large cloud infrastructures,
+to compress humongous amounts of data in various formats and use cases.
+It is also continuously fuzzed for security issues by Google's [oss-fuzz](https://github.com/google/oss-fuzz/tree/master/projects/zstd) program.
 
 ## License
 
-Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
+Zstandard is dual-licensed under [BSD](LICENSE) OR [GPLv2](COPYING).
 
 ## Contributing
 
 The `dev` branch is the one where all contributions are merged before reaching `release`.
-If you plan to propose a patch, please commit into the `dev` branch, or its own feature branch.
 Direct commit to `release` are not permitted.
 For more information, please read [CONTRIBUTING](CONTRIBUTING.md).

SECURITY.md

@@ -0,0 +1,15 @@
+# Reporting and Fixing Security Issues
+
+Please do not open GitHub issues or pull requests - this makes the problem immediately visible to everyone, including malicious actors. Security issues in this open source project can be safely reported via the Meta Bug Bounty program:
+
+https://www.facebook.com/whitehat
+
+Meta's security team will triage your report and determine whether or not is it eligible for a bounty under our program.
+
+# Receiving Vulnerability Notifications
+
+In the case that a significant security vulnerability is reported to us or discovered by us---without being publicly known---we will, at our discretion, notify high-profile, high-exposure users of Zstandard ahead of our public disclosure of the issue and associated fix.
+
+If you believe your project would benefit from inclusion in this list, please reach out to one of the maintainers.
+
+<!-- Note to maintainers: this list is kept [here](https://fburl.com/wiki/cgc1l62x). -->

appveyor.yml

@@ -1,205 +0,0 @@
-# Following tests are run _only_ on `release` branch
-# and on selected feature branch named `appveyorTest` or `visual*`
-
--
-  version: 1.0.{build}
-  branches:
-    only:
-    - release
-    - master
-    - /appveyor*/
-    - /visual*/
-  environment:
-    matrix:
-    - COMPILER: "gcc"
-      HOST:     "mingw"
-      PLATFORM: "x64"
-      SCRIPT:   "make allzstd MOREFLAGS=-static"
-      ARTIFACT: "true"
-      BUILD:    "true"
-    - COMPILER: "gcc"
-      HOST:     "mingw"
-      PLATFORM: "x86"
-      SCRIPT:   "make allzstd MOREFLAGS=-static"
-      ARTIFACT: "true"
-      BUILD:    "true"
-
-    - COMPILER: "clang-cl"
-      HOST:     "cmake-visual"
-      PLATFORM: "x64"
-      CONFIGURATION: "Release"
-      CMAKE_GENERATOR: "Visual Studio 15 2017"
-      CMAKE_GENERATOR_PLATFORM: "x64"
-      CMAKE_GENERATOR_TOOLSET: "LLVM"
-      APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017"
-
-  install:
-  - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
-  - SET PATH_ORIGINAL=%PATH%
-  - if [%HOST%]==[mingw] (
-      SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
-      SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
-      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
-      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
-    )
-  - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
-      SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
-    )
-
-  build_script:
-  - if [%HOST%]==[mingw] (
-      ( if [%PLATFORM%]==[x64] (
-        SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
-      ) else if [%PLATFORM%]==[x86] (
-        SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
-      ) )
-    )
-  - if [%HOST%]==[mingw] if [%BUILD%]==[true] (
-      make -v &&
-      sh -c "%COMPILER% -v" &&
-      ECHO Building zlib to static link &&
-      SET "CC=%COMPILER%" &&
-      sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib" &&
-      sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a"
-      ECHO Building zstd &&
-      SET "CPPFLAGS=-I../../zlib" &&
-      SET "LDFLAGS=../../zlib/libz.a" &&
-      sh -c "%SCRIPT%" &&
-      ( if [%COMPILER%]==[gcc] if [%ARTIFACT%]==[true]
-          ECHO Creating artifacts &&
-          ECHO %cd% &&
-          lib\dll\example\build_package.bat &&
-          make -C programs DEBUGFLAGS= clean zstd &&
-          cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe &&
-          appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip &&
-          cp zstd.exe ..\bin\zstd.exe &&
-          git clone --depth 1 --branch release https://github.com/facebook/zstd &&
-          cd zstd &&
-          git archive --format=tar release -o zstd-src.tar &&
-          ..\zstd -19 zstd-src.tar &&
-          appveyor PushArtifact zstd-src.tar.zst &&
-          certUtil -hashfile zstd-src.tar.zst SHA256 > zstd-src.tar.zst.sha256.sig &&
-          appveyor PushArtifact zstd-src.tar.zst.sha256.sig &&
-          cd ..\..\bin\ &&
-          7z a -tzip -mx9 zstd-win-release-%PLATFORM%.zip * &&
-          appveyor PushArtifact zstd-win-release-%PLATFORM%.zip
-      )
-    )
-  - if [%HOST%]==[cmake-visual] (
-      ECHO *** &&
-      ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% &&
-      PUSHD build\cmake &&
-      cmake -DBUILD_TESTING=ON . &&
-      cmake --build . --config %CONFIGURATION% -j4 &&
-      POPD &&
-      ECHO ***
-    )
-
-  test_script:
-  - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
-  - SET "CC=gcc"
-  - SET "CXX=g++"
-  - if [%TEST%]==[cmake] (
-      mkdir build\cmake\build &&
-      cd build\cmake\build &&
-      SET FUZZERTEST=-T2mn &&
-      SET ZSTREAM_TESTTIME=-T2mn &&
-      cmake -G "Visual Studio 14 2015 Win64" .. &&
-      cd ..\..\.. &&
-      make clean
-    )
-
-
-# The following tests are for regular pushes
-# into `dev` or some feature branch
-# There run less tests, for shorter feedback loop
-
--
-  version: 1.0.{build}
-  environment:
-    matrix:
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "x64"
-      CONFIGURATION: "Debug"
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "Win32"
-      CONFIGURATION: "Debug"
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "x64"
-      CONFIGURATION: "Release"
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "Win32"
-      CONFIGURATION: "Release"
-
-    - COMPILER: "gcc"
-      HOST:     "cygwin"
-      PLATFORM: "x64"
-
-    - COMPILER: "clang-cl"
-      HOST:     "cmake-visual"
-      PLATFORM: "x64"
-      CONFIGURATION: "Release"
-      CMAKE_GENERATOR: "Visual Studio 15 2017"
-      CMAKE_GENERATOR_PLATFORM: "x64"
-      CMAKE_GENERATOR_TOOLSET: "LLVM"
-      APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017"
-
-  install:
-  - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
-  - SET PATH_ORIGINAL=%PATH%
-  - if [%HOST%]==[cygwin] (
-      ECHO Installing Cygwin Packages &&
-      C:\cygwin64\setup-x86_64.exe -qnNdO -R "C:\cygwin64" -g -P ^
-        gcc,^
-        cmake,^
-        make
-    )
-  - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
-      SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
-    )
-
-  build_script:
-  - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
-  - if [%HOST%]==[cygwin] (
-      set CHERE_INVOKING=yes &&
-      set CC=%COMPILER% &&
-      C:\cygwin64\bin\bash --login -c "
-        set -e;
-        cd build/cmake;
-        CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T20s -DZSTD_ZSTREAM_FLAGS=-T20s -DZSTD_FULLBENCH_FLAGS=-i0 .;
-        make VERBOSE=1 -j;
-        ctest -V -L Medium;
-      "
-    )
-  - if [%HOST%]==[cmake-visual] (
-      ECHO *** &&
-      ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% &&
-      PUSHD build\cmake &&
-      cmake -DBUILD_TESTING=ON . &&
-      cmake --build . --config %CONFIGURATION% -j4 &&
-      POPD &&
-      ECHO ***
-    )
-  - if [%HOST%]==[visual] (
-      ECHO *** &&
-      ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
-      ECHO *** &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe
-    )
-
-
-  test_script:
-  - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
-  - SET "FUZZERTEST=-T10s"
-  - if [%HOST%]==[mingw] (
-      set "CC=%COMPILER%" &&
-      make clean &&
-      make check
-    )

build/VS2008/zstd/zstd.vcproj

@@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -122,7 +122,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -197,7 +197,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -275,7 +275,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -356,6 +356,10 @@
 				RelativePath="..\..\..\programs\dibio.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\programs\lorem.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\cover.c"
 				>

build/VS2008/zstdlib/zstdlib.vcproj

@@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -121,7 +121,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -195,7 +195,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -272,7 +272,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"

build/VS2010/datagen/datagen.vcxproj

@@ -157,6 +157,8 @@
   <ItemGroup>
     <ClCompile Include="..\..\..\programs\util.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
+    <ClCompile Include="..\..\..\programs\lorem.c" />
+    <ClCompile Include="..\..\..\tests\loremOut.c" />
     <ClCompile Include="..\..\..\tests\datagencli.c" />
   </ItemGroup>
   <ItemGroup>

build/VS2010/fullbench-dll/fullbench-dll.vcxproj

@@ -1,189 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <ItemGroup Label="ProjectConfigurations">
-    <ProjectConfiguration Include="Debug|Win32">
-      <Configuration>Debug</Configuration>
-      <Platform>Win32</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Debug|x64">
-      <Configuration>Debug</Configuration>
-      <Platform>x64</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Release|Win32">
-      <Configuration>Release</Configuration>
-      <Platform>Win32</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Release|x64">
-      <Configuration>Release</Configuration>
-      <Platform>x64</Platform>
-    </ProjectConfiguration>
-  </ItemGroup>
-  <PropertyGroup Label="Globals">
-    <ProjectGuid>{00000000-1CC8-4FD7-9281-6B8DBB9D3DF8}</ProjectGuid>
-    <Keyword>Win32Proj</Keyword>
-    <RootNamespace>fullbench-dll</RootNamespace>
-    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
-    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
-  </PropertyGroup>
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <UseDebugLibraries>true</UseDebugLibraries>
-    <CharacterSet>MultiByte</CharacterSet>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <UseDebugLibraries>true</UseDebugLibraries>
-    <CharacterSet>MultiByte</CharacterSet>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <UseDebugLibraries>false</UseDebugLibraries>
-    <WholeProgramOptimization>true</WholeProgramOptimization>
-    <CharacterSet>MultiByte</CharacterSet>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <UseDebugLibraries>false</UseDebugLibraries>
-    <WholeProgramOptimization>true</WholeProgramOptimization>
-    <CharacterSet>MultiByte</CharacterSet>
-  </PropertyGroup>
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-  <ImportGroup Label="ExtensionSettings">
-  </ImportGroup>
-  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-  </ImportGroup>
-  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-  </ImportGroup>
-  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-  </ImportGroup>
-  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-  </ImportGroup>
-  <PropertyGroup Label="UserMacros" />
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
-    <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
-    <RunCodeAnalysis>false</RunCodeAnalysis>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
-    <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
-    <RunCodeAnalysis>false</RunCodeAnalysis>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
-    <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
-    <RunCodeAnalysis>false</RunCodeAnalysis>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
-    <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
-    <RunCodeAnalysis>false</RunCodeAnalysis>
-  </PropertyGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
-    <ClCompile>
-      <PrecompiledHeader>
-      </PrecompiledHeader>
-      <WarningLevel>Level4</WarningLevel>
-      <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <TreatWarningAsError>true</TreatWarningAsError>
-      <EnablePREfast>false</EnablePREfast>
-    </ClCompile>
-    <Link>
-      <SubSystem>Console</SubSystem>
-      <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
-      <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
-      <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
-    </Link>
-  </ItemDefinitionGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
-    <ClCompile>
-      <PrecompiledHeader>
-      </PrecompiledHeader>
-      <WarningLevel>Level4</WarningLevel>
-      <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <TreatWarningAsError>true</TreatWarningAsError>
-      <EnablePREfast>false</EnablePREfast>
-    </ClCompile>
-    <Link>
-      <SubSystem>Console</SubSystem>
-      <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
-      <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
-    </Link>
-  </ItemDefinitionGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
-    <ClCompile>
-      <WarningLevel>Level4</WarningLevel>
-      <PrecompiledHeader>
-      </PrecompiledHeader>
-      <Optimization>MaxSpeed</Optimization>
-      <FunctionLevelLinking>true</FunctionLevelLinking>
-      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <EnablePREfast>false</EnablePREfast>
-      <TreatWarningAsError>false</TreatWarningAsError>
-      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
-    </ClCompile>
-    <Link>
-      <SubSystem>Console</SubSystem>
-      <GenerateDebugInformation>true</GenerateDebugInformation>
-      <EnableCOMDATFolding>true</EnableCOMDATFolding>
-      <OptimizeReferences>true</OptimizeReferences>
-      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
-      <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
-      <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
-    </Link>
-  </ItemDefinitionGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
-    <ClCompile>
-      <WarningLevel>Level4</WarningLevel>
-      <PrecompiledHeader>
-      </PrecompiledHeader>
-      <Optimization>MaxSpeed</Optimization>
-      <FunctionLevelLinking>true</FunctionLevelLinking>
-      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <TreatWarningAsError>false</TreatWarningAsError>
-      <EnablePREfast>false</EnablePREfast>
-      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
-    </ClCompile>
-    <Link>
-      <SubSystem>Console</SubSystem>
-      <GenerateDebugInformation>true</GenerateDebugInformation>
-      <EnableCOMDATFolding>true</EnableCOMDATFolding>
-      <OptimizeReferences>true</OptimizeReferences>
-      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
-      <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
-    </Link>
-  </ItemDefinitionGroup>
-  <ItemGroup>
-    <ClCompile Include="..\..\..\lib\common\xxhash.c" />
-    <ClCompile Include="..\..\..\programs\util.c" />
-    <ClCompile Include="..\..\..\programs\timefn.c" />
-    <ClCompile Include="..\..\..\programs\datagen.c" />
-    <ClCompile Include="..\..\..\programs\benchfn.c" />
-    <ClCompile Include="..\..\..\tests\fullbench.c" />
-  </ItemGroup>
-  <ItemGroup>
-    <ClInclude Include="..\..\..\lib\zstd.h" />
-    <ClInclude Include="..\..\..\programs\datagen.h" />
-    <ClInclude Include="..\..\..\programs\benchfn.h" />
-    <ClInclude Include="..\..\..\programs\util.h" />
-  </ItemGroup>
-  <ItemGroup>
-    <ProjectReference Include="..\libzstd-dll\libzstd-dll.vcxproj">
-      <Project>{00000000-94d5-4bf9-8a50-7bd9929a0850}</Project>
-    </ProjectReference>
-  </ItemGroup>
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
-  <ImportGroup Label="ExtensionTargets">
-  </ImportGroup>
-</Project>

build/VS2010/fullbench/fullbench.vcxproj

@@ -170,6 +170,7 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
     <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
@@ -183,6 +184,7 @@
     <ClCompile Include="..\..\..\programs\util.c" />
     <ClCompile Include="..\..\..\programs\timefn.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
+    <ClCompile Include="..\..\..\programs\lorem.c" />
     <ClCompile Include="..\..\..\programs\benchfn.c" />
     <ClCompile Include="..\..\..\tests\fullbench.c" />
   </ItemGroup>

build/VS2010/fuzzer/fuzzer.vcxproj

@@ -170,6 +170,7 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />

build/VS2010/libzstd-dll/libzstd-dll.vcxproj

@@ -34,6 +34,7 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -168,7 +169,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
       <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
@@ -187,7 +188,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -207,7 +208,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
@@ -228,7 +229,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>false</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>

build/VS2010/libzstd/libzstd.vcxproj

@@ -34,6 +34,7 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -161,7 +162,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
       <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
@@ -180,7 +181,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -200,7 +201,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
@@ -221,7 +222,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>false</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>

build/VS2010/zstd.sln

@@ -7,11 +7,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcx
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}"
 EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll\fullbench-dll.vcxproj", "{00000000-1CC8-4FD7-9281-6B8DBB9D3DF8}"
-	ProjectSection(ProjectDependencies) = postProject
-		{00000000-94D5-4BF9-8A50-7BD9929A0850} = {00000000-94D5-4BF9-8A50-7BD9929A0850}
-	EndProjectSection
-EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{037E781E-81A6-494B-B1B3-438AB1200523}"
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libzstd", "libzstd\libzstd.vcxproj", "{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}"

build/VS2010/zstd/zstd.vcxproj

@@ -35,6 +35,7 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -63,6 +64,7 @@
     <ClCompile Include="..\..\..\programs\dibio.c" />
     <ClCompile Include="..\..\..\programs\fileio.c" />
     <ClCompile Include="..\..\..\programs\fileio_asyncio.c" />
+    <ClCompile Include="..\..\..\programs\lorem.c" />
     <ClCompile Include="..\..\..\programs\zstdcli.c" />
     <ClCompile Include="..\..\..\programs\zstdcli_trace.c" />
   </ItemGroup>
@@ -185,7 +187,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
       <EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
@@ -202,7 +204,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
       <EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
@@ -221,7 +223,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <EnablePREfast>false</EnablePREfast>
       <TreatWarningAsError>false</TreatWarningAsError>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@@ -243,7 +245,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>false</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>

build/VS_scripts/build.VSPreview.cmd

@@ -0,0 +1,7 @@
+@echo off
+
+rem build 32-bit
+call "%~p0%build.generic.cmd" preview Win32 Release v143
+
+rem build 64-bit
+call "%~p0%build.generic.cmd" preview x64 Release v143

build/VS_scripts/build.generic.cmd

@@ -2,7 +2,7 @@
 
 IF "%1%" == "" GOTO display_help
 
-SETLOCAL
+SETLOCAL ENABLEDELAYEDEXPANSION
 
 SET msbuild_version=%1
 
@@ -19,39 +19,34 @@ GOTO build
 :display_help
 
 echo Syntax: build.generic.cmd msbuild_version msbuild_platform msbuild_configuration msbuild_toolset
-echo   msbuild_version:          VS installed version (VS2012, VS2013, VS2015, VS2017, VS2019, ...)
+echo   msbuild_version:          VS installed version (latest, VS2012, VS2013, VS2015, VS2017, VS2019, VS2022, ...)
 echo   msbuild_platform:         Platform (x64 or Win32)
 echo   msbuild_configuration:    VS configuration (Release or Debug)
-echo   msbuild_toolset:          Platform Toolset (v100, v110, v120, v140, v141, v142, ...)
+echo   msbuild_toolset:          Platform Toolset (v100, v110, v120, v140, v141, v142, v143, ...)
 
 EXIT /B 1
 
 :build
 
 SET msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe"
-SET msbuild_vs2017community="%programfiles(x86)%\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe"
-SET msbuild_vs2017professional="%programfiles(x86)%\Microsoft Visual Studio\2017\Professional\MSBuild\15.0\Bin\MSBuild.exe"
-SET msbuild_vs2017enterprise="%programfiles(x86)%\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\MSBuild.exe"
 IF %msbuild_version% == VS2013 SET msbuild="%programfiles(x86)%\MSBuild\12.0\Bin\MSBuild.exe"
 IF %msbuild_version% == VS2015 SET msbuild="%programfiles(x86)%\MSBuild\14.0\Bin\MSBuild.exe"
-IF %msbuild_version% == VS2017Community SET msbuild=%msbuild_vs2017community%
-IF %msbuild_version% == VS2017Professional SET msbuild=%msbuild_vs2017professional%
-IF %msbuild_version% == VS2017Enterprise SET msbuild=%msbuild_vs2017enterprise%
-IF %msbuild_version% == VS2017 (
-	IF EXIST %msbuild_vs2017community% SET msbuild=%msbuild_vs2017community%
-	IF EXIST %msbuild_vs2017professional% SET msbuild=%msbuild_vs2017professional%
-	IF EXIST %msbuild_vs2017enterprise% SET msbuild=%msbuild_vs2017enterprise%
-)
+IF %msbuild_version% == VS2017 SET vswhere_params=-version [15,16) -products *
+IF %msbuild_version% == VS2017Community SET vswhere_params=-version [15,16) -products Community
+IF %msbuild_version% == VS2017Enterprise SET vswhere_params=-version [15,16) -products Enterprise
+IF %msbuild_version% == VS2017Professional SET vswhere_params=-version [15,16) -products Professional
+IF %msbuild_version% == VS2019 SET vswhere_params=-version [16,17) -products *
+IF %msbuild_version% == VS2022 SET vswhere_params=-version [17,18) -products *
+REM Add the next Visual Studio version here.
+IF %msbuild_version% == latest SET vswhere_params=-latest -products *
+IF %msbuild_version% == preview SET vswhere_params=-prerelease -products *
 
-:: VS2019
-SET msbuild_vs2019community="%programfiles(x86)%\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe"
-SET msbuild_vs2019professional="%programfiles(x86)%\Microsoft Visual Studio\2019\Professional\MSBuild\Current\Bin\MSBuild.exe"
-SET msbuild_vs2019enterprise="%programfiles(x86)%\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\MSBuild.exe"
-IF %msbuild_version% == VS2019 (
-	IF EXIST %msbuild_vs2019community% SET msbuild=%msbuild_vs2019community%
-	IF EXIST %msbuild_vs2019professional% SET msbuild=%msbuild_vs2019professional%
-	IF EXIST %msbuild_vs2019enterprise% SET msbuild=%msbuild_vs2019enterprise%
+IF NOT DEFINED vswhere_params GOTO skip_vswhere
+SET vswhere="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
+FOR /F "USEBACKQ TOKENS=*" %%F IN (`%vswhere% !vswhere_params! -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe`) DO (
+	SET msbuild="%%F"
 )
+:skip_vswhere
 
 SET project="%~p0\..\VS2010\zstd.sln"
 

build/cmake/CMakeLists.txt

@@ -7,206 +7,75 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR)
-
-# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies.
-# Set and use the newest cmake policies that are validated to work
-set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3")
-set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13") #Policies never changed at PATCH level
-if("${CMAKE_MAJOR_VERSION}" LESS 3)
-  set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
-elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND
-        "${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}")
-    set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
-else()
-    set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0")
-endif()
-cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
-
-set(CMAKE_BUILD_WITH_INSTALL_RPATH on)
+cmake_minimum_required(VERSION 3.10 FATAL_ERROR)
 
+#-----------------------------------------------------------------------------
+# Setup CMake environment
+#-----------------------------------------------------------------------------
+set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
+
+# Define project paths
 set(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
 set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
-# Parse version
-include(GetZstdLibraryVersion)
-GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h zstd_VERSION_MAJOR zstd_VERSION_MINOR zstd_VERSION_PATCH)
 
-if( CMAKE_MAJOR_VERSION LESS 3 )
-  ## Provide cmake 3+ behavior for older versions of cmake
-  project(zstd)
-  set(PROJECT_VERSION_MAJOR ${zstd_VERSION_MAJOR})
-  set(PROJECT_VERSION_MINOR ${zstd_VERSION_MINOR})
-  set(PROJECT_VERSION_PATCH ${zstd_VERSION_PATCH})
-  set(PROJECT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
-  enable_language(C)   # Main library is in C
-  enable_language(ASM) # And ASM
-  enable_language(CXX) # Testing contributed code also utilizes CXX
-else()
-  project(zstd
-    VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}"
-    LANGUAGES C   # Main library is in C
-              ASM # And ASM
-              CXX # Testing contributed code also utilizes CXX
-    )
-endif()
-message(STATUS "ZSTD VERSION: ${zstd_VERSION}")
-set(zstd_HOMEPAGE_URL "https://facebook.github.io/zstd")
-set(zstd_DESCRIPTION  "Zstandard is a real-time compression algorithm, providing high compression ratios.")
+#-----------------------------------------------------------------------------
+# Configure CMake policies and version
+#-----------------------------------------------------------------------------
+include(ZstdVersion)
 
-# Set a default build type if none was specified
+#-----------------------------------------------------------------------------
+# Project declaration
+#-----------------------------------------------------------------------------
+project(zstd
+    VERSION "${ZSTD_FULL_VERSION}"
+    LANGUAGES C   # Main library is in C and ASM, ASM is enabled conditionally
+    HOMEPAGE_URL "${zstd_HOMEPAGE_URL}"
+    DESCRIPTION "${zstd_DESCRIPTION}"
+)
+
+#-----------------------------------------------------------------------------
+# Build type configuration
+#-----------------------------------------------------------------------------
 if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
-  message(STATUS "Setting build type to 'Release' as none was specified.")
-  set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
-  # Set the possible values of build type for cmake-gui
-  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
+    message(STATUS "Setting build type to 'Release' as none was specified.")
+    set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
+    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
 endif()
 
+#-----------------------------------------------------------------------------
+# Include standard modules
+#-----------------------------------------------------------------------------
 include(GNUInstallDirs)
 
 #-----------------------------------------------------------------------------
-# Add extra compilation flags
-#-----------------------------------------------------------------------------
-include(AddZstdCompilationFlags)
-ADD_ZSTD_COMPILATION_FLAGS()
-
-# Always hide XXHash symbols
-add_definitions(-DXXH_NAMESPACE=ZSTD_)
-
-#-----------------------------------------------------------------------------
-# Installation variables
+# Display installation information
 #-----------------------------------------------------------------------------
 message(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
 message(STATUS "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}")
 
 #-----------------------------------------------------------------------------
-# Options
+# Configure build options
 #-----------------------------------------------------------------------------
-
-# Legacy support
-option(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" ON)
-
-if (ZSTD_LEGACY_SUPPORT)
-    message(STATUS "ZSTD_LEGACY_SUPPORT defined!")
-    set(ZSTD_LEGACY_LEVEL 5 CACHE STRING "")
-    add_definitions(-DZSTD_LEGACY_SUPPORT=${ZSTD_LEGACY_LEVEL})
-else ()
-    message(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
-    add_definitions(-DZSTD_LEGACY_SUPPORT=0)
-endif ()
-
-if (ANDROID)
-    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT OFF)
-else()
-    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT ON)
-endif()
-
-# Multi-threading support
-option(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ${ZSTD_MULTITHREAD_SUPPORT_DEFAULT})
-
-if (ZSTD_MULTITHREAD_SUPPORT)
-    message(STATUS "ZSTD_MULTITHREAD_SUPPORT is enabled")
-else ()
-    message(STATUS "ZSTD_MULTITHREAD_SUPPORT is disabled")
-endif ()
-
-option(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
-option(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
-
-# Respect the conventional CMake option for enabling tests if it was specified on the first configure
-if (BUILD_TESTING)
-    set(ZSTD_BUILD_TESTS_default ON)
-else()
-    set(ZSTD_BUILD_TESTS_default OFF)
-endif()
-option(ZSTD_BUILD_TESTS "BUILD TESTS" ${ZSTD_BUILD_TESTS_default})
-if (MSVC)
-    option(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF)
-endif ()
+include(ZstdOptions)
 
 #-----------------------------------------------------------------------------
-# External dependencies
+# Configure compilation flags
 #-----------------------------------------------------------------------------
-if (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
-    set(THREADS_PREFER_PTHREAD_FLAG ON)
-    find_package(Threads REQUIRED)
-    if(CMAKE_USE_PTHREADS_INIT)
-        set(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
-    else()
-        message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
-    endif()
-endif ()
+include(AddZstdCompilationFlags)
+ADD_ZSTD_COMPILATION_FLAGS(ON ZSTD_ENABLE_CXX ON)
 
 #-----------------------------------------------------------------------------
-# Add source directories
+# Configure dependencies
 #-----------------------------------------------------------------------------
-add_subdirectory(lib)
-
-option(ZSTD_PROGRAMS_LINK_SHARED "PROGRAMS LINK SHARED" OFF)
-
-if (ZSTD_BUILD_PROGRAMS)
-    if (NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED)
-        message(SEND_ERROR "You need to build static library to build zstd CLI")
-    elseif(NOT ZSTD_BUILD_SHARED AND ZSTD_PROGRAMS_LINK_SHARED)
-        message(SEND_ERROR "You need to build shared library to build zstd CLI")
-    endif ()
-
-    add_subdirectory(programs)
-endif ()
-
-if (ZSTD_BUILD_TESTS)
-    enable_testing()
-    if (NOT ZSTD_BUILD_STATIC)
-        message(SEND_ERROR "You need to build static library to build tests")
-    endif ()
-
-    add_subdirectory(tests)
-endif ()
-
-if (ZSTD_BUILD_CONTRIB)
-    add_subdirectory(contrib)
-endif ()
+include(ZstdDependencies)
 
 #-----------------------------------------------------------------------------
-# Add clean-all target
+# Configure build targets
 #-----------------------------------------------------------------------------
-add_custom_target(clean-all
-   COMMAND ${CMAKE_BUILD_TOOL} clean
-   COMMAND rm -rf ${CMAKE_BINARY_DIR}/
-)
+include(ZstdBuild)
 
 #-----------------------------------------------------------------------------
-# Generate Package Config files
-#
-# This section is based on the boiler plate code from:
-# https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#creating-packages
+# Configure package generation
 #-----------------------------------------------------------------------------
-include(CMakePackageConfigHelpers)
-write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
-    VERSION ${zstd_VERSION}
-    COMPATIBILITY SameMajorVersion
-    )
-
-# A Package Config file that works from the build directory
-export(EXPORT zstdExports
-    FILE "${CMAKE_CURRENT_BINARY_DIR}/zstdTargets.cmake"
-    NAMESPACE zstd::
-    )
-configure_file(zstdConfig.cmake
-    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
-    COPYONLY
-    )
-
-# A Package Config file that works from the installation directory
-set(ConfigPackageLocation ${CMAKE_INSTALL_LIBDIR}/cmake/zstd)
-install(EXPORT zstdExports
-    FILE zstdTargets.cmake
-    NAMESPACE zstd::
-    DESTINATION ${ConfigPackageLocation}
-    )
-install(FILES
-    zstdConfig.cmake
-    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
-    DESTINATION ${ConfigPackageLocation}
-    )
+include(ZstdPackage)

build/cmake/CMakeModules/AddZstdCompilationFlags.cmake

@@ -1,11 +1,12 @@
-include(CheckCXXCompilerFlag)
 include(CheckCCompilerFlag)
-# VERSION_GREATER_EQUAL requires CMake 3.7 or later.
-# https://cmake.org/cmake/help/latest/command/if.html#version-greater-equal
-if (CMAKE_VERSION VERSION_LESS 3.18)
-    set(ZSTD_HAVE_CHECK_LINKER_FLAG false)
-else ()
+if(CMAKE_CXX_COMPILER)
+    include(CheckCXXCompilerFlag)
+endif()
+
+if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
     set(ZSTD_HAVE_CHECK_LINKER_FLAG true)
+else ()
+    set(ZSTD_HAVE_CHECK_LINKER_FLAG false)
 endif ()
 if (ZSTD_HAVE_CHECK_LINKER_FLAG)
     include(CheckLinkerFlag)
@@ -22,7 +23,7 @@ function(EnableCompilerFlag _flag _C _CXX _LD)
             set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_flag}" PARENT_SCOPE)
         endif ()
     endif ()
-    if (_CXX)
+    if (_CXX AND CMAKE_CXX_COMPILER)
         CHECK_CXX_COMPILER_FLAG(${_flag} CXX_FLAG_${varname})
         if (CXX_FLAG_${varname})
             set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_flag}" PARENT_SCOPE)
@@ -50,49 +51,65 @@ function(EnableCompilerFlag _flag _C _CXX _LD)
     endif ()
 endfunction()
 
-macro(ADD_ZSTD_COMPILATION_FLAGS)
+macro(ADD_ZSTD_COMPILATION_FLAGS _C _CXX _LD)
+    # We set ZSTD_HAS_NOEXECSTACK if we are certain we've set all the required
+    # compiler flags to mark the stack as non-executable.
+    set(ZSTD_HAS_NOEXECSTACK false)
+
     if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" OR MINGW) #Not only UNIX but also WIN32 for MinGW
         # It's possible to select the exact standard used for compilation.
         # It's not necessary, but can be employed for specific purposes.
         # Note that zstd source code is compatible with both C++98 and above
         # and C-gnu90 (c90 + long long + variadic macros ) and above
         # EnableCompilerFlag("-std=c++11" false true) # Set C++ compilation to c++11 standard
-        # EnableCompilerFlag("-std=c99" true false)   # Set C compiation to c99 standard
+        # EnableCompilerFlag("-std=c99" true false)   # Set C compilation to c99 standard
         if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC)
             # clang-cl normally maps -Wall to -Weverything.
-            EnableCompilerFlag("/clang:-Wall" true true false)
+            EnableCompilerFlag("/clang:-Wall" _C _CXX false)
         else ()
-            EnableCompilerFlag("-Wall" true true false)
+            EnableCompilerFlag("-Wall" _C _CXX false)
         endif ()
-        EnableCompilerFlag("-Wextra" true true false)
-        EnableCompilerFlag("-Wundef" true true false)
-        EnableCompilerFlag("-Wshadow" true true false)
-        EnableCompilerFlag("-Wcast-align" true true false)
-        EnableCompilerFlag("-Wcast-qual" true true false)
-        EnableCompilerFlag("-Wstrict-prototypes" true false false)
+        EnableCompilerFlag("-Wextra" _C _CXX false)
+        EnableCompilerFlag("-Wundef" _C _CXX false)
+        EnableCompilerFlag("-Wshadow" _C _CXX false)
+        EnableCompilerFlag("-Wcast-align" _C _CXX false)
+        EnableCompilerFlag("-Wcast-qual" _C _CXX false)
+        EnableCompilerFlag("-Wstrict-prototypes" _C false false)
         # Enable asserts in Debug mode
         if (CMAKE_BUILD_TYPE MATCHES "Debug")
-            EnableCompilerFlag("-DDEBUGLEVEL=1" true true false)
+            EnableCompilerFlag("-DDEBUGLEVEL=1" _C _CXX false)
         endif ()
         # Add noexecstack flags
         # LDFLAGS
-        EnableCompilerFlag("-z noexecstack" false false true)
+        EnableCompilerFlag("-Wl,-z,noexecstack" false false _LD)
         # CFLAGS & CXXFLAGS
-        EnableCompilerFlag("-Qunused-arguments" true true false)
-        EnableCompilerFlag("-Wa,--noexecstack" true true false)
+        EnableCompilerFlag("-Qunused-arguments" _C _CXX false)
+        EnableCompilerFlag("-Wa,--noexecstack" _C _CXX false)
+        # NOTE: Using 3 nested ifs because the variables are sometimes
+        # empty if the condition is false, and sometimes equal to false.
+        # This implicitly converts them to truthy values. There may be
+        # a better way to do this, but this reliably works.
+        if (${LD_FLAG_WL_Z_NOEXECSTACK})
+            if (${C_FLAG_WA_NOEXECSTACK})
+                if (${CXX_FLAG_WA_NOEXECSTACK})
+                    # We've succeeded in marking the stack as non-executable
+                    set(ZSTD_HAS_NOEXECSTACK true)
+                endif()
+            endif()
+        endif()
     elseif (MSVC) # Add specific compilation flags for Windows Visual
 
         set(ACTIVATE_MULTITHREADED_COMPILATION "ON" CACHE BOOL "activate multi-threaded compilation (/MP flag)")
         if (CMAKE_GENERATOR MATCHES "Visual Studio" AND ACTIVATE_MULTITHREADED_COMPILATION)
-            EnableCompilerFlag("/MP" true true false)
+            EnableCompilerFlag("/MP" _C _CXX false)
         endif ()
 
         # UNICODE SUPPORT
-        EnableCompilerFlag("/D_UNICODE" true true false)
-        EnableCompilerFlag("/DUNICODE" true true false)
+        EnableCompilerFlag("/D_UNICODE" _C _CXX false)
+        EnableCompilerFlag("/DUNICODE" _C _CXX false)
         # Enable asserts in Debug mode
         if (CMAKE_BUILD_TYPE MATCHES "Debug")
-            EnableCompilerFlag("/DDEBUGLEVEL=1" true true false)
+            EnableCompilerFlag("/DDEBUGLEVEL=1" _C _CXX false)
         endif ()
     endif ()
 

build/cmake/CMakeModules/ZstdBuild.cmake

@@ -0,0 +1,42 @@
+# ################################################################
+# ZSTD Build Targets Configuration
+# ################################################################
+
+# Always build the library first (this defines ZSTD_BUILD_STATIC/SHARED options)
+add_subdirectory(lib)
+
+# Validate build configuration after lib options are defined
+if(ZSTD_BUILD_PROGRAMS)
+    if(NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED)
+        message(SEND_ERROR "Static library required to build zstd CLI programs")
+    elseif(NOT ZSTD_BUILD_SHARED AND ZSTD_PROGRAMS_LINK_SHARED)
+        message(SEND_ERROR "Shared library required to build zstd CLI programs")
+    endif()
+endif()
+
+if(ZSTD_BUILD_TESTS AND NOT ZSTD_BUILD_STATIC)
+    message(SEND_ERROR "Static library required to build test suite")
+endif()
+
+# Add programs if requested
+if(ZSTD_BUILD_PROGRAMS)
+    add_subdirectory(programs)
+endif()
+
+# Add tests if requested
+if(ZSTD_BUILD_TESTS)
+    enable_testing()
+    add_subdirectory(tests)
+endif()
+
+# Add contrib utilities if requested
+if(ZSTD_BUILD_CONTRIB)
+    add_subdirectory(contrib)
+endif()
+
+# Clean-all target for thorough cleanup
+add_custom_target(clean-all
+    COMMAND ${CMAKE_BUILD_TOOL} clean
+    COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/
+    COMMENT "Performing complete clean including build directory"
+)

build/cmake/CMakeModules/ZstdDependencies.cmake

@@ -0,0 +1,30 @@
+# ################################################################
+# ZSTD Dependencies Configuration
+# ################################################################
+
+# Function to handle HP-UX thread configuration
+function(setup_hpux_threads)
+    find_package(Threads)
+    if(NOT Threads_FOUND)
+        set(CMAKE_USE_PTHREADS_INIT 1 PARENT_SCOPE)
+        set(CMAKE_THREAD_LIBS_INIT -lpthread PARENT_SCOPE)
+        set(CMAKE_HAVE_THREADS_LIBRARY 1 PARENT_SCOPE)
+        set(Threads_FOUND TRUE PARENT_SCOPE)
+    endif()
+endfunction()
+
+# Configure threading support
+if(ZSTD_MULTITHREAD_SUPPORT AND UNIX)
+    if(CMAKE_SYSTEM_NAME MATCHES "HP-UX")
+        setup_hpux_threads()
+    else()
+        set(THREADS_PREFER_PTHREAD_FLAG ON)
+        find_package(Threads REQUIRED)
+    endif()
+    
+    if(CMAKE_USE_PTHREADS_INIT)
+        set(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
+    else()
+        message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
+    endif()
+endif()

build/cmake/CMakeModules/ZstdOptions.cmake

@@ -0,0 +1,68 @@
+# ################################################################
+# ZSTD Build Options Configuration
+# ################################################################
+
+# Legacy support configuration (disabled by default)
+option(ZSTD_LEGACY_SUPPORT "Enable legacy format support" OFF)
+
+if(ZSTD_LEGACY_SUPPORT)
+    message(STATUS "ZSTD_LEGACY_SUPPORT enabled")
+    set(ZSTD_LEGACY_LEVEL 5 CACHE STRING "Legacy support level")
+    add_definitions(-DZSTD_LEGACY_SUPPORT=${ZSTD_LEGACY_LEVEL})
+else()
+    message(STATUS "ZSTD_LEGACY_SUPPORT disabled")
+    add_definitions(-DZSTD_LEGACY_SUPPORT=0)
+endif()
+
+# Platform-specific options
+if(APPLE)
+    option(ZSTD_FRAMEWORK "Build as Apple Framework" OFF)
+endif()
+
+# Android-specific configuration
+if(ANDROID)
+    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT OFF)
+    # Handle old Android API levels
+    if((NOT ANDROID_PLATFORM_LEVEL) OR (ANDROID_PLATFORM_LEVEL VERSION_LESS 24))
+        message(STATUS "Configuring for old Android API - disabling fseeko/ftello")
+        add_compile_definitions(LIBC_NO_FSEEKO)
+    endif()
+else()
+    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT ON)
+endif()
+
+# Multi-threading support
+option(ZSTD_MULTITHREAD_SUPPORT "Enable multi-threading support" ${ZSTD_MULTITHREAD_SUPPORT_DEFAULT})
+
+if(ZSTD_MULTITHREAD_SUPPORT)
+    message(STATUS "Multi-threading support enabled")
+else()
+    message(STATUS "Multi-threading support disabled")
+endif()
+
+# Build component options
+option(ZSTD_BUILD_PROGRAMS "Build command-line programs" ON)
+option(ZSTD_BUILD_CONTRIB "Build contrib utilities" OFF)
+option(ZSTD_PROGRAMS_LINK_SHARED "Link programs against shared library" OFF)
+
+# Test configuration
+if(BUILD_TESTING)
+    set(ZSTD_BUILD_TESTS_default ON)
+else()
+    set(ZSTD_BUILD_TESTS_default OFF)
+endif()
+option(ZSTD_BUILD_TESTS "Build test suite" ${ZSTD_BUILD_TESTS_default})
+
+# MSVC-specific options
+if(MSVC)
+    option(ZSTD_USE_STATIC_RUNTIME "Link to static runtime libraries" OFF)
+endif()
+
+# C++ support (needed for tests)
+set(ZSTD_ENABLE_CXX ${ZSTD_BUILD_TESTS})
+if(ZSTD_ENABLE_CXX)
+    enable_language(CXX)
+endif()
+
+# Set global definitions
+add_definitions(-DXXH_NAMESPACE=ZSTD_)

build/cmake/CMakeModules/ZstdPackage.cmake

@@ -0,0 +1,45 @@
+# ################################################################
+# ZSTD Package Configuration
+# ################################################################
+
+include(CMakePackageConfigHelpers)
+
+# Generate version file
+write_basic_package_version_file(
+    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
+    VERSION ${zstd_VERSION}
+    COMPATIBILITY SameMajorVersion
+)
+
+# Configure package for installation
+set(ConfigPackageLocation ${CMAKE_INSTALL_LIBDIR}/cmake/zstd)
+
+foreach(target_suffix IN ITEMS "_shared" "_static" "")
+    if(TARGET "libzstd${target_suffix}")
+        # Export targets for build directory
+        export(EXPORT "zstdExports${target_suffix}"
+                FILE "${CMAKE_CURRENT_BINARY_DIR}/zstdTargets${target_suffix}.cmake"
+                NAMESPACE zstd::
+        )
+        # Install exported targets
+        install(EXPORT "zstdExports${target_suffix}"
+                FILE "zstdTargets${target_suffix}.cmake"
+                NAMESPACE zstd::
+                DESTINATION ${ConfigPackageLocation}
+        )
+    endif()
+endforeach()
+
+# Configure and install package config file
+configure_package_config_file(
+    zstdConfig.cmake.in
+    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
+    INSTALL_DESTINATION ${ConfigPackageLocation}
+)
+
+# Install config files
+install(FILES
+    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
+    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
+    DESTINATION ${ConfigPackageLocation}
+)

build/cmake/CMakeModules/ZstdVersion.cmake

@@ -0,0 +1,31 @@
+# ################################################################
+# ZSTD Version Configuration
+# ################################################################
+
+# Setup CMake policy version
+set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3")
+set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13")
+
+# Determine appropriate policy version
+if("${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND
+   "${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}")
+    set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
+else()
+    set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0")
+endif()
+
+cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
+
+# Parse version from header file
+include(GetZstdLibraryVersion)
+GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h zstd_VERSION_MAJOR zstd_VERSION_MINOR zstd_VERSION_PATCH)
+
+# Set version variables
+set(ZSTD_SHORT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}")
+set(ZSTD_FULL_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
+
+# Project metadata
+set(zstd_HOMEPAGE_URL "https://facebook.github.io/zstd")
+set(zstd_DESCRIPTION "Zstandard is a real-time compression algorithm, providing high compression ratios.")
+
+message(STATUS "ZSTD VERSION: ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")

build/cmake/README.md

@@ -7,6 +7,15 @@ variables.
 
 ## How to build
 
+You can configure the project from the repository root thanks to the forwarding
+`CMakeLists.txt`:
+```sh
+cmake -S . -B build-cmake
+cmake --build build-cmake
+```
+The historical workflow that starts configuration from `build/cmake` continues
+to work as described below.
+
 As cmake doesn't support command like `cmake clean`, it's recommended to perform an "out of source build".
 To do this, you can create a new directory and build in it:
 ```sh
@@ -41,6 +50,48 @@ cmake -DZSTD_BUILD_TESTS=ON -DZSTD_LEGACY_SUPPORT=OFF ..
 make
 ```
 
+**Apple Frameworks**
+It's generally recommended to have CMake with versions higher than 3.14 for [iOS-derived platforms](https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html#id27).
+```sh
+cmake -S. -B build-cmake -DZSTD_FRAMEWORK=ON -DCMAKE_SYSTEM_NAME=iOS
+```
+Or you can utilize [iOS-CMake](https://github.com/leetal/ios-cmake) toolchain for CMake versions lower than 3.14
+```sh
+cmake -B build -G Xcode -DCMAKE_TOOLCHAIN_FILE=<Path To ios.toolchain.cmake> -DPLATFORM=OS64 -DZSTD_FRAMEWORK=ON
+```
+
+### how to use it with CMake FetchContent
+
+For all options available, you can see it on <https://github.com/facebook/zstd/blob/dev/build/cmake/lib/CMakeLists.txt>
+```cmake
+include(FetchContent)
+
+set(ZSTD_BUILD_STATIC ON)
+set(ZSTD_BUILD_SHARED OFF)
+
+FetchContent_Declare(
+    zstd
+    URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz"
+    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+    SOURCE_SUBDIR build/cmake
+)
+
+FetchContent_MakeAvailable(zstd)
+
+target_link_libraries(
+    ${PROJECT_NAME}
+    PRIVATE
+    libzstd_static
+)
+
+# On windows and macos this is needed
+target_include_directories(
+    ${PROJECT_NAME}
+    PRIVATE
+    ${zstd_SOURCE_DIR}/lib
+)
+```
+
 ### referring
 [Looking for a 'cmake clean' command to clear up CMake output](https://stackoverflow.com/questions/9680420/looking-for-a-cmake-clean-command-to-clear-up-cmake-output)
 

build/cmake/contrib/pzstd/CMakeLists.txt

@@ -18,6 +18,7 @@ set(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
 include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
 
 add_executable(pzstd ${PROGRAMS_DIR}/util.c ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
+target_compile_features(pzstd PRIVATE cxx_std_11)
 set_property(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
 set_property(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
 

build/cmake/lib/CMakeLists.txt

@@ -7,50 +7,76 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-project(libzstd C ASM)
+project(libzstd C) # ASM language is conditionally enabled where supported
 
 set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
 option(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON)
+option(ZSTD_BUILD_COMPRESSION "BUILD COMPRESSION MODULE" ON)
+option(ZSTD_BUILD_DECOMPRESSION "BUILD DECOMPRESSION MODULE" ON)
+option(ZSTD_BUILD_DICTBUILDER "BUILD DICTBUILDER MODULE" ON)
+option(ZSTD_BUILD_DEPRECATED "BUILD DEPRECATED MODULE" OFF)
+
+set(ZSTDLIB_VISIBLE "" CACHE STRING "Visibility for ZSTDLIB API")
+set(ZSTDERRORLIB_VISIBLE "" CACHE STRING "Visibility for ZSTDERRORLIB_VISIBLE API")
+set(ZDICTLIB_VISIBLE "" CACHE STRING "Visibility for ZDICTLIB_VISIBLE API")
+set(ZSTDLIB_STATIC_API "" CACHE STRING "Visibility for ZSTDLIB_STATIC_API API")
+set(ZDICTLIB_STATIC_API "" CACHE STRING "Visibility for ZDICTLIB_STATIC_API API")
+
+set_property(CACHE ZSTDLIB_VISIBLE PROPERTY STRINGS "" "hidden" "default" "protected" "internal")
+set_property(CACHE ZSTDERRORLIB_VISIBLE PROPERTY STRINGS "" "hidden" "default" "protected" "internal")
+set_property(CACHE ZDICTLIB_VISIBLE PROPERTY STRINGS "" "hidden" "default" "protected" "internal")
+set_property(CACHE ZSTDLIB_STATIC_API PROPERTY STRINGS "" "hidden" "default" "protected" "internal")
+set_property(CACHE ZDICTLIB_STATIC_API PROPERTY STRINGS "" "hidden" "default" "protected" "internal")
 
 if(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
     message(SEND_ERROR "You need to build at least one flavor of libzstd")
 endif()
 
-# Define library directory, where sources and header files are located
-include_directories(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
-
 file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c)
 file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c)
+file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
 if (MSVC)
-    file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
     add_compile_options(-DZSTD_DISABLE_ASM)
 else ()
-    file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c ${LIBRARY_DIR}/decompress/*.S)
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|AMD64.*|x86_64.*|X86_64.*" AND ${ZSTD_HAS_NOEXECSTACK})
+        enable_language(ASM)
+        set(DecompressSources ${DecompressSources} ${LIBRARY_DIR}/decompress/huf_decompress_amd64.S)
+    else()
+        add_compile_options(-DZSTD_DISABLE_ASM)
+    endif()
 endif ()
 file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c)
+file(GLOB DeprecatedSources ${LIBRARY_DIR}/deprecated/*.c)
 
-set(Sources
-        ${CommonSources}
-        ${CompressSources}
-        ${DecompressSources}
-        ${DictBuilderSources})
-
+file(GLOB PublicHeaders ${LIBRARY_DIR}/*.h)
 file(GLOB CommonHeaders ${LIBRARY_DIR}/common/*.h)
 file(GLOB CompressHeaders ${LIBRARY_DIR}/compress/*.h)
 file(GLOB DecompressHeaders ${LIBRARY_DIR}/decompress/*.h)
 file(GLOB DictBuilderHeaders ${LIBRARY_DIR}/dictBuilder/*.h)
+file(GLOB DeprecatedHeaders ${LIBRARY_DIR}/deprecated/*.h)
 
-set(Headers
-        ${LIBRARY_DIR}/zstd.h
-        ${CommonHeaders}
-        ${CompressHeaders}
-        ${DecompressHeaders}
-        ${DictBuilderHeaders})
+set(Sources ${CommonSources})
+set(Headers ${PublicHeaders} ${CommonHeaders})
+if (ZSTD_BUILD_COMPRESSION)
+    set(Sources ${Sources} ${CompressSources})
+    set(Headers ${Headers} ${CompressHeaders})
+endif()
+if (ZSTD_BUILD_DECOMPRESSION)
+    set(Sources ${Sources} ${DecompressSources})
+    set(Headers ${Headers} ${DecompressHeaders})
+endif()
+if (ZSTD_BUILD_DICTBUILDER)
+    set(Sources ${Sources} ${DictBuilderSources})
+    set(Headers ${Headers} ${DictBuilderHeaders})
+endif()
+if (ZSTD_BUILD_DEPRECATED)
+    set(Sources ${Sources} ${DeprecatedSources})
+    set(Headers ${Headers} ${DeprecatedHeaders})
+endif()
 
 if (ZSTD_LEGACY_SUPPORT)
     set(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
-    include_directories(${LIBRARY_LEGACY_DIR})
 
     set(Sources ${Sources}
             ${LIBRARY_LEGACY_DIR}/zstd_v01.c
@@ -72,38 +98,84 @@ if (ZSTD_LEGACY_SUPPORT)
             ${LIBRARY_LEGACY_DIR}/zstd_v07.h)
 endif ()
 
-if (MSVC)
+if (MSVC AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang"))
     set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll)
     set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
+else()
+    set(PlatformDependResources)
 endif ()
 
 # Explicitly set the language to C for all files, including ASM files.
 # Our assembly expects to be compiled by a C compiler, and is only enabled for
 # __GNUC__ compatible compilers. Otherwise all the ASM code is disabled by
 # macros.
-set_source_files_properties(${Sources} PROPERTIES LANGUAGE C)
+if(NOT CMAKE_ASM_COMPILER STREQUAL CMAKE_C_COMPILER)
+    set_source_files_properties(${Sources} PROPERTIES LANGUAGE C)
+endif()
 
+macro (add_definition target var)
+    if (NOT ("${${var}}" STREQUAL ""))
+        target_compile_definitions(${target} PUBLIC "${var}=__attribute__((visibility(\"${${var}}\")))")
+    endif ()
+endmacro ()
+
+# Define directories containing the library's public headers
+set(PUBLIC_INCLUDE_DIRS ${LIBRARY_DIR})
+set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} /I \"${LIBRARY_DIR}\"")
 # Split project to static and shared libraries build
-set(library_targets)
 if (ZSTD_BUILD_SHARED)
     add_library(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
-    list(APPEND library_targets libzstd_shared)
+    target_include_directories(libzstd_shared INTERFACE $<BUILD_INTERFACE:${PUBLIC_INCLUDE_DIRS}>)
     if (ZSTD_MULTITHREAD_SUPPORT)
-        set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
+        target_compile_definitions(libzstd_shared PUBLIC ZSTD_MULTITHREAD)
         if (UNIX)
             target_link_libraries(libzstd_shared ${THREADS_LIBS})
         endif ()
-    endif()
+    endif ()
+    add_definition(libzstd_shared ZSTDLIB_VISIBLE)
+    add_definition(libzstd_shared ZSTDERRORLIB_VISIBLE)
+    add_definition(libzstd_shared ZDICTLIB_VISIBLE)
 endif ()
 if (ZSTD_BUILD_STATIC)
     add_library(libzstd_static STATIC ${Sources} ${Headers})
-    list(APPEND library_targets libzstd_static)
+    target_include_directories(libzstd_static INTERFACE $<BUILD_INTERFACE:${PUBLIC_INCLUDE_DIRS}>)
     if (ZSTD_MULTITHREAD_SUPPORT)
-        set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
+        target_compile_definitions(libzstd_static PUBLIC ZSTD_MULTITHREAD)
         if (UNIX)
             target_link_libraries(libzstd_static ${THREADS_LIBS})
         endif ()
     endif ()
+    add_definition(libzstd_static ZSTDLIB_VISIBLE)
+    add_definition(libzstd_static ZSTDERRORLIB_VISIBLE)
+    add_definition(libzstd_static ZDICTLIB_VISIBLE)
+    add_definition(libzstd_static ZSTDLIB_STATIC_API)
+    add_definition(libzstd_static ZDICTLIB_STATIC_API)
+endif ()
+if (ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
+    if (NOT BUILD_SHARED_LIBS)
+        message(WARNING "BUILD_SHARED_LIBS is OFF, but ZSTD_BUILD_SHARED is ON and ZSTD_BUILD_STATIC is OFF, which takes precedence, so libzstd is a shared library")
+    endif ()
+    add_library(libzstd INTERFACE)
+    target_link_libraries(libzstd INTERFACE libzstd_shared)
+endif ()
+if (ZSTD_BUILD_STATIC AND NOT ZSTD_BUILD_SHARED)
+    if (BUILD_SHARED_LIBS)
+        message(WARNING "BUILD_SHARED_LIBS is ON, but ZSTD_BUILD_SHARED is OFF and ZSTD_BUILD_STATIC is ON, which takes precedence, is set so libzstd is a static library")
+    endif ()
+    add_library(libzstd INTERFACE)
+    target_link_libraries(libzstd INTERFACE libzstd_static)
+endif ()
+if (ZSTD_BUILD_SHARED AND ZSTD_BUILD_STATIC)
+    # If both ZSTD_BUILD_SHARED and ZSTD_BUILD_STATIC are set, which is the
+    # default, fallback to using BUILD_SHARED_LIBS to determine whether to
+    # set libzstd to static or shared.
+    if (BUILD_SHARED_LIBS)
+        add_library(libzstd INTERFACE)
+        target_link_libraries(libzstd INTERFACE libzstd_shared)
+    else ()
+        add_library(libzstd INTERFACE)
+        target_link_libraries(libzstd INTERFACE libzstd_static)
+    endif ()
 endif ()
 
 # Add specific compile definitions for MSVC project
@@ -129,8 +201,28 @@ if (ZSTD_BUILD_SHARED)
             libzstd_shared
             PROPERTIES
             OUTPUT_NAME zstd
-            VERSION ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}
+            VERSION ${ZSTD_FULL_VERSION}
             SOVERSION ${zstd_VERSION_MAJOR})
+
+    if (ZSTD_FRAMEWORK)
+        set_target_properties(
+                libzstd_shared
+                PROPERTIES
+                FRAMEWORK TRUE
+                FRAMEWORK_VERSION "${ZSTD_FULL_VERSION}"
+                PRODUCT_BUNDLE_IDENTIFIER "github.com/facebook/zstd"
+                XCODE_ATTRIBUTE_INSTALL_PATH "@rpath"
+                PUBLIC_HEADER "${PublicHeaders}"
+                OUTPUT_NAME "zstd"
+                XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY ""
+                XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO"
+                XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED "NO"
+                MACOSX_FRAMEWORK_IDENTIFIER "github.com/facebook/zstd"
+                MACOSX_FRAMEWORK_BUNDLE_VERSION "${ZSTD_FULL_VERSION}"
+                MACOSX_FRAMEWORK_SHORT_VERSION_STRING "${ZSTD_SHORT_VERSION}"
+                MACOSX_RPATH TRUE
+                RESOURCE ${PublicHeaders})
+    endif ()
 endif ()
 
 if (ZSTD_BUILD_STATIC)
@@ -139,6 +231,26 @@ if (ZSTD_BUILD_STATIC)
             PROPERTIES
             POSITION_INDEPENDENT_CODE On
             OUTPUT_NAME ${STATIC_LIBRARY_BASE_NAME})
+
+    if (ZSTD_FRAMEWORK)
+        set_target_properties(
+                libzstd_static
+                PROPERTIES
+                FRAMEWORK TRUE
+                FRAMEWORK_VERSION "${ZSTD_FULL_VERSION}"
+                PRODUCT_BUNDLE_IDENTIFIER "github.com/facebook/zstd/${STATIC_LIBRARY_BASE_NAME}"
+                XCODE_ATTRIBUTE_INSTALL_PATH "@rpath"
+                PUBLIC_HEADER "${PublicHeaders}"
+                OUTPUT_NAME "${STATIC_LIBRARY_BASE_NAME}"
+                XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY ""
+                XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO"
+                XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED "NO"
+                MACOSX_FRAMEWORK_IDENTIFIER "github.com/facebook/zstd/${STATIC_LIBRARY_BASE_NAME}"
+                MACOSX_FRAMEWORK_BUNDLE_VERSION "${ZSTD_FULL_VERSION}"
+                MACOSX_FRAMEWORK_SHORT_VERSION_STRING "${ZSTD_SHORT_VERSION}"
+                MACOSX_RPATH TRUE
+                RESOURCE ${PublicHeaders})
+    endif ()
 endif ()
 
 # pkg-config
@@ -154,20 +266,22 @@ configure_file("${LIBRARY_DIR}/libzstd.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libzs
 install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 
 # install target
-install(FILES
-    "${LIBRARY_DIR}/zstd.h"
-    "${LIBRARY_DIR}/zdict.h"
-    "${LIBRARY_DIR}/zstd_errors.h"
-    DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
+install(FILES ${PublicHeaders} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
 
-install(TARGETS ${library_targets}
-    EXPORT zstdExports
-    INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
-    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
-    BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}"
-    )
+foreach(target_suffix IN ITEMS "_shared" "_static" "")
+    if(TARGET "libzstd${target_suffix}")
+        install(TARGETS "libzstd${target_suffix}"
+            EXPORT "zstdExports${target_suffix}"
+            INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
+            ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+            LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+            RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
+            BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}"
+            FRAMEWORK DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT runtime OPTIONAL
+            PUBLIC_HEADER DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
+            )
+    endif()
+endforeach()
 
 # uninstall target
 if (NOT TARGET uninstall)

build/cmake/programs/CMakeLists.txt

@@ -32,7 +32,12 @@ if (MSVC)
     set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
 endif ()
 
-add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_asyncio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources})
+file(GLOB ZSTD_PROGRAM_SRCS "${PROGRAMS_DIR}/*.c")
+if (MSVC AND ZSTD_PROGRAMS_LINK_SHARED)
+    list(APPEND ZSTD_PROGRAM_SRCS ${LIBRARY_DIR}/common/pool.c ${LIBRARY_DIR}/common/threading.c)
+endif ()
+
+add_executable(zstd ${ZSTD_PROGRAM_SRCS})
 target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET})
 if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
     target_link_libraries(zstd rt)
@@ -75,7 +80,9 @@ if (UNIX)
         ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1
         DESTINATION "${MAN_INSTALL_DIR}")
 
-    add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_asyncio.c)
+    add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c
+        ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c
+        ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_asyncio.c)
     target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET})
     set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT;ZSTD_NOTRACE")
 endif ()

build/cmake/tests/CMakeLists.txt

@@ -50,18 +50,18 @@ set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
 set(TESTS_DIR ${ZSTD_SOURCE_DIR}/tests)
 include_directories(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
 
-add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
+add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/lorem.c ${TESTS_DIR}/loremOut.c ${TESTS_DIR}/datagencli.c)
 target_link_libraries(datagen libzstd_static)
 
 #
 # fullbench
 #
-add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
+add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/lorem.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
 if (NOT MSVC)
     target_compile_options(fullbench PRIVATE "-Wno-deprecated-declarations")
 endif()
 target_link_libraries(fullbench libzstd_static)
-add_test(NAME fullbench COMMAND fullbench ${ZSTD_FULLBENCH_FLAGS})
+add_test(NAME fullbench COMMAND "$<TARGET_FILE:fullbench>" ${ZSTD_FULLBENCH_FLAGS})
 
 #
 # fuzzer
@@ -73,7 +73,7 @@ endif()
 target_link_libraries(fuzzer libzstd_static)
 AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}"
     "Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)")
-add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS})
+add_test(NAME fuzzer COMMAND "$<TARGET_FILE:fuzzer>" ${ZSTD_FUZZER_FLAGS})
 # Disable the timeout since the run time is too long for the default timeout of
 # 1500 seconds and varies considerably between low-end and high-end CPUs.
 # set_tests_properties(fuzzer PROPERTIES TIMEOUT 0)
@@ -88,7 +88,7 @@ endif()
 target_link_libraries(zstreamtest libzstd_static)
 AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}"
     "Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)")
-add_test(NAME zstreamtest COMMAND zstreamtest ${ZSTD_ZSTREAM_FLAGS})
+add_test(NAME zstreamtest COMMAND "$<TARGET_FILE:zstreamtest>" ${ZSTD_ZSTREAM_FLAGS})
 
 #
 # playTests.sh
@@ -110,7 +110,7 @@ endif()
 # Label the "Medium" set of tests (see TESTING.md)
 set_property(TEST fuzzer zstreamtest playTests APPEND PROPERTY LABELS Medium)
 
-add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
+add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/lorem.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
 if (UNIX)
     target_link_libraries(paramgrill libzstd_static m) #m is math library
 else()

build/cmake/zstdConfig.cmake

@@ -1 +0,0 @@
-include("${CMAKE_CURRENT_LIST_DIR}/zstdTargets.cmake")

build/cmake/zstdConfig.cmake.in

@@ -0,0 +1,13 @@
+@PACKAGE_INIT@
+
+include(CMakeFindDependencyMacro)
+if(@ZSTD_MULTITHREAD_SUPPORT@ AND "@UNIX@")
+  find_dependency(Threads)
+endif()
+
+foreach(lib_suffix IN ITEMS "_shared" "_static")
+    include("${CMAKE_CURRENT_LIST_DIR}/zstdTargets${lib_suffix}.cmake" OPTIONAL)
+endforeach()
+include("${CMAKE_CURRENT_LIST_DIR}/zstdTargets.cmake")
+
+check_required_components("zstd")

build/meson/contrib/pzstd/meson.build

@@ -20,6 +20,6 @@ pzstd = executable('pzstd',
   pzstd_sources,
   cpp_args: pzstd_warning_flags,
   include_directories: pzstd_includes,
-  dependencies: [ libzstd_dep, thread_dep ],
+  dependencies: [ libzstd_internal_dep, thread_dep ],
   override_options: ['b_ndebug=true'],
   install: true)

build/meson/lib/meson.build

@@ -30,6 +30,7 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
   join_paths(zstd_rootdir, 'lib/compress/zstd_compress_literals.c'),
   join_paths(zstd_rootdir, 'lib/compress/zstd_compress_sequences.c'),
   join_paths(zstd_rootdir, 'lib/compress/zstd_compress_superblock.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_preSplit.c'),
   join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'),
   join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'),
   join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'),
@@ -124,7 +125,11 @@ libzstd = library('zstd',
   version: zstd_libversion)
 
 libzstd_dep = declare_dependency(link_with: libzstd,
-  include_directories: libzstd_includes)
+  include_directories: join_paths(zstd_rootdir,'lib')) # Do not expose private headers
+
+if meson.version().version_compare('>=0.54.0')
+  meson.override_dependency('libzstd', libzstd_dep)
+endif
 
 # we link to both:
 # - the shared library (for public symbols)
@@ -134,7 +139,8 @@ libzstd_dep = declare_dependency(link_with: libzstd,
 # -fvisibility=hidden means those cannot be found
 if get_option('default_library') == 'static'
   libzstd_static = libzstd
-  libzstd_internal_dep = libzstd_dep
+  libzstd_internal_dep = declare_dependency(link_with: libzstd,
+      include_directories: libzstd_includes)
 else
   if get_option('default_library') == 'shared'
     libzstd_static = static_library('zstd_objlib',
@@ -147,11 +153,13 @@ else
   if cc_id == compiler_msvc
     # msvc does not actually support linking to both, but errors out with:
     #   error LNK2005: ZSTD_<foo> already defined in zstd.lib(zstd-1.dll)
-    libzstd_internal_dep = declare_dependency(link_with: libzstd_static)
+    libzstd_internal_dep = declare_dependency(link_with: libzstd_static,
+      include_directories: libzstd_includes)
   else
     libzstd_internal_dep = declare_dependency(link_with: libzstd,
       # the static library must be linked after the shared one
-      dependencies: declare_dependency(link_with: libzstd_static))
+      dependencies: declare_dependency(link_with: libzstd_static),
+      include_directories: libzstd_includes)
   endif
 endif
 

build/meson/meson.build

@@ -10,7 +10,7 @@
 
 project('zstd',
   ['c', 'cpp'],
-  license: ['BSD', 'GPLv2'],
+  license: 'BSD-3-Clause OR GPL-2.0-only',
   default_options : [
     # There shouldn't be any need to force a C standard convention for zstd
     # but in case one would want that anyway, this can be done here.
@@ -65,7 +65,6 @@ zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name())
 
 # Built-in options
 use_debug = get_option('debug')
-buildtype = get_option('buildtype')
 default_library_type = get_option('default_library')
 
 # Custom options
@@ -88,8 +87,13 @@ feature_lz4 = get_option('lz4')
 # =============================================================================
 
 libm_dep = cc.find_library('m', required: false)
-thread_dep = dependency('threads', required: feature_multi_thread)
-use_multi_thread = thread_dep.found()
+if host_machine_os == os_windows
+  thread_dep = dependency('', required: false)
+  use_multi_thread = not feature_multi_thread.disabled()
+else
+  thread_dep = dependency('threads', required: feature_multi_thread)
+  use_multi_thread = thread_dep.found()
+endif
 # Arguments in dependency should be equivalent to those passed to pkg-config
 zlib_dep = dependency('zlib', required: feature_zlib)
 use_zlib = zlib_dep.found()
@@ -111,10 +115,16 @@ if [compiler_gcc, compiler_clang].contains(cc_id)
   if cc_id == compiler_clang
     common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation']
   endif
-  cc_compile_flags = cc.get_supported_arguments(common_warning_flags + ['-Wstrict-prototypes'])
-  cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags)
+  noexecstack_flags = ['-Wa,--noexecstack' ]
+  noexecstack_link_flags = ['-Wl,-z,noexecstack']
+  cc_compile_flags = cc.get_supported_arguments(common_warning_flags + noexecstack_flags + ['-Wstrict-prototypes'])
+  cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags + noexecstack_flags)
   add_project_arguments(cc_compile_flags, language : 'c')
   add_project_arguments(cxx_compile_flags, language : 'cpp')
+  cc_link_flags = cc.get_supported_link_arguments(noexecstack_link_flags)
+  cxx_link_flags = cxx.get_supported_link_arguments(noexecstack_link_flags)
+  add_project_link_arguments(cc_link_flags, language: 'c')
+  add_project_link_arguments(cxx_link_flags, language: 'cpp')
 elif cc_id == compiler_msvc
   msvc_compile_flags = [ '/D_UNICODE', '/DUNICODE' ]
   if use_multi_thread

build/meson/meson_options.txt

@@ -10,8 +10,8 @@
 
 # Read guidelines from https://wiki.gnome.org/Initiatives/GnomeGoals/MesonPorting
 
-option('legacy_level', type: 'integer', min: 0, max: 7, value: 5,
-  description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+')
+option('legacy_level', type: 'integer', min: 0, max: 7, value: 0,
+  description: 'Support legacy format: 0=disabled, 1-7=support v0.1+ to v0.7+')
 option('debug_level', type: 'integer', min: 0, max: 9, value: 1,
   description: 'Enable run-time debug. See lib/common/debug.h')
 option('backtrace', type: 'feature', value: 'disabled',
@@ -27,7 +27,7 @@ option('bin_contrib', type: 'boolean', value: false,
   description: 'Enable contrib build')
 
 option('multi_thread', type: 'feature', value: 'enabled',
-  description: 'Enable multi-threading when pthread is detected')
+  description: 'Enable multi-threading when pthread or Windows is detected')
 option('zlib', type: 'feature', value: 'auto',
   description: 'Enable zlib support')
 option('lzma', type: 'feature', value: 'auto',

build/meson/programs/meson.build

@@ -18,6 +18,7 @@ zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
   join_paths(zstd_rootdir, 'programs/benchfn.c'),
   join_paths(zstd_rootdir, 'programs/benchzstd.c'),
   join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/lorem.c'),
   join_paths(zstd_rootdir, 'programs/dibio.c'),
   join_paths(zstd_rootdir, 'programs/zstdcli_trace.c')]
 

build/meson/tests/meson.build

@@ -29,6 +29,7 @@ DECODECORPUS_TESTTIME = '-T30'
 test_includes = [ include_directories(join_paths(zstd_rootdir, 'programs')) ]
 
 testcommon_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/lorem.c'),
   join_paths(zstd_rootdir, 'programs/util.c'),
   join_paths(zstd_rootdir, 'programs/timefn.c'),
   join_paths(zstd_rootdir, 'programs/benchfn.c'),
@@ -43,7 +44,8 @@ testcommon_dep = declare_dependency(link_with: testcommon,
   dependencies: libzstd_deps,
   include_directories: libzstd_includes)
 
-datagen_sources = [join_paths(zstd_rootdir, 'tests/datagencli.c')]
+datagen_sources = [join_paths(zstd_rootdir, 'tests/datagencli.c'),
+  join_paths(zstd_rootdir, 'tests/loremOut.c')]
 datagen = executable('datagen',
   datagen_sources,
   c_args: [ '-DNDEBUG' ],
@@ -91,7 +93,7 @@ roundTripCrash = executable('roundTripCrash',
 longmatch_sources = [join_paths(zstd_rootdir, 'tests/longmatch.c')]
 longmatch = executable('longmatch',
   longmatch_sources,
-  dependencies: [ libzstd_dep ],
+  dependencies: [ libzstd_internal_dep ],
   install: false)
 
 invalidDictionaries_sources = [join_paths(zstd_rootdir, 'tests/invalidDictionaries.c')]

build/meson/tests/valgrindTest.py

@@ -19,7 +19,7 @@ def valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench):
 
   subprocess.check_call([*VALGRIND_ARGS, datagen, '-g50M'], stdout=subprocess.DEVNULL)
 
-  if subprocess.call([*VALGRIND_ARGS, zstd],
+  if subprocess.call([*VALGRIND_ARGS, zstd, '--fake-stdin-is-console'],
                      stdout=subprocess.DEVNULL) == 0:
     raise subprocess.SubprocessError('zstd without argument should have failed')
 

build/single_file_libs/.gitignore

@@ -4,6 +4,7 @@ zstddeclib.c
 zstdenclib.c
 zstd.c
 zstd.h
+zstd_errors.h
 
 # test artifacts
 temp*

build/single_file_libs/build_library_test.sh

@@ -70,6 +70,7 @@ echo "Single file library creation script: PASSED"
 
 # Copy the header to here (for the tests)
 cp "$ZSTD_SRC_ROOT/zstd.h" examples/zstd.h
+cp "$ZSTD_SRC_ROOT/zstd_errors.h" examples/zstd_errors.h
 
 # Compile the generated output
 cc -Wall -Wextra -Werror -Wshadow -pthread -I. -Os -g0 -o $OUT_FILE zstd.c examples/roundtrip.c

build/single_file_libs/zstd-in.c

@@ -69,6 +69,7 @@
 #include "compress/zstd_compress_literals.c"
 #include "compress/zstd_compress_sequences.c"
 #include "compress/zstd_compress_superblock.c"
+#include "compress/zstd_preSplit.c"
 #include "compress/zstd_compress.c"
 #include "compress/zstd_double_fast.c"
 #include "compress/zstd_fast.c"

contrib/externalSequenceProducer/main.c

@@ -27,6 +27,7 @@ do {                                                    \
 } while (0)                                             \
 
 int main(int argc, char *argv[]) {
+    int retn = 0;
     if (argc != 2) {
         printf("Usage: externalSequenceProducer <file>\n");
         return 1;
@@ -96,12 +97,12 @@ int main(int argc, char *argv[]) {
                 break;
             }
         }
-        return 1;
+        retn = 1;
     }
 
     ZSTD_freeCCtx(zc);
     free(src);
     free(dst);
     free(val);
-    return 0;
+    return retn;
 }

contrib/gen_html/Makefile

@@ -40,7 +40,7 @@ gen_html: gen_html.cpp
 
 $(ZSTDMANUAL): gen_html $(ZSTDAPI)
 	echo "Update zstd manual in /doc"
-	./gen_html $(LIBVER) $(ZSTDAPI) $(ZSTDMANUAL)
+	./gen_html$(EXT) $(LIBVER) $(ZSTDAPI) $(ZSTDMANUAL)
 
 .PHONY: manual
 manual: gen_html $(ZSTDMANUAL)

contrib/gen_html/gen_html.cpp

@@ -211,6 +211,7 @@ int main(int argc, char *argv[]) {
 
     ostream << "<html>\n<head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=ISO-8859-1\">\n<title>" << version << "</title>\n</head>\n<body>" << endl;
     ostream << "<h1>" << version << "</h1>\n";
+    ostream << "Note: the content of this file has been automatically generated by parsing \"zstd.h\" \n";
 
     ostream << "<hr>\n<a name=\"Contents\"></a><h2>Contents</h2>\n<ol>\n";
     for (size_t i=0; i<chapters.size(); i++)

contrib/largeNbDicts/largeNbDicts.c

@@ -45,7 +45,9 @@
 #define RUN_TIME_DEFAULT_MS    1000
 #define BENCH_TIME_DEFAULT_MS (BENCH_TIME_DEFAULT_S * RUN_TIME_DEFAULT_MS)
 
-#define DISPLAY_LEVEL_DEFAULT 3
+#ifndef ZSTD_DISPLAY_LEVEL_DEFAULT
+# define ZSTD_DISPLAY_LEVEL_DEFAULT 3
+#endif
 
 #define BENCH_SIZE_MAX (1200 MB)
 
@@ -61,7 +63,7 @@
 
 #define DISPLAY(...)         fprintf(stdout, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
-static int g_displayLevel = DISPLAY_LEVEL_DEFAULT;   /* 0 : no display,  1: errors,  2 : + result + interaction + warnings,  3 : + progression,  4 : + information */
+static int g_displayLevel = ZSTD_DISPLAY_LEVEL_DEFAULT;   /* 0 : no display,  1: errors,  2 : + result + interaction + warnings,  3 : + progression,  4 : + information */
 
 
 /*---  buffer_t  ---*/
@@ -739,6 +741,8 @@ static int benchMem(slice_collection_t dstBlocks, slice_collection_t srcBlocks,
     /* BMK_benchTimedFn may not run exactly nbRounds iterations */
     double speedAggregated =
         aggregateData(speedPerRound, roundNb + 1, metricAggregatePref);
+    free(speedPerRound);
+    
     if (metricAggregatePref == fastest)
       DISPLAY("Fastest Speed : %.1f MB/s \n", speedAggregated);
     else
@@ -1025,7 +1029,7 @@ int main (int argc, const char** argv)
     unsigned nbBlocks = 0; /* determine nbBlocks automatically, from source and blockSize */
     ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto;
     ZSTD_dictAttachPref_e dictAttachPref = ZSTD_dictDefaultAttach;
-    ZSTD_paramSwitch_e prefetchCDictTables = ZSTD_ps_auto;
+    ZSTD_ParamSwitch_e prefetchCDictTables = ZSTD_ps_auto;
     metricAggregatePref_e metricAggregatePref = fastest;
 
     for (int argNb = 1; argNb < argc ; argNb++) {

contrib/linux-kernel/linux.mk

@@ -26,6 +26,7 @@ zstd_compress-y := \
 		compress/zstd_lazy.o \
 		compress/zstd_ldm.o \
 		compress/zstd_opt.o \
+		compress/zstd_preSplit.o \
 
 zstd_decompress-y := \
 		zstd_decompress_module.o \

contrib/linux-kernel/linux_zstd.h

@@ -77,6 +77,30 @@ int zstd_min_clevel(void);
  */
 int zstd_max_clevel(void);
 
+/**
+ * zstd_default_clevel() - default compression level
+ *
+ * Return: Default compression level.
+ */
+int zstd_default_clevel(void);
+
+/**
+ * struct zstd_custom_mem - custom memory allocation
+ */
+typedef ZSTD_customMem zstd_custom_mem;
+
+/**
+ * struct zstd_dict_load_method - Dictionary load method.
+ * See zstd_lib.h.
+ */
+typedef ZSTD_dictLoadMethod_e zstd_dict_load_method;
+
+/**
+ * struct zstd_dict_content_type - Dictionary context type.
+ * See zstd_lib.h.
+ */
+typedef ZSTD_dictContentType_e zstd_dict_content_type;
+
 /* ======   Parameter Selection   ====== */
 
 /**
@@ -136,9 +160,32 @@ typedef ZSTD_parameters zstd_parameters;
 zstd_parameters zstd_get_params(int level,
 	unsigned long long estimated_src_size);
 
-/* ======   Single-pass Compression   ====== */
+/**
+ * zstd_get_cparams() - returns zstd_compression_parameters for selected level
+ * @level:              The compression level
+ * @estimated_src_size: The estimated source size to compress or 0
+ *                      if unknown.
+ * @dict_size:          Dictionary size.
+ *
+ * Return:              The selected zstd_compression_parameters.
+ */
+zstd_compression_parameters zstd_get_cparams(int level,
+	unsigned long long estimated_src_size, size_t dict_size);
 
 typedef ZSTD_CCtx zstd_cctx;
+typedef ZSTD_cParameter zstd_cparameter;
+
+/**
+ * zstd_cctx_set_param() - sets a compression parameter
+ * @cctx:         The context. Must have been initialized with zstd_init_cctx().
+ * @param:        The parameter to set.
+ * @value:        The value to set the parameter to.
+ *
+ * Return:        Zero or an error, which can be checked using zstd_is_error().
+ */
+size_t zstd_cctx_set_param(zstd_cctx *cctx, zstd_cparameter param, int value);
+
+/* ======   Single-pass Compression   ====== */
 
 /**
  * zstd_cctx_workspace_bound() - max memory needed to initialize a zstd_cctx
@@ -153,6 +200,20 @@ typedef ZSTD_CCtx zstd_cctx;
  */
 size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *parameters);
 
+/**
+ * zstd_cctx_workspace_bound_with_ext_seq_prod() - max memory needed to
+ * initialize a zstd_cctx when using the block-level external sequence
+ * producer API.
+ * @parameters: The compression parameters to be used.
+ *
+ * If multiple compression parameters might be used, the caller must call
+ * this function for each set of parameters and use the maximum size.
+ *
+ * Return:      A lower bound on the size of the workspace that is passed to
+ *              zstd_init_cctx().
+ */
+size_t zstd_cctx_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *parameters);
+
 /**
  * zstd_init_cctx() - initialize a zstd compression context
  * @workspace:      The workspace to emplace the context into. It must outlive
@@ -180,6 +241,71 @@ zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size);
 size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
 	const void *src, size_t src_size, const zstd_parameters *parameters);
 
+/**
+ * zstd_create_cctx_advanced() - Create compression context
+ * @custom_mem:   Custom allocator.
+ *
+ * Return:        NULL on error, pointer to compression context otherwise.
+ */
+zstd_cctx *zstd_create_cctx_advanced(zstd_custom_mem custom_mem);
+
+/**
+ * zstd_free_cctx() - Free compression context
+ * @cdict:        Pointer to compression context.
+ *
+ * Return:        Always 0.
+ */
+size_t zstd_free_cctx(zstd_cctx* cctx);
+
+/**
+ * struct zstd_cdict - Compression dictionary.
+ * See zstd_lib.h.
+ */
+typedef ZSTD_CDict zstd_cdict;
+
+/**
+ * zstd_create_cdict_byreference() - Create compression dictionary
+ * @dict:              Pointer to dictionary buffer.
+ * @dict_size:         Size of the dictionary buffer.
+ * @dict_load_method:  Dictionary load method.
+ * @dict_content_type: Dictionary content type.
+ * @custom_mem:        Memory allocator.
+ *
+ * Note, this uses @dict by reference (ZSTD_dlm_byRef), so it should be
+ * free before zstd_cdict is destroyed.
+ *
+ * Return:             NULL on error, pointer to compression dictionary
+ *                     otherwise.
+ */
+zstd_cdict *zstd_create_cdict_byreference(const void *dict, size_t dict_size,
+					  zstd_compression_parameters cparams,
+					  zstd_custom_mem custom_mem);
+
+/**
+ * zstd_free_cdict() - Free compression dictionary
+ * @cdict:        Pointer to compression dictionary.
+ *
+ * Return:        Always 0.
+ */
+size_t zstd_free_cdict(zstd_cdict* cdict);
+
+/**
+ * zstd_compress_using_cdict() - compress src into dst using a dictionary
+ * @cctx:         The context. Must have been initialized with zstd_init_cctx().
+ * @dst:          The buffer to compress src into.
+ * @dst_capacity: The size of the destination buffer. May be any size, but
+ *                ZSTD_compressBound(srcSize) is guaranteed to be large enough.
+ * @src:          The data to compress.
+ * @src_size:     The size of the data to compress.
+ * @cdict:        The dictionary to be used.
+ *
+ * Return:        The compressed size or an error, which can be checked using
+ *                zstd_is_error().
+ */
+size_t zstd_compress_using_cdict(zstd_cctx *cctx, void *dst,
+	size_t dst_capacity, const void *src, size_t src_size,
+	const zstd_cdict *cdict);
+
 /* ======   Single-pass Decompression   ====== */
 
 typedef ZSTD_DCtx zstd_dctx;
@@ -220,6 +346,71 @@ zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size);
 size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity,
 	const void *src, size_t src_size);
 
+/**
+ * struct zstd_ddict - Decompression dictionary.
+ * See zstd_lib.h.
+ */
+typedef ZSTD_DDict zstd_ddict;
+
+/**
+ * zstd_create_ddict_byreference() - Create decompression dictionary
+ * @dict:              Pointer to dictionary buffer.
+ * @dict_size:         Size of the dictionary buffer.
+ * @dict_load_method:  Dictionary load method.
+ * @dict_content_type: Dictionary content type.
+ * @custom_mem:        Memory allocator.
+ *
+ * Note, this uses @dict by reference (ZSTD_dlm_byRef), so it should be
+ * free before zstd_ddict is destroyed.
+ *
+ * Return:             NULL on error, pointer to decompression dictionary
+ *                     otherwise.
+ */
+zstd_ddict *zstd_create_ddict_byreference(const void *dict, size_t dict_size,
+					  zstd_custom_mem custom_mem);
+/**
+ * zstd_free_ddict() - Free decompression dictionary
+ * @dict:         Pointer to the dictionary.
+ *
+ * Return:        Always 0.
+ */
+size_t zstd_free_ddict(zstd_ddict *ddict);
+
+/**
+ * zstd_create_dctx_advanced() - Create decompression context
+ * @custom_mem:   Custom allocator.
+ *
+ * Return:        NULL on error, pointer to decompression context otherwise.
+ */
+zstd_dctx *zstd_create_dctx_advanced(zstd_custom_mem custom_mem);
+
+/**
+ * zstd_free_dctx() -- Free decompression context
+ * @dctx:         Pointer to decompression context.
+ * Return:        Always 0.
+ */
+size_t zstd_free_dctx(zstd_dctx *dctx);
+
+/**
+ * zstd_decompress_using_ddict() - decompress src into dst using a dictionary
+ * @dctx:         The decompression context.
+ * @dst:          The buffer to decompress src into.
+ * @dst_capacity: The size of the destination buffer. Must be at least as large
+ *                as the decompressed size. If the caller cannot upper bound the
+ *                decompressed size, then it's better to use the streaming API.
+ * @src:          The zstd compressed data to decompress. Multiple concatenated
+ *                frames and skippable frames are allowed.
+ * @src_size:     The exact size of the data to decompress.
+ * @ddict:        The dictionary to be used.
+ *
+ * Return:        The decompressed size or an error, which can be checked using
+ *                zstd_is_error().
+ */
+size_t zstd_decompress_using_ddict(zstd_dctx *dctx,
+	void *dst, size_t dst_capacity, const void *src, size_t src_size,
+	const zstd_ddict *ddict);
+
+
 /* ======   Streaming Buffers   ====== */
 
 /**
@@ -257,6 +448,16 @@ typedef ZSTD_CStream zstd_cstream;
  */
 size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams);
 
+/**
+ * zstd_cstream_workspace_bound_with_ext_seq_prod() - memory needed to initialize
+ * a zstd_cstream when using the block-level external sequence producer API.
+ * @cparams: The compression parameters to be used for compression.
+ *
+ * Return:   A lower bound on the size of the workspace that is passed to
+ *           zstd_init_cstream().
+ */
+size_t zstd_cstream_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *cparams);
+
 /**
  * zstd_init_cstream() - initialize a zstd streaming compression context
  * @parameters        The zstd parameters to use for compression.
@@ -416,6 +617,18 @@ size_t zstd_decompress_stream(zstd_dstream *dstream, zstd_out_buffer *output,
  */
 size_t zstd_find_frame_compressed_size(const void *src, size_t src_size);
 
+/**
+ * zstd_register_sequence_producer() - exposes the zstd library function
+ * ZSTD_registerSequenceProducer(). This is used for the block-level external
+ * sequence producer API. See upstream zstd.h for detailed documentation.
+ */
+typedef ZSTD_sequenceProducer_F zstd_sequence_producer_f;
+void zstd_register_sequence_producer(
+  zstd_cctx *cctx,
+  void* sequence_producer_state,
+  zstd_sequence_producer_f sequence_producer
+);
+
 /**
  * struct zstd_frame_params - zstd frame parameters stored in the frame header
  * @frameContentSize: The frame content size, or ZSTD_CONTENTSIZE_UNKNOWN if not
@@ -429,7 +642,7 @@ size_t zstd_find_frame_compressed_size(const void *src, size_t src_size);
  *
  * See zstd_lib.h.
  */
-typedef ZSTD_frameHeader zstd_frame_header;
+typedef ZSTD_FrameHeader zstd_frame_header;
 
 /**
  * zstd_get_frame_header() - extracts parameters from a zstd or skippable frame
@@ -444,4 +657,35 @@ typedef ZSTD_frameHeader zstd_frame_header;
 size_t zstd_get_frame_header(zstd_frame_header *params, const void *src,
 	size_t src_size);
 
+/**
+ * struct zstd_sequence - a sequence of literals or a match
+ *
+ * @offset: The offset of the match
+ * @litLength: The literal length of the sequence
+ * @matchLength: The match length of the sequence
+ * @rep: Represents which repeat offset is used
+ */
+typedef ZSTD_Sequence zstd_sequence;
+
+/**
+ * zstd_compress_sequences_and_literals() - compress an array of zstd_sequence and literals
+ *
+ * @cctx: The zstd compression context.
+ * @dst: The buffer to compress the data into.
+ * @dst_capacity: The size of the destination buffer.
+ * @in_seqs: The array of zstd_sequence to compress.
+ * @in_seqs_size: The number of sequences in in_seqs.
+ * @literals: The literals associated to the sequences to be compressed.
+ * @lit_size: The size of the literals in the literals buffer.
+ * @lit_capacity: The size of the literals buffer.
+ * @decompressed_size: The size of the input data
+ *
+ * Return: The compressed size or an error, which can be checked using
+ * 	   zstd_is_error().
+ */
+size_t zstd_compress_sequences_and_literals(zstd_cctx *cctx, void* dst, size_t dst_capacity,
+					    const zstd_sequence *in_seqs, size_t in_seqs_size,
+					    const void* literals, size_t lit_size, size_t lit_capacity,
+					    size_t decompressed_size);
+
 #endif  /* LINUX_ZSTD_H */

contrib/linux-kernel/mem.h

@@ -15,7 +15,7 @@
 /*-****************************************
 *  Dependencies
 ******************************************/
-#include <asm/unaligned.h>  /* get_unaligned, put_unaligned* */
+#include <linux/unaligned.h>  /* get_unaligned, put_unaligned* */
 #include <linux/compiler.h>  /* inline */
 #include <linux/swab.h>  /* swab32, swab64 */
 #include <linux/types.h>  /* size_t, ptrdiff_t */
@@ -24,6 +24,7 @@
 /*-****************************************
 *  Compiler specifics
 ******************************************/
+#undef MEM_STATIC /* may be already defined from common/compiler.h */
 #define MEM_STATIC static inline
 
 /*-**************************************************************

contrib/linux-kernel/test/include/linux/xxhash.h

@@ -296,7 +296,7 @@ XXH_API void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state
  * - xxHash source repository: https://github.com/Cyan4973/xxHash
  */
 
-#include <asm/unaligned.h>
+#include <linux/unaligned.h>
 #include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/module.h>

contrib/linux-kernel/zstd_compress_module.c

@@ -16,6 +16,7 @@
 
 #include "common/zstd_deps.h"
 #include "common/zstd_internal.h"
+#include "compress/zstd_compress_internal.h"
 
 #define ZSTD_FORWARD_IF_ERR(ret)            \
 	do {                                \
@@ -66,6 +67,12 @@ int zstd_max_clevel(void)
 }
 EXPORT_SYMBOL(zstd_max_clevel);
 
+int zstd_default_clevel(void)
+{
+	return ZSTD_defaultCLevel();
+}
+EXPORT_SYMBOL(zstd_default_clevel);
+
 size_t zstd_compress_bound(size_t src_size)
 {
 	return ZSTD_compressBound(src_size);
@@ -79,12 +86,71 @@ zstd_parameters zstd_get_params(int level,
 }
 EXPORT_SYMBOL(zstd_get_params);
 
+zstd_compression_parameters zstd_get_cparams(int level,
+	unsigned long long estimated_src_size, size_t dict_size)
+{
+	return ZSTD_getCParams(level, estimated_src_size, dict_size);
+}
+EXPORT_SYMBOL(zstd_get_cparams);
+
+size_t zstd_cctx_set_param(zstd_cctx *cctx, ZSTD_cParameter param, int value)
+{
+	return ZSTD_CCtx_setParameter(cctx, param, value);
+}
+EXPORT_SYMBOL(zstd_cctx_set_param);
+
 size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *cparams)
 {
 	return ZSTD_estimateCCtxSize_usingCParams(*cparams);
 }
 EXPORT_SYMBOL(zstd_cctx_workspace_bound);
 
+// Used by zstd_cctx_workspace_bound_with_ext_seq_prod()
+static size_t dummy_external_sequence_producer(
+	void *sequenceProducerState,
+	ZSTD_Sequence *outSeqs, size_t outSeqsCapacity,
+	const void *src, size_t srcSize,
+	const void *dict, size_t dictSize,
+	int compressionLevel,
+	size_t windowSize)
+{
+	(void)sequenceProducerState;
+	(void)outSeqs; (void)outSeqsCapacity;
+	(void)src; (void)srcSize;
+	(void)dict; (void)dictSize;
+	(void)compressionLevel;
+	(void)windowSize;
+	return ZSTD_SEQUENCE_PRODUCER_ERROR;
+}
+
+static void init_cctx_params_from_compress_params(
+	ZSTD_CCtx_params *cctx_params,
+	const zstd_compression_parameters *compress_params)
+{
+	ZSTD_parameters zstd_params;
+	memset(&zstd_params, 0, sizeof(zstd_params));
+	zstd_params.cParams = *compress_params;
+	ZSTD_CCtxParams_init_advanced(cctx_params, zstd_params);
+}
+
+size_t zstd_cctx_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *compress_params)
+{
+	ZSTD_CCtx_params cctx_params;
+	init_cctx_params_from_compress_params(&cctx_params, compress_params);
+	ZSTD_CCtxParams_registerSequenceProducer(&cctx_params, NULL, dummy_external_sequence_producer);
+	return ZSTD_estimateCCtxSize_usingCCtxParams(&cctx_params);
+}
+EXPORT_SYMBOL(zstd_cctx_workspace_bound_with_ext_seq_prod);
+
+size_t zstd_cstream_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *compress_params)
+{
+	ZSTD_CCtx_params cctx_params;
+	init_cctx_params_from_compress_params(&cctx_params, compress_params);
+	ZSTD_CCtxParams_registerSequenceProducer(&cctx_params, NULL, dummy_external_sequence_producer);
+	return ZSTD_estimateCStreamSize_usingCCtxParams(&cctx_params);
+}
+EXPORT_SYMBOL(zstd_cstream_workspace_bound_with_ext_seq_prod);
+
 zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size)
 {
 	if (workspace == NULL)
@@ -93,6 +159,33 @@ zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size)
 }
 EXPORT_SYMBOL(zstd_init_cctx);
 
+zstd_cctx *zstd_create_cctx_advanced(zstd_custom_mem custom_mem)
+{
+	return ZSTD_createCCtx_advanced(custom_mem);
+}
+EXPORT_SYMBOL(zstd_create_cctx_advanced);
+
+size_t zstd_free_cctx(zstd_cctx *cctx)
+{
+	return ZSTD_freeCCtx(cctx);
+}
+EXPORT_SYMBOL(zstd_free_cctx);
+
+zstd_cdict *zstd_create_cdict_byreference(const void *dict, size_t dict_size,
+					  zstd_compression_parameters cparams,
+					  zstd_custom_mem custom_mem)
+{
+	return ZSTD_createCDict_advanced(dict, dict_size, ZSTD_dlm_byRef,
+					 ZSTD_dct_auto, cparams, custom_mem);
+}
+EXPORT_SYMBOL(zstd_create_cdict_byreference);
+
+size_t zstd_free_cdict(zstd_cdict *cdict)
+{
+	return ZSTD_freeCDict(cdict);
+}
+EXPORT_SYMBOL(zstd_free_cdict);
+
 size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
 	const void *src, size_t src_size, const zstd_parameters *parameters)
 {
@@ -101,6 +194,15 @@ size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
 }
 EXPORT_SYMBOL(zstd_compress_cctx);
 
+size_t zstd_compress_using_cdict(zstd_cctx *cctx, void *dst,
+	size_t dst_capacity, const void *src, size_t src_size,
+	const ZSTD_CDict *cdict)
+{
+	return ZSTD_compress_usingCDict(cctx, dst, dst_capacity,
+					src, src_size, cdict);
+}
+EXPORT_SYMBOL(zstd_compress_using_cdict);
+
 size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams)
 {
 	return ZSTD_estimateCStreamSize_usingCParams(*cparams);
@@ -160,5 +262,25 @@ size_t zstd_end_stream(zstd_cstream *cstream, zstd_out_buffer *output)
 }
 EXPORT_SYMBOL(zstd_end_stream);
 
+void zstd_register_sequence_producer(
+  zstd_cctx *cctx,
+  void* sequence_producer_state,
+  zstd_sequence_producer_f sequence_producer
+) {
+	ZSTD_registerSequenceProducer(cctx, sequence_producer_state, sequence_producer);
+}
+EXPORT_SYMBOL(zstd_register_sequence_producer);
+
+size_t zstd_compress_sequences_and_literals(zstd_cctx *cctx, void* dst, size_t dst_capacity,
+					    const zstd_sequence *in_seqs, size_t in_seqs_size,
+					    const void* literals, size_t lit_size, size_t lit_capacity,
+					    size_t decompressed_size)
+{
+	return ZSTD_compressSequencesAndLiterals(cctx, dst, dst_capacity, in_seqs,
+						 in_seqs_size, literals, lit_size,
+						 lit_capacity, decompressed_size);
+}
+EXPORT_SYMBOL(zstd_compress_sequences_and_literals);
+
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DESCRIPTION("Zstd Compressor");

contrib/linux-kernel/zstd_decompress_module.c

@@ -44,6 +44,33 @@ size_t zstd_dctx_workspace_bound(void)
 }
 EXPORT_SYMBOL(zstd_dctx_workspace_bound);
 
+zstd_dctx *zstd_create_dctx_advanced(zstd_custom_mem custom_mem)
+{
+	return ZSTD_createDCtx_advanced(custom_mem);
+}
+EXPORT_SYMBOL(zstd_create_dctx_advanced);
+
+size_t zstd_free_dctx(zstd_dctx *dctx)
+{
+	return ZSTD_freeDCtx(dctx);
+}
+EXPORT_SYMBOL(zstd_free_dctx);
+
+zstd_ddict *zstd_create_ddict_byreference(const void *dict, size_t dict_size,
+					  zstd_custom_mem custom_mem)
+{
+	return ZSTD_createDDict_advanced(dict, dict_size, ZSTD_dlm_byRef,
+					 ZSTD_dct_auto, custom_mem);
+
+}
+EXPORT_SYMBOL(zstd_create_ddict_byreference);
+
+size_t zstd_free_ddict(zstd_ddict *ddict)
+{
+	return ZSTD_freeDDict(ddict);
+}
+EXPORT_SYMBOL(zstd_free_ddict);
+
 zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size)
 {
 	if (workspace == NULL)
@@ -59,6 +86,15 @@ size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity,
 }
 EXPORT_SYMBOL(zstd_decompress_dctx);
 
+size_t zstd_decompress_using_ddict(zstd_dctx *dctx,
+	void *dst, size_t dst_capacity, const void* src, size_t src_size,
+	const zstd_ddict* ddict)
+{
+	return ZSTD_decompress_usingDDict(dctx, dst, dst_capacity, src,
+					  src_size, ddict);
+}
+EXPORT_SYMBOL(zstd_decompress_using_ddict);
+
 size_t zstd_dstream_workspace_bound(size_t max_window_size)
 {
 	return ZSTD_estimateDStreamSize(max_window_size);
@@ -77,7 +113,7 @@ EXPORT_SYMBOL(zstd_init_dstream);
 
 size_t zstd_reset_dstream(zstd_dstream *dstream)
 {
-	return ZSTD_resetDStream(dstream);
+	return ZSTD_DCtx_reset(dstream, ZSTD_reset_session_only);
 }
 EXPORT_SYMBOL(zstd_reset_dstream);
 

contrib/linux-kernel/zstd_deps.h

@@ -115,11 +115,7 @@ static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) {
 #ifndef ZSTD_DEPS_STDINT
 #define ZSTD_DEPS_STDINT
 
-/*
- * The Linux Kernel doesn't provide intptr_t, only uintptr_t, which
- * is an unsigned long.
- */
-typedef long intptr_t;
+/* intptr_t already provided by ZSTD_DEPS_COMMON */
 
 #endif /* ZSTD_DEPS_STDINT */
 #endif /* ZSTD_DEPS_NEED_STDINT */

contrib/premake/zstd.lua

@@ -2,6 +2,7 @@
 -- Basic usage: project_zstd(ZSTD_DIR)
 
 function project_zstd(dir, compression, decompression, deprecated, dictbuilder, legacy)
+	if string.sub(dir, -1, 1) ~= '/' then dir = dir .. '/' end
 	if compression == nil then compression = true end
 	if decompression == nil then decompression = true end
 	if deprecated == nil then deprecated = false end

contrib/pzstd/Logging.h

@@ -10,6 +10,7 @@
 
 #include <cstdio>
 #include <mutex>
+#include <chrono>
 
 namespace pzstd {
 

contrib/pzstd/Makefile

@@ -10,7 +10,7 @@
 # Standard variables for installation
 DESTDIR ?=
 PREFIX  ?= /usr/local
-BINDIR  := $(DESTDIR)$(PREFIX)/bin
+BINDIR  := $(PREFIX)/bin
 
 ZSTDDIR = ../../lib
 PROGDIR = ../../programs
@@ -37,11 +37,8 @@ CFLAGS   += -Wno-deprecated-declarations
 PZSTD_INC  = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
 GTEST_INC  = -isystem googletest/googletest/include
 
-# If default C++ version is older than C++11, explicitly set C++11, which is the
-# minimum required by the code.
-ifeq ($(shell echo "\043if __cplusplus < 201103L\n\043error\n\043endif" | $(CXX) -x c++ -Werror -c - -o /dev/null 2>/dev/null && echo 1 || echo 0),0)
-PZSTD_CXX_STD := -std=c++11
-endif
+# Set the minimum required by gtest
+PZSTD_CXX_STD := -std=c++14
 
 PZSTD_CPPFLAGS  = $(PZSTD_INC)
 PZSTD_CCXXFLAGS =
@@ -112,12 +109,12 @@ check:
 .PHONY: install
 install: PZSTD_CPPFLAGS += -DNDEBUG
 install: pzstd$(EXT)
-	install -d -m 755 $(BINDIR)/
-	install -m 755 pzstd$(EXT) $(BINDIR)/pzstd$(EXT)
+	install -d -m 755 $(DESTDIR)$(BINDIR)/
+	install -m 755 pzstd$(EXT) $(DESTDIR)$(BINDIR)/pzstd$(EXT)
 
 .PHONY: uninstall
 uninstall:
-	$(RM) $(BINDIR)/pzstd$(EXT)
+	$(RM) $(DESTDIR)$(BINDIR)/pzstd$(EXT)
 
 # Targets for many different builds
 .PHONY: all

contrib/pzstd/Options.cpp

@@ -322,7 +322,7 @@ Options::Status Options::parse(int argc, const char **argv) {
   g_utilDisplayLevel = verbosity;
   // Remove local input files that are symbolic links
   if (!followLinks) {
-      std::remove_if(localInputFiles.begin(), localInputFiles.end(),
+	  localInputFiles.erase(std::remove_if(localInputFiles.begin(), localInputFiles.end(),
                      [&](const char *path) {
                         bool isLink = UTIL_isLink(path);
                         if (isLink && verbosity >= 2) {
@@ -332,7 +332,7 @@ Options::Status Options::parse(int argc, const char **argv) {
                                     path);
                         }
                         return isLink;
-                    });
+					 }), localInputFiles.end());
   }
 
   // Translate input files/directories into files to (de)compress

contrib/pzstd/Pzstd.cpp

@@ -269,7 +269,10 @@ static void compress(
     std::shared_ptr<BufferWorkQueue> out,
     size_t maxInputSize) {
   auto& errorHolder = state.errorHolder;
-  auto guard = makeScopeGuard([&] { out->finish(); });
+  auto guard = makeScopeGuard([&] {
+    in->finish();
+    out->finish();
+  });
   // Initialize the CCtx
   auto ctx = state.cStreamPool->get();
   if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_CStream")) {
@@ -431,7 +434,10 @@ static void decompress(
     std::shared_ptr<BufferWorkQueue> in,
     std::shared_ptr<BufferWorkQueue> out) {
   auto& errorHolder = state.errorHolder;
-  auto guard = makeScopeGuard([&] { out->finish(); });
+  auto guard = makeScopeGuard([&] {
+    in->finish();
+    out->finish();
+  });
   // Initialize the DCtx
   auto ctx = state.dStreamPool->get();
   if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_DStream")) {
@@ -578,6 +584,7 @@ std::uint64_t writeFile(
     FILE* outputFd,
     bool decompress) {
   auto& errorHolder = state.errorHolder;
+  auto outsFinishGuard = makeScopeGuard([&outs] { outs.finish(); });
   auto lineClearGuard = makeScopeGuard([&state] {
     state.log.clear(kLogInfo);
   });
@@ -585,6 +592,7 @@ std::uint64_t writeFile(
   std::shared_ptr<BufferWorkQueue> out;
   // Grab the output queue for each decompression job (in order).
   while (outs.pop(out)) {
+    auto outFinishGuard = makeScopeGuard([&out] { out->finish(); });
     if (errorHolder.hasError()) {
       continue;
     }

contrib/pzstd/test/PzstdTest.cpp

@@ -7,9 +7,7 @@
  * in the COPYING file in the root directory of this source tree).
  */
 #include "Pzstd.h"
-extern "C" {
 #include "datagen.h"
-}
 #include "test/RoundTrip.h"
 #include "utils/ScopeGuard.h"
 

contrib/pzstd/test/RoundTripTest.cpp

@@ -6,9 +6,7 @@
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  * in the COPYING file in the root directory of this source tree).
  */
-extern "C" {
 #include "datagen.h"
-}
 #include "Options.h"
 #include "test/RoundTrip.h"
 #include "utils/ScopeGuard.h"

contrib/pzstd/utils/WorkQueue.h

@@ -115,13 +115,14 @@ class WorkQueue {
   }
 
   /**
-   * Promise that `push()` won't be called again, so once the queue is empty
-   * there will never any more work.
+   * Promise that either the reader side or the writer side is done.
+   * If the writer is done, `push()` won't be called again, so once the queue
+   * is empty there will never be any more work. If the reader is done, `pop()`
+   * won't be called again, so further items pushed will just be ignored.
    */
   void finish() {
     {
       std::lock_guard<std::mutex> lock(mutex_);
-      assert(!done_);
       done_ = true;
     }
     readerCv_.notify_all();

contrib/seekable_format/examples/Makefile

@@ -11,6 +11,8 @@
 
 ZSTDLIB_PATH = ../../../lib
 ZSTDLIB_NAME = libzstd.a
+# Parallel tools only make sense against multi-threaded libzstd
+ZSTDLIB_TARGET = $(ZSTDLIB_NAME)-mt
 ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
 
 CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I../../../lib -I../../../lib/common
@@ -28,7 +30,7 @@ all: seekable_compression seekable_decompression seekable_decompression_mem \
 	parallel_processing
 
 $(ZSTDLIB):
-	make -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME)
+	$(MAKE) -C $(ZSTDLIB_PATH) $(ZSTDLIB_TARGET)
 
 seekable_compression : seekable_compression.c $(SEEKABLE_OBJS)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@

contrib/seekable_format/examples/parallel_compression.c

@@ -23,6 +23,8 @@
 
 #include "xxhash.h"
 
+#define ZSTD_MULTITHREAD 1
+#include "threading.h"
 #include "pool.h"      // use zstd thread pool for demo
 
 #include "../zstd_seekable.h"
@@ -72,114 +74,87 @@ static size_t fclose_orDie(FILE* file)
     exit(6);
 }
 
-static void fseek_orDie(FILE* file, long int offset, int origin)
-{
-    if (!fseek(file, offset, origin)) {
-        if (!fflush(file)) return;
-    }
-    /* error */
-    perror("fseek");
-    exit(7);
-}
-
-static long int ftell_orDie(FILE* file)
-{
-    long int off = ftell(file);
-    if (off != -1) return off;
-    /* error */
-    perror("ftell");
-    exit(8);
-}
+struct state {
+    FILE* fout;
+    ZSTD_pthread_mutex_t mutex;
+    size_t nextID;
+    struct job* pending;
+    ZSTD_frameLog* frameLog;
+    const int compressionLevel;
+};
 
 struct job {
-    const void* src;
+    size_t id;
+    struct job* next;
+    struct state* state;
+
+    void* src;
     size_t srcSize;
     void* dst;
     size_t dstSize;
 
     unsigned checksum;
-
-    int compressionLevel;
-    int done;
 };
 
+static void addPending_inmutex(struct state* state, struct job* job)
+{
+    struct job** p = &state->pending;
+    while (*p && (*p)->id < job->id)
+        p = &(*p)->next;
+    job->next = *p;
+    *p = job;
+}
+
+static void flushFrame(struct state* state, struct job* job)
+{
+    fwrite_orDie(job->dst, job->dstSize, state->fout);
+    free(job->dst);
+
+    size_t ret = ZSTD_seekable_logFrame(state->frameLog, (unsigned)job->dstSize, (unsigned)job->srcSize, job->checksum);
+    if (ZSTD_isError(ret)) {
+        fprintf(stderr, "ZSTD_seekable_logFrame() error : %s \n", ZSTD_getErrorName(ret));
+        exit(12);
+    }
+}
+
+static void flushPending_inmutex(struct state* state)
+{
+    while (state->pending && state->pending->id == state->nextID) {
+        struct job* p = state->pending;
+        state->pending = p->next;
+        flushFrame(state, p);
+        free(p);
+        state->nextID++;
+    }
+}
+
+static void finishFrame(struct job* job)
+{
+    struct state *state = job->state;
+    ZSTD_pthread_mutex_lock(&state->mutex);
+    addPending_inmutex(state, job);
+    flushPending_inmutex(state);
+    ZSTD_pthread_mutex_unlock(&state->mutex);
+}
+
 static void compressFrame(void* opaque)
 {
     struct job* job = opaque;
 
-    job->checksum = XXH64(job->src, job->srcSize, 0);
+    job->checksum = (unsigned)XXH64(job->src, job->srcSize, 0);
 
-    size_t ret = ZSTD_compress(job->dst, job->dstSize, job->src, job->srcSize, job->compressionLevel);
+    size_t ret = ZSTD_compress(job->dst, job->dstSize, job->src, job->srcSize, job->state->compressionLevel);
     if (ZSTD_isError(ret)) {
         fprintf(stderr, "ZSTD_compress() error : %s \n", ZSTD_getErrorName(ret));
         exit(20);
     }
-
     job->dstSize = ret;
-    job->done = 1;
-}
 
-static void compressFile_orDie(const char* fname, const char* outName, int cLevel, unsigned frameSize, int nbThreads)
-{
-    POOL_ctx* pool = POOL_create(nbThreads, nbThreads);
-    if (pool == NULL) { fprintf(stderr, "POOL_create() error \n"); exit(9); }
+    // No longer need
+    free(job->src);
+    job->src = NULL;
 
-    FILE* const fin  = fopen_orDie(fname, "rb");
-    FILE* const fout = fopen_orDie(outName, "wb");
-
-    if (ZSTD_compressBound(frameSize) > 0xFFFFFFFFU) { fprintf(stderr, "Frame size too large \n"); exit(10); }
-    unsigned dstSize = ZSTD_compressBound(frameSize);
-
-
-    fseek_orDie(fin, 0, SEEK_END);
-    long int length = ftell_orDie(fin);
-    fseek_orDie(fin, 0, SEEK_SET);
-
-    size_t numFrames = (length + frameSize - 1) / frameSize;
-
-    struct job* jobs = malloc_orDie(sizeof(struct job) * numFrames);
-
-    size_t i;
-    for(i = 0; i < numFrames; i++) {
-        void* in = malloc_orDie(frameSize);
-        void* out = malloc_orDie(dstSize);
-
-        size_t inSize = fread_orDie(in, frameSize, fin);
-
-        jobs[i].src = in;
-        jobs[i].srcSize = inSize;
-        jobs[i].dst = out;
-        jobs[i].dstSize = dstSize;
-        jobs[i].compressionLevel = cLevel;
-        jobs[i].done = 0;
-        POOL_add(pool, compressFrame, &jobs[i]);
-    }
-
-    ZSTD_frameLog* fl = ZSTD_seekable_createFrameLog(1);
-    if (fl == NULL) { fprintf(stderr, "ZSTD_seekable_createFrameLog() failed \n"); exit(11); }
-    for (i = 0; i < numFrames; i++) {
-        while (!jobs[i].done) SLEEP(5); /* wake up every 5 milliseconds to check */
-        fwrite_orDie(jobs[i].dst, jobs[i].dstSize, fout);
-        free((void*)jobs[i].src);
-        free(jobs[i].dst);
-
-        size_t ret = ZSTD_seekable_logFrame(fl, jobs[i].dstSize, jobs[i].srcSize, jobs[i].checksum);
-        if (ZSTD_isError(ret)) { fprintf(stderr, "ZSTD_seekable_logFrame() error : %s \n", ZSTD_getErrorName(ret)); }
-    }
-
-    {   unsigned char seekTableBuff[1024];
-        ZSTD_outBuffer out = {seekTableBuff, 1024, 0};
-        while (ZSTD_seekable_writeSeekTable(fl, &out) != 0) {
-            fwrite_orDie(seekTableBuff, out.pos, fout);
-            out.pos = 0;
-        }
-        fwrite_orDie(seekTableBuff, out.pos, fout);
-    }
-
-    ZSTD_seekable_freeFrameLog(fl);
-    free(jobs);
-    fclose_orDie(fout);
-    fclose_orDie(fin);
+    finishFrame(job);
 }
 
 static const char* createOutFilename_orDie(const char* filename)
@@ -193,6 +168,72 @@ static const char* createOutFilename_orDie(const char* filename)
     return (const char*)outSpace;
 }
 
+static void openInOut_orDie(const char* fname, FILE** fin, FILE** fout) {
+    if (strcmp(fname, "-") == 0) {
+        *fin = stdin;
+        *fout = stdout;
+    } else {
+        *fin = fopen_orDie(fname, "rb");
+        const char* outName = createOutFilename_orDie(fname);
+        *fout = fopen_orDie(outName, "wb");
+    }
+}
+
+static void compressFile_orDie(const char* fname, int cLevel, unsigned frameSize, size_t nbThreads)
+{
+    struct state state = {
+        .nextID = 0,
+        .pending = NULL,
+        .compressionLevel = cLevel,
+    };
+    ZSTD_pthread_mutex_init(&state.mutex, NULL);
+    state.frameLog = ZSTD_seekable_createFrameLog(1);
+    if (state.frameLog == NULL) { fprintf(stderr, "ZSTD_seekable_createFrameLog() failed \n"); exit(11); }
+
+    POOL_ctx* pool = POOL_create(nbThreads, nbThreads);
+    if (pool == NULL) { fprintf(stderr, "POOL_create() error \n"); exit(9); }
+
+    FILE* fin;
+    openInOut_orDie(fname, &fin, &state.fout);
+
+    if (ZSTD_compressBound(frameSize) > 0xFFFFFFFFU) { fprintf(stderr, "Frame size too large \n"); exit(10); }
+    size_t dstSize = ZSTD_compressBound(frameSize);
+
+    for (size_t id = 0; 1; id++) {
+        struct job* job = malloc_orDie(sizeof(struct job));
+        job->id = id;
+        job->next = NULL;
+        job->state = &state;
+        job->src = malloc_orDie(frameSize);
+        job->dst = malloc_orDie(dstSize);
+        job->srcSize = fread_orDie(job->src, frameSize, fin);
+        job->dstSize = dstSize; 
+        POOL_add(pool, compressFrame, job);
+        if (feof(fin))
+            break;
+    }
+
+    POOL_joinJobs(pool);
+    POOL_free(pool);
+    if (state.pending) {
+        fprintf(stderr, "Unexpected leftover output blocks!\n");
+        exit(13);
+    }
+
+    {   unsigned char seekTableBuff[1024];
+        ZSTD_outBuffer out = {seekTableBuff, 1024, 0};
+        while (ZSTD_seekable_writeSeekTable(state.frameLog, &out) != 0) {
+            fwrite_orDie(seekTableBuff, out.pos, state.fout);
+            out.pos = 0;
+        }
+        fwrite_orDie(seekTableBuff, out.pos, state.fout);
+    }
+
+    ZSTD_seekable_freeFrameLog(state.frameLog);
+    fclose_orDie(state.fout);
+    fclose_orDie(fin);
+}
+
 int main(int argc, const char** argv) {
     const char* const exeName = argv[0];
     if (argc!=4) {
@@ -204,10 +245,9 @@ int main(int argc, const char** argv) {
 
     {   const char* const inFileName = argv[1];
         unsigned const frameSize = (unsigned)atoi(argv[2]);
-        int const nbThreads = atoi(argv[3]);
+        size_t const nbThreads = (size_t)atoi(argv[3]);
 
-        const char* const outFileName = createOutFilename_orDie(inFileName);
-        compressFile_orDie(inFileName, outFileName, 5, frameSize, nbThreads);
+        compressFile_orDie(inFileName, 5, frameSize, nbThreads);
     }
 
     return 0;

contrib/seekable_format/examples/parallel_processing.c

@@ -19,7 +19,7 @@
 #define ZSTD_STATIC_LINKING_ONLY
 #include <zstd.h>      // presumes zstd library is installed
 #include <zstd_errors.h>
-#if defined(WIN32) || defined(_WIN32)
+#if defined(_WIN32)
 #  include <windows.h>
 #  define SLEEP(x) Sleep(x)
 #else
@@ -100,13 +100,11 @@ struct sum_job {
     const char* fname;
     unsigned long long sum;
     unsigned frameNb;
-    int done;
 };
 
 static void sumFrame(void* opaque)
 {
     struct sum_job* job = (struct sum_job*)opaque;
-    job->done = 0;
 
     FILE* const fin = fopen_orDie(job->fname, "rb");
 
@@ -128,7 +126,6 @@ static void sumFrame(void* opaque)
         sum += data[i];
     }
     job->sum = sum;
-    job->done = 1;
 
     fclose(fin);
     ZSTD_seekable_free(seekable);
@@ -153,14 +150,14 @@ static void sumFile_orDie(const char* fname, int nbThreads)
 
     unsigned fnb;
     for (fnb = 0; fnb < numFrames; fnb++) {
-        jobs[fnb] = (struct sum_job){ fname, 0, fnb, 0 };
+        jobs[fnb] = (struct sum_job){ fname, 0, fnb };
         POOL_add(pool, sumFrame, &jobs[fnb]);
     }
+    POOL_joinJobs(pool);
 
     unsigned long long total = 0;
 
     for (fnb = 0; fnb < numFrames; fnb++) {
-        while (!jobs[fnb].done) SLEEP(5); /* wake up every 5 milliseconds to check */
         total += jobs[fnb].sum;
     }
 

contrib/seekable_format/tests/.gitignore

@@ -1 +1,2 @@
 seekable_tests
+data.txt

contrib/seekable_format/tests/Makefile

@@ -24,7 +24,7 @@ SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)
 .PHONY: default clean test
 default: test
 
-test: seekable_tests
+test: seekable_tests parallel_compression_test
 	./seekable_tests
 
 $(ZSTDLIB):
@@ -32,7 +32,27 @@ $(ZSTDLIB):
 
 seekable_tests : $(SEEKABLE_OBJS)
 
+EXAMPLES_PATH = ../examples
+PARALLEL_COMPRESSION = $(EXAMPLES_PATH)/parallel_compression
+
+DATAGEN_PATH = ../../../tests
+DATAGEN = $(DATAGEN_PATH)/datagen
+
+$(PARALLEL_COMPRESSION):
+	$(MAKE) -C $(EXAMPLES_PATH) parallel_compression
+
+$(DATAGEN):
+	$(MAKE) -C $(DATAGEN_PATH) datagen
+
+data.txt: $(DATAGEN)
+	$(DATAGEN) -g100M > $@
+
+parallel_compression_test: $(PARALLEL_COMPRESSION) data.txt
+	ulimit -Sv 102400; $(PARALLEL_COMPRESSION) data.txt 1048576 2
+
+.PHONY: parallel_compression_test parallel_comp
+
 clean:
 	@$(RM) core *.o tmp* result* *.zst \
-		seekable_tests
+		seekable_tests data.txt
 	@echo Cleaning completed

contrib/seekable_format/tests/seekable_tests.c

@@ -316,6 +316,10 @@ int main(int argc, const char** argv)
         for (pos = 0; pos < inSize; pos += 2) {
             size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, 1, pos);
             if (decStatus != 1 || outBuffer[0] != inBuffer[pos]) {
+                free(seekBuffer);
+                free(outBuffer);
+                ZSTD_seekable_freeCStream(zscs);
+                ZSTD_seekable_free(stream);
                 goto _test_error;
             }
         }
@@ -323,6 +327,10 @@ int main(int argc, const char** argv)
             /* We read more than the compressed size, meaning there were some rereads.
                This is unneeded because we only seeked forward. */
             printf("Too much data read: %zu read, with compressed size %zu\n", buffWrapper.totalRead, seekSize);
+            free(seekBuffer);
+            free(outBuffer);
+            ZSTD_seekable_freeCStream(zscs);
+            ZSTD_seekable_free(stream);
             goto _test_error;
         }
 
@@ -342,6 +350,10 @@ int main(int argc, const char** argv)
         for (idx = 0; idx < sizeof(tests) / sizeof(tests[0]); idx++) {
             size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, tests[idx].size, tests[idx].offset);
             if (decStatus != tests[idx].size || memcmp(outBuffer, inBuffer + tests[idx].offset, tests[idx].size) != 0) {
+                free(seekBuffer);
+                free(outBuffer);
+                ZSTD_seekable_freeCStream(zscs);
+                ZSTD_seekable_free(stream);
                 goto _test_error;
             }
         }

contrib/seekable_format/zstd_seekable.h

@@ -1,13 +1,13 @@
 #ifndef SEEKABLE_H
 #define SEEKABLE_H
 
+#include <stdio.h>
+#include "zstd.h"   /* ZSTDLIB_API */
+
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
-#include <stdio.h>
-#include "zstd.h"   /* ZSTDLIB_API */
-
 
 #define ZSTD_seekTableFooterSize 9
 

contrib/seekable_format/zstdseek_decompress.c

@@ -77,7 +77,10 @@
 /* ************************************************************
 * Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
 ***************************************************************/
-#if defined(_MSC_VER) && _MSC_VER >= 1400
+#if defined(LIBC_NO_FSEEKO)
+/* Some older libc implementations don't include these functions (e.g. Bionic < 24) */
+#   define LONG_SEEK fseek
+#elif defined(_MSC_VER) && _MSC_VER >= 1400
 #   define LONG_SEEK _fseeki64
 #elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
 #   define LONG_SEEK fseeko
@@ -252,6 +255,8 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
 
 ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs)
 {
+    assert(zs != NULL);
+    if (zs->seekTable.entries == NULL) return NULL;
     ZSTD_seekTable* const st = (ZSTD_seekTable*)malloc(sizeof(ZSTD_seekTable));
     if (st==NULL) return NULL;
 

doc/decompressor_errata.md

@@ -6,12 +6,53 @@ Each entry will contain:
 1. The last affected decompressor versions.
 2. The decompressor components affected.
 2. Whether the compressed frame could ever be produced by the reference compressor.
-3. An example frame.
+3. An example frame (hexadecimal string when it can be short enough, link to golden file otherwise)
 4. A description of the bug.
 
 The document is in reverse chronological order, with the bugs that affect the most recent zstd decompressor versions listed first.
 
 
+No sequence using the 2-bytes format
+------------------------------------------------
+
+**Last affected version**: v1.5.5
+
+**Affected decompressor component(s)**: Library & CLI
+
+**Produced by the reference compressor**: No
+
+**Example Frame**: see zstd/tests/golden-decompression/zeroSeq_2B.zst
+
+The zstd decoder incorrectly expects FSE tables when there are 0 sequences present in the block
+if the value 0 is encoded using the 2-bytes format.
+Instead, it should immediately end the sequence section, and move on to next block.
+
+This situation was never generated by the reference compressor,
+because representing 0 sequences with the 2-bytes format is inefficient
+(the 1-byte format is always used in this case).
+
+
+Compressed block with a size of exactly 128 KB
+------------------------------------------------
+
+**Last affected version**: v1.5.2
+
+**Affected decompressor component(s)**: Library & CLI
+
+**Produced by the reference compressor**: No
+
+**Example Frame**: see zstd/tests/golden-decompression/block-128k.zst
+
+The zstd decoder incorrectly rejected blocks of type `Compressed_Block` when their size was exactly 128 KB.
+Note that `128 KB - 1` was accepted, and `128 KB + 1` is forbidden by the spec.
+
+This type of block was never generated by the reference compressor.
+
+These blocks used to be disallowed by the spec up until spec version 0.3.2 when the restriction was lifted by [PR#1689](https://github.com/facebook/zstd/pull/1689).
+
+> A Compressed_Block has the extra restriction that Block_Size is always strictly less than the decompressed size. If this condition cannot be respected, the block must be sent uncompressed instead (Raw_Block).
+
+
 Compressed block with 0 literals and 0 sequences
 ------------------------------------------------
 
@@ -31,6 +72,7 @@ Additionally, these blocks were disallowed by the spec up until spec version 0.3
 
 > A Compressed_Block has the extra restriction that Block_Size is always strictly less than the decompressed size. If this condition cannot be respected, the block must be sent uncompressed instead (Raw_Block).
 
+
 First block is RLE block
 ------------------------
 
@@ -52,6 +94,7 @@ block.
 
 https://github.com/facebook/zstd/blob/8814aa5bfa74f05a86e55e9d508da177a893ceeb/lib/compress/zstd_compress.c#L3527-L3535
 
+
 Tiny FSE Table & Block
 ----------------------
 
@@ -82,3 +125,24 @@ The total `Block_Content` is `5` bytes, and `Last_Table_Offset` is `2`.
 See the compressor workaround code:
 
 https://github.com/facebook/zstd/blob/8814aa5bfa74f05a86e55e9d508da177a893ceeb/lib/compress/zstd_compress.c#L2667-L2682
+
+Magicless format
+----------------------
+
+**Last affected version**: v1.5.5
+
+**Affected decompressor component(s)**: Library
+
+**Produced by the reference compressor**: Yes (example: https://gist.github.com/embg/9940726094f4cf2cef162cffe9319232)
+
+**Example Frame**: `27 b5 2f fd 00 03 19 00 00 66 6f 6f 3f ba c4 59`
+
+v1.5.6 fixes several bugs in which the magicless-format decoder rejects valid frames.
+These include but are not limited to:
+* Valid frames that happen to begin with a legacy magic number (little-endian)
+* Valid frames that happen to begin with a skippable magic number (little-endian)
+
+If you are affected by this issue and cannot update to v1.5.6 or later, there is a
+workaround to recover affected data. Simply prepend the ZSTD magic number
+`0xFD2FB528` (little-endian) to your data and decompress using the standard-format
+decoder.

doc/decompressor_permissive.md

@@ -0,0 +1,80 @@
+Decompressor Permissiveness to Invalid Data
+===========================================
+
+This document describes the behavior of the reference decompressor in cases
+where it accepts formally invalid data instead of reporting an error.
+
+While the reference decompressor *must* decode any compliant frame following
+the specification, its ability to detect erroneous data is on a best effort
+basis: the decoder may accept input data that would be formally invalid,
+when it causes no risk to the decoder, and which detection would cost too much
+complexity or speed regression.
+
+In practice, the vast majority of invalid data are detected, if only because
+many corruption events are dangerous for the decoder process (such as
+requesting an out-of-bound memory access) and many more are easy to check.
+
+This document lists a few known cases where invalid data was formerly accepted
+by the decoder, and what has changed since.
+
+
+Truncated Huffman states
+------------------------
+
+**Last affected version**: v1.5.6
+
+**Produced by the reference compressor**: No
+
+**Example Frame**: `28b5 2ffd 0000 5500 0072 8001 0420 7e1f 02aa 00`
+
+When using FSE-compressed Huffman weights, the compressed weight bitstream
+could contain fewer bits than necessary to decode the initial states.
+
+The reference decompressor up to v1.5.6 will decode truncated or missing
+initial states as zero, which can result in a valid Huffman tree if only
+the second state is truncated.
+
+In newer versions, truncated initial states are reported as a corruption
+error by the decoder.
+
+
+Offset == 0
+-----------
+
+**Last affected version**: v1.5.5
+
+**Produced by the reference compressor**: No
+
+**Example Frame**: `28b5 2ffd 0000 4500 0008 0002 002f 430b ae`
+
+If a sequence is decoded with `literals_length = 0` and `offset_value = 3`
+while `Repeated_Offset_1 = 1`, the computed offset will be `0`, which is
+invalid.
+
+The reference decompressor up to v1.5.5 processes this case as if the computed
+offset was `1`, including inserting `1` into the repeated offset list.
+This prevents the output buffer from remaining uninitialized, thus denying a
+potential attack vector from an untrusted source.
+However, in the rare case where this scenario would be the outcome of a
+transmission or storage error, the decoder relies on the checksum to detect
+the error.
+
+In newer versions, this case is always detected and reported as a corruption error.
+
+
+Non-zeroes reserved bits
+------------------------
+
+**Last affected version**: v1.5.5
+
+**Produced by the reference compressor**: No
+
+The Sequences section of each block has a header, and one of its elements is a
+byte, which describes the compression mode of each symbol.
+This byte contains 2 reserved bits which must be set to zero.
+
+The reference decompressor up to v1.5.5 just ignores these 2 bits.
+This behavior has no consequence for the rest of the frame decoding process.
+
+In newer versions, the 2 reserved bits are actively checked for value zero,
+and the decoder reports a corruption error if they are not.

doc/educational_decoder/Makefile

@@ -10,7 +10,7 @@
 
 ZSTD ?= zstd   # note: requires zstd installation on local system
 
-UNAME?= $(shell uname)
+UNAME?= $(shell sh -c 'MSYSTEM="MSYS" uname') 
 ifeq ($(UNAME), SunOS)
 DIFF ?= gdiff
 else

doc/educational_decoder/zstd_decompress.c

@@ -997,7 +997,8 @@ static void decompress_sequences(frame_context_t *const ctx,
                                  const size_t num_sequences);
 static sequence_command_t decode_sequence(sequence_states_t *const state,
                                           const u8 *const src,
-                                          i64 *const offset);
+                                          i64 *const offset,
+                                          int lastSequence);
 static void decode_seq_table(FSE_dtable *const table, istream_t *const in,
                                const seq_part_t type, const seq_mode_t mode);
 
@@ -1017,12 +1018,7 @@ static size_t decode_sequences(frame_context_t *const ctx, istream_t *in,
     // This is a variable size field using between 1 and 3 bytes. Let's call its
     // first byte byte0."
     u8 header = IO_read_bits(in, 8);
-    if (header == 0) {
-        // "There are no sequences. The sequence section stops there.
-        // Regenerated content is defined entirely by literals section."
-        *sequences = NULL;
-        return 0;
-    } else if (header < 128) {
+    if (header < 128) {
         // "Number_of_Sequences = byte0 . Uses 1 byte."
         num_sequences = header;
     } else if (header < 255) {
@@ -1033,6 +1029,12 @@ static size_t decode_sequences(frame_context_t *const ctx, istream_t *in,
         num_sequences = IO_read_bits(in, 16) + 0x7F00;
     }
 
+    if (num_sequences == 0) {
+        // "There are no sequences. The sequence section stops there."
+        *sequences = NULL;
+        return 0;
+    }
+
     *sequences = malloc(num_sequences * sizeof(sequence_command_t));
     if (!*sequences) {
         BAD_ALLOC();
@@ -1114,7 +1116,7 @@ static void decompress_sequences(frame_context_t *const ctx, istream_t *in,
 
     for (size_t i = 0; i < num_sequences; i++) {
         // Decode sequences one by one
-        sequences[i] = decode_sequence(&states, src, &bit_offset);
+        sequences[i] = decode_sequence(&states, src, &bit_offset, i==num_sequences-1);
     }
 
     if (bit_offset != 0) {
@@ -1125,7 +1127,8 @@ static void decompress_sequences(frame_context_t *const ctx, istream_t *in,
 // Decode a single sequence and update the state
 static sequence_command_t decode_sequence(sequence_states_t *const states,
                                           const u8 *const src,
-                                          i64 *const offset) {
+                                          i64 *const offset,
+                                          int lastSequence) {
     // "Each symbol is a code in its own context, which specifies Baseline and
     // Number_of_Bits to add. Codes are FSE compressed, and interleaved with raw
     // additional bits in the same bitstream."
@@ -1160,7 +1163,7 @@ static sequence_command_t decode_sequence(sequence_states_t *const states,
     // Literals_Length_State is updated, followed by Match_Length_State, and
     // then Offset_State."
     // If the stream is complete don't read bits to update state
-    if (*offset != 0) {
+    if (!lastSequence) {
         FSE_update_state(&states->ll_table, &states->ll_state, src, offset);
         FSE_update_state(&states->ml_table, &states->ml_state, src, offset);
         FSE_update_state(&states->of_table, &states->of_state, src, offset);
@@ -1210,7 +1213,7 @@ static void decode_seq_table(FSE_dtable *const table, istream_t *const in,
         break;
     }
     case seq_repeat:
-        // "Repeat_Mode : re-use distribution table from previous compressed
+        // "Repeat_Mode : reuse distribution table from previous compressed
         // block."
         // Nothing to do here, table will be unchanged
         if (!table->symbols) {
@@ -1399,7 +1402,7 @@ size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) {
 /******* END OUTPUT SIZE COUNTING *********************************************/
 
 /******* DICTIONARY PARSING ***************************************************/
-dictionary_t* create_dictionary() {
+dictionary_t* create_dictionary(void) {
     dictionary_t* const dict = calloc(1, sizeof(dictionary_t));
     if (!dict) {
         BAD_ALLOC();

doc/zstd_compression_format.md

@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
 
 ### Version
 
-0.3.9 (2023-03-08)
+0.4.5 (2026-05-14)
 
 
 Introduction
@@ -387,20 +387,24 @@ When `Block_Type` is `RLE_Block`, since `Block_Content`’s size is always 1,
 
 __`Block_Content`__ and __`Block_Maximum_Size`__
 
-The size of `Block_Content` is limited by `Block_Maximum_Size`,
-which is the smallest of:
+The size of `Block_Content` is limited by `Block_Maximum_Size`, which is
+determined once for a given frame and is the smallest of:
 -  `Window_Size`
--  128 KB
+-  128 KiB (131.072 bytes)
 
-`Block_Maximum_Size` is constant for a given frame.
-This maximum is applicable to both the decompressed size
-and the compressed size of any block in the frame.
+Both the `Block_Content` and the decompressed size of any block in the frame must
+be no larger than `Block_Maximum_Size`.
 
 The reasoning for this limit is that a decoder can read this information
 at the beginning of a frame and use it to allocate buffers.
 The guarantees on the size of blocks ensure that
 the buffers will be large enough for any following block of the valid frame.
 
+If a compressed block is larger than its uncompressed content, it is recommended
+to send it uncompressed (i.e., a `Raw_Block`). However, as long as
+`Block_Content` is no larger than `Block_Maximum_Size`, it is legal to send such
+a compressed block, even if it's larger than its uncompressed content.
+
 
 Compressed Blocks
 -----------------
@@ -650,13 +654,15 @@ __`Number_of_Sequences`__
 
 This is a variable size field using between 1 and 3 bytes.
 Let's call its first byte `byte0`.
-- `if (byte0 == 0)` : there are no sequences.
-            The sequence section stops there.
-            Decompressed content is defined entirely as Literals Section content.
-            The FSE tables used in `Repeat_Mode` aren't updated.
 - `if (byte0 < 128)` : `Number_of_Sequences = byte0` . Uses 1 byte.
-- `if (byte0 < 255)` : `Number_of_Sequences = ((byte0-128) << 8) + byte1` . Uses 2 bytes.
-- `if (byte0 == 255)`: `Number_of_Sequences = byte1 + (byte2<<8) + 0x7F00` . Uses 3 bytes.
+- `if (byte0 < 255)` : `Number_of_Sequences = ((byte0 - 0x80) << 8) + byte1`. Uses 2 bytes.
+            Note that the 2 bytes format fully overlaps the 1 byte format.
+- `if (byte0 == 255)`: `Number_of_Sequences = byte1 + (byte2<<8) + 0x7F00`. Uses 3 bytes.
+
+`if (Number_of_Sequences == 0)` : there are no sequences.
+            The sequence section stops immediately,
+            FSE tables used in `Repeat_Mode` aren't updated.
+            Block's decompressed content is defined solely by the Literals Section content.
 
 __Symbol compression modes__
 
@@ -927,7 +933,10 @@ There is an exception though, when current sequence's `literals_length = 0`.
 In this case, repeated offsets are shifted by one,
 so an `offset_value` of 1 means `Repeated_Offset2`,
 an `offset_value` of 2 means `Repeated_Offset3`,
-and an `offset_value` of 3 means `Repeated_Offset1 - 1_byte`.
+and an `offset_value` of 3 means `Repeated_Offset1 - 1`.
+
+In the final case, if `Repeated_Offset1 - 1` evaluates to 0, then the
+data is considered corrupted.
 
 For the first block, the starting offset history is populated with following values :
 `Repeated_Offset1`=1, `Repeated_Offset2`=4, `Repeated_Offset3`=8,
@@ -1033,55 +1042,57 @@ and to compress Huffman headers.
 FSE
 ---
 FSE, short for Finite State Entropy, is an entropy codec based on [ANS].
-FSE encoding/decoding involves a state that is carried over between symbols,
-so decoding must be done in the opposite direction as encoding.
+FSE encoding/decoding involves a state that is carried over between symbols.
+Decoding must be done in the opposite direction as encoding.
 Therefore, all FSE bitstreams are read from end to beginning.
 Note that the order of the bits in the stream is not reversed,
-we just read the elements in the reverse order they are written.
+we just read each multi-bits element in the reverse order they are encoded.
 
 For additional details on FSE, see [Finite State Entropy].
 
 [Finite State Entropy]:https://github.com/Cyan4973/FiniteStateEntropy/
 
-FSE decoding involves a decoding table which has a power of 2 size, and contain three elements:
+FSE decoding is directed by a decoding table with a power of 2 size, each row containing three elements:
 `Symbol`, `Num_Bits`, and `Baseline`.
 The `log2` of the table size is its `Accuracy_Log`.
 An FSE state value represents an index in this table.
 
 To obtain the initial state value, consume `Accuracy_Log` bits from the stream as a __little-endian__ value.
-The next symbol in the stream is the `Symbol` indicated in the table for that state.
+The first symbol in the stream is the `Symbol` indicated in the table for that state.
 To obtain the next state value,
 the decoder should consume `Num_Bits` bits from the stream as a __little-endian__ value and add it to `Baseline`.
 
 [ANS]: https://en.wikipedia.org/wiki/Asymmetric_Numeral_Systems
 
 ### FSE Table Description
-To decode FSE streams, it is necessary to construct the decoding table.
-The Zstandard format encodes FSE table descriptions as follows:
+To decode an FSE bitstream, it is necessary to build its FSE decoding table.
+The decoding table is derived from a distribution of Probabilities.
+The Zstandard format encodes distributions of Probabilities as follows:
 
-An FSE distribution table describes the probabilities of all symbols
-from `0` to the last present one (included)
-on a normalized scale of `1 << Accuracy_Log` .
-Note that there must be two or more symbols with nonzero probability.
+The distribution of probabilities is described in a bitstream which is read forward,
+in __little-endian__ fashion.
+The amount of bytes consumed from the bitstream to describe the distribution
+is discovered at the end of the decoding process.
 
-It's a bitstream which is read forward, in __little-endian__ fashion.
-It's not necessary to know bitstream exact size,
-it will be discovered and reported by the decoding process.
-
-The bitstream starts by reporting on which scale it operates.
+The bitstream starts by reporting on which scale the distribution operates.
 Let's `low4Bits` designate the lowest 4 bits of the first byte :
 `Accuracy_Log = low4bits + 5`.
 
-Then follows each symbol value, from `0` to last present one.
-The number of bits used by each field is variable.
+An FSE distribution table describes the probabilities of all symbols
+from `0` to the last present one (included) in natural order.
+The sum of probabilities is normalized to reach a power of 2 total of `1 << Accuracy_Log` .
+There must be two or more symbols with non-zero probabilities.
+
+The number of bits used to decode each probability is variable.
 It depends on :
 
 - Remaining probabilities + 1 :
   __example__ :
   Presuming an `Accuracy_Log` of 8,
-  and presuming 100 probabilities points have already been distributed,
+  and presuming 100 probability points have already been distributed,
   the decoder may read any value from `0` to `256 - 100 + 1 == 157` (inclusive).
-  Therefore, it must read `log2sup(157) == 8` bits.
+  Therefore, it may read up to `log2sup(157) == 8` bits, where `log2sup(N)`
+  is the smallest integer `T` that satisfies `(1 << T) > N`.
 
 - Value decoded : small values use 1 less bit :
   __example__ :
@@ -1092,111 +1103,133 @@ It depends on :
   values from 98 to 157 use 8 bits.
   This is achieved through this scheme :
 
-  | Value read | Value decoded | Number of bits used |
-  | ---------- | ------------- | ------------------- |
-  |   0 -  97  |   0 -  97     |  7                  |
-  |  98 - 127  |  98 - 127     |  8                  |
-  | 128 - 225  |   0 -  97     |  7                  |
-  | 226 - 255  | 128 - 157     |  8                  |
+  | 8-bit field read | Value decoded | Nb of bits consumed |
+  | ---------------- | ------------- | ------------------- |
+  |         0 -  97  |   0 -  97     |  7                  |
+  |        98 - 127  |  98 - 127     |  8                  |
+  |       128 - 225  |   0 -  97     |  7                  |
+  |       226 - 255  | 128 - 157     |  8                  |
 
-Symbols probabilities are read one by one, in order.
+Probability is derived from Value decoded using the following formula:
+`Probality = Value - 1`
 
-Probability is obtained from Value decoded by following formula :
-`Proba = value - 1`
+Consequently, a Probability of `0` is described by a Value `1`.
 
-It means value `0` becomes negative probability `-1`.
-`-1` is a special probability, which means "less than 1".
-Its effect on distribution table is described in the [next section].
-For the purpose of calculating total allocated probability points, it counts as one.
+A Value `0` is used to signal a special case, named "Probability `-1`".
+It describes a probability which should have been "less than 1".
+Its effect on the decoding table building process is described in the [next section].
+For the purpose of counting total allocated probability points, it counts as one.
 
 [next section]:#from-normalized-distribution-to-decoding-tables
 
-When a symbol has a __probability__ of `zero`,
+Symbols probabilities are read one by one, in order.
+After each probability is decoded, the total nb of probability points is updated.
+This is used to determine how many bits must be read to decode the probability of next symbol.
+
+When a symbol has a __probability__ of `zero` (decoded from reading a Value `1`),
 it is followed by a 2-bits repeat flag.
 This repeat flag tells how many probabilities of zeroes follow the current one.
 It provides a number ranging from 0 to 3.
 If it is a 3, another 2-bits repeat flag follows, and so on.
 
-When last symbol reaches cumulated total of `1 << Accuracy_Log`,
-decoding is complete.
-If the last symbol makes cumulated total go above `1 << Accuracy_Log`,
-distribution is considered corrupted.
+When the Probability for a symbol makes cumulated total reach `1 << Accuracy_Log`,
+then it's the last symbol, and decoding is complete.
 
 Then the decoder can tell how many bytes were used in this process,
 and how many symbols are present.
 The bitstream consumes a round number of bytes.
 Any remaining bit within the last byte is just unused.
 
+If this process results in a non-zero probability for a symbol outside of the
+valid range of symbols that the FSE table is defined for, even if that symbol is
+not used, then the data is considered corrupted.
+For the specific case of offset codes,
+a decoder implementation may reject a frame containing a non-zero probability
+for an offset code larger than the largest offset code supported by the decoder
+implementation.
+
 #### From normalized distribution to decoding tables
 
-The distribution of normalized probabilities is enough
+The normalized distribution of probabilities is enough
 to create a unique decoding table.
-
-It follows the following build rule :
+It is generated using the following build rule :
 
 The table has a size of `Table_Size = 1 << Accuracy_Log`.
-Each cell describes the symbol decoded,
-and instructions to get the next state (`Number_of_Bits` and `Baseline`).
+Each row specifies the decoded symbol,
+and instructions to reach the next state (`Number_of_Bits` and `Baseline`).
 
-Symbols are scanned in their natural order for "less than 1" probabilities.
-Symbols with this probability are being attributed a single cell,
+Symbols are first scanned in their natural order for "less than 1" probabilities
+(previously decoded from a Value of `0`).
+Symbols with this special probability are being attributed a single row,
 starting from the end of the table and retreating.
 These symbols define a full state reset, reading `Accuracy_Log` bits.
 
-Then, all remaining symbols, sorted in natural order, are allocated cells.
-Starting from symbol `0` (if it exists), and table position `0`,
-each symbol gets allocated as many cells as its probability.
-Cell allocation is spread, not linear :
-each successor position follows this rule :
+Then, all remaining symbols, sorted in natural order, are allocated rows.
+Starting from smallest present symbol, and table position `0`,
+each symbol gets allocated as many rows as its probability.
 
+Row allocation is not linear, it follows this order, in modular arithmetic:
 ```
 position += (tableSize>>1) + (tableSize>>3) + 3;
 position &= tableSize-1;
 ```
 
-A position is skipped if already occupied by a "less than 1" probability symbol.
-`position` does not reset between symbols, it simply iterates through
-each position in the table, switching to the next symbol when enough
-states have been allocated to the current one.
+Using above ordering rule, each symbol gets allocated as many rows as its probability.
+If a position is already occupied by a "less than 1" probability symbol,
+it is simply skipped, and the next position is allocated instead.
+Once enough rows have been allocated for the current symbol,
+the allocation process continues, using the next symbol, in natural order.
+This process guarantees that the table is entirely and exactly filled.
 
-The process guarantees that the table is entirely filled.
-Each cell corresponds to a state value, which contains the symbol being decoded.
+Each row specifies a decoded symbol, and is accessed by current state value.
+It also specifies `Number_of_Bits` and `Baseline`, which are required to determine next state value.
 
-To add the `Number_of_Bits` and `Baseline` required to retrieve next state,
-it's first necessary to sort all occurrences of each symbol in state order.
-Lower states will need 1 more bit than higher ones.
-The process is repeated for each symbol.
+To correctly set these fields, it's necessary to sort all occurrences of each symbol in state value order,
+and then attribute N+1 bits to lower rows, and N bits to higher rows,
+following the process described below (using an example):
 
 __Example__ :
-Presuming a symbol has a probability of 5,
-it receives 5 cells, corresponding to 5 state values.
-These state values are then sorted in natural order.
+Presuming an `Accuracy_Log` of 7,
+let's imagine a symbol with a Probability of 5:
+it receives 5 rows, corresponding to 5 state values between `0` and `127`.
 
-Next power of 2 after 5 is 8.
-Space of probabilities must be divided into 8 equal parts.
-Presuming the `Accuracy_Log` is 7, it defines a space of 128 states.
-Divided by 8, each share is 16 large.
+In this example, the first state value happens to be `1` (after unspecified previous symbols).
+The next 4 states are then determined using above modular arithmetic rule,
+which specifies to add `64+16+3 = 83` modulo `128` to jump to next position,
+producing the following series: `1`, `84`, `39`, `122`, `77` (modular arithmetic).
+(note: the next symbol will then start at `32`).
 
-In order to reach 8 shares, 8-5=3 lowest states will count "double",
+These state values are then sorted in natural order,
+resulting in the following series: `1`, `39`, `77`, `84`, `122`.
+
+The next power of 2 after 5 is 8.
+Therefore, the probability space will be divided into 8 equal parts.
+Since the probability space is `1<<7 = 128` large, each share is `128/8 = 16` large.
+
+In order to reach 8 shares, the `8-5 = 3` lowest states will count "double",
 doubling their shares (32 in width), hence requiring one more bit.
 
-Baseline is assigned starting from the higher states using fewer bits,
-increasing at each state, then resuming at the first state,
-each state takes its allocated width from Baseline.
+Baseline is assigned starting from the lowest state using fewer bits,
+continuing in natural state order, looping back at the beginning.
+Each state takes its allocated range from Baseline, sized by its `Number_of_Bits`.
 
-| state value      |   1   |  39   |   77   |  84  |  122   |
 | state order      |   0   |   1   |    2   |   3  |    4   |
 | ---------------- | ----- | ----- | ------ | ---- | ------ |
+| state value      |   1   |  39   |   77   |  84  |  122   |
 | width            |  32   |  32   |   32   |  16  |   16   |
 | `Number_of_Bits` |   5   |   5   |    5   |   4  |    4   |
-| range number     |   2   |   4   |    6   |   0  |    1   |
+| allocation order |   3   |   4   |    5   |   1  |    2   |
 | `Baseline`       |  32   |  64   |   96   |   0  |   16   |
 | range            | 32-63 | 64-95 | 96-127 | 0-15 | 16-31  |
 
-During decoding, the next state value is determined from current state value,
-by reading the required `Number_of_Bits`, and adding the specified `Baseline`.
+During decoding, the next state value is determined by using current state value as row number,
+then reading the required `Number_of_Bits` from the bitstream, and adding the specified `Baseline`.
 
-See [Appendix A] for the results of this process applied to the default distributions.
+Note:
+as a trivial example, it follows that, for a symbol with a Probability of `1`,
+`Baseline` is necessarily `0`, and `Number_of_Bits` is necessarily `Accuracy_Log`.
+
+See [Appendix A] to see the outcome of this process applied to the default distributions.
 
 [Appendix A]: #appendix-a---decoding-tables-for-predefined-codes
 
@@ -1241,15 +1274,17 @@ This specification limits maximum code length to 11 bits.
 
 #### Representation
 
-All literal values from zero (included) to last present one (excluded)
+All literal symbols from zero (included) to last present one (excluded)
 are represented by `Weight` with values from `0` to `Max_Number_of_Bits`.
 Transformation from `Weight` to `Number_of_Bits` follows this formula :
 ```
 Number_of_Bits = Weight ? (Max_Number_of_Bits + 1 - Weight) : 0
 ```
-When a literal value is not present, it receives a `Weight` of 0.
+When a literal symbol is not present, it receives a `Weight` of 0.
 The least frequent symbol receives a `Weight` of 1.
-Consequently, the `Weight` 1 is necessarily present.
+If no literal has a `Weight` of 1, then the data is considered corrupted.
+If there are not at least two literals with non-zero `Weight`, then the data
+is considered corrupted.
 The most frequent symbol receives a `Weight` anywhere between 1 and 11 (max).
 The last symbol's `Weight` is deduced from previously retrieved Weights,
 by completing to the nearest power of 2. It's necessarily non 0.
@@ -1262,33 +1297,38 @@ otherwise the representation is considered corrupted.
 __Example__ :
 Let's presume the following Huffman tree must be described :
 
-|  literal value   |  0  |  1  |  2  |  3  |  4  |  5  |
+|  literal symbol  |  A  |  B  |  C  |  D  |  E  |  F  |
 | ---------------- | --- | --- | --- | --- | --- | --- |
 | `Number_of_Bits` |  1  |  2  |  3  |  0  |  4  |  4  |
 
 The tree depth is 4, since its longest elements uses 4 bits
-(longest elements are the one with smallest frequency).
-Literal value `5` will not be listed, as it can be determined from previous values 0-4,
-nor will values above `5` as they are all 0.
-Values from `0` to `4` will be listed using `Weight` instead of `Number_of_Bits`.
+(longest elements are the ones with smallest frequency).
+
+All symbols will now receive a `Weight` instead of `Number_of_Bits`.
 Weight formula is :
 ```
 Weight = Number_of_Bits ? (Max_Number_of_Bits + 1 - Number_of_Bits) : 0
 ```
-It gives the following series of weights :
+It gives the following series of Weights :
 
-| literal value |  0  |  1  |  2  |  3  |  4  |
-| ------------- | --- | --- | --- | --- | --- |
-|   `Weight`    |  4  |  3  |  2  |  0  |  1  |
+| literal symbol |  A  |  B  |  C  |  D  |  E  |  F  |
+| -------------- | --- | --- | --- | --- | --- | --- |
+|   `Weight`     |  4  |  3  |  2  |  0  |  1  |  1  |
+
+This list will be sent to the decoder, with the following modifications:
+
+- `F` will not be listed, because it can be determined from previous symbols
+- nor will symbols above `F` as they are all 0
+- on the other hand, all symbols before `A`, starting with `\0`, will be listed, with a Weight of 0.
 
 The decoder will do the inverse operation :
-having collected weights of literal symbols from `0` to `4`,
-it knows the last literal, `5`, is present with a non-zero `Weight`.
-The `Weight` of `5` can be determined by advancing to the next power of 2.
+having collected weights of literal symbols from `A` to `E`,
+it knows the last literal, `F`, is present with a non-zero `Weight`.
+The `Weight` of `F` can be determined by advancing to the next power of 2.
 The sum of `2^(Weight-1)` (excluding 0's) is :
 `8 + 4 + 2 + 0 + 1 = 15`.
 Nearest larger power of 2 value is 16.
-Therefore, `Max_Number_of_Bits = 4` and `Weight[5] = log_2(16 - 15) + 1 = 1`.
+Therefore, `Max_Number_of_Bits = log2(16) = 4` and `Weight[F] = log_2(16 - 15) + 1 = 1`.
 
 #### Huffman Tree header
 
@@ -1328,7 +1368,7 @@ sharing a single distribution table.
 To decode an FSE bitstream, it is necessary to know its compressed size.
 Compressed size is provided by `headerByte`.
 It's also necessary to know its _maximum possible_ decompressed size,
-which is `255`, since literal values span from `0` to `255`,
+which is `255`, since literal symbols span from `0` to `255`,
 and last symbol's `Weight` is not represented.
 
 An FSE bitstream starts by a header, describing probabilities distribution.
@@ -1350,6 +1390,13 @@ If updating state after decoding a symbol would require more bits than
 remain in the stream, it is assumed that extra bits are 0.  Then,
 symbols for each of the final states are decoded and the process is complete.
 
+If this process would produce more weights than the maximum number of decoded
+weights (255), then the data is considered corrupted.
+
+If either of the 2 initial states are absent or truncated, then the data is
+considered corrupted.  Consequently, it is not possible to encode fewer than
+2 weights using this mode.
+
 #### Conversion from weights to Huffman prefix codes
 
 All present symbols shall now have a `Weight` value.
@@ -1357,26 +1404,28 @@ It is possible to transform weights into `Number_of_Bits`, using this formula:
 ```
 Number_of_Bits = (Weight>0) ? Max_Number_of_Bits + 1 - Weight : 0
 ```
-Symbols are sorted by `Weight`.
-Within same `Weight`, symbols keep natural sequential order.
+In order to determine which prefix code is assigned to each Symbol,
+Symbols are first sorted by `Weight`, then by natural sequential order.
 Symbols with a `Weight` of zero are removed.
-Then, starting from lowest `Weight`, prefix codes are distributed in sequential order.
+Then, starting from lowest `Weight` (hence highest `Number_of_Bits`),
+prefix codes are assigned in ascending order.
 
 __Example__ :
-Let's presume the following list of weights has been decoded :
+Let's assume the following list of weights has been decoded:
 
-| Literal  |  0  |  1  |  2  |  3  |  4  |  5  |
+| Literal  |  A  |  B  |  C  |  D  |  E  |  F  |
 | -------- | --- | --- | --- | --- | --- | --- |
 | `Weight` |  4  |  3  |  2  |  0  |  1  |  1  |
 
 Sorted by weight and then natural sequential order,
-it gives the following distribution :
+it gives the following prefix codes distribution:
 
-| Literal          |  3  |  4  |  5  |  2  |  1  |   0  |
-| ---------------- | --- | --- | --- | --- | --- | ---- |
-| `Weight`         |  0  |  1  |  1  |  2  |  3  |   4  |
-| `Number_of_Bits` |  0  |  4  |  4  |  3  |  2  |   1  |
-| prefix codes     | N/A | 0000| 0001| 001 | 01  |   1  |
+| Literal          |  D  |   E  |   F  |   C  |   B  |   A  |
+| ---------------- | --- | ---- | ---- | ---- | ---- | ---- |
+| `Weight`         |  0  |   1  |   1  |   2  |   3  |   4  |
+| `Number_of_Bits` |  0  |   4  |   4  |   3  |   2  |   1  |
+| prefix code      | N/A | 0000 | 0001 | 001  | 01   | 1    |
+| ascending order  | N/A | 0000 | 0001 | 001x | 01xx | 1xxx |
 
 ### Huffman-coded Streams
 
@@ -1399,10 +1448,10 @@ it's possible to read the bitstream in a __little-endian__ fashion,
 keeping track of already used bits. Since the bitstream is encoded in reverse
 order, starting from the end read symbols in forward order.
 
-For example, if the literal sequence "0145" was encoded using above prefix code,
+For example, if the literal sequence `ABEF` was encoded using above prefix code,
 it would be encoded (in reverse order) as:
 
-|Symbol  |   5  |   4  |  1 | 0 | Padding |
+|Symbol  |   F  |   E  |  B | A | Padding |
 |--------|------|------|----|---|---------|
 |Encoding|`0000`|`0001`|`01`|`1`| `00001` |
 
@@ -1697,6 +1746,12 @@ or at least provide a meaningful error code explaining for which reason it canno
 
 Version changes
 ---------------
+- 0.4.5 : minor clarification regarding Block_Maximum_Size
+- 0.4.4 : minor clarification for block size
+- 0.4.3 : clarifications for Huffman prefix code assignment example
+- 0.4.2 : refactor FSE table construction process, inspired by Donald Pian
+- 0.4.1 : clarifications on a few error scenarios, by Eric Lasota
+- 0.4.0 : fixed imprecise behavior for nbSeq==0, detected by Igor Pavlov
 - 0.3.9 : clarifications for Huffman-compressed literal sizes.
 - 0.3.8 : clarifications for Huffman Blocks and Huffman Tree descriptions.
 - 0.3.7 : clarifications for Repeat_Offsets, matching RFC8878

doc/zstd_manual.html

@@ -1,16 +1,17 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>zstd 1.5.5 Manual</title>
+<title>zstd 1.5.7 Manual</title>
 </head>
 <body>
-<h1>zstd 1.5.5 Manual</h1>
+<h1>zstd 1.5.7 Manual</h1>
+Note: the content of this file has been automatically generated by parsing "zstd.h" 
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
 <li><a href="#Chapter1">Introduction</a></li>
 <li><a href="#Chapter2">Version</a></li>
-<li><a href="#Chapter3">Simple API</a></li>
+<li><a href="#Chapter3">Simple Core API</a></li>
 <li><a href="#Chapter4">Explicit context</a></li>
 <li><a href="#Chapter5">Advanced compression API (Requires v1.4.0+)</a></li>
 <li><a href="#Chapter6">Advanced decompression API (Requires v1.4.0+)</a></li>
@@ -74,7 +75,7 @@
 </b><p>  Return runtime library version, like "1.4.5". Requires v1.3.0+. 
 </p></pre><BR>
 
-<a name="Chapter3"></a><h2>Simple API</h2><pre></pre>
+<a name="Chapter3"></a><h2>Simple Core API</h2><pre></pre>
 
 <pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity,
                 const void* src, size_t srcSize,
@@ -88,44 +89,48 @@
 
 <pre><b>size_t ZSTD_decompress( void* dst, size_t dstCapacity,
                   const void* src, size_t compressedSize);
-</b><p>  `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
-  `dstCapacity` is an upper bound of originalSize to regenerate.
-  If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
-  @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
-            or an errorCode if it fails (which can be tested using ZSTD_isError()). 
+</b><p> `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
+  Multiple compressed frames can be decompressed at once with this method.
+  The result will be the concatenation of all decompressed frames, back to back.
+ `dstCapacity` is an upper bound of originalSize to regenerate.
+  First frame's decompressed size can be extracted using ZSTD_getFrameContentSize().
+  If maximum upper bound isn't known, prefer using streaming mode to decompress data.
+ @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
+           or an errorCode if it fails (which can be tested using ZSTD_isError()). 
 </p></pre><BR>
 
+<h3>Decompression helper functions</h3><pre></pre><b><pre></pre></b><BR>
 <pre><b>#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
 #define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
 unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
-</b><p>  `src` should point to the start of a ZSTD encoded frame.
-  `srcSize` must be at least as large as the frame header.
-            hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
-  @return : - decompressed size of `src` frame content, if known
-            - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
-            - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
-   note 1 : a 0 return value means the frame is valid but "empty".
-   note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
-            When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
-            In which case, it's necessary to use streaming mode to decompress data.
-            Optionally, application can rely on some implicit limit,
-            as ZSTD_decompress() only needs an upper bound of decompressed size.
-            (For example, data could be necessarily cut into blocks <= 16 KB).
-   note 3 : decompressed size is always present when compression is completed using single-pass functions,
-            such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
-   note 4 : decompressed size can be very large (64-bits value),
-            potentially larger than what local system can handle as a single memory segment.
-            In which case, it's necessary to use streaming mode to decompress data.
-   note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
-            Always ensure return value fits within application's authorized limits.
-            Each application can set its own limits.
-   note 6 : This function replaces ZSTD_getDecompressedSize() 
+</b><p> `src` should point to the start of a ZSTD encoded frame.
+ `srcSize` must be at least as large as the frame header.
+           hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
+ @return : - decompressed size of `src` frame content, if known
+           - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+           - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
+  note 1 : a 0 return value means the frame is valid but "empty".
+           When invoking this method on a skippable frame, it will return 0.
+  note 2 : decompressed size is an optional field, it may not be present (typically in streaming mode).
+           When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+           In which case, it's necessary to use streaming mode to decompress data.
+           Optionally, application can rely on some implicit limit,
+           as ZSTD_decompress() only needs an upper bound of decompressed size.
+           (For example, data could be necessarily cut into blocks <= 16 KB).
+  note 3 : decompressed size is always present when compression is completed using single-pass functions,
+           such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
+  note 4 : decompressed size can be very large (64-bits value),
+           potentially larger than what local system can handle as a single memory segment.
+           In which case, it's necessary to use streaming mode to decompress data.
+  note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+           Always ensure return value fits within application's authorized limits.
+           Each application can set its own limits.
+  note 6 : This function replaces ZSTD_getDecompressedSize() 
 </p></pre><BR>
 
 <pre><b>ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
-ZSTDLIB_API
 unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
-</b><p>  NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
+</b><p>  This function is now obsolete, in favor of ZSTD_getFrameContentSize().
   Both functions work the same way, but ZSTD_getDecompressedSize() blends
   "empty", "unknown" and "error" results to the same return value (0),
   while ZSTD_getFrameContentSize() gives them separate return values.
@@ -137,70 +142,78 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
  `srcSize` must be >= first frame size
  @return : the compressed size of the first frame starting at `src`,
            suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
-        or an error code if input is invalid 
+           or an error code if input is invalid
+  Note 1: this method is called _find*() because it's not enough to read the header,
+          it may have to scan through the frame's content, to reach its end.
+  Note 2: this method also works with Skippable Frames. In which case,
+          it returns the size of the complete skippable frame,
+          which is always equal to its content size + 8 bytes for headers. 
 </p></pre><BR>
 
-<h3>Helper functions</h3><pre></pre><b><pre></b>/* ZSTD_compressBound() :<b>
- * maximum compressed size in worst case single-pass scenario.
- * When invoking `ZSTD_compress()` or any other one-pass compression function,
- * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
- * as it eliminates one potential failure scenario,
- * aka not enough room in dst buffer to write the compressed frame.
- * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
- *        In which case, ZSTD_compressBound() will return an error code
- *        which can be tested using ZSTD_isError().
- *
- * ZSTD_COMPRESSBOUND() :
- * same as ZSTD_compressBound(), but as a macro.
- * It can be used to produce constants, which can be useful for static allocation,
- * for example to size a static array on stack.
- * Will produce constant value 0 if srcSize too large.
- */
-#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
+<h3>Compression helper functions</h3><pre></pre><b><pre></pre></b><BR>
+<pre><b>#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U)
 #define ZSTD_COMPRESSBOUND(srcSize)   (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) </b>/* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */<b>
 size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case single-pass scenario */<b>
-</b>/* ZSTD_isError() :<b>
+</b><p> maximum compressed size in worst case single-pass scenario.
+ When invoking `ZSTD_compress()`, or any other one-pass compression function,
+ it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
+ as it eliminates one potential failure scenario,
+ aka not enough room in dst buffer to write the compressed frame.
+ Note : ZSTD_compressBound() itself can fail, if @srcSize >= ZSTD_MAX_INPUT_SIZE .
+        In which case, ZSTD_compressBound() will return an error code
+        which can be tested using ZSTD_isError().
+
+ ZSTD_COMPRESSBOUND() :
+ same as ZSTD_compressBound(), but as a macro.
+ It can be used to produce constants, which can be useful for static allocation,
+ for example to size a static array on stack.
+ Will produce constant value 0 if srcSize is too large.
+ 
+</p></pre><BR>
+
+<h3>Error helper functions</h3><pre></pre><b><pre></b>/* ZSTD_isError() :<b>
  * Most ZSTD_* functions returning a size_t value can be tested for error,
  * using ZSTD_isError().
  * @return 1 if error, 0 otherwise
  */
-unsigned    ZSTD_isError(size_t code);          </b>/*!< tells if a `size_t` function result is an error code */<b>
-const char* ZSTD_getErrorName(size_t code);     </b>/*!< provides readable string from an error code */<b>
-int         ZSTD_minCLevel(void);               </b>/*!< minimum negative compression level allowed, requires v1.4.0+ */<b>
-int         ZSTD_maxCLevel(void);               </b>/*!< maximum compression level available */<b>
-int         ZSTD_defaultCLevel(void);           </b>/*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */<b>
+unsigned     ZSTD_isError(size_t result);      </b>/*!< tells if a `size_t` function result is an error code */<b>
+ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); </b>/* convert a result into an error code, which can be compared to error enum list */<b>
+const char*  ZSTD_getErrorName(size_t result); </b>/*!< provides readable string from a function result */<b>
+int          ZSTD_minCLevel(void);             </b>/*!< minimum negative compression level allowed, requires v1.4.0+ */<b>
+int          ZSTD_maxCLevel(void);             </b>/*!< maximum compression level available */<b>
+int          ZSTD_defaultCLevel(void);         </b>/*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */<b>
 </pre></b><BR>
 <a name="Chapter4"></a><h2>Explicit context</h2><pre></pre>
 
 <h3>Compression context</h3><pre>  When compressing many times,
-  it is recommended to allocate a context just once,
-  and re-use it for each successive compression operation.
-  This will make workload friendlier for system's memory.
+  it is recommended to allocate a compression context just once,
+  and reuse it for each successive compression operation.
+  This will make the workload easier for system's memory.
   Note : re-using context is just a speed / resource optimization.
          It doesn't change the compression ratio, which remains identical.
-  Note 2 : In multi-threaded environments,
-         use one different context per thread for parallel execution.
+  Note 2: For parallel execution in multi-threaded environments,
+         use one different context per thread .
  
 </pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx;
 ZSTD_CCtx* ZSTD_createCCtx(void);
-size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);  </b>/* accept NULL pointer */<b>
+size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);  </b>/* compatible with NULL pointer */<b>
 </pre></b><BR>
 <pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
                          void* dst, size_t dstCapacity,
                    const void* src, size_t srcSize,
                          int compressionLevel);
 </b><p>  Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
-  Important : in order to behave similarly to `ZSTD_compress()`,
-  this function compresses at requested compression level,
-  __ignoring any other parameter__ .
+  Important : in order to mirror `ZSTD_compress()` behavior,
+  this function compresses at the requested compression level,
+  __ignoring any other advanced parameter__ .
   If any advanced parameter was set using the advanced API,
-  they will all be reset. Only `compressionLevel` remains.
+  they will all be reset. Only @compressionLevel remains.
  
 </p></pre><BR>
 
 <h3>Decompression context</h3><pre>  When decompressing many times,
   it is recommended to allocate a context only once,
-  and re-use it for each successive compression operation.
+  and reuse it for each successive compression operation.
   This will make workload friendlier for system's memory.
   Use one context per thread for parallel execution. 
 </pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
@@ -212,7 +225,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
                      const void* src, size_t srcSize);
 </b><p>  Same as ZSTD_decompress(),
   requires an allocated ZSTD_DCtx.
-  Compatible with sticky parameters.
+  Compatible with sticky parameters (see below).
  
 </p></pre><BR>
 
@@ -296,6 +309,19 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
                               * The higher the value of selected strategy, the more complex it is,
                               * resulting in stronger and slower compression.
                               * Special: value 0 means "use default strategy". */
+
+    ZSTD_c_targetCBlockSize=130, </b>/* v1.5.6+<b>
+                                  * Attempts to fit compressed block size into approximately targetCBlockSize.
+                                  * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX.
+                                  * Note that it's not a guarantee, just a convergence target (default:0).
+                                  * No target when targetCBlockSize == 0.
+                                  * This is helpful in low bandwidth streaming environments to improve end-to-end latency,
+                                  * when a client can make use of partial documents (a prominent example being Chrome).
+                                  * Note: this parameter is stable since v1.5.6.
+                                  * It was present as an experimental parameter in earlier versions,
+                                  * but it's not recommended using it with earlier library versions
+                                  * due to massive performance regressions.
+                                  */
     </b>/* LDM mode parameters */<b>
     ZSTD_c_enableLongDistanceMatching=160, </b>/* Enable long distance matching.<b>
                                      * This parameter is designed to improve compression ratio
@@ -375,14 +401,14 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      * ZSTD_c_forceMaxWindow
      * ZSTD_c_forceAttachDict
      * ZSTD_c_literalCompressionMode
-     * ZSTD_c_targetCBlockSize
      * ZSTD_c_srcSizeHint
      * ZSTD_c_enableDedicatedDictSearch
      * ZSTD_c_stableInBuffer
      * ZSTD_c_stableOutBuffer
      * ZSTD_c_blockDelimiters
      * ZSTD_c_validateSequences
-     * ZSTD_c_useBlockSplitter
+     * ZSTD_c_blockSplitterLevel
+     * ZSTD_c_splitAfterSequences
      * ZSTD_c_useRowMatchFinder
      * ZSTD_c_prefetchCDictTables
      * ZSTD_c_enableSeqProducerFallback
@@ -396,7 +422,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      ZSTD_c_experimentalParam3=1000,
      ZSTD_c_experimentalParam4=1001,
      ZSTD_c_experimentalParam5=1002,
-     ZSTD_c_experimentalParam6=1003,
+     </b>/* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */<b>
      ZSTD_c_experimentalParam7=1004,
      ZSTD_c_experimentalParam8=1005,
      ZSTD_c_experimentalParam9=1006,
@@ -409,7 +435,8 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      ZSTD_c_experimentalParam16=1013,
      ZSTD_c_experimentalParam17=1014,
      ZSTD_c_experimentalParam18=1015,
-     ZSTD_c_experimentalParam19=1016
+     ZSTD_c_experimentalParam19=1016,
+     ZSTD_c_experimentalParam20=1017
 } ZSTD_cParameter;
 </b></pre><BR>
 <pre><b>typedef struct {
@@ -483,6 +510,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
                        void* dst, size_t dstCapacity,
                  const void* src, size_t srcSize);
 </b><p>  Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
+  (note that this entry point doesn't even expose a compression level parameter).
   ZSTD_compress2() always starts a new frame.
   Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
   - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
@@ -513,6 +541,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      * ZSTD_d_forceIgnoreChecksum
      * ZSTD_d_refMultipleDDicts
      * ZSTD_d_disableHuffmanAssembly
+     * ZSTD_d_maxBlockSize
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly
      */
@@ -520,7 +549,8 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      ZSTD_d_experimentalParam2=1001,
      ZSTD_d_experimentalParam3=1002,
      ZSTD_d_experimentalParam4=1003,
-     ZSTD_d_experimentalParam5=1004
+     ZSTD_d_experimentalParam5=1004,
+     ZSTD_d_experimentalParam6=1005
 
 } ZSTD_dParameter;
 </b></pre><BR>
@@ -568,14 +598,14 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
   A ZSTD_CStream object is required to track streaming operation.
   Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
   ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
-  It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
+  It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
 
   For parallel execution, use one separate ZSTD_CStream per thread.
 
   note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
 
   Parameters are sticky : when starting a new compression on the same context,
-  it will re-use the same sticky parameters as previous compression session.
+  it will reuse the same sticky parameters as previous compression session.
   When in doubt, it's recommended to fully initialize the context before usage.
   Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
   ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
@@ -666,6 +696,11 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs);  </b>/* accept NULL pointer */<b>
             only ZSTD_e_end or ZSTD_e_flush operations are allowed.
             Before starting a new compression job, or changing compression parameters,
             it is required to fully flush internal buffers.
+  - note: if an operation ends with an error, it may leave @cctx in an undefined state.
+          Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state.
+          In order to be re-employed after an error, a state must be reset,
+          which can be done explicitly (ZSTD_CCtx_reset()),
+          or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx())
  
 </p></pre><BR>
 
@@ -698,7 +733,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 <a name="Chapter9"></a><h2>Streaming decompression - HowTo</h2><pre>
   A ZSTD_DStream object is required to track streaming operations.
   Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
-  ZSTD_DStream objects can be re-used multiple times.
+  ZSTD_DStream objects can be re-employed multiple times.
 
   Use ZSTD_initDStream() to start a new decompression operation.
  @return : recommended first input size
@@ -708,16 +743,21 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
   The function will update both `pos` fields.
   If `input.pos < input.size`, some input has not been consumed.
   It's up to the caller to present again remaining data.
+
   The function tries to flush all data decoded immediately, respecting output buffer size.
   If `output.pos < output.size`, decoder has flushed everything it could.
-  But if `output.pos == output.size`, there might be some data left within internal buffers.,
+
+  However, when `output.pos == output.size`, it's more difficult to know.
+  If @return > 0, the frame is not complete, meaning
+  either there is still some data left to flush within internal buffers,
+  or there is more input to read to complete the frame (or both).
   In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
   Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
  @return : 0 when a frame is completely decoded and fully flushed,
         or an error code, which can be tested using ZSTD_isError(),
         or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
                                 the return value is a suggested next input size (just a hint for better latency)
-                                that will never request more than the remaining frame size.
+                                that will never request more than the remaining content of the compressed frame.
  
 <BR></pre>
 
@@ -743,14 +783,21 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
  Function will update both input and output `pos` fields exposing current state via these fields:
  - `input.pos < input.size`, some input remaining and caller should provide remaining input
    on the next call.
- - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
- - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
-   call ZSTD_decompressStream() again to flush remaining data to output.
+ - `output.pos < output.size`, decoder flushed internal output buffer.
+ - `output.pos == output.size`, unflushed data potentially present in the internal buffers,
+   check ZSTD_decompressStream() @return value,
+   if > 0, invoke it again to flush remaining data to output.
  Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
 
  @return : 0 when a frame is completely decoded and fully flushed,
            or an error code, which can be tested using ZSTD_isError(),
            or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
+
+ Note: when an operation returns with an error code, the @zds state may be left in undefined state.
+       It's UB to invoke `ZSTD_decompressStream()` on such a state.
+       In order to re-use such a state, it must be first reset,
+       which can be done explicitly (`ZSTD_DCtx_reset()`),
+       or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`)
  
 </p></pre><BR>
 
@@ -869,7 +916,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
 <a name="Chapter13"></a><h2>Advanced dictionary and prefix API (Requires v1.4.0+)</h2><pre>
  This API allows dictionaries to be used with ZSTD_compress2(),
  ZSTD_compressStream2(), and ZSTD_decompressDCtx().
- Dictionaries are sticky, they remain valid when same context is re-used,
+ Dictionaries are sticky, they remain valid when same context is reused,
  they only reset when the context is reset
  with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters.
  In contrast, Prefixes are single-use.
@@ -1041,7 +1088,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
                                *
                                * Note: This field is optional. ZSTD_generateSequences() will calculate the value of
                                * 'rep', but repeat offsets do not necessarily need to be calculated from an external
-                               * sequence provider's perspective. For example, ZSTD_compressSequences() does not
+                               * sequence provider perspective. For example, ZSTD_compressSequences() does not
                                * use this 'rep' field at all (as of now).
                                */
 } ZSTD_Sequence;
@@ -1146,14 +1193,14 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 } ZSTD_literalCompressionMode_e;
 </b></pre><BR>
 <pre><b>typedef enum {
-  </b>/* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final<b>
-   * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
-   * or ZSTD_ps_disable allow for a force enable/disable the feature.
+  </b>/* Note: This enum controls features which are conditionally beneficial.<b>
+   * Zstd can take a decision on whether or not to enable the feature (ZSTD_ps_auto),
+   * but setting the switch to ZSTD_ps_enable or ZSTD_ps_disable force enable/disable the feature.
    */
   ZSTD_ps_auto = 0,         </b>/* Let the library automatically determine whether the feature shall be enabled */<b>
   ZSTD_ps_enable = 1,       </b>/* Force-enable the feature */<b>
   ZSTD_ps_disable = 2       </b>/* Do not use the feature */<b>
-} ZSTD_paramSwitch_e;
+} ZSTD_ParamSwitch_e;
 </b></pre><BR>
 <a name="Chapter15"></a><h2>Frame header and size functions</h2><pre></pre>
 
@@ -1196,33 +1243,33 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 </p></pre><BR>
 
 <pre><b>ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
-</b><p>  srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
+</b><p>  srcSize must be large enough, aka >= ZSTD_FRAMEHEADERSIZE_PREFIX.
  @return : size of the Frame Header,
            or an error code (if srcSize is too small) 
 </p></pre><BR>
 
-<pre><b>typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
+<pre><b>typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_FrameType_e;
 </b></pre><BR>
 <pre><b>typedef struct {
     unsigned long long frameContentSize; </b>/* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */<b>
     unsigned long long windowSize;       </b>/* can be very large, up to <= frameContentSize */<b>
     unsigned blockSizeMax;
-    ZSTD_frameType_e frameType;          </b>/* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */<b>
+    ZSTD_FrameType_e frameType;          </b>/* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */<b>
     unsigned headerSize;
-    unsigned dictID;
+    unsigned dictID;                     </b>/* for ZSTD_skippableFrame, contains the skippable magic variant [0-15] */<b>
     unsigned checksumFlag;
     unsigned _reserved1;
     unsigned _reserved2;
-} ZSTD_frameHeader;
+} ZSTD_FrameHeader;
 </b></pre><BR>
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   </b>/**< doesn't consume input */<b>
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize);
 </b>/*! ZSTD_getFrameHeader_advanced() :<b>
  *  same as ZSTD_getFrameHeader(),
  *  with added capability to select a format (like ZSTD_f_zstd1_magicless) */
-ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
-</b><p>  decode Frame Header, or requires larger `srcSize`.
- @return : 0, `zfhPtr` is correctly filled,
-          >0, `srcSize` is too small, value is wanted `srcSize` amount,
+ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
+</b><p>  decode Frame Header into `zfhPtr`, or requires larger `srcSize`.
+ @return : 0 => header is complete, `zfhPtr` is correctly filled,
+          >0 => `srcSize` is too small, @return value is the wanted `srcSize` amount, `zfhPtr` is not filled,
            or an error code, which can be tested using ZSTD_isError() 
 </p></pre><BR>
 
@@ -1272,9 +1319,9 @@ ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr,
 </p></pre><BR>
 
 <pre><b>typedef enum {
-  ZSTD_sf_noBlockDelimiters = 0,         </b>/* Representation of ZSTD_Sequence has no block delimiters, sequences only */<b>
-  ZSTD_sf_explicitBlockDelimiters = 1    </b>/* Representation of ZSTD_Sequence contains explicit block delimiters */<b>
-} ZSTD_sequenceFormat_e;
+  ZSTD_sf_noBlockDelimiters = 0,         </b>/* ZSTD_Sequence[] has no block delimiters, just sequences */<b>
+  ZSTD_sf_explicitBlockDelimiters = 1    </b>/* ZSTD_Sequence[] contains explicit block delimiters */<b>
+} ZSTD_SequenceFormat_e;
 </b></pre><BR>
 <pre><b>ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
 </b><p> `srcSize` : size of the input buffer
@@ -1285,19 +1332,37 @@ ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr,
  
 </p></pre><BR>
 
-<pre><b></b><p> Generate sequences using ZSTD_compress2(), given a source buffer.
+<pre><b>ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()")
+ZSTDLIB_STATIC_API size_t
+ZSTD_generateSequences(ZSTD_CCtx* zc,
+           ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+           const void* src, size_t srcSize);
+</b><p> WARNING: This function is meant for debugging and informational purposes ONLY!
+ Its implementation is flawed, and it will be deleted in a future version.
+ It is not guaranteed to succeed, as there are several cases where it will give
+ up and fail. You should NOT use this function in production code.
+
+ This function is deprecated, and will be removed in a future version.
+
+ Generate sequences using ZSTD_compress2(), given a source buffer.
+
+ @param zc The compression context to be used for ZSTD_compress2(). Set any
+           compression parameters you need on this context.
+ @param outSeqs The output sequences buffer of size @p outSeqsSize
+ @param outSeqsCapacity The size of the output sequences buffer.
+                    ZSTD_sequenceBound(srcSize) is an upper bound on the number
+                    of sequences that can be generated.
+ @param src The source buffer to generate sequences from of size @p srcSize.
+ @param srcSize The size of the source buffer.
 
  Each block will end with a dummy sequence
  with offset == 0, matchLength == 0, and litLength == length of last literals.
  litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
  simply acts as a block delimiter.
 
- @zc can be used to insert custom compression params.
- This function invokes ZSTD_compress2().
-
- The output of this function can be fed into ZSTD_compressSequences() with CCtx
- setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
- @return : number of sequences generated
+ @returns The number of sequences generated, necessarily less than
+          ZSTD_sequenceBound(srcSize), or an error code that can be checked
+          with ZSTD_isError().
  
 </p></pre><BR>
 
@@ -1315,13 +1380,14 @@ ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr,
 </p></pre><BR>
 
 <pre><b>ZSTDLIB_STATIC_API size_t
-ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
-            const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
-            const void* src, size_t srcSize);
+ZSTD_compressSequences(ZSTD_CCtx* cctx,
+           void* dst, size_t dstCapacity,
+     const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+     const void* src, size_t srcSize);
 </b><p> Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
  @src contains the entire input (not just the literals).
  If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
- If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
+ If a dictionary is included, then the cctx should reference the dict (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.).
  The entire source is compressed into a single frame.
 
  The compression behavior changes based on cctx params. In particular:
@@ -1330,11 +1396,17 @@ ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
     the block size derived from the cctx, and sequences may be split. This is the default setting.
 
     If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
-    block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
+    valid block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
 
-    If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
-    behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
-    specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
+    When ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, it's possible to decide generating repcodes
+    using the advanced parameter ZSTD_c_repcodeResolution. Repcodes will improve compression ratio, though the benefit
+    can vary greatly depending on Sequences. On the other hand, repcode resolution is an expensive operation.
+    By default, it's disabled at low (<10) compression levels, and enabled above the threshold (>=10).
+    ZSTD_c_repcodeResolution makes it possible to directly manage this processing in either direction.
+
+    If ZSTD_c_validateSequences == 0, this function blindly accepts the Sequences provided. Invalid Sequences cause undefined
+    behavior. If ZSTD_c_validateSequences == 1, then the function will detect invalid Sequences (see doc/zstd_compression_format.md for
+    specifics regarding offset/matchlength requirements) and then bail out and return an error.
 
     In addition to the two adjustable experimental params, there are other important cctx params.
     - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
@@ -1342,21 +1414,47 @@ ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
     - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
       is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
 
- Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
- Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
-         and cannot emit an RLE block that disagrees with the repcode history
+ Note: Repcodes are, as of now, always re-calculated within this function, ZSTD_Sequence.rep is effectively unused.
+ Dev Note: Once ability to ingest repcodes become available, the explicit block delims mode must respect those repcodes exactly,
+         and cannot emit an RLE block that disagrees with the repcode history.
+ @return : final compressed size, or a ZSTD error code.
+ 
+</p></pre><BR>
+
+<pre><b>ZSTDLIB_STATIC_API size_t
+ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
+                      void* dst, size_t dstCapacity,
+                const ZSTD_Sequence* inSeqs, size_t nbSequences,
+                const void* literals, size_t litSize, size_t litBufCapacity,
+                size_t decompressedSize);
+</b><p> This is a variant of ZSTD_compressSequences() which,
+ instead of receiving (src,srcSize) as input parameter, receives (literals,litSize),
+ aka all the literals, already extracted and laid out into a single continuous buffer.
+ This can be useful if the process generating the sequences also happens to generate the buffer of literals,
+ thus skipping an extraction + caching stage.
+ It's a speed optimization, useful when the right conditions are met,
+ but it also features the following limitations:
+ - Only supports explicit delimiter mode
+ - Currently does not support Sequences validation (so input Sequences are trusted)
+ - Not compatible with frame checksum, which must be disabled
+ - If any block is incompressible, will fail and return an error
+ - @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error.
+ - @litBufCapacity is the size of the underlying buffer into which literals are written, starting at address @literals.
+   @litBufCapacity must be at least 8 bytes larger than @litSize.
+ - @decompressedSize must be correct, and correspond to the sum of all Sequences. Any discrepancy will generate an error.
  @return : final compressed size, or a ZSTD error code.
  
 </p></pre><BR>
 
 <pre><b>ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
-                                const void* src, size_t srcSize, unsigned magicVariant);
+                                 const void* src, size_t srcSize,
+                                       unsigned magicVariant);
 </b><p> Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
 
  Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
  ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
- As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
- the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
+ As such, the parameter magicVariant controls the exact skippable frame magic number variant used,
+ so the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
 
  Returns an error if destination buffer is not large enough, if the source size is not representable
  with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
@@ -1365,13 +1463,14 @@ ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
  
 </p></pre><BR>
 
-<pre><b>size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
-                                const void* src, size_t srcSize);
-</b><p> Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity,
+                                      unsigned* magicVariant,
+                                      const void* src, size_t srcSize);
+</b><p> Retrieves the content of a zstd skippable frame starting at @src, and writes it to @dst buffer.
 
- The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
- i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START.  This can be NULL if the caller is not interested
- in the magicVariant.
+ The parameter @magicVariant will receive the magicVariant that was supplied when the frame was written,
+ i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START.
+ This can be NULL if the caller is not interested in the magicVariant.
 
  Returns an error if destination buffer is not large enough, or if the frame is not skippable.
 
@@ -1379,65 +1478,68 @@ ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
  
 </p></pre><BR>
 
-<pre><b>unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
+<pre><b>ZSTDLIB_STATIC_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
 </b><p>  Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
  
 </p></pre><BR>
 
 <a name="Chapter16"></a><h2>Memory management</h2><pre></pre>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
 </b><p>  These functions make it possible to estimate memory usage
   of a future {D,C}Ctx, before its creation.
+  This is useful in combination with ZSTD_initStatic(),
+  which makes it possible to employ a static buffer for ZSTD_CCtx* state.
 
   ZSTD_estimateCCtxSize() will provide a memory budget large enough
-  for any compression level up to selected one.
-  Note : Unlike ZSTD_estimateCStreamSize*(), this estimate
-         does not include space for a window buffer.
-         Therefore, the estimation is only guaranteed for single-shot compressions, not streaming.
+  to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2()
+  associated with any compression level up to max specified one.
   The estimate will assume the input may be arbitrarily large,
   which is the worst case.
 
+  Note that the size estimation is specific for one-shot compression,
+  it is not valid for streaming (see ZSTD_estimateCStreamSize*())
+  nor other potential ways of using a ZSTD_CCtx* state.
+
   When srcSize can be bound by a known and rather "small" value,
-  this fact can be used to provide a tighter estimation
-  because the CCtx compression context will need less memory.
-  This tighter estimation can be provided by more advanced functions
+  this knowledge can be used to provide a tighter budget estimation
+  because the ZSTD_CCtx* state will need less memory for small inputs.
+  This tighter estimation can be provided by employing more advanced functions
   ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
   and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
   Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
 
   Note : only single-threaded compression is supported.
   ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
-
-  Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
-  Size estimates assume that no external sequence producer is registered.
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
-</b><p>  ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
-  It will also consider src size to be arbitrarily "large", which is worst case.
+</b><p>  ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression
+  using any compression level up to the max specified one.
+  It will also consider src size to be arbitrarily "large", which is a worst case scenario.
   If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
   ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
   ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
   Note : CStream size estimation is only correct for single-threaded compression.
-  ZSTD_DStream memory budget depends on window Size.
+  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+  Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
+  Size estimates assume that no external sequence producer is registered.
+
+  ZSTD_DStream memory budget depends on frame's window Size.
   This information can be passed manually, using ZSTD_estimateDStreamSize,
   or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
+  Any frame requesting a window size larger than max specified one will be rejected.
   Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
          an internal ?Dict will be created, which additional size is not estimated here.
          In this case, get total size by adding ZSTD_estimate?DictSize
-  Note 2 : only single-threaded compression is supported.
-  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
-  Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
-  Size estimates assume that no external sequence producer is registered.
  
 </p></pre><BR>
 
@@ -1483,13 +1585,14 @@ static
 #ifdef __GNUC__
 __attribute__((__unused__))
 #endif
-ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  </b>/**< this constant defers to stdlib's functions */<b>
 </b><p>  These prototypes make it possible to pass your own allocation/free functions.
   ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
   All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
  
 </p></pre><BR>
 
+<pre><b>ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  </b>/**< this constant defers to stdlib's functions */<b>
+</b></pre><BR>
 <pre><b>typedef struct POOL_ctx_s ZSTD_threadPool;
 ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
 ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool);  </b>/* accept NULL pointer */<b>
@@ -1857,7 +1960,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
        explicitly specified.
 
   start a new frame, using same parameters from previous frame.
-  This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
+  This is typically useful to skip dictionary loading stage, since it will reuse it in-place.
   Note that zcs must be init at least once before using ZSTD_resetCStream().
   If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
   If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
@@ -1918,7 +2021,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 </b><p>
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
 
- re-use decompression parameters from previous init; saves dictionary loading
+ reuse decompression parameters from previous init; saves dictionary loading
  
 </p></pre><BR>
 
@@ -1926,7 +2029,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 ZSTD_registerSequenceProducer(
   ZSTD_CCtx* cctx,
   void* sequenceProducerState,
-  ZSTD_sequenceProducer_F* sequenceProducer
+  ZSTD_sequenceProducer_F sequenceProducer
 );
 </b><p> Instruct zstd to use a block-level external sequence producer function.
 
@@ -1948,6 +2051,22 @@ ZSTD_registerSequenceProducer(
  calling this function. 
 </p></pre><BR>
 
+<pre><b>ZSTDLIB_STATIC_API void
+ZSTD_CCtxParams_registerSequenceProducer(
+  ZSTD_CCtx_params* params,
+  void* sequenceProducerState,
+  ZSTD_sequenceProducer_F sequenceProducer
+);
+</b><p> Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params.
+ This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(),
+ which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx().
+
+ If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx()
+ is required, then this function is for you. Otherwise, you probably don't need it.
+
+ See tests/zstreamtest.c for example usage. 
+</p></pre><BR>
+
 <a name="Chapter20"></a><h2>Buffer-less and synchronous inner streaming functions (DEPRECATED)</h2><pre>
   This API is deprecated, and will be removed in a future version.
   It allows streaming (de)compression with user allocated buffers.
@@ -1964,7 +2083,7 @@ ZSTD_registerSequenceProducer(
 <a name="Chapter21"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
   A ZSTD_CCtx object is required to track streaming operations.
   Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
-  ZSTD_CCtx object can be re-used multiple times within successive compression operations.
+  ZSTD_CCtx object can be reused multiple times within successive compression operations.
 
   Start by initializing a context.
   Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
@@ -1985,7 +2104,7 @@ ZSTD_registerSequenceProducer(
   It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
   Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
 
-  `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
+  `ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again.
 <BR></pre>
 
 <h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
@@ -2002,7 +2121,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const Z
 <a name="Chapter22"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
   A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
-  A ZSTD_DCtx object can be re-used multiple times.
+  A ZSTD_DCtx object can be reused multiple times.
 
   First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
   Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
@@ -2012,7 +2131,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const Z
            >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
            errorCode, which can be tested using ZSTD_isError().
 
-  It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
+  It fills a ZSTD_FrameHeader structure with important information to correctly decode the frame,
   such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
   Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
   As a consequence, check that values remain within valid application range.

examples/streaming_compression.c

@@ -42,7 +42,13 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
      */
     CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel) );
     CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads);
+    if (nbThreads > 1) {
+        size_t const r = ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads);
+        if (ZSTD_isError(r)) {
+            fprintf (stderr, "Note: the linked libzstd library doesn't support multithreading. "
+                             "Reverting to single-thread mode. \n");
+        }
+    }
 
     /* This loop read from the input file, compresses that entire chunk,
      * and writes all output produced to the output file.
@@ -117,7 +123,7 @@ int main(int argc, const char** argv)
     }
 
     int cLevel = 1;
-    int nbThreads = 4;
+    int nbThreads = 1;
 
     if (argc >= 3) {
       cLevel = atoi (argv[2]);

lib/BUCK

@@ -57,7 +57,7 @@ cxx_library(
     srcs=glob(['legacy/*.c']),
     deps=[':common'],
     exported_preprocessor_flags=[
-        '-DZSTD_LEGACY_SUPPORT=4',
+        '-DZSTD_LEGACY_SUPPORT=0',
     ],
 )
 

lib/Makefile

@@ -8,6 +8,9 @@
 # You may select, at your option, one of the above-listed licenses.
 # ################################################################
 
+# default target (when running `make` with no argument)
+lib-release:
+
 # Modules
 ZSTD_LIB_COMPRESSION ?= 1
 ZSTD_LIB_DECOMPRESSION ?= 1
@@ -54,13 +57,14 @@ VERSION := $(ZSTD_VERSION)
 # Note: by default, the static library is built single-threaded and dynamic library is built
 # multi-threaded. It is possible to force multi or single threaded builds by appending
 # -mt or -nomt to the build target (like lib-mt for multi-threaded, lib-nomt for single-threaded).
-.PHONY: default
-default: lib-release
+
 
 CPPFLAGS_DYNLIB  += -DZSTD_MULTITHREAD # dynamic library build defaults to multi-threaded
 LDFLAGS_DYNLIB   += -pthread
-CPPFLAGS_STATLIB +=                    # static library build defaults to single-threaded
+CPPFLAGS_STATICLIB +=                  # static library build defaults to single-threaded
 
+# pkg-config Libs.private points to LDFLAGS_DYNLIB
+PCLIB := $(LDFLAGS_DYNLIB)
 
 ifeq ($(findstring GCC,$(CCVER)),GCC)
 decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
@@ -69,13 +73,15 @@ endif
 
 # macOS linker doesn't support -soname, and use different extension
 # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
-ifeq ($(UNAME), Darwin)
+UNAME_TARGET_SYSTEM ?= $(UNAME)
+
+ifeq ($(UNAME_TARGET_SYSTEM), Darwin)
   SHARED_EXT = dylib
   SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
   SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
   SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
 else
-  ifeq ($(UNAME), AIX)
+  ifeq ($(UNAME_TARGET_SYSTEM), AIX)
     SONAME_FLAGS =
   else
     SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
@@ -91,7 +97,7 @@ all: lib
 
 
 .PHONY: libzstd.a  # must be run every time
-libzstd.a: CPPFLAGS += $(CPPFLAGS_STATLIB)
+libzstd.a: CPPFLAGS += $(CPPFLAGS_STATICLIB)
 
 SET_CACHE_DIRECTORY = \
    +$(MAKE) --no-print-directory $@ \
@@ -109,20 +115,20 @@ libzstd.a:
 else
 # BUILD_DIR is defined
 
-ZSTD_STATLIB_DIR := $(BUILD_DIR)/static
-ZSTD_STATLIB := $(ZSTD_STATLIB_DIR)/libzstd.a
-ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
-$(ZSTD_STATLIB): ARFLAGS = rcs
-$(ZSTD_STATLIB): | $(ZSTD_STATLIB_DIR)
-$(ZSTD_STATLIB): $(ZSTD_STATLIB_OBJ)
+ZSTD_STATICLIB_DIR := $(BUILD_DIR)/static
+ZSTD_STATICLIB := $(ZSTD_STATICLIB_DIR)/libzstd.a
+ZSTD_STATICLIB_OBJ := $(addprefix $(ZSTD_STATICLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
+$(ZSTD_STATICLIB): ARFLAGS = rcs
+$(ZSTD_STATICLIB): | $(ZSTD_STATICLIB_DIR)
+$(ZSTD_STATICLIB): $(ZSTD_STATICLIB_OBJ)
   # Check for multithread flag at target execution time
 	$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
     @echo compiling multi-threaded static library $(LIBVER),\
     @echo compiling single-threaded static library $(LIBVER))
 	$(AR) $(ARFLAGS) $@ $^
 
-libzstd.a: $(ZSTD_STATLIB)
-	cp -f $< $@
+libzstd.a: $(ZSTD_STATICLIB)
+	$(CP) $< $@
 
 endif
 
@@ -160,13 +166,13 @@ $(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ)
 	$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
     @echo compiling multi-threaded dynamic library $(LIBVER),\
     @echo compiling single-threaded dynamic library $(LIBVER))
-	$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
+	$(CC) $(FLAGS) $^ $(SONAME_FLAGS) -o $@
 	@echo creating versioned links
-	ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
-	ln -sf $@ libzstd.$(SHARED_EXT)
+	$(LN) -sf $@ libzstd.$(SHARED_EXT_MAJOR)
+	$(LN) -sf $@ libzstd.$(SHARED_EXT)
 
 $(LIBZSTD): $(ZSTD_DYNLIB)
-	cp -f $< $@
+	$(CP) $< $@
 
 endif  # ifndef BUILD_DIR
 endif  # if windows
@@ -182,14 +188,17 @@ lib : libzstd.a libzstd
 # make does not consider implicit pattern rule for .PHONY target
 
 %-mt : CPPFLAGS_DYNLIB  := -DZSTD_MULTITHREAD
-%-mt : CPPFLAGS_STATLIB := -DZSTD_MULTITHREAD
+%-mt : CPPFLAGS_STATICLIB := -DZSTD_MULTITHREAD
 %-mt : LDFLAGS_DYNLIB   := -pthread
+%-mt : PCLIB :=
+%-mt : PCMTLIB := $(LDFLAGS_DYNLIB)
 %-mt : %
 	@echo multi-threaded build completed
 
 %-nomt : CPPFLAGS_DYNLIB  :=
 %-nomt : LDFLAGS_DYNLIB   :=
-%-nomt : CPPFLAGS_STATLIB :=
+%-nomt : CPPFLAGS_STATICLIB :=
+%-nomt : PCLIB :=
 %-nomt : %
 	@echo single-threaded build completed
 
@@ -200,43 +209,53 @@ lib : libzstd.a libzstd
 
 # Generate .h dependencies automatically
 
-DEPFLAGS = -MT $@ -MMD -MP -MF
+# -MMD: compiler generates dependency information as a side-effect of compilation, without system headers
+# -MP: adds phony target for each dependency other than main file.
+DEPFLAGS = -MMD -MP
 
-$(ZSTD_DYNLIB_DIR)/%.o : %.c $(ZSTD_DYNLIB_DIR)/%.d | $(ZSTD_DYNLIB_DIR)
+# ensure that ZSTD_DYNLIB_DIR exists prior to generating %.o
+$(ZSTD_DYNLIB_DIR)/%.o : %.c | $(ZSTD_DYNLIB_DIR)
 	@echo CC $@
-	$(COMPILE.c) $(DEPFLAGS) $(ZSTD_DYNLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
+	$(COMPILE.c) $(DEPFLAGS) $(OUTPUT_OPTION) $<
 
-$(ZSTD_STATLIB_DIR)/%.o : %.c $(ZSTD_STATLIB_DIR)/%.d | $(ZSTD_STATLIB_DIR)
+$(ZSTD_STATICLIB_DIR)/%.o : %.c | $(ZSTD_STATICLIB_DIR)
 	@echo CC $@
-	$(COMPILE.c) $(DEPFLAGS) $(ZSTD_STATLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
+	$(COMPILE.c) $(DEPFLAGS) $(OUTPUT_OPTION) $<
 
 $(ZSTD_DYNLIB_DIR)/%.o : %.S | $(ZSTD_DYNLIB_DIR)
 	@echo AS $@
 	$(COMPILE.S) $(OUTPUT_OPTION) $<
 
-$(ZSTD_STATLIB_DIR)/%.o : %.S | $(ZSTD_STATLIB_DIR)
+$(ZSTD_STATICLIB_DIR)/%.o : %.S | $(ZSTD_STATICLIB_DIR)
 	@echo AS $@
 	$(COMPILE.S) $(OUTPUT_OPTION) $<
 
-MKDIR ?= mkdir
-$(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR):
-	$(MKDIR) -p $@
+MKDIR ?= mkdir -p
+$(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(ZSTD_STATICLIB_DIR):
+	$(MKDIR) $@
 
-DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(ZSTD_STATLIB_OBJ:.o=.d)
+DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(ZSTD_STATICLIB_OBJ:.o=.d)
 $(DEPFILES):
 
-include $(wildcard $(DEPFILES))
+# The leading '-' means: do not fail is include fails (ex: directory does not exist yet)
+-include $(wildcard $(DEPFILES))
 
 
-# Special case : building library in single-thread mode _and_ without zstdmt_compress.c
-ZSTDMT_FILES = compress/zstdmt_compress.c
-ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(ZSTD_FILES))
+# Special case : build library in single-thread mode _and_ without zstdmt_compress.c
+# Note : we still need threading.c and pool.c for the dictionary builder,
+# but they will correctly behave single-threaded.
+ZSTDMT_FILES = zstdmt_compress.c
+ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(notdir $(ZSTD_FILES)))
 libzstd-nomt: CFLAGS += -fPIC -fvisibility=hidden
 libzstd-nomt: LDFLAGS += -shared
 libzstd-nomt: $(ZSTD_NOMT_FILES)
 	@echo compiling single-thread dynamic library $(LIBVER)
 	@echo files : $(ZSTD_NOMT_FILES)
-	$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
+	@if echo "$(ZSTD_NOMT_FILES)" | tr ' ' '\n' | $(GREP) -q zstdmt; then \
+        echo "Error: Found zstdmt in list."; \
+        exit 1; \
+    fi
+	$(CC) $(FLAGS) $^ $(SONAME_FLAGS) -o $@
 
 .PHONY: clean
 clean:
@@ -249,7 +268,7 @@ clean:
 #-----------------------------------------------------------------------------
 # make install is validated only for below listed environments
 #-----------------------------------------------------------------------------
-ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX))
+ifneq (,$(filter $(INSTALL_OS_LIST),$(UNAME)))
 
 lib: libzstd.pc
 
@@ -280,13 +299,21 @@ PCLIBPREFIX := $(if $(findstring $(LIBDIR),$(PCLIBDIR)),,$${exec_prefix})
 # to PREFIX, rather than as a resolved value.
 PCEXEC_PREFIX := $(if $(HAS_EXPLICIT_EXEC_PREFIX),$(EXEC_PREFIX),$${prefix})
 
-ifneq (,$(filter $(UNAME),FreeBSD NetBSD DragonFly))
+
+ifneq ($(MT),)
+  PCLIB :=
+  PCMTLIB := $(LDFLAGS_DYNLIB)
+else
+  PCLIB := $(LDFLAGS_DYNLIB)
+endif
+
+ifneq (,$(filter FreeBSD NetBSD DragonFly,$(UNAME)))
   PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
 else
   PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
 endif
 
-ifneq (,$(filter $(UNAME),SunOS))
+ifneq (,$(filter SunOS,$(UNAME)))
   INSTALL ?= ginstall
 else
   INSTALL ?= install
@@ -296,6 +323,10 @@ INSTALL_PROGRAM ?= $(INSTALL)
 INSTALL_DATA    ?= $(INSTALL) -m 644
 
 
+# pkg-config library define.
+# For static single-threaded library declare -pthread in Libs.private
+# For static multi-threaded library declare -pthread in Libs and Cflags
+.PHONY: libzstd.pc
 libzstd.pc: libzstd.pc.in
 	@echo creating pkgconfig
 	@sed \
@@ -304,7 +335,8 @@ libzstd.pc: libzstd.pc.in
 	        -e 's|@INCLUDEDIR@|$(PCINCPREFIX)$(PCINCDIR)|' \
 	        -e 's|@LIBDIR@|$(PCLIBPREFIX)$(PCLIBDIR)|' \
 	        -e 's|@VERSION@|$(VERSION)|' \
-	        -e 's|@LIBS_PRIVATE@|$(LDFLAGS_DYNLIB)|' \
+	        -e 's|@LIBS_MT@|$(PCMTLIB)|' \
+	        -e 's|@LIBS_PRIVATE@|$(PCLIB)|' \
 	        $< >$@
 
 .PHONY: install
@@ -331,8 +363,8 @@ install-shared:
 	[ -e $(DESTDIR)$(LIBDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
 	@echo Installing shared library
 	$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR)
-	ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
-	ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+	$(LN) -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+	$(LN) -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
 
 .PHONY: install-includes
 install-includes:

lib/README.md

@@ -7,15 +7,30 @@ in order to make it easier to select or exclude features.
 
 #### Building
 
-`Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
+A `Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
 including commands variables, staged install, directory variables and standard targets.
 - `make` : generates both static and dynamic libraries
-- `make install` : install libraries and headers in target system directories
+- `make install` : install libraries, headers and pkg-config in local system directories
 
-`libzstd` default scope is pretty large, including compression, decompression, dictionary builder,
-and support for decoding legacy formats >= v0.5.0.
+`libzstd` default scope includes compression, decompression, and dictionary builder.
+Note: starting v1.6.0, support for decoding legacy formats is disabled by default.
+See _modular build_ below to learn how to enable it.
 The scope can be reduced on demand (see paragraph _modular build_).
 
+#### Multiarch Support
+
+For multiarch systems (like Debian/Ubuntu), libraries should be installed to architecture-specific directories.
+When creating packages for such systems, use the `LIBDIR` variable to specify the correct multiarch path:
+
+```bash
+# For x86_64 systems on Ubuntu/Debian:
+make install PREFIX=/usr LIBDIR=/usr/lib/x86_64-linux-gnu
+
+# For ARM64 systems on Ubuntu/Debian:
+make install PREFIX=/usr LIBDIR=/usr/lib/aarch64-linux-gnu
+```
+
+This will not only install the files in the correct directories, but also generate the correct paths for `pkg-config`.
 
 #### Multithreading support
 
@@ -27,12 +42,16 @@ Enabling multithreading requires 2 conditions :
 
 For convenience, we provide a build target to generate multi and single threaded libraries:
 - Force enable multithreading on both dynamic and static libraries by appending `-mt` to the target, e.g. `make lib-mt`.
+  Note that the `.pc` generated on calling `make lib-mt` will already include the require Libs and Cflags.
 - Force disable multithreading on both dynamic and static libraries by appending `-nomt` to the target, e.g. `make lib-nomt`.
 - By default, as mentioned before, dynamic library is multithreaded, and static library is single-threaded, e.g. `make lib`.
 
 When linking a POSIX program with a multithreaded version of `libzstd`,
 note that it's necessary to invoke the `-pthread` flag during link stage.
 
+The `.pc` generated from `make install` or `make install-pc` always assume a single-threaded static library
+is compiled. To correctly generate a `.pc` for the multi-threaded static library, set `MT=1` as ENV variable.
+
 Multithreading capabilities are exposed
 via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.4.3/lib/zstd.h#L351).
 
@@ -81,14 +100,14 @@ The file structure is designed to make this selection manually achievable for an
         Specifying a number limits versions supported to that version onward.
         For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0".
         Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats".
-        By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`.
+        By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=0` (disabled).
         Decoding supported legacy format is a transparent capability triggered within decompression functions.
         It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`.
         Each version does also provide its own set of advanced API.
         For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
 
 - While invoking `make libzstd`, it's possible to define build macros
-        `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
+        `ZSTD_LIB_COMPRESSION`, `ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
         and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the
         corresponding features. This will also disable compilation of all
         dependencies (e.g. `ZSTD_LIB_COMPRESSION=0` will also disable
@@ -119,6 +138,15 @@ The file structure is designed to make this selection manually achievable for an
   binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and
   `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` (implied by `ZSTD_LIB_MINIFY`).
 
+  On the compressor side, Zstd's compression levels map to several internal
+  strategies. In environments where the higher compression levels aren't used,
+  it is possible to exclude all but the fastest strategy with
+  `ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP=1`. (Note that this will change
+  the behavior of the default compression level.) Or if you want to retain the
+  default compressor as well, you can set
+  `ZSTD_LIB_EXCLUDE_COMPRESSORS_GREEDY_AND_UP=1`, at the cost of an additional
+  ~20KB or so.
+
   For squeezing the last ounce of size out, you can also define
   `ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`,
   which removes the error messages that are otherwise returned by
@@ -136,6 +164,13 @@ The file structure is designed to make this selection manually achievable for an
   will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in
   the shared library, which is now hidden by default.
 
+- The build macro `STATIC_BMI2` can be set to 1 to force usage of `bmi2` instructions.
+  It is generally not necessary to set this build macro,
+  because `STATIC_BMI2` will be automatically set to 1
+  on detecting the presence of the corresponding instruction set in the compilation target.
+  It's nonetheless available as an optional manual toggle for better control,
+  and can also be used to forcefully disable `bmi2` instructions by setting it to 0.
+
 - The build macro `DYNAMIC_BMI2` can be set to 1 or 0 in order to generate binaries
   which can detect at runtime the presence of BMI2 instructions, and use them only if present.
   These instructions contribute to better performance, notably on the decoder side.
@@ -169,6 +204,15 @@ The file structure is designed to make this selection manually achievable for an
   `ZSTDERRORLIB_VSIBILITY`, and `ZDICTLIB_VISIBILITY` if unset, for backwards compatibility
   with the old macro names.
 
+- The C compiler macro `HUF_DISABLE_FAST_DECODE` disables the newer Huffman fast C
+  and assembly decoding loops. You may want to use this macro if these loops are
+  slower on your platform.
+
+- The macro `ZDICT_QSORT` can enforce selection of a specific sorting variant,
+  which is useful when autodetection fails, for example with older versions of `musl`.
+  For this scenario, it can be set as `ZDICT_QSORT=ZDICT_QSORT_C90`.
+  Other selectable suffixes are `_GNU`, `_APPLE`, `_MSVC` and `_C11`.
+
 #### Windows : using MinGW+MSYS to create DLL
 
 DLL can be created using MinGW+MSYS with the `make libzstd` command.

lib/common/allocations.h

@@ -14,7 +14,7 @@
 #define ZSTD_DEPS_NEED_MALLOC
 #include "zstd_deps.h"   /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
 
-#include "mem.h" /* MEM_STATIC */
+#include "compiler.h" /* MEM_STATIC */
 #define ZSTD_STATIC_LINKING_ONLY
 #include "../zstd.h" /* ZSTD_customMem */
 
@@ -33,9 +33,13 @@ MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
 MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
 {
     if (customMem.customAlloc) {
-        /* calloc implemented as malloc+memset;
-         * not as efficient as calloc, but next best guess for custom malloc */
+        /* calloc implemented as malloc+memset */
         void* const ptr = customMem.customAlloc(customMem.opaque, size);
+
+        if (ptr == NULL) {
+            return NULL;
+        }
+
         ZSTD_memset(ptr, 0, size);
         return ptr;
     }

lib/common/bits.h

@@ -21,34 +21,36 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val)
                                                 30, 22, 20, 15, 25, 17, 4, 8,
                                                 31, 27, 13, 23, 21, 19, 16, 7,
                                                 26, 12, 18, 6, 11, 5, 10, 9};
-        return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27];
+        return DeBruijnBytePos[((U32) ((val & (0-val)) * 0x077CB531U)) >> 27];
     }
 }
 
 MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
 {
     assert(val != 0);
-#   if defined(_MSC_VER)
-#       if STATIC_BMI2 == 1
-            return (unsigned)_tzcnt_u32(val);
-#       else
-            if (val != 0) {
-                unsigned long r;
-                _BitScanForward(&r, val);
-                return (unsigned)r;
-            } else {
-                /* Should not reach this code path */
-                __assume(0);
-            }
-#       endif
-#   elif defined(__GNUC__) && (__GNUC__ >= 4)
-        return (unsigned)__builtin_ctz(val);
-#   else
-        return ZSTD_countTrailingZeros32_fallback(val);
-#   endif
+#if defined(_MSC_VER)
+#  if STATIC_BMI2
+    return (unsigned)_tzcnt_u32(val);
+#  else
+    if (val != 0) {
+        unsigned long r;
+        _BitScanForward(&r, val);
+        return (unsigned)r;
+    } else {
+        __assume(0); /* Should not reach this code path */
+    }
+#  endif
+#elif defined(__GNUC__) && (__GNUC__ >= 4)
+    return (unsigned)__builtin_ctz(val);
+#elif defined(__ICCARM__)
+    return (unsigned)__builtin_ctz(val);
+#else
+    return ZSTD_countTrailingZeros32_fallback(val);
+#endif
 }
 
-MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) {
+MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val)
+{
     assert(val != 0);
     {
         static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29,
@@ -67,86 +69,89 @@ MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) {
 MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
 {
     assert(val != 0);
-#   if defined(_MSC_VER)
-#       if STATIC_BMI2 == 1
-            return (unsigned)_lzcnt_u32(val);
-#       else
-            if (val != 0) {
-                unsigned long r;
-                _BitScanReverse(&r, val);
-                return (unsigned)(31 - r);
-            } else {
-                /* Should not reach this code path */
-                __assume(0);
-            }
-#       endif
-#   elif defined(__GNUC__) && (__GNUC__ >= 4)
-        return (unsigned)__builtin_clz(val);
-#   else
-        return ZSTD_countLeadingZeros32_fallback(val);
-#   endif
+#if defined(_MSC_VER)
+#  if STATIC_BMI2
+    return (unsigned)_lzcnt_u32(val);
+#  else
+    if (val != 0) {
+        unsigned long r;
+        _BitScanReverse(&r, val);
+        return (unsigned)(31 - r);
+    } else {
+        __assume(0); /* Should not reach this code path */
+    }
+#  endif
+#elif defined(__GNUC__) && (__GNUC__ >= 4)
+    return (unsigned)__builtin_clz(val);
+#elif defined(__ICCARM__)
+    return (unsigned)__builtin_clz(val);
+#else
+    return ZSTD_countLeadingZeros32_fallback(val);
+#endif
 }
 
 MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
 {
     assert(val != 0);
-#   if defined(_MSC_VER) && defined(_WIN64)
-#       if STATIC_BMI2 == 1
-            return (unsigned)_tzcnt_u64(val);
-#       else
-            if (val != 0) {
-                unsigned long r;
-                _BitScanForward64(&r, val);
-                return (unsigned)r;
-            } else {
-                /* Should not reach this code path */
-                __assume(0);
-            }
-#       endif
-#   elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
-        return (unsigned)__builtin_ctzll(val);
-#   else
-        {
-            U32 mostSignificantWord = (U32)(val >> 32);
-            U32 leastSignificantWord = (U32)val;
-            if (leastSignificantWord == 0) {
-                return 32 + ZSTD_countTrailingZeros32(mostSignificantWord);
-            } else {
-                return ZSTD_countTrailingZeros32(leastSignificantWord);
-            }
+#if defined(_MSC_VER) && defined(_WIN64)
+#  if STATIC_BMI2
+    return (unsigned)_tzcnt_u64(val);
+#  else
+    if (val != 0) {
+        unsigned long r;
+        _BitScanForward64(&r, val);
+        return (unsigned)r;
+    } else {
+        __assume(0); /* Should not reach this code path */
+    }
+#  endif
+#elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
+    return (unsigned)__builtin_ctzll(val);
+#elif defined(__ICCARM__)
+    return (unsigned)__builtin_ctzll(val);
+#else
+    {
+        U32 mostSignificantWord = (U32)(val >> 32);
+        U32 leastSignificantWord = (U32)val;
+        if (leastSignificantWord == 0) {
+            return 32 + ZSTD_countTrailingZeros32(mostSignificantWord);
+        } else {
+            return ZSTD_countTrailingZeros32(leastSignificantWord);
         }
-#   endif
+    }
+#endif
 }
 
 MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
 {
     assert(val != 0);
-#   if defined(_MSC_VER) && defined(_WIN64)
-#       if STATIC_BMI2 == 1
-            return (unsigned)_lzcnt_u64(val);
-#       else
-            if (val != 0) {
-                unsigned long r;
-                _BitScanReverse64(&r, val);
-                return (unsigned)(63 - r);
-            } else {
-                /* Should not reach this code path */
-                __assume(0);
-            }
-#       endif
-#   elif defined(__GNUC__) && (__GNUC__ >= 4)
-        return (unsigned)(__builtin_clzll(val));
-#   else
-        {
-            U32 mostSignificantWord = (U32)(val >> 32);
-            U32 leastSignificantWord = (U32)val;
-            if (mostSignificantWord == 0) {
-                return 32 + ZSTD_countLeadingZeros32(leastSignificantWord);
-            } else {
-                return ZSTD_countLeadingZeros32(mostSignificantWord);
-            }
+#if defined(_MSC_VER) && defined(_WIN64)
+#  if STATIC_BMI2
+    return (unsigned)_lzcnt_u64(val);
+#  else
+    if (val != 0) {
+        unsigned long r;
+        _BitScanReverse64(&r, val);
+        return (unsigned)(63 - r);
+    } else {
+        __assume(0); /* Should not reach this code path */
+    }
+#  endif
+#elif defined(__GNUC__) && (__GNUC__ >= 4)
+    return (unsigned)(__builtin_clzll(val));
+#elif defined(__ICCARM__)
+    return (unsigned)(__builtin_clzll(val));
+#else
+    {
+        U32 mostSignificantWord = (U32)(val >> 32);
+        U32 leastSignificantWord = (U32)val;
+        if (mostSignificantWord == 0) {
+            return 32 + ZSTD_countLeadingZeros32(leastSignificantWord);
+        } else {
+            return ZSTD_countLeadingZeros32(mostSignificantWord);
         }
-#   endif
+    }
+#endif
 }
 
 MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val)

lib/common/bitstream.h

@@ -14,9 +14,6 @@
 #ifndef BITSTREAM_H_MODULE
 #define BITSTREAM_H_MODULE
 
-#if defined (__cplusplus)
-extern "C" {
-#endif
 /*
 *  This API consists of small unitary functions, which must be inlined for best performance.
 *  Since link-time-optimization is not available for all compilers,
@@ -32,7 +29,6 @@ extern "C" {
 #include "error_private.h"  /* error codes and messages */
 #include "bits.h"           /* ZSTD_highbit32 */
 
-
 /*=========================================
 *  Target specific
 =========================================*/
@@ -52,12 +48,13 @@ extern "C" {
 /*-******************************************
 *  bitStream encoding API (write forward)
 ********************************************/
+typedef size_t BitContainerType;
 /* bitStream can mix input from multiple sources.
  * A critical property of these streams is that they encode and decode in **reverse** direction.
  * So the first bit sequence you add will be the last to be read, like a LIFO stack.
  */
 typedef struct {
-    size_t bitContainer;
+    BitContainerType bitContainer;
     unsigned bitPos;
     char*  startPtr;
     char*  ptr;
@@ -65,7 +62,7 @@ typedef struct {
 } BIT_CStream_t;
 
 MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
-MEM_STATIC void   BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
+MEM_STATIC void   BIT_addBits(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits);
 MEM_STATIC void   BIT_flushBits(BIT_CStream_t* bitC);
 MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
 
@@ -74,7 +71,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
 *  `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
 *
 *  bits are first added to a local register.
-*  Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
+*  Local register is BitContainerType, 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
 *  Writing data into memory is an explicit operation, performed by the flushBits function.
 *  Hence keep track how many bits are potentially stored into local register to avoid register overflow.
 *  After a flushBits, a maximum of 7 bits might still be stored into local register.
@@ -91,28 +88,28 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
 *  bitStream decoding API (read backward)
 **********************************************/
 typedef struct {
-    size_t   bitContainer;
+    BitContainerType bitContainer;
     unsigned bitsConsumed;
     const char* ptr;
     const char* start;
     const char* limitPtr;
 } BIT_DStream_t;
 
-typedef enum { BIT_DStream_unfinished = 0,
-               BIT_DStream_endOfBuffer = 1,
-               BIT_DStream_completed = 2,
-               BIT_DStream_overflow = 3 } BIT_DStream_status;  /* result of BIT_reloadDStream() */
-               /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
+typedef enum { BIT_DStream_unfinished = 0,  /* fully refilled */
+               BIT_DStream_endOfBuffer = 1, /* still some bits left in bitstream */
+               BIT_DStream_completed = 2,   /* bitstream entirely consumed, bit-exact */
+               BIT_DStream_overflow = 3     /* user requested more bits than present in bitstream */
+    } BIT_DStream_status;  /* result of BIT_reloadDStream() */
 
 MEM_STATIC size_t   BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
-MEM_STATIC size_t   BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
-MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
+FORCE_INLINE_TEMPLATE BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
+FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
 MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
 
 
 /* Start by invoking BIT_initDStream().
 *  A chunk of the bitStream is then stored into a local register.
-*  Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
+*  Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (BitContainerType).
 *  You can then retrieve bitFields stored into the local register, **in reverse order**.
 *  Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
 *  A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
@@ -124,7 +121,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
 /*-****************************************
 *  unsafe API
 ******************************************/
-MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
+MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits);
 /* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
 
 MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
@@ -162,10 +159,15 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
     return 0;
 }
 
-MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
+FORCE_INLINE_TEMPLATE BitContainerType BIT_getLowerBits(BitContainerType bitContainer, U32 const nbBits)
 {
-#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS)
-    return  _bzhi_u64(bitContainer, nbBits);
+#if STATIC_BMI2 && !defined(ZSTD_NO_INTRINSICS)
+#  if (defined(__x86_64__) || defined(_M_X64)) && !defined(__ILP32__)
+    return _bzhi_u64(bitContainer, nbBits);
+#  else
+    DEBUG_STATIC_ASSERT(sizeof(bitContainer) == sizeof(U32));
+    return _bzhi_u32(bitContainer, nbBits);
+#  endif
 #else
     assert(nbBits < BIT_MASK_SIZE);
     return bitContainer & BIT_mask[nbBits];
@@ -176,7 +178,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 co
  *  can add up to 31 bits into `bitC`.
  *  Note : does not check for register overflow ! */
 MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
-                            size_t value, unsigned nbBits)
+                            BitContainerType value, unsigned nbBits)
 {
     DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32);
     assert(nbBits < BIT_MASK_SIZE);
@@ -189,7 +191,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
  *  works only if `value` is _clean_,
  *  meaning all high bits above nbBits are 0 */
 MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
-                                size_t value, unsigned nbBits)
+                                BitContainerType value, unsigned nbBits)
 {
     assert((value>>nbBits) == 0);
     assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
@@ -236,7 +238,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
     BIT_addBitsFast(bitC, 1, 1);   /* endMark */
     BIT_flushBits(bitC);
     if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
-    return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
+    return (size_t)(bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
 }
 
 
@@ -267,22 +269,22 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
         bitD->bitContainer = *(const BYTE*)(bitD->start);
         switch(srcSize)
         {
-        case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
+        case 7: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
                 ZSTD_FALLTHROUGH;
 
-        case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
+        case 6: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
                 ZSTD_FALLTHROUGH;
 
-        case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
+        case 5: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
                 ZSTD_FALLTHROUGH;
 
-        case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
+        case 4: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[3]) << 24;
                 ZSTD_FALLTHROUGH;
 
-        case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
+        case 3: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[2]) << 16;
                 ZSTD_FALLTHROUGH;
 
-        case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) <<  8;
+        case 2: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[1]) <<  8;
                 ZSTD_FALLTHROUGH;
 
         default: break;
@@ -297,12 +299,12 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
     return srcSize;
 }
 
-MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
+FORCE_INLINE_TEMPLATE BitContainerType BIT_getUpperBits(BitContainerType bitContainer, U32 const start)
 {
     return bitContainer >> start;
 }
 
-MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
+FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits)
 {
     U32 const regMask = sizeof(bitContainer)*8 - 1;
     /* if start > regMask, bitstream is corrupted, and result is undefined */
@@ -312,7 +314,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 c
      * such cpus old (pre-Haswell, 2013) and their performance is not of that
      * importance.
      */
-#if defined(__x86_64__) || defined(_M_X86)
+#if defined(__x86_64__) || defined(_M_X64)
     return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
 #else
     return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
@@ -325,7 +327,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 c
  *  On 32-bits, maxNbBits==24.
  *  On 64-bits, maxNbBits==56.
  * @return : value extracted */
-MEM_STATIC  FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t*  bitD, U32 nbBits)
+FORCE_INLINE_TEMPLATE BitContainerType BIT_lookBits(const BIT_DStream_t*  bitD, U32 nbBits)
 {
     /* arbitrate between double-shift and shift+mask */
 #if 1
@@ -341,14 +343,14 @@ MEM_STATIC  FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t*  bitD, U3
 
 /*! BIT_lookBitsFast() :
  *  unsafe version; only works if nbBits >= 1 */
-MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
+MEM_STATIC BitContainerType BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
 {
     U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
     assert(nbBits >= 1);
     return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);
 }
 
-MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
+FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
 {
     bitD->bitsConsumed += nbBits;
 }
@@ -357,23 +359,38 @@ MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
  *  Read (consume) next n bits from local register and update.
  *  Pay attention to not read more than nbBits contained into local register.
  * @return : extracted value. */
-MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
+FORCE_INLINE_TEMPLATE BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
 {
-    size_t const value = BIT_lookBits(bitD, nbBits);
+    BitContainerType const value = BIT_lookBits(bitD, nbBits);
     BIT_skipBits(bitD, nbBits);
     return value;
 }
 
 /*! BIT_readBitsFast() :
  *  unsafe version; only works if nbBits >= 1 */
-MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
+MEM_STATIC BitContainerType BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
 {
-    size_t const value = BIT_lookBitsFast(bitD, nbBits);
+    BitContainerType const value = BIT_lookBitsFast(bitD, nbBits);
     assert(nbBits >= 1);
     BIT_skipBits(bitD, nbBits);
     return value;
 }
 
+/*! BIT_reloadDStream_internal() :
+ *  Simple variant of BIT_reloadDStream(), with two conditions:
+ *  1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8
+ *  2. look window is valid after shifted down : bitD->ptr >= bitD->start
+ */
+MEM_STATIC BIT_DStream_status BIT_reloadDStream_internal(BIT_DStream_t* bitD)
+{
+    assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);
+    bitD->ptr -= bitD->bitsConsumed >> 3;
+    assert(bitD->ptr >= bitD->start);
+    bitD->bitsConsumed &= 7;
+    bitD->bitContainer = MEM_readLEST(bitD->ptr);
+    return BIT_DStream_unfinished;
+}
+
 /*! BIT_reloadDStreamFast() :
  *  Similar to BIT_reloadDStream(), but with two differences:
  *  1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
@@ -384,31 +401,35 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
 {
     if (UNLIKELY(bitD->ptr < bitD->limitPtr))
         return BIT_DStream_overflow;
-    assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);
-    bitD->ptr -= bitD->bitsConsumed >> 3;
-    bitD->bitsConsumed &= 7;
-    bitD->bitContainer = MEM_readLEST(bitD->ptr);
-    return BIT_DStream_unfinished;
+    return BIT_reloadDStream_internal(bitD);
 }
 
 /*! BIT_reloadDStream() :
  *  Refill `bitD` from buffer previously set in BIT_initDStream() .
- *  This function is safe, it guarantees it will not read beyond src buffer.
+ *  This function is safe, it guarantees it will not never beyond src buffer.
  * @return : status of `BIT_DStream_t` internal register.
  *           when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
-MEM_STATIC FORCE_INLINE_ATTR BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
+FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
 {
-    if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* overflow detected, like end of stream */
+    /* note : once in overflow mode, a bitstream remains in this mode until it's reset */
+    if (UNLIKELY(bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))) {
+        static const BitContainerType zeroFilled = 0;
+        bitD->ptr = (const char*)&zeroFilled; /* aliasing is allowed for char */
+        /* overflow detected, erroneous scenario or end of stream: no update */
         return BIT_DStream_overflow;
+    }
+
+    assert(bitD->ptr >= bitD->start);
 
     if (bitD->ptr >= bitD->limitPtr) {
-        return BIT_reloadDStreamFast(bitD);
+        return BIT_reloadDStream_internal(bitD);
     }
     if (bitD->ptr == bitD->start) {
+        /* reached end of bitStream => no update */
         if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
         return BIT_DStream_completed;
     }
-    /* start < ptr < limitPtr */
+    /* start < ptr < limitPtr => cautious update */
     {   U32 nbBytes = bitD->bitsConsumed >> 3;
         BIT_DStream_status result = BIT_DStream_unfinished;
         if (bitD->ptr - nbBytes < bitD->start) {
@@ -430,8 +451,4 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
     return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
 }
 
-#if defined (__cplusplus)
-}
-#endif
-
 #endif /* BITSTREAM_H_MODULE */