From 6893a6855920a88484549b10a67dfd75df6f1022 Mon Sep 17 00:00:00 2001 From: Daniil Bystrukhin Date: Sat, 21 Feb 2026 23:13:33 -0600 Subject: [PATCH] Add LZMA2 encoding support for 7z writer --- .../SevenZip/SevenZipStreamsCompressor.cs | 67 +++- .../Compressors/LZMA/Lzma2EncoderStream.cs | 294 ++++++++++++++++++ .../Compressors/LZMA/LzmaEncoderProperties.cs | 12 + .../Writers/SevenZip/SevenZipWriter.cs | 10 +- .../SevenZip/SevenZipWriterTests.cs | 150 ++++++++- 5 files changed, 508 insertions(+), 25 deletions(-) create mode 100644 src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs diff --git a/src/SharpCompress/Common/SevenZip/SevenZipStreamsCompressor.cs b/src/SharpCompress/Common/SevenZip/SevenZipStreamsCompressor.cs index 44241c24..6b17a064 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipStreamsCompressor.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipStreamsCompressor.cs @@ -37,22 +37,52 @@ internal sealed class SevenZipStreamsCompressor(Stream outputStream) LzmaEncoderProperties? encoderProperties = null ) { - encoderProperties ??= new LzmaEncoderProperties(eos: true); + encoderProperties ??= new LzmaEncoderProperties(eos: !isLzma2); var outStartOffset = outputStream.Position; - var inStartOffset = inputStream.CanSeek ? inputStream.Position : 0L; // Wrap the output stream in CRC calculator using var outCrcStream = new Crc32Stream(outputStream); - // Create LZMA encoder writing to CRC-wrapped output - using var lzmaStream = LzmaStream.Create(encoderProperties, isLzma2, outCrcStream); - var properties = lzmaStream.Properties; + byte[] properties; + + if (isLzma2) + { + // LZMA2: use Lzma2EncoderStream for chunk-based framing + using var lzma2Stream = new Lzma2EncoderStream( + outCrcStream, + encoderProperties.DictionarySize, + encoderProperties.NumFastBytes + ); + + // Copy input through the LZMA2 encoder while computing input CRC + CopyWithCrc(inputStream, lzma2Stream, out var inputCrc2, out var inputSize2); + + // Flush/finalize (writes remaining buffer + end marker) + lzma2Stream.Dispose(); + + var compressedSize2 = (ulong)(outputStream.Position - outStartOffset); + var uncompressedSize2 = (ulong)inputSize2; + var outputCrc2 = outCrcStream.Crc; + + properties = lzma2Stream.Properties; + + return BuildPackedStream( + isLzma2: true, + properties, + compressedSize2, + uncompressedSize2, + inputCrc2, + outputCrc2 + ); + } + + // LZMA: existing path + using var lzmaStream = LzmaStream.Create(encoderProperties, false, outCrcStream); + properties = lzmaStream.Properties; // Copy input through the LZMA encoder while computing input CRC - uint inputCrc; - long inputSize; - CopyWithCrc(inputStream, lzmaStream, out inputCrc, out inputSize); + CopyWithCrc(inputStream, lzmaStream, out var inputCrc, out var inputSize); // Flush/finalize the LZMA encoder (writes remaining compressed data) lzmaStream.Dispose(); @@ -61,10 +91,27 @@ internal sealed class SevenZipStreamsCompressor(Stream outputStream) var uncompressedSize = (ulong)inputSize; var outputCrc = outCrcStream.Crc; - // Build method ID + return BuildPackedStream( + isLzma2: false, + properties, + compressedSize, + uncompressedSize, + inputCrc, + outputCrc + ); + } + + private static PackedStream BuildPackedStream( + bool isLzma2, + byte[] properties, + ulong compressedSize, + ulong uncompressedSize, + uint inputCrc, + uint? outputCrc + ) + { var methodId = isLzma2 ? CMethodId.K_LZMA2 : CMethodId.K_LZMA; - // Build folder metadata var folder = new CFolder(); folder._coders.Add( new CCoderInfo diff --git a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs new file mode 100644 index 00000000..2d6fe995 --- /dev/null +++ b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs @@ -0,0 +1,294 @@ +using System; +using System.IO; + +namespace SharpCompress.Compressors.LZMA; + +/// +/// Write-only stream that compresses data using the LZMA2 framing format. +/// Buffers input, compresses in chunks, and writes LZMA2-framed output to the underlying stream. +/// Each chunk is independently compressed with a fresh LZMA encoder. +/// +internal sealed class Lzma2EncoderStream : Stream +{ + // Max uncompressed chunk size per LZMA2 spec: (0x1F << 16) + 0xFFFF + 1 = 2MB + private const int MAX_UNCOMPRESSED_CHUNK_SIZE = (0x1F << 16) + 0xFFFF + 1; + + // Max compressed payload per LZMA2 chunk header: 0xFFFF + 1 = 64KB + private const int MAX_COMPRESSED_CHUNK_SIZE = 0xFFFF + 1; + + // Max uncompressed sub-chunk for raw (uncompressed) chunks: 0xFFFF + 1 = 64KB + private const int MAX_UNCOMPRESSED_SUBCHUNK_SIZE = 0xFFFF + 1; + + private readonly Stream _output; + private readonly int _dictionarySize; + private readonly int _numFastBytes; + private readonly byte[] _buffer; + private int _bufferPosition; + private bool _isFirstChunk = true; + private bool _isDisposed; + private byte _lzmaPropertiesByte; + private bool _lzmaPropertiesKnown; + + /// + /// Creates a new LZMA2 encoder stream. + /// + /// The stream to write LZMA2-framed compressed data to. + /// Dictionary size for LZMA compression. + /// Number of fast bytes for LZMA compression. + public Lzma2EncoderStream(Stream output, int dictionarySize, int numFastBytes) + { + _output = output; + _dictionarySize = dictionarySize; + _numFastBytes = numFastBytes; + _buffer = new byte[MAX_UNCOMPRESSED_CHUNK_SIZE]; + _bufferPosition = 0; + } + + /// + /// Gets the 1-byte LZMA2 properties (encoded dictionary size). + /// + public byte[] Properties => [EncodeDictionarySize(_dictionarySize)]; + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + while (count > 0) + { + var toCopy = Math.Min(count, _buffer.Length - _bufferPosition); + Buffer.BlockCopy(buffer, offset, _buffer, _bufferPosition, toCopy); + _bufferPosition += toCopy; + offset += toCopy; + count -= toCopy; + + if (_bufferPosition == _buffer.Length) + { + FlushChunk(); + } + } + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing && !_isDisposed) + { + _isDisposed = true; + + // Flush remaining buffered data + if (_bufferPosition > 0) + { + FlushChunk(); + } + + // Write LZMA2 end marker + _output.WriteByte(0x00); + } + base.Dispose(disposing); + } + + private void FlushChunk() + { + if (_bufferPosition == 0) + { + return; + } + + var uncompressedData = _buffer.AsSpan(0, _bufferPosition); + _bufferPosition = 0; + + // Try compressing the data + byte[] compressed; + try + { + compressed = CompressBlock(uncompressedData); + } + catch + { + // If compression fails, write as uncompressed + WriteUncompressedChunks(uncompressedData); + return; + } + + // Check if compressed output fits in a single chunk and is actually smaller + if (compressed.Length <= MAX_COMPRESSED_CHUNK_SIZE && compressed.Length < uncompressedData.Length) + { + WriteCompressedChunk(uncompressedData.Length, compressed); + } + else + { + WriteUncompressedChunks(uncompressedData); + } + } + + private byte[] CompressBlock(ReadOnlySpan data) + { + var encoderProps = new LzmaEncoderProperties( + eos: false, + _dictionarySize, + _numFastBytes + ); + + var encoder = new Encoder(); + encoder.SetCoderProperties(encoderProps.PropIDs, encoderProps.Properties); + + // Capture the LZMA properties byte (pb/lp/lc encoding) for the chunk header + if (!_lzmaPropertiesKnown) + { + var propBytes = new byte[5]; + encoder.WriteCoderProperties(propBytes); + _lzmaPropertiesByte = propBytes[0]; + _lzmaPropertiesKnown = true; + } + + using var inputMs = new MemoryStream(data.ToArray(), writable: false); + using var outputMs = new MemoryStream(); + + encoder.Code(inputMs, outputMs, data.Length, -1, null); + + var fullCompressed = outputMs.ToArray(); + + // The LZMA range encoder flush writes trailing bytes the decoder doesn't consume. + // Trial-decode to find the exact byte count the decoder needs, so the LZMA2 + // chunk header reports a compressed size that matches what the decoder reads. + var consumed = FindConsumedBytes(fullCompressed, data.Length); + if (consumed < fullCompressed.Length) + { + return fullCompressed.AsSpan(0, consumed).ToArray(); + } + + return fullCompressed; + } + + private int FindConsumedBytes(byte[] compressedData, int uncompressedSize) + { + // Build 5-byte LZMA property header: [pb/lp/lc byte] [dictSize as LE int32] + var props = new byte[5]; + props[0] = _lzmaPropertiesByte; + props[1] = (byte)_dictionarySize; + props[2] = (byte)(_dictionarySize >> 8); + props[3] = (byte)(_dictionarySize >> 16); + props[4] = (byte)(_dictionarySize >> 24); + + var decoder = new Decoder(); + decoder.SetDecoderProperties(props); + + using var input = new MemoryStream(compressedData); + using var output = new MemoryStream(); + decoder.Code(input, output, compressedData.Length, uncompressedSize, null); + + return (int)input.Position; + } + + /// + /// Writes a compressed LZMA2 chunk. + /// Header: [control] [uncompSize_hi] [uncompSize_lo] [compSize_hi] [compSize_lo] [props?] + /// + private void WriteCompressedChunk(int uncompressedSize, byte[] compressedData) + { + var uncompSizeMinus1 = uncompressedSize - 1; + var compSizeMinus1 = compressedData.Length - 1; + + // Each chunk is compressed independently with a fresh LZMA encoder, + // so we must use 0xE0 (full reset: dictionary + state + properties) every time. + // The decoder uses outWindow.Total for literal context and posState; + // 0xE0 triggers outWindow.Reset() which zeros Total, matching the encoder's + // assumption that position starts at 0 for each chunk. + var control = (byte)(0xE0 | ((uncompSizeMinus1 >> 16) & 0x1F)); + _isFirstChunk = false; + + _output.WriteByte(control); + _output.WriteByte((byte)((uncompSizeMinus1 >> 8) & 0xFF)); + _output.WriteByte((byte)(uncompSizeMinus1 & 0xFF)); + _output.WriteByte((byte)((compSizeMinus1 >> 8) & 0xFF)); + _output.WriteByte((byte)(compSizeMinus1 & 0xFF)); + + // 0xE0 (>= 0xC0) requires properties byte + _output.WriteByte(_lzmaPropertiesByte); + + _output.Write(compressedData, 0, compressedData.Length); + } + + /// + /// Writes data as uncompressed LZMA2 sub-chunks (max 64KB each). + /// Header: [control] [size_hi] [size_lo] + /// + private void WriteUncompressedChunks(ReadOnlySpan data) + { + var offset = 0; + while (offset < data.Length) + { + var chunkSize = Math.Min(data.Length - offset, MAX_UNCOMPRESSED_SUBCHUNK_SIZE); + var sizeMinus1 = chunkSize - 1; + + byte control; + if (_isFirstChunk) + { + // 0x01: uncompressed with dictionary reset + control = 0x01; + _isFirstChunk = false; + } + else + { + // 0x02: uncompressed without dictionary reset + control = 0x02; + } + + _output.WriteByte(control); + _output.WriteByte((byte)((sizeMinus1 >> 8) & 0xFF)); + _output.WriteByte((byte)(sizeMinus1 & 0xFF)); + + _output.Write(data.Slice(offset, chunkSize)); + offset += chunkSize; + } + } + + /// + /// Encodes a dictionary size into the 1-byte LZMA2 properties format. + /// Reverse of the decoder formula: dictSize = (2 | (p and 1)) shl ((p shr 1) + 11) + /// Finds the smallest p where the formula result >= target dictSize. + /// + internal static byte EncodeDictionarySize(int dictSize) + { + // Special case: very small dictionary sizes + if (dictSize <= (2 << 11)) + { + return 0; + } + + for (byte p = 0; p < 40; p++) + { + var shift = (p >> 1) + 11; + if (shift >= 31) + { + return p; + } + + var size = (2 | (p & 1)) << shift; + if (size >= dictSize) + { + return p; + } + } + + return 40; + } +} diff --git a/src/SharpCompress/Compressors/LZMA/LzmaEncoderProperties.cs b/src/SharpCompress/Compressors/LZMA/LzmaEncoderProperties.cs index 198ba471..8076ac08 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaEncoderProperties.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaEncoderProperties.cs @@ -12,6 +12,16 @@ public class LzmaEncoderProperties internal ReadOnlySpan Properties => _properties; private readonly object[] _properties; + /// + /// The dictionary size configured for this encoder. + /// + internal int DictionarySize { get; } + + /// + /// The number of fast bytes configured for this encoder. + /// + internal int NumFastBytes { get; } + public LzmaEncoderProperties() : this(false) { } @@ -23,6 +33,8 @@ public class LzmaEncoderProperties public LzmaEncoderProperties(bool eos, int dictionary, int numFastBytes) { + DictionarySize = dictionary; + NumFastBytes = numFastBytes; var posStateBits = 2; var litContextBits = 3; var litPosBits = 0; diff --git a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs index ff92d853..ff98053d 100644 --- a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs +++ b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs @@ -74,18 +74,10 @@ public partial class SevenZipWriter : AbstractWriter } // Compress file data to output stream - // TODO: LZMA2 encoding is not yet implemented in SharpCompress's LzmaStream - if (sevenZipOptions.IsLzma2) - { - throw new ArchiveOperationException( - "LZMA2 encoding is not yet implemented. Use LZMA (IsLzma2 = false) instead." - ); - } - var compressor = new SevenZipStreamsCompressor(OutputStream.NotNull()); var packed = compressor.Compress( progressStream, - isLzma2: false, + isLzma2: sevenZipOptions.IsLzma2, sevenZipOptions.LzmaProperties ); packedStreams.Add(packed); diff --git a/tests/SharpCompress.Test/SevenZip/SevenZipWriterTests.cs b/tests/SharpCompress.Test/SevenZip/SevenZipWriterTests.cs index 5e74bb61..43aa6dc6 100644 --- a/tests/SharpCompress.Test/SevenZip/SevenZipWriterTests.cs +++ b/tests/SharpCompress.Test/SevenZip/SevenZipWriterTests.cs @@ -160,17 +160,155 @@ public class SevenZipWriterTests : TestBase } [Fact] - public void SevenZipWriter_LZMA2_ThrowsNotSupported() + public void SevenZipWriter_LZMA2_SingleFile_RoundTrip() { - // LZMA2 encoding is not yet implemented in SharpCompress's LzmaStream + var content = "Hello, LZMA2 world! This is a test of LZMA2 encoding in the SevenZipWriter."u8.ToArray(); + using var archiveStream = new MemoryStream(); - using var writer = new SevenZipWriter( + + using (var writer = new SevenZipWriter( archiveStream, new SevenZipWriterOptions { IsLzma2 = true } - ); + )) + { + using var source = new MemoryStream(content); + writer.Write("test.txt", source, DateTime.UtcNow); + } - using var source = new MemoryStream("test"u8.ToArray()); - Assert.Throws(() => writer.Write("test.txt", source, DateTime.UtcNow)); + archiveStream.Position = 0; + using (var archive = (SevenZipArchive)SevenZipArchive.OpenArchive(archiveStream)) + { + var entries = archive.Entries.Where(e => !e.IsDirectory).ToList(); + Assert.Single(entries); + Assert.Equal("test.txt", entries[0].Key); + Assert.Equal(content.Length, (int)entries[0].Size); + + using var output = new MemoryStream(); + using (var entryStream = entries[0].OpenEntryStream()) + { + entryStream.CopyTo(output); + } + Assert.Equal(content, output.ToArray()); + } + } + + [Fact] + public void SevenZipWriter_LZMA2_MultipleFiles_RoundTrip() + { + var files = new[] + { + ("file1.txt", "Content of file 1 for LZMA2 testing"), + ("subdir/file2.txt", "Content of file 2 in subdirectory for LZMA2"), + ("file3.bin", "Some binary-ish content with special bytes for LZMA2 testing"), + }; + + using var archiveStream = new MemoryStream(); + + using (var writer = new SevenZipWriter( + archiveStream, + new SevenZipWriterOptions { IsLzma2 = true } + )) + { + foreach (var (name, text) in files) + { + using var source = new MemoryStream(Encoding.UTF8.GetBytes(text)); + writer.Write(name, source, DateTime.UtcNow); + } + } + + archiveStream.Position = 0; + using (var archive = (SevenZipArchive)SevenZipArchive.OpenArchive(archiveStream)) + { + var entries = archive.Entries.Where(e => !e.IsDirectory).ToList(); + Assert.Equal(files.Length, entries.Count); + + for (var i = 0; i < files.Length; i++) + { + var entry = entries.First(e => e.Key == files[i].Item1); + using var output = new MemoryStream(); + using (var entryStream = entry.OpenEntryStream()) + { + entryStream.CopyTo(output); + } + var extractedText = Encoding.UTF8.GetString(output.ToArray()); + Assert.Equal(files[i].Item2, extractedText); + } + } + } + + [Fact] + public void SevenZipWriter_LZMA2_LargerFile_RoundTrip() + { + // Create 3MB of repeating pattern data - forces multi-chunk in LZMA2 + var content = new byte[3 * 1024 * 1024]; + var pattern = Encoding.UTF8.GetBytes("This is a repeating pattern for LZMA2 compression testing. "); + for (var i = 0; i < content.Length; i++) + { + content[i] = pattern[i % pattern.Length]; + } + + using var archiveStream = new MemoryStream(); + + using (var writer = new SevenZipWriter( + archiveStream, + new SevenZipWriterOptions { IsLzma2 = true } + )) + { + using var source = new MemoryStream(content); + writer.Write("large.bin", source, DateTime.UtcNow); + } + + Assert.True(archiveStream.Length < content.Length, "Archive should be smaller than uncompressed data"); + + archiveStream.Position = 0; + using (var archive = (SevenZipArchive)SevenZipArchive.OpenArchive(archiveStream)) + { + var entries = archive.Entries.Where(e => !e.IsDirectory).ToList(); + Assert.Single(entries); + Assert.Equal(content.Length, (int)entries[0].Size); + + using var output = new MemoryStream(); + using (var entryStream = entries[0].OpenEntryStream()) + { + entryStream.CopyTo(output); + } + Assert.Equal(content, output.ToArray()); + } + } + + [Fact] + public void SevenZipWriter_LZMA2_IncompressibleData_RoundTrip() + { + // Random bytes - forces uncompressed fallback in LZMA2 + var content = new byte[100 * 1024]; + var rng = new Random(42); + rng.NextBytes(content); + + using var archiveStream = new MemoryStream(); + + using (var writer = new SevenZipWriter( + archiveStream, + new SevenZipWriterOptions { IsLzma2 = true } + )) + { + using var source = new MemoryStream(content); + writer.Write("random.bin", source, DateTime.UtcNow); + } + + archiveStream.Position = 0; + using (var archive = (SevenZipArchive)SevenZipArchive.OpenArchive(archiveStream)) + { + var entries = archive.Entries.Where(e => !e.IsDirectory).ToList(); + Assert.Single(entries); + Assert.Equal(content.Length, (int)entries[0].Size); + + using var output = new MemoryStream(); + using (var entryStream = entries[0].OpenEntryStream()) + { + entryStream.CopyTo(output); + } + Assert.Equal(content, output.ToArray()); + } } [Fact]