Add LZMA2 encoding support for 7z writer

This commit is contained in:
Daniil Bystrukhin 2026-02-21 23:13:33 -06:00
parent 5e8ea67e21
commit 6893a68559
5 changed files with 508 additions and 25 deletions

View file

@ -37,22 +37,52 @@ internal sealed class SevenZipStreamsCompressor(Stream outputStream)
LzmaEncoderProperties? encoderProperties = null
)
{
encoderProperties ??= new LzmaEncoderProperties(eos: true);
encoderProperties ??= new LzmaEncoderProperties(eos: !isLzma2);
var outStartOffset = outputStream.Position;
var inStartOffset = inputStream.CanSeek ? inputStream.Position : 0L;
// Wrap the output stream in CRC calculator
using var outCrcStream = new Crc32Stream(outputStream);
// Create LZMA encoder writing to CRC-wrapped output
using var lzmaStream = LzmaStream.Create(encoderProperties, isLzma2, outCrcStream);
var properties = lzmaStream.Properties;
byte[] properties;
if (isLzma2)
{
// LZMA2: use Lzma2EncoderStream for chunk-based framing
using var lzma2Stream = new Lzma2EncoderStream(
outCrcStream,
encoderProperties.DictionarySize,
encoderProperties.NumFastBytes
);
// Copy input through the LZMA2 encoder while computing input CRC
CopyWithCrc(inputStream, lzma2Stream, out var inputCrc2, out var inputSize2);
// Flush/finalize (writes remaining buffer + end marker)
lzma2Stream.Dispose();
var compressedSize2 = (ulong)(outputStream.Position - outStartOffset);
var uncompressedSize2 = (ulong)inputSize2;
var outputCrc2 = outCrcStream.Crc;
properties = lzma2Stream.Properties;
return BuildPackedStream(
isLzma2: true,
properties,
compressedSize2,
uncompressedSize2,
inputCrc2,
outputCrc2
);
}
// LZMA: existing path
using var lzmaStream = LzmaStream.Create(encoderProperties, false, outCrcStream);
properties = lzmaStream.Properties;
// Copy input through the LZMA encoder while computing input CRC
uint inputCrc;
long inputSize;
CopyWithCrc(inputStream, lzmaStream, out inputCrc, out inputSize);
CopyWithCrc(inputStream, lzmaStream, out var inputCrc, out var inputSize);
// Flush/finalize the LZMA encoder (writes remaining compressed data)
lzmaStream.Dispose();
@ -61,10 +91,27 @@ internal sealed class SevenZipStreamsCompressor(Stream outputStream)
var uncompressedSize = (ulong)inputSize;
var outputCrc = outCrcStream.Crc;
// Build method ID
return BuildPackedStream(
isLzma2: false,
properties,
compressedSize,
uncompressedSize,
inputCrc,
outputCrc
);
}
private static PackedStream BuildPackedStream(
bool isLzma2,
byte[] properties,
ulong compressedSize,
ulong uncompressedSize,
uint inputCrc,
uint? outputCrc
)
{
var methodId = isLzma2 ? CMethodId.K_LZMA2 : CMethodId.K_LZMA;
// Build folder metadata
var folder = new CFolder();
folder._coders.Add(
new CCoderInfo

View file

@ -0,0 +1,294 @@
using System;
using System.IO;
namespace SharpCompress.Compressors.LZMA;
/// <summary>
/// Write-only stream that compresses data using the LZMA2 framing format.
/// Buffers input, compresses in chunks, and writes LZMA2-framed output to the underlying stream.
/// Each chunk is independently compressed with a fresh LZMA encoder.
/// </summary>
internal sealed class Lzma2EncoderStream : Stream
{
// Max uncompressed chunk size per LZMA2 spec: (0x1F << 16) + 0xFFFF + 1 = 2MB
private const int MAX_UNCOMPRESSED_CHUNK_SIZE = (0x1F << 16) + 0xFFFF + 1;
// Max compressed payload per LZMA2 chunk header: 0xFFFF + 1 = 64KB
private const int MAX_COMPRESSED_CHUNK_SIZE = 0xFFFF + 1;
// Max uncompressed sub-chunk for raw (uncompressed) chunks: 0xFFFF + 1 = 64KB
private const int MAX_UNCOMPRESSED_SUBCHUNK_SIZE = 0xFFFF + 1;
private readonly Stream _output;
private readonly int _dictionarySize;
private readonly int _numFastBytes;
private readonly byte[] _buffer;
private int _bufferPosition;
private bool _isFirstChunk = true;
private bool _isDisposed;
private byte _lzmaPropertiesByte;
private bool _lzmaPropertiesKnown;
/// <summary>
/// Creates a new LZMA2 encoder stream.
/// </summary>
/// <param name="output">The stream to write LZMA2-framed compressed data to.</param>
/// <param name="dictionarySize">Dictionary size for LZMA compression.</param>
/// <param name="numFastBytes">Number of fast bytes for LZMA compression.</param>
public Lzma2EncoderStream(Stream output, int dictionarySize, int numFastBytes)
{
_output = output;
_dictionarySize = dictionarySize;
_numFastBytes = numFastBytes;
_buffer = new byte[MAX_UNCOMPRESSED_CHUNK_SIZE];
_bufferPosition = 0;
}
/// <summary>
/// Gets the 1-byte LZMA2 properties (encoded dictionary size).
/// </summary>
public byte[] Properties => [EncodeDictionarySize(_dictionarySize)];
public override bool CanRead => false;
public override bool CanSeek => false;
public override bool CanWrite => true;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => throw new NotSupportedException();
set => throw new NotSupportedException();
}
public override void Write(byte[] buffer, int offset, int count)
{
while (count > 0)
{
var toCopy = Math.Min(count, _buffer.Length - _bufferPosition);
Buffer.BlockCopy(buffer, offset, _buffer, _bufferPosition, toCopy);
_bufferPosition += toCopy;
offset += toCopy;
count -= toCopy;
if (_bufferPosition == _buffer.Length)
{
FlushChunk();
}
}
}
public override void Flush() { }
public override int Read(byte[] buffer, int offset, int count) =>
throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) =>
throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
protected override void Dispose(bool disposing)
{
if (disposing && !_isDisposed)
{
_isDisposed = true;
// Flush remaining buffered data
if (_bufferPosition > 0)
{
FlushChunk();
}
// Write LZMA2 end marker
_output.WriteByte(0x00);
}
base.Dispose(disposing);
}
private void FlushChunk()
{
if (_bufferPosition == 0)
{
return;
}
var uncompressedData = _buffer.AsSpan(0, _bufferPosition);
_bufferPosition = 0;
// Try compressing the data
byte[] compressed;
try
{
compressed = CompressBlock(uncompressedData);
}
catch
{
// If compression fails, write as uncompressed
WriteUncompressedChunks(uncompressedData);
return;
}
// Check if compressed output fits in a single chunk and is actually smaller
if (compressed.Length <= MAX_COMPRESSED_CHUNK_SIZE && compressed.Length < uncompressedData.Length)
{
WriteCompressedChunk(uncompressedData.Length, compressed);
}
else
{
WriteUncompressedChunks(uncompressedData);
}
}
private byte[] CompressBlock(ReadOnlySpan<byte> data)
{
var encoderProps = new LzmaEncoderProperties(
eos: false,
_dictionarySize,
_numFastBytes
);
var encoder = new Encoder();
encoder.SetCoderProperties(encoderProps.PropIDs, encoderProps.Properties);
// Capture the LZMA properties byte (pb/lp/lc encoding) for the chunk header
if (!_lzmaPropertiesKnown)
{
var propBytes = new byte[5];
encoder.WriteCoderProperties(propBytes);
_lzmaPropertiesByte = propBytes[0];
_lzmaPropertiesKnown = true;
}
using var inputMs = new MemoryStream(data.ToArray(), writable: false);
using var outputMs = new MemoryStream();
encoder.Code(inputMs, outputMs, data.Length, -1, null);
var fullCompressed = outputMs.ToArray();
// The LZMA range encoder flush writes trailing bytes the decoder doesn't consume.
// Trial-decode to find the exact byte count the decoder needs, so the LZMA2
// chunk header reports a compressed size that matches what the decoder reads.
var consumed = FindConsumedBytes(fullCompressed, data.Length);
if (consumed < fullCompressed.Length)
{
return fullCompressed.AsSpan(0, consumed).ToArray();
}
return fullCompressed;
}
private int FindConsumedBytes(byte[] compressedData, int uncompressedSize)
{
// Build 5-byte LZMA property header: [pb/lp/lc byte] [dictSize as LE int32]
var props = new byte[5];
props[0] = _lzmaPropertiesByte;
props[1] = (byte)_dictionarySize;
props[2] = (byte)(_dictionarySize >> 8);
props[3] = (byte)(_dictionarySize >> 16);
props[4] = (byte)(_dictionarySize >> 24);
var decoder = new Decoder();
decoder.SetDecoderProperties(props);
using var input = new MemoryStream(compressedData);
using var output = new MemoryStream();
decoder.Code(input, output, compressedData.Length, uncompressedSize, null);
return (int)input.Position;
}
/// <summary>
/// Writes a compressed LZMA2 chunk.
/// Header: [control] [uncompSize_hi] [uncompSize_lo] [compSize_hi] [compSize_lo] [props?]
/// </summary>
private void WriteCompressedChunk(int uncompressedSize, byte[] compressedData)
{
var uncompSizeMinus1 = uncompressedSize - 1;
var compSizeMinus1 = compressedData.Length - 1;
// Each chunk is compressed independently with a fresh LZMA encoder,
// so we must use 0xE0 (full reset: dictionary + state + properties) every time.
// The decoder uses outWindow.Total for literal context and posState;
// 0xE0 triggers outWindow.Reset() which zeros Total, matching the encoder's
// assumption that position starts at 0 for each chunk.
var control = (byte)(0xE0 | ((uncompSizeMinus1 >> 16) & 0x1F));
_isFirstChunk = false;
_output.WriteByte(control);
_output.WriteByte((byte)((uncompSizeMinus1 >> 8) & 0xFF));
_output.WriteByte((byte)(uncompSizeMinus1 & 0xFF));
_output.WriteByte((byte)((compSizeMinus1 >> 8) & 0xFF));
_output.WriteByte((byte)(compSizeMinus1 & 0xFF));
// 0xE0 (>= 0xC0) requires properties byte
_output.WriteByte(_lzmaPropertiesByte);
_output.Write(compressedData, 0, compressedData.Length);
}
/// <summary>
/// Writes data as uncompressed LZMA2 sub-chunks (max 64KB each).
/// Header: [control] [size_hi] [size_lo]
/// </summary>
private void WriteUncompressedChunks(ReadOnlySpan<byte> data)
{
var offset = 0;
while (offset < data.Length)
{
var chunkSize = Math.Min(data.Length - offset, MAX_UNCOMPRESSED_SUBCHUNK_SIZE);
var sizeMinus1 = chunkSize - 1;
byte control;
if (_isFirstChunk)
{
// 0x01: uncompressed with dictionary reset
control = 0x01;
_isFirstChunk = false;
}
else
{
// 0x02: uncompressed without dictionary reset
control = 0x02;
}
_output.WriteByte(control);
_output.WriteByte((byte)((sizeMinus1 >> 8) & 0xFF));
_output.WriteByte((byte)(sizeMinus1 & 0xFF));
_output.Write(data.Slice(offset, chunkSize));
offset += chunkSize;
}
}
/// <summary>
/// Encodes a dictionary size into the 1-byte LZMA2 properties format.
/// Reverse of the decoder formula: dictSize = (2 | (p and 1)) shl ((p shr 1) + 11)
/// Finds the smallest p where the formula result >= target dictSize.
/// </summary>
internal static byte EncodeDictionarySize(int dictSize)
{
// Special case: very small dictionary sizes
if (dictSize <= (2 << 11))
{
return 0;
}
for (byte p = 0; p < 40; p++)
{
var shift = (p >> 1) + 11;
if (shift >= 31)
{
return p;
}
var size = (2 | (p & 1)) << shift;
if (size >= dictSize)
{
return p;
}
}
return 40;
}
}

View file

@ -12,6 +12,16 @@ public class LzmaEncoderProperties
internal ReadOnlySpan<object> Properties => _properties;
private readonly object[] _properties;
/// <summary>
/// The dictionary size configured for this encoder.
/// </summary>
internal int DictionarySize { get; }
/// <summary>
/// The number of fast bytes configured for this encoder.
/// </summary>
internal int NumFastBytes { get; }
public LzmaEncoderProperties()
: this(false) { }
@ -23,6 +33,8 @@ public class LzmaEncoderProperties
public LzmaEncoderProperties(bool eos, int dictionary, int numFastBytes)
{
DictionarySize = dictionary;
NumFastBytes = numFastBytes;
var posStateBits = 2;
var litContextBits = 3;
var litPosBits = 0;

View file

@ -74,18 +74,10 @@ public partial class SevenZipWriter : AbstractWriter
}
// Compress file data to output stream
// TODO: LZMA2 encoding is not yet implemented in SharpCompress's LzmaStream
if (sevenZipOptions.IsLzma2)
{
throw new ArchiveOperationException(
"LZMA2 encoding is not yet implemented. Use LZMA (IsLzma2 = false) instead."
);
}
var compressor = new SevenZipStreamsCompressor(OutputStream.NotNull());
var packed = compressor.Compress(
progressStream,
isLzma2: false,
isLzma2: sevenZipOptions.IsLzma2,
sevenZipOptions.LzmaProperties
);
packedStreams.Add(packed);

View file

@ -160,17 +160,155 @@ public class SevenZipWriterTests : TestBase
}
[Fact]
public void SevenZipWriter_LZMA2_ThrowsNotSupported()
public void SevenZipWriter_LZMA2_SingleFile_RoundTrip()
{
// LZMA2 encoding is not yet implemented in SharpCompress's LzmaStream
var content = "Hello, LZMA2 world! This is a test of LZMA2 encoding in the SevenZipWriter."u8.ToArray();
using var archiveStream = new MemoryStream();
using var writer = new SevenZipWriter(
using (var writer = new SevenZipWriter(
archiveStream,
new SevenZipWriterOptions { IsLzma2 = true }
);
))
{
using var source = new MemoryStream(content);
writer.Write("test.txt", source, DateTime.UtcNow);
}
using var source = new MemoryStream("test"u8.ToArray());
Assert.Throws<ArchiveOperationException>(() => writer.Write("test.txt", source, DateTime.UtcNow));
archiveStream.Position = 0;
using (var archive = (SevenZipArchive)SevenZipArchive.OpenArchive(archiveStream))
{
var entries = archive.Entries.Where(e => !e.IsDirectory).ToList();
Assert.Single(entries);
Assert.Equal("test.txt", entries[0].Key);
Assert.Equal(content.Length, (int)entries[0].Size);
using var output = new MemoryStream();
using (var entryStream = entries[0].OpenEntryStream())
{
entryStream.CopyTo(output);
}
Assert.Equal(content, output.ToArray());
}
}
[Fact]
public void SevenZipWriter_LZMA2_MultipleFiles_RoundTrip()
{
var files = new[]
{
("file1.txt", "Content of file 1 for LZMA2 testing"),
("subdir/file2.txt", "Content of file 2 in subdirectory for LZMA2"),
("file3.bin", "Some binary-ish content with special bytes for LZMA2 testing"),
};
using var archiveStream = new MemoryStream();
using (var writer = new SevenZipWriter(
archiveStream,
new SevenZipWriterOptions { IsLzma2 = true }
))
{
foreach (var (name, text) in files)
{
using var source = new MemoryStream(Encoding.UTF8.GetBytes(text));
writer.Write(name, source, DateTime.UtcNow);
}
}
archiveStream.Position = 0;
using (var archive = (SevenZipArchive)SevenZipArchive.OpenArchive(archiveStream))
{
var entries = archive.Entries.Where(e => !e.IsDirectory).ToList();
Assert.Equal(files.Length, entries.Count);
for (var i = 0; i < files.Length; i++)
{
var entry = entries.First(e => e.Key == files[i].Item1);
using var output = new MemoryStream();
using (var entryStream = entry.OpenEntryStream())
{
entryStream.CopyTo(output);
}
var extractedText = Encoding.UTF8.GetString(output.ToArray());
Assert.Equal(files[i].Item2, extractedText);
}
}
}
[Fact]
public void SevenZipWriter_LZMA2_LargerFile_RoundTrip()
{
// Create 3MB of repeating pattern data - forces multi-chunk in LZMA2
var content = new byte[3 * 1024 * 1024];
var pattern = Encoding.UTF8.GetBytes("This is a repeating pattern for LZMA2 compression testing. ");
for (var i = 0; i < content.Length; i++)
{
content[i] = pattern[i % pattern.Length];
}
using var archiveStream = new MemoryStream();
using (var writer = new SevenZipWriter(
archiveStream,
new SevenZipWriterOptions { IsLzma2 = true }
))
{
using var source = new MemoryStream(content);
writer.Write("large.bin", source, DateTime.UtcNow);
}
Assert.True(archiveStream.Length < content.Length, "Archive should be smaller than uncompressed data");
archiveStream.Position = 0;
using (var archive = (SevenZipArchive)SevenZipArchive.OpenArchive(archiveStream))
{
var entries = archive.Entries.Where(e => !e.IsDirectory).ToList();
Assert.Single(entries);
Assert.Equal(content.Length, (int)entries[0].Size);
using var output = new MemoryStream();
using (var entryStream = entries[0].OpenEntryStream())
{
entryStream.CopyTo(output);
}
Assert.Equal(content, output.ToArray());
}
}
[Fact]
public void SevenZipWriter_LZMA2_IncompressibleData_RoundTrip()
{
// Random bytes - forces uncompressed fallback in LZMA2
var content = new byte[100 * 1024];
var rng = new Random(42);
rng.NextBytes(content);
using var archiveStream = new MemoryStream();
using (var writer = new SevenZipWriter(
archiveStream,
new SevenZipWriterOptions { IsLzma2 = true }
))
{
using var source = new MemoryStream(content);
writer.Write("random.bin", source, DateTime.UtcNow);
}
archiveStream.Position = 0;
using (var archive = (SevenZipArchive)SevenZipArchive.OpenArchive(archiveStream))
{
var entries = archive.Entries.Where(e => !e.IsDirectory).ToList();
Assert.Single(entries);
Assert.Equal(content.Length, (int)entries[0].Size);
using var output = new MemoryStream();
using (var entryStream = entries[0].OpenEntryStream())
{
entryStream.CopyTo(output);
}
Assert.Equal(content, output.ToArray());
}
}
[Fact]