From 817bd2a95e0355e912e6a256d2ce8e3166cc51cc Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sat, 4 Apr 2026 10:44:23 +0100 Subject: [PATCH 01/20] First pass of implementation --- src/SharpCompress/IO/PooledMemoryStream.cs | 992 ++++++++++++++++++ src/SharpCompress/ThrowHelper.cs | 9 + .../Streams/PooledMemoryStreamTests.cs | 149 +++ 3 files changed, 1150 insertions(+) create mode 100644 src/SharpCompress/IO/PooledMemoryStream.cs create mode 100644 tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs diff --git a/src/SharpCompress/IO/PooledMemoryStream.cs b/src/SharpCompress/IO/PooledMemoryStream.cs new file mode 100644 index 00000000..58bc748f --- /dev/null +++ b/src/SharpCompress/IO/PooledMemoryStream.cs @@ -0,0 +1,992 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace SharpCompress.IO; + +/// +/// MemoryStream implementation backed by pooled byte arrays. +/// +public sealed class PooledMemoryStream : MemoryStream +{ + public const int DefaultBlockSize = 81920; + + private const int MaxStreamLength = int.MaxValue; + + private enum StorageMode + { + Segmented, + Contiguous, + } + + private readonly ArrayPool _arrayPool; + private readonly int _blockSize; + + private readonly List _detachedExposedBuffers = new(); + + private StorageMode _mode; + private List? _blocks; + private byte[]? _contiguousBuffer; + + private bool _contiguousBufferExposed; + private bool _isOpen; + private bool _writable; + private bool _expandable; + private bool _exposable; + + private int _origin; + private int _position; + private int _length; + private int _capacity; + private int _allocatedCapacity; + + public PooledMemoryStream() + : this(0) { } + + public PooledMemoryStream(int capacity) + : this(capacity, DefaultBlockSize, ArrayPool.Shared) { } + + public PooledMemoryStream(int capacity, int blockSize) + : this(capacity, blockSize, ArrayPool.Shared) { } + + public PooledMemoryStream(int capacity, int blockSize, ArrayPool arrayPool) + { + ThrowHelper.ThrowIfNull(arrayPool, nameof(arrayPool)); + ThrowHelper.ThrowIfNegative(capacity, nameof(capacity)); + ThrowHelper.ThrowIfNegativeOrZero(blockSize, nameof(blockSize)); + + _arrayPool = arrayPool; + _blockSize = blockSize; + + _mode = StorageMode.Segmented; + _blocks = new List(); + _isOpen = true; + _writable = true; + _expandable = true; + _exposable = true; + _origin = 0; + _position = 0; + _length = 0; + _capacity = capacity; + + EnsureSegmentedAllocated(capacity); + } + + public PooledMemoryStream(byte[] buffer) + : this(buffer, writable: true) { } + + public PooledMemoryStream(byte[] buffer, bool writable) + : this(buffer, 0, buffer?.Length ?? 0, writable, publiclyVisible: false) { } + + public PooledMemoryStream(byte[] buffer, int index, int count) + : this(buffer, index, count, writable: true, publiclyVisible: false) { } + + public PooledMemoryStream(byte[] buffer, int index, int count, bool writable) + : this(buffer, index, count, writable, publiclyVisible: false) { } + + public PooledMemoryStream( + byte[] buffer, + int index, + int count, + bool writable, + bool publiclyVisible + ) + { + ThrowHelper.ThrowIfNull(buffer, nameof(buffer)); + ThrowHelper.ThrowIfNegative(index, nameof(index)); + ThrowHelper.ThrowIfNegative(count, nameof(count)); + if (buffer.Length - index < count) + { + throw new ArgumentException("Offset and length are out of bounds."); + } + + _arrayPool = ArrayPool.Shared; + _blockSize = DefaultBlockSize; + + _mode = StorageMode.Segmented; + _blocks = new List(); + _origin = 0; + _position = 0; + _length = count; + _capacity = count; + _writable = writable; + _expandable = false; + _exposable = publiclyVisible; + _isOpen = true; + + EnsureSegmentedAllocated(_capacity); + if (count > 0) + { + CopyToSegmented(0, buffer, index, count); + } + } + + public override bool CanRead => _isOpen; + + public override bool CanSeek => _isOpen; + + public override bool CanWrite => _writable; + + public override long Length + { + get + { + EnsureNotClosed(); + return _length - _origin; + } + } + + public override long Position + { + get + { + EnsureNotClosed(); + return _position - _origin; + } + set + { + ThrowHelper.ThrowIfNegative(value, nameof(value)); + EnsureNotClosed(); + ThrowHelper.ThrowIfGreaterThan(value, MaxStreamLength - _origin, nameof(value)); + + _position = _origin + (int)value; + } + } + + public override int Capacity + { + get + { + EnsureNotClosed(); + return _capacity - _origin; + } + set + { + if (value < Length) + { + throw new ArgumentOutOfRangeException(nameof(value)); + } + + EnsureNotClosed(); + + if (!_expandable && value != Capacity) + { + throw new NotSupportedException("Memory stream is not expandable."); + } + + var target = _origin + value; + if (target == _capacity) + { + return; + } + + SetCapacityAbsolute(target); + } + } + + public override void Flush() { } + + public override Task FlushAsync(CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + { + return Task.FromCanceled(cancellationToken); + } + + return Task.CompletedTask; + } + + public override long Seek(long offset, SeekOrigin loc) + { + EnsureNotClosed(); + + var anchor = loc switch + { + SeekOrigin.Begin => _origin, + SeekOrigin.Current => _position, + SeekOrigin.End => _length, + _ => throw new ArgumentException("Invalid seek origin.", nameof(loc)), + }; + + var target = anchor + offset; + if (target < _origin) + { + throw new IOException("Attempted to seek before the beginning of the stream."); + } + + if (target > MaxStreamLength) + { + throw new ArgumentOutOfRangeException(nameof(offset)); + } + + _position = (int)target; + return _position - _origin; + } + + public override void SetLength(long value) + { + ThrowHelper.ThrowIfNegative(value, nameof(value)); + ThrowHelper.ThrowIfGreaterThan(value, MaxStreamLength, nameof(value)); + + EnsureWritable(); + + var newLength = _origin + (int)value; + if (newLength > _capacity) + { + EnsureCapacityForAppend(newLength); + } + + if (newLength > _length) + { + ClearRange(_length, newLength - _length); + } + + _length = newLength; + if (_position > newLength) + { + _position = newLength; + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + ValidateReadWriteBufferArguments(buffer, offset, count); + EnsureNotClosed(); + + var available = _length - _position; + if (available <= 0) + { + return 0; + } + + if (count > available) + { + count = available; + } + + switch (_mode) + { + case StorageMode.Contiguous: + Buffer.BlockCopy(_contiguousBuffer!, _position, buffer, offset, count); + break; + case StorageMode.Segmented: + CopyFromSegmented(_position, buffer, offset, count); + break; + } + + _position += count; + return count; + } + + public override int ReadByte() + { + EnsureNotClosed(); + if (_position >= _length) + { + return -1; + } + + byte value; + switch (_mode) + { + case StorageMode.Contiguous: + value = _contiguousBuffer![_position]; + break; + default: + { + var blockIndex = _position / _blockSize; + var blockOffset = _position % _blockSize; + value = _blocks![blockIndex][blockOffset]; + break; + } + } + + _position++; + return value; + } + + public override void Write(byte[] buffer, int offset, int count) + { + ValidateReadWriteBufferArguments(buffer, offset, count); + EnsureWritable(); + + if (count == 0) + { + return; + } + + var endPosition = _position + count; + if (endPosition < 0) + { + throw new IOException("Stream is too long."); + } + + if (endPosition > _capacity) + { + EnsureCapacityForAppend(endPosition); + } + + if (_position > _length) + { + ClearRange(_length, _position - _length); + } + + switch (_mode) + { + case StorageMode.Contiguous: + Buffer.BlockCopy(buffer, offset, _contiguousBuffer!, _position, count); + break; + case StorageMode.Segmented: + CopyToSegmented(_position, buffer, offset, count); + break; + } + + _position = endPosition; + if (_position > _length) + { + _length = _position; + } + } + + public override void WriteByte(byte value) + { + EnsureWritable(); + + var endPosition = _position + 1; + if (endPosition < 0) + { + throw new IOException("Stream is too long."); + } + + if (endPosition > _capacity) + { + EnsureCapacityForAppend(endPosition); + } + + if (_position > _length) + { + ClearRange(_length, _position - _length); + } + + switch (_mode) + { + case StorageMode.Contiguous: + _contiguousBuffer![_position] = value; + break; + default: + { + var blockIndex = _position / _blockSize; + var blockOffset = _position % _blockSize; + _blocks![blockIndex][blockOffset] = value; + break; + } + } + + _position = endPosition; + if (_position > _length) + { + _length = _position; + } + } + + public override byte[] GetBuffer() + { + EnsureNotClosed(); + if (!_exposable) + { + throw new UnauthorizedAccessException("Memory stream buffer is not publicly visible."); + } + + EnsureContiguous(); + _contiguousBufferExposed = true; + return _contiguousBuffer!; + } + + public override bool TryGetBuffer(out ArraySegment buffer) + { + if (!_exposable) + { + buffer = default; + return false; + } + + EnsureNotClosed(); + + EnsureContiguous(); + _contiguousBufferExposed = true; + buffer = new ArraySegment(_contiguousBuffer!, 0, _length); + return true; + } + + public override byte[] ToArray() + { + EnsureNotClosed(); + + var count = _length - _origin; + if (count == 0) + { + return Array.Empty(); + } + + var copy = new byte[count]; + switch (_mode) + { + case StorageMode.Contiguous: + Buffer.BlockCopy(_contiguousBuffer!, _origin, copy, 0, count); + break; + case StorageMode.Segmented: + CopyFromSegmented(_origin, copy, 0, count); + break; + } + + return copy; + } + + public override void WriteTo(Stream stream) + { + ThrowHelper.ThrowIfNull(stream, nameof(stream)); + EnsureNotClosed(); + + var count = _length - _origin; + if (count == 0) + { + return; + } + + switch (_mode) + { + case StorageMode.Contiguous: + stream.Write(_contiguousBuffer!, _origin, count); + break; + case StorageMode.Segmented: + { + var position = _origin; + var remaining = count; + while (remaining > 0) + { + var blockIndex = position / _blockSize; + var blockOffset = position % _blockSize; + var toWrite = Math.Min(remaining, _blockSize - blockOffset); + stream.Write(_blocks![blockIndex], blockOffset, toWrite); + position += toWrite; + remaining -= toWrite; + } + + break; + } + } + } + + public override Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + if (cancellationToken.IsCancellationRequested) + { + return Task.FromCanceled(cancellationToken); + } + + try + { + return Task.FromResult(Read(buffer, offset, count)); + } + catch (Exception ex) + { + return Task.FromException(ex); + } + } + + public override Task WriteAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + if (cancellationToken.IsCancellationRequested) + { + return Task.FromCanceled(cancellationToken); + } + + try + { + Write(buffer, offset, count); + return Task.CompletedTask; + } + catch (Exception ex) + { + return Task.FromException(ex); + } + } + +#if !LEGACY_DOTNET + public override int Read(Span buffer) + { + EnsureNotClosed(); + + var available = _length - _position; + if (available <= 0) + { + return 0; + } + + var count = Math.Min(available, buffer.Length); + switch (_mode) + { + case StorageMode.Contiguous: + _contiguousBuffer.AsSpan(_position, count).CopyTo(buffer); + break; + case StorageMode.Segmented: + { + var sourcePosition = _position; + var destinationOffset = 0; + var remaining = count; + + while (remaining > 0) + { + var blockIndex = sourcePosition / _blockSize; + var blockOffset = sourcePosition % _blockSize; + var toCopy = Math.Min(remaining, _blockSize - blockOffset); + _blocks! + [blockIndex] + .AsSpan(blockOffset, toCopy) + .CopyTo(buffer.Slice(destinationOffset, toCopy)); + + sourcePosition += toCopy; + destinationOffset += toCopy; + remaining -= toCopy; + } + + break; + } + } + + _position += count; + return count; + } + + public override void Write(ReadOnlySpan buffer) + { + EnsureWritable(); + if (buffer.Length == 0) + { + return; + } + + var endPosition = _position + buffer.Length; + if (endPosition < 0) + { + throw new IOException("Stream is too long."); + } + + if (endPosition > _capacity) + { + EnsureCapacityForAppend(endPosition); + } + + if (_position > _length) + { + ClearRange(_length, _position - _length); + } + + switch (_mode) + { + case StorageMode.Contiguous: + buffer.CopyTo(_contiguousBuffer.AsSpan(_position, buffer.Length)); + break; + case StorageMode.Segmented: + { + var sourceOffset = 0; + var destinationPosition = _position; + var remaining = buffer.Length; + + while (remaining > 0) + { + var blockIndex = destinationPosition / _blockSize; + var blockOffset = destinationPosition % _blockSize; + var toCopy = Math.Min(remaining, _blockSize - blockOffset); + + buffer + .Slice(sourceOffset, toCopy) + .CopyTo(_blocks![blockIndex].AsSpan(blockOffset, toCopy)); + + sourceOffset += toCopy; + destinationPosition += toCopy; + remaining -= toCopy; + } + + break; + } + } + + _position = endPosition; + if (_position > _length) + { + _length = _position; + } + } + + public override ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + if (cancellationToken.IsCancellationRequested) + { + return ValueTask.FromCanceled(cancellationToken); + } + + try + { + return ValueTask.FromResult(Read(buffer.Span)); + } + catch (Exception ex) + { + return ValueTask.FromException(ex); + } + } + + public override ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default + ) + { + if (cancellationToken.IsCancellationRequested) + { + return ValueTask.FromCanceled(cancellationToken); + } + + try + { + Write(buffer.Span); + return ValueTask.CompletedTask; + } + catch (Exception ex) + { + return ValueTask.FromException(ex); + } + } +#endif + + protected override void Dispose(bool disposing) + { + if (_isOpen) + { + _isOpen = false; + _writable = false; + _expandable = false; + + if (disposing) + { + ReturnPooledBuffers(); + } + } + + base.Dispose(disposing); + } + + private void EnsureNotClosed() + { + if (!_isOpen) + { + throw new ObjectDisposedException(nameof(PooledMemoryStream)); + } + } + + private void EnsureWritable() + { + EnsureNotClosed(); + if (!_writable) + { + throw new NotSupportedException("Stream does not support writing."); + } + } + + private void EnsureCapacityForAppend(int requiredLength) + { + if (requiredLength < 0) + { + throw new IOException("Stream is too long."); + } + + if (requiredLength <= _capacity) + { + return; + } + + if (!_expandable) + { + throw new NotSupportedException("Memory stream is not expandable."); + } + + var nextCapacity = RoundUpToBlockBoundary(requiredLength); + SetCapacityAbsolute(nextCapacity); + } + + private void SetCapacityAbsolute(int newCapacity) + { + ThrowHelper.ThrowIfLessThan(newCapacity, _length, nameof(newCapacity)); + + switch (_mode) + { + case StorageMode.Contiguous: + if (newCapacity > _allocatedCapacity) + { + DemoteContiguousToSegmented(); + EnsureSegmentedAllocated(newCapacity); + } + break; + + case StorageMode.Segmented: + EnsureSegmentedAllocated(newCapacity); + break; + } + + _capacity = newCapacity; + if (_length > _capacity) + { + _length = _capacity; + } + if (_position > _capacity) + { + _position = _capacity; + } + } + + private void EnsureContiguous() + { + if (_mode == StorageMode.Contiguous) + { + return; + } + + var requested = Math.Max(_capacity, 1); + var contiguous = _arrayPool.Rent(requested); + if (_length > 0) + { + CopyFromSegmented(0, contiguous, 0, _length); + } + + ReturnSegmentedBlocks(); + + _mode = StorageMode.Contiguous; + _contiguousBuffer = contiguous; + _contiguousBufferExposed = false; + _allocatedCapacity = contiguous.Length; + } + + private void DemoteContiguousToSegmented() + { + var contiguous = _contiguousBuffer; + if (contiguous is null) + { + return; + } + + var requiredCapacity = Math.Max(_capacity, _length); + _mode = StorageMode.Segmented; + _blocks = new List(); + _contiguousBuffer = null; + EnsureSegmentedAllocated(requiredCapacity); + + if (_length > 0) + { + CopyToSegmented(0, contiguous, 0, _length); + } + + if (_contiguousBufferExposed) + { + _detachedExposedBuffers.Add(contiguous); + _contiguousBufferExposed = false; + } + else + { + _arrayPool.Return(contiguous); + } + } + + private void EnsureSegmentedAllocated(int capacity) + { + if (_mode != StorageMode.Segmented) + { + throw new InvalidOperationException( + "Segmented allocation requested while not in segmented mode." + ); + } + + var requiredAllocated = RoundUpToBlockBoundary(capacity); + var requiredBlocks = requiredAllocated == 0 ? 0 : requiredAllocated / _blockSize; + + _blocks ??= new List(); + + while (_blocks.Count < requiredBlocks) + { + _blocks.Add(_arrayPool.Rent(_blockSize)); + } + + while (_blocks.Count > requiredBlocks) + { + var index = _blocks.Count - 1; + var block = _blocks[index]; + _blocks.RemoveAt(index); + _arrayPool.Return(block); + } + + _allocatedCapacity = requiredAllocated; + } + + private int RoundUpToBlockBoundary(int value) + { + if (value <= 0) + { + return 0; + } + + var rounded = ((long)value + _blockSize - 1) / _blockSize * _blockSize; + if (rounded > MaxStreamLength) + { + throw new IOException("Stream is too long."); + } + + return (int)rounded; + } + + private void ClearRange(int absoluteStart, int count) + { + if (count <= 0) + { + return; + } + + switch (_mode) + { + case StorageMode.Contiguous: + Array.Clear(_contiguousBuffer!, absoluteStart, count); + break; + case StorageMode.Segmented: + { + var position = absoluteStart; + var remaining = count; + while (remaining > 0) + { + var blockIndex = position / _blockSize; + var blockOffset = position % _blockSize; + var toClear = Math.Min(remaining, _blockSize - blockOffset); + Array.Clear(_blocks![blockIndex], blockOffset, toClear); + position += toClear; + remaining -= toClear; + } + + break; + } + } + } + + private void CopyFromSegmented( + int absoluteSourcePosition, + byte[] destination, + int offset, + int count + ) + { + var sourcePosition = absoluteSourcePosition; + var destinationOffset = offset; + var remaining = count; + + while (remaining > 0) + { + var blockIndex = sourcePosition / _blockSize; + var blockOffset = sourcePosition % _blockSize; + var toCopy = Math.Min(remaining, _blockSize - blockOffset); + Buffer.BlockCopy( + _blocks![blockIndex], + blockOffset, + destination, + destinationOffset, + toCopy + ); + + sourcePosition += toCopy; + destinationOffset += toCopy; + remaining -= toCopy; + } + } + + private void CopyToSegmented( + int absoluteDestinationPosition, + byte[] source, + int offset, + int count + ) + { + var sourceOffset = offset; + var destinationPosition = absoluteDestinationPosition; + var remaining = count; + + while (remaining > 0) + { + var blockIndex = destinationPosition / _blockSize; + var blockOffset = destinationPosition % _blockSize; + var toCopy = Math.Min(remaining, _blockSize - blockOffset); + Buffer.BlockCopy(source, sourceOffset, _blocks![blockIndex], blockOffset, toCopy); + + sourceOffset += toCopy; + destinationPosition += toCopy; + remaining -= toCopy; + } + } + + private void ReturnSegmentedBlocks() + { + if (_blocks is null) + { + return; + } + + for (var i = 0; i < _blocks.Count; i++) + { + _arrayPool.Return(_blocks[i]); + } + + _blocks.Clear(); + } + + private void ReturnPooledBuffers() + { + if (_mode == StorageMode.Segmented) + { + ReturnSegmentedBlocks(); + _blocks = null; + } + + if (_mode == StorageMode.Contiguous && _contiguousBuffer is not null) + { + _arrayPool.Return(_contiguousBuffer); + _contiguousBuffer = null; + } + + for (var i = 0; i < _detachedExposedBuffers.Count; i++) + { + _arrayPool.Return(_detachedExposedBuffers[i]); + } + + _detachedExposedBuffers.Clear(); + } + + private static void ValidateReadWriteBufferArguments(byte[] buffer, int offset, int count) + { + ThrowHelper.ThrowIfNull(buffer, nameof(buffer)); + ThrowHelper.ThrowIfNegative(offset, nameof(offset)); + ThrowHelper.ThrowIfNegative(count, nameof(count)); + if (buffer.Length - offset < count) + { + throw new ArgumentException("Offset and length are out of bounds."); + } + } +} diff --git a/src/SharpCompress/ThrowHelper.cs b/src/SharpCompress/ThrowHelper.cs index 9d598107..c202d87c 100644 --- a/src/SharpCompress/ThrowHelper.cs +++ b/src/SharpCompress/ThrowHelper.cs @@ -60,6 +60,15 @@ internal static class ThrowHelper } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void ThrowIfGreaterThan(long value, long other, string? paramName = null) + { + if (value > other) + { + throw new ArgumentOutOfRangeException(paramName); + } + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void ThrowIfGreaterThan(uint value, uint other, string? paramName = null) { diff --git a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs new file mode 100644 index 00000000..28af7f1c --- /dev/null +++ b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs @@ -0,0 +1,149 @@ +using System; +using System.Buffers; +using System.IO; +using System.Linq; +using SharpCompress.IO; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class PooledMemoryStreamTests +{ + [Fact] + public void GrowsUsingFixedSizeBlocks() + { + var pool = new TrackingArrayPool(); + + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, arrayPool: pool); + stream.Write(new byte[20], 0, 20); + + Assert.Equal(3, pool.RentRequests.Count); + Assert.All(pool.RentRequests, requested => Assert.Equal(8, requested)); + } + + [Fact] + public void DisposeReturnsRentedBlocksToPool() + { + var pool = new TrackingArrayPool(); + var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, arrayPool: pool); + + stream.Write(new byte[17], 0, 17); + stream.Dispose(); + + Assert.Equal(pool.RentRequests.Count, pool.ReturnedLengths.Count); + Assert.All(pool.ReturnedLengths, length => Assert.Equal(8, length)); + } + + [Fact] + public void GetBufferPromotesToContiguousAndLaterGrowthUsesBlocks() + { + var pool = new TrackingArrayPool(); + + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, arrayPool: pool); + stream.Write(new byte[10], 0, 10); + + var contiguous = stream.GetBuffer(); + Assert.Equal(16, contiguous.Length); + Assert.Contains(16, pool.RentRequests); + + stream.Position = stream.Length; + stream.Write(new byte[10], 0, 10); // grow to 20, forcing demotion + block growth + + var totalBlockRents = pool.RentRequests.Count(size => size == 8); + Assert.True(totalBlockRents >= 3); + } + + [Fact] + public void BufferConstructorCopiesDataAndIsNonExpandable() + { + var backing = Enumerable.Range(0, 10).Select(i => (byte)i).ToArray(); + using var stream = new PooledMemoryStream( + backing, + 2, + 4, + writable: true, + publiclyVisible: true + ); + + Assert.Equal(4, stream.Length); + Assert.Equal(4, stream.Capacity); + + backing[2] = 255; + Assert.Equal(new byte[] { 2, 3, 4, 5 }, stream.ToArray()); + + Assert.Throws(() => stream.Capacity = 5); + } + + [Fact] + public void BufferConstructorTryGetBufferRespectsVisibilityFlag() + { + var backing = new byte[10]; + using var hidden = new PooledMemoryStream( + backing, + 1, + 5, + writable: true, + publiclyVisible: false + ); + Assert.False(hidden.TryGetBuffer(out _)); + + using var visible = new PooledMemoryStream( + backing, + 1, + 5, + writable: true, + publiclyVisible: true + ); + Assert.True(visible.TryGetBuffer(out var segment)); + Assert.Equal(0, segment.Offset); + Assert.Equal(5, segment.Count); + } + + [Fact] + public void SetLengthExtendingClearsGap() + { + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8); + stream.Position = 5; + stream.WriteByte(42); + stream.Position = 0; + + var data = stream.ToArray(); + Assert.Equal(6, data.Length); + Assert.Equal(0, data[0]); + Assert.Equal(0, data[4]); + Assert.Equal(42, data[5]); + } + + [Fact] + public void MethodsThrowAfterDispose() + { + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8); + stream.WriteByte(1); + stream.Dispose(); + + Assert.Throws(() => stream.ReadByte()); + Assert.Throws(() => stream.ToArray()); + Assert.Throws(() => stream.GetBuffer()); + } + + private sealed class TrackingArrayPool : ArrayPool + { + public readonly System.Collections.Generic.List RentRequests = new(); + public readonly System.Collections.Generic.List ReturnedLengths = new(); + + public override byte[] Rent(int minimumLength) + { + RentRequests.Add(minimumLength); + return new byte[minimumLength]; + } + + public override void Return(byte[] array, bool clearArray = false) + { + ReturnedLengths.Add(array.Length); + if (clearArray) + { + Array.Clear(array, 0, array.Length); + } + } + } +} From a26198a6b0488bd730bb942e3cf226ea9f0b0646 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sat, 4 Apr 2026 11:04:51 +0100 Subject: [PATCH 02/20] use pooled memory streams for better performance --- src/SharpCompress/Archives/Tar/TarArchive.Async.cs | 2 +- src/SharpCompress/Archives/Tar/TarArchive.cs | 2 +- src/SharpCompress/Common/SevenZip/ArchiveReader.cs | 2 +- .../Common/SevenZip/SevenZipFilesInfo.cs | 3 ++- .../Compressors/LZMA/Lzma2EncoderStream.cs | 5 +++-- src/SharpCompress/Compressors/PPMd/PpmdStream.cs | 3 ++- .../Compressors/Squeezed/SqueezedStream.Async.cs | 7 ++++--- .../Compressors/Squeezed/SqueezedStream.cs | 5 +++-- src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs | 4 ++-- src/SharpCompress/packages.lock.json | 12 ++++++------ 10 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/SharpCompress/Archives/Tar/TarArchive.Async.cs b/src/SharpCompress/Archives/Tar/TarArchive.Async.cs index ac584e1b..45e5b1ca 100644 --- a/src/SharpCompress/Archives/Tar/TarArchive.Async.cs +++ b/src/SharpCompress/Archives/Tar/TarArchive.Async.cs @@ -141,7 +141,7 @@ public partial class TarArchive using (var entryStream = entry.OpenEntryStream()) { - using var memoryStream = new MemoryStream(); + using var memoryStream = new PooledMemoryStream(); await entryStream.CopyToAsync(memoryStream).ConfigureAwait(false); memoryStream.Position = 0; var bytes = memoryStream.ToArray(); diff --git a/src/SharpCompress/Archives/Tar/TarArchive.cs b/src/SharpCompress/Archives/Tar/TarArchive.cs index fdd8ac3a..44238549 100644 --- a/src/SharpCompress/Archives/Tar/TarArchive.cs +++ b/src/SharpCompress/Archives/Tar/TarArchive.cs @@ -151,7 +151,7 @@ public partial class TarArchive using (var entryStream = entry.OpenEntryStream()) { - using var memoryStream = new MemoryStream(); + using var memoryStream = new PooledMemoryStream(); entryStream.CopyTo(memoryStream, Constants.BufferSize); memoryStream.Position = 0; var bytes = memoryStream.ToArray(); diff --git a/src/SharpCompress/Common/SevenZip/ArchiveReader.cs b/src/SharpCompress/Common/SevenZip/ArchiveReader.cs index 3a4e5622..b75d123c 100644 --- a/src/SharpCompress/Common/SevenZip/ArchiveReader.cs +++ b/src/SharpCompress/Common/SevenZip/ArchiveReader.cs @@ -1182,7 +1182,7 @@ internal partial class ArchiveReader } else { - _stream = new MemoryStream(); + _stream = new PooledMemoryStream(); } _rem = _db._files[index].Size; } diff --git a/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs b/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs index f5fd410e..9162474b 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs @@ -2,6 +2,7 @@ using System; using System.IO; using System.Text; using SharpCompress.Compressors.LZMA.Utilities; +using SharpCompress.IO; namespace SharpCompress.Common.SevenZip; @@ -215,7 +216,7 @@ internal sealed class SevenZipFilesInfoWriter Action writeData ) { - using var dataStream = new MemoryStream(); + using var dataStream = new PooledMemoryStream(); writeData(dataStream); stream.WriteByte((byte)propertyId); diff --git a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs index 1642be44..84bd3f00 100644 --- a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs +++ b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using SharpCompress.IO; namespace SharpCompress.Compressors.LZMA; @@ -158,7 +159,7 @@ internal sealed class Lzma2EncoderStream : Stream } using var inputMs = new MemoryStream(data.ToArray(), writable: false); - using var outputMs = new MemoryStream(); + using var outputMs = new PooledMemoryStream(); encoder.Code(inputMs, outputMs, data.Length, -1, null); @@ -190,7 +191,7 @@ internal sealed class Lzma2EncoderStream : Stream decoder.SetDecoderProperties(props); using var input = new MemoryStream(compressedData); - using var output = new MemoryStream(); + using var output = new PooledMemoryStream(); decoder.Code(input, output, compressedData.Length, uncompressedSize, null); return (int)input.Position; diff --git a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs index 902246d7..210f5aa5 100644 --- a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs +++ b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using SharpCompress.Compressors.LZMA.RangeCoder; using SharpCompress.Compressors.PPMd.H; using SharpCompress.Compressors.PPMd.I1; +using SharpCompress.IO; namespace SharpCompress.Compressors.PPMd; @@ -179,7 +180,7 @@ public class PpmdStream : Stream { if (_compress) { - _model.EncodeBlock(_stream, new MemoryStream(), true); + _model.EncodeBlock(_stream, new PooledMemoryStream(), true); } } base.Dispose(disposing); diff --git a/src/SharpCompress/Compressors/Squeezed/SqueezedStream.Async.cs b/src/SharpCompress/Compressors/Squeezed/SqueezedStream.Async.cs index d8090dca..6a0cfcda 100644 --- a/src/SharpCompress/Compressors/Squeezed/SqueezedStream.Async.cs +++ b/src/SharpCompress/Compressors/Squeezed/SqueezedStream.Async.cs @@ -4,6 +4,7 @@ using System.Threading; using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Compressors.RLE90; +using SharpCompress.IO; namespace SharpCompress.Compressors.Squeezed; @@ -54,14 +55,14 @@ public partial class SqueezeStream if (bytesRead != 2) { - return new MemoryStream(Array.Empty()); + return new PooledMemoryStream(); } int numnodes = numNodesBytes[0] | (numNodesBytes[1] << 8); if (numnodes >= NUMVALS || numnodes == 0) { - return new MemoryStream(Array.Empty()); + return new PooledMemoryStream(); } var dnode = new int[numnodes, 2]; @@ -82,7 +83,7 @@ public partial class SqueezeStream } var bitReader = new BitReader(_stream); - var huffmanDecoded = new MemoryStream(); + var huffmanDecoded = new PooledMemoryStream(); int i = 0; while (true) diff --git a/src/SharpCompress/Compressors/Squeezed/SqueezedStream.cs b/src/SharpCompress/Compressors/Squeezed/SqueezedStream.cs index 09aa8b2c..f6a40f1c 100644 --- a/src/SharpCompress/Compressors/Squeezed/SqueezedStream.cs +++ b/src/SharpCompress/Compressors/Squeezed/SqueezedStream.cs @@ -4,6 +4,7 @@ using System.IO; using System.Text; using SharpCompress.Common; using SharpCompress.Compressors.RLE90; +using SharpCompress.IO; namespace SharpCompress.Compressors.Squeezed; @@ -67,7 +68,7 @@ public partial class SqueezeStream : Stream if (numnodes >= NUMVALS || numnodes == 0) { - return new MemoryStream(Array.Empty()); + return new PooledMemoryStream(); } var dnode = new int[numnodes, 2]; @@ -78,7 +79,7 @@ public partial class SqueezeStream : Stream } var bitReader = new BitReader(_stream); - var huffmanDecoded = new MemoryStream(); + var huffmanDecoded = new PooledMemoryStream(); int i = 0; while (true) diff --git a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs index a785aaf2..501959ee 100644 --- a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs +++ b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs @@ -171,7 +171,7 @@ public partial class SevenZipWriter : AbstractWriter var filesInfo = new SevenZipFilesInfoWriter { Entries = entries.ToArray() }; // Write header to a temporary stream first - using var headerStream = new MemoryStream(); + using var headerStream = new PooledMemoryStream(); ArchiveHeaderWriter.WriteRawHeader(headerStream, mainStreamsInfo, filesInfo); // Optionally compress the header @@ -212,7 +212,7 @@ public partial class SevenZipWriter : AbstractWriter }; // Write encoded header to a second temporary stream - using var encodedHeaderStream = new MemoryStream(); + using var encodedHeaderStream = new PooledMemoryStream(); ArchiveHeaderWriter.WriteEncodedHeader(encodedHeaderStream, headerStreamsInfo); // Write the encoded header to the output diff --git a/src/SharpCompress/packages.lock.json b/src/SharpCompress/packages.lock.json index b44de28b..5059aafe 100644 --- a/src/SharpCompress/packages.lock.json +++ b/src/SharpCompress/packages.lock.json @@ -268,9 +268,9 @@ "net10.0": { "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[10.0.2, )", - "resolved": "10.0.2", - "contentHash": "sXdDtMf2qcnbygw9OdE535c2lxSxrZP8gO4UhDJ0xiJbl1wIqXS1OTcTDFTIJPOFd6Mhcm8gPEthqWGUxBsTqw==" + "requested": "[10.0.0, )", + "resolved": "10.0.0", + "contentHash": "kICGrGYEzCNI3wPzfEXcwNHgTvlvVn9yJDhSdRK+oZQy4jvYH529u7O0xf5ocQKzOMjfS07+3z9PKRIjrFMJDA==" }, "Microsoft.NETFramework.ReferenceAssemblies": { "type": "Direct", @@ -442,9 +442,9 @@ "net8.0": { "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[8.0.23, )", - "resolved": "8.0.23", - "contentHash": "GqHiB1HbbODWPbY/lc5xLQH8siEEhNA0ptpJCC6X6adtAYNEzu5ZlqV3YHA3Gh7fuEwgA8XqVwMtH2KNtuQM1Q==" + "requested": "[8.0.22, )", + "resolved": "8.0.22", + "contentHash": "MhcMithKEiyyNkD2ZfbDZPmcOdi0GheGfg8saEIIEfD/fol3iHmcV8TsZkD4ZYz5gdUuoX4YtlVySUU7Sxl9SQ==" }, "Microsoft.NETFramework.ReferenceAssemblies": { "type": "Direct", From 2bf007542a5bd4cdd74093134f0abdd50bbf0d03 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sat, 4 Apr 2026 11:10:43 +0100 Subject: [PATCH 03/20] cleaned up PooledMemoryStream.cs --- src/SharpCompress/IO/PooledMemoryStream.cs | 157 ++++-------------- .../Streams/PooledMemoryStreamTests.cs | 51 ++---- 2 files changed, 49 insertions(+), 159 deletions(-) diff --git a/src/SharpCompress/IO/PooledMemoryStream.cs b/src/SharpCompress/IO/PooledMemoryStream.cs index 58bc748f..3bbdb330 100644 --- a/src/SharpCompress/IO/PooledMemoryStream.cs +++ b/src/SharpCompress/IO/PooledMemoryStream.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; +using SharpCompress.Common; namespace SharpCompress.IO; @@ -12,8 +13,6 @@ namespace SharpCompress.IO; /// public sealed class PooledMemoryStream : MemoryStream { - public const int DefaultBlockSize = 81920; - private const int MaxStreamLength = int.MaxValue; private enum StorageMode @@ -33,11 +32,6 @@ public sealed class PooledMemoryStream : MemoryStream private bool _contiguousBufferExposed; private bool _isOpen; - private bool _writable; - private bool _expandable; - private bool _exposable; - - private int _origin; private int _position; private int _length; private int _capacity; @@ -47,7 +41,7 @@ public sealed class PooledMemoryStream : MemoryStream : this(0) { } public PooledMemoryStream(int capacity) - : this(capacity, DefaultBlockSize, ArrayPool.Shared) { } + : this(capacity, Constants.BufferSize, ArrayPool.Shared) { } public PooledMemoryStream(int capacity, int blockSize) : this(capacity, blockSize, ArrayPool.Shared) { } @@ -64,10 +58,6 @@ public sealed class PooledMemoryStream : MemoryStream _mode = StorageMode.Segmented; _blocks = new List(); _isOpen = true; - _writable = true; - _expandable = true; - _exposable = true; - _origin = 0; _position = 0; _length = 0; _capacity = capacity; @@ -75,67 +65,18 @@ public sealed class PooledMemoryStream : MemoryStream EnsureSegmentedAllocated(capacity); } - public PooledMemoryStream(byte[] buffer) - : this(buffer, writable: true) { } - - public PooledMemoryStream(byte[] buffer, bool writable) - : this(buffer, 0, buffer?.Length ?? 0, writable, publiclyVisible: false) { } - - public PooledMemoryStream(byte[] buffer, int index, int count) - : this(buffer, index, count, writable: true, publiclyVisible: false) { } - - public PooledMemoryStream(byte[] buffer, int index, int count, bool writable) - : this(buffer, index, count, writable, publiclyVisible: false) { } - - public PooledMemoryStream( - byte[] buffer, - int index, - int count, - bool writable, - bool publiclyVisible - ) - { - ThrowHelper.ThrowIfNull(buffer, nameof(buffer)); - ThrowHelper.ThrowIfNegative(index, nameof(index)); - ThrowHelper.ThrowIfNegative(count, nameof(count)); - if (buffer.Length - index < count) - { - throw new ArgumentException("Offset and length are out of bounds."); - } - - _arrayPool = ArrayPool.Shared; - _blockSize = DefaultBlockSize; - - _mode = StorageMode.Segmented; - _blocks = new List(); - _origin = 0; - _position = 0; - _length = count; - _capacity = count; - _writable = writable; - _expandable = false; - _exposable = publiclyVisible; - _isOpen = true; - - EnsureSegmentedAllocated(_capacity); - if (count > 0) - { - CopyToSegmented(0, buffer, index, count); - } - } - public override bool CanRead => _isOpen; public override bool CanSeek => _isOpen; - public override bool CanWrite => _writable; + public override bool CanWrite => _isOpen; public override long Length { get { EnsureNotClosed(); - return _length - _origin; + return _length; } } @@ -144,15 +85,15 @@ public sealed class PooledMemoryStream : MemoryStream get { EnsureNotClosed(); - return _position - _origin; + return _position; } set { ThrowHelper.ThrowIfNegative(value, nameof(value)); EnsureNotClosed(); - ThrowHelper.ThrowIfGreaterThan(value, MaxStreamLength - _origin, nameof(value)); + ThrowHelper.ThrowIfGreaterThan(value, MaxStreamLength, nameof(value)); - _position = _origin + (int)value; + _position = (int)value; } } @@ -161,23 +102,15 @@ public sealed class PooledMemoryStream : MemoryStream get { EnsureNotClosed(); - return _capacity - _origin; + return _capacity; } set { - if (value < Length) - { - throw new ArgumentOutOfRangeException(nameof(value)); - } + ThrowHelper.ThrowIfLessThan(value, _length, nameof(value)); EnsureNotClosed(); - if (!_expandable && value != Capacity) - { - throw new NotSupportedException("Memory stream is not expandable."); - } - - var target = _origin + value; + var target = value; if (target == _capacity) { return; @@ -205,14 +138,14 @@ public sealed class PooledMemoryStream : MemoryStream var anchor = loc switch { - SeekOrigin.Begin => _origin, + SeekOrigin.Begin => 0, SeekOrigin.Current => _position, SeekOrigin.End => _length, _ => throw new ArgumentException("Invalid seek origin.", nameof(loc)), }; var target = anchor + offset; - if (target < _origin) + if (target < 0) { throw new IOException("Attempted to seek before the beginning of the stream."); } @@ -223,7 +156,7 @@ public sealed class PooledMemoryStream : MemoryStream } _position = (int)target; - return _position - _origin; + return _position; } public override void SetLength(long value) @@ -233,7 +166,7 @@ public sealed class PooledMemoryStream : MemoryStream EnsureWritable(); - var newLength = _origin + (int)value; + var newLength = (int)value; if (newLength > _capacity) { EnsureCapacityForAppend(newLength); @@ -395,11 +328,6 @@ public sealed class PooledMemoryStream : MemoryStream public override byte[] GetBuffer() { EnsureNotClosed(); - if (!_exposable) - { - throw new UnauthorizedAccessException("Memory stream buffer is not publicly visible."); - } - EnsureContiguous(); _contiguousBufferExposed = true; return _contiguousBuffer!; @@ -407,12 +335,6 @@ public sealed class PooledMemoryStream : MemoryStream public override bool TryGetBuffer(out ArraySegment buffer) { - if (!_exposable) - { - buffer = default; - return false; - } - EnsureNotClosed(); EnsureContiguous(); @@ -425,7 +347,7 @@ public sealed class PooledMemoryStream : MemoryStream { EnsureNotClosed(); - var count = _length - _origin; + var count = _length; if (count == 0) { return Array.Empty(); @@ -435,10 +357,10 @@ public sealed class PooledMemoryStream : MemoryStream switch (_mode) { case StorageMode.Contiguous: - Buffer.BlockCopy(_contiguousBuffer!, _origin, copy, 0, count); + Buffer.BlockCopy(_contiguousBuffer!, 0, copy, 0, count); break; case StorageMode.Segmented: - CopyFromSegmented(_origin, copy, 0, count); + CopyFromSegmented(0, copy, 0, count); break; } @@ -450,7 +372,7 @@ public sealed class PooledMemoryStream : MemoryStream ThrowHelper.ThrowIfNull(stream, nameof(stream)); EnsureNotClosed(); - var count = _length - _origin; + var count = _length; if (count == 0) { return; @@ -459,11 +381,11 @@ public sealed class PooledMemoryStream : MemoryStream switch (_mode) { case StorageMode.Contiguous: - stream.Write(_contiguousBuffer!, _origin, count); + stream.Write(_contiguousBuffer!, 0, count); break; case StorageMode.Segmented: { - var position = _origin; + var position = 0; var remaining = count; while (remaining > 0) { @@ -526,7 +448,7 @@ public sealed class PooledMemoryStream : MemoryStream } #if !LEGACY_DOTNET - public override int Read(Span buffer) + public override int Read(Span destination) { EnsureNotClosed(); @@ -536,11 +458,11 @@ public sealed class PooledMemoryStream : MemoryStream return 0; } - var count = Math.Min(available, buffer.Length); + var count = Math.Min(available, destination.Length); switch (_mode) { case StorageMode.Contiguous: - _contiguousBuffer.AsSpan(_position, count).CopyTo(buffer); + _contiguousBuffer.AsSpan(_position, count).CopyTo(destination); break; case StorageMode.Segmented: { @@ -556,7 +478,7 @@ public sealed class PooledMemoryStream : MemoryStream _blocks! [blockIndex] .AsSpan(blockOffset, toCopy) - .CopyTo(buffer.Slice(destinationOffset, toCopy)); + .CopyTo(destination.Slice(destinationOffset, toCopy)); sourcePosition += toCopy; destinationOffset += toCopy; @@ -571,15 +493,15 @@ public sealed class PooledMemoryStream : MemoryStream return count; } - public override void Write(ReadOnlySpan buffer) + public override void Write(ReadOnlySpan source) { EnsureWritable(); - if (buffer.Length == 0) + if (source.Length == 0) { return; } - var endPosition = _position + buffer.Length; + var endPosition = _position + source.Length; if (endPosition < 0) { throw new IOException("Stream is too long."); @@ -598,13 +520,13 @@ public sealed class PooledMemoryStream : MemoryStream switch (_mode) { case StorageMode.Contiguous: - buffer.CopyTo(_contiguousBuffer.AsSpan(_position, buffer.Length)); + source.CopyTo(_contiguousBuffer.AsSpan(_position, source.Length)); break; case StorageMode.Segmented: { var sourceOffset = 0; var destinationPosition = _position; - var remaining = buffer.Length; + var remaining = source.Length; while (remaining > 0) { @@ -612,7 +534,7 @@ public sealed class PooledMemoryStream : MemoryStream var blockOffset = destinationPosition % _blockSize; var toCopy = Math.Min(remaining, _blockSize - blockOffset); - buffer + source .Slice(sourceOffset, toCopy) .CopyTo(_blocks![blockIndex].AsSpan(blockOffset, toCopy)); @@ -633,7 +555,7 @@ public sealed class PooledMemoryStream : MemoryStream } public override ValueTask ReadAsync( - Memory buffer, + Memory destination, CancellationToken cancellationToken = default ) { @@ -644,7 +566,7 @@ public sealed class PooledMemoryStream : MemoryStream try { - return ValueTask.FromResult(Read(buffer.Span)); + return ValueTask.FromResult(Read(destination.Span)); } catch (Exception ex) { @@ -653,7 +575,7 @@ public sealed class PooledMemoryStream : MemoryStream } public override ValueTask WriteAsync( - ReadOnlyMemory buffer, + ReadOnlyMemory source, CancellationToken cancellationToken = default ) { @@ -664,7 +586,7 @@ public sealed class PooledMemoryStream : MemoryStream try { - Write(buffer.Span); + Write(source.Span); return ValueTask.CompletedTask; } catch (Exception ex) @@ -679,8 +601,6 @@ public sealed class PooledMemoryStream : MemoryStream if (_isOpen) { _isOpen = false; - _writable = false; - _expandable = false; if (disposing) { @@ -702,10 +622,6 @@ public sealed class PooledMemoryStream : MemoryStream private void EnsureWritable() { EnsureNotClosed(); - if (!_writable) - { - throw new NotSupportedException("Stream does not support writing."); - } } private void EnsureCapacityForAppend(int requiredLength) @@ -720,11 +636,6 @@ public sealed class PooledMemoryStream : MemoryStream return; } - if (!_expandable) - { - throw new NotSupportedException("Memory stream is not expandable."); - } - var nextCapacity = RoundUpToBlockBoundary(requiredLength); SetCapacityAbsolute(nextCapacity); } diff --git a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs index 28af7f1c..c628821d 100644 --- a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs +++ b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs @@ -54,49 +54,28 @@ public class PooledMemoryStreamTests } [Fact] - public void BufferConstructorCopiesDataAndIsNonExpandable() + public void TryGetBufferReturnsSegmentWhenOpen() { - var backing = Enumerable.Range(0, 10).Select(i => (byte)i).ToArray(); - using var stream = new PooledMemoryStream( - backing, - 2, - 4, - writable: true, - publiclyVisible: true - ); + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8); + stream.Write(new byte[] { 1, 2, 3, 4 }, 0, 4); - Assert.Equal(4, stream.Length); - Assert.Equal(4, stream.Capacity); - - backing[2] = 255; - Assert.Equal(new byte[] { 2, 3, 4, 5 }, stream.ToArray()); - - Assert.Throws(() => stream.Capacity = 5); + Assert.True(stream.TryGetBuffer(out var segment)); + Assert.Equal(0, segment.Offset); + Assert.Equal(4, segment.Count); + Assert.Equal(1, segment.Array![0]); } [Fact] - public void BufferConstructorTryGetBufferRespectsVisibilityFlag() + public void CapacitySetterCanGrowAndShrinkWithinLength() { - var backing = new byte[10]; - using var hidden = new PooledMemoryStream( - backing, - 1, - 5, - writable: true, - publiclyVisible: false - ); - Assert.False(hidden.TryGetBuffer(out _)); + using var stream = new PooledMemoryStream(capacity: 16, blockSize: 8); + stream.Write(new byte[6], 0, 6); - using var visible = new PooledMemoryStream( - backing, - 1, - 5, - writable: true, - publiclyVisible: true - ); - Assert.True(visible.TryGetBuffer(out var segment)); - Assert.Equal(0, segment.Offset); - Assert.Equal(5, segment.Count); + stream.Capacity = 24; + Assert.Equal(24, stream.Capacity); + + stream.Capacity = 8; + Assert.Equal(8, stream.Capacity); } [Fact] From db9bdb20bd931219ac20e0521d8508a5c0eca523 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 5 Apr 2026 14:09:58 +0100 Subject: [PATCH 04/20] Update src/SharpCompress/IO/PooledMemoryStream.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/SharpCompress/IO/PooledMemoryStream.cs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/SharpCompress/IO/PooledMemoryStream.cs b/src/SharpCompress/IO/PooledMemoryStream.cs index 3bbdb330..9cd19696 100644 --- a/src/SharpCompress/IO/PooledMemoryStream.cs +++ b/src/SharpCompress/IO/PooledMemoryStream.cs @@ -325,21 +325,31 @@ public sealed class PooledMemoryStream : MemoryStream } } + private byte[] GetExposableContiguousBuffer() + { + EnsureContiguous(); + + if (_capacity > _length) + { + ClearRange(_length, _capacity - _length); + } + + _contiguousBufferExposed = true; + return _contiguousBuffer!; + } + public override byte[] GetBuffer() { EnsureNotClosed(); - EnsureContiguous(); - _contiguousBufferExposed = true; - return _contiguousBuffer!; + return GetExposableContiguousBuffer(); } public override bool TryGetBuffer(out ArraySegment buffer) { EnsureNotClosed(); - EnsureContiguous(); - _contiguousBufferExposed = true; - buffer = new ArraySegment(_contiguousBuffer!, 0, _length); + var exposableBuffer = GetExposableContiguousBuffer(); + buffer = new ArraySegment(exposableBuffer, 0, _length); return true; } From 6b9a3f8b69a57f999b48b5af3a089045bb28488b Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 5 Apr 2026 14:10:17 +0100 Subject: [PATCH 05/20] Update src/SharpCompress/Compressors/PPMd/PpmdStream.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/SharpCompress/Compressors/PPMd/PpmdStream.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs index 210f5aa5..6f1e7fcb 100644 --- a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs +++ b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs @@ -180,7 +180,7 @@ public class PpmdStream : Stream { if (_compress) { - _model.EncodeBlock(_stream, new PooledMemoryStream(), true); + _model.EncodeBlock(_stream, Stream.Null, true); } } base.Dispose(disposing); From 5e7644bd1857840ef876bbf3317aa7c0a91dfe59 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 5 Apr 2026 14:11:48 +0100 Subject: [PATCH 06/20] Update src/SharpCompress/IO/PooledMemoryStream.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/SharpCompress/IO/PooledMemoryStream.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/SharpCompress/IO/PooledMemoryStream.cs b/src/SharpCompress/IO/PooledMemoryStream.cs index 9cd19696..a832bf69 100644 --- a/src/SharpCompress/IO/PooledMemoryStream.cs +++ b/src/SharpCompress/IO/PooledMemoryStream.cs @@ -892,11 +892,10 @@ public sealed class PooledMemoryStream : MemoryStream _contiguousBuffer = null; } - for (var i = 0; i < _detachedExposedBuffers.Count; i++) - { - _arrayPool.Return(_detachedExposedBuffers[i]); - } - + // Buffers tracked here have been exposed to callers. Returning them to the + // shared pool would allow unrelated code to rent and mutate arrays that may + // still be referenced after the stream is disposed, which breaks + // MemoryStream-compatible expectations for GetBuffer/TryGetBuffer. _detachedExposedBuffers.Clear(); } From c01dd5cd90d21a3ede9e18a0305149d30b1429e0 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 5 Apr 2026 14:16:47 +0100 Subject: [PATCH 07/20] Update tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../Streams/PooledMemoryStreamTests.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs index c628821d..d5e9a192 100644 --- a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs +++ b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs @@ -107,13 +107,22 @@ public class PooledMemoryStreamTests private sealed class TrackingArrayPool : ArrayPool { + private const byte RentedBufferFillValue = 0x5A; + public readonly System.Collections.Generic.List RentRequests = new(); public readonly System.Collections.Generic.List ReturnedLengths = new(); public override byte[] Rent(int minimumLength) { RentRequests.Add(minimumLength); - return new byte[minimumLength]; + + var array = new byte[minimumLength]; + for (var i = 0; i < array.Length; i++) + { + array[i] = RentedBufferFillValue; + } + + return array; } public override void Return(byte[] array, bool clearArray = false) From 8c4fed373b8de60f9650271f7c7a48746fb092b0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 5 Apr 2026 13:21:54 +0000 Subject: [PATCH 08/20] Remove try/catch from async overloads and fix CA1725 parameter naming in PooledMemoryStream Agent-Logs-Url: https://github.com/adamhathcock/sharpcompress/sessions/1f2c3c48-1112-43f1-82ce-efb157fbe0d5 Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com> --- src/SharpCompress/IO/PooledMemoryStream.cs | 64 ++++++---------------- 1 file changed, 18 insertions(+), 46 deletions(-) diff --git a/src/SharpCompress/IO/PooledMemoryStream.cs b/src/SharpCompress/IO/PooledMemoryStream.cs index a832bf69..cf94ad1e 100644 --- a/src/SharpCompress/IO/PooledMemoryStream.cs +++ b/src/SharpCompress/IO/PooledMemoryStream.cs @@ -424,14 +424,7 @@ public sealed class PooledMemoryStream : MemoryStream return Task.FromCanceled(cancellationToken); } - try - { - return Task.FromResult(Read(buffer, offset, count)); - } - catch (Exception ex) - { - return Task.FromException(ex); - } + return Task.FromResult(Read(buffer, offset, count)); } public override Task WriteAsync( @@ -446,19 +439,12 @@ public sealed class PooledMemoryStream : MemoryStream return Task.FromCanceled(cancellationToken); } - try - { - Write(buffer, offset, count); - return Task.CompletedTask; - } - catch (Exception ex) - { - return Task.FromException(ex); - } + Write(buffer, offset, count); + return Task.CompletedTask; } #if !LEGACY_DOTNET - public override int Read(Span destination) + public override int Read(Span buffer) { EnsureNotClosed(); @@ -468,11 +454,11 @@ public sealed class PooledMemoryStream : MemoryStream return 0; } - var count = Math.Min(available, destination.Length); + var count = Math.Min(available, buffer.Length); switch (_mode) { case StorageMode.Contiguous: - _contiguousBuffer.AsSpan(_position, count).CopyTo(destination); + _contiguousBuffer.AsSpan(_position, count).CopyTo(buffer); break; case StorageMode.Segmented: { @@ -488,7 +474,7 @@ public sealed class PooledMemoryStream : MemoryStream _blocks! [blockIndex] .AsSpan(blockOffset, toCopy) - .CopyTo(destination.Slice(destinationOffset, toCopy)); + .CopyTo(buffer.Slice(destinationOffset, toCopy)); sourcePosition += toCopy; destinationOffset += toCopy; @@ -503,15 +489,15 @@ public sealed class PooledMemoryStream : MemoryStream return count; } - public override void Write(ReadOnlySpan source) + public override void Write(ReadOnlySpan buffer) { EnsureWritable(); - if (source.Length == 0) + if (buffer.Length == 0) { return; } - var endPosition = _position + source.Length; + var endPosition = _position + buffer.Length; if (endPosition < 0) { throw new IOException("Stream is too long."); @@ -530,13 +516,13 @@ public sealed class PooledMemoryStream : MemoryStream switch (_mode) { case StorageMode.Contiguous: - source.CopyTo(_contiguousBuffer.AsSpan(_position, source.Length)); + buffer.CopyTo(_contiguousBuffer.AsSpan(_position, buffer.Length)); break; case StorageMode.Segmented: { var sourceOffset = 0; var destinationPosition = _position; - var remaining = source.Length; + var remaining = buffer.Length; while (remaining > 0) { @@ -544,7 +530,7 @@ public sealed class PooledMemoryStream : MemoryStream var blockOffset = destinationPosition % _blockSize; var toCopy = Math.Min(remaining, _blockSize - blockOffset); - source + buffer .Slice(sourceOffset, toCopy) .CopyTo(_blocks![blockIndex].AsSpan(blockOffset, toCopy)); @@ -565,7 +551,7 @@ public sealed class PooledMemoryStream : MemoryStream } public override ValueTask ReadAsync( - Memory destination, + Memory buffer, CancellationToken cancellationToken = default ) { @@ -574,18 +560,11 @@ public sealed class PooledMemoryStream : MemoryStream return ValueTask.FromCanceled(cancellationToken); } - try - { - return ValueTask.FromResult(Read(destination.Span)); - } - catch (Exception ex) - { - return ValueTask.FromException(ex); - } + return ValueTask.FromResult(Read(buffer.Span)); } public override ValueTask WriteAsync( - ReadOnlyMemory source, + ReadOnlyMemory buffer, CancellationToken cancellationToken = default ) { @@ -594,15 +573,8 @@ public sealed class PooledMemoryStream : MemoryStream return ValueTask.FromCanceled(cancellationToken); } - try - { - Write(source.Span); - return ValueTask.CompletedTask; - } - catch (Exception ex) - { - return ValueTask.FromException(ex); - } + Write(buffer.Span); + return ValueTask.CompletedTask; } #endif From c25fec313eba26b7c9d0c1758c18acd3c2d7b686 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 5 Apr 2026 14:39:15 +0100 Subject: [PATCH 09/20] fix PooledMemoryStream.cs --- src/SharpCompress/IO/PooledMemoryStream.cs | 11 +++++++-- .../Streams/PooledMemoryStreamTests.cs | 24 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/SharpCompress/IO/PooledMemoryStream.cs b/src/SharpCompress/IO/PooledMemoryStream.cs index cf94ad1e..aadc2507 100644 --- a/src/SharpCompress/IO/PooledMemoryStream.cs +++ b/src/SharpCompress/IO/PooledMemoryStream.cs @@ -10,7 +10,15 @@ namespace SharpCompress.IO; /// /// MemoryStream implementation backed by pooled byte arrays. +/// Uses to reduce GC pressure for temporary buffers. /// +/// +/// This implementation is not thread-safe. Use appropriate synchronization for concurrent access. +/// Buffers exposed via or will not be +/// returned to the pool on dispose to maintain MemoryStream-compatible semantics. +/// The stream dynamically switches between segmented (multiple blocks) and contiguous storage modes +/// based on usage patterns, optimizing for both memory efficiency and performance. +/// public sealed class PooledMemoryStream : MemoryStream { private const int MaxStreamLength = int.MaxValue; @@ -857,8 +865,7 @@ public sealed class PooledMemoryStream : MemoryStream ReturnSegmentedBlocks(); _blocks = null; } - - if (_mode == StorageMode.Contiguous && _contiguousBuffer is not null) + else if (_mode == StorageMode.Contiguous && _contiguousBuffer is not null) { _arrayPool.Return(_contiguousBuffer); _contiguousBuffer = null; diff --git a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs index d5e9a192..6b12f2a3 100644 --- a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs +++ b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs @@ -105,6 +105,30 @@ public class PooledMemoryStreamTests Assert.Throws(() => stream.GetBuffer()); } + [Fact] + public void MultipleGetBufferCallsReturnSameArray() + { + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8); + stream.Write(new byte[] { 1, 2, 3 }, 0, 3); + + var buffer1 = stream.GetBuffer(); + var buffer2 = stream.GetBuffer(); + + Assert.Same(buffer1, buffer2); + Assert.Equal(1, buffer1[0]); + Assert.Equal(2, buffer1[1]); + Assert.Equal(3, buffer1[2]); + } + + [Fact] + public void SeekBeyondMaxLengthThrows() + { + using var stream = new PooledMemoryStream(); + Assert.Throws(() => + stream.Seek(int.MaxValue + 1L, SeekOrigin.Begin) + ); + } + private sealed class TrackingArrayPool : ArrayPool { private const byte RentedBufferFillValue = 0x5A; From bfe78249fc73470ae3997a1d5ea40d12c380c748 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 5 Apr 2026 14:47:41 +0100 Subject: [PATCH 10/20] remove net5 from target frameworks to avoid issues --- src/SharpCompress/Common/Rar/CryptKey5.cs | 4 +- .../Common/Tar/Headers/TarHeader.Async.cs | 2 +- .../Common/Tar/Headers/TarHeader.cs | 2 +- .../ZStandard/Unsafe/ZstdInternal.cs | 4 +- .../Compressors/ZStandard/Unsafe/ZstdLazy.cs | 6 +-- .../Polyfills/StreamExtensions.cs | 2 +- src/SharpCompress/SharpCompress.csproj | 2 +- src/SharpCompress/packages.lock.json | 42 ------------------- 8 files changed, 11 insertions(+), 53 deletions(-) diff --git a/src/SharpCompress/Common/Rar/CryptKey5.cs b/src/SharpCompress/Common/Rar/CryptKey5.cs index 66f08aa3..490a3c54 100644 --- a/src/SharpCompress/Common/Rar/CryptKey5.cs +++ b/src/SharpCompress/Common/Rar/CryptKey5.cs @@ -35,7 +35,7 @@ internal class CryptKey5 : ICryptKey ) { var passwordBytes = Encoding.UTF8.GetBytes(password); -#if LEGACY_DOTNET || NET5_0 +#if LEGACY_DOTNET using var hmac = new HMACSHA256(passwordBytes); var block = hmac.ComputeHash(salt); #else @@ -50,7 +50,7 @@ internal class CryptKey5 : ICryptKey { for (var i = 1; i < loop[x]; i++) { -#if LEGACY_DOTNET || NET5_0 +#if LEGACY_DOTNET block = hmac.ComputeHash(block); #else block = HMACSHA256.HashData(passwordBytes, block); diff --git a/src/SharpCompress/Common/Tar/Headers/TarHeader.Async.cs b/src/SharpCompress/Common/Tar/Headers/TarHeader.Async.cs index 870c472c..084cd83d 100644 --- a/src/SharpCompress/Common/Tar/Headers/TarHeader.Async.cs +++ b/src/SharpCompress/Common/Tar/Headers/TarHeader.Async.cs @@ -59,7 +59,7 @@ internal sealed partial class TarHeader int splitIndex = -1; for (int i = 0; i < dirSeps.Count; i++) { -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER int count = ArchiveEncoding .GetEncoding() .GetByteCount(fullName.AsSpan(0, dirSeps[i])); diff --git a/src/SharpCompress/Common/Tar/Headers/TarHeader.cs b/src/SharpCompress/Common/Tar/Headers/TarHeader.cs index 2c538067..233970b2 100644 --- a/src/SharpCompress/Common/Tar/Headers/TarHeader.cs +++ b/src/SharpCompress/Common/Tar/Headers/TarHeader.cs @@ -89,7 +89,7 @@ internal sealed partial class TarHeader int splitIndex = -1; for (int i = 0; i < dirSeps.Count; i++) { -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER int count = ArchiveEncoding .GetEncoding() .GetByteCount(fullName.AsSpan(0, dirSeps[i])); diff --git a/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdInternal.cs b/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdInternal.cs index c8b9f15d..4180d6b1 100644 --- a/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdInternal.cs +++ b/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdInternal.cs @@ -5,7 +5,7 @@ using static SharpCompress.Compressors.ZStandard.UnsafeHelper; #if NETCOREAPP3_0_OR_GREATER using System.Runtime.Intrinsics.X86; #endif -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER using System.Runtime.Intrinsics.Arm; #endif @@ -554,7 +554,7 @@ public static unsafe partial class Methods [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void ZSTD_copy16(void* dst, void* src) { -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER if (AdvSimd.IsSupported) { AdvSimd.Store((byte*)dst, AdvSimd.LoadVector128((byte*)src)); diff --git a/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdLazy.cs b/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdLazy.cs index 178160b1..fdf26282 100644 --- a/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdLazy.cs +++ b/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdLazy.cs @@ -6,7 +6,7 @@ using static SharpCompress.Compressors.ZStandard.UnsafeHelper; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; #endif -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER using System.Runtime.Intrinsics.Arm; #endif @@ -1172,7 +1172,7 @@ public static unsafe partial class Methods { assert(rowEntries == 16 || rowEntries == 32 || rowEntries == 64); assert(rowEntries <= 64); -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER if (AdvSimd.IsSupported && BitConverter.IsLittleEndian) { if (rowEntries == 16) @@ -1272,7 +1272,7 @@ public static unsafe partial class Methods } #endif -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER if (AdvSimd.IsSupported && BitConverter.IsLittleEndian) { if (rowEntries == 16) diff --git a/src/SharpCompress/Polyfills/StreamExtensions.cs b/src/SharpCompress/Polyfills/StreamExtensions.cs index 18c5b593..e3caa24f 100644 --- a/src/SharpCompress/Polyfills/StreamExtensions.cs +++ b/src/SharpCompress/Polyfills/StreamExtensions.cs @@ -28,7 +28,7 @@ public static class StreamExtensions public Task SkipAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER return stream.CopyToAsync(Stream.Null, cancellationToken); #else return stream.CopyToAsync(Stream.Null); diff --git a/src/SharpCompress/SharpCompress.csproj b/src/SharpCompress/SharpCompress.csproj index 0a9c440c..bcde6224 100644 --- a/src/SharpCompress/SharpCompress.csproj +++ b/src/SharpCompress/SharpCompress.csproj @@ -6,7 +6,7 @@ 0.0.0.0 0.0.0.0 Adam Hathcock - net48;netstandard2.0;netstandard2.1;net5.0;net6.0;net7.0;net8.0;net9.0;net10.0 + net48;netstandard2.0;netstandard2.1;net6.0;net7.0;net8.0;net9.0;net10.0 SharpCompress ../../SharpCompress.snk true diff --git a/src/SharpCompress/packages.lock.json b/src/SharpCompress/packages.lock.json index 5059aafe..a401c702 100644 --- a/src/SharpCompress/packages.lock.json +++ b/src/SharpCompress/packages.lock.json @@ -313,48 +313,6 @@ "contentHash": "Mk1IMb9q5tahC2NltxYXFkLBtuBvfBoCQ3pIxYQWfzbCE9o1OB9SsHe0hnNGo7lWgTA/ePbFAJLWu6nLL9K17A==" } }, - ".NETCoreApp,Version=v5.0": { - "Microsoft.NETFramework.ReferenceAssemblies": { - "type": "Direct", - "requested": "[1.0.3, )", - "resolved": "1.0.3", - "contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg==", - "dependencies": { - "Microsoft.NETFramework.ReferenceAssemblies.net461": "1.0.3" - } - }, - "Microsoft.SourceLink.GitHub": { - "type": "Direct", - "requested": "[10.0.102, )", - "resolved": "10.0.102", - "contentHash": "Oxq3RCIJSdtpIU4hLqO7XaDe/Ra3HS9Wi8rJl838SAg6Zu1iQjerA0+xXWBgUFYbgknUGCLOU0T+lzMLkvY9Qg==", - "dependencies": { - "Microsoft.Build.Tasks.Git": "10.0.102", - "Microsoft.SourceLink.Common": "10.0.102" - } - }, - "Microsoft.VisualStudio.Threading.Analyzers": { - "type": "Direct", - "requested": "[17.14.15, )", - "resolved": "17.14.15", - "contentHash": "mXQPJsbuUD2ydq4/ffd8h8tSOFCXec+2xJOVNCvXjuMOq/+5EKHq3D2m2MC2+nUaXeFMSt66VS/J4HdKBixgcw==" - }, - "Microsoft.Build.Tasks.Git": { - "type": "Transitive", - "resolved": "10.0.102", - "contentHash": "0i81LYX31U6UiXz4NOLbvc++u+/mVDmOt+PskrM/MygpDxkv9THKQyRUmavBpLK6iBV0abNWnn+CQgSRz//Pwg==" - }, - "Microsoft.NETFramework.ReferenceAssemblies.net461": { - "type": "Transitive", - "resolved": "1.0.3", - "contentHash": "AmOJZwCqnOCNp6PPcf9joyogScWLtwy0M1WkqfEQ0M9nYwyDD7EX9ZjscKS5iYnyvteX7kzSKFCKt9I9dXA6mA==" - }, - "Microsoft.SourceLink.Common": { - "type": "Transitive", - "resolved": "10.0.102", - "contentHash": "Mk1IMb9q5tahC2NltxYXFkLBtuBvfBoCQ3pIxYQWfzbCE9o1OB9SsHe0hnNGo7lWgTA/ePbFAJLWu6nLL9K17A==" - } - }, "net6.0": { "Microsoft.NETFramework.ReferenceAssemblies": { "type": "Direct", From 4f09c6dc0009062ca976397e8524cf42c0e18de0 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 5 Apr 2026 14:57:04 +0100 Subject: [PATCH 11/20] Update tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../Streams/PooledMemoryStreamTests.cs | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs index 6b12f2a3..91f64a40 100644 --- a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs +++ b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs @@ -129,6 +129,59 @@ public class PooledMemoryStreamTests ); } + [Fact] + public void DisposeAfterGetBufferDoesNotReturnExposedArrayToPool() + { + var pool = new TrackingArrayPool(); + byte[] buffer; + + using (var stream = new PooledMemoryStream(pool, capacity: 0, blockSize: 8)) + { + stream.Write(new byte[] { 1, 2, 3 }, 0, 3); + buffer = stream.GetBuffer(); + + Assert.NotNull(buffer); + Assert.NotEmpty(pool.RentRequests); + } + + Assert.Empty(pool.ReturnedLengths); + Assert.Equal(1, buffer[0]); + Assert.Equal(2, buffer[1]); + Assert.Equal(3, buffer[2]); + } + + [Fact] + public void DisposeAfterTryGetBufferDoesNotReturnExposedArrayToPool() + { + var pool = new TrackingArrayPool(); + ArraySegment segment; + + using (var stream = new PooledMemoryStream(pool, capacity: 0, blockSize: 8)) + { + stream.Write(new byte[] { 1, 2, 3 }, 0, 3); + + Assert.True(stream.TryGetBuffer(out segment)); + Assert.NotNull(segment.Array); + Assert.NotEmpty(pool.RentRequests); + } + + Assert.Empty(pool.ReturnedLengths); + Assert.Equal(1, segment.Array![segment.Offset]); + Assert.Equal(2, segment.Array[segment.Offset + 1]); + Assert.Equal(3, segment.Array[segment.Offset + 2]); + } + + [Fact] + public void SetLengthNearIntMaxValueDoesNotThrowIOException() + { + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8); + var length = int.MaxValue - 1L; + + var exception = Record.Exception(() => stream.SetLength(length)); + + Assert.Null(exception); + Assert.Equal(length, stream.Length); + } private sealed class TrackingArrayPool : ArrayPool { private const byte RentedBufferFillValue = 0x5A; From ec46584e545736338c2b9ccd3a52881d9ecac62f Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 5 Apr 2026 14:57:36 +0100 Subject: [PATCH 12/20] Update src/SharpCompress/IO/PooledMemoryStream.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/SharpCompress/IO/PooledMemoryStream.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/SharpCompress/IO/PooledMemoryStream.cs b/src/SharpCompress/IO/PooledMemoryStream.cs index aadc2507..5bfce1dc 100644 --- a/src/SharpCompress/IO/PooledMemoryStream.cs +++ b/src/SharpCompress/IO/PooledMemoryStream.cs @@ -680,6 +680,7 @@ public sealed class PooledMemoryStream : MemoryStream _contiguousBuffer = contiguous; _contiguousBufferExposed = false; _allocatedCapacity = contiguous.Length; + _capacity = _allocatedCapacity; } private void DemoteContiguousToSegmented() From fb785985163ab979f5287bba0b637516fafdf499 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 5 Apr 2026 15:16:00 +0100 Subject: [PATCH 13/20] implement GetBuffer with non-pooled array creation --- src/SharpCompress/IO/PooledMemoryStream.cs | 26 ++++-- .../Streams/PooledMemoryStreamTests.cs | 90 +++++++++++++++---- 2 files changed, 91 insertions(+), 25 deletions(-) diff --git a/src/SharpCompress/IO/PooledMemoryStream.cs b/src/SharpCompress/IO/PooledMemoryStream.cs index aadc2507..e2e7cf6d 100644 --- a/src/SharpCompress/IO/PooledMemoryStream.cs +++ b/src/SharpCompress/IO/PooledMemoryStream.cs @@ -333,30 +333,38 @@ public sealed class PooledMemoryStream : MemoryStream } } - private byte[] GetExposableContiguousBuffer() + private byte[] CreateExposableBuffer() { - EnsureContiguous(); - - if (_capacity > _length) + var exposable = new byte[_capacity]; + if (_length == 0) { - ClearRange(_length, _capacity - _length); + return exposable; } - _contiguousBufferExposed = true; - return _contiguousBuffer!; + switch (_mode) + { + case StorageMode.Contiguous: + Buffer.BlockCopy(_contiguousBuffer!, 0, exposable, 0, _length); + break; + case StorageMode.Segmented: + CopyFromSegmented(0, exposable, 0, _length); + break; + } + + return exposable; } public override byte[] GetBuffer() { EnsureNotClosed(); - return GetExposableContiguousBuffer(); + return CreateExposableBuffer(); } public override bool TryGetBuffer(out ArraySegment buffer) { EnsureNotClosed(); - var exposableBuffer = GetExposableContiguousBuffer(); + var exposableBuffer = CreateExposableBuffer(); buffer = new ArraySegment(exposableBuffer, 0, _length); return true; } diff --git a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs index 6b12f2a3..e7b770e7 100644 --- a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs +++ b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs @@ -1,7 +1,6 @@ using System; using System.Buffers; using System.IO; -using System.Linq; using SharpCompress.IO; using Xunit; @@ -35,22 +34,28 @@ public class PooledMemoryStreamTests } [Fact] - public void GetBufferPromotesToContiguousAndLaterGrowthUsesBlocks() + public void GetBufferReturnsArraySizedToCapacityWithoutTouchingPool() { - var pool = new TrackingArrayPool(); + var pool = new OverRentingArrayPool(extraLength: 8); using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, arrayPool: pool); - stream.Write(new byte[10], 0, 10); + stream.Write(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 0, 10); - var contiguous = stream.GetBuffer(); - Assert.Equal(16, contiguous.Length); - Assert.Contains(16, pool.RentRequests); + var rentsBefore = pool.RentRequests.Count; + var returnsBefore = pool.ReturnedLengths.Count; - stream.Position = stream.Length; - stream.Write(new byte[10], 0, 10); // grow to 20, forcing demotion + block growth + var buffer = stream.GetBuffer(); + Assert.Equal(16, buffer.Length); + Assert.Equal(1, buffer[0]); + Assert.Equal(10, buffer[9]); + Assert.Equal(0, buffer[10]); + Assert.Equal(0, buffer[15]); + Assert.Equal(rentsBefore, pool.RentRequests.Count); + Assert.Equal(returnsBefore, pool.ReturnedLengths.Count); - var totalBlockRents = pool.RentRequests.Count(size => size == 8); - Assert.True(totalBlockRents >= 3); + buffer[0] = 255; + stream.Position = 0; + Assert.Equal(1, stream.ReadByte()); } [Fact] @@ -65,6 +70,33 @@ public class PooledMemoryStreamTests Assert.Equal(1, segment.Array![0]); } + [Fact] + public void TryGetBufferReturnsArraySizedToCapacityWithoutTouchingPool() + { + var pool = new OverRentingArrayPool(extraLength: 8); + + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, arrayPool: pool); + stream.Write(new byte[] { 1, 2, 3, 4 }, 0, 4); + + var rentsBefore = pool.RentRequests.Count; + var returnsBefore = pool.ReturnedLengths.Count; + + Assert.True(stream.TryGetBuffer(out var segment)); + Assert.Equal(0, segment.Offset); + Assert.Equal(4, segment.Count); + Assert.Equal(8, segment.Array!.Length); + Assert.Equal(1, segment.Array[0]); + Assert.Equal(4, segment.Array[3]); + Assert.Equal(0, segment.Array[4]); + Assert.Equal(0, segment.Array[7]); + Assert.Equal(rentsBefore, pool.RentRequests.Count); + Assert.Equal(returnsBefore, pool.ReturnedLengths.Count); + + segment.Array[0] = 255; + stream.Position = 0; + Assert.Equal(1, stream.ReadByte()); + } + [Fact] public void CapacitySetterCanGrowAndShrinkWithinLength() { @@ -106,7 +138,7 @@ public class PooledMemoryStreamTests } [Fact] - public void MultipleGetBufferCallsReturnSameArray() + public void MultipleGetBufferCallsReturnDifferentArrays() { using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8); stream.Write(new byte[] { 1, 2, 3 }, 0, 3); @@ -114,10 +146,8 @@ public class PooledMemoryStreamTests var buffer1 = stream.GetBuffer(); var buffer2 = stream.GetBuffer(); - Assert.Same(buffer1, buffer2); - Assert.Equal(1, buffer1[0]); - Assert.Equal(2, buffer1[1]); - Assert.Equal(3, buffer1[2]); + Assert.NotSame(buffer1, buffer2); + Assert.Equal(buffer1, buffer2); } [Fact] @@ -158,4 +188,32 @@ public class PooledMemoryStreamTests } } } + + private sealed class OverRentingArrayPool : ArrayPool + { + private readonly int _extraLength; + + public OverRentingArrayPool(int extraLength) + { + _extraLength = extraLength; + } + + public readonly System.Collections.Generic.List RentRequests = new(); + public readonly System.Collections.Generic.List ReturnedLengths = new(); + + public override byte[] Rent(int minimumLength) + { + RentRequests.Add(minimumLength); + return new byte[minimumLength + _extraLength]; + } + + public override void Return(byte[] array, bool clearArray = false) + { + ReturnedLengths.Add(array.Length); + if (clearArray) + { + Array.Clear(array, 0, array.Length); + } + } + } } From d1e6173b506bb9ca4cc03da593133d34ae4cf5fa Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 5 Apr 2026 15:25:33 +0100 Subject: [PATCH 14/20] remove contiguous buffer tracking and added overrenting array pool test --- src/SharpCompress/IO/PooledMemoryStream.cs | 325 ++++-------------- .../Streams/PooledMemoryStreamTests.cs | 83 +++++ 2 files changed, 145 insertions(+), 263 deletions(-) diff --git a/src/SharpCompress/IO/PooledMemoryStream.cs b/src/SharpCompress/IO/PooledMemoryStream.cs index e2e7cf6d..8caadd31 100644 --- a/src/SharpCompress/IO/PooledMemoryStream.cs +++ b/src/SharpCompress/IO/PooledMemoryStream.cs @@ -14,36 +14,21 @@ namespace SharpCompress.IO; /// /// /// This implementation is not thread-safe. Use appropriate synchronization for concurrent access. -/// Buffers exposed via or will not be -/// returned to the pool on dispose to maintain MemoryStream-compatible semantics. -/// The stream dynamically switches between segmented (multiple blocks) and contiguous storage modes -/// based on usage patterns, optimizing for both memory efficiency and performance. +/// Buffers exposed via or are allocated as +/// fresh non-pooled arrays to avoid exposing pooled memory. /// public sealed class PooledMemoryStream : MemoryStream { private const int MaxStreamLength = int.MaxValue; - private enum StorageMode - { - Segmented, - Contiguous, - } - private readonly ArrayPool _arrayPool; private readonly int _blockSize; - private readonly List _detachedExposedBuffers = new(); - - private StorageMode _mode; private List? _blocks; - private byte[]? _contiguousBuffer; - - private bool _contiguousBufferExposed; private bool _isOpen; private int _position; private int _length; private int _capacity; - private int _allocatedCapacity; public PooledMemoryStream() : this(0) { } @@ -63,7 +48,6 @@ public sealed class PooledMemoryStream : MemoryStream _arrayPool = arrayPool; _blockSize = blockSize; - _mode = StorageMode.Segmented; _blocks = new List(); _isOpen = true; _position = 0; @@ -208,15 +192,7 @@ public sealed class PooledMemoryStream : MemoryStream count = available; } - switch (_mode) - { - case StorageMode.Contiguous: - Buffer.BlockCopy(_contiguousBuffer!, _position, buffer, offset, count); - break; - case StorageMode.Segmented: - CopyFromSegmented(_position, buffer, offset, count); - break; - } + CopyFromSegmented(_position, buffer, offset, count); _position += count; return count; @@ -230,20 +206,9 @@ public sealed class PooledMemoryStream : MemoryStream return -1; } - byte value; - switch (_mode) - { - case StorageMode.Contiguous: - value = _contiguousBuffer![_position]; - break; - default: - { - var blockIndex = _position / _blockSize; - var blockOffset = _position % _blockSize; - value = _blocks![blockIndex][blockOffset]; - break; - } - } + var blockIndex = _position / _blockSize; + var blockOffset = _position % _blockSize; + var value = _blocks![blockIndex][blockOffset]; _position++; return value; @@ -275,15 +240,7 @@ public sealed class PooledMemoryStream : MemoryStream ClearRange(_length, _position - _length); } - switch (_mode) - { - case StorageMode.Contiguous: - Buffer.BlockCopy(buffer, offset, _contiguousBuffer!, _position, count); - break; - case StorageMode.Segmented: - CopyToSegmented(_position, buffer, offset, count); - break; - } + CopyToSegmented(_position, buffer, offset, count); _position = endPosition; if (_position > _length) @@ -312,19 +269,9 @@ public sealed class PooledMemoryStream : MemoryStream ClearRange(_length, _position - _length); } - switch (_mode) - { - case StorageMode.Contiguous: - _contiguousBuffer![_position] = value; - break; - default: - { - var blockIndex = _position / _blockSize; - var blockOffset = _position % _blockSize; - _blocks![blockIndex][blockOffset] = value; - break; - } - } + var blockIndex = _position / _blockSize; + var blockOffset = _position % _blockSize; + _blocks![blockIndex][blockOffset] = value; _position = endPosition; if (_position > _length) @@ -341,15 +288,7 @@ public sealed class PooledMemoryStream : MemoryStream return exposable; } - switch (_mode) - { - case StorageMode.Contiguous: - Buffer.BlockCopy(_contiguousBuffer!, 0, exposable, 0, _length); - break; - case StorageMode.Segmented: - CopyFromSegmented(0, exposable, 0, _length); - break; - } + CopyFromSegmented(0, exposable, 0, _length); return exposable; } @@ -380,15 +319,7 @@ public sealed class PooledMemoryStream : MemoryStream } var copy = new byte[count]; - switch (_mode) - { - case StorageMode.Contiguous: - Buffer.BlockCopy(_contiguousBuffer!, 0, copy, 0, count); - break; - case StorageMode.Segmented: - CopyFromSegmented(0, copy, 0, count); - break; - } + CopyFromSegmented(0, copy, 0, count); return copy; } @@ -404,27 +335,16 @@ public sealed class PooledMemoryStream : MemoryStream return; } - switch (_mode) + var position = 0; + var remaining = count; + while (remaining > 0) { - case StorageMode.Contiguous: - stream.Write(_contiguousBuffer!, 0, count); - break; - case StorageMode.Segmented: - { - var position = 0; - var remaining = count; - while (remaining > 0) - { - var blockIndex = position / _blockSize; - var blockOffset = position % _blockSize; - var toWrite = Math.Min(remaining, _blockSize - blockOffset); - stream.Write(_blocks![blockIndex], blockOffset, toWrite); - position += toWrite; - remaining -= toWrite; - } - - break; - } + var blockIndex = position / _blockSize; + var blockOffset = position % _blockSize; + var toWrite = Math.Min(remaining, _blockSize - blockOffset); + stream.Write(_blocks![blockIndex], blockOffset, toWrite); + position += toWrite; + remaining -= toWrite; } } @@ -471,34 +391,23 @@ public sealed class PooledMemoryStream : MemoryStream } var count = Math.Min(available, buffer.Length); - switch (_mode) + var sourcePosition = _position; + var destinationOffset = 0; + var remaining = count; + + while (remaining > 0) { - case StorageMode.Contiguous: - _contiguousBuffer.AsSpan(_position, count).CopyTo(buffer); - break; - case StorageMode.Segmented: - { - var sourcePosition = _position; - var destinationOffset = 0; - var remaining = count; + var blockIndex = sourcePosition / _blockSize; + var blockOffset = sourcePosition % _blockSize; + var toCopy = Math.Min(remaining, _blockSize - blockOffset); + _blocks! + [blockIndex] + .AsSpan(blockOffset, toCopy) + .CopyTo(buffer.Slice(destinationOffset, toCopy)); - while (remaining > 0) - { - var blockIndex = sourcePosition / _blockSize; - var blockOffset = sourcePosition % _blockSize; - var toCopy = Math.Min(remaining, _blockSize - blockOffset); - _blocks! - [blockIndex] - .AsSpan(blockOffset, toCopy) - .CopyTo(buffer.Slice(destinationOffset, toCopy)); - - sourcePosition += toCopy; - destinationOffset += toCopy; - remaining -= toCopy; - } - - break; - } + sourcePosition += toCopy; + destinationOffset += toCopy; + remaining -= toCopy; } _position += count; @@ -529,34 +438,23 @@ public sealed class PooledMemoryStream : MemoryStream ClearRange(_length, _position - _length); } - switch (_mode) + var sourceOffset = 0; + var destinationPosition = _position; + var remaining = buffer.Length; + + while (remaining > 0) { - case StorageMode.Contiguous: - buffer.CopyTo(_contiguousBuffer.AsSpan(_position, buffer.Length)); - break; - case StorageMode.Segmented: - { - var sourceOffset = 0; - var destinationPosition = _position; - var remaining = buffer.Length; + var blockIndex = destinationPosition / _blockSize; + var blockOffset = destinationPosition % _blockSize; + var toCopy = Math.Min(remaining, _blockSize - blockOffset); - while (remaining > 0) - { - var blockIndex = destinationPosition / _blockSize; - var blockOffset = destinationPosition % _blockSize; - var toCopy = Math.Min(remaining, _blockSize - blockOffset); + buffer + .Slice(sourceOffset, toCopy) + .CopyTo(_blocks![blockIndex].AsSpan(blockOffset, toCopy)); - buffer - .Slice(sourceOffset, toCopy) - .CopyTo(_blocks![blockIndex].AsSpan(blockOffset, toCopy)); - - sourceOffset += toCopy; - destinationPosition += toCopy; - remaining -= toCopy; - } - - break; - } + sourceOffset += toCopy; + destinationPosition += toCopy; + remaining -= toCopy; } _position = endPosition; @@ -642,20 +540,7 @@ public sealed class PooledMemoryStream : MemoryStream { ThrowHelper.ThrowIfLessThan(newCapacity, _length, nameof(newCapacity)); - switch (_mode) - { - case StorageMode.Contiguous: - if (newCapacity > _allocatedCapacity) - { - DemoteContiguousToSegmented(); - EnsureSegmentedAllocated(newCapacity); - } - break; - - case StorageMode.Segmented: - EnsureSegmentedAllocated(newCapacity); - break; - } + EnsureSegmentedAllocated(newCapacity); _capacity = newCapacity; if (_length > _capacity) @@ -668,67 +553,8 @@ public sealed class PooledMemoryStream : MemoryStream } } - private void EnsureContiguous() - { - if (_mode == StorageMode.Contiguous) - { - return; - } - - var requested = Math.Max(_capacity, 1); - var contiguous = _arrayPool.Rent(requested); - if (_length > 0) - { - CopyFromSegmented(0, contiguous, 0, _length); - } - - ReturnSegmentedBlocks(); - - _mode = StorageMode.Contiguous; - _contiguousBuffer = contiguous; - _contiguousBufferExposed = false; - _allocatedCapacity = contiguous.Length; - } - - private void DemoteContiguousToSegmented() - { - var contiguous = _contiguousBuffer; - if (contiguous is null) - { - return; - } - - var requiredCapacity = Math.Max(_capacity, _length); - _mode = StorageMode.Segmented; - _blocks = new List(); - _contiguousBuffer = null; - EnsureSegmentedAllocated(requiredCapacity); - - if (_length > 0) - { - CopyToSegmented(0, contiguous, 0, _length); - } - - if (_contiguousBufferExposed) - { - _detachedExposedBuffers.Add(contiguous); - _contiguousBufferExposed = false; - } - else - { - _arrayPool.Return(contiguous); - } - } - private void EnsureSegmentedAllocated(int capacity) { - if (_mode != StorageMode.Segmented) - { - throw new InvalidOperationException( - "Segmented allocation requested while not in segmented mode." - ); - } - var requiredAllocated = RoundUpToBlockBoundary(capacity); var requiredBlocks = requiredAllocated == 0 ? 0 : requiredAllocated / _blockSize; @@ -746,8 +572,6 @@ public sealed class PooledMemoryStream : MemoryStream _blocks.RemoveAt(index); _arrayPool.Return(block); } - - _allocatedCapacity = requiredAllocated; } private int RoundUpToBlockBoundary(int value) @@ -773,27 +597,16 @@ public sealed class PooledMemoryStream : MemoryStream return; } - switch (_mode) + var position = absoluteStart; + var remaining = count; + while (remaining > 0) { - case StorageMode.Contiguous: - Array.Clear(_contiguousBuffer!, absoluteStart, count); - break; - case StorageMode.Segmented: - { - var position = absoluteStart; - var remaining = count; - while (remaining > 0) - { - var blockIndex = position / _blockSize; - var blockOffset = position % _blockSize; - var toClear = Math.Min(remaining, _blockSize - blockOffset); - Array.Clear(_blocks![blockIndex], blockOffset, toClear); - position += toClear; - remaining -= toClear; - } - - break; - } + var blockIndex = position / _blockSize; + var blockOffset = position % _blockSize; + var toClear = Math.Min(remaining, _blockSize - blockOffset); + Array.Clear(_blocks![blockIndex], blockOffset, toClear); + position += toClear; + remaining -= toClear; } } @@ -868,22 +681,8 @@ public sealed class PooledMemoryStream : MemoryStream private void ReturnPooledBuffers() { - if (_mode == StorageMode.Segmented) - { - ReturnSegmentedBlocks(); - _blocks = null; - } - else if (_mode == StorageMode.Contiguous && _contiguousBuffer is not null) - { - _arrayPool.Return(_contiguousBuffer); - _contiguousBuffer = null; - } - - // Buffers tracked here have been exposed to callers. Returning them to the - // shared pool would allow unrelated code to rent and mutate arrays that may - // still be referenced after the stream is disposed, which breaks - // MemoryStream-compatible expectations for GetBuffer/TryGetBuffer. - _detachedExposedBuffers.Clear(); + ReturnSegmentedBlocks(); + _blocks = null; } private static void ValidateReadWriteBufferArguments(byte[] buffer, int offset, int count) diff --git a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs index e7b770e7..9c9571ba 100644 --- a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs +++ b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs @@ -33,6 +33,53 @@ public class PooledMemoryStreamTests Assert.All(pool.ReturnedLengths, length => Assert.Equal(8, length)); } + [Fact] + public void OverRentedBlocksUseLogicalBlockSize() + { + var pool = new FilledOverRentingArrayPool(extraLength: 8, fillValue: 0x5A); + + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, arrayPool: pool); + stream.Write(new byte[] { 1, 2, 3, 4, 5 }, 0, 5); + + stream.Position = 10; + stream.Write(new byte[] { 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }, 0, 10); + + Assert.Equal(3, pool.RentRequests.Count); + Assert.All(pool.RentRequests, requested => Assert.Equal(8, requested)); + Assert.All(pool.RentedLengths, length => Assert.Equal(16, length)); + + var expected = new byte[] + { + 1, + 2, + 3, + 4, + 5, + 0, + 0, + 0, + 0, + 0, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + }; + + Assert.Equal(expected, stream.ToArray()); + + stream.Position = 0; + var roundTrip = new byte[expected.Length]; + Assert.Equal(expected.Length, stream.Read(roundTrip, 0, roundTrip.Length)); + Assert.Equal(expected, roundTrip); + } + [Fact] public void GetBufferReturnsArraySizedToCapacityWithoutTouchingPool() { @@ -216,4 +263,40 @@ public class PooledMemoryStreamTests } } } + + private sealed class FilledOverRentingArrayPool : ArrayPool + { + private readonly int _extraLength; + private readonly byte _fillValue; + + public FilledOverRentingArrayPool(int extraLength, byte fillValue) + { + _extraLength = extraLength; + _fillValue = fillValue; + } + + public readonly System.Collections.Generic.List RentRequests = new(); + public readonly System.Collections.Generic.List RentedLengths = new(); + + public override byte[] Rent(int minimumLength) + { + RentRequests.Add(minimumLength); + + var array = new byte[minimumLength + _extraLength]; + RentedLengths.Add(array.Length); + for (var i = 0; i < array.Length; i++) + { + array[i] = _fillValue; + } + return array; + } + + public override void Return(byte[] array, bool clearArray = false) + { + if (clearArray) + { + Array.Clear(array, 0, array.Length); + } + } + } } From b36689af20e0e0d101dc1b21e9d69e0ccf952c42 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 19 Apr 2026 11:48:54 +0100 Subject: [PATCH 15/20] format --- tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs index 9cb8f2ca..2864209d 100644 --- a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs +++ b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs @@ -259,6 +259,7 @@ public class PooledMemoryStreamTests Assert.Null(exception); Assert.Equal(length, stream.Length); } + private sealed class TrackingArrayPool : ArrayPool { private const byte RentedBufferFillValue = 0x5A; From ad3c5dafd70e8794da3160462931e74662d38ee4 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 19 Apr 2026 12:11:50 +0100 Subject: [PATCH 16/20] update tests --- src/SharpCompress/packages.lock.json | 12 +++++----- .../Streams/PooledMemoryStreamTests.cs | 22 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/SharpCompress/packages.lock.json b/src/SharpCompress/packages.lock.json index a401c702..e82196f8 100644 --- a/src/SharpCompress/packages.lock.json +++ b/src/SharpCompress/packages.lock.json @@ -268,9 +268,9 @@ "net10.0": { "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[10.0.0, )", - "resolved": "10.0.0", - "contentHash": "kICGrGYEzCNI3wPzfEXcwNHgTvlvVn9yJDhSdRK+oZQy4jvYH529u7O0xf5ocQKzOMjfS07+3z9PKRIjrFMJDA==" + "requested": "[10.0.6, )", + "resolved": "10.0.6", + "contentHash": "QKuvS0LWX4fjFqeDkyM7Kqt8P3wYTiPD4nwU+9y59n0sCiG714fxDgbbN82vDnzq89AF/PiHl92TP2C4aFDUQA==" }, "Microsoft.NETFramework.ReferenceAssemblies": { "type": "Direct", @@ -400,9 +400,9 @@ "net8.0": { "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[8.0.22, )", - "resolved": "8.0.22", - "contentHash": "MhcMithKEiyyNkD2ZfbDZPmcOdi0GheGfg8saEIIEfD/fol3iHmcV8TsZkD4ZYz5gdUuoX4YtlVySUU7Sxl9SQ==" + "requested": "[8.0.26, )", + "resolved": "8.0.26", + "contentHash": "o7/yVssM2r9Wyln2s9edBd5ANZXqdSdBI+g7JqXkyJmXrhs2WsJp25K5yPnYrTgdKBCjKB8bg+O2oew4sgzFaA==" }, "Microsoft.NETFramework.ReferenceAssemblies": { "type": "Direct", diff --git a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs index 2864209d..d2b0996c 100644 --- a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs +++ b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs @@ -209,10 +209,10 @@ public class PooledMemoryStreamTests [Fact] public void DisposeAfterGetBufferDoesNotReturnExposedArrayToPool() { - var pool = new TrackingArrayPool(); + var pool = new OverRentingArrayPool(extraLength: 8); byte[] buffer; - using (var stream = new PooledMemoryStream(pool, capacity: 0, blockSize: 8)) + using (var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, pool)) { stream.Write(new byte[] { 1, 2, 3 }, 0, 3); buffer = stream.GetBuffer(); @@ -221,7 +221,7 @@ public class PooledMemoryStreamTests Assert.NotEmpty(pool.RentRequests); } - Assert.Empty(pool.ReturnedLengths); + Assert.DoesNotContain(buffer, pool.ReturnedArrays); Assert.Equal(1, buffer[0]); Assert.Equal(2, buffer[1]); Assert.Equal(3, buffer[2]); @@ -230,10 +230,10 @@ public class PooledMemoryStreamTests [Fact] public void DisposeAfterTryGetBufferDoesNotReturnExposedArrayToPool() { - var pool = new TrackingArrayPool(); + var pool = new OverRentingArrayPool(extraLength: 8); ArraySegment segment; - using (var stream = new PooledMemoryStream(pool, capacity: 0, blockSize: 8)) + using (var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, pool)) { stream.Write(new byte[] { 1, 2, 3 }, 0, 3); @@ -242,22 +242,20 @@ public class PooledMemoryStreamTests Assert.NotEmpty(pool.RentRequests); } - Assert.Empty(pool.ReturnedLengths); + Assert.DoesNotContain(segment.Array!, pool.ReturnedArrays); Assert.Equal(1, segment.Array![segment.Offset]); Assert.Equal(2, segment.Array[segment.Offset + 1]); Assert.Equal(3, segment.Array[segment.Offset + 2]); } [Fact] - public void SetLengthNearIntMaxValueDoesNotThrowIOException() + public void SetLengthNearIntMaxValueThrowsIOExceptionWhenBlockRoundingOverflows() { using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8); var length = int.MaxValue - 1L; - var exception = Record.Exception(() => stream.SetLength(length)); - - Assert.Null(exception); - Assert.Equal(length, stream.Length); + Assert.Throws(() => stream.SetLength(length)); + Assert.Equal(0, stream.Length); } private sealed class TrackingArrayPool : ArrayPool @@ -301,6 +299,7 @@ public class PooledMemoryStreamTests public readonly System.Collections.Generic.List RentRequests = new(); public readonly System.Collections.Generic.List ReturnedLengths = new(); + public readonly System.Collections.Generic.List ReturnedArrays = new(); public override byte[] Rent(int minimumLength) { @@ -311,6 +310,7 @@ public class PooledMemoryStreamTests public override void Return(byte[] array, bool clearArray = false) { ReturnedLengths.Add(array.Length); + ReturnedArrays.Add(array); if (clearArray) { Array.Clear(array, 0, array.Length); From 891d1268fb6bb03ce3f22cc592fd36552baf4e54 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 19 Apr 2026 12:18:47 +0100 Subject: [PATCH 17/20] Update src/SharpCompress/IO/PooledMemoryStream.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/SharpCompress/IO/PooledMemoryStream.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SharpCompress/IO/PooledMemoryStream.cs b/src/SharpCompress/IO/PooledMemoryStream.cs index 8caadd31..3fabeb0b 100644 --- a/src/SharpCompress/IO/PooledMemoryStream.cs +++ b/src/SharpCompress/IO/PooledMemoryStream.cs @@ -81,8 +81,8 @@ public sealed class PooledMemoryStream : MemoryStream } set { - ThrowHelper.ThrowIfNegative(value, nameof(value)); EnsureNotClosed(); + ThrowHelper.ThrowIfNegative(value, nameof(value)); ThrowHelper.ThrowIfGreaterThan(value, MaxStreamLength, nameof(value)); _position = (int)value; From e0ccbbb7c7c108b70099ac85b9a117ad4cfb178c Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 19 Apr 2026 12:19:06 +0100 Subject: [PATCH 18/20] Update src/SharpCompress/IO/PooledMemoryStream.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/SharpCompress/IO/PooledMemoryStream.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/SharpCompress/IO/PooledMemoryStream.cs b/src/SharpCompress/IO/PooledMemoryStream.cs index 3fabeb0b..8111945e 100644 --- a/src/SharpCompress/IO/PooledMemoryStream.cs +++ b/src/SharpCompress/IO/PooledMemoryStream.cs @@ -153,11 +153,10 @@ public sealed class PooledMemoryStream : MemoryStream public override void SetLength(long value) { - ThrowHelper.ThrowIfNegative(value, nameof(value)); - ThrowHelper.ThrowIfGreaterThan(value, MaxStreamLength, nameof(value)); - EnsureWritable(); + ThrowHelper.ThrowIfNegative(value, nameof(value)); + ThrowHelper.ThrowIfGreaterThan(value, MaxStreamLength, nameof(value)); var newLength = (int)value; if (newLength > _capacity) { From 49a7e15a8b3d669dd4f2bb375f35855f5325b4cb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Apr 2026 11:20:33 +0000 Subject: [PATCH 19/20] Add EnsureNotClosed() to Flush and FlushAsync in PooledMemoryStream Agent-Logs-Url: https://github.com/adamhathcock/sharpcompress/sessions/0ce9533f-f529-4c07-b641-6bf0883a0643 Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com> --- src/SharpCompress/IO/PooledMemoryStream.cs | 6 +++++- src/SharpCompress/packages.lock.json | 12 ++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/SharpCompress/IO/PooledMemoryStream.cs b/src/SharpCompress/IO/PooledMemoryStream.cs index 8111945e..3116386f 100644 --- a/src/SharpCompress/IO/PooledMemoryStream.cs +++ b/src/SharpCompress/IO/PooledMemoryStream.cs @@ -112,7 +112,10 @@ public sealed class PooledMemoryStream : MemoryStream } } - public override void Flush() { } + public override void Flush() + { + EnsureNotClosed(); + } public override Task FlushAsync(CancellationToken cancellationToken) { @@ -121,6 +124,7 @@ public sealed class PooledMemoryStream : MemoryStream return Task.FromCanceled(cancellationToken); } + EnsureNotClosed(); return Task.CompletedTask; } diff --git a/src/SharpCompress/packages.lock.json b/src/SharpCompress/packages.lock.json index e82196f8..03c03a9a 100644 --- a/src/SharpCompress/packages.lock.json +++ b/src/SharpCompress/packages.lock.json @@ -268,9 +268,9 @@ "net10.0": { "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[10.0.6, )", - "resolved": "10.0.6", - "contentHash": "QKuvS0LWX4fjFqeDkyM7Kqt8P3wYTiPD4nwU+9y59n0sCiG714fxDgbbN82vDnzq89AF/PiHl92TP2C4aFDUQA==" + "requested": "[10.0.5, )", + "resolved": "10.0.5", + "contentHash": "A+5ZuQ0f449tM+MQrhf6R9ZX7lYpjk/ODEwLYKrnF6111rtARx8fVsm4YznUnQiKnnXfaXNBqgxmil6RW3L3SA==" }, "Microsoft.NETFramework.ReferenceAssemblies": { "type": "Direct", @@ -400,9 +400,9 @@ "net8.0": { "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[8.0.26, )", - "resolved": "8.0.26", - "contentHash": "o7/yVssM2r9Wyln2s9edBd5ANZXqdSdBI+g7JqXkyJmXrhs2WsJp25K5yPnYrTgdKBCjKB8bg+O2oew4sgzFaA==" + "requested": "[8.0.25, )", + "resolved": "8.0.25", + "contentHash": "sqX4nmBft05ivqKvUT4nxaN8rT3apCLt9SWFkfRrQPwra1zPwFknQAw1lleuMCKOCLvVmOWwrC2iPSm9RiXZUg==" }, "Microsoft.NETFramework.ReferenceAssemblies": { "type": "Direct", From f8485b0e8c15a13067f03f672ba5bba80a0b5e7c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Apr 2026 11:33:40 +0000 Subject: [PATCH 20/20] Compute CRC without GetBuffer() in SevenZipWriter to avoid allocation Agent-Logs-Url: https://github.com/adamhathcock/sharpcompress/sessions/77f37cc4-3538-4df1-a816-aa7391065878 Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com> --- .../Writers/SevenZip/SevenZipWriter.cs | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs index 501959ee..7e2a058e 100644 --- a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs +++ b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs @@ -220,12 +220,10 @@ public partial class SevenZipWriter : AbstractWriter encodedHeaderStream.Position = 0; encodedHeaderStream.CopyTo(output); - // Compute CRC of the encoded header - var headerCrc = Crc32Stream.Compute( - Crc32Stream.DEFAULT_POLYNOMIAL, - Crc32Stream.DEFAULT_SEED, - encodedHeaderStream.GetBuffer().AsSpan(0, (int)encodedHeaderStream.Length) - ); + // Compute CRC of the encoded header without allocating a contiguous buffer + var encodedHeaderCrcSink = new Crc32Stream(Stream.Null); + encodedHeaderStream.WriteTo(encodedHeaderCrcSink); + var headerCrc = encodedHeaderCrcSink.Crc; // Back-patch signature header var nextHeaderOffset = (ulong)(headerStartPos - SevenZipSignatureHeaderWriter.HeaderSize); @@ -251,12 +249,10 @@ public partial class SevenZipWriter : AbstractWriter rawHeaderStream.Position = 0; rawHeaderStream.CopyTo(output); - // Compute CRC of the raw header - var headerCrc = Crc32Stream.Compute( - Crc32Stream.DEFAULT_POLYNOMIAL, - Crc32Stream.DEFAULT_SEED, - rawHeaderStream.GetBuffer().AsSpan(0, (int)rawHeaderStream.Length) - ); + // Compute CRC of the raw header without allocating a contiguous buffer + var rawHeaderCrcSink = new Crc32Stream(Stream.Null); + rawHeaderStream.WriteTo(rawHeaderCrcSink); + var headerCrc = rawHeaderCrcSink.Crc; // Back-patch signature header var nextHeaderOffset = (ulong)(headerStartPos - SevenZipSignatureHeaderWriter.HeaderSize);