diff --git a/src/SharpCompress/Common/Rar/CryptKey5.cs b/src/SharpCompress/Common/Rar/CryptKey5.cs index 66f08aa3..490a3c54 100644 --- a/src/SharpCompress/Common/Rar/CryptKey5.cs +++ b/src/SharpCompress/Common/Rar/CryptKey5.cs @@ -35,7 +35,7 @@ internal class CryptKey5 : ICryptKey ) { var passwordBytes = Encoding.UTF8.GetBytes(password); -#if LEGACY_DOTNET || NET5_0 +#if LEGACY_DOTNET using var hmac = new HMACSHA256(passwordBytes); var block = hmac.ComputeHash(salt); #else @@ -50,7 +50,7 @@ internal class CryptKey5 : ICryptKey { for (var i = 1; i < loop[x]; i++) { -#if LEGACY_DOTNET || NET5_0 +#if LEGACY_DOTNET block = hmac.ComputeHash(block); #else block = HMACSHA256.HashData(passwordBytes, block); diff --git a/src/SharpCompress/Common/SevenZip/ArchiveReader.cs b/src/SharpCompress/Common/SevenZip/ArchiveReader.cs index 3a4e5622..b75d123c 100644 --- a/src/SharpCompress/Common/SevenZip/ArchiveReader.cs +++ b/src/SharpCompress/Common/SevenZip/ArchiveReader.cs @@ -1182,7 +1182,7 @@ internal partial class ArchiveReader } else { - _stream = new MemoryStream(); + _stream = new PooledMemoryStream(); } _rem = _db._files[index].Size; } diff --git a/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs b/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs index f5fd410e..9162474b 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs @@ -2,6 +2,7 @@ using System; using System.IO; using System.Text; using SharpCompress.Compressors.LZMA.Utilities; +using SharpCompress.IO; namespace SharpCompress.Common.SevenZip; @@ -215,7 +216,7 @@ internal sealed class SevenZipFilesInfoWriter Action writeData ) { - using var dataStream = new MemoryStream(); + using var dataStream = new PooledMemoryStream(); writeData(dataStream); stream.WriteByte((byte)propertyId); diff --git a/src/SharpCompress/Common/Tar/Headers/TarHeader.Async.cs b/src/SharpCompress/Common/Tar/Headers/TarHeader.Async.cs index 7ca51bc1..812a75eb 100644 --- a/src/SharpCompress/Common/Tar/Headers/TarHeader.Async.cs +++ b/src/SharpCompress/Common/Tar/Headers/TarHeader.Async.cs @@ -59,7 +59,7 @@ internal sealed partial class TarHeader int splitIndex = -1; for (int i = 0; i < dirSeps.Count; i++) { -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER int count = ArchiveEncoding .GetEncoding() .GetByteCount(fullName.AsSpan(0, dirSeps[i])); diff --git a/src/SharpCompress/Common/Tar/Headers/TarHeader.cs b/src/SharpCompress/Common/Tar/Headers/TarHeader.cs index 0e27f2e9..897e5871 100644 --- a/src/SharpCompress/Common/Tar/Headers/TarHeader.cs +++ b/src/SharpCompress/Common/Tar/Headers/TarHeader.cs @@ -102,7 +102,7 @@ internal sealed partial class TarHeader int splitIndex = -1; for (int i = 0; i < dirSeps.Count; i++) { -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER int count = ArchiveEncoding .GetEncoding() .GetByteCount(fullName.AsSpan(0, dirSeps[i])); diff --git a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs index 1642be44..84bd3f00 100644 --- a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs +++ b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using SharpCompress.IO; namespace SharpCompress.Compressors.LZMA; @@ -158,7 +159,7 @@ internal sealed class Lzma2EncoderStream : Stream } using var inputMs = new MemoryStream(data.ToArray(), writable: false); - using var outputMs = new MemoryStream(); + using var outputMs = new PooledMemoryStream(); encoder.Code(inputMs, outputMs, data.Length, -1, null); @@ -190,7 +191,7 @@ internal sealed class Lzma2EncoderStream : Stream decoder.SetDecoderProperties(props); using var input = new MemoryStream(compressedData); - using var output = new MemoryStream(); + using var output = new PooledMemoryStream(); decoder.Code(input, output, compressedData.Length, uncompressedSize, null); return (int)input.Position; diff --git a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs index 902246d7..6f1e7fcb 100644 --- a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs +++ b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using SharpCompress.Compressors.LZMA.RangeCoder; using SharpCompress.Compressors.PPMd.H; using SharpCompress.Compressors.PPMd.I1; +using SharpCompress.IO; namespace SharpCompress.Compressors.PPMd; @@ -179,7 +180,7 @@ public class PpmdStream : Stream { if (_compress) { - _model.EncodeBlock(_stream, new MemoryStream(), true); + _model.EncodeBlock(_stream, Stream.Null, true); } } base.Dispose(disposing); diff --git a/src/SharpCompress/Compressors/Squeezed/SqueezedStream.Async.cs b/src/SharpCompress/Compressors/Squeezed/SqueezedStream.Async.cs index d8090dca..6a0cfcda 100644 --- a/src/SharpCompress/Compressors/Squeezed/SqueezedStream.Async.cs +++ b/src/SharpCompress/Compressors/Squeezed/SqueezedStream.Async.cs @@ -4,6 +4,7 @@ using System.Threading; using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Compressors.RLE90; +using SharpCompress.IO; namespace SharpCompress.Compressors.Squeezed; @@ -54,14 +55,14 @@ public partial class SqueezeStream if (bytesRead != 2) { - return new MemoryStream(Array.Empty()); + return new PooledMemoryStream(); } int numnodes = numNodesBytes[0] | (numNodesBytes[1] << 8); if (numnodes >= NUMVALS || numnodes == 0) { - return new MemoryStream(Array.Empty()); + return new PooledMemoryStream(); } var dnode = new int[numnodes, 2]; @@ -82,7 +83,7 @@ public partial class SqueezeStream } var bitReader = new BitReader(_stream); - var huffmanDecoded = new MemoryStream(); + var huffmanDecoded = new PooledMemoryStream(); int i = 0; while (true) diff --git a/src/SharpCompress/Compressors/Squeezed/SqueezedStream.cs b/src/SharpCompress/Compressors/Squeezed/SqueezedStream.cs index 09aa8b2c..f6a40f1c 100644 --- a/src/SharpCompress/Compressors/Squeezed/SqueezedStream.cs +++ b/src/SharpCompress/Compressors/Squeezed/SqueezedStream.cs @@ -4,6 +4,7 @@ using System.IO; using System.Text; using SharpCompress.Common; using SharpCompress.Compressors.RLE90; +using SharpCompress.IO; namespace SharpCompress.Compressors.Squeezed; @@ -67,7 +68,7 @@ public partial class SqueezeStream : Stream if (numnodes >= NUMVALS || numnodes == 0) { - return new MemoryStream(Array.Empty()); + return new PooledMemoryStream(); } var dnode = new int[numnodes, 2]; @@ -78,7 +79,7 @@ public partial class SqueezeStream : Stream } var bitReader = new BitReader(_stream); - var huffmanDecoded = new MemoryStream(); + var huffmanDecoded = new PooledMemoryStream(); int i = 0; while (true) diff --git a/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdInternal.cs b/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdInternal.cs index c8b9f15d..4180d6b1 100644 --- a/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdInternal.cs +++ b/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdInternal.cs @@ -5,7 +5,7 @@ using static SharpCompress.Compressors.ZStandard.UnsafeHelper; #if NETCOREAPP3_0_OR_GREATER using System.Runtime.Intrinsics.X86; #endif -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER using System.Runtime.Intrinsics.Arm; #endif @@ -554,7 +554,7 @@ public static unsafe partial class Methods [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void ZSTD_copy16(void* dst, void* src) { -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER if (AdvSimd.IsSupported) { AdvSimd.Store((byte*)dst, AdvSimd.LoadVector128((byte*)src)); diff --git a/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdLazy.cs b/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdLazy.cs index 178160b1..fdf26282 100644 --- a/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdLazy.cs +++ b/src/SharpCompress/Compressors/ZStandard/Unsafe/ZstdLazy.cs @@ -6,7 +6,7 @@ using static SharpCompress.Compressors.ZStandard.UnsafeHelper; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; #endif -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER using System.Runtime.Intrinsics.Arm; #endif @@ -1172,7 +1172,7 @@ public static unsafe partial class Methods { assert(rowEntries == 16 || rowEntries == 32 || rowEntries == 64); assert(rowEntries <= 64); -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER if (AdvSimd.IsSupported && BitConverter.IsLittleEndian) { if (rowEntries == 16) @@ -1272,7 +1272,7 @@ public static unsafe partial class Methods } #endif -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER if (AdvSimd.IsSupported && BitConverter.IsLittleEndian) { if (rowEntries == 16) diff --git a/src/SharpCompress/IO/PooledMemoryStream.cs b/src/SharpCompress/IO/PooledMemoryStream.cs new file mode 100644 index 00000000..3116386f --- /dev/null +++ b/src/SharpCompress/IO/PooledMemoryStream.cs @@ -0,0 +1,701 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Common; + +namespace SharpCompress.IO; + +/// +/// MemoryStream implementation backed by pooled byte arrays. +/// Uses to reduce GC pressure for temporary buffers. +/// +/// +/// This implementation is not thread-safe. Use appropriate synchronization for concurrent access. +/// Buffers exposed via or are allocated as +/// fresh non-pooled arrays to avoid exposing pooled memory. +/// +public sealed class PooledMemoryStream : MemoryStream +{ + private const int MaxStreamLength = int.MaxValue; + + private readonly ArrayPool _arrayPool; + private readonly int _blockSize; + + private List? _blocks; + private bool _isOpen; + private int _position; + private int _length; + private int _capacity; + + public PooledMemoryStream() + : this(0) { } + + public PooledMemoryStream(int capacity) + : this(capacity, Constants.BufferSize, ArrayPool.Shared) { } + + public PooledMemoryStream(int capacity, int blockSize) + : this(capacity, blockSize, ArrayPool.Shared) { } + + public PooledMemoryStream(int capacity, int blockSize, ArrayPool arrayPool) + { + ThrowHelper.ThrowIfNull(arrayPool, nameof(arrayPool)); + ThrowHelper.ThrowIfNegative(capacity, nameof(capacity)); + ThrowHelper.ThrowIfNegativeOrZero(blockSize, nameof(blockSize)); + + _arrayPool = arrayPool; + _blockSize = blockSize; + + _blocks = new List(); + _isOpen = true; + _position = 0; + _length = 0; + _capacity = capacity; + + EnsureSegmentedAllocated(capacity); + } + + public override bool CanRead => _isOpen; + + public override bool CanSeek => _isOpen; + + public override bool CanWrite => _isOpen; + + public override long Length + { + get + { + EnsureNotClosed(); + return _length; + } + } + + public override long Position + { + get + { + EnsureNotClosed(); + return _position; + } + set + { + EnsureNotClosed(); + ThrowHelper.ThrowIfNegative(value, nameof(value)); + ThrowHelper.ThrowIfGreaterThan(value, MaxStreamLength, nameof(value)); + + _position = (int)value; + } + } + + public override int Capacity + { + get + { + EnsureNotClosed(); + return _capacity; + } + set + { + ThrowHelper.ThrowIfLessThan(value, _length, nameof(value)); + + EnsureNotClosed(); + + var target = value; + if (target == _capacity) + { + return; + } + + SetCapacityAbsolute(target); + } + } + + public override void Flush() + { + EnsureNotClosed(); + } + + public override Task FlushAsync(CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + { + return Task.FromCanceled(cancellationToken); + } + + EnsureNotClosed(); + return Task.CompletedTask; + } + + public override long Seek(long offset, SeekOrigin loc) + { + EnsureNotClosed(); + + var anchor = loc switch + { + SeekOrigin.Begin => 0, + SeekOrigin.Current => _position, + SeekOrigin.End => _length, + _ => throw new ArgumentException("Invalid seek origin.", nameof(loc)), + }; + + var target = anchor + offset; + if (target < 0) + { + throw new IOException("Attempted to seek before the beginning of the stream."); + } + + if (target > MaxStreamLength) + { + throw new ArgumentOutOfRangeException(nameof(offset)); + } + + _position = (int)target; + return _position; + } + + public override void SetLength(long value) + { + EnsureWritable(); + + ThrowHelper.ThrowIfNegative(value, nameof(value)); + ThrowHelper.ThrowIfGreaterThan(value, MaxStreamLength, nameof(value)); + var newLength = (int)value; + if (newLength > _capacity) + { + EnsureCapacityForAppend(newLength); + } + + if (newLength > _length) + { + ClearRange(_length, newLength - _length); + } + + _length = newLength; + if (_position > newLength) + { + _position = newLength; + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + ValidateReadWriteBufferArguments(buffer, offset, count); + EnsureNotClosed(); + + var available = _length - _position; + if (available <= 0) + { + return 0; + } + + if (count > available) + { + count = available; + } + + CopyFromSegmented(_position, buffer, offset, count); + + _position += count; + return count; + } + + public override int ReadByte() + { + EnsureNotClosed(); + if (_position >= _length) + { + return -1; + } + + var blockIndex = _position / _blockSize; + var blockOffset = _position % _blockSize; + var value = _blocks![blockIndex][blockOffset]; + + _position++; + return value; + } + + public override void Write(byte[] buffer, int offset, int count) + { + ValidateReadWriteBufferArguments(buffer, offset, count); + EnsureWritable(); + + if (count == 0) + { + return; + } + + var endPosition = _position + count; + if (endPosition < 0) + { + throw new IOException("Stream is too long."); + } + + if (endPosition > _capacity) + { + EnsureCapacityForAppend(endPosition); + } + + if (_position > _length) + { + ClearRange(_length, _position - _length); + } + + CopyToSegmented(_position, buffer, offset, count); + + _position = endPosition; + if (_position > _length) + { + _length = _position; + } + } + + public override void WriteByte(byte value) + { + EnsureWritable(); + + var endPosition = _position + 1; + if (endPosition < 0) + { + throw new IOException("Stream is too long."); + } + + if (endPosition > _capacity) + { + EnsureCapacityForAppend(endPosition); + } + + if (_position > _length) + { + ClearRange(_length, _position - _length); + } + + var blockIndex = _position / _blockSize; + var blockOffset = _position % _blockSize; + _blocks![blockIndex][blockOffset] = value; + + _position = endPosition; + if (_position > _length) + { + _length = _position; + } + } + + private byte[] CreateExposableBuffer() + { + var exposable = new byte[_capacity]; + if (_length == 0) + { + return exposable; + } + + CopyFromSegmented(0, exposable, 0, _length); + + return exposable; + } + + public override byte[] GetBuffer() + { + EnsureNotClosed(); + return CreateExposableBuffer(); + } + + public override bool TryGetBuffer(out ArraySegment buffer) + { + EnsureNotClosed(); + + var exposableBuffer = CreateExposableBuffer(); + buffer = new ArraySegment(exposableBuffer, 0, _length); + return true; + } + + public override byte[] ToArray() + { + EnsureNotClosed(); + + var count = _length; + if (count == 0) + { + return Array.Empty(); + } + + var copy = new byte[count]; + CopyFromSegmented(0, copy, 0, count); + + return copy; + } + + public override void WriteTo(Stream stream) + { + ThrowHelper.ThrowIfNull(stream, nameof(stream)); + EnsureNotClosed(); + + var count = _length; + if (count == 0) + { + return; + } + + var position = 0; + var remaining = count; + while (remaining > 0) + { + var blockIndex = position / _blockSize; + var blockOffset = position % _blockSize; + var toWrite = Math.Min(remaining, _blockSize - blockOffset); + stream.Write(_blocks![blockIndex], blockOffset, toWrite); + position += toWrite; + remaining -= toWrite; + } + } + + public override Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + if (cancellationToken.IsCancellationRequested) + { + return Task.FromCanceled(cancellationToken); + } + + return Task.FromResult(Read(buffer, offset, count)); + } + + public override Task WriteAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + if (cancellationToken.IsCancellationRequested) + { + return Task.FromCanceled(cancellationToken); + } + + Write(buffer, offset, count); + return Task.CompletedTask; + } + +#if !LEGACY_DOTNET + public override int Read(Span buffer) + { + EnsureNotClosed(); + + var available = _length - _position; + if (available <= 0) + { + return 0; + } + + var count = Math.Min(available, buffer.Length); + var sourcePosition = _position; + var destinationOffset = 0; + var remaining = count; + + while (remaining > 0) + { + var blockIndex = sourcePosition / _blockSize; + var blockOffset = sourcePosition % _blockSize; + var toCopy = Math.Min(remaining, _blockSize - blockOffset); + _blocks! + [blockIndex] + .AsSpan(blockOffset, toCopy) + .CopyTo(buffer.Slice(destinationOffset, toCopy)); + + sourcePosition += toCopy; + destinationOffset += toCopy; + remaining -= toCopy; + } + + _position += count; + return count; + } + + public override void Write(ReadOnlySpan buffer) + { + EnsureWritable(); + if (buffer.Length == 0) + { + return; + } + + var endPosition = _position + buffer.Length; + if (endPosition < 0) + { + throw new IOException("Stream is too long."); + } + + if (endPosition > _capacity) + { + EnsureCapacityForAppend(endPosition); + } + + if (_position > _length) + { + ClearRange(_length, _position - _length); + } + + var sourceOffset = 0; + var destinationPosition = _position; + var remaining = buffer.Length; + + while (remaining > 0) + { + var blockIndex = destinationPosition / _blockSize; + var blockOffset = destinationPosition % _blockSize; + var toCopy = Math.Min(remaining, _blockSize - blockOffset); + + buffer + .Slice(sourceOffset, toCopy) + .CopyTo(_blocks![blockIndex].AsSpan(blockOffset, toCopy)); + + sourceOffset += toCopy; + destinationPosition += toCopy; + remaining -= toCopy; + } + + _position = endPosition; + if (_position > _length) + { + _length = _position; + } + } + + public override ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + if (cancellationToken.IsCancellationRequested) + { + return ValueTask.FromCanceled(cancellationToken); + } + + return ValueTask.FromResult(Read(buffer.Span)); + } + + public override ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default + ) + { + if (cancellationToken.IsCancellationRequested) + { + return ValueTask.FromCanceled(cancellationToken); + } + + Write(buffer.Span); + return ValueTask.CompletedTask; + } +#endif + + protected override void Dispose(bool disposing) + { + if (_isOpen) + { + _isOpen = false; + + if (disposing) + { + ReturnPooledBuffers(); + } + } + + base.Dispose(disposing); + } + + private void EnsureNotClosed() + { + if (!_isOpen) + { + throw new ObjectDisposedException(nameof(PooledMemoryStream)); + } + } + + private void EnsureWritable() + { + EnsureNotClosed(); + } + + private void EnsureCapacityForAppend(int requiredLength) + { + if (requiredLength < 0) + { + throw new IOException("Stream is too long."); + } + + if (requiredLength <= _capacity) + { + return; + } + + var nextCapacity = RoundUpToBlockBoundary(requiredLength); + SetCapacityAbsolute(nextCapacity); + } + + private void SetCapacityAbsolute(int newCapacity) + { + ThrowHelper.ThrowIfLessThan(newCapacity, _length, nameof(newCapacity)); + + EnsureSegmentedAllocated(newCapacity); + + _capacity = newCapacity; + if (_length > _capacity) + { + _length = _capacity; + } + if (_position > _capacity) + { + _position = _capacity; + } + } + + private void EnsureSegmentedAllocated(int capacity) + { + var requiredAllocated = RoundUpToBlockBoundary(capacity); + var requiredBlocks = requiredAllocated == 0 ? 0 : requiredAllocated / _blockSize; + + _blocks ??= new List(); + + while (_blocks.Count < requiredBlocks) + { + _blocks.Add(_arrayPool.Rent(_blockSize)); + } + + while (_blocks.Count > requiredBlocks) + { + var index = _blocks.Count - 1; + var block = _blocks[index]; + _blocks.RemoveAt(index); + _arrayPool.Return(block); + } + } + + private int RoundUpToBlockBoundary(int value) + { + if (value <= 0) + { + return 0; + } + + var rounded = ((long)value + _blockSize - 1) / _blockSize * _blockSize; + if (rounded > MaxStreamLength) + { + throw new IOException("Stream is too long."); + } + + return (int)rounded; + } + + private void ClearRange(int absoluteStart, int count) + { + if (count <= 0) + { + return; + } + + var position = absoluteStart; + var remaining = count; + while (remaining > 0) + { + var blockIndex = position / _blockSize; + var blockOffset = position % _blockSize; + var toClear = Math.Min(remaining, _blockSize - blockOffset); + Array.Clear(_blocks![blockIndex], blockOffset, toClear); + position += toClear; + remaining -= toClear; + } + } + + private void CopyFromSegmented( + int absoluteSourcePosition, + byte[] destination, + int offset, + int count + ) + { + var sourcePosition = absoluteSourcePosition; + var destinationOffset = offset; + var remaining = count; + + while (remaining > 0) + { + var blockIndex = sourcePosition / _blockSize; + var blockOffset = sourcePosition % _blockSize; + var toCopy = Math.Min(remaining, _blockSize - blockOffset); + Buffer.BlockCopy( + _blocks![blockIndex], + blockOffset, + destination, + destinationOffset, + toCopy + ); + + sourcePosition += toCopy; + destinationOffset += toCopy; + remaining -= toCopy; + } + } + + private void CopyToSegmented( + int absoluteDestinationPosition, + byte[] source, + int offset, + int count + ) + { + var sourceOffset = offset; + var destinationPosition = absoluteDestinationPosition; + var remaining = count; + + while (remaining > 0) + { + var blockIndex = destinationPosition / _blockSize; + var blockOffset = destinationPosition % _blockSize; + var toCopy = Math.Min(remaining, _blockSize - blockOffset); + Buffer.BlockCopy(source, sourceOffset, _blocks![blockIndex], blockOffset, toCopy); + + sourceOffset += toCopy; + destinationPosition += toCopy; + remaining -= toCopy; + } + } + + private void ReturnSegmentedBlocks() + { + if (_blocks is null) + { + return; + } + + for (var i = 0; i < _blocks.Count; i++) + { + _arrayPool.Return(_blocks[i]); + } + + _blocks.Clear(); + } + + private void ReturnPooledBuffers() + { + ReturnSegmentedBlocks(); + _blocks = null; + } + + private static void ValidateReadWriteBufferArguments(byte[] buffer, int offset, int count) + { + ThrowHelper.ThrowIfNull(buffer, nameof(buffer)); + ThrowHelper.ThrowIfNegative(offset, nameof(offset)); + ThrowHelper.ThrowIfNegative(count, nameof(count)); + if (buffer.Length - offset < count) + { + throw new ArgumentException("Offset and length are out of bounds."); + } + } +} diff --git a/src/SharpCompress/Polyfills/StreamExtensions.cs b/src/SharpCompress/Polyfills/StreamExtensions.cs index 18c5b593..e3caa24f 100644 --- a/src/SharpCompress/Polyfills/StreamExtensions.cs +++ b/src/SharpCompress/Polyfills/StreamExtensions.cs @@ -28,7 +28,7 @@ public static class StreamExtensions public Task SkipAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); -#if NET5_0_OR_GREATER +#if NET6_0_OR_GREATER return stream.CopyToAsync(Stream.Null, cancellationToken); #else return stream.CopyToAsync(Stream.Null); diff --git a/src/SharpCompress/SharpCompress.csproj b/src/SharpCompress/SharpCompress.csproj index 0a9c440c..bcde6224 100644 --- a/src/SharpCompress/SharpCompress.csproj +++ b/src/SharpCompress/SharpCompress.csproj @@ -6,7 +6,7 @@ 0.0.0.0 0.0.0.0 Adam Hathcock - net48;netstandard2.0;netstandard2.1;net5.0;net6.0;net7.0;net8.0;net9.0;net10.0 + net48;netstandard2.0;netstandard2.1;net6.0;net7.0;net8.0;net9.0;net10.0 SharpCompress ../../SharpCompress.snk true diff --git a/src/SharpCompress/ThrowHelper.cs b/src/SharpCompress/ThrowHelper.cs index 9d598107..c202d87c 100644 --- a/src/SharpCompress/ThrowHelper.cs +++ b/src/SharpCompress/ThrowHelper.cs @@ -60,6 +60,15 @@ internal static class ThrowHelper } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void ThrowIfGreaterThan(long value, long other, string? paramName = null) + { + if (value > other) + { + throw new ArgumentOutOfRangeException(paramName); + } + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void ThrowIfGreaterThan(uint value, uint other, string? paramName = null) { diff --git a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs index a785aaf2..7e2a058e 100644 --- a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs +++ b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.cs @@ -171,7 +171,7 @@ public partial class SevenZipWriter : AbstractWriter var filesInfo = new SevenZipFilesInfoWriter { Entries = entries.ToArray() }; // Write header to a temporary stream first - using var headerStream = new MemoryStream(); + using var headerStream = new PooledMemoryStream(); ArchiveHeaderWriter.WriteRawHeader(headerStream, mainStreamsInfo, filesInfo); // Optionally compress the header @@ -212,7 +212,7 @@ public partial class SevenZipWriter : AbstractWriter }; // Write encoded header to a second temporary stream - using var encodedHeaderStream = new MemoryStream(); + using var encodedHeaderStream = new PooledMemoryStream(); ArchiveHeaderWriter.WriteEncodedHeader(encodedHeaderStream, headerStreamsInfo); // Write the encoded header to the output @@ -220,12 +220,10 @@ public partial class SevenZipWriter : AbstractWriter encodedHeaderStream.Position = 0; encodedHeaderStream.CopyTo(output); - // Compute CRC of the encoded header - var headerCrc = Crc32Stream.Compute( - Crc32Stream.DEFAULT_POLYNOMIAL, - Crc32Stream.DEFAULT_SEED, - encodedHeaderStream.GetBuffer().AsSpan(0, (int)encodedHeaderStream.Length) - ); + // Compute CRC of the encoded header without allocating a contiguous buffer + var encodedHeaderCrcSink = new Crc32Stream(Stream.Null); + encodedHeaderStream.WriteTo(encodedHeaderCrcSink); + var headerCrc = encodedHeaderCrcSink.Crc; // Back-patch signature header var nextHeaderOffset = (ulong)(headerStartPos - SevenZipSignatureHeaderWriter.HeaderSize); @@ -251,12 +249,10 @@ public partial class SevenZipWriter : AbstractWriter rawHeaderStream.Position = 0; rawHeaderStream.CopyTo(output); - // Compute CRC of the raw header - var headerCrc = Crc32Stream.Compute( - Crc32Stream.DEFAULT_POLYNOMIAL, - Crc32Stream.DEFAULT_SEED, - rawHeaderStream.GetBuffer().AsSpan(0, (int)rawHeaderStream.Length) - ); + // Compute CRC of the raw header without allocating a contiguous buffer + var rawHeaderCrcSink = new Crc32Stream(Stream.Null); + rawHeaderStream.WriteTo(rawHeaderCrcSink); + var headerCrc = rawHeaderCrcSink.Crc; // Back-patch signature header var nextHeaderOffset = (ulong)(headerStartPos - SevenZipSignatureHeaderWriter.HeaderSize); diff --git a/src/SharpCompress/packages.lock.json b/src/SharpCompress/packages.lock.json index a3e65b16..e82196f8 100644 --- a/src/SharpCompress/packages.lock.json +++ b/src/SharpCompress/packages.lock.json @@ -313,48 +313,6 @@ "contentHash": "Mk1IMb9q5tahC2NltxYXFkLBtuBvfBoCQ3pIxYQWfzbCE9o1OB9SsHe0hnNGo7lWgTA/ePbFAJLWu6nLL9K17A==" } }, - ".NETCoreApp,Version=v5.0": { - "Microsoft.NETFramework.ReferenceAssemblies": { - "type": "Direct", - "requested": "[1.0.3, )", - "resolved": "1.0.3", - "contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg==", - "dependencies": { - "Microsoft.NETFramework.ReferenceAssemblies.net461": "1.0.3" - } - }, - "Microsoft.SourceLink.GitHub": { - "type": "Direct", - "requested": "[10.0.102, )", - "resolved": "10.0.102", - "contentHash": "Oxq3RCIJSdtpIU4hLqO7XaDe/Ra3HS9Wi8rJl838SAg6Zu1iQjerA0+xXWBgUFYbgknUGCLOU0T+lzMLkvY9Qg==", - "dependencies": { - "Microsoft.Build.Tasks.Git": "10.0.102", - "Microsoft.SourceLink.Common": "10.0.102" - } - }, - "Microsoft.VisualStudio.Threading.Analyzers": { - "type": "Direct", - "requested": "[17.14.15, )", - "resolved": "17.14.15", - "contentHash": "mXQPJsbuUD2ydq4/ffd8h8tSOFCXec+2xJOVNCvXjuMOq/+5EKHq3D2m2MC2+nUaXeFMSt66VS/J4HdKBixgcw==" - }, - "Microsoft.Build.Tasks.Git": { - "type": "Transitive", - "resolved": "10.0.102", - "contentHash": "0i81LYX31U6UiXz4NOLbvc++u+/mVDmOt+PskrM/MygpDxkv9THKQyRUmavBpLK6iBV0abNWnn+CQgSRz//Pwg==" - }, - "Microsoft.NETFramework.ReferenceAssemblies.net461": { - "type": "Transitive", - "resolved": "1.0.3", - "contentHash": "AmOJZwCqnOCNp6PPcf9joyogScWLtwy0M1WkqfEQ0M9nYwyDD7EX9ZjscKS5iYnyvteX7kzSKFCKt9I9dXA6mA==" - }, - "Microsoft.SourceLink.Common": { - "type": "Transitive", - "resolved": "10.0.102", - "contentHash": "Mk1IMb9q5tahC2NltxYXFkLBtuBvfBoCQ3pIxYQWfzbCE9o1OB9SsHe0hnNGo7lWgTA/ePbFAJLWu6nLL9K17A==" - } - }, "net6.0": { "Microsoft.NETFramework.ReferenceAssemblies": { "type": "Direct", diff --git a/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs new file mode 100644 index 00000000..d2b0996c --- /dev/null +++ b/tests/SharpCompress.Test/Streams/PooledMemoryStreamTests.cs @@ -0,0 +1,356 @@ +using System; +using System.Buffers; +using System.IO; +using SharpCompress.IO; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class PooledMemoryStreamTests +{ + [Fact] + public void GrowsUsingFixedSizeBlocks() + { + var pool = new TrackingArrayPool(); + + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, arrayPool: pool); + stream.Write(new byte[20], 0, 20); + + Assert.Equal(3, pool.RentRequests.Count); + Assert.All(pool.RentRequests, requested => Assert.Equal(8, requested)); + } + + [Fact] + public void DisposeReturnsRentedBlocksToPool() + { + var pool = new TrackingArrayPool(); + var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, arrayPool: pool); + + stream.Write(new byte[17], 0, 17); + stream.Dispose(); + + Assert.Equal(pool.RentRequests.Count, pool.ReturnedLengths.Count); + Assert.All(pool.ReturnedLengths, length => Assert.Equal(8, length)); + } + + [Fact] + public void OverRentedBlocksUseLogicalBlockSize() + { + var pool = new FilledOverRentingArrayPool(extraLength: 8, fillValue: 0x5A); + + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, arrayPool: pool); + stream.Write(new byte[] { 1, 2, 3, 4, 5 }, 0, 5); + + stream.Position = 10; + stream.Write(new byte[] { 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }, 0, 10); + + Assert.Equal(3, pool.RentRequests.Count); + Assert.All(pool.RentRequests, requested => Assert.Equal(8, requested)); + Assert.All(pool.RentedLengths, length => Assert.Equal(16, length)); + + var expected = new byte[] + { + 1, + 2, + 3, + 4, + 5, + 0, + 0, + 0, + 0, + 0, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + }; + + Assert.Equal(expected, stream.ToArray()); + + stream.Position = 0; + var roundTrip = new byte[expected.Length]; + Assert.Equal(expected.Length, stream.Read(roundTrip, 0, roundTrip.Length)); + Assert.Equal(expected, roundTrip); + } + + [Fact] + public void GetBufferReturnsArraySizedToCapacityWithoutTouchingPool() + { + var pool = new OverRentingArrayPool(extraLength: 8); + + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, arrayPool: pool); + stream.Write(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 0, 10); + + var rentsBefore = pool.RentRequests.Count; + var returnsBefore = pool.ReturnedLengths.Count; + + var buffer = stream.GetBuffer(); + Assert.Equal(16, buffer.Length); + Assert.Equal(1, buffer[0]); + Assert.Equal(10, buffer[9]); + Assert.Equal(0, buffer[10]); + Assert.Equal(0, buffer[15]); + Assert.Equal(rentsBefore, pool.RentRequests.Count); + Assert.Equal(returnsBefore, pool.ReturnedLengths.Count); + + buffer[0] = 255; + stream.Position = 0; + Assert.Equal(1, stream.ReadByte()); + } + + [Fact] + public void TryGetBufferReturnsSegmentWhenOpen() + { + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8); + stream.Write(new byte[] { 1, 2, 3, 4 }, 0, 4); + + Assert.True(stream.TryGetBuffer(out var segment)); + Assert.Equal(0, segment.Offset); + Assert.Equal(4, segment.Count); + Assert.Equal(1, segment.Array![0]); + } + + [Fact] + public void TryGetBufferReturnsArraySizedToCapacityWithoutTouchingPool() + { + var pool = new OverRentingArrayPool(extraLength: 8); + + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, arrayPool: pool); + stream.Write(new byte[] { 1, 2, 3, 4 }, 0, 4); + + var rentsBefore = pool.RentRequests.Count; + var returnsBefore = pool.ReturnedLengths.Count; + + Assert.True(stream.TryGetBuffer(out var segment)); + Assert.Equal(0, segment.Offset); + Assert.Equal(4, segment.Count); + Assert.Equal(8, segment.Array!.Length); + Assert.Equal(1, segment.Array[0]); + Assert.Equal(4, segment.Array[3]); + Assert.Equal(0, segment.Array[4]); + Assert.Equal(0, segment.Array[7]); + Assert.Equal(rentsBefore, pool.RentRequests.Count); + Assert.Equal(returnsBefore, pool.ReturnedLengths.Count); + + segment.Array[0] = 255; + stream.Position = 0; + Assert.Equal(1, stream.ReadByte()); + } + + [Fact] + public void CapacitySetterCanGrowAndShrinkWithinLength() + { + using var stream = new PooledMemoryStream(capacity: 16, blockSize: 8); + stream.Write(new byte[6], 0, 6); + + stream.Capacity = 24; + Assert.Equal(24, stream.Capacity); + + stream.Capacity = 8; + Assert.Equal(8, stream.Capacity); + } + + [Fact] + public void SetLengthExtendingClearsGap() + { + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8); + stream.Position = 5; + stream.WriteByte(42); + stream.Position = 0; + + var data = stream.ToArray(); + Assert.Equal(6, data.Length); + Assert.Equal(0, data[0]); + Assert.Equal(0, data[4]); + Assert.Equal(42, data[5]); + } + + [Fact] + public void MethodsThrowAfterDispose() + { + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8); + stream.WriteByte(1); + stream.Dispose(); + + Assert.Throws(() => stream.ReadByte()); + Assert.Throws(() => stream.ToArray()); + Assert.Throws(() => stream.GetBuffer()); + } + + [Fact] + public void MultipleGetBufferCallsReturnDifferentArrays() + { + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8); + stream.Write(new byte[] { 1, 2, 3 }, 0, 3); + + var buffer1 = stream.GetBuffer(); + var buffer2 = stream.GetBuffer(); + + Assert.NotSame(buffer1, buffer2); + Assert.Equal(buffer1, buffer2); + } + + [Fact] + public void SeekBeyondMaxLengthThrows() + { + using var stream = new PooledMemoryStream(); + Assert.Throws(() => + stream.Seek(int.MaxValue + 1L, SeekOrigin.Begin) + ); + } + + [Fact] + public void DisposeAfterGetBufferDoesNotReturnExposedArrayToPool() + { + var pool = new OverRentingArrayPool(extraLength: 8); + byte[] buffer; + + using (var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, pool)) + { + stream.Write(new byte[] { 1, 2, 3 }, 0, 3); + buffer = stream.GetBuffer(); + + Assert.NotNull(buffer); + Assert.NotEmpty(pool.RentRequests); + } + + Assert.DoesNotContain(buffer, pool.ReturnedArrays); + Assert.Equal(1, buffer[0]); + Assert.Equal(2, buffer[1]); + Assert.Equal(3, buffer[2]); + } + + [Fact] + public void DisposeAfterTryGetBufferDoesNotReturnExposedArrayToPool() + { + var pool = new OverRentingArrayPool(extraLength: 8); + ArraySegment segment; + + using (var stream = new PooledMemoryStream(capacity: 0, blockSize: 8, pool)) + { + stream.Write(new byte[] { 1, 2, 3 }, 0, 3); + + Assert.True(stream.TryGetBuffer(out segment)); + Assert.NotNull(segment.Array); + Assert.NotEmpty(pool.RentRequests); + } + + Assert.DoesNotContain(segment.Array!, pool.ReturnedArrays); + Assert.Equal(1, segment.Array![segment.Offset]); + Assert.Equal(2, segment.Array[segment.Offset + 1]); + Assert.Equal(3, segment.Array[segment.Offset + 2]); + } + + [Fact] + public void SetLengthNearIntMaxValueThrowsIOExceptionWhenBlockRoundingOverflows() + { + using var stream = new PooledMemoryStream(capacity: 0, blockSize: 8); + var length = int.MaxValue - 1L; + + Assert.Throws(() => stream.SetLength(length)); + Assert.Equal(0, stream.Length); + } + + private sealed class TrackingArrayPool : ArrayPool + { + private const byte RentedBufferFillValue = 0x5A; + + public readonly System.Collections.Generic.List RentRequests = new(); + public readonly System.Collections.Generic.List ReturnedLengths = new(); + + public override byte[] Rent(int minimumLength) + { + RentRequests.Add(minimumLength); + + var array = new byte[minimumLength]; + for (var i = 0; i < array.Length; i++) + { + array[i] = RentedBufferFillValue; + } + + return array; + } + + public override void Return(byte[] array, bool clearArray = false) + { + ReturnedLengths.Add(array.Length); + if (clearArray) + { + Array.Clear(array, 0, array.Length); + } + } + } + + private sealed class OverRentingArrayPool : ArrayPool + { + private readonly int _extraLength; + + public OverRentingArrayPool(int extraLength) + { + _extraLength = extraLength; + } + + public readonly System.Collections.Generic.List RentRequests = new(); + public readonly System.Collections.Generic.List ReturnedLengths = new(); + public readonly System.Collections.Generic.List ReturnedArrays = new(); + + public override byte[] Rent(int minimumLength) + { + RentRequests.Add(minimumLength); + return new byte[minimumLength + _extraLength]; + } + + public override void Return(byte[] array, bool clearArray = false) + { + ReturnedLengths.Add(array.Length); + ReturnedArrays.Add(array); + if (clearArray) + { + Array.Clear(array, 0, array.Length); + } + } + } + + private sealed class FilledOverRentingArrayPool : ArrayPool + { + private readonly int _extraLength; + private readonly byte _fillValue; + + public FilledOverRentingArrayPool(int extraLength, byte fillValue) + { + _extraLength = extraLength; + _fillValue = fillValue; + } + + public readonly System.Collections.Generic.List RentRequests = new(); + public readonly System.Collections.Generic.List RentedLengths = new(); + + public override byte[] Rent(int minimumLength) + { + RentRequests.Add(minimumLength); + + var array = new byte[minimumLength + _extraLength]; + RentedLengths.Add(array.Length); + for (var i = 0; i < array.Length; i++) + { + array[i] = _fillValue; + } + return array; + } + + public override void Return(byte[] array, bool clearArray = false) + { + if (clearArray) + { + Array.Clear(array, 0, array.Length); + } + } + } +}