lancommander-ref/SharpCompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #887 from majorro/improve-rar-memory-usage

Browse source 
Improve rar memory usage
This commit is contained in:
Adam Hathcock 2025-01-14 08:58:18 +00:00 • committed by GitHub
commit fa1d440947
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 129 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Directory.Packages.props
View file

@ -7,6 +7,7 @@
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
<PackageVersion Include="Mono.Posix.NETStandard" Version="1.0.0" />
<PackageVersion Include="SimpleExec" Version="12.0.0" />
<PackageVersion Include="System.Buffers" Version="4.6.0" />
<PackageVersion Include="System.Memory" Version="4.5.5" />
<PackageVersion Include="System.Text.Encoding.CodePages" Version="8.0.0" />
<PackageVersion Include="xunit" Version="2.9.0" />
@ -15,4 +16,4 @@
<PackageVersion Include="ZstdSharp.Port" Version="0.8.1" />
<GlobalPackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
</ItemGroup>
</Project>
</Project>

15
src/SharpCompress/Archives/Rar/RarArchive.cs
View file

@ -14,6 +14,7 @@ namespace SharpCompress.Archives.Rar;
public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
{
private bool _disposed;
internal Lazy<IRarUnpack> UnpackV2017 { get; } =
new(() => new Compressors.Rar.UnpackV2017.Unpack());
internal Lazy<IRarUnpack> UnpackV1 { get; } = new(() => new Compressors.Rar.UnpackV1.Unpack());
@ -25,6 +26,20 @@ public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
private RarArchive(SourceStream sourceStream)
: base(ArchiveType.Rar, sourceStream) { }
public override void Dispose()
{
if (!_disposed)
{
if (UnpackV1.IsValueCreated && UnpackV1.Value is IDisposable unpackV1)
{
unpackV1.Dispose();
}
_disposed = true;
base.Dispose();
}
}
protected override IEnumerable<RarArchiveEntry> LoadEntries(IEnumerable<RarVolume> volumes) =>
RarArchiveEntryFactory.GetEntries(this, volumes, ReaderOptions);

33
src/SharpCompress/BufferPool.cs
View file

@ -1,33 +0,0 @@
using System.Buffers;
namespace SharpCompress.Helpers;
internal static class BufferPool
{
/// <summary>
/// gets a buffer from the pool
/// </summary>
/// <param name="bufferSize">size of the buffer</param>
/// <returns>the buffer</returns>
public static byte[] Rent(int bufferSize)
{
#if NETCOREAPP || NETSTANDARD2_1_OR_GREATER
return ArrayPool<byte>.Shared.Rent(bufferSize);
#else
return new byte[bufferSize];
#endif
}
/// <summary>
/// returns a buffer to the pool
/// </summary>
/// <param name="buffer">the buffer to return</param>
public static void Return(byte[] buffer)
{
#if NETCOREAPP || NETSTANDARD2_1_OR_GREATER
ArrayPool<byte>.Shared.Return(buffer);
#else
// no-op
#endif
}
}

2
src/SharpCompress/Compressors/Rar/RarCRC.cs
View file

@ -9,7 +9,7 @@ internal static class RarCRC
public static uint CheckCrc(uint startCrc, byte b) =>
(crcTab[((int)startCrc ^ b) & 0xff] ^ (startCrc >> 8));
public static uint CheckCrc(uint startCrc, byte[] data, int offset, int count)
public static uint CheckCrc(uint startCrc, ReadOnlySpan<byte> data, int offset, int count)
{
var size = Math.Min(data.Length - offset, count);

9
src/SharpCompress/Compressors/Rar/RarStream.cs
View file

@ -1,6 +1,7 @@
#nullable disable
using System;
using System.Buffers;
using System.IO;
using SharpCompress.Common.Rar.Headers;
@ -14,7 +15,7 @@ internal class RarStream : Stream
private bool fetch;
private byte[] tmpBuffer = BufferPool.Rent(65536);
private byte[] tmpBuffer = ArrayPool<byte>.Shared.Rent(65536);
private int tmpOffset;
private int tmpCount;
@ -42,7 +43,7 @@ internal class RarStream : Stream
{
if (disposing)
{
BufferPool.Return(this.tmpBuffer);
ArrayPool<byte>.Shared.Return(this.tmpBuffer);
this.tmpBuffer = null;
}
isDisposed = true;
@ -143,11 +144,11 @@ internal class RarStream : Stream
this.tmpBuffer.Length * 2 > this.tmpCount + count
? this.tmpBuffer.Length * 2
: this.tmpCount + count;
var newBuffer = BufferPool.Rent(newLength);
var newBuffer = ArrayPool<byte>.Shared.Rent(newLength);
Buffer.BlockCopy(this.tmpBuffer, 0, newBuffer, 0, this.tmpCount);
var oldBuffer = this.tmpBuffer;
this.tmpBuffer = newBuffer;
BufferPool.Return(oldBuffer);
ArrayPool<byte>.Shared.Return(oldBuffer);
}
}
}

59
src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs
View file

@ -1,6 +1,7 @@
#nullable disable
using System;
using System.Buffers;
using System.Collections.Generic;
using System.IO;
using SharpCompress.Common;
@ -12,14 +13,28 @@ using SharpCompress.Compressors.Rar.VM;
namespace SharpCompress.Compressors.Rar.UnpackV1;
internal sealed partial class Unpack : BitInput, IRarUnpack
internal sealed partial class Unpack : BitInput, IRarUnpack, IDisposable
{
private readonly BitInput Inp;
private bool disposed;
public Unpack() =>
// to ease in porting Unpack50.cs
Inp = this;
public void Dispose()
{
if (!disposed)
{
if (!externalWindow)
{
ArrayPool<byte>.Shared.Return(window);
window = null;
}
disposed = true;
}
}
public bool FileExtracted { get; private set; }
public long DestSize
@ -74,7 +89,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack
private BlockTypes unpBlockType;
//private bool externalWindow;
private bool externalWindow;
private long writtenFileSize;
@ -111,15 +126,14 @@ internal sealed partial class Unpack : BitInput, IRarUnpack
private void Init(byte[] window)
{
if (window is null)
if (this.window is null && window is null)
{
this.window = new byte[PackDef.MAXWINSIZE];
this.window = ArrayPool<byte>.Shared.Rent(PackDef.MAXWINSIZE);
}
else
else if (window is not null)
{
this.window = window;
//externalWindow = true;
externalWindow = true;
}
inAddr = 0;
UnpInitData(false);
@ -175,31 +189,24 @@ internal sealed partial class Unpack : BitInput, IRarUnpack
private void UnstoreFile()
{
var buffer = new byte[0x10000];
while (true)
Span<byte> buffer = stackalloc byte[(int)Math.Min(0x10000, destUnpSize)];
do
{
var code = readStream.Read(buffer, 0, (int)Math.Min(buffer.Length, destUnpSize));
var code = readStream.Read(buffer);
if (code == 0 || code == -1)
{
break;
}
code = code < destUnpSize ? code : (int)destUnpSize;
writeStream.Write(buffer, 0, code);
if (destUnpSize >= 0)
{
destUnpSize -= code;
}
if (suspended)
{
return;
}
}
writeStream.Write(buffer.Slice(0, code));
destUnpSize -= code;
} while (!suspended && destUnpSize > 0);
}
private void Unpack29(bool solid)
{
var DDecode = new int[PackDef.DC];
var DBits = new byte[PackDef.DC];
Span<int> DDecode = stackalloc int[PackDef.DC];
Span<byte> DBits = stackalloc byte[PackDef.DC];
int Bits;
@ -859,9 +866,9 @@ internal sealed partial class Unpack : BitInput, IRarUnpack
private bool ReadTables()
{
var bitLength = new byte[PackDef.BC];
Span<byte> bitLength = stackalloc byte[PackDef.BC];
Span<byte> table = stackalloc byte[PackDef.HUFF_TABLE_SIZE];
var table = new byte[PackDef.HUFF_TABLE_SIZE];
if (inAddr > readTop - 25)
{
if (!unpReadBuf())
@ -989,7 +996,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack
// memcpy(unpOldTable,table,sizeof(unpOldTable));
Buffer.BlockCopy(table, 0, unpOldTable, 0, unpOldTable.Length);
table.CopyTo(unpOldTable);
return (true);
}
@ -1190,7 +1197,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack
{
return (false);
}
var VMCode = new byte[VMCodeSize];
Span<byte> VMCode = stackalloc byte[VMCodeSize];
for (var I = 0; I < VMCodeSize; I++)
{
if (Inp.Overflow(3))

4
src/SharpCompress/Compressors/Rar/UnpackV1/Unpack20.cs
View file

@ -391,8 +391,8 @@ internal partial class Unpack
private bool ReadTables20()
{
var BitLength = new byte[PackDef.BC20];
var Table = new byte[PackDef.MC20 * 4];
Span<byte> BitLength = stackalloc byte[PackDef.BC20];
Span<byte> Table = stackalloc byte[PackDef.MC20 * 4];
int TableSize,
N,
I;

7
src/SharpCompress/Compressors/Rar/UnpackV1/UnpackUtility.cs
View file

@ -181,7 +181,12 @@ internal static class UnpackUtility
return (dec.DecodeNum[N]);
}
internal static void makeDecodeTables(byte[] lenTab, int offset, Decode.Decode dec, int size)
internal static void makeDecodeTables(
Span<byte> lenTab,
int offset,
Decode.Decode dec,
int size
)
{
Span<int> lenCount = stackalloc int[16];
Span<int> tmpPos = stackalloc int[16];

6
src/SharpCompress/Compressors/Rar/VM/RarVM.cs
View file

@ -776,14 +776,14 @@ internal sealed class RarVM : BitInput
}
}
public void prepare(byte[] code, int codeSize, VMPreparedProgram prg)
public void prepare(ReadOnlySpan<byte> code, int codeSize, VMPreparedProgram prg)
{
InitBitInput();
var cpLength = Math.Min(MAX_SIZE, codeSize);
// memcpy(inBuf,Code,Min(CodeSize,BitInput::MAX_SIZE));
Buffer.BlockCopy(code, 0, InBuf, 0, cpLength);
code.Slice(0, cpLength).CopyTo(InBuf);
byte xorSum = 0;
for (var i = 1; i < codeSize; i++)
{
@ -1105,7 +1105,7 @@ internal sealed class RarVM : BitInput
}
}
private VMStandardFilters IsStandardFilter(byte[] code, int codeSize)
private VMStandardFilters IsStandardFilter(ReadOnlySpan<byte> code, int codeSize)
{
VMStandardFilterSignature[] stdList =
{

2
src/SharpCompress/Readers/AbstractReader.cs
View file

@ -44,7 +44,7 @@ public abstract class AbstractReader<TEntry, TVolume> : IReader, IReaderExtracti
#region IDisposable Members
public void Dispose()
public virtual void Dispose()
{
_entriesForCurrentReadStream?.Dispose();
Volume?.Dispose();

15
src/SharpCompress/Readers/Rar/RarReader.cs
View file

@ -13,6 +13,7 @@ namespace SharpCompress.Readers.Rar;
/// </summary>
public abstract class RarReader : AbstractReader<RarReaderEntry, RarVolume>
{
private bool _disposed;
private RarVolume? volume;
private Lazy<IRarUnpack> UnpackV2017 { get; } =
new(() => new Compressors.Rar.UnpackV2017.Unpack());
@ -21,6 +22,20 @@ public abstract class RarReader : AbstractReader<RarReaderEntry, RarVolume>
internal RarReader(ReaderOptions options)
: base(options, ArchiveType.Rar) { }
public override void Dispose()
{
if (!_disposed)
{
if (UnpackV1.IsValueCreated && UnpackV1.Value is IDisposable unpackV1)
{
unpackV1.Dispose();
}
_disposed = true;
base.Dispose();
}
}
protected abstract void ValidateArchive(RarVolume archive);
public override RarVolume? Volume => volume;

1
src/SharpCompress/SharpCompress.csproj
View file

@ -33,6 +33,7 @@
<IsTrimmable>true</IsTrimmable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.Buffers" />
<PackageReference Include="ZstdSharp.Port" />
</ItemGroup>
<ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.1' ">

47
src/SharpCompress/packages.lock.json
View file

@ -30,6 +30,12 @@
"Microsoft.SourceLink.Common": "8.0.0"
}
},
"System.Buffers": {
"type": "Direct",
"requested": "[4.6.0, )",
"resolved": "4.6.0",
"contentHash": "lN6tZi7Q46zFzAbRYXTIvfXcyvQQgxnY7Xm6C6xQ9784dEL1amjM6S6Iw4ZpsvesAKnRVsM4scrDQaDqSClkjA=="
},
"System.Memory": {
"type": "Direct",
"requested": "[4.5.5, )",
@ -76,11 +82,6 @@
"resolved": "8.0.0",
"contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw=="
},
"System.Buffers": {
"type": "Transitive",
"resolved": "4.5.1",
"contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg=="
},
"System.Numerics.Vectors": {
"type": "Transitive",
"resolved": "4.5.0",
@ -129,6 +130,12 @@
"Microsoft.NETCore.Platforms": "1.1.0"
}
},
"System.Buffers": {
"type": "Direct",
"requested": "[4.6.0, )",
"resolved": "4.6.0",
"contentHash": "lN6tZi7Q46zFzAbRYXTIvfXcyvQQgxnY7Xm6C6xQ9784dEL1amjM6S6Iw4ZpsvesAKnRVsM4scrDQaDqSClkjA=="
},
"System.Memory": {
"type": "Direct",
"requested": "[4.5.5, )",
@ -175,11 +182,6 @@
"resolved": "8.0.0",
"contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw=="
},
"System.Buffers": {
"type": "Transitive",
"resolved": "4.5.1",
"contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg=="
},
"System.Numerics.Vectors": {
"type": "Transitive",
"resolved": "4.4.0",
@ -216,6 +218,12 @@
"Microsoft.SourceLink.Common": "8.0.0"
}
},
"System.Buffers": {
"type": "Direct",
"requested": "[4.6.0, )",
"resolved": "4.6.0",
"contentHash": "lN6tZi7Q46zFzAbRYXTIvfXcyvQQgxnY7Xm6C6xQ9784dEL1amjM6S6Iw4ZpsvesAKnRVsM4scrDQaDqSClkjA=="
},
"System.Text.Encoding.CodePages": {
"type": "Direct",
"requested": "[8.0.0, )",
@ -245,11 +253,6 @@
"resolved": "8.0.0",
"contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw=="
},
"System.Buffers": {
"type": "Transitive",
"resolved": "4.5.1",
"contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg=="
},
"System.Numerics.Vectors": {
"type": "Transitive",
"resolved": "4.4.0",
@ -283,6 +286,12 @@
"Microsoft.SourceLink.Common": "8.0.0"
}
},
"System.Buffers": {
"type": "Direct",
"requested": "[4.6.0, )",
"resolved": "4.6.0",
"contentHash": "lN6tZi7Q46zFzAbRYXTIvfXcyvQQgxnY7Xm6C6xQ9784dEL1amjM6S6Iw4ZpsvesAKnRVsM4scrDQaDqSClkjA=="
},
"ZstdSharp.Port": {
"type": "Direct",
"requested": "[0.8.1, )",
@ -317,6 +326,12 @@
"Microsoft.SourceLink.Common": "8.0.0"
}
},
"System.Buffers": {
"type": "Direct",
"requested": "[4.6.0, )",
"resolved": "4.6.0",
"contentHash": "lN6tZi7Q46zFzAbRYXTIvfXcyvQQgxnY7Xm6C6xQ9784dEL1amjM6S6Iw4ZpsvesAKnRVsM4scrDQaDqSClkjA=="
},
"ZstdSharp.Port": {
"type": "Direct",
"requested": "[0.8.1, )",
@ -335,4 +350,4 @@
}
}
}
}
}

19
tests/SharpCompress.Test/packages.lock.json
View file

@ -97,11 +97,6 @@
"System.Reflection.Metadata": "1.6.0"
}
},
"System.Buffers": {
"type": "Transitive",
"resolved": "4.5.1",
"contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg=="
},
"System.Collections.Immutable": {
"type": "Transitive",
"resolved": "1.5.0",
@ -182,6 +177,7 @@
"type": "Project",
"dependencies": {
"Microsoft.Bcl.AsyncInterfaces": "[8.0.0, )",
"System.Buffers": "[4.6.0, )",
"System.Memory": "[4.5.5, )",
"System.Text.Encoding.CodePages": "[8.0.0, )",
"ZstdSharp.Port": "[0.8.1, )"
@ -196,6 +192,12 @@
"System.Threading.Tasks.Extensions": "4.5.4"
}
},
"System.Buffers": {
"type": "CentralTransitive",
"requested": "[4.6.0, )",
"resolved": "4.6.0",
"contentHash": "lN6tZi7Q46zFzAbRYXTIvfXcyvQQgxnY7Xm6C6xQ9784dEL1amjM6S6Iw4ZpsvesAKnRVsM4scrDQaDqSClkjA=="
},
"System.Memory": {
"type": "CentralTransitive",
"requested": "[4.5.5, )",
@ -394,9 +396,16 @@
"sharpcompress": {
"type": "Project",
"dependencies": {
"System.Buffers": "[4.6.0, )",
"ZstdSharp.Port": "[0.8.1, )"
}
},
"System.Buffers": {
"type": "CentralTransitive",
"requested": "[4.6.0, )",
"resolved": "4.6.0",
"contentHash": "lN6tZi7Q46zFzAbRYXTIvfXcyvQQgxnY7Xm6C6xQ9784dEL1amjM6S6Iw4ZpsvesAKnRVsM4scrDQaDqSClkjA=="
},
"ZstdSharp.Port": {
"type": "CentralTransitive",
"requested": "[0.8.1, )",