Merge pull request #658 from Nanook/master

Added Split archive support with unit tests. …

src/SharpCompress/Archives/ArchiveFactory.cs

@@ -1,11 +1,14 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using SharpCompress.Archives.GZip;
 using SharpCompress.Archives.Rar;
 using SharpCompress.Archives.SevenZip;
 using SharpCompress.Archives.Tar;
 using SharpCompress.Archives.Zip;
 using SharpCompress.Common;
+using SharpCompress.IO;
 using SharpCompress.Readers;
 
 namespace SharpCompress.Archives
@@ -26,34 +29,25 @@ namespace SharpCompress.Archives
                 throw new ArgumentException("Stream should be readable and seekable");
             }
             readerOptions ??= new ReaderOptions();
-            if (ZipArchive.IsZipFile(stream, null))
+
+            ArchiveType? type;
+            IsArchive(stream, out type); //test and reset stream position
+
+            if (type != null)
             {
-                stream.Seek(0, SeekOrigin.Begin);
-                return ZipArchive.Open(stream, readerOptions);
-            }
-            stream.Seek(0, SeekOrigin.Begin);
-            if (SevenZipArchive.IsSevenZipFile(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return SevenZipArchive.Open(stream, readerOptions);
-            }
-            stream.Seek(0, SeekOrigin.Begin);
-            if (GZipArchive.IsGZipFile(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return GZipArchive.Open(stream, readerOptions);
-            }
-            stream.Seek(0, SeekOrigin.Begin);
-            if (RarArchive.IsRarFile(stream, readerOptions))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return RarArchive.Open(stream, readerOptions);
-            }
-            stream.Seek(0, SeekOrigin.Begin);
-            if (TarArchive.IsTarFile(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return TarArchive.Open(stream, readerOptions);
+                switch (type.Value)
+                {
+                    case ArchiveType.Zip:
+                        return ZipArchive.Open(stream, readerOptions);
+                    case ArchiveType.SevenZip:
+                        return SevenZipArchive.Open(stream, readerOptions);
+                    case ArchiveType.GZip:
+                        return GZipArchive.Open(stream, readerOptions);
+                    case ArchiveType.Rar:
+                        return RarArchive.Open(stream, readerOptions);
+                    case ArchiveType.Tar:
+                        return TarArchive.Open(stream, readerOptions);
+                }
             }
             throw new InvalidOperationException("Cannot determine compressed stream type. Supported Archive Formats: Zip, GZip, Tar, Rar, 7Zip, LZip");
         }
@@ -90,30 +84,78 @@ namespace SharpCompress.Archives
             fileInfo.CheckNotNull(nameof(fileInfo));
             options ??= new ReaderOptions { LeaveStreamOpen = false };
 
-            using var stream = fileInfo.OpenRead();
-            if (ZipArchive.IsZipFile(stream, null))
+            ArchiveType? type;
+            using (Stream stream = fileInfo.OpenRead())
             {
-                return ZipArchive.Open(fileInfo, options);
+                IsArchive(stream, out type); //test and reset stream position
+
+                if (type != null)
+                {
+                    switch (type.Value)
+                    {
+                        case ArchiveType.Zip:
+                            return ZipArchive.Open(fileInfo, options);
+                        case ArchiveType.SevenZip:
+                            return SevenZipArchive.Open(fileInfo, options);
+                        case ArchiveType.GZip:
+                            return GZipArchive.Open(fileInfo, options);
+                        case ArchiveType.Rar:
+                            return RarArchive.Open(fileInfo, options);
+                        case ArchiveType.Tar:
+                            return TarArchive.Open(fileInfo, options);
+                    }
+                }
             }
-            stream.Seek(0, SeekOrigin.Begin);
-            if (SevenZipArchive.IsSevenZipFile(stream))
+            throw new InvalidOperationException("Cannot determine compressed stream type. Supported Archive Formats: Zip, GZip, Tar, Rar, 7Zip");
+        }
+
+        /// <summary>
+        /// Constructor with IEnumerable FileInfo objects, multi and split support.
+        /// </summary>
+        /// <param name="fileInfos"></param>
+        /// <param name="options"></param>
+        public static IArchive Open(IEnumerable<FileInfo> fileInfos, ReaderOptions? options = null)
+        {
+            fileInfos.CheckNotNull(nameof(fileInfos));
+            FileInfo[] files = fileInfos.ToArray();
+            if (files.Length == 0)
+                throw new InvalidOperationException("No files to open");
+            FileInfo fileInfo = files[0];
+            if (files.Length == 1)
+                return Open(fileInfo, options);
+
+
+            fileInfo.CheckNotNull(nameof(fileInfo));
+            options ??= new ReaderOptions { LeaveStreamOpen = false };
+
+            ArchiveType? type;
+            using (Stream stream = fileInfo.OpenRead())
+                IsArchive(stream, out type); //test and reset stream position
+
+            if (type != null)
             {
-                return SevenZipArchive.Open(fileInfo, options);
-            }
-            stream.Seek(0, SeekOrigin.Begin);
-            if (GZipArchive.IsGZipFile(stream))
-            {
-                return GZipArchive.Open(fileInfo, options);
-            }
-            stream.Seek(0, SeekOrigin.Begin);
-            if (RarArchive.IsRarFile(stream, options))
-            {
-                return RarArchive.Open(fileInfo, options);
-            }
-            stream.Seek(0, SeekOrigin.Begin);
-            if (TarArchive.IsTarFile(stream))
-            {
-                return TarArchive.Open(fileInfo, options);
+                switch (type.Value)
+                {
+                    case ArchiveType.Zip:
+                        return ZipArchive.Open(new SplitStream(files), options);
+                    case ArchiveType.SevenZip:
+                        return SevenZipArchive.Open(new SplitStream(files), options);
+                    case ArchiveType.GZip:
+                        return GZipArchive.Open(new SplitStream(files), options);
+                    case ArchiveType.Rar:
+                        using (Stream prt2 = files[1].OpenRead())
+                        {
+                            try
+                            {
+                                if (RarArchive.IsRarFile(prt2, options)) //if part2 is a rar then it's multi not split
+                                    return RarArchive.Open(files.Select(f => File.OpenRead(f.FullName))); //multipart read
+                            }
+                            catch { }
+                        }
+                        return RarArchive.Open(new SplitStream(files), options);
+                    case ArchiveType.Tar:
+                        return TarArchive.Open(new SplitStream(files), options);
+                }
             }
             throw new InvalidOperationException("Cannot determine compressed stream type. Supported Archive Formats: Zip, GZip, Tar, Rar, 7Zip");
         }
@@ -130,5 +172,52 @@ namespace SharpCompress.Archives
                 entry.WriteToDirectory(destinationDirectory, options);
             }
         }
+
+        public static bool IsArchive(string filePath, out ArchiveType? type)
+        {
+            filePath.CheckNotNullOrEmpty(nameof(filePath));
+            using (Stream s = File.OpenRead(filePath))
+                return IsArchive(s, out type);
+        }
+
+        private static bool IsArchive(Stream stream, out ArchiveType? type)
+        {
+            type = null;
+            stream.CheckNotNull(nameof(stream));
+
+            if (!stream.CanRead || !stream.CanSeek)
+            {
+                throw new ArgumentException("Stream should be readable and seekable");
+            }
+            if (ZipArchive.IsZipFile(stream, null))
+                type = ArchiveType.Zip;
+            stream.Seek(0, SeekOrigin.Begin);
+            if (type == null)
+            {
+                if (SevenZipArchive.IsSevenZipFile(stream))
+                    type = ArchiveType.SevenZip;
+                stream.Seek(0, SeekOrigin.Begin);
+            }
+            if (type == null)
+            {
+                if (GZipArchive.IsGZipFile(stream))
+                    type = ArchiveType.GZip;
+                stream.Seek(0, SeekOrigin.Begin);
+            }
+            if (type == null)
+            {
+                if (RarArchive.IsRarFile(stream))
+                    type = ArchiveType.Rar;
+                stream.Seek(0, SeekOrigin.Begin);
+            }
+            if (type == null)
+            {
+                if (TarArchive.IsTarFile(stream))
+                    type = ArchiveType.Tar;
+                stream.Seek(0, SeekOrigin.Begin);
+            }
+
+            return type != null;
+        }
     }
 }

src/SharpCompress/IO/SplitStream.cs

@@ -0,0 +1,150 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace SharpCompress.IO
+{
+    public class SplitStream : Stream
+    {
+        private int _idx;
+        private long _size;
+        private bool _isSplit;
+        private long _prevSize;
+        private long _pos;
+        private FileInfo[] _files;
+        private Stream _stream;
+
+
+        public SplitStream(IEnumerable<FileInfo> files)
+        {
+            _files = files.ToArray();
+            _isSplit = _files.Length > 1;
+            _size = _files.Sum(a => a.Length);
+            _idx = 0;
+            _prevSize = 0;
+            _stream = openStream(0);
+        }
+
+        private Stream openStream(int idx)
+        {
+            if (_stream != null)
+                _stream.Dispose();
+
+            _stream = File.OpenRead(_files[idx].FullName);
+            _idx = idx;
+            _pos = 0;
+            return _stream;
+        }
+
+        public override bool CanRead => true;
+
+        public override bool CanSeek => true;
+
+        public override bool CanWrite => false;
+
+        public override long Length => _size;
+
+        public override long Position
+        {
+            get => _prevSize + _pos;
+            set => Seek(value, SeekOrigin.Begin);
+        }
+
+        public override void Flush()
+        {
+            _stream.Flush();
+        }
+
+        public override int Read(byte[] buffer, int offset, int count)
+        {
+            count = (int)Math.Min(count, _size - this.Position);
+
+            if (count <= 0)
+                return 0;
+
+
+            int total = count;
+            int r = -1;
+
+            while (count != 0 && r != 0)
+            {
+                r = _stream.Read(buffer, offset, count);
+                _pos += (long)r;
+                count -= r;
+                offset += r;
+
+                if (_isSplit && _pos == _files[_idx].Length)
+                    Seek(0, SeekOrigin.Current); //will load next file
+            }
+
+            return total - count;
+        }
+
+        public override long Seek(long offset, SeekOrigin origin)
+        {
+            long pos = this.Position;
+            switch (origin)
+            {
+                case SeekOrigin.Begin: pos = offset; break;
+                case SeekOrigin.Current: pos += offset; break;
+                case SeekOrigin.End: pos = Length + offset; break;
+            }
+
+            if (_isSplit)
+            {
+                _prevSize = 0;
+                for (int i = 0; i < _files.Length; i++)
+                {
+                    if (_prevSize + _files[i].Length > pos)
+                    {
+                        if (_idx != i)
+                            _stream = openStream(i);
+                        break;
+                    }
+                    _prevSize += _files[i].Length;
+                }
+            }
+
+            _pos = pos - _prevSize;
+
+            if (_pos != _stream.Position && this.Position != this.Length)
+                _stream.Seek(_pos, SeekOrigin.Begin);
+            return pos;
+        }
+
+        public override void SetLength(long value)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override void Write(byte[] buffer, int offset, int count)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override void Close()
+        {
+            try
+            {
+                if (_stream != null)
+                    _stream.Close();
+            }
+            catch { }
+            base.Close();
+        }
+
+        protected override void Dispose(bool disposing)
+        {
+            try
+            {
+                if (_stream != null)
+                    _stream.Dispose();
+            }
+            catch { }
+            base.Dispose(disposing);
+        }
+    }
+}

tests/SharpCompress.Test/ArchiveTests.cs

@@ -95,6 +95,68 @@ namespace SharpCompress.Test
             }
         }
 
+        protected void ArchiveStreamSplitRead(ReaderOptions readerOptions = null, params string[] testArchives)
+        {
+            ArchiveStreamSplitRead(readerOptions, testArchives.Select(x => Path.Combine(TEST_ARCHIVES_PATH, x)));
+        }
+
+        protected void ArchiveStreamSplitRead(ReaderOptions readerOptions, IEnumerable<string> testArchives)
+        {
+            using (SplitStream stream = new SplitStream(testArchives.Select(f => new FileInfo(f))))
+            {
+                using (var archive = ArchiveFactory.Open(stream, readerOptions))
+                {
+                    try
+                    {
+                        foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory))
+                        {
+                            entry.WriteToDirectory(SCRATCH_FILES_PATH,
+                                                    new ExtractionOptions()
+                                                    {
+                                                        ExtractFullPath = true,
+                                                        Overwrite = true
+                                                    });
+                        }
+                    }
+                    catch (IndexOutOfRangeException)
+                    {
+                        throw;
+                    }
+                }
+                VerifyFiles();
+            }
+        }
+
+        protected void ArchiveOpenStreamRead(ReaderOptions readerOptions = null, params string[] testArchives)
+        {
+            ArchiveOpenStreamRead(readerOptions, testArchives.Select(x => Path.Combine(TEST_ARCHIVES_PATH, x)));
+        }
+
+        protected void ArchiveOpenStreamRead(ReaderOptions readerOptions, IEnumerable<string> testArchives)
+        {
+            using (var archive = ArchiveFactory.Open(testArchives.Select(f => new FileInfo(f)), null))
+            {
+                try
+                {
+                    foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory))
+                    {
+                        entry.WriteToDirectory(SCRATCH_FILES_PATH,
+                                               new ExtractionOptions()
+                                               {
+                                                   ExtractFullPath = true,
+                                                   Overwrite = true
+                                               });
+                    }
+                }
+                catch (IndexOutOfRangeException)
+                {
+                    throw;
+                }
+            }
+            VerifyFiles();
+        }
+
+
         protected void ArchiveFileRead(string testArchive, ReaderOptions readerOptions = null)
         {
             testArchive = Path.Combine(TEST_ARCHIVES_PATH, testArchive);

tests/SharpCompress.Test/Rar/RarArchiveTests.cs

@@ -386,6 +386,71 @@ namespace SharpCompress.Test.Rar
         }
 
         [Fact]
+        public void Rar4_Multi_ArchiveFileRead()
+        {
+            ArchiveFileRead("Rar4.multi.part01.rar");
+        }
+
+        [Fact]
+        public void Rar4_ArchiveFileRead()
+        {
+            ArchiveFileRead("Rar4.rar");
+        }
+
+        [Fact]
+        public void Rar4_Multi_ArchiveStreamRead()
+        {
+            DoRar_Multi_ArchiveStreamRead(new string[] {
+                "Rar4.multi.part01.rar",
+                "Rar4.multi.part02.rar",
+                "Rar4.multi.part03.rar",
+                "Rar4.multi.part04.rar",
+                "Rar4.multi.part05.rar",
+                "Rar4.multi.part06.rar",
+                "Rar4.multi.part07.rar"}, false);
+        }
+
+        //no extension to test the lib identifies the archive by content not ext
+        [Fact]
+        public void Rar4_Split_ArchiveStreamRead()
+        {
+            ArchiveStreamSplitRead(null, new string[] {
+                "Rar4.split.001",
+                "Rar4.split.002",
+                "Rar4.split.003",
+                "Rar4.split.004",
+                "Rar4.split.005",
+                "Rar4.split.006"});
+        }
+
+        //open with ArchiveFactory.Open and stream
+        [Fact]
+        public void Rar4_Split_ArchiveOpen()
+        {
+            ArchiveOpenStreamRead(null,
+                "Rar4.split.001",
+                "Rar4.split.002",
+                "Rar4.split.003",
+                "Rar4.split.004",
+                "Rar4.split.005",
+                "Rar4.split.006");
+        }
+
+        //open with ArchiveFactory.Open and stream
+        [Fact]
+        public void Rar4_Multi_ArchiveOpen()
+        {
+            ArchiveOpenStreamRead(null,
+                "Rar4.multi.part01.rar",
+                "Rar4.multi.part02.rar",
+                "Rar4.multi.part03.rar",
+                "Rar4.multi.part04.rar",
+                "Rar4.multi.part05.rar",
+                "Rar4.multi.part06.rar",
+                "Rar4.multi.part07.rar");
+        }
+
+    [Fact]
         public void Rar_Multi_ArchiveFileRead()
         {
             ArchiveFileRead("Rar.multi.part01.rar");

tests/SharpCompress.Test/SevenZip/SevenZipArchiveTests.cs

@@ -101,5 +101,19 @@ namespace SharpCompress.Test.SevenZip
                                                                             "Original.7z.006",
                                                                             "Original.7z.007"));
         }
+
+        //Same as archive as Original.7z.001 ... 007 files without the root directory 'Original\' in the archive - this caused the verify to fail
+        [Fact]
+        public void SevenZipArchive_BZip2_Split_Working()
+        {
+            ArchiveStreamSplitRead(null, "7Zip.BZip2.split.001",
+                                        "7Zip.BZip2.split.002",
+                                        "7Zip.BZip2.split.003",
+                                        "7Zip.BZip2.split.004",
+                                        "7Zip.BZip2.split.005",
+                                        "7Zip.BZip2.split.006",
+                                        "7Zip.BZip2.split.007");
+        }
+
     }
 }

tests/SharpCompress.Test/Zip/ZipArchiveTests.cs

@@ -108,6 +108,16 @@ namespace SharpCompress.Test.Zip
             ArchiveFileRead("Zip.deflate.zip");
         }
         [Fact]
+        public void Zip_Deflate_Split_ArchiveFileRead()
+        {
+            ArchiveStreamSplitRead(null, "Zip.deflate.split.001",
+                                        "Zip.deflate.split.002",
+                                        "Zip.deflate.split.003",
+                                        "Zip.deflate.split.004",
+                                        "Zip.deflate.split.005",
+                                        "Zip.deflate.split.006");
+        }
+        [Fact]
         public void Zip_Deflate64_ArchiveFileRead()
         {
             ArchiveFileRead("Zip.deflate64.zip");
@@ -616,5 +626,6 @@ namespace SharpCompress.Test.Zip
                 }
             }
         }
+
     }
 }