diff --git a/dep/K4os.Compression.LZ4/K4os.Compression.LZ4.License b/dep/K4os.Compression.LZ4/K4os.Compression.LZ4.License
new file mode 100644
index 00000000..f691b2ae
--- /dev/null
+++ b/dep/K4os.Compression.LZ4/K4os.Compression.LZ4.License
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Milosz Krajewski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/dep/K4os.Compression.LZ4/K4os.Compression.LZ4.dll b/dep/K4os.Compression.LZ4/K4os.Compression.LZ4.dll
new file mode 100644
index 00000000..9b8a0ae6
Binary files /dev/null and b/dep/K4os.Compression.LZ4/K4os.Compression.LZ4.dll differ
diff --git a/dep/K4os.Compression.LZ4/K4os.Compression.LZ4.xml b/dep/K4os.Compression.LZ4/K4os.Compression.LZ4.xml
new file mode 100644
index 00000000..2ccb4269
--- /dev/null
+++ b/dep/K4os.Compression.LZ4/K4os.Compression.LZ4.xml
@@ -0,0 +1,1211 @@
+
+
+
+ K4os.Compression.LZ4
+
+
+
+
+ Action performed by encoder using FlushAndEncode method.
+
+
+
+ Nothing has happened, most likely loading 0 bytes.
+
+
+ Some bytes has been loaded into encoder.
+
+
+ Compression was not possible so bytes has been copied.
+
+
+ Compression succeeded.
+
+
+
+ Interface of LZ4 decoder used by LZ4 streams.
+
+
+
+ Block size.
+
+
+ Bytes already decoded and available to be read.
+ Always smaller than
+
+
+
+ Decodes previously compressed block and caches decompressed block in decoder.
+ Returns number of bytes decoded. These bytes can be read with .
+
+ Points to compressed block.
+ Length of compressed block.
+ Size of the block. Value 0 indicates default block size.
+ Number of decoded bytes.
+
+
+
+ Inject already decompressed block and caches it in decoder.
+ Used with uncompressed-yet-chained blocks and pre-made dictionaries.
+ These bytes can be read with .
+
+ Points to uncompressed block.
+ Length of uncompressed block.
+ Number of decoded bytes.
+
+
+
+ Reads previously decoded bytes. Please note, should be
+ negative number, pointing to bytes before current head.
+
+ Buffer to write to.
+ Offset in source buffer relatively to current head.
+ Please note, it should be negative value.
+ Number of bytes to read.
+
+
+
+ Peeks at previously decoded bytes. Please note, should be
+ negative number, pointing to bytes before current head.
+
+ Offset in source buffer relatively to current head.
+ Please note, it should be negative value.
+
+
+
+ Interface of LZ4 encoder used by LZ4 streams.
+
+
+
+ Block size.
+
+
+ Number of bytes read for compression.
+ Always smaller than
+
+
+ Adds bytes to internal buffer. Increases
+ Source buffer.
+ Source buffer length.
+ Number of bytes topped up. If this function returns 0 it means that buffer
+ is full ( equals ) and
+ should be called to flush it.
+
+
+
+ Encodes bytes in internal buffer (see: , ).
+ If is true then if encoded buffer is bigger than
+ source buffer source bytes are copied instead. In such case returned length is negative.
+
+ Target buffer.
+ Target buffer length.
+ Indicates if copying is allowed.
+ Length of encoded buffer. Negative if bytes are just copied.
+
+
+
+ LZ4 decoder used with independent blocks mode. Please note, that it will fail
+ if input data has been compressed with chained blocks
+ ( and )
+
+
+
+ Creates new instance of block decoder.
+ Block size. Must be equal or greater to one used for compression.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Independent block encoder. Produces larger files but uses less memory and
+ gives better performance.
+
+
+
+ Creates new instance of
+ Compression level.
+ Block size.
+
+
+
+
+
+
+
+
+ LZ4 decoder handling dependent blocks.
+
+
+ Creates new instance of .
+ Block size.
+ Number of extra blocks.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Static class with factory methods to create LZ4 decoders.
+
+
+
+ Creates appropriate decoder for given parameters.
+ Dependent blocks.
+ Block size.
+ Number of extra blocks.
+ LZ4 decoder.
+
+
+
+ Static class with factory method to create LZ4 encoders.
+
+
+
+ Creates appropriate decoder for given parameters.
+ Dependent blocks.
+ Compression level.
+ Block size.
+ Number of extra blocks.
+ LZ4 encoder.
+
+
+
+ Base class for LZ4 encoders. Provides basic functionality shared by
+ , ,
+ and encoders. Do not used directly.
+
+
+
+ Creates new instance of encoder.
+ Needs to be true if using dependent blocks.
+ Block size.
+ Number of extra blocks.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Encodes single block using appropriate algorithm.
+ Source buffer.
+ Source buffer length.
+ Target buffer.
+ Target buffer length.
+ Number of bytes actually written to target buffer.
+
+
+ Copies current dictionary.
+ Target buffer.
+ Dictionary length.
+ Dictionary length.
+
+
+
+
+
+
+ Functionality of encoders added on top of fixed interface.
+
+
+
+ Tops encoder up with some data.
+ Encoder.
+ Buffer pointer, will be shifted after operation by the number of
+ bytes actually loaded.
+ Length of buffer.
+ true if buffer was topped up, false if no bytes were loaded.
+
+
+ Tops encoder up with some data.
+ Encoder.
+ Buffer.
+ Buffer offset.
+ Length of buffer.
+ Number of bytes actually loaded.
+
+
+ Tops encoder up with some data.
+ Encoder.
+ Buffer.
+ Buffer offset, will be increased after operation by the number
+ of bytes actually loaded.
+ Length of buffer.
+ true if buffer was topped up, false if no bytes were loaded.
+
+
+ Encodes all bytes currently stored in encoder into target buffer.
+ Encoder.
+ Target buffer.
+ Offset in target buffer.
+ Length of target buffer.
+ if true copying bytes is allowed.
+ Number of bytes encoder. If bytes were copied than this value is negative.
+
+
+ Encodes all bytes currently stored in encoder into target buffer.
+ Encoder.
+ Target buffer.
+ Offset in target buffer. Will be updated after operation.
+ Length of target buffer.
+ if true copying bytes is allowed.
+ Result of this action. Bytes can be Copied (),
+ Encoded () or nothing could have
+ happened ().
+
+
+ Encodes all bytes currently stored in encoder into target buffer.
+ Encoder.
+ Target buffer. Will be updated after operation.
+ Length of buffer.
+ if true copying bytes is allowed.
+ Result of this action. Bytes can be Copied (),
+ Encoded () or nothing could have
+ happened ().
+
+
+ Tops encoder and encodes content.
+ Encoder.
+ Source buffer (used to top up from).
+ Source buffer length.
+ Target buffer (used to encode into)
+ Target buffer length.
+ Forces encoding even if encoder is not full.
+ Allows to copy bytes if compression was not possible.
+ Number of bytes loaded (topped up)
+ Number if bytes encoded or copied.
+ Value is 0 if no encoding was done.
+ Action performed.
+
+
+ Tops encoder and encodes content.
+ Encoder.
+ Source buffer (used to top up from).
+ Offset within source buffer.
+ Source buffer length.
+ Target buffer (used to encode into)
+ Offset within target buffer.
+ Target buffer length.
+ Forces encoding even if encoder is not full.
+ Allows to copy bytes if compression was not possible.
+ Number of bytes loaded (topped up)
+ Number if bytes encoded or copied.
+ Value is 0 if no encoding was done.
+ Action performed.
+
+
+ Tops encoder and encodes content.
+ Encoder.
+ Source buffer (used to top up from).
+ Target buffer (used to encode into)
+ Forces encoding even if encoder is not full.
+ Allows to copy bytes if compression was not possible.
+ Number of bytes loaded (topped up)
+ Number if bytes encoded or copied.
+ Value is 0 if no encoding was done.
+ Action performed.
+
+
+ Encoded remaining bytes in encoder.
+ Encoder.
+ Target buffer.
+ Target buffer length.
+ Allows to copy bytes if compression was not possible.
+ Number if bytes encoded or copied.
+ Value is 0 if no encoding was done.
+ Action performed.
+
+
+ Encoded remaining bytes in encoder.
+ Encoder.
+ Target buffer.
+ Offset within target buffer.
+ Target buffer length.
+ Allows to copy bytes if compression was not possible.
+ Number if bytes encoded or copied.
+ Value is 0 if no encoding was done.
+ Action performed.
+
+
+ Encoded remaining bytes in encoder.
+ Encoder.
+ Target buffer.
+ Allows to copy bytes if compression was not possible.
+ Number if bytes encoded or copied.
+ Value is 0 if no encoding was done.
+ Action performed.
+
+
+ Drains decoder by reading all bytes which are ready.
+ Decoder.
+ Target buffer.
+ Offset within target buffer.
+ Offset in decoder relatively to decoder's head.
+ Please note, it should be negative value.
+ Number of bytes.
+
+
+ Drains decoder by reading all bytes which are ready.
+ Decoder.
+ Target buffer.
+ Offset in decoder relatively to decoder's head.
+ Please note, it should be negative value.
+ Number of bytes.
+
+
+ Decodes data and immediately drains it into target buffer.
+ Decoder.
+ Source buffer (with compressed data, to be decoded).
+ Source buffer length.
+ Target buffer (to drained into).
+ Target buffer length.
+ Number of bytes actually decoded.
+ true decoder was drained, false otherwise.
+
+
+ Decodes data and immediately drains it into target buffer.
+ Decoder.
+ Source buffer (with compressed data, to be decoded).
+ Offset within source buffer.
+ Source buffer length.
+ Target buffer (to drained into).
+ Offset within target buffer.
+ Target buffer length.
+ Number of bytes actually decoded.
+ true decoder was drained, false otherwise.
+
+
+ Decodes data and immediately drains it into target buffer.
+ Decoder.
+ Source buffer (with compressed data, to be decoded).
+ Target buffer (to drained into).
+ Number of bytes actually decoded.
+ true decoder was drained, false otherwise.
+
+
+
+ Inject already decompressed block and caches it in decoder.
+ Used with uncompressed-yet-chained blocks and pre-made dictionaries.
+ See .
+
+ Decoder.
+ Uncompressed block.
+ Offset in uncompressed block.
+ Length of uncompressed block.
+ Number of decoded bytes.
+
+
+
+ Decodes previously compressed block and caches decompressed block in decoder.
+ Returns number of bytes decoded.
+ See .
+
+ Decoder.
+ Compressed block.
+ Offset in compressed block.
+ Length of compressed block.
+ Size of the block. Value 0 indicates default block size.
+ Number of decoded bytes.
+
+
+
+ LZ4 encoder using dependent blocks with fast compression.
+
+
+
+ Creates new instance of
+ Block size.
+ Number of extra blocks.
+
+
+
+
+
+
+
+
+
+
+
+
+ LZ4 encoder using dependent blocks with high compression.
+
+
+
+ Creates new instance of
+ Compression level.
+ Block size.
+ Number of extra blocks.
+
+
+
+
+
+
+
+
+
+
+
+ Algorithm selection.
+
+
+ Intel and ARMv7 version of 32 bit algorithm.
+
+
+ Intel version of 64 bit algorithm.
+
+
+ Checks what algorithm should be used (32 vs 64 bit).
+
+
+
+ Existence of this class is an admission of failure.
+ I failed to export internals to test assemblies.
+ Using InternalsVisibleTo work, of course, but with signing (which was requested
+ in https://github.com/MiloszKrajewski/K4os.Compression.LZ4/issues/9) it is
+ absolute PITA. So no, I'm not using InternalsVisibleTo I will just expose this
+ little class with some "redirects" to real internals.
+
+
+
+ Pubternal wrapper for LZ4_stream_t.
+
+
+ Creates new instance of wrapper for LZ4_stream_t.
+
+
+
+
+
+
+ Compresses chunk of data using LZ4_compress_fast_continue.
+
+ Wrapper for LZ4_stream_t
+ Source block address.
+ Target block address.
+ Source block length.
+ Target block length.
+ Acceleration.
+ Number of bytes actually written to target.
+
+
+
+ Naive wrapper around ArrayPool. Makes calls if something should be pooled.
+
+
+
+ Minimum size of the buffer that can be pooled.
+
+
+ Allocate temporary buffer to store decompressed data.
+ Minimum size of the buffer.
+ Clear all data.
+ Allocated buffer.
+
+
+
+ Determines if buffer was pooled or not.
+ The logic is quite simple: if buffer is smaller than 512 bytes are pooled.
+
+ Buffer.
+ true if buffer was pooled; false otherwise
+
+
+ Releases allocated buffer.
+ Previously allocated buffer.
+
+
+ Utility class with memory related functions.
+
+
+ 1 KiB
+
+
+ 2 KiB
+
+
+ 4 KiB
+
+
+ 8 KiB
+
+
+ 16 KiB
+
+
+ 32 KiB
+
+
+ 64 KiB
+
+
+ 128 KiB
+
+
+ 256 KiB
+
+
+ 512 KiB
+
+
+ 1 MiB
+
+
+ 4 MiB
+
+
+ Empty byte array.
+
+
+ Checks if process is ran in 32-bit mode.
+
+
+ Rounds integer value up to nearest multiple of step.
+ A value.
+ A step.
+ Value rounded up.
+
+
+
+ Copies memory block for to .
+
+ The target block address.
+ The source block address.
+ Length in bytes.
+
+
+
+ Fills memory block with predefined .
+
+ The target block address.
+ Value to be used.
+ Length in bytes.
+
+
+
+ Copies memory block for to .
+
+ The target block address.
+ The source block address.
+ Length in bytes.
+
+
+
+ Copies memory block for to .
+ It handle "move" semantic properly handling overlapping blocks properly.
+
+ The target block address.
+ The source block address.
+ Length in bytes.
+
+
+ Allocated block of memory. It is NOT initialized with zeroes.
+ Size in bytes.
+ Pointer to allocated block.
+
+
+ Fill block of memory with zeroes.
+ Address.
+ Length.
+ Original pointer.
+
+
+ Fills memory block with repeating pattern of a single byte.
+ Address.
+ A pattern.
+ Length.
+ Original pointer.
+
+
+ Allocates block of memory and fills it with zeroes.
+ Size in bytes.
+ Pointer to allocated block.
+
+
+ Free memory allocated previously with .
+ Pointer to allocated block.
+
+
+ Clones managed array to unmanaged one.
+ Allows quicker yet less safe unchecked access.
+ Input array.
+ Cloned array.
+
+
+ Reads exactly 1 byte from given address.
+ Address.
+ Byte at given address.
+
+
+ Writes exactly 1 byte to given address.
+ Address.
+ Value.
+
+
+ Reads exactly 2 bytes from given address.
+ Address.
+ 2 bytes at given address.
+
+
+ Writes exactly 2 bytes to given address.
+ Address.
+ Value.
+
+
+ Reads exactly 4 bytes from given address.
+ Address.
+ 4 bytes at given address.
+
+
+ Writes exactly 4 bytes to given address.
+ Address.
+ Value.
+
+
+ Reads exactly 8 bytes from given address.
+ Address.
+ 8 bytes at given address.
+
+
+ Writes exactly 8 bytes to given address.
+ Address.
+ Value.
+
+
+ Copies exactly 1 byte from source to target.
+ Target address.
+ Source address.
+
+
+ Copies exactly 2 bytes from source to target.
+ Target address.
+ Source address.
+
+
+ Copies exactly 4 bytes from source to target.
+ Target address.
+ Source address.
+
+
+ Copies exactly 8 bytes from source to target.
+ Target address.
+ Source address.
+
+
+
+ Represents pinned memory.
+ It either points to unmanaged memory or block of memory from shared array pool.
+ When disposed, it handles it appropriately.
+
+
+
+
+ Maximum size of the buffer that can be pooled from shared array pool.
+
+
+
+ Pointer to block of bytes.
+
+
+ Pointer to block of bytes as span.
+
+
+ Pointer to block of bytes.
+
+
+
+ Allocates pinned block of memory, depending on the size it tries to use shared array pool.
+
+ Size in bytes.
+ Indicates if block should be zeroed.
+ Allocated .
+
+
+
+
+ Allocates pinned block of memory, depending on the size it tries to use shared array pool.
+
+ Pinned memory pointer.
+ Size in bytes.
+ Indicates if block should be zeroed.
+ Allocated .
+
+
+
+
+ Allocates pinned block of memory for one item from shared array pool.
+
+ PinnedMemory pointer.
+ Indicates if block should be zeroed.
+ Type of item.
+
+
+ Fill allocated block of memory with zeros.
+
+
+
+ Releases the memory.
+
+
+
+
+ Skeleton for class with unmanaged resources.
+ Implements but also handles proper release in
+ case was not called.
+
+
+
+ Determines if object was already disposed.
+
+
+ Throws exception is object has been disposed already. Convenience method.
+ Thrown if object is already disposed.
+
+
+ Method releasing unmanaged resources.
+
+
+ Method releasing managed resources.
+
+
+
+ Disposed resources.
+
+ true if dispose was explicitly called,
+ false if called from GC.
+
+
+
+
+
+ Destructor.
+
+
+ Unsafe memory operations.
+
+
+ Reads 4 bytes from given address.
+ Address.
+ 4 bytes at given address.
+
+
+ Writes 4 or 8 bytes to given address.
+ Address.
+ Value.
+
+
+ Copies exactly 16 bytes from source to target.
+ Target address.
+ Source address.
+
+
+ Copies exactly 18 bytes from source to target.
+ Target address.
+ Source address.
+
+
+
+ Copies memory block for to
+ up to (around) .
+ It does not handle overlapping blocks and may copy up to 8 bytes more than expected.
+
+ The target block address.
+ The source block address.
+ The limit (in target block).
+
+
+
+ Copies memory block for to
+ up to (around) .
+ It does not handle overlapping blocks and may copy up to 32 bytes more than expected.
+ This version copies two times 16 bytes (instead of one time 32 bytes)
+ because it must be compatible with offsets >= 16.
+
+ The target block address.
+ The source block address.
+ The limit (in target block).
+
+
+ Unsafe memory operations.
+
+
+ Reads exactly 2 bytes from given address.
+ Address.
+ 2 bytes at given address.
+
+
+ Writes exactly 2 bytes to given address.
+ Address.
+ Value.
+
+
+ Reads exactly 4 bytes from given address.
+ Address.
+ 4 bytes at given address.
+
+
+ Writes exactly 4 bytes to given address.
+ Address.
+ Value.
+
+
+ Copies exactly 1 byte from source to target.
+ Target address.
+ Source address.
+
+
+ Copies exactly 2 bytes from source to target.
+ Target address.
+ Source address.
+
+
+ Copies exactly 4 bytes from source to target.
+ Target address.
+ Source address.
+
+
+ Reads exactly 8 bytes from given address.
+ Address.
+ 8 bytes at given address.
+
+
+ Writes exactly 8 bytes to given address.
+ Address.
+ Value.
+
+
+ Copies exactly 8 bytes from source to target.
+ Target address.
+ Source address.
+
+
+ Reads 8 bytes from given address.
+ Address.
+ 8 bytes at given address.
+
+
+ Writes 8 bytes to given address.
+ Address.
+ Value.
+
+
+ Copies exactly 16 bytes from source to target.
+ Target address.
+ Source address.
+
+
+ Copies exactly 18 bytes from source to target.
+ Target address.
+ Source address.
+
+
+
+ Copies memory block for to
+ up to (around) .
+ It does not handle overlapping blocks and may copy up to 8 bytes more than expected.
+
+ The target block address.
+ The source block address.
+ The limit (in target block).
+
+
+
+ Copies memory block for to
+ up to (around) .
+ It does not handle overlapping blocks and may copy up to 32 bytes more than expected.
+ This version copies two times 16 bytes (instead of one time 32 bytes)
+ because it must be compatible with offsets >= 16.
+
+ The target block address.
+ The source block address.
+ The limit (in target block).
+
+
+
+ Static class exposing LZ4 block compression methods.
+
+
+
+ Version of LZ4 implementation.
+
+
+
+ Enforces 32-bit compression/decompression algorithm even on 64-bit systems.
+ Please note, this property should not be used on regular basis, it just allows
+ to workaround some problems on platforms which do not support 64-bit the same was
+ as Intel (for example: unaligned read/writes).
+
+
+
+ Maximum size after compression.
+ Length of input buffer.
+ Maximum length after compression.
+
+
+ Compresses data from one buffer into another.
+ Input buffer.
+ Length of input buffer.
+ Output buffer.
+ Output buffer length.
+ Compression level.
+ Number of bytes written, or negative value if output buffer is too small.
+
+
+ Compresses data from one buffer into another.
+ Input buffer.
+ Output buffer.
+ Compression level.
+ Number of bytes written, or negative value if output buffer is too small.
+
+
+ Compresses data from one buffer into another.
+ Input buffer.
+ Input buffer offset.
+ Input buffer length.
+ Output buffer.
+ Output buffer offset.
+ Output buffer length.
+ Compression level.
+ Number of bytes written, or negative value if output buffer is too small.
+
+
+ Decompresses data from given buffer.
+ Input buffer.
+ Input buffer length.
+ Output buffer.
+ Output buffer length.
+ Number of bytes written, or negative value if output buffer is too small.
+
+
+ Decompresses data from given buffer.
+ Input buffer.
+ Input buffer length.
+ Output buffer.
+ Output buffer length.
+ Dictionary buffer.
+ Dictionary buffer length.
+ Number of bytes written, or negative value if output buffer is too small.
+
+
+ Decompresses data from given buffer.
+ Input buffer.
+ Output buffer.
+ Number of bytes written, or negative value if output buffer is too small.
+
+
+ Decompresses data from given buffer.
+ Input buffer.
+ Output buffer.
+ Dictionary buffer.
+ Number of bytes written, or negative value if output buffer is too small.
+
+
+ Decompresses data from given buffer.
+ Input buffer.
+ Input buffer offset.
+ Input buffer length.
+ Output buffer.
+ Output buffer offset.
+ Output buffer length.
+ Number of bytes written, or negative value if output buffer is too small.
+
+
+ Decompresses data from given buffer.
+ Input buffer.
+ Input buffer offset.
+ Input buffer length.
+ Output buffer.
+ Output buffer offset.
+ Output buffer length.
+ Dictionary buffer.
+ Dictionary buffer offset.
+ Dictionary buffer length.
+ Number of bytes written, or negative value if output buffer is too small.
+
+
+ Compression level.
+
+
+ Fast compression.
+
+
+ High compression, level 3.
+
+
+ High compression, level 4.
+
+
+ High compression, level 5.
+
+
+ High compression, level 6.
+
+
+ High compression, level 7.
+
+
+ High compression, level 8.
+
+
+ High compression, level 9.
+
+
+ Optimal compression, level 10.
+
+
+ Optimal compression, level 11.
+
+
+ Maximum compression, level 12.
+
+
+
+ Pickling support with LZ4 compression.
+
+
+ Pickling support with LZ4 compression.
+
+
+
+ Compresses input buffer into self-contained package.
+ Input buffer.
+ Compression level.
+ Output buffer.
+
+
+ Compresses input buffer into self-contained package.
+ Input buffer.
+ Input buffer offset.
+ Input buffer length.
+ Compression level.
+ Output buffer.
+
+
+ Compresses input buffer into self-contained package.
+ Input buffer.
+ Length of input data.
+ Compression level.
+ Output buffer.
+
+
+ Compresses input buffer into self-contained package.
+ Input buffer.
+ Compression level.
+ Output buffer.
+
+
+ Compresses input buffer into self-contained package.
+ Input buffer.
+ Where the compressed data is written.
+ Compression level.
+ Output buffer.
+
+
+ Compresses input buffer into self-contained package.
+ Input buffer.
+ Where the compressed data is written.
+ Compression level.
+ Output buffer.
+
+
+ Decompresses previously pickled buffer (see: .
+ Input buffer.
+ Output buffer.
+
+
+ Decompresses previously pickled buffer (see: .
+ Input buffer.
+ Input buffer offset.
+ Input buffer length.
+ Output buffer.
+
+
+ Decompresses previously pickled buffer (see: .
+ Input buffer.
+ Input buffer length.
+ Output buffer.
+
+
+ Decompresses previously pickled buffer (see: .
+ Input buffer.
+ Output buffer.
+
+
+ Decompresses previously pickled buffer (see: .
+ Input buffer.
+ Where the decompressed data is written.
+
+
+ Decompresses previously pickled buffer (see: .
+ Input buffer.
+ Where the decompressed data is written.
+
+
+
+ Returns the uncompressed size of a chunk of compressed data.
+
+ The data to inspect.
+ The size in bytes of the data once unpickled.
+
+
+
+ Returns the uncompressed size of a chunk of compressed data.
+
+ Decoded header.
+ The size in bytes of the data once unpickled.
+
+
+ Decompresses previously pickled buffer (see: .
+ Input buffer.
+ Where the decompressed data is written.
+
+ You obtain the size of the output buffer by calling .
+
+
+
+
diff --git a/docs/Setup.md b/docs/Setup.md
index 905c2649..a850a1eb 100644
--- a/docs/Setup.md
+++ b/docs/Setup.md
@@ -18,6 +18,8 @@ After getting the client, you need to set up a web server to serve SiteConfig.xm
6. Compile MHServerEmu with Visual Studio or any other tool you prefer.
+7. Copy `Calligraphy.sip` and `mu_cdata.sip` located in `Marvel Heroes\Data\Game` to `MHServerEmu\Assets\GPAK`.
+
Now you can actually start everything and get in-game.
1. Start Apache by running ```Apache24\bin\httpd.exe```.
diff --git a/src/MHServerEmu/Assets/GPAK/IMPORTANT b/src/MHServerEmu/Assets/GPAK/IMPORTANT
new file mode 100644
index 00000000..356d3e5f
--- /dev/null
+++ b/src/MHServerEmu/Assets/GPAK/IMPORTANT
@@ -0,0 +1 @@
+Copy Calligraphy.sip and mu_cdata.sip from Marvel Heroes\Data\Game to here
\ No newline at end of file
diff --git a/src/MHServerEmu/GameServer/Data/Database.cs b/src/MHServerEmu/GameServer/Data/Database.cs
index 91edb6ce..84bc7089 100644
--- a/src/MHServerEmu/GameServer/Data/Database.cs
+++ b/src/MHServerEmu/GameServer/Data/Database.cs
@@ -1,5 +1,6 @@
using System.Text;
using MHServerEmu.Common;
+using MHServerEmu.GameServer.Data.Gpak;
namespace MHServerEmu.GameServer.Data
{
@@ -8,6 +9,9 @@ namespace MHServerEmu.GameServer.Data
private static readonly Logger Logger = LogManager.CreateLogger();
public static bool IsInitialized { get; private set; }
+ public static GpakFile Calligraphy { get; private set; }
+ public static GpakFile Resource { get; private set; }
+
public static Dictionary PrototypeDataDict { get; private set; }
public static ulong[] GlobalEnumRefTable { get; private set; }
@@ -16,17 +20,23 @@ namespace MHServerEmu.GameServer.Data
static Database()
{
- Logger.Info("Loading prototypes...");
+ long startTime = ((DateTimeOffset)DateTime.UtcNow).ToUnixTimeMilliseconds();
+
+ Calligraphy = new("Calligraphy.sip");
+ Resource = new("mu_cdata.sip");
+
+ Logger.Trace("Loading prototypes...");
PrototypeDataDict = LoadPrototypeData($"{Directory.GetCurrentDirectory()}\\Assets\\PrototypeDataTable.bin");
+ Logger.Info($"Loaded {PrototypeDataDict.Count} prototypes");
GlobalEnumRefTable = LoadPrototypeEnumRefTable($"{Directory.GetCurrentDirectory()}\\Assets\\GlobalEnumRefTable.bin");
ResourceEnumRefTable = LoadPrototypeEnumRefTable($"{Directory.GetCurrentDirectory()}\\Assets\\ResourceEnumRefTable.bin");
PropertyIdPowerRefTable = LoadPrototypeEnumRefTable($"{Directory.GetCurrentDirectory()}\\Assets\\PropertyIdPowerRefTable.bin");
- if (PrototypeDataDict.Count > 0 && GlobalEnumRefTable.Length > 0 && ResourceEnumRefTable.Length > 0 && PropertyIdPowerRefTable.Length > 0)
+ if (VerifyData())
{
- // -1 is here because the first entry is 0 to offset values and align with the data we get from the game
- Logger.Info($"Loaded {PrototypeDataDict.Count} prototypes");
+ long loadTime = ((DateTimeOffset)DateTime.UtcNow).ToUnixTimeMilliseconds() - startTime;
+ Logger.Info($"Finished loading in {loadTime} ms");
IsInitialized = true;
}
else
@@ -36,6 +46,24 @@ namespace MHServerEmu.GameServer.Data
}
}
+ public static void ExportGpakEntries()
+ {
+ Logger.Info("Exporting Calligraphy entries...");
+ Calligraphy.ExportEntries("Calligraphy.tsv");
+ Logger.Info("Exporting Resource entries...");
+ Resource.ExportEntries("mu_cdata.tsv");
+ Logger.Info("Finished exporting GPAK entries");
+ }
+
+ public static void ExportGpakData()
+ {
+ Logger.Info("Exporting Calligraphy data...");
+ Calligraphy.ExportData();
+ Logger.Info("Exporting Resource data...");
+ Resource.ExportData();
+ Logger.Info("Finished exporting GPAK data");
+ }
+
private static Dictionary LoadPrototypeData(string path)
{
Dictionary prototypeDict = new();
@@ -97,5 +125,15 @@ namespace MHServerEmu.GameServer.Data
return Array.Empty();
}
}
+
+ private static bool VerifyData()
+ {
+ return Calligraphy.Entries.Length > 0
+ && Resource.Entries.Length > 0
+ && PrototypeDataDict.Count > 0
+ && GlobalEnumRefTable.Length > 0
+ && ResourceEnumRefTable.Length > 0
+ && PropertyIdPowerRefTable.Length > 0;
+ }
}
}
diff --git a/src/MHServerEmu/GameServer/Data/Gpak/GpakFile.cs b/src/MHServerEmu/GameServer/Data/Gpak/GpakFile.cs
index ea484bb4..ac70fddb 100644
--- a/src/MHServerEmu/GameServer/Data/Gpak/GpakFile.cs
+++ b/src/MHServerEmu/GameServer/Data/Gpak/GpakFile.cs
@@ -1,4 +1,6 @@
-using System.Text;
+using System.IO;
+using System.Text;
+using K4os.Compression.LZ4;
using MHServerEmu.Common;
namespace MHServerEmu.GameServer.Data.Gpak
@@ -10,7 +12,7 @@ namespace MHServerEmu.GameServer.Data.Gpak
public int Header { get; } // KAPG
public int Field1 { get; }
- public GpakEntry[] Entries { get; }
+ public GpakEntry[] Entries { get; } = Array.Empty();
public GpakFile(string gpakFileName)
{
@@ -18,15 +20,18 @@ namespace MHServerEmu.GameServer.Data.Gpak
if (File.Exists(path))
{
+ Logger.Trace($"Loading {gpakFileName}...");
+
using (FileStream fileStream = File.OpenRead(path))
{
byte[] buffer = new byte[4096];
+ // Header
Header = ReadInt(fileStream, buffer);
Field1 = ReadInt(fileStream, buffer);
+ Entries = new GpakEntry[ReadInt(fileStream, buffer)];
- Entries = new GpakEntry[ReadInt(fileStream, buffer)];
-
+ // Entry metadata
for (int i = 0; i < Entries.Length; i++)
{
ulong id = ReadUlong(fileStream, buffer);
@@ -42,12 +47,46 @@ namespace MHServerEmu.GameServer.Data.Gpak
Entries[i] = new(id, filePath, field2, offset, compressedSize, uncompressedSize);
}
- Logger.Debug($"Loaded {Entries.Length} GPAK entries");
+ // Entry data
+ foreach (GpakEntry entry in Entries)
+ {
+ byte[] compressedData = new byte[entry.CompressedSize];
+ byte[] uncompressedData = new byte[entry.UncompressedSize];
+ fileStream.Read(compressedData, 0, compressedData.Length);
+ LZ4Codec.Decode(compressedData, uncompressedData);
+ entry.Data = uncompressedData;
+ }
}
+
+ Logger.Info($"Loaded {Entries.Length} GPAK entries from {gpakFileName}");
}
else
{
- Logger.Warn($"{gpakFileName} not found");
+ Logger.Error($"{gpakFileName} not found");
+ }
+ }
+
+ public void ExportEntries(string fileName)
+ {
+ using (StreamWriter streamWriter = new($"{GpakDirectory}\\{fileName}"))
+ {
+ foreach (GpakEntry entry in Entries)
+ {
+ string entryString = $"{entry.Id}\t{entry.FilePath}\t{entry.Field2}\t{entry.Offset}\t{entry.CompressedSize}\t{entry.UncompressedSize}";
+ streamWriter.WriteLine(entryString);
+ }
+ }
+ }
+
+ public void ExportData()
+ {
+ foreach (GpakEntry entry in Entries)
+ {
+ string uncompressedFilePath = $"{GpakDirectory}\\{entry.FilePath}";
+ string uncompressedFileDirectory = Path.GetDirectoryName(uncompressedFilePath);
+
+ if (Directory.Exists(uncompressedFileDirectory) == false) Directory.CreateDirectory(uncompressedFileDirectory);
+ File.WriteAllBytes($"{GpakDirectory}\\{entry.FilePath}", entry.Data);
}
}
diff --git a/src/MHServerEmu/MHServerEmu.csproj b/src/MHServerEmu/MHServerEmu.csproj
index 1455c165..f3698090 100644
--- a/src/MHServerEmu/MHServerEmu.csproj
+++ b/src/MHServerEmu/MHServerEmu.csproj
@@ -15,9 +15,15 @@
..\..\dep\protobuf-csharp\Google.ProtocolBuffers.dll
+
+ ..\..\dep\K4os.Compression.LZ4\K4os.Compression.LZ4.dll
+
+
+ PreserveNewest
+
PreserveNewest
diff --git a/src/MHServerEmu/Program.cs b/src/MHServerEmu/Program.cs
index f196ac9f..e5e532da 100644
--- a/src/MHServerEmu/Program.cs
+++ b/src/MHServerEmu/Program.cs
@@ -47,6 +47,10 @@ namespace MHServerEmu
_frontendServer.Shutdown();
else if (input.Equals("parse", StringComparison.OrdinalIgnoreCase))
PacketHelper.ParseServerMessagesFromAllPacketFiles();
+ else if (input.Equals("exportgpakentries", StringComparison.OrdinalIgnoreCase))
+ Database.ExportGpakEntries();
+ else if (input.Equals("exportgpakdata", StringComparison.OrdinalIgnoreCase))
+ Database.ExportGpakData();
}
_frontendServer.Shutdown();