diff --git a/docs/GameData/DataReferences.md b/docs/GameData/DataReferences.md index 00d06c17..ce108377 100644 --- a/docs/GameData/DataReferences.md +++ b/docs/GameData/DataReferences.md @@ -42,4 +42,6 @@ For resource prototypes: Example: `Resource/Encounters/CH09Norway/OM_NordicRuins_CowBosses.encounter` should be formatted as `&Resource/Encounters/CH09Norway/OM_NordicRuins_CowBosses.encounter`. +[Pak file](./PakFile.md) entry hashes are generated the same way, but without any additional formatting. + GUID generation algorithms are currently unknown. diff --git a/docs/GameData/PakFile.md b/docs/GameData/PakFile.md index 8518d180..b962a33e 100644 --- a/docs/GameData/PakFile.md +++ b/docs/GameData/PakFile.md @@ -2,7 +2,7 @@ A pak file (also known as `GPAK` by its signature and `sip` by its extension) is an archive that contains game data. Early versions of the game used a SQLite database for storage, but later on it was replaced with a custom format. -Initially all data was stored in a single `mu_cdata.sip` file located in `%GameDirectory%\UnrealEngine3\Binaries\Win32\Data`. Eventually data was separated into two files (`Calligraphy.sip` and `mu_cdata.sip`) located in `%GameDirectory%\Data\Game`. +Initially all data was stored in a single `mu_cdata.sip` file located in `%GameDirectory%\UnrealEngine3\Binaries\Win32\Data`. Eventually data was separated into two files (`Calligraphy.sip` and `mu_cdata.sip`), and then moved to `%GameDirectory%\Data\Game`. ## SQLite Paks @@ -12,7 +12,7 @@ The `data_tbl` table has the following columns: | Name | Type | Constraints | Description | | ---- | --------- | ----------------------- | -------------- | -| `i` | `INTEGER` | | File id / hash | +| `i` | `INTEGER` | | File name hash | | `n` | `TEXT` | `UNIQUE COLLATE NOCASE` | File name | | `b` | `BLOB` | | Data | | `l` | `INTEGER` | | Data size | @@ -29,7 +29,7 @@ There are two known versions of these SQLite-based paks: | Format Version | Note | Client Version | | -------------- | --------------------------------------------------------------------------- | -------------- | -| 1.5 | Added back index for names for non-maxload optimization | 1.9-1.22 | +| 1.5 | Added back index for names for non-maxload optimization | 1.9-1.21 | | 1.6 | Added lz4 compression for stored data to speed up shipping client load time | 1.22-1.28 | ## Custom Gazillion Paks @@ -68,6 +68,8 @@ struct PakEntry } ``` +`FileHash` is `FileName` hashed using the same algorithm as Calligraphy / resource [data references](./DataReferences.md), but without any additional formatting. Files are sorted by their hash value. + The entries are followed by raw data compressed using the [LZ4](https://github.com/lz4/lz4) algorithm. The offsets specified in the entries are from where the raw data begins. -Data from these custom paks can be extracted and parsed with [MHDataParser](https://github.com/Crypto137/MHDataParser). +These custom paks can be unpacked and packed with the [MHPakTool](./../../Tools/MHPakTool) included in this repository. Data can be parsed with [MHDataParser](https://github.com/Crypto137/MHDataParser), diff --git a/tools/MHPakTool/Extensions.cs b/tools/MHPakTool/Extensions.cs new file mode 100644 index 00000000..a191972b --- /dev/null +++ b/tools/MHPakTool/Extensions.cs @@ -0,0 +1,24 @@ +using System.Text; + +namespace MHPakTool +{ + public static class Extensions + { + /// + /// Reads a fixed-length string preceded by its length as a 32-bit signed integer. + /// + public static string ReadFixedString32(this BinaryReader reader) + { + return Encoding.UTF8.GetString(reader.ReadBytes(reader.ReadInt32())); + } + + /// + /// Writes a fixed-length string preceded by its length as a 32-bit signed integer. + /// + public static void WriteFixedString32(this BinaryWriter writer, string @string) + { + writer.Write(@string.Length); + writer.Write(Encoding.UTF8.GetBytes(@string)); + } + } +} diff --git a/tools/MHPakTool/HashHelper.cs b/tools/MHPakTool/HashHelper.cs new file mode 100644 index 00000000..5aa73756 --- /dev/null +++ b/tools/MHPakTool/HashHelper.cs @@ -0,0 +1,38 @@ +using System.Text; + +namespace MHPakTool +{ + public class HashHelper + { + public static uint Adler32(string str) + { + const int mod = 65521; + uint a = 1, b = 0; + foreach (char c in str) + { + a = (a + c) % mod; + b = (b + a) % mod; + } + return (b << 16) | a; + } + + public static uint Crc32(byte[] bytes) + { + byte[] hash = System.IO.Hashing.Crc32.Hash(bytes); + return BitConverter.ToUInt32(hash); + } + + public static uint Crc32(string str) => Crc32(Encoding.UTF8.GetBytes(str)); + + /// + /// Hashes a path with Adler32 and Crc32. + /// + public static ulong HashPath(string path) + { + path = path.ToLower(); + ulong adler = Adler32(path); + ulong crc = Crc32(path); + return (adler | (crc << 32)) - 1; + } + } +} diff --git a/tools/MHPakTool/MHPakTool.csproj b/tools/MHPakTool/MHPakTool.csproj new file mode 100644 index 00000000..9e63aa4f --- /dev/null +++ b/tools/MHPakTool/MHPakTool.csproj @@ -0,0 +1,19 @@ + + + + Exe + net6.0 + enable + disable + + + + + ..\..\dep\K4os.Compression.LZ4\K4os.Compression.LZ4.dll + + + ..\..\dep\System.IO.Hashing\System.IO.Hashing.dll + + + + diff --git a/tools/MHPakTool/PakEntry.cs b/tools/MHPakTool/PakEntry.cs new file mode 100644 index 00000000..415a9cb8 --- /dev/null +++ b/tools/MHPakTool/PakEntry.cs @@ -0,0 +1,59 @@ +using K4os.Compression.LZ4; + +namespace MHPakTool +{ + public class PakEntry + { + private static readonly byte[] CompressionBuffer = new byte[1024 * 1024 * 8]; + + public ulong FileHash { get; } + public string FilePath { get; } + public int ModTime { get; } + public int Offset { get; set; } + public int CompressedSize { get; } + public int UncompressedSize { get; } + public byte[] UncompressedData { get; set; } + public byte[] CompressedData { get; set; } + + /// + /// Constructor for reading existing pak file. + /// + public PakEntry(BinaryReader reader) + { + FileHash = reader.ReadUInt64(); + FilePath = reader.ReadFixedString32(); + ModTime = reader.ReadInt32(); + Offset = reader.ReadInt32(); + CompressedSize = reader.ReadInt32(); + UncompressedSize = reader.ReadInt32(); + } + + /// + /// Constructor for adding entries to a new pak file. + /// + public PakEntry(string relativeFilePath, byte[] uncompressedData) + { + // Hash file path + FilePath = relativeFilePath; + FileHash = HashHelper.HashPath(relativeFilePath); + + ModTime = 1717986918; // ffff from Calligraphy.sip + + // Compress data + UncompressedData = uncompressedData; + UncompressedSize = uncompressedData.Length; + CompressedSize = LZ4Codec.Encode(uncompressedData, CompressionBuffer); // Output doesn't match original sips 1 to 1, but it seems to work fine + CompressedData = CompressionBuffer.Take(CompressedSize).ToArray(); + } + + public void WriteMetadata(BinaryWriter writer) + { + writer.Write(FileHash); + writer.WriteFixedString32(FilePath); + writer.Write(ModTime); + writer.Write(Offset); + writer.Write(CompressedSize); + writer.Write(UncompressedSize); + } + } +} diff --git a/tools/MHPakTool/PakFile.cs b/tools/MHPakTool/PakFile.cs new file mode 100644 index 00000000..edf472a3 --- /dev/null +++ b/tools/MHPakTool/PakFile.cs @@ -0,0 +1,148 @@ +using K4os.Compression.LZ4; + +namespace MHPakTool +{ + public class PakFile + { + private const uint Signature = 1196441931; // KAPG + private const uint Version = 1; + + private readonly Dictionary _fileDict = new(); + + public List EntryList { get; private set; } = new(); + + /// + /// Read a pak file from the specified path. + /// + public PakFile(string pakFilePath) + { + // Make sure the specified file exists + if (File.Exists(pakFilePath) == false) + { + Console.WriteLine($"{Path.GetFileName(pakFilePath)} not found"); + return; + } + + // Read pak file + using (FileStream stream = File.OpenRead(pakFilePath)) + using (BinaryReader reader = new(stream)) + { + // Read file header + uint signature = reader.ReadUInt32(); + if (signature != Signature) + { + Console.WriteLine($"Invalid pak file signature {signature}, expected {Signature}"); + return; + } + + uint version = reader.ReadUInt32(); + if (version != Version) + { + Console.WriteLine($"Invalid pak file version {version}, expected {Version}"); + return; + } + + // Read all entries + int numEntries = reader.ReadInt32(); + for (int i = 0; i < numEntries; i++) + EntryList.Add(new(reader)); + + // Read and decompress the actual data + foreach (PakEntry entry in EntryList) + { + entry.CompressedData = new byte[entry.CompressedSize]; + entry.UncompressedData = new byte[entry.UncompressedSize]; + + stream.Read(entry.CompressedData, 0, entry.CompressedSize); + LZ4Codec.Decode(entry.CompressedData, 0, entry.CompressedSize, + entry.UncompressedData, 0, entry.UncompressedData.Length); + + _fileDict.Add(entry.FilePath, entry.UncompressedData); // Add data lookup + } + } + + Console.WriteLine($"Loaded {EntryList.Count} entries from {Path.GetFileName(pakFilePath)}"); + } + + /// + /// Create a new empty pak file. + /// + public PakFile() { } + + public byte[] GetFile(string filePath) + { + if (_fileDict.TryGetValue(filePath, out var file) == false) + { + Console.WriteLine($"File {filePath} not found"); + return Array.Empty(); + } + + return file; + } + + public void ExtractData(string outputDirectory) + { + Console.WriteLine("Extracting pak data..."); + + foreach (PakEntry entry in EntryList) + { + Console.WriteLine($"Extracting {entry.FilePath}..."); + + string filePath = Path.Combine(outputDirectory, entry.FilePath); + string directory = Path.GetDirectoryName(filePath); // Paks have their own directory structure that we need to keep in mind. + + if (Directory.Exists(directory) == false) Directory.CreateDirectory(directory); + File.WriteAllBytes(filePath, entry.UncompressedData); + } + } + + public void AddDirectory(string path) + { + Console.WriteLine($"Adding files from {path}..."); + + string root = Path.GetFullPath(Path.Combine(path, "..")); // Get root of the directory that's being added + + int start = EntryList.Count; + + // Iterate through all files + foreach (string file in Directory.GetFiles(path, "*.*", SearchOption.AllDirectories)) + { + string relativePath = Path.GetRelativePath(root, file).Replace('\\', '/'); // Use forward slashes, same as original pak files + Console.WriteLine($"Adding {relativePath}..."); + EntryList.Add(new(relativePath, File.ReadAllBytes(file))); + } + + Console.WriteLine($"Added {EntryList.Count - start} files"); + } + + public void WritePak(string filePath) + { + Console.WriteLine($"Writing pak to {filePath}..."); + + // Sort by hash + EntryList = EntryList.OrderBy(o => o.FileHash).ToList(); + + // Set offsets + int offset = 0; + foreach (PakEntry entry in EntryList) + { + entry.Offset = offset; + offset += entry.CompressedSize; + } + + using (FileStream stream = File.OpenWrite(filePath)) + using (BinaryWriter writer = new(stream)) + { + writer.Write(Signature); + writer.Write(Version); + writer.Write(EntryList.Count); + + foreach (PakEntry entry in EntryList) + entry.WriteMetadata(writer); + + foreach (PakEntry entry in EntryList) + writer.Write(entry.CompressedData); + } + } + } +} diff --git a/tools/MHPakTool/Program.cs b/tools/MHPakTool/Program.cs new file mode 100644 index 00000000..9a057835 --- /dev/null +++ b/tools/MHPakTool/Program.cs @@ -0,0 +1,46 @@ +namespace MHPakTool +{ + internal class Program + { + static void Main(string[] args) + { + if (args.Length < 1) + { + Console.WriteLine("Drag and drop a .sip file on this tool to unpack it or a folder to pack it to .sip."); + Console.ReadLine(); + return; + } + + string path = args[0]; + + if ((Directory.Exists(path) || File.Exists(path)) == false) + { + Console.WriteLine($"{path} is not a valid path"); + Console.ReadLine(); + return; + } + + if (File.GetAttributes(path).HasFlag(FileAttributes.Directory)) + WritePak(path); + else + ReadPak(path); + + Console.WriteLine("Finished"); + Console.ReadLine(); + } + + private static void ReadPak(string path) + { + PakFile pak = new(path); + pak.ExtractData(Path.GetDirectoryName(path)); + } + + private static void WritePak(string path) + { + Console.WriteLine(path); + PakFile pak = new(); + pak.AddDirectory(path); + pak.WritePak(Path.Combine(Directory.GetCurrentDirectory(), $"{new DirectoryInfo(path).Name}.sip")); + } + } +} diff --git a/tools/Tools.sln b/tools/Tools.sln index 44c4492a..7482c1aa 100644 --- a/tools/Tools.sln +++ b/tools/Tools.sln @@ -5,6 +5,8 @@ VisualStudioVersion = 17.8.34316.72 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MHExecutableAnalyzer", "MHExecutableAnalyzer\MHExecutableAnalyzer.csproj", "{F2A52386-D324-40C5-910B-970908C1349E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MHPakTool", "MHPakTool\MHPakTool.csproj", "{8B76417F-BE52-45B6-B167-0DD8EEF60956}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -15,6 +17,10 @@ Global {F2A52386-D324-40C5-910B-970908C1349E}.Debug|Any CPU.Build.0 = Debug|Any CPU {F2A52386-D324-40C5-910B-970908C1349E}.Release|Any CPU.ActiveCfg = Release|Any CPU {F2A52386-D324-40C5-910B-970908C1349E}.Release|Any CPU.Build.0 = Release|Any CPU + {8B76417F-BE52-45B6-B167-0DD8EEF60956}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8B76417F-BE52-45B6-B167-0DD8EEF60956}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8B76417F-BE52-45B6-B167-0DD8EEF60956}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8B76417F-BE52-45B6-B167-0DD8EEF60956}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE