lancommander-ref/MHServerEmu
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement MHPakTool

Browse source
This commit is contained in:
Crypto137 2023-12-01 00:10:03 +03:00
parent 1cd87ca6fa
commit b0ade50d52
9 changed files with 348 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docs/GameData/DataReferences.md
View file

@ -42,4 +42,6 @@ For resource prototypes:
Example: `Resource/Encounters/CH09Norway/OM_NordicRuins_CowBosses.encounter` should be formatted as `&Resource/Encounters/CH09Norway/OM_NordicRuins_CowBosses.encounter`.
[Pak file](./PakFile.md) entry hashes are generated the same way, but without any additional formatting.
GUID generation algorithms are currently unknown.

10
docs/GameData/PakFile.md
View file

@ -2,7 +2,7 @@
A pak file (also known as `GPAK` by its signature and `sip` by its extension) is an archive that contains game data. Early versions of the game used a SQLite database for storage, but later on it was replaced with a custom format.
Initially all data was stored in a single `mu_cdata.sip` file located in `%GameDirectory%\UnrealEngine3\Binaries\Win32\Data`. Eventually data was separated into two files (`Calligraphy.sip` and `mu_cdata.sip`) located in `%GameDirectory%\Data\Game`.
Initially all data was stored in a single `mu_cdata.sip` file located in `%GameDirectory%\UnrealEngine3\Binaries\Win32\Data`. Eventually data was separated into two files (`Calligraphy.sip` and `mu_cdata.sip`), and then moved to `%GameDirectory%\Data\Game`.
## SQLite Paks
@ -12,7 +12,7 @@ The `data_tbl` table has the following columns:
| Name | Type | Constraints | Description |
| ---- | --------- | ----------------------- | -------------- |
| `i` | `INTEGER` | | File id / hash |
| `i` | `INTEGER` | | File name hash |
| `n` | `TEXT` | `UNIQUE COLLATE NOCASE` | File name |
| `b` | `BLOB` | | Data |
| `l` | `INTEGER` | | Data size |
@ -29,7 +29,7 @@ There are two known versions of these SQLite-based paks:
| Format Version | Note | Client Version |
| -------------- | --------------------------------------------------------------------------- | -------------- |
| 1.5 | Added back index for names for non-maxload optimization | 1.9-1.22 |
| 1.5 | Added back index for names for non-maxload optimization | 1.9-1.21 |
| 1.6 | Added lz4 compression for stored data to speed up shipping client load time | 1.22-1.28 |
## Custom Gazillion Paks
@ -68,6 +68,8 @@ struct PakEntry
}
```
`FileHash` is `FileName` hashed using the same algorithm as Calligraphy / resource [data references](./DataReferences.md), but without any additional formatting. Files are sorted by their hash value.
The entries are followed by raw data compressed using the [LZ4](https://github.com/lz4/lz4) algorithm. The offsets specified in the entries are from where the raw data begins.
Data from these custom paks can be extracted and parsed with [MHDataParser](https://github.com/Crypto137/MHDataParser).
These custom paks can be unpacked and packed with the [MHPakTool](./../../Tools/MHPakTool) included in this repository. Data can be parsed with [MHDataParser](https://github.com/Crypto137/MHDataParser),

24
tools/MHPakTool/Extensions.cs Normal file
View file

@ -0,0 +1,24 @@
using System.Text;
namespace MHPakTool
{
public static class Extensions
{
/// <summary>
/// Reads a fixed-length string preceded by its length as a 32-bit signed integer.
/// </summary>
public static string ReadFixedString32(this BinaryReader reader)
{
return Encoding.UTF8.GetString(reader.ReadBytes(reader.ReadInt32()));
}
/// <summary>
/// Writes a fixed-length string preceded by its length as a 32-bit signed integer.
/// </summary>
public static void WriteFixedString32(this BinaryWriter writer, string @string)
{
writer.Write(@string.Length);
writer.Write(Encoding.UTF8.GetBytes(@string));
}
}
}

38
tools/MHPakTool/HashHelper.cs Normal file
View file

@ -0,0 +1,38 @@
using System.Text;
namespace MHPakTool
{
public class HashHelper
{
public static uint Adler32(string str)
{
const int mod = 65521;
uint a = 1, b = 0;
foreach (char c in str)
{
a = (a + c) % mod;
b = (b + a) % mod;
}
return (b << 16) | a;
}
public static uint Crc32(byte[] bytes)
{
byte[] hash = System.IO.Hashing.Crc32.Hash(bytes);
return BitConverter.ToUInt32(hash);
}
public static uint Crc32(string str) => Crc32(Encoding.UTF8.GetBytes(str));
/// <summary>
/// Hashes a path with Adler32 and Crc32.
/// </summary>
public static ulong HashPath(string path)
{
path = path.ToLower();
ulong adler = Adler32(path);
ulong crc = Crc32(path);
return (adler | (crc << 32)) - 1;
}
}
}

19
tools/MHPakTool/MHPakTool.csproj Normal file
View file

@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>disable</Nullable>
</PropertyGroup>
<ItemGroup>
<Reference Include="K4os.Compression.LZ4">
<HintPath>..\..\dep\K4os.Compression.LZ4\K4os.Compression.LZ4.dll</HintPath>
</Reference>
<Reference Include="System.IO.Hashing">
<HintPath>..\..\dep\System.IO.Hashing\System.IO.Hashing.dll</HintPath>
</Reference>
</ItemGroup>
</Project>

59
tools/MHPakTool/PakEntry.cs Normal file
View file

@ -0,0 +1,59 @@
using K4os.Compression.LZ4;
namespace MHPakTool
{
public class PakEntry
{
private static readonly byte[] CompressionBuffer = new byte[1024 * 1024 * 8];
public ulong FileHash { get; }
public string FilePath { get; }
public int ModTime { get; }
public int Offset { get; set; }
public int CompressedSize { get; }
public int UncompressedSize { get; }
public byte[] UncompressedData { get; set; }
public byte[] CompressedData { get; set; }
/// <summary>
/// Constructor for reading existing pak file.
/// </summary>
public PakEntry(BinaryReader reader)
{
FileHash = reader.ReadUInt64();
FilePath = reader.ReadFixedString32();
ModTime = reader.ReadInt32();
Offset = reader.ReadInt32();
CompressedSize = reader.ReadInt32();
UncompressedSize = reader.ReadInt32();
}
/// <summary>
/// Constructor for adding entries to a new pak file.
/// </summary>
public PakEntry(string relativeFilePath, byte[] uncompressedData)
{
// Hash file path
FilePath = relativeFilePath;
FileHash = HashHelper.HashPath(relativeFilePath);
ModTime = 1717986918; // ffff from Calligraphy.sip
// Compress data
UncompressedData = uncompressedData;
UncompressedSize = uncompressedData.Length;
CompressedSize = LZ4Codec.Encode(uncompressedData, CompressionBuffer); // Output doesn't match original sips 1 to 1, but it seems to work fine
CompressedData = CompressionBuffer.Take(CompressedSize).ToArray();
}
public void WriteMetadata(BinaryWriter writer)
{
writer.Write(FileHash);
writer.WriteFixedString32(FilePath);
writer.Write(ModTime);
writer.Write(Offset);
writer.Write(CompressedSize);
writer.Write(UncompressedSize);
}
}
}

148
tools/MHPakTool/PakFile.cs Normal file
View file

@ -0,0 +1,148 @@
using K4os.Compression.LZ4;
namespace MHPakTool
{
public class PakFile
{
private const uint Signature = 1196441931; // KAPG
private const uint Version = 1;
private readonly Dictionary<string, byte[]> _fileDict = new();
public List<PakEntry> EntryList { get; private set; } = new();
/// <summary>
/// Read a pak file from the specified path.
/// </summary>
public PakFile(string pakFilePath)
{
// Make sure the specified file exists
if (File.Exists(pakFilePath) == false)
{
Console.WriteLine($"{Path.GetFileName(pakFilePath)} not found");
return;
}
// Read pak file
using (FileStream stream = File.OpenRead(pakFilePath))
using (BinaryReader reader = new(stream))
{
// Read file header
uint signature = reader.ReadUInt32();
if (signature != Signature)
{
Console.WriteLine($"Invalid pak file signature {signature}, expected {Signature}");
return;
}
uint version = reader.ReadUInt32();
if (version != Version)
{
Console.WriteLine($"Invalid pak file version {version}, expected {Version}");
return;
}
// Read all entries
int numEntries = reader.ReadInt32();
for (int i = 0; i < numEntries; i++)
EntryList.Add(new(reader));
// Read and decompress the actual data
foreach (PakEntry entry in EntryList)
{
entry.CompressedData = new byte[entry.CompressedSize];
entry.UncompressedData = new byte[entry.UncompressedSize];
stream.Read(entry.CompressedData, 0, entry.CompressedSize);
LZ4Codec.Decode(entry.CompressedData, 0, entry.CompressedSize,
entry.UncompressedData, 0, entry.UncompressedData.Length);
_fileDict.Add(entry.FilePath, entry.UncompressedData); // Add data lookup
}
}
Console.WriteLine($"Loaded {EntryList.Count} entries from {Path.GetFileName(pakFilePath)}");
}
/// <summary>
/// Create a new empty pak file.
/// </summary>
public PakFile() { }
public byte[] GetFile(string filePath)
{
if (_fileDict.TryGetValue(filePath, out var file) == false)
{
Console.WriteLine($"File {filePath} not found");
return Array.Empty<byte>();
}
return file;
}
public void ExtractData(string outputDirectory)
{
Console.WriteLine("Extracting pak data...");
foreach (PakEntry entry in EntryList)
{
Console.WriteLine($"Extracting {entry.FilePath}...");
string filePath = Path.Combine(outputDirectory, entry.FilePath);
string directory = Path.GetDirectoryName(filePath); // Paks have their own directory structure that we need to keep in mind.
if (Directory.Exists(directory) == false) Directory.CreateDirectory(directory);
File.WriteAllBytes(filePath, entry.UncompressedData);
}
}
public void AddDirectory(string path)
{
Console.WriteLine($"Adding files from {path}...");
string root = Path.GetFullPath(Path.Combine(path, "..")); // Get root of the directory that's being added
int start = EntryList.Count;
// Iterate through all files
foreach (string file in Directory.GetFiles(path, "*.*", SearchOption.AllDirectories))
{
string relativePath = Path.GetRelativePath(root, file).Replace('\\', '/'); // Use forward slashes, same as original pak files
Console.WriteLine($"Adding {relativePath}...");
EntryList.Add(new(relativePath, File.ReadAllBytes(file)));
}
Console.WriteLine($"Added {EntryList.Count - start} files");
}
public void WritePak(string filePath)
{
Console.WriteLine($"Writing pak to {filePath}...");
// Sort by hash
EntryList = EntryList.OrderBy(o => o.FileHash).ToList();
// Set offsets
int offset = 0;
foreach (PakEntry entry in EntryList)
{
entry.Offset = offset;
offset += entry.CompressedSize;
}
using (FileStream stream = File.OpenWrite(filePath))
using (BinaryWriter writer = new(stream))
{
writer.Write(Signature);
writer.Write(Version);
writer.Write(EntryList.Count);
foreach (PakEntry entry in EntryList)
entry.WriteMetadata(writer);
foreach (PakEntry entry in EntryList)
writer.Write(entry.CompressedData);
}
}
}
}

46
tools/MHPakTool/Program.cs Normal file
View file

@ -0,0 +1,46 @@
namespace MHPakTool
{
internal class Program
{
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Drag and drop a .sip file on this tool to unpack it or a folder to pack it to .sip.");
Console.ReadLine();
return;
}
string path = args[0];
if ((Directory.Exists(path) || File.Exists(path)) == false)
{
Console.WriteLine($"{path} is not a valid path");
Console.ReadLine();
return;
}
if (File.GetAttributes(path).HasFlag(FileAttributes.Directory))
WritePak(path);
else
ReadPak(path);
Console.WriteLine("Finished");
Console.ReadLine();
}
private static void ReadPak(string path)
{
PakFile pak = new(path);
pak.ExtractData(Path.GetDirectoryName(path));
}
private static void WritePak(string path)
{
Console.WriteLine(path);
PakFile pak = new();
pak.AddDirectory(path);
pak.WritePak(Path.Combine(Directory.GetCurrentDirectory(), $"{new DirectoryInfo(path).Name}.sip"));
}
}
}

6
tools/Tools.sln
View file

@ -5,6 +5,8 @@ VisualStudioVersion = 17.8.34316.72
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MHExecutableAnalyzer", "MHExecutableAnalyzer\MHExecutableAnalyzer.csproj", "{F2A52386-D324-40C5-910B-970908C1349E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MHPakTool", "MHPakTool\MHPakTool.csproj", "{8B76417F-BE52-45B6-B167-0DD8EEF60956}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -15,6 +17,10 @@ Global
{F2A52386-D324-40C5-910B-970908C1349E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F2A52386-D324-40C5-910B-970908C1349E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F2A52386-D324-40C5-910B-970908C1349E}.Release|Any CPU.Build.0 = Release|Any CPU
{8B76417F-BE52-45B6-B167-0DD8EEF60956}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8B76417F-BE52-45B6-B167-0DD8EEF60956}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8B76417F-BE52-45B6-B167-0DD8EEF60956}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8B76417F-BE52-45B6-B167-0DD8EEF60956}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE