mirror of
https://github.com/mehah/otclient
synced 2026-08-15 16:29:06 -04:00
fix(cache): canonicalize manifest line endings (#1802)
fix(cache): canonicalize vcpkg manifest line endings Normalize manifest line endings when generating shared build-cache fingerprints so equivalent vcpkg configurations use the same dependency pool regardless of Git checkout settings. Root cause: - Equivalent Git content can be checked out using CRLF or LF depending on core.autocrlf and local Git configuration. - The previous shared-cache resolver hashed raw manifest bytes. - Two otherwise equivalent worktrees could therefore resolve to different dependency fingerprints and separate vcpkg pools. Main changes: - Normalize CRLF and LF when fingerprinting vcpkg.json. - Normalize CRLF and LF when fingerprinting vcpkg-configuration.json. - Keep registry, overlay, port, patch, and arbitrary payload trees byte-signatured because their contents may be line-ending-sensitive. - Advance the shared-cache contract from schema v3 to schema v4. - Retain recognition of schemas v1 through v3 for auditing and transient cleanup. - Require an exact full-fingerprint metadata match before cleaning fingerprint-specific transient data. - Keep shortened cache directory names backed by full SHA-256 identity metadata. - Update shared-cache setup and cleanup tooling for the v4 layout. - Update documentation with v4 fingerprinting behavior, migration rules, cleanup procedures, and cache paths. Windows build updates: - Move Windows CI to the Visual Studio 2026 hosted runner. - Verify that Visual Studio 2026 provides the native C++ v145 toolset. - Configure MSBuild discovery specifically for the Visual Studio 18.x prerelease range. - Remove the previous Chocolatey-based Visual Studio toolset installation path. Validation: - Reproduced the Release x64 dependency contract in two worktrees with different manifest line endings. - Verified both worktrees now resolve to the same schema v4 dependency fingerprint. - Ran SharedBuildCache.cmake in CMake script mode. - Verified the windows-release preset listing. - Ran PowerShell parser checks. - Ran JSON parsing checks. - Ran documentation portability checks. - Ran git diff --check. This keeps shared vcpkg cache identity stable across CRLF and LF checkouts while preserving byte-sensitive hashing for dependency content where physical file representation can affect behavior.
This commit is contained in:
Eduardo Dantas
GitHub
parent
44a980b07e
commit
4a2de8de5f
4 changed files with 96 additions and 49 deletions
56
.github/workflows/reusable-build-windows.yml
vendored
56
.github/workflows/reusable-build-windows.yml
vendored
|
|
@ -29,7 +29,7 @@ jobs:
|
|||
type: Solution
|
||||
configuration: DirectX
|
||||
artifact: windows-solution-directx
|
||||
runs-on: windows-2025
|
||||
runs-on: windows-2025-vs2026
|
||||
env:
|
||||
VCPKG_BINARY_CACHE_ACCESS: ${{ secrets.VCPKG_PACKAGES_TOKEN != '' && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'workflow_dispatch') && 'readwrite' || 'read' }}
|
||||
VCPKG_BINARY_SOURCES: "clear;nuget,https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json,${{ secrets.VCPKG_PACKAGES_TOKEN != '' && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'workflow_dispatch') && 'readwrite' || 'read' }};nugettimeout,600"
|
||||
|
|
@ -38,48 +38,50 @@ jobs:
|
|||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
SCCACHE_DIR: ${{ github.workspace }}\.sccache
|
||||
steps:
|
||||
- name: Install VS 2026 Build Tools (v145 toolset)
|
||||
- name: Verify VS 2026 v145 toolset
|
||||
if: matrix.type == 'Solution'
|
||||
shell: pwsh
|
||||
run: |
|
||||
$acceptedExitCodes = @(0, 3010)
|
||||
for ($attempt = 1; $attempt -le 3; $attempt++) {
|
||||
choco install visualstudio2026-workload-vctools --yes --ignore-package-exit-codes=3010 --no-progress
|
||||
if ($acceptedExitCodes -contains $LASTEXITCODE) {
|
||||
exit 0
|
||||
}
|
||||
|
||||
Write-Warning "Chocolatey install failed with exit code $LASTEXITCODE on attempt $attempt."
|
||||
if ($attempt -lt 3) {
|
||||
Start-Sleep -Seconds (30 * $attempt)
|
||||
}
|
||||
$vswhere = Join-Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe"
|
||||
if (-not (Test-Path -LiteralPath $vswhere -PathType Leaf)) {
|
||||
throw "vswhere.exe is not available on the hosted Visual Studio runner"
|
||||
}
|
||||
|
||||
Write-Warning "Chocolatey could not install the VS 2026 v145 workload. Continuing so the next step can use any runner-provided toolset."
|
||||
$installationPath = & $vswhere `
|
||||
-latest `
|
||||
-prerelease `
|
||||
-products "*" `
|
||||
-version '[18.0,19.0)' `
|
||||
-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 `
|
||||
-property installationPath
|
||||
if ([string]::IsNullOrWhiteSpace($installationPath)) {
|
||||
throw "Visual Studio 2026 with the native C++ toolset is required"
|
||||
}
|
||||
|
||||
$toolsetVersionPath = Join-Path $installationPath "VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt"
|
||||
if (-not (Test-Path -LiteralPath $toolsetVersionPath -PathType Leaf)) {
|
||||
throw "The Visual Studio C++ toolset version file is missing: $toolsetVersionPath"
|
||||
}
|
||||
|
||||
$toolsetVersion = (Get-Content -LiteralPath $toolsetVersionPath -Raw).Trim()
|
||||
if ($toolsetVersion -notmatch '^14\.5') {
|
||||
throw "Visual Studio 2026 must provide the v145 toolset; found $toolsetVersion"
|
||||
}
|
||||
|
||||
- name: Setup MSBuild.exe
|
||||
if: matrix.type == 'Solution'
|
||||
uses: microsoft/setup-msbuild@v3
|
||||
with:
|
||||
vs-prerelease: true
|
||||
vs-version: "latest"
|
||||
vs-version: "[18.0,19.0)"
|
||||
|
||||
- name: Verify v145 platform toolset
|
||||
if: matrix.type == 'Solution'
|
||||
shell: pwsh
|
||||
run: |
|
||||
$programFilesX86 = [Environment]::GetFolderPath("ProgramFilesX86")
|
||||
$vswhere = Join-Path $programFilesX86 "Microsoft Visual Studio\Installer\vswhere.exe"
|
||||
if (-not (Test-Path $vswhere)) {
|
||||
throw "vswhere.exe not found"
|
||||
}
|
||||
|
||||
$installPath = & $vswhere -latest -prerelease -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath
|
||||
if (-not $installPath) {
|
||||
throw "Visual Studio with VC tools was not found"
|
||||
}
|
||||
|
||||
$vcMsbuildRoot = Join-Path $installPath "MSBuild\Microsoft\VC"
|
||||
$vswhere = Join-Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe"
|
||||
$installationPath = & $vswhere -latest -prerelease -products "*" -version '[18.0,19.0)' -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath
|
||||
$vcMsbuildRoot = Join-Path $installationPath "MSBuild\Microsoft\VC"
|
||||
$toolset = Get-ChildItem -Path $vcMsbuildRoot -Directory -Recurse -Filter v145 -ErrorAction SilentlyContinue |
|
||||
Where-Object { $_.FullName -match "\\PlatformToolsets\\v145$" } |
|
||||
Select-Object -First 1
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
include_guard(GLOBAL)
|
||||
|
||||
set(CANARY_SHARED_CACHE_SCHEMA
|
||||
"v3"
|
||||
"v4"
|
||||
)
|
||||
|
||||
file(
|
||||
|
|
@ -31,6 +31,14 @@ function(
|
|||
path
|
||||
output
|
||||
)
|
||||
cmake_parse_arguments(
|
||||
signature
|
||||
"NORMALIZE_LINE_ENDINGS"
|
||||
""
|
||||
""
|
||||
${ARGN}
|
||||
)
|
||||
|
||||
if(NOT CMAKE_SCRIPT_MODE_FILE)
|
||||
get_filename_component(
|
||||
candidate_directory
|
||||
|
|
@ -54,11 +62,31 @@ function(
|
|||
endif()
|
||||
|
||||
if(EXISTS "${path}")
|
||||
file(
|
||||
SHA256
|
||||
"${path}"
|
||||
file_hash
|
||||
)
|
||||
if(signature_NORMALIZE_LINE_ENDINGS)
|
||||
# The manifest JSON grammar treats physical CRLF and LF line endings
|
||||
# identically. Hash their canonical form so Git checkout settings do
|
||||
# not split an otherwise identical dependency pool. Generic port and
|
||||
# registry trees intentionally remain byte-signatured because
|
||||
# patches and other arbitrary payloads can be line-ending-sensitive.
|
||||
file(
|
||||
READ
|
||||
"${path}"
|
||||
file_contents
|
||||
)
|
||||
string(
|
||||
REPLACE "\r\n"
|
||||
"\n"
|
||||
file_contents
|
||||
"${file_contents}"
|
||||
)
|
||||
string(SHA256 file_hash "${file_contents}")
|
||||
else()
|
||||
file(
|
||||
SHA256
|
||||
"${path}"
|
||||
file_hash
|
||||
)
|
||||
endif()
|
||||
if(NOT CMAKE_SCRIPT_MODE_FILE)
|
||||
set_property(
|
||||
DIRECTORY
|
||||
|
|
@ -1759,6 +1787,7 @@ canary_shared_cache_file_signature(
|
|||
"manifest/vcpkg.json"
|
||||
"${manifest_root}/vcpkg.json"
|
||||
manifest_signature
|
||||
NORMALIZE_LINE_ENDINGS
|
||||
)
|
||||
string(
|
||||
APPEND
|
||||
|
|
@ -1769,6 +1798,7 @@ canary_shared_cache_file_signature(
|
|||
"manifest/vcpkg-configuration.json"
|
||||
"${manifest_root}/vcpkg-configuration.json"
|
||||
manifest_configuration_signature
|
||||
NORMALIZE_LINE_ENDINGS
|
||||
)
|
||||
string(
|
||||
APPEND
|
||||
|
|
|
|||
|
|
@ -80,13 +80,13 @@ pwsh -File tools/configure_shared_build_cache.ps1 -CleanTransientVcpkg
|
|||
|
||||
The helper refuses cleanup while build-related processes are active and holds the vcpkg root lock during deletion. It preserves installed trees, downloads, binary packages, and fingerprint-specific pools. Cleanup mode does not register repositories, persist environment variables, create the shared layout, detect compilers, or regenerate Solution contracts.
|
||||
|
||||
To reclaim only the disposable `buildtrees` and `packages` data for one known schema-v3 pool, pass its full dependency SHA-256:
|
||||
To reclaim only the disposable `buildtrees` and `packages` data for one known fingerprint pool, pass its full dependency SHA-256:
|
||||
|
||||
```text
|
||||
pwsh -File tools/configure_shared_build_cache.ps1 -CleanSharedFingerprintTransients <full-dependency-fingerprint>
|
||||
```
|
||||
|
||||
This narrower cleanup validates the full-hash identity metadata, confines both targets to the verified local cache root, holds the registry operation lock and that installed tree's vcpkg lock, and refuses to run while build processes are active. It never removes the expanded installed tree, metadata, downloads, or binary cache, and it performs no setup side effects. Because it does not prune persistent data, it remains suitable when the global consumer audit is incomplete; pruning an installed fingerprint still requires the complete audit described below.
|
||||
This narrower cleanup validates the full-hash identity metadata, confines both targets to the verified local cache root, holds the registry operation lock and that installed tree's vcpkg lock, and refuses to run while build processes are active. A full dependency fingerprint must match exactly one schema metadata record: the schema and normalized module identity are part of the fingerprint, so an ambiguous match is unsafe and is not cleaned. It never removes the expanded installed tree, metadata, downloads, or binary cache, and it performs no setup side effects. Because it does not prune persistent data, it remains suitable when the global consumer audit is incomplete; pruning an installed fingerprint still requires the complete audit described below.
|
||||
|
||||
## Non-Windows setup
|
||||
|
||||
|
|
@ -106,9 +106,9 @@ cmake --build --preset <build-preset>
|
|||
`cmake/SharedBuildCache.cmake` runs before the first `project()` call. When it can prove the complete installation contract, it selects:
|
||||
|
||||
```text
|
||||
<cache-root>/vcpkg-installed/v3/<dependency-fingerprint>
|
||||
<cache-root>/vcpkg-buildtrees/v3/<dependency-fingerprint>
|
||||
<cache-root>/vcpkg-packages/v3/<dependency-fingerprint>
|
||||
<cache-root>/vcpkg-installed/v4/<24-hex-dependency-fingerprint-prefix>
|
||||
<cache-root>/vcpkg-buildtrees/v4/<24-hex-dependency-fingerprint-prefix>
|
||||
<cache-root>/vcpkg-packages/v4/<24-hex-dependency-fingerprint-prefix>
|
||||
```
|
||||
|
||||
The first directory is persistent. The latter two are transient and are cleaned after successful dependency builds by the preset's vcpkg options.
|
||||
|
|
@ -228,13 +228,15 @@ The consumer fingerprint includes the dependency fingerprint and then adds eithe
|
|||
|
||||
The module disables sharing when any required identity or the local-filesystem guarantee is ambiguous. Absolute worktree paths do not participate. Content paths inside manifests and configurations still participate through the files themselves, while referenced local trees are hashed using relative file names and contents.
|
||||
|
||||
The module's normalized SHA-256 is part of schema `v3`. Copies in different forks must therefore be byte-equivalent after newline normalization to converge. A divergent implementation selects another fingerprint even if a maintainer forgets to bump the schema.
|
||||
The module's normalized SHA-256 is part of schema `v4`. Copies in different forks must therefore be byte-equivalent after newline normalization to converge. A divergent implementation selects another fingerprint even if a maintainer forgets to bump the schema.
|
||||
|
||||
Schema `v4` canonicalizes CRLF and LF only while signing `vcpkg.json` and an optional `vcpkg-configuration.json`: JSON treats those physical line endings identically, so a Git checkout setting cannot split an otherwise identical dependency pool. Files in registry and overlay trees remain byte-signatured; arbitrary ports, patches, and payloads can be line-ending-sensitive and must not be treated as interchangeable.
|
||||
|
||||
An existing configured preset never changes fingerprint or falls back in place. Cached package variables could retain paths into the old pool, so the module stops before `project()` and requests `cmake --fresh --preset <configure-preset>`.
|
||||
|
||||
Fingerprint input files and trees are CMake configure dependencies. Adding, removing, or changing a manifest, registry, overlay, triplet, compiler, or toolchain input requests regeneration.
|
||||
|
||||
The full dependency SHA-256 and non-local metadata are written below `<cache-root>/metadata/v3`. Directory names use the first 24 hexadecimal characters to limit Windows path length; the metadata lock verifies the full hash before a shortened directory is accepted.
|
||||
The full dependency SHA-256 and non-local metadata are written below `<cache-root>/metadata/v4`. Directory names use the first 24 hexadecimal characters to limit Windows path length; the metadata lock verifies the full hash before a shortened directory is accepted.
|
||||
|
||||
## Concurrency
|
||||
|
||||
|
|
@ -256,7 +258,7 @@ Verify its `CMakeCache.txt`:
|
|||
CANARY_SHARED_VCPKG_ACTIVE:INTERNAL=true
|
||||
CANARY_VCPKG_DEPENDENCY_FINGERPRINT:INTERNAL=<full-dependency-fingerprint>
|
||||
CANARY_VCPKG_CONSUMER_FINGERPRINT:INTERNAL=<full-consumer-fingerprint>
|
||||
VCPKG_INSTALLED_DIR:PATH=<cache-root>/vcpkg-installed/v3/<dependency-fingerprint>
|
||||
VCPKG_INSTALLED_DIR:PATH=<cache-root>/vcpkg-installed/v4/<24-hex-dependency-fingerprint-prefix>
|
||||
```
|
||||
|
||||
Also confirm that `CMakeCache.txt` and the generated native build files contain neither a legacy local installed path nor another global fingerprint. Inspect `build.ninja` for a Ninja preset; inspect the generated `.sln` and `.vcxproj` files for a Visual Studio preset. Complete a build against the refreshed preset before deleting the old local tree, then build again after deletion. The final invocation must not recreate the local installation.
|
||||
|
|
@ -275,7 +277,7 @@ Before pruning a fingerprint:
|
|||
|
||||
An audit that reports an unregistered, unavailable, partially enumerated, or malformed repository/configure tree, a non-local/reparse root, or a missing/mismatched full-hash identity is incomplete and exits with failure. Do not prune any global fingerprint until every registered family is available and the audit succeeds.
|
||||
|
||||
Schema migrations intentionally create a new pool. Keep the previous schema until every registered configured build has migrated, built successfully, and stopped referencing it.
|
||||
Schema migrations intentionally create a new pool. Keep the previous schema until every registered configured build has migrated, built successfully, and stopped referencing it. In particular, schema `v4` replaces schema `v3` to make manifest JSON signatures independent of Git's CRLF/LF checkout conversion; do not redirect or rename an existing v3 directory by hand.
|
||||
|
||||
If one fingerprint becomes corrupt, stop all its consumers, remove only that exact directory, and reconfigure one existing preset. vcpkg recreates it from the binary cache. Do not delete downloads or the global binary cache during normal recovery.
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ param(
|
|||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
$CurrentSharedCacheSchema = "v4"
|
||||
$KnownSharedCacheSchemas = @("v1", "v2", "v3", $CurrentSharedCacheSchema)
|
||||
|
||||
if ($env:OS -ne "Windows_NT") {
|
||||
throw "This helper persists Windows user environment variables. Follow docs/development/shared-build-cache.md for non-Windows setup."
|
||||
|
|
@ -864,6 +866,10 @@ if (-not $AuditOnly) {
|
|||
(Join-Path $CacheRoot "vcpkg-buildtrees\v3"),
|
||||
(Join-Path $CacheRoot "vcpkg-packages\v3"),
|
||||
(Join-Path $CacheRoot "metadata\v3"),
|
||||
(Join-Path $CacheRoot "vcpkg-installed\$CurrentSharedCacheSchema"),
|
||||
(Join-Path $CacheRoot "vcpkg-buildtrees\$CurrentSharedCacheSchema"),
|
||||
(Join-Path $CacheRoot "vcpkg-packages\$CurrentSharedCacheSchema"),
|
||||
(Join-Path $CacheRoot "metadata\$CurrentSharedCacheSchema"),
|
||||
(Join-Path $CacheRoot "registry\v2"),
|
||||
$binaryCache,
|
||||
$downloadsRoot
|
||||
|
|
@ -1031,13 +1037,20 @@ if (-not $AuditOnly) {
|
|||
if ($fingerprint -notmatch "^[0-9a-f]{64}$") {
|
||||
throw "A shared fingerprint cleanup target must be a full 64-character SHA-256: $fingerprintInput"
|
||||
}
|
||||
if (-not (Test-SharedFingerprintIdentity -CacheRoot $CacheRoot -Schema "v3" -Fingerprint $fingerprint)) {
|
||||
throw "The shared fingerprint identity is missing or does not match: $fingerprint"
|
||||
$matchingSchemas = @(
|
||||
$KnownSharedCacheSchemas |
|
||||
Where-Object {
|
||||
Test-SharedFingerprintIdentity -CacheRoot $CacheRoot -Schema $_ -Fingerprint $fingerprint
|
||||
}
|
||||
)
|
||||
if ($matchingSchemas.Count -ne 1) {
|
||||
throw "The shared fingerprint identity must match exactly one known schema: $fingerprint"
|
||||
}
|
||||
$schema = $matchingSchemas[0]
|
||||
|
||||
$shortFingerprint = $fingerprint.Substring(0, 24)
|
||||
$installedRoot = Get-FullPath -Path (Join-Path $CacheRoot "vcpkg-installed\v3\$shortFingerprint")
|
||||
$installedParent = Get-FullPath -Path (Join-Path $CacheRoot "vcpkg-installed\v3")
|
||||
$installedRoot = Get-FullPath -Path (Join-Path $CacheRoot "vcpkg-installed\$schema\$shortFingerprint")
|
||||
$installedParent = Get-FullPath -Path (Join-Path $CacheRoot "vcpkg-installed\$schema")
|
||||
if (
|
||||
(Get-FullPath -Path (Split-Path -Parent $installedRoot)) -ne $installedParent -or
|
||||
(Split-Path -Leaf $installedRoot) -ne $shortFingerprint -or
|
||||
|
|
@ -1060,7 +1073,7 @@ if (-not $AuditOnly) {
|
|||
)
|
||||
|
||||
foreach ($directoryName in @("vcpkg-buildtrees", "vcpkg-packages")) {
|
||||
$schemaRoot = Get-FullPath -Path (Join-Path $CacheRoot "$directoryName\v3")
|
||||
$schemaRoot = Get-FullPath -Path (Join-Path $CacheRoot "$directoryName\$schema")
|
||||
$transientRoot = Get-FullPath -Path (Join-Path $schemaRoot $shortFingerprint)
|
||||
$transientItem = Get-Item -LiteralPath $transientRoot -Force -ErrorAction SilentlyContinue
|
||||
if (
|
||||
|
|
@ -1188,7 +1201,7 @@ $configureTrees = foreach ($worktreeRoot in $worktreeRoots) {
|
|||
if ($dependencyFingerprint -notmatch "^[0-9a-fA-F]{64}$") {
|
||||
$sharedContractValid = $false
|
||||
} else {
|
||||
foreach ($candidateSchema in @("v1", "v2", "v3")) {
|
||||
foreach ($candidateSchema in $KnownSharedCacheSchemas) {
|
||||
$shortFingerprint = $dependencyFingerprint.Substring(0, 24).ToLowerInvariant()
|
||||
$expectedInstalledRoot = Get-FullPath -Path (Join-Path $CacheRoot "vcpkg-installed\$candidateSchema\$shortFingerprint")
|
||||
if ((Get-FullPath -Path $installedRoot).Equals($expectedInstalledRoot, [StringComparison]::OrdinalIgnoreCase)) {
|
||||
|
|
@ -1357,7 +1370,7 @@ $solutionTrees = foreach ($worktreeRoot in $worktreeRoots) {
|
|||
$packagesRoot = [string] $propertyGroup.CanarySharedVcpkgPackagesRoot
|
||||
|
||||
if (
|
||||
$schema -ne "v3" -or
|
||||
$schema -notin $KnownSharedCacheSchemas -or
|
||||
$dependencyFingerprint -notmatch "^[0-9a-fA-F]{64}$" -or
|
||||
$consumerFingerprint -notmatch "^[0-9a-fA-F]{64}$" -or
|
||||
[string]::IsNullOrWhiteSpace($configurationName)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue