diff --git a/ArmPkg/ArmPkg.dec b/ArmPkg/ArmPkg.dec index 8166fdc658..da7028bb15 100644 --- a/ArmPkg/ArmPkg.dec +++ b/ArmPkg/ArmPkg.dec @@ -98,6 +98,10 @@ gEdkiiTpmEventLogDescHobGuid = { 0x360c4a06, 0x146c, 0x11f0, { 0xb1, 0x73, 0x6b, 0xd5, 0x98, 0xff, 0x54, 0xd6 }} + ## SMMU config data + # Include/Guid/SmmuConfig.h + gSmmuConfigHobGuid = { 0xcd56ec8f, 0x75f1, 0x440a, { 0xaa, 0x48, 0x09, 0x58, 0xb1, 0x1c, 0x9a, 0xa7 } } + [Protocols.common] ## Arm System Control and Management Interface(SCMI) Base protocol ## ArmPkg/Include/Protocol/ArmScmiBaseProtocol.h diff --git a/ArmPkg/ArmPkg.dsc b/ArmPkg/ArmPkg.dsc index 810efcd7c6..6ad6a81376 100644 --- a/ArmPkg/ArmPkg.dsc +++ b/ArmPkg/ArmPkg.dsc @@ -152,6 +152,7 @@ ArmPkg/Universal/Smbios/OemMiscLibNull/OemMiscLibNull.inf ArmPkg/Drivers/MmCommunicationPei/MmCommunicationPei.inf + ArmPkg/Drivers/SmmuDxe/SmmuDxe.inf ArmPkg/Library/FmpDevicePsaFwuLib/FmpDevicePsaFwuLib.inf diff --git a/ArmPkg/Drivers/SmmuDxe/IoMmu.c b/ArmPkg/Drivers/SmmuDxe/IoMmu.c new file mode 100644 index 0000000000..44ecbfed45 --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/IoMmu.c @@ -0,0 +1,1613 @@ +/** @file IoMmu.c + + This file contains functions for the IoMmu protocol for use with the SMMU driver. + This driver provides a generic interface for mapping host memory to device memory. + Maintains a 4-level (0-3) page table for mapping virtual addresses to physical addresses. + The mapping is identity mapped. + + Copyright (c) Microsoft Corporation. + SPDX-License-Identifier: BSD-2-Clause-Patent + +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "SmmuV3.h" + +/** + IOMMU Mapping structure used to store the mapping information. + Used to pass between IoMmuMap, IoMmuUnmap and IoMmuSetAttribute. +**/ +typedef struct IOMMU_MAP_INFO { + UINTN NumberOfBytes; + UINT64 DeviceAddress; + UINT64 HostAddress; + EDKII_IOMMU_OPERATION Operation; + SMMU_INFO *SmmuInfo; + PAGE_TABLE *Root; +} IOMMU_MAP_INFO; + +/** + Decode the (PageTableRoot, Vmid) currently programmed in a Valid STAGE_2 + STE. The STE is the single source of truth for per-stream translation + state; the (Root, Vmid) pair returned here is exactly what the SMMU is + using to translate this StreamID. + + Caller must have already confirmed Ste->Bits.Valid != 0 (a not-yet-promoted + STE is a normal initial state, not an error condition). + + @param [in] Ste STE slot. + @param [out] Root Receives the stage-2 page-table root encoded in S2Ttb. + @param [out] Vmid Receives the VMID encoded in S2Vmid. + + @retval EFI_SUCCESS (Root, Vmid) decoded. + @retval EFI_INVALID_PARAMETER Any of Ste / Root / Vmid is NULL. +**/ +STATIC +EFI_STATUS +SmmuV3DecodeSteStage2 ( + IN SMMUV3_STREAM_TABLE_ENTRY *Ste, + OUT PAGE_TABLE **Root, + OUT UINT16 *Vmid + ) +{ + if ((Ste == NULL) || (Root == NULL) || (Vmid == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameter\n", __func__)); + ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); + return EFI_INVALID_PARAMETER; + } + + *Root = (PAGE_TABLE *)(UINTN)((UINT64)Ste->Bits.S2Ttb << SMMUV3_STREAM_TABLE_ENTRY_S2TTB_OFFSET); + *Vmid = Ste->Bits.S2Vmid; + return EFI_SUCCESS; +} + +/** + Allocate the next per-stream VMID for SmmuInfo. VMID 0 is reserved as + "unassigned"; the allocator hands out 1..MaxVmid (width depends on + IDR0.VMID16) and never reuses a VMID within the same boot. + + @param [in] SmmuInfo SMMU instance. + @param [out] OutVmid Receives the newly allocated VMID. + + @retval EFI_SUCCESS VMID allocated. + @retval EFI_OUT_OF_RESOURCES VMID space exhausted. +**/ +STATIC +EFI_STATUS +SmmuV3AllocateVmid ( + IN SMMU_INFO *SmmuInfo, + OUT UINT16 *OutVmid + ) +{ + UINT16 MaxVmid; + + MaxVmid = SmmuInfo->Vmid16Supported ? MAX_UINT16 : MAX_UINT8; + if (SmmuInfo->NextVmid == SMMU_VMID_RESERVED) { + // wrapped past the max + DEBUG ((DEBUG_ERROR, "%a: VMID space exhausted on SMMU 0x%llx\n", __func__, SmmuInfo->SmmuBase)); + ASSERT (SmmuInfo->NextVmid != SMMU_VMID_RESERVED); + return EFI_OUT_OF_RESOURCES; + } + + *OutVmid = SmmuInfo->NextVmid; + if (SmmuInfo->NextVmid == MaxVmid) { + SmmuInfo->NextVmid = SMMU_VMID_RESERVED; // mark exhausted; next allocation will fail above + } else { + SmmuInfo->NextVmid++; + } + + return EFI_SUCCESS; +} + +/** + Allocate the next per-stream ASID for SmmuInfo. Mirrors + SmmuV3AllocateVmid. ASID 0 is reserved as "unassigned"; the allocator + hands out 1..MaxAsid and never reuses within the same boot. + + @param [in] SmmuInfo SMMU instance. + @param [out] OutAsid Receives the newly allocated ASID. + + @retval EFI_SUCCESS ASID allocated. + @retval EFI_OUT_OF_RESOURCES ASID space exhausted. +**/ +STATIC +EFI_STATUS +SmmuV3AllocateAsid ( + IN SMMU_INFO *SmmuInfo, + OUT UINT16 *OutAsid + ) +{ + UINT16 MaxAsid; + + MaxAsid = SmmuInfo->Asid16Supported ? MAX_UINT16 : MAX_UINT8; + if (SmmuInfo->NextAsid == SMMU_ASID_RESERVED) { + DEBUG ((DEBUG_ERROR, "%a: ASID space exhausted on SMMU 0x%llx\n", __func__, SmmuInfo->SmmuBase)); + ASSERT (SmmuInfo->NextAsid != SMMU_ASID_RESERVED); + return EFI_OUT_OF_RESOURCES; + } + + *OutAsid = SmmuInfo->NextAsid; + if (SmmuInfo->NextAsid == MaxAsid) { + SmmuInfo->NextAsid = SMMU_ASID_RESERVED; + } else { + SmmuInfo->NextAsid++; + } + + return EFI_SUCCESS; +} + +/** + Decode the (CD, PageTableRoot, Asid) programmed in a Valid STAGE_1 STE. + Dereferences S1ContextPtr to reach the CD, which holds the Stage 1 + root (CD.Ttb0) and ASID (CD.Asid). Caller must have already confirmed + Ste->Bits.Valid != 0. + + @param [in] Ste STE slot. + @param [out] OutCd Receives the CD pointer. + @param [out] Root Receives the Stage 1 page-table root. + @param [out] Asid Receives the ASID. + + @retval EFI_SUCCESS Decoded. + @retval EFI_INVALID_PARAMETER Any parameter is NULL, or S1ContextPtr = 0. +**/ +STATIC +EFI_STATUS +SmmuV3DecodeSteStage1 ( + IN SMMUV3_STREAM_TABLE_ENTRY *Ste, + OUT SMMUV3_CONTEXT_DESCRIPTOR **OutCd, + OUT PAGE_TABLE **Root, + OUT UINT16 *Asid + ) +{ + SMMUV3_CONTEXT_DESCRIPTOR *Cd; + + if ((Ste == NULL) || (OutCd == NULL) || (Root == NULL) || (Asid == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameter\n", __func__)); + ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); + return EFI_INVALID_PARAMETER; + } + + Cd = (SMMUV3_CONTEXT_DESCRIPTOR *)(UINTN)((UINT64)Ste->Bits.S1ContextPtr << SMMUV3_STREAM_TABLE_ENTRY_S1CONTEXTPTR_OFFSET); + if (Cd == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Valid Stage 1 STE with S1ContextPtr == 0\n", __func__)); + ASSERT (Cd != NULL); + return EFI_INVALID_PARAMETER; + } + + *OutCd = Cd; + *Root = (PAGE_TABLE *)(UINTN)((UINT64)Cd->Bits.Ttb0 << SMMUV3_CD_TTB0_OFFSET); + *Asid = (UINT16)Cd->Bits.Asid; + return EFI_SUCCESS; +} + +/** + Ensure a stage-2 page-table root exists for the given StreamID. On first + call for a StreamID, allocates a fresh root + VMID and promotes the + corresponding STE from Invalid to a Valid STAGE_2_TRANSLATE entry using + break-before-make. Subsequent calls read the (Root, Vmid) back out of the + live STE (the single source of truth). + + @param [in] SmmuInfo SMMU instance. + @param [in] StreamId StreamID. + @param [out] OutRoot Receives the stage-2 page-table root. + @param [out] OutVmid Receives the VMID tag installed in the STE. +**/ +STATIC +EFI_STATUS +SmmuV3StreamGetOrCreateStage2 ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 StreamId, + OUT PAGE_TABLE **OutRoot, + OUT UINT16 *OutVmid + ) +{ + EFI_STATUS Status; + SMMUV3_STREAM_TABLE_ENTRY *Ste; + SMMUV3_STREAM_TABLE_ENTRY *NewL2; + PAGE_TABLE *NewRoot; + UINT16 NewVmid; + EFI_TPL OldTpl; + BOOLEAN NewL2Consumed; + + Ste = SmmuV3GetSteSlot (SmmuInfo, StreamId); + if (Ste == NULL) { + DEBUG ((DEBUG_ERROR, "%a: No STE slot for SmmuBase=0x%llx StreamId 0x%x\n", __func__, SmmuInfo->SmmuBase, StreamId)); + ASSERT (Ste != NULL); + return EFI_INVALID_PARAMETER; + } + + // Already promoted -> read the (Root, Vmid) the SMMU is actively using. + if (Ste->Bits.Valid != 0) { + Status = SmmuV3DecodeSteStage2 (Ste, OutRoot, OutVmid); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to decode STE for SmmuBase=0x%llx StreamId 0x%x\n", __func__, SmmuInfo->SmmuBase, StreamId)); + ASSERT_EFI_ERROR (Status); + } + + return Status; + } + + NewRoot = SmmuV3AllocatePageTableRoot (SmmuInfo); + if (NewRoot == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate page-table root for SmmuBase=0x%llx StreamId 0x%x\n", __func__, SmmuInfo->SmmuBase, StreamId)); + ASSERT (NewRoot != NULL); + return EFI_OUT_OF_RESOURCES; + } + + NewL2 = (SMMUV3_STREAM_TABLE_ENTRY *)AllocatePages (1); + if (NewL2 == NULL) { + SmmuV3FreePageTableTree (SmmuInfo, 0, NewRoot); + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate L2 page for SmmuBase=0x%llx StreamId 0x%x\n", __func__, SmmuInfo->SmmuBase, StreamId)); + ASSERT (NewL2 != NULL); + return EFI_OUT_OF_RESOURCES; + } + + ZeroMem (NewL2, EFI_PAGE_SIZE); + NewL2Consumed = FALSE; + + // Raise TPL to prevent concurrent updates to the SMMU STEs. + OldTpl = gBS->RaiseTPL (TPL_HIGH_LEVEL); + + Status = SmmuV3AllocateVmid (SmmuInfo, &NewVmid); + if (EFI_ERROR (Status)) { + gBS->RestoreTPL (OldTpl); + SmmuV3FreePageTableTree (SmmuInfo, 0, NewRoot); + FreePages (NewL2, 1); + ASSERT_EFI_ERROR (Status); + return Status; + } + + Status = SmmuV3PromoteSteToStage2Translate (SmmuInfo, StreamId, NewVmid, NewRoot, NewL2, &NewL2Consumed); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to promote STE for SmmuBase=0x%llx StreamId 0x%x: %r\n", __func__, SmmuInfo->SmmuBase, StreamId, Status)); + gBS->RestoreTPL (OldTpl); + SmmuV3FreePageTableTree (SmmuInfo, 0, NewRoot); + if (!NewL2Consumed) { + FreePages (NewL2, 1); + } + + ASSERT_EFI_ERROR (Status); + return Status; + } + + // Restore TPL after STE modifications are complete and the SMMU has been notified of the change. + gBS->RestoreTPL (OldTpl); + + // If SmmuV3PromoteSteToStage2Translate did not install NewL2 into an L1 + // descriptor (linear stream tables, or the covering L1 was already split), + // the page is unused and must be freed here to avoid leaking it. + if (!NewL2Consumed) { + FreePages (NewL2, 1); + } + + *OutRoot = NewRoot; + *OutVmid = NewVmid; + return EFI_SUCCESS; +} + +/** + Stage 1 counterpart of SmmuV3StreamGetOrCreateStage2. On first call for + a StreamID, allocates a CD + ASID + Stage 1 page-table root, populates + the CD and promotes the STE from Invalid to STAGE_1_TRANSLATE / + STAGE_2_BYPASS. Subsequent calls read (Root, Asid) back out of the live + STE via CD dereference. + + @param [in] SmmuInfo SMMU instance. + @param [in] StreamId StreamID. + @param [out] OutRoot Receives the Stage 1 page-table root. + @param [out] OutAsid Receives the ASID installed in the CD. +**/ +STATIC +EFI_STATUS +SmmuV3StreamGetOrCreateStage1 ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 StreamId, + OUT PAGE_TABLE **OutRoot, + OUT UINT16 *OutAsid + ) +{ + EFI_STATUS Status; + SMMUV3_STREAM_TABLE_ENTRY *Ste; + SMMUV3_STREAM_TABLE_ENTRY *NewL2; + SMMUV3_CONTEXT_DESCRIPTOR *NewCd; + SMMUV3_CONTEXT_DESCRIPTOR *ExistingCd; + PAGE_TABLE *NewRoot; + UINT16 NewAsid; + EFI_TPL OldTpl; + BOOLEAN NewL2Consumed; + + Ste = SmmuV3GetSteSlot (SmmuInfo, StreamId); + if (Ste == NULL) { + DEBUG ((DEBUG_ERROR, "%a: No STE slot for SmmuBase=0x%llx StreamId 0x%x\n", __func__, SmmuInfo->SmmuBase, StreamId)); + ASSERT (Ste != NULL); + return EFI_INVALID_PARAMETER; + } + + // Already promoted -> decode CD to recover (Root, Asid). + if (Ste->Bits.Valid != 0) { + Status = SmmuV3DecodeSteStage1 (Ste, &ExistingCd, OutRoot, OutAsid); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to decode Stage 1 STE for SmmuBase=0x%llx StreamId 0x%x\n", __func__, SmmuInfo->SmmuBase, StreamId)); + ASSERT_EFI_ERROR (Status); + } + + return Status; + } + + NewRoot = SmmuV3AllocatePageTableRoot (SmmuInfo); + if (NewRoot == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate Stage 1 page-table root for SmmuBase=0x%llx StreamId 0x%x\n", __func__, SmmuInfo->SmmuBase, StreamId)); + ASSERT (NewRoot != NULL); + return EFI_OUT_OF_RESOURCES; + } + + NewCd = SmmuV3AllocateContextDescriptor (); + if (NewCd == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate CD for SmmuBase=0x%llx StreamId 0x%x\n", __func__, SmmuInfo->SmmuBase, StreamId)); + SmmuV3FreePageTableTree (SmmuInfo, 0, NewRoot); + ASSERT (NewCd != NULL); + return EFI_OUT_OF_RESOURCES; + } + + NewL2 = (SMMUV3_STREAM_TABLE_ENTRY *)AllocatePages (1); + if (NewL2 == NULL) { + SmmuV3FreeContextDescriptor (NewCd); + SmmuV3FreePageTableTree (SmmuInfo, 0, NewRoot); + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate L2 page for SmmuBase=0x%llx StreamId 0x%x\n", __func__, SmmuInfo->SmmuBase, StreamId)); + ASSERT (NewL2 != NULL); + return EFI_OUT_OF_RESOURCES; + } + + ZeroMem (NewL2, EFI_PAGE_SIZE); + NewL2Consumed = FALSE; + + // Raise TPL to prevent concurrent updates to the SMMU STEs. + OldTpl = gBS->RaiseTPL (TPL_HIGH_LEVEL); + + Status = SmmuV3AllocateAsid (SmmuInfo, &NewAsid); + if (EFI_ERROR (Status)) { + gBS->RestoreTPL (OldTpl); + SmmuV3FreeContextDescriptor (NewCd); + SmmuV3FreePageTableTree (SmmuInfo, 0, NewRoot); + FreePages (NewL2, 1); + ASSERT_EFI_ERROR (Status); + return Status; + } + + Status = SmmuV3BuildStage1ContextDescriptor (SmmuInfo, NewRoot, NewAsid, NewCd); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to build CD for SmmuBase=0x%llx StreamId 0x%x: %r\n", __func__, SmmuInfo->SmmuBase, StreamId, Status)); + gBS->RestoreTPL (OldTpl); + SmmuV3FreeContextDescriptor (NewCd); + SmmuV3FreePageTableTree (SmmuInfo, 0, NewRoot); + FreePages (NewL2, 1); + ASSERT_EFI_ERROR (Status); + return Status; + } + + // Publish CD writes before the SMMU can observe them via STE promotion. + ArmDataSynchronizationBarrier (); + + Status = SmmuV3PromoteSteToStage1Translate (SmmuInfo, StreamId, NewCd, NewL2, &NewL2Consumed); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to promote Stage 1 STE for SmmuBase=0x%llx StreamId 0x%x: %r\n", __func__, SmmuInfo->SmmuBase, StreamId, Status)); + gBS->RestoreTPL (OldTpl); + SmmuV3FreeContextDescriptor (NewCd); + SmmuV3FreePageTableTree (SmmuInfo, 0, NewRoot); + if (!NewL2Consumed) { + FreePages (NewL2, 1); + } + + ASSERT_EFI_ERROR (Status); + return Status; + } + + gBS->RestoreTPL (OldTpl); + + if (!NewL2Consumed) { + FreePages (NewL2, 1); + } + + *OutRoot = NewRoot; + *OutAsid = NewAsid; + return EFI_SUCCESS; +} + +/** + Ensure a per-stream page-table root exists for the given StreamID and + dispatch to the stage-appropriate allocator. + + For Stage 2 SMMUs the returned tag is the VMID that identifies the + page-table root in the STE. For Stage 1 SMMUs the returned tag is the + ASID stored in the per-stream CD. In both cases the tag can be fed to + the stage-appropriate TLB invalidation helper. + + @param [in] SmmuInfo SMMU instance. + @param [in] StreamId StreamID. + @param [out] OutRoot Receives the per-stream page-table root. + @param [out] OutTagId Receives the per-stream ID tag (VMID for Stage 2, + ASID for Stage 1). + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameters. + @retval EFI_OUT_OF_RESOURCES Allocation failed / VMID/ASID space exhausted. + @retval Other STE promotion failure. +**/ +EFI_STATUS +SmmuV3StreamGetOrCreate ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 StreamId, + OUT PAGE_TABLE **OutRoot, + OUT UINT16 *OutTagId + ) +{ + if ((SmmuInfo == NULL) || (OutRoot == NULL) || (OutTagId == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameter\n", __func__)); + ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); + return EFI_INVALID_PARAMETER; + } + + if (SmmuInfo->TranslationStage == SmmuTranslationStage1) { + return SmmuV3StreamGetOrCreateStage1 (SmmuInfo, StreamId, OutRoot, OutTagId); + } + + return SmmuV3StreamGetOrCreateStage2 (SmmuInfo, StreamId, OutRoot, OutTagId); +} + +/** + Bind a StreamID's STE to share an existing primary stream's stage-2 + page-table root and VMID. Used when a single device exposes multiple + StreamIDs that must all see the same translations. + + For Stage 2 the shared state is (root, VMID) encoded in the STE. For + Stage 1 both STEs point at the same CD via S1ContextPtr. A no-op if the + alias STE already encodes the same state; a configuration error if it + is already promoted with a different primary. + + @param [in] SmmuInfo SMMU instance. + @param [in] PrimaryStreamId Primary StreamID (used to recover the + Stage 1 CD via S1ContextPtr). + @param [in] AliasStreamId StreamID that should alias the primary. + @param [in] PrimaryRoot Primary's page-table root (non-NULL). + @param [in] PrimaryTagId Primary's tag: VMID for Stage 2, ASID for + Stage 1 (non-zero). + + @retval EFI_SUCCESS Alias bound. + @retval EFI_INVALID_PARAMETER Invalid parameters. + @retval EFI_ALREADY_STARTED Alias STE already points elsewhere. + @retval Other STE-promotion failure. +**/ +STATIC +EFI_STATUS +SmmuV3StreamAlias ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 PrimaryStreamId, + IN UINT32 AliasStreamId, + IN PAGE_TABLE *PrimaryRoot, + IN UINT16 PrimaryTagId + ) +{ + EFI_STATUS Status; + SMMUV3_STREAM_TABLE_ENTRY *AliasSte; + SMMUV3_STREAM_TABLE_ENTRY *PrimarySte; + SMMUV3_STREAM_TABLE_ENTRY *NewL2; + SMMUV3_CONTEXT_DESCRIPTOR *PrimaryCd; + SMMUV3_CONTEXT_DESCRIPTOR *ExistingCd; + PAGE_TABLE *ExistingRoot; + UINT16 ExistingTagId; + EFI_TPL OldTpl; + BOOLEAN NewL2Consumed; + + if ((SmmuInfo == NULL) || (PrimaryRoot == NULL) || (PrimaryTagId == 0)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameter\n", __func__)); + ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); + return EFI_INVALID_PARAMETER; + } + + AliasSte = SmmuV3GetSteSlot (SmmuInfo, AliasStreamId); + if (AliasSte == NULL) { + DEBUG ((DEBUG_ERROR, "%a: No STE slot for SmmuBase=0x%llx StreamId 0x%x\n", __func__, SmmuInfo->SmmuBase, AliasStreamId)); + ASSERT (AliasSte != NULL); + return EFI_INVALID_PARAMETER; + } + + if (SmmuInfo->TranslationStage == SmmuTranslationStage1) { + // Stage 1: alias by pointing the alias STE at the primary's CD. + PrimarySte = SmmuV3GetSteSlot (SmmuInfo, PrimaryStreamId); + if ((PrimarySte == NULL) || (PrimarySte->Bits.Valid == 0)) { + DEBUG (( + DEBUG_ERROR, + "%a: Primary StreamId 0x%x on SmmuBase=0x%llx has no valid STE; cannot alias 0x%x to it\n", + __func__, + PrimaryStreamId, + SmmuInfo->SmmuBase, + AliasStreamId + )); + ASSERT ((PrimarySte != NULL) && (PrimarySte->Bits.Valid != 0)); + return EFI_INVALID_PARAMETER; + } + + Status = SmmuV3DecodeSteStage1 (PrimarySte, &PrimaryCd, &ExistingRoot, &ExistingTagId); + if (EFI_ERROR (Status)) { + ASSERT_EFI_ERROR (Status); + return Status; + } + + // If the alias STE is already promoted, compare CDs. + if (AliasSte->Bits.Valid != 0) { + Status = SmmuV3DecodeSteStage1 (AliasSte, &ExistingCd, &ExistingRoot, &ExistingTagId); + if (EFI_ERROR (Status)) { + ASSERT_EFI_ERROR (Status); + return Status; + } + + if (ExistingCd == PrimaryCd) { + return EFI_SUCCESS; + } + + DEBUG (( + DEBUG_ERROR, + "%a: Stage 1 alias StreamId 0x%x already has its own CD %p; cannot alias to %p\n", + __func__, + AliasStreamId, + ExistingCd, + PrimaryCd + )); + ASSERT (ExistingCd == PrimaryCd); + return EFI_ALREADY_STARTED; + } + + NewL2 = (SMMUV3_STREAM_TABLE_ENTRY *)AllocatePages (1); + if (NewL2 == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate L2 page for SmmuBase=0x%llx alias StreamId 0x%x\n", __func__, SmmuInfo->SmmuBase, AliasStreamId)); + ASSERT (NewL2 != NULL); + return EFI_OUT_OF_RESOURCES; + } + + ZeroMem (NewL2, EFI_PAGE_SIZE); + NewL2Consumed = FALSE; + + OldTpl = gBS->RaiseTPL (TPL_HIGH_LEVEL); + + Status = SmmuV3PromoteSteToStage1Translate (SmmuInfo, AliasStreamId, PrimaryCd, NewL2, &NewL2Consumed); + if (EFI_ERROR (Status)) { + DEBUG (( + DEBUG_ERROR, + "%a: Stage 1 STE promotion failed for alias StreamId 0x%x: %r\n", + __func__, + AliasStreamId, + Status + )); + ASSERT_EFI_ERROR (Status); + gBS->RestoreTPL (OldTpl); + if (!NewL2Consumed) { + FreePages (NewL2, 1); + } + + return Status; + } + + gBS->RestoreTPL (OldTpl); + + if (!NewL2Consumed) { + FreePages (NewL2, 1); + } + + DEBUG (( + DEBUG_VERBOSE, + "%a: Aliased Stage 1 StreamId 0x%x (CD=%p ASID=0x%x)\n", + __func__, + AliasStreamId, + PrimaryCd, + PrimaryTagId + )); + + return EFI_SUCCESS; + } + + // Stage 2 (default): alias by sharing S2Ttb + VMID. + // If the alias STE is already promoted, compare against the primary. + if (AliasSte->Bits.Valid != 0) { + Status = SmmuV3DecodeSteStage2 (AliasSte, &ExistingRoot, &ExistingTagId); + if (EFI_ERROR (Status)) { + ASSERT_EFI_ERROR (Status); + return Status; + } + + if ((ExistingRoot == PrimaryRoot) && (ExistingTagId == PrimaryTagId)) { + return EFI_SUCCESS; + } + + if (ExistingRoot != PrimaryRoot) { + DEBUG (( + DEBUG_ERROR, + "%a: StreamId 0x%x already has its own root %p; cannot alias to %p\n", + __func__, + AliasStreamId, + ExistingRoot, + PrimaryRoot + )); + ASSERT (ExistingRoot == PrimaryRoot); + return EFI_ALREADY_STARTED; + } + } + + NewL2 = (SMMUV3_STREAM_TABLE_ENTRY *)AllocatePages (1); + if (NewL2 == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate L2 page for SmmuBase=0x%llx alias StreamId 0x%x\n", __func__, SmmuInfo->SmmuBase, AliasStreamId)); + ASSERT (NewL2 != NULL); + return EFI_OUT_OF_RESOURCES; + } + + ZeroMem (NewL2, EFI_PAGE_SIZE); + NewL2Consumed = FALSE; + + // Raise TPL to prevent concurrent updates to the SMMU STEs. + OldTpl = gBS->RaiseTPL (TPL_HIGH_LEVEL); + + Status = SmmuV3PromoteSteToStage2Translate ( + SmmuInfo, + AliasStreamId, + PrimaryTagId, + PrimaryRoot, + NewL2, + &NewL2Consumed + ); + if (EFI_ERROR (Status)) { + DEBUG (( + DEBUG_ERROR, + "%a: STE promotion failed for SmmuBase=0x%llx alias StreamId 0x%x: %r\n", + __func__, + SmmuInfo->SmmuBase, + AliasStreamId, + Status + )); + ASSERT_EFI_ERROR (Status); + gBS->RestoreTPL (OldTpl); + if (!NewL2Consumed) { + FreePages (NewL2, 1); + } + + return Status; + } + + // Restore TPL before promoting the alias STE. + gBS->RestoreTPL (OldTpl); + + // If SmmuV3PromoteSteToStage2Translate did not install NewL2 into an L1 + // descriptor (linear stream tables, or the covering L1 was already split), + // the page is unused and must be freed here to avoid leaking it. + if (!NewL2Consumed) { + FreePages (NewL2, 1); + } + + DEBUG (( + DEBUG_VERBOSE, + "%a: Aliased StreamId 0x%x (root=%p VMID=0x%x)\n", + __func__, + AliasStreamId, + PrimaryRoot, + PrimaryTagId + )); + + return EFI_SUCCESS; +} + +/** + Update the mapping of a virtual address to a physical address in the page table. + + Iterates through the page table levels to find the leaf entry for the given virtual address and + validates entries along the way as needed. The leaf entry is then updated with the physical address along + with appropriate attributes and valid bit set. + + Break-before-make does not apply here because we are only switching between invalid/valid, + no other Entry bits are changing. If the entry is already valid, it must have the same + PA and attributes to be considered a match; otherwise it's an error because we don't expect + multiple mappings for the same VA. + + @param [in] SmmuInfo SMMU instance whose translation parameters drive the page-table walk. + @param [in] Root Pointer to the root page table. + @param [in] VirtualAddress Virtual address to map. + @param [in] PhysicalAddress Physical address to map to. + @param [in] Attributes Attributes to set for the mapping. Must be a valid Stage 2 Translation Table attribute (12 bit or less). + @param [in] Valid Boolean to indicate if the entry is valid. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameter. + @retval EFI_OUT_OF_RESOURCES Out of resources. +**/ +STATIC +EFI_STATUS +UpdateMapping ( + IN SMMU_INFO CONST *CONST SmmuInfo, + IN PAGE_TABLE *Root, + IN UINT64 VirtualAddress, + IN UINT64 PhysicalAddress, + IN UINT16 Attributes, + IN BOOLEAN Valid + ) +{ + EFI_STATUS Status; + UINT8 Level; + UINT32 Index; + PAGE_TABLE *Current; + PAGE_TABLE *NewPage; + UINT64 Entry; + EFI_TPL OldTpl; + UINT16 Stage1Ap; + PAGE_TABLE *NewPageList[PAGE_TABLE_DEPTH]; + + // Attributes must be a valid Stage 2 Translation Table attribute (12 bits or less) + if ((Root == NULL) || (SmmuInfo == NULL) || ((Attributes & ~PAGE_TABLE_BLOCK_MASK) != 0) || (PhysicalAddress == 0)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameter.\n", __func__)); + ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); + return EFI_INVALID_PARAMETER; + } + + Status = EFI_SUCCESS; + Current = Root; + ZeroMem (NewPageList, sizeof (NewPageList)); + + // Read-only pre-walk of the page table for VirtualAddress to determine + // which intermediate levels actually need a new page-table page. Allocate + // only those levels. + for (Level = SmmuInfo->TranslationStartingLevel; Level < PAGE_TABLE_DEPTH - 1; Level++) { + Index = PAGE_TABLE_INDEX (VirtualAddress, Level, SmmuInfo->OutputAddressWidth, SmmuInfo->TranslationStartingLevel, SmmuInfo->PageTableRootConcatenated); + Entry = Current->Entries[Index]; + + if (Entry == 0) { + // This level is empty; allocate a page for it. Then follow the + // freshly zeroed page in subsequent iterations so every deeper level + // also reads Entry == 0 and gets a page allocated. + NewPageList[Level] = (PAGE_TABLE *)AllocatePages (1); + if (NewPageList[Level] == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed allocating page.\n", __func__)); + // Free any pages we already preallocated at shallower levels, then bail. + while (Level > SmmuInfo->TranslationStartingLevel) { + Level--; + if (NewPageList[Level] != NULL) { + FreePages (NewPageList[Level], 1); + NewPageList[Level] = NULL; + } + } + + ASSERT_EFI_ERROR (EFI_OUT_OF_RESOURCES); + return EFI_OUT_OF_RESOURCES; + } + + ZeroMem ((VOID *)NewPageList[Level], EFI_PAGE_SIZE); + Current = NewPageList[Level]; + continue; + } + + Current = (PAGE_TABLE *)((UINTN)Entry & ~PAGE_TABLE_BLOCK_MASK); + } + + // Reset Current for the locked walk below. + Current = Root; + + // Raise TPL to prevent concurrent access to the SMMU page table. + OldTpl = gBS->RaiseTPL (TPL_HIGH_LEVEL); + + // Traverse the page table to the leaf level. + for (Level = SmmuInfo->TranslationStartingLevel; Level < PAGE_TABLE_DEPTH - 1; Level++) { + Index = PAGE_TABLE_INDEX (VirtualAddress, Level, SmmuInfo->OutputAddressWidth, SmmuInfo->TranslationStartingLevel, SmmuInfo->PageTableRootConcatenated); + NewPage = NewPageList[Level]; + + if (Current->Entries[Index] == 0) { + // This level is empty; consume this level's preallocated page. + ASSERT (NewPage != NULL); + Entry = (PAGE_TABLE_ENTRY)(UINTN)NewPage | TT_AF | PAGE_TABLE_DESCRIPTOR | PAGE_TABLE_ENTRY_VALID_BIT; + + // Issue a data synchronization barrier to ensure that the page is correctly pre-allocated and zeroed before updating the page table entry. + ArmDataSynchronizationBarrier (); + + Current->Entries[Index] = Entry; + + // Mark this level as consumed so the cleanup loop below (run after TPL is restored) + // does not free a page that is now live in the page table. + NewPageList[Level] = NULL; + } + + Current = (PAGE_TABLE *)((UINTN)Current->Entries[Index] & ~PAGE_TABLE_BLOCK_MASK); + } + + // Current should not be NULL at this point, as we have traversed the page table to the leaf level. + ASSERT (Current != 0); + + if (Current != 0) { + Index = PAGE_TABLE_INDEX (VirtualAddress, Level, SmmuInfo->OutputAddressWidth, SmmuInfo->TranslationStartingLevel, SmmuInfo->PageTableRootConcatenated); + + if (Valid) { + Entry = (PhysicalAddress & ~PAGE_TABLE_BLOCK_MASK); + // + // Stage 1 uses AttrIndx (bits [4:2], indexes MAIR in the CD) and + // AP[2:1] (bit [7:6]); Stage 2 uses MemAttr (bits [5:2]) and S2AP + // (bits [7:6]). The Attributes parameter carries the Stage 2 R/W + // encoding; for Stage 1 translate the write bit into AP[2:1]. + // + if (SmmuInfo->TranslationStage == SmmuTranslationStage1) { + Stage1Ap = ((Attributes & PAGE_TABLE_WRITE_BIT) != 0) ? TT_AP_RW_RW : TT_AP_RO_RO; + Entry |= PAGE_TABLE_S1_ATTRINDX0 | + Stage1Ap | + TT_SH_INNER_SHAREABLE | + TT_AF | + PAGE_TABLE_DESCRIPTOR | + PAGE_TABLE_ENTRY_VALID_BIT; + } else { + // validate entry and set leaf level attributes (Stage 2) + Entry |= Attributes | PAGE_TABLE_S2_MEMATTR_NORMAL_WB | TT_SH_INNER_SHAREABLE | TT_AF | PAGE_TABLE_DESCRIPTOR | PAGE_TABLE_ENTRY_VALID_BIT; + } + + // Break-before-make does not apply here because we are only switching between invalid/valid, no other Entry bits are changing. + // If the entry is already valid, it must have the same PA and attributes to be considered a match; otherwise it's an error because we don't expect multiple mappings for the same VA. + if ((Current->Entries[Index] & PAGE_TABLE_ENTRY_VALID_BIT) != 0) { + DEBUG ((DEBUG_INFO, "%a: Page already mapped with valid Entry. VirtualAddress = 0x%llx PhysicalAddress=0x%llx\n", __func__, VirtualAddress, PhysicalAddress)); + if (Current->Entries[Index] != Entry) { + DEBUG ((DEBUG_ERROR, "%a: Page already mapped with different PA or attributes. OldEntry = 0x%llx NewEntry = 0x%llx\n", __func__, Current->Entries[Index], Entry)); + Status = EFI_DEVICE_ERROR; + } + + goto End; + } + + Current->Entries[Index] = Entry; + } else { + Entry = Current->Entries[Index] & ~PAGE_TABLE_ENTRY_VALID_BIT; + Current->Entries[Index] = Entry; // only invalidate leaf entry + } + } + +End: + // Restore TPL after the page table update is complete. + gBS->RestoreTPL (OldTpl); + + // Free any preallocated pages that were not consumed during the walk + // (slot was already populated, so this level's page is unused). + for (Level = SmmuInfo->TranslationStartingLevel; Level < PAGE_TABLE_DEPTH - 1; Level++) { + if (NewPageList[Level] != NULL) { + FreePages (NewPageList[Level], 1); + } + } + + ASSERT_EFI_ERROR (Status); + return Status; +} + +/** + Update the page table mapping with the given physical address and attributes. + + @param [in] SmmuInfo SMMU instance. + @param [in] Root Pointer to the root page table. + @param [in] TagId Per-stream tag whose TLB entries + should be invalidated on unmap. + VMID for Stage 2 SMMUs, ASID for + Stage 1 SMMUs. + @param [in] PhysicalAddress Physical address to map. + @param [in] Bytes Number of bytes to map. + @param [in] Attributes Attributes to set for the mapping. Must be a valid Stage 2 Translation Table attribute (12 bits or less). + @param [in] Valid Boolean to indicate if the entry is valid. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameter. + @retval EFI_OUT_OF_RESOURCES Out of resources. +**/ +EFI_STATUS +UpdatePageTable ( + IN SMMU_INFO *SmmuInfo, + IN PAGE_TABLE *Root, + IN UINT16 TagId, + IN UINT64 PhysicalAddress, + IN UINT64 Bytes, + IN UINT16 Attributes, + IN BOOLEAN Valid + ) +{ + EFI_STATUS Status; + EFI_PHYSICAL_ADDRESS PhysicalAddressEnd; + EFI_PHYSICAL_ADDRESS CurPhysicalAddress; + + if ((Root == NULL) || (SmmuInfo == NULL) || ((Attributes & ~PAGE_TABLE_BLOCK_MASK) != 0) || (PhysicalAddress == 0) || (Bytes == 0)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameter\n", __func__)); + Status = EFI_INVALID_PARAMETER; + goto End; + } + + CurPhysicalAddress = PhysicalAddress; + PhysicalAddressEnd = ALIGN_VALUE (PhysicalAddress + Bytes, EFI_PAGE_SIZE); + + while (CurPhysicalAddress < PhysicalAddressEnd) { + Status = UpdateMapping (SmmuInfo, Root, CurPhysicalAddress, CurPhysicalAddress, Attributes, Valid); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to update page table mapping\n", __func__)); + goto End; + } + + CurPhysicalAddress += EFI_PAGE_SIZE; + } + + // Only invalidate the TLB if we are unmapping the page table entries because + // we are only swapping between valid and invalid entries, and no other bits are changing in the entry. + if (!Valid) { + if (SmmuInfo->TranslationStage == SmmuTranslationStage1) { + Status = SmmuV3TLBInvalidateAllStage1 (SmmuInfo, TagId); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to invalidate Stage 1 TLB for ASID 0x%x\n", __func__, TagId)); + goto End; + } + } else { + Status = SmmuV3TLBInvalidateAllStage2 (SmmuInfo, TagId); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to invalidate TLB for Vmid 0x%llx\n", __func__, TagId)); + goto End; + } + } + } + +End: + ASSERT_EFI_ERROR (Status); + return Status; +} + +/** + Map a host address to a device address. + Remaps and copies the DMA buffer using a bounce buffer if the buffer is not aligned to a 4KB boundary + or if the buffer is above 4GB and the device cannot handle DMA above 4GB. + + @param [in] This Pointer to the IOMMU protocol instance. + @param [in] Operation The type of IOMMU operation. + @param [in] HostAddress The host address to map. + @param [in, out] NumberOfBytes On input, the number of bytes to map. On output, the number of bytes mapped. + @param [out] DeviceAddress The resulting device address. + @param [out] Mapping A handle to the mapping. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameter. + @retval EFI_OUT_OF_RESOURCES Out of resources. +**/ +EFI_STATUS +EFIAPI +IoMmuMap ( + IN EDKII_IOMMU_PROTOCOL *This, + IN EDKII_IOMMU_OPERATION Operation, + IN VOID *HostAddress, + IN OUT UINTN *NumberOfBytes, + OUT EFI_PHYSICAL_ADDRESS *DeviceAddress, + OUT VOID **Mapping + ) +{ + EFI_STATUS Status; + IOMMU_MAP_INFO *MapInfo; + EFI_PHYSICAL_ADDRESS PhysicalAddress; + BOOLEAN NeedRemap; + EFI_PHYSICAL_ADDRESS DmaMemoryTop; + + Status = EFI_SUCCESS; + NeedRemap = FALSE; + + if ((This == NULL) || + (HostAddress == NULL) || + (NumberOfBytes == NULL) || + (*NumberOfBytes == 0) || + (DeviceAddress == NULL) || + (Mapping == NULL)) + { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameter\n", __func__)); + Status = EFI_INVALID_PARAMETER; + goto End; + } + + // Allocate and fill the IOMMU_MAP_INFO structure with mapping information + MapInfo = (IOMMU_MAP_INFO *)AllocateZeroPool (sizeof (IOMMU_MAP_INFO)); + if (MapInfo == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate IOMMU_MAP_INFO structure\n", __func__)); + Status = EFI_OUT_OF_RESOURCES; + goto End; + } + + DmaMemoryTop = MAX_UINTN; + PhysicalAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress; + + if ((Operation != EdkiiIoMmuOperationBusMasterCommonBuffer) && (Operation != EdkiiIoMmuOperationBusMasterCommonBuffer64)) { + if ((*NumberOfBytes != ALIGN_VALUE (*NumberOfBytes, SIZE_4KB)) || (PhysicalAddress != ALIGN_VALUE (PhysicalAddress, SIZE_4KB))) { + // If the buffer start and end is not aligned to a 4KB boundary, we need to remap it. + NeedRemap = TRUE; + } + + if ((((Operation != EdkiiIoMmuOperationBusMasterRead64) && + (Operation != EdkiiIoMmuOperationBusMasterWrite64))) && + ((PhysicalAddress + *NumberOfBytes) > SIZE_4GB)) + { + // + // If the root bridge or the device cannot handle performing DMA above + // 4GB but any part of the DMA transfer being mapped is above 4GB, then + // remap the DMA transfer to a buffer below 4GB. + // + NeedRemap = TRUE; + DmaMemoryTop = SIZE_4GB - 1; + } + } + + MapInfo->NumberOfBytes = *NumberOfBytes; + MapInfo->DeviceAddress = DmaMemoryTop; + MapInfo->HostAddress = PhysicalAddress; + MapInfo->Operation = Operation; + + // Bounce buffer case + if (NeedRemap) { + Status = gBS->AllocatePages ( + AllocateMaxAddress, + EfiBootServicesData, + EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes), + &MapInfo->DeviceAddress + ); + if (EFI_ERROR (Status)) { + FreePool (MapInfo); + *NumberOfBytes = 0; + DEBUG ((DEBUG_ERROR, "%a: %r\n", __func__, Status)); + ASSERT_EFI_ERROR (Status); + return Status; + } + + // + // If this is a read operation from the Bus Master's point of view, + // then copy the contents of the real buffer into the mapped buffer + // so the Bus Master can read the contents of the real buffer. + // + if ((Operation == EdkiiIoMmuOperationBusMasterRead) || (Operation == EdkiiIoMmuOperationBusMasterRead64)) { + CopyMem ((VOID *)(UINTN)MapInfo->DeviceAddress, (VOID *)(UINTN)MapInfo->HostAddress, MapInfo->NumberOfBytes); + } + } else { + MapInfo->DeviceAddress = MapInfo->HostAddress; + } + + *DeviceAddress = MapInfo->DeviceAddress; + *Mapping = MapInfo; + +End: + ASSERT_EFI_ERROR (Status); + return Status; +} + +/** + Based on the Mapping info provided, copies back the buffer if a bounce buffer was used and frees the bounce buffer. + + @param [in] This Pointer to the IOMMU protocol instance. + @param [in] Mapping The mapping to unmap. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameter. + @retval EFI_OUT_OF_RESOURCES Out of resources. + @retval EFI_TIMEOUT Timeout. +**/ +EFI_STATUS +EFIAPI +IoMmuUnmap ( + IN EDKII_IOMMU_PROTOCOL *This, + IN VOID *Mapping + ) +{ + IOMMU_MAP_INFO *MapInfo; + + if ((This == NULL) || (Mapping == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameter\n", __func__)); + ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); + return EFI_INVALID_PARAMETER; + } + + MapInfo = (IOMMU_MAP_INFO *)Mapping; + + // Bounce buffer case + if (MapInfo->DeviceAddress != MapInfo->HostAddress) { + if ((MapInfo->DeviceAddress == 0) || (MapInfo->HostAddress == 0) || (MapInfo->NumberOfBytes == 0)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid fields in MapInfo struct.\n", __func__)); + ASSERT ((MapInfo->DeviceAddress != 0) && (MapInfo->HostAddress != 0) && (MapInfo->NumberOfBytes != 0)); + return EFI_INVALID_PARAMETER; + } + + // + // If this is a write operation from the Bus Master's point of view, + // then copy the contents of the mapped buffer into the real buffer + // so the processor can read the contents of the real buffer. + // + if ((MapInfo->Operation == EdkiiIoMmuOperationBusMasterWrite) || (MapInfo->Operation == EdkiiIoMmuOperationBusMasterWrite64)) { + CopyMem ( + (VOID *)(UINTN)MapInfo->HostAddress, + (VOID *)(UINTN)MapInfo->DeviceAddress, + MapInfo->NumberOfBytes + ); + } + + // + // Free the mapped buffer and the MAP_INFO structure. + // + gBS->FreePages (MapInfo->DeviceAddress, EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes)); + } + + // Ensure that the SMMU info and root page table are valid before checking if the address range is still mapped. + ASSERT (MapInfo->SmmuInfo != NULL); + ASSERT (MapInfo->Root != NULL); + + // Ensure that the address range is no longer mapped in the SMMU page tables. + // By the time Unmap() is called, SetAttribute() with IoMmuAccess = 0 should have already been called to invalidate the mapping, so this should always be FALSE. + ASSERT (SmmuV3IsAddressRangeMapped (MapInfo->SmmuInfo, MapInfo->Root, MapInfo->DeviceAddress, EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes)) == FALSE); + + // Free the mapping structure allocated in IoMmuMap + FreePool (Mapping); + + return EFI_SUCCESS; +} + +/** + Free a buffer allocated by IoMmuAllocateBuffer. + + @param [in] This Pointer to the IOMMU protocol instance. + @param [in] Pages The number of pages to free. + @param [in] HostAddress The host address to free. + + @retval EFI_SUCCESS The requested pages were freed. + @retval EFI_INVALID_PARAMETER Memory is not a page-aligned address or Pages is invalid. + @retval EFI_NOT_FOUND The requested memory pages were not allocated with AllocatePages(). +**/ +EFI_STATUS +EFIAPI +IoMmuFreeBuffer ( + IN EDKII_IOMMU_PROTOCOL *This, + IN UINTN Pages, + IN VOID *HostAddress + ) +{ + EFI_STATUS Status; + + if ((This == NULL) || (HostAddress == NULL) || (Pages == 0)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameter\n", __func__)); + Status = EFI_INVALID_PARAMETER; + goto End; + } + + Status = gBS->FreePages ((EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress, Pages); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to free pages\n", __func__)); + goto End; + } + +End: + ASSERT_EFI_ERROR (Status); + return Status; +} + +/** + Allocate a buffer for use with the IOMMU. + + @param [in] This Pointer to the IOMMU protocol instance. + @param [in] Type The type of allocation to perform. + @param [in] MemoryType The type of memory to allocate. + @param [in] Pages The number of pages to allocate. + @param [in, out] HostAddress On input, the desired host address. On output, the allocated host address. + @param [in] Attributes The memory attributes to use for the allocation. + + @retval EFI_SUCCESS The requested pages were allocated. + @retval EFI_INVALID_PARAMETER Invalid parameter. + @retval EFI_OUT_OF_RESOURCES The pages could not be allocated. + @retval EFI_NOT_FOUND The requested pages could not be found. +**/ +EFI_STATUS +EFIAPI +IoMmuAllocateBuffer ( + IN EDKII_IOMMU_PROTOCOL *This, + IN EFI_ALLOCATE_TYPE Type, + IN EFI_MEMORY_TYPE MemoryType, + IN UINTN Pages, + IN OUT VOID **HostAddress, + IN UINT64 Attributes + ) +{ + EFI_STATUS Status; + EFI_PHYSICAL_ADDRESS PhysicalAddress; + + if ((This == NULL) || (Pages == 0) || (HostAddress == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameter\n", __func__)); + Status = EFI_INVALID_PARAMETER; + goto End; + } + + if ((Attributes & EDKII_IOMMU_ATTRIBUTE_DUAL_ADDRESS_CYCLE) == 0) { + // Limit allocations to memory below 4GB + PhysicalAddress = SIZE_4GB - 1; + Type = AllocateMaxAddress; + } + + Status = gBS->AllocatePages ( + Type, + MemoryType, + Pages, + &PhysicalAddress + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate pages\n", __func__)); + goto End; + } + + *HostAddress = (VOID *)(UINTN)PhysicalAddress; + +End: + ASSERT_EFI_ERROR (Status); + return Status; +} + +/** + Shared back-end for IoMmuSetAttribute / IoMmuSetAttributeById. + + Resolves OwningSmmuBase to an enabled SMMU instance, ensures a stage-2 + page-table root + VMID exist for PrimaryStreamId on it, optionally aliases + additional StreamIDs (every node after the head of StreamIdList) to that + root + VMID, then updates the page table with the requested mapping / + permissions. + + @param [in] OwningSmmuBase Base MMIO address of the SMMU that owns + PrimaryStreamId. Resolved to an enabled + SMMU_INFO instance internally. + @param [in] PrimaryStreamId Primary StreamID whose page-table root is + ensured / used for the mapping update. + @param [in] StreamIdList OPTIONAL. Full StreamID list whose first node + is the primary; every subsequent node is + aliased to the primary's root + VMID. Pass + NULL when there are no aliases to bind. + @param [in] MapInfo Mapping info from Map(). + @param [in] IoMmuAccess R/W access bits. + + @retval EFI_SUCCESS Success. + @retval EFI_NOT_FOUND No enabled SMMU matches OwningSmmuBase. + @retval EFI_OUT_OF_RESOURCES Out of resources. + @retval EFI_DEVICE_ERROR The IOMMU device reported an error. + @retval Other Page-table / alias setup failure. +**/ +STATIC +EFI_STATUS +IoMmuSetAttributeHelper ( + IN UINT64 OwningSmmuBase, + IN UINT32 PrimaryStreamId, + IN LIST_ENTRY *StreamIdList OPTIONAL, + IN IOMMU_MAP_INFO *MapInfo, + IN UINT64 IoMmuAccess + ) +{ + EFI_STATUS Status; + SMMU_INFO *TargetSmmu; + PAGE_TABLE *PrimaryRoot; + UINT16 PrimaryVmid; + LIST_ENTRY *Link; + UINT32 SmmuIndex; + + if ((MapInfo == NULL) || (OwningSmmuBase == 0)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameter.\n", __func__)); + ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); + return EFI_INVALID_PARAMETER; + } + + TargetSmmu = NULL; + for (SmmuIndex = 0; SmmuIndex < mIoMmu->SmmuCount; SmmuIndex++) { + if (mIoMmu->SmmuInfo[SmmuIndex].Enabled && + (mIoMmu->SmmuInfo[SmmuIndex].SmmuBase == OwningSmmuBase)) + { + TargetSmmu = &mIoMmu->SmmuInfo[SmmuIndex]; + break; + } + } + + if (TargetSmmu == NULL) { + DEBUG (( + DEBUG_ERROR, + "%a: No enabled SMMU with base 0x%llx (StreamId 0x%x)\n", + __func__, + OwningSmmuBase, + PrimaryStreamId + )); + ASSERT (TargetSmmu != NULL); + return EFI_NOT_FOUND; + } + + // Ensure / allocate root + VMID for the primary StreamID. + PrimaryRoot = NULL; + PrimaryVmid = 0; + Status = SmmuV3StreamGetOrCreate (TargetSmmu, PrimaryStreamId, &PrimaryRoot, &PrimaryVmid); + if (EFI_ERROR (Status) || (PrimaryRoot == NULL) || (PrimaryVmid == 0)) { + DEBUG (( + DEBUG_ERROR, + "%a: Failed to ensure page-table root for StreamId 0x%x on SMMU 0x%llx: %r\n", + __func__, + PrimaryStreamId, + TargetSmmu->SmmuBase, + Status + )); + if (!EFI_ERROR (Status)) { + Status = EFI_OUT_OF_RESOURCES; + } + + ASSERT_EFI_ERROR (Status); + return Status; + } + + DEBUG (( + DEBUG_VERBOSE, + "%a: SmmuBase=0x%llx PrimaryStreamId=0x%x IoMmuAccess=0x%llx HostAddress=0x%llx DeviceAddress=0x%llx Bytes=0x%llx Root=%p VMID=0x%x\n", + __func__, + TargetSmmu->SmmuBase, + PrimaryStreamId, + IoMmuAccess, + MapInfo->HostAddress, + MapInfo->DeviceAddress, + (UINT64)MapInfo->NumberOfBytes, + PrimaryRoot, + PrimaryVmid + )); + + // Bind any alias StreamIDs (every node after the head of StreamIdList) to + // the primary's root + VMID so a single page-table update below covers DMA + // from all of them. + if ((StreamIdList != NULL) && !IsListEmpty (StreamIdList)) { + for (Link = GetNextNode (StreamIdList, GetFirstNode (StreamIdList)); + !IsNull (StreamIdList, Link); + Link = GetNextNode (StreamIdList, Link)) + { + SMMU_STREAM_ID_ENTRY *AliasEntry; + + AliasEntry = BASE_CR (Link, SMMU_STREAM_ID_ENTRY, Link); + Status = SmmuV3StreamAlias ( + TargetSmmu, + PrimaryStreamId, + AliasEntry->StreamId, + PrimaryRoot, + PrimaryVmid + ); + if (EFI_ERROR (Status)) { + DEBUG (( + DEBUG_ERROR, + "%a: Failed to alias StreamId 0x%x -> primary 0x%x on SMMU 0x%llx: %r\n", + __func__, + AliasEntry->StreamId, + PrimaryStreamId, + TargetSmmu->SmmuBase, + Status + )); + ASSERT_EFI_ERROR (Status); + return Status; + } + } + } + + for (SmmuIndex = 0; SmmuIndex < mIoMmu->SmmuCount; SmmuIndex++) { + if (mIoMmu->SmmuInfo[SmmuIndex].Enabled) { + SmmuV3LogErrors (&mIoMmu->SmmuInfo[SmmuIndex]); + } + } + + Status = UpdatePageTable ( + TargetSmmu, + PrimaryRoot, + PrimaryVmid, + MapInfo->DeviceAddress, + MapInfo->NumberOfBytes, + PAGE_TABLE_READ_WRITE_FROM_IOMMU_ACCESS ((EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE)), + (IoMmuAccess != 0) + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to update page table.\n", __func__)); + } + + MapInfo->SmmuInfo = TargetSmmu; + MapInfo->Root = PrimaryRoot; + + ASSERT_EFI_ERROR (Status); + return Status; +} + +/** + Set the R/W access attributes for Mapping in the Page Table. + + @param [in] This Pointer to the IOMMU protocol instance. + @param [in] DeviceHandle The device handle to set attributes for. + @param [in] Mapping The mapping to set attributes for. + @param [in] IoMmuAccess The IOMMU access attributes for R/W. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameter. + @retval EFI_OUT_OF_RESOURCES Out of resources. +**/ +EFI_STATUS +EFIAPI +IoMmuSetAttribute ( + IN EDKII_IOMMU_PROTOCOL *This, + IN EFI_HANDLE DeviceHandle, + IN VOID *Mapping, + IN UINT64 IoMmuAccess + ) +{ + EFI_STATUS Status; + IOMMU_MAP_INFO *MapInfo; + LIST_ENTRY StreamIdList; + SMMU_STREAM_ID_ENTRY *StreamIdEntry; + UINT32 PrimaryStreamId; + UINT64 OwningSmmuBase; + + if ((This == NULL) || (Mapping == NULL) || ((IoMmuAccess & ~(EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE)) != 0)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameter\n", __func__)); + ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); + return EFI_INVALID_PARAMETER; + } + + MapInfo = (IOMMU_MAP_INFO *)Mapping; + InitializeListHead (&StreamIdList); + + // + // PCI-only lazy mapping path: + // 1. Resolve the DeviceHandle to its IORT-derived StreamID(s) *and* the + // base address of the SMMUv3 node that owns them (via the matched + // RC ID-mapping's OutputReference, or the platform NC table for + // NonDiscoverable devices). + // 2. Locate the SMMU_INFO whose SmmuBase matches. + // 3. Ensure the *primary* StreamID has a per-stream stage-2 page-table + // root on that SMMU (allocates + promotes the STE on first call). + // 4. For any additional StreamIDs reported for this device, alias them + // to share the primary's root + VMID so a single page-table update + // covers all of them. + // 5. Update that shared root once with the requested mapping / + // permissions. + // + // The current IoMmu protocol does not have the DeviceHandle on Map(), + // so all real page-table mutation happens here in SetAttribute(). + // + if ((mIortData == NULL) || (DeviceHandle == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: IORT/DeviceHandle not available; cannot resolve StreamID\n", __func__)); + Status = EFI_UNSUPPORTED; + goto End; + } + + OwningSmmuBase = 0; + Status = DeviceHandleToStreamId (mIortData, DeviceHandle, &StreamIdList, &OwningSmmuBase); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: DeviceHandleToStreamId failed: %r\n", __func__, Status)); + goto End; + } + + if (IsListEmpty (&StreamIdList)) { + DEBUG ((DEBUG_ERROR, "%a: DeviceHandleToStreamId returned no StreamIDs\n", __func__)); + Status = EFI_DEVICE_ERROR; + goto End; + } + + StreamIdEntry = BASE_CR (GetFirstNode (&StreamIdList), SMMU_STREAM_ID_ENTRY, Link); + PrimaryStreamId = StreamIdEntry->StreamId; + + if (OwningSmmuBase == 0) { + DEBUG ((DEBUG_ERROR, "%a: IORT did not name an owning SMMU for StreamId 0x%x\n", __func__, PrimaryStreamId)); + Status = EFI_NOT_FOUND; + goto End; + } + + Status = IoMmuSetAttributeHelper ( + OwningSmmuBase, + PrimaryStreamId, + &StreamIdList, + MapInfo, + IoMmuAccess + ); + +End: + SmmuStreamIdListFree (&StreamIdList); + ASSERT_EFI_ERROR (Status); + return Status; +} + +/** + Set the R/W access attributes for Mapping in the Page Table for a caller + that explicitly specifies (IommuBase, DmaId) rather than supplying an + EFI_HANDLE. + + Intended for firmware-internal DMA agents that have no UEFI DeviceHandle, so + DeviceHandleToStreamId() cannot resolve them. The caller MUST pass the + base address of the owning SMMU and the DMA identifier (StreamID on Arm SMMU) + emitted by the device. + + Only the single DmaId provided is programmed; no alias resolution is + performed. + + @param [in] This Pointer to the IOMMU protocol instance. + @param [in] IommuBase Base MMIO address of the IOMMU that owns DmaId. + For Arm this is the SmmuV3 base address. + @param [in] DmaId DMA identifier emitted by the calling agent + (StreamID on Arm SMMU, RequesterID on VT-d). + @param [in] Mapping The mapping returned from Map(). + @param [in] IoMmuAccess The IOMMU access attributes (R/W bits). + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameter. + @retval EFI_NOT_FOUND No enabled SMMU matches IommuBase. + @retval EFI_OUT_OF_RESOURCES Out of resources. + @retval EFI_DEVICE_ERROR The IOMMU device reported an error. +**/ +EFI_STATUS +EFIAPI +IoMmuSetAttributeById ( + IN EDKII_IOMMU_PROTOCOL *This, + IN UINT64 IommuBase, + IN UINT32 DmaId, + IN VOID *Mapping, + IN UINT64 IoMmuAccess + ) +{ + EFI_STATUS Status; + + if ((This == NULL) || (Mapping == NULL) || + ((IoMmuAccess & ~(EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE)) != 0)) + { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameter\n", __func__)); + ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); + return EFI_INVALID_PARAMETER; + } + + Status = IoMmuSetAttributeHelper ( + IommuBase, + DmaId, + NULL, + (IOMMU_MAP_INFO *)Mapping, + IoMmuAccess + ); + + ASSERT_EFI_ERROR (Status); + return Status; +} + +// IOMMU Protocol instance for the SMMU. +EDKII_IOMMU_PROTOCOL SmmuIoMmu = { + EDKII_IOMMU_PROTOCOL_REVISION, + IoMmuSetAttribute, + IoMmuMap, + IoMmuUnmap, + IoMmuAllocateBuffer, + IoMmuFreeBuffer, + IoMmuSetAttributeById, +}; + +/** + Installs the IOMMU Protocol on this SMMU instance. + + @retval EFI_SUCCESS All the protocol interface was installed. + @retval EFI_OUT_OF_RESOURCES There was not enough memory in pool to install all the protocols. + @retval EFI_ALREADY_STARTED A Device Path Protocol instance was passed in that is already present in + the handle database. + @retval EFI_INVALID_PARAMETER Handle is NULL. + @retval EFI_INVALID_PARAMETER Protocol is already installed on the handle specified by Handle. +**/ +EFI_STATUS +IoMmuInit ( + VOID + ) +{ + EFI_STATUS Status; + EFI_HANDLE Handle; + + Handle = NULL; + Status = gBS->InstallMultipleProtocolInterfaces ( + &Handle, + &gEdkiiIoMmuProtocolGuid, + &SmmuIoMmu, + NULL + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to install gEdkiiIoMmuProtocolGuid\n", __func__)); + ASSERT_EFI_ERROR (Status); + } + + return Status; +} diff --git a/ArmPkg/Drivers/SmmuDxe/IoMmu.h b/ArmPkg/Drivers/SmmuDxe/IoMmu.h new file mode 100644 index 0000000000..db3b9f9461 --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/IoMmu.h @@ -0,0 +1,57 @@ +/** @file IoMmu.h + + This file is the IoMmu header file for SMMU driver. + + Copyright (c) Microsoft Corporation. + SPDX-License-Identifier: BSD-2-Clause-Patent + +**/ + +#pragma once + +/** + Page Table bit definitions used by the Smmu/IoMmu for mapping. + See ARM ARM section D8.3.1 (VMSAv8-64 descriptor formats): + + + Note: shared architectural bit definitions (access flag, inner shareable, + AP[2:1], etc.) are pulled from (via + ) as TT_AF, TT_SH_INNER_SHAREABLE, TT_AP_RW_RW, + TT_AP_RO_RO. Only Stage-2 specific and IOMMU-specific helpers are + defined here. +**/ +#define PAGE_TABLE_ENTRY_VALID_BIT 0x1 +#define PAGE_TABLE_BLOCK_MASK 0xFFF +#define PAGE_TABLE_DESCRIPTOR (0x1 << 1) +#define PAGE_TABLE_READ_WRITE_FROM_IOMMU_ACCESS(IoMmuAccess) (IoMmuAccess << 6) +#define PAGE_TABLE_WRITE_BIT (0x1 << 7) +#define PAGE_TABLE_S2_MEMATTR_NORMAL_WB (0xF << 2) + +// Stage 1 VMSAv8-64 leaf descriptor AttrIndex bits (used when the SMMU +// is configured for Stage 1 translation). AttrIndex 0 selects MAIR[0] +// in the CD (Normal Inner+Outer WBWA per SmmuV3BuildStage1ContextDescriptor). +#define PAGE_TABLE_S1_ATTRINDX0 (0x0 << 2) + +typedef UINT64 PAGE_TABLE_ENTRY; + +#define PAGE_TABLE_SIZE (EFI_PAGE_SIZE / sizeof(PAGE_TABLE_ENTRY)) // Number of entries in a page table + +// Page Table Structure used by SMMU +typedef struct _PAGE_TABLE { + PAGE_TABLE_ENTRY Entries[PAGE_TABLE_SIZE]; +} PAGE_TABLE; + +/** + Installs the IOMMU Protocol on this SMMU instance. + + @retval EFI_SUCCESS All the protocol interface was installed. + @retval EFI_OUT_OF_RESOURCES There was not enough memory in pool to install all the protocols. + @retval EFI_ALREADY_STARTED A Device Path Protocol instance was passed in that is already present in + the handle database. + @retval EFI_INVALID_PARAMETER Handle is NULL. + @retval EFI_INVALID_PARAMETER Protocol is already installed on the handle specified by Handle. +**/ +EFI_STATUS +IoMmuInit ( + VOID + ); diff --git a/ArmPkg/Drivers/SmmuDxe/README.md b/ArmPkg/Drivers/SmmuDxe/README.md new file mode 100644 index 0000000000..64140cbefe --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/README.md @@ -0,0 +1,838 @@ +# SmmuDxe Driver + +This document describes the System Memory Management Unit (SMMU) driver implementation, and how it integrates with the +DMA capable subsystem. The driver configures the SMMUv3 hardware and implements the IOMMU protocol to provide address +translation and memory protection for DMA operations. + +## Architecture Overview + +The SmmuDxe driver will consume the SMMU_CONFIG HOB with the IORT data to configure the SMMUs found on the platform. +It will set them up for Stage 2 Translation by default, or Stage 1 Translation when the platform explicitly opts in +via `SMMU_CONFIG->TranslationStage` (see [Translation Stage](#translation-stage)). SmmuDxe will install the IoMmu +Protocol. Translation table mapping can be done by leveraging the IoMmu Protocol. The protocol functions are outlined +below. Seperatley, an IoMmuLib is provided for platforms to use as an abstraction for locating and using the IoMmu +protocol. SmmuDxe will install the IORT ACPI Table. Platform should not install the IORT, but instead pass in the +IORT data with the SMMU_CONFIG HOB. + +The system consists of four main components working together: + +1. **PCI I/O Protocol**: Provides interface for PCI / NonDiscoverable device access and DMA operations (handle-aware path) +2. **DmaLib (`CoherentDmaLib` / `NonCoherentDmaLib`)**: Handle-less DMA path for firmware-internal agents; forwards + caller-supplied `(IommuBase, DmaId)` to `IoMmuSetAttributeById` +3. **IOMMU Protocol**: Implements DMA remapping and memory protection (`SetAttribute` and `SetAttributeById` entry points) +4. **SMMU Hardware Driver**: Configures and manages the SMMU hardware + +## IOMMU Protocol Integration + +1. **PCI / NonDiscoverable Driver Initiates DMA** (handle-aware path): + - Device driver calls `PciIo->Map()` / `PciIo->Unmap()` + - SmmuDxe resolves the controller `EFI_HANDLE` → IORT → StreamID(s) + owning SMMU and programs the page table + via `IoMmuSetAttribute`. + +2. **Handle-less DMA Agent via DmaLib** (handle-less path): + - Firmware-internal agents (no `EFI_HANDLE`, not in the IORT) call `DmaMap()` / `DmaUnmap()` from + `EmbeddedPkg`'s `CoherentDmaLib` / `NonCoherentDmaLib`: + + ```c + EFI_STATUS + EFIAPI + DmaMap ( + IN DMA_MAP_OPERATION Operation, + IN VOID *HostAddress, + IN OUT UINTN *NumberOfBytes, + IN UINT64 IommuBase, // SmmuV3 base address + IN UINT32 DmaId, // StreamID on Arm SMMU + OUT PHYSICAL_ADDRESS *DeviceAddress, + OUT VOID **Mapping + ); + ``` + + - DmaLib calls `IoMmuMap` for bookkeeping, then forwards `(IommuBase, DmaId)` to `IoMmuSetAttributeById`. + +3. **IOMMU Protocol Setup**: + - Implements the IOMMU protocol: + + ```c + struct _EDKII_IOMMU_PROTOCOL { + UINT64 Revision; + EDKII_IOMMU_SET_ATTRIBUTE SetAttribute; + EDKII_IOMMU_MAP Map; + EDKII_IOMMU_UNMAP Unmap; + EDKII_IOMMU_ALLOCATE_BUFFER AllocateBuffer; + EDKII_IOMMU_FREE_BUFFER FreeBuffer; + EDKII_IOMMU_SET_ATTRIBUTE_BY_ID SetAttributeById; + }; + ``` + + - SmmuDxe will handle Translation Table initialization + +## DMA Mapping with IoMmuLib and IoMmu Protocol + +- Maintains up to a 4-level page table, depending on configuration, to map HostAddress and DeviceAddress +- Identity Mapped + +1. **IoMmu Map**: + + ```c + EFI_STATUS + EFIAPI + IoMmuMap ( + IN EDKII_IOMMU_OPERATION Operation, + IN VOID *HostAddress, + IN OUT UINTN *NumberOfBytes, + OUT EFI_PHYSICAL_ADDRESS *DeviceAddress, + OUT VOID **Mapping + ); + ``` + +- Maps HostAddress to DeviceAddress +- Validates operation type +- Called by PciIo protocol for mapping + +### Bounce Buffering + +In certain conditions, `IoMmuMap` will allocate a bounce buffer instead of using the original host address directly. +A bounce buffer is a temporary intermediate buffer that is used when the original DMA buffer +cannot be used directly by the device. + +**Bounce Buffer Conditions:** + +A bounce buffer is allocated when the operation is NOT `EdkiiIoMmuOperationBusMasterCommonBuffer` or +`EdkiiIoMmuOperationBusMasterCommonBuffer64`, AND any of the following conditions are met: + +1. **Alignment Requirements Not Met:** + - Alignment check for the start and end of the DMA buffer are needed so that we don't over-map + a buffer in the page table. Page table mappings can only be done page-by-page. + - The `NumberOfBytes` is not 4KB aligned, OR + - The `HostAddress` (PhysicalAddress) is not 4KB aligned + +2. **32-bit DMA Limitation:** + - The operation is a 32-bit DMA operation (`EdkiiIoMmuOperationBusMasterRead` or + `EdkiiIoMmuOperationBusMasterWrite`), AND + - Any part of the DMA transfer range (`PhysicalAddress + NumberOfBytes`) exceeds 4GB + +**Bounce Buffer Behavior:** + +- When a bounce buffer is needed due to **alignment issues only** (with 64-bit operations), memory can be allocated + anywhere in the address space +- When a bounce buffer is needed due to the **32-bit DMA limitation**, memory is allocated below 4GB using + `AllocateMaxAddress` with `DmaMemoryTop` set to `SIZE_4GB - 1` +- The `DeviceAddress` returned points to the bounce buffer, not the original host address +- The `Mapping` handle stores information about both the original host address and the bounce buffer address + +**CopyMem on Map (Host → Bounce Buffer):** + +A `CopyMem` from the host buffer to the bounce buffer is performed during `IoMmuMap` when: + +- A bounce buffer was allocated (NeedRemap is TRUE), AND +- The operation is a **read operation** from the Bus Master's perspective: + - `EdkiiIoMmuOperationBusMasterRead`, OR + - `EdkiiIoMmuOperationBusMasterRead64` + +This copy ensures the Bus Master can read the correct data from the bounce buffer during the DMA operation. +For write operations, no copy is needed on Map since the Bus Master will write new data into the bounce buffer. + +## DMA Unmapping with IoMmuLib and IoMmu Protocol + +1. **PCI Driver Completes DMA**: + - Calls PciIo->Unmap() + - Provides mapping handle + +2. **IoMmu Unmap**: + + ```c + EFI_STATUS + EFIAPI + IoMmuUnmap ( + IN VOID *Mapping + ); + ``` + + - Only does bounce buffer cleanup. + - Caller must call SetAttributes(0) before IoMmuUnmap to remove R|W attributes to effectivley unmap the page table. + +### Bounce Buffer Handling on Unmap + +When unmapping a DMA operation that used a bounce buffer (i.e., `DeviceAddress != HostAddress`): + +**CopyMem on Unmap (Bounce Buffer → Host):** + +A `CopyMem` from the bounce buffer back to the host buffer is performed during `IoMmuUnmap` when: + +- A bounce buffer was used (`DeviceAddress != HostAddress`), AND +- The operation is a **write operation** from the Bus Master's perspective: + - `EdkiiIoMmuOperationBusMasterWrite`, OR + - `EdkiiIoMmuOperationBusMasterWrite64` + +This copy ensures the processor can access the data that the Bus Master wrote into the bounce buffer. +For read operations, no copy is needed on Unmap since the Bus Master only read data and did not modify it. + +**Cleanup:** + +- The bounce buffer pages are freed using `FreePages` +- The mapping information structure is freed + +For direct mappings (no bounce buffer), only the mapping information structure is freed. + +### DMA Access Attributes with IoMmuLib and IoMmu Protocol + +1. Setting R/W permissions + - After mapping an address with IoMmuMap() + - Clearning R/W permissions before unmmapping an address with IoMmuUnmap() + - Sets access permissions based on IoMmuAccess type: + - When IoMmuAccess is not 0, R|W permissions are set. + - When IoMmuAccess is 0, no read or write access is permitted. + +2. IoMmu SetAttribute + + ```c + EFI_STATUS + EFIAPI + IoMmuSetAttribute ( + IN EFI_HANDLE DeviceHandle, + IN VOID *Mapping, + IN UINT64 IoMmuAccess + ); + ``` + + `SetAttribute` is the **worker** of the IOMMU protocol - `IoMmuMap` only allocates the `IOMMU_MAP_INFO` + bookkeeping (and a bounce buffer if needed); it does **not** touch the SMMU page tables. All real page-table + mutation happens here, on the `SetAttribute` call that follows `Map` (and on the `SetAttribute(..., 0)` that + precedes `Unmap`). + + The `EFI_HANDLE DeviceHandle` parameter is how SmmuDxe knows *which* device (and therefore which StreamID(s) and + which SMMU) to program: + + 1. `gBS->HandleProtocol (DeviceHandle, &gEfiPciIoProtocolGuid, ...)` retrieves the `EFI_PCI_IO_PROTOCOL` instance + installed on the handle. Both real PCIe devices (installed by `PciBusDxe`) and non-discoverable devices + (installed by `NonDiscoverablePciDeviceDxe`) expose `PciIo` on the same handle the driver bound to. + 2. `PciIo->GetLocation (&Seg, &Bus, &Dev, &Func)` returns the BDF - for non-discoverable devices this is the + synthesized `(0xFF, UniqueId>>5, UniqueId&0x1F, 0)`. + 3. `DeviceHandleToStreamId` uses `Seg` to dispatch: real PCIe goes through the IORT Root-Complex ID-mapping path, + Segment `0xFF` goes through the NonDiscoverable lookup table + IORT Named Component path + (see [Per-StreamID Isolation](#per-streamid-isolation)). + 4. The resolved primary StreamID's per-stream page-table root and tag (VMID on Stage 2, or ASID inside a + Context Descriptor on Stage 1) are then used to actually install / invalidate the mapping for + `MapInfo->DeviceAddress` with the requested `IoMmuAccess` permissions (or to tear it down when + `IoMmuAccess == 0`). + + This is also why the controller handle had to be plumbed through `NonDiscoverablePciDeviceDxe` - its + `PciIoMap`/`PciIoUnmap` now pass `Dev->Handle` to `IoMmuSetAttribute` instead of the old `NULL`, so SmmuDxe can do + the resolution above for non-discoverable devices the same way it does for real PCIe. + +3. IoMmu SetAttributeById + + ```c + EFI_STATUS + EFIAPI + IoMmuSetAttributeById ( + IN EDKII_IOMMU_PROTOCOL *This, + IN UINT64 IommuBase, + IN UINT32 DmaId, + IN VOID *Mapping, + IN UINT64 IoMmuAccess + ); + ``` + + `SetAttributeById` is for callers that **do not** have an `EFI_HANDLE` for the DMA agent - for example, firmware + internal MMIO/DMA agents that are not described in the IORT and have no UEFI device handle. The caller is + responsible for supplying the owning SMMU/IOMMU base address (`IommuBase`) and the `DmaId` (StreamID on Arm SMMU, + RequesterID on VT-d) emitted by the device. Only that single `DmaId` is programmed; no IORT lookup or alias + resolution is performed. + + The protocol now exposes both entry points: + + ```c + struct _EDKII_IOMMU_PROTOCOL { + UINT64 Revision; // EDKII_IOMMU_PROTOCOL_REVISION == 0x00010001 + EDKII_IOMMU_SET_ATTRIBUTE SetAttribute; + EDKII_IOMMU_MAP Map; + EDKII_IOMMU_UNMAP Unmap; + EDKII_IOMMU_ALLOCATE_BUFFER AllocateBuffer; + EDKII_IOMMU_FREE_BUFFER FreeBuffer; + EDKII_IOMMU_SET_ATTRIBUTE_BY_ID SetAttributeById; // Optional; callers must NULL-check. + }; + ``` + + `EDKII_IOMMU_PROTOCOL_REVISION` was bumped to `0x00010001` to advertise `SetAttributeById`. Legacy producers that do + not implement `SetAttributeById` leave the field `NULL`; callers (and `IoMmuLib::IoMmuSetAttributeById`) must check + before invoking. + +## Per-StreamID Isolation + +Each DMA-capable device gets its own per-stream page-table root and per-stream tag. For Stage 2 the tag is a +**VMID** encoded directly in the STE; for Stage 1 the tag is an **ASID** stored in a per-stream Context Descriptor +(CD) whose address the STE's `S1ContextPtr` field points at. Mappings made for one device are **not** visible to +any other device. Conceptually, on every `IoMmuSetAttribute` / `IoMmuSetAttributeById` call the driver: + +1. Resolves the caller (device handle, or a caller-supplied `(IommuBase, DmaId)` for the handle-less path) to: + - One or more StreamIDs. + - The base address of the SMMU node that owns them. + + For handle-aware callers this resolution walks the platform IORT (and the optional NonDiscoverable lookup table - + see [Non-Discoverable Device Integration](#non-discoverable-device-integration)). +2. Selects the matching SMMU instance by that base address. +3. For the **primary** (first) StreamID, lazily allocates a per-stream page-table root plus: + - **Stage 2:** a VMID, then promotes the Stream Table Entry (STE) from `INVALID` to `STAGE_2_TRANSLATE` + (break-before-make). + - **Stage 1:** a Context Descriptor plus an ASID (the CD's `Ttb0` points at the root, `Asid` holds the tag), + then promotes the STE from `INVALID` to `STAGE_1_TRANSLATE / STAGE_2_BYPASS` with `S1ContextPtr` pointing at + the CD. +4. Aliases every additional StreamID reported for the same device onto the primary's per-stream state: + - **Stage 2:** the alias STE is promoted to carry the same `S2Ttb + S2VMID` as the primary. + - **Stage 1:** the alias STE is promoted to point at the **same CD** as the primary via `S1ContextPtr`, so all + aliases share the same ASID + Stage 1 root. + + A single page-table update therefore covers DMA from all StreamIDs that belong to one logical device. +5. Updates the page table with the requested permissions (or tears the mapping down when permissions are cleared). + +### Page table root and per-stream tag + +Per-stream page tables on the SMMU are tagged by VMID (Stage 2) or ASID (Stage 1). The SMMU caches TLB entries +keyed by that tag, and the STE (Stage 2) or CD (Stage 1) carries the tag field that selects which page tables the +SMMU walks for that stream. Two streams with two different page-table roots **must** have two different tags. + +Each page-table root therefore needs its own tag. Each per-stream context gets a unique tag unless the driver +explicitly wants to share that root with another stream (the alias case below): + +- **Stage 2**: The STE itself is the single source of truth for the `(Root, Vmid)` binding. On the first mapping + for a StreamID a fresh VMID and root are allocated and the STE is promoted; on subsequent mappings the existing + binding is read back from the live STE's `S2Ttb` / `S2Vmid` fields. VMID `0` is reserved as "unassigned". +- **Stage 1**: The STE + CD together are the source of truth. `S1ContextPtr` locates the CD; the CD's `Ttb0` / + `Asid` fields hold the root pointer and the ASID tag. ASID `0` is reserved as "unassigned". Per SMMUv3 §5.2, + when only Stage 1 is enabled the STE's `S2VMID` field is ignored and Stage 1 TLB entries are tagged with + VMID = 0; TLB invalidation uses `CMD_TLBI_NH_ASID` with VMID = 0. +- Tag width comes from the SMMU's `IDR0.VMID16` (Stage 2) or `IDR0.ASID16` (Stage 1) capability: 8-bit SMMUs use + values `1..0xFF`, 16-bit SMMUs use `1..0xFFFF`. A tag is never reused for a different root within the same boot. +- Aliasing does the inverse: given the **primary's** state, the alias STE is promoted in place to reference the + same state (same `S2Ttb + S2VMID` on Stage 2, same `S1ContextPtr` on Stage 1). If the alias STE already encodes + the same state this is a no-op. That's how multiple StreamIDs share one mapping - they all resolve through the + same tag, so a single TLB invalidation covers every alias. +- On unmap the driver invalidates by VMID (Stage 2) or ASID (Stage 1). Because the tag is unique per page-table + root, that invalidation can't accidentally evict another device's translations. + +### End-to-end Flow Diagram + +#### Driver Initialization + +```mermaid +flowchart TD + Entry[Driver entry point] --> Hob[Read SMMU_CONFIG HOB
gSmmuConfigHobGuid] + Hob --> Ver[Validate config structure version] + Ver --> Loc[Locate ACPI Table + GIC interrupt protocols] + Loc --> Evt[Register ExitBootServices callback] + Evt --> Cfg[Allocate IOMMU state] + Cfg --> Parse[Parse IORT
discover SMMU nodes] + Parse --> SaveIort[Save IORT pointer for
runtime StreamID resolution] + SaveIort --> NcTbl{NonDiscoverable
lookup table
present in HOB?} + NcTbl -- Yes --> NcOn[Save NC device table pointer] + NcTbl -- No --> NcOff[NC StreamID resolution disabled] + NcOn --> Inst + NcOff --> Inst[Install IORT as an ACPI table] + Inst --> Dis[Apply SmmuDisabledList
mark each SMMU enabled/disabled] + Dis --> EnLoop["For each enabled SMMU:
Program SMMU for Stage-2 (default)
or Stage-1 Translation"] + EnLoop --> DisLoop[For each disabled SMMU:
disable translation, set global bypass] + DisLoop --> Install[Install IOMMU protocol] + Install --> Ready([SMMU ready;
STEs sit in INVALID until promoted
by the first mapping call]) +``` + +After init the SMMU hardware is configured but **no device is mapped yet**. Every STE is in `INVALID` until +promoted, so any unsolicited DMA is dropped. STEs are promoted (`STAGE_2_TRANSLATE` by default, or +`STAGE_1_TRANSLATE / STAGE_2_BYPASS` when the platform opted into Stage 1) lazily on the first +`IoMmuSetAttribute` / `IoMmuSetAttributeById` call for that StreamID. + +#### Runtime Map / SetAttribute / Unmap + +```mermaid +flowchart TD + Drv["PCI / NonDiscoverable
device driver"] -->|PciIo->Map| Map[IoMmuMap
bookkeeping only] + DmaDrv["Handle-less DMA agent
(no EFI_HANDLE)"] -->|DmaMap IommuBase, DmaId| Map + Map -->|allocate mapping info
+ optional bounce buffer| MapDone[(Begin stream configuration
for DMA isolation)] + MapDone -->|PciIo path| SA[IoMmuSetAttribute
DeviceHandle, Mapping, IoMmuAccess] + MapDone -->|DmaLib path| SAX[IoMmuSetAttributeById] + + SA --> H[Retrieve PciIo on DeviceHandle] + H --> GL["Query device location
Seg/Bus/Dev/Func"] + GL -->|Seg != 0xFF| RC[IORT Root Complex
RID -> ID mapping -> StreamID + SMMU base] + GL -->|Seg == 0xFF| NC["Reconstruct UniqueId
from synthesized BDF"] + NC --> Tbl[NC lookup table
UniqueId -> ObjectName] + Tbl --> Nc2[IORT Named Component node
ID mappings -> StreamIDs + SMMU base] + + RC --> Sel[Find SMMU matching resolved base] + Nc2 --> Sel + SAX -->|caller-supplied SMMU base + StreamId| Sel + + Sel --> Pri{Primary StreamID
context exists?} + Pri -- No --> Alloc[Allocate per-stream root
+ assign VMID/ASID
+ CD on Stage 1] + Alloc --> Promote["Promote STE
INVALID -> STAGE_2_TRANSLATE or STAGE_1_TRANSLATE
break-before-make"] + Promote --> Aliases + Pri -- Yes --> Aliases[For each alias StreamID:
bind STE to primary's root + tag
Stage 2: shared S2Ttb + VMID
Stage 1: shared CD via S1ContextPtr
SetAttributeById skips this step] + + Aliases --> PT{IoMmuAccess != 0?} + PT -- Yes --> Upd[Update page table
identity-map DeviceAddress
set R/W flags] + PT -- No --> Inv["Invalidate page-table entry
+ TLB invalidate per-VMID (Stage 2)
or per-ASID (Stage 1)"] + + Upd --> Done([Return]) + Inv --> Done + + Done --> Unmap[IoMmuUnmap
preceded by SetAttribute / SetAttributeById with IoMmuAccess=0] + Unmap --> Free["Free bounce buffer if any
Free mapping info
STE/page-table stay until reboot"] +``` + +The diagram shows three caller paths that converge on the same per-StreamID isolation logic: + +- **Map only does bookkeeping.** It allocates the mapping info (and a bounce buffer if needed) and returns a + handle. No STE or page-table is touched. The `DmaLib` wrappers (`CoherentDmaLib` / `NonCoherentDmaLib`) call + straight into `IoMmuMap` and then forward the caller-supplied `(IommuBase, DmaId)` to `IoMmuSetAttributeById` + instead of using a handle. +- **SetAttribute is the worker for handle-aware callers.** It resolves the handle to StreamID(s) and SMMU base via + the IORT, lazily creates a per-stream page-table root and tag (VMID for Stage 2, ASID + CD for Stage 1) and + promotes the STE on first use, aliases additional StreamIDs onto that same state, then either installs + (`IoMmuAccess != 0`) or tears down (`IoMmuAccess == 0`) the identity mapping for the DMA address. +- **SetAttributeById is the handle-less variant** used by `DmaLib` (and any other firmware-internal DMA agent that + isn't described in the IORT). It skips both the handle-to-location lookup and the alias-binding loop - only the + single `(IommuBase, DmaId)` the caller provided is promoted and programmed. +- **Unmap** is purely cleanup of the mapping info and any bounce buffer. The STE and page tables stay promoted for + the lifetime of UEFI; on `ExitBootServices` the SMMU is flipped to global bypass + BME-off so the OS can take + over cleanly. + +## Non-Discoverable Device Integration + +Non-PCI MMIO devices registered through `MdeModulePkg/Library/NonDiscoverableDeviceRegistrationLib` are bound by +`NonDiscoverablePciDeviceDxe`, which synthesizes a `PciIo` instance over them. The driver no longer auto-assigns a +counter-based BDF; instead the platform supplies a deterministic `UniqueId` at registration time: + +```c +EFI_STATUS +EFIAPI +RegisterNonDiscoverableMmioDevice ( + IN UINTN UniqueId, // platform-assigned, must be unique + IN NON_DISCOVERABLE_DEVICE_TYPE Type, + IN NON_DISCOVERABLE_DEVICE_DMA_TYPE DmaType, + IN NON_DISCOVERABLE_DEVICE_INIT InitFunc, + IN OUT EFI_HANDLE *Handle OPTIONAL, + IN UINTN NumMmioResources, + ... + ); +``` + +The `UniqueId` is stored on the `NON_DISCOVERABLE_DEVICE` protocol and copied into the `NON_DISCOVERABLE_PCI_DEVICE` +instance by `NonDiscoverablePciDeviceDxe`. `PciIo->GetLocation()` then returns a deterministic +`(Segment = 0xFF, Bus = UniqueId >> 5, Device = UniqueId & 0x1F, Function = 0)`. SmmuDxe uses Segment `0xFF` to +detect a non-discoverable device and switch from the PCI Root-Complex lookup path to the Named Component lookup +path. + +`NonDiscoverablePciDeviceDxe` also remembers the controller `EFI_HANDLE` on the `NON_DISCOVERABLE_PCI_DEVICE` and +passes it (instead of `NULL`) to `IoMmuSetAttribute` from its `PciIoMap` / `PciIoUnmap` implementations, so SmmuDxe +can resolve the DMA identifier through the IORT. + +### Mapping a Non-Discoverable Device to an IORT Named Component + +To enable SMMU translation for a non-discoverable device, the platform must: + +1. Add a **Named Component** node to the IORT whose `ObjectName` matches the device (e.g. `"USB"`) and whose + ID mappings produce the StreamIDs the device emits. The Named Component's `OutputReference` must point at the + SMMUv3 node that owns those StreamIDs. +2. Publish a **NonDiscoverable lookup table** in the `SMMU_CONFIG` HOB that maps each registered `UniqueId` to the + matching IORT NC `ObjectName`: + + ```c + typedef struct _SMMU_NC_DEVICE_ENTRY { + UINT64 UniqueId; // Same UniqueId passed to RegisterNonDiscoverableMmioDevice(). + CHAR8 ObjName[SMMU_NC_DEVICE_OBJNAME_MAX]; // IORT Named Component ObjectName (NUL-terminated). + } SMMU_NC_DEVICE_ENTRY; + ``` + + Example - one xHCI controller registered with `UniqueId == 1` whose IORT Named Component node is named `"USB"`, + plus an SATA AHCI controller registered with `UniqueId == 2`: + + ```c + STATIC CONST SMMU_NC_DEVICE_ENTRY NcDeviceTable[] = { + // { UniqueId, IORT NC ObjectName } + { 0x1, "USB" }, + { 0x2, "SATA_AHCI" }, + }; + ``` + + Each entry's `UniqueId` is exactly the value the platform passed to `RegisterNonDiscoverableMmioDevice`, and the + `ObjName` must character-for-character match the `ObjectName` field of an `EFI_ACPI_IORT_TYPE_NAMED_COMP` node in + the IORT blob. SmmuDxe uses this two-step lookup (UniqueId → ObjectName → IORT NC node → StreamID list + + owning SMMU base) every time a non-discoverable device calls `IoMmuSetAttribute`. + +3. Point the new `NcDeviceListSize`/`NcDeviceListOffset` fields in `SMMU_CONFIG` at that table (see + [SMMU_CONFIG HOB Layout](#smmu_config-hob-layout)). + +At translation time, when `PciIo->GetLocation()` returns Segment `0xFF`, SmmuDxe reconstructs the original +`UniqueId` from `(Bus << 5) | (Device & 0x1F)`, looks it up in the NC table to get an `ObjectName`, walks the IORT +for a Named Component node with that name, and **uses every StreamID in that node's ID mappings** - not just one. + +This is where the alias logic from [Per-StreamID Isolation](#per-streamid-isolation) begins: the first StreamID +returned becomes the **primary** and gets its own per-stream page-table root and tag (a VMID on Stage 2, or an +ASID inside a Context Descriptor on Stage 1). Every other StreamID emitted by the same IORT NC node is then +**aliased** so its STE points at the *same* root and uses the *same* tag - on Stage 2 the alias STE carries the +same `S2Ttb + S2VMID` as the primary, on Stage 1 the alias STE's `S1ContextPtr` points at the primary's CD, so +all aliases share the same ASID + Stage 1 root. The device therefore sees one unified mapping no matter which of +its StreamIDs the transaction was tagged with. + +```text +NC HOB IORT Named Component SMMU per-stream state ++-----------+ match +---------------------+ ID maps +-------------------------------------------+ +| UniqueId | --------> | ObjectName = "USB" | ----------> | Primary SID -> Root R, tag T +| ObjName | | - StreamID 0x10 | | Alias SID -> Root R, tag T (shared) ++-----------+ | - StreamID 0x11 | | Alias SID -> Root R, tag T (shared) + +---------------------+ +-------------------------------------------+ + tag = VMID (Stage 2, in STE) + = ASID (Stage 1, in shared CD via S1ContextPtr) +``` + +Adding or removing alias StreamIDs is therefore purely an IORT edit - the platform NC table stays small and only +needs one entry per logical device. + +## SMMU_CONFIG HOB Layout + +The platform builds one HOB (`gSmmuConfigHobGuid`) containing the `SMMU_CONFIG` header followed (in any order) by the +IORT blob, the optional SMMU disable list, and the optional NonDiscoverable device lookup table. Each section is +located via its offset and size in the header: + +```c +typedef struct _SMMU_CONFIG { + UINT32 VersionMajor; + UINT32 VersionMinor; + UINT32 SmmuDisabledListSize; // Size of SmmuDisabledList in bytes. + UINT32 SmmuDisabledListOffset; // Offset to the SmmuDisabledList from the start of the HOB. + UINT32 IortSize; + UINT32 IortOffset; // Offset to the IORT table from the start of the HOB. + UINT32 NcDeviceListSize; // Size of the NonDiscoverable lookup array, in bytes. + UINT32 NcDeviceListOffset; // Offset to the NonDiscoverable lookup array. 0 if absent. + SMMU_TRANSLATION_STAGE TranslationStage; // SmmuTranslationStage2 (default) or SmmuTranslationStage1. + // Applies to every SMMU described by this HOB. +} SMMU_CONFIG; +``` + +`SMMU_TRANSLATION_STAGE` is a shared enum defined in ``: + +```c +typedef enum _SMMU_TRANSLATION_STAGE { + SmmuTranslationStage2 = 0, // Stage 2 translate, Stage 1 bypass (default) + SmmuTranslationStage1 = 1, // Stage 1 translate, Stage 2 bypass +} SMMU_TRANSLATION_STAGE; +``` + +Any value other than `SmmuTranslationStage1` is coerced to `SmmuTranslationStage2` by SmmuDxe. + +Visual layout (one possible ordering): + +```text ++----------------------------+ +| SMMU_CONFIG | header ++----------------------------+ <-- IortOffset +| IORT table data | IortSize bytes ++----------------------------+ <-- NcDeviceListOffset +| SMMU_NC_DEVICE_ENTRY[ N ] | NcDeviceListSize bytes (multiple of sizeof(SMMU_NC_DEVICE_ENTRY)) ++----------------------------+ <-- SmmuDisabledListOffset (optional) +| UINT64 disabledBases[] | SmmuDisabledListSize bytes ++----------------------------+ +``` + +## SMMU Configuration + +### 1. SMMUv3 Hardware Setup + +The SMMU is configured in **Stage 2 translation** mode by default, or in **Stage 1 translation** mode when the +platform sets `SMMU_CONFIG->TranslationStage = SmmuTranslationStage1`. Every SMMU described by the HOB is +configured for the same stage (see [Translation Stage](#translation-stage)). Both modes bring up: + +- Stream table for device ID mapping +- Command queue for SMMU operations, like TLB management +- Event queue for error handling +- 4KB translation granule + +Stage 1 additionally allocates a per-stream Context Descriptor (CD) whose `S1ContextPtr` is installed in the STE +so the SMMU can locate the CD (and through it, the Stage 1 root and ASID) when translating that stream. + +### Translation Stage + +- **Stage 2 (default, `SmmuTranslationStage2`)**: The SMMU installs `STAGE_2_TRANSLATE / STAGE_1_BYPASS` STEs. + Requires `IDR0.S2p`. Each StreamID's page-table root is encoded directly in the STE (`S2Ttb`, `S2VMID`). +- **Stage 1 (`SmmuTranslationStage1`)**: The SMMU installs `STAGE_1_TRANSLATE / STAGE_2_BYPASS` STEs. Requires + `IDR0.S1p`. Each StreamID's page-table root and ASID live in a Context Descriptor (64-byte, page-aligned); the + STE's `S1ContextPtr` points at the CD. Stage 1 TLB entries are tagged with VMID = 0 (SMMUv3 §5.2) and + invalidation uses `CMD_TLBI_NH_ASID`. Concatenation at the starting level is not architecturally allowed for + Stage 1, so the walker uses a single-page root and dynamically picks L0 (OAS > 39 b) or L1 (OAS ≤ 39 b) as the + starting level. + +Choice of stage is a policy decision between platforms - the identity-mapping semantics that DMA agents observe +are identical. + +### 2. Page Table Structure + +The IOMMU uses up to a 4-level page table structure for DMA address translation: + + +Depending on configuration from the SMMU registers, the starting level of translation is chosen. Stage 2 supports +Concatenated Translation Tables at the starting level for wide OAS (Arm ARM D8.2.2); Stage 1 is architecturally +forbidden from concatenating at the starting level (Arm ARM D8.5.2), so on Stage 1 the walker always installs a +single-page root and picks the starting level dynamically: + +- **Stage 2 (default)**: 16-page concatenated root; L1 start for OAS < 44 b, L0 start for OAS ≥ 44 b. +- **Stage 1**: single-page root; L1 start for OAS ≤ 39 b, L0 start otherwise. Full 48-bit input is covered by a + single L0 page (512 entries × 512 GB). + +### 3. Address Translation Process + +1. **Device Issues DMA**: + - Device uses IOVA (I/O Virtual Address) + - SMMU intercepts access + +2. **SMMU Translation**: + - Looks up Stream Table Entry (STE) + - 2 level or Linear Stream Table. Depending on configurable maximum StreamId via IORT. + - Stage 2: walks up to 4-level Stage 2 page tables directly from `STE.S2Ttb`. + - Stage 1: dereferences `STE.S1ContextPtr` to fetch the Context Descriptor, then walks up to 4-level Stage 1 + page tables from `CD.Ttb0`. + - Converts IOVA to PA (Physical Address) + +### Leaf Page Table Entry Updates and Break-Before-Make + +ARM's break-before-make (BBM) rule applies when a *live* leaf entry's PA / attributes change in place - without an +Invalid+TLBI step in between, a concurrent walk can observe a torn entry. `UpdateMapping` (`IoMmu.c`) skips BBM +because the only transitions it performs on a leaf are **Invalid ↔ Valid**; PA and flags are never rewritten on a +Valid entry. + +That invariant is enforced on the map path: if the leaf is already valid, the new `Entry` must be byte-for-byte +identical, otherwise `EFI_DEVICE_ERROR` is returned instead of overwriting in place. + +### TLB Invalidation + +The SMMU TLB caches translations tagged by the per-stream tag (VMID for Stage 2, ASID for Stage 1). The driver +therefore issues a different invalidation command depending on the configured translation stage. + +| Stage | Helper | Command | Notes | +|---------|-------------------------------|------------------------------------------------------|-------| +| Stage 2 | `SmmuV3TLBInvalidateAllStage2` | `CMD_TLBI_S12_VMALL(Vmid)` | Invalidates **all** Stage 1 + Stage 2 TLB entries owned by the given `VMID`. VMID is read from the live STE (`S2Vmid`). VMID `0` is reserved and rejected. | +| Stage 1 | `SmmuV3TLBInvalidateAllStage1` | `CMD_TLBI_NH_ASID(VMID = 0, Asid)` | The invalidation scope is equivalent to that of ASIDE1: when issued from the Non-secure Command queue, invalidates stage 1 NS-EL1 non-global entries by ASID and VMID. When only Stage 1 is enabled the STE's `S2VMID` is ignored and Stage 1 TLB entries are tagged with `VMID = 0`, so the invalidation always targets `VMID = 0`. ASID `0` is reserved and rejected. | + +Because per-stream tags (VMID/ASID) are unique per page-table root within the boot +(see [Page table root and per-stream tag](#page-table-root-and-per-stream-tag)), an invalidation for one device +cannot evict another device's translations even though the command targets an entire tag rather than a single IPA. + +## Memory Protection + +The IOMMU protocol provides several protection mechanisms: + +1. **Access Control**: + - Read/Write permissions per mapping + - Device isolation through Stream IDs + +2. **Address Range Protection**: + - Validates DMA addresses + - Prevents access outside mapped regions + +3. **Error Handling**: + - Translation faults logged to Event Queue + - See [SMMU Fault Reporting via GIC Interrupts](#smmu-fault-reporting-via-gic-interrupts) for how faults are + surfaced at runtime. + +### SMMU Fault Reporting via GIC Interrupts + +SmmuDxe hooks the SMMU's two error-reporting interrupts so translation faults and command/global errors are surfaced +the moment they happen instead of being noticed only on the next mapping call: + +- **EVTQ IRQ** - fires when the SMMU pushes an entry into its **Event Queue** (e.g. a Stage 1 or Stage 2 + translation fault for some StreamID, a forbidden access, a CD/STE fetch fault, etc.). Each SMMU carries the + per-SMMU `EvtqIrqNum` parsed from the IORT SMMUv3 node. +- **GERR IRQ** - fires on **Global Errors** reported through `GERROR` / `GERRORN` (command-queue consumer faults, + MSI write errors, queue overflow, etc.). Tracked as `GerrIrqNum`. + +#### Configuration path + +1. During driver init the SMMU driver locates the platform's GIC interrupt producer (the EDK II + `HardwareInterrupt2` protocol) and caches it. If the protocol isn't present, IRQ hookup is skipped + (non-fatal) - faults will only be surfaced on the next mapping call. +2. For each enabled SMMU, both the EVTQ IRQ and the GERR IRQ are registered with a shared ISR: the source is bound + to the handler, configured as edge-rising, and enabled at the GIC. +3. The SMMU itself is programmed (in the same configure path) to deliver these IRQs as wired SPIs through the GIC. + MSI delivery is not used here - the IORT-supplied SPI numbers are routed directly. + +#### Runtime handling + +The shared ISR runs at GIC dispatch: + +1. Walks the driver's SMMU list to find the enabled SMMU whose EVTQ IRQ or GERR IRQ matches the firing source. +2. Drains and pretty-prints every queued Event Queue entry plus the latched `GERROR` bits, then asserts so the + failure is loud during development. +3. Always signals end-of-interrupt to the GIC to deassert the IRQ. + +The same error-logging is also invoked opportunistically from `IoMmuSetAttribute` / `IoMmuSetAttributeById` so +anything that slipped past the IRQ path (or was raised while interrupts were masked) is still surfaced on the next +mapping call. + +## Continuous Protection Through the OS + +### SMMU Initialization Pre-DXE + +To ensure continuous protection, the SMMU should be set in abort mode during the pre-DXE phase. This means that the SMMU +will block all transactions unless explicitly configured to allow them. This setup ensures that no unauthorized DMA +operations can occur before the SMMU is fully configured. + +### Interaction with PcdDisableBMEonEBS + +The PcdDisableBMEonEBS (Disable Bus Master Enable on Exit Boot Services) setting plays a crucial role in maintaining +system security during the transition from firmware to the operating system. + +The following steps outline the end-to-end interaction: + +1. Pre-DXE Initialization: + + The SMMU is set to abort mode to block all transactions. This ensures that no DMA operations can bypass the SMMU's + protection mechanisms. + +2. DMA Remapping with SMMU: + + The SMMU is configured to translate addresses and manage memory protection. + The SMMU's page tables and stream tables are set up to allow only authorized DMA operations. + +3. Exit Boot Services (EBS): + + `SmmuV3ExitBootServices` runs at `TPL_NOTIFY` for every enabled SMMU and decides per-SMMU between two terminal + states based on `SMMU_INFO->EBSBehaviorAbort`: + + - **Default - `SmmuV3GlobalAbort`.** The SMMU is flipped into global-abort: every StreamID, including those that + were translated during UEFI, has its transactions aborted. Translation is then disabled + (`SmmuV3DisableTranslation`). This is the safe default: nothing can DMA until the OS reprograms the SMMU. + - **Opt-out for SMMUs with RMR-mapped streams - `SmmuV3SetGlobalBypass`.** If any IORT RMR range was pre-mapped + under this SMMU during init (`EBSBehaviorAbort == FALSE`), the SMMU is set to global-bypass instead so the + reserved-memory devices the firmware handed off to the OS (UEFI runtime framebuffer, etc.) keep working until + the OS rebinds them. Translation is still disabled. **Other SMMUs without RMRs are unaffected - their + StreamIDs are still protected** until the OS reconfigures them. + - In addition, `PcdDisableBMEonEBS` controls clearing the PCI Bus Master Enable bit on PCI controllers, which + prevents any PCIe device from initiating new DMA in the abort window. + + The net effect: SMMUs that must keep RMR streams alive go to bypass; every other SMMU stays in abort. There is no + point in the handoff where a previously-protected StreamID can DMA freely against memory it didn't have a mapping + for during UEFI. + +4. OS Reconfiguration: + + The operating system reconfigures the SMMU as needed for its own DMA remapping and memory protection requirements. + By following these steps, the system ensures that the SMMU provides continuous protection from the pre-DXE phase + through to the operating system's reconfiguration. This approach helps prevent unauthorized DMA operations and + maintains system security. + +## Limitations + +Current implementation constraints: + +1. Supports cache coherent SMMUs only +2. Fixed 4KB granule size +3. 48-bit address space limit +4. Single translation stage per HOB + - The platform picks Stage 2 (default) or Stage 1 via `SMMU_CONFIG->TranslationStage`, and every SMMU described + by the HOB is configured for the same stage. Nested Stage 1 + Stage 2 is not supported. + - By only ever programming one stage, firmware hands off a translation regime that both a hypervisor and a + bare-metal OS can re-use for live handoff scenarios. +5. Only Identity-mapped page tables are supported (one per StreamID, allocated lazily on first + `IoMmuSetAttribute` for that StreamID) + +## Future Enhancements + +Potential improvements: + +1. Multiple translation granule support +2. Different page table mapping schemes besides Identiy-Mapping +3. Updated IoMmu Protocol to optimize redundancies +4. Bounce Buffer Optimization + - Remove the `NumberOfBytes` end address alignment check in `IoMmuMap` + - Update individual drivers to allocate their DMA buffers by page (page-aligned and page-sized) + - This eliminates one case for bounce buffering, improving performance by avoiding unnecessary + buffer copies and allocations when only the end address is unaligned + +## Configuration Options + +Key SMMU settings controlled through the SMMU config HOB: + +- **IORT data:** The complete IORT table data that the SMMU(s) will be configured with. Located by + `IortOffset` / `IortSize`. SmmuDxe walks this both at init (to discover SMMU nodes / RMR ranges) and at runtime + (to resolve `DeviceHandle` → StreamID for `IoMmuSetAttribute`). + +- **SmmuDisabledList:** Provides the platform the ability to individually disable/bypass an SMMU if needed. + This list is a `UINT64[]` of SMMU base addresses that the platform wants to disable/bypass, located by + `SmmuDisabledListOffset` / `SmmuDisabledListSize`. By default every SMMU found in the IORT is configured for the + translation stage picked by `TranslationStage` (Stage 2 by default); any SMMU whose base appears in this list is + instead set to global-bypass. + +- **NonDiscoverable device lookup table (`NcDeviceListOffset` / `NcDeviceListSize`):** Maps each platform-assigned + `UniqueId` (the value passed to `RegisterNonDiscoverableMmioDevice`) to the matching IORT Named Component + `ObjectName`. SmmuDxe uses this two-step lookup + (`UniqueId → ObjectName → IORT NC node → StreamID(s) + owning SMMU base`) every time a non-discoverable device + calls `IoMmuSetAttribute`. Omit the table (or leave the offset/size as `0`) only if the platform has no + non-discoverable DMA-capable devices; otherwise NC StreamID resolution will fail. See + [Mapping a Non-Discoverable Device to an IORT Named Component](#mapping-a-non-discoverable-device-to-an-iort-named-component) + for the entry format and an example. + +- **TranslationStage:** Chooses the translation stage every SMMU in the HOB is configured for. Default + `SmmuTranslationStage2` matches previous behavior (Stage 2 translate, Stage 1 bypass); + `SmmuTranslationStage1` opts into Stage 1 translation (Stage 2 bypass). Any other value is coerced to + `SmmuTranslationStage2`. Requires the SMMU to advertise `IDR0.S1p` when Stage 1 is selected; otherwise + `SmmuV3Configure` will return `EFI_UNSUPPORTED` and driver init will fail. See + [Translation Stage](#translation-stage) for behavior differences. + +## Platform Integration Instructions + +Generic Platform Integration: + +1. **Build the IORT.** Include one SMMUv3 node, the necessary ITS / Root Complex nodes for PCI, and one + `EFI_ACPI_IORT_TYPE_NAMED_COMP` node per non-discoverable DMA-capable device. Each Named Component's ID mappings + must produce the StreamIDs the device emits and reference the owning SMMUv3 node. + +2. **Register non-discoverable devices with a stable `UniqueId`** using `RegisterNonDiscoverableMmioDevice`. The same + `UniqueId` is what SmmuDxe will reconstruct from `PciIo->GetLocation()` later, so it must be unique across all + registrations on the platform. + +3. **Publish the SMMU_CONFIG HOB** (`gSmmuConfigHobGuid`). Append (in any order) the IORT blob, the NC device lookup + table (see [Non-Discoverable Device Integration](#non-discoverable-device-integration)), and the optional SMMU + disable list, then point the offsets/sizes in the `SMMU_CONFIG` header at them. Leave `TranslationStage` at + `SmmuTranslationStage2` (default) unless the platform explicitly wants Stage 1 - see + [Translation Stage](#translation-stage). Example: + + ```c + STATIC CONST SMMU_NC_DEVICE_ENTRY NcDeviceTable[] = { + { /* UniqueId */ 0x1, /* IORT NC ObjectName */ "USB" }, + }; + + SmmuConfig = AllocateZeroPool (sizeof (SMMU_CONFIG) + sizeof (IortData) + sizeof (NcDeviceTable)); + SmmuConfig->VersionMajor = CURRENT_SMMU_CONFIG_VERSION_MAJOR; + SmmuConfig->VersionMinor = CURRENT_SMMU_CONFIG_VERSION_MINOR; + SmmuConfig->IortSize = sizeof (IortData); + SmmuConfig->IortOffset = sizeof (SMMU_CONFIG); + SmmuConfig->NcDeviceListSize = sizeof (NcDeviceTable); + SmmuConfig->NcDeviceListOffset = sizeof (SMMU_CONFIG) + sizeof (IortData); + SmmuConfig->TranslationStage = SmmuTranslationStage2; // or SmmuTranslationStage1 to opt into Stage 1 + + CopyMem ((UINT8 *)SmmuConfig + SmmuConfig->IortOffset, &IortData, sizeof (IortData)); + CopyMem ((UINT8 *)SmmuConfig + SmmuConfig->NcDeviceListOffset, NcDeviceTable, sizeof (NcDeviceTable)); + + BuildGuidDataHob (&gSmmuConfigHobGuid, SmmuConfig, sizeof (SMMU_CONFIG) + sizeof (IortData) + sizeof (NcDeviceTable)); + ``` + +4. **Do not install the IORT yourself** - SmmuDxe installs it as an ACPI table after parsing. + +5. **Disable specific SMMUs (optional).** Append a `UINT64[]` of SMMU base addresses to put in bypass mode and point + `SmmuDisabledListOffset` / `SmmuDisabledListSize` at it. + +6. **Handle-less DMA agents (optional).** Firmware components that have no UEFI device handle (and therefore are not + in the IORT) can program the SMMU directly through `IoMmuLib::IoMmuSetAttributeById (IommuBase, DmaId, ...)`. The + caller is responsible for supplying the correct SMMU base and DMA identifier. + +Integration with Qemu: + +- SMMU is supported on Qemu v9.1.50 and above. + +## Relevant Docs + +- SMMUv3 specification +- Useful ARM SMMU documentation - +- Arm AArch64 memory management guide - +- ARM a_a-profile_architecture_reference_manual +- Intel IOMMU for DMA protection in UEFI +- IORT documentation diff --git a/ArmPkg/Drivers/SmmuDxe/SmmuDxe.c b/ArmPkg/Drivers/SmmuDxe/SmmuDxe.c new file mode 100644 index 0000000000..907db29e94 --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/SmmuDxe.c @@ -0,0 +1,2416 @@ +/** @file SmmuDxe.c + + This file contains functions for the SMMU driver. + + This driver consumes a SMMU_CONFIG Hob structure defined by the platform to configure the SMMU hardware. + Initializes the SmmuV3 hardware to enable stage 2 translation and dma remapping. + Installs the IORT to describe the SMMU configuration to the OS. + Implements the IoMmu protocol to provide a generic interface for mapping host memory to device memory. + + Copyright (c) Microsoft Corporation. + SPDX-License-Identifier: BSD-2-Clause-Patent + +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "IoMmu.h" +#include "SmmuV3.h" + +// Global IOMMU/SMMU instance +IOMMU_CONFIG *mIoMmu; + +// GIC interrupt protocol used to register the SMMU EVTQ / GERR ISRs. +EFI_HARDWARE_INTERRUPT2_PROTOCOL *mGicInterrupt = NULL; + +// Global IORT data pointer - saved for lookups during runtime +VOID *mIortData = NULL; + +// IORT byte length paired with mIortData, used by the deferred IORT install path. +UINT32 mIortSize = 0; + +/** + Add the IORT ACPI table. + + @param [in] AcpiTableProtocol Pointer to the ACPI Table Protocol. + @param [in] IortData Pointer to the IORT. + @param [in] IortSize Size of the IORT table. + + @retval EFI_SUCCESS Success. + @retval EFI_OUT_OF_RESOURCES Out of resources. + @retval EFI_INVALID_PARAMETER Invalid parameter. +**/ +STATIC +EFI_STATUS +AddIortTable ( + IN EFI_ACPI_TABLE_PROTOCOL *AcpiTable, + IN VOID *IortData, + IN UINT32 IortSize + ) +{ + EFI_STATUS Status; + UINTN TableHandle; + + if ((AcpiTable == NULL) || (IortData == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Status = AcpiTable->InstallAcpiTable ( + AcpiTable, + (EFI_ACPI_COMMON_HEADER *)(UINTN)IortData, + IortSize, + &TableHandle + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to install IORT table\n", __func__)); + } + + return Status; +} + +/** + Protocol-notify callback fired when a platform installs + gEfiAcpiTableProtocolGuid after SmmuDxe already checked for it at entry. + Uses mIortData / mIortSize saved during driver init. + On success, CloseEvent unregisters the notify so it never fires again + + @param [in] Event The event that fired. + @param [in] Context Unused. +**/ +STATIC +VOID +EFIAPI +SmmuV3AcpiTableProtocolNotify ( + IN EFI_EVENT Event, + IN VOID *Context + ) +{ + EFI_STATUS Status; + EFI_ACPI_TABLE_PROTOCOL *AcpiTable; + + Status = gBS->LocateProtocol (&gEfiAcpiTableProtocolGuid, NULL, (VOID **)&AcpiTable); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: ACPI Table Protocol notify fired but LocateProtocol failed: %r\n", __func__, Status)); + ASSERT_EFI_ERROR (Status); + gBS->CloseEvent (Event); + return; + } + + Status = AddIortTable (AcpiTable, mIortData, mIortSize); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Deferred IORT install failed: %r\n", __func__, Status)); + ASSERT_EFI_ERROR (Status); + } else { + DEBUG ((DEBUG_INFO, "%a: IORT installed via ACPI Table Protocol notify\n", __func__)); + } + + gBS->CloseEvent (Event); +} + +/** + Install the IORT table via the ACPI Table Protocol if available; + otherwise arm a protocol-notify to install it if/when the protocol arrives. + + Locates gEfiAcpiTableProtocolGuid internally. If the protocol is not yet + produced (non-ACPI platform or produced later), + SmmuV3AcpiTableProtocolNotify is registered as the notify callback and + consumes mIortData / mIortSize if/when the protocol is produced. + + @param [in] IortData Pointer to the IORT table data. + @param [in] IortSize Size of the IORT table data in bytes. + + @retval EFI_SUCCESS IORT installed or notify armed. + @retval Other Install / event / notify-registration failure. +**/ +STATIC +EFI_STATUS +SmmuV3InstallOrDeferIortTable ( + IN VOID *IortData, + IN UINT32 IortSize + ) +{ + EFI_STATUS Status; + EFI_ACPI_TABLE_PROTOCOL *AcpiTable; + EFI_EVENT AcpiNotifyEvent; + VOID *AcpiNotifyRegistration; + + // ACPI Table Protocol is optional. Non-ACPI (e.g. Device-Tree-only) + // platforms may not publish it; SMMU HW configuration still proceeds. + // If a producer arrives after us, the notify below installs IORT then. + Status = gBS->LocateProtocol ( + &gEfiAcpiTableProtocolGuid, + NULL, + (VOID **)&AcpiTable + ); + if (EFI_ERROR (Status)) { + AcpiTable = NULL; + } + + if (AcpiTable != NULL) { + Status = AddIortTable (AcpiTable, IortData, IortSize); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to add IORT table\n", __func__)); + } + + return Status; + } + + Status = gBS->CreateEvent ( + EVT_NOTIFY_SIGNAL, + TPL_CALLBACK, + SmmuV3AcpiTableProtocolNotify, + NULL, + &AcpiNotifyEvent + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to create ACPI notify event (%r); IORT will not be installed\n", __func__, Status)); + ASSERT_EFI_ERROR (Status); + return Status; + } + + Status = gBS->RegisterProtocolNotify ( + &gEfiAcpiTableProtocolGuid, + AcpiNotifyEvent, + &AcpiNotifyRegistration + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to arm ACPI protocol notify (%r); IORT will not be installed\n", __func__, Status)); + gBS->CloseEvent (AcpiNotifyEvent); + ASSERT_EFI_ERROR (Status); + return Status; + } + + DEBUG ((DEBUG_INFO, "%a: Deferred IORT install: waiting on ACPI Table Protocol\n", __func__)); + return EFI_SUCCESS; +} + +/** + Initialize a page table. Only initializes the root page table. + UpdateMapping() will allocate entries on the fly as needed. + + Root size depends on SmmuInfo->TranslationStage: Stage 2 allocates the + full concatenated span so any OAS up to 44 bits can be indexed from L1. + Stage 1 allocates a single-page root; concatenation at the starting + level is not architecturally allowed for Stage 1, so the walker starts + at L1 for OAS <= 39 bits and at L0 for wider OAS (see + SmmuV3SetTranslationStartingLevel). + + @param [in] SmmuInfo SMMU instance whose TranslationStage picks the + allocation size. + + @retval A pointer to the initialized page table, or NULL on failure. +**/ +PAGE_TABLE * +SmmuV3AllocatePageTableRoot ( + IN SMMU_INFO *SmmuInfo + ) +{ + PAGE_TABLE *PageTable; + UINTN NumPages; + + if (SmmuInfo == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return NULL; + } + + NumPages = PAGE_TABLE_ROOT_PAGES (SmmuInfo); + + // Align the base address of the first translation table to the sum of the + // size of the memory occupied by the concatenated translation tables. For + // Stage 1 (no concatenation) this simplifies to a single 4 KB alignment. + PageTable = (PAGE_TABLE *)AllocateAlignedPages ( + NumPages, + EFI_PAGES_TO_SIZE (NumPages) + ); + + if (PageTable == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate page table\n", __func__)); + return NULL; + } + + ZeroMem (PageTable, EFI_PAGES_TO_SIZE (NumPages)); + + return PageTable; +} + +/** + Recursivley deinitialize and free a page table for all previously + allocated entries, given its level and pointer. + + @param [in] SmmuInfo SMMU instance the tree belongs to. + @param [in] Level The level of the page table to deinitialize. + @param [in] PageTable The page table to deinitialize. +**/ +VOID +SmmuV3FreePageTableTree ( + IN SMMU_INFO *SmmuInfo, + IN UINT8 Level, + IN PAGE_TABLE *PageTable + ) +{ + UINTN Index; + PAGE_TABLE_ENTRY Entry; + PAGE_TABLE *PageTableAddress; + + if ((SmmuInfo == NULL) || (Level >= PAGE_TABLE_DEPTH) || (PageTable == NULL)) { + return; + } + + for (Index = 0; Index < PAGE_TABLE_SIZE; Index++) { + Entry = PageTable->Entries[Index]; + PageTableAddress = (PAGE_TABLE *)((UINTN)Entry & ~PAGE_TABLE_BLOCK_MASK); + + if (Entry != 0) { + SmmuV3FreePageTableTree (SmmuInfo, Level + 1, PageTableAddress); + } + } + + // Root level allocation size depends on TranslationStage; deeper levels + // are always single pages allocated by UpdateMapping(). + if (Level == 0) { + FreePages (PageTable, PAGE_TABLE_ROOT_PAGES (SmmuInfo)); + } else { + FreePages (PageTable, 1); + } +} + +/** + Allocate an event queue for SMMUv3. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [out] QueueLog2Size Pointer to store the log2 size of the queue. + @param [out] EventQueueBase Pointer to store the base address of the allocated event queue. + + @retval EFI_SUCCESS The event queue was allocated successfully. + @retval EFI_INVALID_PARAMETER One or more parameters are invalid. + @retval EFI_OUT_OF_RESOURCES Allocation failed due to insufficient resources. +**/ +STATIC +EFI_STATUS +SmmuV3AllocateEventQueue ( + IN SMMU_INFO *SmmuInfo, + OUT UINT32 *QueueLog2Size, + OUT VOID **EventQueueBase + ) +{ + UINT32 QueueSize; + SMMUV3_IDR1 Idr1; + UINT32 Pages; + + if ((SmmuInfo == NULL) || (QueueLog2Size == NULL) || (EventQueueBase == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Idr1.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR1); + + *QueueLog2Size = MIN (Idr1.Bits.EventQs, SMMUV3_EVENT_QUEUE_LOG2ENTRIES); + QueueSize = SMMUV3_EVENT_QUEUE_SIZE_FROM_LOG2 (*QueueLog2Size); + Pages = EFI_SIZE_TO_PAGES (QueueSize); + *EventQueueBase = AllocatePages (Pages); + + if (*EventQueueBase == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Allocation failed\n", __func__)); + return EFI_OUT_OF_RESOURCES; + } + + ZeroMem (*EventQueueBase, EFI_PAGES_TO_SIZE (Pages)); + return EFI_SUCCESS; +} + +/** + Allocate a command queue for SMMUv3. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [out] QueueLog2Size Pointer to store the log2 size of the queue. + @param [out] CmdQueueBase Pointer to store the base address of the allocated command queue. + + @retval EFI_SUCCESS The command queue was allocated successfully. + @retval EFI_INVALID_PARAMETER One or more parameters are invalid. + @retval EFI_OUT_OF_RESOURCES Allocation failed due to insufficient resources. +**/ +STATIC +EFI_STATUS +SmmuV3AllocateCommandQueue ( + IN SMMU_INFO *SmmuInfo, + OUT UINT32 *QueueLog2Size, + OUT VOID **CmdQueueBase + ) +{ + UINT32 QueueSize; + SMMUV3_IDR1 Idr1; + UINT32 Pages; + + if ((SmmuInfo == NULL) || (QueueLog2Size == NULL) || (CmdQueueBase == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Idr1.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR1); + + *QueueLog2Size = MIN (Idr1.Bits.CmdQs, SMMUV3_COMMAND_QUEUE_LOG2ENTRIES); + QueueSize = SMMUV3_COMMAND_QUEUE_SIZE_FROM_LOG2 (*QueueLog2Size); + Pages = EFI_SIZE_TO_PAGES (QueueSize); + *CmdQueueBase = AllocatePages (Pages); + + if (*CmdQueueBase == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Allocation failed\n", __func__)); + return EFI_OUT_OF_RESOURCES; + } + + ZeroMem (*CmdQueueBase, EFI_PAGES_TO_SIZE (Pages)); + return EFI_SUCCESS; +} + +/** + Free a previously allocated event queue. + + @param [in] QueuePtr Pointer to the queue to free. + @param [in] Log2Size Log2 of the queue entry count, used to recover the + original allocation size. +**/ +STATIC +VOID +SmmuV3FreeEventQueue ( + IN VOID *QueuePtr, + IN UINT32 Log2Size + ) +{ + UINT32 Size; + + if (QueuePtr == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameters. QueuePtr == NULL\n", __func__)); + } else { + Size = SMMUV3_EVENT_QUEUE_SIZE_FROM_LOG2 (Log2Size); + FreePages ((VOID *)QueuePtr, EFI_SIZE_TO_PAGES (Size)); + } +} + +/** + Free a previously allocated command queue. + + @param [in] QueuePtr Pointer to the queue to free. + @param [in] Log2Size Log2 of the queue entry count, used to recover the + original allocation size. +**/ +STATIC +VOID +SmmuV3FreeCommandQueue ( + IN VOID *QueuePtr, + IN UINT32 Log2Size + ) +{ + UINT32 Size; + + if (QueuePtr == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameters. QueuePtr == NULL\n", __func__)); + } else { + Size = SMMUV3_COMMAND_QUEUE_SIZE_FROM_LOG2 (Log2Size); + FreePages ((VOID *)QueuePtr, EFI_SIZE_TO_PAGES (Size)); + } +} + +/** + Build an invalid stream-table entry used at init for every STE + slot before any device has been mapped. + + Implemented by reusing SmmuV3BuildStage2TranslateStreamTableEntry (or + the Stage 1 counterpart when TranslationStage == SmmuTranslationStage1) + with PageTableRoot / Cd == NULL, VMID/ASID = 0, and VALID = 0. + + @param [in] SmmuInfo SMMU instance (needed for IDR-derived fields). + @param [out] StreamEntry STE buffer to populate. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameters. + @retval Other Failure from the underlying stage-specific STE builder. +**/ +EFI_STATUS +SmmuV3BuildInvalidStreamTableEntry ( + IN SMMU_INFO *SmmuInfo, + OUT SMMUV3_STREAM_TABLE_ENTRY *StreamEntry + ) +{ + if ((SmmuInfo == NULL) || (StreamEntry == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + if (SmmuInfo->TranslationStage == SmmuTranslationStage1) { + // Cd == NULL and VALID == 0. + return SmmuV3BuildStage1TranslateStreamTableEntry (SmmuInfo, NULL, StreamEntry); + } + + // PageTableRoot==NULL VMID==0 and VALID==0. + return SmmuV3BuildStage2TranslateStreamTableEntry (SmmuInfo, NULL, 0, StreamEntry); +} + +/** + Build a Valid STAGE_2_TRANSLATE stream-table entry using the + given page-table root. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] PageTableRoot Page-table root the STE should point at. + @param [in] Vmid VMID tag to install in the STE's S2VMID field. + @param [out] StreamEntry STE buffer to populate. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameter. +**/ +EFI_STATUS +SmmuV3BuildStage2TranslateStreamTableEntry ( + IN SMMU_INFO *SmmuInfo, + IN PAGE_TABLE *PageTableRoot, + IN UINT16 Vmid, + OUT SMMUV3_STREAM_TABLE_ENTRY *StreamEntry + ) +{ + EFI_STATUS Status; + SMMUV3_IDR1 Idr1; + SMMUV3_IDR5 Idr5; + UINT32 CCA; + UINT8 CPM; + UINT8 DACS; + UINT64 S2Sl0; + + if ((SmmuInfo == NULL) || (StreamEntry == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + CCA = SMMUV3_STREAM_TABLE_ENTRY_CCA; + CPM = SMMUV3_STREAM_TABLE_ENTRY_CPM; + DACS = SMMUV3_STREAM_TABLE_ENTRY_DACS; + + ZeroMem ((VOID *)StreamEntry, sizeof (*StreamEntry)); + + Idr1.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR1); + Idr5.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR5); + + StreamEntry->Bits.Config = SMMUV3_STREAM_TABLE_ENTRY_CONFIG_STAGE_2_TRANSLATE_STAGE_1_BYPASS; + // ATS is not supported in this implementation, so set the ATS field to 0. + StreamEntry->Bits.Eats = SMMUV3_STREAM_TABLE_ENTRY_EATS_NOT_SUPPORTED; + StreamEntry->Bits.S2Vmid = Vmid; // Per-stream VMID (allocated by SmmuV3StreamGetOrCreate). + StreamEntry->Bits.S2Tg = SMMUV3_STREAM_TABLE_ENTRY_S2TG_4KB; + StreamEntry->Bits.S2Aa64 = 1; // AArch64 S2 translation tables + if (PageTableRoot != NULL) { + StreamEntry->Bits.S2Ttb = (UINT64)(UINTN)PageTableRoot >> SMMUV3_STREAM_TABLE_ENTRY_S2TTB_OFFSET; + } else { + StreamEntry->Bits.S2Ttb = 0; // For abort STEs, S2Ttb is set to 0 so any access will fault since it is not a valid page-table root. + } + + // Set the maximum output address width. + // SmmuDxe does not support output address widths greater than 48 bits. + SmmuInfo->OutputAddressWidth = SmmuV3DecodeAddressWidth (Idr5.Bits.Oas); + + if (SmmuInfo->OutputAddressWidth < SMMUV3_STREAM_TABLE_ENTRY_OUTPUT_ADDRESS_MAX) { + StreamEntry->Bits.S2Ps = SmmuV3EncodeAddressWidth (SmmuInfo->OutputAddressWidth); + } else { + DEBUG ((DEBUG_INFO, "%a: Advertised OutputAddressWidth >= 48. Capping the width to 48 per the SMMU spec.\n", __func__)); + StreamEntry->Bits.S2Ps = SmmuV3EncodeAddressWidth (SMMUV3_STREAM_TABLE_ENTRY_OUTPUT_ADDRESS_MAX); + SmmuInfo->OutputAddressWidth = SMMUV3_STREAM_TABLE_ENTRY_OUTPUT_ADDRESS_MAX; + } + + Status = SmmuV3SetTranslationStartingLevel (SmmuInfo, SmmuInfo->OutputAddressWidth, &S2Sl0); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to set translation starting level\n", __func__)); + return Status; + } + + // S2SL0 Meaning + // + // Starting level of the stage 2 translation lookup, controlled by VTCR_EL2. The meaning of this field depends on the value of VTCR_EL2.TG0. + // 0x2: + // If VTCR_EL2.TG0 is 0b00 (4KB granule): + // If FEAT_LPA2 is not implemented, start at level 0. + // If FEAT_LPA2 is implemented and VTCR_EL2.SL2 is 0b0, start at level 0. + // If FEAT_LPA2 is implemented, the combination of VTCR_EL2.SL0 == 10 and VTCR_EL2.SL2 == 1 is reserved. + // If VTCR_EL2.TG0 is 0b10 (16KB granule) or 0b01 (64KB granule), start at level 1. + // + StreamEntry->Bits.S2Sl0 = S2Sl0; + StreamEntry->Bits.S2T0Sz = 64 - SmmuInfo->OutputAddressWidth; + + // Set cache attributes as: + // - Inner/Outer cacheability -> Write-back-cacheable (WBC), + // Read-Allocate (RA), Write-Allocate (WA) + // - Shareability -> Inner-shareable. + StreamEntry->Bits.S2Ir0 = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; + StreamEntry->Bits.S2Or0 = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; + StreamEntry->Bits.S2Sh0 = ARM64_SHATTR_INNER_SHAREABLE; + + StreamEntry->Bits.S2Rs = SMMUV3_STREAM_TABLE_ENTRY_S2RS_RECORD_FAULTS; + + if (Idr1.Bits.AttrTypesOvr != 0) { + StreamEntry->Bits.ShCfg = SMMUV3_STREAM_TABLE_ENTRY_SHCFG_INCOMING_SHAREABILITY; + } + + // If the device requires memory attribute overrides, then hard-code it to + // Inner+Outer write-back cached and Inner-shareable (IWB-OWB-ISH) as + // given by the IORT spec. + if ((Idr1.Bits.AttrTypesOvr != 0) && ((CCA == 1) && (CPM == 1) && (DACS == 0))) { + StreamEntry->Bits.Mtcfg = SMMUV3_STREAM_TABLE_ENTRY_MTCFG; + StreamEntry->Bits.MemAttr = SMMUV3_STREAM_TABLE_ENTRY_MEMATTR_INNER_OUTTER_WRITEBACK_CACHED; + StreamEntry->Bits.ShCfg = SMMUV3_STREAM_TABLE_ENTRY_SHCFG_INNER_SHAREABLE; + } + + if (PageTableRoot != NULL) { + StreamEntry->Bits.Valid = SMMUV3_STREAM_TABLE_ENTRY_VALID; + } else { + StreamEntry->Bits.Valid = 0; + } + + return Status; +} + +/** + 2-level stream tables only: if the L1 descriptor covering StreamId still + points at the shared-ABORT L2 page (SmmuInfo->SharedAbortL2), seed the + caller-provided L2 page (NewL2) with copies of the shared ABORT STE and + rewrite the L1 descriptor to point at it. + The caller owns the lifetime of NewL2; this function does not allocate or free it. + + Each L1 index that sees at least one promotion gets its own private L2. + Other L1 indices keep aliasing the shared page until they too see + a promotion. + + Must be issued with break-before-make for every STE in the L1's coverage + because the L1 descriptor change invalidates the SMMU's cached STE + pointers for that whole range. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] StreamId StreamID whose L1 slot may need splitting. + @param [in] NewL2 Caller-provided L2 page (page-sized, writable) + that the L1 descriptor will be swung to point at. + Used only if a split is actually needed. + @param [out] NewL2Consumed Set to TRUE if NewL2 was installed into the L1 + descriptor (ownership has transferred to the + SMMU and the caller must NOT free it), FALSE + if no split was performed (caller is free to + release NewL2). + + @retval EFI_SUCCESS No split needed, or split succeeded. + @retval EFI_INVALID_PARAMETER Invalid parameters. + @retval Other Command-queue failure. +**/ +STATIC +EFI_STATUS +SmmuV3SplitL1IfShared ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 StreamId, + IN SMMUV3_STREAM_TABLE_ENTRY *NewL2, + OUT BOOLEAN *NewL2Consumed + ) +{ + EFI_STATUS Status; + SMMUV3_L1_STREAM_TABLE_DESCRIPTOR NewDesc; + SMMUV3_L1_STREAM_TABLE_DESCRIPTOR *L1Table; + SMMUV3_L1_STREAM_TABLE_DESCRIPTOR *L1Desc; + UINT64 SharedAbortL2Encoded; + UINT32 L1Index; + UINT32 BaseStreamId; + SMMUV3_CMD_GENERIC Command; + + if ((SmmuInfo == NULL) || (SmmuInfo->StreamTable == NULL) || (NewL2Consumed == NULL)) { + return EFI_INVALID_PARAMETER; + } + + // Assume no split until we actually publish NewL2 into the L1 descriptor. + *NewL2Consumed = FALSE; + + // Linear stream-table mode: no L1 indirection to split. + if (SmmuInfo->SharedAbortL2 == NULL) { + return EFI_SUCCESS; + } + + L1Index = StreamId >> SMMUV3_STR_TAB_BASE_CFG_SPLIT; + BaseStreamId = L1Index << SMMUV3_STR_TAB_BASE_CFG_SPLIT; + + L1Table = (SMMUV3_L1_STREAM_TABLE_DESCRIPTOR *)SmmuInfo->StreamTable; + L1Desc = &L1Table[L1Index]; + + // If this L1 descriptor has already been split onto a private L2 page + // (i.e. it no longer points at the shared-ABORT L2), there is nothing + // more to do: the caller can safely write its STE in place because + // sibling slots in this L2 either still hold the abort STE or are + // already-promoted translating STEs that belong to this same caller's + // sequence. + SharedAbortL2Encoded = ((UINT64)(UINTN)SmmuInfo->SharedAbortL2) >> SMMUV3_STR_TAB_BASE_L2_PTR_OFFSET; + if (L1Desc->Bits.L2Ptr != SharedAbortL2Encoded) { + return EFI_SUCCESS; + } + + // Seed the new L2 from the shared-ABORT L2 so every unpromoted slot in + // this L1 range starts out faulting. + CopyMem (NewL2, SmmuInfo->SharedAbortL2, EFI_PAGE_SIZE); + + // Ensure all STE writes in the new L2 page are visible to the SMMU + ArmDataSynchronizationBarrier (); + + // Atomically swing the L1 descriptor onto the private L2 + // so the SMMU cannot observe a torn L1STD. + NewDesc.AsUINT64 = 0; + NewDesc.Bits.L2Ptr = (UINT64)(UINTN)NewL2 >> SMMUV3_STR_TAB_BASE_L2_PTR_OFFSET; + NewDesc.Bits.Span = SMMUV3_STR_TAB_BASE_CFG_SPLIT + 1; + + L1Desc->AsUINT64 = NewDesc.AsUINT64; + + *NewL2Consumed = TRUE; + + // Ensure the L1STD write is visible to the SMMU before issuing the CFGI_STE_RANGE command. + ArmDataSynchronizationBarrier (); + + // + // The old L1STD was the shared-ABORT entry with full Span = SPLIT+1, so + // its L2 page and STEs could already be cached. Invalidate the full + // L1 range (2^SPLIT STEs anchored at BaseStreamId) using CFGI_STE_RANGE. + // CFGI_STE_RANGE invalidates 2^(Range+1) STEs, so Range = SPLIT - 1. + SMMUV3_BUILD_CMD_CFGI_STE_RANGE (&Command, BaseStreamId, SMMUV3_STR_TAB_BASE_CFG_SPLIT - 1); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CFGI_STE_RANGE failed: %r\n", __func__, Status)); + return Status; + } + + SMMUV3_BUILD_CMD_SYNC_NO_INTERRUPT (&Command); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CMD_SYNC failed: %r\n", __func__, Status)); + return Status; + } + + DEBUG (( + DEBUG_INFO, + "%a: Split L1[0x%x] on SmmuBase=0x%llx for StreamId=0x%x (range base 0x%x, %u entries) onto private L2 0x%p\n", + __func__, + L1Index, + SmmuInfo->SmmuBase, + StreamId, + BaseStreamId, + (1 << SMMUV3_STR_TAB_BASE_CFG_SPLIT), + NewL2 + )); + + return EFI_SUCCESS; +} + +/** + Locate the STE slot in the SMMU's stream table for a given StreamID. + + Supports both linear and 2-level stream tables. For 2-level tables, walks + the L1 descriptor array by (StreamId >> SPLIT), follows the L2Ptr, then + indexes the L2 table by (StreamId & ((1 << SPLIT) - 1)). The L2 table is + shared across all L1 entries (allocated once in SmmuV3Configure) so any + StreamID within range resolves to a real slot. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] StreamId The StreamID. + + @retval Pointer to the STE slot, or NULL on out-of-range. +**/ +SMMUV3_STREAM_TABLE_ENTRY * +SmmuV3GetSteSlot ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 StreamId + ) +{ + BOOLEAN TwoLevel; + SMMUV3_L1_STREAM_TABLE_DESCRIPTOR *L1Table; + SMMUV3_L1_STREAM_TABLE_DESCRIPTOR *L1Desc; + SMMUV3_STREAM_TABLE_ENTRY *L2Table; + UINT32 L1Index; + UINT32 L2Index; + UINT32 L2EntriesPerTable; + + if ((SmmuInfo == NULL) || (SmmuInfo->StreamTable == NULL)) { + return NULL; + } + + if (StreamId > SmmuInfo->StreamTableEntryMax) { + DEBUG ((DEBUG_ERROR, "%a: StreamId 0x%x out of range (max 0x%x)\n", __func__, StreamId, SmmuInfo->StreamTableEntryMax)); + return NULL; + } + + TwoLevel = (SmmuInfo->StreamTableEntryMax >= (EFI_PAGE_SIZE / sizeof (SMMUV3_STREAM_TABLE_ENTRY))); + if (!SmmuInfo->TwoLevelStreamTableSupported) { + TwoLevel = FALSE; + DEBUG ((DEBUG_VERBOSE, "%a: SMMU does not support 2-level stream tables. Falling back to linear stream table.\n", __func__)); + } + + if (!TwoLevel) { + return &((SMMUV3_STREAM_TABLE_ENTRY *)SmmuInfo->StreamTable)[StreamId]; + } + + // 2-level: L1 index = top bits above SPLIT, L2 index = low SPLIT bits. + L2EntriesPerTable = 1u << SMMUV3_STR_TAB_BASE_CFG_SPLIT; + L1Index = StreamId >> SMMUV3_STR_TAB_BASE_CFG_SPLIT; + L2Index = StreamId & (L2EntriesPerTable - 1); + + L1Table = (SMMUV3_L1_STREAM_TABLE_DESCRIPTOR *)SmmuInfo->StreamTable; + L1Desc = &L1Table[L1Index]; + if (L1Desc->Bits.L2Ptr == 0) { + DEBUG ((DEBUG_ERROR, "%a: L1[%u] has no L2 table for StreamId 0x%x\n", __func__, L1Index, StreamId)); + return NULL; + } + + L2Table = (SMMUV3_STREAM_TABLE_ENTRY *)(UINTN)((UINT64)L1Desc->Bits.L2Ptr << SMMUV3_STR_TAB_BASE_L2_PTR_OFFSET); + return &L2Table[L2Index]; +} + +/** + Promote the STE for StreamId from Invalid to Valid with the given + page-table root, using the SMMU break-before-make sequence required by the + SMMUv3 spec for STE Config changes: + + 1. Write the STE with V=0. + 2. DSB + CFGI_STE(StreamId) + CMD_SYNC. + 3. Write the full new STE contents (S2Ttb etc., Config=S2_TRANSLATE, V=1). + 4. DSB + CFGI_STE(StreamId) + CMD_SYNC. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] StreamId The StreamID whose STE is being promoted. + @param [in] Vmid VMID tag to install in the STE's S2VMID field. + @param [in] PageTableRoot Page-table root to install in the STE. + @param [in] NewL2 Caller-provided L2 page used by + SmmuV3SplitL1IfShared() when the covering L1 + descriptor still points at the shared-ABORT + L2. Because the caller owns the PageTableRoot and NewL2, + this function does not allocate or free them. + @param [out] NewL2Consumed Set to TRUE if NewL2 was installed into an L1 + descriptor by the split step (the caller must + NOT free it), FALSE otherwise (the caller + should free NewL2). + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameters. + @retval Other Command-queue / sync failure. +**/ +EFI_STATUS +SmmuV3PromoteSteToStage2Translate ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 StreamId, + IN UINT16 Vmid, + IN PAGE_TABLE *PageTableRoot, + IN SMMUV3_STREAM_TABLE_ENTRY *NewL2, + OUT BOOLEAN *NewL2Consumed + ) +{ + EFI_STATUS Status; + SMMUV3_STREAM_TABLE_ENTRY *SteSlot; + SMMUV3_STREAM_TABLE_ENTRY NewEntry; + SMMUV3_CMD_GENERIC Command; + UINTN Index; + + if ((SmmuInfo == NULL) || (PageTableRoot == NULL) || (NewL2 == NULL) || (NewL2Consumed == NULL)) { + return EFI_INVALID_PARAMETER; + } + + // Assume NewL2 is not consumed until SplitL1IfShared reports otherwise. + *NewL2Consumed = FALSE; + + // 2-level only: if this L1 index still points at the shared-ABORT L2, + // copy-on-write it onto a private L2 page before mutating any STE. This + // is what prevents StreamID collisions across L1 indices that originally + // shared one L2. For linear stream tables this is a no-op (SharedAbortL2 + // is NULL) and NewL2 stays unconsumed. + if (SmmuInfo->SharedAbortL2 != NULL) { + Status = SmmuV3SplitL1IfShared (SmmuInfo, StreamId, NewL2, NewL2Consumed); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: SplitL1IfShared failed for StreamId 0x%x: %r\n", __func__, StreamId, Status)); + return Status; + } + } + + SteSlot = SmmuV3GetSteSlot (SmmuInfo, StreamId); + if (SteSlot == NULL) { + return EFI_INVALID_PARAMETER; + } + + // Build the full STAGE_2_TRANSLATE STE (V=1, Config=S2_TRANSLATE, + // S2Ttb, attrs, etc.) into a local. We then publish it into the live + // slot following the invalid -> valid sequence from the SMMU spec. + // + // The init-time STE template installed by SmmuV3Configure has Valid=0, + // so every promotion is an invalid -> valid transition. + Status = SmmuV3BuildStage2TranslateStreamTableEntry (SmmuInfo, PageTableRoot, Vmid, &NewEntry); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Build translate STE failed for StreamId 0x%x: %r\n", __func__, StreamId, Status)); + return Status; + } + + // + // 1. Write all STE UINT64's except Index 0 (which holds Valid + Config). + // + for (Index = 1; Index < (sizeof (SMMUV3_STREAM_TABLE_ENTRY) / sizeof (UINT64)); Index++) { + SteSlot->AsUINT64[Index] = NewEntry.AsUINT64[Index]; + } + + // + // 2. DSB so the SteSlot[1..7] writes are observable, then CFGI_STE + SYNC + // so the SMMU drops any cached STE state derived from the old + // contents before we publish Valid=1. + // + ArmDataSynchronizationBarrier (); + + SMMUV3_BUILD_CMD_CFGI_STE (&Command, StreamId, 1); // Leaf = 1 + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CFGI_STE (pre-valid) failed for StreamId 0x%x: %r\n", __func__, StreamId, Status)); + return Status; + } + + SMMUV3_BUILD_CMD_SYNC_NO_INTERRUPT (&Command); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CMD_SYNC (pre-valid) failed for StreamId 0x%x: %r\n", __func__, StreamId, Status)); + return Status; + } + + // + // 3. Publish SteSlot[0] (Valid + Config) last with a single + // atomic 64-bit write. + // + SteSlot->AsUINT64[0] = NewEntry.AsUINT64[0]; + + // + // 4. Final DSB + CFGI_STE + SYNC so the SMMU re-fetches the now-valid + // STE and picks up Config=S2_TRANSLATE for this StreamID. + // + ArmDataSynchronizationBarrier (); + + SMMUV3_BUILD_CMD_CFGI_STE (&Command, StreamId, 1); // Leaf = 1 + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CFGI_STE (post-valid) failed for StreamId 0x%x: %r\n", __func__, StreamId, Status)); + return Status; + } + + SMMUV3_BUILD_CMD_SYNC_NO_INTERRUPT (&Command); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CMD_SYNC (post-valid) failed for StreamId 0x%x: %r\n", __func__, StreamId, Status)); + return Status; + } + + DEBUG (( + DEBUG_INFO, + "%a: Promoted STE StreamId=0x%x VMID=0x%x on SmmuBase=0x%llx to STAGE_2_TRANSLATE, root=0x%p\n", + __func__, + StreamId, + Vmid, + SmmuInfo->SmmuBase, + PageTableRoot + )); + + return EFI_SUCCESS; +} + +/** + Allocate a Context Descriptor (CD) for use as a Stage 1 STE's + S1ContextPtr. The CD is 64-byte aligned as required by SMMUv3 spec + section 5.4 and zero-initialized (V = 0). We allocate a whole page so + there is room to grow to substream-based CDs later. + + @retval Pointer to the zeroed CD, or NULL on failure. +**/ +SMMUV3_CONTEXT_DESCRIPTOR * +SmmuV3AllocateContextDescriptor ( + VOID + ) +{ + SMMUV3_CONTEXT_DESCRIPTOR *Cd; + + Cd = (SMMUV3_CONTEXT_DESCRIPTOR *)AllocateAlignedPages (1, EFI_PAGE_SIZE); + if (Cd == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate CD\n", __func__)); + return NULL; + } + + ZeroMem (Cd, EFI_PAGE_SIZE); + return Cd; +} + +/** + Free a Context Descriptor previously returned by + SmmuV3AllocateContextDescriptor. + + @param [in] Cd Context Descriptor to free. May be NULL. +**/ +VOID +SmmuV3FreeContextDescriptor ( + IN SMMUV3_CONTEXT_DESCRIPTOR *Cd + ) +{ + if (Cd == NULL) { + return; + } + + FreeAlignedPages ((VOID *)Cd, 1); +} + +/** + Populate a Context Descriptor for Stage 1 identity-mapped translation. + + Sets Ttb0 to PageTableRoot, ASID to the supplied tag, T0Sz / TG0 / IPS / + IR0 / OR0 / SH0 from the SMMU's OAS + coherency configuration, MAIR0 to + AttrIndex 0 = Normal WBWA (matching Stage 1 leaf PTEs), disables TTBR1, + and sets AArch64 + Valid = 1. PageTableRoot = NULL builds an invalid + (V = 0) CD. + + @param [in] SmmuInfo SMMU instance (needed for IDR-derived fields). + @param [in] PageTableRoot Stage 1 page-table root to install in Ttb0, + or NULL for an invalid CD. + @param [in] Asid ASID tag installed in CD.Asid. + @param [out] Cd CD buffer to populate. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameters. + @retval Other Failure from starting-level computation. +**/ +EFI_STATUS +SmmuV3BuildStage1ContextDescriptor ( + IN SMMU_INFO *SmmuInfo, + IN PAGE_TABLE *PageTableRoot, + IN UINT16 Asid, + OUT SMMUV3_CONTEXT_DESCRIPTOR *Cd + ) +{ + EFI_STATUS Status; + SMMUV3_IDR5 Idr5; + UINT32 InputAddressWidth; + UINT64 S2Sl0Unused; + + if ((SmmuInfo == NULL) || (Cd == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + ZeroMem ((VOID *)Cd, sizeof (SMMUV3_CONTEXT_DESCRIPTOR)); + + // + // Stage 1 4KB derives the walk starting level from CD.T0Sz per Arm ARM + // Table D8-6 (no SL0 field in the CD). Concatenation at the starting + // level is not architecturally allowed for Stage 1 (Arm ARM D8.5.2), + // so the shared helper picks L0 for wide inputs and L1 otherwise, + // always with PageTableRootConcatenated = FALSE. CD.T0Sz encodes the + // input width, and CD.IPS is driven off IDR5.OAS. + // + Idr5.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR5); + InputAddressWidth = SmmuV3DecodeAddressWidth (Idr5.Bits.Oas); + if (InputAddressWidth > SMMUV3_STREAM_TABLE_ENTRY_OUTPUT_ADDRESS_MAX) { + InputAddressWidth = SMMUV3_STREAM_TABLE_ENTRY_OUTPUT_ADDRESS_MAX; + } + + SmmuInfo->OutputAddressWidth = InputAddressWidth; + + Status = SmmuV3SetTranslationStartingLevel (SmmuInfo, InputAddressWidth, &S2Sl0Unused); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to set translation starting level for Stage 1\n", __func__)); + return Status; + } + + Cd->Bits.T0Sz = 64 - InputAddressWidth; + Cd->Bits.Tg0 = SMMUV3_CD_TG0_4KB; + Cd->Bits.Ips = SmmuV3EncodeAddressWidth (InputAddressWidth); + Cd->Bits.Epd1 = SMMUV3_CD_EPD1; // Disable TTBR1 walk. + Cd->Bits.Aa64 = SMMUV3_CD_AA64; + Cd->Bits.Ars = SMMUV3_CD_ARS_ABORT_RECORD; + Cd->Bits.Asid = Asid; + Cd->Bits.Mair0 = SMMUV3_CD_MAIR0_NORMAL_WBWA; // AttrIndex 0 = Normal WBWA; AttrIndex 1 = Device. + + // Match translation-table walk attributes to the Stage 2 STE builder + // above (always Inner+Outer WBWA + Inner shareable in this driver). + Cd->Bits.Ir0 = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; + Cd->Bits.Or0 = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; + Cd->Bits.Sh0 = ARM64_SHATTR_INNER_SHAREABLE; + + if (PageTableRoot != NULL) { + Cd->Bits.Ttb0 = (UINT64)(UINTN)PageTableRoot >> SMMUV3_CD_TTB0_OFFSET; + Cd->Bits.Valid = SMMUV3_STREAM_TABLE_ENTRY_VALID; + } else { + Cd->Bits.Ttb0 = 0; + Cd->Bits.Valid = 0; + } + + return EFI_SUCCESS; +} + +/** + Build a STAGE_1_TRANSLATE / STAGE_2_BYPASS stream-table entry that + points at the supplied Context Descriptor. Passing Cd = NULL builds an + invalid (Valid = 0) STE suitable for the init-time template. + + Selects linear single-CD format (S1Fmt = 0, S1CdMax = 0). The stream + translation regime (STRW) is left at the ZeroMem-implicit value 0 + (Non-secure EL1&0), matching the Stage 2 STE builder. That pairs with + CMD_TLBI_NH_ASID (Stage 1) / CMD_TLBI_S12_VMALL (Stage 2), which flush + the NH partition. STRW = 0 is also universally implementable + (independent of IDR0.HYP / HTTU) and matches how bare-metal / hypervisor + operating systems re-program Stage 1 STEs after ExitBootServices. + STE.S2Vmid is fixed to SMMUV3_STREAM_TABLE_ENTRY_S1_ONLY_VMID per + SMMUv3 spec §5.2. + + @param [in] SmmuInfo SMMU instance. + @param [in] Cd CD the STE's S1ContextPtr should point at, or + NULL to build an invalid STE. + @param [out] StreamEntry STE buffer to populate. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameters. +**/ +EFI_STATUS +SmmuV3BuildStage1TranslateStreamTableEntry ( + IN SMMU_INFO *SmmuInfo, + IN SMMUV3_CONTEXT_DESCRIPTOR *Cd, + OUT SMMUV3_STREAM_TABLE_ENTRY *StreamEntry + ) +{ + SMMUV3_IDR0 Idr0; + SMMUV3_IDR1 Idr1; + + if ((SmmuInfo == NULL) || (StreamEntry == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + ZeroMem ((VOID *)StreamEntry, sizeof (SMMUV3_STREAM_TABLE_ENTRY)); + + Idr0.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR0); + Idr1.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR1); + + StreamEntry->Bits.Config = SMMUV3_STREAM_TABLE_ENTRY_CONFIG_STAGE_1_TRANSLATE_STAGE_2_BYPASS; + StreamEntry->Bits.S1Fmt = SMMUV3_STREAM_TABLE_ENTRY_S1FMT_LINEAR; + StreamEntry->Bits.S1CdMax = SMMUV3_STREAM_TABLE_ENTRY_S1CDMAX_SINGLE_CD; + StreamEntry->Bits.Eats = SMMUV3_STREAM_TABLE_ENTRY_EATS_NOT_SUPPORTED; + + // Defense in depth: abort any DMA that arrives with SSID != 0 even + // though S1CdMax = 0 disables SubStreamIDs. + StreamEntry->Bits.S1Dss = SMMUV3_STREAM_TABLE_ENTRY_S1DSS_ABORT; + + // Only meaningful when both stall and terminate are supported (StallModel=0b00); + // strict IPs may C_BAD_STE if we set this while stalls aren't optional. + if (Idr0.Bits.StallModel == 0) { + StreamEntry->Bits.S1StallD = SMMUV3_STREAM_TABLE_ENTRY_S1STALLD_TERMINATE; + } + + // Memory attributes for CD / Stage 1 translation-table walks. Match the + // Stage 2 STE builder (Inner+Outer WBWA + Inner shareable). + StreamEntry->Bits.S1Cir = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; + StreamEntry->Bits.S1Cor = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; + StreamEntry->Bits.S1Csh = ARM64_SHATTR_INNER_SHAREABLE; + + if (Idr1.Bits.AttrTypesOvr != 0) { + StreamEntry->Bits.ShCfg = SMMUV3_STREAM_TABLE_ENTRY_SHCFG_INCOMING_SHAREABILITY; + } + + if (Cd != NULL) { + StreamEntry->Bits.S1ContextPtr = (UINT64)(UINTN)Cd >> SMMUV3_STREAM_TABLE_ENTRY_S1CONTEXTPTR_OFFSET; + StreamEntry->Bits.Valid = SMMUV3_STREAM_TABLE_ENTRY_VALID; + } else { + StreamEntry->Bits.S1ContextPtr = 0; + StreamEntry->Bits.Valid = 0; + } + + return EFI_SUCCESS; +} + +/** + Promote the STE for StreamId from ABORT to STAGE_1_TRANSLATE / + STAGE_2_BYPASS with the supplied CD, using the break-before-make + sequence required for STE Config changes: write STE[1..7], DSB + + CFGI_STE(Leaf = 0) + SYNC, then publish STE[0] atomically, then DSB + + CFGI_STE + SYNC. + + Mirrors SmmuV3PromoteSteToStage2Translate wrt the shared-ABORT L2 + split-on-write path: for 2-level stream tables the caller passes a + page-sized NewL2 and receives back NewL2Consumed, indicating whether + it was installed by SmmuV3SplitL1IfShared. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] StreamId The StreamID whose STE is being promoted. + @param [in] Cd Context Descriptor to install in the STE. + @param [in] NewL2 Caller-provided L2 page for the shared-L2 + split step. Not allocated or freed here. + @param [out] NewL2Consumed TRUE if NewL2 was installed in an L1 + descriptor; FALSE otherwise. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameters. + @retval Other Command-queue / sync failure. +**/ +EFI_STATUS +SmmuV3PromoteSteToStage1Translate ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 StreamId, + IN SMMUV3_CONTEXT_DESCRIPTOR *Cd, + IN SMMUV3_STREAM_TABLE_ENTRY *NewL2, + OUT BOOLEAN *NewL2Consumed + ) +{ + EFI_STATUS Status; + SMMUV3_STREAM_TABLE_ENTRY *SteSlot; + SMMUV3_STREAM_TABLE_ENTRY NewEntry; + SMMUV3_CMD_GENERIC Command; + UINTN Index; + + if ((SmmuInfo == NULL) || (Cd == NULL) || (NewL2 == NULL) || (NewL2Consumed == NULL)) { + return EFI_INVALID_PARAMETER; + } + + *NewL2Consumed = FALSE; + + // 2-level only: split any L1 slot that still points at the shared-ABORT + // L2 before mutating the STE. + if (SmmuInfo->SharedAbortL2 != NULL) { + Status = SmmuV3SplitL1IfShared (SmmuInfo, StreamId, NewL2, NewL2Consumed); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: SmmuV3SplitL1IfShared failed for StreamId 0x%x: %r\n", __func__, StreamId, Status)); + return Status; + } + } + + SteSlot = SmmuV3GetSteSlot (SmmuInfo, StreamId); + if (SteSlot == NULL) { + return EFI_INVALID_PARAMETER; + } + + Status = SmmuV3BuildStage1TranslateStreamTableEntry (SmmuInfo, Cd, &NewEntry); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Build Stage 1 STE failed for StreamId 0x%x: %r\n", __func__, StreamId, Status)); + return Status; + } + + // + // 1. Write STE UINT64[1..7]; leave [0] (Valid + Config) untouched. + // + for (Index = 1; Index < (sizeof (SMMUV3_STREAM_TABLE_ENTRY) / sizeof (UINT64)); Index++) { + SteSlot->AsUINT64[Index] = NewEntry.AsUINT64[Index]; + } + + // + // 2. DSB + CFGI_STE(Leaf = 0) + SYNC so the SMMU drops any cached + // STE / CD state derived from the old contents before we publish + // Valid = 1. + // + ArmDataSynchronizationBarrier (); + + SMMUV3_BUILD_CMD_CFGI_STE (&Command, StreamId, 0); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CFGI_STE (pre-valid) failed for StreamId 0x%x: %r\n", __func__, StreamId, Status)); + return Status; + } + + SMMUV3_BUILD_CMD_SYNC_NO_INTERRUPT (&Command); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CMD_SYNC (pre-valid) failed for StreamId 0x%x: %r\n", __func__, StreamId, Status)); + return Status; + } + + // + // 3. Publish SteSlot[0] (Valid + Config) with a single atomic 64-bit write. + // + SteSlot->AsUINT64[0] = NewEntry.AsUINT64[0]; + + // + // 4. Final DSB + CFGI_STE + SYNC. + // + ArmDataSynchronizationBarrier (); + + SMMUV3_BUILD_CMD_CFGI_STE (&Command, StreamId, 0); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CFGI_STE (post-valid) failed for StreamId 0x%x: %r\n", __func__, StreamId, Status)); + return Status; + } + + SMMUV3_BUILD_CMD_SYNC_NO_INTERRUPT (&Command); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CMD_SYNC (post-valid) failed for StreamId 0x%x: %r\n", __func__, StreamId, Status)); + return Status; + } + + DEBUG (( + DEBUG_INFO, + "%a: Promoted STE StreamId=0x%x ASID=0x%x on SmmuBase=0x%llx to STAGE_1_TRANSLATE, CD=0x%p\n", + __func__, + StreamId, + (UINT32)Cd->Bits.Asid, + SmmuInfo->SmmuBase, + Cd + )); + + return EFI_SUCCESS; +} + +/** + Allocate a linear or 2-Level stream table for SMMUv3. + + For allocating a 2-level or linear stream table, the stream table alignment + requirements per SMMUv3 spec: + - For 2-level table, the table needs to be aligned to the larger of L1 + table size or 64 bytes. + - For linear table, the table needs to be aligned to its size. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] TwoLevelStreamTable Flag to indicate if a two-level stream table is used. + @param [out] Log2Size Pointer to store the log2 size of the stream table. + @param [out] Size Pointer to store the size of the stream table. + + @retval Pointer to the allocated stream table, or NULL on failure. +**/ +STATIC +VOID * +SmmuV3AllocateStreamTable ( + IN SMMU_INFO *SmmuInfo, + IN BOOLEAN TwoLevelStreamTable, + OUT UINT32 *Log2Size, + OUT UINT32 *Size + ) +{ + UINT32 MaxStreamId; + UINT32 SidMsb; + UINT32 L1Bits; + UINT32 Alignment; + UINTN Pages; + VOID *AllocatedAddress; + + if ((SmmuInfo == NULL) || (Log2Size == NULL) || (Size == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return NULL; + } + + // The max stream id is calculated as the output base + the number of stream ids + MaxStreamId = SmmuInfo->StreamTableEntryMax; + if (TwoLevelStreamTable && (MaxStreamId < (EFI_PAGE_SIZE / sizeof (SMMUV3_STREAM_TABLE_ENTRY)))) { + DEBUG ((DEBUG_ERROR, "%a: Invalid MaxStreamId for 2-Level table%u\n", __func__, MaxStreamId)); + return NULL; + } + + SidMsb = HighBitSet32 (MaxStreamId); + *Log2Size = SidMsb + 1; + *Size = SMMUV3_LINEAR_STREAM_TABLE_SIZE_FROM_LOG2 (*Log2Size); + if (TwoLevelStreamTable) { + L1Bits = *Log2Size - SMMUV3_STR_TAB_BASE_CFG_SPLIT; // L1 table log2 size + *Size = SMMUV3_L1_STREAM_TABLE_SIZE_FROM_LOG2 (L1Bits); + } + + *Size = ALIGN_VALUE (*Size, EFI_PAGE_SIZE); + Alignment = *Size; // Aligned to the size of the table, linear stream table + Pages = EFI_SIZE_TO_PAGES (*Size); + AllocatedAddress = AllocateAlignedPages (Pages, Alignment); + if (AllocatedAddress == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Allocation failed for stream table\n", __func__)); + return NULL; + } + + ZeroMem (AllocatedAddress, *Size); + return AllocatedAddress; +} + +/** + Free the allocated stream table for SMMUv3. + + @param [in] StreamTablePtr Pointer to the stream table entry. + @param [in] Size Size of the stream table. +**/ +STATIC +VOID +SmmuV3FreeStreamTable ( + IN VOID *StreamTablePtr, + IN UINT32 Size + ) +{ + UINTN Pages; + + if ((StreamTablePtr == NULL) || (Size == 0)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return; + } + + Pages = EFI_SIZE_TO_PAGES (Size); + + FreeAlignedPages ((VOID *)StreamTablePtr, Pages); +} + +/** + Allocate the stream table for the SMMU and populate every STE slot with + the init-time "abort-equivalent" STE template. For 2-level stream tables + this also allocates the shared-ABORT L2 page and points every L1 + descriptor at it. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [out] TwoLevelStreamTable Set to TRUE if a 2-level stream table + was allocated, FALSE for linear. + + @retval EFI_SUCCESS Success. + @retval EFI_OUT_OF_RESOURCES Allocation failure. + @retval Other Failure building the STE template. +**/ +STATIC +EFI_STATUS +SmmuV3InitStreamTable ( + IN SMMU_INFO *SmmuInfo, + OUT BOOLEAN *TwoLevelStreamTable + ) +{ + EFI_STATUS Status; + UINT32 Index; + SMMUV3_STREAM_TABLE_ENTRY *StreamTableEntryPtr; + SMMUV3_STREAM_TABLE_ENTRY *L2StreamTablePtr; + SMMUV3_L1_STREAM_TABLE_DESCRIPTOR *L1Table; + SMMUV3_STREAM_TABLE_ENTRY TemplateEntry; + + *TwoLevelStreamTable = (SmmuInfo->StreamTableEntryMax >= (EFI_PAGE_SIZE / sizeof (SMMUV3_STREAM_TABLE_ENTRY))); + if (!SmmuInfo->TwoLevelStreamTableSupported) { + *TwoLevelStreamTable = FALSE; + DEBUG ((DEBUG_INFO, "%a: SMMU does not support 2-level stream tables. Falling back to linear stream table.\n", __func__)); + } + + SmmuInfo->StreamTable = SmmuV3AllocateStreamTable (SmmuInfo, *TwoLevelStreamTable, &SmmuInfo->StreamTableLog2Size, &SmmuInfo->StreamTableSize); + if (SmmuInfo->StreamTable == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Error allocating stream table\n", __func__)); + return EFI_OUT_OF_RESOURCES; + } + + // Build the init-time STE template. This is a STAGE_2_TRANSLATE STE with + // S2Ttb = 0 (no page-table root yet) and Valid = 0, so any DMA from a non-promoted + // StreamID will trigger a SMMU fault and be recorded in + // the event queue. The first IoMmu Map/SetAttribute for a StreamID + // publishes a real S2Ttb in-place via SmmuV3PromoteSteToStage2Translate(). + Status = SmmuV3BuildInvalidStreamTableEntry (SmmuInfo, &TemplateEntry); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error building init STE template\n", __func__)); + return Status; + } + + if (*TwoLevelStreamTable) { + L2StreamTablePtr = (SMMUV3_STREAM_TABLE_ENTRY *)AllocatePages (1); + if (L2StreamTablePtr == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Error allocating L2 stream table\n", __func__)); + return EFI_OUT_OF_RESOURCES; + } + + ZeroMem (L2StreamTablePtr, EFI_PAGE_SIZE); + + for (Index = 0; Index < (EFI_PAGE_SIZE / sizeof (SMMUV3_STREAM_TABLE_ENTRY)); Index++) { + CopyMem (&L2StreamTablePtr[Index], &TemplateEntry, sizeof (SMMUV3_STREAM_TABLE_ENTRY)); + } + + // Remember this shared-ABORT L2. Every L1 descriptor initially points + // here. The first IoMmu promotion that targets an L1 index whose L2Ptr + // still equals SharedAbortL2 will copy-on-write a private L2 page so + // distinct StreamIDs that happen to share L1 bits (same StreamId >> + // SPLIT) don't alias to the same STE slot. + SmmuInfo->SharedAbortL2 = L2StreamTablePtr; + + L1Table = (SMMUV3_L1_STREAM_TABLE_DESCRIPTOR *)SmmuInfo->StreamTable; + for (Index = 0; Index < (SMMUV3_L1_STREAM_TABLE_SIZE_FROM_LOG2 (SmmuInfo->StreamTableLog2Size - SMMUV3_STR_TAB_BASE_CFG_SPLIT) / sizeof (UINT64)); Index++) { + L1Table[Index].Bits.L2Ptr = (UINT64)(UINTN)L2StreamTablePtr >> SMMUV3_STR_TAB_BASE_L2_PTR_OFFSET; + // Per SmmuV3 spec: Span must be within the range of 0 to (SMMU_STRTAB_BASE_CFG.SPLIT + 1) + // That is it must stay within the bounds of the Stream table split point. + // Cannot have Span of 0, means invalid L2 table ptr in the L1 table entry. + L1Table[Index].Bits.Span = SMMUV3_STR_TAB_BASE_CFG_SPLIT + 1; + } + } else { + StreamTableEntryPtr = (SMMUV3_STREAM_TABLE_ENTRY *)SmmuInfo->StreamTable; + for (Index = 0; Index <= SmmuInfo->StreamTableEntryMax; Index++) { + CopyMem (&StreamTableEntryPtr[Index], &TemplateEntry, sizeof (SMMUV3_STREAM_TABLE_ENTRY)); + } + } + + return EFI_SUCCESS; +} + +/** + Program the Stream Table, Command Queue and Event Queue base registers. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] TwoLevelStreamTable TRUE if a 2-level stream table is used. +**/ +STATIC +VOID +SmmuV3ProgramBaseRegisters ( + IN SMMU_INFO *SmmuInfo, + IN BOOLEAN TwoLevelStreamTable + ) +{ + SMMUV3_STRTAB_BASE StrTabBase; + SMMUV3_STRTAB_BASE_CFG StrTabBaseCfg; + SMMUV3_CMDQ_BASE CommandQueueBase; + SMMUV3_EVENTQ_BASE EventQueueBase; + + // Configure Stream Table Base + StrTabBaseCfg.AsUINT32 = 0; + StrTabBaseCfg.Bits.Fmt = SMMUV3_STR_TAB_BASE_CFG_FMT_LINEAR; + if (TwoLevelStreamTable) { + StrTabBaseCfg.Bits.Fmt = SMMUV3_STR_TAB_BASE_CFG_FMT_2LEVEL; + StrTabBaseCfg.Bits.Split = SMMUV3_STR_TAB_BASE_CFG_SPLIT; + } + + StrTabBaseCfg.Bits.Log2Size = SmmuInfo->StreamTableLog2Size; + + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_STRTAB_BASE_CFG, StrTabBaseCfg.AsUINT32); + + StrTabBase.AsUINT64 = 0; + StrTabBase.Bits.Ra = 1; + StrTabBase.Bits.Addr = ((UINT64)(UINTN)SmmuInfo->StreamTable) >> SMMUV3_STR_TAB_BASE_ADDR_OFFSET; + SmmuV3WriteRegister64 (SmmuInfo->SmmuBase, SMMU_STRTAB_BASE, StrTabBase.AsUINT64); + + // Configure Command Queue Base + CommandQueueBase.AsUINT64 = 0; + CommandQueueBase.Bits.Log2Size = SmmuInfo->CommandQueueLog2Size; + CommandQueueBase.Bits.Addr = ((UINT64)(UINTN)SmmuInfo->CommandQueue) >> SMMUV3_STR_TAB_BASE_CMDQ_OFFSET; + CommandQueueBase.Bits.Ra = 1; + SmmuV3WriteRegister64 (SmmuInfo->SmmuBase, SMMU_CMDQ_BASE, CommandQueueBase.AsUINT64); + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_PROD, 0); + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_CONS, 0); + SmmuInfo->CachedConsumer = 0; + SmmuInfo->CachedProducer = 0; + + // Configure Event Queue Base + EventQueueBase.AsUINT64 = 0; + EventQueueBase.Bits.Log2Size = SmmuInfo->EventQueueLog2Size; + EventQueueBase.Bits.Addr = ((UINT64)(UINTN)SmmuInfo->EventQueue) >> SMMUV3_STR_TAB_BASE_EVENTQ_OFFSET; + EventQueueBase.Bits.Wa = 1; + SmmuV3WriteRegister64 (SmmuInfo->SmmuBase, SMMU_EVENTQ_BASE, EventQueueBase.AsUINT64); + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase + SMMUV3_PAGE_1_OFFSET, SMMU_EVENTQ_PROD, 0); + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase + SMMUV3_PAGE_1_OFFSET, SMMU_EVENTQ_CONS, 0); +} + +/** + Configure the SMMU CR1 and CR2 control registers. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. +**/ +STATIC +VOID +SmmuV3ConfigureControlRegisters ( + IN SMMU_INFO *SmmuInfo + ) +{ + SMMUV3_CR1 Cr1; + SMMUV3_CR2 Cr2; + SMMUV3_IDR0 Idr0; + + // Configure CR1 + Cr1.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CR1); + Cr1.AsUINT32 &= ~SMMUV3_CR1_VALID_MASK; + Cr1.Bits.QueueIc = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; + Cr1.Bits.QueueOc = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; + Cr1.Bits.QueueSh = ARM64_SHATTR_INNER_SHAREABLE; + + Cr1.Bits.TableIc = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; + Cr1.Bits.TableOc = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; + Cr1.Bits.TableSh = ARM64_SHATTR_INNER_SHAREABLE; + + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CR1, Cr1.AsUINT32); + + // Configure CR2 + Cr2.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CR2); + Cr2.AsUINT32 &= ~SMMUV3_CR2_VALID_MASK; + // Set E2h to match HCR_EL2.E2H in the host PE + Cr2.Bits.E2h = (ArmReadHcr () & ARM_HCR_E2H) ? 1 : 0; + Cr2.Bits.RecInvSid = SMMUV3_CR2_REC_INV_SID; // Record C_BAD_STREAMID for invalid input streams. + + // + // If broadcast TLB maintenance (BTM) is not enabled, then configure + // private TLB maintenance (PTM). Per SMMU spec (section 6.3.12), the PTM bit is + // only valid when BTM is indicated as supported. + // + Idr0.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR0); + if (Idr0.Bits.Btm == 1) { + Cr2.Bits.Ptm = SMMUV3_CR2_PTM; // Private TLB maintenance. + } + + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CR2, Cr2.AsUINT32); +} + +/** + Enable the SMMU event and command queues (CR0 part 1) and then invalidate + all cached configuration and TLB entries. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + + @retval EFI_SUCCESS Success. + @retval Other Command-queue / poll failure. +**/ +STATIC +EFI_STATUS +SmmuV3EnableQueuesAndInvalidate ( + IN SMMU_INFO *SmmuInfo + ) +{ + EFI_STATUS Status; + SMMUV3_CR0 Cr0; + SMMUV3_CMD_GENERIC Command; + + // Issue a DSB to ensure that all previous writes are observable to the SMMU before configuring CR0. + ArmDataSynchronizationBarrier (); + + // Configure CR0 part1 + Cr0.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CR0); + Cr0.Bits.EventQEn = SMMUV3_CR0_EVENTQ_EN; + Cr0.Bits.CmdQEn = SMMUV3_CR0_CMDQ_EN; + + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CR0, Cr0.AsUINT32); + Status = SmmuV3Poll (SmmuInfo->SmmuBase, SMMU_CR0ACK, 0xC, 0xC); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error polling register: 0x%lx\n", __func__, SmmuInfo->SmmuBase + SMMU_CR0ACK)); + return Status; + } + + // + // Invalidate all cached configuration and TLB entries + // + SMMUV3_BUILD_CMD_CFGI_ALL (&Command); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error sending command.\n", __func__)); + return Status; + } + + SMMUV3_BUILD_CMD_TLBI_NSNH_ALL (&Command); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error sending command.\n", __func__)); + return Status; + } + + // Issue a CMD_SYNC command to guarantee that any previously issued TLB + // invalidations (CMD_TLBI_*) are completed (SMMUv3 spec section 4.6.3). + SMMUV3_BUILD_CMD_SYNC_NO_INTERRUPT (&Command); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error sending command.\n", __func__)); + return Status; + } + + return EFI_SUCCESS; +} + +/** + Enable SMMU translation (CR0 part 2) and check for any global errors + reported after enablement. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + + @retval EFI_SUCCESS Success. + @retval EFI_DEVICE_ERROR Global SMMU error reported. + @retval Other Poll failure. +**/ +STATIC +EFI_STATUS +SmmuV3EnableSmmuTranslation ( + IN SMMU_INFO *SmmuInfo + ) +{ + EFI_STATUS Status; + SMMUV3_CR0 Cr0; + SMMUV3_IDR0 Idr0; + SMMUV3_GERROR GError; + + // Configure CR0 part2 + Cr0.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CR0); + + // Issue a DSB to ensure that all previous writes are observable to the SMMU before configuring CR0. + ArmDataSynchronizationBarrier (); + + Cr0.AsUINT32 = Cr0.AsUINT32 & ~SMMUV3_CR0_VALID_MASK; + Cr0.Bits.SmmuEn = SMMUV3_CR0_SMMU_EN; + Cr0.Bits.EventQEn = SMMUV3_CR0_EVENTQ_EN; + Cr0.Bits.CmdQEn = SMMUV3_CR0_CMDQ_EN; + Cr0.Bits.PriQEn = SMMUV3_CR0_PRIQ_EN_DISABLED; + Cr0.Bits.Vmw = SMMUV3_CR0_VMW_DISABLED; // Disable VMID wildcard matching. + Idr0.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR0); + if (Idr0.Bits.Ats != 0) { + Cr0.Bits.AtsChk = SMMUV3_CR0_ATS_CHK_DISABLE; // Disable bypass for ATS translated traffic. + } + + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CR0, Cr0.AsUINT32); + Status = SmmuV3Poll (SmmuInfo->SmmuBase, SMMU_CR0ACK, SMMUV3_CR0_SMMU_EN_MASK, SMMUV3_CR0_SMMU_EN_MASK); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error polling register: 0x%lx\n", __func__, SmmuInfo->SmmuBase + SMMU_CR0ACK)); + return Status; + } + + // Issue a DSB to ensure that all previous writes are observable to the SMMU. + ArmDataSynchronizationBarrier (); + + GError.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_GERROR); + if (GError.AsUINT32 != 0) { + DEBUG ((DEBUG_ERROR, "%a: Global SMMU Error detected: 0x%lx\n", __func__, GError.AsUINT32)); + return EFI_DEVICE_ERROR; + } + + return EFI_SUCCESS; +} + +/** + Configure the SMMUv3 based on the provided configuration per the SmmuV3 specification. + Main configuration function for smmu hardware. Creates and enables a stream table, page table, + event queue, and command queue. Enables stage 2 translation and dma remapping. + + + + + At init time every STE is built in "abort-equivalent" mode (S2 translate + with S2Ttb=0), so no global page-table root is needed; per-StreamID roots + are allocated lazily on the first IoMmu map. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameter. + @retval EFI_OUT_OF_RESOURCES Out of resources. + @retval EFI_TIMEOUT Timeout. + @retval EFI_DEVICE_ERROR Device error. + @retval Others Failure. +**/ +STATIC +EFI_STATUS +SmmuV3Configure ( + IN SMMU_INFO *SmmuInfo + ) +{ + EFI_STATUS Status; + UINT32 CommandQueueLog2Size; + UINT32 EventQueueLog2Size; + SMMUV3_IDR0 Idr0; + SMMUV3_IDR3 Idr3; + VOID *CommandQueue; + VOID *EventQueue; + BOOLEAN TwoLevelStreamTable; + + if (SmmuInfo == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Idr0.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR0); + // This implementation only supports cache coherent SMMUs + if (Idr0.Bits.Cohacc == 0) { + DEBUG ((DEBUG_ERROR, "%a: Non-coherent access to translation tables not supported.\n", __func__)); + return EFI_UNSUPPORTED; + } + + // Check translation-stage support based on the platform's configured + // TranslationStage. + if (SmmuInfo->TranslationStage == SmmuTranslationStage1) { + if (Idr0.Bits.S1p == 0) { + DEBUG ((DEBUG_ERROR, "%a: SMMU 0x%llx does not support stage 1 translation.\n", __func__, SmmuInfo->SmmuBase)); + return EFI_UNSUPPORTED; + } + } else { + // Check for Stage 2 translation support + if (Idr0.Bits.S2p == 0) { + DEBUG ((DEBUG_ERROR, "%a: SMMU does not support stage 2 translation.\n", __func__)); + return EFI_UNSUPPORTED; + } + } + + // Check for 2-level stream table support. + SmmuInfo->TwoLevelStreamTableSupported = (Idr0.Bits.StLevel != 0); + + // Cache VMID/ASID width and seed the allocators. Both 0 values are + // reserved as "unassigned" so the allocator starts at 1. + SmmuInfo->Vmid16Supported = (Idr0.Bits.Vmid16 != 0); + SmmuInfo->NextVmid = 1; + SmmuInfo->Asid16Supported = (Idr0.Bits.Asid16 != 0); + SmmuInfo->NextAsid = 1; + + DEBUG (( + DEBUG_VERBOSE, + "%a: SMMU 0x%llx VMID=%u bits ASID=%u bits TranslationStage=%u\n", + __func__, + SmmuInfo->SmmuBase, + SmmuInfo->Vmid16Supported ? 16u : 8u, + SmmuInfo->Asid16Supported ? 16u : 8u, + (UINT32)SmmuInfo->TranslationStage + )); + + // Disable SMMU before configuring + Status = SmmuV3DisableTranslation (SmmuInfo->SmmuBase); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error disabling translation\n", __func__)); + goto End; + } + + Status = SmmuV3DisableInterrupts (SmmuInfo->SmmuBase, TRUE); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error disabling interrupts\n", __func__)); + goto End; + } + + Status = SmmuV3InitStreamTable (SmmuInfo, &TwoLevelStreamTable); + if (EFI_ERROR (Status)) { + goto End; + } + + Status = SmmuV3AllocateCommandQueue (SmmuInfo, &CommandQueueLog2Size, &CommandQueue); + if (EFI_ERROR (Status) || (CommandQueue == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Error allocating SMMU Command Queue\n", __func__)); + goto End; + } + + Status = SmmuV3AllocateEventQueue (SmmuInfo, &EventQueueLog2Size, &EventQueue); + if (EFI_ERROR (Status) || (EventQueue == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Error allocating SMMU Event Queue\n", __func__)); + goto End; + } + + SmmuInfo->CommandQueue = CommandQueue; + SmmuInfo->CommandQueueLog2Size = CommandQueueLog2Size; + SmmuInfo->EventQueue = EventQueue; + SmmuInfo->EventQueueLog2Size = EventQueueLog2Size; + + SmmuV3ProgramBaseRegisters (SmmuInfo, TwoLevelStreamTable); + + // Register EVTQ + GERR ISRs with the GIC so SMMU faults are surfaced. + if (mGicInterrupt != NULL) { + Status = SmmuV3RegisterGicIsr (mGicInterrupt, SmmuInfo); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error registering SMMU GIC ISR\n", __func__)); + goto End; + } + + DEBUG ((DEBUG_INFO, "%a: Registered SMMU GIC ISR for SmmuBase=0x%llx\n", __func__, SmmuInfo->SmmuBase)); + } else { + DEBUG ((DEBUG_ERROR, "%a: SMMU GIC ISR for SmmuBase=0x%llx not registered.\n", __func__, SmmuInfo->SmmuBase)); + } + + // Check if Range-based invalidation and level hint are supported. + Idr3.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR3); + SmmuInfo->RangeInvalidationSupported = (Idr3.Bits.Ril != 0); + + // Enable GError and event interrupts + Status = SmmuV3EnableInterrupts (SmmuInfo->SmmuBase); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error enabling interrupts\n", __func__)); + goto End; + } + + SmmuV3ConfigureControlRegisters (SmmuInfo); + + Status = SmmuV3EnableQueuesAndInvalidate (SmmuInfo); + if (EFI_ERROR (Status)) { + goto End; + } + + Status = SmmuV3EnableSmmuTranslation (SmmuInfo); + if (EFI_ERROR (Status)) { + goto End; + } + + // Only logs errors if errors are found after SMMU is enabled. + SmmuV3LogErrors (SmmuInfo); + +End: + return Status; +} + +/** + Retrieve the SMMU configuration data from the HOB. + + @return Pointer to the SMMU_CONFIG structure, or NULL if not found. +**/ +STATIC +SMMU_CONFIG * +GetSmmuConfigHobData ( + VOID + ) +{ + VOID *GuidHob; + + GuidHob = GetFirstGuidHob (&gSmmuConfigHobGuid); + + if (GuidHob != NULL) { + return (SMMU_CONFIG *)GET_GUID_HOB_DATA (GuidHob); + } + + return NULL; +} + +/** + Check if the SMMU_CONFIG structure is compatible with the current driver version. + Backwards compatibility is currently not supported. + + @param [in] SmmuConfig Pointer to the SMMU_CONFIG structure. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameter. + @retval EFI_INCOMPATIBLE_VERSION Incompatible version. +**/ +STATIC +EFI_STATUS +CheckSmmuConfigStructure ( + IN SMMU_CONFIG *SmmuConfig + ) +{ + if (SmmuConfig == NULL) { + DEBUG ((DEBUG_ERROR, "%a: SMMU_CONFIG structure is NULL\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + if ((SmmuConfig->VersionMajor == CURRENT_SMMU_CONFIG_VERSION_MAJOR) && (SmmuConfig->VersionMinor == CURRENT_SMMU_CONFIG_VERSION_MINOR)) { + return EFI_SUCCESS; + } + + DEBUG (( + DEBUG_ERROR, + "%a: SMMU_CONFIG version mismatch. Expected: %u.%u Got: %u.%u\n", + __func__, + CURRENT_SMMU_CONFIG_VERSION_MAJOR, + CURRENT_SMMU_CONFIG_VERSION_MINOR, + SmmuConfig->VersionMajor, + SmmuConfig->VersionMinor + )); + return EFI_INCOMPATIBLE_VERSION; +} + +/** + Initialize the IOMMU_CONFIG structure. + + @param [out] IoMmu Pointer to receive the allocated IOMMU_CONFIG structure. + + @retval EFI_SUCCESS The IOMMU_CONFIG structure was allocated. + @retval EFI_OUT_OF_RESOURCES Failed to allocate the IOMMU_CONFIG structure. +**/ +EFI_STATUS +IoMmuConfigInit ( + OUT IOMMU_CONFIG **IoMmu + ) +{ + *IoMmu = (IOMMU_CONFIG *)AllocateZeroPool (sizeof (IOMMU_CONFIG)); + if (*IoMmu == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate IOMMU_CONFIG structure\n", __func__)); + return EFI_OUT_OF_RESOURCES; + } + + return EFI_SUCCESS; +} + +/** + Free every per-StreamID stage-2 page-table root reachable from this SMMU's + stream table. + + The STEs are the single source of truth for what roots exist. Multiple + StreamIDs can alias the same root (a device's secondary StreamIDs share + its primary's S2Ttb / VMID), so this walks every STE, dedupes the S2Ttb + addresses it finds, and frees each unique root exactly once via + SmmuV3FreePageTableTree. + + If the dedupe-tracking buffer cannot be allocated, the function leaks the + per-stream roots rather than risking a double free. + + Caller must have already disabled translation / driven the SMMU into + ABORT so no in-flight DMA can still reference these roots. + + @param [in] SmmuInfo SMMU instance whose per-stream roots should be + freed. No-op if StreamTable is NULL. +**/ +STATIC +VOID +SmmuV3FreePerStreamPageTableRoots ( + IN SMMU_INFO *SmmuInfo + ) +{ + PAGE_TABLE **FreedRoots; + SMMUV3_CONTEXT_DESCRIPTOR **FreedCds; + UINTN MaxRoots; + UINTN FreedCount; + UINTN FreedCdCount; + UINTN MaxStreamId; + UINTN StreamId; + UINTN DupIdx; + SMMUV3_STREAM_TABLE_ENTRY *Ste; + SMMUV3_CONTEXT_DESCRIPTOR *Cd; + PAGE_TABLE *Root; + BOOLEAN IsDup; + BOOLEAN IsStage1; + + if ((SmmuInfo == NULL) || (SmmuInfo->StreamTable == NULL)) { + return; + } + + IsStage1 = (SmmuInfo->TranslationStage == SmmuTranslationStage1); + + // Upper bound on unique roots = number of tags handed out by the + // per-stream allocator (each call hands out one VMID / ASID before + // allocating a fresh root). If the counter wrapped to the reserved 0, + // every tag in the configured width is in use. + if (IsStage1) { + MaxRoots = (SmmuInfo->NextAsid == SMMU_ASID_RESERVED) + ? (SmmuInfo->Asid16Supported ? MAX_UINT16 : MAX_UINT8) + : (SmmuInfo->NextAsid - 1); + } else { + MaxRoots = (SmmuInfo->NextVmid == SMMU_VMID_RESERVED) + ? (SmmuInfo->Vmid16Supported ? MAX_UINT16 : MAX_UINT8) + : (SmmuInfo->NextVmid - 1); + } + + FreedRoots = NULL; + FreedCds = NULL; + FreedCount = 0; + FreedCdCount = 0; + if (MaxRoots > 0) { + FreedRoots = (PAGE_TABLE **)AllocateZeroPool (MaxRoots * sizeof (PAGE_TABLE *)); + if (IsStage1) { + FreedCds = (SMMUV3_CONTEXT_DESCRIPTOR **)AllocateZeroPool (MaxRoots * sizeof (SMMUV3_CONTEXT_DESCRIPTOR *)); + } + } + + // Walk every STE slot. SmmuV3GetSteSlot transparently handles both + // linear and 2-level (including the shared-ABORT L2). If we cannot + // allocate the dedupe buffer, leak rather than risk double-free. + if ((MaxRoots == 0) || ((FreedRoots != NULL) && (!IsStage1 || (FreedCds != NULL)))) { + MaxStreamId = SmmuInfo->StreamTableEntryMax; + for (StreamId = 0; StreamId <= MaxStreamId; StreamId++) { + Ste = SmmuV3GetSteSlot (SmmuInfo, (UINT32)StreamId); + if ((Ste == NULL) || (Ste->Bits.Valid == 0)) { + continue; + } + + if (IsStage1) { + if (Ste->Bits.S1ContextPtr == 0) { + continue; + } + + Cd = (SMMUV3_CONTEXT_DESCRIPTOR *)(UINTN)((UINT64)Ste->Bits.S1ContextPtr << SMMUV3_STREAM_TABLE_ENTRY_S1CONTEXTPTR_OFFSET); + if (Cd == NULL) { + continue; + } + + Root = (PAGE_TABLE *)(UINTN)((UINT64)Cd->Bits.Ttb0 << SMMUV3_CD_TTB0_OFFSET); + } else { + if (Ste->Bits.S2Ttb == 0) { + continue; + } + + Cd = NULL; + Root = (PAGE_TABLE *)(UINTN)((UINT64)Ste->Bits.S2Ttb << SMMUV3_STREAM_TABLE_ENTRY_S2TTB_OFFSET); + } + + if (Root != NULL) { + IsDup = FALSE; + for (DupIdx = 0; DupIdx < FreedCount; DupIdx++) { + if (FreedRoots[DupIdx] == Root) { + IsDup = TRUE; + break; + } + } + + if (!IsDup) { + if ((FreedRoots != NULL) && (FreedCount < MaxRoots)) { + FreedRoots[FreedCount++] = Root; + } + + SmmuV3FreePageTableTree (SmmuInfo, 0, Root); + } + } + + if (IsStage1 && (Cd != NULL)) { + IsDup = FALSE; + for (DupIdx = 0; DupIdx < FreedCdCount; DupIdx++) { + if (FreedCds[DupIdx] == Cd) { + IsDup = TRUE; + break; + } + } + + if (!IsDup) { + if ((FreedCds != NULL) && (FreedCdCount < MaxRoots)) { + FreedCds[FreedCdCount++] = Cd; + } + + SmmuV3FreeContextDescriptor (Cd); + } + } + } + } else { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate dedupe buffer; leaking per-stream roots on SMMU 0x%llx\n", __func__, SmmuInfo->SmmuBase)); + } + + if (FreedRoots != NULL) { + FreePool (FreedRoots); + } + + if (FreedCds != NULL) { + FreePool (FreedCds); + } +} + +/** + Release the stream table itself plus any auxiliary L2 stream-table pages + attached to it. + + For 2-level stream tables this frees every private L2 page installed by + split-on-write (any L1 descriptor whose L2Ptr no longer points at the + shared-ABORT L2) and then the shared-ABORT L2 page. For linear stream + tables the L2 handling is a no-op. In both cases the top-level stream + table allocation is released last via SmmuV3FreeStreamTable. + + Must run after SmmuV3FreePerStreamPageTableRoots so the S2Ttb pointers + in each STE remain valid while roots are being reclaimed. + + @param [in] SmmuInfo SMMU instance whose stream table + L2 pages + should be freed. No-op if StreamTable is NULL. +**/ +STATIC +VOID +SmmuV3FreeStreamTableAndL2Pages ( + IN SMMU_INFO *SmmuInfo + ) +{ + SMMUV3_L1_STREAM_TABLE_DESCRIPTOR *L1Tbl; + UINTN L1Count; + UINTN L1Idx; + UINT64 SharedEnc; + + if ((SmmuInfo == NULL) || (SmmuInfo->StreamTable == NULL)) { + return; + } + + // Free any private L2 stream-table pages allocated by split-on-write. + // The shared-ABORT L2 page is freed separately below. + if (SmmuInfo->SharedAbortL2 != NULL) { + L1Tbl = (SMMUV3_L1_STREAM_TABLE_DESCRIPTOR *)SmmuInfo->StreamTable; + L1Count = SmmuInfo->StreamTableSize / sizeof (SMMUV3_L1_STREAM_TABLE_DESCRIPTOR); + SharedEnc = (UINT64)(UINTN)SmmuInfo->SharedAbortL2 >> SMMUV3_STR_TAB_BASE_L2_PTR_OFFSET; + for (L1Idx = 0; L1Idx < L1Count; L1Idx++) { + if ((L1Tbl[L1Idx].Bits.L2Ptr != 0) && ((UINT64)L1Tbl[L1Idx].Bits.L2Ptr != SharedEnc)) { + FreePages ( + (VOID *)(UINTN)((UINT64)L1Tbl[L1Idx].Bits.L2Ptr << SMMUV3_STR_TAB_BASE_L2_PTR_OFFSET), + 1 + ); + L1Tbl[L1Idx].Bits.L2Ptr = 0; + } + } + + FreePages (SmmuInfo->SharedAbortL2, 1); + SmmuInfo->SharedAbortL2 = NULL; + } + + SmmuV3FreeStreamTable (SmmuInfo->StreamTable, SmmuInfo->StreamTableSize); + SmmuInfo->StreamTable = NULL; +} + +/** + Deinitialize and free the SMMU_INFO structure and everything inside. + Also disables SMMU translation and sets global abort. + + @param [in] IoMmu Pointer to the IOMMU_CONFIG structure to deinitialize. +**/ +STATIC +VOID +IoMmuDeInit ( + IN IOMMU_CONFIG *IoMmu + ) +{ + EFI_STATUS Status; + UINT32 SmmuIndex; + + if (IoMmu == NULL) { + ASSERT (IoMmu != NULL); + return; + } + + for (SmmuIndex = 0; SmmuIndex < IoMmu->SmmuCount; SmmuIndex++) { + Status = SmmuV3DisableTranslation (IoMmu->SmmuInfo[SmmuIndex].SmmuBase); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to disable SMMUv3 translation 0x%llx\n", __func__, IoMmu->SmmuInfo[SmmuIndex].SmmuBase)); + } + + Status = SmmuV3GlobalAbort (IoMmu->SmmuInfo[SmmuIndex].SmmuBase); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to global abort SMMUv3 0x%llx\n", __func__, IoMmu->SmmuInfo[SmmuIndex].SmmuBase)); + } + + // Free any per-StreamID page-table roots installed by lazy STE + // promotion, then release the stream table (and its auxiliary L2 + // pages, if 2-level). + SmmuV3FreePerStreamPageTableRoots (&IoMmu->SmmuInfo[SmmuIndex]); + SmmuV3FreeStreamTableAndL2Pages (&IoMmu->SmmuInfo[SmmuIndex]); + + if (IoMmu->SmmuInfo[SmmuIndex].CommandQueue != NULL) { + SmmuV3FreeCommandQueue (IoMmu->SmmuInfo[SmmuIndex].CommandQueue, IoMmu->SmmuInfo[SmmuIndex].CommandQueueLog2Size); + IoMmu->SmmuInfo[SmmuIndex].CommandQueue = NULL; + } + + if (IoMmu->SmmuInfo[SmmuIndex].EventQueue != NULL) { + SmmuV3FreeEventQueue (IoMmu->SmmuInfo[SmmuIndex].EventQueue, IoMmu->SmmuInfo[SmmuIndex].EventQueueLog2Size); + IoMmu->SmmuInfo[SmmuIndex].EventQueue = NULL; + } + } + + FreePool (IoMmu->SmmuInfo); + FreePool (IoMmu); +} + +/** + Disable SMMU translation and set SMMU to global abort or bypass during ExitBootServices + depending on the EBSBehaviorAbort flag in the SMMU_INFO structure. + + @param [in] Event The event that triggered this notification function. + @param [in] Context Pointer to the notification function's context. +**/ +STATIC +VOID +SmmuV3ExitBootServices ( + IN EFI_EVENT Event, + IN VOID *Context + ) +{ + EFI_STATUS Status; + UINT32 SmmuIndex; + + if (Event == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Event\n", __func__)); + ASSERT (Event != NULL); + return; + } + + if ((mIoMmu == NULL) || (mIoMmu->SmmuInfo == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: IOMMU_CONFIG/SMMU_INFO structure is NULL\n", __func__)); + ASSERT (mIoMmu != NULL); + ASSERT (mIoMmu->SmmuInfo != NULL); + return; + } + + for (SmmuIndex = 0; SmmuIndex < mIoMmu->SmmuCount; SmmuIndex++) { + if (mIoMmu->SmmuInfo[SmmuIndex].Enabled) { + if (mIoMmu->SmmuInfo[SmmuIndex].EBSBehaviorAbort) { + Status = SmmuV3GlobalAbort (mIoMmu->SmmuInfo[SmmuIndex].SmmuBase); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to global abort smmu 0x%llx.\n", __func__, mIoMmu->SmmuInfo[SmmuIndex].SmmuBase)); + ASSERT_EFI_ERROR (Status); + } + } else { + Status = SmmuV3SetGlobalBypass (mIoMmu->SmmuInfo[SmmuIndex].SmmuBase); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to set smmu 0x%llx global bypass.\n", __func__, mIoMmu->SmmuInfo[SmmuIndex].SmmuBase)); + ASSERT_EFI_ERROR (Status); + } + } + + Status = SmmuV3DisableTranslation (mIoMmu->SmmuInfo[SmmuIndex].SmmuBase); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to disable smmu 0x%llx translation.\n", __func__, mIoMmu->SmmuInfo[SmmuIndex].SmmuBase)); + ASSERT_EFI_ERROR (Status); + } + } + } + + gBS->CloseEvent (Event); +} + +/** + Entrypoint for SmmuDxe driver. + Configures IORT, and SMMUv3 hardware for Stage 1 or 2 translation + based on the configuration data from gSmmuConfigHobGuid HOB. + Initializes IoMmu Protocol. + + @param [in] ImageHandle The firmware allocated handle for the EFI image. + @param [in] SystemTable A pointer to the EFI System Table. + + @retval EFI_SUCCESS The entry point is executed successfully. + @retval EFI_OUT_OF_RESOURCES Not enough resources to initialize the driver. + @retval EFI_NOT_FOUND The SMMU configuration data is not found. + @retval EFI_INVALID_PARAMETER Invalid parameter. + @retval EFI_OUT_OF_RESOURCES Out of resources. + @retval EFI_TIMEOUT Timeout. + @retval EFI_DEVICE_ERROR Device error. + @retval EFI_INCOMPATIBLE_VERSION Incompatible version. + @retval Others Some error occurs when executing this entry point. +**/ +EFI_STATUS +InitializeSmmuDxe ( + IN EFI_HANDLE ImageHandle, + IN EFI_SYSTEM_TABLE *SystemTable + ) +{ + EFI_STATUS Status; + EFI_EVENT Event; + UINT32 SmmuIndex; + UINT32 SmmuStatusIndex; + UINT32 SmmuDisabledCount; + UINT64 *SmmuDisabledList; + SMMU_CONFIG *SmmuConfig; + VOID *IortData; + + // Get SMMU configuration data from HOB + SmmuConfig = GetSmmuConfigHobData (); + if (SmmuConfig == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to get SMMU config data from gSmmuConfigHobGuid\n", __func__)); + return EFI_NOT_FOUND; + } + + // Check SMMU_CONFIG version, return error if incompatible. Backwards compatibility not supported. + Status = CheckSmmuConfigStructure (SmmuConfig); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: SMMU_CONFIG version check failed\n", __func__)); + return Status; + } + + // Retrieve GIC interrupt registration interface so we can hook SMMU + // EVTQ / GERR interrupts later in SmmuV3Configure. Treat absence as + // non-fatal so SMMU configuration still proceeds without ISRs. + Status = gBS->LocateProtocol ( + &gHardwareInterrupt2ProtocolGuid, + NULL, + (VOID **)&mGicInterrupt + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_WARN, "%a: HardwareInterrupt2 protocol not found (%r); SMMU IRQs will not be hooked\n", __func__, Status)); + mGicInterrupt = NULL; + } + + // Create an event callback to disable SMMUv3 translation and set global abort during ExitBootServices + Status = gBS->CreateEventEx ( + EVT_NOTIFY_SIGNAL, + TPL_CALLBACK, + SmmuV3ExitBootServices, + NULL, + &gEfiEventExitBootServicesGuid, + &Event + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to create ExitBootServices event\n", __func__)); + return Status; + } + + Status = IoMmuConfigInit (&mIoMmu); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to initialize IoMmu Config\n", __func__)); + return Status; + } + + IortData = (VOID *)((UINTN)SmmuConfig + (UINTN)SmmuConfig->IortOffset); + + Status = SmmuV3ParseIort (IortData, &mIoMmu->SmmuInfo, &mIoMmu->SmmuCount); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to parse IORT for SMMU\n", __func__)); + return Status; + } + + DEBUG ((DEBUG_VERBOSE, "%a: Found %u SMMUs\n", __func__, mIoMmu->SmmuCount)); + + // Save IORT data pointer for StreamID lookups after PCI enumeration. + mIortData = IortData; + mIortSize = SmmuConfig->IortSize; + + Status = SmmuV3InstallOrDeferIortTable (IortData, SmmuConfig->IortSize); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to install/defer IORT table\n", __func__)); + goto Error; + } + + // Pick up the platform's NonDiscoverable {UniqueId -> Named Component Obj Name} + // lookup table, if provided. SmmuDxe uses this in IoMmuSetAttribute to + // resolve NC DeviceHandles reliably. + if ((SmmuConfig->NcDeviceListSize >= sizeof (SMMU_NC_DEVICE_ENTRY)) && + (SmmuConfig->NcDeviceListOffset != 0)) + { + mIoMmu->NcDeviceList = (SMMU_NC_DEVICE_ENTRY *)((UINTN)SmmuConfig + (UINTN)SmmuConfig->NcDeviceListOffset); + mIoMmu->NcDeviceCount = SmmuConfig->NcDeviceListSize / sizeof (SMMU_NC_DEVICE_ENTRY); + DEBUG ((DEBUG_VERBOSE, "%a: NonDiscoverable lookup table: %u entries\n", __func__, mIoMmu->NcDeviceCount)); + } else { + mIoMmu->NcDeviceList = NULL; + mIoMmu->NcDeviceCount = 0; + DEBUG ((DEBUG_WARN, "%a: No NonDiscoverable lookup table - NC StreamID resolution will fail\n", __func__)); + } + + // Set SMMUs' Enabled status based on the SmmuDisabledList in the SMMU_CONFIG HOB structure. + SmmuDisabledCount = SmmuConfig->SmmuDisabledListSize / sizeof (UINT64); + SmmuDisabledList = (UINT64 *)((UINTN)SmmuConfig + (UINTN)SmmuConfig->SmmuDisabledListOffset); + + for (SmmuIndex = 0; SmmuIndex < mIoMmu->SmmuCount; SmmuIndex++) { + mIoMmu->SmmuInfo[SmmuIndex].Enabled = TRUE; + for (SmmuStatusIndex = 0; SmmuStatusIndex < SmmuDisabledCount; SmmuStatusIndex++) { + if (mIoMmu->SmmuInfo[SmmuIndex].SmmuBase == SmmuDisabledList[SmmuStatusIndex]) { + mIoMmu->SmmuInfo[SmmuIndex].Enabled = FALSE; + } + } + } + + // Configure SMMUv3 hardware + for (SmmuIndex = 0; SmmuIndex < mIoMmu->SmmuCount; SmmuIndex++) { + if (mIoMmu->SmmuInfo[SmmuIndex].Enabled) { + mIoMmu->SmmuInfo[SmmuIndex].TranslationStage = (SmmuConfig->TranslationStage == SmmuTranslationStage1) + ? SmmuTranslationStage1 + : SmmuTranslationStage2; + Status = SmmuV3Configure (&mIoMmu->SmmuInfo[SmmuIndex]); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to configure SMMUv3 hardware\n", __func__)); + goto Error; + } + + DEBUG (( + DEBUG_INFO, + "%a: SMMUv3 0x%llx is configured for %a Translation with starting level 0x%llx\n", + __func__, + mIoMmu->SmmuInfo[SmmuIndex].SmmuBase, + (mIoMmu->SmmuInfo[SmmuIndex].TranslationStage == SmmuTranslationStage1) ? "Stage1" : "Stage2", + mIoMmu->SmmuInfo[SmmuIndex].TranslationStartingLevel + )); + + // Pre-map any IORT RMR ranges into the per-StreamID page tables that the RMR's IdMappings cover. + Status = SmmuV3AddRMRMapping (&mIoMmu->SmmuInfo[SmmuIndex]); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to add RMR mappings for SMMU 0x%llx: %r\n", __func__, mIoMmu->SmmuInfo[SmmuIndex].SmmuBase, Status)); + goto Error; + } + } + } + + // Disable any SMMU that is not enabled. + // Disables translation and sets global bypass. + for (SmmuIndex = 0; SmmuIndex < mIoMmu->SmmuCount; SmmuIndex++) { + if (mIoMmu->SmmuInfo[SmmuIndex].Enabled == FALSE) { + Status = SmmuV3DisableTranslation (mIoMmu->SmmuInfo[SmmuIndex].SmmuBase); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to disable smmu 0x%llx translation.\n", __func__, mIoMmu->SmmuInfo[SmmuIndex].SmmuBase)); + ASSERT_EFI_ERROR (Status); + } + + Status = SmmuV3SetGlobalBypass (mIoMmu->SmmuInfo[SmmuIndex].SmmuBase); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to set smmu 0x%llx global bypass.\n", __func__, mIoMmu->SmmuInfo[SmmuIndex].SmmuBase)); + ASSERT_EFI_ERROR (Status); + } + + DEBUG ((DEBUG_INFO, "%a: SMMUv3 0x%llx is disabled/global bypass.\n", __func__, mIoMmu->SmmuInfo[SmmuIndex].SmmuBase)); + } + } + + // Initialize IoMmu Protocol + Status = IoMmuInit (); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to intall IoMmuProtocol\n", __func__)); + goto Error; + } + + DEBUG ((DEBUG_INFO, "%a: Status = %llx\n", __func__, Status)); + + return Status; + +Error: + DEBUG ((DEBUG_ERROR, "%a: SMMU DMA protection failed to initialize. Status = %llx\n", __func__, Status)); + IoMmuDeInit (mIoMmu); + mIoMmu = NULL; + ASSERT_EFI_ERROR (Status); + return Status; +} diff --git a/ArmPkg/Drivers/SmmuDxe/SmmuDxe.inf b/ArmPkg/Drivers/SmmuDxe/SmmuDxe.inf new file mode 100644 index 0000000000..c82832a9c7 --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/SmmuDxe.inf @@ -0,0 +1,53 @@ +## @file SmmuDxe.inf +# This driver initializes the SmmuV3 hardware to enable stage 2 translation and DMA remapping. +# The IOMMU protocol is implemented assuming a 4-level Page Table Structure. +# +# For more info see README.md +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +[Defines] + INF_VERSION = 0x00010006 + BASE_NAME = SmmuDxe + FILE_GUID = BE506866-85F2-45EC-AC01-9BBF39D4CA78 + MODULE_TYPE = DXE_DRIVER + VERSION_STRING = 1.0 + ENTRY_POINT = InitializeSmmuDxe + +[Sources] + IoMmu.c + IoMmu.h + SmmuDxe.c + SmmuV3.h + SmmuV3Util.c + +[Guids] + gSmmuConfigHobGuid + gEfiEventExitBootServicesGuid + +[Packages] + ArmPkg/ArmPkg.dec + EmbeddedPkg/EmbeddedPkg.dec + MdeModulePkg/MdeModulePkg.dec + MdePkg/MdePkg.dec + +[LibraryClasses] + ArmLib + BaseLib + BaseMemoryLib + IoLib + HobLib + MemoryAllocationLib + TimerLib + UefiDriverEntryPoint + +[Protocols] + gEdkiiIoMmuProtocolGuid ## PRODUCES + gEfiAcpiTableProtocolGuid ## CONSUMES + gEfiPciIoProtocolGuid ## CONSUMES + gHardwareInterrupt2ProtocolGuid ## CONSUMES + +[Depex] + gHardwareInterrupt2ProtocolGuid diff --git a/ArmPkg/Drivers/SmmuDxe/SmmuV3.h b/ArmPkg/Drivers/SmmuDxe/SmmuV3.h new file mode 100644 index 0000000000..52cf386d03 --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/SmmuV3.h @@ -0,0 +1,981 @@ +/** @file SmmuV3.h + + This file is the SmmuV3 header file for SMMU driver compliant with the Smmu spec: + + + + Copyright (c) Microsoft Corporation. + SPDX-License-Identifier: BSD-2-Clause-Patent + +**/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include "IoMmu.h" + +// Number of levels in the page table +#define PAGE_TABLE_DEPTH 4 +// If the starting level is 1, and the address width exceeds 39 bits, then the page table is concatenated +#define PAGE_TABLE_CONCATENATED_PAGES_BITS_CUTOFF 39 +// Page Table Index macro to calculate the index of the page table entry based on the level and address width, supports concatenated page tables +#define PAGE_TABLE_INDEX(VirtualAddress, Level, OutputAddressWidth, TranslationStartingLevel, PageTableRootConcatenated) \ + (PageTableRootConcatenated && (Level == 1) && (TranslationStartingLevel == Level)) ? \ + (((VirtualAddress) >> (12 + (9 * ((PAGE_TABLE_DEPTH - 1) - (Level))))) & \ + ((1 << (9 + ((OutputAddressWidth) - PAGE_TABLE_CONCATENATED_PAGES_BITS_CUTOFF))) - 1)) : \ + (((VirtualAddress) >> (12 + (9 * ((PAGE_TABLE_DEPTH - 1) - (Level))))) & 0x1FF) + +#define PAGE_TABLE_4_LEVEL_OUTPUT_ADDRESS_WIDTH_MIN 44 +#define PAGE_TABLE_OUTPUT_ADDRESS_WIDTH_MAX 48 +#define PAGE_TABLE_OUTPUT_ADDRESS_WIDTH_MIN 32 +#define PAGE_TABLE_ROOT_CONCATENATED_PAGES_MAX 16 + +#define PAGE_TABLE_ROOT_STAGE1_PAGES 1 +#define PAGE_TABLE_ROOT_STAGE2_PAGES PAGE_TABLE_ROOT_CONCATENATED_PAGES_MAX + +#define PAGE_TABLE_ROOT_PAGES(SmmuInfo) \ + (((SmmuInfo)->TranslationStage == SmmuTranslationStage1) \ + ? PAGE_TABLE_ROOT_STAGE1_PAGES \ + : PAGE_TABLE_ROOT_STAGE2_PAGES) + +// Cacheability and Shareability attributes +#define ARM64_RGNCACHEATTR_NONCACHEABLE 0 +#define ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE 1 +#define ARM64_RGNCACHEATTR_WRITETHROUGH 2 +#define ARM64_RGNCACHEATTR_WRITEBACK_NOWRITEALLOCATE 3 + +#define ARM64_SHATTR_NON_SHAREABLE 0 +#define ARM64_SHATTR_OUTER_SHAREABLE 2 +#define ARM64_SHATTR_INNER_SHAREABLE 3 + +#define SMMUV3_PAGE_1_OFFSET 0x10000 + +// log2 size of the command queue +#define SMMUV3_COMMAND_QUEUE_LOG2ENTRIES (8) + +// +// Define the size of each entry in the command queue. +// +#define SMMUV3_COMMAND_QUEUE_ENTRY_SIZE (sizeof(SMMUV3_CMD_GENERIC)) + +// +// Macros to compute command queue size given its Log2 size. +// +#define SMMUV3_COMMAND_QUEUE_SIZE_FROM_LOG2(QueueLog2Size) \ + ((UINT32)(1UL << (QueueLog2Size)) * \ + (UINT16)(SMMUV3_COMMAND_QUEUE_ENTRY_SIZE)) + +// log2 size of the event queue +#define SMMUV3_EVENT_QUEUE_LOG2ENTRIES (7) + +// +// Define the size of each entry in the event queue. +// +#define SMMUV3_EVENT_QUEUE_ENTRY_SIZE (sizeof(SMMUV3_FAULT_RECORD)) + +// +// Macros to compute event queue size given its Log2 size. +// +#define SMMUV3_EVENT_QUEUE_SIZE_FROM_LOG2(QueueLog2Size) \ + ((UINT32)(1UL << (QueueLog2Size)) * (UINT16)(SMMUV3_EVENT_QUEUE_ENTRY_SIZE)) + +#define SMMUV3_COUNT_FROM_LOG2(Log2Size) (1UL << (Log2Size)) + +// +// Macro to determine if a queue is empty. It is empty if the producer and +// consumer indices are equal and their wrap bits are also equal. +// +#define SMMUV3_IS_QUEUE_EMPTY(ProducerIndex, \ + ProducerWrap, \ + ConsumerIndex, \ + ConsumerWrap) \ + \ + (((ProducerIndex) == (ConsumerIndex)) && ((ProducerWrap) == (ConsumerWrap))) + +// +// Macro to determine if a queue is full. It is full if the producer and +// consumer indices are equal and their wrap bits are different. +// +#define SMMUV3_IS_QUEUE_FULL(ProducerIndex, \ + ProducerWrap, \ + ConsumerIndex, \ + ConsumerWrap) \ + \ + (((ProducerIndex) == (ConsumerIndex)) && ((ProducerWrap) != (ConsumerWrap))) + +// +// SMMUV3 Stream Table Entry bit definitions +// +#define SMMUV3_STREAM_TABLE_ENTRY_CCA 1 // Cache Coherent Attribute +#define SMMUV3_STREAM_TABLE_ENTRY_CPM 1 // Coherent Path to Memory +#define SMMUV3_STREAM_TABLE_ENTRY_DACS 1 // Device attributes are Cacheable and Inner-Shareable +#define SMMUV3_STREAM_TABLE_ENTRY_CONFIG_STAGE_2_TRANSLATE_STAGE_1_BYPASS 0x6 // Stage 2 Translate, Stage 1 Bypass +#define SMMUV3_STREAM_TABLE_ENTRY_CONFIG_STAGE_1_TRANSLATE_STAGE_2_BYPASS 0x5 // Stage 1 Translate, Stage 2 Bypass +#define SMMUV3_STREAM_TABLE_ENTRY_CONFIG_STAGE_2_BYPASS_STAGE_1_BYPASS 0x4 // Stage 2 Bypass, Stage 1 Bypass +#define SMMUV3_STREAM_TABLE_ENTRY_EATS_NOT_SUPPORTED 0 // ATS not supported +#define SMMUV3_STREAM_TABLE_ENTRY_S2VMID 1 // Pick non-zero value +#define SMMUV3_STREAM_TABLE_ENTRY_S2TG_4KB 0 // Granule size 4KB +#define SMMUV3_STREAM_TABLE_ENTRY_S2AA64 1 // AA64 +#define SMMUV3_STREAM_TABLE_ENTRY_S2TTB_OFFSET 4 // S2TTB offset >> 4 +#define SMMUV3_STREAM_TABLE_ENTRY_S2PTW 1 // S2PTW bit +#define SMMUV3_STREAM_TABLE_ENTRY_S2SL0 2 // Encoded level of page table walk, 2 -> 4-level page table starting from 0 +#define SMMUV3_STREAM_TABLE_ENTRY_OUTPUT_ADDRESS_MAX 48 // 48 bit output address width max +#define SMMUV3_STREAM_TABLE_ENTRY_S2RS_RECORD_FAULTS 2 // Record faults +#define SMMUV3_STREAM_TABLE_ENTRY_SHCFG_INCOMING_SHAREABILITY 1 // Incoming shareability attribute +#define SMMUV3_STREAM_TABLE_ENTRY_SHCFG_INNER_SHAREABLE 3 // Inner shareable +#define SMMUV3_STREAM_TABLE_ENTRY_MTCFG 1 // MTCFG bit +#define SMMUV3_STREAM_TABLE_ENTRY_MEMATTR_INNER_OUTTER_WRITEBACK_CACHED 0xF // Inner+Outer write-back cached +#define SMMUV3_STREAM_TABLE_ENTRY_VALID 1 // Entry is valid + +// +// SMMUV3 Stage 1 Stream Table Entry bit definitions (used when +// Config = STAGE_1_TRANSLATE_STAGE_2_BYPASS). The S2VMID field is +// ignored when only Stage 1 is enabled and TLB entries +// are tagged with VMID = 0, so STE.S2Vmid is left 0 and every Stage 1 +// TLB invalidation targets VMID = 0. +// +#define SMMUV3_STREAM_TABLE_ENTRY_S1CONTEXTPTR_OFFSET 6 // S1ContextPtr address is stored shifted right by 6 +#define SMMUV3_STREAM_TABLE_ENTRY_S1FMT_LINEAR 0 // Linear single-CD format +#define SMMUV3_STREAM_TABLE_ENTRY_S1CDMAX_SINGLE_CD 0 // 2^0 = 1 CD, no SubStreamID support +#define SMMUV3_STREAM_TABLE_ENTRY_S1DSS_ABORT 0x2 // Default SubStreamID handling: abort untagged DMA +#define SMMUV3_STREAM_TABLE_ENTRY_S1STALLD_TERMINATE 1 // Force Stage 1 stalling faults to terminate +#define SMMUV3_STREAM_TABLE_ENTRY_S1_ONLY_VMID 0 // Per §5.2: S2Vmid ignored, TLB tagged VMID 0 + +// +// SMMUV3 Context Descriptor field values used by the Stage 1 CD. +// +#define SMMUV3_CD_TG0_4KB 0 // 4KB granule +#define SMMUV3_CD_AA64 1 // AArch64 translation regime +#define SMMUV3_CD_TTB0_OFFSET 4 // Ttb0 is stored shifted right by 4 +#define SMMUV3_CD_EPD1 1 // Disable TTBR1 (TTBR0-only walk) + +// +// CD.Ars packed { A(bit2), R(bit1), S(bit0) }: 0x6 = abort + record fault +// event (visible in event queue). +// +#define SMMUV3_CD_ARS_ABORT_RECORD 0x6 +#define SMMUV3_CD_MAIR_ATTR0_NORMAL_WBWA 0xFFULL // MAIR[0] = Normal Inner+Outer WBWA +#define SMMUV3_CD_MAIR_ATTR1_DEVICE_NGNRNE 0x00ULL // MAIR[1] = Device-nGnRnE +#define SMMUV3_CD_MAIR0_NORMAL_WBWA (SMMUV3_CD_MAIR_ATTR0_NORMAL_WBWA | (SMMUV3_CD_MAIR_ATTR1_DEVICE_NGNRNE << 8)) + +// VMID 0 is not a valid VMID so whenever we wrap past the max VMID, we set it to 0 to mark exhaustion. +// (Applies to both 8/16-bit VMID width). +#define SMMU_VMID_RESERVED 0 + +// ASID 0 is treated the same way as VMID but for the Stage 1 ASID allocator. +#define SMMU_ASID_RESERVED 0 + +// +// SMMUV3 Configuration bit definitions +// +#define SMMUV3_STR_TAB_BASE_CFG_FMT_LINEAR 0 // Linear Stream Table format +#define SMMUV3_STR_TAB_BASE_CFG_FMT_2LEVEL 1 // 2-Level Stream Table format +#define SMMUV3_STR_TAB_BASE_CFG_SPLIT 6 // Split bit for 2-Level Stream Table +#define SMMUV3_STR_TAB_BASE_L2_PTR_OFFSET 6 // Offset of L2 pointer in the L1 stream table entry +#define SMMUV3_STR_TAB_BASE_ADDR_OFFSET 6 // Stream Table base address offset +#define SMMUV3_STR_TAB_BASE_CMDQ_OFFSET 5 // Command queue base address offset +#define SMMUV3_STR_TAB_BASE_EVENTQ_OFFSET 5 // Event queue base address offset +#define SMMUV3_CR2_E2H 0 // E2H bit 0 +#define SMMUV3_CR2_REC_INV_SID 1 // Record C_BAD_STREAMID for invalid input streams +#define SMMUV3_CR2_PTM 1 // PTM bit +#define SMMUV3_CR0_EVENTQ_EN 1 // Event queue enable +#define SMMUV3_CR0_CMDQ_EN 1 // Command queue enable +#define SMMUV3_CR0_SMMU_EN 1 // SMMU enable +#define SMMUV3_CR0_EVENTQ_EN 1 // Event queue enable +#define SMMUV3_CR0_CMDQ_EN 1 // Command queue enable +#define SMMUV3_CR0_PRIQ_EN_DISABLED 0 // Disable PRI queue +#define SMMUV3_CR0_VMW_DISABLED 0 // Disable VMID wildcard matching +#define SMMUV3_CR0_ATS_CHK_DISABLE 1 // Disable bypass for ATS translated traffic + +typedef enum _SMMU_ADDRESS_SIZE_TYPE { + SmmuAddressSize32Bit = 0, + SmmuAddressSize36Bit = 1, + SmmuAddressSize40Bit = 2, + SmmuAddressSize42Bit = 3, + SmmuAddressSize44Bit = 4, + SmmuAddressSize48Bit = 5, + SmmuAddressSize52Bit = 6, +} SMMU_ADDRESS_SIZE_TYPE; + +typedef struct _RMR_NODE_INFO { + EFI_ACPI_6_0_IO_REMAPPING_RMR_NODE *RmrNode; // Pointer to the RMR node + LIST_ENTRY Link; // Link to the RMR node in the list +} RMR_NODE_INFO; + +// Single node in a StreamID list returned by DeviceHandleToStreamId. +// Allocated from pool; freed by SmmuStreamIdListFree. +typedef struct { + LIST_ENTRY Link; + UINT32 StreamId; +} SMMU_STREAM_ID_ENTRY; + +// General SMMU Information for a SMMU instance +typedef struct _SMMU_INFO { + VOID *SharedAbortL2; // 2-level only: shared L2 page of all-ABORT STEs. L1 entries point here until split-on-write. + VOID *StreamTable; + VOID *CommandQueue; + VOID *EventQueue; + LIST_ENTRY RmrNodeList; + UINT64 SmmuBase; + UINT64 CachedProducer; + UINT64 CachedConsumer; + UINT32 StreamTableSize; + UINT32 StreamTableEntryMax; + UINT32 Flags; + UINT32 CommandQueueSize; + UINT32 EventQueueSize; + UINT32 StreamTableLog2Size; + UINT32 CommandQueueLog2Size; + UINT32 EventQueueLog2Size; + UINT32 OutputAddressWidth; + UINT8 TranslationStartingLevel; + BOOLEAN PageTableRootConcatenated; + BOOLEAN RangeInvalidationSupported; + BOOLEAN EBSBehaviorAbort; + BOOLEAN Enabled; + BOOLEAN TwoLevelStreamTableSupported; // Whether the SMMU supports 2-level stream tables, which allows more entries than can fit in a single page. + BOOLEAN Vmid16Supported; // IDR0.VMID16. FALSE = 8-bit VMIDs only. + UINT16 NextVmid; // Next per-stream VMID to hand out. Starts at 1. + // + // Per-SMMU translation stage. Defaults to SmmuTranslationStage2. + // + SMMU_TRANSLATION_STAGE TranslationStage; + // + // Stage 1-only fields populated during SmmuV3Configure when + // TranslationStage == SmmuTranslationStage1. + // + BOOLEAN Asid16Supported; // IDR0.ASID16. FALSE = 8-bit ASIDs. + UINT16 NextAsid; // Next per-stream ASID to hand out. Starts at 1. + UINTN EvtqIrqNum; + UINTN GerrIrqNum; +} SMMU_INFO; + +// IoMmu configuration structure +typedef struct _IOMMU_CONFIG { + UINT32 SmmuCount; + SMMU_INFO *SmmuInfo; + // Platform-provided UniqueId -> Named Component ObjectName mapping for NonDiscoverable devices. + SMMU_NC_DEVICE_ENTRY *NcDeviceList; + UINT32 NcDeviceCount; +} IOMMU_CONFIG; + +// IOMMU/SMMU instance +extern IOMMU_CONFIG *mIoMmu; +extern EFI_HARDWARE_INTERRUPT2_PROTOCOL *mGicInterrupt; +extern VOID *mIortData; // Global IORT data pointer + +/** + Decode the address width from the given address size type. + + @param [in] AddressSizeType The address size type. + + @return The decoded address width. 0 if the address size type is invalid. +**/ +UINT32 +SmmuV3DecodeAddressWidth ( + IN UINT32 AddressSizeType + ); + +/** + Encode the address width to the corresponding address size type. + + @param [in] AddressWidth The address width. + + @return The encoded address size type. 0 if the address width is invalid. +**/ +UINT8 +SmmuV3EncodeAddressWidth ( + IN UINT32 AddressWidth + ); + +/** + Set the translation starting level for SMMUv3 page tables. + Only 3 and 4 level paging are supported. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] OutputAddressWidth The output address width. + @param [out] S2Sl0 The starting level for stage 2 translation. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameter. +**/ +EFI_STATUS +SmmuV3SetTranslationStartingLevel ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 OutputAddressWidth, + OUT UINT64 *S2Sl0 + ); + +/** + Read a 32-bit value from the specified SMMU register. + + @param [in] SmmuBase The base address of the SMMU. + @param [in] Register The offset of the register. + + @return The 32-bit value read from the register. 0 if the SMMU base address is invalid. +**/ +UINT32 +SmmuV3ReadRegister32 ( + IN UINT64 SmmuBase, + IN UINT64 Register + ); + +/** + Read a 64-bit value from the specified SMMU register. + + @param [in] SmmuBase The base address of the SMMU. + @param [in] Register The offset of the register. + + @return The 64-bit value read from the register. 0 if the SMMU base address is invalid. +**/ +UINT64 +SmmuV3ReadRegister64 ( + IN UINT64 SmmuBase, + IN UINT64 Register + ); + +/** + Write a 32-bit value to the specified SMMU register. + + @param [in] SmmuBase The base address of the SMMU. + @param [in] Register The offset of the register. + @param [in] Value The 32-bit value to write. + + @return The 32-bit value written to the register, or 0 if the SMMU base address is invalid. +**/ +UINT32 +SmmuV3WriteRegister32 ( + IN UINT64 SmmuBase, + IN UINT64 Register, + IN UINT32 Value + ); + +/** + Write a 64-bit value to the specified SMMU register. + + @param [in] SmmuBase The base address of the SMMU. + @param [in] Register The offset of the register. + @param [in] Value The 64-bit value to write. + + @return The 64-bit value written to the register, or 0 if the SMMU base address is invalid. +**/ +UINT64 +SmmuV3WriteRegister64 ( + IN UINT64 SmmuBase, + IN UINT64 Register, + IN UINT64 Value + ); + +/** + Disable interrupts for the SMMUv3. + + @param [in] SmmuBase The base address of the SMMU. + @param [in] ClearStaleErrors Whether to clear stale errors. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval EFI_TIMEOUT Timeout. +**/ +EFI_STATUS +SmmuV3DisableInterrupts ( + IN UINT64 SmmuBase, + IN BOOLEAN ClearStaleErrors + ); + +/** + Enable interrupts for the SMMUv3. + + @param [in] SmmuBase The base address of the SMMU. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval EFI_TIMEOUT Timeout. +**/ +EFI_STATUS +SmmuV3EnableInterrupts ( + IN UINT64 SmmuBase + ); + +/** + Disable translation for the SMMUv3. + + @param [in] SmmuBase The base address of the SMMU. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval EFI_TIMEOUT Timeout. +**/ +EFI_STATUS +SmmuV3DisableTranslation ( + IN UINT64 SmmuBase + ); + +/** + Set the Smmu in ABORT mode and stop DMA. + + @param [in] SmmuReg Base address of the SMMUv3. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval EFI_TIMEOUT Timeout. +**/ +EFI_STATUS +SmmuV3GlobalAbort ( + IN UINT64 SmmuBase + ); + +/** + Set all streams to bypass the SMMU. + + @param [in] SmmuReg Base address of the SMMUv3. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +EFI_STATUS +SmmuV3SetGlobalBypass ( + IN UINT64 SmmuBase + ); + +/** + Poll the SMMU register and test the value based on the mask. + + @param [in] SmmuBase Base address of the SMMU. + @param [in] SmmuReg The SMMU register to poll. + @param [in] Mask Mask of register bits to monitor. + @param [in] Value Expected value. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +EFI_STATUS +SmmuV3Poll ( + IN UINT64 SmmuBase, + IN UINT64 SmmuReg, + IN UINT32 Mask, + IN UINT32 Value + ); + +/** + Consume the event queue for errors and retrieve the fault record. + Clears the outputted FaultRecord if the queue is empty. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [out] FaultRecord Pointer to the fault record structure. + @param [out] IsEmpty Flag to indicate if the queue is empty. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +EFI_STATUS +SmmuV3ConsumeEventQueueForErrors ( + IN SMMU_INFO *SmmuInfo, + OUT SMMUV3_FAULT_RECORD *FaultRecord, + OUT BOOLEAN *IsEmpty + ); + +/** + Dump the page table entries for a given virtual address. + Dumps PTE's for all levels regardless of the starting level chosen for translation. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] VirtualAddress The virtual address to dump. + @param [in] Root Pointer to the root page table. +**/ +VOID +SmmuV3DumpPageTableEntries ( + IN SMMU_INFO *SmmuInfo, + IN UINT64 VirtualAddress, + IN PAGE_TABLE *Root + ); + +/** + Check if an entire address range has a valid identity mapping in the + stage-2 translation table. + + Walks Root using SmmuInfo's translation parameters for every 4 KB page in + [Address, Address + Pages * EFI_PAGE_SIZE). Address is rounded down to the + enclosing page. Since this driver identity-maps DMA a page counts as mapped only when: + - every intermediate level has a non-zero descriptor, + - the leaf entry has PAGE_TABLE_ENTRY_VALID_BIT set, and + - the leaf entry's encoded physical address matches the page address. + + Returns TRUE only if every page in the range satisfies the above; FALSE on + the first page that fails (short-circuit), bad parameters, or Pages == 0. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure whose translation + parameters drive the walk. + @param [in] Root Pointer to the root page table. + @param [in] Address Start address of the range (same value used as both + VA and PA under identity mapping). Aligned down to + the enclosing 4 KB page. + @param [in] Pages Number of 4 KB pages to check, starting at Address. + + @retval TRUE Every page in the range has a valid identity-mapped leaf. + @retval FALSE At least one page is not mapped, encodes a different PA, or + a required parameter is invalid. +**/ +BOOLEAN +SmmuV3IsAddressRangeMapped ( + IN SMMU_INFO *SmmuInfo, + IN PAGE_TABLE *Root, + IN UINT64 Address, + IN UINTN Pages + ); + +/** + Log the errors if found from the SMMUv3. Prints Event Queue entries and GError register. + Does nothing if no errors found. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + + @retval EFI_SUCCESS No SMMU errors found. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval EFI_DEVICE_ERROR SMMU error found. +**/ +EFI_STATUS +SmmuV3LogErrors ( + IN SMMU_INFO *SmmuInfo + ); + +/** + Register GIC interrupt sources for SmmuV3 EVTQ and GERR interrupts. + + @param[in] GicInterrupt Pointer to the GIC interrupt protocol. + @param[in] SmmuInfo Pointer to the SMMU_INFO structure. + + @retval EFI_SUCCESS The interrupt sources were registered successfully. + @retval EFI_INVALID_PARAMETER The GicInterrupt or SmmuInfo is NULL. +**/ +EFI_STATUS +SmmuV3RegisterGicIsr ( + IN EFI_HARDWARE_INTERRUPT2_PROTOCOL *GicInterrupt, + IN SMMU_INFO *SmmuInfo + ); + +/** + Send a SMMUV3_CMD_GENERIC command to the SMMUv3. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] Command Pointer to the command to send. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +EFI_STATUS +SmmuV3SendCommand ( + IN SMMU_INFO *SmmuInfo, + IN SMMUV3_CMD_GENERIC *Command + ); + +/** + Invalidate all Stage 2 TLB entries owned by the given VMID on this SMMU. + Uses CMD_TLBI_S12_VMALL. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] Vmid The VMID to invalidate. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +EFI_STATUS +SmmuV3TLBInvalidateAllStage2 ( + IN SMMU_INFO *SmmuInfo, + IN UINT16 Vmid + ); + +/** + Invalidate all Stage 1 TLB entries owned by the given ASID on this SMMU. + Uses CMD_TLBI_NH_ASID with VMID = 0 (Stage 1 only mode tags TLBs with VMID = 0). + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] Asid ASID to invalidate. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +EFI_STATUS +SmmuV3TLBInvalidateAllStage1 ( + IN SMMU_INFO *SmmuInfo, + IN UINT16 Asid + ); + +/** + * Add RMR mappings for each SMMU node in the SmmuInfo structure. + * This function iterates through the RMR nodes and updates the page table + * for each memory range described in the RMR node. + * + * @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + * + * @retval EFI_SUCCESS Success. + * @retval EFI_INVALID_PARAMETER Invalid Parameters. + * @retval Other RMR mapping update failure. + */ +EFI_STATUS +SmmuV3AddRMRMapping ( + IN SMMU_INFO *SmmuInfo + ); + +/** + * Parse IORT table and extract SMMU information + * + * @param[in] IortTable Pointer to the IORT table + * @param[out] SmmuInfo Pointer to store the array of SMMU_INFO structures + * @param[out] SmmuCount Pointer to store the number of SMMU nodes found + * + * @return EFI_SUCCESS on success + * @return EFI_INVALID_PARAMETER if any parameter is NULL + * @return EFI_OUT_OF_RESOURCES if memory allocation fails + * @return EFI_NOT_FOUND if no SMMU nodes are found + * @return EFI_UNSUPPORTED if the IORT table is not supported + */ +EFI_STATUS +SmmuV3ParseIort ( + IN VOID *IortTable, + OUT SMMU_INFO **SmmuInfo, + OUT UINT32 *SmmuCount + ); + +/** + Allocate a per-stream page-table root suitable for use as an STE's S2TTB + (Stage 2) or CD.TTB0 (Stage 1). The root size is derived from + SmmuInfo->TranslationStage: Stage 2 allocates + PAGE_TABLE_ROOT_STAGE2_PAGES for concatenation at the starting level; + Stage 1 allocates PAGE_TABLE_ROOT_STAGE1_PAGES since concatenation is + not architecturally allowed. + + @param [in] SmmuInfo SMMU instance whose TranslationStage decides how + many pages the root occupies. + + @retval Pointer to the zeroed root, or NULL on failure. +**/ +PAGE_TABLE * +SmmuV3AllocatePageTableRoot ( + IN SMMU_INFO *SmmuInfo + ); + +/** + Recursively free a per-stream page-table tree previously returned by + SmmuV3AllocatePageTableRoot(). SmmuInfo->TranslationStage is consulted + at Level == 0 to release the correct number of pages for the root. + + @param [in] SmmuInfo SMMU instance the tree was allocated for. + @param [in] Level Current level (caller must pass 0 for the root). + @param [in] PageTable The page-table tree to free. May be NULL. +**/ +VOID +SmmuV3FreePageTableTree ( + IN SMMU_INFO *SmmuInfo, + IN UINT8 Level, + IN PAGE_TABLE *PageTable + ); + +/** + Ensure a stage-2 page-table root exists for the given StreamID. If this is + the first call for the StreamID on this SMMU, allocate a fresh root + VMID + and promote the corresponding STE from ABORT to STAGE_2_TRANSLATE using a + break-before-make sequence (CFGI_STE + CMD_SYNC twice). Subsequent calls + return the (Root, Vmid) already encoded in the live STE. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] StreamId The StreamID. + @param [out] OutRoot Receives the stage-2 page-table root. + @param [out] OutVmid Receives the VMID tag installed in the STE. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameters. + @retval EFI_OUT_OF_RESOURCES Allocation failed / VMID space exhausted. + @retval Other STE promotion failure. +**/ +EFI_STATUS +SmmuV3StreamGetOrCreate ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 StreamId, + OUT PAGE_TABLE **OutRoot, + OUT UINT16 *OutVmid + ); + +/** + Free every SMMU_STREAM_ID_ENTRY hanging off the given list head and + re-initialize the head as empty. Safe to call on an already-empty list. + + @param[in,out] StreamIdList List head previously populated by + DeviceHandleToStreamId. +**/ +VOID +SmmuStreamIdListFree ( + IN OUT LIST_ENTRY *StreamIdList + ); + +/** + Resolve a DeviceHandle to its IORT-derived StreamID(s). + + For real PCIe devices (Segment != 0xFF): + PciIo->GetLocation() -> RID -> IORT RC node ID mapping -> single StreamID. + The matched mapping's OutputReference identifies the owning SMMUv3 node, + whose base address is returned in *SmmuBase. + + For NonDiscoverable devices (Segment == 0xFF): + UniqueId -> platform NC table entry -> IORT Named Component node -> + full StreamID list (every mapping expanded, including ranges). + + @param[in] IortTable Pointer to the IORT ACPI table. + @param[in] DeviceHandle The device handle to resolve. + @param[in,out] StreamIdList Caller-supplied, initialized-empty list head. + On success contains one SMMU_STREAM_ID_ENTRY + per resolved StreamID, in IORT order (first + entry is the primary; rest are aliases that + share the primary's stage-2 page table and + VMID). Caller must release via + SmmuStreamIdListFree. + @param[out] SmmuBase Optional. If non-NULL, receives the base + address of the SMMUv3 node that owns these + StreamIDs (0 if unknown). + + @retval EFI_SUCCESS StreamIDs resolved. + @retval EFI_INVALID_PARAMETER One or more required parameters are NULL. + @retval EFI_UNSUPPORTED DeviceHandle has no PciIo protocol. + @retval EFI_NOT_FOUND No IORT mapping found. + @retval EFI_OUT_OF_RESOURCES Allocation failure while building the list. +**/ +EFI_STATUS +DeviceHandleToStreamId ( + IN VOID *IortTable, + IN EFI_HANDLE DeviceHandle, + IN OUT LIST_ENTRY *StreamIdList, + OUT UINT64 *SmmuBase + ); + +/** + Build a default "abort-equivalent" stream-table entry. Equivalent to a + STAGE_2_TRANSLATE STE with S2Ttb = 0 — stage 2 is enabled but no + translations are installed, so DMA misses fault and are recorded. Used at + init to populate every STE slot before any device has been mapped. + + @param [in] SmmuInfo SMMU instance (needed for IDR-derived fields). + @param [out] StreamEntry STE buffer to populate. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameters. + @retval Other Failure from the underlying translate STE builder. +**/ +EFI_STATUS +SmmuV3BuildInvalidStreamTableEntry ( + IN SMMU_INFO *SmmuInfo, + OUT SMMUV3_STREAM_TABLE_ENTRY *StreamEntry + ); + +/** + Build a STAGE_2_TRANSLATE stream-table entry using the given page-table root. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] PageTableRoot Page-table root the STE should point at. + @param [in] Vmid VMID tag to install in the STE's S2VMID field. + @param [out] StreamEntry STE buffer to populate. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameters. +**/ +EFI_STATUS +SmmuV3BuildStage2TranslateStreamTableEntry ( + IN SMMU_INFO *SmmuInfo, + IN PAGE_TABLE *PageTableRoot, + IN UINT16 Vmid, + OUT SMMUV3_STREAM_TABLE_ENTRY *StreamEntry + ); + +/** + Locate the STE slot in the SMMU's stream table for a given StreamID. + Supports both linear and 2-level stream tables. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] StreamId The StreamID. + + @retval Pointer to the STE slot, or NULL if out-of-range. +**/ +SMMUV3_STREAM_TABLE_ENTRY * +SmmuV3GetSteSlot ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 StreamId + ); + +/** + Promote the STE for StreamId from Invalid to Valid with the given + page-table root, using the SMMU break-before-make sequence required by the + SMMUv3 spec for STE Config changes: + + 1. Write the STE with V=0. + 2. DSB + CFGI_STE(StreamId) + CMD_SYNC. + 3. Write the full new STE contents (S2Ttb etc., Config=S2_TRANSLATE, V=1). + 4. DSB + CFGI_STE(StreamId) + CMD_SYNC. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] StreamId The StreamID whose STE is being promoted. + @param [in] Vmid VMID tag to install in the STE's S2VMID field. + @param [in] PageTableRoot Page-table root to install in the STE. + @param [in] NewL2 Caller-provided L2 page used by + SmmuV3SplitL1IfShared() when the covering L1 + descriptor still points at the shared-ABORT + L2. Because the caller owns the PageTableRoot and NewL2, + this function does not allocate or free them. + @param [out] NewL2Consumed Set to TRUE if NewL2 was installed into an L1 + descriptor by the split step (the caller must + NOT free it), FALSE otherwise (the caller + should free NewL2). + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameters. + @retval Other Command-queue / sync failure. +**/ +EFI_STATUS +SmmuV3PromoteSteToStage2Translate ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 StreamId, + IN UINT16 Vmid, + IN PAGE_TABLE *PageTableRoot, + IN SMMUV3_STREAM_TABLE_ENTRY *NewL2, + OUT BOOLEAN *NewL2Consumed + ); + +/** + Allocate a Context Descriptor (CD) suitable for use as a Stage 1 STE's + S1ContextPtr. The CD is 64-byte aligned as required by the SMMUv3 spec + and zero-initialized (V=0). The caller populates Ttb0 / Asid / TCR + fields via SmmuV3BuildStage1ContextDescriptor before publishing it via + SmmuV3PromoteSteToStage1Translate. + + @retval Pointer to the zeroed CD, or NULL on failure. +**/ +SMMUV3_CONTEXT_DESCRIPTOR * +SmmuV3AllocateContextDescriptor ( + VOID + ); + +/** + Free a Context Descriptor previously returned by + SmmuV3AllocateContextDescriptor. + + @param [in] Cd Context Descriptor to free. May be NULL. +**/ +VOID +SmmuV3FreeContextDescriptor ( + IN SMMUV3_CONTEXT_DESCRIPTOR *Cd + ); + +/** + Populate a Context Descriptor for Stage 1 identity-mapped translation. + Sets Ttb0 to the given page-table root, ASID to the supplied per-stream + tag, T0Sz / TG0 / IPS / IR0 / OR0 / SH0 based on the SMMU's output + address width and the platform's coherency configuration, MAIR to + attribute index 0 = Normal WB Inner+Outer, disables TTBR1, and sets + AArch64 + Valid = 1. + + @param [in] SmmuInfo SMMU instance (needed for IDR-derived fields). + @param [in] PageTableRoot Stage 1 page-table root to install in Ttb0. + NULL builds an invalid (V = 0) CD suitable + for the init-time template. + @param [in] Asid ASID tag installed in CD.Asid. + @param [out] Cd CD buffer to populate. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameters. + @retval Other Failure from starting-level computation. +**/ +EFI_STATUS +SmmuV3BuildStage1ContextDescriptor ( + IN SMMU_INFO *SmmuInfo, + IN PAGE_TABLE *PageTableRoot, + IN UINT16 Asid, + OUT SMMUV3_CONTEXT_DESCRIPTOR *Cd + ); + +/** + Build a STAGE_1_TRANSLATE / STAGE_2_BYPASS stream-table entry that + points at the supplied Context Descriptor. Passing Cd = NULL builds an + invalid (Valid = 0) STE suitable for the init-time template. + + @param [in] SmmuInfo SMMU instance. + @param [in] Cd CD the STE's S1ContextPtr should point at, or + NULL to build an invalid STE. + @param [out] StreamEntry STE buffer to populate. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameters. +**/ +EFI_STATUS +SmmuV3BuildStage1TranslateStreamTableEntry ( + IN SMMU_INFO *SmmuInfo, + IN SMMUV3_CONTEXT_DESCRIPTOR *Cd, + OUT SMMUV3_STREAM_TABLE_ENTRY *StreamEntry + ); + +/** + Promote the STE for StreamId from ABORT to STAGE_1_TRANSLATE / + STAGE_2_BYPASS with the supplied CD, using the SMMU break-before-make + sequence required for STE Config changes. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] StreamId The StreamID whose STE is being promoted. + @param [in] Cd Context Descriptor to install in the STE. + @param [in] NewL2 Caller-provided L2 page used by + SmmuV3SplitL1IfShared() when the covering L1 + descriptor still points at the shared-ABORT + L2. Because the caller owns Cd and NewL2, + this function does not allocate or free them. + @param [out] NewL2Consumed Set to TRUE if NewL2 was installed into an L1 + descriptor by the split step (the caller must + NOT free it), FALSE otherwise (the caller + should free NewL2). + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameters. + @retval Other Command-queue / sync failure. +**/ +EFI_STATUS +SmmuV3PromoteSteToStage1Translate ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 StreamId, + IN SMMUV3_CONTEXT_DESCRIPTOR *Cd, + IN SMMUV3_STREAM_TABLE_ENTRY *NewL2, + OUT BOOLEAN *NewL2Consumed + ); + +/** + Update the page table mapping with the given physical address and attributes. + + @param [in] SmmuInfo SMMU instance. + @param [in] Root Pointer to the root page table. + @param [in] TagId Per-stream tag whose TLB entries + should be invalidated on unmap. + VMID for Stage 2 SMMUs, ASID for + Stage 1 SMMUs. + @param [in] PhysicalAddress Physical address to map. + @param [in] Bytes Number of bytes to map. + @param [in] Attributes Attributes to set for the mapping. Must be a valid Stage 2 Translation Table attribute (12 bits or less). + @param [in] Valid Boolean to indicate if the entry is valid. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameter. + @retval EFI_OUT_OF_RESOURCES Out of resources. +**/ +EFI_STATUS +UpdatePageTable ( + IN SMMU_INFO *SmmuInfo, + IN PAGE_TABLE *Root, + IN UINT16 TagId, + IN UINT64 PhysicalAddress, + IN UINT64 Bytes, + IN UINT16 Attributes, + IN BOOLEAN Valid + ); diff --git a/ArmPkg/Drivers/SmmuDxe/SmmuV3Util.c b/ArmPkg/Drivers/SmmuDxe/SmmuV3Util.c new file mode 100644 index 0000000000..c48053c0da --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/SmmuV3Util.c @@ -0,0 +1,2384 @@ +/** @file Smmuv3Util.c + + This file contains util functions for the SMMU driver. + All functions are derived from the SMMU spec: + + Copyright (c) Microsoft Corporation. + SPDX-License-Identifier: BSD-2-Clause-Patent + +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "SmmuV3.h" + +/** + Decode the address width from the given address size type. + + @param [in] AddressSizeType The address size type. + + @return The decoded address width. 0 if the address size type is invalid. +**/ +UINT32 +SmmuV3DecodeAddressWidth ( + IN UINT32 AddressSizeType + ) +{ + UINT32 Length; + + switch (AddressSizeType) { + case SmmuAddressSize32Bit: + Length = 32; + break; + case SmmuAddressSize36Bit: + Length = 36; + break; + case SmmuAddressSize40Bit: + Length = 40; + break; + case SmmuAddressSize42Bit: + Length = 42; + break; + case SmmuAddressSize44Bit: + Length = 44; + break; + case SmmuAddressSize48Bit: + Length = 48; + break; + case SmmuAddressSize52Bit: + Length = 52; + break; + default: + DEBUG ((DEBUG_ERROR, "%a: Invalid Address Size Type: 0x%lx\n", __func__, AddressSizeType)); + Length = 0; + break; + } + + return Length; +} + +/** + Encode the address width to the corresponding address size type. + + @param [in] AddressWidth The address width. + + @return The encoded address size type. 0 if the address width is invalid. +**/ +UINT8 +SmmuV3EncodeAddressWidth ( + IN UINT32 AddressWidth + ) +{ + UINT8 Encoding; + + switch (AddressWidth) { + case 32: + Encoding = SmmuAddressSize32Bit; + break; + case 36: + Encoding = SmmuAddressSize36Bit; + break; + case 40: + Encoding = SmmuAddressSize40Bit; + break; + case 42: + Encoding = SmmuAddressSize42Bit; + break; + case 44: + Encoding = SmmuAddressSize44Bit; + break; + case 48: + Encoding = SmmuAddressSize48Bit; + break; + case 52: + Encoding = SmmuAddressSize52Bit; + break; + default: + DEBUG ((DEBUG_ERROR, "%a: Invalid Address Width: 0x%lx\n", __func__, AddressWidth)); + Encoding = 0; + break; + } + + return Encoding; +} + +/** + Set the translation starting level for SMMUv3 page tables. + Only 3 and 4 level paging are supported. + + Stage 2: dynamically picks between L0 (>= 44 bits OAS) and concatenated + L1 (< 44 bits OAS) to minimize walk depth. `*S2Sl0` receives the S2SL0 + encoding for the chosen starting level. + + Stage 1: concatenation at the starting level is not architecturally + allowed (Arm ARM D8.5.2). A non-concatenated single-page root covers at + most `PAGE_TABLE_CONCATENATED_PAGES_BITS_CUTOFF` bits from L1, so the + helper picks L0 for wider inputs and L1 otherwise, and forces + `PageTableRootConcatenated = FALSE`. `*S2Sl0` is zeroed because the CD + has no SL0 field (starting level is inferred from CD.T0Sz). + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] OutputAddressWidth The output address width. + @param [out] S2Sl0 The starting level for stage 2 translation. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid parameter. +**/ +EFI_STATUS +SmmuV3SetTranslationStartingLevel ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 OutputAddressWidth, + OUT UINT64 *S2Sl0 + ) +{ + if ((OutputAddressWidth > PAGE_TABLE_OUTPUT_ADDRESS_WIDTH_MAX) || (OutputAddressWidth < PAGE_TABLE_OUTPUT_ADDRESS_WIDTH_MIN)) { + DEBUG ((DEBUG_ERROR, "%a: OutputAddressWidth %d not supported.\n", __func__, OutputAddressWidth)); + return EFI_INVALID_PARAMETER; + } + + if (SmmuInfo->TranslationStage == SmmuTranslationStage1) { + // + // Stage 1: no concatenation. Single-page L1 root covers up to + // PAGE_TABLE_CONCATENATED_PAGES_BITS_CUTOFF bits; L0 covers the + // full 48-bit input at 4KB granule. + // + if (OutputAddressWidth > PAGE_TABLE_CONCATENATED_PAGES_BITS_CUTOFF) { + SmmuInfo->TranslationStartingLevel = 0; // 4-level paging (L0..L3) + } else { + SmmuInfo->TranslationStartingLevel = 1; // 3-level paging (L1..L3) + } + + SmmuInfo->PageTableRootConcatenated = FALSE; + *S2Sl0 = 0; // unused for Stage 1 (no SL0 in CD) + return EFI_SUCCESS; + } + + // Stage 2: + // Per the Arm ARM VMSA spec, >= 44 bits of address width requires 4 level paging. + // Otherwise, 3 level paging is used. + if (OutputAddressWidth >= PAGE_TABLE_4_LEVEL_OUTPUT_ADDRESS_WIDTH_MIN) { + SmmuInfo->TranslationStartingLevel = 0; // 4 level paging + *S2Sl0 = 0x2; + } else { + SmmuInfo->TranslationStartingLevel = 1; // 3 level paging + *S2Sl0 = 0x1; + // If the output address width is greater than PAGE_TABLE_CONCATENATED_PAGES_BITS_CUTOFF, the page table root must be concatenated. + SmmuInfo->PageTableRootConcatenated = (OutputAddressWidth > PAGE_TABLE_CONCATENATED_PAGES_BITS_CUTOFF); + } + + return EFI_SUCCESS; +} + +/** + Read a 32-bit value from the specified SMMU register. + + @param [in] SmmuBase The base address of the SMMU. + @param [in] Register The offset of the register. + + @return The 32-bit value read from the register. 0 if the SMMU base address is invalid. +**/ +UINT32 +SmmuV3ReadRegister32 ( + IN UINT64 SmmuBase, + IN UINT64 Register + ) +{ + if (SmmuBase == 0) { + DEBUG ((DEBUG_ERROR, "%a: Invalid SMMU base address\n", __func__)); + ASSERT (SmmuBase != 0); + return 0; + } + + return MmioRead32 (SmmuBase + Register); +} + +/** + Read a 64-bit value from the specified SMMU register. + + @param [in] SmmuBase The base address of the SMMU. + @param [in] Register The offset of the register. + + @return The 64-bit value read from the register. 0 if the SMMU base address is invalid. +**/ +UINT64 +SmmuV3ReadRegister64 ( + IN UINT64 SmmuBase, + IN UINT64 Register + ) +{ + if (SmmuBase == 0) { + DEBUG ((DEBUG_ERROR, "%a: Invalid SMMU base address\n", __func__)); + ASSERT (SmmuBase != 0); + return 0; + } + + return MmioRead64 (SmmuBase + Register); +} + +/** + Write a 32-bit value to the specified SMMU register. + + @param [in] SmmuBase The base address of the SMMU. + @param [in] Register The offset of the register. + @param [in] Value The 32-bit value to write. + + @return The 32-bit value written to the register, or 0 if the SMMU base address is invalid. +**/ +UINT32 +SmmuV3WriteRegister32 ( + IN UINT64 SmmuBase, + IN UINT64 Register, + IN UINT32 Value + ) +{ + if (SmmuBase == 0) { + DEBUG ((DEBUG_ERROR, "%a: Invalid SMMU base address\n", __func__)); + ASSERT (SmmuBase != 0); + return 0; + } + + return MmioWrite32 (SmmuBase + Register, Value); +} + +/** + Write a 64-bit value to the specified SMMU register. + + @param [in] SmmuBase The base address of the SMMU. + @param [in] Register The offset of the register. + @param [in] Value The 64-bit value to write. + + @return The 64-bit value written to the register, or 0 if the SMMU base address is invalid. +**/ +UINT64 +SmmuV3WriteRegister64 ( + IN UINT64 SmmuBase, + IN UINT64 Register, + IN UINT64 Value + ) +{ + if (SmmuBase == 0) { + DEBUG ((DEBUG_ERROR, "%a: Invalid SMMU base address\n", __func__)); + ASSERT (SmmuBase != 0); + return 0; + } + + return MmioWrite64 (SmmuBase + Register, Value); +} + +/** + Disable interrupts for the SMMUv3. + + @param [in] SmmuBase The base address of the SMMU. + @param [in] ClearStaleErrors Whether to clear stale errors. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval EFI_TIMEOUT Timeout. +**/ +EFI_STATUS +SmmuV3DisableInterrupts ( + IN UINT64 SmmuBase, + IN BOOLEAN ClearStaleErrors + ) +{ + EFI_STATUS Status; + SMMUV3_IRQ_CTRL IrqControl; + SMMUV3_GERROR GlobalErrors; + + if (SmmuBase == 0) { + DEBUG ((DEBUG_ERROR, "%a: Invalid SMMU base address\n", __func__)); + ASSERT (SmmuBase != 0); + return EFI_INVALID_PARAMETER; + } + + IrqControl.AsUINT32 = SmmuV3ReadRegister32 (SmmuBase, SMMU_IRQ_CTRL); + if ((IrqControl.AsUINT32 & SMMUV3_IRQ_CTRL_GLOBAL_PRIQ_EVTQ_EN_MASK) != 0) { + IrqControl.AsUINT32 &= ~SMMUV3_IRQ_CTRL_GLOBAL_PRIQ_EVTQ_EN_MASK; + SmmuV3WriteRegister32 (SmmuBase, SMMU_IRQ_CTRL, IrqControl.AsUINT32); + Status = SmmuV3Poll (SmmuBase, SMMU_IRQ_CTRLACK, SMMUV3_IRQ_CTRL_GLOBAL_PRIQ_EVTQ_EN_MASK, 0); + if (Status != EFI_SUCCESS) { + DEBUG ((DEBUG_ERROR, "%a: Error polling register: 0x%lx\n", __func__, SmmuBase + SMMU_IRQ_CTRLACK)); + return Status; + } + } + + if (ClearStaleErrors) { + GlobalErrors.AsUINT32 = SmmuV3ReadRegister32 (SmmuBase, SMMU_GERROR); + GlobalErrors.AsUINT32 = GlobalErrors.AsUINT32 & SMMUV3_GERROR_VALID_MASK; + SmmuV3WriteRegister32 (SmmuBase, SMMU_GERRORN, GlobalErrors.AsUINT32); + } + + return EFI_SUCCESS; +} + +/** + Enable interrupts for the SMMUv3. + + @param [in] SmmuBase The base address of the SMMU. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval EFI_TIMEOUT Timeout. +**/ +EFI_STATUS +SmmuV3EnableInterrupts ( + IN UINT64 SmmuBase + ) +{ + EFI_STATUS Status; + SMMUV3_IRQ_CTRL IrqControl; + + if (SmmuBase == 0) { + DEBUG ((DEBUG_ERROR, "%a: Invalid SMMU base address\n", __func__)); + ASSERT (SmmuBase != 0); + return EFI_INVALID_PARAMETER; + } + + IrqControl.AsUINT32 = SmmuV3ReadRegister32 (SmmuBase, SMMU_IRQ_CTRL); + IrqControl.AsUINT32 &= ~SMMUV3_IRQ_CTRL_GLOBAL_PRIQ_EVTQ_EN_MASK; + IrqControl.Bits.GlobalErrorIrqEn = 1; + IrqControl.Bits.EventqIrqEn = 1; + SmmuV3WriteRegister32 (SmmuBase, SMMU_IRQ_CTRL, IrqControl.AsUINT32); + Status = SmmuV3Poll (SmmuBase, SMMU_IRQ_CTRLACK, 0x5, 0x5); + if (Status != EFI_SUCCESS) { + DEBUG ((DEBUG_ERROR, "%a: Error polling register: 0x%lx\n", __func__, SmmuBase + SMMU_IRQ_CTRLACK)); + } + + return Status; +} + +/** + Disable translation for the SMMUv3. + + @param [in] SmmuBase The base address of the SMMU. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval EFI_TIMEOUT Timeout. +**/ +EFI_STATUS +SmmuV3DisableTranslation ( + IN UINT64 SmmuBase + ) +{ + SMMUV3_CR0 Cr0; + EFI_STATUS Status; + + if (SmmuBase == 0) { + DEBUG ((DEBUG_ERROR, "%a: Invalid SMMU base address\n", __func__)); + ASSERT (SmmuBase != 0); + return EFI_INVALID_PARAMETER; + } + + Cr0.AsUINT32 = SmmuV3ReadRegister32 (SmmuBase, SMMU_CR0); + if ((Cr0.AsUINT32 & SMMUV3_CR0_SMMU_CMDQ_EVTQ_PRIQ_EN_MASK) != 0) { + Cr0.AsUINT32 = Cr0.AsUINT32 & ~SMMUV3_CR0_SMMU_CMDQ_EVTQ_PRIQ_EN_MASK; + SmmuV3WriteRegister32 (SmmuBase, SMMU_CR0, Cr0.AsUINT32); + Status = SmmuV3Poll (SmmuBase, SMMU_CR0ACK, SMMUV3_CR0_SMMU_CMDQ_EVTQ_PRIQ_EN_MASK, 0); + if (Status != EFI_SUCCESS) { + DEBUG ((DEBUG_ERROR, "%a: Error polling register: 0x%lx\n", __func__, SmmuBase + SMMU_CR0ACK)); + return Status; + } + } + + return EFI_SUCCESS; +} + +/** + Set the Smmu in ABORT mode and stop DMA. + + @param [in] SmmuBase Base address of the SMMUv3. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval EFI_TIMEOUT Timeout. +**/ +EFI_STATUS +SmmuV3GlobalAbort ( + IN UINT64 SmmuBase + ) +{ + EFI_STATUS Status; + UINT32 RegVal; + + if (SmmuBase == 0) { + DEBUG ((DEBUG_ERROR, "%a: Invalid SMMU base address\n", __func__)); + ASSERT (SmmuBase != 0); + return EFI_INVALID_PARAMETER; + } + + // Attribute update has completed when SMMU_(S)_GBPA.Update bit is 0. + Status = SmmuV3Poll (SmmuBase, SMMU_GBPA, SMMU_GBPA_UPDATE, 0); + if (EFI_ERROR (Status)) { + return Status; + } + + // SMMU_(S)_CR0 resets to zero with all streams bypassing the SMMU, + // so just abort all incoming transactions. + RegVal = SmmuV3ReadRegister32 (SmmuBase, SMMU_GBPA); + + // Set the SMMU_GBPA.ABORT and SMMU_GBPA.UPDATE. + RegVal |= (SMMU_GBPA_ABORT | SMMU_GBPA_UPDATE); + + SmmuV3WriteRegister32 (SmmuBase, SMMU_GBPA, RegVal); + + // Attribute update has completed when SMMU_(S)_GBPA.Update bit is 0. + Status = SmmuV3Poll (SmmuBase, SMMU_GBPA, SMMU_GBPA_UPDATE, 0); + if (EFI_ERROR (Status)) { + return Status; + } + + // Sanity check to see if abort is set + Status = SmmuV3Poll (SmmuBase, SMMU_GBPA, SMMU_GBPA_ABORT, SMMU_GBPA_ABORT); + if (EFI_ERROR (Status)) { + return Status; + } + + return EFI_SUCCESS; +} + +/** + Set all streams to bypass the SMMU. + + @param [in] SmmuBase Base address of the SMMUv3. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +EFI_STATUS +SmmuV3SetGlobalBypass ( + IN UINT64 SmmuBase + ) +{ + EFI_STATUS Status; + UINT32 RegVal; + + if (SmmuBase == 0) { + DEBUG ((DEBUG_ERROR, "%a: Invalid SMMU base address\n", __func__)); + ASSERT (SmmuBase != 0); + return EFI_INVALID_PARAMETER; + } + + // Attribute update has completed when SMMU_(S)_GBPA.Update bit is 0. + Status = SmmuV3Poll (SmmuBase, SMMU_GBPA, SMMU_GBPA_UPDATE, 0); + if (EFI_ERROR (Status)) { + return Status; + } + + // SMMU_(S)_CR0 resets to zero with all streams bypassing the SMMU + RegVal = SmmuV3ReadRegister32 (SmmuBase, SMMU_GBPA); + + // Clear the SMMU_GBPA.ABORT to allow Non-secure streams to bypass + // the SMMU. + RegVal &= ~SMMU_GBPA_ABORT; + RegVal |= SMMU_GBPA_UPDATE; + + SmmuV3WriteRegister32 (SmmuBase, SMMU_GBPA, RegVal); + + Status = SmmuV3Poll (SmmuBase, SMMU_GBPA, SMMU_GBPA_UPDATE, 0); + if (EFI_ERROR (Status)) { + return Status; + } + + return EFI_SUCCESS; +} + +/** + Poll the SMMU register and test the value based on the mask. + + @param [in] SmmuBase Base address of the SMMU. + @param [in] SmmuReg The SMMU register to poll. + @param [in] Mask Mask of register bits to monitor. + @param [in] Value Expected value. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +EFI_STATUS +SmmuV3Poll ( + IN UINT64 SmmuBase, + IN UINT64 SmmuReg, + IN UINT32 Mask, + IN UINT32 Value + ) +{ + UINT32 RegVal; + UINTN Count; + + if (SmmuBase == 0) { + DEBUG ((DEBUG_ERROR, "%a: Invalid SMMU base address\n", __func__)); + ASSERT (SmmuBase != 0); + return EFI_INVALID_PARAMETER; + } + + // Set 0.1ms timeout value. + Count = 10; + do { + RegVal = SmmuV3ReadRegister32 (SmmuBase, SmmuReg); + if ((RegVal & Mask) == Value) { + return EFI_SUCCESS; + } + + MicroSecondDelay (10); + } while ((--Count) > 0); + + DEBUG (( + DEBUG_ERROR, + "%a: Timeout polling SMMUv3 register @%p Read value 0x%x " + "expected 0x%x\n", + __func__, + SmmuReg, + RegVal, + ((Value == 0) ? (RegVal & ~Mask) : (RegVal | Mask)) + )); + + return EFI_TIMEOUT; +} + +/** + Consume the event queue for errors and retrieve the fault record. + Clears the outputted FaultRecord if the queue is empty. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [out] FaultRecord Pointer to the fault record structure. + @param [out] IsEmpty Flag to indicate if the queue is empty. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +EFI_STATUS +SmmuV3ConsumeEventQueueForErrors ( + IN SMMU_INFO *SmmuInfo, + OUT SMMUV3_FAULT_RECORD *FaultRecord, + OUT BOOLEAN *IsEmpty + ) +{ + SMMUV3_EVENTQ_CONS Consumer; + UINT32 ConsumerIndex; + UINT32 ConsumerWrap; + SMMUV3_FAULT_RECORD *NextFault; + SMMUV3_EVENTQ_PROD Producer; + UINT32 ProducerIndex; + UINT32 ProducerWrap; + BOOLEAN QueueEmpty; + UINT32 QueueMask; + UINT32 TotalQueueEntries; + UINT32 WrapMask; + + if ((SmmuInfo == NULL) || ((FaultRecord == NULL) || (IsEmpty == NULL))) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + TotalQueueEntries = SMMUV3_COUNT_FROM_LOG2 (SmmuInfo->EventQueueLog2Size); + WrapMask = TotalQueueEntries; + QueueMask = TotalQueueEntries - 1; + + Producer.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase + SMMUV3_PAGE_1_OFFSET, SMMU_EVENTQ_PROD); + Consumer.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase + SMMUV3_PAGE_1_OFFSET, SMMU_EVENTQ_CONS); + + ProducerIndex = Producer.Bits.WriteIndex & QueueMask; + ProducerWrap = Producer.Bits.WriteIndex & WrapMask; + ConsumerIndex = Consumer.Bits.ReadIndex & QueueMask; + ConsumerWrap = Consumer.Bits.ReadIndex & WrapMask; + QueueEmpty = SMMUV3_IS_QUEUE_EMPTY ( + ProducerIndex, + ProducerWrap, + ConsumerIndex, + ConsumerWrap + ); + + if (QueueEmpty != FALSE) { + *IsEmpty = TRUE; + goto End; + } + + *IsEmpty = FALSE; + NextFault = (SMMUV3_FAULT_RECORD *)SmmuInfo->EventQueue + ConsumerIndex; + CopyMem (FaultRecord, NextFault, SMMUV3_EVENT_QUEUE_ENTRY_SIZE); + + ConsumerIndex += 1; + if (ConsumerIndex == TotalQueueEntries) { + ConsumerIndex = 0; + ConsumerWrap = ConsumerWrap ^ WrapMask; + } + + Consumer.Bits.ReadIndex = ConsumerIndex | ConsumerWrap; + + ArmDataSynchronizationBarrier (); + + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase + SMMUV3_PAGE_1_OFFSET, SMMU_EVENTQ_CONS, Consumer.AsUINT32); + +End: + return EFI_SUCCESS; +} + +/** + Dump the page table entries for a given virtual address. + Dumps PTE's for all levels regardless of the starting level chosen for translation. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] VirtualAddress The virtual address to dump. + @param [in] Root Pointer to the root page table. +**/ +VOID +SmmuV3DumpPageTableEntries ( + IN SMMU_INFO *SmmuInfo, + IN UINT64 VirtualAddress, + IN PAGE_TABLE *Root + ) +{ + UINTN Index; + UINT8 Level; + PAGE_TABLE *Current; + + if ((SmmuInfo == NULL) || (Root == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: No page-table root to dump (SmmuInfo=%p Root=%p)\n", __func__, SmmuInfo, Root)); + return; + } + + Current = Root; + + for (Level = SmmuInfo->TranslationStartingLevel; Level < PAGE_TABLE_DEPTH; Level++) { + UINT64 Entry; + + Index = PAGE_TABLE_INDEX (VirtualAddress, Level, SmmuInfo->OutputAddressWidth, SmmuInfo->TranslationStartingLevel, SmmuInfo->PageTableRootConcatenated); + Entry = Current->Entries[Index]; + + if (Entry == 0) { + DEBUG ((DEBUG_ERROR, "%a: Invalid entry at level %d, index %d\n", __func__, Level, Index)); + break; + } + + DEBUG ((DEBUG_INFO, "%a: VirtualAddress = %llx Level = %d Current->Entries[%d] = 0x%llx\n", __func__, VirtualAddress, Level, Index, Entry)); + Current = (PAGE_TABLE *)((UINTN)Entry & ~0xFFF); + } +} + +/** + Check if an entire address range has a valid identity mapping in the + stage-2 translation table. + + Walks Root using SmmuInfo's translation parameters for every 4 KB page in + [Address, Address + Pages * EFI_PAGE_SIZE). Address is rounded down to the + enclosing page. Since this driver identity-maps DMA a page counts as mapped only when: + - every intermediate level has a non-zero descriptor, + - the leaf entry has PAGE_TABLE_ENTRY_VALID_BIT set, and + - the leaf entry's encoded physical address matches the page address. + + Returns TRUE only if every page in the range satisfies the above; FALSE on + the first page that fails (short-circuit), bad parameters, or Pages == 0. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure whose translation + parameters drive the walk. + @param [in] Root Pointer to the root page table. + @param [in] Address Start address of the range (same value used as both + VA and PA under identity mapping). Aligned down to + the enclosing 4 KB page. + @param [in] Pages Number of 4 KB pages to check, starting at Address. + + @retval TRUE Every page in the range has a valid identity-mapped leaf. + @retval FALSE At least one page is not mapped, encodes a different PA, or + a required parameter is invalid. +**/ +BOOLEAN +SmmuV3IsAddressRangeMapped ( + IN SMMU_INFO *SmmuInfo, + IN PAGE_TABLE *Root, + IN UINT64 Address, + IN UINTN Pages + ) +{ + UINT8 Level; + UINT32 Index; + PAGE_TABLE *Current; + UINT64 Entry; + UINT64 PageAddress; + UINTN PageIndex; + + if ((SmmuInfo == NULL) || (Root == NULL) || (Pages == 0)) { + return FALSE; + } + + PageAddress = Address & ~(UINT64)PAGE_TABLE_BLOCK_MASK; + + for (PageIndex = 0; PageIndex < Pages; PageIndex++) { + Current = Root; + + // Walk intermediate levels. A missing descriptor means this page is not mapped. + for (Level = SmmuInfo->TranslationStartingLevel; Level < PAGE_TABLE_DEPTH - 1; Level++) { + Index = PAGE_TABLE_INDEX (PageAddress, Level, SmmuInfo->OutputAddressWidth, SmmuInfo->TranslationStartingLevel, SmmuInfo->PageTableRootConcatenated); + Entry = Current->Entries[Index]; + + if (Entry == 0) { + return FALSE; + } + + Current = (PAGE_TABLE *)((UINTN)Entry & ~PAGE_TABLE_BLOCK_MASK); + } + + ASSERT (Current != NULL); + + // Leaf level: mapped iff the valid bit is set AND the encoded PA matches + // PageAddress (identity mapping). + Index = PAGE_TABLE_INDEX (PageAddress, Level, SmmuInfo->OutputAddressWidth, SmmuInfo->TranslationStartingLevel, SmmuInfo->PageTableRootConcatenated); + Entry = Current->Entries[Index]; + + if (((Entry & PAGE_TABLE_ENTRY_VALID_BIT) == 0) || + ((Entry & ~PAGE_TABLE_BLOCK_MASK) != PageAddress)) + { + return FALSE; + } + + PageAddress += EFI_PAGE_SIZE; + } + + return TRUE; +} + +/** + Log the errors if found from the SMMUv3 and asserts. Prints Event Queue entries and GError register. + Does nothing if no errors found. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + + @retval EFI_SUCCESS No SMMU errors found. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval EFI_DEVICE_ERROR SMMU error found. +**/ +EFI_STATUS +SmmuV3LogErrors ( + IN SMMU_INFO *SmmuInfo + ) +{ + SMMUV3_GERROR GError; + SMMUV3_FAULT_RECORD FaultRecord; + UINTN Index; + BOOLEAN IsEmpty; + EFI_STATUS Status; + EFI_STATUS EventQueueStatus; + EFI_TPL OldTpl; + UINT8 FaultType; + BOOLEAN IsTranslErr; + BOOLEAN IsConfigErr; + SMMUV3_STREAM_TABLE_ENTRY *SteSlot; + + if (SmmuInfo == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Status = EFI_SUCCESS; + + do { + // Only consumes one entry at a time, so we loop until empty + OldTpl = gBS->RaiseTPL (TPL_HIGH_LEVEL); + EventQueueStatus = SmmuV3ConsumeEventQueueForErrors (SmmuInfo, &FaultRecord, &IsEmpty); + gBS->RestoreTPL (OldTpl); + if (EFI_ERROR (EventQueueStatus)) { + DEBUG ((DEBUG_ERROR, "%a: Error consuming event queue\n", __func__)); + return EventQueueStatus; + } + + if (IsEmpty == FALSE) { + Status = EFI_DEVICE_ERROR; + DEBUG ((DEBUG_ERROR, "%a: SmmuBase=0x%llx StreamId=0x%x FaultRecord:\n", __func__, SmmuInfo->SmmuBase, FaultRecord.Translation.StreamId)); + for (Index = 0; Index < sizeof (FaultRecord.Fault) / sizeof (FaultRecord.Fault[0]); Index++) { + DEBUG ((DEBUG_ERROR, "0x%llx\n", FaultRecord.Fault[Index])); + } + + FaultType = (UINT8)(FaultRecord.Fault[0] & 0xFF); + IsTranslErr = (FaultType == 0x10) || (FaultType == 0x11) || (FaultType == 0x12) || (FaultType == 0x13); + // Config-error class: C_BAD_STREAMID / C_BAD_STE / C_BAD_SUBSTREAMID / C_BAD_CD. + IsConfigErr = (FaultType == 0x02) || (FaultType == 0x04) || (FaultType == 0x08) || (FaultType == 0x0A); + + // On config-error faults dump raw STE bytes and IDR fingerprint so the offending field is visible. + if (IsConfigErr) { + UINT32 Idr0Raw; + UINT32 Idr1Raw; + UINT32 Idr3Raw; + UINT32 Idr5Raw; + + Idr0Raw = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR0); + Idr1Raw = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR1); + Idr3Raw = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR3); + Idr5Raw = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR5); + DEBUG (( + DEBUG_ERROR, + "%a: SMMU 0x%llx IDR0=0x%08x IDR1=0x%08x IDR3=0x%08x IDR5=0x%08x TranslationStage=%u Flags=0x%x\n", + __func__, + SmmuInfo->SmmuBase, + Idr0Raw, + Idr1Raw, + Idr3Raw, + Idr5Raw, + (UINT32)SmmuInfo->TranslationStage, + SmmuInfo->Flags + )); + + OldTpl = gBS->RaiseTPL (TPL_HIGH_LEVEL); + SteSlot = SmmuV3GetSteSlot (SmmuInfo, FaultRecord.Translation.StreamId); + if (SteSlot != NULL) { + DEBUG (( + DEBUG_ERROR, + "%a: STE bytes for StreamId=0x%x SteSlot=%p:\n" + " [0]=0x%016llx [1]=0x%016llx [2]=0x%016llx [3]=0x%016llx\n" + " [4]=0x%016llx [5]=0x%016llx [6]=0x%016llx [7]=0x%016llx\n", + __func__, + FaultRecord.Translation.StreamId, + SteSlot, + SteSlot->AsUINT64[0], + SteSlot->AsUINT64[1], + SteSlot->AsUINT64[2], + SteSlot->AsUINT64[3], + SteSlot->AsUINT64[4], + SteSlot->AsUINT64[5], + SteSlot->AsUINT64[6], + SteSlot->AsUINT64[7] + )); + + if ((SmmuInfo->TranslationStage == SmmuTranslationStage1) && (SteSlot->Bits.S1ContextPtr != 0)) { + SMMUV3_CONTEXT_DESCRIPTOR *DumpCd; + + DumpCd = (SMMUV3_CONTEXT_DESCRIPTOR *)(UINTN)((UINT64)SteSlot->Bits.S1ContextPtr << SMMUV3_STREAM_TABLE_ENTRY_S1CONTEXTPTR_OFFSET); + DEBUG (( + DEBUG_ERROR, + "%a: CD bytes at %p:\n" + " [0]=0x%016llx [1]=0x%016llx [2]=0x%016llx [3]=0x%016llx\n" + " [4]=0x%016llx [5]=0x%016llx [6]=0x%016llx [7]=0x%016llx\n", + __func__, + DumpCd, + DumpCd->AsUINT64[0], + DumpCd->AsUINT64[1], + DumpCd->AsUINT64[2], + DumpCd->AsUINT64[3], + DumpCd->AsUINT64[4], + DumpCd->AsUINT64[5], + DumpCd->AsUINT64[6], + DumpCd->AsUINT64[7] + )); + } + } else { + DEBUG ((DEBUG_ERROR, "%a: Active STE for StreamId=0x%x: not found\n", __func__, FaultRecord.Translation.StreamId)); + } + + gBS->RestoreTPL (OldTpl); + } + + // Dump PTE's if translation related fault. Stage 2: STE.S2Ttb. + // Stage 1: STE.S1ContextPtr -> CD.Ttb0. + if (IsTranslErr) { + UINT32 FaultStreamId; + PAGE_TABLE *FaultRoot; + SMMUV3_STRTAB_BASE StrTabBaseReg; + VOID *HwStreamTableBase; + SMMUV3_CONTEXT_DESCRIPTOR *FaultCd; + + FaultStreamId = FaultRecord.Translation.StreamId; + FaultRoot = NULL; + FaultCd = NULL; + HwStreamTableBase = NULL; + + // Read back SMMU_STRTAB_BASE from hardware and compare its decoded + // stream-table base with SmmuInfo->StreamTable. + StrTabBaseReg.AsUINT64 = SmmuV3ReadRegister64 (SmmuInfo->SmmuBase, SMMU_STRTAB_BASE); + HwStreamTableBase = (VOID *)(UINTN)(StrTabBaseReg.Bits.Addr << SMMUV3_STR_TAB_BASE_ADDR_OFFSET); + DEBUG (( + DEBUG_ERROR, + "%a: STRTAB_BASE reg=0x%llx decoded base=%p software base=%p\n", + __func__, + StrTabBaseReg.AsUINT64, + HwStreamTableBase, + SmmuInfo->StreamTable + )); + if (HwStreamTableBase != SmmuInfo->StreamTable) { + DEBUG (( + DEBUG_ERROR, + "%a: Stream table base mismatch: hw=%p sw=%p\n", + __func__, + HwStreamTableBase, + SmmuInfo->StreamTable + )); + } + + OldTpl = gBS->RaiseTPL (TPL_HIGH_LEVEL); + // Dump the currently active STE fields for this StreamID and use + // the stage-appropriate root (Stage 2: S2Ttb; Stage 1: CD.Ttb0) + // to walk the page table. + SteSlot = SmmuV3GetSteSlot (SmmuInfo, FaultStreamId); + if (SteSlot != NULL) { + if (SmmuInfo->TranslationStage == SmmuTranslationStage1) { + FaultCd = (SMMUV3_CONTEXT_DESCRIPTOR *)(UINTN)((UINT64)SteSlot->Bits.S1ContextPtr << SMMUV3_STREAM_TABLE_ENTRY_S1CONTEXTPTR_OFFSET); + if (FaultCd != NULL) { + FaultRoot = (PAGE_TABLE *)(UINTN)((UINT64)FaultCd->Bits.Ttb0 << SMMUV3_CD_TTB0_OFFSET); + } + + DEBUG (( + DEBUG_ERROR, + "%a: Active Stage 1 STE for StreamId=0x%x: Valid=%u Config=0x%x S1ContextPtr=0x%llx CD=%p ASID=0x%x Ttb0=0x%llx Root=%p\n", + __func__, + FaultStreamId, + SteSlot->Bits.Valid, + SteSlot->Bits.Config, + (UINT64)SteSlot->Bits.S1ContextPtr, + FaultCd, + (FaultCd != NULL) ? (UINT32)FaultCd->Bits.Asid : 0, + (FaultCd != NULL) ? (UINT64)FaultCd->Bits.Ttb0 : 0, + FaultRoot + )); + } else { + FaultRoot = (PAGE_TABLE *)(UINTN)((UINT64)SteSlot->Bits.S2Ttb << SMMUV3_STREAM_TABLE_ENTRY_S2TTB_OFFSET); + DEBUG (( + DEBUG_ERROR, + "%a: Active Stage 2 STE for StreamId=0x%x: Valid=%u Config=0x%x S2Ttb=0x%llx S2Vmid=0x%x Root=%p\n", + __func__, + FaultStreamId, + SteSlot->Bits.Valid, + SteSlot->Bits.Config, + (UINT64)SteSlot->Bits.S2Ttb, + (UINT32)SteSlot->Bits.S2Vmid, + FaultRoot + )); + } + } else { + DEBUG ((DEBUG_ERROR, "%a: Active STE for StreamId=0x%x: not found\n", __func__, FaultStreamId)); + } + + if ((SteSlot != NULL) && (SteSlot->Bits.Valid != 0) && (FaultRoot != NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Dumping page table for StreamId=0x%x Root=%p\n", __func__, FaultStreamId, FaultRoot)); + SmmuV3DumpPageTableEntries (SmmuInfo, FaultRecord.Fault[2], FaultRoot); + } else if ((SteSlot != NULL) && (SteSlot->Bits.Valid == 0)) { + DEBUG ((DEBUG_ERROR, "%a: StreamId=0x%x has no page-table root (STE still in invalid mode)\n", __func__, FaultStreamId)); + } else if (SteSlot != NULL) { + // Valid = 1 but per-stream root could not be decoded (e.g., Stage 1 + // STE with S1ContextPtr = 0, or Stage 2 STE with S2Ttb = 0). + DEBUG ((DEBUG_ERROR, "%a: StreamId=0x%x STE Valid=1 but per-stream root could not be decoded\n", __func__, FaultStreamId)); + } + + gBS->RestoreTPL (OldTpl); + } + } + } while (IsEmpty == FALSE); + + GError.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_GERROR); + if (GError.AsUINT32 != 0) { + Status = EFI_DEVICE_ERROR; + DEBUG ((DEBUG_ERROR, "%a: %llx GError: 0x%lx\n", __func__, SmmuInfo->SmmuBase, GError.AsUINT32)); + } + + // Assert if we found any errors in the event queue or GError register + ASSERT_EFI_ERROR (Status); + return Status; +} + +/** + Write commands to the SMMUv3 command queue. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] StartingIndex The starting index in the command queue. + @param [in] CommandCount The number of commands to write. + @param [in] Commands Pointer to the commands to write. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +STATIC +EFI_STATUS +SmmuV3WriteCommands ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 StartingIndex, + IN UINT32 CommandCount, + IN SMMUV3_CMD_GENERIC *Commands + ) +{ + UINT32 Index; + UINT32 ProducerIndex; + UINT32 QueueMask; + UINT32 WrapMask; + SMMUV3_CMD_GENERIC *CommandQueue; + + if ((SmmuInfo == NULL) || (Commands == NULL) || (CommandCount == 0)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + WrapMask = (1UL << SmmuInfo->CommandQueueLog2Size); + QueueMask = WrapMask - 1; + CommandQueue = (SMMUV3_CMD_GENERIC *)SmmuInfo->CommandQueue; + for (Index = 0; Index < CommandCount; Index += 1) { + ProducerIndex = (UINT32)((StartingIndex + Index) & QueueMask); + CommandQueue[ProducerIndex] = Commands[Index]; + } + + // This DSB ensures that all commands written to the command queue before this point will be visible + // before we update the producer index register to actually trigger processing of the commands. + ArmDataSynchronizationBarrier (); + + return EFI_SUCCESS; +} + +/** + Update the cached consumer index for the SMMUv3 command queue. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] QueueMask The queue mask. + @param [in] WrapMask The wrap mask. + @param [in] TotalQueueEntries The total number of entries in the queue. + @param [out] ConsumerIndexOut Pointer to store the consumer index. + @param [out] ConsumerWrapOut Pointer to store the consumer wrap. +**/ +VOID +SmmuV3CmdQueueUpdateCachedConsumer ( + IN SMMU_INFO *SmmuInfo, + IN UINT32 QueueMask, + IN UINT32 WrapMask, + IN UINT32 TotalQueueEntries, + OUT UINT32 *ConsumerIndexOut, + OUT UINT32 *ConsumerWrapOut + ) +{ + SMMUV3_CMDQ_CONS Consumer; + UINT32 ConsumerIndex; + UINT32 ConsumerWrap; + UINT64 CachedConsumerWrap; + + if ((SmmuInfo == NULL) || (ConsumerIndexOut == NULL) || (ConsumerWrapOut == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); + return; + } + + Consumer.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_CONS); + ConsumerIndex = Consumer.Bits.ReadIndex & QueueMask; + ConsumerWrap = Consumer.Bits.ReadIndex & WrapMask; + *ConsumerIndexOut = ConsumerIndex; + *ConsumerWrapOut = ConsumerWrap; + + CachedConsumerWrap = SmmuInfo->CachedConsumer & WrapMask; + SmmuInfo->CachedConsumer = (SmmuInfo->CachedConsumer & ~QueueMask) | ConsumerIndex; + + if (CachedConsumerWrap != ConsumerWrap) { + SmmuInfo->CachedConsumer += TotalQueueEntries; + } +} + +/** + SMMU ISR Handler. Drains the event queue and dumps GError / fault records + for the SMMU instance whose EVTQ or GERR interrupt fired. + + @param[in] Source The interrupt source. + @param[in] SystemContext The system context. +**/ +STATIC +VOID +EFIAPI +SmmuV3IsrHandler ( + IN HARDWARE_INTERRUPT_SOURCE Source, + IN EFI_SYSTEM_CONTEXT SystemContext + ) +{ + SMMU_INFO *SmmuInfo; + UINT32 SmmuIndex; + EFI_STATUS Status; + + DEBUG ((DEBUG_INFO, "%a: ISR received for source %d\n", __func__, Source)); + + if ((mIoMmu == NULL) || (mIoMmu->SmmuInfo == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: IOMMU_CONFIG/SMMU_INFO structure is NULL\n", __func__)); + return; + } + + for (SmmuIndex = 0; SmmuIndex < mIoMmu->SmmuCount; SmmuIndex++) { + SmmuInfo = &mIoMmu->SmmuInfo[SmmuIndex]; + if (SmmuInfo->Enabled) { + if ((Source == SmmuInfo->EvtqIrqNum) || (Source == SmmuInfo->GerrIrqNum)) { + SmmuV3LogErrors (SmmuInfo); + break; + } + } + } + + Status = mGicInterrupt->EndOfInterrupt (mGicInterrupt, Source); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error ending interrupt\n", __func__)); + return; + } + + DEBUG ((DEBUG_INFO, "%a: ISR handled for source %d\n", __func__, Source)); +} + +/** + Register a single GIC interrupt source with the SmmuV3 ISR handler. + + @param[in] GicInterrupt Pointer to the GIC interrupt protocol. + @param[in] Source The interrupt source to register. + + @retval EFI_SUCCESS The interrupt source was registered successfully. + @retval EFI_INVALID_PARAMETER GicInterrupt or Source is invalid. +**/ +STATIC +EFI_STATUS +SmmuV3RegisterInterruptSource ( + IN EFI_HARDWARE_INTERRUPT2_PROTOCOL *GicInterrupt, + IN HARDWARE_INTERRUPT_SOURCE Source + ) +{ + EFI_STATUS Status; + BOOLEAN InterruptState; + + InterruptState = FALSE; + + DEBUG ((DEBUG_VERBOSE, "%a: Registering GIC interrupt for source %d\n", __func__, Source)); + + if ((GicInterrupt == NULL) || (Source == 0)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Status = GicInterrupt->RegisterInterruptSource ( + GicInterrupt, + Source, + SmmuV3IsrHandler + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error registering interrupt source\n", __func__)); + return Status; + } + + Status = GicInterrupt->SetTriggerType ( + GicInterrupt, + Source, + EFI_HARDWARE_INTERRUPT2_TRIGGER_EDGE_RISING + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error setting trigger type\n", __func__)); + return Status; + } + + Status = GicInterrupt->GetInterruptSourceState ( + GicInterrupt, + Source, + &InterruptState + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error getting interrupt state\n", __func__)); + return Status; + } + + if (InterruptState == FALSE) { + Status = GicInterrupt->EnableInterruptSource (GicInterrupt, Source); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error enabling interrupt source\n", __func__)); + return Status; + } + } + + return EFI_SUCCESS; +} + +/** + Register GIC interrupt sources for SmmuV3 EVTQ and GERR interrupts. + + @param[in] GicInterrupt Pointer to the GIC interrupt protocol. + @param[in] SmmuInfo Pointer to the SMMU_INFO structure. + + @retval EFI_SUCCESS The interrupt sources were registered successfully. + @retval EFI_INVALID_PARAMETER GicInterrupt or SmmuInfo is NULL. +**/ +EFI_STATUS +SmmuV3RegisterGicIsr ( + IN EFI_HARDWARE_INTERRUPT2_PROTOCOL *GicInterrupt, + IN SMMU_INFO *SmmuInfo + ) +{ + EFI_STATUS Status; + + if ((GicInterrupt == NULL) || (SmmuInfo == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Status = SmmuV3RegisterInterruptSource (GicInterrupt, SmmuInfo->EvtqIrqNum); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error registering EVTQ interrupt\n", __func__)); + return Status; + } + + Status = SmmuV3RegisterInterruptSource (GicInterrupt, SmmuInfo->GerrIrqNum); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error registering GERR interrupt\n", __func__)); + return Status; + } + + return EFI_SUCCESS; +} + +/** + Send a SMMUV3_CMD_GENERIC command to the SMMUv3. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] Command Pointer to the command to send. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +EFI_STATUS +SmmuV3SendCommand ( + IN SMMU_INFO *SmmuInfo, + IN SMMUV3_CMD_GENERIC *Command + ) +{ + UINT32 QueueMask; + UINT32 WrapMask; + UINT32 TotalQueueEntries; + UINT32 ProducerIndex; + UINT32 ConsumerIndex; + UINT32 ProducerWrap; + UINT32 ConsumerWrap; + SMMUV3_CMDQ_PROD Producer; + SMMUV3_CMDQ_CONS Consumer; + EFI_STATUS Status; + EFI_TPL OldTpl; + UINT64 NewProducer; + + if ((SmmuInfo == NULL) || (Command == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + TotalQueueEntries = SMMUV3_COUNT_FROM_LOG2 (SmmuInfo->CommandQueueLog2Size); + WrapMask = TotalQueueEntries; + QueueMask = WrapMask - 1; + + // We need to synchronize the the entire command queue write and producer update with the TPL locks. + // As a result we don't just lock SmmuV3CmdQueueUpdateCachedConsumer but the entire process of writing the command + // and updating the producer index. + OldTpl = gBS->RaiseTPL (TPL_HIGH_LEVEL); + + // Loop until there is space in the command queue + do { + Producer.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_PROD); + ProducerWrap = Producer.Bits.WriteIndex & WrapMask; + ProducerIndex = Producer.Bits.WriteIndex & QueueMask; + + SmmuV3CmdQueueUpdateCachedConsumer (SmmuInfo, QueueMask, WrapMask, TotalQueueEntries, &ConsumerIndex, &ConsumerWrap); + } while (SMMUV3_IS_QUEUE_FULL (ProducerIndex, ProducerWrap, ConsumerIndex, ConsumerWrap) != FALSE); + + Status = SmmuV3WriteCommands (SmmuInfo, ProducerIndex, 1, Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Error writing command to queue\n", __func__)); + gBS->RestoreTPL (OldTpl); + return Status; + } + + SmmuInfo->CachedProducer += 1; + NewProducer = SmmuInfo->CachedProducer; + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_PROD, (UINT32)(NewProducer & (WrapMask | QueueMask))); + + gBS->RestoreTPL (OldTpl); + + // Loop until the command is consumed + do { + // SmmuV3CmdQueueUpdateCachedConsumer needs to be within the scope of this lock because we want to make sure we have + // the synchronized view of the consumer index when checking against the current local producer index. + OldTpl = gBS->RaiseTPL (TPL_HIGH_LEVEL); + SmmuV3CmdQueueUpdateCachedConsumer (SmmuInfo, QueueMask, WrapMask, TotalQueueEntries, &ConsumerIndex, &ConsumerWrap); + + // Check for command queue errors before continuing to poll + Consumer.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_CONS); + if (Consumer.Bits.Err != 0) { + DEBUG ((DEBUG_ERROR, "%a: Command queue error detected: CONS.ERR=0x%x\n", __func__, Consumer.Bits.Err)); + gBS->RestoreTPL (OldTpl); + ASSERT_EFI_ERROR (EFI_DEVICE_ERROR); + return EFI_DEVICE_ERROR; + } + + gBS->RestoreTPL (OldTpl); + } while (SmmuInfo->CachedConsumer < NewProducer); + + return Status; +} + +/** + Invalidate all Stage 2 TLB entries owned by the given VMID on this SMMU. + Uses CMD_TLBI_S12_VMALL. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] Vmid The VMID to invalidate. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +EFI_STATUS +SmmuV3TLBInvalidateAllStage2 ( + IN SMMU_INFO *SmmuInfo, + IN UINT16 Vmid + ) +{ + SMMUV3_CMD_GENERIC Command; + EFI_STATUS Status; + + if ((SmmuInfo == NULL) || (Vmid == 0)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + // DSB before invalidating TLBs + ArmDataSynchronizationBarrier (); + + // Invalidate TLBI Command by Vmid + SMMUV3_BUILD_CMD_TLBI_S12_VMALL (&Command, Vmid); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CMD_TLBI_S12_VMALL failed for Vmid 0x%llx.\n", __func__, Vmid)); + return Status; + } + + // Issue a CMD_SYNC command to guarantee that any previously issued TLB + // invalidations (CMD_TLBI_*) are completed + SMMUV3_BUILD_CMD_SYNC_NO_INTERRUPT (&Command); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CMD_SYNC_NO_INTERRUPT failed.\n", __func__)); + return Status; + } + + ArmDataSynchronizationBarrier (); + + return Status; +} + +/** + Invalidate all Stage 1 TLB entries owned by the given ASID on this SMMU. + Per SMMUv3 §5.2, Stage 1 TLB entries are tagged with VMID = 0 when + only Stage 1 is enabled, so the invalidation targets VMID = 0. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] Asid ASID to invalidate. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +EFI_STATUS +SmmuV3TLBInvalidateAllStage1 ( + IN SMMU_INFO *SmmuInfo, + IN UINT16 Asid + ) +{ + SMMUV3_CMD_GENERIC Command; + EFI_STATUS Status; + + if ((SmmuInfo == NULL) || (Asid == SMMU_ASID_RESERVED)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + ArmDataSynchronizationBarrier (); + + // Target the (VMID = 0, ASID) TLB entries. + SMMUV3_BUILD_CMD_TLBI_NH_ASID (&Command, SMMUV3_STREAM_TABLE_ENTRY_S1_ONLY_VMID, Asid); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CMD_TLBI_NH_ASID failed for Asid 0x%x.\n", __func__, Asid)); + return Status; + } + + SMMUV3_BUILD_CMD_SYNC_NO_INTERRUPT (&Command); + Status = SmmuV3SendCommand (SmmuInfo, &Command); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: CMD_SYNC_NO_INTERRUPT failed.\n", __func__)); + return Status; + } + + ArmDataSynchronizationBarrier (); + + return Status; +} + +/** + Get SMMUV3 node information from the IORT table. + + @param [in] IortTable Pointer to the IORT table. + @param [out] SmmuInfoArray Pointer to the array of SMMU_INFO structures. + @param [out] SmmuNodePtrs Pointer to the array of SMMU node pointers. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. +**/ +EFI_STATUS +SmmuV3GetNodeInfo ( + IN VOID *IortTable, + OUT SMMU_INFO *SmmuInfoArray, + OUT VOID **SmmuNodePtrs + ) +{ + EFI_ACPI_6_0_IO_REMAPPING_TABLE *Iort; + EFI_ACPI_6_0_IO_REMAPPING_NODE *Node; + EFI_ACPI_6_0_IO_REMAPPING_SMMU3_NODE *SmmuNode; + UINT32 SmmuIndex; + UINT32 Count; + + if ((IortTable == NULL) || (SmmuInfoArray == NULL) || (SmmuNodePtrs == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Iort = (EFI_ACPI_6_0_IO_REMAPPING_TABLE *)IortTable; + Node = (EFI_ACPI_6_0_IO_REMAPPING_NODE *)((UINT8 *)Iort + Iort->NodeOffset); + SmmuIndex = 0; + + for (Count = 0; Count < Iort->NumNodes; Count++) { + if (Node->Type == EFI_ACPI_IORT_TYPE_SMMUv3) { + InitializeListHead (&SmmuInfoArray[SmmuIndex].RmrNodeList); + SmmuNode = (EFI_ACPI_6_0_IO_REMAPPING_SMMU3_NODE *)Node; + SmmuInfoArray[SmmuIndex].SmmuBase = SmmuNode->Base; + SmmuInfoArray[SmmuIndex].Flags = SmmuNode->Flags; + SmmuInfoArray[SmmuIndex].EvtqIrqNum = SmmuNode->Event; + SmmuInfoArray[SmmuIndex].GerrIrqNum = SmmuNode->Gerr; + SmmuInfoArray[SmmuIndex].StreamTableEntryMax = 0; // Initialize max stream ID to 0 + SmmuInfoArray[SmmuIndex].EBSBehaviorAbort = TRUE; // Initialize EBS behavior to Abort by default + SmmuNodePtrs[SmmuIndex] = (VOID *)SmmuNode; + SmmuIndex++; + } + + // Move to the next node + Node = (EFI_ACPI_6_0_IO_REMAPPING_NODE *)((UINT8 *)Node + Node->Length); + } + + return EFI_SUCCESS; +} + +/** + Get the number of SMMUV3 nodes in the IORT table. + + @param [in] IortTable Pointer to the IORT table. + @param [out] SmmuNodeCount Pointer to store the number of SMMU nodes found. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval EFI_NOT_FOUND IORT table not found. +**/ +EFI_STATUS +SmmuV3NodeCount ( + IN VOID *IortTable, + OUT UINT32 *SmmuNodeCount + ) +{ + EFI_ACPI_6_0_IO_REMAPPING_TABLE *Iort; + EFI_ACPI_6_0_IO_REMAPPING_NODE *Node; + UINT32 Counter; + + if ((IortTable == NULL) || (SmmuNodeCount == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + // Cast the void* to the proper IORT structure + Iort = (EFI_ACPI_6_0_IO_REMAPPING_TABLE *)IortTable; + if (Iort == NULL) { + DEBUG ((DEBUG_ERROR, "%a: NULL IORT table\n", __func__)); + return EFI_NOT_FOUND; + } + + DEBUG ((DEBUG_VERBOSE, "%a: IORT contains %d nodes\n", __func__, Iort->NumNodes)); + + // First pass: count SMMU nodes + *SmmuNodeCount = 0; + Node = (EFI_ACPI_6_0_IO_REMAPPING_NODE *)((UINT8 *)Iort + Iort->NodeOffset); + + for (Counter = 0; Counter < Iort->NumNodes; Counter++) { + if (Node->Type == EFI_ACPI_IORT_TYPE_SMMUv3) { + (*SmmuNodeCount)++; + } + + // Move to the next node + Node = (EFI_ACPI_6_0_IO_REMAPPING_NODE *)((UINT8 *)Node + Node->Length); + } + + DEBUG ((DEBUG_VERBOSE, "%a: Found %d SMMU nodes\n", __func__, *SmmuNodeCount)); + + return EFI_SUCCESS; +} + +/** + Get the max stream ID for each SMMU. + + @param [in] IortTable Pointer to the IORT table. + @param [in] SmmuNodePtrs Pointer to the array of SMMU node pointers. + @param [in] SmmuNodeCount Number of SMMU nodes. + @param [out] SmmuInfoArray Pointer to the array of SMMU_INFO structures. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval EFI_NOT_FOUND SMMU node not found. +**/ +EFI_STATUS +SmmuV3GetMaxStreamIds ( + IN VOID *IortTable, + IN VOID **SmmuNodePtrs, + IN UINT32 SmmuNodeCount, + OUT SMMU_INFO *SmmuInfoArray + ) +{ + EFI_ACPI_6_0_IO_REMAPPING_TABLE *Iort; + EFI_ACPI_6_0_IO_REMAPPING_NODE *Node; + EFI_ACPI_6_0_IO_REMAPPING_ID_TABLE *IdMapping; + VOID *OutputNode; + UINT32 ByteOffset; + BOOLEAN Found; + UINT32 Count; + UINT32 IdMappingIndex; + UINT32 SmmuIndex; + UINT32 CurMaxMappingStreamId; + + if ((IortTable == NULL) || (SmmuNodePtrs == NULL) || (SmmuInfoArray == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Iort = (EFI_ACPI_6_0_IO_REMAPPING_TABLE *)IortTable; + Node = (EFI_ACPI_6_0_IO_REMAPPING_NODE *)((UINT8 *)Iort + Iort->NodeOffset); + + for (Count = 0; Count < Iort->NumNodes; Count++) { + if (((Node->Type == EFI_ACPI_IORT_TYPE_ROOT_COMPLEX) || (Node->Type == EFI_ACPI_IORT_TYPE_NAMED_COMP)) && (Node->NumIdMappings > 0)) { + // Get the ID mapping array + IdMapping = (EFI_ACPI_6_0_IO_REMAPPING_ID_TABLE *)((UINT8 *)Node + Node->IdReference); + + for (IdMappingIndex = 0; IdMappingIndex < Node->NumIdMappings; IdMappingIndex++) { + // Calculate the absolute offset of the output reference + ByteOffset = IdMapping[IdMappingIndex].OutputReference; + OutputNode = (VOID *)((UINT8 *)Iort + ByteOffset); + + // Check if the output reference points to an SMMU node + Found = FALSE; + for (SmmuIndex = 0; SmmuIndex < SmmuNodeCount; SmmuIndex++) { + if (OutputNode == SmmuNodePtrs[SmmuIndex]) { + // This ID mapping references an SMMU node + // Calculate the max Stream ID for this mapping: OutputBase + NumIds + CurMaxMappingStreamId = IdMapping[IdMappingIndex].OutputBase + IdMapping[IdMappingIndex].NumIds; + + // Update MaxStreamId if this mapping has a higher value + if (CurMaxMappingStreamId > SmmuInfoArray[SmmuIndex].StreamTableEntryMax) { + SmmuInfoArray[SmmuIndex].StreamTableEntryMax = CurMaxMappingStreamId; + DEBUG (( + DEBUG_VERBOSE, + "%a: Updated MaxStreamId for SMMU[0x%llx] to 0x%x (from mapping: InputBase=0x%x, NumIds=0x%x, OutputBase=0x%x)\n", + __func__, + SmmuInfoArray[SmmuIndex].SmmuBase, + SmmuInfoArray[SmmuIndex].StreamTableEntryMax, + IdMapping[IdMappingIndex].InputBase, + IdMapping[IdMappingIndex].NumIds, + IdMapping[IdMappingIndex].OutputBase + )); + } + + Found = TRUE; + break; + } + } + + if (!Found) { + DEBUG (( + DEBUG_ERROR, + "%a: ID mapping references a non-SMMU node (offset: 0x%x)\n", + __func__, + ByteOffset + )); + return EFI_NOT_FOUND; + } + } + } + + // Move to the next node + Node = (EFI_ACPI_6_0_IO_REMAPPING_NODE *)((UINT8 *)Node + Node->Length); + } + + return EFI_SUCCESS; +} + +/** + Add a new RMR node to the SMMU_INFO structure's RMR node list. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + @param [in] RmrNode Pointer to the RMR node to add. + + @return EFI_SUCCESS on success, or EFI_OUT_OF_RESOURCES on failure. +**/ +EFI_STATUS +SmmuV3AddRmrNodeToList ( + IN SMMU_INFO *SmmuInfo, + IN EFI_ACPI_6_0_IO_REMAPPING_RMR_NODE *RmrNode + ) +{ + RMR_NODE_INFO *Item; + + if ((SmmuInfo == NULL) || (RmrNode == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Item = AllocateZeroPool (sizeof (RMR_NODE_INFO)); + if (Item == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate memory for RMR_NODE_INFO\n", __func__)); + return EFI_OUT_OF_RESOURCES; + } + + Item->RmrNode = RmrNode; + InsertTailList (&SmmuInfo->RmrNodeList, &Item->Link); + return EFI_SUCCESS; +} + +/** + Collect RMR Node information for each SMMU and add it to the RmrNodeList. + + @param [in] IortTable Pointer to the IORT table. + @param [in] SmmuNodePtrs Pointer to the array of SMMU node pointers. + @param [in] SmmuNodeCount Number of SMMU nodes. + @param [out] SmmuInfoArray Pointer to the array of SMMU_INFO structures. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval EFI_NOT_FOUND SMMU node not found. +**/ +EFI_STATUS +SmmuV3GetRMRNodeInfo ( + IN VOID *IortTable, + IN VOID **SmmuNodePtrs, + IN UINT32 SmmuNodeCount, + OUT SMMU_INFO *SmmuInfoArray + ) +{ + EFI_ACPI_6_0_IO_REMAPPING_TABLE *Iort; + EFI_ACPI_6_0_IO_REMAPPING_NODE *Node; + EFI_ACPI_6_0_IO_REMAPPING_ID_TABLE *IdMapping; + VOID *OutputNode; + UINT32 ByteOffset; + UINT32 SmmuIndex; + UINT32 IdMappingIndex; + UINT32 Count; + BOOLEAN Found; + EFI_STATUS Status; + + if ((IortTable == NULL) || (SmmuNodePtrs == NULL) || (SmmuInfoArray == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Iort = (EFI_ACPI_6_0_IO_REMAPPING_TABLE *)IortTable; + Node = (EFI_ACPI_6_0_IO_REMAPPING_NODE *)((UINT8 *)Iort + Iort->NodeOffset); + + for (Count = 0; Count < Iort->NumNodes; Count++) { + if (Node->Type == EFI_ACPI_IORT_TYPE_RMR) { + if (Node->NumIdMappings > 0) { + // Get the ID mapping array + IdMapping = (EFI_ACPI_6_0_IO_REMAPPING_ID_TABLE *)((UINT8 *)Node + Node->IdReference); + + for (IdMappingIndex = 0; IdMappingIndex < Node->NumIdMappings; IdMappingIndex++) { + // Calculate the absolute offset of the output reference + ByteOffset = IdMapping[IdMappingIndex].OutputReference; + OutputNode = (VOID *)((UINT8 *)Iort + ByteOffset); + + // Check if the output reference points to an SMMU node + Found = FALSE; + for (SmmuIndex = 0; SmmuIndex < SmmuNodeCount; SmmuIndex++) { + if (OutputNode == SmmuNodePtrs[SmmuIndex]) { + // This ID mapping references an SMMU node + // If RMR Node store the RMR node pointer for this SMMU + Status = SmmuV3AddRmrNodeToList (&SmmuInfoArray[SmmuIndex], (EFI_ACPI_6_0_IO_REMAPPING_RMR_NODE *)Node); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to add RMR node to list for SMMU[0x%llx]\n", __func__, SmmuInfoArray[SmmuIndex].SmmuBase)); + return Status; + } + + Found = TRUE; + break; + } + } + + if (!Found) { + DEBUG (( + DEBUG_ERROR, + "%a: ID mapping references a non-SMMU node (offset: 0x%x)\n", + __func__, + ByteOffset + )); + return EFI_NOT_FOUND; + } + } + } + } + + // Move to the next node + Node = (EFI_ACPI_6_0_IO_REMAPPING_NODE *)((UINT8 *)Node + Node->Length); + } + + return EFI_SUCCESS; +} + +/** + Add RMR mappings for each SMMU node in the SmmuInfo structure. + + For every RMR node attached to this SMMU, walk its IdMappings to discover + the StreamIDs the reserved memory range applies to. For each such StreamID + we ensure a per-stream stage-2 page-table root exists (allocating it and + promoting the STE out of abort if needed via + SmmuV3StreamGetOrCreate), then identity-map the RMR ranges into + that per-stream root using its allocated VMID. + + Must be called AFTER SmmuV3Configure has finished initializing the + stream table for this SMMU. + + @param [in] SmmuInfo Pointer to the SMMU_INFO structure. + + @retval EFI_SUCCESS Success. + @retval EFI_INVALID_PARAMETER Invalid Parameters. + @retval Other RMR mapping update failure. +**/ +EFI_STATUS +SmmuV3AddRMRMapping ( + IN SMMU_INFO *SmmuInfo + ) +{ + EFI_ACPI_6_0_IO_REMAPPING_MEM_RANGE_DESC *IortMemRangeDesc; + EFI_ACPI_6_0_IO_REMAPPING_ID_TABLE *IdMapping; + UINT32 NumMemRangeDesc; + UINT32 IdMapIdx; + UINT32 IdInRange; + UINT32 StreamId; + PAGE_TABLE *Root; + UINT16 Vmid; + LIST_ENTRY *Entry; + RMR_NODE_INFO *Item; + EFI_STATUS Status; + + if (SmmuInfo == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Invalid Parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Entry = GetFirstNode (&SmmuInfo->RmrNodeList); + while (!IsNull (&SmmuInfo->RmrNodeList, Entry)) { + Item = BASE_CR (Entry, RMR_NODE_INFO, Link); + Entry = GetNextNode (&SmmuInfo->RmrNodeList, Entry); + + if ((Item == NULL) || (Item->RmrNode == NULL)) { + continue; + } + + IortMemRangeDesc = (EFI_ACPI_6_0_IO_REMAPPING_MEM_RANGE_DESC *)((UINT8 *)Item->RmrNode + Item->RmrNode->MemRangeDescRef); + IdMapping = (EFI_ACPI_6_0_IO_REMAPPING_ID_TABLE *)((UINT8 *)Item->RmrNode + Item->RmrNode->Node.IdReference); + + for (IdMapIdx = 0; IdMapIdx < Item->RmrNode->Node.NumIdMappings; IdMapIdx++) { + for (IdInRange = 0; IdInRange < IdMapping[IdMapIdx].NumIds; IdInRange++) { + StreamId = IdMapping[IdMapIdx].OutputBase + IdInRange; + + // Allocate/promote a per-stream page-table root for this StreamID. + Root = NULL; + Vmid = 0; + Status = SmmuV3StreamGetOrCreate (SmmuInfo, StreamId, &Root, &Vmid); + if (EFI_ERROR (Status)) { + DEBUG (( + DEBUG_ERROR, + "%a: Failed to ensure per-stream root for SMMU[0x%llx] StreamId=0x%x: %r\n", + __func__, + SmmuInfo->SmmuBase, + StreamId, + Status + )); + return Status; + } + + for (NumMemRangeDesc = 0; NumMemRangeDesc < Item->RmrNode->NumMemRangeDesc; NumMemRangeDesc++) { + if ((IortMemRangeDesc[NumMemRangeDesc].Base == 0) || (IortMemRangeDesc[NumMemRangeDesc].Length == 0)) { + continue; + } + + SmmuInfo->EBSBehaviorAbort = FALSE; // At least one RMR mapping exists, set EBS behavior to bypass + DEBUG (( + DEBUG_INFO, + "%a: Adding RMR mapping for SMMU[0x%llx] StreamId=0x%x Vmid=0x%x: Base=0x%llx, Length=0x%llx\n", + __func__, + SmmuInfo->SmmuBase, + StreamId, + Vmid, + IortMemRangeDesc[NumMemRangeDesc].Base, + IortMemRangeDesc[NumMemRangeDesc].Length + )); + Status = UpdatePageTable ( + SmmuInfo, + Root, + Vmid, + IortMemRangeDesc[NumMemRangeDesc].Base, + IortMemRangeDesc[NumMemRangeDesc].Length, + PAGE_TABLE_READ_WRITE_FROM_IOMMU_ACCESS ((EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE)), + TRUE + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to update RMR mapping.\n", __func__)); + return Status; + } + } + } + } + + RemoveEntryList (&Item->Link); + FreePool (Item); + } + + return EFI_SUCCESS; +} + +/** + Parse IORT table and extract SMMU information. + + @param [in] IortTable Pointer to the IORT table. + @param [out] SmmuInfo Pointer to store the array of SMMU_INFO structures. + @param [out] SmmuCount Pointer to store the number of SMMU nodes found. + + @return EFI_SUCCESS on success + @return EFI_INVALID_PARAMETER if any parameter is NULL + @return EFI_OUT_OF_RESOURCES if memory allocation fails + @return EFI_NOT_FOUND if no SMMU nodes are found + @return EFI_UNSUPPORTED if the IORT table is not supported +**/ +EFI_STATUS +SmmuV3ParseIort ( + IN VOID *IortTable, + OUT SMMU_INFO **SmmuInfo, + OUT UINT32 *SmmuCount + ) +{ + EFI_STATUS Status; + EFI_ACPI_6_0_IO_REMAPPING_TABLE *Iort; + SMMU_INFO *SmmuInfoArray; + VOID **SmmuNodePtrs; + UINT32 SmmuNodeCount; + + if ((IortTable == NULL) || (SmmuInfo == NULL) || (SmmuCount == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + SmmuInfoArray = NULL; + SmmuNodePtrs = NULL; + + // Cast the void* to the IORT structure + Iort = (EFI_ACPI_6_0_IO_REMAPPING_TABLE *)IortTable; + + // Verify IORT signature + if (Iort->Header.Signature != EFI_ACPI_6_0_IO_REMAPPING_TABLE_SIGNATURE) { + DEBUG (( + DEBUG_ERROR, + "%a: Invalid IORT signature: 0x%08X, expected: 0x%08X\n", + __func__, + Iort->Header.Signature, + EFI_ACPI_6_0_IO_REMAPPING_TABLE_SIGNATURE + )); + return EFI_UNSUPPORTED; + } + + if ((Iort->Header.Revision != EFI_ACPI_IO_REMAPPING_TABLE_REVISION_00) && (Iort->Header.Revision != EFI_ACPI_IO_REMAPPING_TABLE_REVISION_06)) { + DEBUG (( + DEBUG_ERROR, + "%a: Unsupported IORT revision: %d, expected: [%d, %d]\n", + __func__, + Iort->Header.Revision, + EFI_ACPI_IO_REMAPPING_TABLE_REVISION_00, + EFI_ACPI_IO_REMAPPING_TABLE_REVISION_06 + )); + return EFI_UNSUPPORTED; + } + + // First pass: get the number of SMMU nodes + Status = SmmuV3NodeCount (IortTable, &SmmuNodeCount); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to get IORT node count\n", __func__)); + return Status; + } + + if (SmmuNodeCount == 0) { + *SmmuCount = 0; + *SmmuInfo = NULL; + return EFI_NOT_FOUND; + } + + // Allocate memory for SMMU info array + SmmuInfoArray = AllocateZeroPool (SmmuNodeCount * sizeof (SMMU_INFO)); + if (SmmuInfoArray == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate memory for SMMU info array\n", __func__)); + Status = EFI_OUT_OF_RESOURCES; + goto Error; + } + + // Allocate memory for SMMU node pointers (for output reference lookup) + SmmuNodePtrs = AllocateZeroPool (SmmuNodeCount * sizeof (VOID *)); + if (SmmuNodePtrs == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to allocate memory for SMMU node pointers\n", __func__)); + Status = EFI_OUT_OF_RESOURCES; + goto Error; + } + + // Second pass: collect SMMU information + Status = SmmuV3GetNodeInfo (IortTable, SmmuInfoArray, SmmuNodePtrs); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to get SMMU node info\n", __func__)); + goto Error; + } + + // Third pass: calculate max Stream ID for each SMMU node + Status = SmmuV3GetMaxStreamIds (IortTable, SmmuNodePtrs, SmmuNodeCount, SmmuInfoArray); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to get max Stream ID for SMMU nodes\n", __func__)); + goto Error; + } + + // Fourth pass: collect per Stream ID range info like CCA, CPM, DACS for each RC/NamedComp node + Status = SmmuV3GetRMRNodeInfo (IortTable, SmmuNodePtrs, SmmuNodeCount, SmmuInfoArray); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to get Stream ID info for SMMU nodes\n", __func__)); + goto Error; + } + + FreePool (SmmuNodePtrs); + *SmmuInfo = SmmuInfoArray; + *SmmuCount = SmmuNodeCount; + return Status; + +Error: + if (SmmuInfoArray != NULL) { + FreePool (SmmuInfoArray); + } + + if (SmmuNodePtrs != NULL) { + FreePool (SmmuNodePtrs); + } + + return Status; +} + +/** + Allocate a SMMU_STREAM_ID_ENTRY for StreamId and append it to the list. + + @param[in,out] StreamIdList List head to append to. + @param[in] StreamId StreamID value. + + @retval EFI_SUCCESS Appended. + @retval EFI_OUT_OF_RESOURCES Allocation failed. +**/ +STATIC +EFI_STATUS +AppendStreamId ( + IN OUT LIST_ENTRY *StreamIdList, + IN UINT32 StreamId + ) +{ + SMMU_STREAM_ID_ENTRY *Entry; + + if (StreamIdList == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Invalid StreamIdList pointer\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Entry = (SMMU_STREAM_ID_ENTRY *)AllocatePool (sizeof (SMMU_STREAM_ID_ENTRY)); + if (Entry == NULL) { + return EFI_OUT_OF_RESOURCES; + } + + Entry->StreamId = StreamId; + InsertTailList (StreamIdList, &Entry->Link); + return EFI_SUCCESS; +} + +/** + Free all entries in a StreamID list. + + @param [in, out] StreamIdList The StreamID list to free. +**/ +VOID +SmmuStreamIdListFree ( + IN OUT LIST_ENTRY *StreamIdList + ) +{ + LIST_ENTRY *Link; + SMMU_STREAM_ID_ENTRY *Entry; + + if (StreamIdList == NULL) { + return; + } + + while (!IsListEmpty (StreamIdList)) { + Link = GetFirstNode (StreamIdList); + Entry = BASE_CR (Link, SMMU_STREAM_ID_ENTRY, Link); + RemoveEntryList (Link); + FreePool (Entry); + } +} + +/** + Try to resolve a real PCIe device handle to a StreamID. + + Gets BDF via PciIo->GetLocation(), computes RID, finds the matching IORT + Root Complex node by PCI Segment, and applies the ID mapping formula. + + @param[in] Iort Parsed IORT table pointer. + @param[in] Seg PCI Segment from GetLocation(). + @param[in] Bus PCI Bus from GetLocation(). + @param[in] Dev PCI Device from GetLocation(). + @param[in] Func PCI Function from GetLocation(). + @param[out] StreamId Resolved StreamID. + @param[out] SmmuBase Base address of the SMMU that owns the StreamID. + + @retval EFI_SUCCESS StreamID found. + @retval EFI_NOT_FOUND No matching IORT RC node or ID mapping. +**/ +STATIC +EFI_STATUS +ResolvePcieStreamId ( + IN EFI_ACPI_6_0_IO_REMAPPING_TABLE *Iort, + IN UINTN Seg, + IN UINTN Bus, + IN UINTN Dev, + IN UINTN Func, + OUT UINT32 *StreamId, + OUT UINT64 *SmmuBase + ) +{ + EFI_ACPI_6_0_IO_REMAPPING_NODE *Node; + EFI_ACPI_6_0_IO_REMAPPING_RC_NODE *RcNode; + EFI_ACPI_6_0_IO_REMAPPING_ID_TABLE *IdMapping; + EFI_ACPI_6_0_IO_REMAPPING_SMMU3_NODE *SmmuNode; + UINT32 Count; + UINT32 IdIdx; + UINT32 Rid; + + if ((Iort == NULL) || (StreamId == NULL) || (SmmuBase == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + Rid = ((UINT32)Bus << 8) | ((UINT32)Dev << 3) | (UINT32)Func; + Node = (EFI_ACPI_6_0_IO_REMAPPING_NODE *)((UINT8 *)Iort + Iort->NodeOffset); + + for (Count = 0; Count < Iort->NumNodes; Count++) { + if (Node->Type == EFI_ACPI_IORT_TYPE_ROOT_COMPLEX) { + RcNode = (EFI_ACPI_6_0_IO_REMAPPING_RC_NODE *)Node; + + if (RcNode->PciSegmentNumber == (UINT16)Seg) { + IdMapping = (EFI_ACPI_6_0_IO_REMAPPING_ID_TABLE *)((UINT8 *)Node + Node->IdReference); + + for (IdIdx = 0; IdIdx < Node->NumIdMappings; IdIdx++) { + if ((Rid >= IdMapping[IdIdx].InputBase) && + (Rid <= (IdMapping[IdIdx].InputBase + IdMapping[IdIdx].NumIds))) + { + *StreamId = Rid - IdMapping[IdIdx].InputBase + IdMapping[IdIdx].OutputBase; + + // The OutputReference is an offset from the start of the IORT + // table to the destination node, which for an RC mapping is + // the SMMUv3 node owning this StreamID. + SmmuNode = (EFI_ACPI_6_0_IO_REMAPPING_SMMU3_NODE *)((UINT8 *)Iort + IdMapping[IdIdx].OutputReference); + if (SmmuNode->Node.Type == EFI_ACPI_IORT_TYPE_SMMUv3) { + *SmmuBase = SmmuNode->Base; + } else { + DEBUG ((DEBUG_ERROR, "%a: OutputReference does not point to an SMMUv3 node (type=%u)\n", __func__, SmmuNode->Node.Type)); + *SmmuBase = 0; + return EFI_NOT_FOUND; + } + + return EFI_SUCCESS; + } + } + + // Found the RC node but RID not in any mapping range + return EFI_NOT_FOUND; + } + } + + Node = (EFI_ACPI_6_0_IO_REMAPPING_NODE *)((UINT8 *)Node + Node->Length); + } + + return EFI_NOT_FOUND; +} + +/** + Resolve a NonDiscoverable DeviceHandle to its IORT Named Component node + and append every StreamID from that node onto StreamIdList. + + The platform-supplied NC HOB table only maps each NON_DISCOVERABLE_DEVICE + UniqueId to an IORT Named Component ObjectName. All + StreamIDs and the owning SMMUv3 base are read from the matching NC node + in the IORT itself, so adding/removing alias StreamIDs is purely an IORT + edit and the platform table stays tiny. + + @param[in] Iort Parsed IORT table pointer. + @param[in] Bus Bus number from PciIo->GetLocation(). + @param[in] Dev Device number from PciIo->GetLocation(). + @param[in,out] StreamIdList List head to append SMMU_STREAM_ID_ENTRY + nodes to (one per resolved StreamID). + @param[out] SmmuBase Optional. Receives owning SMMUv3 base. + + @retval EFI_SUCCESS Match found and list populated. + @retval EFI_UNSUPPORTED No platform NC table available. + @retval EFI_NOT_FOUND UniqueId not in platform table, or NC node + could not be located in the IORT. + @retval EFI_OUT_OF_RESOURCES Allocation failure while building the list. +**/ +STATIC +EFI_STATUS +ResolveNonDiscoverableStreamId ( + IN EFI_ACPI_6_0_IO_REMAPPING_TABLE *Iort, + IN UINTN Bus, + IN UINTN Dev, + IN OUT LIST_ENTRY *StreamIdList, + OUT UINT64 *SmmuBase + ) +{ + EFI_STATUS Status; + UINT64 UniqueId; + UINT32 Idx; + CONST CHAR8 *WantedName; + EFI_ACPI_6_0_IO_REMAPPING_NODE *Node; + EFI_ACPI_6_0_IO_REMAPPING_NAMED_COMP_NODE *NcNode; + EFI_ACPI_6_0_IO_REMAPPING_ID_TABLE *IdMapping; + EFI_ACPI_6_0_IO_REMAPPING_SMMU3_NODE *SmmuNode; + CONST CHAR8 *NodeName; + UINTN NameMax; + UINT32 NodeIdx; + UINT32 MapIdx; + UINT32 PrimaryStreamId; + UINT32 TotalCount; + + if ((Iort == NULL) || (StreamIdList == NULL) || (SmmuBase == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + if ((mIoMmu == NULL) || (mIoMmu->NcDeviceList == NULL) || (mIoMmu->NcDeviceCount == 0)) { + DEBUG ((DEBUG_WARN, "%a: Platform did not publish a NonDiscoverable lookup table\n", __func__)); + return EFI_UNSUPPORTED; + } + + // PciIoGetLocation in NonDiscoverablePciDeviceIo.c sets: + // *BusNumber = Dev->UniqueId >> 5; + // *DeviceNumber = Dev->UniqueId & 0x1F; + // so reconstruct the original UniqueId from (Bus, Dev). + UniqueId = ((UINT64)Bus << 5) | ((UINT64)Dev & 0x1F); + WantedName = NULL; + + for (Idx = 0; Idx < mIoMmu->NcDeviceCount; Idx++) { + if (mIoMmu->NcDeviceList[Idx].UniqueId == UniqueId) { + WantedName = mIoMmu->NcDeviceList[Idx].ObjName; + break; + } + } + + if (WantedName == NULL) { + DEBUG ((DEBUG_ERROR, "%a: UniqueId=0x%llx not in platform NC table\n", __func__, UniqueId)); + return EFI_NOT_FOUND; + } + + // Walk the IORT for a Named Component node whose ObjectName matches. + // The ObjectName lives at offset sizeof(NC node header) and runs up to + // IdReference (which points at the first ID mapping that follows). + Node = (EFI_ACPI_6_0_IO_REMAPPING_NODE *)((UINT8 *)Iort + Iort->NodeOffset); + for (NodeIdx = 0; NodeIdx < Iort->NumNodes; NodeIdx++) { + if (Node->Type == EFI_ACPI_IORT_TYPE_NAMED_COMP) { + NcNode = (EFI_ACPI_6_0_IO_REMAPPING_NAMED_COMP_NODE *)Node; + NodeName = (CONST CHAR8 *)((UINT8 *)NcNode + sizeof (EFI_ACPI_6_0_IO_REMAPPING_NAMED_COMP_NODE)); + NameMax = Node->IdReference - sizeof (EFI_ACPI_6_0_IO_REMAPPING_NAMED_COMP_NODE); + + if ((Node->IdReference > sizeof (EFI_ACPI_6_0_IO_REMAPPING_NAMED_COMP_NODE)) && + (NameMax > 0) && (AsciiStrnCmp (NodeName, WantedName, NameMax) == 0) && + (AsciiStrLen (WantedName) < NameMax)) + { + if (Node->NumIdMappings == 0) { + DEBUG ((DEBUG_ERROR, "%a: NC node \"%a\" has no ID mappings\n", __func__, WantedName)); + return EFI_NOT_FOUND; + } + + IdMapping = (EFI_ACPI_6_0_IO_REMAPPING_ID_TABLE *)((UINT8 *)Node + Node->IdReference); + + // + // Expand every ID mapping into one or more StreamIDs. + // + // Per the IORT spec each mapping describes either: + // - A SINGLE entry (Flags & EFI_ACPI_IORT_ID_MAPPING_FLAGS_SINGLE): + // exactly one StreamID == OutputBase. NumIds is ignored. + // - A range: input IDs in [InputBase, InputBase + NumIds] map to + // output IDs in [OutputBase, OutputBase + NumIds], inclusive + // on both ends -> NumIds + 1 StreamIDs total. (This matches the + // <= in ResolvePcieStreamId above.) + // + // For a Named Component every output StreamID in the union of all + // mappings is one this device can present at the SMMU, so they all + // must share the primary's stage-2 page-table + VMID. + // + TotalCount = 0; + PrimaryStreamId = 0; + for (MapIdx = 0; MapIdx < Node->NumIdMappings; MapIdx++) { + UINT32 RangeCount; + UINT32 RangeIdx; + UINT32 StreamId; + + if ((IdMapping[MapIdx].Flags & EFI_ACPI_IORT_ID_MAPPING_FLAGS_SINGLE) != 0) { + RangeCount = 1; + } else { + // NumIds is the max offset, so the range covers NumIds + 1 IDs. + RangeCount = IdMapping[MapIdx].NumIds + 1; + } + + for (RangeIdx = 0; RangeIdx < RangeCount; RangeIdx++) { + StreamId = IdMapping[MapIdx].OutputBase + RangeIdx; + Status = AppendStreamId (StreamIdList, StreamId); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: NC \"%a\" failed to append StreamId 0x%x: %r\n", __func__, WantedName, StreamId, Status)); + return Status; + } + + if (TotalCount == 0) { + PrimaryStreamId = StreamId; + } + + TotalCount++; + } + } + + if (SmmuBase != NULL) { + SmmuNode = (EFI_ACPI_6_0_IO_REMAPPING_SMMU3_NODE *)((UINT8 *)Iort + IdMapping[0].OutputReference); + if (SmmuNode->Node.Type == EFI_ACPI_IORT_TYPE_SMMUv3) { + *SmmuBase = SmmuNode->Base; + } else { + DEBUG ((DEBUG_ERROR, "%a: NC \"%a\" OutputReference is not an SMMUv3 node (type=%u)\n", __func__, WantedName, SmmuNode->Node.Type)); + *SmmuBase = 0; + return EFI_DEVICE_ERROR; + } + } + + DEBUG (( + DEBUG_VERBOSE, + "%a: NonDiscoverable UniqueId=0x%llx \"%a\" -> %u StreamID(s) (primary=0x%x) SmmuBase=0x%llx\n", + __func__, + UniqueId, + WantedName, + TotalCount, + PrimaryStreamId, + (SmmuBase != NULL) ? *SmmuBase : 0ULL + )); + return EFI_SUCCESS; + } + } + + Node = (EFI_ACPI_6_0_IO_REMAPPING_NODE *)((UINT8 *)Node + Node->Length); + } + + DEBUG ((DEBUG_ERROR, "%a: UniqueId=0x%llx mapped to \"%a\" but no matching IORT NC node\n", __func__, UniqueId, WantedName)); + return EFI_NOT_FOUND; +} + +/** + Resolve a DeviceHandle to a StreamID using the IORT table. + + For real PCIe devices (Segment != 0xFF): + PciIo->GetLocation() → RID → IORT RC node ID mapping → StreamID + + For NonDiscoverable devices (Segment == 0xFF): + NON_DISCOVERABLE_DEVICE MMIO base → match against IORT Named Component → StreamID + + @param[in] IortTable Pointer to the IORT ACPI table. + @param[in] DeviceHandle The device handle to resolve. + @param[in, out] StreamIdList List to receive the resolved StreamID(s). + @param[out] SmmuBase Base address of the SMMU that owns the StreamID. + + @retval EFI_SUCCESS StreamID resolved. + @retval EFI_INVALID_PARAMETER One or more parameters are NULL. + @retval EFI_UNSUPPORTED DeviceHandle has no PciIo protocol. + @retval EFI_NOT_FOUND No IORT mapping found. +**/ +EFI_STATUS +DeviceHandleToStreamId ( + IN VOID *IortTable, + IN EFI_HANDLE DeviceHandle, + IN OUT LIST_ENTRY *StreamIdList, + OUT UINT64 *SmmuBase + ) +{ + EFI_STATUS Status; + EFI_PCI_IO_PROTOCOL *PciIo; + UINTN Seg; + UINTN Bus; + UINTN Dev; + UINTN Func; + EFI_ACPI_6_0_IO_REMAPPING_TABLE *Iort; + UINT32 PcieStreamId; + + if ((IortTable == NULL) || (DeviceHandle == NULL) || (StreamIdList == NULL) || (SmmuBase == NULL)) { + DEBUG ((DEBUG_ERROR, "%a: Invalid parameters\n", __func__)); + return EFI_INVALID_PARAMETER; + } + + // + // Get PciIo from the handle - both real PCIe and NonDiscoverable have it. + // + Status = gBS->HandleProtocol (DeviceHandle, &gEfiPciIoProtocolGuid, (VOID **)&PciIo); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: No PciIo on handle\n", __func__)); + return EFI_UNSUPPORTED; + } + + Status = PciIo->GetLocation (PciIo, &Seg, &Bus, &Dev, &Func); + if (EFI_ERROR (Status)) { + return EFI_UNSUPPORTED; + } + + Iort = (EFI_ACPI_6_0_IO_REMAPPING_TABLE *)IortTable; + + // + // Dispatch based on Segment: + // - Real PCIe: Segment is a valid PCI segment number (0x0000-0xFFFE) + // - NonDiscoverable: Segment is hardcoded to 0xFF by NonDiscoverablePciDeviceIo.c + // + if (Seg != 0xFF) { + Status = ResolvePcieStreamId (Iort, Seg, Bus, Dev, Func, &PcieStreamId, SmmuBase); + if (EFI_ERROR (Status)) { + return Status; + } + + Status = AppendStreamId (StreamIdList, PcieStreamId); + if (EFI_ERROR (Status)) { + return Status; + } + + DEBUG (( + DEBUG_VERBOSE, + "%a: PCIe S%llx B%llx D%llx F%llx --> StreamID=0x%x SmmuBase=0x%llx\n", + __func__, + Seg, + Bus, + Dev, + Func, + PcieStreamId, + (SmmuBase != NULL) ? *SmmuBase : 0ULL + )); + return EFI_SUCCESS; + } + + // + // NonDiscoverable device path. + // + DEBUG ((DEBUG_VERBOSE, "%a: NonDiscoverable device (Seg=0xFF Bus=%llx Dev=%llx)\n", __func__, Bus, Dev)); + return ResolveNonDiscoverableStreamId (Iort, Bus, Dev, StreamIdList, SmmuBase); +} diff --git a/ArmPkg/Include/Guid/SmmuConfig.h b/ArmPkg/Include/Guid/SmmuConfig.h new file mode 100644 index 0000000000..a0493fff61 --- /dev/null +++ b/ArmPkg/Include/Guid/SmmuConfig.h @@ -0,0 +1,73 @@ +/** @file + File for SMMU config structures. + + This SMMU_CONFIG structure is used to pass the SMMU configuration data from + the platform to the SMMU driver. The Smmu driver will use this data to install + the IORT table and configure the SMMU hardware. + + Given the IORT is configurable and platform dependent, the SMMU_CONFIG structure contains + all info relevant to the IORT table and SMMUv3 platform specific configuration. + + See for IORT spec. + + Copyright (c) Microsoft Corporation. + SPDX-License-Identifier: BSD-2-Clause-Patent + +**/ + +#pragma once + +// Increment SMMU_CONFIG version when the structure changes. +// Backwards compatibility is currently not supported. +// Future backwards compatibility is only possible if new fields are added to the end of the structure and existing fields are not modified. +// SmmuDxe driver will check and enforce the version of the SMMU_CONFIG structure to this current version set here. +#define CURRENT_SMMU_CONFIG_VERSION_MAJOR 1 +#define CURRENT_SMMU_CONFIG_VERSION_MINOR 1 + +#pragma pack(push, 1) + +// Maximum length (including NULL) of the ACPI namespace path stored for a +// NonDiscoverable device entry. Must be large enough to hold the +// IORT Named Component ObjectName +#define SMMU_NC_DEVICE_OBJNAME_MAX 32 + +// Per-SMMU translation stage selection. Every SMMU described by the HOB is +// configured for the same stage. +typedef enum _SMMU_TRANSLATION_STAGE { + SmmuTranslationStage2 = 0, // Stage 2 translate, Stage 1 bypass + SmmuTranslationStage1 = 1, // Stage 1 translate, Stage 2 bypass +} SMMU_TRANSLATION_STAGE; + +// Platform-provided lookup entry mapping a NonDiscoverable device's +// PciIo->GetLocation()-derived UniqueId to the SMMU StreamID's associated +// with the NamedComponent node in the IORT. +// Each NonDiscoverable device exposes a UniqueId from the NonDiscoverableDeviceRegistrationLib. +// This is used to determine a determinstic PciIo->GetLocation(). +typedef struct _SMMU_NC_DEVICE_ENTRY { + UINT64 UniqueId; // Value from the NON_DISCOVERABLE_DEVICE_UNIQUE_ID protocol on the handle. + CHAR8 ObjName[SMMU_NC_DEVICE_OBJNAME_MAX]; // IORT Named Component ObjectName (NUL-terminated). SmmuDxe walks the IORT for the matching NC node to recover the owning SMMU base and the full StreamID list. +} SMMU_NC_DEVICE_ENTRY; + +// SMMU_CONFIG structure to pass the SMMU configuration data from the platform to the SMMU driver. +// Platform will configure SmmuDisabledList size and offset to the SMMU disabled list appropriatley +// for any SMMU that needs be disabled in UEFI and set to bypass. +typedef struct _SMMU_CONFIG { + UINT32 VersionMajor; + UINT32 VersionMinor; + UINT32 SmmuDisabledListSize; // Size of SmmuDisabledList in bytes. + UINT32 SmmuDisabledListOffset; // Offset in bytes to the SmmuDisabledList from the start of the HOB structure. + UINT32 IortSize; + UINT32 IortOffset; // Offset in bytes to the IORT table from the start of the HOB structure. + UINT32 NcDeviceListSize; // Size of the NonDiscoverable device lookup array in bytes (multiple of sizeof(SMMU_NC_DEVICE_ENTRY)). + UINT32 NcDeviceListOffset; // Offset in bytes to the NonDiscoverable device lookup array from the start of the HOB structure. 0 if absent. + SMMU_TRANSLATION_STAGE TranslationStage; // SmmuTranslationStage2 (default) or SmmuTranslationStage1. + // SmmuDxe will configure every SMMU described by this HOB to the same stage. + // Any value other than SmmuTranslationStage1 is treated as SmmuTranslationStage2. +} SMMU_CONFIG; + +#pragma pack(pop) + +#define SMMU_CONFIG_HOB_GUID \ + { 0xcd56ec8f, 0x75f1, 0x440a, { 0xaa, 0x48, 0x09, 0x58, 0xb1, 0x1c, 0x9a, 0xa7 } } + +extern EFI_GUID gSmmuConfigHobGuid; diff --git a/ArmPkg/Include/Register/SmmuV3Registers.h b/ArmPkg/Include/Register/SmmuV3Registers.h new file mode 100644 index 0000000000..a1c6ea4b6b --- /dev/null +++ b/ArmPkg/Include/Register/SmmuV3Registers.h @@ -0,0 +1,1838 @@ +/** @file SmmuV3Registers.h + + This file is the SmmuV3Registers header file for SMMU driver. + + Contains all relevant register definitions for SMMUv3 found per the spec: + + + Copyright (c) Microsoft Corporation. + SPDX-License-Identifier: BSD-2-Clause-Patent + +**/ + +#pragma once + +typedef enum { + SmmuV3Rev0, + SmmuV3Rev1, + SmmuV3Rev2, + SmmuV3Rev3, +} SMMUV3_REVISION; + +// +// ------------------------------------------------------ Data Type Definitions +// + +// +// SMMUv3 ID registers (IDR0 - IDR5, IIDR, AIDR). +// + +typedef union { + struct { + UINT32 S2p : 1; + UINT32 S1p : 1; + UINT32 Ttf : 2; + UINT32 Cohacc : 1; + UINT32 Btm : 1; + UINT32 Httu : 2; + UINT32 DormHint : 1; + UINT32 Hyp : 1; + UINT32 Ats : 1; + UINT32 Ns1Ats : 1; + UINT32 Asid16 : 1; + UINT32 Msi : 1; + UINT32 Sev : 1; + UINT32 Atos : 1; + UINT32 Pri : 1; + UINT32 Vmw : 1; + UINT32 Vmid16 : 1; + UINT32 Cd2L : 1; + UINT32 Vatos : 1; + UINT32 Ttendian : 2; + UINT32 Reserved0 : 1; + UINT32 StallModel : 2; + UINT32 TermModel : 1; + UINT32 StLevel : 2; + UINT32 Reserved1 : 3; + } Bits; + UINT32 AsUINT32; +} SMMUV3_IDR0; + +typedef union { + struct { + UINT32 SidSize : 6; + UINT32 SSidSize : 5; + UINT32 PriQs : 5; + UINT32 EventQs : 5; + UINT32 CmdQs : 5; + UINT32 AttrPermsOvr : 1; + UINT32 AttrTypesOvr : 1; + UINT32 Rel : 1; + UINT32 QueuesPreset : 1; + UINT32 TablesPreset : 1; + UINT32 Ecmdq : 1; + } Bits; + UINT32 AsUINT32; +} SMMUV3_IDR1; + +#define SMMUV3_VATOS_REGION_OFFSET 0x20000 +#define SMMUV3_VATOS_REGION_UNIT_SIZE 0x10000 +#define SMMUV3_VATOS_REGION_TOTAL_SIZE 0x10000 + +typedef union { + struct { + UINT32 BaVatos : 10; + UINT32 Reserved : 22; + } Bits; + UINT32 AsUINT32; +} SMMUV3_IDR2; + +typedef union { + struct { + UINT32 Reserved0 : 2; + UINT32 Had : 1; + UINT32 Pbha : 1; + UINT32 Xnx : 1; + UINT32 Pps : 1; + UINT32 Reserved1 : 1; + UINT32 Mpam : 1; + UINT32 Fwb : 1; + UINT32 Stt : 1; + UINT32 Ril : 1; + UINT32 Bbml : 2; + UINT32 Reserved2 : 19; + } Bits; + UINT32 AsUINT32; +} SMMUV3_IDR3; + +typedef union { + struct { + UINT32 Impl; + } Bits; + UINT32 AsUINT32; +} SMMUV3_IDR4; + +typedef union { + struct { + UINT32 Oas : 3; + UINT32 Reserved0 : 1; + UINT32 Gran4k : 1; + UINT32 Gran16k : 1; + UINT32 Gran64k : 1; + UINT32 Reserved1 : 3; + UINT32 Vax : 2; + UINT32 Reserved2 : 4; + UINT32 StallMax : 16; + } Bits; + UINT32 AsUINT32; +} SMMUV3_IDR5; + +typedef union { + struct { + UINT32 Implementer : 12; + UINT32 Revision : 4; + UINT32 Variant : 4; + UINT32 ProductId : 12; + } Bits; + UINT32 AsUINT32; +} SMMUV3_IIDR; + +typedef union { + struct { + UINT32 ArchMinorRev : 4; + UINT32 ArchMajorRev : 4; + UINT32 Reserved : 24; + } Bits; + UINT32 AsUINT32; +} SMMUV3_AIDR; + +// +// SMMUv3 control registers (CR0 - CR2). +// + +#define SMMUV3_CR0_VALID_MASK (0x5FUL) +#define SMMUV3_CR0_SMMU_CMDQ_EVTQ_PRIQ_EN_MASK (0xFUL) +#define SMMUV3_CR0_SMMU_EN_MASK (0x1UL) + +typedef union { + struct { + UINT32 SmmuEn : 1; + UINT32 PriQEn : 1; + UINT32 EventQEn : 1; + UINT32 CmdQEn : 1; + UINT32 AtsChk : 1; + UINT32 Reserved0 : 1; + UINT32 Vmw : 3; + UINT32 Reserved : 23; + } Bits; + UINT32 AsUINT32; +} SMMUV3_CR0; + +// +// The CR0ACK register has the same format as CR0. +// + +typedef SMMUV3_CR0 SMMUV3_CR0ACK; + +#define SMMUV3_CR1_VALID_MASK (0x3FUL) + +typedef union { + struct { + UINT32 QueueIc : 2; + UINT32 QueueOc : 2; + UINT32 QueueSh : 2; + UINT32 TableIc : 2; + UINT32 TableOc : 2; + UINT32 TableSh : 2; + UINT32 Reserved : 20; + } Bits; + UINT32 AsUINT32; +} SMMUV3_CR1; + +#define SMMUV3_CR2_VALID_MASK (0x7UL) + +typedef union { + struct { + UINT32 E2h : 1; + UINT32 RecInvSid : 1; + UINT32 Ptm : 1; + UINT32 Reserved : 29; + } Bits; + UINT32 AsUINT32; +} SMMUV3_CR2; + +typedef union { + struct { + UINT32 MemAttr : 4; + UINT32 Mtcfg : 1; + UINT32 Reserved0 : 3; + UINT32 AllocCfg : 4; + UINT32 ShCfg : 2; + UINT32 Reserved1 : 2; + UINT32 PrivCfg : 2; + UINT32 InstCfg : 2; + UINT32 Abort : 1; + UINT32 Reserved2 : 10; + UINT32 Update : 1; + } Bits; + UINT32 AsUINT32; +} SMMUV3_GBPA; + +typedef struct { + UINT32 Impl; +} SMMUV3_AGBPA; + +typedef union { + struct { + UINT32 Dormant : 1; + UINT32 Reserved : 31; + } Bits; + UINT32 AsUINT32; +} SMMUV3_STATUSR; + +// +// Global error control and IRQ configuration registers. +// + +#define SMMUV3_IRQ_CTRL_GLOBAL_PRIQ_EVTQ_EN_MASK (0x7UL) + +typedef union { + struct { + UINT32 GlobalErrorIrqEn : 1; + UINT32 PriqIrqEn : 1; + UINT32 EventqIrqEn : 1; + UINT32 Reserved : 29; + } Bits; + UINT32 AsUINT32; +} SMMUV3_IRQ_CTRL; + +typedef SMMUV3_IRQ_CTRL SMMUV3_IRQ_CTRLACK; + +// +// Define a mask of the valid bits within the GERROR register. +// + +#define SMMUV3_GERROR_VALID_MASK (0x1FDUL) +#define SMMUV3_GERROR_SFM_ERROR_MASK (0x100UL) + +typedef union { + struct { + UINT32 CmdqErr : 1; + UINT32 Reserved0 : 1; + UINT32 EventqAbtErr : 1; + UINT32 PriqAbtErr : 1; + UINT32 MsiCmdqAbtErr : 1; + UINT32 MsiEventqAbtErr : 1; + UINT32 MsiPriqAbtErr : 1; + UINT32 MsiGerrorAbtErr : 1; + UINT32 SfmErr : 1; + UINT32 Reserved : 23; + } Bits; + UINT32 AsUINT32; +} SMMUV3_GERROR; + +typedef SMMUV3_GERROR SMMUV3_GERRORN; + +typedef union { + struct { + UINT64 Reserved : 2; + UINT64 Addr : 50; + UINT64 Reserved1 : 12; + } Bits; + UINT64 AsUINT64; +} SMMUV3_GERROR_IRQ_CFG0; + +typedef struct { + UINT32 Data; +} SMMUV3_GERROR_IRQ_CFG1; + +typedef union { + struct { + UINT32 MemAttr : 4; + UINT32 Sh : 2; + UINT32 Reserved : 26; + } Bits; + UINT32 AsUINT32; +} SMMUV3_GERROR_IRQ_CFG2; + +// +// Stream table base and configuration registers. +// + +typedef union { + struct { + UINT64 Reserved0 : 6; + UINT64 Addr : 46; + UINT64 Reserved1 : 10; + UINT64 Ra : 1; + UINT64 Reserved2 : 1; + } Bits; + UINT64 AsUINT64; +} SMMUV3_STRTAB_BASE; + +typedef union { + struct { + UINT32 Log2Size : 6; + UINT32 Split : 5; + UINT32 Reserved0 : 5; + UINT32 Fmt : 2; + UINT32 Reserved2 : 14; + } Bits; + UINT32 AsUINT32; +} SMMUV3_STRTAB_BASE_CFG; + +// +// Command queue base, producer and consumer index registers. +// + +typedef union { + struct { + UINT64 Log2Size : 5; + UINT64 Addr : 47; + UINT64 Reserved0 : 10; + UINT64 Ra : 1; + UINT64 Reserved1 : 1; + } Bits; + UINT64 AsUINT64; +} SMMUV3_CMDQ_BASE; + +typedef union { + struct { + UINT32 ReadIndex : 20; + UINT32 Reserved0 : 4; + UINT32 Err : 7; + UINT32 Reserved1 : 1; + } Bits; + UINT32 AsUINT32; +} SMMUV3_CMDQ_CONS; + +typedef union { + struct { + UINT32 WriteIndex : 20; + UINT32 Reserved : 12; + } Bits; + UINT32 AsUINT32; +} SMMUV3_CMDQ_PROD; + +// +// Event queue base, producer/consumer, and IRQ configuration registers. +// + +typedef union { + struct { + UINT64 Log2Size : 5; + UINT64 Addr : 47; + UINT64 Reserved0 : 10; + UINT64 Wa : 1; + UINT64 Reserved1 : 1; + } Bits; + UINT64 AsUINT64; +} SMMUV3_EVENTQ_BASE; + +typedef union { + struct { + UINT32 ReadIndex : 20; + UINT32 Reserved : 11; + UINT32 OvAckFlag : 1; + } Bits; + UINT32 AsUINT32; +} SMMUV3_EVENTQ_CONS; + +typedef union { + struct { + UINT32 WriteIndex : 20; + UINT32 Reserved : 11; + UINT32 OvFlag : 1; + } Bits; + UINT32 AsUINT32; +} SMMUV3_EVENTQ_PROD; + +typedef union { + struct { + UINT64 Reserved : 2; + UINT64 Addr : 50; + UINT64 Reserved1 : 12; + } Bits; + UINT64 AsUINT64; +} SMMUV3_EVENTQ_IRQ_CFG0; + +typedef struct { + UINT32 Data; +} SMMUV3_EVENTQ_IRQ_CFG1; + +typedef union { + struct { + UINT32 MemAttr : 4; + UINT32 Sh : 2; + UINT32 Reserved : 26; + } Bits; + UINT32 AsUINT32; +} SMMUV3_EVENTQ_IRQ_CFG2; + +// +// PRI queue base, producer/consumer, and IRQ configuration registers. +// + +typedef union { + struct { + UINT64 Log2Size : 5; + UINT64 Addr : 47; + UINT64 Reserved0 : 10; + UINT64 Wa : 1; + UINT64 Reserved1 : 1; + } Bits; + UINT64 AsUINT64; +} SMMUV3_PRIQ_BASE; + +typedef union { + struct { + UINT32 ReadIndex : 20; + UINT32 Reserved : 11; + UINT32 OvAckFlg : 1; + } Bits; + UINT32 AsUINT32; +} SMMUV3_PRIQ_CONS; + +typedef union { + struct { + UINT32 WriteIndex : 20; + UINT32 Reserved : 11; + UINT32 OvFlg : 1; + } Bits; + UINT32 AsUINT32; +} SMMUV3_PRIQ_PROD; + +typedef union { + struct { + UINT64 Reserved : 2; + UINT64 Addr : 50; + UINT64 Reserved1 : 12; + } Bits; + UINT64 AsUINT64; +} SMMUV3_PRIQ_IRQ_CFG0; + +typedef struct { + UINT32 Data; +} SMMUV3_PRIQ_IRQ_CFG1; + +typedef union { + struct { + UINT32 MemAttr : 4; + UINT32 Sh : 2; + UINT32 Reserved : 26; + } Bits; + UINT32 AsUINT32; +} SMMUV3_PRIQ_IRQ_CFG2; + +// +// ATOS (Address Translation) control and configuration registers. +// + +typedef union { + struct { + UINT32 Run : 1; + UINT32 Reserved : 31; + } Bits; + UINT32 AsUINT32; +} SMMUV3_GATOS_CTRL; + +typedef union { + struct { + UINT64 StreamId : 32; + UINT64 SubstreamId : 20; + UINT64 SsidValid : 1; + UINT64 Reserved : 11; + } Bits; + UINT64 AsUINT64; +} SMMUV3_GATOS_SID; + +typedef union { + struct { + UINT64 Reserved : 6; + UINT64 HttuI : 1; + UINT64 InstructionNotData : 1; + UINT64 ReadNotWrite : 1; + UINT64 PrivilegedNotUnprivileged : 1; + UINT64 Type : 2; + UINT64 Addr : 52; + } Bits; + UINT64 AsUINT64; +} SMMUV3_GATOS_ADDR; + +typedef union { + struct { + UINT64 Fault : 1; + UINT64 Reserved0 : 7; + UINT64 Sh : 2; + UINT64 Reserved1 : 1; + UINT64 Size : 1; + UINT64 Addr : 40; + UINT64 Reserved2 : 4; + UINT64 Attr : 8; + } NoFault; + + struct { + UINT64 Fault : 1; + UINT64 Reason : 2; + UINT64 Reserved0 : 1; + UINT64 FaultCode : 8; + UINT64 Faddr : 40; + UINT64 Reserved1 : 8; + UINT64 ImpDef : 4; + } Fault; + + UINT64 AsUINT64; +} SMMUV3_GATOS_PAR; + +typedef union { + struct { + UINT32 PartIdMax : 16; + UINT32 PmgMax : 8; + UINT32 Reserved : 8; + } Bits; + UINT32 AsUINT32; +} SMMUV3_MPAMIDR; + +typedef union { + struct { + UINT32 SoPartId : 16; + UINT32 SoPmg : 8; + UINT32 Reserved : 7; + UINT32 Update : 1; + } Bits; + UINT32 AsUINT32; +} SMMUV3_GMPAM; + +typedef union { + struct { + UINT32 GbpPartId : 16; + UINT32 GbpPmg : 8; + UINT32 Reserved : 7; + UINT32 Update : 1; + } Bits; + UINT32 AsUINT32; +} SMMUV3_GBPMPAM; + +typedef union { + struct { + UINT32 Vmid : 16; + UINT32 Reserved : 16; + } Bits; + UINT32 AsUINT32; +} SMMUV3_VATOS_SEL; + +// +// SMMUv3 (component and peripheral) ID registers. These are laid out as follows +// in Page 0: +// Offsets 0xFF0 - 0xFFC: CIDR0 to CIDR3 +// Offsets 0xFD0 - 0xFDC - PIDR4 to PIDR7 +// Offsets 0xFE0 - 0xFEC - PIDR0 to PIDR3 +// + +typedef struct { + UINT32 Cidr0; + UINT32 Cidr1; + UINT32 Cidr2; + UINT32 Cidr3; +} SMMUV3_CIDRS; + +typedef union { + struct { + UINT64 Des2 : 4; + UINT64 Size : 4; + UINT64 Reserved0 : 24; + UINT64 Reserved1 : 32; + } Bits; + + UINT64 AsUINT64; +} SMMUV3_PIDR_4_5; + +typedef union { + struct { + UINT64 Part0 : 8; + UINT64 Reserved0 : 24; + UINT64 Part1 : 4; + UINT64 Des0 : 4; + UINT64 Reserved1 : 24; + } Bits; + + UINT64 AsUINT64; +} SMMUV3_PIDR_0_1; + +typedef union { + struct { + UINT64 Des1 : 3; + UINT64 JedecId : 1; + UINT64 Revision : 4; + UINT64 Reserved0 : 24; + UINT64 Cmod : 4; + UINT64 Revand : 4; + UINT64 Reserved1 : 24; + } Bits; + + UINT64 AsUINT64; +} SMMUV3_PIDR_2_3; + +typedef struct { + SMMUV3_PIDR_4_5 Pidr4_5; + UINT64 Pidr6_7; + SMMUV3_PIDR_0_1 Pidr0_1; + SMMUV3_PIDR_2_3 Pidr2_3; +} SMMUV3_PIDRS; + +typedef struct { + SMMUV3_PIDRS Pidrs; + SMMUV3_CIDRS Cidrs; +} SMMUV3_ID_REGS; + +// +// Page 0 SMMUv3 register layout. +// + +typedef struct { + SMMUV3_IDR0 Idr0; + SMMUV3_IDR1 Idr1; + SMMUV3_IDR2 Idr2; + SMMUV3_IDR3 Idr3; + SMMUV3_IDR4 Idr4; + SMMUV3_IDR5 Idr5; + SMMUV3_IIDR Iidr; + SMMUV3_AIDR Aidr; + SMMUV3_CR0 Cr0; + SMMUV3_CR0ACK Cr0Ack; + SMMUV3_CR1 Cr1; + SMMUV3_CR2 Cr2; + UINT32 Reserved0[4]; + SMMUV3_STATUSR StatusR; + SMMUV3_GBPA Gbpa; + SMMUV3_AGBPA Agbpa; + UINT32 Reserved1[1]; + SMMUV3_IRQ_CTRL IrqCtrl; + SMMUV3_IRQ_CTRLACK IrqCtrlAck; + UINT32 Reserved2[2]; + SMMUV3_GERROR GError; + SMMUV3_GERRORN GErrorN; + SMMUV3_GERROR_IRQ_CFG0 GErrorIrqCfg0; + SMMUV3_GERROR_IRQ_CFG1 GErrorIrqCfg1; + SMMUV3_GERROR_IRQ_CFG2 GErrorIrqCfg2; + UINT32 Reserved3[2]; + SMMUV3_STRTAB_BASE StrTabBase; + SMMUV3_STRTAB_BASE_CFG StrTabBaseCfg; + UINT32 Reserved4[1]; + SMMUV3_CMDQ_BASE CmdQBase; + SMMUV3_CMDQ_PROD CmdQProd; + SMMUV3_CMDQ_CONS CmdQCons; + SMMUV3_EVENTQ_BASE EventQBase; + UINT32 Reserved5[2]; // Aliases SMMU_EVENTQ_PROD + SMMU_EVENTQ_CONS + SMMUV3_EVENTQ_IRQ_CFG0 EventQIrqCfg0; + SMMUV3_EVENTQ_IRQ_CFG1 EventQIrqCfg1; + SMMUV3_EVENTQ_IRQ_CFG2 EventQIrqCfg2; + SMMUV3_PRIQ_BASE PriQBase; + UINT32 Reserved6[2]; // Aliases SMMU_PRIQ_PROD + SMMU_PRIQ_CONS + SMMUV3_PRIQ_IRQ_CFG0 PriQIrqCfg0; + SMMUV3_PRIQ_IRQ_CFG1 PriQIrqCfg1; + SMMUV3_PRIQ_IRQ_CFG2 PriQIrqCfg2; + UINT32 Reserved7[8]; + SMMUV3_GATOS_CTRL GatosCtrl; + UINT32 Reserved8[1]; + SMMUV3_GATOS_SID GatosSid; + SMMUV3_GATOS_ADDR GatosAddr; + SMMUV3_GATOS_PAR GatosPar; + UINT32 Reserved9[4]; + SMMUV3_MPAMIDR Mpamidr; + SMMUV3_GMPAM Gmpam; + SMMUV3_GBPMPAM Gbpmpam; + UINT32 Reserved10[17]; + SMMUV3_VATOS_SEL VatosSel; + UINT32 Reserved11[799]; + UINT32 Impl0[64]; + UINT32 Reserved12[52]; + SMMUV3_ID_REGS IdRegs; + + // + // Rest are implementation defined registers and registers for secure + // state management. Left undefined unless needed. + // +} SMMUV3_REGISTER_LAYOUT_PAGE0; + +// +// Page 1 SMMUv3 register layout. +// + +typedef struct { + UINT32 Reserved0[42]; + SMMUV3_EVENTQ_PROD EventQProd; + SMMUV3_EVENTQ_CONS EventQCons; + UINT32 Reserved1[6]; + SMMUV3_PRIQ_PROD PriQProd; + SMMUV3_PRIQ_CONS PriQCons; +} SMMUV3_REGISTER_LAYOUT_PAGE1; + +// +// Level 1 Stream table descriptor +// + +typedef union { + struct { + UINT64 Span : 5; + UINT64 Reserved0 : 1; + UINT64 L2Ptr : 46; + UINT64 Reserved1 : 12; + } Bits; + UINT64 AsUINT64; +} SMMUV3_L1_STREAM_TABLE_DESCRIPTOR; + +typedef union { + struct { + UINT64 Valid : 1; + UINT64 Config : 3; + UINT64 S1Fmt : 2; + UINT64 S1ContextPtr : 46; + UINT64 Reserved0 : 7; + UINT64 S1CdMax : 5; + UINT64 S1Dss : 2; + UINT64 S1Cir : 2; + UINT64 S1Cor : 2; + UINT64 S1Csh : 2; + UINT64 S2Hwu59 : 1; + UINT64 S2Hwu60 : 1; + UINT64 S2Hwu61 : 1; + UINT64 S2Hwu62 : 1; + UINT64 Dre : 1; + UINT64 Cont : 4; + UINT64 Dcp : 1; + UINT64 Ppar : 1; + UINT64 Mev : 1; + UINT64 ResSw : 4; + UINT64 Reserved1 : 1; + UINT64 S2Fwb : 1; + UINT64 S1Mpam : 1; + UINT64 S1StallD : 1; + UINT64 Eats : 2; + UINT64 Strw : 2; + UINT64 MemAttr : 4; + UINT64 Mtcfg : 1; + UINT64 AllocCfg : 4; + UINT64 Reserved2 : 3; + UINT64 ShCfg : 2; + UINT64 NsCfg : 2; + UINT64 PrivCfg : 2; + UINT64 InstCfg : 2; + UINT64 Impl0 : 12; + UINT64 S2Vmid : 16; + UINT64 Impl1 : 16; + UINT64 S2T0Sz : 6; + UINT64 S2Sl0 : 2; + UINT64 S2Ir0 : 2; + UINT64 S2Or0 : 2; + UINT64 S2Sh0 : 2; + UINT64 S2Tg : 2; + UINT64 S2Ps : 3; + UINT64 S2Aa64 : 1; + UINT64 S2Endi : 1; + UINT64 S2Affd : 1; + UINT64 S2Ptw : 1; + UINT64 S2Had : 2; + UINT64 S2Rs : 2; + UINT64 Reserved3 : 5; + UINT64 S2Nsw : 1; + UINT64 S2Nsa : 1; + UINT64 Reserved4 : 2; + UINT64 S2Ttb : 48; + UINT64 Reserved5 : 12; + UINT64 Impl2 : 16; + UINT64 PartId : 16; + UINT64 Reserved6 : 32; + UINT64 Pmg : 8; + UINT64 Reserved7 : 4; + UINT64 VmsPtr : 40; + UINT64 Reserved8 : 12; + UINT64 Reserved9; + UINT64 Reserved10; + } Bits; + UINT64 AsUINT64[8]; +} SMMUV3_STREAM_TABLE_ENTRY; + +STATIC_ASSERT (sizeof (SMMUV3_STREAM_TABLE_ENTRY) == 64, "Invalid size for SMMUV3_STREAM_TABLE_ENTRY"); + +// +// Define an enumeration of valid values for stream entry config field +// (SMMUV3_STREAM_TABLE_ENTRY.Config). +// + +typedef enum { + StreamEntryConfigS1BlockedS2Blocked = 0, + StreamEntryConfigS1BypassS2Bypass = 4, + StreamEntryConfigS1TranslateS2Bypass, + StreamEntryConfigS1BypassS2Translate, + StreamEntryConfigS1TranslateS2Translate +} SMMUV3_STREAM_ENTRY_CONFIG_TYPE; + +// +// Level 1 Context descriptor +// + +typedef union { + struct { + UINT64 Valid : 1; + UINT64 Reserved0 : 11; + UINT64 L2Ptr : 40; + UINT64 Reserved1 : 12; + } Bits; + UINT64 AsUINT64; +} SMMUV3_L1_CONTEXT_DESCRIPTOR; + +// +// Context descriptor +// + +typedef union { + struct { + UINT64 T0Sz : 6; + UINT64 Tg0 : 2; + UINT64 Ir0 : 2; + UINT64 Or0 : 2; + UINT64 Sh0 : 2; + UINT64 Epd0 : 1; + UINT64 Endi : 1; + UINT64 T1Sz : 6; + UINT64 Tg1 : 2; + UINT64 Ir1 : 2; + UINT64 Or1 : 2; + UINT64 Sh1 : 2; + UINT64 Epd1 : 1; + UINT64 Valid : 1; + UINT64 Ips : 3; + UINT64 Affd : 1; + UINT64 Wxn : 1; + UINT64 UWxn : 1; + UINT64 Tbi0 : 1; + UINT64 Tbi1 : 1; + UINT64 Pan : 1; + UINT64 Aa64 : 1; + UINT64 Had : 2; + UINT64 Ars : 3; + UINT64 Aset : 1; + UINT64 Asid : 16; + UINT64 NsCfg0 : 1; + UINT64 Had0 : 1; + UINT64 Reserved0 : 2; + UINT64 Ttb0 : 48; + UINT64 Reserved1 : 8; + UINT64 Hwu059 : 1; + UINT64 Hwu060 : 1; + UINT64 Hwu061 : 1; + UINT64 Hwu062 : 1; + UINT64 NsCfg1 : 1; + UINT64 Had1 : 1; + UINT64 Reserved2 : 2; + UINT64 Ttb1 : 48; + UINT64 Reserved3 : 8; + UINT64 Hwu159 : 1; + UINT64 Hwu160 : 1; + UINT64 Hwu161 : 1; + UINT64 Hwu162 : 1; + UINT64 Mair0 : 32; + UINT64 Mair1 : 32; + UINT64 AMair0 : 32; + UINT64 AMair1 : 32; + UINT64 Impl : 32; + UINT64 PartId : 16; + UINT64 Pmg : 8; + UINT64 Reserved4 : 8; + UINT64 Reserved5[2]; + } Bits; + UINT64 AsUINT64[8]; +} SMMUV3_CONTEXT_DESCRIPTOR; + +// +// Command opcodes and their formats. +// + +typedef enum { + CmdPrefetchConfig = 0x1, + CmdPrefetchAddr, + CmdCfgiSte, + CmdCfgiSteRange, + CmdCfgiCd, + CmdCfgiCdAll, + CmdCfgiVmsPidm, + CmdTlbiNhAll = 0x10, + CmdTlbiNhAsid, + CmdTlbiNhVa, + CmdTlbiNhVaa, + CmdTlbiEl3All = 0x18, + CmdTlbiEl3Va = 0x1A, + CmdTlbiEl2All = 0x20, + CmdTlbiEl2Asid, + CmdTlbiEl2Va, + CmdTlbiEl2Vaa, + CmdTlbiS12VmAll = 0x28, + CmdTlbiS2Ipa = 0x2A, + CmdTlbiNsnhAll = 0x30, + CmdAtcInv = 0x40, + CmdPriResp = 0x41, + CmdResume = 0x44, + CmdSync = 0x46 +} SMMUV3_COMMAND_OPCODE; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 21; + UINT32 StreamId; + UINT64 Leaf : 1; + UINT64 Reserved2 : 63; +} SMMUV3_CMD_CFGI_STE; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 21; + UINT32 StreamId; + UINT64 Range : 5; + UINT64 Reserved2 : 59; +} SMMUV3_CMD_CFGI_STE_RANGE; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 1; + UINT32 SubStreamId : 20; + UINT32 StreamId; + UINT64 Leaf : 1; + UINT64 Reserved2 : 63; +} SMMUV3_CMD_CFGI_CD; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 21; + UINT32 StreamId; + UINT64 Reserved2; +} SMMUV3_CMD_CFGI_CD_ALL; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 21; + UINT32 Vmid : 16; + UINT32 Reserved2 : 16; + UINT64 Reserved3; +} SMMUV3_CMD_CFGI_VMS_PIDM; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 21; + UINT32 Ignored; + UINT64 Range : 5; + UINT64 Reserved2 : 59; +} SMMUV3_CMD_CFGI_ALL; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 24; + UINT32 Vmid : 16; + UINT32 Reserved1 : 16; + UINT64 Reserved2; +} SMMUV3_CMD_TLBI_NH_ALL; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 24; + UINT32 Vmid : 16; + UINT32 Asid : 16; + UINT64 Reserved1; +} SMMUV3_CMD_TLBI_NH_ASID; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 4; + UINT32 Num : 5; + UINT32 Reserved1 : 3; + UINT32 Scale : 5; + UINT32 Reserved2 : 7; + UINT32 Vmid : 16; + UINT32 Reserved3 : 16; + UINT64 Leaf : 1; + UINT64 Reserved4 : 7; + UINT64 Ttl : 2; + UINT64 Tg : 2; + UINT64 Address : 52; +} SMMUV3_CMD_TLBI_NH_VAA; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 4; + UINT32 Num : 5; + UINT32 Reserved1 : 3; + UINT32 Scale : 5; + UINT32 Reserved2 : 7; + UINT32 Vmid : 16; + UINT32 Asid : 16; + UINT64 Leaf : 1; + UINT64 Reserved3 : 7; + UINT64 Ttl : 2; + UINT64 Tg : 2; + UINT64 Address : 52; +} SMMUV3_CMD_TLBI_NH_VA; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 24; + UINT32 Reserved1; + UINT64 Reserved2; +} SMMUV3_CMD_TLBI_EL2_ALL; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 4; + UINT32 Num : 5; + UINT32 Reserved1 : 3; + UINT32 Scale : 5; + UINT32 Reserved2 : 7; + UINT32 Reserved3 : 16; + UINT32 Asid : 16; + UINT64 Leaf : 1; + UINT64 Reserved4 : 7; + UINT64 Ttl : 2; + UINT64 Tg : 2; + UINT64 Address : 52; +} SMMUV3_CMD_TLBI_EL2_VA; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 4; + UINT32 Num : 5; + UINT32 Reserved1 : 3; + UINT32 Scale : 5; + UINT32 Reserved2 : 7; + UINT32 Reserved3; + UINT64 Leaf : 1; + UINT64 Reserved4 : 7; + UINT64 Ttl : 2; + UINT64 Tg : 2; + UINT64 Address : 52; +} SMMUV3_CMD_TLBI_EL2_VAA; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 24; + UINT32 Reserved1 : 16; + UINT32 Asid : 16; + UINT64 Reserved2; +} SMMUV3_CMD_TLBI_EL2_ASID; + +// +// Stage 2 TLB invalidation commands. +// + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 4; + UINT32 Num : 5; + UINT32 Reserved1 : 3; + UINT32 Scale : 5; + UINT32 Reserved2 : 7; + UINT32 Vmid : 16; + UINT32 Reserved3 : 16; + UINT64 Leaf : 1; + UINT64 Reserved4 : 7; + UINT64 Ttl : 2; + UINT64 Tg : 2; + UINT64 Address : 40; + UINT64 Reserved5 : 12; +} SMMUV3_CMD_TLBI_S2_IPA; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 24; + UINT32 Vmid : 16; + UINT32 Reserved1 : 16; + UINT64 Reserved2; +} SMMUV3_CMD_TLBI_S12_VMALL; + +// +// Common TLB invalidation commands. +// + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 24; + UINT32 Reserved1; + UINT64 Reserved2; +} SMMUV3_CMD_TLBI_NSNH_ALL; + +// +// Fault response and synchronization commands. +// + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 1; + UINT32 Ac : 1; + UINT32 Ab : 1; + UINT32 Reserved2 : 18; + UINT32 StreamId; + UINT64 Stag : 16; + UINT64 Reserved3 : 48; +} SMMUV3_CMD_RESUME; + +typedef struct { + UINT32 Opcode : 8; + UINT32 Reserved0 : 4; + UINT32 Cs : 2; + UINT32 Reserved1 : 8; + UINT32 Msh : 2; + UINT32 MsiAttr : 4; + UINT32 Reserved2 : 4; + UINT32 MsiData; + UINT64 Reserved3 : 2; + UINT64 MsiAddress : 50; + UINT64 Reserved4 : 12; +} SMMUV3_CMD_SYNC; + +typedef union { + SMMUV3_CMD_CFGI_STE CfgiSte; + SMMUV3_CMD_CFGI_STE_RANGE CfgiSteRange; + SMMUV3_CMD_CFGI_CD CfgiCd; + SMMUV3_CMD_CFGI_CD_ALL CfgiCdAll; + SMMUV3_CMD_CFGI_VMS_PIDM CfgiVmsPidm; + SMMUV3_CMD_CFGI_ALL CfgiAll; + SMMUV3_CMD_TLBI_NH_ALL TlbiNhAll; + SMMUV3_CMD_TLBI_NH_ASID TlbiNhAsid; + SMMUV3_CMD_TLBI_NH_VAA TlbiNhVaa; + SMMUV3_CMD_TLBI_NH_VA TlbiNhVa; + SMMUV3_CMD_TLBI_EL2_ALL TlbiEl2All; + SMMUV3_CMD_TLBI_EL2_VA TlbiEl2Va; + SMMUV3_CMD_TLBI_EL2_VAA TlbiEl2Vaa; + SMMUV3_CMD_TLBI_EL2_ASID TlbiEl2Asid; + SMMUV3_CMD_TLBI_S2_IPA TlbiS2Ipa; + SMMUV3_CMD_TLBI_S12_VMALL TlbiS12VmAll; + SMMUV3_CMD_TLBI_NSNH_ALL TlbiNsnhAll; + SMMUV3_CMD_RESUME Resume; + SMMUV3_CMD_SYNC Sync; + + struct { + UINT64 CmdLow; + UINT64 CmdHigh; + } Raw; +} SMMUV3_CMD_GENERIC; + +// +// Event/Fault types and their formats. +// + +typedef enum { + FaultTypeUnsupportedUpstreamTransaction = 0x1, + FaultTypeStartingFault = FaultTypeUnsupportedUpstreamTransaction, + FaultTypeBadStreamId, + FaultTypeStreamEntryFetchAbort, + FaultTypeBadStreamEntry, + FaultTypeBadAtsTranslationRequest, + FaultTypeStreamDisabled, + FaultTypeTranslationForbidden, + FaultTypeBadSubstreamId, + FaultTypeContextDescriptorFetchAbort, + FaultTypeBadContextDescriptor, + FaultTypeTranslationWalkExternalAbort, + FaultTypeTranslation = 0x10, + FaultTypeAddressSize, + FaultTypeAccessFlag, + FaultTypePermission, + FaultTypeTlbConflict = 0x20, + FaultTypeConfigurationCacheConflict, + FaultTypePageRequest = 0x24, + FaultTypeVmsFetchAbort, + FaultTypeImplDefinedFaultStart = 0xE0, + FaultTypeImplDefinedFaultEnd = 0xEF, + FaultTypeEndingFault = FaultTypeImplDefinedFaultEnd, +} SMMUV3_FAULT_TYPE; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason : 16; + UINT32 Reserved1 : 16; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 28; + UINT64 InputAddress; + UINT64 Reserved4; +} SMMUV3_UNSUPPORTED_UPSTREAM_TRANSACTION_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT64 Reserved1; + UINT64 Reserved2; + UINT64 Reserved3; +} SMMUV3_BAD_STREAM_ID_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason : 16; + UINT32 Reserved1 : 16; + UINT32 Reserved2; + UINT64 Reserved3; + UINT64 Reserved4 : 3; + UINT64 FetchAddress : 49; + UINT64 Reserved5 : 12; +} SMMUV3_STREAM_ENTRY_FETCH_ABORT_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT64 Reserved1; + UINT64 Reserved2; + UINT64 Reserved3; +} SMMUV3_BAD_STREAM_ENTRY_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Span : 4; + UINT32 Reserved1 : 24; + UINT32 Privilege : 1; + UINT32 Execute : 1; + UINT32 Write : 1; + UINT32 Read : 1; + UINT32 Reserved2; + UINT64 Reserved3 : 12; + UINT64 InputAddress : 52; + UINT64 Reserved4; +} SMMUV3_BAD_ATS_TRANSLATION_REQUEST; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 24; + UINT32 StreamId; + UINT64 Reserved1; + UINT64 Reserved2; + UINT64 Reserved3; +} SMMUV3_STREAM_DISABLED_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 24; + UINT32 StreamId; + UINT32 Reserved1; + UINT32 Reserved2 : 3; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 28; + UINT64 InputAddress; + UINT64 Reserved4; +} SMMUV3_TRANSLATION_FORBIDDEN_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 4; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT64 Reserved1; + UINT64 Reserved2; + UINT64 Reserved3; +} SMMUV3_BAD_SUBSTREAM_ID_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason : 16; + UINT32 Reserved1 : 16; + UINT32 Reserved2; + UINT64 Reserved3; + UINT64 Reserved4 : 3; + UINT64 FetchAddress : 49; + UINT64 Reserved5 : 12; +} SMMUV3_CONTEXT_DESCRIPTOR_FETCH_ABORT_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT64 Reserved1; + UINT64 Reserved2; + UINT64 Reserved3; +} SMMUV3_BAD_CONTEXT_DESCRIPTOR_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason : 16; + UINT32 Reserved1 : 16; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 2; + UINT32 NonSecureIpa : 1; + UINT32 Stage2 : 1; + UINT32 Class : 2; + UINT32 Reserved4 : 22; + UINT64 InputAddress; + UINT64 Reserved5 : 3; + UINT64 FetchAddress : 49; + UINT64 Reserved6 : 12; +} SMMUV3_TRANSLATION_WALK_EXTERNAL_ABORT_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Stag : 16; + UINT32 Reserved1 : 15; + UINT32 Stall : 1; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 2; + UINT32 NonSecureIpa : 1; + UINT32 Stage2 : 1; + UINT32 Class : 2; + UINT32 Reserved4 : 6; + UINT32 ImplDefined : 16; + UINT64 InputAddress; + UINT64 Reserved5 : 12; + UINT64 Ipa : 40; + UINT64 Reserved6 : 12; +} SMMUV3_TRANSLATION_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Stag : 16; + UINT32 Reserved1 : 15; + UINT32 Stall : 1; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 2; + UINT32 NonSecureIpa : 1; + UINT32 Stage2 : 1; + UINT32 Class : 2; + UINT32 Reserved4 : 6; + UINT32 ImplDefined : 16; + UINT64 InputAddress; + UINT64 Reserved5 : 12; + UINT64 Ipa : 40; + UINT64 Reserved6 : 12; +} SMMUV3_ADDRESS_SIZE_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Stag : 16; + UINT32 Reserved1 : 15; + UINT32 Stall : 1; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 2; + UINT32 NonSecureIpa : 1; + UINT32 Stage2 : 1; + UINT32 Class : 2; + UINT32 Reserved4 : 6; + UINT32 ImplDefined : 16; + UINT64 InputAddress; + UINT64 Reserved5 : 12; + UINT64 Ipa : 40; + UINT64 Reserved6 : 12; +} SMMUV3_ACCESS_FLAG_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Stag : 16; + UINT32 Reserved1 : 15; + UINT32 Stall : 1; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 2; + UINT32 NonSecureIpa : 1; + UINT32 Stage2 : 1; + UINT32 Class : 2; + UINT32 Reserved4 : 6; + UINT32 ImplDefined : 16; + UINT64 InputAddress; + UINT64 Reserved5 : 12; + UINT64 Ipa : 40; + UINT64 Reserved6 : 12; +} SMMUV3_PERMISSION_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason; + UINT32 Reserved1 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved2 : 2; + UINT32 NonSecureIpa : 1; + UINT32 Stage2 : 1; + UINT32 Reserved3 : 24; + UINT64 InputAddress; + UINT64 Reserved5 : 12; + UINT64 Ipa : 40; + UINT64 Reserved6 : 12; +} SMMUV3_TLB_CONFLICT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason; + UINT32 Reserved1; + UINT64 Reserved2; + UINT64 Reserved3; +} SMMUV3_CONFIGURATION_CACHE_CONFLICT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reserved1; + UINT32 Reserved2 : 1; + UINT32 UserExecute : 1; + UINT32 UserWrite : 1; + UINT32 UserRead : 1; + UINT32 Reserved3 : 1; + UINT32 PrivilegedExecute : 1; + UINT32 PrivilegedWrite : 1; + UINT32 PrivilegedRead : 1; + UINT32 Reserved4 : 4; + UINT32 Span : 8; + UINT32 Reserved5 : 12; + UINT64 Reserved6 : 12; + UINT64 InputAddress : 52; + UINT64 Reserved7; +} SMMUV3_PAGE_REQUEST_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason : 16; + UINT32 Reserved1 : 16; + UINT32 Reserved2; + UINT64 Reserved3; + UINT64 Reserved4 : 3; + UINT64 FetchAddress : 49; + UINT64 Reserved5 : 12; +} SMMUV3_VMS_FETCH_ABORT_FAULT; + +typedef struct { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reserved1; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 3; + UINT32 Stage2 : 1; + UINT32 Class : 2; + UINT32 Reserved4 : 22; + UINT64 InputAddress; + UINT64 Reserved5; +} SMMUV3_FAULT_GENERIC; + +typedef union { + SMMUV3_UNSUPPORTED_UPSTREAM_TRANSACTION_FAULT UnsupportedUpstream; + SMMUV3_BAD_STREAM_ID_FAULT BadStreamdId; + SMMUV3_STREAM_ENTRY_FETCH_ABORT_FAULT StreamEntryFetchAbort; + SMMUV3_BAD_STREAM_ENTRY_FAULT BadStreamEntry; + SMMUV3_BAD_ATS_TRANSLATION_REQUEST BadAtsRequest; + SMMUV3_STREAM_DISABLED_FAULT StreamDisabled; + SMMUV3_TRANSLATION_FORBIDDEN_FAULT TranslationForbidden; + SMMUV3_BAD_SUBSTREAM_ID_FAULT BadSubstreamId; + SMMUV3_CONTEXT_DESCRIPTOR_FETCH_ABORT_FAULT CdFetchAbort; + SMMUV3_BAD_CONTEXT_DESCRIPTOR_FAULT CdFault; + SMMUV3_TRANSLATION_WALK_EXTERNAL_ABORT_FAULT WalkAbort; + SMMUV3_TRANSLATION_FAULT Translation; + SMMUV3_ADDRESS_SIZE_FAULT AddressSize; + SMMUV3_ACCESS_FLAG_FAULT AccessFlag; + SMMUV3_PERMISSION_FAULT Permission; + SMMUV3_TLB_CONFLICT TlbConflict; + SMMUV3_CONFIGURATION_CACHE_CONFLICT ConfigCache; + SMMUV3_PAGE_REQUEST_FAULT PageRequest; + SMMUV3_VMS_FETCH_ABORT_FAULT VmsFetchAbort; + SMMUV3_FAULT_GENERIC Generic; + UINT64 Fault[4]; +} SMMUV3_FAULT_RECORD; + +// +// --------------------------------------------------------------------- Macros +// + +// +// Define mask for command queue opcodes. +// + +#define SMMUV3_COMMAND_OPCODE_MASK (0xFF) +#define SMMUV3_LAST_VALID_COMMAND_OPCODE (CmdSync) + +// +// Define macros to build various cache, TLB and fault response commands. +// + +#define SMMUV3_BUILD_CMD_CFGI_STE(Command, InputStreamId, InputLeaf) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->CfgiSte.Opcode = CmdCfgiSte; \ + (Command)->CfgiSte.StreamId = InputStreamId; \ + (Command)->CfgiSte.Leaf = InputLeaf; \ +} + +#define SMMUV3_BUILD_CMD_CFGI_STE_RANGE(Command, InputStreamId, InputRange) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->CfgiSteRange.Opcode = CmdCfgiSteRange; \ + (Command)->CfgiSteRange.StreamId = InputStreamId; \ + (Command)->CfgiSteRange.Range = InputRange; \ +} + +#define SMMUV3_BUILD_CMD_CFGI_CD(Command, InputStreamId, InputLeaf) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->CfgiCd.Opcode = CmdCfgiCd; \ + (Command)->CfgiCd.StreamId = InputStreamId; \ + (Command)->CfgiCd.Leaf = InputLeaf; \ +} + +#define SMMUV3_BUILD_CMD_CFGI_CD_ALL(Command, InputStreamId) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->CfgiCdAll.Opcode = CmdCfgiCdAll; \ + (Command)->CfgiCdAll.StreamId = InputStreamId; \ +} + +#define SMMUV3_BUILD_CMD_CFGI_ALL(Command) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->CfgiSteRange.Opcode = CmdCfgiSteRange; \ + (Command)->CfgiSteRange.Range = 31; \ +} + +#define SMMUV3_BUILD_CMD_TLBI_EL2_ALL(Command) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->TlbiEl2All.Opcode = CmdTlbiEl2All; \ +} + +#define SMMUV3_BUILD_CMD_TLBI_NSNH_ALL(Command) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->TlbiNsnhAll.Opcode = CmdTlbiNsnhAll; \ +} + +#define SMMUV3_BUILD_CMD_SYNC_NO_INTERRUPT(Command) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->Sync.Opcode = CmdSync; \ +} + +#define SMMUV3_BUILD_CMD_TLBI_NH_ALL(Command, InputVmid) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->TlbiNhAll.Opcode = CmdTlbiNhAll; \ + (Command)->TlbiNhAsid.Vmid = InputVmid; \ +} + +#define SMMUV3_BUILD_CMD_TLBI_NH_ASID(Command, InputVmid, InputAsid) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->TlbiNhAsid.Opcode = CmdTlbiNhAsid; \ + (Command)->TlbiNhAsid.Vmid = InputVmid; \ + (Command)->TlbiNhAsid.Asid = InputAsid; \ +} + +#define SMMUV3_BUILD_CMD_TLBI_NH_VA(Command, \ + InputVmid, \ + InputAsid, \ + InputAddress) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->TlbiNhVa.Opcode = CmdTlbiNhVa; \ + (Command)->TlbiNhVa.Vmid = InputVmid; \ + (Command)->TlbiNhVa.Asid = InputAsid; \ + (Command)->TlbiNhVa.Address = ((InputAddress) >> 12); \ +} + +#define SMMUV3_BUILD_CMD_TLBI_NH_VAA(Command, \ + InputVmid, \ + InputAddress) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->TlbiNhVaa.Opcode = CmdTlbiNhVaa; \ + (Command)->TlbiNhVaa.Vmid = InputVmid; \ + (Command)->TlbiNhVaa.Address = ((InputAddress) >> 12); \ +} + +// +// Stage-2 TLB invalidation macros. +// + +#define SMMUV3_BUILD_CMD_TLBI_S12_VMALL(Command, InputVmid) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->TlbiS12VmAll.Opcode = CmdTlbiS12VmAll; \ + (Command)->TlbiS12VmAll.Vmid = InputVmid; \ +} + +#define SMMUV3_BUILD_CMD_TLBI_S2_IPA(Command, InputVmid, InputAddress) \ +{ \ + (Command)->Raw.CmdLow = 0; \ + (Command)->Raw.CmdHigh = 0; \ + (Command)->TlbiS2Ipa.Opcode = CmdTlbiS2Ipa; \ + (Command)->TlbiS2Ipa.Vmid = InputVmid; \ + (Command)->TlbiS2Ipa.Address = ((InputAddress) >> 12); \ +} + +#define SMMUV3_LINEAR_STREAM_TABLE_SIZE_FROM_LOG2(Log2Size) \ + (UINT32)((UINT32)(1UL << (Log2Size)) * \ + (UINT16)sizeof(SMMUV3_STREAM_TABLE_ENTRY)) + +#define SMMUV3_L1_STREAM_TABLE_SIZE_FROM_LOG2(Log2Size) \ + (UINT32)((UINT32)(1UL << (Log2Size)) * \ + (UINT32)sizeof(UINT64)) + +// +// Register offsets for Page0. +// +#define SMMU_IDR0 0x0000 +#define SMMU_IDR1 0x0004 +#define SMMU_IDR2 0x0008 +#define SMMU_IDR3 0x000C +#define SMMU_IDR4 0x0010 +#define SMMU_IDR5 0x0014 +#define SMMU_IIDR 0x0018 +#define SMMU_AIDR 0x001C +#define SMMU_CR0 0x0020 +#define SMMU_CR0ACK 0x0024 +#define SMMU_CR1 0x0028 +#define SMMU_CR2 0x002C +#define SMMU_STATUSR 0x0040 +#define SMMU_GBPA 0x0044 +#define SMMU_AGBPA 0x0048 +#define SMMU_IRQ_CTRL 0x0050 +#define SMMU_IRQ_CTRLACK 0x0054 +#define SMMU_GERROR 0x0060 +#define SMMU_GERRORN 0x0064 +#define SMMU_GERROR_IRQ_CFG0 0x0068 +#define SMMU_GERROR_IRQ_CFG1 0x0070 +#define SMMU_GERROR_IRQ_CFG2 0x0074 +#define SMMU_STRTAB_BASE 0x0080 +#define SMMU_STRTAB_BASE_CFG 0x0088 +#define SMMU_CMDQ_BASE 0x0090 +#define SMMU_CMDQ_PROD 0x0098 +#define SMMU_CMDQ_CONS 0x009C +#define SMMU_EVENTQ_BASE 0x00A0 +#define SMMU_EVENTQ_PROD 0x00A8 +#define SMMU_EVENTQ_CONS 0x00AC +#define SMMU_EVENTQ_IRQ_CFG0 0x00B0 +#define SMMU_EVENTQ_IRQ_CFG1 0x00B8 +#define SMMU_EVENTQ_IRQ_CFG2 0x00BC +#define SMMU_PRIQ_BASE 0x00C0 +#define SMMU_PRIQ_PROD 0x00C8 +#define SMMU_PRIQ_CONS 0x00CC +#define SMMU_PRIQ_IRQ_CFG0 0x00D0 +#define SMMU_PRIQ_IRQ_CFG1 0x00D8 +#define SMMU_PRIQ_IRQ_CFG2 0x00DC +#define SMMU_GATOS_CTRL 0x0100 +#define SMMU_GATOS_SID 0x0108 +#define SMMU_GATOS_ADDR 0x0110 +#define SMMU_GATOS_PAR 0x0118 +#define SMMU_MPAMIDR 0x0130 +#define SMMU_GMPAM 0x0138 +#define SMMU_GBPMPAM 0x013C +#define SMMU_VATOS_SEL 0x0180 +#define SMMU_IDR6 0x0190 +#define SMMU_DPT_BASE 0x0200 +#define SMMU_DPT_BASE_CFG 0x0208 +#define SMMU_DPT_CFG_FAR 0x0210 + +// Implementation Defined Registers +#define SMMU_IMPL_DEF_START 0x0E00 +#define SMMU_IMPL_DEF_END 0x0EFF +#define SMMU_ID_REGS_START 0x0FD0 +#define SMMU_ID_REGS_END 0x0FFC +#define SMMU_IMPL_DEF2_START 0x1000 +#define SMMU_IMPL_DEF2_END 0x3FFF + +// Command Queue Control Page Registers +#define SMMU_CMDQ_CONTROL_PAGE_BASE(n) (0x4000 + 32 * (n)) +#define SMMU_CMDQ_CONTROL_PAGE_CFG(n) (0x4008 + 32 * (n)) +#define SMMU_CMDQ_CONTROL_PAGE_STATUS(n) (0x400C + 32 * (n)) + +// Secure Registers +#define SMMU_S_IDR0 0x8000 +#define SMMU_S_IDR1 0x8004 +#define SMMU_S_IDR2 0x8008 +#define SMMU_S_IDR3 0x800C +#define SMMU_S_IDR4 0x8010 +#define SMMU_S_CR0 0x8020 +#define SMMU_S_CR0ACK 0x8024 +#define SMMU_S_CR1 0x8028 +#define SMMU_S_CR2 0x802C +#define SMMU_S_INIT 0x803C +#define SMMU_S_GBPA 0x8044 +#define SMMU_S_AGBPA 0x8048 +#define SMMU_S_IRQ_CTRL 0x8050 +#define SMMU_S_IRQ_CTRLACK 0x8054 +#define SMMU_S_GERROR 0x8060 +#define SMMU_S_GERRORN 0x8064 +#define SMMU_S_GERROR_IRQ_CFG0 0x8068 +#define SMMU_S_GERROR_IRQ_CFG1 0x8070 +#define SMMU_S_GERROR_IRQ_CFG2 0x8074 +#define SMMU_S_STRTAB_BASE 0x8080 +#define SMMU_S_STRTAB_BASE_CFG 0x8088 +#define SMMU_S_CMDQ_BASE 0x8090 +#define SMMU_S_CMDQ_PROD 0x8098 +#define SMMU_S_CMDQ_CONS 0x809C +#define SMMU_S_EVENTQ_BASE 0x80A0 +#define SMMU_S_EVENTQ_PROD 0x80A8 +#define SMMU_S_EVENTQ_CONS 0x80AC +#define SMMU_S_EVENTQ_IRQ_CFG0 0x80B0 +#define SMMU_S_EVENTQ_IRQ_CFG1 0x80B8 +#define SMMU_S_EVENTQ_IRQ_CFG2 0x80BC +#define SMMU_S_GATOS_CTRL 0x8100 +#define SMMU_S_GATOS_SID 0x8108 +#define SMMU_S_GATOS_ADDR 0x8110 +#define SMMU_S_GATOS_PAR 0x8118 +#define SMMU_S_MPAMIDR 0x8130 +#define SMMU_S_GMPAM 0x8138 +#define SMMU_S_GBPMPAM 0x813C +#define SMMU_S_VATOS_SEL 0x8180 +#define SMMU_S_IDR6 0x8190 + +// Secure Implementation Defined Registers +#define SMMU_S_IMPL_DEF_START 0x8E00 +#define SMMU_S_IMPL_DEF_END 0x8EFF +#define SMMU_S_IMPL_DEF2_START 0x9000 +#define SMMU_S_IMPL_DEF2_END 0xBFFF + +// Secure Command Queue Control Page Registers +#define SMMU_S_CMDQ_CONTROL_PAGE_BASE(n) (0xC000 + 32 * (n)) +#define SMMU_S_CMDQ_CONTROL_PAGE_CFG(n) (0xC008 + 32 * (n)) +#define SMMU_S_CMDQ_CONTROL_PAGE_STATUS(n) (0xC00C + 32 * (n)) + +// SMMU_GBPA register fields. +#define SMMU_GBPA_UPDATE BIT31 +#define SMMU_GBPA_ABORT BIT20