/** @file | |
Copyright (c) 2022, Intel Corporation. All rights reserved.<BR> | |
SPDX-License-Identifier: BSD-2-Clause-Patent | |
**/ | |
#include <Library/BaseLib.h> | |
#include <Library/BaseMemoryLib.h> | |
#include <Library/DebugLib.h> | |
#include <Library/MemoryAllocationLib.h> | |
#include <Library/MemEncryptSevLib.h> | |
#include <Library/MemEncryptTdxLib.h> | |
#include <Library/PcdLib.h> | |
#include <Library/SynchronizationLib.h> | |
#include <Library/UefiBootServicesTableLib.h> | |
#include "IoMmuInternal.h" | |
extern BOOLEAN mReservedSharedMemSupported; | |
#define SIZE_OF_MEM_RANGE(MemRange) (MemRange->HeaderSize + MemRange->DataSize) | |
#define RESERVED_MEM_BITMAP_4K_MASK 0xf | |
#define RESERVED_MEM_BITMAP_32K_MASK 0xff0 | |
#define RESERVED_MEM_BITMAP_128K_MASK 0x3000 | |
#define RESERVED_MEM_BITMAP_1M_MASK 0x40000 | |
#define RESERVED_MEM_BITMAP_2M_MASK 0x180000 | |
#define RESERVED_MEM_BITMAP_MASK 0x1fffff | |
/** | |
* mReservedMemRanges describes the layout of the reserved memory. | |
* The reserved memory consists of disfferent size of memory region. | |
* The pieces of memory with the same size are managed by one entry | |
* in the mReservedMemRanges. All the pieces of memories are managed by | |
* mReservedMemBitmap which is a UINT32. It means it can manage at most | |
* 32 pieces of memory. Because of the layout of CommonBuffer | |
* (1-page header + n-page data), a piece of reserved memory consists of | |
* 2 parts: Header + Data. | |
* | |
* So put all these together, mReservedMemRanges and mReservedMemBitmap | |
* are designed to manage the reserved memory. | |
* | |
* Use the second entry of mReservedMemRanges as an example. | |
* { RESERVED_MEM_BITMAP_32K_MASK, 4, 8, SIZE_32KB, SIZE_4KB, 0 }, | |
* - RESERVED_MEM_BITMAP_32K_MASK is 0xff0. It means bit4-11 in mReservedMemBitmap | |
* is reserved for 32K size memory. | |
* - 4 is the shift of mReservedMemBitmap. | |
* - 8 means there are 8 pieces of 32K size memory. | |
* - SIZE_32KB indicates the size of Data part. | |
* - SIZE_4KB is the size of Header part. | |
* - 0 is the start address of this memory range which will be populated when | |
* the reserved memory is initialized. | |
* | |
* The size and count of the memory region are derived from the experience. For | |
* a typical grub boot, there are about 5100 IoMmu/DMA operation. Most of these | |
* DMA operation require the memory with size less than 32K (~5080). But we find | |
* in grub boot there may be 2 DMA operation which require for the memory larger | |
* than 1M. And these 2 DMA operation occur concurrently. So we reserve 2 pieces | |
* of memory with size of SIZE_2MB. This is for the best boot performance. | |
* | |
* If all the reserved memory are exausted, then it will fall back to the legacy | |
* memory allocation as before. | |
*/ | |
STATIC IOMMU_RESERVED_MEM_RANGE mReservedMemRanges[] = { | |
{ RESERVED_MEM_BITMAP_4K_MASK, 0, 4, SIZE_4KB, SIZE_4KB, 0 }, | |
{ RESERVED_MEM_BITMAP_32K_MASK, 4, 8, SIZE_32KB, SIZE_4KB, 0 }, | |
{ RESERVED_MEM_BITMAP_128K_MASK, 12, 2, SIZE_128KB, SIZE_4KB, 0 }, | |
{ RESERVED_MEM_BITMAP_1M_MASK, 14, 1, SIZE_1MB, SIZE_4KB, 0 }, | |
{ RESERVED_MEM_BITMAP_2M_MASK, 15, 2, SIZE_2MB, SIZE_4KB, 0 }, | |
}; | |
// | |
// Bitmap of the allocation of reserved memory. | |
// | |
STATIC UINT32 mReservedMemBitmap = 0; | |
// | |
// Start address of the reserved memory region. | |
// | |
STATIC EFI_PHYSICAL_ADDRESS mReservedSharedMemAddress = 0; | |
// | |
// Total size of the reserved memory region. | |
// | |
STATIC UINT32 mReservedSharedMemSize = 0; | |
/** | |
* Calculate the size of reserved memory. | |
* | |
* @retval UINT32 Size of the reserved memory | |
*/ | |
STATIC | |
UINT32 | |
CalcuateReservedMemSize ( | |
VOID | |
) | |
{ | |
UINT32 Index; | |
IOMMU_RESERVED_MEM_RANGE *MemRange; | |
if (mReservedSharedMemSize != 0) { | |
return mReservedSharedMemSize; | |
} | |
for (Index = 0; Index < ARRAY_SIZE (mReservedMemRanges); Index++) { | |
MemRange = &mReservedMemRanges[Index]; | |
mReservedSharedMemSize += (SIZE_OF_MEM_RANGE (MemRange) * MemRange->Slots); | |
} | |
return mReservedSharedMemSize; | |
} | |
/** | |
* Allocate a memory region and convert it to be shared. This memory region will be | |
* used in the DMA operation. | |
* | |
* The pre-alloc memory contains pieces of memory regions with different size. The | |
* allocation of the shared memory regions are indicated by a 32-bit bitmap (mReservedMemBitmap). | |
* | |
* The memory regions are consumed by IoMmuAllocateBuffer (in which CommonBuffer is allocated) and | |
* IoMmuMap (in which bounce buffer is allocated). | |
* | |
* The CommonBuffer contains 2 parts, one page for CommonBufferHeader which is private memory, | |
* the other part is shared memory. So the layout of a piece of memory region after initialization | |
* looks like: | |
* | |
* |------------|----------------------------| | |
* | Header | Data | <-- a piece of pre-alloc memory region | |
* | 4k, private| 4k/32k/128k/etc, shared | | |
* |-----------------------------------------| | |
* | |
* @retval EFI_SUCCESS Successfully initialize the reserved memory. | |
* @retval EFI_UNSUPPORTED This feature is not supported. | |
*/ | |
EFI_STATUS | |
IoMmuInitReservedSharedMem ( | |
VOID | |
) | |
{ | |
EFI_STATUS Status; | |
UINT32 Index1, Index2; | |
UINTN TotalPages; | |
IOMMU_RESERVED_MEM_RANGE *MemRange; | |
EFI_PHYSICAL_ADDRESS PhysicalAddress; | |
UINT64 SharedAddress; | |
if (!mReservedSharedMemSupported) { | |
return EFI_UNSUPPORTED; | |
} | |
TotalPages = EFI_SIZE_TO_PAGES (CalcuateReservedMemSize ()); | |
PhysicalAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)AllocatePages (TotalPages); | |
DEBUG (( | |
DEBUG_VERBOSE, | |
"%a: ReservedMem (%d pages) address = 0x%llx\n", | |
__func__, | |
TotalPages, | |
PhysicalAddress | |
)); | |
mReservedMemBitmap = 0; | |
mReservedSharedMemAddress = PhysicalAddress; | |
for (Index1 = 0; Index1 < ARRAY_SIZE (mReservedMemRanges); Index1++) { | |
MemRange = &mReservedMemRanges[Index1]; | |
MemRange->StartAddressOfMemRange = PhysicalAddress; | |
for (Index2 = 0; Index2 < MemRange->Slots; Index2++) { | |
SharedAddress = (UINT64)(UINTN)(MemRange->StartAddressOfMemRange + Index2 * SIZE_OF_MEM_RANGE (MemRange) + MemRange->HeaderSize); | |
if (CC_GUEST_IS_SEV (PcdGet64 (PcdConfidentialComputingGuestAttr))) { | |
Status = MemEncryptSevClearPageEncMask ( | |
0, | |
SharedAddress, | |
EFI_SIZE_TO_PAGES (MemRange->DataSize) | |
); | |
ASSERT (!EFI_ERROR (Status)); | |
} else if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) { | |
Status = MemEncryptTdxSetPageSharedBit ( | |
0, | |
SharedAddress, | |
EFI_SIZE_TO_PAGES (MemRange->DataSize) | |
); | |
ASSERT (!EFI_ERROR (Status)); | |
} else { | |
ASSERT (FALSE); | |
} | |
} | |
PhysicalAddress += (MemRange->Slots * SIZE_OF_MEM_RANGE (MemRange)); | |
} | |
return EFI_SUCCESS; | |
} | |
/** | |
* Release the pre-alloc shared memory. | |
* | |
* @retval EFI_SUCCESS Successfully release the shared memory | |
*/ | |
EFI_STATUS | |
IoMmuReleaseReservedSharedMem ( | |
BOOLEAN MemoryMapLocked | |
) | |
{ | |
EFI_STATUS Status; | |
UINT32 Index1, Index2; | |
IOMMU_RESERVED_MEM_RANGE *MemRange; | |
UINT64 SharedAddress; | |
if (!mReservedSharedMemSupported) { | |
return EFI_SUCCESS; | |
} | |
for (Index1 = 0; Index1 < ARRAY_SIZE (mReservedMemRanges); Index1++) { | |
MemRange = &mReservedMemRanges[Index1]; | |
for (Index2 = 0; Index2 < MemRange->Slots; Index2++) { | |
SharedAddress = (UINT64)(UINTN)(MemRange->StartAddressOfMemRange + Index2 * SIZE_OF_MEM_RANGE (MemRange) + MemRange->HeaderSize); | |
if (CC_GUEST_IS_SEV (PcdGet64 (PcdConfidentialComputingGuestAttr))) { | |
Status = MemEncryptSevSetPageEncMask ( | |
0, | |
SharedAddress, | |
EFI_SIZE_TO_PAGES (MemRange->DataSize) | |
); | |
ASSERT (!EFI_ERROR (Status)); | |
} else if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) { | |
Status = MemEncryptTdxClearPageSharedBit ( | |
0, | |
SharedAddress, | |
EFI_SIZE_TO_PAGES (MemRange->DataSize) | |
); | |
ASSERT (!EFI_ERROR (Status)); | |
} else { | |
ASSERT (FALSE); | |
} | |
} | |
} | |
if (!MemoryMapLocked) { | |
FreePages ((VOID *)(UINTN)mReservedSharedMemAddress, EFI_SIZE_TO_PAGES (CalcuateReservedMemSize ())); | |
mReservedSharedMemAddress = 0; | |
mReservedMemBitmap = 0; | |
} | |
mReservedSharedMemSupported = FALSE; | |
return EFI_SUCCESS; | |
} | |
/** | |
* Allocate from the reserved memory pool. | |
* If the reserved shared memory is exausted or there is no suitalbe size, it turns | |
* to the LegacyAllocateBuffer. | |
* | |
* @param Type Allocate type | |
* @param MemoryType The memory type to be allocated | |
* @param Pages Pages to be allocated. | |
* @param ReservedMemBitmap Bitmap of the allocated memory region | |
* @param PhysicalAddress Pointer to the data part of allocated memory region | |
* | |
* @retval EFI_SUCCESS Successfully allocate the buffer | |
* @retval Other As the error code indicates | |
*/ | |
STATIC | |
EFI_STATUS | |
InternalAllocateBuffer ( | |
IN EFI_ALLOCATE_TYPE Type, | |
IN EFI_MEMORY_TYPE MemoryType, | |
IN UINTN Pages, | |
OUT UINT32 *ReservedMemBit, | |
IN OUT EFI_PHYSICAL_ADDRESS *PhysicalAddress | |
) | |
{ | |
UINT32 MemBitmap; | |
UINT32 ReservedMemBitmap; | |
UINT8 Index; | |
IOMMU_RESERVED_MEM_RANGE *MemRange; | |
UINTN PagesOfLastMemRange; | |
*ReservedMemBit = 0; | |
if (Pages == 0) { | |
ASSERT (FALSE); | |
return EFI_INVALID_PARAMETER; | |
} | |
if (!mReservedSharedMemSupported) { | |
goto LegacyAllocateBuffer; | |
} | |
if (mReservedSharedMemAddress == 0) { | |
goto LegacyAllocateBuffer; | |
} | |
PagesOfLastMemRange = 0; | |
for (Index = 0; Index < ARRAY_SIZE (mReservedMemRanges); Index++) { | |
if ((Pages > PagesOfLastMemRange) && (Pages <= EFI_SIZE_TO_PAGES (mReservedMemRanges[Index].DataSize))) { | |
break; | |
} | |
PagesOfLastMemRange = EFI_SIZE_TO_PAGES (mReservedMemRanges[Index].DataSize); | |
} | |
if (Index == ARRAY_SIZE (mReservedMemRanges)) { | |
// There is no suitable size of reserved memory. Turn to legacy allocate. | |
goto LegacyAllocateBuffer; | |
} | |
MemRange = &mReservedMemRanges[Index]; | |
do { | |
ReservedMemBitmap = mReservedMemBitmap; | |
if ((ReservedMemBitmap & MemRange->BitmapMask) == MemRange->BitmapMask) { | |
// The reserved memory is exhausted. Turn to legacy allocate. | |
goto LegacyAllocateBuffer; | |
} | |
MemBitmap = (ReservedMemBitmap & MemRange->BitmapMask) >> MemRange->Shift; | |
for (Index = 0; Index < MemRange->Slots; Index++) { | |
if ((MemBitmap & (UINT8)(1<<Index)) == 0) { | |
break; | |
} | |
} | |
ASSERT (Index != MemRange->Slots); | |
*PhysicalAddress = MemRange->StartAddressOfMemRange + Index * SIZE_OF_MEM_RANGE (MemRange) + MemRange->HeaderSize; | |
*ReservedMemBit = (UINT32)(1 << (Index + MemRange->Shift)); | |
} while (ReservedMemBitmap != InterlockedCompareExchange32 ( | |
&mReservedMemBitmap, | |
ReservedMemBitmap, | |
ReservedMemBitmap | *ReservedMemBit | |
)); | |
DEBUG (( | |
DEBUG_VERBOSE, | |
"%a: range-size: %lx, start-address=0x%llx, pages=0x%llx, bits=0x%lx, bitmap: %lx => %lx\n", | |
__func__, | |
MemRange->DataSize, | |
*PhysicalAddress, | |
Pages, | |
*ReservedMemBit, | |
ReservedMemBitmap, | |
ReservedMemBitmap | *ReservedMemBit | |
)); | |
return EFI_SUCCESS; | |
LegacyAllocateBuffer: | |
*ReservedMemBit = 0; | |
return gBS->AllocatePages (Type, MemoryType, Pages, PhysicalAddress); | |
} | |
/** | |
* Allocate reserved shared memory for bounce buffer. | |
* | |
* @param Type Allocate type | |
* @param MemoryType The memory type to be allocated | |
* @param MapInfo Pointer to the MAP_INFO | |
* | |
* @retval EFI_SUCCESS Successfully allocate the bounce buffer | |
* @retval Other As the error code indicates | |
*/ | |
EFI_STATUS | |
IoMmuAllocateBounceBuffer ( | |
IN EFI_ALLOCATE_TYPE Type, | |
IN EFI_MEMORY_TYPE MemoryType, | |
IN OUT MAP_INFO *MapInfo | |
) | |
{ | |
EFI_STATUS Status; | |
Status = InternalAllocateBuffer ( | |
Type, | |
MemoryType, | |
MapInfo->NumberOfPages, | |
&MapInfo->ReservedMemBitmap, | |
&MapInfo->PlainTextAddress | |
); | |
ASSERT (Status == EFI_SUCCESS); | |
return Status; | |
} | |
/** | |
* Clear a bit in the reserved memory bitmap in a thread safe manner | |
* | |
* @param ReservedMemBit The bit to clear | |
*/ | |
STATIC | |
VOID | |
ClearReservedMemBit ( | |
IN UINT32 ReservedMemBit | |
) | |
{ | |
UINT32 ReservedMemBitmap; | |
do { | |
ReservedMemBitmap = mReservedMemBitmap; | |
} while (ReservedMemBitmap != InterlockedCompareExchange32 ( | |
&mReservedMemBitmap, | |
ReservedMemBitmap, | |
ReservedMemBitmap & ~ReservedMemBit | |
)); | |
} | |
/** | |
* Free the bounce buffer allocated in IoMmuAllocateBounceBuffer. | |
* | |
* @param MapInfo Pointer to the MAP_INFO | |
* @return EFI_SUCCESS Successfully free the bounce buffer. | |
*/ | |
EFI_STATUS | |
IoMmuFreeBounceBuffer ( | |
IN OUT MAP_INFO *MapInfo | |
) | |
{ | |
if (MapInfo->ReservedMemBitmap == 0) { | |
gBS->FreePages (MapInfo->PlainTextAddress, MapInfo->NumberOfPages); | |
} else { | |
DEBUG (( | |
DEBUG_VERBOSE, | |
"%a: PlainTextAddress=0x%Lx, bits=0x%Lx, bitmap: %Lx => %Lx\n", | |
__func__, | |
MapInfo->PlainTextAddress, | |
MapInfo->ReservedMemBitmap, | |
mReservedMemBitmap, | |
mReservedMemBitmap & ((UINT32)(~MapInfo->ReservedMemBitmap)) | |
)); | |
ClearReservedMemBit (MapInfo->ReservedMemBitmap); | |
MapInfo->PlainTextAddress = 0; | |
MapInfo->ReservedMemBitmap = 0; | |
} | |
return EFI_SUCCESS; | |
} | |
/** | |
* Allocate CommonBuffer from pre-allocated shared memory. | |
* | |
* @param MemoryType Memory type | |
* @param CommonBufferPages Pages of CommonBuffer | |
* @param PhysicalAddress Allocated physical address | |
* @param ReservedMemBitmap Bitmap which indicates the allocation of reserved memory | |
* | |
* @retval EFI_SUCCESS Successfully allocate the common buffer | |
* @retval Other As the error code indicates | |
*/ | |
EFI_STATUS | |
IoMmuAllocateCommonBuffer ( | |
IN EFI_MEMORY_TYPE MemoryType, | |
IN UINTN CommonBufferPages, | |
OUT EFI_PHYSICAL_ADDRESS *PhysicalAddress, | |
OUT UINT32 *ReservedMemBitmap | |
) | |
{ | |
EFI_STATUS Status; | |
Status = InternalAllocateBuffer ( | |
AllocateMaxAddress, | |
MemoryType, | |
CommonBufferPages, | |
ReservedMemBitmap, | |
PhysicalAddress | |
); | |
ASSERT (Status == EFI_SUCCESS); | |
if (*ReservedMemBitmap != 0) { | |
*PhysicalAddress -= SIZE_4KB; | |
} | |
return Status; | |
} | |
/** | |
* Free CommonBuffer which is allocated by IoMmuAllocateCommonBuffer(). | |
* | |
* @param CommonBufferHeader Pointer to the CommonBufferHeader | |
* @param CommonBufferPages Pages of CommonBuffer | |
* | |
* @retval EFI_SUCCESS Successfully free the common buffer | |
* @retval Other As the error code indicates | |
*/ | |
EFI_STATUS | |
IoMmuFreeCommonBuffer ( | |
IN COMMON_BUFFER_HEADER *CommonBufferHeader, | |
IN UINTN CommonBufferPages | |
) | |
{ | |
if (!mReservedSharedMemSupported) { | |
goto LegacyFreeCommonBuffer; | |
} | |
if (CommonBufferHeader->ReservedMemBitmap == 0) { | |
goto LegacyFreeCommonBuffer; | |
} | |
DEBUG (( | |
DEBUG_VERBOSE, | |
"%a: CommonBuffer=0x%Lx, bits=0x%Lx, bitmap: %Lx => %Lx\n", | |
__func__, | |
(UINT64)(UINTN)CommonBufferHeader + SIZE_4KB, | |
CommonBufferHeader->ReservedMemBitmap, | |
mReservedMemBitmap, | |
mReservedMemBitmap & ((UINT32)(~CommonBufferHeader->ReservedMemBitmap)) | |
)); | |
ClearReservedMemBit (CommonBufferHeader->ReservedMemBitmap); | |
return EFI_SUCCESS; | |
LegacyFreeCommonBuffer: | |
return gBS->FreePages ((UINTN)CommonBufferHeader, CommonBufferPages); | |
} |