Add Pagetable manager and auxtable support.
Change-Id: I429b52a70c77fc304c65d28d42b18f99b01116d5
diff --git a/Source/GmmLib/CMakeLists.txt b/Source/GmmLib/CMakeLists.txt
index dbb3d5e..db4373d 100644
--- a/Source/GmmLib/CMakeLists.txt
+++ b/Source/GmmLib/CMakeLists.txt
@@ -191,6 +191,7 @@
${BS_DIR_GMMLIB}/inc/External/Common/GmmConst.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmDebug.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmFormatTable.h
+ ${BS_DIR_GMMLIB}/inc/External/Common/GmmHw.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmInfo.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmInfoExt.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmInternal.h
@@ -205,6 +206,7 @@
${BS_DIR_GMMLIB}/inc/External/Common/GmmTextureExt.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmUtil.h
${BS_DIR_GMMLIB}/inc/External/Linux/GmmResourceInfoLin.h
+ ${BS_DIR_GMMLIB}/inc/External/Linux/GmmResourceInfoLinExt.h
${BS_DIR_GMMLIB}/inc/Internal/Common/Platform/GmmGen10Platform.h
${BS_DIR_GMMLIB}/inc/Internal/Common/Platform/GmmGen11Platform.h
${BS_DIR_GMMLIB}/inc/Internal/Common/Platform/GmmGen12Platform.h
@@ -225,6 +227,7 @@
set(UMD_HEADERS
${HEADERS_}
+ ${BS_DIR_GMMLIB}/inc/External/Common/GmmPageTableMgr.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmClientContext.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmLibDll.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmLibDllName.h
@@ -250,6 +253,7 @@
${BS_DIR_GMMLIB}/Resource/GmmResourceInfoCommon.cpp
${BS_DIR_GMMLIB}/Resource/GmmResourceInfoCommonEx.cpp
${BS_DIR_GMMLIB}/Resource/GmmRestrictions.cpp
+ ${BS_DIR_GMMLIB}/Resource/Linux/GmmResourceInfoLinCWrapper.cpp
${BS_DIR_GMMLIB}/Texture/GmmGen7Texture.cpp
${BS_DIR_GMMLIB}/Texture/GmmGen8Texture.cpp
${BS_DIR_GMMLIB}/Texture/GmmGen9Texture.cpp
@@ -269,6 +273,9 @@
set(UMD_SOURCES
${SOURCES_}
+ ${BS_DIR_GMMLIB}/TranslationTable/GmmAuxTable.cpp
+ ${BS_DIR_GMMLIB}/TranslationTable/GmmPageTableMgr.cpp
+ ${BS_DIR_GMMLIB}/TranslationTable/GmmUmdTranslationTable.cpp
${BS_DIR_GMMLIB}/GlobalInfo/GmmClientContext.cpp
${BS_DIR_GMMLIB}/GlobalInfo/GmmOldApi.cpp
${BS_DIR_GMMLIB}/GlobalInfo/GmmLibDllMain.cpp
@@ -287,13 +294,27 @@
source_group("Source Files\\Global" ${BS_DIR_GMMLIB}/GlobalInfo/.*)
source_group("Source Files\\Platform" ${BS_DIR_GMMLIB}/Platform/.*)
source_group("Source Files\\Texture" ${BS_DIR_GMMLIB}/Texture/.*)
+source_group("Source Files\\Translation Table" ${BS_DIR_GMMLIB}/TranslationTable/.*)
source_group("Source Files\\Utility" ${BS_DIR_GMMLIB}/Utility/.*)
source_group("Source Files\\Resource" FILES
${BS_DIR_GMMLIB}/Resource/GmmResourceInfo.cpp
${BS_DIR_GMMLIB}/Resource/GmmResourceInfoCommon.cpp
+ ${BS_DIR_GMMLIB}/Resource/GmmResourceInfoCommonEx.cpp
${BS_DIR_GMMLIB}/Resource/GmmRestrictions.cpp)
+source_group("Source Files\\Resource\\Linux" FILES
+ ${BS_DIR_GMMLIB}/Resource/Linux/GmmResourceInfoLinCWrapper.cpp
+ )
+
+source_group("Source Files\\TranslationTable\\Windows" FILES
+ ${BS_DIR_GMMLIB}/TranslationTable/GmmAuxTable.cpp
+ ${BS_DIR_GMMLIB}/TranslationTable/GmmPageTableMgr.cpp
+ ${BS_DIR_GMMLIB}/TranslationTable/GmmUmdTranslationTable.cpp)
+
+source_group("Source Files\\TranslationTable" FILES
+ ${BS_DIR_GMMLIB}/TranslationTable/GmmUmdTranslationTable.h)
+
source_group("Header Files\\External\\Common" FILES
${BS_DIR_GMMLIB}/inc/External/Common/GmmCachePolicy.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmCachePolicyCommon.h
@@ -302,6 +323,7 @@
${BS_DIR_GMMLIB}/inc/External/Common/GmmConst.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmDebug.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmFormatTable.h
+ ${BS_DIR_GMMLIB}/inc/External/Common/GmmHw.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmInfo.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmInfoExt.h
${BS_DIR_GMMLIB}/inc/External/Common/GmmInternal.h
@@ -331,6 +353,7 @@
source_group("Header Files\\External\\Linux" FILES
${BS_DIR_GMMLIB}/inc/External/Linux/GmmResourceInfoLin.h
+ ${BS_DIR_GMMLIB}/inc/External/Linux/GmmResourceInfoLinExt.h
)
source_group("Header Files\\Internal\\Common" FILES
diff --git a/Source/GmmLib/GlobalInfo/GmmClientContext.cpp b/Source/GmmLib/GlobalInfo/GmmClientContext.cpp
index 655f9ff..51258f0 100644
--- a/Source/GmmLib/GlobalInfo/GmmClientContext.cpp
+++ b/Source/GmmLib/GlobalInfo/GmmClientContext.cpp
@@ -27,6 +27,7 @@
#include "..\..\inc\common\gfxEscape.h"
#include "..\..\..\miniport\LHDM\inc\gmmEscape.h"
#include "Internal\Windows\GmmResourceInfoWinInt.h"
+#include "../TranslationTable/GmmUmdTranslationTable.h"
#endif
/////////////////////////////////////////////////////////////////////////////////////
@@ -465,6 +466,40 @@
}
}
+/////////////////////////////////////////////////////////////////////////////////////
+/// Member function of ClientContext class for creation of PAgeTableMgr Object .
+/// @see GmmLib::GMM_PAGETABLE_MGR::GMM_PAGETABLE_MGR
+///
+/// @param[in] pDevCb: Pointer to GMM_DEVICE_CALLBACKS_INT
+/// @param[in] pTTCB: Pointer to GMM_TRANSLATIONTABLE_CALLBACKS
+/// @param[in] TTFags
+/// @return Pointer to GMM_PAGETABLE_MGR class.
+//TBD: move the code to new overloaded the API and remove this API once all clients are moved to new API.
+/////////////////////////////////////////////////////////////////////////////////////
+GMM_PAGETABLE_MGR* GMM_STDCALL GmmLib::GmmClientContext::CreatePageTblMgrObject(GMM_DEVICE_CALLBACKS_INT* pDevCb,
+ uint32_t TTFlags)
+{
+ GMM_PAGETABLE_MGR* pPageTableMgr = NULL;
+
+ pPageTableMgr = new GMM_PAGETABLE_MGR(pDevCb, TTFlags, this);
+
+ return pPageTableMgr;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// Member function of ClientContext class for destroy of PageTableMgr Object .
+///
+/// @param[in] pPageTableMgr: Pointer to GMM_PAGETABLE_MGR
+/// @return void.
+/////////////////////////////////////////////////////////////////////////////////////
+void GMM_STDCALL GmmLib::GmmClientContext::DestroyPageTblMgrObject(GMM_PAGETABLE_MGR* pPageTableMgr)
+{
+ if (pPageTableMgr)
+ {
+ delete pPageTableMgr;
+ }
+}
+
#ifdef GMM_LIB_DLL
/////////////////////////////////////////////////////////////////////////////////////
/// Member function of ClientContext class for creation of ResourceInfo Object .
@@ -568,6 +603,48 @@
}
}
#endif
+/////////////////////////////////////////////////////////////////////////////////////
+/// Member function of ClientContext class for creation of PAgeTableMgr Object .
+/// @see GmmLib::GMM_PAGETABLE_MGR::GMM_PAGETABLE_MGR
+///
+/// @param[in] pDevCb: Pointer to GMM_DEVICE_CALLBACKS_INT
+/// @param[in] pTTCB: Pointer to GMM_TRANSLATIONTABLE_CALLBACKS
+/// @param[in] TTFags
+/// @return Pointer to GMM_PAGETABLE_MGR class.
+/// TBD: move the code to new overloaded the API and remove this API once all clients are moved to new API.
+/////////////////////////////////////////////////////////////////////////////////////
+GMM_PAGETABLE_MGR* GMM_STDCALL GmmLib::GmmClientContext::CreatePageTblMgrObject(
+ GMM_DEVICE_CALLBACKS_INT* pDevCb,
+ uint32_t TTFlags,
+ GmmClientAllocationCallbacks* pAllocCbs)
+{
+ if (!pAllocCbs || !pAllocCbs->pfnAllocation)
+ {
+ return CreatePageTblMgrObject(
+ pDevCb,
+ TTFlags);
+ }
+ else
+ {
+ GMM_PAGETABLE_MGR* pPageTableMgr = NULL;
+ return pPageTableMgr;
+ }
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// Member function of ClientContext class for destroy of PageTableMgr Object .
+///
+/// @param[in] pPageTableMgr: Pointer to GMM_PAGETABLE_MGR
+/// @return void.
+/////////////////////////////////////////////////////////////////////////////////////
+void GMM_STDCALL GmmLib::GmmClientContext::DestroyPageTblMgrObject(GMM_PAGETABLE_MGR* pPageTableMgr,
+ GmmClientAllocationCallbacks* pAllocCbs)
+{
+ if (!pAllocCbs || !pAllocCbs->pfnFree)
+ {
+ return DestroyPageTblMgrObject(pPageTableMgr);
+ }
+}
/////////////////////////////////////////////////////////////////////////////////////
/// Gmm lib DLL exported C wrapper for creating GmmLib::GmmClientContext object
diff --git a/Source/GmmLib/Resource/GmmRestrictions.cpp b/Source/GmmLib/Resource/GmmRestrictions.cpp
index 5f591fc..aca889d 100644
--- a/Source/GmmLib/Resource/GmmRestrictions.cpp
+++ b/Source/GmmLib/Resource/GmmRestrictions.cpp
@@ -577,7 +577,7 @@
if(pTexinfo->Flags.Info.RenderCompressed ||
pTexinfo->Flags.Info.MediaCompressed)
{
- Restrictions.Alignment = GFX_ALIGN(Restrictions.Alignment, GMM_KBYTE(64));
+ Restrictions.Alignment = GFX_ALIGN(Restrictions.Alignment, (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16)));
}
GMM_DPF_EXIT;
diff --git a/Source/GmmLib/Resource/Linux/GmmResourceInfoLinCWrapper.cpp b/Source/GmmLib/Resource/Linux/GmmResourceInfoLinCWrapper.cpp
new file mode 100644
index 0000000..546815f
--- /dev/null
+++ b/Source/GmmLib/Resource/Linux/GmmResourceInfoLinCWrapper.cpp
@@ -0,0 +1,179 @@
+/*==============================================================================
+Copyright(c) 2019 Intel Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files(the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and / or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+============================================================================*/
+
+#include "Internal/Common/GmmLibInc.h"
+
+#if defined(__linux__) && !LHDM
+#include "Internal/Linux/GmmResourceInfoLinInt.h"
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// This is an overloaded function to call DeviceCb Function for CreateAllocation
+///
+/// @param[in] ClientType
+/// @param[in] pDeviceCb: Pointer to GMM_DEVICE_CALLBACKS_INT Struct
+/// @param[in] pAllocate: Pointer to GMM_DDI_ALLOCATE Union
+/// @return Status of CreateAllocation call.
+/////////////////////////////////////////////////////////////////////////////////////
+int GmmDeviceCallback(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DDI_ALLOCATE *pAllocate)
+{
+ int Status = 0;
+ void * pBo = NULL;
+ void * pCpuAddr = NULL;
+ uint64_t gpuAddr = 0ULL;
+
+ Status = pDeviceCb->DevCbPtrs_.pfnAllocate(pDeviceCb->pBufMgr,
+ pAllocate->size,
+ pAllocate->alignment,
+ &pBo,
+ &pCpuAddr,
+ &gpuAddr);
+ pAllocate->bo = pBo;
+ pAllocate->cpuAddr = pCpuAddr;
+ pAllocate->gfxAddr = gpuAddr;
+
+ return Status;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// This is an overloaded function to call DeviceCb Function for DestroyAllocation
+///
+/// @param[in] ClientType
+/// @param[in] pDeviceCb: Pointer to GMM_DEVICE_CALLBACKS_INT Struct
+/// @param[in] pAllocate: Pointer to GMM_DDI_DEALLOCATE Union
+/// @return Status of DestroyAllocation call.
+/////////////////////////////////////////////////////////////////////////////////////
+int GmmDeviceCallback(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DDI_DEALLOCATE *pDeallocate)
+{
+ int Status = 0;
+
+ pDeviceCb->DevCbPtrs_.pfnDeallocate(pDeallocate->bo);
+
+ return Status;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// This is an overloaded function to call DeviceCb Function for WaitForSyncObjFromCpu
+///
+/// @param[in] ClientType
+/// @param[in] pDeviceCb: Pointer to GMM_DEVICE_CALLBACKS_INT Struct
+/// @param[in] pAllocate: Pointer to GMM_DDI_WAITFORSYNCHRONIZATIONOBJECTFROMCPU Union
+/// @return Status of WaitForSyncObjFromCpu call.
+/////////////////////////////////////////////////////////////////////////////////////
+int GmmDeviceCallback(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DDI_WAITFORSYNCHRONIZATIONOBJECTFROMCPU *pWait)
+{
+ int Status = 0;
+
+ pDeviceCb->DevCbPtrs_.pfnWaitFromCpu(pWait->bo);
+
+ return Status;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// This function checks for Null DeviceCb Function pointer
+///
+/// @param[in] ClientType
+/// @param[in] pDeviceCb: Pointer to GMM_DEVICE_CALLBACKS_INT Struct
+/// @param[in] CallBackType Enum @GMM_DEVICE_CALLBACKS_TYPE
+/// @return True if not Null.
+/////////////////////////////////////////////////////////////////////////////////////
+int GmmCheckForNullDevCbPfn(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DEVICE_CALLBACKS_TYPE CallBackType)
+{
+ int Status = 0;
+
+ switch(CallBackType)
+ {
+ case GMM_DEV_CB_ALLOC:
+ Status = (pDeviceCb->DevCbPtrs_.pfnAllocate != 0);
+ break;
+ case GMM_DEV_CB_DEALLOC:
+ Status = (pDeviceCb->DevCbPtrs_.pfnDeallocate != 0);
+ break;
+ case GMM_DEV_CB_WAIT_FROM_CPU:
+ Status = (pDeviceCb->DevCbPtrs_.pfnWaitFromCpu != 0);
+ break;
+ default:
+ Status = 0;
+ break;
+ }
+
+ return Status;
+}
+
+// Dummy Translation Table Callback for reusing ..
+static inline int DummyPrologTranslationTable(void *pDeviceHandle)
+{
+ return 0;
+}
+
+static inline int DummyWriteL1Entries(void * pDeviceHandle,
+ const uint32_t NumEntries,
+ GMM_GFX_ADDRESS GfxAddress,
+ uint32_t * Data)
+{
+ return 0;
+}
+
+static inline int DummyWriteL2L3Entry(void * pDeviceHandle,
+ GMM_GFX_ADDRESS GfxAddress,
+ uint64_t Data)
+{
+ return 0;
+}
+
+static inline int DummyWriteFenceID(void * pDeviceHandle,
+ GMM_GFX_ADDRESS GfxAddress,
+ uint64_t Data)
+{
+ return 0;
+}
+
+static inline int DummyEpilogTranslationTable(void * pDeviceHandle,
+ uint8_t ForceFlush)
+{
+ return 0;
+}
+
+static inline int DummyCopyL1Entry(void * pDeviceHandle,
+ GMM_GFX_ADDRESS DstGfxAddress,
+ GMM_GFX_ADDRESS SrcGfxAddress)
+{
+ return 0;
+}
+
+static inline int DummyWriteL3Adr(void * pDeviceHandle,
+ GMM_GFX_ADDRESS L3GfxAddress,
+ uint64_t RegOffset)
+{
+ return 0;
+}
+
+GMM_TRANSLATIONTABLE_CALLBACKS DummyTTCB = {
+.pfPrologTranslationTable = DummyPrologTranslationTable,
+.pfWriteL1Entries = DummyWriteL1Entries,
+.pfWriteL2L3Entry = DummyWriteL2L3Entry,
+.pfWriteFenceID = DummyWriteFenceID,
+.pfEpilogTranslationTable = DummyEpilogTranslationTable,
+.pfCopyL1Entry = DummyCopyL1Entry,
+.pfWriteL3Adr = DummyWriteL3Adr,
+};
+
+#endif /*__linux__*/
diff --git a/Source/GmmLib/Texture/GmmGen11Texture.cpp b/Source/GmmLib/Texture/GmmGen11Texture.cpp
index 63c9986..7e6d60b 100644
--- a/Source/GmmLib/Texture/GmmGen11Texture.cpp
+++ b/Source/GmmLib/Texture/GmmGen11Texture.cpp
@@ -321,9 +321,12 @@
GMM_GFX_SIZE_T TileHeight = pPlatform->TileInfo[pTexInfo->TileMode].LogicalTileHeight;
GMM_GFX_SIZE_T TileWidth = pPlatform->TileInfo[pTexInfo->TileMode].LogicalTileWidth;
- //U/V must be aligned to AuxT granularity, for 16K AuxT- 4x pitchalign enforces it,
- //add extra padding for 64K AuxT
- TileHeight *= 1;
+ if(GFX_GET_CURRENT_RENDERCORE(pPlatform->Platform) > IGFX_GEN11LP_CORE)
+ {
+ //U/V must be aligned to AuxT granularity, for 16K AuxT- 4x pitchalign enforces it,
+ //add extra padding for 64K AuxT
+ TileHeight *= (!GMM_IS_64KB_TILE(pTexInfo->Flags) && !WA16K) ? 4 : 1;
+ }
*pUOffsetX = GFX_ALIGN(*pUOffsetX, TileWidth);
*pUOffsetY = GFX_ALIGN(*pUOffsetY, TileHeight);
diff --git a/Source/GmmLib/Texture/GmmGen12Texture.cpp b/Source/GmmLib/Texture/GmmGen12Texture.cpp
index d6b18a9..450770a 100644
--- a/Source/GmmLib/Texture/GmmGen12Texture.cpp
+++ b/Source/GmmLib/Texture/GmmGen12Texture.cpp
@@ -838,7 +838,7 @@
//U/V must be aligned to AuxT granularity, 4x pitchalign enforces 16K-align,
//add extra padding for 64K AuxT
- TileHeight *= (!GMM_IS_64KB_TILE(pTexInfo->Flags)) ? 4 : 1;
+ TileHeight *= (!GMM_IS_64KB_TILE(pTexInfo->Flags) && !WA16K) ? 4 : 1;
if(pTexInfo->Format == GMM_FORMAT_IMC2 || // IMC2, IMC4 needs even tile columns
pTexInfo->Format == GMM_FORMAT_IMC4)
diff --git a/Source/GmmLib/TranslationTable/GmmAuxTable.cpp b/Source/GmmLib/TranslationTable/GmmAuxTable.cpp
new file mode 100644
index 0000000..4c3d04b
--- /dev/null
+++ b/Source/GmmLib/TranslationTable/GmmAuxTable.cpp
@@ -0,0 +1,949 @@
+/*==============================================================================
+Copyright(c) 2019 Intel Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files(the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and / or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+Description: AUX-Table management functions
+ (contains functions to assign memory to
+ AUX-Tables with valid entries,
+ and update their entries on request)
+
+============================================================================*/
+
+#include "Internal/Common/GmmLibInc.h"
+#include "../TranslationTable/GmmUmdTranslationTable.h"
+
+#if !defined(__GMM_KMD__)
+
+//=============================================================================
+//
+// Function: MapNullCCS
+//
+// Desc: Maps given resource, with dummy null-ccs chain, on Aux Table
+//
+// Caller: UpdateAuxTable (map op for null-tiles)
+//
+// Parameters:
+// UmdContext: Caller-thread specific info (regarding BB for TR-Aux udpate, cmdQ to use etc)
+// BaseAdr: Start adr of main surface
+// Size: Main-surface size in bytes
+// PartialL1e: Aux-metadata other than AuxVA
+// DoNotWait: 1 for CPU update, 0 for async(Gpu) update
+//-----------------------------------------------------------------------------
+GMM_STATUS GmmLib::AuxTable::MapNullCCS(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T Size, uint64_t PartialL1e, uint8_t DoNotWait)
+{
+ GMM_STATUS Status = GMM_SUCCESS;
+ GMM_GFX_SIZE_T L1TableSize = (GMM_L1_SIZE(AUXTT, pGmmGlobalContext)) * (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16)); //Each AuxTable entry maps 16K main-surface
+ GMM_GFX_ADDRESS Addr = 0;
+ GMM_GFX_ADDRESS L3GfxAddress = 0;
+ GMM_CLIENT ClientType;
+
+ GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
+
+ EnterCriticalSection(&TTLock);
+
+ DoNotWait |= (!UmdContext || !UmdContext->pCommandQueueHandle);
+
+ if(TTL3.L3Handle)
+ {
+ L3GfxAddress = TTL3.GfxAddress;
+ }
+ else
+ {
+ LeaveCriticalSection(&TTLock);
+ return GMM_ERROR;
+ }
+
+ if(!DoNotWait)
+ {
+ PageTableMgr->TTCb.pfPrologTranslationTable(
+ UmdContext->pCommandQueueHandle);
+ }
+
+ // For each L1 table
+ for(Addr = GFX_ALIGN_FLOOR(BaseAdr, L1TableSize); // Start at begining of L1 table
+ Addr < BaseAdr + Size;
+ Addr += L1TableSize) // Increment by 1 L1 table
+ {
+ GMM_GFX_ADDRESS L1GfxAddress, L2GfxAddress;
+ GMM_GFX_ADDRESS L1CPUAddress, L2CPUAddress;
+ GMM_GFX_ADDRESS StartAddress = 0;
+ GMM_GFX_ADDRESS EndAddress = 0;
+ GMM_GFX_ADDRESS TileAddr = 0;
+ GMM_GFX_SIZE_T L2eIdx = 0;
+
+ StartAddress = Addr < BaseAdr ? BaseAdr : Addr;
+ EndAddress = Addr + L1TableSize;
+ if(EndAddress > BaseAdr + Size)
+ {
+ EndAddress = BaseAdr + Size;
+ }
+
+ GetL1L2TableAddr(StartAddress,
+ &L1GfxAddress,
+ &L2GfxAddress);
+
+ // If tables are not there, then they are already invalidated as part of
+ // AUX-TT initialization or other APIs.
+ if(L2GfxAddress == GMM_NO_TABLE ||
+ L1GfxAddress == GMM_NO_TABLE)
+ {
+ //Clear Valid-bit for L3Entry or L2Entry
+ uint64_t Data = 0;
+ GMM_GFX_ADDRESS TableGfxAddress = (L2GfxAddress == GMM_NO_TABLE) ? L3GfxAddress : L2GfxAddress;
+ GMM_GFX_ADDRESS TableCPUAddress = (L2GfxAddress == GMM_NO_TABLE) ? TTL3.CPUAddress : pTTL2[GMM_L3_ENTRY_IDX(AUXTT, StartAddress)].GetCPUAddress();
+ uint32_t TableEntryIdx = (L2GfxAddress == GMM_NO_TABLE) ? static_cast<uint32_t>(GMM_L3_ENTRY_IDX(AUXTT, StartAddress)) : static_cast<uint32_t>(GMM_L2_ENTRY_IDX(AUXTT, StartAddress));
+ L2CPUAddress = (L2GfxAddress == GMM_NO_TABLE) ? 0 : TableCPUAddress;
+
+ if(!NullL1Table || !NullL2Table)
+ {
+ AllocateDummyTables(&NullL2Table, &NullL1Table);
+ if(!NullL1Table || !NullL2Table)
+ {
+ //report error
+ LeaveCriticalSection(&TTLock);
+ return GMM_OUT_OF_MEMORY;
+ }
+ else
+ {
+ //Initialize dummy table entries (one-time)
+ GMM_GFX_ADDRESS TableAddr = NullL2Table->GetCPUAddress();
+ GMM_AUXTTL2e L2e = {0};
+ L2e.Valid = 1;
+ L2e.L1GfxAddr = (NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()) >> 13;
+ for(int i = 0; i < GMM_AUX_L2_SIZE; i++)
+ {
+ //initialize L2e ie clear Valid bit for all entries
+ ((GMM_AUXTTL2e *)TableAddr)[i].Value = L2e.Value;
+ }
+
+ TableAddr = NullL1Table->GetCPUAddress();
+
+ GMM_AUXTTL1e L1e = {0};
+ L1e.Valid = 1;
+ L1e.GfxAddress = (NullCCSTile >> 8);
+ for(int i = 0; i < GMM_AUX_L1_SIZE(pGmmGlobalContext); i++)
+ {
+ //initialize L1e with null ccs tile
+ ((GMM_AUXTTL1e *)TableAddr)[i].Value = L1e.Value;
+ }
+ }
+ }
+
+ if(L2GfxAddress == GMM_NO_TABLE)
+ {
+ GMM_AUXTTL3e L3e = {0};
+ L3e.Valid = 1;
+ L3e.L2GfxAddr = (NullL2Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL2Table->GetNodeIdx()) >> 15;
+ Data = L3e.Value;
+ }
+ else
+ {
+ GMM_AUXTTL2e L2e = {0};
+ L2e.Valid = 1;
+ L2e.L1GfxAddr = (NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()) >> 13;
+ Data = L2e.Value;
+ }
+
+ if(DoNotWait)
+ {
+ //Sync update on CPU
+ ((GMM_AUXTTL2e *)TableCPUAddress)[TableEntryIdx].Value = Data;
+ }
+ else
+ {
+ if(L2GfxAddress != GMM_NO_TABLE)
+ {
+ pTTL2[GMM_L3_ENTRY_IDX(AUXTT, StartAddress)].UpdatePoolFence(UmdContext, false);
+ }
+ PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
+ TableGfxAddress + TableEntryIdx * GMM_AUX_L2e_SIZE,
+ Data);
+ }
+ continue;
+ }
+ else
+ {
+ uint32_t L3eIdx = static_cast<uint32_t>(GMM_L3_ENTRY_IDX(AUXTT, StartAddress));
+ L2CPUAddress = pTTL2[L3eIdx].GetCPUAddress();
+
+ L2eIdx = GMM_L2_ENTRY_IDX(AUXTT, StartAddress);
+ if(DoNotWait)
+ {
+ //Sync update on CPU
+ ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].Valid = 1; //set Valid bit
+ ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].L2GfxAddr = L2GfxAddress >> 15;
+
+ ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Valid = 1; //set Valid bit
+ ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].L1GfxAddr = L1GfxAddress >> 13;
+ }
+ else
+ {
+ GMM_AUXTTL3e L3e = {0};
+ L3e.Valid = 1;
+ L3e.L2GfxAddr = L2GfxAddress >> 15;
+ PageTableMgr->TTCb.pfWriteL2L3Entry(
+ UmdContext->pCommandQueueHandle,
+ L3GfxAddress + (L3eIdx * GMM_AUX_L3e_SIZE),
+ L3e.Value);
+
+ pTTL2[L3eIdx].UpdatePoolFence(UmdContext, false);
+
+ GMM_AUXTTL2e L2e = {0};
+ L2e.Valid = 1;
+ L2e.L1GfxAddr = L1GfxAddress >> 13;
+ PageTableMgr->TTCb.pfWriteL2L3Entry(
+ UmdContext->pCommandQueueHandle,
+ L2GfxAddress + (L2eIdx * GMM_AUX_L2e_SIZE),
+ L2e.Value);
+ }
+ }
+
+ // For each 64KB or 16KB of main surface (entry) in L1 table
+ for(TileAddr = StartAddress;
+ TileAddr < EndAddress;
+ TileAddr += (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16)))
+ {
+ uint64_t Data = PartialL1e | NullCCSTile | __BIT(0);
+ GMM_GFX_SIZE_T L1eIdx = GMM_L1_ENTRY_IDX(AUXTT, TileAddr, pGmmGlobalContext);
+ GmmLib::LastLevelTable *pL1Tbl = NULL;
+
+ pL1Tbl = pTTL2[GMM_AUX_L3_ENTRY_IDX(TileAddr)].GetL1Table(L2eIdx, NULL);
+ L1CPUAddress = pL1Tbl->GetCPUAddress();
+ if(DoNotWait)
+ {
+ //Sync update on CPU
+ ((GMM_AUXTTL1e *)L1CPUAddress)[L1eIdx].Value = Data;
+
+ GMM_DPF(GFXDBG_CRITICAL, "Null-Map | Table Entry: [0x%06x] L2Addr[0x%016llX] Value[0x%016llX] :: [0x%06x] L1Addr[0x%016llX] Value[0x%016llX]\n", L2eIdx, ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx], ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].L1GfxAddr << 13, L1eIdx, &((GMM_AUXTTL1e *)L1CPUAddress)[L1eIdx], Data);
+ }
+ else
+ {
+ pL1Tbl->UpdatePoolFence(UmdContext, false);
+
+ /* PageTableMgr->TTCb.pfWriteL1Entries(
+ UmdContext->pCommandQueueHandle,
+ 2,
+ L1GfxAddress + (L1eIdx * GMM_AUX_L1e_SIZE),
+ (uint32_t*)(&Data));*/ //**********REQUIRE UMD CHANGE TO UPDATE 64-bit ENTRY - both DWORDs must be updated atomically*******/
+ PageTableMgr->TTCb.pfWriteL2L3Entry(
+ UmdContext->pCommandQueueHandle,
+ L1GfxAddress + (L1eIdx * GMM_AUX_L1e_SIZE),
+ Data);
+ }
+
+ if(pL1Tbl->TrackTableUsage(AUXTT, true, TileAddr, true))
+ { // L1 Table is not being used anymore
+ GMM_AUXTTL2e L2e = {0};
+ GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
+ GmmLib::LastLevelTable * pL1Tbl = NULL, *Prev = NULL;
+
+ pL1Tbl = pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].GetL1Table(L2eIdx, &Prev);
+ // Map L2-entry to Null-L1Table
+ L2e.Valid = 1;
+ L2e.L1GfxAddr = (NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()) >> 13;
+ if(DoNotWait)
+ {
+ //Sync update on CPU
+ ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Value = L2e.Value;
+ }
+ else
+ {
+ pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].UpdatePoolFence(UmdContext, false);
+ PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
+ L2GfxAddress + L2eIdx * GMM_AUX_L2e_SIZE,
+ L2e.Value);
+ }
+ //Update usage for PoolNode assigned to L1Table, and free L1Tbl
+ if(pL1Tbl)
+ {
+ PoolElem = pL1Tbl->GetPool();
+ if(PoolElem)
+ {
+ if(pL1Tbl->GetBBInfo().BBQueueHandle)
+ {
+ PoolElem->GetNodeBBInfoAtIndex(pL1Tbl->GetNodeIdx()) = pL1Tbl->GetBBInfo();
+ }
+ DEASSIGN_POOLNODE(PageTableMgr, UmdContext, PoolElem, pL1Tbl->GetNodeIdx(), AUX_L1TABLE_SIZE_IN_POOLNODES)
+ }
+ pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].DeleteFromList(pL1Tbl, Prev);
+ }
+
+ // The L1 table is unused -- meaning everything else in this table is
+ // already invalid. So, break early.
+ break;
+ }
+ }
+ }
+
+ if(!DoNotWait)
+ {
+ PageTableMgr->TTCb.pfEpilogTranslationTable(
+ UmdContext->pCommandQueueHandle,
+ 1); // ForceFlush
+ }
+ LeaveCriticalSection(&TTLock);
+
+ return Status;
+}
+
+//=============================================================================
+//
+// Function: InvalidateTable (InvalidateMappings)
+//
+// Desc: Unmaps given resource from Aux Table; and marks affected entries as invalid
+//
+// Caller: UpdateAuxTable (unmap op)
+//
+// Parameters:
+// UmdContext: Caller-thread specific info (regarding BB for Aux udpate, cmdQ to use etc)
+// BaseAdr: Start adr of main surface
+// Size: Main-surface size in bytes? (or take GmmResInfo?)
+// DoNotWait: 1 for CPU update, 0 for async(Gpu) update
+//-----------------------------------------------------------------------------
+GMM_STATUS GmmLib::AuxTable::InvalidateTable(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T Size, uint8_t DoNotWait)
+{
+ GMM_STATUS Status = GMM_SUCCESS;
+ GMM_GFX_SIZE_T L1TableSize = (GMM_L1_SIZE(AUXTT, pGmmGlobalContext)) * (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16)); //Each AuxTable entry maps 16K main-surface
+ GMM_GFX_ADDRESS Addr = 0;
+ GMM_GFX_ADDRESS L3GfxAddress = 0;
+ uint8_t isTRVA = 0;
+
+ GMM_CLIENT ClientType;
+
+ GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
+
+ //NullCCSTile isn't initialized, disable TRVA path
+ isTRVA = (NullCCSTile ? isTRVA : 0);
+
+ EnterCriticalSection(&TTLock);
+
+ DoNotWait |= (!UmdContext || !UmdContext->pCommandQueueHandle);
+
+ if(TTL3.L3Handle)
+ {
+ L3GfxAddress = TTL3.GfxAddress;
+ }
+ else
+ {
+ LeaveCriticalSection(&TTLock);
+ return GMM_ERROR;
+ }
+
+ if(!DoNotWait)
+ {
+ PageTableMgr->TTCb.pfPrologTranslationTable(
+ UmdContext->pCommandQueueHandle);
+ }
+
+ // For each L1 table
+ for(Addr = GFX_ALIGN_FLOOR(BaseAdr, L1TableSize); // Start at begining of L1 table
+ Addr < BaseAdr + Size;
+ Addr += L1TableSize) // Increment by 1 L1 table
+ {
+ GMM_GFX_ADDRESS L1GfxAddress, L2GfxAddress;
+ GMM_GFX_ADDRESS L1CPUAddress, L2CPUAddress;
+ GMM_GFX_ADDRESS StartAddress = 0;
+ GMM_GFX_ADDRESS EndAddress = 0;
+ GMM_GFX_ADDRESS TileAddr = 0;
+ GMM_GFX_SIZE_T L2eIdx = 0;
+
+ StartAddress = Addr < BaseAdr ? BaseAdr : Addr;
+ EndAddress = Addr + L1TableSize;
+ if(EndAddress > BaseAdr + Size)
+ {
+ EndAddress = BaseAdr + Size;
+ }
+
+ GetL1L2TableAddr(StartAddress,
+ &L1GfxAddress,
+ &L2GfxAddress);
+
+ // If tables are not there, then they are already invalidated as part of
+ // AUX-TT initialization or other APIs.
+ if(L2GfxAddress == GMM_NO_TABLE ||
+ L1GfxAddress == GMM_NO_TABLE)
+ {
+ //Clear Valid-bit for L3Entry or L2Entry
+ GMM_AUXTTL2e L2e = {0}; //AUXTT L3e is identical to L2e, reuse.
+ GMM_GFX_ADDRESS TableGfxAddress = (L2GfxAddress == GMM_NO_TABLE) ? L3GfxAddress : L2GfxAddress;
+ GMM_GFX_ADDRESS TableCPUAddress = (L2GfxAddress == GMM_NO_TABLE) ? TTL3.CPUAddress : pTTL2[GMM_L3_ENTRY_IDX(AUXTT, StartAddress)].GetCPUAddress();
+ uint32_t TableEntryIdx = (L2GfxAddress == GMM_NO_TABLE) ? static_cast<uint32_t>(GMM_L3_ENTRY_IDX(AUXTT, StartAddress)) : static_cast<uint32_t>(GMM_L2_ENTRY_IDX(AUXTT, StartAddress));
+ L2CPUAddress = (L2GfxAddress == GMM_NO_TABLE) ? 0 : TableCPUAddress;
+
+ if(isTRVA && NullL2Table && NullL1Table)
+ {
+ //invalidate if request spans entire stretch ie TileAdr aligns L1TableSize*GMM_L2_SIZE
+ uint64_t Data = 0;
+ if(L2GfxAddress == GMM_NO_TABLE)
+ {
+ GMM_AUXTTL3e L3e = {0};
+ L3e.Valid = 1;
+ L3e.L2GfxAddr = (NullL2Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL2Table->GetNodeIdx()) >> 15;
+ Data = L3e.Value;
+ }
+ else
+ {
+ GMM_AUXTTL2e L2e = {0};
+ L2e.Valid = 1;
+ L2e.L1GfxAddr = (NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()) >> 13;
+ Data = L2e.Value;
+ }
+ L2e.Value = Data;
+ }
+ else
+ {
+ L2e.Valid = 0;
+ }
+ if(DoNotWait)
+ {
+ //Sync update on CPU
+ ((GMM_AUXTTL2e *)TableCPUAddress)[TableEntryIdx].Value = L2e.Value;
+ }
+ else
+ {
+ if(L2GfxAddress != GMM_NO_TABLE)
+ {
+ pTTL2[GMM_L3_ENTRY_IDX(AUXTT, StartAddress)].UpdatePoolFence(UmdContext, false);
+ }
+ PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
+ TableGfxAddress + TableEntryIdx * GMM_AUX_L2e_SIZE,
+ L2e.Value);
+ }
+ continue;
+ }
+ else
+ {
+ uint32_t L3eIdx = static_cast<uint32_t>(GMM_L3_ENTRY_IDX(AUXTT, StartAddress));
+ L2CPUAddress = pTTL2[L3eIdx].GetCPUAddress();
+
+ L2eIdx = GMM_L2_ENTRY_IDX(AUXTT, StartAddress);
+ if(DoNotWait)
+ {
+ //Sync update on CPU
+ ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].Valid = 1; //set Valid bit
+ ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].L2GfxAddr = L2GfxAddress >> 15;
+
+ ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Valid = 1; //set Valid bit
+ ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].L1GfxAddr = L1GfxAddress >> 13;
+ }
+ else
+ {
+ GMM_AUXTTL3e L3e = {0};
+ L3e.Valid = 1;
+ L3e.L2GfxAddr = L2GfxAddress >> 15;
+ PageTableMgr->TTCb.pfWriteL2L3Entry(
+ UmdContext->pCommandQueueHandle,
+ L3GfxAddress + (L3eIdx * GMM_AUX_L3e_SIZE),
+ L3e.Value);
+
+ pTTL2[L3eIdx].UpdatePoolFence(UmdContext, false);
+
+ GMM_AUXTTL2e L2e = {0};
+ L2e.Valid = 1;
+ L2e.L1GfxAddr = L1GfxAddress >> 13;
+ PageTableMgr->TTCb.pfWriteL2L3Entry(
+ UmdContext->pCommandQueueHandle,
+ L2GfxAddress + (L2eIdx * GMM_AUX_L2e_SIZE),
+ L2e.Value);
+ }
+ }
+
+ // For each 64KB or 16KB of main surface (entry) in L1 table
+ for(TileAddr = StartAddress;
+ TileAddr < EndAddress;
+ TileAddr += (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16)))
+ {
+ //Invalidation of requested range irrespective of TRVA
+ uint64_t Data = GMM_INVALID_AUX_ENTRY;
+ GMM_GFX_SIZE_T L1eIdx = GMM_L1_ENTRY_IDX(AUXTT, TileAddr, pGmmGlobalContext);
+ GmmLib::LastLevelTable *pL1Tbl = NULL;
+
+ pL1Tbl = pTTL2[GMM_AUX_L3_ENTRY_IDX(TileAddr)].GetL1Table(L2eIdx, NULL);
+ L1CPUAddress = pL1Tbl->GetCPUAddress();
+ if(DoNotWait)
+ {
+ //Sync update on CPU
+ ((GMM_AUXTTL1e *)L1CPUAddress)[L1eIdx].Value = Data;
+
+ GMM_DPF(GFXDBG_CRITICAL, "UnMap | Table Entry: [0x%06x] L2Addr[0x%016llX] Value[0x%016llX] :: [0x%06x] L1Addr[0x%016llX] Value[0x%016llX]\n", L2eIdx, ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx], ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].L1GfxAddr << 13, L1eIdx, &((GMM_AUXTTL1e *)L1CPUAddress)[L1eIdx], Data);
+ }
+ else
+ {
+ pL1Tbl->UpdatePoolFence(UmdContext, false);
+
+ /* PageTableMgr->TTCb.pfWriteL1Entries(
+ UmdContext->pCommandQueueHandle,
+ 2,
+ L1GfxAddress + (L1eIdx * GMM_AUX_L1e_SIZE),
+ (uint32_t*)(&Data));*/ //**********REQUIRE UMD CHANGE TO UPDATE 64-bit ENTRY - both DWORDs must be updated atomically*******/
+ PageTableMgr->TTCb.pfWriteL2L3Entry(
+ UmdContext->pCommandQueueHandle,
+ L1GfxAddress + (L1eIdx * GMM_AUX_L1e_SIZE),
+ Data);
+ }
+
+ if(pL1Tbl->TrackTableUsage(AUXTT, true, TileAddr, true))
+ { // L1 Table is not being used anymore
+ GMM_AUXTTL2e L2e = {0};
+ GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
+ GmmLib::LastLevelTable * pL1Tbl = NULL, *Prev = NULL;
+
+ pL1Tbl = pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].GetL1Table(L2eIdx, &Prev);
+
+ if(isTRVA && NullL1Table &&
+ ((TileAddr > GFX_ALIGN_FLOOR(BaseAdr, L1TableSize) && TileAddr < GFX_ALIGN_NP2(BaseAdr, L1TableSize)) ||
+ (TileAddr > GFX_ALIGN_FLOOR(BaseAdr + Size, L1TableSize) && TileAddr < GFX_ALIGN_NP2(BaseAdr + Size, L1TableSize))))
+ {
+ //Invalidation affects entries out of requested range, null-map for TR
+ L2e.Valid = 1;
+ L2e.L1GfxAddr = (NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()) >> 13;
+ }
+ else
+ {
+ // Clear valid bit of L2 entry
+ L2e.Valid = 0;
+ ((GMM_AUXTTL2e *)L2GfxAddress)[L2eIdx].Valid = 0;
+ }
+ if(DoNotWait)
+ {
+ //Sync update on CPU
+ ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Value = L2e.Value;
+ }
+ else
+ {
+ pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].UpdatePoolFence(UmdContext, false);
+ PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
+ L2GfxAddress + L2eIdx * GMM_AUX_L2e_SIZE,
+ L2e.Value);
+ }
+ //Update usage for PoolNode assigned to L1Table, and free L1Tbl
+ if(pL1Tbl)
+ {
+ PoolElem = pL1Tbl->GetPool();
+ if(PoolElem)
+ {
+ if(pL1Tbl->GetBBInfo().BBQueueHandle)
+ {
+ PoolElem->GetNodeBBInfoAtIndex(pL1Tbl->GetNodeIdx()) = pL1Tbl->GetBBInfo();
+ }
+ DEASSIGN_POOLNODE(PageTableMgr, UmdContext, PoolElem, pL1Tbl->GetNodeIdx(), AUX_L1TABLE_SIZE_IN_POOLNODES)
+ }
+ pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].DeleteFromList(pL1Tbl, Prev);
+ }
+
+ // The L1 table is unused -- meaning everything else in this table is
+ // already invalid. So, break early.
+ break;
+ }
+ }
+ }
+
+ if(!DoNotWait)
+ {
+ PageTableMgr->TTCb.pfEpilogTranslationTable(
+ UmdContext->pCommandQueueHandle,
+ 1); // ForceFlush
+ }
+
+ LeaveCriticalSection(&TTLock);
+
+ return Status;
+}
+
+//=============================================================================
+//
+// Function: MapValidEntry
+//
+// Desc: Maps given main-surface, on Aux-Table, to get the exact CCS cacheline tied to
+// different 4x4K pages of main-surface
+//
+// Caller: UpdateAuxTable (map op)
+//
+// Parameters:
+// UmdContext: ptr to thread-data
+// BaseAdr: Start adr of main-surface
+// BaseSize: main-surface Size in bytes
+// BaseResInfo: main surface ResInfo
+// AuxVA: Start adr of Aux-surface
+// AuxResInfo: Aux surface ResInfo
+// PartialData: Aux L1 partial data (ie w/o address)
+// DoNotWait: true for CPU update, false for async(Gpu) update
+//-----------------------------------------------------------------------------
+GMM_STATUS GmmLib::AuxTable::MapValidEntry(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T BaseSize,
+ GMM_RESOURCE_INFO *BaseResInfo, GMM_GFX_ADDRESS AuxVA, GMM_RESOURCE_INFO *AuxResInfo, uint64_t PartialData, uint8_t DoNotWait)
+{
+ GMM_STATUS Status = GMM_SUCCESS;
+ GMM_GFX_ADDRESS Addr = 0, L3TableAdr = GMM_NO_TABLE;
+ GMM_GFX_SIZE_T L1TableSize = GMM_AUX_L1_SIZE(pGmmGlobalContext) * (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16));
+ GMM_GFX_SIZE_T CCS$Adr = AuxVA;
+ uint8_t isTRVA =0 ;
+
+ GMM_CLIENT ClientType;
+
+ GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
+
+ //NullCCSTile isn't initialized, disable TRVA path
+ isTRVA = (NullCCSTile ? isTRVA : 0);
+
+ EnterCriticalSection(&TTLock);
+ if(!TTL3.L3Handle || (!DoNotWait && !UmdContext))
+ {
+ Status = GMM_ERROR;
+ }
+ else
+ {
+ L3TableAdr = TTL3.GfxAddress;
+
+ if(!DoNotWait)
+ {
+ PageTableMgr->TTCb.pfPrologTranslationTable(UmdContext->pCommandQueueHandle);
+ }
+
+ GMM_DPF(GFXDBG_CRITICAL, "Mapping surface: GPUVA=0x%016llX Size=0x%08X Aux_GPUVA=0x%016llX\n", BaseAdr, BaseSize, AuxVA);
+ for(Addr = GFX_ALIGN_FLOOR(BaseAdr, L1TableSize); Addr < BaseAdr + BaseSize; Addr += L1TableSize)
+ {
+ GMM_GFX_ADDRESS StartAdr, EndAdr, TileAdr;
+ GMM_GFX_ADDRESS L1TableAdr = GMM_NO_TABLE, L2TableAdr = GMM_NO_TABLE;
+ GMM_GFX_ADDRESS L1TableCPUAdr = GMM_NO_TABLE, L2TableCPUAdr = GMM_NO_TABLE;
+ GMM_GFX_SIZE_T L2eIdx = 0;
+ GMM_GFX_SIZE_T L3eIdx = 0;
+ bool AllocateL1 = false, AllocateL2 = false;
+
+ EndAdr = Addr + L1TableSize;
+ EndAdr = EndAdr > BaseAdr + BaseSize ? BaseAdr + BaseSize : EndAdr;
+ StartAdr = Addr < BaseAdr ? BaseAdr : Addr;
+
+ L2eIdx = GMM_L2_ENTRY_IDX(AUXTT, StartAdr);
+ L3eIdx = GMM_L3_ENTRY_IDX(AUXTT, StartAdr);
+
+ //Allocate L2/L1 Table -- get L2 Table Adr for <StartAdr,EndAdr>
+ GetL1L2TableAddr(Addr, &L1TableAdr, &L2TableAdr);
+ if(L2TableAdr == GMM_NO_TABLE || L1TableAdr == GMM_NO_TABLE)
+ {
+ AllocateL1 = GMM_NO_TABLE == L1TableAdr;
+ AllocateL2 = GMM_NO_TABLE == L2TableAdr;
+ AllocateL1L2Table(Addr, &L1TableAdr, &L2TableAdr);
+
+ if(L2TableAdr == GMM_NO_TABLE || L1TableAdr == GMM_NO_TABLE)
+ {
+ LeaveCriticalSection(&TTLock);
+ return GMM_OUT_OF_MEMORY;
+ }
+
+ if(AllocateL2)
+ {
+ uint32_t i = 0;
+ GMM_AUXTTL2e InvalidEntry;
+ InvalidEntry.Value = 0;
+ if(isTRVA && NullL1Table)
+ {
+ InvalidEntry.Valid = 1;
+ InvalidEntry.L1GfxAddr = (NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()) >> 13;
+ }
+
+ if(DoNotWait)
+ {
+ L2TableCPUAdr = pTTL2[L3eIdx].GetCPUAddress();
+
+ ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].Value = 0;
+ ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].L2GfxAddr = L2TableAdr >> 15;
+ ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].Valid = 1;
+ for(i = 0; i < GMM_AUX_L2_SIZE; i++)
+ {
+ //initialize L2e ie clear Valid bit for all entries
+ ((GMM_AUXTTL2e *)L2TableCPUAdr)[i].Value = InvalidEntry.Value;
+ }
+ }
+ else
+ {
+ GMM_AUXTTL3e L3e = {0};
+ L3e.Valid = 1;
+ L3e.L2GfxAddr = L2TableAdr >> 15;
+ PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
+ L3TableAdr + L3eIdx * GMM_AUX_L3e_SIZE,
+ L3e.Value);
+
+ //initialize L2e ie clear valid bit for all entries
+ for(i = 0; i < GMM_AUX_L2_SIZE; i++)
+ {
+ PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
+ L2TableAdr + i * GMM_AUX_L2e_SIZE,
+ InvalidEntry.Value);
+ }
+ }
+ }
+
+ if(AllocateL1)
+ {
+ uint64_t InvalidEntry = (!isTRVA) ? GMM_INVALID_AUX_ENTRY : (NullCCSTile | __BIT(0));
+ uint32_t i = 0;
+
+ if(DoNotWait)
+ {
+ GmmLib::LastLevelTable *pL1Tbl = NULL;
+ pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx, NULL);
+ L2TableCPUAdr = pTTL2[L3eIdx].GetCPUAddress();
+ L1TableCPUAdr = pL1Tbl->GetCPUAddress();
+ //Sync update on CPU
+ ((GMM_AUXTTL2e *)L2TableCPUAdr)[L2eIdx].Value = 0;
+ ((GMM_AUXTTL2e *)L2TableCPUAdr)[L2eIdx].L1GfxAddr = L1TableAdr >> 13;
+ ((GMM_AUXTTL2e *)L2TableCPUAdr)[L2eIdx].Valid = 1;
+ for(i = 0; i < (uint32_t)GMM_AUX_L1_SIZE(pGmmGlobalContext); i++)
+ {
+ //initialize L1e ie mark all entries with Null tile value
+ ((GMM_AUXTTL1e *)L1TableCPUAdr)[i].Value = InvalidEntry;
+ }
+ }
+ else
+ {
+ GMM_AUXTTL2e L2e = {0};
+ L2e.Valid = 1;
+ L2e.L1GfxAddr = L1TableAdr >> 13;
+ pTTL2[L3eIdx].UpdatePoolFence(UmdContext, false);
+ PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
+ L2TableAdr + L2eIdx * GMM_AUX_L2e_SIZE,
+ L2e.Value);
+
+ //initialize all L1e with invalid entries
+ for(i = 0; i < (uint32_t)GMM_AUX_L1_SIZE(pGmmGlobalContext); i++)
+ {
+ PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
+ L1TableAdr + i * sizeof(uint64_t),
+ InvalidEntry);
+ }
+ }
+ }
+ }
+
+ GMM_DPF(GFXDBG_CRITICAL, "Mapping surface: GPUVA=0x%016llx Size=0x%08x Aux_GPUVA=0x%016llx", StartAdr, BaseSize, CCS$Adr);
+
+ for(TileAdr = StartAdr; TileAdr < EndAdr; TileAdr += (!WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16)),
+ CCS$Adr += (pGmmGlobalContext->GetSkuTable().FtrLinearCCS ?
+ (!WA16K ? GMM_BYTES(256) : GMM_BYTES(64)) :
+ 0))
+ {
+ GMM_GFX_SIZE_T L1eIdx = GMM_L1_ENTRY_IDX(AUXTT, TileAdr, pGmmGlobalContext);
+ GMM_AUXTTL1e L1e = {0};
+ L1e.Value = PartialData;
+ L1e.Valid = 1;
+
+ CCS$Adr = (pGmmGlobalContext->GetSkuTable().FtrLinearCCS ? CCS$Adr :
+ __GetCCSCacheline(BaseResInfo, BaseAdr, AuxResInfo, AuxVA, TileAdr - BaseAdr));
+
+ if(!WA16K)
+ {
+ __GMM_ASSERT((CCS$Adr & 0xFF) == 0x0);
+ __GMM_ASSERT(GFX_IS_ALIGNED(CCS$Adr, GMM_BYTES(256)));
+ __GMM_ASSERT(GFX_IS_ALIGNED(TileAdr, GMM_KBYTE(64)));
+ L1e.GfxAddress = CCS$Adr >> 8; /*********** 256B-aligned CCS adr *****/
+ }
+ else
+ {
+ L1e.Reserved2 = CCS$Adr >> 6; /*********** 2 lsbs of 64B-aligned CCS adr *****/
+ L1e.GfxAddress = CCS$Adr >> 8; /*********** 256B-aligned CCS adr *****/
+ }
+
+ //GMM_DPF(GFXDBG_CRITICAL, "Map | L1=0x%016llx[0x%016llx] L1e=[0x%016llx] | [ GPUVA=0x%016llx[0x%08x] Aux_GPUVA=0x%016llx [%s]", L1TableAdr + L1eIdx << 3, L1eIdx << 3, L1e.Value, TileAdr, TileAdr - StartAdr, CCS$Adr, L1e.Valid ? "V" : " ");
+
+ GmmLib::LastLevelTable *pL1Tbl = NULL;
+
+ pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx, NULL);
+ L1TableCPUAdr = pL1Tbl->GetCPUAddress();
+ if(DoNotWait)
+ {
+ //Sync update on CPU
+ ((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx].Value = L1e.Value;
+
+ //GMM_DPF(GFXDBG_CRITICAL, "Map | Table Entry: [0x%06x] L2Addr[0x%016llX] Value[0x%016llX] :: [0x%06x] L1Addr[0x%016llX] Value[0x%016llX] -> GPUVA: 0x%016llX[0x%06X] Aux_GPUVA: 0x%016llX [%s]\n", L2eIdx, &((GMM_AUXTTL2e *)L2TableCPUAdr)[L2eIdx], ((GMM_AUXTTL2e *)L2TableCPUAdr)[L2eIdx].L1GfxAddr << 13, L1eIdx, &((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx] /*L1TableAdr + L1eIdx * GMM_AUX_L1e_SIZE*/, L1e.Value, TileAdr, TileAdr - BaseAdr, ((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx].GfxAddress << 6, ((GMM_AUXTTL1e *)L1CPUAdr)[L1eIdx].Valid ? "V" : " ");
+ }
+ else
+ {
+ pL1Tbl->UpdatePoolFence(UmdContext, false);
+ PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
+ L1TableAdr + L1eIdx * GMM_AUX_L1e_SIZE,
+ L1e.Value);
+ }
+
+ // Since we are mapping a non-null entry, no need to check whether
+ // L1 table is unused.
+ pL1Tbl->TrackTableUsage(AUXTT, true, TileAdr, false);
+ }
+ }
+ if(!DoNotWait)
+ {
+ PageTableMgr->TTCb.pfEpilogTranslationTable(
+ UmdContext->pCommandQueueHandle,
+ 1);
+ }
+ }
+
+ LeaveCriticalSection(&TTLock);
+
+ return Status;
+}
+
+GMM_AUXTTL1e GmmLib::AuxTable::CreateAuxL1Data(GMM_RESOURCE_INFO *BaseResInfo)
+{
+ GMM_FORMAT_ENTRY FormatInfo = pGmmGlobalContext->GetPlatformInfo().FormatTable[BaseResInfo->GetResourceFormat()];
+ GMM_AUXTTL1e L1ePartial = {0};
+#define GMM_REGISTRY_UMD_PATH "SOFTWARE\\Intel\\IGFX\\GMM\\"
+#define GMM_E2EC_OVERRIDEDEPTH16BPPTO12 "ForceYUV16To12BPP"
+
+ L1ePartial.Mode = BaseResInfo->GetResFlags().Info.RenderCompressed ? 0x1 : 0x0; //MC on VCS supports all compression modes,
+ //MC on Render pipe only 128B compr (until B-step)
+ //Recognize which .MC surfaces needs Render pipe access
+ if(pGmmGlobalContext->GetWaTable().WaLimit128BMediaCompr)
+ {
+ L1ePartial.Mode = 0x1; //Limit media compression to 128B (same as RC) on gen12LP A0
+ }
+
+ //L1ePartial.Lossy = 0; // when to set it
+ L1ePartial.TileMode = BaseResInfo->GetResFlags().Info.TiledYs ? 0 : 1;
+
+ L1ePartial.Format = FormatInfo.CompressionFormat.AuxL1eFormat;
+ L1ePartial.LumaChroma = GmmIsPlanar(BaseResInfo->GetResourceFormat());
+
+ if(pGmmGlobalContext->GetWaTable().WaUntypedBufferCompression && BaseResInfo->GetResourceType() == RESOURCE_BUFFER)
+ {
+ //Gen12LP WA to support untyped raw buffer compression on HDC ie MLC(machine-learning compression)
+ L1ePartial.TileMode = 0;
+ L1ePartial.Depth = 0x6;
+ L1ePartial.Format = GMM_E2ECOMP_FORMAT_RGBAFLOAT16;
+ }
+
+ __GMM_ASSERT(L1ePartial.Format > GMM_E2ECOMP_MIN_FORMAT && //Are we going to reuse 0x00 for uncompressed indication? CCS contains that info, but only known by HW
+ L1ePartial.Format <= GMM_E2ECOMP_MAX_FORMAT); //Could SW use it as surface-wide uncompressed state indicator? If so, remove teh assert (Need to make sure, all format encodings are correct)
+
+ if(BaseResInfo->GetResFlags().Info.RenderCompressed)
+ {
+ if(BaseResInfo->GetResourceType() != RESOURCE_BUFFER)
+ {
+ switch(FormatInfo.Element.BitsPer)
+ {
+ case 8:
+ L1ePartial.Depth = 0x4;
+ break;
+ case 16:
+ L1ePartial.Depth = 0x0;
+ break;
+ case 32:
+ L1ePartial.Depth = 0x5;
+ break;
+ case 64:
+ L1ePartial.Depth = 0x6;
+ break;
+ case 128:
+ L1ePartial.Depth = 0x7;
+ break;
+ default:
+ L1ePartial.Depth = 0x3;
+ }
+ }
+ }
+ else
+ {
+ switch(BaseResInfo->GetResourceFormat())
+ {
+ case GMM_FORMAT_P012:
+ case GMM_FORMAT_Y412:
+ case GMM_FORMAT_Y212: //which format encoding for Y212, Y412, P012?
+ L1ePartial.Depth = 0x2;
+ break;
+ case GMM_FORMAT_P010:
+ //case GMM_FORMAT_Y410:
+ case GMM_FORMAT_Y210: //which format encoding for Y210?
+ L1ePartial.Depth = 0x1;
+ break;
+ case GMM_FORMAT_P016: //per HAS, separate encoding than P010, but a comment says to use P010 in AuxTable?
+ case GMM_FORMAT_Y416:
+ case GMM_FORMAT_Y216:
+ L1ePartial.Depth = 0x0;
+ break;
+ default:
+ L1ePartial.Depth = 0x3; //For MC, bpp got from format encoding
+ }
+
+ if(L1ePartial.Format == GMM_E2ECOMP_FORMAT_R10G10B10A2_UNORM)
+ {
+ L1ePartial.Format = GMM_E2ECOMP_FORMAT_RGB10b;
+ }
+ }
+
+ return L1ePartial;
+}
+
+GMM_GFX_ADDRESS GMM_INLINE GmmLib::AuxTable::__GetCCSCacheline(GMM_RESOURCE_INFO *BaseResInfo, GMM_GFX_ADDRESS BaseAdr,
+ GMM_RESOURCE_INFO *AuxResInfo, GMM_GFX_ADDRESS AuxVA, GMM_GFX_SIZE_T AdrOffset)
+{
+ GMM_GFX_ADDRESS CCSChunkAdr = 0xFFFFFFF0;
+ uint32_t x = 0, y = 0;
+ uint32_t i = 0, j = 0;
+ uint32_t CCSXTile = 0, CCSYTile = 0;
+ GMM_UNREFERENCED_PARAMETER(BaseAdr);
+
+ bool BaseIsYF = BaseResInfo->GetResFlags().Info.TiledYf ? true : false;
+ uint32_t BasePitchInTiles = BaseResInfo->GetRenderPitchTiles();
+
+ //Find YF/YS TileId <x,y> for given main surface 16K-chunk
+ //and CCS$Id <i,j> corresponding to main's <x,y>
+ AdrOffset >>= 14; //AdrOffset must be 16K-aligned chunk, since mapping unit is 4 YF pages
+ if(BaseIsYF)
+ {
+ uint32_t PitchIn4YF = BasePitchInTiles / 4; //Base Pitch is physically padded to 4x1 YF width
+ i = static_cast<uint32_t>(AdrOffset % PitchIn4YF);
+ j = static_cast<uint32_t>(AdrOffset / PitchIn4YF);
+ }
+ else if(BasePitchInTiles != 0) //TileYs
+ {
+ x = static_cast<uint32_t>(AdrOffset >> 2); //YS-tile count
+ y = x / BasePitchInTiles; //YS- tile id <x,y>
+ x = x % BasePitchInTiles;
+ i = 2 * x;
+ j = 2 * y;
+ switch(AdrOffset % 4) //YS : XYXY [XYXY YF] ie 2x2 16K-units in Y-major
+ {
+ case 0:
+ break;
+ case 1:
+ j++;
+ break;
+ case 2:
+ i++;
+ break;
+ case 3:
+ i++;
+ j++;
+ break;
+ }
+ }
+
+ //Compute CCS$ address for <i,j>
+ CCSXTile = (i >= 8) ? i / 8 : 0; //8x8 CLs make one CCS Tile; get TileOffset
+ CCSYTile = (j >= 8) ? j / 8 : 0;
+ i %= 8;
+ j %= 8;
+
+ uint32_t AuxPitchInTiles = AuxResInfo ? AuxResInfo->GetRenderPitchTiles() : BaseResInfo->GetRenderAuxPitchTiles();
+ CCSChunkAdr = AuxVA + ((CCSXTile + CCSYTile * AuxPitchInTiles) * GMM_KBYTE(4)) + (8 * GMM_BYTES(64) * i) + (GMM_BYTES(64) * j);
+
+ return CCSChunkAdr;
+}
+
+#endif /*!__GMM_KMD__*/
diff --git a/Source/GmmLib/TranslationTable/GmmPageTableMgr.cpp b/Source/GmmLib/TranslationTable/GmmPageTableMgr.cpp
new file mode 100644
index 0000000..55e6a33
--- /dev/null
+++ b/Source/GmmLib/TranslationTable/GmmPageTableMgr.cpp
@@ -0,0 +1,730 @@
+/*==============================================================================
+Copyright(c) 2019 Intel Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files(the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and / or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+Description: UMD-TT manager (manages both TR-TT and AUX-TT in user mode space)
+
+============================================================================*/
+
+#include "Internal/Common/GmmLibInc.h"
+#include "External/Common/GmmPageTableMgr.h"
+#include "../TranslationTable/GmmUmdTranslationTable.h"
+#include "External/Common/GmmClientContext.h"
+
+#if defined(__linux__)
+#include "Internal/Linux/GmmResourceInfoLinInt.h"
+#endif
+
+#define ENTER_CRITICAL_SECTION \
+ if(AuxTTObj) \
+ { \
+ EnterCriticalSection(&PoolLock); \
+ }
+
+#define EXIT_CRITICAL_SECTION \
+ if(AuxTTObj) \
+ { \
+ LeaveCriticalSection(&PoolLock); \
+ }
+
+#if defined(__linux__)
+GMM_STATUS GmmLib::__GmmDeviceAlloc(GmmClientContext * pClientContext,
+ GMM_DEVICE_CALLBACKS_INT *pDeviceCbInt,
+ GMM_DEVICE_ALLOC * pAlloc)
+{
+ GMM_CLIENT ClientType;
+ GMM_DDI_ALLOCATE Alloc = {0};
+ int err;
+
+ GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
+
+ __GMM_ASSERTPTR(GmmCheckForNullDevCbPfn(ClientType, pDeviceCbInt, GMM_DEV_CB_ALLOC), GMM_INVALIDPARAM);
+
+ if(GmmCheckForNullDevCbPfn(ClientType, pDeviceCbInt, GMM_DEV_CB_ALLOC))
+ {
+ Alloc.size = pAlloc->Size;
+ Alloc.alignment = pAlloc->Alignment;
+
+ err = GmmDeviceCallback(ClientType, pDeviceCbInt, &Alloc);
+ if(err)
+ {
+ return GMM_OUT_OF_MEMORY;
+ }
+
+ pAlloc->GfxVA = Alloc.gfxAddr;
+ pAlloc->CPUVA = Alloc.gfxAddr;
+ pAlloc->Handle = (HANDLE)Alloc.bo;
+ }
+
+ return GMM_SUCCESS;
+}
+
+GMM_STATUS GmmLib::__GmmDeviceDealloc(GMM_CLIENT ClientType,
+ GMM_DEVICE_CALLBACKS_INT *DeviceCb,
+ GMM_DEVICE_DEALLOC * pDealloc)
+{
+ GMM_DDI_DEALLOCATE DeAlloc = {0};
+ int err = 0;
+
+ __GMM_ASSERTPTR(GmmCheckForNullDevCbPfn(ClientType, DeviceCb, GMM_DEV_CB_DEALLOC), GMM_INVALIDPARAM);
+
+ if(GmmCheckForNullDevCbPfn(ClientType, DeviceCb, GMM_DEV_CB_DEALLOC))
+ {
+ DeAlloc.bo = pDealloc->Handle;
+
+ err = GmmDeviceCallback(ClientType, DeviceCb, &DeAlloc);
+ }
+
+ return (err == 0) ? GMM_SUCCESS : GMM_ERROR;
+}
+#endif
+
+//=============================================================================
+//
+// Function: __AllocateNodePool
+//
+// Desc: Allocates (always resident SVM) memory for new Pool node, and updates PageTableMgr object
+//
+// Parameters:
+// AddrAlignment: Pool allocation address alignment
+//
+// Returns:
+// S_OK on success,
+//-----------------------------------------------------------------------------
+GmmLib::GMM_PAGETABLEPool *GmmLib::GmmPageTableMgr::__AllocateNodePool(uint32_t AddrAlignment, GmmLib::POOL_TYPE Type)
+{
+ GMM_STATUS Status = GMM_SUCCESS;
+ GMM_RESOURCE_INFO *pGmmResInfo = NULL;
+ GMM_PAGETABLEPool *pTTPool = NULL;
+ HANDLE PoolHnd = 0;
+ GMM_CLIENT ClientType;
+ GMM_DEVICE_ALLOC Alloc = {0};
+
+ ENTER_CRITICAL_SECTION
+
+ //Allocate pool, sized PAGETABLE_POOL_MAX_NODES pages, assignable to TR/Aux L1/L2 tables
+ //SVM allocation, always resident
+ Alloc.Size = PAGETABLE_POOL_SIZE;
+ Alloc.Alignment = AddrAlignment;
+ Alloc.hCsr = hCsr;
+
+ Status = __GmmDeviceAlloc(pClientContext, &DeviceCbInt, &Alloc);
+
+ if(Status != GMM_SUCCESS)
+ {
+ __GMM_ASSERT(0);
+ EXIT_CRITICAL_SECTION
+ return NULL;
+ }
+
+ PoolHnd = Alloc.Handle;
+ pGmmResInfo = (GMM_RESOURCE_INFO *)Alloc.Priv;
+
+ pTTPool = new GMM_PAGETABLEPool(PoolHnd, pGmmResInfo, Alloc.GfxVA, Alloc.CPUVA, Type);
+
+ __GMM_ASSERTPTR(pTTPool, NULL);
+
+ if(pTTPool)
+ {
+ if(pPool)
+ {
+ NumNodePoolElements++;
+ if(Type == POOL_TYPE_TRTTL2) // TRTT-L2 not 1st node in Pool LinkedList, place it at beginning
+ {
+ pPool = pPool->InsertInListAtBegin(pTTPool);
+ }
+ else
+ {
+ pTTPool = pPool->InsertInList(pTTPool);
+ }
+ }
+ else
+ {
+ NumNodePoolElements = 1;
+ pPool = pTTPool;
+ }
+ }
+ else
+ {
+ Status = GMM_OUT_OF_MEMORY;
+ }
+
+ EXIT_CRITICAL_SECTION
+ return (Status == GMM_SUCCESS) ? pTTPool : NULL;
+}
+
+//=============================================================================
+//
+// Function: __ReleaseUnusedPool
+//
+// Desc: Frees up unused PageTablePools once residency limit is hit
+//
+// Parameters:
+// UmdContext: pointer to caller thread's context (containing BBHandle/Fence info)
+//
+//-----------------------------------------------------------------------------
+void GmmLib::GmmPageTableMgr::__ReleaseUnusedPool(GMM_UMD_SYNCCONTEXT *UmdContext)
+{
+ GMM_STATUS Status = GMM_SUCCESS;
+ GMM_GFX_SIZE_T PoolSizeToFree = {0};
+ GMM_GFX_SIZE_T FreedSize = {0};
+ GmmLib::GMM_PAGETABLEPool *Pool = NULL, *PrevPool = NULL;
+ uint32_t i = 0;
+ GMM_CLIENT ClientType;
+ GMM_DEVICE_DEALLOC Dealloc;
+
+ GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
+
+ ENTER_CRITICAL_SECTION
+ if(pPool->__IsUnusedTRTTPoolOverLimit(&PoolSizeToFree))
+ {
+ for(i = 0; i < NumNodePoolElements && FreedSize < PoolSizeToFree; i++)
+ {
+ Pool = (PrevPool) ? PrevPool->GetNextPool() : pPool;
+
+ if(Pool->IsPoolInUse(UmdContext ? SyncInfo(UmdContext->BBFenceObj, UmdContext->BBLastFence) : SyncInfo()))
+ {
+ PrevPool = Pool;
+ continue;
+ }
+
+ if(GmmCheckForNullDevCbPfn(ClientType, &DeviceCbInt, GMM_DEV_CB_WAIT_FROM_CPU))
+ {
+ GMM_DDI_WAITFORSYNCHRONIZATIONOBJECTFROMCPU Wait = {0};
+ Wait.bo = Pool->GetPoolHandle();
+ GmmDeviceCallback(ClientType, &DeviceCbInt, &Wait);
+ }
+
+ Dealloc.Handle = Pool->GetPoolHandle();
+ Dealloc.GfxVA = Pool->GetGfxAddress();
+ Dealloc.Priv = Pool->GetGmmResInfo();
+ Dealloc.hCsr = hCsr;
+
+ Status = __GmmDeviceDealloc(ClientType, &DeviceCbInt, &Dealloc);
+
+ __GMM_ASSERT(GMM_SUCCESS == Status);
+
+ if(PrevPool)
+ {
+ PrevPool->GetNextPool() = Pool->GetNextPool();
+ }
+ else
+ {
+ pPool = Pool->GetNextPool();
+ }
+ delete Pool;
+ FreedSize += PAGETABLE_POOL_SIZE;
+ }
+ }
+ EXIT_CRITICAL_SECTION
+}
+
+//=============================================================================
+//
+// Function: __GetFreePoolNode
+//
+// Desc: Finds free node within existing PageTablePool(s), if no such node found,
+// allocates new PageTablePool. Caller should update Pool Node usage
+//
+// Parameters:
+// FreePoolNodeIdx: pointer to return Pool's free Node index
+// PoolType: AuxTT_L1/L2 pool
+//
+// Returns:
+// PageTablePool element and FreePoolNodeIdx that should be used for L2/L1 assignment
+// NULL, if no free node exists and new pool allocation failed
+//-----------------------------------------------------------------------------
+GmmLib::GMM_PAGETABLEPool *GmmLib::GmmPageTableMgr::__GetFreePoolNode(uint32_t *FreePoolNodeIdx, POOL_TYPE PoolType)
+{
+ uint32_t PoolNode = -1, i = 0, j = 0, DWdivisor = 1, IdxMultiplier = 1;
+ bool PoolNodeFound = false, TRTTPool = false;
+
+ ENTER_CRITICAL_SECTION
+ GmmLib::GMM_PAGETABLEPool *Pool = pPool;
+
+ Pool = (PoolType == POOL_TYPE_TRTTL2) ? Pool : //1st pool reserved for TRTT-L2, since TRTT-L2 pruning not supported yet,
+ (Pool ? Pool->GetNextPool() : NULL); //other pools can be TR-L1/Aux-L1/Aux-L2 (and support dynamic pruning)
+ TRTTPool = (PoolType == POOL_TYPE_TRTTL2 || PoolType == POOL_TYPE_TRTTL1) ? true : false;
+ DWdivisor = TRTTPool ? 8 * sizeof(uint32_t) : (PoolType == POOL_TYPE_AUXTTL2) ? 8 * sizeof(uint32_t) * AUX_L2TABLE_SIZE_IN_POOLNODES : 8 * sizeof(uint32_t) * AUX_L1TABLE_SIZE_IN_POOLNODES;
+ IdxMultiplier = TRTTPool ? 1 : (PoolType == POOL_TYPE_AUXTTL2) ? AUX_L2TABLE_SIZE_IN_POOLNODES : AUX_L1TABLE_SIZE_IN_POOLNODES;
+
+ //Scan existing PageTablePools for free pool node
+ for(i = (PoolType == POOL_TYPE_TRTTL2) ? 0 : 1; Pool && i < NumNodePoolElements; i++)
+ {
+ if(Pool->GetNumFreeNode() > 0 && Pool->GetPoolType() == PoolType)
+ {
+ PoolNodeFound = true;
+ *FreePoolNodeIdx = 0;
+ for(; j < PAGETABLE_POOL_MAX_NODES / DWdivisor; j++)
+ {
+ if(_BitScanForward((uint32_t *)&PoolNode, (uint32_t) ~(Pool->GetNodeUsageAtIndex(j)))) // Get LSB that has value 0
+ {
+ *FreePoolNodeIdx += PoolNode * IdxMultiplier;
+ PoolNodeFound = true;
+ break;
+ }
+ PoolNodeFound = false;
+ *FreePoolNodeIdx += DWdivisor; //DWORD size in bits
+ }
+ }
+ if(PoolNodeFound)
+ {
+ __GMM_ASSERT(Pool->GetPoolType() == PoolType);
+ EXIT_CRITICAL_SECTION
+ return Pool;
+ }
+ Pool = Pool->GetNextPool();
+ }
+
+ //No free pool node, allocate new
+ if(!PoolNodeFound)
+ {
+ GMM_PAGETABLEPool *Pool = NULL;
+ if(Pool = __AllocateNodePool(IdxMultiplier * PAGE_SIZE, PoolType))
+ {
+ __GMM_ASSERT(Pool->GetPoolType() == PoolType);
+
+ *FreePoolNodeIdx = 0;
+ EXIT_CRITICAL_SECTION
+ return Pool;
+ }
+ }
+
+ EXIT_CRITICAL_SECTION
+ return NULL;
+}
+
+
+
+/**********************************************************************************
+** Class GmmPageTableMgr functions **
+***********************************************************************************/
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// Instantiates GmmPageTableMgr, allocating memory for root-tables, copies provided
+/// device-callback function pointers
+///
+/// @param[in] DeviceCb: pointer sharing device-callback function pointers
+/// @param[in] TTFlags: Flags specifying which PageTables are required by client
+/// @return GmmPageTableMgr*
+/////////////////////////////////////////////////////////////////////////////////////
+GmmLib::GmmPageTableMgr::GmmPageTableMgr(GMM_DEVICE_CALLBACKS_INT *DeviceCB, uint32_t TTFlags, GmmClientContext *pClientContextIn)
+ : GmmPageTableMgr()
+{
+ GMM_PAGETABLE_MGR *ptr = NULL;
+ GMM_STATUS status = GMM_SUCCESS;
+ GMM_CLIENT ClientType;
+
+ if(pClientContextIn)
+ {
+ ClientType = pClientContextIn->GetClientType();
+ }
+ else
+ {
+ goto ERROR_CASE;
+ }
+
+ // this is needed if there is an error case and destructor gets called on ptr
+ this->pClientContext = pClientContextIn;
+
+ // Currently coping the code below to GMMOldAPi.cpp for backward compatible.
+ // Any changes here should be copied there.
+ //Initialize PageTableMgr further, only if PageTable creation succeeded
+ try
+ {
+ ptr = new GmmPageTableMgr();
+ ptr->pClientContext = pClientContextIn;
+ memcpy(&ptr->DeviceCbInt, DeviceCB, sizeof(GMM_DEVICE_CALLBACKS_INT));
+
+ if(pGmmGlobalContext->GetSkuTable().FtrE2ECompression)
+ {
+ __GMM_ASSERT(TTFlags & AUXTT); //Aux-TT is mandatory
+ ptr->AuxTTObj = new AuxTable();
+ if(!ptr->AuxTTObj)
+ {
+ goto ERROR_CASE;
+ }
+ ptr->AuxTTObj->PageTableMgr = ptr;
+ ptr->AuxTTObj->pClientContext = pClientContextIn;
+ status = ptr->AuxTTObj->AllocateL3Table(8 * PAGE_SIZE, 8 * PAGE_SIZE);
+
+ if(status != GMM_SUCCESS)
+ {
+ InitializeCriticalSection(&(ptr->PoolLock));
+ goto ERROR_CASE;
+ }
+ }
+ }
+
+ catch(...)
+ {
+ __GMM_ASSERT(false);
+ if(ptr && (AuxTTObj))
+ {
+ InitializeCriticalSection(&(ptr->PoolLock));
+ }
+ goto ERROR_CASE;
+ }
+
+ if(status == GMM_SUCCESS && !(AuxTTObj))
+ {
+ if(ptr->AuxTTObj)
+ {
+ ptr->AuxTTObj->PageTableMgr = this;
+ }
+ *this = *ptr;
+ //Don't initialize PoolLock until any of AuxTable object created
+ if(ptr->AuxTTObj )
+ {
+ InitializeCriticalSection(&PoolLock);
+
+ }
+ //Delete temporary ptr, but don't release allocated PageTable Obj.
+ ptr->AuxTTObj = NULL;
+ }
+
+ERROR_CASE:
+ delete ptr;
+ ptr = NULL;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// Returns Root-table address for Aux-table
+///
+/// @return GMM_GFX_ADDRESS if Aux-Table was created; NULL otherwise
+/////////////////////////////////////////////////////////////////////////////////////
+GMM_GFX_ADDRESS GmmLib::GmmPageTableMgr::GetAuxL3TableAddr()
+{
+ return AuxTTObj ? AuxTTObj->GetL3Address() : NULL;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// Queues commands to initialize Aux-Table registers in the HW context image
+///
+/// @param[in] initialBBHandle: pointer to BatchBuffer for queuing commands
+/// @param[in] engType: specifes engine on which the context would run
+/// @return GMM_SUCCESS if queuing succeeded; GMM_ERROR otherwise
+/////////////////////////////////////////////////////////////////////////////////////
+GMM_STATUS GmmLib::GmmPageTableMgr::InitContextAuxTableRegister(HANDLE CmdQHandle, GMM_ENGINE_TYPE engType)
+{
+ GMM_GFX_ADDRESS MaskedL3GfxAddress = 0ULL;
+ GMM_UNREFERENCED_PARAMETER(engType);
+
+ //Check FtrE2ECompression = 1
+ if(pGmmGlobalContext->GetSkuTable().FtrE2ECompression && AuxTTObj != NULL)
+ {
+ EnterCriticalSection(&AuxTTObj->TTLock);
+ if(CmdQHandle)
+ {
+ //engType = ENGINE_TYPE_RCS; //use correct offset based on engType (once per-eng offsets known)
+ uint64_t RegOffset = 0, L3AdrReg = 0;
+ GET_L3ADROFFSET(0, L3AdrReg);
+
+ RegOffset = (L3AdrReg + sizeof(uint32_t));
+ RegOffset = L3AdrReg | (RegOffset << 0x20);
+
+ MaskedL3GfxAddress = AuxTTObj->GetL3Address();
+
+ //TTCb.pfPrologTranslationTable(CmdQHandle); //MI_FLUSH, TLBInv not required since its called during context-init
+
+ TTCb.pfWriteL3Adr(CmdQHandle, MaskedL3GfxAddress, RegOffset);
+
+ GMM_DPF(GFXDBG_CRITICAL, "AuxTT Map Address: GPUVA=0x%016llX\n", MaskedL3GfxAddress);
+
+ //TTCb.pfEpilogTranslationTable(CmdQHandle, 0);
+
+ AuxTTObj->GetRegisterStatus() = 0;
+ }
+ else
+ {
+ __GMM_ASSERT(false);
+ LeaveCriticalSection(&AuxTTObj->TTLock);
+ return GMM_INVALIDPARAM;
+ }
+ LeaveCriticalSection(&AuxTTObj->TTLock);
+ }
+ return GMM_SUCCESS;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// Updates the Aux-PageTables, for given base resource, with appropriate mappings
+///
+/// @param[in] Details of AuxTable update request
+/// @return GMM_STATUS
+/////////////////////////////////////////////////////////////////////////////////////
+GMM_STATUS GmmLib::GmmPageTableMgr::UpdateAuxTable(const GMM_DDI_UPDATEAUXTABLE *UpdateReq)
+{
+ if(GetAuxL3TableAddr() == NULL)
+ {
+ GMM_ASSERTDPF(0, "Invalid AuxTable update request, AuxTable is not initialized");
+ return GMM_INVALIDPARAM;
+ }
+
+ if(!((UpdateReq->BaseResInfo->GetResFlags().Info.RenderCompressed ||
+ UpdateReq->BaseResInfo->GetResFlags().Info.MediaCompressed) &&
+ ((!UpdateReq->AuxResInfo && UpdateReq->BaseResInfo->GetResFlags().Gpu.UnifiedAuxSurface) ||
+ (UpdateReq->AuxResInfo && UpdateReq->AuxResInfo->GetResFlags().Gpu.CCS))))
+ /*(UpdateReq->BaseResInfo->GetResFlags().Gpu.TiledResource ||
+ UpdateReq->BaseResInfo->GetResFlags().Gpu.Depth) */
+ //Allow Separate Aux for Depth/TR/MSAA/others?
+ {
+ GMM_ASSERTDPF(0, "Invalid AuxTable update request");
+ return GMM_INVALIDPARAM;
+ }
+
+ if(UpdateReq->Map && !(!UpdateReq->BaseResInfo->GetResFlags().Gpu.TiledResource || (UpdateReq->BaseResInfo->GetResFlags().Gpu.TiledResource && UpdateReq->UmdContext && UpdateReq->UmdContext->pCommandQueueHandle)))
+ {
+ //GMM_DPF_CRITICAL("TiledResources must Gpu-update AuxTable, proceeding with CPU-update...");
+
+ //Allowing CPU-update if requested so..
+ if(!UpdateReq->DoNotWait)
+ {
+ return GMM_INVALIDPARAM;
+ }
+ }
+
+ ENTER_CRITICAL_SECTION
+
+ if(UpdateReq->Map)
+ {
+ //Get AuxL1e data (other than CCS-adr) from main surface
+ uint64_t PartialL1e = AuxTTObj->CreateAuxL1Data(UpdateReq->BaseResInfo).Value;
+ GMM_STATUS Status = GMM_SUCCESS;
+
+ if(UpdateReq->BaseResInfo->GetResFlags().Gpu.TiledResource)
+ {
+ //Aux-TT is sparsely updated, for TRs, upon change in mapping state ie
+ // null->non-null must be mapped
+ // non-null->null invalidated on AuxTT
+ uint8_t CpuUpdate = UpdateReq->DoNotWait || !(UpdateReq->UmdContext && UpdateReq->UmdContext->pCommandQueueHandle);
+
+ GMM_GFX_ADDRESS AuxVA = UpdateReq->AuxSurfVA;
+ if(UpdateReq->BaseResInfo->GetResFlags().Gpu.UnifiedAuxSurface)
+ {
+ GMM_UNIFIED_AUX_TYPE AuxType = GMM_AUX_CCS;
+ AuxType = (UpdateReq->BaseResInfo->GetResFlags().Gpu.Depth && UpdateReq->BaseResInfo->GetResFlags().Gpu.CCS) ? GMM_AUX_ZCS : AuxType;
+ AuxVA = UpdateReq->BaseGpuVA + GmmResGetAuxSurfaceOffset(UpdateReq->BaseResInfo, AuxType);
+ }
+
+ }
+ else
+ {
+ GMM_GFX_ADDRESS AuxVA = {0};
+ GMM_GFX_ADDRESS UVAuxVA = {0};
+ GMM_GFX_SIZE_T YPlaneSize = 0;
+ uint32_t MaxPlanes = 1;
+
+ if(!UpdateReq->AuxResInfo && UpdateReq->BaseResInfo->GetResFlags().Gpu.UnifiedAuxSurface)
+ {
+ GMM_UNIFIED_AUX_TYPE AuxType = GMM_AUX_CCS;
+ AuxType = (UpdateReq->BaseResInfo->GetResFlags().Gpu.Depth &&
+ UpdateReq->BaseResInfo->GetResFlags().Gpu.CCS) ?
+ GMM_AUX_ZCS :
+ AuxType;
+
+ AuxVA = UpdateReq->BaseGpuVA + GmmResGetAuxSurfaceOffset(UpdateReq->BaseResInfo, AuxType);
+
+ //For UV Packed, Gen12 e2e compr supported formats have 2 planes per surface
+ //Each has distinct Aux surface, Y-plane/UV-plane must be mapped to respective Y/UV Aux surface
+ if(GmmIsPlanar(UpdateReq->BaseResInfo->GetResourceFormat()))
+ {
+ GMM_REQ_OFFSET_INFO ReqInfo = {0};
+ ReqInfo.Plane = GMM_PLANE_U;
+ ReqInfo.ReqRender = 1;
+
+ MaxPlanes = 2;
+ UpdateReq->BaseResInfo->GetOffset(ReqInfo);
+ YPlaneSize = ReqInfo.Render.Offset64;
+
+ UVAuxVA = UpdateReq->BaseGpuVA + GmmResGetAuxSurfaceOffset(UpdateReq->BaseResInfo, GMM_AUX_UV_CCS);
+ }
+ }
+
+ //Per-plane Aux-TT map called with per-plane base/Aux address/size
+ for(uint32_t i = 0; i < MaxPlanes; i++)
+ {
+ GMM_GFX_SIZE_T SurfSize = (MaxPlanes > 1 && UpdateReq->BaseResInfo->GetArraySize() > 1) ?
+ (UpdateReq->BaseResInfo->GetQPitchPlanar(GMM_NO_PLANE) * UpdateReq->BaseResInfo->GetRenderPitch()) :
+ UpdateReq->BaseResInfo->GetSizeMainSurface();
+ GMM_GFX_SIZE_T MapSize = (i == 0) ? ((MaxPlanes > 1) ? YPlaneSize : SurfSize) : SurfSize - YPlaneSize;
+
+ GMM_GFX_ADDRESS BaseSurfVA = (UpdateReq->AuxResInfo || i == 0) ? UpdateReq->BaseGpuVA :
+ UpdateReq->BaseGpuVA + YPlaneSize;
+ GMM_GFX_ADDRESS AuxSurfVA = (UpdateReq->AuxResInfo) ? UpdateReq->AuxSurfVA : (i > 0 ? UVAuxVA : AuxVA);
+
+ //Luma plane reset LumaChroma bit
+ ((GMM_AUXTTL1e *)&PartialL1e)->LumaChroma = (i == 0) ? 0 : 1;
+ uint32_t ArrayEle = GFX_MAX(((MaxPlanes > 1) ?
+ UpdateReq->BaseResInfo->GetArraySize() :
+ 1),
+ 1);
+
+ for(uint32_t j = 0; j < ArrayEle; j++)
+ {
+ BaseSurfVA += ((j > 0) ? (UpdateReq->BaseResInfo->GetQPitchPlanar(GMM_PLANE_Y) * UpdateReq->BaseResInfo->GetRenderPitch()) : 0);
+ AuxSurfVA += (UpdateReq->AuxResInfo ?
+ ((j > 0) ? (UpdateReq->AuxResInfo->GetQPitchPlanar(GMM_PLANE_Y) * UpdateReq->BaseResInfo->GetRenderPitch()) : 0) :
+ ((j > 0) ? UpdateReq->BaseResInfo->GetAuxQPitch() : 0));
+
+ //(Flat mapping): Remove main/aux resInfo from params
+ Status = AuxTTObj->MapValidEntry(UpdateReq->UmdContext, BaseSurfVA, MapSize, UpdateReq->BaseResInfo,
+ AuxSurfVA, UpdateReq->AuxResInfo, PartialL1e, 1);
+ if(Status != GMM_SUCCESS)
+ {
+ GMM_ASSERTDPF(0, "Insufficient memory, free resources and try again");
+ EXIT_CRITICAL_SECTION
+ return Status;
+ }
+ }
+ }
+ }
+ }
+ else
+ {
+ //Invalidate all mappings for given main surface
+ AuxTTObj->InvalidateTable(UpdateReq->UmdContext, UpdateReq->BaseGpuVA, UpdateReq->BaseResInfo->GetSizeMainSurface(), UpdateReq->DoNotWait);
+ }
+
+ EXIT_CRITICAL_SECTION
+ return GMM_SUCCESS;
+}
+
+#if defined(__linux__) && !_WIN32
+/////////////////////////////////////////////////////////////////////////////////////
+/// Gets size of PageTable buffer object (BOs) list
+///
+/// @param[in] TTFlags: Flags specifying PageTable-type for which BO-count required
+/// @return non-zero if BO list is created. Zero otherwise.
+/////////////////////////////////////////////////////////////////////////////////////
+int GmmLib::GmmPageTableMgr::GetNumOfPageTableBOs(uint8_t TTFlags)
+{
+ int NumBO = 0;
+
+ __GMM_ASSERTPTR(TTFlags & AUXTT, 0);
+
+ ENTER_CRITICAL_SECTION
+
+ if(AuxTTObj && AuxTTObj->GetL3Handle())
+ NumBO++;
+
+ NumBO += NumNodePoolElements;
+
+ EXIT_CRITICAL_SECTION
+
+ return NumBO;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// Gets size of PageTable buffer object (BOs) list
+///
+/// @param[in] TTFlags: Flags specifying PageTable-type for which BO-count required
+/// @param[in][out] BOList: pointer to memory where PageTable BO*(s) must be sent
+/// @return non-zero if BO list is created. Zero otherwise.
+/////////////////////////////////////////////////////////////////////////////////////
+int GmmLib::GmmPageTableMgr::GetPageTableBOList(uint8_t TTFlags, void *BOList)
+{
+ int NumBO = GetNumOfPageTableBOs(TTFlags);
+ HANDLE * Handles = (HANDLE *)BOList;
+ GmmLib::GMM_PAGETABLEPool *Pool;
+
+ __GMM_ASSERTPTR(TTFlags & AUXTT, 0);
+ __GMM_ASSERTPTR(BOList, 0);
+ __GMM_ASSERTPTR(NumBO, 0);
+
+ ENTER_CRITICAL_SECTION
+
+ if(AuxTTObj && AuxTTObj->GetL3Handle())
+ Handles[0] = AuxTTObj->GetL3Handle();
+
+ Pool = pPool;
+
+ for(int i = 0; i < NumNodePoolElements; i++)
+ {
+ if(Pool)
+ {
+ Handles[i + 1] = Pool->GetPoolHandle();
+ Pool = Pool->GetNextPool();
+ }
+ }
+
+ EXIT_CRITICAL_SECTION
+
+ return NumBO;
+}
+#endif
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// Releases GmmPageTableMgr, deleting root-tables and existing page-table pools
+/////////////////////////////////////////////////////////////////////////////////////
+GmmLib::GmmPageTableMgr::~GmmPageTableMgr()
+{
+ GMM_CLIENT ClientType;
+
+ GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
+
+
+
+ if(pPool)
+ {
+ ENTER_CRITICAL_SECTION
+ pPool->__DestroyPageTablePool(&DeviceCbInt, hCsr);
+ NumNodePoolElements = 0;
+ EXIT_CRITICAL_SECTION
+ }
+
+ if(AuxTTObj)
+ {
+ DeleteCriticalSection(&PoolLock);
+
+ if(AuxTTObj)
+ {
+ if(AuxTTObj->NullL1Table)
+ {
+ delete AuxTTObj->NullL1Table;
+ }
+ if(AuxTTObj->NullL2Table)
+ {
+ delete AuxTTObj->NullL2Table;
+ }
+ AuxTTObj->DestroyL3Table();
+ delete AuxTTObj;
+ AuxTTObj = NULL;
+ }
+ }
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// Instantiates and zeroes out GmmPageTableMgr
+///
+/// @return GmmPageTableMgr*
+/////////////////////////////////////////////////////////////////////////////////////
+GmmLib::GmmPageTableMgr::GmmPageTableMgr()
+{
+ this->AuxTTObj = NULL;
+ this->pPool = NULL;
+ this->NumNodePoolElements = 0;
+ this->pClientContext = NULL;
+ this->hCsr = NULL;
+
+ memset(&DeviceCb, 0, sizeof(GMM_DEVICE_CALLBACKS));
+ memset(&DeviceCbInt, 0, sizeof(GMM_DEVICE_CALLBACKS_INT));
+}
+
+
diff --git a/Source/GmmLib/TranslationTable/GmmUmdTranslationTable.cpp b/Source/GmmLib/TranslationTable/GmmUmdTranslationTable.cpp
new file mode 100644
index 0000000..3d94fcc
--- /dev/null
+++ b/Source/GmmLib/TranslationTable/GmmUmdTranslationTable.cpp
@@ -0,0 +1,670 @@
+/*==============================================================================
+Copyright(c) 2019 Intel Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files(the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and / or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+Description: Contains functions of internal classes
+ (ie PageTablePool, PageTable, Table), that support
+ user mode page table management
+
+============================================================================*/
+
+#include "Internal/Common/GmmLibInc.h"
+#include "../TranslationTable/GmmUmdTranslationTable.h"
+#include "Internal/Common/Texture/GmmTextureCalc.h"
+
+#if !defined(__GMM_KMD)
+
+#if defined(__linux__)
+#include "Internal/Linux/GmmResourceInfoLinInt.h"
+
+#define _aligned_free(ptr) free(ptr)
+
+#endif
+
+//=============================================================================
+//
+// Function: AllocateL3Table
+//
+// Desc: Allocates (always resident SVM) memory for AUXTT\L3 Table, and updates AUXTT object
+//
+// Parameters:
+// pAUXTT_Obj: per-device AUX-TT object. Contains AUXTT node info
+//
+// Returns:
+// GMM_SUCCESS on success,
+// GMM_INVALIDPARAM on invalid parameter(s)
+// GMM_OUT_OF_MEMORY on memory allocation failure, failure to make resident
+//-----------------------------------------------------------------------------
+GMM_STATUS GmmLib::PageTable::AllocateL3Table(uint32_t L3TableSize, uint32_t L3AddrAlignment)
+{
+ GMM_STATUS Status = GMM_SUCCESS;
+ GMM_DEVICE_ALLOC Alloc = {0};
+
+ __GMM_ASSERTPTR(PageTableMgr, GMM_INVALIDPARAM);
+
+ EnterCriticalSection(&TTLock);
+
+ Alloc.Size = L3TableSize;
+ Alloc.Alignment = L3AddrAlignment;
+ Alloc.hCsr = PageTableMgr->hCsr;
+
+ Status = __GmmDeviceAlloc(pClientContext, &PageTableMgr->DeviceCbInt, &Alloc);
+ if(Status != GMM_SUCCESS)
+ {
+ LeaveCriticalSection(&TTLock);
+ return Status;
+ }
+
+ TTL3.GfxAddress = GMM_GFX_ADDRESS_CANONIZE(Alloc.GfxVA);
+ TTL3.CPUAddress = Alloc.CPUVA;
+ TTL3.NeedRegisterUpdate = true;
+ TTL3.L3Handle = (HANDLE)(uintptr_t)Alloc.Handle;
+ TTL3.pGmmResInfo = (GMM_RESOURCE_INFO *)Alloc.Priv;
+
+ // Invalidate L3e's
+ for(int i = 0; i < (GMM_L3_SIZE(TTType)); i++)
+ {
+ //initialize L3e ie mark all entries with Null tile/invalid value
+ ((GMM_AUXTTL3e *)TTL3.CPUAddress)[i].Value = 0;
+
+ }
+
+ LeaveCriticalSection(&TTLock);
+ return Status;
+}
+
+//=============================================================================
+//
+// Function: __IsUnusedTRTTPoolOverLimit
+//
+// Desc: Checks if unused TRTTPools have reached residency limit and must be freed.
+//
+// Parameters:
+// pTRTT_Obj: per-device TT object. Contains TT node info
+// OverLimitSize: Size in bytes that can be freed
+//
+// Returns:
+// True, if unused TTPool reached max. residency limit
+// False, otherwise
+//-----------------------------------------------------------------------------
+bool GmmLib::GmmPageTablePool::__IsUnusedTRTTPoolOverLimit(GMM_GFX_SIZE_T *OverLimitSize)
+{
+ GMM_GFX_SIZE_T UnusedTrTTPoolSize = 0;
+ GmmLib::GMM_PAGETABLEPool *Pool = NULL;
+
+ Pool = this;
+
+ while(Pool)
+ {
+ if(Pool->NumFreeNodes == PAGETABLE_POOL_MAX_NODES)
+ {
+ UnusedTrTTPoolSize += PAGETABLE_POOL_SIZE;
+ }
+ Pool = Pool->NextPool;
+ }
+
+ *OverLimitSize = (UnusedTrTTPoolSize > PAGETABLE_POOL_MAX_UNUSED_SIZE) ? (UnusedTrTTPoolSize - PAGETABLE_POOL_MAX_UNUSED_SIZE) : 0;
+ return (UnusedTrTTPoolSize > PAGETABLE_POOL_MAX_UNUSED_SIZE) ? true : false;
+}
+
+//=============================================================================
+//
+// Function: AllocateL1L2Table
+//
+// Desc: Assigns pages from AUXTTPool for L1/L2Table for translation of given TRVA
+//
+// Parameters:
+// pAUXTT_Obj: per-device AUX-TT object. Contains AUXTT node info
+// TileAddr: Tiled Resource Virtual address
+//
+// Returns:
+// L2Table/L1Table Address
+//-----------------------------------------------------------------------------
+void GmmLib::PageTable::AllocateL1L2Table(GMM_GFX_ADDRESS TileAddr, GMM_GFX_ADDRESS *L1TableAdr, GMM_GFX_ADDRESS *L2TableAdr)
+{
+ GMM_GFX_ADDRESS L3TableAdr = GMM_NO_TABLE;
+ uint32_t L3eIdx = static_cast<uint32_t>(GMM_L3_ENTRY_IDX(TTType, TileAddr));
+ uint32_t L2eIdx = static_cast<uint32_t>(GMM_L2_ENTRY_IDX(TTType, TileAddr));
+ GmmLib::LastLevelTable *pL1Tbl = NULL;
+
+ *L2TableAdr = GMM_NO_TABLE;
+ *L1TableAdr = GMM_NO_TABLE;
+
+ if(TTL3.L3Handle)
+ {
+ L3TableAdr = TTL3.GfxAddress;
+ }
+ else
+ {
+ //Should never hit -- L3Table is allocated during device creation
+ __GMM_ASSERT(false);
+ }
+
+ if(pTTL2[L3eIdx].GetPool())
+ {
+ GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
+ PoolElem = pTTL2[L3eIdx].GetPool();
+ *L2TableAdr = (PoolElem != NULL) ? PoolElem->GetGfxAddress() + (PAGE_SIZE * pTTL2[L3eIdx].GetNodeIdx()) : GMM_NO_TABLE;
+ }
+ else
+ {
+ uint32_t PoolNodeIdx = PAGETABLE_POOL_MAX_NODES;
+ GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
+ POOL_TYPE PoolType = POOL_TYPE_AUXTTL2;
+ PoolElem = PageTableMgr->__GetFreePoolNode(&PoolNodeIdx, PoolType);
+ if(PoolElem)
+ {
+ pTTL2[L3eIdx] = MidLevelTable(PoolElem, PoolNodeIdx, PoolElem->GetNodeBBInfoAtIndex(PoolNodeIdx));
+ *L2TableAdr = PoolElem->GetGfxAddress() + PAGE_SIZE * PoolNodeIdx; //PoolNodeIdx must be multiple of 8 (Aux L2) and multiple of 2 (Aux L1)
+ ASSIGN_POOLNODE(PoolElem, PoolNodeIdx, NodesPerTable)
+ }
+ }
+
+ pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx);
+ if(pL1Tbl)
+ {
+ GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
+ PoolElem = pL1Tbl->GetPool();
+ *L1TableAdr = (PoolElem != NULL) ? PoolElem->GetGfxAddress() + (PAGE_SIZE * pL1Tbl->GetNodeIdx()) : GMM_NO_TABLE;
+ }
+ else
+ {
+ //Allocate L1 Table
+ uint32_t PoolNodeIdx = PAGETABLE_POOL_MAX_NODES;
+ GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
+ POOL_TYPE PoolType = POOL_TYPE_AUXTTL1;
+
+ PoolElem = PageTableMgr->__GetFreePoolNode(&PoolNodeIdx, PoolType); //Recognize if Aux-L1 being allocated
+ if(PoolElem)
+ {
+ pL1Tbl = new GmmLib::LastLevelTable(PoolElem, PoolNodeIdx, GMM_L1_SIZE_DWORD(TTType, pGmmGlobalContext), L2eIdx); //use Aux L1_Size_DWORD
+
+ if(pL1Tbl)
+ {
+ *L1TableAdr = PoolElem->GetGfxAddress() + PAGE_SIZE * PoolNodeIdx; //PoolNodeIdx should reflect 1 node per Tr-table and 2 nodes per AUX L1 TABLE
+ if(PoolNodeIdx != PAGETABLE_POOL_MAX_NODES)
+ {
+ uint32_t PerTableNodes = (TTType == AUXTT) ? AUX_L1TABLE_SIZE_IN_POOLNODES : 1;
+ ASSIGN_POOLNODE(PoolElem, PoolNodeIdx, PerTableNodes)
+ }
+ pTTL2[L3eIdx].InsertInList(pL1Tbl);
+ }
+ }
+ }
+}
+
+//=============================================================================
+//
+// Function: AllocateDummyTables
+//
+// Desc: Assigns pages from AUXTTPool for Dummy L1/L2Table
+//
+// Parameters:
+// L2Table - Ptr to initiatize dummy L2Table
+// L1Table - Ptr to initiatize dummy L1Table
+//
+// Returns:
+// L2Table/L1Tables
+//-----------------------------------------------------------------------------
+void GmmLib::PageTable::AllocateDummyTables(GmmLib::Table **L2Table, GmmLib::Table **L1Table)
+{
+ GMM_GFX_ADDRESS L3TableAdr = GMM_NO_TABLE;
+ GmmLib::LastLevelTable *pL1Tbl = NULL;
+
+ if(TTL3.L3Handle)
+ {
+ L3TableAdr = TTL3.GfxAddress;
+ }
+ else
+ {
+ //Should never hit -- L3Table is allocated during device creation
+ __GMM_ASSERT(false);
+ }
+
+ //Allocate dummy L2Table
+ {
+ uint32_t PoolNodeIdx = PAGETABLE_POOL_MAX_NODES;
+ GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
+ POOL_TYPE PoolType = POOL_TYPE_AUXTTL2;
+ PoolElem = PageTableMgr->__GetFreePoolNode(&PoolNodeIdx, PoolType);
+ if(PoolElem)
+ {
+ *L2Table = new GmmLib::MidLevelTable(PoolElem, PoolNodeIdx, PoolElem->GetNodeBBInfoAtIndex(PoolNodeIdx));
+ ASSIGN_POOLNODE(PoolElem, PoolNodeIdx, NodesPerTable)
+ }
+ }
+
+ //Allocate dummy L1Table
+ {
+ uint32_t PoolNodeIdx = PAGETABLE_POOL_MAX_NODES;
+ GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
+ POOL_TYPE PoolType = POOL_TYPE_AUXTTL1;
+
+ PoolElem = PageTableMgr->__GetFreePoolNode(&PoolNodeIdx, PoolType); //Recognize if Aux-L1 being allocated
+ if(PoolElem)
+ {
+ *L1Table = new GmmLib::LastLevelTable(PoolElem, PoolNodeIdx, GMM_L1_SIZE_DWORD(TTType, pGmmGlobalContext), 0); //use Aux L1_Size_DWORD
+
+ if(*L1Table)
+ {
+ if(PoolNodeIdx != PAGETABLE_POOL_MAX_NODES)
+ {
+ uint32_t PerTableNodes = (TTType == AUXTT) ? AUX_L1TABLE_SIZE_IN_POOLNODES : 1;
+ ASSIGN_POOLNODE(PoolElem, PoolNodeIdx, PerTableNodes)
+ }
+ }
+ }
+ }
+}
+
+//=============================================================================
+//
+// Function: GetL1L2TableAddr
+//
+// Desc: For given tile address, returns L1/L2 Table address if the table exists
+//
+// Parameters:
+// pAUXTT_Obj: per-device AUX-TT object. Contains AXUTT node info
+// TileAddr: Tiled Resource Virtual address
+//
+// Returns:
+// L2Table/L1Table Address
+//-----------------------------------------------------------------------------
+void GmmLib::PageTable::GetL1L2TableAddr(GMM_GFX_ADDRESS TileAddr, GMM_GFX_ADDRESS *L1TableAdr, GMM_GFX_ADDRESS *L2TableAdr)
+{
+ GMM_GFX_SIZE_T L3eIdx, L2eIdx, L1eIdx;
+ GMM_GFX_ADDRESS L3TableAdr = GMM_NO_TABLE;
+ *L2TableAdr = GMM_NO_TABLE;
+ *L1TableAdr = GMM_NO_TABLE;
+
+ L3eIdx = GMM_L3_ENTRY_IDX(TTType, TileAddr);
+ L2eIdx = GMM_L2_ENTRY_IDX(TTType, TileAddr);
+ L1eIdx = GMM_L1_ENTRY_IDX(TTType, TileAddr, pGmmGlobalContext);
+
+ __GMM_ASSERT(TTL3.L3Handle);
+ L3TableAdr = TTL3.GfxAddress;
+ if(pTTL2[L3eIdx].GetPool())
+ {
+ GmmLib::GMM_PAGETABLEPool *Pool = NULL;
+ GmmLib::LastLevelTable * pL1Tbl = NULL;
+ Pool = pTTL2[L3eIdx].GetPool();
+
+ if(Pool)
+ {
+ __GMM_ASSERT(Pool->GetNumFreeNode() != PAGETABLE_POOL_MAX_NODES);
+ __GMM_ASSERT(pTTL2[L3eIdx].GetNodeIdx() < PAGETABLE_POOL_MAX_NODES);
+ __GMM_ASSERT(Pool->GetNodeUsageAtIndex(pTTL2[L3eIdx].GetNodeIdx() / (32 * NodesPerTable)) != 0);
+
+ *L2TableAdr = Pool->GetGfxAddress() + PAGE_SIZE * (pTTL2[L3eIdx].GetNodeIdx());
+ }
+
+ pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx);
+ if(pL1Tbl && pL1Tbl->GetPool())
+ {
+ Pool = NULL;
+ Pool = pL1Tbl->GetPool();
+ if(Pool)
+ {
+ uint32_t PerTableNodes = (TTType == AUXTT) ? AUX_L1TABLE_SIZE_IN_POOLNODES : 1;
+ __GMM_ASSERT(Pool->GetNumFreeNode() != PAGETABLE_POOL_MAX_NODES);
+ __GMM_ASSERT(pL1Tbl->GetNodeIdx() < PAGETABLE_POOL_MAX_NODES);
+ __GMM_ASSERT(Pool->GetNodeUsageAtIndex(pL1Tbl->GetNodeIdx() / (32 * PerTableNodes)) != 0);
+
+ *L1TableAdr = Pool->GetGfxAddress() + PAGE_SIZE * (pL1Tbl->GetNodeIdx());
+ }
+ }
+ }
+}
+
+//=============================================================================
+//
+// Function: GetMappingType
+//
+// Desc: For given gfx address and size, returns MappingType (null/non-null or
+// Valid/Invalid) of GfxVA and first gfx address have reverse mapping
+//
+/// @param[in] GfxVA: Gfx Address whose mapping type is being queried
+/// @param[in] Size: Size of interested Gfx address range
+/// @param[out] LastAddr : 1st Gfx Address having reverse mapping type
+//
+/// @return 1/0 : for non-null/valid vs null/invalid mapping
+//-----------------------------------------------------------------------------
+uint8_t GmmLib::PageTable::GetMappingType(GMM_GFX_ADDRESS GfxVA, GMM_GFX_SIZE_T Size, GMM_GFX_ADDRESS &LastAddr)
+{
+ GMM_GFX_SIZE_T L3eIdx, L2eIdx, L1eIdx, L1EntrySize;
+ uint8_t MapType = 0; //true for non-null, false for null mapped
+ bool bFoundLastVA = false, bTerminate = false;
+ GMM_GFX_ADDRESS TileAddr = GfxVA;
+
+ L3eIdx = GMM_L3_ENTRY_IDX(TTType, GfxVA);
+ L2eIdx = GMM_L2_ENTRY_IDX(TTType, GfxVA);
+ L1eIdx = GMM_L1_ENTRY_IDX(TTType, GfxVA, pGmmGlobalContext);
+ L1EntrySize = !WA16K ? GMM_KBYTE(64) : GMM_KBYTE(16);
+
+ EnterCriticalSection(&TTLock);
+ __GMM_ASSERT(TTL3.L3Handle);
+
+#define GET_NEXT_L1TABLE(L1eIdx, L2eIdx, L3eIdx) \
+ { \
+ L1eIdx = 0; \
+ L2eIdx++; \
+ if(L2eIdx == (GMM_L2_SIZE(TTType))) \
+ { \
+ L2eIdx = 0; \
+ L3eIdx++; \
+ if(L3eIdx == (GMM_L3_SIZE(TTType))) \
+ { \
+ bTerminate = true; \
+ } \
+ } \
+ }
+
+#define GET_NEXT_L2TABLE(L1eIdx, L2eIdx, L3eIdx) \
+ { \
+ L1eIdx = L2eIdx = 0; \
+ L3eIdx++; \
+ if(L3eIdx == (GMM_L3_SIZE(TTType))) \
+ { \
+ bTerminate = true; \
+ } \
+ }
+
+ while(!(bFoundLastVA || bTerminate) && (TileAddr < GfxVA + Size))
+ {
+ if(pTTL2[L3eIdx].GetPool())
+ {
+ GmmLib::LastLevelTable *pL1Tbl = NULL;
+ pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx);
+ if(pL1Tbl && pL1Tbl->GetPool())
+ {
+ uint32_t LastBit = 0;
+ uint32_t i = static_cast<uint32_t>(L1eIdx) / 32;
+
+ while(!bFoundLastVA && i < (uint32_t)(GMM_L1_SIZE_DWORD(TTType, pGmmGlobalContext)))
+ {
+ uint32_t UsageDW = pL1Tbl->GetUsedEntries()[i++];
+ uint32_t BitNum = 31;
+ if(GfxVA == TileAddr)
+ {
+ BitNum = L1eIdx % 32;
+ MapType = ((UsageDW & __BIT(BitNum)) ? 0x1 : 0x0); //true for non-null, false for null mapped
+ UsageDW = (!MapType) ? UsageDW : ~UsageDW;
+ UsageDW = ((uint64_t)UsageDW >> (BitNum + 1)) << (BitNum + 1); // clear lsb <= BitNum
+ }
+ else
+ {
+ UsageDW = (!MapType) ? UsageDW : ~UsageDW;
+ }
+
+ if(_BitScanForward((uint32_t *)&LastBit, UsageDW)) // Gets lsb > BitNum, having reverse mapType
+ {
+ bFoundLastVA = true;
+ uint32_t NumTiles = (GfxVA == TileAddr) ? (LastBit - BitNum) : LastBit;
+ LastAddr = TileAddr + NumTiles * L1EntrySize;
+ }
+ else
+ {
+ uint32_t NumTiles = (GfxVA == TileAddr) ? (32 - BitNum) : 32;
+ TileAddr += NumTiles * L1EntrySize;
+ }
+ }
+ if(!bFoundLastVA)
+ {
+ GET_NEXT_L1TABLE(L1eIdx, L2eIdx, L3eIdx);
+ }
+ }
+ else //L2Entry is NULL
+ {
+ if(MapType) //First hit null-map
+ {
+ LastAddr = TileAddr;
+ bFoundLastVA = true;
+ }
+ else
+ {
+ GMM_GFX_SIZE_T NumTiles = GMM_L1_SIZE(TTType, pGmmGlobalContext);
+ if(GfxVA == TileAddr)
+ {
+ MapType = false;
+ NumTiles -= L1eIdx;
+ }
+ TileAddr += NumTiles * L1EntrySize;
+ GET_NEXT_L1TABLE(L1eIdx, L2eIdx, L3eIdx)
+ }
+ }
+ }
+ else //L3entry is NULL
+ {
+ if(MapType) //First hit null-map
+ {
+ LastAddr = TileAddr;
+ bFoundLastVA = true;
+ }
+ else
+ {
+ GMM_GFX_SIZE_T NumTiles = 0;
+ if(GfxVA == TileAddr)
+ {
+ MapType = false;
+ NumTiles = (GMM_L2_SIZE(TTType) - L2eIdx) * (GMM_L1_SIZE(TTType, pGmmGlobalContext) - L1eIdx);
+ }
+ else
+ {
+ NumTiles = ((GMM_L2_SIZE(TTType)) * (GMM_L1_SIZE(TTType, pGmmGlobalContext)));
+ }
+ TileAddr += NumTiles * L1EntrySize;
+ GET_NEXT_L2TABLE(L1eIdx, L2eIdx, L3eIdx)
+ }
+ }
+ }
+
+ if(!bFoundLastVA)
+ {
+ LastAddr = TileAddr;
+ }
+
+ LeaveCriticalSection(&TTLock);
+ return MapType;
+}
+
+//=============================================================================
+//
+// Function: TrackTableUsage
+//
+// Desc: For given tile address, updates Table Usage.If Table has all Nullmappings
+// then its pool node can be unassigned
+//
+// Parameters:
+// Type: Translation Table type (Aux)
+// IsL1: Is called for L1table or L2 Table
+// TileAddr: Tiled Resource Virtual address
+// NullMapped: true if given tiled adr was null mapped, otherwise false
+//
+// Returns:
+// true, if Table for given tile adr is all null mapped
+// false,if Table does not exist or has non-null mapping
+//-----------------------------------------------------------------------------
+bool GmmLib::Table::TrackTableUsage(TT_TYPE Type, bool IsL1, GMM_GFX_ADDRESS TileAdr, bool NullMapped)
+{
+ uint32_t EntryIdx;
+ uint32_t ElemNum = 0, BitNum = 0;
+
+ EntryIdx = IsL1 ? static_cast<uint32_t>(GMM_L1_ENTRY_IDX(Type, TileAdr, pGmmGlobalContext)) : static_cast<uint32_t>(GMM_L2_ENTRY_IDX(Type, TileAdr));
+
+ ElemNum = EntryIdx / (sizeof(UsedEntries[0]) * 8);
+ BitNum = EntryIdx % (sizeof(UsedEntries[0]) * 8);
+
+ if(NullMapped)
+ {
+ UsedEntries[ElemNum] &= ~(1 << BitNum);
+ }
+ else
+ {
+ UsedEntries[ElemNum] |= (1 << BitNum);
+ }
+
+ if(NullMapped)
+ {
+ int TableDWSize = IsL1 ? static_cast<int>(GMM_L1_SIZE_DWORD(Type, pGmmGlobalContext)) : static_cast<int>(GMM_L2_SIZE_DWORD(Type));
+ for(int i = 0; i < TableDWSize; i++)
+ {
+ if(UsedEntries[i])
+ {
+ return false;
+ }
+ }
+ }
+ return NullMapped ? true : false;
+}
+
+//=============================================================================
+//
+// Function: __IsTableNullMapped
+//
+// Desc: For given tile address, checks if given Table has all Nullmappings
+// then its pool node can be unassigned
+//
+// Parameters:
+// Type: Translation Table type (TR or Aux)
+// IsL1: Is called for L1table or L2 Table
+// TileAddr: Tiled Resource Virtual address
+//
+// Returns:
+// true, if Table for given tile adr is all null mapped
+// false,if Table has non-null mapping
+//-----------------------------------------------------------------------------
+bool GmmLib::Table::IsTableNullMapped(TT_TYPE Type, bool IsL1, GMM_GFX_ADDRESS TileAdr)
+{
+ GMM_UNREFERENCED_PARAMETER(TileAdr);
+ int TableDWSize = IsL1 ? static_cast<int>(GMM_L1_SIZE_DWORD(Type, pGmmGlobalContext)) : static_cast<int>(GMM_L2_SIZE_DWORD(Type));
+ for(int i = 0; i < TableDWSize; i++)
+ {
+ if(UsedEntries[i])
+ {
+ return false;
+ }
+ }
+ return true;
+}
+
+//=============================================================================
+//
+// Function: __UpdatePoolFence
+//
+// Desc: Updates AUXTTPool's or Table's BBFenceObj/value with current BB fence
+//
+// Parameters:
+// pAUXTT_Obj: per-device AUX-TT object. Contains AUXTT node info
+// Table: L1/L2 table pointer
+// L1Table: true for L1 Table, else false
+// ClearNode: if true, Fence info is cleared for table
+// false, Fence info is updated for table and pool
+//-----------------------------------------------------------------------------
+void GmmLib::Table::UpdatePoolFence(GMM_UMD_SYNCCONTEXT *UmdContext, bool ClearNode)
+{
+ if(!ClearNode)
+ {
+ //update both node and pool with current fence/handle
+ PoolElem->GetPoolBBInfo().BBQueueHandle =
+ BBInfo.BBQueueHandle = UmdContext->BBFenceObj;
+ PoolElem->GetPoolBBInfo().BBFence =
+ BBInfo.BBFence = UmdContext->BBLastFence + 1; //Save incremented fence value, since DX does it during submission
+ }
+ else
+ {
+ //Clear node fence/handle
+ BBInfo.BBQueueHandle = 0;
+ BBInfo.BBFence = 0;
+ }
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// Releases all PageTable Pool(s) existing in Linked List
+///
+/// @param[in] DeviceCallbacks pointer to device callbacks structure
+/// @return GMM_STATUS
+/////////////////////////////////////////////////////////////////////////////////////
+GMM_STATUS GmmLib::GmmPageTablePool::__DestroyPageTablePool(void *DeviceCallbacks, HANDLE hCsr)
+{
+ GMM_STATUS Status = GMM_SUCCESS;
+ GMM_DEVICE_CALLBACKS_INT *DeviceCb = static_cast<GMM_DEVICE_CALLBACKS_INT *>(DeviceCallbacks);
+
+ GMM_PAGETABLEPool *Node = this, *Next = NULL;
+ GMM_CLIENT ClientType;
+ GMM_DEVICE_DEALLOC Dealloc = {0};
+
+ //Evict/Free gpu Va is implictly done by OS when de-allocating
+ while(Node)
+ {
+ Next = Node->NextPool;
+
+ GET_GMM_CLIENT_TYPE(Node->pClientContext, ClientType);
+
+ Dealloc.Handle = Node->PoolHandle;
+ Dealloc.GfxVA = Node->PoolGfxAddress;
+ Dealloc.Priv = Node->pGmmResInfo;
+ Dealloc.hCsr = hCsr;
+
+ Status = __GmmDeviceDealloc(ClientType, DeviceCb, &Dealloc);
+
+ Node->PoolHandle = NULL;
+ Node->PoolGfxAddress = 0;
+ delete Node;
+ Node = Next;
+ }
+
+ return Status;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+/// Releases memory allocated to PageTable's Root-table
+///
+/// @return GMM_STATUS
+/////////////////////////////////////////////////////////////////////////////////////
+GMM_STATUS GmmLib::PageTable::DestroyL3Table()
+{
+ GMM_STATUS Status = GMM_SUCCESS;
+ uint8_t hr = GMM_SUCCESS;
+ GMM_CLIENT ClientType;
+ GMM_DEVICE_DEALLOC Dealloc = {0};
+
+ GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
+
+ EnterCriticalSection(&TTLock);
+
+ if(TTL3.L3Handle)
+ {
+ Dealloc.Handle = TTL3.L3Handle;
+ Dealloc.GfxVA = TTL3.GfxAddress;
+ Dealloc.Priv = TTL3.pGmmResInfo;
+ Dealloc.hCsr = PageTableMgr->hCsr;
+
+ Status = __GmmDeviceDealloc(ClientType, &PageTableMgr->DeviceCbInt, &Dealloc);
+
+ TTL3.L3Handle = NULL;
+ TTL3.GfxAddress = 0;
+ TTL3.CPUAddress = 0;
+ }
+
+ LeaveCriticalSection(&TTLock);
+ return Status;
+}
+
+#endif //!__GMM_KMD
diff --git a/Source/GmmLib/TranslationTable/GmmUmdTranslationTable.h b/Source/GmmLib/TranslationTable/GmmUmdTranslationTable.h
new file mode 100644
index 0000000..4844054
--- /dev/null
+++ b/Source/GmmLib/TranslationTable/GmmUmdTranslationTable.h
@@ -0,0 +1,531 @@
+/*==============================================================================
+Copyright(c) 2019 Intel Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files(the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and / or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+Description: This file contains the class definitions for GmmPageTablePool
+ PageTable, and low-level Tables for user-mode PageTable management,
+ that is common for both Linux and Windows.
+
+======================= end_copyright_notice ==================================*/
+#pragma once
+#include "External/Common/GmmPageTableMgr.h"
+
+#ifdef __linux__
+#include <pthread.h>
+#include <string.h>
+
+// Internal Linux version of MSDK APIs.
+static inline void InitializeCriticalSection(pthread_mutex_t *mutex)
+{
+ pthread_mutexattr_t Attr;
+ pthread_mutexattr_init(&Attr);
+ pthread_mutexattr_settype(&Attr, PTHREAD_MUTEX_RECURSIVE);
+ pthread_mutex_init(mutex, &Attr);
+}
+
+static inline void DeleteCriticalSection(pthread_mutex_t *mutex)
+{
+ pthread_mutex_destroy(mutex);
+}
+
+static inline void EnterCriticalSection(pthread_mutex_t *mutex)
+{
+ pthread_mutex_lock(mutex);
+}
+
+static inline void LeaveCriticalSection(pthread_mutex_t *mutex)
+{
+ pthread_mutex_unlock(mutex);
+}
+
+#ifndef _BitScanForwardDefined
+static inline int _BitScanForward(uint32_t *index, uint32_t mask)
+{
+ int i;
+
+ i = ffsl(mask);
+ if(i > 0)
+ {
+ *index = (uint32_t)(i - 1);
+ return i;
+ }
+ return 0;
+}
+#endif
+#endif
+
+#define GMM_L1_SIZE(TTType, pGmmLibContext) GMM_AUX_L1_SIZE(pGmmLibContext)
+#define GMM_L1_SIZE_DWORD(TTType, pGmmLibContext) GMM_AUX_L1_SIZE_DWORD(pGmmLibContext)
+#define GMM_L2_SIZE(TTType) GMM_AUX_L2_SIZE
+#define GMM_L2_SIZE_DWORD(TTType) GMM_AUX_L2_SIZE_DWORD
+#define GMM_L3_SIZE(TTType) GMM_AUX_L3_SIZE
+#define GMM_L1_ENTRY_IDX(TTType, GfxAddress, pGmmLibContext) GMM_AUX_L1_ENTRY_IDX((GfxAddress), (pGmmLibContext))
+#define GMM_L2_ENTRY_IDX(TTType, GfxAddress) GMM_AUX_L2_ENTRY_IDX((GfxAddress))
+#define GMM_L3_ENTRY_IDX(TTType, GfxAddress) GMM_AUX_L3_ENTRY_IDX((GfxAddress))
+
+#ifdef GMM_ULT
+#define GMM_L1_ENTRY_IDX_EXPORTED(TTType, GfxAddress, WA64KEx) GMM_AUX_L1_ENTRY_IDX_EXPORTED((GfxAddress), WA64KEx)
+#endif
+
+#ifdef __cplusplus
+#include "External/Common/GmmMemAllocator.hpp"
+
+//HW provides single-set of TR/Aux-TT registers for non-privileged programming
+//Engine-specific offsets are HW-updated with programmed values.
+#define GET_L3ADROFFSET(TRTT, L3AdrOffset) \
+ L3AdrOffset = 0x4200;
+
+
+#define ASSIGN_POOLNODE(Pool, NodeIdx, PerTableNodes) { \
+ (Pool)->GetNodeUsageAtIndex((NodeIdx) / (32 *(PerTableNodes))) |= __BIT(((NodeIdx) / (PerTableNodes)) % 32); \
+ (Pool)->GetNodeBBInfoAtIndex((NodeIdx)/ (PerTableNodes)) = SyncInfo(); \
+ (Pool)->GetNumFreeNode() -= (PerTableNodes); \
+ }
+
+#define DEASSIGN_POOLNODE(PageTableMgr, UmdContext, Pool, NodeIdx, PerTableNodes) { \
+ (Pool)->GetNodeUsageAtIndex((NodeIdx) / (32 * (PerTableNodes))) &= ~__BIT(((NodeIdx) / (PerTableNodes)) % 32 ); \
+ (Pool)->GetNumFreeNode() += (PerTableNodes); \
+ if((Pool)->GetNumFreeNode() == PAGETABLE_POOL_MAX_NODES) { \
+ PageTableMgr->__ReleaseUnusedPool((UmdContext)); \
+ } \
+ }
+
+namespace GmmLib
+{
+#define PAGETABLE_POOL_MAX_NODES 512 //Max. number of L2/L1 tables pool contains
+#define PAGETABLE_POOL_SIZE_IN_DWORD PAGETABLE_POOL_MAX_NODES / 32
+#define PAGETABLE_POOL_SIZE PAGETABLE_POOL_MAX_NODES * PAGE_SIZE //Pool for L2/L1 table allocation
+#define AUX_L2TABLE_SIZE_IN_POOLNODES 8 //Aux L2 is 32KB
+#define AUX_L1TABLE_SIZE_IN_POOLNODES 2 //Aux L1 is 8KB
+#define PAGETABLE_POOL_MAX_UNUSED_SIZE GMM_MBYTE(16) //Max. size of unused pool, driver keeps resident
+
+
+ //////////////////////////////////////////////////////////////////////////////////////////////
+ /// Contains functions and members for GmmPageTablePool.
+ /// PageTablePool is a Linked-list, provides common location for both Aux TT and TR-TT pages
+ /// Separate NodePool (linked-list element) kept for each PoolType, for cleaner management in
+ /// per-table size
+ /////////////////////////////////////////////////////////////////////////////////////////////
+ class GmmPageTablePool
+ {
+ private:
+ //PageTablePool allocation descriptor
+ GMM_RESOURCE_INFO* pGmmResInfo;
+ HANDLE PoolHandle;
+ GMM_GFX_ADDRESS PoolGfxAddress;
+ GMM_GFX_ADDRESS CPUAddress; //LMEM-cpuvisible adr
+
+ POOL_TYPE PoolType; //Separate Node-pools for TR-L2, TR-L1, Aux-L2, Aux-L1 usages-
+
+ //PageTablePool usage descriptors
+ int NumFreeNodes; //has value {0 to Pool_Max_nodes}
+ uint32_t* NodeUsage; //destined node state (updated during node assignment and removed based on destined state of L1/L2 Table
+ //that used the pool node)
+ //Aux-Pool node-usage tracked at every eighth/second node(for L2 vs L1)
+ //ie 1b per node for TR-table, 1b per 8-nodes for Aux-L2table, 1b per 2-nodes for AuxL1-table
+ //array size= POOL_SIZE_IN_DWORD for TR, =POOL_SIZE_IN_DWORD/8 for AuxL2, POOL_SIZE_IN_DWORD/2 for AuxL1
+
+ SyncInfo* NodeBBInfo; //BB info for pending Gpu usage of each pool node
+ //array of size MaxPoolNodes for TR, =MaxPoolNodes / 8 for Aux, MaxPoolNodes / 2 for AuxL1
+
+ SyncInfo PoolBBInfo; //BB info for Gpu usage of the Pool (most recent of pool node BB info)
+
+ GmmPageTablePool* NextPool; //Next node-Pool in the LinkedList
+ GmmClientContext *pClientContext; ///< ClientContext of the client creating this Object
+ public:
+ GmmPageTablePool() :
+ pGmmResInfo(NULL),
+ PoolHandle(),
+ PoolGfxAddress(0x0),
+ CPUAddress(0x0),
+ PoolType(POOL_TYPE_TRTTL1),
+ NumFreeNodes(PAGETABLE_POOL_MAX_NODES),
+ NodeUsage(NULL),
+ NodeBBInfo(NULL),
+ PoolBBInfo(),
+ NextPool(NULL),
+ pClientContext(NULL)
+ {
+
+ }
+ GmmPageTablePool(HANDLE hAlloc, GMM_RESOURCE_INFO* pGmmRes, GMM_GFX_ADDRESS SysMem, POOL_TYPE Type) :
+ GmmPageTablePool()
+ {
+ PoolHandle = hAlloc;
+ pGmmResInfo = pGmmRes;
+ PoolGfxAddress = SysMem;
+ CPUAddress = PoolGfxAddress;
+ NextPool = NULL;
+ NumFreeNodes = PAGETABLE_POOL_MAX_NODES;
+ PoolType = Type;
+ int DwordPoolSize = (Type == POOL_TYPE_AUXTTL1) ? PAGETABLE_POOL_SIZE_IN_DWORD / AUX_L1TABLE_SIZE_IN_POOLNODES
+ : (Type == POOL_TYPE_AUXTTL2) ? PAGETABLE_POOL_SIZE_IN_DWORD / AUX_L2TABLE_SIZE_IN_POOLNODES
+ : PAGETABLE_POOL_SIZE_IN_DWORD;
+ NodeUsage = new uint32_t[DwordPoolSize]();
+ NodeBBInfo = new SyncInfo[DwordPoolSize * 32]();
+ if (pGmmResInfo)
+ {
+ pClientContext = pGmmResInfo->GetGmmClientContext();
+ }
+ }
+ GmmPageTablePool(HANDLE hAlloc, GMM_RESOURCE_INFO* pGmmRes, GMM_GFX_ADDRESS GfxAdr, GMM_GFX_ADDRESS CPUAdr, POOL_TYPE Type) :
+ GmmPageTablePool(hAlloc, pGmmRes, GfxAdr, Type)
+ {
+ CPUAddress = (CPUAdr != GfxAdr) ? CPUAdr : GfxAdr;
+ }
+ ~GmmPageTablePool()
+ {
+ delete[] NodeUsage;
+ delete[] NodeBBInfo;
+ }
+
+ GmmPageTablePool* InsertInList(GmmPageTablePool* NewNode)
+ {
+ GmmPageTablePool *Node = this;
+ while (Node->NextPool)
+ {
+ Node = Node->NextPool;
+ }
+ Node->NextPool = NewNode;
+ return Node->NextPool;
+ }
+
+ GmmPageTablePool* InsertInListAtBegin(GmmPageTablePool* NewNode)
+ {
+ GmmPageTablePool *Node = this;
+ NewNode->NextPool = Node;
+ return NewNode;
+ }
+
+ GmmPageTablePool* &GetNextPool() { return NextPool; }
+ HANDLE& GetPoolHandle() { return PoolHandle; }
+ POOL_TYPE& GetPoolType() { return PoolType; }
+ int& GetNumFreeNode() { return NumFreeNodes; }
+ SyncInfo& GetPoolBBInfo() { return PoolBBInfo; }
+ uint32_t& GetNodeUsageAtIndex(int j) { return NodeUsage[j]; }
+ SyncInfo& GetNodeBBInfoAtIndex(int j) { return NodeBBInfo[j]; }
+ GMM_GFX_ADDRESS GetGfxAddress() { return PoolGfxAddress; }
+ GMM_GFX_ADDRESS GetCPUAddress() { return CPUAddress; }
+ GMM_RESOURCE_INFO* &GetGmmResInfo() { return pGmmResInfo; }
+ bool IsPoolInUse(SyncInfo BBInfo) {
+ if (NumFreeNodes < PAGETABLE_POOL_MAX_NODES ||
+ (PoolBBInfo.BBQueueHandle == BBInfo.BBQueueHandle &&
+ PoolBBInfo.BBFence == BBInfo.BBFence + 1)) //Pool will be used by next BB submission, freeing it will cause page fault
+ {
+ return true;
+ }
+ return false;
+ }
+ bool __IsUnusedTRTTPoolOverLimit(GMM_GFX_SIZE_T * OverLimitSize);
+ void ClearBBReference(void * BBQHandle);
+ GMM_STATUS __DestroyPageTablePool(void * DeviceCallbacks,HANDLE hCsr);
+ };
+
+ //////////////////////////////////////////////////////////////////////////////////////////////
+ /// Contains functions and members for Table.
+ /// Table defines basic building block for tables at different page-table levels
+ /////////////////////////////////////////////////////////////////////////////////////////////
+ class Table
+ {
+ protected:
+ GMM_PAGETABLEPool *PoolElem; //L2 Pool ptr different for L2Tables when Pool_nodes <512
+ int PoolNodeIdx; //pool node idx used for L2 Table
+ SyncInfo BBInfo; //BB Handle/fence using Table
+ uint32_t* UsedEntries; //Tracks which L1/L2 entries are being used
+ //array size GMM_L1_SIZE_DWORD(TT-type) for LastLevelTable, MidLeveltable(??)
+ //array of 1024/32=32 DWs for TR-table, 4096/32 =512 for Aux-Table
+ public:
+ Table() :
+ PoolElem(NULL),
+ PoolNodeIdx(),
+ BBInfo(),
+ UsedEntries(NULL)
+ {
+ }
+ int& GetNodeIdx() { return PoolNodeIdx; }
+ GmmPageTablePool* &GetPool() { return PoolElem; }
+ GMM_GFX_ADDRESS GetCPUAddress() { return (PoolElem->GetCPUAddress() + (PoolNodeIdx * PAGE_SIZE)); }
+ SyncInfo& GetBBInfo() { return BBInfo; }
+ uint32_t* &GetUsedEntries() { return UsedEntries; }
+ bool TrackTableUsage(TT_TYPE Type, bool IsL1, GMM_GFX_ADDRESS TileAdr, bool NullMapped);
+ bool IsTableNullMapped(TT_TYPE Type, bool IsL1, GMM_GFX_ADDRESS TileAdr);
+ void UpdatePoolFence(GMM_UMD_SYNCCONTEXT * UmdContext, bool ClearNode);
+ };
+
+ //////////////////////////////////////////////////////////////////////////////////////////////
+ /// Contains functions and members for LastLevelTable.
+ /// LastLevelTable defines leaf level tables in multi-level pageTable structure
+ /////////////////////////////////////////////////////////////////////////////////////////////
+ class LastLevelTable : public Table
+ {
+ private:
+ uint32_t L2eIdx;
+ LastLevelTable *pNext;
+
+ public:
+ LastLevelTable() : Table(),
+ L2eIdx() //Pass in Aux vs TR table's GMM_L2_SIZE and initialize L2eIdx?
+ {
+ pNext = NULL;
+ }
+
+ LastLevelTable(GMM_PAGETABLEPool* Elem, int NodeIdx, int DwordL1e, int L2eIndex)
+ : LastLevelTable()
+ {
+ PoolElem = Elem;
+ PoolNodeIdx = NodeIdx;
+ BBInfo = Elem->GetNodeBBInfoAtIndex(NodeIdx);
+ L2eIdx = L2eIndex;
+ pNext = NULL;
+ UsedEntries = new uint32_t[DwordL1e]();
+ }
+ ~LastLevelTable()
+ {
+ delete[] UsedEntries;
+ }
+
+ int GetL2eIdx() {
+ return L2eIdx;
+ }
+
+ LastLevelTable* &Next() {
+ return pNext;
+ }
+ };
+
+ //////////////////////////////////////////////////////////////////////////////////////////////
+ /// Contains functions and members for MidLevelTable.
+ /// MidLevelTable defines secondary level tables in multi-level pageTable structure
+ /////////////////////////////////////////////////////////////////////////////////////////////
+ class MidLevelTable : public Table
+ {
+ private:
+ LastLevelTable *pTTL1; //linked list of L1 tables
+
+ public:
+ MidLevelTable() :Table()
+ {
+ pTTL1 = NULL;
+ }
+ MidLevelTable(GMM_PAGETABLEPool *Pool, int NodeIdx, SyncInfo Info) : MidLevelTable()
+ {
+ PoolElem = Pool;
+ BBInfo = Info;
+ PoolNodeIdx = NodeIdx;
+ }
+ ~MidLevelTable()
+ {
+ if (pTTL1)
+ {
+ LastLevelTable* item = pTTL1;
+
+ while (item)
+ {
+ LastLevelTable* nextItem = item->Next();
+ delete item;
+ item = nextItem;
+ }
+
+ pTTL1 = NULL;
+ }
+ }
+ LastLevelTable* GetL1Table(GMM_GFX_SIZE_T L2eIdx, LastLevelTable** Prev = NULL)
+ {
+ LastLevelTable* pL1Tbl = pTTL1;
+ LastLevelTable* PrevL1Tbl = NULL;
+
+ while (pL1Tbl)
+ {
+ if (pL1Tbl->GetL2eIdx() == L2eIdx)
+ {
+ break;
+ }
+ PrevL1Tbl = pL1Tbl;
+ pL1Tbl = pL1Tbl->Next();
+ }
+
+ //if requested, save previous node in linked-list
+ if (Prev)
+ {
+ *Prev = PrevL1Tbl;
+ }
+ return pL1Tbl;
+ }
+ void InsertInList(LastLevelTable* pL1Tbl)
+ {
+ LastLevelTable* Prev = pTTL1;
+
+ //Insert at end
+ while (Prev && Prev->Next())
+ {
+ Prev = Prev->Next();
+ }
+
+ if (Prev)
+ {
+ Prev->Next() = pL1Tbl;
+ }
+ else
+ {
+ pTTL1 = pL1Tbl;
+ }
+ }
+ void DeleteFromList(LastLevelTable* pL1Tbl, LastLevelTable* PrevL1Tbl)
+ {
+ //Save next L1Table in list, before deleting current one
+ if (pL1Tbl)
+ {
+ if (PrevL1Tbl)
+ {
+ PrevL1Tbl->Next() = pL1Tbl->Next();
+ }
+ else
+ {
+ pTTL1 = pL1Tbl->Next();
+ }
+
+ delete pL1Tbl;
+ }
+ }
+ };
+
+ /////////////////////////////////////////////////////
+ /// Contains functions and members for PageTable.
+ /// PageTable defines multi-level pageTable
+ /////////////////////////////////////////////////////
+ class PageTable :
+ public GmmMemAllocator
+ {
+ protected:
+ const TT_TYPE TTType; //PageTable is AuxTT
+ const int NodesPerTable; //Aux L2/L3 has 32KB size, Aux L1 has 4KB -can't use as selector for PageTable is AuxTT
+ // 1 node for TR-table, 8 nodes for Aux-Table L2, 2 nodes for Aux-table L1
+ //Root Table structure
+ struct
+ {
+ GMM_RESOURCE_INFO* pGmmResInfo;
+ HANDLE L3Handle;
+ GMM_GFX_ADDRESS GfxAddress; //L3 Table Adr CPU equivalent GPU addr
+ GMM_GFX_ADDRESS CPUAddress; //LMEM-cpuvisible adr
+ bool NeedRegisterUpdate; //True @ L3 allocation, False when L3AdrRegWrite done
+ SyncInfo BBInfo;
+ } TTL3;
+
+ MidLevelTable* pTTL2; //array of L2-Tables
+
+ public:
+#ifdef _WIN32
+ CRITICAL_SECTION TTLock; //synchronized access of PageTable obj
+#elif defined __linux__
+ pthread_mutex_t TTLock;
+#endif
+
+ GmmPageTableMgr* PageTableMgr;
+ GmmClientContext *pClientContext;
+
+ PageTable(int Size, int NumL3e, TT_TYPE flag) :
+ NodesPerTable(Size / PAGE_SIZE),
+ TTType(flag)
+ {
+ memset(&TTL3, 0, sizeof(TTL3));
+
+ InitializeCriticalSection(&TTLock);
+
+ pTTL2 = new MidLevelTable[NumL3e];
+ }
+
+ ~PageTable()
+ {
+ delete[] pTTL2;
+
+ DeleteCriticalSection(&TTLock);
+ }
+
+ GMM_GFX_ADDRESS GetL3Address() { return TTL3.GfxAddress; }
+ bool &GetRegisterStatus() { return TTL3.NeedRegisterUpdate; }
+ GMM_STATUS AllocateL3Table(uint32_t L3TableSize, uint32_t L3AddrAlignment);
+ GMM_STATUS DestroyL3Table();
+ void AllocateL1L2Table(GMM_GFX_ADDRESS TileAddr, GMM_GFX_ADDRESS * L1TableAdr, GMM_GFX_ADDRESS * L2TableAdr);
+ void AllocateDummyTables(GmmLib::Table **L2Table, GmmLib::Table **L1Table);
+ void GetL1L2TableAddr(GMM_GFX_ADDRESS TileAddr, GMM_GFX_ADDRESS * L1TableAdr, GMM_GFX_ADDRESS* L2TableAdr);
+ uint8_t GetMappingType(GMM_GFX_ADDRESS GfxVA, GMM_GFX_SIZE_T Size, GMM_GFX_ADDRESS& LastAddr);
+ HANDLE GetL3Handle() { return TTL3.L3Handle; }
+ };
+
+ //////////////////////////////////////////////////////////////////////////////////////////////
+ /// Contains functions and members for AuxTable.
+ /// AuxTable defines PageTable for translating VA->AuxVA, ie defines page-walk to get address
+ /// of CCS-cacheline containing auxiliary data (compression tag, etc) for some resource
+ /////////////////////////////////////////////////////////////////////////////////////////////
+ class AuxTable : public PageTable
+ {
+ public:
+ const int L1Size;
+ Table* NullL2Table;
+ Table* NullL1Table;
+ GMM_GFX_ADDRESS NullCCSTile;
+ AuxTable() : PageTable(8 * PAGE_SIZE, GMM_AUX_L3_SIZE, TT_TYPE::AUXTT),
+ L1Size(2 * PAGE_SIZE)
+ {
+ NullL2Table = nullptr;
+ NullL1Table = nullptr;
+ NullCCSTile = 0;
+ }
+ GMM_STATUS InvalidateTable(GMM_UMD_SYNCCONTEXT * UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T Size, uint8_t DoNotWait);
+
+ GMM_STATUS MapValidEntry(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T BaseSize,
+ GMM_RESOURCE_INFO* BaseResInfo, GMM_GFX_ADDRESS AuxVA, GMM_RESOURCE_INFO* AuxResInfo, uint64_t PartialData, uint8_t DoNotWait);
+
+ GMM_STATUS MapNullCCS(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T Size, uint64_t PartialL1e, uint8_t DoNotWait);
+
+ GMM_AUXTTL1e CreateAuxL1Data(GMM_RESOURCE_INFO* BaseResInfo);
+ GMM_GFX_ADDRESS GMM_INLINE __GetCCSCacheline(GMM_RESOURCE_INFO* BaseResInfo, GMM_GFX_ADDRESS BaseAdr, GMM_RESOURCE_INFO* AuxResInfo,
+ GMM_GFX_ADDRESS AuxVA, GMM_GFX_SIZE_T AdrOffset);
+
+ };
+
+typedef struct _GMM_DEVICE_ALLOC {
+ uint32_t Size;
+ uint32_t Alignment;
+ HANDLE Handle;
+ GMM_GFX_ADDRESS GfxVA;
+ GMM_GFX_ADDRESS CPUVA;
+ void * Priv;
+ HANDLE hCsr;
+} GMM_DEVICE_ALLOC;
+
+typedef struct _GMM_DEVICE_DEALLOC {
+ HANDLE Handle;
+ GMM_GFX_ADDRESS GfxVA;
+ void * Priv;
+ HANDLE hCsr;
+} GMM_DEVICE_DEALLOC;
+
+GMM_STATUS __GmmDeviceAlloc(GmmClientContext *pClientContext,
+ GMM_DEVICE_CALLBACKS_INT *pDeviceCbInt,
+ GMM_DEVICE_ALLOC *pAlloc);
+
+GMM_STATUS __GmmDeviceDealloc(GMM_CLIENT ClientType,
+ GMM_DEVICE_CALLBACKS_INT *DeviceCb,
+ GMM_DEVICE_DEALLOC *pDealloc);
+
+}
+#endif // #ifdef __cplusplus
+
diff --git a/Source/GmmLib/ULT/CMakeLists.txt b/Source/GmmLib/ULT/CMakeLists.txt
index 6f51725..e5c2f21 100644
--- a/Source/GmmLib/ULT/CMakeLists.txt
+++ b/Source/GmmLib/ULT/CMakeLists.txt
@@ -1,141 +1,152 @@
-# Copyright(c) 2017 Intel Corporation
-
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files(the "Software"),
-# to deal in the Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish, distribute, sublicense,
-# and / or sell copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following conditions:
-
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-# OTHER DEALINGS IN THE SOFTWARE.
-
-set (EXE_NAME GMMULT)
-
-set(GMMULT_HEADERS
- GmmCachePolicyULT.h
- GmmCommonULT.h
- GmmGen10CachePolicyULT.h
- GmmGen10ResourceULT.h
- GmmGen11CachePolicyULT.h
- GmmGen11ResourceULT.h
- GmmGen12ResourceULT.h
- GmmGen12CachePolicyULT.h
- GmmGen9CachePolicyULT.h
- GmmGen9ResourceULT.h
- GmmResourceULT.h
- stdafx.h
- targetver.h
- )
-
-set(GMMULT_SOURCES
- GmmCachePolicyULT.cpp
- GmmCommonULT.cpp
- GmmGen10CachePolicyULT.cpp
- GmmGen10ResourceULT.cpp
- GmmGen11CachePolicyULT.cpp
- GmmGen12CachePolicyULT.cpp
- GmmGen11ResourceULT.cpp
- GmmGen12ResourceULT.cpp
- GmmGen9CachePolicyULT.cpp
- GmmGen9ResourceULT.cpp
- GmmResourceCpuBltULT.cpp
- GmmResourceULT.cpp
- googletest/src/gtest-all.cc
- GmmULT.cpp
-)
-
-source_group("Source Files\\Cache Policy" FILES
- GmmCachePolicyULT.cpp
- GmmGen9CachePolicyULT.cpp
- GmmGen10CachePolicyULT.cpp
- GmmGen11CachePolicyULT.cpp
- GmmGen12CachePolicyULT.cpp
- )
-
-source_group("Source Files\\Resource" FILES
- GmmGen10ResourceULT.cpp
- GmmGen11ResourceULT.cpp
- GmmGen12ResourceULT.cpp
- GmmGen9ResourceULT.cpp
- GmmResourceCpuBltULT.cpp
- GmmResourceULT.cpp
- )
-
-source_group("Header Files\\Cache Policy" FILES
- GmmCachePolicyULT.h
- GmmGen10CachePolicyULT.h
- GmmGen11CachePolicyULT.h
- GmmGen12CachePolicyULT.h
- GmmGen9CachePolicyULT.h
- )
-
-source_group("Header Files\\Resource" FILES
- GmmGen12ResourceULT.h
- GmmGen11ResourceULT.h
- GmmGen10ResourceULT.h
- GmmGen9ResourceULT.h
- GmmResourceULT.h
- )
-
-source_group("gtest" FILES
- googletest/gtest/gtest.h
- googletest/src/gtest-all.cc
- )
-
-include_directories(BEFORE ./)
-
-include_directories(BEFORE ${PROJECT_SOURCE_DIR})
-
-include_directories(
- googletest
- googletest/gtest
- ${BS_DIR_INC}/umKmInc
- ${BS_DIR_INC}
- ${BS_DIR_GMMLIB}/inc
- ${BS_DIR_INC}/common
- )
-
-macro(GmmLibULTSetTargetConfig ultTarget)
- if (TARGET ${ultTarget})
- set_property(TARGET ${ultTarget} APPEND PROPERTY COMPILE_DEFINITIONS
- $<$<CONFIG:Release>: _RELEASE>
- $<$<CONFIG:ReleaseInternal>: _RELEASE_INTERNAL>
- $<$<CONFIG:Debug>: _DEBUG>
- )
- endif()
-
-endmacro()
-
-add_executable(${EXE_NAME} ${GMMULT_HEADERS} ${GMMULT_SOURCES})
-
-GmmLibULTSetTargetConfig(${EXE_NAME})
-
-set_property(TARGET ${EXE_NAME} APPEND PROPERTY COMPILE_DEFINITIONS __GMM GMM_LIB_DLL __UMD)
-
-if(NOT TARGET igfx_gmmumd_dll)
- add_subdirectory("${BS_DIR_GMMLIB}" "${CMAKE_BINARY_DIR}/gmmlib/ult")
-endif()
-target_link_libraries(${EXE_NAME} igfx_gmmumd_dll)
-
-target_link_libraries(${EXE_NAME}
- pthread
- dl
-)
-
-add_custom_target(Run_ULT ALL DEPENDS GMMULT)
-
-add_custom_command(
- TARGET Run_ULT
- POST_BUILD
- COMMAND echo running ULTs
- COMMAND "${CMAKE_COMMAND}" -E env "LD_LIBRARY_PATH=$<TARGET_FILE_DIR:igfx_gmmumd_dll>" ${CMAKE_CFG_INTDIR}/${EXE_NAME} --gtest_filter=CTest*
-)
\ No newline at end of file
+# Copyright(c) 2017 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files(the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and / or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+set (EXE_NAME GMMULT)
+
+set(GMMULT_HEADERS
+ GmmCachePolicyULT.h
+ GmmCommonULT.h
+ GmmGen10CachePolicyULT.h
+ GmmGen10ResourceULT.h
+ GmmGen11CachePolicyULT.h
+ GmmGen11ResourceULT.h
+ GmmGen12ResourceULT.h
+ GmmGen12CachePolicyULT.h
+ GmmGen9CachePolicyULT.h
+ GmmGen9ResourceULT.h
+ GmmResourceULT.h
+ GmmAuxTableULT.h
+ stdafx.h
+ targetver.h
+ )
+
+set(GMMULT_SOURCES
+ GmmCachePolicyULT.cpp
+ GmmCommonULT.cpp
+ GmmGen10CachePolicyULT.cpp
+ GmmGen10ResourceULT.cpp
+ GmmGen11CachePolicyULT.cpp
+ GmmGen12CachePolicyULT.cpp
+ GmmGen11ResourceULT.cpp
+ GmmGen12ResourceULT.cpp
+ GmmGen9CachePolicyULT.cpp
+ GmmGen9ResourceULT.cpp
+ GmmResourceCpuBltULT.cpp
+ GmmResourceULT.cpp
+ GmmAuxTableULT.cpp
+ googletest/src/gtest-all.cc
+ GmmULT.cpp
+)
+
+source_group("Source Files\\Cache Policy" FILES
+ GmmCachePolicyULT.cpp
+ GmmGen9CachePolicyULT.cpp
+ GmmGen10CachePolicyULT.cpp
+ GmmGen11CachePolicyULT.cpp
+ GmmGen12CachePolicyULT.cpp
+ )
+
+source_group("Source Files\\Resource" FILES
+ GmmGen10ResourceULT.cpp
+ GmmGen11ResourceULT.cpp
+ GmmGen12ResourceULT.cpp
+
+ GmmGen9ResourceULT.cpp
+ GmmResourceCpuBltULT.cpp
+ GmmResourceULT.cpp
+ )
+
+source_group("Source Files\\TranslationTable" FILES
+ GmmAuxTableULT.cpp
+ )
+
+source_group("Header Files\\TranslationTable" FILES
+ GmmAuxTableULT.h
+ )
+
+source_group("Header Files\\Cache Policy" FILES
+ GmmCachePolicyULT.h
+ GmmGen10CachePolicyULT.h
+ GmmGen11CachePolicyULT.h
+ GmmGen12CachePolicyULT.h
+ GmmGen9CachePolicyULT.h
+ )
+
+source_group("Header Files\\Resource" FILES
+ GmmGen10ResourceULT.h
+ GmmGen11ResourceULT.h
+ GmmGen12ResourceULT.h
+ GmmGen9ResourceULT.h
+ GmmResourceULT.h
+ )
+
+source_group("gtest" FILES
+ googletest/gtest/gtest.h
+ googletest/src/gtest-all.cc
+ )
+
+include_directories(BEFORE ./)
+
+include_directories(BEFORE ${PROJECT_SOURCE_DIR})
+
+include_directories(
+ googletest
+ googletest/gtest
+ ${BS_DIR_INC}/umKmInc
+ ${BS_DIR_INC}
+ ${BS_DIR_GMMLIB}/inc
+ ${BS_DIR_INC}/common
+ )
+
+macro(GmmLibULTSetTargetConfig ultTarget)
+ if (TARGET ${ultTarget})
+ set_property(TARGET ${ultTarget} APPEND PROPERTY COMPILE_DEFINITIONS
+ $<$<CONFIG:Release>: _RELEASE>
+ $<$<CONFIG:ReleaseInternal>: _RELEASE_INTERNAL>
+ $<$<CONFIG:Debug>: _DEBUG>
+ )
+ endif()
+
+endmacro()
+
+add_executable(${EXE_NAME} ${GMMULT_HEADERS} ${GMMULT_SOURCES})
+
+GmmLibULTSetTargetConfig(${EXE_NAME})
+
+set_property(TARGET ${EXE_NAME} APPEND PROPERTY COMPILE_DEFINITIONS __GMM GMM_LIB_DLL __UMD)
+
+if(NOT TARGET igfx_gmmumd_dll)
+ add_subdirectory("${BS_DIR_GMMLIB}" "${CMAKE_BINARY_DIR}/gmmlib/ult")
+endif()
+target_link_libraries(${EXE_NAME} igfx_gmmumd_dll)
+
+target_link_libraries(${EXE_NAME}
+ pthread
+ dl
+)
+
+add_custom_target(Run_ULT ALL DEPENDS GMMULT)
+
+add_custom_command(
+ TARGET Run_ULT
+ POST_BUILD
+ COMMAND echo running ULTs
+ COMMAND "${CMAKE_COMMAND}" -E env "LD_LIBRARY_PATH=$<TARGET_FILE_DIR:igfx_gmmumd_dll>" ${CMAKE_CFG_INTDIR}/${EXE_NAME} --gtest_filter=CTest*
+)
diff --git a/Source/GmmLib/ULT/GmmAuxTableULT.cpp b/Source/GmmLib/ULT/GmmAuxTableULT.cpp
new file mode 100644
index 0000000..5bb8b81
--- /dev/null
+++ b/Source/GmmLib/ULT/GmmAuxTableULT.cpp
@@ -0,0 +1,265 @@
+/*==============================================================================
+Copyright(c) 2019 Intel Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files(the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and / or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+============================================================================*/
+
+#ifdef __linux__
+
+#include "GmmAuxTableULT.h"
+
+using namespace std;
+using namespace GmmLib;
+
+static GMM_DEVICE_CALLBACKS_INT DeviceCBInt;
+
+CTestAuxTable::CTestAuxTable()
+{
+}
+
+CTestAuxTable::~CTestAuxTable()
+{
+}
+
+int CTestAuxTable::allocCB(void *bufMgr, size_t size, size_t alignment, void **bo, void **cpuAddr, uint64_t *gpuAddr)
+{
+ if(bufMgr != (void *)0xdeadbeef)
+ return -1;
+
+ if(!bo || !cpuAddr || !gpuAddr)
+ return -2;
+
+ *cpuAddr = aligned_alloc(alignment, ALIGN(size, alignment));
+ if(!*cpuAddr)
+ return -3;
+
+ *bo = *cpuAddr;
+ *gpuAddr = (uint64_t)*cpuAddr;
+
+ return 0;
+}
+
+void CTestAuxTable::freeCB(void *bo)
+{
+ ASSERT_TRUE(bo != NULL);
+
+ free(bo);
+}
+
+void CTestAuxTable::waitFromCpuCB(void *bo)
+{
+}
+
+void CTestAuxTable::SetUpTestCase()
+{
+ GfxPlatform.eProductFamily = IGFX_TIGERLAKE_LP;
+ GfxPlatform.eRenderCoreFamily = IGFX_GEN12_CORE;
+
+ DeviceCBInt.pBufMgr = (void *)0xdeadbeef;
+ DeviceCBInt.DevCbPtrs_.pfnAllocate = CTestAuxTable::allocCB;
+ DeviceCBInt.DevCbPtrs_.pfnDeallocate = CTestAuxTable::freeCB;
+ DeviceCBInt.DevCbPtrs_.pfnWaitFromCpu = CTestAuxTable::waitFromCpuCB;
+
+ if(GfxPlatform.eProductFamily == IGFX_UNKNOWN ||
+ GfxPlatform.eRenderCoreFamily == IGFX_UNKNOWN_CORE)
+ {
+ GfxPlatform.eProductFamily = IGFX_BROADWELL;
+ GfxPlatform.eRenderCoreFamily = IGFX_GEN8_CORE;
+ }
+
+ AllocateAdapterInfo();
+ if(pGfxAdapterInfo)
+ {
+ pGfxAdapterInfo->SkuTable.FtrE2ECompression = true;
+ pGfxAdapterInfo->SkuTable.FtrLinearCCS = true;
+ }
+
+ CommonULT::SetUpTestCase();
+}
+
+void CTestAuxTable::TearDownTestCase()
+{
+ CommonULT::TearDownTestCase();
+}
+
+TEST_F(CTestAuxTable, TestUpdateAuxTableCompressedSurface)
+{
+ GmmPageTableMgr *mgr = pGmmULTClientContext->CreatePageTblMgrObject(&DeviceCBInt, TT_TYPE::AUXTT);
+
+ ASSERT_TRUE(mgr != NULL);
+
+ Surface *surf = new Surface(7680, 4320);
+
+ ASSERT_TRUE(surf != NULL && surf->init());
+
+ GMM_DDI_UPDATEAUXTABLE updateReq = {0};
+
+ updateReq.BaseResInfo = surf->getGMMResourceInfo();
+ updateReq.BaseGpuVA = surf->getGfxAddress(GMM_PLANE_Y);
+ updateReq.Map = 1;
+
+ GMM_STATUS res = mgr->UpdateAuxTable(&updateReq);
+ ASSERT_TRUE(res == GMM_SUCCESS);
+
+ delete surf;
+ pGmmULTClientContext->DestroyPageTblMgrObject(mgr);
+}
+
+TEST_F(CTestAuxTable, DISABLED_TestUpdateAuxTableNonCompressedSurface)
+{
+ GmmPageTableMgr *mgr = pGmmULTClientContext->CreatePageTblMgrObject(&DeviceCBInt, TT_TYPE::AUXTT);
+
+ ASSERT_TRUE(mgr != NULL);
+
+ Surface *surf = new Surface(7680, 4320, false);
+
+ ASSERT_TRUE(surf != NULL && surf->init());
+
+ GMM_DDI_UPDATEAUXTABLE updateReq = {0};
+
+ memset(&updateReq, 0, sizeof(GMM_DDI_UPDATEAUXTABLE));
+
+ updateReq.BaseResInfo = surf->getGMMResourceInfo();
+ updateReq.BaseGpuVA = surf->getGfxAddress(GMM_PLANE_Y);
+ updateReq.Map = 1;
+
+ GMM_STATUS res = mgr->UpdateAuxTable(&updateReq);
+ ASSERT_TRUE(res != GMM_SUCCESS);
+
+ delete surf;
+ pGmmULTClientContext->DestroyPageTblMgrObject(mgr);
+}
+
+TEST_F(CTestAuxTable, TestInvalidateAuxTable)
+{
+ GmmPageTableMgr *mgr = pGmmULTClientContext->CreatePageTblMgrObject(&DeviceCBInt, TT_TYPE::AUXTT);
+
+ ASSERT_TRUE(mgr != NULL);
+
+ Surface *surf = new Surface(7680, 4320);
+
+ ASSERT_TRUE(surf != NULL && surf->init());
+
+ GMM_DDI_UPDATEAUXTABLE updateReq = {0};
+
+ updateReq.BaseResInfo = surf->getGMMResourceInfo();
+ updateReq.BaseGpuVA = surf->getGfxAddress(GMM_PLANE_Y);
+ updateReq.Map = 1;
+
+ GMM_STATUS res = mgr->UpdateAuxTable(&updateReq);
+ ASSERT_TRUE(res == GMM_SUCCESS);
+
+ memset(&updateReq, 0, sizeof(updateReq));
+
+ updateReq.BaseResInfo = surf->getGMMResourceInfo();
+ updateReq.Map = 0;
+
+ res = mgr->UpdateAuxTable(&updateReq);
+ ASSERT_TRUE(res == GMM_SUCCESS);
+
+ delete surf;
+ pGmmULTClientContext->DestroyPageTblMgrObject(mgr);
+}
+
+TEST_F(CTestAuxTable, DISABLED_TestUpdateAuxTableStress)
+{
+ const int num_surf = 1000;
+ Surface * surfaces[num_surf];
+ Surface * surf;
+ int i;
+
+ GmmPageTableMgr *mgr = pGmmULTClientContext->CreatePageTblMgrObject(&DeviceCBInt, TT_TYPE::AUXTT);
+
+ ASSERT_TRUE(mgr != NULL);
+
+ for(i = 0; i < num_surf; i++)
+ {
+ surf = new Surface(7680, 4320);
+ surfaces[i] = surf;
+
+ ASSERT_TRUE(surf != NULL && surf->init());
+
+ GMM_DDI_UPDATEAUXTABLE updateReq = {0};
+
+ updateReq.BaseResInfo = surf->getGMMResourceInfo();
+ updateReq.BaseGpuVA = surf->getGfxAddress(GMM_PLANE_Y);
+ updateReq.Map = 1;
+
+ mgr->UpdateAuxTable(&updateReq);
+ }
+
+ for(i = 0; i < num_surf; i++)
+ {
+ surf = surfaces[i];
+ delete surf;
+ }
+
+ pGmmULTClientContext->DestroyPageTblMgrObject(mgr);
+}
+
+TEST_F(CTestAuxTable, TestAuxTableContent)
+{
+ GmmPageTableMgr *mgr = pGmmULTClientContext->CreatePageTblMgrObject(&DeviceCBInt, TT_TYPE::AUXTT);
+
+ ASSERT_TRUE(mgr != NULL);
+
+ Surface *surf = new Surface(720, 480);
+
+ ASSERT_TRUE(surf != NULL && surf->init());
+
+ GMM_DDI_UPDATEAUXTABLE updateReq = {0};
+
+ updateReq.BaseResInfo = surf->getGMMResourceInfo();
+ updateReq.BaseGpuVA = surf->getGfxAddress(GMM_PLANE_Y);
+ updateReq.Map = 1;
+
+ GMM_STATUS res = mgr->UpdateAuxTable(&updateReq);
+ ASSERT_TRUE(res == GMM_SUCCESS);
+
+ Walker *ywalker = new Walker(surf->getGfxAddress(GMM_PLANE_Y),
+ surf->getAuxGfxAddress(GMM_AUX_CCS),
+ mgr->GetAuxL3TableAddr());
+
+ for(size_t i = 0; i < surf->getSurfaceSize(GMM_PLANE_Y); i++)
+ {
+ GMM_GFX_ADDRESS addr = surf->getGfxAddress(GMM_PLANE_Y) + i;
+ GMM_GFX_ADDRESS val = ywalker->walk(addr);
+ GMM_GFX_ADDRESS expected = ywalker->expected(addr);
+ ASSERT_EQ(expected, val);
+ }
+
+ Walker *uvwalker = new Walker(surf->getGfxAddress(GMM_PLANE_U),
+ surf->getAuxGfxAddress(GMM_AUX_UV_CCS),
+ mgr->GetAuxL3TableAddr());
+
+ for(size_t i = 0; i < surf->getSurfaceSize(GMM_PLANE_U); i++)
+ {
+ GMM_GFX_ADDRESS addr = surf->getGfxAddress(GMM_PLANE_U) + i;
+ GMM_GFX_ADDRESS val = uvwalker->walk(addr);
+ GMM_GFX_ADDRESS expected = uvwalker->expected(addr);
+ ASSERT_EQ(expected, val);
+ }
+
+ delete uvwalker;
+ delete ywalker;
+ delete surf;
+ pGmmULTClientContext->DestroyPageTblMgrObject(mgr);
+}
+
+#endif /* __linux__ */
diff --git a/Source/GmmLib/ULT/GmmAuxTableULT.h b/Source/GmmLib/ULT/GmmAuxTableULT.h
new file mode 100644
index 0000000..3a40f1d
--- /dev/null
+++ b/Source/GmmLib/ULT/GmmAuxTableULT.h
@@ -0,0 +1,252 @@
+/*==============================================================================
+Copyright(c) 2019 Intel Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files(the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and / or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+============================================================================*/
+
+#pragma once
+
+#ifdef __linux__
+
+#ifndef _ISOC11_SOURCE
+#define _ISOC11_SOURCE 1
+#endif
+
+#include "GmmGen10ResourceULT.h"
+#include <stdlib.h>
+#include <malloc.h>
+
+#ifndef ALIGN
+#define ALIGN(v, a) (((v) + ((a)-1)) & ~((a)-1))
+#endif
+
+class CTestAuxTable : public CTestGen10Resource
+{
+
+public:
+ static void SetUpTestCase();
+ static void TearDownTestCase();
+
+ CTestAuxTable();
+ ~CTestAuxTable();
+
+ static int allocCB(void *bufMgr, size_t size, size_t alignment, void **bo, void **cpuAddr, uint64_t *gpuAddr);
+ static void freeCB(void *bo);
+ static void waitFromCpuCB(void *bo);
+
+ class Surface
+ {
+ public:
+ Surface(unsigned int width, unsigned int height, bool mmc = true)
+ : mWidth(width), mHeight(height), mMMC(mmc), mResInfo(0), mBuf(0)
+ {
+ }
+
+ ~Surface()
+ {
+ deinit();
+ }
+
+ bool init()
+ {
+ size_t size;
+ size_t alignment;
+
+ GMM_RESCREATE_PARAMS gmmParams = {};
+ gmmParams.Type = RESOURCE_2D;
+ gmmParams.Format = GMM_FORMAT_NV12;
+ gmmParams.BaseWidth = mWidth;
+ gmmParams.BaseHeight = mHeight;
+ gmmParams.Depth = 0x1;
+ gmmParams.ArraySize = 1;
+ gmmParams.Flags.Info.TiledY = 1;
+ gmmParams.Flags.Info.MediaCompressed = mMMC ? 1 : 0;
+ //gmmParams.Flags.Gpu.CCS = mmc ? 1 : 0;
+ gmmParams.Flags.Gpu.MMC = mMMC ? 1 : 0;
+ gmmParams.Flags.Gpu.Texture = 1;
+ gmmParams.Flags.Gpu.RenderTarget = 1;
+ gmmParams.Flags.Gpu.UnifiedAuxSurface = mMMC ? 1 : 0;
+ //gmmParams.Flags.Gpu.Depth = 1;
+ gmmParams.Flags.Gpu.Video = true;
+
+ mResInfo = pGmmULTClientContext->CreateResInfoObject(&gmmParams);
+
+ size = mResInfo->GetSizeSurface();
+
+ alignment = mResInfo->GetResFlags().Info.TiledYf ? GMM_KBYTE(16) : GMM_KBYTE(64);
+
+ mBuf = aligned_alloc(alignment, ALIGN(size, alignment));
+
+ if(!mResInfo || !mBuf)
+ return false;
+
+ mYBase = (GMM_GFX_ADDRESS)mBuf;
+ mUVBase = 0;
+ mAuxYBase = mYBase + mResInfo->GetUnifiedAuxSurfaceOffset(GMM_AUX_CCS);
+ mAuxUVBase = 0;
+ mYPlaneSize = mResInfo->GetSizeMainSurface();
+
+ if(pGmmULTClientContext->IsPlanar(mResInfo->GetResourceFormat()))
+ {
+ GMM_REQ_OFFSET_INFO ReqInfo = {0};
+ ReqInfo.Plane = GMM_PLANE_U;
+ ReqInfo.ReqRender = 1;
+
+ mResInfo->GetOffset(ReqInfo);
+ mYPlaneSize = ReqInfo.Render.Offset64;
+
+ mUVBase = mYBase + mYPlaneSize;
+ mAuxUVBase = mYBase + mResInfo->GetUnifiedAuxSurfaceOffset(GMM_AUX_UV_CCS);
+ }
+
+ mUVPlaneSize = mResInfo->GetSizeMainSurface() - mYPlaneSize;
+
+ return true;
+ }
+
+ void deinit()
+ {
+ if(mBuf)
+ {
+ free(mBuf);
+ mBuf = NULL;
+ }
+
+ if(mResInfo)
+ {
+ pGmmULTClientContext->DestroyResInfoObject(mResInfo);
+ mResInfo = NULL;
+ }
+ }
+
+ GMM_GFX_ADDRESS getGfxAddress(GMM_YUV_PLANE plane)
+ {
+ switch(plane)
+ {
+ case GMM_PLANE_Y:
+ return mYBase;
+ case GMM_PLANE_U:
+ case GMM_PLANE_V:
+ return mUVBase;
+ default:
+ throw;
+ }
+ }
+
+ GMM_GFX_ADDRESS getAuxGfxAddress(GMM_UNIFIED_AUX_TYPE auxType)
+ {
+ switch(auxType)
+ {
+ case GMM_AUX_CCS:
+ case GMM_AUX_Y_CCS:
+ return mAuxYBase;
+ case GMM_AUX_UV_CCS:
+ return mAuxUVBase;
+ default:
+ throw;
+ }
+ }
+
+ GMM_RESOURCE_INFO *getGMMResourceInfo()
+ {
+ return mResInfo;
+ }
+
+ size_t getSurfaceSize(GMM_YUV_PLANE plane)
+ {
+ switch(plane)
+ {
+ case GMM_PLANE_Y:
+ return mYPlaneSize;
+ case GMM_PLANE_U:
+ case GMM_PLANE_V:
+ return mUVPlaneSize;
+ default:
+ throw;
+ }
+ }
+
+ private:
+ unsigned int mWidth;
+ unsigned int mHeight;
+ bool mMMC;
+ GMM_RESOURCE_INFO *mResInfo;
+ void * mBuf;
+ GMM_GFX_ADDRESS mYBase;
+ GMM_GFX_ADDRESS mUVBase;
+ GMM_GFX_ADDRESS mAuxYBase;
+ GMM_GFX_ADDRESS mAuxUVBase;
+ size_t mYPlaneSize;
+ size_t mUVPlaneSize;
+ };
+
+ class Walker
+ {
+ public:
+ Walker(GMM_GFX_ADDRESS mainBase, GMM_GFX_ADDRESS auxBase,
+ GMM_GFX_ADDRESS l3Base)
+ {
+ mMainBase = mainBase;
+ mAuxBase = (auxBase >> 6) << 6;
+ mL3Base = (uint64_t *)l3Base;
+ }
+
+ GMM_GFX_ADDRESS expected(GMM_GFX_ADDRESS addr)
+ {
+ uint8_t Is64KChunk = (const_cast<WA_TABLE &>(pGfxAdapterInfo->WaTable).WaAuxTable16KGranular) ? 0 : 1;
+ uint32_t count = (addr - mMainBase) / (Is64KChunk ? GMM_KBYTE(64) : GMM_KBYTE(16));
+ return mAuxBase + (Is64KChunk ? 256 : 64) * count;
+ }
+
+ GMM_GFX_ADDRESS walk(GMM_GFX_ADDRESS addr)
+ {
+ uint64_t mask = (const_cast<WA_TABLE &>(pGfxAdapterInfo->WaTable).WaAuxTable16KGranular) ? 0x0000ffffffffffc0 : 0x0000ffffffffff00;
+ uint32_t idx = l3Index(addr);
+ uint64_t *l2Base = (uint64_t *)((mL3Base[idx] >> 15) << 15);
+ idx = l2Index(addr);
+ uint64_t *l1Base = (uint64_t *)((l2Base[idx] >> 13) << 13);
+ idx = l1Index(addr);
+ uint64_t auxAddr = (uint64_t)(l1Base[idx] & mask);
+ return auxAddr;
+ }
+
+ public:
+ static inline uint32_t l3Index(GMM_GFX_ADDRESS addr)
+ {
+ return GMM_AUX_L3_ENTRY_IDX(addr);
+ }
+
+ static inline uint32_t l2Index(GMM_GFX_ADDRESS addr)
+ {
+ return GMM_AUX_L2_ENTRY_IDX(addr);
+ }
+
+ static inline uint32_t l1Index(GMM_GFX_ADDRESS addr)
+ {
+ return GMM_AUX_L1_ENTRY_IDX_EXPORTED(addr, !(const_cast<WA_TABLE &>(pGfxAdapterInfo->WaTable).WaAuxTable16KGranular));
+ }
+
+ private:
+ GMM_GFX_ADDRESS mMainBase;
+ GMM_GFX_ADDRESS mAuxBase;
+ uint64_t * mL3Base;
+ };
+};
+
+#endif /* __linux__ */
diff --git a/Source/GmmLib/ULT/GmmCommonULT.cpp b/Source/GmmLib/ULT/GmmCommonULT.cpp
index d3eaef7..0498c78 100644
--- a/Source/GmmLib/ULT/GmmCommonULT.cpp
+++ b/Source/GmmLib/ULT/GmmCommonULT.cpp
@@ -25,6 +25,7 @@
#include <dlfcn.h>
#endif
+
ADAPTER_INFO * CommonULT::pGfxAdapterInfo = NULL;
PLATFORM CommonULT::GfxPlatform = {};
GMM_CLIENT_CONTEXT *CommonULT::pGmmULTClientContext = NULL;
diff --git a/Source/GmmLib/ULT/stdafx.h b/Source/GmmLib/ULT/stdafx.h
index 9a89212..717b5b9 100644
--- a/Source/GmmLib/ULT/stdafx.h
+++ b/Source/GmmLib/ULT/stdafx.h
@@ -59,6 +59,7 @@
#include "../inc/External/Common/GmmInfoExt.h"
#include "../inc/External/Common/GmmInfo.h"
#include "../inc/External/Common/GmmClientContext.h"
+#include "../inc/External/Common/GmmPageTableMgr.h"
#include "../inc/External/Common/GmmLibDll.h"
#include "../inc/External/Common/GmmLibDllName.h"
diff --git a/Source/GmmLib/inc/External/Common/GmmClientContext.h b/Source/GmmLib/inc/External/Common/GmmClientContext.h
index 47f2ff2..8dcca1f 100644
--- a/Source/GmmLib/inc/External/Common/GmmClientContext.h
+++ b/Source/GmmLib/inc/External/Common/GmmClientContext.h
@@ -122,7 +122,7 @@
GMM_VIRTUAL GMM_RESOURCE_INFO* GMM_STDCALL CopyResInfoObject(GMM_RESOURCE_INFO *pSrcRes);
GMM_VIRTUAL void GMM_STDCALL ResMemcpy(void *pDst, void *pSrc);
GMM_VIRTUAL void GMM_STDCALL DestroyResInfoObject(GMM_RESOURCE_INFO *pResInfo);
-
+
#ifdef GMM_LIB_DLL
/* ResourceInfo and PageTableMgr Create and Destroy APIs with Client provided Memory Allocators */
GMM_VIRTUAL GMM_RESOURCE_INFO* GMM_STDCALL CreateResInfoObject(GMM_RESCREATE_PARAMS *pCreateParams,
@@ -130,6 +130,18 @@
GMM_VIRTUAL void GMM_STDCALL DestroyResInfoObject(GMM_RESOURCE_INFO *pResInfo,
GmmClientAllocationCallbacks *pAllocCbs);
#endif
+
+ /* PageTableMgr Creation and Destroy API's */
+ GMM_VIRTUAL GMM_PAGETABLE_MGR* GMM_STDCALL CreatePageTblMgrObject(GMM_DEVICE_CALLBACKS_INT *pDevCb, uint32_t TTFlags);
+ /* PageTableMgr Creation and Destroy API's */
+ GMM_VIRTUAL void GMM_STDCALL DestroyPageTblMgrObject(GMM_PAGETABLE_MGR *pPageTableMgr);
+
+ GMM_VIRTUAL GMM_PAGETABLE_MGR* GMM_STDCALL CreatePageTblMgrObject(
+ GMM_DEVICE_CALLBACKS_INT* pDevCb,
+ uint32_t TTFlags,
+ GmmClientAllocationCallbacks* pAllocCbs);
+ GMM_VIRTUAL void GMM_STDCALL DestroyPageTblMgrObject(GMM_PAGETABLE_MGR* pPageTableMgr,
+ GmmClientAllocationCallbacks* pAllocCbs);
};
}
diff --git a/Source/GmmLib/inc/External/Common/GmmCommonExt.h b/Source/GmmLib/inc/External/Common/GmmCommonExt.h
index d0b7c1e..689ddab 100644
--- a/Source/GmmLib/inc/External/Common/GmmCommonExt.h
+++ b/Source/GmmLib/inc/External/Common/GmmCommonExt.h
@@ -72,6 +72,10 @@
#define GMM_NO_FENCE_REG 0xDEADBEEF
#define GMM_MAX_DISPLAYS 3
+#if defined __linux__
+typedef void* HANDLE;
+#endif
+
//===========================================================================
// typedef:
// GMM_GFX_ADDRESS/etc.
diff --git a/Source/GmmLib/inc/External/Common/GmmHw.h b/Source/GmmLib/inc/External/Common/GmmHw.h
new file mode 100644
index 0000000..81b4cf6
--- /dev/null
+++ b/Source/GmmLib/inc/External/Common/GmmHw.h
@@ -0,0 +1,150 @@
+/*==============================================================================
+Copyright(c) 2019 Intel Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files(the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and / or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+============================================================================*/
+
+
+#pragma once
+#include "gfxmacro.h"
+
+
+
+////////////////////// Auxiliary Translation Table definitions//////////////////////////////////////////
+//===========================================================================
+// typedef:
+// GMM_AUXTTL3e
+//
+// Description:
+// Struct for Auxiliary Translation-Table L3 entry
+//--------------------------------------------------------------------------
+typedef union GMM_AUXTTL3e_REC
+{
+ struct {
+ uint64_t Valid : 1;
+ uint64_t Reserved0 : 14;
+ uint64_t L2GfxAddr : 33;
+ uint64_t Reserved1 : 16;
+ };
+ uint64_t Value;
+} GMM_AUXTTL3e;
+C_ASSERT(sizeof(GMM_AUXTTL3e) == 8);
+
+// Get the L2GfxAddr bit field as a full L2 graphics address
+#define GMM_FULL_GFXADDR_FROM_AUX_L3e_L2GFXADDR(L2GfxAddr) ((L2GfxAddr) << 16)
+
+// Set the L2GfxAddr bit field given a full L2 graphics address
+#define GMM_TO_AUX_L3e_L2GFXADDR(L2GfxAddress) ((L2GfxAddress) >> 16)
+
+//===========================================================================
+// typedef:
+// GMM_AUXTTL2e
+//
+// Description:
+// Struct for Auxiliary Translation-Table L2 entry
+//--------------------------------------------------------------------------
+typedef union GMM_AUXTTL2e_REC
+{
+ struct
+ {
+ uint64_t Valid : 1;
+ uint64_t Reserved0 : 12;
+ uint64_t L1GfxAddr : 35;
+ uint64_t Reserved1 : 16;
+ };
+ uint64_t Value;
+} GMM_AUXTTL2e;
+C_ASSERT(sizeof(GMM_AUXTTL2e) == 8);
+
+// Get the L1GfxAddr bit field as a full L1 graphics address
+#define GMM_FULL_GFXADDR_FROM_AUX_L2e_L1GFXADDR(L1GfxAddr) ((L1GfxAddr) << 16)
+
+// Set the L1GfxAddr bit field given a full L1 graphics address
+#define GMM_TO_AUX_L2e_L1GFXADDR(L1GfxAddress) ((L1GfxAddress) >> 16)
+
+typedef union GMM_AUXTTL1e_REC
+{
+ struct
+ {
+ uint64_t Valid : 1;
+ uint64_t Mode : 2; //Compression ratio (128B compr ie 2:1 for RC, 256B compr ie 4:n compr for MC)
+ uint64_t Lossy : 1; //Lossy Compression
+ uint64_t Reserved0 : 2;
+ uint64_t Reserved2 : 2; //LSbs of 64B-aligned CCS chunk/cacheline address
+ uint64_t GfxAddress : 40; //256B-aligned CCS chunk address
+ uint64_t Reserved1 : 4;
+ uint64_t TileMode : 2; //Ys = 0, Y=1, Reserved=(2-3)
+ uint64_t Depth : 3; //Packed/Planar bit-depth for MC; Bpp for RC
+ uint64_t LumaChroma : 1; //Planar Y=0 or Cr=1
+ uint64_t Format : 6; //Format encoding shared with Vivante/Internal CC/DEC units to recognize surafce formats
+ };
+ uint64_t Value;
+} GMM_AUXTTL1e;
+C_ASSERT(sizeof(GMM_AUXTTL1e) == 8);
+
+#define GMM_NO_TABLE ((GMM_GFX_ADDRESS)(-1L)) //common
+
+#define GMM_INVALID_AUX_ENTRY ~__BIT(0)
+
+#define GMM_AUX_L1e_SIZE (sizeof(GMM_AUXTTL1e))
+#define GMM_AUX_L2e_SIZE (sizeof(GMM_AUXTTL2e))
+#define GMM_AUX_L3e_SIZE (sizeof(GMM_AUXTTL3e))
+
+#define GMM_AUX_L1_LOW_BIT (14)
+#define GMM_AUX_L1_HIGH_BIT (23)
+#define GMM_AUX_L2_LOW_BIT (24)
+#define GMM_AUX_L2_HIGH_BIT (35)
+#define GMM_AUX_L3_LOW_BIT (36)
+#define GMM_AUX_L3_HIGH_BIT (47)
+
+//For perf, AuxTable granularity changed to 64K
+#define WA16K (pGmmGlobalContext->GetWaTable().WaAuxTable16KGranular)
+
+// #L1 entries, i.e. 1024; 16K-granular ie 4 consequtive pages share Aux-cacheline;
+// HW only tracks the distinct entries;
+// Handle WA where HW chicken bit forces 64K-granularity
+#define GMM_AUX_L1_SIZE(pGmmGlobalContext) ((1 << (GMM_AUX_L1_HIGH_BIT - GMM_AUX_L1_LOW_BIT + 1)) / (!(WA16K) ? 4 : 1))
+#define GMM_AUX_L1_SIZE_DWORD(pGmmGlobalContext) (GFX_CEIL_DIV(GMM_AUX_L1_SIZE(pGmmGlobalContext), 32))
+
+// #L2 entries, i.e. 4096
+#define GMM_AUX_L2_SIZE (1 << (GMM_AUX_L2_HIGH_BIT - GMM_AUX_L2_LOW_BIT + 1))
+#define GMM_AUX_L2_SIZE_DWORD (GFX_CEIL_DIV(GMM_AUX_L2_SIZE, 32))
+
+// #L3 entries, i.e. 4096
+#define GMM_AUX_L3_SIZE (1 << (GMM_AUX_L3_HIGH_BIT - GMM_AUX_L3_LOW_BIT + 1))
+
+#define GMM_AUX_L1_ENTRY_IDX(GfxAddress,pGmmGlobalContext) \
+ ((((GfxAddress) & GFX_MASK_LARGE(GMM_AUX_L1_LOW_BIT, GMM_AUX_L1_HIGH_BIT)) >> \
+ (uint64_t)GMM_AUX_L1_LOW_BIT) / (!(WA16K) ? 4 : 1))
+
+
+#define GMM_AUX_L1_ENTRY_IDX_EXPORTED(GfxAddress,WA64KEx) \
+ ((((GfxAddress) & GFX_MASK_LARGE(GMM_AUX_L1_LOW_BIT, GMM_AUX_L1_HIGH_BIT)) >> \
+ (uint64_t)GMM_AUX_L1_LOW_BIT) / ((WA64KEx) ? 4 : 1))
+
+
+#define GMM_AUX_L2_ENTRY_IDX(GfxAddress) \
+ (((GfxAddress) & GFX_MASK_LARGE(GMM_AUX_L2_LOW_BIT, GMM_AUX_L2_HIGH_BIT)) >> \
+ (uint64_t)GMM_AUX_L2_LOW_BIT)
+
+#define GMM_AUX_L3_ENTRY_IDX(GfxAddress) \
+ (((GfxAddress) & GFX_MASK_LARGE(GMM_AUX_L3_LOW_BIT, GMM_AUX_L3_HIGH_BIT)) >> \
+ (uint64_t)GMM_AUX_L3_LOW_BIT)
+
+////////////////////// Auxiliary Translation Table definitions end//////////////////////////////////////////
diff --git a/Source/GmmLib/inc/External/Common/GmmPageTableMgr.h b/Source/GmmLib/inc/External/Common/GmmPageTableMgr.h
new file mode 100644
index 0000000..bc73f00
--- /dev/null
+++ b/Source/GmmLib/inc/External/Common/GmmPageTableMgr.h
@@ -0,0 +1,165 @@
+/*==============================================================================
+Copyright(c) 2019 Intel Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files(the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and / or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+Description: This file contains the class definitions for GmmPageTableMgr
+ for user-mode PageTable management, that is common for both
+ Linux and Windows.
+
+============================================================================*/
+
+#pragma once
+#include "GmmHw.h"
+
+#ifdef __linux__
+#include <pthread.h>
+#endif
+
+typedef enum _GMM_ENGINE_TYPE
+{
+ ENGINE_TYPE_RCS = 0, //RCS
+ ENGINE_TYPE_COMPUTE = 1, //Compute-CS
+ ENGINE_TYPE_BCS, //BLT
+ ENGINE_TYPE_VD0,
+ ENGINE_TYPE_VD1,
+ ENGINE_TYPE_VE0
+ //Add all engines supporting AUX-TT
+} GMM_ENGINE_TYPE;
+
+typedef enum TT_Flags
+{
+ AUXTT = 1, //Indicate TT request for AUX i.e. e2e compression
+} TT_TYPE;
+
+
+#if !(defined(__GMM_KMD__))
+// Shared Structure for both Windows and Linux
+typedef struct __GMM_DDI_UPDATEAUXTABLE
+{
+ GMM_UMD_SYNCCONTEXT * UmdContext; // [in] pointer to thread-specific data, specifying BBQHandle/Fence etc
+ GMM_RESOURCE_INFO * BaseResInfo; // [in] GmmResourceInfo ptr for compressed resource
+ GMM_RESOURCE_INFO * AuxResInfo; // [in] GmmResourceInfo ptr for separate Auxiliary resource
+ GMM_GFX_ADDRESS BaseGpuVA; // [in] GPUVA where compressed resource has been mapped
+ GMM_GFX_ADDRESS AuxSurfVA; // [in] GPUVA where separate Auxiliary resource has been mapped
+ uint8_t Map; // [in] specifies if resource is being mapped or unmapped
+ uint8_t DoNotWait; // [in] specifies if PageTable update be done on CPU (true) or GPU (false)
+}GMM_DDI_UPDATEAUXTABLE;
+
+#ifdef __cplusplus
+#include "GmmMemAllocator.hpp"
+
+namespace GmmLib
+{
+ class SyncInfoLin { //dummy class
+ public:
+ HANDLE BBQueueHandle;
+ uint64_t BBFence;
+ SyncInfoLin() {}
+ SyncInfoLin(HANDLE Handle, uint64_t Fence) {}
+ };
+ typedef class SyncInfoLin SyncInfo;
+
+
+ //Forward class declarations
+ class AuxTable;
+ class GmmPageTablePool;
+ typedef class GmmPageTablePool GMM_PAGETABLEPool;
+
+ typedef enum POOL_TYPE_REC
+ {
+ POOL_TYPE_TRTTL1 = 0,
+ POOL_TYPE_TRTTL2 = 1,
+ POOL_TYPE_AUXTTL1 = 2,
+ POOL_TYPE_AUXTTL2 = 3,
+ } POOL_TYPE;
+
+ //////////////////////////////////////////////////////////////////////////////////////////////
+ /// Contains functions and members for GMM_PAGETABLE_MGR, clients must place its pointer in
+ /// their device object. Clients call GmmLib to initialize the instance and use it for mapping
+ /// /unmapping on GmmLib managed page tables (TR-TT for SparseResources, AUX-TT for compression)
+ //////////////////////////////////////////////////////////////////////////////////////////////
+ class GMM_LIB_API NON_PAGED_SECTION GmmPageTableMgr :
+ public GmmMemAllocator
+ {
+ private:
+ GMM_ENGINE_TYPE EngType; //PageTable managed @ device-level (specifies engine associated with the device)
+
+ AuxTable* AuxTTObj; //Auxiliary Translation Table obj
+
+ GMM_PAGETABLEPool *pPool; //Common page table pool
+ uint32_t NumNodePoolElements;
+ GmmClientContext *pClientContext; ///< ClientContext of the client creating this Object
+
+ //OS-specific defn
+#if defined __linux__
+ pthread_mutex_t PoolLock;
+#endif
+ public:
+ GMM_DEVICE_CALLBACKS DeviceCb; //OS-specific defn: Will be used by Clients to send as input arguments for TR-TT APIs
+ GMM_DEVICE_CALLBACKS_INT DeviceCbInt; //OS-specific defn: Will be used internally GMM lib
+ GMM_TRANSLATIONTABLE_CALLBACKS TTCb; //OS-specific defn
+ HANDLE hCsr; // OCL per-device command stream receiver handle for aubcapture
+ public:
+ GmmPageTableMgr();
+ GmmPageTableMgr(GMM_DEVICE_CALLBACKS_INT *, uint32_t TTFlags, GmmClientContext *pClientContextIn); // Allocates memory for indicate TT’s root-tables, initializes common node-pool
+
+
+ //GMM_VIRTUAL GMM_GFX_ADDRESS GetTRL3TableAddr();
+ GMM_VIRTUAL GMM_GFX_ADDRESS GetAuxL3TableAddr();
+
+ //Update TT root table address in context-image
+ GMM_VIRTUAL GMM_STATUS InitContextAuxTableRegister(HANDLE initialBBHandle, GMM_ENGINE_TYPE engType); //Clients call it to update Aux-Table pointer in context-image, engType reqd. if @ context level
+
+ //Aux TT management API
+ GMM_VIRTUAL GMM_STATUS UpdateAuxTable(const GMM_DDI_UPDATEAUXTABLE*); //new API for updating Aux-Table to point to correct 16B-chunk
+ //for given host page VA when base/Aux surf is mapped/unmapped
+ GMM_VIRTUAL void __ReleaseUnusedPool(GMM_UMD_SYNCCONTEXT *UmdContext);
+ GMM_VIRTUAL GMM_PAGETABLEPool * __GetFreePoolNode(uint32_t * FreePoolNodeIdx, POOL_TYPE PoolType);
+
+
+#if defined __linux__
+ //returns number of BOs for indicated TTs = NumNodePoolElements+1 BOs for root table and pools
+ GMM_VIRTUAL int GetNumOfPageTableBOs(uint8_t TTFlags);
+ //returns BO* list for indicated TT in client allocated memory
+ GMM_VIRTUAL int GetPageTableBOList(uint8_t TTFlags, void* BOList);
+#endif
+
+ //Destructor
+ GMM_VIRTUAL ~GmmPageTableMgr(); //Clean-up page table structures
+
+ // Inline Functions
+ /////////////////////////////////////////////////////////////////////////////////////
+ /// Returns GmmClientContext associated with this PageTableMgr
+ /// @return ::GmmClientContext
+ /////////////////////////////////////////////////////////////////////////////////////
+ GMM_INLINE_VIRTUAL GmmClientContext* GetClientContext()
+ {
+ return pClientContext;
+ }
+
+ private:
+ GMM_PAGETABLEPool * __AllocateNodePool(uint32_t AddrAlignment, POOL_TYPE Type);
+
+ };
+
+
+}
+#endif // #ifdef __cplusplus
+#endif
diff --git a/Source/GmmLib/inc/External/Common/GmmResourceInfo.h b/Source/GmmLib/inc/External/Common/GmmResourceInfo.h
index 7c49411..972e4cf 100644
--- a/Source/GmmLib/inc/External/Common/GmmResourceInfo.h
+++ b/Source/GmmLib/inc/External/Common/GmmResourceInfo.h
@@ -100,4 +100,5 @@
#ifdef __cplusplus
}
-#endif /*__cplusplus*/
\ No newline at end of file
+#endif /*__cplusplus*/
+
diff --git a/Source/GmmLib/inc/External/Common/GmmResourceInfoCommon.h b/Source/GmmLib/inc/External/Common/GmmResourceInfoCommon.h
index 5c96507..0966d89 100644
--- a/Source/GmmLib/inc/External/Common/GmmResourceInfoCommon.h
+++ b/Source/GmmLib/inc/External/Common/GmmResourceInfoCommon.h
@@ -90,6 +90,28 @@
GMM_VIRTUAL bool ReAdjustPlaneProperties(bool IsAuxSurf);
GMM_VIRTUAL const GMM_PLATFORM_INFO& GetPlatformInfo();
+ /////////////////////////////////////////////////////////////////////////////////////
+ /// Returns tile mode for SURFACE_STATE programming.
+ /// @return Tiled Mode
+ /////////////////////////////////////////////////////////////////////////////////////
+ GMM_INLINE uint32_t GetTileModeSurfaceState(const GMM_TEXTURE_INFO *pTextureInfo) const
+ {
+ uint32_t TiledMode = 0;
+
+ if(GMM_IS_TILEY)
+ {
+ TiledMode =
+ pTextureInfo->Flags.Info.Linear ? 0 :
+ pTextureInfo->Flags.Info.TiledW ? 1 :
+ pTextureInfo->Flags.Info.TiledX ? 2 :
+ /* Y/YF/YS */ 3;
+
+ __GMM_ASSERT((TiledMode != 3) || (pTextureInfo->Flags.Info.TiledY || pTextureInfo->Flags.Info.TiledYf || pTextureInfo->Flags.Info.TiledYs));
+ }
+
+ return TiledMode;
+ }
+
public:
/* Constructors */
GmmResourceInfoCommon():
@@ -138,6 +160,7 @@
ClientType = rhs.ClientType;
Surf = rhs.Surf;
AuxSurf = rhs.AuxSurf;
+ AuxSecSurf = rhs.AuxSecSurf;
RotateInfo = rhs.RotateInfo;
ExistingSysMem = rhs.ExistingSysMem;
SvmAddress = rhs.SvmAddress;
@@ -1292,12 +1315,15 @@
}
else
{
- switch (GetHAlign())
+ if(GMM_IS_TILEY)
{
- case 4: HAlign = 1; break;
- case 8: HAlign = 2; break;
- case 16: HAlign = 3; break;
- default: HAlign = 1;
+ switch (GetHAlign())
+ {
+ case 4: HAlign = 1; break;
+ case 8: HAlign = 2; break;
+ case 16: HAlign = 3; break;
+ default: HAlign = 1; // TODO(Benign): Change back to 0 + assert after packed YUV handling corrected.
+ }
}
}
}
@@ -1362,16 +1388,16 @@
/////////////////////////////////////////////////////////////////////////////////////
GMM_INLINE_VIRTUAL GMM_INLINE_EXPORTED uint32_t GMM_STDCALL GetTileModeSurfaceState()
{
- uint32_t TiledMode = 0;
-
- TiledMode =
- Surf.Flags.Info.Linear ? 0 :
- Surf.Flags.Info.TiledX ? 2 :
- /* Y/YF/YS */ 3;
-
- __GMM_ASSERT((TiledMode != 3) || (Surf.Flags.Info.TiledY || Surf.Flags.Info.TiledYf || Surf.Flags.Info.TiledYs));
-
- return TiledMode;
+ return GetTileModeSurfaceState(&Surf);
+ }
+
+ /////////////////////////////////////////////////////////////////////////////////////
+ /// Returns tile mode for AUX SURFACE_STATE programming.
+ /// @return Tiled Mode
+ /////////////////////////////////////////////////////////////////////////////////////
+ GMM_INLINE_VIRTUAL GMM_INLINE_EXPORTED uint32_t GMM_STDCALL GetAuxTileModeSurfaceState()
+ {
+ return GetTileModeSurfaceState(&AuxSurf);
}
/////////////////////////////////////////////////////////////////////////////////////
diff --git a/Source/GmmLib/inc/External/Common/GmmResourceInfoExt.h b/Source/GmmLib/inc/External/Common/GmmResourceInfoExt.h
index 9b8098f..63fbd98 100644
--- a/Source/GmmLib/inc/External/Common/GmmResourceInfoExt.h
+++ b/Source/GmmLib/inc/External/Common/GmmResourceInfoExt.h
@@ -181,7 +181,9 @@
// TBD - Place holder for GMM_RESOURCE_FLAG definition.
//---------------------------------------------------------------------------
#include "GmmResourceFlags.h"
-
+#if defined __linux__
+ #include "External/Linux/GmmResourceInfoLinExt.h"
+#endif
//==========================================================================
// typedef:
diff --git a/Source/GmmLib/inc/External/Linux/GmmResourceInfoLinExt.h b/Source/GmmLib/inc/External/Linux/GmmResourceInfoLinExt.h
new file mode 100644
index 0000000..20d3b95
--- /dev/null
+++ b/Source/GmmLib/inc/External/Linux/GmmResourceInfoLinExt.h
@@ -0,0 +1,96 @@
+/*==============================================================================
+Copyright(c) 2019 Intel Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files(the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and / or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+============================================================================*/
+#pragma once
+
+#ifdef __linux__
+
+#ifdef __cplusplus
+extern "C" {
+#endif /*__cplusplus*/
+
+typedef struct {
+ int(*pfnAllocate)(void *bufMgr, size_t size, size_t alignment, void **bo, void **cpuAddr, uint64_t *gpuAddr);
+ void(*pfnDeallocate)(void *bo);
+ void(*pfnWaitFromCpu)(void *bo);;
+} GMM_DEVICE_CB_PTRS;
+
+typedef struct _GMM_DEVICE_CALLBACKS_INT
+{
+ void *pBufMgr;
+ GMM_DEVICE_CB_PTRS DevCbPtrs_;
+} GMM_DEVICE_CALLBACKS_INT;
+
+
+// Add the definition to compatible.
+typedef struct GMM_TRANSLATIONTABLE_CALLBACKS_REC
+{
+ int (*pfPrologTranslationTable)(void *pDeviceHandle);
+
+ int (*pfWriteL1Entries)(void *pDeviceHandle,
+ const uint32_t NumEntries,
+ GMM_GFX_ADDRESS GfxAddress,
+ uint32_t *Data);
+
+ int (*pfWriteL2L3Entry)(void *pDeviceHandle,
+ GMM_GFX_ADDRESS GfxAddress,
+ uint64_t Data);
+
+ int (*pfWriteFenceID)(void *pDeviceHandle,
+ GMM_GFX_ADDRESS GfxAddress,
+ uint64_t Data);
+
+ int (*pfEpilogTranslationTable)(void *pDeviceHandle,
+ uint8_t ForceFlush);
+
+ int (*pfCopyL1Entry)(void *pDeviceHandle,
+ GMM_GFX_ADDRESS DstGfxAddress,
+ GMM_GFX_ADDRESS SrcGfxAddress);
+
+ int (*pfWriteL3Adr)(void *pDeviceHandle,
+ GMM_GFX_ADDRESS L3GfxAddress,
+ uint64_t RegOffset);
+} GMM_TRANSLATIONTABLE_CALLBACKS;
+
+typedef struct _GMM_DEVICE_CALLBACKS
+{
+ void *pBufferMgr;
+ int FuncDevice;
+ int(*pfnAllocate)(void *bufMgr, size_t size, size_t alignment, void **bo, void **cpuAddr, uint64_t *gpuAddr);
+ void(*pfnDeallocate)(void *bo);
+ void(*pfnWaitFromCpu)(void *bo);;
+} GMM_DEVICE_CALLBACKS;
+
+
+// This definition is only for code sharing.
+typedef struct GMM_UMD_SYNCCONTEXT_REC
+{
+ void *pCommandQueueHandle; // pointer to command queue handle
+ void *pUpdateGpuVaInfo;
+ HANDLE BBFenceObj; // BatchBuffer Last Fence, for CPU to wait on before destroying TT pages
+ uint64_t BBLastFence; // BatchBuffer Last Fence for TT
+} GMM_UMD_SYNCCONTEXT;
+
+#ifdef __cplusplus
+}
+#endif /*__cplusplus*/
+
+#endif /*__linux__*/
diff --git a/Source/GmmLib/inc/Internal/Common/GmmLibInc.h b/Source/GmmLib/inc/Internal/Common/GmmLibInc.h
index a66365b..9f7d9e5 100644
--- a/Source/GmmLib/inc/Internal/Common/GmmLibInc.h
+++ b/Source/GmmLib/inc/Internal/Common/GmmLibInc.h
@@ -29,6 +29,7 @@
#include "External/Common/GmmConst.h"
#include "External/Common/GmmUtil.h"
+#include "External/Common/GmmHw.h"
#include "External/Common/GmmCommonExt.h"
#include "External/Common/GmmPlatformExt.h"
@@ -58,6 +59,7 @@
#include "External/Common/GmmInfoExt.h"
#include "External/Common/GmmInfo.h"
#include "../Utility/GmmUtility.h"
+#include "External/Common/GmmPageTableMgr.h"
#include "External/Common/GmmDebug.h" // Unified Definitions of GMM_ASSERT and GMM_DEBUG Macros
diff --git a/Source/GmmLib/inc/Internal/Linux/GmmResourceInfoLinInt.h b/Source/GmmLib/inc/Internal/Linux/GmmResourceInfoLinInt.h
new file mode 100644
index 0000000..750d537
--- /dev/null
+++ b/Source/GmmLib/inc/Internal/Linux/GmmResourceInfoLinInt.h
@@ -0,0 +1,71 @@
+/*==============================================================================
+Copyright(c) 2019 Intel Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files(the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and / or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+============================================================================*/
+
+#pragma once
+
+#if defined(__linux__) && !LHDM
+
+// GMM Lib Client Exports
+#include "External/Common/GmmCommonExt.h"
+#include "External/Common/GmmUtil.h"
+#include "External/Common/GmmResourceFlags.h"
+#include "External/Common/GmmCachePolicy.h"
+#include "External/Common/GmmCachePolicyExt.h"
+#include "External/Common/GmmResourceInfoExt.h"
+#include "External/Common/GmmPlatformExt.h"
+#include "External/Common/GmmTextureExt.h"
+#include "External/Common/GmmInfoExt.h"
+#include "External/Common/GmmResourceInfo.h"
+#include "External/Common/GmmInfo.h"
+
+typedef struct {
+ size_t size;
+ size_t alignment;
+ void *bo;
+ void *cpuAddr;
+ uint64_t gfxAddr;
+} GMM_DDI_ALLOCATE;
+
+typedef struct {
+ void *bo;
+} GMM_DDI_DEALLOCATE;
+
+typedef struct {
+ void *bo;
+}GMM_DDI_WAITFORSYNCHRONIZATIONOBJECTFROMCPU;
+
+typedef enum GMM_DEVICE_CALLBACKS_TYPE_REC
+{
+ GMM_DEV_CB_ALLOC = 0,
+ GMM_DEV_CB_DEALLOC,
+ GMM_DEV_CB_WAIT_FROM_CPU,
+} GMM_DEVICE_CALLBACKS_TYPE;
+
+int GmmDeviceCallback(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DDI_ALLOCATE *pAllocate);
+int GmmDeviceCallback(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DDI_DEALLOCATE *pDeallocate);
+int GmmDeviceCallback(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DDI_WAITFORSYNCHRONIZATIONOBJECTFROMCPU *pWait);
+
+int GmmCheckForNullDevCbPfn(GMM_CLIENT ClientType, GMM_DEVICE_CALLBACKS_INT *pDeviceCb, GMM_DEVICE_CALLBACKS_TYPE CallBackType);
+
+extern GMM_TRANSLATIONTABLE_CALLBACKS DummyTTCB;
+
+#endif /*__linux__*/
diff --git a/Source/inc/common/sku_wa.h b/Source/inc/common/sku_wa.h
index e9e46a7..e83c1a6 100644
--- a/Source/inc/common/sku_wa.h
+++ b/Source/inc/common/sku_wa.h
@@ -457,6 +457,24 @@
WA_BUG_TYPE_CORRUPTION,
WA_BUG_PERF_IMPACT_UNKNOWN, WA_COMPONENT_GMM)
+ WA_DECLARE(
+ WaAuxTable16KGranular,
+ "AuxTable map granularity changed to 16K ",
+ WA_BUG_TYPE_PERF,
+ WA_BUG_PERF_IMPACT_UNKNOWN, WA_COMPONENT_UNKNOWN)
+
+ WA_DECLARE(
+ WaLimit128BMediaCompr,
+ "WA to limit media decompression on Render pipe to 128B (2CLs) 4:n.",
+ WA_BUG_TYPE_FUNCTIONAL,
+ WA_BUG_PERF_IMPACT_UNKNOWN, WA_COMPONENT_GMM)
+
+ WA_DECLARE(
+ WaUntypedBufferCompression,
+ "WA to allow untyped raw buffer AuxTable mapping",
+ WA_BUG_TYPE_FUNCTIONAL,
+ WA_BUG_PERF_IMPACT_UNKNOWN, WA_COMPONENT_GMM)
+
} WA_TABLE, *PWA_TABLE;
//********************************** SKU/WA Macros *************************************
