Skip to content

Commit 75edea8

Browse files
Virtual address space partitioning on Linux [2/n]
Move selectHeap from Wddm to MemoryManager. Set DrmAllocation::origin. Change-Id: I5d412e35d524d1f31174893b9ce1d3b1e98eee96
1 parent 4139e88 commit 75edea8

File tree

15 files changed

+137
-119
lines changed

15 files changed

+137
-119
lines changed

runtime/memory_manager/graphics_allocation.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,8 @@ class GraphicsAllocation : public IDNode<GraphicsAllocation> {
115115
return gpuAddress + allocationOffset - gpuBaseAddress;
116116
}
117117

118-
bool isCoherent() { return coherent; };
119-
void setCoherent(bool coherentIn) { this->coherent = coherentIn; };
118+
bool isCoherent() const { return coherent; }
119+
void setCoherent(bool coherentIn) { this->coherent = coherentIn; }
120120
void setSize(size_t size) { this->size = size; }
121121
osHandle peekSharedHandle() { return sharedHandle; }
122122

runtime/memory_manager/memory_manager.cpp

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include "runtime/gmm_helper/resource_info.h"
1616
#include "runtime/helpers/aligned_memory.h"
1717
#include "runtime/helpers/basic_math.h"
18+
#include "runtime/helpers/hw_info.h"
1819
#include "runtime/helpers/kernel_commands.h"
1920
#include "runtime/helpers/options.h"
2021
#include "runtime/memory_manager/deferrable_allocation_deletion.h"
@@ -374,4 +375,24 @@ void MemoryManager::unlockResource(GraphicsAllocation *graphicsAllocation) {
374375
unlockResourceImpl(*graphicsAllocation);
375376
graphicsAllocation->unlock();
376377
}
378+
379+
HeapIndex MemoryManager::selectHeap(const GraphicsAllocation *allocation, const void *ptr, const HardwareInfo &hwInfo) {
380+
if (allocation) {
381+
if (allocation->origin == AllocationOrigin::INTERNAL_ALLOCATION) {
382+
return internalHeapIndex;
383+
} else if (allocation->is32BitAllocation) {
384+
return HeapIndex::HEAP_EXTERNAL;
385+
}
386+
}
387+
if (hwInfo.capabilityTable.gpuAddressSpace == MemoryConstants::max48BitAddress) {
388+
if (ptr) {
389+
return HeapIndex::HEAP_SVM;
390+
}
391+
if (allocation && GraphicsAllocation::isCpuAccessRequired(allocation->getAllocationType())) {
392+
return HeapIndex::HEAP_STANDARD64Kb;
393+
}
394+
return HeapIndex::HEAP_STANDARD;
395+
}
396+
return HeapIndex::HEAP_LIMITED;
397+
}
377398
} // namespace OCLRT

runtime/memory_manager/memory_manager.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ class Gmm;
2727
class GraphicsAllocation;
2828
class HostPtrManager;
2929
class OsContext;
30+
struct HardwareInfo;
3031
struct ImageInfo;
3132

3233
using CsrContainer = std::vector<std::vector<std::unique_ptr<CommandStreamReceiver>>>;
@@ -187,6 +188,8 @@ class MemoryManager {
187188
HostPtrManager *getHostPtrManager() const { return hostPtrManager.get(); }
188189
void setDefaultEngineIndex(uint32_t index) { defaultEngineIndex = index; }
189190

191+
static HeapIndex selectHeap(const GraphicsAllocation *allocation, const void *ptr, const HardwareInfo &hwInfo);
192+
190193
protected:
191194
struct AllocationData {
192195
union {

runtime/memory_manager/os_agnostic_memory_manager.h

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
#pragma once
99
#include "runtime/helpers/basic_math.h"
1010
#include "runtime/memory_manager/memory_manager.h"
11-
#include <map>
1211

1312
namespace OCLRT {
1413
constexpr size_t bigAllocation = 1 * MB;
@@ -31,18 +30,15 @@ class MemoryAllocation : public GraphicsAllocation {
3130
void overrideMemoryPool(MemoryPool::Type pool);
3231
};
3332

34-
typedef std::map<void *, MemoryAllocation *> PointerMap;
35-
3633
class OsAgnosticMemoryManager : public MemoryManager {
3734
public:
3835
using MemoryManager::allocateGraphicsMemory;
39-
using MemoryManager::createGraphicsAllocationFromSharedHandle;
4036

4137
OsAgnosticMemoryManager(bool enable64kbPages, bool enableLocalMemory, ExecutionEnvironment &executionEnvironment) : OsAgnosticMemoryManager(enable64kbPages, enableLocalMemory, false, executionEnvironment){};
4238

4339
OsAgnosticMemoryManager(bool enable64kbPages, bool enableLocalMemory, bool aubUsage, ExecutionEnvironment &executionEnvironment) : MemoryManager(enable64kbPages, enableLocalMemory, executionEnvironment) {
4440
allocator32Bit = std::unique_ptr<Allocator32bit>(create32BitAllocator(aubUsage));
45-
};
41+
}
4642

4743
~OsAgnosticMemoryManager() override;
4844
GraphicsAllocation *allocateGraphicsMemoryForNonSvmHostPtr(size_t size, void *cpuPtr) override;
@@ -71,8 +67,8 @@ class OsAgnosticMemoryManager : public MemoryManager {
7167
GraphicsAllocation *allocateGraphicsMemory64kb(AllocationData allocationData) override;
7268
GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr<Gmm> gmm) override;
7369

74-
void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) override { return ptrOffset(graphicsAllocation.getUnderlyingBuffer(), static_cast<size_t>(graphicsAllocation.allocationOffset)); };
75-
void unlockResourceImpl(GraphicsAllocation &graphicsAllocation) override{};
70+
void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) override { return ptrOffset(graphicsAllocation.getUnderlyingBuffer(), static_cast<size_t>(graphicsAllocation.allocationOffset)); }
71+
void unlockResourceImpl(GraphicsAllocation &graphicsAllocation) override {}
7672
GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) override;
7773
GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override;
7874

runtime/os_interface/linux/drm_memory_manager.cpp

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,9 @@ DrmAllocation *DrmMemoryManager::allocateGraphicsMemoryWithAlignment(const Alloc
241241
if (forcePinEnabled && pinBB != nullptr && allocationData.flags.forcePin && allocationData.size >= this->pinThreshold) {
242242
pinBB->pin(&bo, 1, getDefaultCommandStreamReceiver(0)->getOsContext().get()->getDrmContextId());
243243
}
244-
return new DrmAllocation(bo, res, cSize, MemoryPool::System4KBPages, allocationData.flags.multiOsContextCapable);
244+
auto allocation = new DrmAllocation(bo, res, cSize, MemoryPool::System4KBPages, allocationData.flags.multiOsContextCapable);
245+
allocation->origin = allocationData.allocationOrigin;
246+
return allocation;
245247
}
246248

247249
DrmAllocation *DrmMemoryManager::allocateGraphicsMemoryWithHostPtr(const AllocationData &allocationData) {
@@ -256,7 +258,7 @@ DrmAllocation *DrmMemoryManager::allocateGraphicsMemoryWithHostPtr(const Allocat
256258
}
257259

258260
DrmAllocation *DrmMemoryManager::allocateGraphicsMemoryForNonSvmHostPtr(size_t size, void *cpuPtr) {
259-
if ((size == 0) || !cpuPtr)
261+
if (size == 0 || !cpuPtr)
260262
return nullptr;
261263

262264
auto alignedPtr = alignDown(reinterpret_cast<char *>(cpuPtr), MemoryConstants::pageSize);
@@ -329,6 +331,7 @@ GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemoryForImageImpl(const A
329331
auto allocation = new DrmAllocation(bo, nullptr, (uint64_t)gpuRange, allocationData.imgInfo->size, MemoryPool::SystemCpuInaccessible, false);
330332
bo->setAllocationType(allocatorType);
331333
allocation->gmm = gmm.release();
334+
allocation->origin = allocationData.allocationOrigin;
332335
return allocation;
333336
}
334337

@@ -363,6 +366,7 @@ DrmAllocation *DrmMemoryManager::allocate32BitGraphicsMemoryImpl(const Allocatio
363366
allocationSize, MemoryPool::System4KBPagesWith32BitGpuAddressing, false);
364367
drmAllocation->is32BitAllocation = true;
365368
drmAllocation->gpuBaseAddress = allocatorToUse->getBase();
369+
drmAllocation->origin = allocationData.allocationOrigin;
366370
return drmAllocation;
367371
}
368372

@@ -417,6 +421,7 @@ DrmAllocation *DrmMemoryManager::allocate32BitGraphicsMemoryImpl(const Allocatio
417421
drmAllocation->is32BitAllocation = true;
418422
drmAllocation->gpuBaseAddress = allocatorToUse->getBase();
419423
drmAllocation->driverAllocatedCpuPointer = ptrAlloc;
424+
drmAllocation->origin = allocationData.allocationOrigin;
420425
return drmAllocation;
421426
}
422427

runtime/os_interface/linux/drm_memory_manager.h

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,6 @@ class Drm;
2121

2222
class DrmMemoryManager : public MemoryManager {
2323
public:
24-
using MemoryManager::allocateGraphicsMemory;
25-
using MemoryManager::createGraphicsAllocationFromSharedHandle;
26-
2724
DrmMemoryManager(Drm *drm, gemCloseWorkerMode mode, bool forcePinAllowed, bool validateHostPtrMemory, ExecutionEnvironment &executionEnvironment);
2825
~DrmMemoryManager() override;
2926

runtime/os_interface/windows/wddm/wddm.cpp

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "runtime/gmm_helper/gmm_helper.h"
1212
#include "runtime/gmm_helper/resource_info.h"
1313
#include "runtime/gmm_helper/page_table_mngr.h"
14+
#include "runtime/memory_manager/memory_manager.h"
1415
#include "runtime/os_interface/windows/wddm/wddm.h"
1516
#include "runtime/os_interface/hw_info_config.h"
1617
#include "runtime/os_interface/windows/gdi_interface.h"
@@ -288,15 +289,15 @@ bool Wddm::makeResident(D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFur
288289
bool Wddm::mapGpuVirtualAddress(WddmAllocation *allocation, void *cpuPtr) {
289290
void *mapPtr = allocation->getReservedAddress() != nullptr ? allocation->getReservedAddress() : cpuPtr;
290291
return mapGpuVirtualAddressImpl(allocation->gmm, allocation->handle, mapPtr, allocation->gpuPtr,
291-
selectHeap(allocation, mapPtr));
292+
MemoryManager::selectHeap(allocation, mapPtr, *hardwareInfoTable[gfxPlatform->eProductFamily]));
292293
}
293294

294295
bool Wddm::mapGpuVirtualAddress(AllocationStorageData *allocationStorageData) {
295296
return mapGpuVirtualAddressImpl(allocationStorageData->osHandleStorage->gmm,
296297
allocationStorageData->osHandleStorage->handle,
297298
const_cast<void *>(allocationStorageData->cpuPtr),
298299
allocationStorageData->osHandleStorage->gpuPtr,
299-
selectHeap(nullptr, allocationStorageData->cpuPtr));
300+
MemoryManager::selectHeap(nullptr, allocationStorageData->cpuPtr, *hardwareInfoTable[gfxPlatform->eProductFamily]));
300301
}
301302

302303
bool Wddm::mapGpuVirtualAddressImpl(Gmm *gmm, D3DKMT_HANDLE handle, void *cpuPtr, D3DGPU_VIRTUAL_ADDRESS &gpuPtr, HeapIndex heapIndex) {
@@ -994,24 +995,4 @@ std::unique_lock<SpinLock> Wddm::acquireLock(SpinLock &lock) {
994995
return std::unique_lock<SpinLock>{lock};
995996
}
996997

997-
HeapIndex Wddm::selectHeap(const WddmAllocation *allocation, const void *ptr) const {
998-
if (allocation) {
999-
if (allocation->origin == AllocationOrigin::INTERNAL_ALLOCATION) {
1000-
return internalHeapIndex;
1001-
} else if (allocation->is32BitAllocation) {
1002-
return HeapIndex::HEAP_EXTERNAL;
1003-
}
1004-
}
1005-
if (hardwareInfoTable[gfxPlatform->eProductFamily]->capabilityTable.gpuAddressSpace == MemoryConstants::max48BitAddress) {
1006-
if (ptr) {
1007-
return HeapIndex::HEAP_SVM;
1008-
}
1009-
if (allocation && GraphicsAllocation::isCpuAccessRequired(allocation->getAllocationType())) {
1010-
return HeapIndex::HEAP_STANDARD64Kb;
1011-
}
1012-
return HeapIndex::HEAP_STANDARD;
1013-
}
1014-
return HeapIndex::HEAP_LIMITED;
1015-
}
1016-
1017998
} // namespace OCLRT

runtime/os_interface/windows/wddm/wddm.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,6 @@ class Wddm {
148148
MOCKABLE_VIRTUAL EvictionStatus evictTemporaryResource(WddmAllocation &allocation);
149149
MOCKABLE_VIRTUAL void applyBlockingMakeResident(WddmAllocation &allocation);
150150
MOCKABLE_VIRTUAL std::unique_lock<SpinLock> acquireLock(SpinLock &lock);
151-
HeapIndex selectHeap(const WddmAllocation *allocation, const void *cpuPtr) const;
152151

153152
protected:
154153
bool initialized = false;

runtime/os_interface/windows/wddm_memory_manager.cpp

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryForImageImpl(const
4848
auto allocation = std::make_unique<WddmAllocation>(nullptr, allocationData.imgInfo->size, nullptr, MemoryPool::SystemCpuInaccessible, false);
4949
allocation->gmm = gmm.get();
5050
allocation->origin = allocationData.allocationOrigin;
51-
if (!WddmMemoryManager::createWddmAllocation(allocation.get())) {
51+
if (!createWddmAllocation(allocation.get())) {
5252
return nullptr;
5353
}
5454

@@ -60,11 +60,10 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryForImageImpl(const
6060

6161
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemory64kb(AllocationData allocationData) {
6262
size_t sizeAligned = alignUp(allocationData.size, MemoryConstants::pageSize64k);
63-
Gmm *gmm = nullptr;
6463

6564
auto wddmAllocation = std::make_unique<WddmAllocation>(nullptr, sizeAligned, nullptr, MemoryPool::System64KBPages, !!allocationData.flags.multiOsContextCapable);
6665

67-
gmm = new Gmm(nullptr, sizeAligned, false, allocationData.flags.preferRenderCompressed, true, {});
66+
auto gmm = new Gmm(nullptr, sizeAligned, false, allocationData.flags.preferRenderCompressed, true, {});
6867
wddmAllocation->gmm = gmm;
6968

7069
if (!wddm->createAllocation64k(wddmAllocation.get())) {
@@ -77,7 +76,7 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemory64kb(AllocationData
7776
// 64kb map is not needed
7877
auto status = wddm->mapGpuVirtualAddress(wddmAllocation.get(), cpuPtr);
7978
DEBUG_BREAK_IF(!status);
80-
wddmAllocation->setCpuPtrAndGpuAddress(cpuPtr, (uint64_t)wddmAllocation->gpuPtr);
79+
wddmAllocation->setCpuPtrAndGpuAddress(cpuPtr, wddmAllocation->gpuPtr);
8180

8281
DebugManager.logAllocation(wddmAllocation.get());
8382
return wddmAllocation.release();
@@ -280,15 +279,15 @@ void WddmMemoryManager::removeAllocationFromHostPtrManager(GraphicsAllocation *g
280279

281280
void *WddmMemoryManager::lockResourceImpl(GraphicsAllocation &graphicsAllocation) {
282281
return wddm->lockResource(static_cast<WddmAllocation &>(graphicsAllocation));
283-
};
282+
}
284283
void WddmMemoryManager::unlockResourceImpl(GraphicsAllocation &graphicsAllocation) {
285284
auto &wddmAllocation = static_cast<WddmAllocation &>(graphicsAllocation);
286285
wddm->unlockResource(wddmAllocation);
287286
if (wddmAllocation.needsMakeResidentBeforeLock) {
288287
auto evictionStatus = wddm->evictTemporaryResource(wddmAllocation);
289288
DEBUG_BREAK_IF(evictionStatus == EvictionStatus::FAILED);
290289
}
291-
};
290+
}
292291

293292
void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) {
294293
WddmAllocation *input = static_cast<WddmAllocation *>(gfxAllocation);
@@ -450,7 +449,7 @@ uint64_t WddmMemoryManager::getMaxApplicationAddress() {
450449
}
451450

452451
uint64_t WddmMemoryManager::getInternalHeapBaseAddress() {
453-
return this->wddm->getGfxPartition().Heap32[static_cast<uint32_t>(internalHeapIndex)].Base;
452+
return wddm->getGfxPartition().Heap32[static_cast<uint32_t>(internalHeapIndex)].Base;
454453
}
455454

456455
bool WddmMemoryManager::mapAuxGpuVA(GraphicsAllocation *graphicsAllocation) {

runtime/os_interface/windows/wddm_memory_manager.h

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,7 @@ using OsContextWin = OsContext::OsContextImpl;
2424

2525
class WddmMemoryManager : public MemoryManager {
2626
public:
27-
using MemoryManager::allocateGraphicsMemory;
28-
using MemoryManager::createGraphicsAllocationFromSharedHandle;
29-
30-
~WddmMemoryManager();
27+
~WddmMemoryManager() override;
3128
WddmMemoryManager(bool enable64kbPages, bool enableLocalMemory, Wddm *wddm, ExecutionEnvironment &executionEnvironment);
3229

3330
WddmMemoryManager(const WddmMemoryManager &) = delete;

0 commit comments

Comments
 (0)