From 1fe058fa33e905c0bff78de92632ee051d1c0a09 Mon Sep 17 00:00:00 2001 From: "Chandio, Bibrak Qamar" Date: Thu, 6 Nov 2025 12:11:27 -0800 Subject: [PATCH 1/3] feature: multi gpu SVM madvise Implements madvise SVM allocated memory for multi-gpu Signed-off-by: Chandio, Bibrak Qamar --- level_zero/core/source/cmdlist/cmdlist_hw.inl | 14 ++++++++-- shared/source/memory_manager/memory_manager.h | 2 +- .../memory_manager/unified_memory_manager.cpp | 6 ++-- .../memory_manager/unified_memory_manager.h | 2 +- .../os_interface/linux/drm_memory_manager.cpp | 18 +++++++----- .../os_interface/linux/drm_memory_manager.h | 2 +- .../source/os_interface/linux/ioctl_helper.h | 6 ++-- .../linux/ioctl_helper_prelim.cpp | 2 +- .../linux/ioctl_helper_upstream.cpp | 2 +- .../os_interface/linux/xe/ioctl_helper_xe.cpp | 20 +++++++++---- .../os_interface/linux/xe/ioctl_helper_xe.h | 2 +- .../test/common/mocks/mock_memory_manager.h | 4 +-- .../memory_manager/memory_manager_tests.cpp | 2 +- .../unified_memory_manager_tests.cpp | 2 +- .../linux/drm_memory_manager_tests.cpp | 2 +- .../linux/xe/ioctl_helper_xe_tests.cpp | 28 +++++++++++-------- third_party/uapi/drm-next/xe/xe_drm.h | 4 +-- 17 files changed, 71 insertions(+), 47 deletions(-) diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.inl b/level_zero/core/source/cmdlist/cmdlist_hw.inl index 9e5471b5fb514..e2b6a00333c3d 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.inl +++ b/level_zero/core/source/cmdlist/cmdlist_hw.inl @@ -1355,10 +1355,19 @@ ze_result_t CommandListCoreFamily::executeMemAdvise(ze_device_han ze_memory_advice_t advice) { auto driverHandle = device->getDriverHandle(); + auto callingNEODevice = device->getNEODevice(); auto allocData = driverHandle->getSvmAllocsManager()->getSVMAlloc(ptr); if (!allocData) { - if (device->getNEODevice()->areSharedSystemAllocationsAllowed()) { + if (callingNEODevice->areSharedSystemAllocationsAllowed()) { + + DeviceImp *targetDeviceImp = static_cast((L0::Device::fromHandle(hDevice))); + auto targetNEODevice = targetDeviceImp->getNEODevice(); + + if (!targetNEODevice->areSharedSystemAllocationsAllowed()) { + return ZE_RESULT_ERROR_INVALID_ARGUMENT; + } + NEO::MemAdvise memAdviseOp = NEO::MemAdvise::invalidAdvise; switch (advice) { @@ -1384,10 +1393,9 @@ ze_result_t CommandListCoreFamily::executeMemAdvise(ze_device_han return ZE_RESULT_SUCCESS; } - DeviceImp *deviceImp = static_cast((L0::Device::fromHandle(hDevice))); auto unifiedMemoryManager = driverHandle->getSvmAllocsManager(); - unifiedMemoryManager->sharedSystemMemAdvise(*deviceImp->getNEODevice(), memAdviseOp, ptr, size); + unifiedMemoryManager->sharedSystemMemAdvise(*callingNEODevice, *targetNEODevice, memAdviseOp, ptr, size); return ZE_RESULT_SUCCESS; } else { diff --git a/shared/source/memory_manager/memory_manager.h b/shared/source/memory_manager/memory_manager.h index 748db794ff592..1c3f36ac0230b 100644 --- a/shared/source/memory_manager/memory_manager.h +++ b/shared/source/memory_manager/memory_manager.h @@ -287,7 +287,7 @@ class MemoryManager { virtual AllocationStatus registerLocalMemAlloc(GraphicsAllocation *allocation, uint32_t rootDeviceIndex); virtual bool setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags, uint32_t rootDeviceIndex) { return true; } - virtual bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) { return true; } + virtual bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t callingRootDeviceIndex, uint32_t targetRootDeviceIndex) { return true; } virtual bool setMemPrefetch(GraphicsAllocation *gfxAllocation, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) { return true; } virtual bool prefetchSharedSystemAlloc(const void *ptr, const size_t size, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) { return true; } virtual bool setAtomicAccess(GraphicsAllocation *gfxAllocation, size_t size, AtomicAccessMode mode, uint32_t rootDeviceIndex) { return true; } diff --git a/shared/source/memory_manager/unified_memory_manager.cpp b/shared/source/memory_manager/unified_memory_manager.cpp index 1db435cdacfb9..9be4829397c0a 100644 --- a/shared/source/memory_manager/unified_memory_manager.cpp +++ b/shared/source/memory_manager/unified_memory_manager.cpp @@ -1141,12 +1141,12 @@ static NEO::SubDeviceIdsVec getSubDeviceIds(CommandStreamReceiver &csr) { return subDeviceIds; }; -void SVMAllocsManager::sharedSystemMemAdvise(Device &device, MemAdvise memAdviseOp, const void *ptr, const size_t size) { +void SVMAllocsManager::sharedSystemMemAdvise(Device &callingDevice, Device &targetDevice, MemAdvise memAdviseOp, const void *ptr, const size_t size) { // All vm_ids on a single device for shared system USM allocation - auto subDeviceIds = NEO::SubDevice::getSubDeviceIdsFromDevice(device); + auto subDeviceIds = NEO::SubDevice::getSubDeviceIdsFromDevice(targetDevice); - memoryManager->setSharedSystemMemAdvise(ptr, size, memAdviseOp, subDeviceIds, device.getRootDeviceIndex()); + memoryManager->setSharedSystemMemAdvise(ptr, size, memAdviseOp, subDeviceIds, callingDevice.getRootDeviceIndex(), targetDevice.getRootDeviceIndex()); } void SVMAllocsManager::prefetchMemory(Device &device, CommandStreamReceiver &commandStreamReceiver, const void *ptr, const size_t size) { diff --git a/shared/source/memory_manager/unified_memory_manager.h b/shared/source/memory_manager/unified_memory_manager.h index 957f0ac5b77e0..690d6f578b6d1 100644 --- a/shared/source/memory_manager/unified_memory_manager.h +++ b/shared/source/memory_manager/unified_memory_manager.h @@ -304,7 +304,7 @@ class SVMAllocsManager { std::atomic allocationsCounter = 0; MOCKABLE_VIRTUAL void makeIndirectAllocationsResident(CommandStreamReceiver &commandStreamReceiver, TaskCountType taskCount); void prepareIndirectAllocationForDestruction(SvmAllocationData *allocationData, bool isNonBlockingFree); - void sharedSystemMemAdvise(Device &device, MemAdvise memAdviseOp, const void *ptr, const size_t size); + void sharedSystemMemAdvise(Device &callingDevice, Device &targetDevice, MemAdvise memAdviseOp, const void *ptr, const size_t size); MOCKABLE_VIRTUAL void prefetchMemory(Device &device, CommandStreamReceiver &commandStreamReceiver, const void *ptr, const size_t size); void prefetchSVMAllocs(Device &device, CommandStreamReceiver &commandStreamReceiver); void sharedSystemAtomicAccess(Device &device, AtomicAccessMode mode, const void *ptr, const size_t size); diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index 0f50d798318e9..6f31a7850c676 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -308,22 +308,26 @@ bool DrmMemoryManager::setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdvise return drmAllocation->setMemAdvise(&this->getDrm(rootDeviceIndex), flags); } -bool DrmMemoryManager::setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) { +bool DrmMemoryManager::setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t callingRootDeviceIndex, uint32_t targetRootDeviceIndex) { - auto &drm = this->getDrm(rootDeviceIndex); - auto ioctlHelper = drm.getIoctlHelper(); + auto &targetDeviceDrm = this->getDrm(targetRootDeviceIndex); + auto targetDeviceIoctlHelper = targetDeviceDrm.getIoctlHelper(); + auto targetDeviceFd = targetDeviceDrm.getFileDescriptor(); - uint32_t attribute = ioctlHelper->getPreferredLocationAdvise(); - uint64_t param = ioctlHelper->getPreferredLocationArgs(memAdviseOp); + uint32_t attribute = targetDeviceIoctlHelper->getPreferredLocationAdvise(); + uint64_t param = targetDeviceIoctlHelper->getPreferredLocationArgs(targetDeviceFd, memAdviseOp); // Apply the shared system USM IOCTL to all the VMs of the device std::vector vmIds; vmIds.reserve(subDeviceIds.size()); for (auto subDeviceId : subDeviceIds) { - vmIds.push_back(drm.getVirtualMemoryAddressSpace(subDeviceId)); + vmIds.push_back(targetDeviceDrm.getVirtualMemoryAddressSpace(subDeviceId)); } - auto result = ioctlHelper->setVmSharedSystemMemAdvise(reinterpret_cast(ptr), size, attribute, param, vmIds); + auto &callingDeviceDrm = this->getDrm(callingRootDeviceIndex); + auto callingDeviceIoctlHelper = callingDeviceDrm.getIoctlHelper(); + + auto result = callingDeviceIoctlHelper->setVmSharedSystemMemAdvise(reinterpret_cast(ptr), size, attribute, param, vmIds); return result; } diff --git a/shared/source/os_interface/linux/drm_memory_manager.h b/shared/source/os_interface/linux/drm_memory_manager.h index 0dff0fc780d34..5ad2beee8f2ca 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.h +++ b/shared/source/os_interface/linux/drm_memory_manager.h @@ -90,7 +90,7 @@ class DrmMemoryManager : public MemoryManager { bool isKmdMigrationAvailable(uint32_t rootDeviceIndex) override; bool setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags, uint32_t rootDeviceIndex) override; - bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override; + bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t callingRootDeviceIndex, uint32_t targetRootDeviceIndex) override; bool setMemPrefetch(GraphicsAllocation *gfxAllocation, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override; bool prefetchSharedSystemAlloc(const void *ptr, const size_t size, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override; bool setAtomicAccess(GraphicsAllocation *gfxAllocation, size_t size, AtomicAccessMode mode, uint32_t rootDeviceIndex) override; diff --git a/shared/source/os_interface/linux/ioctl_helper.h b/shared/source/os_interface/linux/ioctl_helper.h index f939b5345fe3b..9d2434411e237 100644 --- a/shared/source/os_interface/linux/ioctl_helper.h +++ b/shared/source/os_interface/linux/ioctl_helper.h @@ -129,7 +129,7 @@ class IoctlHelper { bool userInterrupt, uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) = 0; virtual uint32_t getAtomicAdvise(bool isNonAtomic) = 0; virtual uint32_t getAtomicAccess(AtomicAccessMode mode) = 0; - virtual uint64_t getPreferredLocationArgs(MemAdvise memAdviseOp) = 0; + virtual uint64_t getPreferredLocationArgs(int deviceFd, MemAdvise memAdviseOp) = 0; virtual uint32_t getPreferredLocationAdvise() = 0; virtual std::optional getPreferredLocationRegion(PreferredLocation memoryLocation, uint32_t memoryInstance) = 0; virtual bool setVmBoAdvise(int32_t handle, uint32_t attribute, void *region) = 0; @@ -328,7 +328,7 @@ class IoctlHelperUpstream : public IoctlHelperI915 { bool userInterrupt, uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) override; uint32_t getAtomicAdvise(bool isNonAtomic) override; uint32_t getAtomicAccess(AtomicAccessMode mode) override; - uint64_t getPreferredLocationArgs(MemAdvise memAdviseOp) override; + uint64_t getPreferredLocationArgs(int deviceFd, MemAdvise memAdviseOp) override; uint32_t getPreferredLocationAdvise() override; std::optional getPreferredLocationRegion(PreferredLocation memoryLocation, uint32_t memoryInstance) override; bool setVmBoAdvise(int32_t handle, uint32_t attribute, void *region) override; @@ -404,7 +404,7 @@ class IoctlHelperPrelim20 : public IoctlHelperI915 { bool userInterrupt, uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) override; uint32_t getAtomicAdvise(bool isNonAtomic) override; uint32_t getAtomicAccess(AtomicAccessMode mode) override; - uint64_t getPreferredLocationArgs(MemAdvise memAdviseOp) override; + uint64_t getPreferredLocationArgs(int deviceFd, MemAdvise memAdviseOp) override; uint32_t getPreferredLocationAdvise() override; std::optional getPreferredLocationRegion(PreferredLocation memoryLocation, uint32_t memoryInstance) override; bool setVmBoAdvise(int32_t handle, uint32_t attribute, void *region) override; diff --git a/shared/source/os_interface/linux/ioctl_helper_prelim.cpp b/shared/source/os_interface/linux/ioctl_helper_prelim.cpp index ef2aeac6f4615..f5abe50174a32 100644 --- a/shared/source/os_interface/linux/ioctl_helper_prelim.cpp +++ b/shared/source/os_interface/linux/ioctl_helper_prelim.cpp @@ -345,7 +345,7 @@ uint32_t IoctlHelperPrelim20::getAtomicAdvise(bool isNonAtomic) { return isNonAtomic ? PRELIM_I915_VM_ADVISE_ATOMIC_NONE : PRELIM_I915_VM_ADVISE_ATOMIC_SYSTEM; } -uint64_t IoctlHelperPrelim20::getPreferredLocationArgs(MemAdvise memAdviseOp) { +uint64_t IoctlHelperPrelim20::getPreferredLocationArgs(int deviceFd, MemAdvise memAdviseOp) { return 0; } diff --git a/shared/source/os_interface/linux/ioctl_helper_upstream.cpp b/shared/source/os_interface/linux/ioctl_helper_upstream.cpp index 15d0954460978..4421189672121 100644 --- a/shared/source/os_interface/linux/ioctl_helper_upstream.cpp +++ b/shared/source/os_interface/linux/ioctl_helper_upstream.cpp @@ -140,7 +140,7 @@ uint32_t IoctlHelperUpstream::getAtomicAccess(AtomicAccessMode mode) { return 0; } -uint64_t IoctlHelperUpstream::getPreferredLocationArgs(MemAdvise memAdviseOp) { +uint64_t IoctlHelperUpstream::getPreferredLocationArgs(int deviceFd, MemAdvise memAdviseOp) { return 0; } diff --git a/shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp b/shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp index 8b4e1e0e9ebb4..8c0cc1d600583 100644 --- a/shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp +++ b/shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp @@ -794,24 +794,30 @@ uint32_t IoctlHelperXe::getAtomicAccess(AtomicAccessMode mode) { return retVal; } -uint64_t IoctlHelperXe::getPreferredLocationArgs(MemAdvise memAdviseOp) { +uint64_t IoctlHelperXe::getPreferredLocationArgs(int deviceFd, MemAdvise memAdviseOp) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); uint64_t param = 0; switch (memAdviseOp) { - case MemAdvise::setPreferredLocation: case MemAdvise::clearPreferredLocation: - case MemAdvise::clearSystemMemoryPreferredLocation: { // Assumes that the default location is Device VRAM. const auto preferredLocation = static_cast(getDrmParamValue(DrmParam::memoryAdviseLocationDevice)); const auto policy = static_cast(getDrmParamValue(DrmParam::memoryAdviseMigrationPolicyAllPages)); - param = (preferredLocation << 32) | policy; + const auto regionInstance = static_cast(0); + param = (preferredLocation << 32) | (policy << 16) | regionInstance; + } break; + case MemAdvise::setPreferredLocation: { + const auto preferredLocation = static_cast(deviceFd); + const auto policy = static_cast(getDrmParamValue(DrmParam::memoryAdviseMigrationPolicyAllPages)); + const auto regionInstance = static_cast(1); + param = (preferredLocation << 32) | (policy << 16) | regionInstance; } break; case MemAdvise::setSystemMemoryPreferredLocation: { const auto preferredLocation = static_cast(getDrmParamValue(DrmParam::memoryAdviseLocationSystem)); const auto policy = static_cast(getDrmParamValue(DrmParam::memoryAdviseMigrationPolicySystemPages)); - param = (preferredLocation << 32) | policy; + const auto regionInstance = static_cast(0); + param = (preferredLocation << 32) | (policy << 16) | regionInstance; } break; default: xeLog(" Invalid advise operation %s\n", __FUNCTION__); @@ -837,10 +843,12 @@ bool IoctlHelperXe::setVmBoAdvise(int32_t handle, uint32_t attribute, void *regi inline void setMemoryAdvisePreferredLocationParam(drm_xe_madvise &vmAdvise, const uint64_t param) { uint32_t devmemFd = static_cast(param >> 32); - uint16_t migrationPolicy = static_cast(param & 0xFFFF); + uint16_t migrationPolicy = static_cast((param >> 16) & 0xFFFF); + uint16_t regionInstance = static_cast(param & 0xFFFF); vmAdvise.preferred_mem_loc.devmem_fd = devmemFd; vmAdvise.preferred_mem_loc.migration_policy = migrationPolicy; + vmAdvise.preferred_mem_loc.region_instance = regionInstance; } inline void setMemoryAdviseAtomicParam(drm_xe_madvise &vmAdvise, const uint64_t param) { diff --git a/shared/source/os_interface/linux/xe/ioctl_helper_xe.h b/shared/source/os_interface/linux/xe/ioctl_helper_xe.h index a53343fb71644..db779f70719d9 100644 --- a/shared/source/os_interface/linux/xe/ioctl_helper_xe.h +++ b/shared/source/os_interface/linux/xe/ioctl_helper_xe.h @@ -56,7 +56,7 @@ class IoctlHelperXe : public IoctlHelper { bool userInterrupt, uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) override; uint32_t getAtomicAdvise(bool isNonAtomic) override; uint32_t getAtomicAccess(AtomicAccessMode mode) override; - uint64_t getPreferredLocationArgs(MemAdvise memAdviseOp) override; + uint64_t getPreferredLocationArgs(int deviceFd, MemAdvise memAdviseOp) override; uint32_t getPreferredLocationAdvise() override; std::optional getPreferredLocationRegion(PreferredLocation memoryLocation, uint32_t memoryInstance) override; bool setVmBoAdvise(int32_t handle, uint32_t attribute, void *region) override; diff --git a/shared/test/common/mocks/mock_memory_manager.h b/shared/test/common/mocks/mock_memory_manager.h index ff23985e49342..d9d3eddc06b71 100644 --- a/shared/test/common/mocks/mock_memory_manager.h +++ b/shared/test/common/mocks/mock_memory_manager.h @@ -219,13 +219,13 @@ class MockMemoryManager : public MemoryManagerCreate { return MemoryManager::setMemAdvise(gfxAllocation, flags, rootDeviceIndex); } - bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override { + bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t callingRootDeviceIndex, uint32_t targetRootDeviceIndex) override { setSharedSystemMemAdviseCalledCount++; setSharedSystemMemAdviseCalled = true; if (failSetSharedSystemMemAdvise) { return false; } - return MemoryManager::setSharedSystemMemAdvise(ptr, size, memAdviseOp, subDeviceIds, rootDeviceIndex); + return MemoryManager::setSharedSystemMemAdvise(ptr, size, memAdviseOp, subDeviceIds, callingRootDeviceIndex, targetRootDeviceIndex); } bool setMemPrefetch(GraphicsAllocation *gfxAllocation, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override { diff --git a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp index 4a4846002e11e..2e7dc5b4754b6 100644 --- a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp +++ b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp @@ -59,7 +59,7 @@ TEST(MemoryManagerTest, WhenCallingSetSharedSystemMemAdviseThenReturnTrue) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); OsAgnosticMemoryManager memoryManager(executionEnvironment); auto subDeviceId = SubDeviceIdsVec{0}; - EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, MemAdvise::invalidAdvise, subDeviceId, 0u)); + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, MemAdvise::invalidAdvise, subDeviceId, 0u, 0u)); } TEST(MemoryManagerTest, WhenCallingSetSharedSystemAtomicAccessThenReturnTrue) { diff --git a/shared/test/unit_test/memory_manager/unified_memory_manager_tests.cpp b/shared/test/unit_test/memory_manager/unified_memory_manager_tests.cpp index f62d2a7ed5ed6..6c90c36b8b0c8 100644 --- a/shared/test/unit_test/memory_manager/unified_memory_manager_tests.cpp +++ b/shared/test/unit_test/memory_manager/unified_memory_manager_tests.cpp @@ -384,7 +384,7 @@ TEST_F(SVMLocalMemoryAllocatorTest, givenSharedSystemAllocationWhenSharedSystemM auto ptr = malloc(4096); EXPECT_NE(nullptr, ptr); - svmManager->sharedSystemMemAdvise(*device, memAdviseOp, ptr, 4096); + svmManager->sharedSystemMemAdvise(*device, *device, memAdviseOp, ptr, 4096); auto mockMemoryManager = static_cast(device->getMemoryManager()); EXPECT_TRUE(mockMemoryManager->setSharedSystemMemAdviseCalled); diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index 38b0ea4a2e6c9..d96c436d189f1 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -6190,7 +6190,7 @@ HWTEST_TEMPLATED_F(DrmMemoryManagerTest, givenDrmMemoryManagerWhenSetSharedSyste auto subDeviceIds = NEO::SubDeviceIdsVec{0}; MemAdvise memAdviseOp = MemAdvise::setPreferredLocation; - EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, subDeviceIds, 0u)); + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, subDeviceIds, 0u, 0u)); EXPECT_EQ(1u, mockIoctlHelper->setVmSharedSystemMemAdviseCalled); } diff --git a/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp b/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp index f11b0b4ddb8be..8c64d7c9fecc5 100644 --- a/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp @@ -377,30 +377,34 @@ TEST_F(IoctlHelperXeTest, givenIoctlHelperXeWhenCallGetPreferredLocationArgsCorr DrmMock drm{*executionEnvironment->rootDeviceEnvironments[0]}; auto xeIoctlHelper = std::make_unique(drm); + auto deviceFd = drm.getFileDescriptor(); uint64_t expectedParam = 0; + auto preferredLocation = static_cast(xeIoctlHelper->getDrmParamValue(DrmParam::memoryAdviseLocationDevice)); auto policy = static_cast(xeIoctlHelper->getDrmParamValue(DrmParam::memoryAdviseMigrationPolicyAllPages)); - expectedParam = (preferredLocation << 32) | policy; - - auto memAdviseOp = MemAdvise::setPreferredLocation; - EXPECT_EQ(expectedParam, xeIoctlHelper->getPreferredLocationArgs(memAdviseOp)); - - memAdviseOp = MemAdvise::clearPreferredLocation; - EXPECT_EQ(expectedParam, xeIoctlHelper->getPreferredLocationArgs(memAdviseOp)); + const auto regionInstance = static_cast(1); + expectedParam = (preferredLocation << 32) | (policy << 16) | regionInstance; + auto memAdviseOp = MemAdvise::clearPreferredLocation; + EXPECT_EQ(expectedParam, xeIoctlHelper->getPreferredLocationArgs(deviceFd, memAdviseOp)); memAdviseOp = MemAdvise::clearSystemMemoryPreferredLocation; - EXPECT_EQ(expectedParam, xeIoctlHelper->getPreferredLocationArgs(memAdviseOp)); + EXPECT_EQ(expectedParam, xeIoctlHelper->getPreferredLocationArgs(deviceFd, memAdviseOp)); + + preferredLocation = static_cast(deviceFd); + policy = static_cast(xeIoctlHelper->getDrmParamValue(DrmParam::memoryAdviseMigrationPolicyAllPages)); + expectedParam = (preferredLocation << 32) | (policy << 16) | regionInstance; + memAdviseOp = MemAdvise::setPreferredLocation; + EXPECT_EQ(expectedParam, xeIoctlHelper->getPreferredLocationArgs(deviceFd, memAdviseOp)); preferredLocation = static_cast(xeIoctlHelper->getDrmParamValue(DrmParam::memoryAdviseLocationSystem)); policy = static_cast(xeIoctlHelper->getDrmParamValue(DrmParam::memoryAdviseMigrationPolicySystemPages)); - expectedParam = (preferredLocation << 32) | policy; - + expectedParam = (preferredLocation << 32) | (policy << 16) | regionInstance; memAdviseOp = MemAdvise::setSystemMemoryPreferredLocation; - EXPECT_EQ(expectedParam, xeIoctlHelper->getPreferredLocationArgs(memAdviseOp)); + EXPECT_EQ(expectedParam, xeIoctlHelper->getPreferredLocationArgs(deviceFd, memAdviseOp)); memAdviseOp = MemAdvise::invalidAdvise; expectedParam = 0; - EXPECT_EQ(expectedParam, xeIoctlHelper->getPreferredLocationArgs(memAdviseOp)); + EXPECT_EQ(expectedParam, xeIoctlHelper->getPreferredLocationArgs(deviceFd, memAdviseOp)); } TEST_F(IoctlHelperXeTest, givenIoctlHelperXeWhenCallingGetAtomicAccessThenCorrectValueIsReturned) { diff --git a/third_party/uapi/drm-next/xe/xe_drm.h b/third_party/uapi/drm-next/xe/xe_drm.h index 5c2d63abf4135..11efd7bd01d83 100644 --- a/third_party/uapi/drm-next/xe/xe_drm.h +++ b/third_party/uapi/drm-next/xe/xe_drm.h @@ -2079,8 +2079,8 @@ struct drm_xe_madvise { /** @preferred_mem_loc.migration_policy: Page migration policy */ __u16 migration_policy; - /** @preferred_mem_loc.pad : MBZ */ - __u16 pad; + /** @preferred_mem_loc.region_instance : Region instance */ + __u16 region_instance; /** @preferred_mem_loc.reserved : Reserved */ __u64 reserved; From 2465fe3d3a599999a82f1b6a075770697dcf2e8d Mon Sep 17 00:00:00 2001 From: "Chandio, Bibrak Qamar" Date: Thu, 6 Nov 2025 12:11:27 -0800 Subject: [PATCH 2/3] feature: multi gpu SVM madvise Implements madvise SVM allocated memory for multi-gpu Signed-off-by: Chandio, Bibrak Qamar --- level_zero/core/source/cmdlist/cmdlist_hw.inl | 2 +- .../sources/cmdlist/test_cmdlist_1.cpp | 33 ++++++++++++++++++- .../linux/xe/ioctl_helper_xe_tests.cpp | 4 ++- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.inl b/level_zero/core/source/cmdlist/cmdlist_hw.inl index e2b6a00333c3d..c042cd1a6b3f8 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.inl +++ b/level_zero/core/source/cmdlist/cmdlist_hw.inl @@ -1344,7 +1344,7 @@ ze_result_t CommandListCoreFamily::appendMemAdvise(ze_device_hand const void *ptr, size_t size, ze_memory_advice_t advice) { - this->memAdviseOperations.push_back(MemAdviseOperation(hDevice, ptr, size, advice)); + this->memAdviseOperations.emplace_back(hDevice, ptr, size, advice); return ZE_RESULT_SUCCESS; } diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp index cc34ae70d2c37..29a01e6dccea4 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp @@ -234,7 +234,7 @@ TEST_F(CommandListCreateTests, givenValidDeviceMemPtrWhenExecuteMemAdviseFailsTh ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListCreateTests, givenValidSystemAlloctedPtrAndNotSharedSystemAllocationsAllowedWhenExecuteMemAdviseFailsThenReturnError) { +TEST_F(CommandListCreateTests, givenValidSystemAlloctedPtrAndNotSharedSystemAllocationsAllowedForTheCallingDeviceWhenExecuteMemAdviseFailsThenReturnError) { DebugManagerStateRestore restorer; debugManager.flags.EnableSharedSystemUsmSupport.set(1u); @@ -261,6 +261,37 @@ TEST_F(CommandListCreateTests, givenValidSystemAlloctedPtrAndNotSharedSystemAllo free(ptr); } +TEST_F(CommandListCreateTests, givenValidSystemAlloctedPtrAndNotSharedSystemAllocationsAllowedForTheTargetDeviceWhenExecuteMemAdviseFailsThenReturnError) { + + DebugManagerStateRestore restorer; + debugManager.flags.EnableSharedSystemUsmSupport.set(1u); + debugManager.flags.EnableRecoverablePageFaults.set(1u); + + ze_result_t returnValue; + std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)); + ASSERT_NE(nullptr, commandList); + + // Create a new device with modified capabilities + auto *neoMockDevice = NEO::MockDevice::createWithNewExecutionEnvironment(NEO::defaultHwInfo.get(), rootDeviceIndex); + MockDeviceImp targetDevice(neoMockDevice); + + auto &hwInfo = *targetDevice.getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo(); + VariableBackup sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities}; + + sharedSystemMemCapabilities = 0; // enables return false for Device::areSharedSystemAllocationsAllowed() + + size_t size = 10; + void *ptr = nullptr; + + ptr = malloc(size); + EXPECT_NE(nullptr, ptr); + + auto res = commandList->executeMemAdvise(targetDevice.toHandle(), ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); + + free(ptr); +} + TEST_F(CommandListCreateTests, givenValidDeviceMemPtrWhenAppendMemAdviseSuccedsThenMemAdviseOperationsGrows) { size_t size = 10; diff --git a/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp b/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp index 8c64d7c9fecc5..1b35575d41467 100644 --- a/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp @@ -382,7 +382,7 @@ TEST_F(IoctlHelperXeTest, givenIoctlHelperXeWhenCallGetPreferredLocationArgsCorr auto preferredLocation = static_cast(xeIoctlHelper->getDrmParamValue(DrmParam::memoryAdviseLocationDevice)); auto policy = static_cast(xeIoctlHelper->getDrmParamValue(DrmParam::memoryAdviseMigrationPolicyAllPages)); - const auto regionInstance = static_cast(1); + auto regionInstance = static_cast(0); expectedParam = (preferredLocation << 32) | (policy << 16) | regionInstance; auto memAdviseOp = MemAdvise::clearPreferredLocation; EXPECT_EQ(expectedParam, xeIoctlHelper->getPreferredLocationArgs(deviceFd, memAdviseOp)); @@ -392,12 +392,14 @@ TEST_F(IoctlHelperXeTest, givenIoctlHelperXeWhenCallGetPreferredLocationArgsCorr preferredLocation = static_cast(deviceFd); policy = static_cast(xeIoctlHelper->getDrmParamValue(DrmParam::memoryAdviseMigrationPolicyAllPages)); + regionInstance = static_cast(1); expectedParam = (preferredLocation << 32) | (policy << 16) | regionInstance; memAdviseOp = MemAdvise::setPreferredLocation; EXPECT_EQ(expectedParam, xeIoctlHelper->getPreferredLocationArgs(deviceFd, memAdviseOp)); preferredLocation = static_cast(xeIoctlHelper->getDrmParamValue(DrmParam::memoryAdviseLocationSystem)); policy = static_cast(xeIoctlHelper->getDrmParamValue(DrmParam::memoryAdviseMigrationPolicySystemPages)); + regionInstance = static_cast(0); expectedParam = (preferredLocation << 32) | (policy << 16) | regionInstance; memAdviseOp = MemAdvise::setSystemMemoryPreferredLocation; EXPECT_EQ(expectedParam, xeIoctlHelper->getPreferredLocationArgs(deviceFd, memAdviseOp)); From e2708dfb4a9ea05da7ab789f82656b4d75aad2f1 Mon Sep 17 00:00:00 2001 From: "Chandio, Bibrak Qamar" Date: Mon, 10 Nov 2025 10:27:14 -0800 Subject: [PATCH 3/3] feature: multi gpu SVM madvise Implements madvise SVM allocated memory for multi-gpu Signed-off-by: Chandio, Bibrak Qamar Signed-off-by: Falkowski, John --- shared/source/memory_manager/memory_manager.h | 2 +- .../source/memory_manager/unified_memory_manager.cpp | 5 +---- .../source/os_interface/linux/drm_memory_manager.cpp | 11 +++++++---- shared/source/os_interface/linux/drm_memory_manager.h | 2 +- shared/test/common/mocks/mock_memory_manager.h | 4 ++-- .../unit_test/memory_manager/memory_manager_tests.cpp | 5 +++-- .../os_interface/linux/drm_memory_manager_tests.cpp | 5 +++-- 7 files changed, 18 insertions(+), 16 deletions(-) diff --git a/shared/source/memory_manager/memory_manager.h b/shared/source/memory_manager/memory_manager.h index 1c3f36ac0230b..6360d459a535d 100644 --- a/shared/source/memory_manager/memory_manager.h +++ b/shared/source/memory_manager/memory_manager.h @@ -287,7 +287,7 @@ class MemoryManager { virtual AllocationStatus registerLocalMemAlloc(GraphicsAllocation *allocation, uint32_t rootDeviceIndex); virtual bool setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags, uint32_t rootDeviceIndex) { return true; } - virtual bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t callingRootDeviceIndex, uint32_t targetRootDeviceIndex) { return true; } + virtual bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, Device &callingDevice, Device &targetDevice) { return true; } virtual bool setMemPrefetch(GraphicsAllocation *gfxAllocation, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) { return true; } virtual bool prefetchSharedSystemAlloc(const void *ptr, const size_t size, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) { return true; } virtual bool setAtomicAccess(GraphicsAllocation *gfxAllocation, size_t size, AtomicAccessMode mode, uint32_t rootDeviceIndex) { return true; } diff --git a/shared/source/memory_manager/unified_memory_manager.cpp b/shared/source/memory_manager/unified_memory_manager.cpp index 9be4829397c0a..b186b05dd0990 100644 --- a/shared/source/memory_manager/unified_memory_manager.cpp +++ b/shared/source/memory_manager/unified_memory_manager.cpp @@ -1143,10 +1143,7 @@ static NEO::SubDeviceIdsVec getSubDeviceIds(CommandStreamReceiver &csr) { void SVMAllocsManager::sharedSystemMemAdvise(Device &callingDevice, Device &targetDevice, MemAdvise memAdviseOp, const void *ptr, const size_t size) { - // All vm_ids on a single device for shared system USM allocation - auto subDeviceIds = NEO::SubDevice::getSubDeviceIdsFromDevice(targetDevice); - - memoryManager->setSharedSystemMemAdvise(ptr, size, memAdviseOp, subDeviceIds, callingDevice.getRootDeviceIndex(), targetDevice.getRootDeviceIndex()); + memoryManager->setSharedSystemMemAdvise(ptr, size, memAdviseOp, callingDevice, targetDevice); } void SVMAllocsManager::prefetchMemory(Device &device, CommandStreamReceiver &commandStreamReceiver, const void *ptr, const size_t size) { diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index 6f31a7850c676..47e758ca82ed7 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -308,23 +308,26 @@ bool DrmMemoryManager::setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdvise return drmAllocation->setMemAdvise(&this->getDrm(rootDeviceIndex), flags); } -bool DrmMemoryManager::setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t callingRootDeviceIndex, uint32_t targetRootDeviceIndex) { +bool DrmMemoryManager::setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, Device &callingDevice, Device &targetDevice) { - auto &targetDeviceDrm = this->getDrm(targetRootDeviceIndex); + auto &targetDeviceDrm = this->getDrm(targetDevice.getRootDeviceIndex()); auto targetDeviceIoctlHelper = targetDeviceDrm.getIoctlHelper(); auto targetDeviceFd = targetDeviceDrm.getFileDescriptor(); uint32_t attribute = targetDeviceIoctlHelper->getPreferredLocationAdvise(); uint64_t param = targetDeviceIoctlHelper->getPreferredLocationArgs(targetDeviceFd, memAdviseOp); - // Apply the shared system USM IOCTL to all the VMs of the device + // All vm_ids on a single device for shared system USM allocation + auto subDeviceIds = NEO::SubDevice::getSubDeviceIdsFromDevice(targetDevice); + + // Need to apply the shared system USM IOCTL to all the VMs of the device std::vector vmIds; vmIds.reserve(subDeviceIds.size()); for (auto subDeviceId : subDeviceIds) { vmIds.push_back(targetDeviceDrm.getVirtualMemoryAddressSpace(subDeviceId)); } - auto &callingDeviceDrm = this->getDrm(callingRootDeviceIndex); + auto &callingDeviceDrm = this->getDrm(callingDevice.getRootDeviceIndex()); auto callingDeviceIoctlHelper = callingDeviceDrm.getIoctlHelper(); auto result = callingDeviceIoctlHelper->setVmSharedSystemMemAdvise(reinterpret_cast(ptr), size, attribute, param, vmIds); diff --git a/shared/source/os_interface/linux/drm_memory_manager.h b/shared/source/os_interface/linux/drm_memory_manager.h index 5ad2beee8f2ca..6afe3b380d26d 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.h +++ b/shared/source/os_interface/linux/drm_memory_manager.h @@ -90,7 +90,7 @@ class DrmMemoryManager : public MemoryManager { bool isKmdMigrationAvailable(uint32_t rootDeviceIndex) override; bool setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags, uint32_t rootDeviceIndex) override; - bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t callingRootDeviceIndex, uint32_t targetRootDeviceIndex) override; + bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, Device &callingDevice, Device &targetDevice) override; bool setMemPrefetch(GraphicsAllocation *gfxAllocation, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override; bool prefetchSharedSystemAlloc(const void *ptr, const size_t size, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override; bool setAtomicAccess(GraphicsAllocation *gfxAllocation, size_t size, AtomicAccessMode mode, uint32_t rootDeviceIndex) override; diff --git a/shared/test/common/mocks/mock_memory_manager.h b/shared/test/common/mocks/mock_memory_manager.h index d9d3eddc06b71..45e67d596d3ba 100644 --- a/shared/test/common/mocks/mock_memory_manager.h +++ b/shared/test/common/mocks/mock_memory_manager.h @@ -219,13 +219,13 @@ class MockMemoryManager : public MemoryManagerCreate { return MemoryManager::setMemAdvise(gfxAllocation, flags, rootDeviceIndex); } - bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, SubDeviceIdsVec &subDeviceIds, uint32_t callingRootDeviceIndex, uint32_t targetRootDeviceIndex) override { + bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, Device &callingDevice, Device &targetDevice) override { setSharedSystemMemAdviseCalledCount++; setSharedSystemMemAdviseCalled = true; if (failSetSharedSystemMemAdvise) { return false; } - return MemoryManager::setSharedSystemMemAdvise(ptr, size, memAdviseOp, subDeviceIds, callingRootDeviceIndex, targetRootDeviceIndex); + return MemoryManager::setSharedSystemMemAdvise(ptr, size, memAdviseOp, callingDevice, targetDevice); } bool setMemPrefetch(GraphicsAllocation *gfxAllocation, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override { diff --git a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp index 2e7dc5b4754b6..26228315b695c 100644 --- a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp +++ b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp @@ -58,8 +58,9 @@ using namespace NEO; TEST(MemoryManagerTest, WhenCallingSetSharedSystemMemAdviseThenReturnTrue) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); OsAgnosticMemoryManager memoryManager(executionEnvironment); - auto subDeviceId = SubDeviceIdsVec{0}; - EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, MemAdvise::invalidAdvise, subDeviceId, 0u, 0u)); + MockDevice callingDevice; + MockDevice targetDevice; + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, MemAdvise::invalidAdvise, callingDevice, targetDevice)); } TEST(MemoryManagerTest, WhenCallingSetSharedSystemAtomicAccessThenReturnTrue) { diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index d96c436d189f1..070ec5788f071 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -6188,9 +6188,10 @@ HWTEST_TEMPLATED_F(DrmMemoryManagerTest, givenDrmMemoryManagerWhenSetSharedSyste auto &drm = static_cast(memoryManager.getDrm(mockRootDeviceIndex)); drm.ioctlHelper.reset(mockIoctlHelper); - auto subDeviceIds = NEO::SubDeviceIdsVec{0}; + MockDevice callingDevice; + MockDevice targetDevice; MemAdvise memAdviseOp = MemAdvise::setPreferredLocation; - EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, subDeviceIds, 0u, 0u)); + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, callingDevice, targetDevice)); EXPECT_EQ(1u, mockIoctlHelper->setVmSharedSystemMemAdviseCalled); }