Copy user buffers when not accepted by Kernel

When performing copy operations to or from buffers allocated by the
user, it could happen that the buffer address is not accepted by
kernel, even though the buffer is valid. In those ocassions, then
allocate a new graphics allocation and copy the user buffer.

Change-Id: I6b1b6f2ef5fea0acf32c868bc87eafe8746f9a79
Signed-off: Jaime Arteaga <jaime.a.arteaga.molina@intel.com>

Author: Jaime Arteaga

level_zero/core/source/device/device_imp.cpp

@@ -735,12 +735,18 @@ NEO::GraphicsAllocation *DeviceImp::allocateManagedMemoryFromHostPtr(void *buffe
 }
 
 NEO::GraphicsAllocation *DeviceImp::allocateMemoryFromHostPtr(const void *buffer, size_t size) {
-    NEO::AllocationProperties properties = {getRootDeviceIndex(), false, size, NEO::GraphicsAllocation::AllocationType::EXTERNAL_HOST_PTR, false, neoDevice->getDeviceBitfield()};
+    NEO::AllocationProperties properties = {getRootDeviceIndex(), false, size,
+                                            NEO::GraphicsAllocation::AllocationType::EXTERNAL_HOST_PTR,
+                                            false, neoDevice->getDeviceBitfield()};
     properties.flags.flushL3RequiredForRead = properties.flags.flushL3RequiredForWrite = true;
     auto allocation = neoDevice->getMemoryManager()->allocateGraphicsMemoryWithProperties(properties,
                                                                                           buffer);
-
-    UNRECOVERABLE_IF(allocation == nullptr);
+    if (allocation == nullptr) {
+        allocation = neoDevice->getMemoryManager()->allocateInternalGraphicsMemoryWithHostCopy(neoDevice->getRootDeviceIndex(),
+                                                                                               neoDevice->getDeviceBitfield(),
+                                                                                               buffer,
+                                                                                               size);
+    }
 
     return allocation;
 }

level_zero/core/test/unit_tests/sources/device/test_device.cpp

@@ -90,6 +90,65 @@ TEST_F(DeviceTest, givenEmptySVmAllocStorageWhenAllocateMemoryFromHostPtrThenVal
     neoDevice->getMemoryManager()->freeGraphicsMemory(allocation);
 }
 
+struct MemoryManagerHostPointer : public NEO::OsAgnosticMemoryManager {
+    MemoryManagerHostPointer(NEO::ExecutionEnvironment &executionEnvironment) : OsAgnosticMemoryManager(const_cast<NEO::ExecutionEnvironment &>(executionEnvironment)) {}
+    GraphicsAllocation *allocateGraphicsMemoryWithProperties(const AllocationProperties &properties,
+                                                             const void *ptr) override {
+        return nullptr;
+    }
+};
+
+struct DeviceHostPointerTest : public ::testing::Test {
+    void SetUp() override {
+        executionEnvironment = new NEO::ExecutionEnvironment();
+        executionEnvironment->prepareRootDeviceEnvironments(numRootDevices);
+        for (uint32_t i = 0; i < numRootDevices; i++) {
+            executionEnvironment->rootDeviceEnvironments[i]->setHwInfo(NEO::defaultHwInfo.get());
+        }
+
+        memoryManager = new MemoryManagerHostPointer(*executionEnvironment);
+        executionEnvironment->memoryManager.reset(memoryManager);
+
+        neoDevice = NEO::MockDevice::create<NEO::MockDevice>(executionEnvironment, rootDeviceIndex);
+        std::vector<std::unique_ptr<NEO::Device>> devices;
+        devices.push_back(std::unique_ptr<NEO::Device>(neoDevice));
+
+        driverHandle = std::make_unique<Mock<L0::DriverHandleImp>>();
+        driverHandle->initialize(std::move(devices));
+
+        device = driverHandle->devices[0];
+    }
+    void TearDown() override {
+    }
+
+    NEO::ExecutionEnvironment *executionEnvironment = nullptr;
+    std::unique_ptr<Mock<L0::DriverHandleImp>> driverHandle;
+    NEO::MockDevice *neoDevice = nullptr;
+    L0::Device *device = nullptr;
+    MemoryManagerHostPointer *memoryManager = nullptr;
+    const uint32_t rootDeviceIndex = 1u;
+    const uint32_t numRootDevices = 2u;
+};
+
+TEST_F(DeviceHostPointerTest, givenHostPointerNotAcceptedByKernelThenNewAllocationIsCreatedAndHostPointerCopied) {
+    size_t size = 55;
+    uint64_t *buffer = new uint64_t[size];
+    for (uint32_t i = 0; i < size; i++) {
+        buffer[i] = i + 10;
+    }
+
+    auto allocation = device->allocateMemoryFromHostPtr(buffer, size);
+    EXPECT_NE(nullptr, allocation);
+    EXPECT_EQ(NEO::GraphicsAllocation::AllocationType::INTERNAL_HOST_MEMORY, allocation->getAllocationType());
+    EXPECT_EQ(rootDeviceIndex, allocation->getRootDeviceIndex());
+    EXPECT_NE(allocation->getUnderlyingBuffer(), reinterpret_cast<void *>(buffer));
+    EXPECT_EQ(allocation->getUnderlyingBufferSize(), size);
+    EXPECT_EQ(0, memcmp(buffer, allocation->getUnderlyingBuffer(), size));
+
+    neoDevice->getMemoryManager()->freeGraphicsMemory(allocation);
+    delete[] buffer;
+}
+
 TEST_F(DeviceTest, givenKernelPropertiesStructureWhenKernelPropertiesCalledThenAllPropertiesAreAssigned) {
     const auto &hardwareInfo = this->neoDevice->getHardwareInfo();
 

shared/source/command_stream/command_stream_receiver.cpp

@@ -483,12 +483,10 @@ bool CommandStreamReceiver::createAllocationForHostSurface(HostPtrSurface &surfa
         allocation.reset(memoryManager->allocateGraphicsMemoryWithProperties(properties, surface.getMemoryPointer()));
         if (allocation == nullptr && surface.peekIsPtrCopyAllowed()) {
             // Try with no host pointer allocation and copy
-            AllocationProperties copyProperties{rootDeviceIndex, surface.getSurfaceSize(), GraphicsAllocation::AllocationType::INTERNAL_HOST_MEMORY, internalAllocationStorage->getDeviceBitfield()};
-            copyProperties.alignment = MemoryConstants::pageSize;
-            allocation.reset(memoryManager->allocateGraphicsMemoryWithProperties(copyProperties));
-            if (allocation) {
-                memcpy_s(allocation->getUnderlyingBuffer(), allocation->getUnderlyingBufferSize(), surface.getMemoryPointer(), surface.getSurfaceSize());
-            }
+            allocation.reset(memoryManager->allocateInternalGraphicsMemoryWithHostCopy(rootDeviceIndex,
+                                                                                       internalAllocationStorage->getDeviceBitfield(),
+                                                                                       surface.getMemoryPointer(),
+                                                                                       surface.getSurfaceSize()));
         }
     }
 

shared/source/memory_manager/memory_manager.cpp

@@ -422,6 +422,22 @@ GraphicsAllocation *MemoryManager::allocateGraphicsMemoryInPreferredPool(const A
     return allocation;
 }
 
+GraphicsAllocation *MemoryManager::allocateInternalGraphicsMemoryWithHostCopy(uint32_t rootDeviceIndex,
+                                                                              DeviceBitfield bitField,
+                                                                              const void *ptr,
+                                                                              size_t size) {
+    NEO::AllocationProperties copyProperties{rootDeviceIndex,
+                                             size,
+                                             NEO::GraphicsAllocation::AllocationType::INTERNAL_HOST_MEMORY,
+                                             bitField};
+    copyProperties.alignment = MemoryConstants::pageSize;
+    auto allocation = this->allocateGraphicsMemoryWithProperties(copyProperties);
+    if (allocation) {
+        memcpy_s(allocation->getUnderlyingBuffer(), allocation->getUnderlyingBufferSize(), ptr, size);
+    }
+    return allocation;
+}
+
 bool MemoryManager::mapAuxGpuVA(GraphicsAllocation *graphicsAllocation) {
     auto index = graphicsAllocation->getRootDeviceIndex();
     if (executionEnvironment.rootDeviceEnvironments[index]->pageTableManager.get()) {

shared/source/memory_manager/memory_manager.h

@@ -76,6 +76,8 @@ class MemoryManager {
         return allocateGraphicsMemoryInPreferredPool(properties, ptr);
     }
 
+    GraphicsAllocation *allocateInternalGraphicsMemoryWithHostCopy(uint32_t rootDeviceIndex, DeviceBitfield bitField, const void *ptr, size_t size);
+
     MOCKABLE_VIRTUAL GraphicsAllocation *allocateGraphicsMemoryInPreferredPool(const AllocationProperties &properties, const void *hostPtr);
 
     virtual bool verifyHandle(osHandle handle, uint32_t rootDeviceIndex, bool) { return true; }