Add support for making buffer resident for their lifetime.

Change-Id: Idde4c561b0968858202a78719942f30ab89ffde6
Signed-off-by: Michal Mrozek <michal.mrozek@intel.com>

Author: MichalMrozek

core/debug_settings/debug_variables_base.inl

@@ -110,6 +110,7 @@ DECLARE_DEBUG_VARIABLE(bool, EnableFreeMemory, false, "Enable freeMemory in memo
 DECLARE_DEBUG_VARIABLE(bool, ForceSamplerLowFilteringPrecision, false, "Force Low Filtering Precision Sampler mode")
 DECLARE_DEBUG_VARIABLE(bool, UseBindlessBuffers, false, "Force compiler to use bindless buffer addressing instead of stateful one")
 DECLARE_DEBUG_VARIABLE(bool, UseBindlessImages, false, "Force compiler to use bindless image addressing instead of stateful one")
+DECLARE_DEBUG_VARIABLE(bool, MakeAllBuffersResident, false, "Make all buffers resident after creation")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableIntelVme, -1, "-1: default, 0: disabled, 1: Enables cl_intel_motion_estimation extension")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableIntelAdvancedVme, -1, "-1: default, 0: disabled, 1: Enables cl_intel_advanced_motion_estimation extension")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableBlitterOperationsSupport, -1, "-1: default, 0: disable, 1: enable")

runtime/mem_obj/buffer.cpp

@@ -8,6 +8,7 @@
 #include "runtime/mem_obj/buffer.h"
 
 #include "core/debug_settings/debug_settings_manager.h"
+#include "core/execution_environment/root_device_environment.h"
 #include "core/gmm_helper/gmm.h"
 #include "core/gmm_helper/gmm_helper.h"
 #include "core/helpers/aligned_memory.h"
@@ -18,6 +19,7 @@
 #include "core/helpers/timestamp_packet.h"
 #include "core/memory_manager/host_ptr_manager.h"
 #include "core/memory_manager/memory_manager.h"
+#include "core/memory_manager/memory_operations_handler.h"
 #include "core/memory_manager/unified_memory_manager.h"
 #include "runtime/command_queue/command_queue.h"
 #include "runtime/command_stream/command_stream_receiver.h"
@@ -320,6 +322,11 @@ Buffer *Buffer::create(Context *context,
         return nullptr;
     }
 
+    if (DebugManager.flags.MakeAllBuffersResident.get()) {
+        auto graphicsAllocation = pBuffer->getGraphicsAllocation();
+        context->getDevice(0u)->getRootDeviceEnvironment().memoryOperationsInterface->makeResident(ArrayRef<GraphicsAllocation *>(&graphicsAllocation, 1));
+    }
+
     return pBuffer;
 }
 

unit_tests/mem_obj/buffer_tests.cpp

@@ -11,9 +11,11 @@
 #include "core/helpers/array_count.h"
 #include "core/helpers/hw_helper.h"
 #include "core/memory_manager/allocations_list.h"
+#include "core/memory_manager/memory_operations_handler.h"
 #include "core/memory_manager/unified_memory_manager.h"
 #include "core/os_interface/os_context.h"
 #include "core/unit_tests/helpers/debug_manager_state_restore.h"
+#include "core/unit_tests/helpers/ult_hw_config.h"
 #include "core/unit_tests/utilities/base_object_utils.h"
 #include "public/cl_ext_private.h"
 #include "runtime/command_queue/command_queue_hw.h"
@@ -29,6 +31,7 @@
 #include "unit_tests/gen_common/matchers.h"
 #include "unit_tests/helpers/hw_parse.h"
 #include "unit_tests/helpers/unit_test_helper.h"
+#include "unit_tests/helpers/variable_backup.h"
 #include "unit_tests/mocks/mock_buffer.h"
 #include "unit_tests/mocks/mock_command_queue.h"
 #include "unit_tests/mocks/mock_context.h"
@@ -1831,6 +1834,24 @@ TEST(SharedBuffersTest, whenBuffersIsCreatedWithSharingHandlerThenItIsSharedBuff
     buffer->release();
 }
 
+TEST(ResidencyTests, whenBuffersIsCreatedWithMakeResidentFlagThenItSuccessfulyCreates) {
+    VariableBackup<UltHwConfig> backup(&ultHwConfig);
+    ultHwConfig.useMockedGetDevicesFunc = false;
+    DebugManagerStateRestore restorer;
+    DebugManager.flags.MakeAllBuffersResident.set(true);
+
+    size_t numRootDevices = 0u;
+    getDevices(numRootDevices, *platform()->peekExecutionEnvironment());
+    platform()->initialize(1u, 0u);
+    auto device = platform()->getClDevice(0u);
+
+    MockContext context(device, false);
+    auto retValue = CL_SUCCESS;
+    auto clBuffer = clCreateBuffer(&context, 0u, 4096u, nullptr, &retValue);
+    ASSERT_EQ(retValue, CL_SUCCESS);
+    clReleaseMemObject(clBuffer);
+}
+
 class BufferTests : public ::testing::Test {
   protected:
     void SetUp() override {

unit_tests/os_interface/windows/wddm_residency_handler_tests.cpp

@@ -6,9 +6,14 @@
  */
 
 #include "core/os_interface/windows/wddm_memory_operations_handler.h"
+#include "core/unit_tests/helpers/debug_manager_state_restore.h"
+#include "core/unit_tests/helpers/ult_hw_config.h"
 #include "core/utilities/stackvec.h"
+#include "runtime/device/cl_device.h"
 #include "test.h"
+#include "unit_tests/helpers/variable_backup.h"
 #include "unit_tests/mocks/mock_allocation_properties.h"
+#include "unit_tests/mocks/mock_context.h"
 #include "unit_tests/mocks/mock_wddm.h"
 #include "unit_tests/os_interface/windows/mock_wddm_allocation.h"
 #include "unit_tests/os_interface/windows/wddm_fixture.h"
@@ -84,3 +89,29 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenVariousAllocationsWhenEvictingResid
     EXPECT_EQ(wddmMemoryOperationsHandler->evict(wddmFragmentedAllocation), MemoryOperationsStatus::SUCCESS);
     EXPECT_EQ(wddmMemoryOperationsHandler->isResident(wddmFragmentedAllocation), MemoryOperationsStatus::MEMORY_NOT_FOUND);
 }
+
+TEST(WddmResidentBufferTests, whenBuffersIsCreatedWithMakeResidentFlagSetThenItIsMadeResidentUponCreation) {
+    VariableBackup<UltHwConfig> backup(&ultHwConfig);
+    ultHwConfig.useMockedGetDevicesFunc = false;
+    DebugManagerStateRestore restorer;
+    DebugManager.flags.MakeAllBuffersResident.set(true);
+
+    size_t numRootDevices = 0u;
+    getDevices(numRootDevices, *platform()->peekExecutionEnvironment());
+    platform()->initialize(1u, 0u);
+    auto device = platform()->getClDevice(0u);
+
+    MockContext context(device, false);
+    auto retValue = CL_SUCCESS;
+    auto clBuffer = clCreateBuffer(&context, 0u, 4096u, nullptr, &retValue);
+    ASSERT_EQ(retValue, CL_SUCCESS);
+
+    auto memoryOperationsHandler = context.getDevice(0)->getRootDeviceEnvironment().memoryOperationsInterface.get();
+    auto neoBuffer = castToObject<MemObj>(clBuffer);
+    auto bufferAllocation = neoBuffer->getGraphicsAllocation();
+    auto status = memoryOperationsHandler->isResident(*bufferAllocation);
+
+    EXPECT_EQ(status, MemoryOperationsStatus::SUCCESS);
+
+    clReleaseMemObject(clBuffer);
+}

unit_tests/test_files/igdrcl.config

@@ -128,4 +128,5 @@ UseBindlessImages = 0
 PrintProgramBinaryProcessingTime = 0
 OverrideGpuAddressSpace = -1
 OverrideMaxWorkgroupSize = -1
-DisableTimestampPacketOptimizations = 0
+DisableTimestampPacketOptimizations = 0
+MakeAllBuffersResident = 0