Add direct submission termination mechanism

Signed-off-by: Lukasz Jobczyk <lukasz.jobczyk@intel.com>

Author: LukaszJobczyk

opencl/test/unit_test/command_stream/command_stream_receiver_tests.cpp

@@ -28,6 +28,7 @@
 #include "shared/test/common/mocks/ult_device_factory.h"
 #include "shared/test/common/test_macros/matchers.h"
 #include "shared/test/common/test_macros/test_checks_shared.h"
+#include "shared/test/unit_test/direct_submission/direct_submission_controller_mock.h"
 
 #include "opencl/source/command_stream/definitions/command_stream_receiver_simulated_hw.h"
 #include "opencl/source/mem_obj/buffer.h"
@@ -393,6 +394,37 @@ struct InitDirectSubmissionFixture {
 
 using InitDirectSubmissionTest = Test<InitDirectSubmissionFixture>;
 
+HWTEST_F(InitDirectSubmissionTest, givenDirectSubmissionControllerEnabledWhenInitDirectSubmissionThenCsrIsRegistered) {
+    DebugManagerStateRestore restorer;
+    DebugManager.flags.EnableDirectSubmissionController.set(1);
+
+    auto csr = std::make_unique<CommandStreamReceiverHw<FamilyType>>(*device->executionEnvironment, device->getRootDeviceIndex(), device->getDeviceBitfield());
+    std::unique_ptr<OsContext> osContext(OsContext::create(device->getExecutionEnvironment()->rootDeviceEnvironments[0]->osInterface.get(), 0,
+                                                           EngineDescriptorHelper::getDefaultDescriptor({aub_stream::ENGINE_RCS, EngineUsage::Regular},
+                                                                                                        PreemptionMode::ThreadGroup, device->getDeviceBitfield())));
+
+    auto controller = static_cast<DirectSubmissionControllerMock *>(device->executionEnvironment->getDirectSubmissionController());
+    controller->keepControlling.store(false);
+    EXPECT_EQ(controller->directSubmissions.size(), 0u);
+
+    osContext->ensureContextInitialized();
+    osContext->setDefaultContext(true);
+    auto hwInfo = device->getRootDeviceEnvironment().getMutableHardwareInfo();
+    hwInfo->capabilityTable.directSubmissionEngines.data[aub_stream::ENGINE_RCS].engineSupported = true;
+    hwInfo->capabilityTable.directSubmissionEngines.data[aub_stream::ENGINE_RCS].submitOnInit = false;
+
+    bool ret = csr->initDirectSubmission(*device, *osContext.get());
+    EXPECT_TRUE(ret);
+    EXPECT_TRUE(csr->isDirectSubmissionEnabled());
+    EXPECT_FALSE(csr->isBlitterDirectSubmissionEnabled());
+
+    EXPECT_EQ(controller->directSubmissions.size(), 1u);
+    EXPECT_TRUE(controller->directSubmissions.find(csr.get()) != controller->directSubmissions.end());
+
+    csr.reset();
+    EXPECT_EQ(controller->directSubmissions.size(), 0u);
+}
+
 HWTEST_F(InitDirectSubmissionTest, whenDirectSubmissionEnabledOnRcsThenExpectFeatureAvailable) {
     auto csr = std::make_unique<CommandStreamReceiverHw<FamilyType>>(*device->executionEnvironment, device->getRootDeviceIndex(), device->getDeviceBitfield());
     std::unique_ptr<OsContext> osContext(OsContext::create(device->getExecutionEnvironment()->rootDeviceEnvironments[0]->osInterface.get(), 0,

opencl/test/unit_test/execution_environment/execution_environment_tests.cpp

@@ -11,6 +11,7 @@
 #include "shared/source/command_stream/preemption.h"
 #include "shared/source/compiler_interface/compiler_interface.h"
 #include "shared/source/device/device.h"
+#include "shared/source/direct_submission/direct_submission_controller.h"
 #include "shared/source/execution_environment/execution_environment.h"
 #include "shared/source/gmm_helper/gmm_helper.h"
 #include "shared/source/helpers/hw_helper.h"
@@ -159,6 +160,24 @@ TEST(ExecutionEnvironment, givenExecutionEnvironmentWhenInitializeMemoryManagerI
     EXPECT_EQ(enableLocalMemory, executionEnvironment->memoryManager->isLocalMemorySupported(device->getRootDeviceIndex()));
 }
 
+TEST(ExecutionEnvironment, givenEnableDirectSubmissionControllerSetWhenGetDirectSubmissionControllerThenNotNull) {
+    DebugManagerStateRestore restorer;
+    DebugManager.flags.EnableDirectSubmissionController.set(1);
+
+    auto controller = platform()->peekExecutionEnvironment()->getDirectSubmissionController();
+
+    EXPECT_NE(controller, nullptr);
+}
+
+TEST(ExecutionEnvironment, givenEnableDirectSubmissionControllerSetZeroWhenGetDirectSubmissionControllerThenNull) {
+    DebugManagerStateRestore restorer;
+    DebugManager.flags.EnableDirectSubmissionController.set(0);
+
+    auto controller = platform()->peekExecutionEnvironment()->getDirectSubmissionController();
+
+    EXPECT_EQ(controller, nullptr);
+}
+
 TEST(ExecutionEnvironment, givenExecutionEnvironmentWhenInitializeMemoryManagerIsCalledThenItIsInitalized) {
     ExecutionEnvironment *executionEnvironment = platform()->peekExecutionEnvironment();
     executionEnvironment->initializeMemoryManager();
@@ -167,16 +186,20 @@ TEST(ExecutionEnvironment, givenExecutionEnvironmentWhenInitializeMemoryManagerI
 static_assert(sizeof(ExecutionEnvironment) == sizeof(std::unique_ptr<HardwareInfo>) +
                                                   sizeof(std::vector<RootDeviceEnvironment>) +
                                                   sizeof(std::unique_ptr<OsEnvironment>) +
+                                                  sizeof(std::unique_ptr<DirectSubmissionController>) +
                                                   sizeof(bool) +
                                                   (is64bit ? 23 : 15),
               "New members detected in ExecutionEnvironment, please ensure that destruction sequence of objects is correct");
 
 TEST(ExecutionEnvironment, givenExecutionEnvironmentWithVariousMembersWhenItIsDestroyedThenDeleteSequenceIsSpecified) {
     uint32_t destructorId = 0u;
 
-    struct MemoryMangerMock : public DestructorCounted<MockMemoryManager, 7> {
+    struct MemoryMangerMock : public DestructorCounted<MockMemoryManager, 8> {
         MemoryMangerMock(uint32_t &destructorId, ExecutionEnvironment &executionEnvironment) : DestructorCounted(destructorId, executionEnvironment) {}
     };
+    struct DirectSubmissionControllerMock : public DestructorCounted<DirectSubmissionController, 7> {
+        DirectSubmissionControllerMock(uint32_t &destructorId) : DestructorCounted(destructorId) {}
+    };
     struct GmmHelperMock : public DestructorCounted<GmmHelper, 6> {
         GmmHelperMock(uint32_t &destructorId, const HardwareInfo *hwInfo) : DestructorCounted(destructorId, nullptr, hwInfo) {}
     };
@@ -212,9 +235,10 @@ TEST(ExecutionEnvironment, givenExecutionEnvironmentWithVariousMembersWhenItIsDe
     executionEnvironment->rootDeviceEnvironments[0]->builtins = std::make_unique<BuiltinsMock>(destructorId);
     executionEnvironment->rootDeviceEnvironments[0]->compilerInterface = std::make_unique<CompilerInterfaceMock>(destructorId);
     executionEnvironment->rootDeviceEnvironments[0]->debugger = std::make_unique<SourceLevelDebuggerMock>(destructorId);
+    executionEnvironment->directSubmissionController = std::make_unique<DirectSubmissionControllerMock>(destructorId);
 
     executionEnvironment.reset(nullptr);
-    EXPECT_EQ(8u, destructorId);
+    EXPECT_EQ(9u, destructorId);
 }
 
 TEST(ExecutionEnvironment, givenMultipleRootDevicesWhenTheyAreCreatedThenReuseMemoryManager) {

opencl/test/unit_test/test_files/igdrcl.config

@@ -98,6 +98,8 @@ DirectSubmissionNewResourceTlbFlush = -1
 DirectSubmissionDisableCacheFlush = -1
 DirectSubmissionDisableMonitorFence = -1
 USMEvictAfterMigration = 1
+EnableDirectSubmissionController = -1
+DirectSubmissionControllerTimeout = -1
 UseVmBind = -1
 PassBoundBOToExec = -1
 EnableNullHardware = 0

shared/source/command_stream/command_stream_receiver.h

@@ -241,6 +241,8 @@ class CommandStreamReceiver {
         return false;
     }
 
+    virtual void stopDirectSubmission() {}
+
     bool isStaticWorkPartitioningEnabled() const {
         return staticWorkPartitioningEnabled;
     }

shared/source/command_stream/command_stream_receiver_hw.h

@@ -53,7 +53,9 @@ class CommandStreamReceiverHw : public CommandStreamReceiver {
     static void addBatchBufferEnd(LinearStream &commandStream, void **patchLocation);
     void programEndingCmd(LinearStream &commandStream, Device &device, void **patchLocation, bool directSubmissionEnabled);
     void addBatchBufferStart(MI_BATCH_BUFFER_START *commandBufferMemory, uint64_t startAddress, bool secondary);
+
     static void alignToCacheLine(LinearStream &commandStream);
+    static void emitNoop(LinearStream &commandStream, size_t bytesToUpdate);
 
     size_t getRequiredStateBaseAddressSize(const Device &device) const;
     size_t getRequiredCmdStreamSize(const DispatchFlags &dispatchFlags, Device &device);
@@ -119,6 +121,8 @@ class CommandStreamReceiverHw : public CommandStreamReceiver {
         return blitterDirectSubmission.get() != nullptr;
     }
 
+    void stopDirectSubmission() override;
+
     virtual bool isKmdWaitModeActive() { return true; }
 
     bool initDirectSubmission(Device &device, OsContext &osContext) override;
@@ -157,8 +161,6 @@ class CommandStreamReceiverHw : public CommandStreamReceiver {
     uint64_t getScratchPatchAddress();
     void createScratchSpaceController();
 
-    static void emitNoop(LinearStream &commandStream, size_t bytesToUpdate);
-
     bool detectInitProgrammingFlagsRequired(const DispatchFlags &dispatchFlags) const;
     bool checkPlatformSupportsNewResourceImplicitFlush() const;
     bool checkPlatformSupportsGpuIdleImplicitFlush() const;

shared/source/command_stream/command_stream_receiver_hw_base.inl

@@ -13,6 +13,7 @@
 #include "shared/source/command_stream/stream_properties.h"
 #include "shared/source/debug_settings/debug_settings_manager.h"
 #include "shared/source/device/device.h"
+#include "shared/source/direct_submission/direct_submission_controller.h"
 #include "shared/source/direct_submission/direct_submission_hw.h"
 #include "shared/source/execution_environment/root_device_environment.h"
 #include "shared/source/gmm_helper/page_table_mngr.h"
@@ -38,7 +39,12 @@
 namespace NEO {
 
 template <typename GfxFamily>
-CommandStreamReceiverHw<GfxFamily>::~CommandStreamReceiverHw() = default;
+CommandStreamReceiverHw<GfxFamily>::~CommandStreamReceiverHw() {
+    auto directSubmissionController = executionEnvironment.getDirectSubmissionController();
+    if (directSubmissionController) {
+        directSubmissionController->unregisterDirectSubmission(this);
+    }
+}
 
 template <typename GfxFamily>
 CommandStreamReceiverHw<GfxFamily>::CommandStreamReceiverHw(ExecutionEnvironment &executionEnvironment,
@@ -1390,6 +1396,15 @@ inline size_t CommandStreamReceiverHw<GfxFamily>::getCmdSizeForPrologue() const
     return 0u;
 }
 
+template <typename GfxFamily>
+inline void CommandStreamReceiverHw<GfxFamily>::stopDirectSubmission() {
+    if (EngineHelpers::isBcs(this->osContext->getEngineType())) {
+        this->blitterDirectSubmission->stopRingBuffer();
+    } else {
+        this->directSubmission->stopRingBuffer();
+    }
+}
+
 template <typename GfxFamily>
 inline bool CommandStreamReceiverHw<GfxFamily>::initDirectSubmission(Device &device, OsContext &osContext) {
     bool ret = true;
@@ -1398,16 +1413,19 @@ inline bool CommandStreamReceiverHw<GfxFamily>::initDirectSubmission(Device &dev
     auto startDirect = osContext.isDirectSubmissionAvailable(device.getHardwareInfo(), submitOnInit);
 
     if (startDirect) {
-        if (EngineHelpers::isBcs(osContext.getEngineType())) {
-            if (!this->isBlitterDirectSubmissionEnabled()) {
+        if (!this->isBlitterDirectSubmissionEnabled() && !this->isDirectSubmissionEnabled()) {
+            if (EngineHelpers::isBcs(osContext.getEngineType())) {
                 blitterDirectSubmission = DirectSubmissionHw<GfxFamily, BlitterDispatcher<GfxFamily>>::create(device, osContext);
                 ret = blitterDirectSubmission->initialize(submitOnInit);
-            }
-        } else {
-            if (!this->isDirectSubmissionEnabled()) {
+
+            } else {
                 directSubmission = DirectSubmissionHw<GfxFamily, RenderDispatcher<GfxFamily>>::create(device, osContext);
                 ret = directSubmission->initialize(submitOnInit);
             }
+            auto directSubmissionController = executionEnvironment.getDirectSubmissionController();
+            if (directSubmissionController) {
+                directSubmissionController->registerDirectSubmission(this);
+            }
         }
         osContext.setDirectSubmissionActive();
     }

shared/source/debug_settings/debug_variables_base.inl

@@ -235,6 +235,8 @@ DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionDisableCacheFlush, -1, "-1: driv
 DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionNewResourceTlbFlush, -1, "-1: driver default - flush when new resource is bound, 0: disabled, 1: enabled")
 DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionDisableMonitorFence, -1, "Disable dispatching monitor fence commands")
 DECLARE_DEBUG_VARIABLE(bool, USMEvictAfterMigration, true, "Evict USM allocation after implicit migration to GPU")
+DECLARE_DEBUG_VARIABLE(int32_t, EnableDirectSubmissionController, -1, "Enable direct submission terminating after given timeout, -1: default, 0: disabled, 1: enabled")
+DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionControllerTimeout, -1, "Set direct submission controller timeout, -1: default 5 ms, >=0: timeout in ms")
 
 /*FEATURE FLAGS*/
 DECLARE_DEBUG_VARIABLE(bool, EnableNV12, true, "Enables NV12 extension")

shared/source/direct_submission/CMakeLists.txt

@@ -1,11 +1,13 @@
 #
-# Copyright (C) 2020 Intel Corporation
+# Copyright (C) 2020-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 #
 
 set(NEO_CORE_DIRECT_SUBMISSION
     ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
+    ${CMAKE_CURRENT_SOURCE_DIR}/direct_submission_controller.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/direct_submission_controller.h
     ${CMAKE_CURRENT_SOURCE_DIR}/direct_submission_hw.h
     ${CMAKE_CURRENT_SOURCE_DIR}/direct_submission_hw.inl
     ${CMAKE_CURRENT_SOURCE_DIR}/direct_submission_hw_diagnostic_mode.cpp

shared/source/direct_submission/direct_submission_controller.cpp

@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2019-2021 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/direct_submission/direct_submission_controller.h"
+
+#include "shared/source/command_stream/command_stream_receiver.h"
+
+#include <chrono>
+
+namespace NEO {
+
+DirectSubmissionController::DirectSubmissionController() {
+    timeout = 5;
+
+    if (DebugManager.flags.DirectSubmissionControllerTimeout.get() != -1) {
+        timeout = DebugManager.flags.DirectSubmissionControllerTimeout.get();
+    }
+
+    directSubmissionControllingThread = std::thread(&DirectSubmissionController::controlDirectSubmissionsState, this);
+};
+
+DirectSubmissionController::~DirectSubmissionController() {
+    keepControlling.store(false);
+    if (directSubmissionControllingThread.joinable()) {
+        directSubmissionControllingThread.join();
+    }
+}
+
+void DirectSubmissionController::registerDirectSubmission(CommandStreamReceiver *csr) {
+    std::lock_guard<std::mutex> lock(directSubmissionsMutex);
+    directSubmissions.insert(std::make_pair(csr, DirectSubmissionState{}));
+}
+
+void DirectSubmissionController::unregisterDirectSubmission(CommandStreamReceiver *csr) {
+    std::lock_guard<std::mutex> lock(directSubmissionsMutex);
+    directSubmissions.erase(csr);
+}
+
+void DirectSubmissionController::controlDirectSubmissionsState() {
+    while (true) {
+
+        auto start = std::chrono::steady_clock::now();
+        int diff = 0u;
+        do {
+            if (!keepControlling.load()) {
+                return;
+            }
+
+            auto timestamp = std::chrono::steady_clock::now();
+            diff = static_cast<int>(std::chrono::duration_cast<std::chrono::milliseconds>(timestamp - start).count());
+        } while (diff <= timeout);
+
+        this->checkNewSubmissions();
+    }
+}
+
+void DirectSubmissionController::checkNewSubmissions() {
+    std::lock_guard<std::mutex> lock(this->directSubmissionsMutex);
+
+    for (auto &directSubmission : this->directSubmissions) {
+        auto csr = directSubmission.first;
+        auto &state = directSubmission.second;
+
+        auto taskCount = csr->peekTaskCount();
+        if (taskCount <= *csr->getTagAddress()) {
+            if (taskCount == state.taskCount) {
+                if (state.isStopped) {
+                    continue;
+                } else {
+                    auto lock = csr->obtainUniqueOwnership();
+                    csr->stopDirectSubmission();
+                    state.isStopped = true;
+                }
+            } else {
+                state.isStopped = false;
+                state.taskCount = taskCount;
+            }
+        } else {
+            state.isStopped = false;
+        }
+    }
+}
+
+} // namespace NEO

shared/source/direct_submission/direct_submission_controller.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2019-2021 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include <atomic>
+#include <mutex>
+#include <thread>
+#include <unordered_map>
+
+namespace NEO {
+class MemoryManager;
+class CommandStreamReceiver;
+
+class DirectSubmissionController {
+  public:
+    DirectSubmissionController();
+    virtual ~DirectSubmissionController();
+
+    void registerDirectSubmission(CommandStreamReceiver *csr);
+    void unregisterDirectSubmission(CommandStreamReceiver *csr);
+
+  protected:
+    struct DirectSubmissionState {
+        bool isStopped = false;
+        uint32_t taskCount = 0u;
+    };
+
+    void controlDirectSubmissionsState();
+    void checkNewSubmissions();
+
+    std::unordered_map<CommandStreamReceiver *, DirectSubmissionState> directSubmissions;
+    std::mutex directSubmissionsMutex;
+
+    std::thread directSubmissionControllingThread;
+    std::atomic_bool keepControlling = true;
+
+    int timeout = 5;
+};
+} // namespace NEO