Turn ON DrmGemCloseWorker.

- For every command buffer that we submit, pass it to gem close worker.
- Gem close worker will do asynchronous cleanup if this resource is meant to
be destroyed.
- if the resource is not meant to be destroyed we will call IOCTL wait for
this batch buffer.
- This will result in bumping up GPU clocks and better performance.

Change-Id: If9f181e411d7748573f31682e875a97c5355abe5

Author: MichalMrozek

runtime/os_interface/linux/drm_command_stream.h

@@ -48,7 +48,7 @@ class DrmCommandStreamReceiver : public DeviceCommandStreamReceiver<GfxFamily> {
   public:
     // When drm is null default implementation is used. In this case DrmCommandStreamReceiver is responsible to free drm.
     // When drm is passed, DCSR will not free it at destruction
-    DrmCommandStreamReceiver(const HardwareInfo &hwInfoIn, Drm *drm, gemCloseWorkerMode mode = gemCloseWorkerMode::gemCloseWorkerInactive);
+    DrmCommandStreamReceiver(const HardwareInfo &hwInfoIn, Drm *drm, gemCloseWorkerMode mode = gemCloseWorkerMode::gemCloseWorkerActive);
 
     FlushStamp flush(BatchBuffer &batchBuffer, EngineType engineType, ResidencyContainer *allocationsForResidency) override;
     void makeResident(GraphicsAllocation &gfxAllocation) override;

runtime/os_interface/linux/drm_command_stream.inl

@@ -79,6 +79,11 @@ FlushStamp DrmCommandStreamReceiver<GfxFamily>::flush(BatchBuffer &batchBuffer,
                  batchBuffer.low_priority);
 
         bb->getResidency()->clear();
+
+        if (this->gemCloseWorkerOperationMode == gemCloseWorkerActive) {
+            bb->reference();
+            this->getMemoryManager()->peekGemCloseWorker()->push(bb);
+        }
     }
 
     return flushStamp;

runtime/os_interface/linux/drm_memory_manager.h

@@ -75,6 +75,8 @@ class DrmMemoryManager : public MemoryManager {
         return validateHostPtrMemory;
     }
 
+    DrmGemCloseWorker *peekGemCloseWorker() { return this->gemCloseWorker.get(); }
+
   protected:
     BufferObject *findAndReferenceSharedBufferObject(int boHandle);
     BufferObject *createSharedBufferObject(int boHandle, size_t size, bool requireSpecificBitness);

unit_tests/os_interface/linux/device_command_stream_tests.cpp

@@ -47,13 +47,13 @@ HWTEST_F(DeviceCommandStreamLeaksTest, Create) {
 HWTEST_F(DeviceCommandStreamLeaksTest, givenDefaultDrmCsrWhenItIsCreatedThenGemCloseWorkerInactiveModeIsSelected) {
     std::unique_ptr<CommandStreamReceiver> ptr(DeviceCommandStreamReceiver<FamilyType>::create(*platformDevices[0], false));
     auto drmCsr = (DrmCommandStreamReceiver<FamilyType> *)ptr.get();
-    EXPECT_EQ(drmCsr->peekGemCloseWorkerOperationMode(), gemCloseWorkerMode::gemCloseWorkerInactive);
+    EXPECT_EQ(drmCsr->peekGemCloseWorkerOperationMode(), gemCloseWorkerMode::gemCloseWorkerActive);
 }
 
 HWTEST_F(DeviceCommandStreamLeaksTest, givenDefaultDrmCsrWithAubDumWhenItIsCreatedThenGemCloseWorkerInactiveModeIsSelected) {
     std::unique_ptr<CommandStreamReceiver> ptr(DeviceCommandStreamReceiver<FamilyType>::create(*platformDevices[0], true));
     auto drmCsrWithAubDump = (CommandStreamReceiverWithAUBDump<DrmCommandStreamReceiver<FamilyType>> *)ptr.get();
-    EXPECT_EQ(drmCsrWithAubDump->peekGemCloseWorkerOperationMode(), gemCloseWorkerMode::gemCloseWorkerInactive);
+    EXPECT_EQ(drmCsrWithAubDump->peekGemCloseWorkerOperationMode(), gemCloseWorkerMode::gemCloseWorkerActive);
     auto aubCSR = static_cast<CommandStreamReceiverWithAUBDump<DrmCommandStreamReceiver<FamilyType>> *>(ptr.get())->aubCSR;
     EXPECT_NE(nullptr, aubCSR);
 }

unit_tests/os_interface/linux/drm_command_stream_tests.cpp

@@ -50,7 +50,7 @@ using namespace OCLRT;
 class DrmCommandStreamFixture {
   public:
     DeviceCommandStreamReceiver<DEFAULT_TEST_FAMILY_NAME> *csr = nullptr;
-    MemoryManager *mm = nullptr;
+    DrmMemoryManager *mm = nullptr;
     DrmMockImpl *mock;
     const int mockFd = 33;
 
@@ -62,13 +62,13 @@ class DrmCommandStreamFixture {
 
         this->mock = new DrmMockImpl(mockFd);
 
-        csr = new DrmCommandStreamReceiver<DEFAULT_TEST_FAMILY_NAME>(*platformDevices[0], mock, gemCloseWorkerMode::gemCloseWorkerInactive);
+        csr = new DrmCommandStreamReceiver<DEFAULT_TEST_FAMILY_NAME>(*platformDevices[0], mock, gemCloseWorkerMode::gemCloseWorkerActive);
         ASSERT_NE(nullptr, csr);
 
         // Memory manager creates pinBB with ioctl, expect one call
         EXPECT_CALL(*mock, ioctl(::testing::_, ::testing::_))
             .Times(1);
-        mm = csr->createMemoryManager(false);
+        mm = static_cast<DrmMemoryManager *>(csr->createMemoryManager(false));
         ::testing::Mock::VerifyAndClearExpectations(mock);
 
         //assert we have memory manager
@@ -77,6 +77,7 @@ class DrmCommandStreamFixture {
 
     void TearDown() {
         mm->waitForDeletions();
+        mm->peekGemCloseWorker()->close(true);
         delete csr;
         ::testing::Mock::VerifyAndClearExpectations(mock);
         // Memory manager closes pinBB with ioctl, expect one call
@@ -199,7 +200,7 @@ TEST_F(DrmCommandStreamTest, Flush) {
         .WillRepeatedly(::testing::Return(0))
         .RetiresOnSaturation();
     EXPECT_CALL(*mock, ioctl(DRM_IOCTL_I915_GEM_WAIT, ::testing::_))
-        .Times(1)
+        .Times(2)
         .RetiresOnSaturation();
     EXPECT_CALL(*mock, ioctl(DRM_IOCTL_GEM_CLOSE, ::testing::_))
         .Times(1)
@@ -235,7 +236,7 @@ TEST_F(DrmCommandStreamTest, FlushWithLowPriorityContext) {
         .WillRepeatedly(::testing::Return(0))
         .RetiresOnSaturation();
     EXPECT_CALL(*mock, ioctl(DRM_IOCTL_I915_GEM_WAIT, ::testing::_))
-        .Times(1)
+        .Times(2)
         .RetiresOnSaturation();
     EXPECT_CALL(*mock, ioctl(DRM_IOCTL_GEM_CLOSE, ::testing::_))
         .Times(1)
@@ -298,7 +299,7 @@ TEST_F(DrmCommandStreamTest, FlushNotEmptyBB) {
         .WillRepeatedly(::testing::Return(0))
         .RetiresOnSaturation();
     EXPECT_CALL(*mock, ioctl(DRM_IOCTL_I915_GEM_WAIT, ::testing::_))
-        .Times(1)
+        .Times(2)
         .RetiresOnSaturation();
     EXPECT_CALL(*mock, ioctl(DRM_IOCTL_GEM_CLOSE, ::testing::_))
         .Times(1)
@@ -327,7 +328,7 @@ TEST_F(DrmCommandStreamTest, FlushNotEmptyNotPaddedBB) {
         .WillRepeatedly(::testing::Return(0))
         .RetiresOnSaturation();
     EXPECT_CALL(*mock, ioctl(DRM_IOCTL_I915_GEM_WAIT, ::testing::_))
-        .Times(1)
+        .Times(2)
         .RetiresOnSaturation();
     EXPECT_CALL(*mock, ioctl(DRM_IOCTL_GEM_CLOSE, ::testing::_))
         .Times(1)
@@ -364,7 +365,7 @@ TEST_F(DrmCommandStreamTest, FlushNotAligned) {
     EXPECT_CALL(*mock, ioctl(DRM_IOCTL_GEM_CLOSE, ::testing::_))
         .Times(1);
     EXPECT_CALL(*mock, ioctl(DRM_IOCTL_I915_GEM_WAIT, ::testing::_))
-        .Times(1);
+        .Times(2);
 
     csr->addBatchBufferEnd(cs, nullptr);
     csr->alignToCacheLine(cs);
@@ -447,7 +448,7 @@ TEST_F(DrmCommandStreamTest, CheckDrmFree) {
     EXPECT_CALL(*mock, ioctl(DRM_IOCTL_GEM_CLOSE, GemCloseEq(17u)))
         .Times(1);
     EXPECT_CALL(*mock, ioctl(DRM_IOCTL_I915_GEM_WAIT, ::testing::_))
-        .Times(1);
+        .Times(2);
 
     DrmAllocation allocation(nullptr, nullptr, 1024);
 
@@ -489,7 +490,7 @@ TEST_F(DrmCommandStreamTest, CheckDrmFreeCloseFailed) {
         .Times(1)
         .WillOnce(::testing::Return(-1));
     EXPECT_CALL(*mock, ioctl(DRM_IOCTL_I915_GEM_WAIT, ::testing::_))
-        .Times(1);
+        .Times(2);
     DrmAllocation allocation(nullptr, nullptr, 1024);
 
     csr->makeResident(allocation);
@@ -787,17 +788,11 @@ TEST_F(DrmCommandStreamGemWorkerTests, givenDefaultDrmCSRWhenItIsCreatedThenGemC
     EXPECT_EQ(gemCloseWorkerMode::gemCloseWorkerInactive, tCsr->peekGemCloseWorkerOperationMode());
 }
 
-TEST_F(DrmCommandStreamGemWorkerTests, givenCommandStreamWhenItIsFlushedWithGemCloseWorkerInactiveModeThenCsIsNotNulled) {
-    tCsr->overrideGemCloseWorkerOperationMode(gemCloseWorkerMode::gemCloseWorkerInactive);
-
+TEST_F(DrmCommandStreamGemWorkerTests, givenCommandStreamWhenItIsFlushedWithGemCloseWorkerInDefaultModeThenWorkerDecreasesTheRefCount) {
     auto commandBuffer = mm->allocateGraphicsMemory(1024, 4096);
-    auto dummyAllocation = mm->allocateGraphicsMemory(1024, 4096);
     ASSERT_NE(nullptr, commandBuffer);
-    ASSERT_EQ(0u, reinterpret_cast<uintptr_t>(commandBuffer->getUnderlyingBuffer()) & 0xFFF);
     LinearStream cs(commandBuffer);
 
-    csr->makeResident(*dummyAllocation);
-
     csr->addBatchBufferEnd(cs, nullptr);
     csr->alignToCacheLine(cs);
     auto storedBase = cs.getCpuBase();
@@ -813,12 +808,14 @@ TEST_F(DrmCommandStreamGemWorkerTests, givenCommandStreamWhenItIsFlushedWithGemC
     //no allocations should be connected
     EXPECT_EQ(bo->getResidency()->size(), 0u);
 
-    mm->freeGraphicsMemory(dummyAllocation);
+    //spin until gem close worker finishes execution
+    while (bo->getRefCount() > 1)
+        ;
+
     mm->freeGraphicsMemory(commandBuffer);
 }
 
 TEST_F(DrmCommandStreamGemWorkerTests, givenTaskThatRequiresLargeResourceCountWhenItIsFlushedThenExecStorageIsResized) {
-    tCsr->overrideGemCloseWorkerOperationMode(gemCloseWorkerMode::gemCloseWorkerInactive);
     std::vector<GraphicsAllocation *> graphicsAllocations;
 
     auto &execStorage = tCsr->getExecStorage();
@@ -847,9 +844,6 @@ TEST_F(DrmCommandStreamGemWorkerTests, givenTaskThatRequiresLargeResourceCountWh
 }
 
 TEST_F(DrmCommandStreamGemWorkerTests, givenGemCloseWorkerInactiveModeWhenMakeResidentIsCalledThenRefCountsAreNotUpdated) {
-
-    tCsr->overrideGemCloseWorkerOperationMode(gemCloseWorkerMode::gemCloseWorkerInactive);
-
     auto dummyAllocation = mm->allocateGraphicsMemory(1024, 4096);
 
     auto bo = dummyAllocation->getBO();
@@ -867,8 +861,6 @@ TEST_F(DrmCommandStreamGemWorkerTests, givenGemCloseWorkerInactiveModeWhenMakeRe
 }
 
 TEST_F(DrmCommandStreamGemWorkerTests, givenCommandStreamWithDuplicatesWhenItIsFlushedWithGemCloseWorkerInactiveModeThenCsIsNotNulled) {
-    tCsr->overrideGemCloseWorkerOperationMode(gemCloseWorkerMode::gemCloseWorkerInactive);
-
     auto commandBuffer = mm->allocateGraphicsMemory(1024, 4096);
     auto dummyAllocation = mm->allocateGraphicsMemory(1024, 4096);
     ASSERT_NE(nullptr, commandBuffer);
@@ -917,7 +909,6 @@ class DrmCommandStreamBatchingTests : public Test<DrmCommandStreamEnhancedFixtur
 };
 
 TEST_F(DrmCommandStreamBatchingTests, givenCSRWhenFlushIsCalledThenProperFlagsArePassed) {
-    tCsr->overrideGemCloseWorkerOperationMode(gemCloseWorkerMode::gemCloseWorkerInactive);
     auto commandBuffer = mm->allocateGraphicsMemory(1024, 4096);
     auto dummyAllocation = mm->allocateGraphicsMemory(1024, 4096);
     ASSERT_NE(nullptr, commandBuffer);
@@ -944,7 +935,6 @@ TEST_F(DrmCommandStreamBatchingTests, givenCSRWhenFlushIsCalledThenProperFlagsAr
 
 TEST_F(DrmCommandStreamBatchingTests, givenCsrWhenDispatchPolicyIsSetToBatchingThenCommandBufferIsNotSubmitted) {
     tCsr->overrideDispatchPolicy(DispatchMode::BatchedDispatch);
-    tCsr->overrideGemCloseWorkerOperationMode(gemCloseWorkerMode::gemCloseWorkerInactive);
 
     auto mockedSubmissionsAggregator = new mockSubmissionsAggregator();
     tCsr->overrideSubmissionAggregator(mockedSubmissionsAggregator);
@@ -1005,7 +995,6 @@ TEST_F(DrmCommandStreamBatchingTests, givenCsrWhenDispatchPolicyIsSetToBatchingT
 
 TEST_F(DrmCommandStreamBatchingTests, givenRecordedCommandBufferWhenItIsSubmittedThenFlushTaskIsProperlyCalled) {
     tCsr->overrideDispatchPolicy(DispatchMode::BatchedDispatch);
-    tCsr->overrideGemCloseWorkerOperationMode(gemCloseWorkerMode::gemCloseWorkerInactive);
 
     auto mockedSubmissionsAggregator = new mockSubmissionsAggregator();
     tCsr->overrideSubmissionAggregator(mockedSubmissionsAggregator);

unit_tests/os_interface/linux/drm_memory_manager_tests.cpp

@@ -375,25 +375,13 @@ TEST_F(DrmMemoryManagerTest, UnreferenceNullPtr) {
 }
 
 TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDrmMemoryManagerCreatedWithGemCloseWorkerModeInactiveThenGemCloseWorkerIsNotCreated) {
-    class MyTestedDrmMemoryManager : public DrmMemoryManager {
-      public:
-        MyTestedDrmMemoryManager(Drm *drm, gemCloseWorkerMode mode) : DrmMemoryManager(drm, mode, false, false) {}
-        DrmGemCloseWorker *getgemCloseWorker() { return this->gemCloseWorker.get(); }
-    };
-
-    MyTestedDrmMemoryManager drmMemoryManger(this->mock, gemCloseWorkerMode::gemCloseWorkerInactive);
-    EXPECT_EQ(nullptr, drmMemoryManger.getgemCloseWorker());
+    DrmMemoryManager drmMemoryManger(this->mock, gemCloseWorkerMode::gemCloseWorkerInactive, false, false);
+    EXPECT_EQ(nullptr, drmMemoryManger.peekGemCloseWorker());
 }
 
 TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDrmMemoryManagerCreatedWithGemCloseWorkerActiveThenGemCloseWorkerIsCreated) {
-    class MyTestedDrmMemoryManager : public DrmMemoryManager {
-      public:
-        MyTestedDrmMemoryManager(Drm *drm, gemCloseWorkerMode mode) : DrmMemoryManager(drm, mode, false, false) {}
-        DrmGemCloseWorker *getgemCloseWorker() { return this->gemCloseWorker.get(); }
-    };
-
-    MyTestedDrmMemoryManager drmMemoryManger(this->mock, gemCloseWorkerMode::gemCloseWorkerActive);
-    EXPECT_NE(nullptr, drmMemoryManger.getgemCloseWorker());
+    DrmMemoryManager drmMemoryManger(this->mock, gemCloseWorkerMode::gemCloseWorkerActive, false, false);
+    EXPECT_NE(nullptr, drmMemoryManger.peekGemCloseWorker());
 }
 
 TEST_F(DrmMemoryManagerTest, AllocateThenFree) {