fix: correct signaling of partitioned aggregated event 2

BartoszDunajski · Compute-Runtime-Automation · commit e88b4d7b4c51 · 2025-09-29T16:21:38.000+02:00
Related-To: NEO-14557

Signed-off-by: Bartosz Dunajski &lt;bartosz.dunajski@intel.com&gt;
diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.inl b/level_zero/core/source/cmdlist/cmdlist_hw.inl
@@ -183,7 +183,7 @@ void CommandListCoreFamily<gfxCoreFamily>::handleInOrderDependencyCounter(Event
 
     this->addResidency(inOrderExecInfo->getDeviceCounterAllocation(), inOrderExecInfo->getHostCounterAllocation());
 
-    if (signalEvent && signalEvent->getInOrderIncrementValue() == 0) {
+    if (signalEvent && signalEvent->getInOrderIncrementValue(this->partitionCount) == 0) {
         if (signalEvent->isCounterBased() || nonWalkerInOrderCmdsChaining || (isImmediateType() && this->duplicatedInOrderCounterStorageEnabled)) {
             assignInOrderExecInfoToEvent(signalEvent);
         } else {
@@ -3322,8 +3322,10 @@ void CommandListCoreFamily<gfxCoreFamily>::appendSignalAggregatedEventAtomic(Eve
     using ATOMIC_OPCODES = typename GfxFamily::MI_ATOMIC::ATOMIC_OPCODES;
     using DATA_SIZE = typename GfxFamily::MI_ATOMIC::DATA_SIZE;
 
+    auto incValue = event.getInOrderIncrementValue(this->partitionCount);
+
     NEO::EncodeAtomic<GfxFamily>::programMiAtomic(*commandContainer.getCommandStream(), event.getInOrderExecInfo()->getBaseDeviceAddress(), ATOMIC_OPCODES::ATOMIC_8B_ADD,
-                                                  DATA_SIZE::DATA_SIZE_QWORD, 0, 0, event.getInOrderIncrementValue(), 0);
+                                                  DATA_SIZE::DATA_SIZE_QWORD, 0, 0, incValue, 0);
 }
 
 template <GFXCORE_FAMILY gfxCoreFamily>
diff --git a/level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl b/level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl
@@ -1962,7 +1962,7 @@ ze_result_t CommandListCoreFamilyImmediate<gfxCoreFamily>::appendStagingMemoryCo
         return ret;
     }
 
-    if (event && event->isCounterBased() && event->getInOrderIncrementValue() == 0) {
+    if (event && event->isCounterBased() && event->getInOrderIncrementValue(this->partitionCount) == 0) {
         this->assignInOrderExecInfoToEvent(event);
     } else if (event && !event->isCounterBased() && !event->isEventTimestampFlagSet()) {
         ret = this->appendBarrier(hSignalEvent, 0, nullptr, relaxedOrdering);
diff --git a/level_zero/core/source/cmdlist/cmdlist_hw_xehp_and_later.inl b/level_zero/core/source/cmdlist/cmdlist_hw_xehp_and_later.inl
@@ -352,9 +352,9 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendLaunchKernelWithParams(K
                     inOrderExecInfo = this->inOrderExecInfo.get();
                     if (eventForInOrderExec && eventForInOrderExec->isCounterBased()) {
                         isCounterBasedEvent = true;
-                        if (eventForInOrderExec->getInOrderIncrementValue() > 0) {
+                        if (eventForInOrderExec->getInOrderIncrementValue(this->partitionCount) > 0) {
                             inOrderIncrementGpuAddress = eventForInOrderExec->getInOrderExecInfo()->getBaseDeviceAddress();
-                            inOrderIncrementValue = eventForInOrderExec->getInOrderIncrementValue();
+                            inOrderIncrementValue = eventForInOrderExec->getInOrderIncrementValue(this->partitionCount);
                         }
                     }
                 }
diff --git a/level_zero/core/source/device/bcs_split.h b/level_zero/core/source/device/bcs_split.h
@@ -93,10 +93,10 @@ struct BcsSplit {
         uint64_t aggregatedEventIncrementVal = 1;
 
         const bool useSignalEventForSubcopy = aggregatedEventsMode && cmdList->isUsingAdditionalBlitProperties() && Event::isAggregatedEvent(signalEvent) &&
-                                              (signalEvent->getInOrderIncrementValue() % engineCount == 0);
+                                              (signalEvent->getInOrderIncrementValue(1) % engineCount == 0);
 
         if (useSignalEventForSubcopy) {
-            aggregatedEventIncrementVal = signalEvent->getInOrderIncrementValue() / engineCount;
+            aggregatedEventIncrementVal = signalEvent->getInOrderIncrementValue(1) / engineCount;
         } else {
             auto markerEventIndexRet = this->events.obtainForSplit(Context::fromHandle(cmdList->getCmdListContext()), maxEventCountInPool);
             if (!markerEventIndexRet.has_value()) {
diff --git a/level_zero/core/source/event/event.cpp b/level_zero/core/source/event/event.cpp
@@ -659,6 +659,11 @@ uint64_t Event::getInOrderExecSignalValueWithSubmissionCounter() const {
     return (inOrderExecSignalValue + appendCounter);
 }
 
+uint64_t Event::getInOrderIncrementValue(uint32_t partitionCount) const {
+    DEBUG_BREAK_IF(inOrderIncrementValue % partitionCount != 0);
+    return (inOrderIncrementValue / partitionCount);
+}
+
 void Event::setLatestUsedCmdQueue(CommandQueue *newCmdQ) {
     this->latestUsedCmdQueue = newCmdQ;
 }
diff --git a/level_zero/core/source/event/event.h b/level_zero/core/source/event/event.h
@@ -307,7 +307,7 @@ struct Event : _ze_event_handle_t {
     uint64_t getInOrderExecSignalValueWithSubmissionCounter() const;
     uint64_t getInOrderExecBaseSignalValue() const { return inOrderExecSignalValue; }
     uint32_t getInOrderAllocationOffset() const { return inOrderAllocationOffset; }
-    uint64_t getInOrderIncrementValue() const { return inOrderIncrementValue; }
+    uint64_t getInOrderIncrementValue(uint32_t partitionCount) const;
     void setLatestUsedCmdQueue(CommandQueue *newCmdQ);
     NEO::TimeStampData *peekReferenceTs() {
         return static_cast<NEO::TimeStampData *>(ptrOffset(getHostAddress(), getMaxPacketsCount() * getSinglePacketSize()));
@@ -349,7 +349,7 @@ struct Event : _ze_event_handle_t {
         this->isEventOnBarrierOptimized = value;
     }
 
-    static bool isAggregatedEvent(const Event *event) { return (event && event->getInOrderIncrementValue() > 0); }
+    static bool isAggregatedEvent(const Event *event) { return (event && event->getInOrderIncrementValue(1) > 0); }
 
     CommandList *getRecordedSignalFrom() const {
         return this->recordedSignalFrom;
diff --git a/level_zero/core/source/mutable_cmdlist/mutable_cmdlist_hw.inl b/level_zero/core/source/mutable_cmdlist/mutable_cmdlist_hw.inl
@@ -828,7 +828,7 @@ void MutableCommandListCoreFamily<gfxCoreFamily>::storeSignalEventVariable(Mutab
             if (CommandListImp::isInOrderExecutionEnabled()) {
                 mutableEventParams.eventInsideInOrder = true;
                 mutableEventParams.counterBasedEvent = event->isCounterBased();
-                mutableEventParams.inOrderIncrementEvent = event->getInOrderIncrementValue() > 0;
+                mutableEventParams.inOrderIncrementEvent = event->getInOrderIncrementValue(this->partitionCount) > 0;
                 if (mutableEventParams.counterBasedEvent && CommandListCoreFamily<gfxCoreFamily>::duplicatedInOrderCounterStorageEnabled) {
                     mutableEventParams.counterBasedTimestampEvent = event->isEventTimestampFlagSet();
                 }
diff --git a/level_zero/core/source/mutable_cmdlist/variable.cpp b/level_zero/core/source/mutable_cmdlist/variable.cpp
@@ -121,7 +121,7 @@ ze_result_t Variable::setAsSignalEvent(Event *event, MutableComputeWalker *walke
     this->eventValue.event = event;
     this->eventValue.eventPoolAllocation = event->getAllocation(cmdList->getBase()->getDevice());
     this->eventValue.counterBasedEvent = event->isCounterBased();
-    this->eventValue.inOrderIncrementEvent = event->getInOrderIncrementValue() > 0;
+    this->eventValue.inOrderIncrementEvent = event->getInOrderIncrementValue(cmdList->getBase()->getPartitionCount()) > 0;
     this->eventValue.walkerCmd = walkerCmd;
     this->eventValue.postSyncCmd = postSyncCmd;
     this->eventValue.kernelCount = event->getKernelCount();
diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_blit.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_blit.cpp
@@ -1251,7 +1251,7 @@ HWTEST2_F(AggregatedBcsSplitTests, whenObtainCalledThenAggregatedEventsCreated,
         EXPECT_EQ(0u, *bcsSplit->events.subcopy[i]->getInOrderExecInfo()->getBaseHostAddress());
         EXPECT_FALSE(bcsSplit->events.subcopy[i]->isSignalScope(ZE_EVENT_SCOPE_FLAG_HOST));
         EXPECT_TRUE(bcsSplit->events.subcopy[i]->isSignalScope(ZE_EVENT_SCOPE_FLAG_DEVICE));
-        EXPECT_EQ(1u, bcsSplit->events.subcopy[i]->getInOrderIncrementValue());
+        EXPECT_EQ(1u, bcsSplit->events.subcopy[i]->getInOrderIncrementValue(1));
         EXPECT_EQ(static_cast<uint64_t>(bcsSplit->cmdLists.size()), bcsSplit->events.subcopy[i]->getInOrderExecBaseSignalValue());
 
         EXPECT_EQ(nullptr, bcsSplit->events.marker[i]->getInOrderExecInfo());
@@ -1296,7 +1296,7 @@ HWTEST2_F(AggregatedBcsSplitTests, whenObtainCalledThenAggregatedEventsCreated,
 
     for (auto &event : bcsSplit->events.subcopy) {
         EXPECT_TRUE(event->isCounterBased());
-        EXPECT_EQ(1u, event->getInOrderIncrementValue());
+        EXPECT_EQ(1u, event->getInOrderIncrementValue(1));
         EXPECT_EQ(static_cast<uint64_t>(bcsSplit->cmdLists.size()), event->getInOrderExecSignalValueWithSubmissionCounter());
     }
 }
diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_in_order_cmdlist_1.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_in_order_cmdlist_1.cpp
@@ -5716,7 +5716,7 @@ HWTEST_F(InOrderCmdListTests, givenExternalSyncStorageWhenCreatingCounterBasedEv
 
     auto inOrderExecInfo = eventObj->getInOrderExecInfo();
 
-    EXPECT_EQ(incValue, eventObj->getInOrderIncrementValue());
+    EXPECT_EQ(incValue, eventObj->getInOrderIncrementValue(1));
     EXPECT_EQ(counterValue, inOrderExecInfo->getCounterValue());
     EXPECT_EQ(castToUint64(externalStorageAllocProperties.deviceAddress), inOrderExecInfo->getBaseDeviceAddress());
     EXPECT_NE(nullptr, inOrderExecInfo->getDeviceCounterAllocation());
@@ -5775,7 +5775,7 @@ HWTEST_F(InOrderCmdListTests, givenExternalSyncStorageWhenCallingAppendThenDontR
     EXPECT_EQ(inOrderExecInfo, eventObj->getInOrderExecInfo());
     EXPECT_EQ(counterValue, eventObj->getInOrderExecInfo()->getCounterValue());
     EXPECT_EQ(counterValue, eventObj->getInOrderExecSignalValueWithSubmissionCounter());
-    EXPECT_EQ(incValue, eventObj->getInOrderIncrementValue());
+    EXPECT_EQ(incValue, eventObj->getInOrderIncrementValue(1));
 
     context->freeMem(devAddress);
 }
diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_in_order_cmdlist_2.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_in_order_cmdlist_2.cpp
@@ -3593,6 +3593,115 @@ HWTEST2_F(MultiTileInOrderCmdListTests, givenStandaloneEventAndCopyOnlyCmdListWh
     context->freeMem(hostAddress);
 }
 
+HWTEST2_F(MultiTileInOrderCmdListTests, givenExternalSyncStorageWhenCallingAppendSignalInOrderDependencyCounterThenProgramAtomicOperation, IsAtLeastXeHpcCore) {
+    using MI_ATOMIC = typename FamilyType::MI_ATOMIC;
+    using ATOMIC_OPCODES = typename FamilyType::MI_ATOMIC::ATOMIC_OPCODES;
+    using DATA_SIZE = typename FamilyType::MI_ATOMIC::DATA_SIZE;
+
+    const uint64_t incValue = (static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1234) * partitionCount;
+    const uint64_t counterValue = incValue * 2;
+    const uint64_t programmedIncValue = incValue / partitionCount;
+
+    auto devAddress = reinterpret_cast<uint64_t *>(allocDeviceMem(sizeof(uint64_t)));
+
+    auto immCmdList = createImmCmdList<FamilyType::gfxCoreFamily>();
+
+    auto eventObj = createExternalSyncStorageEvent(counterValue, incValue, devAddress);
+
+    auto cmdStream = immCmdList->getCmdContainer().getCommandStream();
+    immCmdList->inOrderAtomicSignalingEnabled = false;
+    immCmdList->appendSignalInOrderDependencyCounter(eventObj.get(), false, false, false, false);
+
+    GenCmdList cmdList;
+    ASSERT_TRUE(FamilyType::Parse::parseCommandBuffer(cmdList, cmdStream->getCpuBase(), cmdStream->getUsed()));
+
+    auto it = find<MI_ATOMIC *>(cmdList.begin(), cmdList.end());
+    ASSERT_NE(cmdList.end(), it);
+
+    auto miAtomic = genCmdCast<MI_ATOMIC *>(*it);
+    EXPECT_EQ(ATOMIC_OPCODES::ATOMIC_8B_ADD, miAtomic->getAtomicOpcode());
+    EXPECT_EQ(DATA_SIZE::DATA_SIZE_QWORD, miAtomic->getDataSize());
+    EXPECT_EQ(getLowPart(programmedIncValue), miAtomic->getOperand1DataDword0());
+    EXPECT_EQ(getHighPart(programmedIncValue), miAtomic->getOperand1DataDword1());
+
+    EXPECT_EQ(castToUint64(devAddress), NEO::UnitTestHelper<FamilyType>::getAtomicMemoryAddress(*miAtomic));
+
+    context->freeMem(devAddress);
+}
+
+HWTEST2_F(MultiTileInOrderCmdListTests, givenExternalSyncStorageAndCopyOnlyCmdListWhenCallingAppendMemoryCopyWithDisabledInOrderSignalingThenSignalAtomicStorage, IsAtLeastXeHpcCore) {
+    using MI_ATOMIC = typename FamilyType::MI_ATOMIC;
+    using ATOMIC_OPCODES = typename FamilyType::MI_ATOMIC::ATOMIC_OPCODES;
+    using DATA_SIZE = typename FamilyType::MI_ATOMIC::DATA_SIZE;
+
+    const uint64_t incValue = (static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1234) * partitionCount;
+    const uint64_t counterValue = incValue * 2;
+    const uint64_t programmedIncValue = incValue / partitionCount;
+
+    auto devAddress = reinterpret_cast<uint64_t *>(allocDeviceMem(sizeof(uint64_t)));
+
+    auto immCmdList = createCopyOnlyImmCmdList<FamilyType::gfxCoreFamily>();
+
+    auto eventObj = createExternalSyncStorageEvent(counterValue, incValue, devAddress);
+
+    auto cmdStream = immCmdList->getCmdContainer().getCommandStream();
+
+    auto offset = cmdStream->getUsed();
+    uint32_t copyData = 0;
+    copyParams.forceDisableCopyOnlyInOrderSignaling = true;
+
+    {
+        immCmdList->appendMemoryCopy(&copyData, &copyData, 1, eventObj->toHandle(), 0, nullptr, copyParams);
+
+        GenCmdList cmdList;
+        ASSERT_TRUE(FamilyType::Parse::parseCommandBuffer(cmdList, ptrOffset(cmdStream->getCpuBase(), offset), cmdStream->getUsed() - offset));
+
+        auto it = find<MI_ATOMIC *>(cmdList.begin(), cmdList.end());
+
+        if (immCmdList->useAdditionalBlitProperties) {
+            EXPECT_EQ(cmdList.end(), it);
+        } else {
+            ASSERT_NE(cmdList.end(), it);
+
+            auto miAtomic = genCmdCast<MI_ATOMIC *>(*it);
+            EXPECT_EQ(ATOMIC_OPCODES::ATOMIC_8B_ADD, miAtomic->getAtomicOpcode());
+            EXPECT_EQ(DATA_SIZE::DATA_SIZE_QWORD, miAtomic->getDataSize());
+            EXPECT_EQ(getLowPart(programmedIncValue), miAtomic->getOperand1DataDword0());
+            EXPECT_EQ(getHighPart(programmedIncValue), miAtomic->getOperand1DataDword1());
+
+            EXPECT_EQ(castToUint64(devAddress), NEO::UnitTestHelper<FamilyType>::getAtomicMemoryAddress(*miAtomic));
+        }
+    }
+
+    offset = cmdStream->getUsed();
+
+    {
+        ze_copy_region_t region = {0, 0, 0, 1, 1, 1};
+
+        immCmdList->appendMemoryCopyRegion(&copyData, &region, 1, 1, &copyData, &region, 1, 1, eventObj->toHandle(), 0, nullptr, copyParams);
+
+        GenCmdList cmdList;
+        ASSERT_TRUE(FamilyType::Parse::parseCommandBuffer(cmdList, ptrOffset(cmdStream->getCpuBase(), offset), cmdStream->getUsed() - offset));
+
+        auto it = find<MI_ATOMIC *>(cmdList.begin(), cmdList.end());
+        if (immCmdList->useAdditionalBlitProperties) {
+            EXPECT_EQ(cmdList.end(), it);
+        } else {
+            ASSERT_NE(cmdList.end(), it);
+
+            auto miAtomic = genCmdCast<MI_ATOMIC *>(*it);
+            EXPECT_EQ(ATOMIC_OPCODES::ATOMIC_8B_ADD, miAtomic->getAtomicOpcode());
+            EXPECT_EQ(DATA_SIZE::DATA_SIZE_QWORD, miAtomic->getDataSize());
+            EXPECT_EQ(getLowPart(programmedIncValue), miAtomic->getOperand1DataDword0());
+            EXPECT_EQ(getHighPart(programmedIncValue), miAtomic->getOperand1DataDword1());
+
+            EXPECT_EQ(castToUint64(devAddress), NEO::UnitTestHelper<FamilyType>::getAtomicMemoryAddress(*miAtomic));
+        }
+    }
+
+    context->freeMem(devAddress);
+}
+
 HWTEST2_F(MultiTileInOrderCmdListTests, givenDebugFlagSetWhenAskingForAtomicSignallingThenReturnTrue, IsAtLeastXeCore) {
     auto immCmdList = createMultiTileImmCmdList<FamilyType::gfxCoreFamily>();
     auto heaplessEnabled = immCmdList->isHeaplessModeEnabled();

Original file line number	Diff line number	Diff line change
`@@ -1962,7 +1962,7 @@ ze_result_t CommandListCoreFamilyImmediate<gfxCoreFamily>::appendStagingMemoryCo`
`1962`	`1962`	`return ret;`
`1963`	`1963`	`}`
`1964`	`1964`
`1965`		`- if (event && event->isCounterBased() && event->getInOrderIncrementValue() == 0) {`
	`1965`	`+ if (event && event->isCounterBased() && event->getInOrderIncrementValue(this->partitionCount) == 0) {`
`1966`	`1966`	`this->assignInOrderExecInfoToEvent(event);`
`1967`	`1967`	`} else if (event && !event->isCounterBased() && !event->isEventTimestampFlagSet()) {`
`1968`	`1968`	`ret = this->appendBarrier(hSignalEvent, 0, nullptr, relaxedOrdering);`
Original file line number	Diff line number	Diff line change
`@@ -352,9 +352,9 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendLaunchKernelWithParams(K`
`352`	`352`	`inOrderExecInfo = this->inOrderExecInfo.get();`
`353`	`353`	`if (eventForInOrderExec && eventForInOrderExec->isCounterBased()) {`
`354`	`354`	`isCounterBasedEvent = true;`
`355`		`- if (eventForInOrderExec->getInOrderIncrementValue() > 0) {`
	`355`	`+ if (eventForInOrderExec->getInOrderIncrementValue(this->partitionCount) > 0) {`
`356`	`356`	`inOrderIncrementGpuAddress = eventForInOrderExec->getInOrderExecInfo()->getBaseDeviceAddress();`
`357`		`- inOrderIncrementValue = eventForInOrderExec->getInOrderIncrementValue();`
	`357`	`+ inOrderIncrementValue = eventForInOrderExec->getInOrderIncrementValue(this->partitionCount);`
`358`	`358`	`}`
`359`	`359`	`}`
`360`	`360`	`}`
Original file line number	Diff line number	Diff line change
`@@ -659,6 +659,11 @@ uint64_t Event::getInOrderExecSignalValueWithSubmissionCounter() const {`
`659`	`659`	`return (inOrderExecSignalValue + appendCounter);`
`660`	`660`	`}`
`661`	`661`
	`662`	`+uint64_t Event::getInOrderIncrementValue(uint32_t partitionCount) const {`
	`663`	`+ DEBUG_BREAK_IF(inOrderIncrementValue % partitionCount != 0);`
	`664`	`+ return (inOrderIncrementValue / partitionCount);`
	`665`	`+}`
	`666`	`+`
`662`	`667`	`void Event::setLatestUsedCmdQueue(CommandQueue *newCmdQ) {`
`663`	`668`	`this->latestUsedCmdQueue = newCmdQ;`
`664`	`669`	`}`
Original file line number	Diff line number	Diff line change
`@@ -828,7 +828,7 @@ void MutableCommandListCoreFamily<gfxCoreFamily>::storeSignalEventVariable(Mutab`
`828`	`828`	`if (CommandListImp::isInOrderExecutionEnabled()) {`
`829`	`829`	`mutableEventParams.eventInsideInOrder = true;`
`830`	`830`	`mutableEventParams.counterBasedEvent = event->isCounterBased();`
`831`		`- mutableEventParams.inOrderIncrementEvent = event->getInOrderIncrementValue() > 0;`
	`831`	`+ mutableEventParams.inOrderIncrementEvent = event->getInOrderIncrementValue(this->partitionCount) > 0;`
`832`	`832`	`if (mutableEventParams.counterBasedEvent && CommandListCoreFamily<gfxCoreFamily>::duplicatedInOrderCounterStorageEnabled) {`
`833`	`833`	`mutableEventParams.counterBasedTimestampEvent = event->isEventTimestampFlagSet();`
`834`	`834`	`}`