Roll out the big gun, the MultiTimelineEventHandler

I forgot that the acquire returns on CPU before the presentation is finished, so technically I can't get rid of resources needed for the previous acquire-release cycle on the same image index when a new acquire returns on the CPU
also there's nothing in the spec saying that I can't acquire/return multiple times immediately on the CPU, getting the same image index every time and just spam my submission queue with acquire-present pairs

Author: devshgraphicsprogramming

include/nbl/video/ISwapchain.h

@@ -361,11 +361,10 @@ class ISwapchain : public IBackendObject
                 case ACQUIRE_IMAGE_RESULT::SUCCESS: [[fallthrough]];
                 case ACQUIRE_IMAGE_RESULT::SUBOPTIMAL:
                     m_acquireCounter++;
+                    // Now hold onto the signal-on-acquire semaphores for this image index till the acquire actually takes place
                     {
-                        auto semaphoreArray = core::make_refctd_dynamic_array<core::smart_refctd_dynamic_array<void_refctd_ptr>>(info.signalSemaphores.size());
-                        for (auto i=0ull; i<info.signalSemaphores.size(); i++)
-                            semaphoreArray->operator[](i) = void_refctd_ptr(info.signalSemaphores[i].semaphore);
-                        m_frameResources[*out_imgIx] = std::move(semaphoreArray);
+                        const auto& lastSignal = info.signalSemaphores.back();
+                        m_frameResources[*out_imgIx]->latch({.semaphore=lastSignal.semaphore,.value=lastSignal.value},DeferredFrameResourceDrop(info.signalSemaphores));
                     }
                     if (m_oldSwapchain && m_acquireCounter>core::max(m_oldSwapchain->getImageCount(),m_imageCount) && !m_oldSwapchain->acquiredImagesAwaitingPresent())
                         m_oldSwapchain = nullptr;
@@ -408,19 +407,15 @@ class ISwapchain : public IBackendObject
                 threadsafeQ->m.unlock();
 
             // If not FATAL_ERROR then semaphore wait will actually occur in the future and resources should be released, either:
-            // - on the next acquire of the same index
+            // - on the next SIGNALLED acquire of the same index
             // - when dropping the swapchain entirely, but need to wait on all previous presents to finish (some manageable UB)
             if (retval!=PRESENT_RESULT::FATAL_ERROR)
             {
-                auto frameResources = core::make_refctd_dynamic_array<core::smart_refctd_dynamic_array<void_refctd_ptr>>(info.waitSemaphores.size()+2);
-                frameResources->operator[](0) = std::move(m_frameResources[info.imgIndex]);
-                frameResources->operator[](1) = std::move(_frameResources);
-                for (auto i=0ull; i<info.waitSemaphores.size(); i++)
-                    frameResources->operator[](i+2) = void_refctd_ptr(info.waitSemaphores[i].semaphore);
-                m_frameResources[info.imgIndex] = std::move(frameResources);
+                auto& lastWait = info.waitSemaphores.back();
+                m_frameResources[info.imgIndex]->latch({.semaphore=lastWait.semaphore,.value=lastWait.value},DeferredFrameResourceDrop(info.waitSemaphores,std::move(_frameResources)));
             }
-            else
-                m_frameResources[info.imgIndex] = nullptr;
+            // kill a few frame resources
+            m_frameResources[info.imgIndex]->poll();
             return retval;
         }
 
@@ -433,7 +428,7 @@ class ISwapchain : public IBackendObject
             for (uint8_t i=0; i<m_imageCount; i++)
             {
                 if (unacquired(i))
-                    m_frameResources[i] = nullptr;
+                    m_frameResources[i]->abortAll();
                 else
                     return true;
             }
@@ -470,7 +465,56 @@ class ISwapchain : public IBackendObject
 
         // Vulkan: const VkSwapchainKHR*
         virtual const void* getNativeHandle() const = 0;
-
+        
+		// only public because MultiTimelineEventHandlerST needs to know about it
+		class DeferredFrameResourceDrop final
+		{
+				core::smart_refctd_dynamic_array<void_refctd_ptr> m_frameResources;
+
+			public:
+				inline DeferredFrameResourceDrop(const std::span<const IQueue::SSubmitInfo::SSemaphoreInfo> _semaphores, void_refctd_ptr&& _frameResources=nullptr)
+				{
+                    m_frameResources = core::make_refctd_dynamic_array<decltype(m_frameResources)>(_semaphores.size()+(_frameResources ? 1:0));
+                    for (auto i=0ull; i< _semaphores.size(); i++)
+                        m_frameResources->operator[](i) = void_refctd_ptr(_semaphores[i].semaphore);
+                    if (_frameResources)
+                        m_frameResources->back() = std::move(_frameResources);
+				}
+                DeferredFrameResourceDrop(const DeferredFrameResourceDrop& other) = delete;
+				inline DeferredFrameResourceDrop(DeferredFrameResourceDrop&& other) : m_frameResources(nullptr)
+				{
+					this->operator=(std::move(other));
+				}
+
+                DeferredFrameResourceDrop& operator=(const DeferredFrameResourceDrop& other) = delete;
+				inline DeferredFrameResourceDrop& operator=(DeferredFrameResourceDrop&& other)
+				{
+                    m_frameResources = std::move(other.m_frameResources);
+					m_frameResources = nullptr;
+					return *this;
+				}
+
+				struct single_poll_t {};
+				static inline single_poll_t single_poll;
+				inline bool operator()(single_poll_t _single_poll)
+				{
+					operator()();
+					return true;
+				}
+
+				struct exhaustive_poll_t {};
+				static inline exhaustive_poll_t exhaustive_poll;
+				inline bool operator()(exhaustive_poll_t _exhaustive_poll)
+				{
+					operator()();
+					return false;
+				}
+
+                inline void operator()()
+                {
+                    m_frameResources = nullptr;
+                }
+		};
     protected:
         ISwapchain(core::smart_refctd_ptr<const ILogicalDevice>&& dev, SCreationParams&& params, const uint8_t imageCount, core::smart_refctd_ptr<ISwapchain>&& oldSwapchain);
         virtual inline ~ISwapchain()
@@ -513,8 +557,8 @@ class ISwapchain : public IBackendObject
         std::array<uint8_t,ILogicalDevice::MaxQueueFamilies> m_queueFamilies;
         const uint8_t m_imageCount;
         uint64_t m_acquireCounter = 0;
-        // resource to hold onto until a frame is done rendering (between; just before presenting, and next acquire of the same image index)
-        std::array<void_refctd_ptr,ILogicalDevice::MaxQueueFamilies> m_frameResources = {};
+        // Resources to hold onto until a frame is done rendering (between; just before presenting, and next acquire of the same image index)
+        std::array<std::unique_ptr<MultiTimelineEventHandlerST<DeferredFrameResourceDrop>>,ILogicalDevice::MaxQueueFamilies> m_frameResources;
 };
 
 }

src/nbl/video/ISwapchain.cpp

@@ -26,6 +26,12 @@ ISwapchain::ISwapchain(core::smart_refctd_ptr<const ILogicalDevice>&& dev, SCrea
 
     std::copy(m_params.queueFamilyIndices.begin(),m_params.queueFamilyIndices.end(),m_queueFamilies.data());
     m_params.queueFamilyIndices = {m_queueFamilies.data(),m_params.queueFamilyIndices.size()};
-}
+
+    for (auto i=0; i<m_imageCount; i++)
+    {
+        auto device = core::smart_refctd_ptr<ILogicalDevice>(const_cast<ILogicalDevice*>(getOriginDevice()));
+        m_frameResources[i] = std::make_unique<MultiTimelineEventHandlerST<DeferredFrameResourceDrop>>(std::move(device));
+    }
+} 
 
 }