Fix Vulkan swapchain wait for acquired backbuffer

Move backbuffer submit waiting until after swapchain image acquire, and track the submitted fence counter per backbuffer to avoid waiting on stale command buffer objects. This preserves multi-buffered presentation and fixes iOS/MoltenVK frame pacing drops caused by waiting on the wrong swapchain image.

Source/Engine/GraphicsDevice/Vulkan/GPUSwapChainVulkan.cpp

@@ -25,16 +25,23 @@ void BackBufferVulkan::Setup(GPUSwapChainVulkan* window, VkImage backbuffer, Pix
     ImageAcquiredSemaphore = New<SemaphoreVulkan>(Device);
 }
 
+void BackBufferVulkan::WaitForSubmit()
+{
+    if (SubmitCmdBuffer)
+    {
+        if (SubmitCmdBufferFenceCounter == SubmitCmdBuffer->GetFenceSignaledCounter())
+            SubmitCmdBuffer->Wait();
+        SubmitCmdBuffer = nullptr;
+        SubmitCmdBufferFenceCounter = 0;
+    }
+}
+
 void BackBufferVulkan::Release()
 {
+    WaitForSubmit();
     Handle.Release();
     Delete(RenderingDoneSemaphore);
     Delete(ImageAcquiredSemaphore);
-    if (SubmitCmdBuffer)
-    {
-        SubmitCmdBuffer->Wait();
-        SubmitCmdBuffer = nullptr;
-    }
     Device = nullptr;
 }
 
@@ -121,7 +128,14 @@ GPUTextureView* GPUSwapChainVulkan::GetBackBufferView()
         ASSERT(_acquiredImageIndex != -1);
 
         auto context = _device->MainContext;
-        const auto backBuffer = &_backBuffers[_acquiredImageIndex].Handle;
+
+        // Wait for prior GPU work that used this acquired image before recording
+        // commands against it again. Waiting before acquire can target a different image
+        // and unnecessarily serialize frames when the swapchain has multiple images.
+        auto& acquiredBackBuffer = _backBuffers[_acquiredImageIndex];
+        acquiredBackBuffer.WaitForSubmit();
+
+        const auto backBuffer = &acquiredBackBuffer.Handle;
 
         auto cmdBufferManager = context->GetCmdBufferManager();
         auto cmdBuffer = cmdBufferManager->GetCmdBuffer();
@@ -142,17 +156,6 @@ GPUTextureView* GPUSwapChainVulkan::GetBackBufferView()
 void GPUSwapChainVulkan::Begin(RenderTask* task)
 {
     GPUSwapChain::Begin(task);
-
-    // Wait for the backbuffer to be available
-    if (_currentImageIndex != -1)
-    {
-        auto& backBuffer = _backBuffers[_currentImageIndex];
-        if (backBuffer.SubmitCmdBuffer)
-        {
-            backBuffer.SubmitCmdBuffer->Wait();
-            backBuffer.SubmitCmdBuffer = nullptr;
-        }
-    }
 }
 
 bool GPUSwapChainVulkan::Resize(int32 width, int32 height)
@@ -589,6 +592,7 @@ void GPUSwapChainVulkan::Present(bool vsync)
     acquiredBackBuffer.SubmitCmdBuffer = context->GetCmdBufferManager()->GetActiveCmdBuffer();
 
     context->GetCmdBufferManager()->SubmitActiveCmdBuffer(_backBuffers[_acquiredImageIndex].RenderingDoneSemaphore);
+    acquiredBackBuffer.SubmitCmdBufferFenceCounter = acquiredBackBuffer.SubmitCmdBuffer->GetSubmittedFenceCounter();
 
     // Present the back buffer to the viewport window
     const auto result = TryPresent(DoPresent, _device->PresentQueue, true);

Source/Engine/GraphicsDevice/Vulkan/GPUSwapChainVulkan.h

@@ -34,6 +34,11 @@ public:
     /// </summary>
     CmdBufferVulkan* SubmitCmdBuffer = nullptr;
 
+    /// <summary>
+    /// The fence counter value for SubmitCmdBuffer at the time it was submitted.
+    /// </summary>
+    uint64 SubmitCmdBufferFenceCounter = 0;
+
     /// <summary>
     /// The render target surface handle.
     /// </summary>
@@ -41,6 +46,7 @@ public:
 
 public:
     void Setup(GPUSwapChainVulkan* window, VkImage backbuffer, PixelFormat format, VkExtent3D extent);
+    void WaitForSubmit();
     void Release();
 
 public:

Source/Engine/Platform/iOS/iOSPlatform.cpp

@@ -350,7 +350,13 @@ MessagePipeline MainThreadPipeline;
 
     // Create UI thread update callback
     self.displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(UIThreadMain)];
-    self.displayLink.preferredFramesPerSecond = 60;
+    const int32 targetFrameRate = 60;
+#if __IPHONE_OS_VERSION_MAX_ALLOWED >= 150000
+    if (@available(iOS 15.0, *))
+        self.displayLink.preferredFrameRateRange = CAFrameRateRangeMake(targetFrameRate, targetFrameRate, targetFrameRate);
+    else
+#endif
+        self.displayLink.preferredFramesPerSecond = targetFrameRate;
     [self.displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes];
 
     // Run engine on a separate main thread

Source/Platforms/iOS/Binaries/Project/FlaxGame.xcodeproj/project.pbxproj

@@ -295,6 +295,7 @@ ${PBXResourcesGroup}
 				GENERATE_INFOPLIST_FILE = YES;
 				INFOPLIST_FILE = FlaxGame/Info.plist;
 				INFOPLIST_KEY_CFBundleDisplayName = "${ProjectName}";
+				INFOPLIST_KEY_CADisableMinimumFrameDurationOnPhone = YES;
 				INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.games";
 				INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
 				INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen;
@@ -336,6 +337,7 @@ ${PBXResourcesGroup}
 				GENERATE_INFOPLIST_FILE = YES;
 				INFOPLIST_FILE = FlaxGame/Info.plist;
 				INFOPLIST_KEY_CFBundleDisplayName = "${ProjectName}";
+				INFOPLIST_KEY_CADisableMinimumFrameDurationOnPhone = YES;
 				INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.games";
 				INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
 				INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen;

Source/Platforms/iOS/Binaries/Project/FlaxGame/Info.plist

@@ -2,6 +2,8 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
 <dict>
+	<key>CADisableMinimumFrameDurationOnPhone</key>
+	<true/>
 	<key>UIApplicationSceneManifest</key>
 	<dict>
 		<key>UIApplicationSupportsMultipleScenes</key>