examples: Prevent invalid semaphore reuse across frames (#1009)

MarijnS95 · web-flow · commit 96becc8565af · 2025-11-03T00:30:43.000+01:00
* examples: Have a unique semaphore per swapchain image This resolves `VUID-vkQueueSubmit-pSignalSemaphores-00067` as explained at https://docs.vulkan.org/guide/latest/swapchain_semaphore_reuse.html A unique semaphore should be used per swapchain image, because the presentation engine may still reference and use it before the image index it was used with is returned out of `acquire_next_image()` again. Alternatives include waiting for the `Fence` from presentation in `VkSwapchainPresentFenceInfoKHR`, added by the `VK_EXT_swapchain_maintenance1`/`VK_KHR_swapchain_maintenance1` extension but this is less performant than just using a unique semaphore per swapchain image. * examples: Wait earlier for command-buffer fence to prevent semaphore reuse This resolves `VUID-vkAcquireNextImageKHR-semaphore-01779` where `acquire_next_image()` is asked to signal a semaphore that may still be waited on by the currently-running submit. Since the next submit is reusing the existing command buffer and resources as well it must inevitably wait on a fence from the previous frame/submit, pulling out that fence to wait on it is the most practical approach despite teaching bad/slow practices. Note that this fence was also used to "reuse" the setup command buffer in the `texture` sample; this was instead generalized to use another command buffer without waiting for and signaling any fence. * examples: Implement triple-buffering This removes a per-frame stall and demonstrates how users should utilize Vulkan to keep the GPU fed with rendering work, as long as that work (and corresponding presentation requests) complete before we're getting 3 frames ahead. Note that this does not implement proper frame throttling, and may render lots of discarded frames. It is an alternative solution to solving `VUID-vkAcquireNextImageKHR-semaphore-01779` by also using a unique `Semaphore` that we know is no longer waited on via previously waiting on the relevant `Fence`; although similar in nature to the previous solution.
diff --git a/ash-examples/src/bin/texture.rs b/ash-examples/src/bin/texture.rs
@@ -352,8 +352,8 @@ fn main() -> Result<(), Box<dyn Error>> {
 
         record_submit_commandbuffer(
             &base.device,
-            base.setup_command_buffer,
-            base.setup_commands_reuse_fence,
+            base.app_setup_command_buffer,
+            vk::Fence::null(),
             base.present_queue,
             &[],
             &[],
@@ -687,13 +687,19 @@ fn main() -> Result<(), Box<dyn Error>> {
 
         let graphic_pipeline = graphics_pipelines[0];
 
-        let _ = base.render_loop(|| {
+        let _ = base.render_loop(|frame_index| {
+            let present_complete_semaphore =
+                base.present_complete_semaphores[frame_index % MAX_FRAME_LATENCY];
+            let draw_commands_reuse_fence =
+                base.draw_commands_reuse_fences[frame_index % MAX_FRAME_LATENCY];
+            let draw_command_buffer = base.draw_command_buffers[frame_index % MAX_FRAME_LATENCY];
+
             let (present_index, _) = base
                 .swapchain_loader
                 .acquire_next_image(
                     base.swapchain,
                     u64::MAX,
-                    base.present_complete_semaphore,
+                    present_complete_semaphore,
                     vk::Fence::null(),
                 )
                 .unwrap();
@@ -711,6 +717,9 @@ fn main() -> Result<(), Box<dyn Error>> {
                 },
             ];
 
+            let rendering_complete_semaphore =
+                base.rendering_complete_semaphores[present_index as usize];
+
             let render_pass_begin_info = vk::RenderPassBeginInfo::default()
                 .render_pass(renderpass)
                 .framebuffer(framebuffers[present_index as usize])
@@ -719,12 +728,12 @@ fn main() -> Result<(), Box<dyn Error>> {
 
             record_submit_commandbuffer(
                 &base.device,
-                base.draw_command_buffer,
-                base.draw_commands_reuse_fence,
+                draw_command_buffer,
+                draw_commands_reuse_fence,
                 base.present_queue,
                 &[vk::PipelineStageFlags::BOTTOM_OF_PIPE],
-                &[base.present_complete_semaphore],
-                &[base.rendering_complete_semaphore],
+                &[present_complete_semaphore],
+                &[rendering_complete_semaphore],
                 |device, draw_command_buffer| {
                     device.cmd_begin_render_pass(
                         draw_command_buffer,
@@ -773,7 +782,7 @@ fn main() -> Result<(), Box<dyn Error>> {
             );
             let present_info = vk::PresentInfoKHR {
                 wait_semaphore_count: 1,
-                p_wait_semaphores: &base.rendering_complete_semaphore,
+                p_wait_semaphores: &rendering_complete_semaphore,
                 swapchain_count: 1,
                 p_swapchains: &base.swapchain,
                 p_image_indices: &present_index,
diff --git a/ash-examples/src/bin/triangle.rs b/ash-examples/src/bin/triangle.rs
@@ -349,13 +349,19 @@ fn main() -> Result<(), Box<dyn Error>> {
 
         let graphic_pipeline = graphics_pipelines[0];
 
-        let _ = base.render_loop(|| {
+        let _ = base.render_loop(|frame_index| {
+            let present_complete_semaphore =
+                base.present_complete_semaphores[frame_index % MAX_FRAME_LATENCY];
+            let draw_commands_reuse_fence =
+                base.draw_commands_reuse_fences[frame_index % MAX_FRAME_LATENCY];
+            let draw_command_buffer = base.draw_command_buffers[frame_index % MAX_FRAME_LATENCY];
+
             let (present_index, _) = base
                 .swapchain_loader
                 .acquire_next_image(
                     base.swapchain,
                     u64::MAX,
-                    base.present_complete_semaphore,
+                    present_complete_semaphore,
                     vk::Fence::null(),
                 )
                 .unwrap();
@@ -373,6 +379,9 @@ fn main() -> Result<(), Box<dyn Error>> {
                 },
             ];
 
+            let rendering_complete_semaphore =
+                base.rendering_complete_semaphores[present_index as usize];
+
             let render_pass_begin_info = vk::RenderPassBeginInfo::default()
                 .render_pass(renderpass)
                 .framebuffer(framebuffers[present_index as usize])
@@ -381,12 +390,12 @@ fn main() -> Result<(), Box<dyn Error>> {
 
             record_submit_commandbuffer(
                 &base.device,
-                base.draw_command_buffer,
-                base.draw_commands_reuse_fence,
+                draw_command_buffer,
+                draw_commands_reuse_fence,
                 base.present_queue,
                 &[vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT],
-                &[base.present_complete_semaphore],
-                &[base.rendering_complete_semaphore],
+                &[present_complete_semaphore],
+                &[rendering_complete_semaphore],
                 |device, draw_command_buffer| {
                     device.cmd_begin_render_pass(
                         draw_command_buffer,
@@ -425,11 +434,11 @@ fn main() -> Result<(), Box<dyn Error>> {
                     device.cmd_end_render_pass(draw_command_buffer);
                 },
             );
-            let wait_semaphors = [base.rendering_complete_semaphore];
+            let wait_semaphores = [rendering_complete_semaphore];
             let swapchains = [base.swapchain];
             let image_indices = [present_index];
             let present_info = vk::PresentInfoKHR::default()
-                .wait_semaphores(&wait_semaphors) // &base.rendering_complete_semaphore)
+                .wait_semaphores(&wait_semaphores)
                 .swapchains(&swapchains)
                 .image_indices(&image_indices);
 
diff --git a/ash-examples/src/lib.rs b/ash-examples/src/lib.rs
@@ -25,6 +25,9 @@ use winit::{
     window::WindowBuilder,
 };
 
+// The maximum number of frames we allow to be in flight at any given time
+pub const MAX_FRAME_LATENCY: usize = 3;
+
 // Simple offset_of macro akin to C++ offsetof
 #[macro_export]
 macro_rules! offset_of {
@@ -36,9 +39,7 @@ macro_rules! offset_of {
         }
     }};
 }
-/// Helper function for submitting command buffers. Immediately waits for the fence before the command buffer
-/// is executed. That way we can delay the waiting for the fences by 1 frame which is good for performance.
-/// Make sure to create the fence in a signaled state on the first use.
+
 #[allow(clippy::too_many_arguments)]
 pub fn record_submit_commandbuffer<F: FnOnce(&Device, vk::CommandBuffer)>(
     device: &Device,
@@ -51,14 +52,6 @@ pub fn record_submit_commandbuffer<F: FnOnce(&Device, vk::CommandBuffer)>(
     f: F,
 ) {
     unsafe {
-        device
-            .wait_for_fences(&[command_buffer_reuse_fence], true, u64::MAX)
-            .expect("Wait for fence failed.");
-
-        device
-            .reset_fences(&[command_buffer_reuse_fence])
-            .expect("Reset fences failed.");
-
         device
             .reset_command_buffer(
                 command_buffer,
@@ -143,6 +136,7 @@ pub struct ExampleBase {
     pub debug_utils_loader: debug_utils::Instance,
     pub window: winit::window::Window,
     pub event_loop: RefCell<EventLoop<()>>,
+    pub frame_index: RefCell<usize>,
     pub debug_call_back: vk::DebugUtilsMessengerEXT,
 
     pub pdevice: vk::PhysicalDevice,
@@ -159,22 +153,22 @@ pub struct ExampleBase {
     pub present_image_views: Vec<vk::ImageView>,
 
     pub pool: vk::CommandPool,
-    pub draw_command_buffer: vk::CommandBuffer,
+    pub draw_command_buffers: [vk::CommandBuffer; MAX_FRAME_LATENCY],
     pub setup_command_buffer: vk::CommandBuffer,
+    pub app_setup_command_buffer: vk::CommandBuffer,
 
     pub depth_image: vk::Image,
     pub depth_image_view: vk::ImageView,
     pub depth_image_memory: vk::DeviceMemory,
 
-    pub present_complete_semaphore: vk::Semaphore,
-    pub rendering_complete_semaphore: vk::Semaphore,
+    pub present_complete_semaphores: [vk::Semaphore; MAX_FRAME_LATENCY],
+    pub rendering_complete_semaphores: Vec<vk::Semaphore>,
 
-    pub draw_commands_reuse_fence: vk::Fence,
-    pub setup_commands_reuse_fence: vk::Fence,
+    pub draw_commands_reuse_fences: [vk::Fence; MAX_FRAME_LATENCY],
 }
 
 impl ExampleBase {
-    pub fn render_loop<F: Fn()>(&self, f: F) -> Result<(), impl Error> {
+    pub fn render_loop<F: Fn(usize)>(&self, f: F) -> Result<(), impl Error> {
         self.event_loop.borrow_mut().run_on_demand(|event, elwp| {
             elwp.set_control_flow(ControlFlow::Poll);
             match event {
@@ -194,7 +188,24 @@ impl ExampleBase {
                 } => {
                     elwp.exit();
                 }
-                Event::AboutToWait => f(),
+                Event::AboutToWait => {
+                    let mut frame_index = self.frame_index.borrow_mut();
+
+                    // The fence from 3 frames ago, that will also be signaled this frame
+                    let draw_commands_reuse_fence =
+                        self.draw_commands_reuse_fences[*frame_index % MAX_FRAME_LATENCY];
+                    unsafe {
+                        self.device
+                            .wait_for_fences(&[draw_commands_reuse_fence], true, u64::MAX)
+                    }
+                    .expect("Wait for fence failed.");
+
+                    unsafe { self.device.reset_fences(&[draw_commands_reuse_fence]) }
+                        .expect("Reset fences failed.");
+
+                    f(*frame_index);
+                    *frame_index += 1;
+                }
                 _ => (),
             }
         })
@@ -400,15 +411,18 @@ impl ExampleBase {
             let pool = device.create_command_pool(&pool_create_info, None).unwrap();
 
             let command_buffer_allocate_info = vk::CommandBufferAllocateInfo::default()
-                .command_buffer_count(2)
+                .command_buffer_count(2 + MAX_FRAME_LATENCY as u32)
                 .command_pool(pool)
                 .level(vk::CommandBufferLevel::PRIMARY);
 
             let command_buffers = device
                 .allocate_command_buffers(&command_buffer_allocate_info)
                 .unwrap();
             let setup_command_buffer = command_buffers[0];
-            let draw_command_buffer = command_buffers[1];
+            let app_setup_command_buffer = command_buffers[1];
+            let draw_command_buffers = command_buffers[2..][..MAX_FRAME_LATENCY]
+                .try_into()
+                .unwrap();
 
             let present_images = swapchain_loader.get_swapchain_images(swapchain).unwrap();
             let present_image_views: Vec<vk::ImageView> = present_images
@@ -467,20 +481,10 @@ impl ExampleBase {
                 .bind_image_memory(depth_image, depth_image_memory, 0)
                 .expect("Unable to bind depth image memory");
 
-            let fence_create_info =
-                vk::FenceCreateInfo::default().flags(vk::FenceCreateFlags::SIGNALED);
-
-            let draw_commands_reuse_fence = device
-                .create_fence(&fence_create_info, None)
-                .expect("Create fence failed.");
-            let setup_commands_reuse_fence = device
-                .create_fence(&fence_create_info, None)
-                .expect("Create fence failed.");
-
             record_submit_commandbuffer(
                 &device,
                 setup_command_buffer,
-                setup_commands_reuse_fence,
+                vk::Fence::null(),
                 present_queue,
                 &[],
                 &[],
@@ -530,15 +534,31 @@ impl ExampleBase {
 
             let semaphore_create_info = vk::SemaphoreCreateInfo::default();
 
-            let present_complete_semaphore = device
-                .create_semaphore(&semaphore_create_info, None)
-                .unwrap();
-            let rendering_complete_semaphore = device
-                .create_semaphore(&semaphore_create_info, None)
-                .unwrap();
+            let present_complete_semaphores = std::array::from_fn(|_| {
+                device
+                    .create_semaphore(&semaphore_create_info, None)
+                    .unwrap()
+            });
+            let rendering_complete_semaphores = (0..present_images.len())
+                .map(|_| {
+                    device
+                        .create_semaphore(&semaphore_create_info, None)
+                        .unwrap()
+                })
+                .collect();
+
+            let fence_create_info =
+                vk::FenceCreateInfo::default().flags(vk::FenceCreateFlags::SIGNALED);
+
+            let draw_commands_reuse_fences = std::array::from_fn(|_| {
+                device
+                    .create_fence(&fence_create_info, None)
+                    .expect("Create fence failed.")
+            });
 
             Ok(Self {
                 event_loop: RefCell::new(event_loop),
+                frame_index: RefCell::new(0),
                 entry,
                 instance,
                 device,
@@ -555,14 +575,14 @@ impl ExampleBase {
                 present_images,
                 present_image_views,
                 pool,
-                draw_command_buffer,
+                draw_command_buffers,
                 setup_command_buffer,
+                app_setup_command_buffer,
                 depth_image,
                 depth_image_view,
-                present_complete_semaphore,
-                rendering_complete_semaphore,
-                draw_commands_reuse_fence,
-                setup_commands_reuse_fence,
+                present_complete_semaphores,
+                rendering_complete_semaphores,
+                draw_commands_reuse_fences,
                 surface,
                 debug_call_back,
                 debug_utils_loader,
@@ -576,18 +596,19 @@ impl Drop for ExampleBase {
     fn drop(&mut self) {
         unsafe {
             self.device.device_wait_idle().unwrap();
-            self.device
-                .destroy_semaphore(self.present_complete_semaphore, None);
-            self.device
-                .destroy_semaphore(self.rendering_complete_semaphore, None);
-            self.device
-                .destroy_fence(self.draw_commands_reuse_fence, None);
-            self.device
-                .destroy_fence(self.setup_commands_reuse_fence, None);
+            for &semaphore in &self.present_complete_semaphores {
+                self.device.destroy_semaphore(semaphore, None);
+            }
+            for &semaphore in &self.rendering_complete_semaphores {
+                self.device.destroy_semaphore(semaphore, None);
+            }
+            for &fence in &self.draw_commands_reuse_fences {
+                self.device.destroy_fence(fence, None);
+            }
             self.device.free_memory(self.depth_image_memory, None);
             self.device.destroy_image_view(self.depth_image_view, None);
             self.device.destroy_image(self.depth_image, None);
-            for &image_view in self.present_image_views.iter() {
+            for &image_view in &self.present_image_views {
                 self.device.destroy_image_view(image_view, None);
             }
             self.device.destroy_command_pool(self.pool, None);
