From 9f986031216db6a9141e8944054339d9c2224236 Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 12:48:02 +0200
Subject: [PATCH 01/22] vulkan: move OnDrop out of vulkan module

---
 src/gfx_apis/vulkan.rs                            |  1 -
 src/gfx_apis/vulkan/bo_allocator.rs               |  4 ++--
 src/gfx_apis/vulkan/device.rs                     |  2 +-
 src/gfx_apis/vulkan/image.rs                      |  4 ++--
 src/gfx_apis/vulkan/instance.rs                   |  5 ++++-
 src/gfx_apis/vulkan/pipeline.rs                   |  9 ++++++---
 src/gfx_apis/vulkan/shm_image.rs                  |  3 +--
 src/gfx_apis/vulkan/staging.rs                    | 12 +++++++-----
 src/utils.rs                                      |  1 +
 src/{gfx_apis/vulkan/util.rs => utils/on_drop.rs} |  0
 10 files changed, 24 insertions(+), 17 deletions(-)
 rename src/{gfx_apis/vulkan/util.rs => utils/on_drop.rs} (100%)

diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs
index 8b1cae35..49f3dac0 100644
--- a/src/gfx_apis/vulkan.rs
+++ b/src/gfx_apis/vulkan.rs
@@ -14,7 +14,6 @@ mod semaphore;
 mod shaders;
 mod shm_image;
 mod staging;
-mod util;
 
 use {
     crate::{
diff --git a/src/gfx_apis/vulkan/bo_allocator.rs b/src/gfx_apis/vulkan/bo_allocator.rs
index 1874c72f..f89da55b 100644
--- a/src/gfx_apis/vulkan/bo_allocator.rs
+++ b/src/gfx_apis/vulkan/bo_allocator.rs
@@ -8,9 +8,9 @@ use {
         gfx_apis::vulkan::{
             allocator::VulkanAllocator, command::VulkanCommandBuffer, device::VulkanDevice,
             format::VulkanFormat, renderer::image_barrier, staging::VulkanStagingBuffer,
-            util::OnDrop, VulkanError,
+            VulkanError,
         },
-        utils::errorfmt::ErrorFmt,
+        utils::{errorfmt::ErrorFmt, on_drop::OnDrop},
         video::{
             dmabuf::{DmaBuf, DmaBufIds, DmaBufPlane, PlaneVec},
             drm::Drm,
diff --git a/src/gfx_apis/vulkan/device.rs b/src/gfx_apis/vulkan/device.rs
index 8e23fabe..00f3b67b 100644
--- a/src/gfx_apis/vulkan/device.rs
+++ b/src/gfx_apis/vulkan/device.rs
@@ -7,9 +7,9 @@ use {
                 map_extension_properties, ApiVersionDisplay, Extensions, VulkanInstance,
                 API_VERSION,
             },
-            util::OnDrop,
             VulkanError,
         },
+        utils::on_drop::OnDrop,
         video::{
             drm::{sync_obj::SyncObjCtx, Drm},
             gbm::GbmDevice,
diff --git a/src/gfx_apis/vulkan/image.rs b/src/gfx_apis/vulkan/image.rs
index d92ad53a..ee0555a5 100644
--- a/src/gfx_apis/vulkan/image.rs
+++ b/src/gfx_apis/vulkan/image.rs
@@ -4,10 +4,10 @@ use {
         gfx_api::{GfxApiOpt, GfxError, GfxFramebuffer, GfxImage, GfxTexture, SyncFile},
         gfx_apis::vulkan::{
             allocator::VulkanAllocation, device::VulkanDevice, format::VulkanModifierLimits,
-            renderer::VulkanRenderer, shm_image::VulkanShmImage, util::OnDrop, VulkanError,
+            renderer::VulkanRenderer, shm_image::VulkanShmImage, VulkanError,
         },
         theme::Color,
-        utils::clonecell::CloneCell,
+        utils::{clonecell::CloneCell, on_drop::OnDrop},
         video::dmabuf::{DmaBuf, PlaneVec},
     },
     ash::vk::{
diff --git a/src/gfx_apis/vulkan/instance.rs b/src/gfx_apis/vulkan/instance.rs
index aba36851..bb39f3c3 100644
--- a/src/gfx_apis/vulkan/instance.rs
+++ b/src/gfx_apis/vulkan/instance.rs
@@ -1,5 +1,8 @@
 use {
-    crate::gfx_apis::vulkan::{util::OnDrop, VulkanError, VULKAN_VALIDATION},
+    crate::{
+        gfx_apis::vulkan::{VulkanError, VULKAN_VALIDATION},
+        utils::on_drop::OnDrop,
+    },
     ahash::{AHashMap, AHashSet},
     ash::{
         ext::{debug_utils, validation_features},
diff --git a/src/gfx_apis/vulkan/pipeline.rs b/src/gfx_apis/vulkan/pipeline.rs
index 037e1692..3d495f39 100644
--- a/src/gfx_apis/vulkan/pipeline.rs
+++ b/src/gfx_apis/vulkan/pipeline.rs
@@ -1,7 +1,10 @@
 use {
-    crate::gfx_apis::vulkan::{
-        descriptor::VulkanDescriptorSetLayout, device::VulkanDevice, shaders::VulkanShader,
-        util::OnDrop, VulkanError,
+    crate::{
+        gfx_apis::vulkan::{
+            descriptor::VulkanDescriptorSetLayout, device::VulkanDevice, shaders::VulkanShader,
+            VulkanError,
+        },
+        utils::on_drop::OnDrop,
     },
     arrayvec::ArrayVec,
     ash::{
diff --git a/src/gfx_apis/vulkan/shm_image.rs b/src/gfx_apis/vulkan/shm_image.rs
index e272eb87..81f6e568 100644
--- a/src/gfx_apis/vulkan/shm_image.rs
+++ b/src/gfx_apis/vulkan/shm_image.rs
@@ -9,11 +9,10 @@ use {
             image::{VulkanImage, VulkanImageMemory},
             renderer::{image_barrier, VulkanRenderer},
             staging::VulkanStagingBuffer,
-            util::OnDrop,
             VulkanError,
         },
         rect::Rect,
-        utils::errorfmt::ErrorFmt,
+        utils::{errorfmt::ErrorFmt, on_drop::OnDrop},
     },
     ash::vk::{
         AccessFlags2, BufferImageCopy2, BufferMemoryBarrier2, CommandBufferBeginInfo,
diff --git a/src/gfx_apis/vulkan/staging.rs b/src/gfx_apis/vulkan/staging.rs
index 7e205a81..b8ccc2ef 100644
--- a/src/gfx_apis/vulkan/staging.rs
+++ b/src/gfx_apis/vulkan/staging.rs
@@ -1,9 +1,11 @@
 use {
-    crate::gfx_apis::vulkan::{
-        allocator::{VulkanAllocation, VulkanAllocator},
-        device::VulkanDevice,
-        util::OnDrop,
-        VulkanError,
+    crate::{
+        gfx_apis::vulkan::{
+            allocator::{VulkanAllocation, VulkanAllocator},
+            device::VulkanDevice,
+            VulkanError,
+        },
+        utils::on_drop::OnDrop,
     },
     ash::vk::{Buffer, BufferCreateInfo, BufferUsageFlags, MappedMemoryRange},
     gpu_alloc::UsageFlags,
diff --git a/src/utils.rs b/src/utils.rs
index d2afdf17..9aed5327 100644
--- a/src/utils.rs
+++ b/src/utils.rs
@@ -27,6 +27,7 @@ pub mod nonblock;
 pub mod num_cpus;
 pub mod numcell;
 pub mod on_change;
+pub mod on_drop;
 pub mod once;
 pub mod opaque;
 pub mod opaque_cell;
diff --git a/src/gfx_apis/vulkan/util.rs b/src/utils/on_drop.rs
similarity index 100%
rename from src/gfx_apis/vulkan/util.rs
rename to src/utils/on_drop.rs

From 92f7cb56fd5df49c81a3895cbdcf3b4b69907c5b Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 16:54:25 +0200
Subject: [PATCH 02/22] damage: add damage queue

---
 src/rect.rs        |  4 ++-
 src/rect/region.rs | 67 +++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 63 insertions(+), 8 deletions(-)

diff --git a/src/rect.rs b/src/rect.rs
index df9dedd7..a5cd711c 100644
--- a/src/rect.rs
+++ b/src/rect.rs
@@ -3,6 +3,8 @@ mod region;
 #[cfg(test)]
 mod tests;
 
+#[expect(unused_imports)]
+pub use region::DamageQueue;
 pub use region::RegionBuilder;
 use {
     jay_algorithms::rect::RectRaw,
@@ -16,7 +18,7 @@ pub struct Rect {
     raw: RectRaw,
 }
 
-#[derive(Clone, Eq, PartialEq, Debug)]
+#[derive(Clone, Eq, PartialEq, Debug, Default)]
 pub struct Region {
     rects: SmallVec<[RectRaw; 1]>,
     extents: Rect,
diff --git a/src/rect/region.rs b/src/rect/region.rs
index 262950d3..ade706c1 100644
--- a/src/rect/region.rs
+++ b/src/rect/region.rs
@@ -1,11 +1,17 @@
 use {
-    crate::rect::{Rect, Region},
+    crate::{
+        rect::{Rect, Region},
+        utils::{
+            array,
+            ptr_ext::{MutPtrExt, PtrExt},
+        },
+    },
     jay_algorithms::rect::{
         region::{extents, rects_to_bands, subtract, union},
         RectRaw,
     },
     smallvec::SmallVec,
-    std::{mem, ops::Deref, rc::Rc},
+    std::{cell::UnsafeCell, mem, ops::Deref, rc::Rc},
 };
 
 thread_local! {
@@ -18,12 +24,16 @@ thread_local! {
 
 impl Region {
     pub fn new(rect: Rect) -> Rc<Self> {
+        Rc::new(Self::new2(rect))
+    }
+
+    pub fn new2(rect: Rect) -> Self {
         let mut rects = SmallVec::new();
         rects.push(rect.raw);
-        Rc::new(Self {
+        Self {
             rects,
             extents: rect,
-        })
+        }
     }
 
     pub fn empty() -> Rc<Self> {
@@ -34,16 +44,23 @@ impl Region {
         if rects.is_empty() {
             return Self::empty();
         }
+        Rc::new(Self::from_rects2(rects))
+    }
+
+    pub fn from_rects2(rects: &[Rect]) -> Self {
+        if rects.is_empty() {
+            return Self::default();
+        }
         if rects.len() == 1 {
-            return Self::new(rects[0]);
+            return Self::new2(rects[0]);
         }
         let rects = rects_to_bands(unsafe { mem::transmute::<&[Rect], &[RectRaw]>(rects) });
-        Rc::new(Self {
+        Self {
             extents: Rect {
                 raw: extents(&rects),
             },
             rects,
-        })
+        }
     }
 
     pub fn union(self: &Rc<Self>, other: &Rc<Self>) -> Rc<Self> {
@@ -173,3 +190,39 @@ impl RegionBuilder {
         self.pending.clear();
     }
 }
+
+pub struct DamageQueue {
+    this: usize,
+    datas: Rc<UnsafeCell<Vec<Vec<Rect>>>>,
+}
+
+impl DamageQueue {
+    #[expect(dead_code)]
+    pub fn new<const N: usize>() -> [DamageQueue; N] {
+        let datas = Rc::new(UnsafeCell::new(vec![vec!(); N]));
+        array::from_fn(|this| DamageQueue {
+            this,
+            datas: datas.clone(),
+        })
+    }
+
+    #[expect(dead_code)]
+    pub fn damage(&self, rects: &[Rect]) {
+        let datas = unsafe { self.datas.get().deref_mut() };
+        for data in datas {
+            data.extend(rects);
+        }
+    }
+
+    #[expect(dead_code)]
+    pub fn clear(&self) {
+        let data = unsafe { &mut self.datas.get().deref_mut()[self.this] };
+        data.clear();
+    }
+
+    #[expect(dead_code)]
+    pub fn get(&self) -> Region {
+        let data = unsafe { &self.datas.get().deref()[self.this] };
+        Region::from_rects2(data)
+    }
+}

From 604974c927aee0c410774cdec3caea74e6ac907a Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 17:00:08 +0200
Subject: [PATCH 03/22] clientmem: store more information about mappings

---
 src/cli/input.rs                           |  5 +-
 src/clientmem.rs                           | 58 +++++++++++++++++++---
 src/ifs/jay_input.rs                       |  4 +-
 src/ifs/wl_seat/zwp_virtual_keyboard_v1.rs |  2 +-
 src/ifs/wl_shm_pool.rs                     |  5 +-
 src/it/tests/t0040_virtual_keyboard.rs     | 11 ++--
 6 files changed, 67 insertions(+), 18 deletions(-)

diff --git a/src/cli/input.rs b/src/cli/input.rs
index b41c2c4b..46625bd5 100644
--- a/src/cli/input.rs
+++ b/src/cli/input.rs
@@ -353,8 +353,9 @@ impl Input {
     async fn handle_keymap(&self, input: JayInputId) -> Vec<u8> {
         let data = Rc::new(RefCell::new(Vec::new()));
         jay_input::Keymap::handle(&self.tc, input, data.clone(), |d, map| {
-            let mem = Rc::new(ClientMem::new(map.keymap.raw(), map.keymap_len as _, true).unwrap())
-                .offset(0);
+            let mem =
+                Rc::new(ClientMem::new(&map.keymap, map.keymap_len as _, true, None).unwrap())
+                    .offset(0);
             mem.read(d.borrow_mut().deref_mut()).unwrap();
         });
         self.tc.round_trip().await;
diff --git a/src/clientmem.rs b/src/clientmem.rs
index f85dc5e7..1c8e9c98 100644
--- a/src/clientmem.rs
+++ b/src/clientmem.rs
@@ -1,5 +1,5 @@
 use {
-    crate::utils::vec_ext::VecExt,
+    crate::{client::Client, utils::vec_ext::VecExt},
     std::{
         cell::Cell,
         mem::MaybeUninit,
@@ -8,7 +8,10 @@ use {
         sync::atomic::{compiler_fence, Ordering},
     },
     thiserror::Error,
-    uapi::{c, c::raise},
+    uapi::{
+        c::{self, raise},
+        OwnedFd,
+    },
 };
 
 #[derive(Debug, Error)]
@@ -22,6 +25,7 @@ pub enum ClientMemError {
 }
 
 pub struct ClientMem {
+    fd: Rc<OwnedFd>,
     failed: Cell<bool>,
     sigbus_impossible: bool,
     data: *const [Cell<u8>],
@@ -30,19 +34,34 @@ pub struct ClientMem {
 #[derive(Clone)]
 pub struct ClientMemOffset {
     mem: Rc<ClientMem>,
+    offset: usize,
     data: *const [Cell<u8>],
 }
 
 impl ClientMem {
-    pub fn new(fd: i32, len: usize, read_only: bool) -> Result<Self, ClientMemError> {
+    pub fn new(
+        fd: &Rc<OwnedFd>,
+        len: usize,
+        read_only: bool,
+        client: Option<&Client>,
+    ) -> Result<Self, ClientMemError> {
         let mut sigbus_impossible = false;
-        if let Ok(seals) = uapi::fcntl_get_seals(fd) {
+        if let Ok(seals) = uapi::fcntl_get_seals(fd.raw()) {
             if seals & c::F_SEAL_SHRINK != 0 {
-                if let Ok(stat) = uapi::fstat(fd) {
+                if let Ok(stat) = uapi::fstat(fd.raw()) {
                     sigbus_impossible = stat.st_size as u64 >= len as u64;
                 }
             }
         }
+        if !sigbus_impossible {
+            if let Some(client) = client {
+                log::debug!(
+                    "Client {} ({}) has created a shm buffer that might cause SIGBUS",
+                    client.pid_info.comm,
+                    client.id,
+                );
+            }
+        }
         let data = if len == 0 {
             &mut [][..]
         } else {
@@ -51,7 +70,7 @@ impl ClientMem {
                 false => c::PROT_READ | c::PROT_WRITE,
             };
             unsafe {
-                let data = c::mmap64(ptr::null_mut(), len, prot, c::MAP_SHARED, fd, 0);
+                let data = c::mmap64(ptr::null_mut(), len, prot, c::MAP_SHARED, fd.raw(), 0);
                 if data == c::MAP_FAILED {
                     return Err(ClientMemError::MmapFailed(uapi::Errno::default().into()));
                 }
@@ -59,6 +78,7 @@ impl ClientMem {
             }
         };
         Ok(Self {
+            fd: fd.clone(),
             failed: Cell::new(false),
             sigbus_impossible,
             data,
@@ -73,12 +93,38 @@ impl ClientMem {
         let mem = unsafe { &*self.data };
         ClientMemOffset {
             mem: self.clone(),
+            offset,
             data: &mem[offset..],
         }
     }
+
+    #[expect(dead_code)]
+    pub fn fd(&self) -> &Rc<OwnedFd> {
+        &self.fd
+    }
+
+    #[expect(dead_code)]
+    pub fn sigbus_impossible(&self) -> bool {
+        self.sigbus_impossible
+    }
 }
 
 impl ClientMemOffset {
+    #[expect(dead_code)]
+    pub fn pool(&self) -> &ClientMem {
+        &self.mem
+    }
+
+    #[expect(dead_code)]
+    pub fn offset(&self) -> usize {
+        self.offset
+    }
+
+    #[expect(dead_code)]
+    pub fn ptr(&self) -> *const [Cell<u8>] {
+        self.data
+    }
+
     pub fn access<T, F: FnOnce(&[Cell<u8>]) -> T>(&self, f: F) -> Result<T, ClientMemError> {
         unsafe {
             if self.mem.sigbus_impossible {
diff --git a/src/ifs/jay_input.rs b/src/ifs/jay_input.rs
index f0362763..8d196063 100644
--- a/src/ifs/jay_input.rs
+++ b/src/ifs/jay_input.rs
@@ -165,11 +165,11 @@ impl JayInput {
         }
     }
 
-    fn set_keymap_impl<F>(&self, keymap: &OwnedFd, len: u32, f: F) -> Result<(), JayInputError>
+    fn set_keymap_impl<F>(&self, keymap: &Rc<OwnedFd>, len: u32, f: F) -> Result<(), JayInputError>
     where
         F: FnOnce(&Rc<XkbKeymap>) -> Result<(), JayInputError>,
     {
-        let cm = Rc::new(ClientMem::new(keymap.raw(), len as _, true)?).offset(0);
+        let cm = Rc::new(ClientMem::new(keymap, len as _, true, Some(&self.client))?).offset(0);
         let mut map = vec![];
         cm.read(&mut map)?;
         self.or_error(|| {
diff --git a/src/ifs/wl_seat/zwp_virtual_keyboard_v1.rs b/src/ifs/wl_seat/zwp_virtual_keyboard_v1.rs
index 2dde71f4..914ac247 100644
--- a/src/ifs/wl_seat/zwp_virtual_keyboard_v1.rs
+++ b/src/ifs/wl_seat/zwp_virtual_keyboard_v1.rs
@@ -56,7 +56,7 @@ impl ZwpVirtualKeyboardV1RequestHandler for ZwpVirtualKeyboardV1 {
         if req.size > MAX_SIZE {
             return Err(ZwpVirtualKeyboardV1Error::OversizedKeymap);
         }
-        let client_mem = ClientMem::new(req.fd.raw(), req.size as usize - 1, true)
+        let client_mem = ClientMem::new(&req.fd, req.size as usize - 1, true, Some(&self.client))
             .map(Rc::new)
             .map_err(ZwpVirtualKeyboardV1Error::MapKeymap)?;
         let mut map = vec![];
diff --git a/src/ifs/wl_shm_pool.rs b/src/ifs/wl_shm_pool.rs
index 30687f6c..6ea0c670 100644
--- a/src/ifs/wl_shm_pool.rs
+++ b/src/ifs/wl_shm_pool.rs
@@ -34,7 +34,7 @@ impl WlShmPool {
         Ok(Self {
             id,
             client: client.clone(),
-            mem: CloneCell::new(Rc::new(ClientMem::new(fd.raw(), len, false)?)),
+            mem: CloneCell::new(Rc::new(ClientMem::new(&fd, len, false, Some(client))?)),
             fd,
             tracker: Default::default(),
             version,
@@ -82,9 +82,10 @@ impl WlShmPoolRequestHandler for WlShmPool {
             return Err(WlShmPoolError::CannotShrink);
         }
         self.mem.set(Rc::new(ClientMem::new(
-            self.fd.raw(),
+            &self.fd,
             req.size as usize,
             false,
+            Some(&self.client),
         )?));
         Ok(())
     }
diff --git a/src/it/tests/t0040_virtual_keyboard.rs b/src/it/tests/t0040_virtual_keyboard.rs
index bf6519f2..bc0627c3 100644
--- a/src/it/tests/t0040_virtual_keyboard.rs
+++ b/src/it/tests/t0040_virtual_keyboard.rs
@@ -7,6 +7,7 @@ use {
     },
     bstr::ByteSlice,
     std::rc::Rc,
+    uapi::OwnedFd,
 };
 
 testcase!();
@@ -15,7 +16,7 @@ async fn test(run: Rc<TestRun>) -> TestResult {
     let virtual_keymap_str = {
         let xkb = XkbContext::new()?;
         let map = xkb.keymap_from_str(VIRTUAL_KEYMAP).unwrap();
-        read_keymap(map.map.raw(), map.map_len)
+        read_keymap(&map.map, map.map_len)
     };
 
     let ds = run.create_default_setup().await?;
@@ -51,7 +52,7 @@ async fn test(run: Rc<TestRun>) -> TestResult {
     s_client.sync().await;
     let (start, keymap) = s_keymap.next().expect("virtual keymap");
     tassert_eq!(
-        &read_keymap(keymap.fd.raw(), keymap.size as _),
+        &read_keymap(&keymap.fd, keymap.size as _),
         &virtual_keymap_str
     );
     {
@@ -119,7 +120,7 @@ async fn test(run: Rc<TestRun>) -> TestResult {
     s_client.sync().await;
     let (pos, keymap) = s_keymap.next().expect("seat keymap");
     tassert_eq!(pos, start + 8);
-    tassert!(read_keymap(keymap.fd.raw(), keymap.size as _) != virtual_keymap_str);
+    tassert!(read_keymap(&keymap.fd, keymap.size as _) != virtual_keymap_str);
     {
         let (pos, mods) = s_modifiers.next().expect("mods 0");
         tassert_eq!(pos, start + 9);
@@ -147,8 +148,8 @@ async fn test(run: Rc<TestRun>) -> TestResult {
     Ok(())
 }
 
-fn read_keymap(fd: i32, size: usize) -> String {
-    let client_mem = ClientMem::new(fd, size - 1, true).unwrap();
+fn read_keymap(fd: &Rc<OwnedFd>, size: usize) -> String {
+    let client_mem = ClientMem::new(fd, size - 1, true, None).unwrap();
     let client_mem = Rc::new(client_mem).offset(0);
     let mut v = vec![];
     client_mem.read(&mut v).unwrap();

From 69c0cf4031030ebbaed3d8eeab45d9c4513f3946 Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 17:01:22 +0200
Subject: [PATCH 04/22] fdclosor: assign name to thread

---
 src/utils/fdcloser.rs | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/src/utils/fdcloser.rs b/src/utils/fdcloser.rs
index fa5b4cac..32bb80aa 100644
--- a/src/utils/fdcloser.rs
+++ b/src/utils/fdcloser.rs
@@ -16,20 +16,23 @@ impl FdCloser {
             cv: Condvar::new(),
         });
         let slf2 = slf.clone();
-        std::thread::spawn(move || {
-            let mut fds = vec![];
-            let mut lock = slf2.fds.lock();
-            loop {
-                mem::swap(&mut *lock, &mut fds);
-                if fds.len() > 0 {
-                    drop(lock);
-                    fds.clear();
-                    lock = slf2.fds.lock();
-                } else {
-                    slf2.cv.wait(&mut lock);
+        std::thread::Builder::new()
+            .name("fd closer".to_string())
+            .spawn(move || {
+                let mut fds = vec![];
+                let mut lock = slf2.fds.lock();
+                loop {
+                    mem::swap(&mut *lock, &mut fds);
+                    if fds.len() > 0 {
+                        drop(lock);
+                        fds.clear();
+                        lock = slf2.fds.lock();
+                    } else {
+                        slf2.cv.wait(&mut lock);
+                    }
                 }
-            }
-        });
+            })
+            .unwrap();
         slf
     }
 

From e1d1fe7fda34eb8bb7f1ba722c840008e27fcefe Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 17:02:28 +0200
Subject: [PATCH 05/22] util: add OnDrop2 for non-copy closures

---
 src/utils/on_drop.rs | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/src/utils/on_drop.rs b/src/utils/on_drop.rs
index a7a12d79..15557f0a 100644
--- a/src/utils/on_drop.rs
+++ b/src/utils/on_drop.rs
@@ -1,4 +1,4 @@
-use std::mem;
+use std::{mem, mem::ManuallyDrop};
 
 pub struct OnDrop<F>(pub F)
 where
@@ -15,3 +15,34 @@ impl<F: FnMut() + Copy> Drop for OnDrop<F> {
         (self.0)();
     }
 }
+
+pub struct OnDrop2<F>
+where
+    F: FnOnce(),
+{
+    f: ManuallyDrop<F>,
+}
+
+impl<F: FnOnce()> OnDrop2<F> {
+    #[expect(dead_code)]
+    pub fn new(f: F) -> Self {
+        Self {
+            f: ManuallyDrop::new(f),
+        }
+    }
+
+    #[expect(dead_code)]
+    pub fn forget(mut self) {
+        unsafe {
+            ManuallyDrop::drop(&mut self.f);
+        }
+        mem::forget(self);
+    }
+}
+
+impl<F: FnOnce()> Drop for OnDrop2<F> {
+    fn drop(&mut self) {
+        let f = unsafe { ManuallyDrop::take(&mut self.f) };
+        f();
+    }
+}

From 6cbf6119de53eaddb6bfd6e9d472d7bafada683c Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 17:05:43 +0200
Subject: [PATCH 06/22] io_uring: use strongly typed ids

---
 src/io_uring.rs                  | 35 ++++++++++++++++++++------------
 src/io_uring/ops/accept.rs       |  6 +++---
 src/io_uring/ops/async_cancel.rs | 12 +++++------
 src/io_uring/ops/connect.rs      |  8 ++++----
 src/io_uring/ops/poll.rs         |  6 +++---
 src/io_uring/ops/read_write.rs   |  6 +++---
 src/io_uring/ops/recvmsg.rs      |  8 ++++----
 src/io_uring/ops/sendmsg.rs      |  8 ++++----
 src/io_uring/ops/timeout.rs      |  6 +++---
 src/io_uring/ops/timeout_link.rs |  6 +++---
 10 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/io_uring.rs b/src/io_uring.rs
index baf4c934..23b8749d 100644
--- a/src/io_uring.rs
+++ b/src/io_uring.rs
@@ -22,7 +22,6 @@ use {
             copyhashmap::CopyHashMap,
             errorfmt::ErrorFmt,
             mmap::{mmap, Mmapped},
-            numcell::NumCell,
             oserror::OsError,
             ptr_ext::{MutPtrExt, PtrExt},
             stack::Stack,
@@ -254,10 +253,10 @@ struct IoUringData {
 
     cqes_consumed: AsyncEvent,
 
-    next: NumCell<u64>,
-    to_encode: SyncQueue<u64>,
-    pending_in_kernel: CopyHashMap<u64, ()>,
-    tasks: CopyHashMap<u64, Box<dyn Task>>,
+    next: IoUringTaskIds,
+    to_encode: SyncQueue<IoUringTaskId>,
+    pending_in_kernel: CopyHashMap<IoUringTaskId, ()>,
+    tasks: CopyHashMap<IoUringTaskId, Box<dyn Task>>,
 
     pending_results: PendingResults,
 
@@ -276,7 +275,7 @@ struct IoUringData {
 }
 
 unsafe trait Task {
-    fn id(&self) -> u64;
+    fn id(&self) -> IoUringTaskId;
     fn complete(self: Box<Self>, ring: &IoUringData, res: i32);
     fn encode(&self, sqe: &mut io_uring_sqe);
 
@@ -347,8 +346,9 @@ impl IoUringData {
                 let entry = self.cqmap.deref()[idx].get();
                 head = head.wrapping_add(1);
                 self.cqhead.deref().store(head, Release);
-                if let Some(pending) = self.tasks.remove(&entry.user_data) {
-                    self.pending_in_kernel.remove(&entry.user_data);
+                let id = IoUringTaskId(entry.user_data);
+                if let Some(pending) = self.tasks.remove(&id) {
+                    self.pending_in_kernel.remove(&id);
                     pending.complete(self, entry.res);
                 }
             }
@@ -384,7 +384,7 @@ impl IoUringData {
                 let sqe = self.sqesmap.deref()[idx].get().deref_mut();
                 self.sqmap.deref()[idx].set(idx as _);
                 *sqe = Default::default();
-                sqe.user_data = id;
+                sqe.user_data = id.raw();
                 task.encode(sqe);
                 if has_timeout {
                     sqe.flags |= IOSQE_IO_LINK;
@@ -404,11 +404,11 @@ impl IoUringData {
         }
     }
 
-    fn id_raw(&self) -> u64 {
-        self.next.fetch_add(1)
+    fn id_raw(&self) -> IoUringTaskId {
+        self.next.next()
     }
 
-    fn cancel_task(&self, id: u64) {
+    fn cancel_task(&self, id: IoUringTaskId) {
         if !self.tasks.contains(&id) {
             return;
         }
@@ -466,8 +466,17 @@ impl IoUringData {
     }
 }
 
+linear_ids!(IoUringTaskIds, IoUringTaskId, u64);
+
+#[expect(clippy::derivable_impls)]
+impl Default for IoUringTaskId {
+    fn default() -> Self {
+        Self(0)
+    }
+}
+
 struct Cancellable<'a> {
-    id: u64,
+    id: IoUringTaskId,
     data: &'a IoUringData,
 }
 
diff --git a/src/io_uring/ops/accept.rs b/src/io_uring/ops/accept.rs
index 6f751dcf..149592ea 100644
--- a/src/io_uring/ops/accept.rs
+++ b/src/io_uring/ops/accept.rs
@@ -2,7 +2,7 @@ use {
     crate::io_uring::{
         pending_result::PendingResult,
         sys::{io_uring_sqe, IORING_OP_ACCEPT},
-        IoUring, IoUringData, IoUringError, Task, TaskResultExt,
+        IoUring, IoUringData, IoUringError, IoUringTaskId, Task, TaskResultExt,
     },
     std::rc::Rc,
     uapi::{c, OwnedFd},
@@ -39,14 +39,14 @@ struct Data {
 
 #[derive(Default)]
 pub struct AcceptTask {
-    id: u64,
+    id: IoUringTaskId,
     fd: i32,
     flags: u32,
     data: Option<Data>,
 }
 
 unsafe impl Task for AcceptTask {
-    fn id(&self) -> u64 {
+    fn id(&self) -> IoUringTaskId {
         self.id
     }
 
diff --git a/src/io_uring/ops/async_cancel.rs b/src/io_uring/ops/async_cancel.rs
index d322fd3b..3874ab98 100644
--- a/src/io_uring/ops/async_cancel.rs
+++ b/src/io_uring/ops/async_cancel.rs
@@ -2,7 +2,7 @@ use {
     crate::{
         io_uring::{
             sys::{io_uring_sqe, IORING_OP_ASYNC_CANCEL},
-            IoUringData, Task,
+            IoUringData, IoUringTaskId, Task,
         },
         utils::errorfmt::ErrorFmt,
     },
@@ -11,12 +11,12 @@ use {
 
 #[derive(Default)]
 pub struct AsyncCancelTask {
-    id: u64,
-    target: u64,
+    id: IoUringTaskId,
+    target: IoUringTaskId,
 }
 
 impl IoUringData {
-    pub fn cancel_task_in_kernel(&self, target: u64) {
+    pub fn cancel_task_in_kernel(&self, target: IoUringTaskId) {
         let id = self.id_raw();
         let mut task = self.cached_cancels.pop().unwrap_or_default();
         task.id = id;
@@ -26,7 +26,7 @@ impl IoUringData {
 }
 
 unsafe impl Task for AsyncCancelTask {
-    fn id(&self) -> u64 {
+    fn id(&self) -> IoUringTaskId {
         self.id
     }
 
@@ -41,7 +41,7 @@ unsafe impl Task for AsyncCancelTask {
 
     fn encode(&self, sqe: &mut io_uring_sqe) {
         sqe.opcode = IORING_OP_ASYNC_CANCEL;
-        sqe.u2.addr = self.target;
+        sqe.u2.addr = self.target.raw();
     }
 
     fn is_cancel(&self) -> bool {
diff --git a/src/io_uring/ops/connect.rs b/src/io_uring/ops/connect.rs
index 0aca6829..a23d460e 100644
--- a/src/io_uring/ops/connect.rs
+++ b/src/io_uring/ops/connect.rs
@@ -2,7 +2,7 @@ use {
     crate::io_uring::{
         pending_result::PendingResult,
         sys::{io_uring_sqe, IORING_OP_CONNECT},
-        IoUring, IoUringData, IoUringError, Task, TaskResultExt,
+        IoUring, IoUringData, IoUringError, IoUringTaskId, Task, TaskResultExt,
     },
     std::{mem, ptr, rc::Rc},
     uapi::{c, OwnedFd, SockAddr},
@@ -37,7 +37,7 @@ struct Data {
 }
 
 pub struct ConnectTask {
-    id: u64,
+    id: IoUringTaskId,
     fd: i32,
     sockaddr: c::sockaddr_storage,
     addrlen: u64,
@@ -47,7 +47,7 @@ pub struct ConnectTask {
 impl Default for ConnectTask {
     fn default() -> Self {
         Self {
-            id: 0,
+            id: Default::default(),
             fd: 0,
             sockaddr: uapi::pod_zeroed(),
             addrlen: 0,
@@ -57,7 +57,7 @@ impl Default for ConnectTask {
 }
 
 unsafe impl Task for ConnectTask {
-    fn id(&self) -> u64 {
+    fn id(&self) -> IoUringTaskId {
         self.id
     }
 
diff --git a/src/io_uring/ops/poll.rs b/src/io_uring/ops/poll.rs
index 6edab2d4..1ac45281 100644
--- a/src/io_uring/ops/poll.rs
+++ b/src/io_uring/ops/poll.rs
@@ -3,7 +3,7 @@ use {
         ops::TaskResult,
         pending_result::PendingResult,
         sys::{io_uring_sqe, IORING_OP_POLL_ADD},
-        IoUring, IoUringData, IoUringError, Task, TaskResultExt,
+        IoUring, IoUringData, IoUringError, IoUringTaskId, Task, TaskResultExt,
     },
     std::rc::Rc,
     uapi::{c, OwnedFd},
@@ -45,14 +45,14 @@ struct Data {
 
 #[derive(Default)]
 pub struct PollTask {
-    id: u64,
+    id: IoUringTaskId,
     events: u16,
     fd: i32,
     data: Option<Data>,
 }
 
 unsafe impl Task for PollTask {
-    fn id(&self) -> u64 {
+    fn id(&self) -> IoUringTaskId {
         self.id
     }
 
diff --git a/src/io_uring/ops/read_write.rs b/src/io_uring/ops/read_write.rs
index 9ebf2c1d..f8db92cb 100644
--- a/src/io_uring/ops/read_write.rs
+++ b/src/io_uring/ops/read_write.rs
@@ -3,7 +3,7 @@ use {
         io_uring::{
             pending_result::PendingResult,
             sys::{io_uring_sqe, IORING_OP_READ, IORING_OP_WRITE},
-            IoUring, IoUringData, IoUringError, Task, TaskResultExt,
+            IoUring, IoUringData, IoUringError, IoUringTaskId, Task, TaskResultExt,
         },
         time::Time,
         utils::buf::Buf,
@@ -66,7 +66,7 @@ struct ReadWriteTaskData {
 
 #[derive(Default)]
 pub struct ReadWriteTask {
-    id: u64,
+    id: IoUringTaskId,
     has_timeout: bool,
     fd: c::c_int,
     buf: usize,
@@ -76,7 +76,7 @@ pub struct ReadWriteTask {
 }
 
 unsafe impl Task for ReadWriteTask {
-    fn id(&self) -> u64 {
+    fn id(&self) -> IoUringTaskId {
         self.id
     }
 
diff --git a/src/io_uring/ops/recvmsg.rs b/src/io_uring/ops/recvmsg.rs
index 4196f8d4..6bea0d09 100644
--- a/src/io_uring/ops/recvmsg.rs
+++ b/src/io_uring/ops/recvmsg.rs
@@ -3,7 +3,7 @@ use {
         io_uring::{
             pending_result::PendingResult,
             sys::{io_uring_sqe, IORING_OP_RECVMSG},
-            IoUring, IoUringData, IoUringError, Task,
+            IoUring, IoUringData, IoUringError, IoUringTaskId, Task,
         },
         utils::buf::Buf,
     },
@@ -85,7 +85,7 @@ struct Data {
 }
 
 pub struct RecvmsgTask {
-    id: u64,
+    id: IoUringTaskId,
     fd: c::c_int,
     bufs: Vec<Buf>,
     iovecs: Vec<c::iovec>,
@@ -97,7 +97,7 @@ pub struct RecvmsgTask {
 impl Default for RecvmsgTask {
     fn default() -> Self {
         RecvmsgTask {
-            id: 0,
+            id: Default::default(),
             fd: 0,
             bufs: vec![],
             iovecs: vec![],
@@ -109,7 +109,7 @@ impl Default for RecvmsgTask {
 }
 
 unsafe impl Task for RecvmsgTask {
-    fn id(&self) -> u64 {
+    fn id(&self) -> IoUringTaskId {
         self.id
     }
 
diff --git a/src/io_uring/ops/sendmsg.rs b/src/io_uring/ops/sendmsg.rs
index 23cb7a8f..67d7b4e8 100644
--- a/src/io_uring/ops/sendmsg.rs
+++ b/src/io_uring/ops/sendmsg.rs
@@ -3,7 +3,7 @@ use {
         io_uring::{
             pending_result::PendingResult,
             sys::{io_uring_sqe, IORING_OP_SENDMSG},
-            IoUring, IoUringData, IoUringError, Task,
+            IoUring, IoUringData, IoUringError, IoUringTaskId, Task,
         },
         time::Time,
         utils::{buf::Buf, vec_ext::UninitVecExt},
@@ -91,7 +91,7 @@ struct SendmsgTaskData {
 }
 
 pub struct SendmsgTask {
-    id: u64,
+    id: IoUringTaskId,
     iovecs: Vec<c::iovec>,
     msghdr: c::msghdr,
     bufs: Vec<Buf>,
@@ -106,7 +106,7 @@ impl Default for SendmsgTask {
     fn default() -> Self {
         unsafe {
             SendmsgTask {
-                id: 0,
+                id: Default::default(),
                 iovecs: vec![],
                 msghdr: MaybeUninit::zeroed().assume_init(),
                 bufs: vec![],
@@ -121,7 +121,7 @@ impl Default for SendmsgTask {
 }
 
 unsafe impl Task for SendmsgTask {
-    fn id(&self) -> u64 {
+    fn id(&self) -> IoUringTaskId {
         self.id
     }
 
diff --git a/src/io_uring/ops/timeout.rs b/src/io_uring/ops/timeout.rs
index 3567a56f..e8067c61 100644
--- a/src/io_uring/ops/timeout.rs
+++ b/src/io_uring/ops/timeout.rs
@@ -2,7 +2,7 @@ use {
     crate::io_uring::{
         pending_result::PendingResult,
         sys::{io_uring_sqe, IORING_OP_TIMEOUT, IORING_TIMEOUT_ABS},
-        IoUring, IoUringData, IoUringError, Task,
+        IoUring, IoUringData, IoUringError, IoUringTaskId, Task,
     },
     uapi::c,
 };
@@ -16,7 +16,7 @@ pub(super) struct timespec64 {
 
 #[derive(Default)]
 pub struct TimeoutTask {
-    id: u64,
+    id: IoUringTaskId,
     timespec: timespec64,
     pr: Option<PendingResult>,
 }
@@ -42,7 +42,7 @@ impl IoUring {
 }
 
 unsafe impl Task for TimeoutTask {
-    fn id(&self) -> u64 {
+    fn id(&self) -> IoUringTaskId {
         self.id
     }
 
diff --git a/src/io_uring/ops/timeout_link.rs b/src/io_uring/ops/timeout_link.rs
index aabaa421..023fbd99 100644
--- a/src/io_uring/ops/timeout_link.rs
+++ b/src/io_uring/ops/timeout_link.rs
@@ -2,14 +2,14 @@ use crate::{
     io_uring::{
         ops::timeout::timespec64,
         sys::{io_uring_sqe, IORING_OP_LINK_TIMEOUT, IORING_TIMEOUT_ABS},
-        IoUring, IoUringData, Task,
+        IoUring, IoUringData, IoUringTaskId, Task,
     },
     time::Time,
 };
 
 #[derive(Default)]
 pub struct TimeoutLinkTask {
-    id: u64,
+    id: IoUringTaskId,
     timespec: timespec64,
 }
 
@@ -27,7 +27,7 @@ impl IoUring {
 }
 
 unsafe impl Task for TimeoutLinkTask {
-    fn id(&self) -> u64 {
+    fn id(&self) -> IoUringTaskId {
         self.id
     }
 

From 5b36980e72669b4850fa0c06f18597dbc85a1b30 Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 17:09:12 +0200
Subject: [PATCH 07/22] io_uring: allow manually cancelling tasks

---
 src/io_uring.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/io_uring.rs b/src/io_uring.rs
index 23b8749d..f5bd6b68 100644
--- a/src/io_uring.rs
+++ b/src/io_uring.rs
@@ -228,6 +228,11 @@ impl IoUring {
         self.ring.kill();
         res
     }
+
+    #[expect(dead_code)]
+    pub fn cancel(&self, id: IoUringTaskId) {
+        self.ring.cancel_task(id);
+    }
 }
 
 struct IoUringData {

From 874d0d0c59d41e1506493fbc1a4cb1e1f328a89c Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 17:11:19 +0200
Subject: [PATCH 08/22] io_uring: add ops for non-owning read/write operations

---
 src/io_uring.rs                               |   5 +-
 src/io_uring/ops.rs                           |   1 +
 src/io_uring/ops/read_write_no_cancel.rs      | 139 ++++++++++++++++++
 .../ops/read_write_no_cancel/tests.rs         |  50 +++++++
 4 files changed, 194 insertions(+), 1 deletion(-)
 create mode 100644 src/io_uring/ops/read_write_no_cancel.rs
 create mode 100644 src/io_uring/ops/read_write_no_cancel/tests.rs

diff --git a/src/io_uring.rs b/src/io_uring.rs
index f5bd6b68..865634e4 100644
--- a/src/io_uring.rs
+++ b/src/io_uring.rs
@@ -5,7 +5,8 @@ use {
         io_uring::{
             ops::{
                 accept::AcceptTask, async_cancel::AsyncCancelTask, connect::ConnectTask,
-                poll::PollTask, read_write::ReadWriteTask, recvmsg::RecvmsgTask,
+                poll::PollTask, read_write::ReadWriteTask,
+                read_write_no_cancel::ReadWriteNoCancelTask, recvmsg::RecvmsgTask,
                 sendmsg::SendmsgTask, timeout::TimeoutTask, timeout_link::TimeoutLinkTask,
             },
             pending_result::PendingResults,
@@ -205,6 +206,7 @@ impl IoUring {
             tasks: Default::default(),
             pending_results: Default::default(),
             cached_read_writes: Default::default(),
+            cached_read_writes_no_cancel: Default::default(),
             cached_cancels: Default::default(),
             cached_polls: Default::default(),
             cached_sendmsg: Default::default(),
@@ -266,6 +268,7 @@ struct IoUringData {
     pending_results: PendingResults,
 
     cached_read_writes: Stack<Box<ReadWriteTask>>,
+    cached_read_writes_no_cancel: Stack<Box<ReadWriteNoCancelTask>>,
     cached_cancels: Stack<Box<AsyncCancelTask>>,
     cached_polls: Stack<Box<PollTask>>,
     cached_sendmsg: Stack<Box<SendmsgTask>>,
diff --git a/src/io_uring/ops.rs b/src/io_uring/ops.rs
index 64ea36b7..15fc1235 100644
--- a/src/io_uring/ops.rs
+++ b/src/io_uring/ops.rs
@@ -5,6 +5,7 @@ pub mod async_cancel;
 pub mod connect;
 pub mod poll;
 pub mod read_write;
+pub mod read_write_no_cancel;
 pub mod recvmsg;
 pub mod sendmsg;
 pub mod timeout;
diff --git a/src/io_uring/ops/read_write_no_cancel.rs b/src/io_uring/ops/read_write_no_cancel.rs
new file mode 100644
index 00000000..16729de7
--- /dev/null
+++ b/src/io_uring/ops/read_write_no_cancel.rs
@@ -0,0 +1,139 @@
+#[cfg(test)]
+mod tests;
+
+use {
+    crate::{
+        io_uring::{
+            pending_result::PendingResult,
+            sys::{io_uring_sqe, IORING_OP_READ, IORING_OP_WRITE},
+            IoUring, IoUringData, IoUringError, IoUringTaskId, Task, TaskResultExt,
+        },
+        time::Time,
+        utils::on_drop::OnDrop,
+    },
+    uapi::{c, Fd},
+};
+
+impl IoUring {
+    #[expect(dead_code)]
+    pub async fn read_no_cancel(
+        &self,
+        fd: Fd,
+        offset: usize,
+        buf: &mut [u8],
+        cancel: impl FnOnce(IoUringTaskId),
+    ) -> Result<usize, IoUringError> {
+        self.perform_no_cancel(
+            fd,
+            offset,
+            buf.as_mut_ptr(),
+            buf.len(),
+            None,
+            IORING_OP_READ,
+            cancel,
+        )
+        .await
+    }
+
+    #[expect(dead_code)]
+    pub async fn write_no_cancel(
+        &self,
+        fd: Fd,
+        offset: usize,
+        buf: &[u8],
+        timeout: Option<Time>,
+        cancel: impl FnOnce(IoUringTaskId),
+    ) -> Result<usize, IoUringError> {
+        self.perform_no_cancel(
+            fd,
+            offset,
+            buf.as_ptr() as _,
+            buf.len(),
+            timeout,
+            IORING_OP_WRITE,
+            cancel,
+        )
+        .await
+    }
+
+    async fn perform_no_cancel(
+        &self,
+        fd: Fd,
+        offset: usize,
+        buf: *mut u8,
+        len: usize,
+        timeout: Option<Time>,
+        opcode: u8,
+        cancel: impl FnOnce(IoUringTaskId),
+    ) -> Result<usize, IoUringError> {
+        self.ring.check_destroyed()?;
+        let id = self.ring.id();
+        let pr = self.ring.pending_results.acquire();
+        {
+            let mut pw = self
+                .ring
+                .cached_read_writes_no_cancel
+                .pop()
+                .unwrap_or_default();
+            pw.opcode = opcode;
+            pw.id = id.id;
+            pw.has_timeout = timeout.is_some();
+            pw.fd = fd.raw();
+            pw.offset = offset;
+            pw.buf = buf as _;
+            pw.len = len;
+            pw.data = Some(ReadWriteTaskData { res: pr.clone() });
+            self.ring.schedule(pw);
+            if let Some(time) = timeout {
+                self.schedule_timeout_link(time);
+            }
+        }
+        let panic = OnDrop(|| panic!("Operation cannot be cancelled from userspace"));
+        cancel(id.id);
+        let res = Ok(pr.await.map(|v| v as usize)).merge();
+        panic.forget();
+        res
+    }
+}
+
+struct ReadWriteTaskData {
+    res: PendingResult,
+}
+
+#[derive(Default)]
+pub struct ReadWriteNoCancelTask {
+    id: IoUringTaskId,
+    has_timeout: bool,
+    fd: c::c_int,
+    offset: usize,
+    buf: usize,
+    len: usize,
+    data: Option<ReadWriteTaskData>,
+    opcode: u8,
+}
+
+unsafe impl Task for ReadWriteNoCancelTask {
+    fn id(&self) -> IoUringTaskId {
+        self.id
+    }
+
+    fn complete(mut self: Box<Self>, ring: &IoUringData, res: i32) {
+        if let Some(data) = self.data.take() {
+            data.res.complete(res);
+        }
+        ring.cached_read_writes_no_cancel.push(self);
+    }
+
+    fn encode(&self, sqe: &mut io_uring_sqe) {
+        sqe.opcode = self.opcode;
+        sqe.fd = self.fd as _;
+        sqe.u1.off = self.offset as _;
+        sqe.u2.addr = self.buf as _;
+        sqe.u3.rw_flags = 0;
+        sqe.len = self.len as _;
+    }
+
+    fn has_timeout(&self) -> bool {
+        self.has_timeout
+    }
+}
diff --git a/src/io_uring/ops/read_write_no_cancel/tests.rs b/src/io_uring/ops/read_write_no_cancel/tests.rs
new file mode 100644
index 00000000..d334426a
--- /dev/null
+++ b/src/io_uring/ops/read_write_no_cancel/tests.rs
@@ -0,0 +1,50 @@
+use {
+    crate::{
+        async_engine::AsyncEngine,
+        io_uring::{IoUring, IoUringError},
+        utils::{oserror::OsError, queue::AsyncQueue},
+        wheel::Wheel,
+    },
+    std::rc::Rc,
+    uapi::c::ECANCELED,
+};
+
+fn cancel(timeout: bool) {
+    let eng = AsyncEngine::new();
+    let ring = IoUring::new(&eng, 32).unwrap();
+    let ring2 = ring.clone();
+    let ring3 = ring.clone();
+    let wheel = Wheel::new(&eng, &ring).unwrap();
+    let queue = Rc::new(AsyncQueue::new());
+    let queue2 = queue.clone();
+    let _fut1 = eng.spawn(async move {
+        let (read, _write) = uapi::pipe().unwrap();
+        let mut buf = [10];
+        let res = ring
+            .read_no_cancel(read.borrow(), !0, &mut buf, |id| queue.push(id))
+            .await;
+        assert!(matches!(
+            res.unwrap_err(),
+            IoUringError::OsError(OsError(ECANCELED))
+        ));
+        ring.stop();
+    });
+    let _fut2 = eng.spawn(async move {
+        let id = queue2.pop().await;
+        if timeout {
+            wheel.timeout(1).await.unwrap();
+        }
+        ring2.cancel(id);
+    });
+    ring3.run().unwrap();
+}
+
+#[test]
+fn cancel_in_kernel() {
+    cancel(true);
+}
+
+#[test]
+fn cancel_in_userspace() {
+    cancel(true);
+}

From ed4ef3c8e761a05f35bbcb7fc5a1281c05bff5fd Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 17:18:50 +0200
Subject: [PATCH 09/22] cpu_worker: add worker-thread framework

---
 src/compositor.rs                        |   5 +
 src/cpu_worker.rs                        | 432 +++++++++++++++++++++++
 src/cpu_worker/jobs.rs                   |   1 +
 src/cpu_worker/jobs/read_write.rs        | 153 ++++++++
 src/cpu_worker/tests.rs                  | 117 ++++++
 src/io_uring.rs                          |   1 -
 src/io_uring/ops/read_write_no_cancel.rs |   2 -
 src/main.rs                              |   1 +
 src/state.rs                             |   3 +
 src/utils/queue.rs                       |   6 +
 10 files changed, 718 insertions(+), 3 deletions(-)
 create mode 100644 src/cpu_worker.rs
 create mode 100644 src/cpu_worker/jobs.rs
 create mode 100644 src/cpu_worker/jobs/read_write.rs
 create mode 100644 src/cpu_worker/tests.rs

diff --git a/src/compositor.rs b/src/compositor.rs
index ad5d5720..1a93f071 100644
--- a/src/compositor.rs
+++ b/src/compositor.rs
@@ -13,6 +13,7 @@ use {
         client::{ClientId, Clients},
         clientmem::{self, ClientMemError},
         config::ConfigProxy,
+        cpu_worker::{CpuWorker, CpuWorkerError},
         damage::{visualize_damage, DamageVisualizer},
         dbus::Dbus,
         ei::ei_client::EiClients,
@@ -107,6 +108,8 @@ pub enum CompositorError {
     WheelError(#[from] WheelError),
     #[error("Could not create an io-uring")]
     IoUringError(#[from] IoUringError),
+    #[error("Could not create cpu worker")]
+    CpuWorkerError(#[from] CpuWorkerError),
 }
 
 pub const WAYLAND_DISPLAY: &str = "WAYLAND_DISPLAY";
@@ -143,6 +146,7 @@ fn start_compositor2(
     let node_ids = NodeIds::default();
     let scales = RefCounted::default();
     scales.add(Scale::from_int(1));
+    let cpu_worker = Rc::new(CpuWorker::new(&ring, &engine)?);
     let state = Rc::new(State {
         xkb_ctx,
         backend: CloneCell::new(Rc::new(DummyBackend)),
@@ -258,6 +262,7 @@ fn start_compositor2(
         enable_ei_acceptor: Default::default(),
         ei_clients: EiClients::new(),
         slow_ei_clients: Default::default(),
+        cpu_worker,
     });
     state.tracker.register(ClientId::from_raw(0));
     create_dummy_output(&state);
diff --git a/src/cpu_worker.rs b/src/cpu_worker.rs
new file mode 100644
index 00000000..d3d6adca
--- /dev/null
+++ b/src/cpu_worker.rs
@@ -0,0 +1,432 @@
+pub mod jobs;
+#[cfg(test)]
+mod tests;
+
+use {
+    crate::{
+        async_engine::{AsyncEngine, SpawnedFuture},
+        io_uring::IoUring,
+        utils::{
+            buf::TypedBuf, copyhashmap::CopyHashMap, errorfmt::ErrorFmt, oserror::OsError,
+            ptr_ext::MutPtrExt, queue::AsyncQueue, stack::Stack,
+        },
+    },
+    parking_lot::Mutex,
+    std::{
+        any::Any,
+        cell::{Cell, RefCell},
+        collections::VecDeque,
+        mem,
+        ptr::NonNull,
+        rc::Rc,
+        sync::Arc,
+        thread,
+    },
+    thiserror::Error,
+    uapi::{c, OwnedFd},
+};
+
+pub trait CpuJob {
+    fn work(&mut self) -> &mut dyn CpuWork;
+    fn completed(self: Box<Self>);
+}
+
+pub trait CpuWork: Send {
+    fn run(&mut self) -> Option<Box<dyn AsyncCpuWork>>;
+
+    fn cancel_async(&mut self, ring: &Rc<IoUring>) {
+        let _ = ring;
+        unreachable!();
+    }
+
+    fn async_work_done(&mut self, work: Box<dyn AsyncCpuWork>) {
+        let _ = work;
+        unreachable!();
+    }
+}
+
+pub trait AsyncCpuWork {
+    fn run(
+        self: Box<Self>,
+        eng: &Rc<AsyncEngine>,
+        ring: &Rc<IoUring>,
+        completion: WorkCompletion,
+    ) -> SpawnedFuture<CompletedWork>;
+
+    fn into_any(self: Box<Self>) -> Box<dyn Any>;
+}
+
+pub struct WorkCompletion {
+    worker: Rc<Worker>,
+    id: CpuJobId,
+}
+
+pub struct CompletedWork(());
+
+impl WorkCompletion {
+    pub fn complete(self, work: Box<dyn AsyncCpuWork>) -> CompletedWork {
+        let job = self.worker.async_jobs.remove(&self.id).unwrap();
+        unsafe {
+            job.work.deref_mut().async_work_done(work);
+        }
+        self.worker.send_completion(self.id);
+        CompletedWork(())
+    }
+}
+
+pub struct CpuWorker {
+    data: Rc<CpuWorkerData>,
+    _completions_listener: SpawnedFuture<()>,
+    _job_enqueuer: SpawnedFuture<()>,
+}
+
+#[must_use]
+pub struct PendingJob {
+    id: CpuJobId,
+    thread_data: Rc<CpuWorkerData>,
+    job_data: Rc<PendingJobData>,
+}
+
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Default)]
+enum PendingJobState {
+    #[default]
+    Waiting,
+    Abandoned,
+    Completed,
+}
+
+#[derive(Default)]
+struct PendingJobData {
+    job: Cell<Option<NonNull<dyn CpuJob>>>,
+    state: Cell<PendingJobState>,
+}
+
+enum Job {
+    New {
+        id: CpuJobId,
+        work: *mut dyn CpuWork,
+    },
+    Cancel {
+        id: CpuJobId,
+    },
+}
+
+unsafe impl Send for Job {}
+
+struct CpuWorkerData {
+    next: CpuJobIds,
+    jobs_to_enqueue: AsyncQueue<Job>,
+    new_jobs: Arc<Mutex<VecDeque<Job>>>,
+    have_new_jobs: Rc<OwnedFd>,
+    completed_jobs_remote: Arc<Mutex<VecDeque<CpuJobId>>>,
+    completed_jobs_local: RefCell<VecDeque<CpuJobId>>,
+    have_completed_jobs: Rc<OwnedFd>,
+    pending_jobs: CopyHashMap<CpuJobId, Rc<PendingJobData>>,
+    ring: Rc<IoUring>,
+    _stop: OwnedFd,
+    pending_job_data_cache: Stack<Rc<PendingJobData>>,
+}
+
+linear_ids!(CpuJobIds, CpuJobId, u64);
+
+#[derive(Debug, Error)]
+pub enum CpuWorkerError {
+    #[error("Could not create a pipe")]
+    Pipe(#[source] OsError),
+    #[error("Could not create an eventfd")]
+    EventFd(#[source] OsError),
+    #[error("Could not dup an eventfd")]
+    Dup(#[source] OsError),
+}
+
+impl PendingJob {
+    #[expect(dead_code)]
+    pub fn detach(self) {
+        match self.job_data.state.get() {
+            PendingJobState::Waiting => {
+                self.job_data.state.set(PendingJobState::Abandoned);
+            }
+            PendingJobState::Abandoned => {
+                unreachable!();
+            }
+            PendingJobState::Completed => {}
+        }
+    }
+}
+
+impl Drop for CpuWorker {
+    fn drop(&mut self) {
+        self.data.do_equeue_jobs();
+        if self.data.pending_jobs.is_not_empty() {
+            log::warn!("CpuWorker dropped with pending jobs. Completed jobs will not be triggered.")
+        }
+    }
+}
+
+impl Drop for PendingJob {
+    fn drop(&mut self) {
+        match self.job_data.state.get() {
+            PendingJobState::Waiting => {
+                log::warn!("PendingJob dropped before completion. Blocking.");
+                let data = &self.thread_data;
+                let id = self.id;
+                self.job_data.state.set(PendingJobState::Abandoned);
+                data.jobs_to_enqueue.push(Job::Cancel { id });
+                data.do_equeue_jobs();
+                let mut buf = 0u64;
+                while data.pending_jobs.contains(&id) {
+                    if let Err(e) = uapi::read(data.have_completed_jobs.raw(), &mut buf) {
+                        panic!("Could not wait for job completions: {}", ErrorFmt(e));
+                    }
+                    data.dispatch_completions();
+                }
+            }
+            PendingJobState::Abandoned => {}
+            PendingJobState::Completed => {
+                self.thread_data
+                    .pending_job_data_cache
+                    .push(self.job_data.clone());
+            }
+        }
+    }
+}
+
+impl CpuWorkerData {
+    async fn wait_for_completions(self: Rc<Self>) {
+        let mut buf = TypedBuf::<u64>::new();
+        loop {
+            if let Err(e) = self.ring.read(&self.have_completed_jobs, buf.buf()).await {
+                log::error!("Could not wait for job completions: {}", ErrorFmt(e));
+                return;
+            }
+            self.dispatch_completions();
+        }
+    }
+
+    fn dispatch_completions(&self) {
+        let completions = &mut *self.completed_jobs_local.borrow_mut();
+        mem::swap(completions, &mut *self.completed_jobs_remote.lock());
+        while let Some(id) = completions.pop_front() {
+            let job_data = self.pending_jobs.remove(&id).unwrap();
+            let job = job_data.job.take().unwrap();
+            let job = unsafe { Box::from_raw(job.as_ptr()) };
+            match job_data.state.get() {
+                PendingJobState::Waiting => {
+                    job_data.state.set(PendingJobState::Completed);
+                    job.completed();
+                }
+                PendingJobState::Abandoned => {
+                    self.pending_job_data_cache.push(job_data);
+                }
+                PendingJobState::Completed => {
+                    unreachable!();
+                }
+            }
+        }
+    }
+
+    async fn equeue_jobs(self: Rc<Self>) {
+        loop {
+            self.jobs_to_enqueue.non_empty().await;
+            self.do_equeue_jobs();
+        }
+    }
+
+    fn do_equeue_jobs(&self) {
+        self.jobs_to_enqueue.move_to(&mut self.new_jobs.lock());
+        if let Err(e) = uapi::eventfd_write(self.have_new_jobs.raw(), 1) {
+            panic!("Could not signal eventfd: {}", ErrorFmt(e));
+        }
+    }
+}
+
+impl CpuWorker {
+    pub fn new(ring: &Rc<IoUring>, eng: &Rc<AsyncEngine>) -> Result<Self, CpuWorkerError> {
+        let new_jobs: Arc<Mutex<VecDeque<Job>>> = Default::default();
+        let completed_jobs: Arc<Mutex<VecDeque<CpuJobId>>> = Default::default();
+        let (stop_read, stop_write) =
+            uapi::pipe2(c::O_CLOEXEC).map_err(|e| CpuWorkerError::Pipe(e.into()))?;
+        let have_new_jobs =
+            uapi::eventfd(0, c::EFD_CLOEXEC).map_err(|e| CpuWorkerError::EventFd(e.into()))?;
+        let have_completed_jobs =
+            uapi::eventfd(0, c::EFD_CLOEXEC).map_err(|e| CpuWorkerError::EventFd(e.into()))?;
+        thread::Builder::new()
+            .name("cpu worker".to_string())
+            .spawn({
+                let new_jobs = new_jobs.clone();
+                let completed_jobs = completed_jobs.clone();
+                let have_new_jobs = uapi::fcntl_dupfd_cloexec(have_new_jobs.raw(), 0)
+                    .map_err(|e| CpuWorkerError::Dup(e.into()))?;
+                let have_completed_jobs = uapi::fcntl_dupfd_cloexec(have_completed_jobs.raw(), 0)
+                    .map_err(|e| CpuWorkerError::Dup(e.into()))?;
+                move || {
+                    work(
+                        new_jobs,
+                        completed_jobs,
+                        stop_write,
+                        have_new_jobs,
+                        have_completed_jobs,
+                    )
+                }
+            })
+            .unwrap();
+        let data = Rc::new(CpuWorkerData {
+            next: Default::default(),
+            jobs_to_enqueue: Default::default(),
+            new_jobs,
+            have_new_jobs: Rc::new(have_new_jobs),
+            completed_jobs_remote: completed_jobs,
+            completed_jobs_local: Default::default(),
+            have_completed_jobs: Rc::new(have_completed_jobs),
+            pending_jobs: Default::default(),
+            ring: ring.clone(),
+            _stop: stop_read,
+            pending_job_data_cache: Default::default(),
+        });
+        Ok(Self {
+            _completions_listener: eng.spawn(data.clone().wait_for_completions()),
+            _job_enqueuer: eng.spawn(data.clone().equeue_jobs()),
+            data,
+        })
+    }
+
+    #[expect(dead_code)]
+    pub fn submit(&self, job: Box<dyn CpuJob>) -> PendingJob {
+        let mut job = NonNull::from(Box::leak(job));
+        let id = self.data.next.next();
+        self.data.jobs_to_enqueue.push(Job::New {
+            id,
+            work: unsafe { job.as_mut().work() },
+        });
+        let job_data = self.data.pending_job_data_cache.pop().unwrap_or_default();
+        job_data.job.set(Some(job));
+        job_data.state.set(PendingJobState::Waiting);
+        self.data.pending_jobs.set(id, job_data.clone());
+        PendingJob {
+            id,
+            thread_data: self.data.clone(),
+            job_data,
+        }
+    }
+}
+
+fn work(
+    new_jobs: Arc<Mutex<VecDeque<Job>>>,
+    completed_jobs: Arc<Mutex<VecDeque<CpuJobId>>>,
+    stop: OwnedFd,
+    have_new_jobs: OwnedFd,
+    have_completed_jobs: OwnedFd,
+) {
+    let eng = AsyncEngine::new();
+    let ring = IoUring::new(&eng, 32).unwrap();
+    let worker = Rc::new(Worker {
+        eng,
+        ring,
+        completed_jobs,
+        have_completed_jobs,
+        async_jobs: Default::default(),
+        stopped: Cell::new(false),
+    });
+    let _stop_listener = worker.eng.spawn(worker.clone().handle_stop(stop));
+    let _new_job_listener = worker
+        .eng
+        .spawn(worker.clone().handle_new_jobs(new_jobs, have_new_jobs));
+    if let Err(e) = worker.ring.run() {
+        panic!("io_uring failed: {}", ErrorFmt(e));
+    }
+}
+
+struct Worker {
+    eng: Rc<AsyncEngine>,
+    ring: Rc<IoUring>,
+    completed_jobs: Arc<Mutex<VecDeque<CpuJobId>>>,
+    have_completed_jobs: OwnedFd,
+    async_jobs: CopyHashMap<CpuJobId, AsyncJob>,
+    stopped: Cell<bool>,
+}
+
+struct AsyncJob {
+    _future: SpawnedFuture<CompletedWork>,
+    work: *mut dyn CpuWork,
+}
+
+impl Worker {
+    async fn handle_stop(self: Rc<Self>, stop: OwnedFd) {
+        let stop = Rc::new(stop);
+        if let Err(e) = self.ring.poll(&stop, 0).await {
+            log::error!(
+                "Could not wait for stop fd to become readable: {}",
+                ErrorFmt(e)
+            );
+        } else {
+            assert!(self.async_jobs.is_empty());
+            self.stopped.set(true);
+            self.ring.stop();
+        }
+    }
+
+    async fn handle_new_jobs(
+        self: Rc<Self>,
+        jobs_remote: Arc<Mutex<VecDeque<Job>>>,
+        new_jobs: OwnedFd,
+    ) {
+        let mut buf = TypedBuf::<u64>::new();
+        let new_jobs = Rc::new(new_jobs);
+        let mut jobs = VecDeque::new();
+        loop {
+            if let Err(e) = self.ring.read(&new_jobs, buf.buf()).await {
+                if self.stopped.get() {
+                    return;
+                }
+                panic!(
+                    "Could not wait for new jobs fd to be signaled: {}",
+                    ErrorFmt(e),
+                );
+            }
+            mem::swap(&mut jobs, &mut *jobs_remote.lock());
+            while let Some(job) = jobs.pop_front() {
+                self.handle_new_job(job);
+            }
+        }
+    }
+
+    fn handle_new_job(self: &Rc<Self>, job: Job) {
+        match job {
+            Job::Cancel { id } => {
+                let mut jobs = self.async_jobs.lock();
+                if let Some(job) = jobs.get_mut(&id) {
+                    unsafe {
+                        job.work.deref_mut().cancel_async(&self.ring);
+                    }
+                }
+            }
+            Job::New { id, work } => match unsafe { work.deref_mut() }.run() {
+                None => {
+                    self.send_completion(id);
+                    return;
+                }
+                Some(w) => {
+                    let completion = WorkCompletion {
+                        worker: self.clone(),
+                        id,
+                    };
+                    let future = w.run(&self.eng, &self.ring, completion);
+                    self.async_jobs.set(
+                        id,
+                        AsyncJob {
+                            _future: future,
+                            work,
+                        },
+                    );
+                }
+            },
+        }
+    }
+
+    fn send_completion(&self, id: CpuJobId) {
+        self.completed_jobs.lock().push_back(id);
+        if let Err(e) = uapi::eventfd_write(self.have_completed_jobs.raw(), 1) {
+            panic!("Could not signal job completion: {}", ErrorFmt(e));
+        }
+    }
+}
diff --git a/src/cpu_worker/jobs.rs b/src/cpu_worker/jobs.rs
new file mode 100644
index 00000000..aadb998f
--- /dev/null
+++ b/src/cpu_worker/jobs.rs
@@ -0,0 +1 @@
+pub mod read_write;
diff --git a/src/cpu_worker/jobs/read_write.rs b/src/cpu_worker/jobs/read_write.rs
new file mode 100644
index 00000000..c8790529
--- /dev/null
+++ b/src/cpu_worker/jobs/read_write.rs
@@ -0,0 +1,153 @@
+use {
+    crate::{
+        async_engine::{AsyncEngine, SpawnedFuture},
+        cpu_worker::{AsyncCpuWork, CompletedWork, CpuWork, WorkCompletion},
+        io_uring::{IoUring, IoUringError, IoUringTaskId},
+    },
+    std::{
+        any::Any,
+        ptr,
+        rc::Rc,
+        slice,
+        sync::{
+            atomic::{AtomicBool, AtomicU64, Ordering::Relaxed},
+            Arc,
+        },
+    },
+    thiserror::Error,
+    uapi::{c, Fd},
+};
+
+#[derive(Debug, Error)]
+pub enum ReadWriteJobError {
+    #[error("An io_uring error occurred")]
+    IoUring(#[source] IoUringError),
+    #[error("The job was cancelled")]
+    Cancelled,
+    #[error("Tried to operate outside the bounds of the file descriptor")]
+    OutOfBounds,
+}
+
+pub struct ReadWriteWork {
+    cancel: Arc<CancelState>,
+    config: Option<Box<ReadWriteWorkConfig>>,
+}
+
+unsafe impl Send for ReadWriteWork {}
+
+impl ReadWriteWork {
+    #[expect(dead_code)]
+    pub unsafe fn new() -> Self {
+        let cancel = Arc::new(CancelState::default());
+        ReadWriteWork {
+            cancel: cancel.clone(),
+            config: Some(Box::new(ReadWriteWorkConfig {
+                fd: -1,
+                offset: 0,
+                ptr: ptr::null_mut(),
+                len: 0,
+                write: false,
+                cancel,
+                result: None,
+            })),
+        }
+    }
+
+    #[expect(dead_code)]
+    pub fn config(&mut self) -> &mut ReadWriteWorkConfig {
+        self.config.as_mut().unwrap()
+    }
+}
+
+pub struct ReadWriteWorkConfig {
+    pub fd: c::c_int,
+    pub offset: usize,
+    pub ptr: *mut u8,
+    pub len: usize,
+    pub write: bool,
+    pub result: Option<Result<(), ReadWriteJobError>>,
+    cancel: Arc<CancelState>,
+}
+
+#[derive(Default)]
+struct CancelState {
+    cancelled: AtomicBool,
+    cancel_id: AtomicU64,
+}
+
+impl CpuWork for ReadWriteWork {
+    fn run(&mut self) -> Option<Box<dyn AsyncCpuWork>> {
+        self.cancel.cancelled.store(false, Relaxed);
+        self.cancel.cancel_id.store(0, Relaxed);
+        self.config.take().map(|b| b as _)
+    }
+
+    fn cancel_async(&mut self, ring: &Rc<IoUring>) {
+        self.cancel.cancelled.store(true, Relaxed);
+        let id = self.cancel.cancel_id.load(Relaxed);
+        if id != 0 {
+            ring.cancel(IoUringTaskId::from_raw(id));
+        }
+    }
+
+    fn async_work_done(&mut self, work: Box<dyn AsyncCpuWork>) {
+        let work = work.into_any().downcast().unwrap();
+        self.config = Some(work);
+    }
+}
+
+impl AsyncCpuWork for ReadWriteWorkConfig {
+    fn run(
+        mut self: Box<Self>,
+        eng: &Rc<AsyncEngine>,
+        ring: &Rc<IoUring>,
+        completion: WorkCompletion,
+    ) -> SpawnedFuture<CompletedWork> {
+        let ring = ring.clone();
+        eng.spawn(async move {
+            let res = loop {
+                if self.cancel.cancelled.load(Relaxed) {
+                    break Err(ReadWriteJobError::Cancelled);
+                }
+                if self.len == 0 {
+                    break Ok(());
+                };
+                let res = if self.write {
+                    ring.write_no_cancel(
+                        Fd::new(self.fd),
+                        self.offset,
+                        unsafe { slice::from_raw_parts(self.ptr, self.len) },
+                        None,
+                        |id| self.cancel.cancel_id.store(id.raw(), Relaxed),
+                    )
+                    .await
+                } else {
+                    ring.read_no_cancel(
+                        Fd::new(self.fd),
+                        self.offset,
+                        unsafe { slice::from_raw_parts_mut(self.ptr, self.len) },
+                        |id| self.cancel.cancel_id.store(id.raw(), Relaxed),
+                    )
+                    .await
+                };
+                match res {
+                    Ok(0) => break Err(ReadWriteJobError::OutOfBounds),
+                    Ok(n) => {
+                        self.len -= n;
+                        self.offset += n;
+                        unsafe {
+                            self.ptr = self.ptr.add(n);
+                        }
+                    }
+                    Err(e) => break Err(ReadWriteJobError::IoUring(e)),
+                }
+            };
+            self.result = Some(res);
+            completion.complete(self)
+        })
+    }
+
+    fn into_any(self: Box<Self>) -> Box<dyn Any> {
+        self
+    }
+}
diff --git a/src/cpu_worker/tests.rs b/src/cpu_worker/tests.rs
new file mode 100644
index 00000000..6b0bb826
--- /dev/null
+++ b/src/cpu_worker/tests.rs
@@ -0,0 +1,117 @@
+use {
+    crate::{
+        async_engine::{AsyncEngine, SpawnedFuture},
+        cpu_worker::{AsyncCpuWork, CompletedWork, CpuJob, CpuWork, CpuWorker, WorkCompletion},
+        io_uring::IoUring,
+        utils::asyncevent::AsyncEvent,
+        wheel::Wheel,
+    },
+    std::{any::Any, future::pending, rc::Rc, sync::Arc},
+    uapi::{c::EFD_CLOEXEC, OwnedFd},
+};
+
+struct Job {
+    ae: Rc<AsyncEvent>,
+    work: Work,
+    cancel: bool,
+}
+struct Work(Arc<OwnedFd>);
+struct AsyncWork(Arc<OwnedFd>);
+
+impl CpuJob for Job {
+    fn work(&mut self) -> &mut dyn CpuWork {
+        &mut self.work
+    }
+
+    fn completed(self: Box<Self>) {
+        if self.cancel {
+            unreachable!();
+        } else {
+            self.ae.trigger();
+        }
+    }
+}
+
+impl Drop for Job {
+    fn drop(&mut self) {
+        if self.cancel {
+            self.ae.trigger();
+        }
+    }
+}
+
+impl CpuWork for Work {
+    fn run(&mut self) -> Option<Box<dyn AsyncCpuWork>> {
+        Some(Box::new(AsyncWork(self.0.clone())))
+    }
+
+    fn cancel_async(&mut self, _ring: &Rc<IoUring>) {
+        uapi::eventfd_write(self.0.raw(), 1).unwrap();
+    }
+
+    fn async_work_done(&mut self, work: Box<dyn AsyncCpuWork>) {
+        let _ = work;
+    }
+}
+
+impl AsyncCpuWork for AsyncWork {
+    fn run(
+        self: Box<Self>,
+        eng: &Rc<AsyncEngine>,
+        ring: &Rc<IoUring>,
+        completion: WorkCompletion,
+    ) -> SpawnedFuture<CompletedWork> {
+        let ring = ring.clone();
+        eng.spawn(async move {
+            let mut buf = [0; 8];
+            let res = ring
+                .read_no_cancel(self.0.borrow(), 0, &mut buf, |_| ())
+                .await;
+            res.unwrap();
+            completion.complete(self)
+        })
+    }
+
+    fn into_any(self: Box<Self>) -> Box<dyn Any> {
+        self
+    }
+}
+
+fn run(cancel: bool) {
+    let eng = AsyncEngine::new();
+    let ring = IoUring::new(&eng, 32).unwrap();
+    let ring2 = ring.clone();
+    let wheel = Wheel::new(&eng, &ring).unwrap();
+    let cpu = Rc::new(CpuWorker::new(&ring, &eng).unwrap());
+    let ae = Rc::new(AsyncEvent::default());
+    let eventfd = Arc::new(uapi::eventfd(0, EFD_CLOEXEC).unwrap());
+    let pending_job = cpu.submit(Box::new(Job {
+        ae: ae.clone(),
+        work: Work(eventfd.clone()),
+        cancel,
+    }));
+    let _fut1 = eng.spawn(async move {
+        wheel.timeout(1).await.unwrap();
+        if cancel {
+            drop(pending_job);
+        } else {
+            uapi::eventfd_write(eventfd.raw(), 1).unwrap();
+            pending::<()>().await;
+        }
+    });
+    let _fut2 = eng.spawn(async move {
+        ae.triggered().await;
+        ring2.stop();
+    });
+    ring.run().unwrap();
+}
+
+#[test]
+fn cancel() {
+    run(true);
+}
+
+#[test]
+fn complete() {
+    run(false);
+}
diff --git a/src/io_uring.rs b/src/io_uring.rs
index 865634e4..60c5291a 100644
--- a/src/io_uring.rs
+++ b/src/io_uring.rs
@@ -231,7 +231,6 @@ impl IoUring {
         res
     }
 
-    #[expect(dead_code)]
     pub fn cancel(&self, id: IoUringTaskId) {
         self.ring.cancel_task(id);
     }
diff --git a/src/io_uring/ops/read_write_no_cancel.rs b/src/io_uring/ops/read_write_no_cancel.rs
index 16729de7..02cc2107 100644
--- a/src/io_uring/ops/read_write_no_cancel.rs
+++ b/src/io_uring/ops/read_write_no_cancel.rs
@@ -15,7 +15,6 @@ use {
 };
 
 impl IoUring {
-    #[expect(dead_code)]
     pub async fn read_no_cancel(
         &self,
         fd: Fd,
@@ -35,7 +34,6 @@ impl IoUring {
         .await
     }
 
-    #[expect(dead_code)]
     pub async fn write_no_cancel(
         &self,
         fd: Fd,
diff --git a/src/main.rs b/src/main.rs
index 82655149..4e47f8dd 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -54,6 +54,7 @@ mod client;
 mod clientmem;
 mod compositor;
 mod config;
+mod cpu_worker;
 mod cursor;
 mod cursor_user;
 mod damage;
diff --git a/src/state.rs b/src/state.rs
index 98636a3d..fb44f9f6 100644
--- a/src/state.rs
+++ b/src/state.rs
@@ -13,6 +13,7 @@ use {
         clientmem::ClientMemOffset,
         compositor::LIBEI_SOCKET,
         config::ConfigProxy,
+        cpu_worker::CpuWorker,
         cursor::{Cursor, ServerCursors},
         cursor_user::{CursorUserGroup, CursorUserGroupId, CursorUserGroupIds, CursorUserIds},
         damage::DamageVisualizer,
@@ -214,6 +215,8 @@ pub struct State {
     pub enable_ei_acceptor: Cell<bool>,
     pub ei_clients: EiClients,
     pub slow_ei_clients: AsyncQueue<Rc<EiClient>>,
+    #[expect(dead_code)]
+    pub cpu_worker: Rc<CpuWorker>,
 }
 
 // impl Drop for State {
diff --git a/src/utils/queue.rs b/src/utils/queue.rs
index 8e55ba6b..c39a28d5 100644
--- a/src/utils/queue.rs
+++ b/src/utils/queue.rs
@@ -64,6 +64,12 @@ impl<T> AsyncQueue<T> {
         }
         self.waiter.take();
     }
+
+    pub fn move_to(&self, other: &mut VecDeque<T>) {
+        unsafe {
+            other.append(self.data.get().deref_mut());
+        }
+    }
 }
 
 pub struct AsyncQueuePop<'a, T> {

From c96802490577f8e45c8fe30bab8d554bcd0b762d Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 20:22:30 +0200
Subject: [PATCH 10/22] gfx: add ShmGfxTexture

---
 src/cursor.rs                       |  3 ++-
 src/gfx_api.rs                      |  8 ++++++--
 src/gfx_apis/gl/renderer/context.rs |  8 ++++----
 src/gfx_apis/gl/renderer/texture.rs |  8 +++++++-
 src/gfx_apis/vulkan.rs              |  8 ++++----
 src/gfx_apis/vulkan/image.rs        | 10 +++++++++-
 src/ifs/wl_buffer.rs                |  2 +-
 src/ifs/wl_surface.rs               |  5 +++--
 src/it/test_gfx_api.rs              | 13 ++++++++++---
 src/text.rs                         |  8 +++++---
 10 files changed, 51 insertions(+), 22 deletions(-)

diff --git a/src/cursor.rs b/src/cursor.rs
index 6d8e1d58..8f8d1775 100644
--- a/src/cursor.rs
+++ b/src/cursor.rs
@@ -328,7 +328,8 @@ impl CursorImageScaled {
             extents: Rect::new_sized(-xhot, -yhot, width, height).unwrap(),
             tex: ctx
                 .clone()
-                .shmem_texture(None, data, ARGB8888, width, height, width * 4, None)?,
+                .shmem_texture(None, data, ARGB8888, width, height, width * 4, None)?
+                .into_texture(),
         }))
     }
 }
diff --git a/src/gfx_api.rs b/src/gfx_api.rs
index d8100264..659821bf 100644
--- a/src/gfx_api.rs
+++ b/src/gfx_api.rs
@@ -531,6 +531,10 @@ pub trait GfxTexture: Debug {
     fn format(&self) -> &'static Format;
 }
 
+pub trait ShmGfxTexture: GfxTexture {
+    fn into_texture(self: Rc<Self>) -> Rc<dyn GfxTexture>;
+}
+
 pub trait GfxContext: Debug {
     fn reset_status(&self) -> Option<ResetStatus>;
 
@@ -546,14 +550,14 @@ pub trait GfxContext: Debug {
 
     fn shmem_texture(
         self: Rc<Self>,
-        old: Option<Rc<dyn GfxTexture>>,
+        old: Option<Rc<dyn ShmGfxTexture>>,
         data: &[Cell<u8>],
         format: &'static Format,
         width: i32,
         height: i32,
         stride: i32,
         damage: Option<&[Rect]>,
-    ) -> Result<Rc<dyn GfxTexture>, GfxError>;
+    ) -> Result<Rc<dyn ShmGfxTexture>, GfxError>;
 
     fn allocator(&self) -> Rc<dyn Allocator>;
 
diff --git a/src/gfx_apis/gl/renderer/context.rs b/src/gfx_apis/gl/renderer/context.rs
index 4d9568af..1948ac16 100644
--- a/src/gfx_apis/gl/renderer/context.rs
+++ b/src/gfx_apis/gl/renderer/context.rs
@@ -4,7 +4,7 @@ use {
         format::{Format, XRGB8888},
         gfx_api::{
             BufferResvUser, GfxApiOpt, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage,
-            GfxTexture, ResetStatus,
+            ResetStatus, ShmGfxTexture,
         },
         gfx_apis::gl::{
             egl::{context::EglContext, display::EglDisplay, image::EglImage},
@@ -265,17 +265,17 @@ impl GfxContext for GlRenderContext {
 
     fn shmem_texture(
         self: Rc<Self>,
-        _old: Option<Rc<dyn GfxTexture>>,
+        _old: Option<Rc<dyn ShmGfxTexture>>,
         data: &[Cell<u8>],
         format: &'static Format,
         width: i32,
         height: i32,
         stride: i32,
         _damage: Option<&[Rect]>,
-    ) -> Result<Rc<dyn GfxTexture>, GfxError> {
+    ) -> Result<Rc<dyn ShmGfxTexture>, GfxError> {
         (&self)
             .shmem_texture(data, format, width, height, stride)
-            .map(|w| w as Rc<dyn GfxTexture>)
+            .map(|w| w as Rc<dyn ShmGfxTexture>)
             .map_err(|e| e.into())
     }
 
diff --git a/src/gfx_apis/gl/renderer/texture.rs b/src/gfx_apis/gl/renderer/texture.rs
index 003fe6be..e4a97575 100644
--- a/src/gfx_apis/gl/renderer/texture.rs
+++ b/src/gfx_apis/gl/renderer/texture.rs
@@ -1,7 +1,7 @@
 use {
     crate::{
         format::Format,
-        gfx_api::{GfxError, GfxTexture},
+        gfx_api::{GfxError, GfxTexture, ShmGfxTexture},
         gfx_apis::gl::{
             gl::texture::GlTexture,
             renderer::{context::GlRenderContext, framebuffer::Framebuffer},
@@ -82,3 +82,9 @@ impl GfxTexture for Texture {
         self.format
     }
 }
+
+impl ShmGfxTexture for Texture {
+    fn into_texture(self: Rc<Self>) -> Rc<dyn GfxTexture> {
+        self
+    }
+}
diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs
index 49f3dac0..09827d4e 100644
--- a/src/gfx_apis/vulkan.rs
+++ b/src/gfx_apis/vulkan.rs
@@ -21,7 +21,7 @@ use {
         async_engine::AsyncEngine,
         format::Format,
         gfx_api::{
-            GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage, GfxTexture, ResetStatus,
+            GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage, ResetStatus, ShmGfxTexture,
         },
         gfx_apis::vulkan::{
             image::VulkanImageMemory, instance::VulkanInstance, renderer::VulkanRenderer,
@@ -249,16 +249,16 @@ impl GfxContext for Context {
 
     fn shmem_texture(
         self: Rc<Self>,
-        old: Option<Rc<dyn GfxTexture>>,
+        old: Option<Rc<dyn ShmGfxTexture>>,
         data: &[Cell<u8>],
         format: &'static Format,
         width: i32,
         height: i32,
         stride: i32,
         damage: Option<&[Rect]>,
-    ) -> Result<Rc<dyn GfxTexture>, GfxError> {
+    ) -> Result<Rc<dyn ShmGfxTexture>, GfxError> {
         if let Some(old) = old {
-            let old = old.into_vk(&self.0.device.device);
+            let old = old.into_texture().into_vk(&self.0.device.device);
             let shm = match &old.ty {
                 VulkanImageMemory::DmaBuf(_) => unreachable!(),
                 VulkanImageMemory::Internal(shm) => shm,
diff --git a/src/gfx_apis/vulkan/image.rs b/src/gfx_apis/vulkan/image.rs
index ee0555a5..29469a79 100644
--- a/src/gfx_apis/vulkan/image.rs
+++ b/src/gfx_apis/vulkan/image.rs
@@ -1,7 +1,9 @@
 use {
     crate::{
         format::Format,
-        gfx_api::{GfxApiOpt, GfxError, GfxFramebuffer, GfxImage, GfxTexture, SyncFile},
+        gfx_api::{
+            GfxApiOpt, GfxError, GfxFramebuffer, GfxImage, GfxTexture, ShmGfxTexture, SyncFile,
+        },
         gfx_apis::vulkan::{
             allocator::VulkanAllocation, device::VulkanDevice, format::VulkanModifierLimits,
             renderer::VulkanRenderer, shm_image::VulkanShmImage, VulkanError,
@@ -530,3 +532,9 @@ impl GfxTexture for VulkanImage {
         self.format
     }
 }
+
+impl ShmGfxTexture for VulkanImage {
+    fn into_texture(self: Rc<Self>) -> Rc<dyn GfxTexture> {
+        self
+    }
+}
diff --git a/src/ifs/wl_buffer.rs b/src/ifs/wl_buffer.rs
index ffe03415..268f72a8 100644
--- a/src/ifs/wl_buffer.rs
+++ b/src/ifs/wl_buffer.rs
@@ -201,7 +201,7 @@ impl WlBuffer {
         match &*self.storage.borrow() {
             None => None,
             Some(s) => match s {
-                WlBufferStorage::Shm { .. } => surface.shm_texture.get(),
+                WlBufferStorage::Shm { .. } => surface.shm_texture.get().map(|t| t.into_texture()),
                 WlBufferStorage::Dmabuf { tex, .. } => tex.clone(),
             },
         }
diff --git a/src/ifs/wl_surface.rs b/src/ifs/wl_surface.rs
index d04b82b2..48422742 100644
--- a/src/ifs/wl_surface.rs
+++ b/src/ifs/wl_surface.rs
@@ -23,7 +23,8 @@ use {
         drm_feedback::DrmFeedback,
         fixed::Fixed,
         gfx_api::{
-            AcquireSync, BufferResv, BufferResvUser, GfxTexture, ReleaseSync, SampleRect, SyncFile,
+            AcquireSync, BufferResv, BufferResvUser, ReleaseSync, SampleRect, ShmGfxTexture,
+            SyncFile,
         },
         ifs::{
             wl_buffer::WlBuffer,
@@ -271,7 +272,7 @@ pub struct WlSurface {
     pub buffer: CloneCell<Option<Rc<SurfaceBuffer>>>,
     buffer_presented: Cell<bool>,
     buffer_had_frame_request: Cell<bool>,
-    pub shm_texture: CloneCell<Option<Rc<dyn GfxTexture>>>,
+    pub shm_texture: CloneCell<Option<Rc<dyn ShmGfxTexture>>>,
     pub buf_x: NumCell<i32>,
     pub buf_y: NumCell<i32>,
     pub children: RefCell<Option<Box<ParentData>>>,
diff --git a/src/it/test_gfx_api.rs b/src/it/test_gfx_api.rs
index 19f55018..ef37188c 100644
--- a/src/it/test_gfx_api.rs
+++ b/src/it/test_gfx_api.rs
@@ -4,7 +4,8 @@ use {
         format::{Format, ARGB8888, XRGB8888},
         gfx_api::{
             CopyTexture, FillRect, FramebufferRect, GfxApiOpt, GfxContext, GfxError, GfxFormat,
-            GfxFramebuffer, GfxImage, GfxTexture, GfxWriteModifier, ResetStatus, SyncFile,
+            GfxFramebuffer, GfxImage, GfxTexture, GfxWriteModifier, ResetStatus, ShmGfxTexture,
+            SyncFile,
         },
         rect::Rect,
         theme::Color,
@@ -108,14 +109,14 @@ impl GfxContext for TestGfxCtx {
 
     fn shmem_texture(
         self: Rc<Self>,
-        _old: Option<Rc<dyn GfxTexture>>,
+        _old: Option<Rc<dyn ShmGfxTexture>>,
         data: &[Cell<u8>],
         format: &'static Format,
         width: i32,
         height: i32,
         stride: i32,
         _damage: Option<&[Rect]>,
-    ) -> Result<Rc<dyn GfxTexture>, GfxError> {
+    ) -> Result<Rc<dyn ShmGfxTexture>, GfxError> {
         assert!(stride >= width * 4);
         let size = (stride * height) as usize;
         assert!(data.len() >= size);
@@ -301,6 +302,12 @@ impl GfxTexture for TestGfxImage {
     }
 }
 
+impl ShmGfxTexture for TestGfxImage {
+    fn into_texture(self: Rc<Self>) -> Rc<dyn GfxTexture> {
+        self
+    }
+}
+
 impl GfxImage for TestGfxImage {
     fn to_framebuffer(self: Rc<Self>) -> Result<Rc<dyn GfxFramebuffer>, GfxError> {
         Ok(Rc::new(TestGfxFb {
diff --git a/src/text.rs b/src/text.rs
index 77dfbacf..104e8776 100644
--- a/src/text.rs
+++ b/src/text.rs
@@ -1,7 +1,7 @@
 use {
     crate::{
         format::ARGB8888,
-        gfx_api::{GfxContext, GfxError, GfxTexture},
+        gfx_api::{GfxContext, GfxError, GfxTexture, ShmGfxTexture},
         pango::{
             consts::{
                 CAIRO_FORMAT_ARGB32, CAIRO_OPERATOR_SOURCE, PANGO_ELLIPSIZE_END, PANGO_SCALE,
@@ -73,6 +73,7 @@ impl<'a> Config<'a> {
 #[derive(Clone)]
 pub struct TextTexture {
     config: Rc<Config<'static>>,
+    shm_texture: Rc<dyn ShmGfxTexture>,
     pub texture: Rc<dyn GfxTexture>,
 }
 
@@ -210,7 +211,7 @@ fn render2(
         Ok(d) => d,
         Err(e) => return Err(TextError::ImageData(e)),
     };
-    let old = old.map(|o| o.texture);
+    let old = old.map(|o| o.shm_texture);
     match ctx.clone().shmem_texture(
         old,
         bytes,
@@ -222,7 +223,8 @@ fn render2(
     ) {
         Ok(t) => Ok(TextTexture {
             config: Rc::new(config.to_static()),
-            texture: t,
+            texture: t.clone().into_texture(),
+            shm_texture: t,
         }),
         Err(e) => Err(TextError::RenderError(e)),
     }

From f213372b8ef3aaa6ad688a00a60c1366aac86e54 Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 17:30:32 +0200
Subject: [PATCH 11/22] gfx: add async shm api

---
 src/gfx_api.rs                      | 66 ++++++++++++++++++++++++++++-
 src/gfx_apis/gl/renderer/context.rs | 16 ++++++-
 src/gfx_apis/vulkan.rs              | 15 ++++++-
 src/it/test_gfx_api.rs              | 18 ++++++--
 4 files changed, 108 insertions(+), 7 deletions(-)

diff --git a/src/gfx_api.rs b/src/gfx_api.rs
index 659821bf..e0c4c28e 100644
--- a/src/gfx_api.rs
+++ b/src/gfx_api.rs
@@ -1,11 +1,13 @@
 use {
     crate::{
         allocator::Allocator,
+        clientmem::ClientMemOffset,
+        cpu_worker::CpuWorker,
         cursor::Cursor,
         damage::DamageVisualizer,
         fixed::Fixed,
         format::Format,
-        rect::Rect,
+        rect::{Rect, Region},
         renderer::{renderer_base::RendererBase, RenderResult, Renderer},
         scale::Scale,
         state::State,
@@ -535,6 +537,45 @@ pub trait ShmGfxTexture: GfxTexture {
     fn into_texture(self: Rc<Self>) -> Rc<dyn GfxTexture>;
 }
 
+#[expect(dead_code)]
+pub trait AsyncShmGfxTextureCallback {
+    fn completed(self: Rc<Self>, res: Result<(), GfxError>);
+}
+
+pub trait AsyncShmGfxTextureUploadCancellable {
+    fn cancel(&self, id: u64);
+}
+
+pub struct PendingShmUpload {
+    cancel: Rc<dyn AsyncShmGfxTextureUploadCancellable>,
+    id: u64,
+}
+
+pub trait AsyncShmGfxTexture: GfxTexture {
+    #[expect(dead_code)]
+    fn async_upload(
+        self: Rc<Self>,
+        callback: Rc<dyn AsyncShmGfxTextureCallback>,
+        mem: &Rc<ClientMemOffset>,
+        damage: Region,
+    ) -> Result<Option<PendingShmUpload>, GfxError>;
+
+    #[expect(dead_code)]
+    fn sync_upload(self: Rc<Self>, shm: &[Cell<u8>], damage: Region) -> Result<(), GfxError>;
+
+    #[expect(dead_code)]
+    fn compatible_with(
+        &self,
+        format: &'static Format,
+        width: i32,
+        height: i32,
+        stride: i32,
+    ) -> bool;
+
+    #[expect(dead_code)]
+    fn into_texture(self: Rc<Self>) -> Rc<dyn GfxTexture>;
+}
+
 pub trait GfxContext: Debug {
     fn reset_status(&self) -> Option<ResetStatus>;
 
@@ -559,6 +600,16 @@ pub trait GfxContext: Debug {
         damage: Option<&[Rect]>,
     ) -> Result<Rc<dyn ShmGfxTexture>, GfxError>;
 
+    #[expect(dead_code)]
+    fn async_shmem_texture(
+        self: Rc<Self>,
+        format: &'static Format,
+        width: i32,
+        height: i32,
+        stride: i32,
+        cpu_worker: &Rc<CpuWorker>,
+    ) -> Result<Rc<dyn AsyncShmGfxTexture>, GfxError>;
+
     fn allocator(&self) -> Rc<dyn Allocator>;
 
     fn gfx_api(&self) -> GfxApi;
@@ -637,3 +688,16 @@ pub fn cross_intersect_formats(
     }
     res
 }
+
+impl PendingShmUpload {
+    #[expect(dead_code)]
+    pub fn new(cancel: Rc<dyn AsyncShmGfxTextureUploadCancellable>, id: u64) -> Self {
+        Self { cancel, id }
+    }
+}
+
+impl Drop for PendingShmUpload {
+    fn drop(&mut self) {
+        self.cancel.cancel(self.id);
+    }
+}
diff --git a/src/gfx_apis/gl/renderer/context.rs b/src/gfx_apis/gl/renderer/context.rs
index 1948ac16..8ff76a27 100644
--- a/src/gfx_apis/gl/renderer/context.rs
+++ b/src/gfx_apis/gl/renderer/context.rs
@@ -1,10 +1,11 @@
 use {
     crate::{
         allocator::Allocator,
+        cpu_worker::CpuWorker,
         format::{Format, XRGB8888},
         gfx_api::{
-            BufferResvUser, GfxApiOpt, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage,
-            ResetStatus, ShmGfxTexture,
+            AsyncShmGfxTexture, BufferResvUser, GfxApiOpt, GfxContext, GfxError, GfxFormat,
+            GfxFramebuffer, GfxImage, ResetStatus, ShmGfxTexture,
         },
         gfx_apis::gl::{
             egl::{context::EglContext, display::EglDisplay, image::EglImage},
@@ -279,6 +280,17 @@ impl GfxContext for GlRenderContext {
             .map_err(|e| e.into())
     }
 
+    fn async_shmem_texture(
+        self: Rc<Self>,
+        _format: &'static Format,
+        _width: i32,
+        _height: i32,
+        _stride: i32,
+        _cpu_worker: &Rc<CpuWorker>,
+    ) -> Result<Rc<dyn AsyncShmGfxTexture>, GfxError> {
+        todo!()
+    }
+
     fn allocator(&self) -> Rc<dyn Allocator> {
         self.gbm.clone()
     }
diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs
index 09827d4e..5e24fc8d 100644
--- a/src/gfx_apis/vulkan.rs
+++ b/src/gfx_apis/vulkan.rs
@@ -19,9 +19,11 @@ use {
     crate::{
         allocator::{Allocator, AllocatorError},
         async_engine::AsyncEngine,
+        cpu_worker::CpuWorker,
         format::Format,
         gfx_api::{
-            GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage, ResetStatus, ShmGfxTexture,
+            AsyncShmGfxTexture, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage,
+            ResetStatus, ShmGfxTexture,
         },
         gfx_apis::vulkan::{
             image::VulkanImageMemory, instance::VulkanInstance, renderer::VulkanRenderer,
@@ -278,6 +280,17 @@ impl GfxContext for Context {
         Ok(tex as _)
     }
 
+    fn async_shmem_texture(
+        self: Rc<Self>,
+        _format: &'static Format,
+        _width: i32,
+        _height: i32,
+        _stride: i32,
+        _cpu_worker: &Rc<CpuWorker>,
+    ) -> Result<Rc<dyn AsyncShmGfxTexture>, GfxError> {
+        todo!()
+    }
+
     fn allocator(&self) -> Rc<dyn Allocator> {
         self.0.device.gbm.clone()
     }
diff --git a/src/it/test_gfx_api.rs b/src/it/test_gfx_api.rs
index ef37188c..abae7a07 100644
--- a/src/it/test_gfx_api.rs
+++ b/src/it/test_gfx_api.rs
@@ -1,11 +1,12 @@
 use {
     crate::{
         allocator::{Allocator, AllocatorError, BufferObject, BufferUsage},
+        cpu_worker::CpuWorker,
         format::{Format, ARGB8888, XRGB8888},
         gfx_api::{
-            CopyTexture, FillRect, FramebufferRect, GfxApiOpt, GfxContext, GfxError, GfxFormat,
-            GfxFramebuffer, GfxImage, GfxTexture, GfxWriteModifier, ResetStatus, ShmGfxTexture,
-            SyncFile,
+            AsyncShmGfxTexture, CopyTexture, FillRect, FramebufferRect, GfxApiOpt, GfxContext,
+            GfxError, GfxFormat, GfxFramebuffer, GfxImage, GfxTexture, GfxWriteModifier,
+            ResetStatus, ShmGfxTexture, SyncFile,
         },
         rect::Rect,
         theme::Color,
@@ -133,6 +134,17 @@ impl GfxContext for TestGfxCtx {
         })))
     }
 
+    fn async_shmem_texture(
+        self: Rc<Self>,
+        _format: &'static Format,
+        _width: i32,
+        _height: i32,
+        _stride: i32,
+        _cpu_worker: &Rc<CpuWorker>,
+    ) -> Result<Rc<dyn AsyncShmGfxTexture>, GfxError> {
+        todo!()
+    }
+
     fn allocator(&self) -> Rc<dyn Allocator> {
         self.allocator.clone()
     }

From 0a0caf38007c6a30653c7356020974785998fdcd Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 21:35:21 +0200
Subject: [PATCH 12/22] opengl: implement AsyncShmGfxTexture

---
 src/gfx_api.rs                      |  3 +-
 src/gfx_apis/gl.rs                  |  7 +++
 src/gfx_apis/gl/gl/texture.rs       |  6 +++
 src/gfx_apis/gl/renderer/context.rs | 29 +++++++++--
 src/gfx_apis/gl/renderer/texture.rs | 74 ++++++++++++++++++++++++++++-
 5 files changed, 111 insertions(+), 8 deletions(-)

diff --git a/src/gfx_api.rs b/src/gfx_api.rs
index e0c4c28e..a3556206 100644
--- a/src/gfx_api.rs
+++ b/src/gfx_api.rs
@@ -537,8 +537,8 @@ pub trait ShmGfxTexture: GfxTexture {
     fn into_texture(self: Rc<Self>) -> Rc<dyn GfxTexture>;
 }
 
-#[expect(dead_code)]
 pub trait AsyncShmGfxTextureCallback {
+    #[expect(dead_code)]
     fn completed(self: Rc<Self>, res: Result<(), GfxError>);
 }
 
@@ -560,7 +560,6 @@ pub trait AsyncShmGfxTexture: GfxTexture {
         damage: Region,
     ) -> Result<Option<PendingShmUpload>, GfxError>;
 
-    #[expect(dead_code)]
     fn sync_upload(self: Rc<Self>, shm: &[Cell<u8>], damage: Region) -> Result<(), GfxError>;
 
     #[expect(dead_code)]
diff --git a/src/gfx_apis/gl.rs b/src/gfx_apis/gl.rs
index d8b96c05..22d2a511 100644
--- a/src/gfx_apis/gl.rs
+++ b/src/gfx_apis/gl.rs
@@ -67,6 +67,7 @@ macro_rules! dynload {
 
 use {
     crate::{
+        clientmem::ClientMemError,
         gfx_api::{
             AcquireSync, CopyTexture, FillRect, GfxApiOpt, GfxContext, GfxError, GfxTexture,
             ReleaseSync, SyncFile,
@@ -196,6 +197,8 @@ enum RenderError {
     WaitSync,
     #[error("Buffer format {0} is not supported for shm buffers in OpenGL context")]
     UnsupportedShmFormat(&'static str),
+    #[error("Could not access the client memory")]
+    AccessFailed(#[source] ClientMemError),
 }
 
 #[derive(Default)]
@@ -318,6 +321,10 @@ fn fill_boxes3(ctx: &GlRenderContext, boxes: &[[f32; 2]], color: &Color) {
 
 fn render_texture(ctx: &GlRenderContext, tex: &CopyTexture) {
     let texture = tex.tex.as_gl();
+    if !texture.gl.contents_valid.get() {
+        log::error!("Ignoring texture with invalid contents");
+        return;
+    }
     assert!(rc_eq(&ctx.ctx, &texture.ctx.ctx));
     let gles = ctx.ctx.dpy.gles;
     unsafe {
diff --git a/src/gfx_apis/gl/gl/texture.rs b/src/gfx_apis/gl/gl/texture.rs
index c1c5e943..d6ab3e02 100644
--- a/src/gfx_apis/gl/gl/texture.rs
+++ b/src/gfx_apis/gl/gl/texture.rs
@@ -21,8 +21,10 @@ pub struct GlTexture {
     pub tex: GLuint,
     pub width: i32,
     pub height: i32,
+    pub stride: i32,
     pub external_only: bool,
     pub format: &'static Format,
+    pub contents_valid: Cell<bool>,
 }
 
 pub fn image_target(external_only: bool) -> GLenum {
@@ -60,8 +62,10 @@ impl GlTexture {
             tex,
             width: img.dmabuf.width,
             height: img.dmabuf.height,
+            stride: 0,
             external_only: img.external_only,
             format: img.dmabuf.format,
+            contents_valid: Cell::new(true),
         })
     }
 
@@ -108,8 +112,10 @@ impl GlTexture {
             tex,
             width,
             height,
+            stride,
             external_only: false,
             format,
+            contents_valid: Cell::new(true),
         })
     }
 }
diff --git a/src/gfx_apis/gl/renderer/context.rs b/src/gfx_apis/gl/renderer/context.rs
index 8ff76a27..f242800c 100644
--- a/src/gfx_apis/gl/renderer/context.rs
+++ b/src/gfx_apis/gl/renderer/context.rs
@@ -282,13 +282,32 @@ impl GfxContext for GlRenderContext {
 
     fn async_shmem_texture(
         self: Rc<Self>,
-        _format: &'static Format,
-        _width: i32,
-        _height: i32,
-        _stride: i32,
+        format: &'static Format,
+        width: i32,
+        height: i32,
+        stride: i32,
         _cpu_worker: &Rc<CpuWorker>,
     ) -> Result<Rc<dyn AsyncShmGfxTexture>, GfxError> {
-        todo!()
+        let tex = self.ctx.with_current(|| unsafe {
+            let mut tex = 0;
+            (self.ctx.dpy.gles.glGenTextures)(1, &mut tex);
+            Ok(tex)
+        })?;
+        Ok(Rc::new(Texture {
+            gl: GlTexture {
+                ctx: self.ctx.clone(),
+                img: None,
+                tex,
+                width,
+                height,
+                stride,
+                external_only: false,
+                format,
+                contents_valid: Cell::new(false),
+            },
+            ctx: self,
+            format,
+        }))
     }
 
     fn allocator(&self) -> Rc<dyn Allocator> {
diff --git a/src/gfx_apis/gl/renderer/texture.rs b/src/gfx_apis/gl/renderer/texture.rs
index e4a97575..efa45b84 100644
--- a/src/gfx_apis/gl/renderer/texture.rs
+++ b/src/gfx_apis/gl/renderer/texture.rs
@@ -1,12 +1,21 @@
 use {
     crate::{
+        clientmem::ClientMemOffset,
         format::Format,
-        gfx_api::{GfxError, GfxTexture, ShmGfxTexture},
+        gfx_api::{
+            AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxError, GfxTexture, PendingShmUpload,
+            ShmGfxTexture,
+        },
         gfx_apis::gl::{
             gl::texture::GlTexture,
             renderer::{context::GlRenderContext, framebuffer::Framebuffer},
+            sys::{
+                GLint, GL_CLAMP_TO_EDGE, GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_TEXTURE_WRAP_T,
+                GL_UNPACK_ROW_LENGTH_EXT,
+            },
             RenderError,
         },
+        rect::Region,
         video::dmabuf::DmaBuf,
     },
     std::{
@@ -88,3 +97,66 @@ impl ShmGfxTexture for Texture {
         self
     }
 }
+
+impl AsyncShmGfxTexture for Texture {
+    fn async_upload(
+        self: Rc<Self>,
+        _callback: Rc<dyn AsyncShmGfxTextureCallback>,
+        mem: &Rc<ClientMemOffset>,
+        damage: Region,
+    ) -> Result<Option<PendingShmUpload>, GfxError> {
+        mem.access(|data| self.clone().sync_upload(data, damage))
+            .map_err(RenderError::AccessFailed)??;
+        Ok(None)
+    }
+
+    fn sync_upload(self: Rc<Self>, data: &[Cell<u8>], _damage: Region) -> Result<(), GfxError> {
+        let shm_info = self.format.shm_info.as_ref().unwrap();
+        if (self.gl.stride * self.gl.height) as usize > data.len() {
+            return Err(RenderError::SmallImageBuffer.into());
+        }
+        let gles = self.ctx.ctx.dpy.gles;
+        self.ctx.ctx.with_current(|| unsafe {
+            (gles.glBindTexture)(GL_TEXTURE_2D, self.gl.tex);
+            (gles.glTexParameteri)(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+            (gles.glTexParameteri)(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+            (gles.glPixelStorei)(
+                GL_UNPACK_ROW_LENGTH_EXT,
+                self.gl.stride / shm_info.bpp as GLint,
+            );
+            (gles.glTexImage2D)(
+                GL_TEXTURE_2D,
+                0,
+                shm_info.gl_format,
+                self.gl.width,
+                self.gl.height,
+                0,
+                shm_info.gl_format as _,
+                shm_info.gl_type as _,
+                data.as_ptr() as _,
+            );
+            (gles.glPixelStorei)(GL_UNPACK_ROW_LENGTH_EXT, 0);
+            (gles.glBindTexture)(GL_TEXTURE_2D, 0);
+            Ok(())
+        })?;
+        self.gl.contents_valid.set(true);
+        Ok(())
+    }
+
+    fn compatible_with(
+        &self,
+        format: &'static Format,
+        width: i32,
+        height: i32,
+        stride: i32,
+    ) -> bool {
+        format == self.gl.format
+            && width == self.gl.width
+            && height == self.gl.height
+            && stride == self.gl.stride
+    }
+
+    fn into_texture(self: Rc<Self>) -> Rc<dyn GfxTexture> {
+        self
+    }
+}

From cf74756c348b2bcee47d48559497750ec435551a Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 20:48:09 +0200
Subject: [PATCH 13/22] it: implement AsyncShmGfxTexture

---
 src/it/test_gfx_api.rs | 71 ++++++++++++++++++++++++++++++++++++------
 1 file changed, 62 insertions(+), 9 deletions(-)

diff --git a/src/it/test_gfx_api.rs b/src/it/test_gfx_api.rs
index abae7a07..8ef52878 100644
--- a/src/it/test_gfx_api.rs
+++ b/src/it/test_gfx_api.rs
@@ -1,14 +1,15 @@
 use {
     crate::{
         allocator::{Allocator, AllocatorError, BufferObject, BufferUsage},
+        clientmem::{ClientMemError, ClientMemOffset},
         cpu_worker::CpuWorker,
         format::{Format, ARGB8888, XRGB8888},
         gfx_api::{
-            AsyncShmGfxTexture, CopyTexture, FillRect, FramebufferRect, GfxApiOpt, GfxContext,
-            GfxError, GfxFormat, GfxFramebuffer, GfxImage, GfxTexture, GfxWriteModifier,
-            ResetStatus, ShmGfxTexture, SyncFile,
+            AsyncShmGfxTexture, AsyncShmGfxTextureCallback, CopyTexture, FillRect, FramebufferRect,
+            GfxApiOpt, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage, GfxTexture,
+            GfxWriteModifier, PendingShmUpload, ResetStatus, ShmGfxTexture, SyncFile,
         },
-        rect::Rect,
+        rect::{Rect, Region},
         theme::Color,
         video::{dmabuf::DmaBuf, drm::sync_obj::SyncObjCtx, LINEAR_MODIFIER},
     },
@@ -33,6 +34,8 @@ enum TestGfxError {
     MapDmaBuf(#[source] AllocatorError),
     #[error("Could not import dmabuf")]
     ImportDmaBuf(#[source] AllocatorError),
+    #[error("Could not access the client memory")]
+    AccessFailed(#[source] ClientMemError),
 }
 
 impl From<TestGfxError> for GfxError {
@@ -136,13 +139,21 @@ impl GfxContext for TestGfxCtx {
 
     fn async_shmem_texture(
         self: Rc<Self>,
-        _format: &'static Format,
-        _width: i32,
-        _height: i32,
-        _stride: i32,
+        format: &'static Format,
+        width: i32,
+        height: i32,
+        stride: i32,
         _cpu_worker: &Rc<CpuWorker>,
     ) -> Result<Rc<dyn AsyncShmGfxTexture>, GfxError> {
-        todo!()
+        assert!(stride >= width * 4);
+        let size = (stride * height) as usize;
+        Ok(Rc::new(TestGfxImage::Shm(TestShmGfxImage {
+            data: RefCell::new(vec![0; size]),
+            width,
+            height,
+            stride,
+            format,
+        })))
     }
 
     fn allocator(&self) -> Rc<dyn Allocator> {
@@ -320,6 +331,48 @@ impl ShmGfxTexture for TestGfxImage {
     }
 }
 
+impl AsyncShmGfxTexture for TestGfxImage {
+    fn async_upload(
+        self: Rc<Self>,
+        _callback: Rc<dyn AsyncShmGfxTextureCallback>,
+        mem: &Rc<ClientMemOffset>,
+        damage: Region,
+    ) -> Result<Option<PendingShmUpload>, GfxError> {
+        mem.access(|d| self.clone().sync_upload(d, damage))
+            .map_err(TestGfxError::AccessFailed)??;
+        Ok(None)
+    }
+
+    fn sync_upload(self: Rc<Self>, mem: &[Cell<u8>], _damage: Region) -> Result<(), GfxError> {
+        let TestGfxImage::Shm(shm) = &*self else {
+            unreachable!();
+        };
+        let data = &mut *shm.data.borrow_mut();
+        assert!(mem.len() >= data.len());
+        unsafe {
+            ptr::copy_nonoverlapping(mem.as_ptr() as _, data.as_mut_ptr(), data.len());
+        }
+        Ok(())
+    }
+
+    fn compatible_with(
+        &self,
+        format: &'static Format,
+        width: i32,
+        height: i32,
+        stride: i32,
+    ) -> bool {
+        let TestGfxImage::Shm(shm) = &self else {
+            unreachable!();
+        };
+        shm.format == format && shm.width == width && shm.height == height && shm.stride == stride
+    }
+
+    fn into_texture(self: Rc<Self>) -> Rc<dyn GfxTexture> {
+        self
+    }
+}
+
 impl GfxImage for TestGfxImage {
     fn to_framebuffer(self: Rc<Self>) -> Result<Rc<dyn GfxFramebuffer>, GfxError> {
         Ok(Rc::new(TestGfxFb {

From 1f169a0d7b79dcef238825abb06a705c8d951047 Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Fri, 6 Sep 2024 18:35:48 +0200
Subject: [PATCH 14/22] vulkan: prevent creating new async processes after the
 renderer has been dropped

---
 src/gfx_apis/vulkan.rs           |  2 ++
 src/gfx_apis/vulkan/renderer.rs  | 11 +++++++++++
 src/gfx_apis/vulkan/shm_image.rs |  1 +
 3 files changed, 14 insertions(+)

diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs
index 5e24fc8d..59923d50 100644
--- a/src/gfx_apis/vulkan.rs
+++ b/src/gfx_apis/vulkan.rs
@@ -197,6 +197,8 @@ pub enum VulkanError {
     WaitIdle(#[source] vk::Result),
     #[error("Could not dup a DRM device")]
     DupDrm(#[source] DrmError),
+    #[error("Graphics context has already been dropped")]
+    Defunct,
 }
 
 impl From<VulkanError> for GfxError {
diff --git a/src/gfx_apis/vulkan/renderer.rs b/src/gfx_apis/vulkan/renderer.rs
index d884d76f..6613b9ae 100644
--- a/src/gfx_apis/vulkan/renderer.rs
+++ b/src/gfx_apis/vulkan/renderer.rs
@@ -78,6 +78,7 @@ pub struct VulkanRenderer {
     pub(super) tex_frag_mult_opaque_shader: Rc<VulkanShader>,
     pub(super) tex_frag_mult_alpha_shader: Rc<VulkanShader>,
     pub(super) tex_descriptor_set_layout: Rc<VulkanDescriptorSetLayout>,
+    pub(super) defunct: Cell<bool>,
 }
 
 pub(super) struct UsedTexture {
@@ -195,6 +196,7 @@ impl VulkanDevice {
             tex_frag_mult_opaque_shader,
             tex_frag_mult_alpha_shader,
             tex_descriptor_set_layout,
+            defunct: Cell::new(false),
         });
         render.get_or_create_pipelines(XRGB8888.vk_format)?;
         Ok(render)
@@ -996,6 +998,7 @@ impl VulkanRenderer {
         opts: &[GfxApiOpt],
         clear: Option<&Color>,
     ) -> Result<(), VulkanError> {
+        self.check_defunct()?;
         let buf = self.allocate_command_buffer()?;
         self.collect_memory(opts);
         self.begin_command_buffer(buf.buffer)?;
@@ -1025,6 +1028,7 @@ impl VulkanRenderer {
     }
 
     pub fn on_drop(&self) {
+        self.defunct.set(true);
         let mut pending_frames = self.pending_frames.lock();
         let mut pending_uploads = self.pending_uploads.lock();
         if pending_frames.is_not_empty() || pending_uploads.is_not_empty() {
@@ -1037,6 +1041,13 @@ impl VulkanRenderer {
         pending_frames.clear();
         pending_uploads.clear();
     }
+
+    pub(super) fn check_defunct(&self) -> Result<(), VulkanError> {
+        match self.defunct.get() {
+            true => Err(VulkanError::Defunct),
+            false => Ok(()),
+        }
+    }
 }
 
 impl Debug for VulkanRenderer {
diff --git a/src/gfx_apis/vulkan/shm_image.rs b/src/gfx_apis/vulkan/shm_image.rs
index 81f6e568..a21698a3 100644
--- a/src/gfx_apis/vulkan/shm_image.rs
+++ b/src/gfx_apis/vulkan/shm_image.rs
@@ -41,6 +41,7 @@ impl VulkanShmImage {
         buffer: &[Cell<u8>],
         damage: Option<&[Rect]>,
     ) -> Result<(), VulkanError> {
+        img.renderer.check_defunct()?;
         if let Some(damage) = damage {
             if damage.is_empty() {
                 return Ok(());

From fe8238421fd4af7e650e37fac8c978d88c1a9ca1 Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Fri, 6 Sep 2024 19:47:34 +0200
Subject: [PATCH 15/22] vulkan: only use coherency functions for non-coherent
 memory

---
 src/gfx_apis/vulkan/allocator.rs |  7 ++++++-
 src/gfx_apis/vulkan/staging.rs   | 24 ++++++++++++++----------
 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/src/gfx_apis/vulkan/allocator.rs b/src/gfx_apis/vulkan/allocator.rs
index 83f661ba..4f2568e4 100644
--- a/src/gfx_apis/vulkan/allocator.rs
+++ b/src/gfx_apis/vulkan/allocator.rs
@@ -4,7 +4,7 @@ use {
         utils::{numcell::NumCell, ptr_ext::MutPtrExt},
     },
     ash::vk::{DeviceMemory, DeviceSize, MemoryRequirements},
-    gpu_alloc::{Config, GpuAllocator, MemoryBlock, Request, UsageFlags},
+    gpu_alloc::{Config, GpuAllocator, MemoryBlock, MemoryPropertyFlags, Request, UsageFlags},
     gpu_alloc_ash::AshMemoryDevice,
     std::{
         cell::{Cell, UnsafeCell},
@@ -25,6 +25,7 @@ pub struct VulkanAllocation {
     pub(super) offset: DeviceSize,
     pub(super) mem: Option<*mut u8>,
     pub(super) size: DeviceSize,
+    pub(super) coherency_mask: Option<u64>,
     block: Cell<Option<MemoryBlock<DeviceMemory>>>,
 }
 
@@ -111,6 +112,10 @@ impl VulkanAllocator {
             offset: block.offset(),
             mem: ptr,
             size: block.size(),
+            coherency_mask: match block.props().contains(MemoryPropertyFlags::HOST_COHERENT) {
+                true => None,
+                false => Some(self.non_coherent_atom_mask),
+            },
             block: Cell::new(Some(block)),
         })
     }
diff --git a/src/gfx_apis/vulkan/staging.rs b/src/gfx_apis/vulkan/staging.rs
index b8ccc2ef..d09d705a 100644
--- a/src/gfx_apis/vulkan/staging.rs
+++ b/src/gfx_apis/vulkan/staging.rs
@@ -72,27 +72,31 @@ impl VulkanStagingBuffer {
         F: FnOnce(*mut u8, usize) -> T,
     {
         let t = f(self.allocation.mem.unwrap(), self.size as usize);
-        let range = self.range();
-        let res = unsafe { self.device.device.flush_mapped_memory_ranges(&[range]) };
-        res.map_err(VulkanError::FlushMemory).map(|_| t)
+        if let Some(mask) = self.allocation.coherency_mask {
+            let range = self.incoherent_range(mask);
+            let res = unsafe { self.device.device.flush_mapped_memory_ranges(&[range]) };
+            res.map_err(VulkanError::FlushMemory)?;
+        }
+        Ok(t)
     }
 
     pub fn download<T, F>(&self, f: F) -> Result<T, VulkanError>
     where
         F: FnOnce(*const u8, usize) -> T,
     {
-        let range = self.range();
-        let res = unsafe { self.device.device.invalidate_mapped_memory_ranges(&[range]) };
-        res.map_err(VulkanError::FlushMemory)?;
+        if let Some(mask) = self.allocation.coherency_mask {
+            let range = self.incoherent_range(mask);
+            let res = unsafe { self.device.device.invalidate_mapped_memory_ranges(&[range]) };
+            res.map_err(VulkanError::FlushMemory)?;
+        }
         Ok(f(self.allocation.mem.unwrap(), self.size as usize))
     }
 
-    fn range(&self) -> MappedMemoryRange {
-        let atom_mask = self.allocation.allocator.non_coherent_atom_mask;
+    fn incoherent_range(&self, mask: u64) -> MappedMemoryRange {
         MappedMemoryRange::default()
             .memory(self.allocation.memory)
-            .offset(self.allocation.offset & !atom_mask)
-            .size((self.allocation.size + atom_mask) & !atom_mask)
+            .offset(self.allocation.offset & !mask)
+            .size((self.allocation.size + mask) & !mask)
     }
 }
 

From 37fb45df00cb25ad56649effae09c891bfba55d7 Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 17:38:12 +0200
Subject: [PATCH 16/22] vulkan: add an async allocator

---
 src/allocator.rs                 |   2 +-
 src/cpu_worker.rs                |   1 -
 src/gfx_api.rs                   |   2 +-
 src/gfx_apis/vulkan/allocator.rs | 376 ++++++++++++++++++++++++++-----
 src/gfx_apis/vulkan/device.rs    |   5 +-
 src/gfx_apis/vulkan/renderer.rs  |   9 +-
 6 files changed, 329 insertions(+), 66 deletions(-)

diff --git a/src/allocator.rs b/src/allocator.rs
index 6354bdb7..f836a471 100644
--- a/src/allocator.rs
+++ b/src/allocator.rs
@@ -13,7 +13,7 @@ use {
 
 #[derive(Debug, Error)]
 #[error(transparent)]
-pub struct AllocatorError(#[from] pub Box<dyn Error>);
+pub struct AllocatorError(#[from] pub Box<dyn Error + Send>);
 
 bitflags! {
     BufferUsage: u32;
diff --git a/src/cpu_worker.rs b/src/cpu_worker.rs
index d3d6adca..3875401f 100644
--- a/src/cpu_worker.rs
+++ b/src/cpu_worker.rs
@@ -290,7 +290,6 @@ impl CpuWorker {
         })
     }
 
-    #[expect(dead_code)]
     pub fn submit(&self, job: Box<dyn CpuJob>) -> PendingJob {
         let mut job = NonNull::from(Box::leak(job));
         let id = self.data.next.next();
diff --git a/src/gfx_api.rs b/src/gfx_api.rs
index a3556206..5ca68519 100644
--- a/src/gfx_api.rs
+++ b/src/gfx_api.rs
@@ -642,7 +642,7 @@ pub struct GfxFormat {
 
 #[derive(Error)]
 #[error(transparent)]
-pub struct GfxError(pub Box<dyn Error>);
+pub struct GfxError(pub Box<dyn Error + Send>);
 
 impl Debug for GfxError {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
diff --git a/src/gfx_apis/vulkan/allocator.rs b/src/gfx_apis/vulkan/allocator.rs
index 4f2568e4..77ffced0 100644
--- a/src/gfx_apis/vulkan/allocator.rs
+++ b/src/gfx_apis/vulkan/allocator.rs
@@ -1,26 +1,55 @@
 use {
     crate::{
-        gfx_apis::vulkan::{device::VulkanDevice, instance::API_VERSION, VulkanError},
+        cpu_worker::{AsyncCpuWork, CpuJob, CpuWork, CpuWorker},
+        gfx_apis::vulkan::{
+            device::VulkanDevice, instance::API_VERSION, renderer::VulkanRenderer, VulkanError,
+        },
         utils::{numcell::NumCell, ptr_ext::MutPtrExt},
     },
-    ash::vk::{DeviceMemory, DeviceSize, MemoryRequirements},
+    ash::{
+        vk::{DeviceMemory, DeviceSize, MemoryRequirements},
+        Device,
+    },
     gpu_alloc::{Config, GpuAllocator, MemoryBlock, MemoryPropertyFlags, Request, UsageFlags},
     gpu_alloc_ash::AshMemoryDevice,
+    parking_lot::Mutex,
     std::{
         cell::{Cell, UnsafeCell},
         rc::Rc,
+        sync::Arc,
     },
 };
 
-pub struct VulkanAllocator {
-    pub(super) device: Rc<VulkanDevice>,
-    pub(super) non_coherent_atom_mask: u64,
+pub struct SyncAllocatorStorage {
+    allocator: Arc<Mutex<GpuAllocator<DeviceMemory>>>,
+    device: Rc<VulkanDevice>,
+}
+
+pub struct UnsyncAllocatorStorage {
     allocator: UnsafeCell<GpuAllocator<DeviceMemory>>,
+    device: Rc<VulkanDevice>,
+}
+
+pub struct VulkanAllocatorType<T> {
+    storage: T,
+    non_coherent_atom_mask: u64,
     total: NumCell<u64>,
 }
 
+pub type VulkanAllocator = VulkanAllocatorType<UnsyncAllocatorStorage>;
+pub type VulkanThreadedAllocator = VulkanAllocatorType<SyncAllocatorStorage>;
+
+enum AllocatorType {
+    Local(Rc<VulkanAllocator>),
+    Threaded {
+        allocator: Rc<VulkanThreadedAllocator>,
+        renderer: Rc<VulkanRenderer>,
+        cpu: Rc<CpuWorker>,
+    },
+}
+
 pub struct VulkanAllocation {
-    pub(super) allocator: Rc<VulkanAllocator>,
+    allocator: AllocatorType,
     pub(super) memory: DeviceMemory,
     pub(super) offset: DeviceSize,
     pub(super) mem: Option<*mut u8>,
@@ -29,26 +58,65 @@ pub struct VulkanAllocation {
     block: Cell<Option<MemoryBlock<DeviceMemory>>>,
 }
 
+impl VulkanAllocation {
+    unsafe fn free_locally<T>(
+        &self,
+        allocator: &VulkanAllocatorType<T>,
+        device: &VulkanDevice,
+        gpu: &mut GpuAllocator<DeviceMemory>,
+    ) {
+        allocator.total.fetch_sub(self.size);
+        let block = self.block.take().unwrap();
+        do_free(gpu, &device.device, block, self.mem);
+    }
+}
+
 impl Drop for VulkanAllocation {
     fn drop(&mut self) {
         unsafe {
-            self.allocator.total.fetch_sub(self.size);
-            let mut block = self.block.take().unwrap();
-            if let Some(_ptr) = self.mem {
-                // log::info!("free = {:?} - {:?} ({})", ptr, ptr.add(block.size() as usize), block.size());
-                block.unmap(AshMemoryDevice::wrap(&self.allocator.device.device));
+            match &self.allocator {
+                AllocatorType::Local(a) => self.free_locally(a, &a.storage.device, a.allocator()),
+                AllocatorType::Threaded {
+                    allocator,
+                    renderer,
+                    cpu,
+                } => {
+                    if renderer.defunct.get() {
+                        self.free_locally(
+                            allocator,
+                            &allocator.storage.device,
+                            &mut allocator.storage.allocator.lock(),
+                        );
+                    } else {
+                        let id = renderer.allocate_point();
+                        let job = FreeJob {
+                            id,
+                            renderer: renderer.clone(),
+                            allocator: allocator.clone(),
+                            size: self.size,
+                            work: FreeWork {
+                                device: allocator.storage.device.device.clone(),
+                                allocator: allocator.storage.allocator.clone(),
+                                allocation: Some(UnsafeAllocation {
+                                    block: self.block.take().unwrap(),
+                                    ptr: self.mem,
+                                }),
+                            },
+                        };
+                        let pending = cpu.submit(Box::new(job));
+                        renderer.pending_cpu_jobs.set(id, pending);
+                    }
+                }
             }
-            self.allocator
-                .allocator
-                .get()
-                .deref_mut()
-                .dealloc(AshMemoryDevice::wrap(&self.allocator.device.device), block);
         }
     }
 }
 
 impl VulkanDevice {
-    pub fn create_allocator(self: &Rc<Self>) -> Result<Rc<VulkanAllocator>, VulkanError> {
+    fn create_allocator_<T>(
+        self: &Rc<Self>,
+        map: impl FnOnce(GpuAllocator<DeviceMemory>) -> T,
+    ) -> Result<Rc<VulkanAllocatorType<T>>, VulkanError> {
         let config = Config::i_am_prototyping();
         let props = unsafe {
             gpu_alloc_ash::device_properties(
@@ -61,18 +129,56 @@ impl VulkanDevice {
         props.buffer_device_address = false;
         let non_coherent_atom_size = props.non_coherent_atom_size;
         let allocator = GpuAllocator::new(config, props);
-        Ok(Rc::new(VulkanAllocator {
-            device: self.clone(),
+        Ok(Rc::new(VulkanAllocatorType {
             non_coherent_atom_mask: non_coherent_atom_size - 1,
-            allocator: UnsafeCell::new(allocator),
+            storage: map(allocator),
             total: Default::default(),
         }))
     }
+
+    pub fn create_allocator(self: &Rc<Self>) -> Result<Rc<VulkanAllocator>, VulkanError> {
+        self.create_allocator_(|a| UnsyncAllocatorStorage {
+            allocator: UnsafeCell::new(a),
+            device: self.clone(),
+        })
+    }
+
+    pub fn create_threaded_allocator(
+        self: &Rc<Self>,
+    ) -> Result<Rc<VulkanThreadedAllocator>, VulkanError> {
+        self.create_allocator_(|a| SyncAllocatorStorage {
+            allocator: Arc::new(Mutex::new(a)),
+            device: self.clone(),
+        })
+    }
+}
+
+impl<T> VulkanAllocatorType<T> {
+    fn commit_allocation(
+        self: &Rc<Self>,
+        ua: UnsafeAllocation,
+        allocator: AllocatorType,
+    ) -> VulkanAllocation {
+        let UnsafeAllocation { block, ptr } = ua;
+        self.total.fetch_add(block.size());
+        VulkanAllocation {
+            allocator,
+            memory: *block.memory(),
+            offset: block.offset(),
+            mem: ptr,
+            size: block.size(),
+            coherency_mask: match block.props().contains(MemoryPropertyFlags::HOST_COHERENT) {
+                true => None,
+                false => Some(self.non_coherent_atom_mask),
+            },
+            block: Cell::new(Some(block)),
+        }
+    }
 }
 
 impl VulkanAllocator {
     fn allocator(&self) -> &mut GpuAllocator<DeviceMemory> {
-        unsafe { self.allocator.get().deref_mut() }
+        unsafe { self.storage.allocator.get().deref_mut() }
     }
 
     pub fn alloc(
@@ -81,53 +187,203 @@ impl VulkanAllocator {
         usage: UsageFlags,
         map: bool,
     ) -> Result<VulkanAllocation, VulkanError> {
-        let request = Request {
-            size: req.size,
-            align_mask: req.alignment - 1,
+        let ua = do_alloc(
+            self.allocator(),
+            &self.storage.device.device,
+            req,
             usage,
-            memory_types: req.memory_type_bits,
-        };
-        let block = unsafe {
-            self.allocator()
-                .alloc(AshMemoryDevice::wrap(&self.device.device), request)
-        };
-        let mut block = block.map_err(VulkanError::AllocateMemory2)?;
-        let ptr = match map {
-            true => {
-                let ptr = unsafe {
-                    block.map(
-                        AshMemoryDevice::wrap(&self.device.device),
-                        0,
-                        block.size() as usize,
-                    )
-                };
-                Some(ptr.map_err(VulkanError::MapMemory)?.as_ptr())
-            }
-            false => None,
-        };
-        self.total.fetch_add(block.size());
-        Ok(VulkanAllocation {
+            map,
+        )?;
+        Ok(self.commit_allocation(ua, AllocatorType::Local(self.clone())))
+    }
+}
+
+impl VulkanThreadedAllocator {
+    #[expect(dead_code)]
+    pub fn async_alloc(
+        self: &Rc<Self>,
+        renderer: &Rc<VulkanRenderer>,
+        cpu: &Rc<CpuWorker>,
+        req: MemoryRequirements,
+        usage: UsageFlags,
+        map: bool,
+        cb: impl FnOnce(Result<VulkanAllocation, VulkanError>) + 'static,
+    ) -> Result<(), VulkanError> {
+        renderer.check_defunct()?;
+        let id = renderer.allocate_point();
+        let job = AllocJob {
+            id,
+            renderer: renderer.clone(),
+            cpu: cpu.clone(),
             allocator: self.clone(),
-            memory: *block.memory(),
-            offset: block.offset(),
-            mem: ptr,
-            size: block.size(),
-            coherency_mask: match block.props().contains(MemoryPropertyFlags::HOST_COHERENT) {
-                true => None,
-                false => Some(self.non_coherent_atom_mask),
+            cb: Some(cb),
+            work: AllocWork {
+                req,
+                usage,
+                map,
+                device: self.storage.device.device.clone(),
+                allocator: self.storage.allocator.clone(),
+                res: None,
             },
-            block: Cell::new(Some(block)),
-        })
+        };
+        let pending = cpu.submit(Box::new(job));
+        renderer.pending_cpu_jobs.set(id, pending);
+        Ok(())
+    }
+}
+
+struct AllocJob<T> {
+    id: u64,
+    renderer: Rc<VulkanRenderer>,
+    cpu: Rc<CpuWorker>,
+    allocator: Rc<VulkanThreadedAllocator>,
+    cb: Option<T>,
+    work: AllocWork,
+}
+
+struct AllocWork {
+    req: MemoryRequirements,
+    usage: UsageFlags,
+    map: bool,
+    device: Arc<Device>,
+    allocator: Arc<Mutex<GpuAllocator<DeviceMemory>>>,
+    res: Option<Result<UnsafeAllocation, VulkanError>>,
+}
+
+impl CpuWork for AllocWork {
+    fn run(&mut self) -> Option<Box<dyn AsyncCpuWork>> {
+        let r = do_alloc(
+            &mut self.allocator.lock(),
+            &self.device,
+            &self.req,
+            self.usage,
+            self.map,
+        );
+        self.res = Some(r);
+        None
+    }
+}
+
+impl<T> CpuJob for AllocJob<T>
+where
+    T: FnOnce(Result<VulkanAllocation, VulkanError>),
+{
+    fn work(&mut self) -> &mut dyn CpuWork {
+        &mut self.work
+    }
+
+    fn completed(mut self: Box<Self>) {
+        self.renderer.pending_cpu_jobs.remove(&self.id);
+        let res = self.work.res.take().unwrap().map(|ua| {
+            self.allocator.commit_allocation(
+                ua,
+                AllocatorType::Threaded {
+                    allocator: self.allocator.clone(),
+                    renderer: self.renderer.clone(),
+                    cpu: self.cpu.clone(),
+                },
+            )
+        });
+        self.cb.take().unwrap()(res);
+    }
+}
+
+struct FreeJob {
+    id: u64,
+    renderer: Rc<VulkanRenderer>,
+    allocator: Rc<VulkanThreadedAllocator>,
+    size: u64,
+    work: FreeWork,
+}
+
+struct FreeWork {
+    device: Arc<Device>,
+    allocator: Arc<Mutex<GpuAllocator<DeviceMemory>>>,
+    allocation: Option<UnsafeAllocation>,
+}
+
+impl CpuWork for FreeWork {
+    fn run(&mut self) -> Option<Box<dyn AsyncCpuWork>> {
+        let ua = self.allocation.take().unwrap();
+        unsafe {
+            do_free(&mut self.allocator.lock(), &self.device, ua.block, ua.ptr);
+        }
+        None
+    }
+}
+
+impl CpuJob for FreeJob {
+    fn work(&mut self) -> &mut dyn CpuWork {
+        &mut self.work
+    }
+
+    fn completed(self: Box<Self>) {
+        self.renderer.pending_cpu_jobs.remove(&self.id);
+        self.allocator.total.fetch_sub(self.size);
+    }
+}
+
+pub struct UnsafeAllocation {
+    block: MemoryBlock<DeviceMemory>,
+    ptr: Option<*mut u8>,
+}
+
+unsafe impl Send for UnsafeAllocation {}
+
+fn do_alloc(
+    allocator: &mut GpuAllocator<DeviceMemory>,
+    device: &Device,
+    req: &MemoryRequirements,
+    usage: UsageFlags,
+    map: bool,
+) -> Result<UnsafeAllocation, VulkanError> {
+    let request = Request {
+        size: req.size,
+        align_mask: req.alignment - 1,
+        usage,
+        memory_types: req.memory_type_bits,
+    };
+    let device = AshMemoryDevice::wrap(device);
+    let block = unsafe { allocator.alloc(device, request) };
+    let mut block = block.map_err(VulkanError::AllocateMemory2)?;
+    let ptr = match map {
+        true => {
+            let ptr = unsafe { block.map(device, 0, block.size() as usize) };
+            Some(ptr.map_err(VulkanError::MapMemory)?.as_ptr())
+        }
+        false => None,
+    };
+    Ok(UnsafeAllocation { block, ptr })
+}
+
+unsafe fn do_free(
+    gpu: &mut GpuAllocator<DeviceMemory>,
+    device: &Device,
+    mut block: MemoryBlock<DeviceMemory>,
+    ptr: Option<*mut u8>,
+) {
+    let device = AshMemoryDevice::wrap(device);
+    if let Some(_ptr) = ptr {
+        // log::info!("free = {:?} - {:?} ({})", ptr, ptr.add(block.size() as usize), block.size());
+        block.unmap(device);
+    }
+    gpu.dealloc(device, block);
+}
+
+impl Drop for UnsyncAllocatorStorage {
+    fn drop(&mut self) {
+        let device = AshMemoryDevice::wrap(&self.device.device);
+        unsafe {
+            self.allocator.get_mut().cleanup(device);
+        }
     }
 }
 
-impl Drop for VulkanAllocator {
+impl Drop for SyncAllocatorStorage {
     fn drop(&mut self) {
+        let device = AshMemoryDevice::wrap(&self.device.device);
         unsafe {
-            self.allocator
-                .get()
-                .deref_mut()
-                .cleanup(AshMemoryDevice::wrap(&self.device.device));
+            self.allocator.lock().cleanup(device);
         }
     }
 }
diff --git a/src/gfx_apis/vulkan/device.rs b/src/gfx_apis/vulkan/device.rs
index 00f3b67b..8792a8ac 100644
--- a/src/gfx_apis/vulkan/device.rs
+++ b/src/gfx_apis/vulkan/device.rs
@@ -42,6 +42,7 @@ use {
     std::{
         ffi::{CStr, CString},
         rc::Rc,
+        sync::Arc,
     },
     uapi::Ustr,
 };
@@ -52,7 +53,7 @@ pub struct VulkanDevice {
     pub(super) gbm: Rc<GbmDevice>,
     pub(super) sync_ctx: Rc<SyncObjCtx>,
     pub(super) instance: Rc<VulkanInstance>,
-    pub(super) device: Device,
+    pub(super) device: Arc<Device>,
     pub(super) external_memory_fd: external_memory_fd::Device,
     pub(super) external_semaphore_fd: external_semaphore_fd::Device,
     pub(super) external_fence_fd: external_fence_fd::Device,
@@ -292,7 +293,7 @@ impl VulkanInstance {
             sync_ctx: Rc::new(SyncObjCtx::new(gbm.drm.fd())),
             gbm: Rc::new(gbm),
             instance: self.clone(),
-            device,
+            device: Arc::new(device),
             external_memory_fd,
             external_semaphore_fd,
             external_fence_fd,
diff --git a/src/gfx_apis/vulkan/renderer.rs b/src/gfx_apis/vulkan/renderer.rs
index 6613b9ae..05982da7 100644
--- a/src/gfx_apis/vulkan/renderer.rs
+++ b/src/gfx_apis/vulkan/renderer.rs
@@ -1,13 +1,14 @@
 use {
     crate::{
         async_engine::{AsyncEngine, SpawnedFuture},
+        cpu_worker::PendingJob,
         format::{Format, XRGB8888},
         gfx_api::{
             AcquireSync, BufferResv, BufferResvUser, GfxApiOpt, GfxFormat, GfxFramebuffer,
             GfxTexture, GfxWriteModifier, ReleaseSync, SyncFile,
         },
         gfx_apis::vulkan::{
-            allocator::VulkanAllocator,
+            allocator::{VulkanAllocator, VulkanThreadedAllocator},
             command::{VulkanCommandBuffer, VulkanCommandPool},
             descriptor::VulkanDescriptorSetLayout,
             device::VulkanDevice,
@@ -79,6 +80,9 @@ pub struct VulkanRenderer {
     pub(super) tex_frag_mult_alpha_shader: Rc<VulkanShader>,
     pub(super) tex_descriptor_set_layout: Rc<VulkanDescriptorSetLayout>,
     pub(super) defunct: Cell<bool>,
+    pub(super) pending_cpu_jobs: CopyHashMap<u64, PendingJob>,
+    #[expect(dead_code)]
+    pub(super) shm_allocator: Rc<VulkanThreadedAllocator>,
 }
 
 pub(super) struct UsedTexture {
@@ -173,6 +177,7 @@ impl VulkanDevice {
             })
             .collect();
         let allocator = self.create_allocator()?;
+        let shm_allocator = self.create_threaded_allocator()?;
         let render = Rc::new(VulkanRenderer {
             formats: Rc::new(formats),
             device: self.clone(),
@@ -197,6 +202,8 @@ impl VulkanDevice {
             tex_frag_mult_alpha_shader,
             tex_descriptor_set_layout,
             defunct: Cell::new(false),
+            pending_cpu_jobs: Default::default(),
+            shm_allocator,
         });
         render.get_or_create_pipelines(XRGB8888.vk_format)?;
         Ok(render)

From b57d86c1bcc638a5148c1ec5bf886a7cbf838754 Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 17:42:29 +0200
Subject: [PATCH 17/22] vulkan: add async staging buffer allocation

---
 src/gfx_apis/vulkan/allocator.rs |  1 -
 src/gfx_apis/vulkan/renderer.rs  |  1 -
 src/gfx_apis/vulkan/staging.rs   | 91 +++++++++++++++++++++++++-------
 src/utils/on_drop.rs             |  2 -
 4 files changed, 72 insertions(+), 23 deletions(-)

diff --git a/src/gfx_apis/vulkan/allocator.rs b/src/gfx_apis/vulkan/allocator.rs
index 77ffced0..6cff0834 100644
--- a/src/gfx_apis/vulkan/allocator.rs
+++ b/src/gfx_apis/vulkan/allocator.rs
@@ -199,7 +199,6 @@ impl VulkanAllocator {
 }
 
 impl VulkanThreadedAllocator {
-    #[expect(dead_code)]
     pub fn async_alloc(
         self: &Rc<Self>,
         renderer: &Rc<VulkanRenderer>,
diff --git a/src/gfx_apis/vulkan/renderer.rs b/src/gfx_apis/vulkan/renderer.rs
index 05982da7..71877a7f 100644
--- a/src/gfx_apis/vulkan/renderer.rs
+++ b/src/gfx_apis/vulkan/renderer.rs
@@ -81,7 +81,6 @@ pub struct VulkanRenderer {
     pub(super) tex_descriptor_set_layout: Rc<VulkanDescriptorSetLayout>,
     pub(super) defunct: Cell<bool>,
     pub(super) pending_cpu_jobs: CopyHashMap<u64, PendingJob>,
-    #[expect(dead_code)]
     pub(super) shm_allocator: Rc<VulkanThreadedAllocator>,
 }
 
diff --git a/src/gfx_apis/vulkan/staging.rs b/src/gfx_apis/vulkan/staging.rs
index d09d705a..e2d6b39f 100644
--- a/src/gfx_apis/vulkan/staging.rs
+++ b/src/gfx_apis/vulkan/staging.rs
@@ -1,11 +1,13 @@
 use {
     crate::{
+        cpu_worker::CpuWorker,
         gfx_apis::vulkan::{
             allocator::{VulkanAllocation, VulkanAllocator},
             device::VulkanDevice,
+            renderer::VulkanRenderer,
             VulkanError,
         },
-        utils::on_drop::OnDrop,
+        utils::on_drop::{OnDrop, OnDrop2},
     },
     ash::vk::{Buffer, BufferCreateInfo, BufferUsageFlags, MappedMemoryRange},
     gpu_alloc::UsageFlags,
@@ -28,24 +30,8 @@ impl VulkanDevice {
         download: bool,
         transient: bool,
     ) -> Result<VulkanStagingBuffer, VulkanError> {
-        let mut vk_usage = BufferUsageFlags::empty();
-        let mut usage = UsageFlags::empty();
-        if upload {
-            vk_usage |= BufferUsageFlags::TRANSFER_SRC;
-            usage |= UsageFlags::UPLOAD;
-        }
-        if download {
-            vk_usage |= BufferUsageFlags::TRANSFER_DST;
-            usage |= UsageFlags::DOWNLOAD;
-        }
-        if transient {
-            usage |= UsageFlags::TRANSIENT;
-        }
-        let buffer = {
-            let create_info = BufferCreateInfo::default().size(size).usage(vk_usage);
-            let buffer = unsafe { self.device.create_buffer(&create_info, None) };
-            buffer.map_err(VulkanError::CreateBuffer)?
-        };
+        let (vk_usage, usage) = get_usage(upload, download, transient);
+        let buffer = self.create_buffer(size, vk_usage)?;
         let destroy_buffer = OnDrop(|| unsafe { self.device.destroy_buffer(buffer, None) });
         let memory_requirements = unsafe { self.device.get_buffer_memory_requirements(buffer) };
         let allocation = allocator.alloc(&memory_requirements, usage, true)?;
@@ -64,6 +50,73 @@ impl VulkanDevice {
             size,
         })
     }
+
+    #[expect(dead_code)]
+    pub(super) fn create_shm_staging(
+        self: &Rc<Self>,
+        renderer: &Rc<VulkanRenderer>,
+        cpu: &Rc<CpuWorker>,
+        size: u64,
+        upload: bool,
+        download: bool,
+        cb: impl FnOnce(Result<VulkanStagingBuffer, VulkanError>) + 'static,
+    ) -> Result<(), VulkanError> {
+        let (vk_usage, usage) = get_usage(upload, download, false);
+        let buffer = self.create_buffer(size, vk_usage)?;
+        let memory_requirements = unsafe { self.device.get_buffer_memory_requirements(buffer) };
+        let slf = self.clone();
+        let destroy_buffer =
+            OnDrop2::new(move || unsafe { slf.device.destroy_buffer(buffer, None) });
+        let slf = self.clone();
+        let finish_allocation = move |res| {
+            let allocation: VulkanAllocation = res?;
+            {
+                let res = unsafe {
+                    slf.device
+                        .bind_buffer_memory(buffer, allocation.memory, allocation.offset)
+                };
+                res.map_err(VulkanError::BindBufferMemory)?;
+            }
+            destroy_buffer.forget();
+            Ok(VulkanStagingBuffer {
+                device: slf.clone(),
+                allocation,
+                buffer,
+                size,
+            })
+        };
+        renderer.shm_allocator.async_alloc(
+            renderer,
+            cpu,
+            memory_requirements,
+            usage,
+            true,
+            move |res| cb(finish_allocation(res)),
+        )
+    }
+
+    fn create_buffer(&self, size: u64, usage: BufferUsageFlags) -> Result<Buffer, VulkanError> {
+        let create_info = BufferCreateInfo::default().size(size).usage(usage);
+        let buffer = unsafe { self.device.create_buffer(&create_info, None) };
+        buffer.map_err(VulkanError::CreateBuffer)
+    }
+}
+
+fn get_usage(upload: bool, download: bool, transient: bool) -> (BufferUsageFlags, UsageFlags) {
+    let mut vk_usage = BufferUsageFlags::empty();
+    let mut usage = UsageFlags::empty();
+    if upload {
+        vk_usage |= BufferUsageFlags::TRANSFER_SRC;
+        usage |= UsageFlags::UPLOAD;
+    }
+    if download {
+        vk_usage |= BufferUsageFlags::TRANSFER_DST;
+        usage |= UsageFlags::DOWNLOAD;
+    }
+    if transient {
+        usage |= UsageFlags::TRANSIENT;
+    }
+    (vk_usage, usage)
 }
 
 impl VulkanStagingBuffer {
diff --git a/src/utils/on_drop.rs b/src/utils/on_drop.rs
index 15557f0a..f7d7644d 100644
--- a/src/utils/on_drop.rs
+++ b/src/utils/on_drop.rs
@@ -24,14 +24,12 @@ where
 }
 
 impl<F: FnOnce()> OnDrop2<F> {
-    #[expect(dead_code)]
     pub fn new(f: F) -> Self {
         Self {
             f: ManuallyDrop::new(f),
         }
     }
 
-    #[expect(dead_code)]
     pub fn forget(mut self) {
         unsafe {
             ManuallyDrop::drop(&mut self.f);

From c712efcd35baeb74b4c4dac7c087b48b258612bb Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 17:48:08 +0200
Subject: [PATCH 18/22] vulkan: implement async shm textures

---
 src/clientmem.rs                  |   5 -
 src/cpu_worker/jobs.rs            |   1 +
 src/cpu_worker/jobs/img_copy.rs   |  62 +++++
 src/cpu_worker/jobs/read_write.rs |   2 -
 src/gfx_api.rs                    |   2 -
 src/gfx_apis/vulkan.rs            |  31 ++-
 src/gfx_apis/vulkan/image.rs      |  63 ++++-
 src/gfx_apis/vulkan/renderer.rs   |   8 +
 src/gfx_apis/vulkan/shm_image.rs  | 409 ++++++++++++++++++++++++++++--
 src/gfx_apis/vulkan/staging.rs    |   1 -
 10 files changed, 548 insertions(+), 36 deletions(-)
 create mode 100644 src/cpu_worker/jobs/img_copy.rs

diff --git a/src/clientmem.rs b/src/clientmem.rs
index 1c8e9c98..8b7d506f 100644
--- a/src/clientmem.rs
+++ b/src/clientmem.rs
@@ -98,29 +98,24 @@ impl ClientMem {
         }
     }
 
-    #[expect(dead_code)]
     pub fn fd(&self) -> &Rc<OwnedFd> {
         &self.fd
     }
 
-    #[expect(dead_code)]
     pub fn sigbus_impossible(&self) -> bool {
         self.sigbus_impossible
     }
 }
 
 impl ClientMemOffset {
-    #[expect(dead_code)]
     pub fn pool(&self) -> &ClientMem {
         &self.mem
     }
 
-    #[expect(dead_code)]
     pub fn offset(&self) -> usize {
         self.offset
     }
 
-    #[expect(dead_code)]
     pub fn ptr(&self) -> *const [Cell<u8>] {
         self.data
     }
diff --git a/src/cpu_worker/jobs.rs b/src/cpu_worker/jobs.rs
index aadb998f..63b5b257 100644
--- a/src/cpu_worker/jobs.rs
+++ b/src/cpu_worker/jobs.rs
@@ -1 +1,2 @@
+pub mod img_copy;
 pub mod read_write;
diff --git a/src/cpu_worker/jobs/img_copy.rs b/src/cpu_worker/jobs/img_copy.rs
new file mode 100644
index 00000000..414c1afe
--- /dev/null
+++ b/src/cpu_worker/jobs/img_copy.rs
@@ -0,0 +1,62 @@
+use {
+    crate::{
+        cpu_worker::{AsyncCpuWork, CpuWork},
+        rect::Rect,
+    },
+    std::ptr,
+};
+
+#[expect(clippy::manual_non_exhaustive)]
+pub struct ImgCopyWork {
+    pub src: *mut u8,
+    pub dst: *mut u8,
+    pub width: i32,
+    pub stride: i32,
+    pub bpp: i32,
+    pub rects: Vec<Rect>,
+    _priv: (),
+}
+
+unsafe impl Send for ImgCopyWork {}
+
+impl ImgCopyWork {
+    pub unsafe fn new() -> Self {
+        Self {
+            src: ptr::null_mut(),
+            dst: ptr::null_mut(),
+            width: 0,
+            stride: 0,
+            bpp: 0,
+            rects: vec![],
+            _priv: (),
+        }
+    }
+}
+
+impl CpuWork for ImgCopyWork {
+    fn run(&mut self) -> Option<Box<dyn AsyncCpuWork>> {
+        for rect in &self.rects {
+            let mut offset = rect.y1() * self.stride + rect.x1() * self.bpp;
+            if rect.width() == self.width {
+                let offset = offset as usize;
+                let len = rect.height() * self.stride;
+                unsafe {
+                    ptr::copy_nonoverlapping(self.src.add(offset), self.dst.add(offset), len as _);
+                }
+            } else {
+                let len = rect.width() * self.bpp;
+                for _ in 0..rect.height() {
+                    unsafe {
+                        ptr::copy_nonoverlapping(
+                            self.src.add(offset as _),
+                            self.dst.add(offset as _),
+                            len as _,
+                        );
+                    }
+                    offset += self.stride;
+                }
+            }
+        }
+        None
+    }
+}
diff --git a/src/cpu_worker/jobs/read_write.rs b/src/cpu_worker/jobs/read_write.rs
index c8790529..791ee22f 100644
--- a/src/cpu_worker/jobs/read_write.rs
+++ b/src/cpu_worker/jobs/read_write.rs
@@ -36,7 +36,6 @@ pub struct ReadWriteWork {
 unsafe impl Send for ReadWriteWork {}
 
 impl ReadWriteWork {
-    #[expect(dead_code)]
     pub unsafe fn new() -> Self {
         let cancel = Arc::new(CancelState::default());
         ReadWriteWork {
@@ -53,7 +52,6 @@ impl ReadWriteWork {
         }
     }
 
-    #[expect(dead_code)]
     pub fn config(&mut self) -> &mut ReadWriteWorkConfig {
         self.config.as_mut().unwrap()
     }
diff --git a/src/gfx_api.rs b/src/gfx_api.rs
index 5ca68519..23ac5bab 100644
--- a/src/gfx_api.rs
+++ b/src/gfx_api.rs
@@ -538,7 +538,6 @@ pub trait ShmGfxTexture: GfxTexture {
 }
 
 pub trait AsyncShmGfxTextureCallback {
-    #[expect(dead_code)]
     fn completed(self: Rc<Self>, res: Result<(), GfxError>);
 }
 
@@ -689,7 +688,6 @@ pub fn cross_intersect_formats(
 }
 
 impl PendingShmUpload {
-    #[expect(dead_code)]
     pub fn new(cancel: Rc<dyn AsyncShmGfxTextureUploadCancellable>, id: u64) -> Self {
         Self { cancel, id }
     }
diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs
index 59923d50..4562377b 100644
--- a/src/gfx_apis/vulkan.rs
+++ b/src/gfx_apis/vulkan.rs
@@ -19,7 +19,7 @@ use {
     crate::{
         allocator::{Allocator, AllocatorError},
         async_engine::AsyncEngine,
-        cpu_worker::CpuWorker,
+        cpu_worker::{jobs::read_write::ReadWriteJobError, CpuWorker},
         format::Format,
         gfx_api::{
             AsyncShmGfxTexture, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage,
@@ -199,6 +199,10 @@ pub enum VulkanError {
     DupDrm(#[source] DrmError),
     #[error("Graphics context has already been dropped")]
     Defunct,
+    #[error("Could not perform an async copy to the staging buffer")]
+    AsyncCopyToStaging(#[source] ReadWriteJobError),
+    #[error("The async shm texture is busy")]
+    AsyncCopyBusy,
 }
 
 impl From<VulkanError> for GfxError {
@@ -278,19 +282,28 @@ impl GfxContext for Context {
         }
         let tex = self
             .0
-            .create_shm_texture(format, width, height, stride, data, false)?;
+            .create_shm_texture(format, width, height, stride, data, false, None)?;
         Ok(tex as _)
     }
 
     fn async_shmem_texture(
         self: Rc<Self>,
-        _format: &'static Format,
-        _width: i32,
-        _height: i32,
-        _stride: i32,
-        _cpu_worker: &Rc<CpuWorker>,
+        format: &'static Format,
+        width: i32,
+        height: i32,
+        stride: i32,
+        cpu_worker: &Rc<CpuWorker>,
     ) -> Result<Rc<dyn AsyncShmGfxTexture>, GfxError> {
-        todo!()
+        let tex = self.0.create_shm_texture(
+            format,
+            width,
+            height,
+            stride,
+            &[],
+            false,
+            Some(cpu_worker),
+        )?;
+        Ok(tex)
     }
 
     fn allocator(&self) -> Rc<dyn Allocator> {
@@ -310,7 +323,7 @@ impl GfxContext for Context {
     ) -> Result<Rc<dyn GfxFramebuffer>, GfxError> {
         let fb = self
             .0
-            .create_shm_texture(format, width, height, stride, &[], true)?;
+            .create_shm_texture(format, width, height, stride, &[], true, None)?;
         Ok(fb)
     }
 
diff --git a/src/gfx_apis/vulkan/image.rs b/src/gfx_apis/vulkan/image.rs
index 29469a79..53c26af4 100644
--- a/src/gfx_apis/vulkan/image.rs
+++ b/src/gfx_apis/vulkan/image.rs
@@ -1,13 +1,17 @@
 use {
     crate::{
+        clientmem::ClientMemOffset,
         format::Format,
         gfx_api::{
-            GfxApiOpt, GfxError, GfxFramebuffer, GfxImage, GfxTexture, ShmGfxTexture, SyncFile,
+            AsyncShmGfxTexture, AsyncShmGfxTextureCallback, AsyncShmGfxTextureUploadCancellable,
+            GfxApiOpt, GfxError, GfxFramebuffer, GfxImage, GfxTexture, PendingShmUpload,
+            ShmGfxTexture, SyncFile,
         },
         gfx_apis::vulkan::{
             allocator::VulkanAllocation, device::VulkanDevice, format::VulkanModifierLimits,
             renderer::VulkanRenderer, shm_image::VulkanShmImage, VulkanError,
         },
+        rect::Region,
         theme::Color,
         utils::{clonecell::CloneCell, on_drop::OnDrop},
         video::dmabuf::{DmaBuf, PlaneVec},
@@ -53,6 +57,7 @@ pub struct VulkanImage {
     pub(super) render_view: Option<ImageView>,
     pub(super) image: Image,
     pub(super) is_undefined: Cell<bool>,
+    pub(super) contents_are_undefined: Cell<bool>,
     pub(super) ty: VulkanImageMemory,
     pub(super) render_ops: CloneCell<Vec<GfxApiOpt>>,
     pub(super) bridge: Option<VulkanFramebufferBridge>,
@@ -380,6 +385,7 @@ impl VulkanDmaBufImageTemplate {
             }),
             format: self.dmabuf.format,
             is_undefined: Cell::new(true),
+            contents_are_undefined: Cell::new(false),
             bridge,
         }))
     }
@@ -538,3 +544,58 @@ impl ShmGfxTexture for VulkanImage {
         self
     }
 }
+
+impl AsyncShmGfxTexture for VulkanImage {
+    fn async_upload(
+        self: Rc<Self>,
+        callback: Rc<dyn AsyncShmGfxTextureCallback>,
+        mem: &Rc<ClientMemOffset>,
+        damage: Region,
+    ) -> Result<Option<PendingShmUpload>, GfxError> {
+        let VulkanImageMemory::Internal(shm) = &self.ty else {
+            unreachable!();
+        };
+        let pending = shm.async_upload(&self, mem, damage, callback)?;
+        Ok(pending)
+    }
+
+    fn sync_upload(self: Rc<Self>, mem: &[Cell<u8>], damage: Region) -> Result<(), GfxError> {
+        let VulkanImageMemory::Internal(shm) = &self.ty else {
+            unreachable!();
+        };
+        if shm.async_data.as_ref().unwrap().busy.get() {
+            return Err(VulkanError::AsyncCopyBusy.into());
+        }
+        shm.upload(&self, mem, Some(damage.rects()))?;
+        Ok(())
+    }
+
+    fn compatible_with(
+        &self,
+        format: &'static Format,
+        width: i32,
+        height: i32,
+        stride: i32,
+    ) -> bool {
+        self.format == format
+            && self.width == width as u32
+            && self.height == height as u32
+            && self.stride == stride as u32
+    }
+
+    fn into_texture(self: Rc<Self>) -> Rc<dyn GfxTexture> {
+        self
+    }
+}
+
+impl AsyncShmGfxTextureUploadCancellable for VulkanImage {
+    fn cancel(&self, id: u64) {
+        let VulkanImageMemory::Internal(shm) = &self.ty else {
+            unreachable!();
+        };
+        let data = shm.async_data.as_ref().unwrap();
+        if data.callback_id.get() == id {
+            data.callback.take();
+        }
+    }
+}
diff --git a/src/gfx_apis/vulkan/renderer.rs b/src/gfx_apis/vulkan/renderer.rs
index 71877a7f..0018700b 100644
--- a/src/gfx_apis/vulkan/renderer.rs
+++ b/src/gfx_apis/vulkan/renderer.rs
@@ -279,6 +279,9 @@ impl VulkanRenderer {
         for cmd in opts {
             if let GfxApiOpt::CopyTexture(c) = cmd {
                 let tex = c.tex.clone().into_vk(&self.device.device);
+                if tex.contents_are_undefined.get() {
+                    continue;
+                }
                 if let VulkanImageMemory::DmaBuf(_) = &tex.ty {
                     memory.sample.push(tex.clone())
                 }
@@ -458,6 +461,10 @@ impl VulkanRenderer {
                 }
                 GfxApiOpt::CopyTexture(c) => {
                     let tex = c.tex.as_vk(&self.device.device);
+                    if tex.contents_are_undefined.get() {
+                        log::warn!("Ignoring undefined texture");
+                        continue;
+                    }
                     let copy_type = match c.alpha.is_some() {
                         true => TexCopyType::Multiply,
                         false => TexCopyType::Identity,
@@ -807,6 +814,7 @@ impl VulkanRenderer {
             stride as i32,
             &[],
             true,
+            None,
         )?;
         (&*tmp_tex as &dyn GfxFramebuffer)
             .copy_texture(
diff --git a/src/gfx_apis/vulkan/shm_image.rs b/src/gfx_apis/vulkan/shm_image.rs
index a21698a3..24ef74e5 100644
--- a/src/gfx_apis/vulkan/shm_image.rs
+++ b/src/gfx_apis/vulkan/shm_image.rs
@@ -1,7 +1,15 @@
 use {
     crate::{
+        clientmem::ClientMemOffset,
+        cpu_worker::{
+            jobs::{
+                img_copy::ImgCopyWork,
+                read_write::{ReadWriteJobError, ReadWriteWork},
+            },
+            CpuJob, CpuWork, CpuWorker,
+        },
         format::{Format, FormatShmInfo},
-        gfx_api::SyncFile,
+        gfx_api::{AsyncShmGfxTextureCallback, PendingShmUpload, SyncFile},
         gfx_apis::vulkan::{
             allocator::VulkanAllocation,
             command::VulkanCommandBuffer,
@@ -11,8 +19,8 @@ use {
             staging::VulkanStagingBuffer,
             VulkanError,
         },
-        rect::Rect,
-        utils::{errorfmt::ErrorFmt, on_drop::OnDrop},
+        rect::{Rect, Region},
+        utils::{clonecell::CloneCell, errorfmt::ErrorFmt, on_drop::OnDrop},
     },
     ash::vk::{
         AccessFlags2, BufferImageCopy2, BufferMemoryBarrier2, CommandBufferBeginInfo,
@@ -24,14 +32,32 @@ use {
     },
     gpu_alloc::UsageFlags,
     isnt::std_1::primitive::IsntSliceExt,
-    std::{cell::Cell, ptr, rc::Rc, slice},
+    std::{
+        cell::{Cell, RefCell},
+        ptr,
+        rc::Rc,
+        slice,
+    },
+    uapi::OwnedFd,
 };
 
 pub struct VulkanShmImage {
-    pub(super) _size: DeviceSize,
+    pub(super) size: DeviceSize,
     pub(super) stride: u32,
     pub(super) _allocation: VulkanAllocation,
     pub(super) shm_info: &'static FormatShmInfo,
+    pub(super) async_data: Option<VulkanShmImageAsyncData>,
+}
+
+pub struct VulkanShmImageAsyncData {
+    pub(super) busy: Cell<bool>,
+    pub(super) io_job: Cell<Option<Box<IoUploadJob>>>,
+    pub(super) copy_job: Cell<Option<Box<CopyUploadJob>>>,
+    pub(super) staging: CloneCell<Option<Rc<VulkanStagingBuffer>>>,
+    pub(super) callback: Cell<Option<Rc<dyn AsyncShmGfxTextureCallback>>>,
+    pub(super) callback_id: Cell<u64>,
+    pub(super) regions: RefCell<Vec<BufferImageCopy2<'static>>>,
+    pub(super) cpu: Rc<CpuWorker>,
 }
 
 impl VulkanShmImage {
@@ -133,6 +159,30 @@ impl VulkanShmImage {
                 ptr::copy_nonoverlapping(buf, mem, total_size as usize);
             }
         })?;
+        let Some((cmd, fence, sync_file, point)) =
+            self.submit_buffer_to_image_copy(img, &staging, cpy)?
+        else {
+            return Ok(());
+        };
+        let future = img.renderer.eng.spawn(await_upload(
+            point,
+            img.clone(),
+            cmd,
+            sync_file,
+            fence,
+            staging,
+        ));
+        img.renderer.pending_uploads.set(point, future);
+        Ok(())
+    }
+
+    fn submit_buffer_to_image_copy(
+        &self,
+        img: &Rc<VulkanImage>,
+        staging: &VulkanStagingBuffer,
+        regions: &[BufferImageCopy2],
+    ) -> Result<Option<(Rc<VulkanCommandBuffer>, Rc<VulkanFence>, SyncFile, u64)>, VulkanError>
+    {
         let memory_barrier = |sam, ssm, dam, dsm| {
             BufferMemoryBarrier2::default()
                 .buffer(staging.buffer)
@@ -185,7 +235,7 @@ impl VulkanShmImage {
             .src_buffer(staging.buffer)
             .dst_image(img.image)
             .dst_image_layout(ImageLayout::TRANSFER_DST_OPTIMAL)
-            .regions(cpy);
+            .regions(regions);
         let cmd = img.renderer.allocate_command_buffer()?;
         let dev = &img.renderer.device.device;
         let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd.buffer);
@@ -210,35 +260,341 @@ impl VulkanShmImage {
             .map_err(VulkanError::Submit)?;
         }
         img.is_undefined.set(false);
+        img.contents_are_undefined.set(false);
         let release_sync_file = match release_fence.export_sync_file() {
             Ok(s) => s,
             Err(e) => {
                 log::error!("Could not export sync file from fence: {}", ErrorFmt(e));
                 img.renderer.block();
-                return Ok(());
+                return Ok(None);
             }
         };
         let point = img.renderer.allocate_point();
-        let future = img.renderer.eng.spawn(await_upload(
+        Ok(Some((cmd, release_fence, release_sync_file, point)))
+    }
+}
+
+async fn await_upload(
+    id: u64,
+    img: Rc<VulkanImage>,
+    buf: Rc<VulkanCommandBuffer>,
+    sync_file: SyncFile,
+    _fence: Rc<VulkanFence>,
+    _staging: VulkanStagingBuffer,
+) {
+    let res = img.renderer.ring.readable(&sync_file.0).await;
+    if let Err(e) = res {
+        log::error!(
+            "Could not wait for sync file to become readable: {}",
+            ErrorFmt(e)
+        );
+        img.renderer.block();
+    }
+    img.renderer.command_buffers.push(buf);
+    img.renderer.pending_uploads.remove(&id);
+}
+
+impl VulkanShmImageAsyncData {
+    fn complete(&self, result: Result<(), VulkanError>) {
+        self.busy.set(false);
+        if let Some(cb) = self.callback.take() {
+            cb.completed(result.map_err(|e| e.into()));
+        }
+    }
+}
+
+impl VulkanShmImage {
+    pub fn async_upload(
+        &self,
+        img: &Rc<VulkanImage>,
+        client_mem: &Rc<ClientMemOffset>,
+        damage: Region,
+        callback: Rc<dyn AsyncShmGfxTextureCallback>,
+    ) -> Result<Option<PendingShmUpload>, VulkanError> {
+        let data = self.async_data.as_ref().unwrap();
+        let res = self.try_async_upload(img, data, client_mem, damage);
+        match res {
+            Ok(()) => {
+                let id = img.renderer.allocate_point();
+                data.callback_id.set(id);
+                data.callback.set(Some(callback));
+                Ok(Some(PendingShmUpload::new(img.clone(), id)))
+            }
+            Err(e) => Err(e),
+        }
+    }
+
+    fn try_async_upload(
+        &self,
+        img: &Rc<VulkanImage>,
+        data: &VulkanShmImageAsyncData,
+        client_mem: &Rc<ClientMemOffset>,
+        mut damage: Region,
+    ) -> Result<(), VulkanError> {
+        if data.busy.get() {
+            return Err(VulkanError::AsyncCopyBusy);
+        }
+        if self.size > client_mem.ptr().len() as u64 {
+            return Err(VulkanError::InvalidBufferSize);
+        }
+        data.busy.set(true);
+        if img.contents_are_undefined.get() {
+            damage = Region::new2(Rect::new_sized(0, 0, img.width as _, img.height as _).unwrap())
+        }
+
+        let copies = &mut *data.regions.borrow_mut();
+        copies.clear();
+
+        let mut copy = |x, y, width, height| {
+            let buffer_offset = (y as u32 * img.stride + x as u32 * self.shm_info.bpp) as u64;
+            let copy = BufferImageCopy2::default()
+                .buffer_offset(buffer_offset)
+                .image_offset(Offset3D { x, y, z: 0 })
+                .image_extent(Extent3D {
+                    width,
+                    height,
+                    depth: 1,
+                })
+                .image_subresource(ImageSubresourceLayers {
+                    aspect_mask: ImageAspectFlags::COLOR,
+                    mip_level: 0,
+                    base_array_layer: 0,
+                    layer_count: 1,
+                })
+                .buffer_image_height(img.height)
+                .buffer_row_length(img.stride / self.shm_info.bpp);
+            copies.push(copy);
+        };
+        for damage in damage.rects() {
+            let Some(damage) = Rect::new(
+                damage.x1().max(0),
+                damage.y1().max(0),
+                damage.x2().min(img.width as i32),
+                damage.y2().min(img.height as i32),
+            ) else {
+                continue;
+            };
+            if damage.is_empty() {
+                continue;
+            }
+            copy(
+                damage.x1(),
+                damage.y1(),
+                damage.width() as u32,
+                damage.height() as u32,
+            );
+        }
+
+        if let Some(staging) = data.staging.get() {
+            return self.async_upload_initiate_copy(img, data, &staging, copies, client_mem);
+        }
+
+        let img2 = img.clone();
+        let client_mem = client_mem.clone();
+        img.renderer.device.create_shm_staging(
+            &img.renderer,
+            &data.cpu,
+            self.size,
+            true,
+            false,
+            move |res| {
+                let VulkanImageMemory::Internal(shm) = &img2.ty else {
+                    unreachable!();
+                };
+                if let Err(e) = shm.async_upload_after_allocation(&img2, &client_mem, res) {
+                    shm.async_data.as_ref().unwrap().complete(Err(e));
+                }
+            },
+        )
+    }
+
+    fn async_upload_after_allocation(
+        &self,
+        img: &Rc<VulkanImage>,
+        client_mem: &Rc<ClientMemOffset>,
+        res: Result<VulkanStagingBuffer, VulkanError>,
+    ) -> Result<(), VulkanError> {
+        let staging = Rc::new(res?);
+        let data = self.async_data.as_ref().unwrap();
+        data.staging.set(Some(staging.clone()));
+        let copies = &*data.regions.borrow();
+        self.async_upload_initiate_copy(img, data, &staging, copies, client_mem)
+    }
+
+    fn async_upload_initiate_copy(
+        &self,
+        img: &Rc<VulkanImage>,
+        data: &VulkanShmImageAsyncData,
+        staging: &VulkanStagingBuffer,
+        copies: &[BufferImageCopy2],
+        client_mem: &Rc<ClientMemOffset>,
+    ) -> Result<(), VulkanError> {
+        img.renderer.check_defunct()?;
+
+        let id = img.renderer.allocate_point();
+        let pending;
+        if client_mem.pool().sigbus_impossible() {
+            let mut job = data.copy_job.take().unwrap_or_else(|| {
+                Box::new(CopyUploadJob {
+                    img: None,
+                    id,
+                    _mem: None,
+                    work: unsafe { ImgCopyWork::new() },
+                })
+            });
+            job.id = id;
+            job.img = Some(img.clone());
+            job._mem = Some(client_mem.clone());
+            job.work.src = client_mem.ptr() as _;
+            job.work.dst = staging.allocation.mem.unwrap();
+            job.work.width = img.width as _;
+            job.work.stride = img.stride as _;
+            job.work.bpp = self.shm_info.bpp as _;
+            job.work.rects.clear();
+            for copy in copies {
+                job.work.rects.push(
+                    Rect::new_sized(
+                        copy.image_offset.x as _,
+                        copy.image_offset.y as _,
+                        copy.image_extent.width as _,
+                        copy.image_extent.height as _,
+                    )
+                    .unwrap(),
+                );
+            }
+            pending = data.cpu.submit(job);
+        } else {
+            let mut min_offset = client_mem.ptr().len() as u64;
+            let mut max_offset = 0;
+            for copy in copies {
+                min_offset = min_offset.min(copy.buffer_offset);
+                let len = img.stride * (copy.image_extent.height - 1)
+                    + copy.image_extent.width * self.shm_info.bpp;
+                max_offset = max_offset.max(copy.buffer_offset + len as u64);
+            }
+            let mut job = data.io_job.take().unwrap_or_else(|| {
+                Box::new(IoUploadJob {
+                    img: None,
+                    id,
+                    _mem: None,
+                    work: unsafe { ReadWriteWork::new() },
+                    fd: None,
+                })
+            });
+            job.id = id;
+            job.img = Some(img.clone());
+            job._mem = Some(client_mem.clone());
+            job.fd = Some(client_mem.pool().fd().clone());
+            unsafe {
+                let config = job.work.config();
+                config.fd = client_mem.pool().fd().raw();
+                config.offset = client_mem.offset() + min_offset as usize;
+                config.ptr = staging.allocation.mem.unwrap().add(min_offset as _);
+                config.len = max_offset.saturating_sub(min_offset) as usize;
+                config.write = false;
+            }
+            pending = data.cpu.submit(job);
+        }
+
+        img.renderer.pending_cpu_jobs.set(id, pending);
+
+        Ok(())
+    }
+
+    fn async_upload_copy_buffer_to_image(
+        &self,
+        img: &Rc<VulkanImage>,
+        data: &VulkanShmImageAsyncData,
+        res: Result<(), ReadWriteJobError>,
+    ) -> Result<(), VulkanError> {
+        if let Err(e) = res {
+            return Err(VulkanError::AsyncCopyToStaging(e));
+        }
+        img.renderer.check_defunct()?;
+        let regions = &*data.regions.borrow();
+        let staging = data.staging.get().unwrap();
+        staging.upload(|_, _| ())?;
+        let Some((cmd, fence, sync_file, point)) =
+            self.submit_buffer_to_image_copy(img, &staging, regions)?
+        else {
+            return Ok(());
+        };
+        let future = img.renderer.eng.spawn(await_async_upload(
             point,
             img.clone(),
             cmd,
-            release_sync_file,
-            release_fence,
-            staging,
+            fence,
+            sync_file,
         ));
         img.renderer.pending_uploads.set(point, future);
         Ok(())
     }
 }
 
-async fn await_upload(
+pub(super) struct IoUploadJob {
+    img: Option<Rc<VulkanImage>>,
+    id: u64,
+    _mem: Option<Rc<ClientMemOffset>>,
+    fd: Option<Rc<OwnedFd>>,
+    work: ReadWriteWork,
+}
+
+pub(super) struct CopyUploadJob {
+    img: Option<Rc<VulkanImage>>,
+    id: u64,
+    _mem: Option<Rc<ClientMemOffset>>,
+    work: ImgCopyWork,
+}
+
+impl CpuJob for IoUploadJob {
+    fn work(&mut self) -> &mut dyn CpuWork {
+        &mut self.work
+    }
+
+    fn completed(mut self: Box<Self>) {
+        self._mem = None;
+        self.fd = None;
+        let img = self.img.take().unwrap();
+        let res = self.work.config().result.take().unwrap();
+        complete_async_upload(&img, self.id, res, |data| data.io_job.set(Some(self)));
+    }
+}
+
+impl CpuJob for CopyUploadJob {
+    fn work(&mut self) -> &mut dyn CpuWork {
+        &mut self.work
+    }
+
+    fn completed(mut self: Box<Self>) {
+        self._mem = None;
+        let img = self.img.take().unwrap();
+        complete_async_upload(&img, self.id, Ok(()), |data| data.copy_job.set(Some(self)));
+    }
+}
+
+fn complete_async_upload(
+    img: &Rc<VulkanImage>,
+    id: u64,
+    res: Result<(), ReadWriteJobError>,
+    store: impl FnOnce(&VulkanShmImageAsyncData),
+) {
+    img.renderer.pending_cpu_jobs.remove(&id);
+    let VulkanImageMemory::Internal(shm) = &img.ty else {
+        unreachable!();
+    };
+    let data = shm.async_data.as_ref().unwrap();
+    store(data);
+    if let Err(e) = shm.async_upload_copy_buffer_to_image(img, data, res) {
+        data.complete(Err(e));
+    }
+}
+
+async fn await_async_upload(
     id: u64,
     img: Rc<VulkanImage>,
     buf: Rc<VulkanCommandBuffer>,
-    sync_file: SyncFile,
     _fence: Rc<VulkanFence>,
-    _staging: VulkanStagingBuffer,
+    sync_file: SyncFile,
 ) {
     let res = img.renderer.ring.readable(&sync_file.0).await;
     if let Err(e) = res {
@@ -250,6 +606,11 @@ async fn await_upload(
     }
     img.renderer.command_buffers.push(buf);
     img.renderer.pending_uploads.remove(&id);
+    let VulkanImageMemory::Internal(shm) = &img.ty else {
+        unreachable!();
+    };
+    let data = shm.async_data.as_ref().unwrap();
+    data.complete(Ok(()));
 }
 
 impl VulkanRenderer {
@@ -261,6 +622,7 @@ impl VulkanRenderer {
         stride: i32,
         data: &[Cell<u8>],
         for_download: bool,
+        cpu_worker: Option<&Rc<CpuWorker>>,
     ) -> Result<Rc<VulkanImage>, VulkanError> {
         let Some(shm_info) = &format.shm_info else {
             return Err(VulkanError::UnsupportedShmFormat(format.name));
@@ -283,7 +645,7 @@ impl VulkanRenderer {
         if width > shm.limits.max_width || height > shm.limits.max_height {
             return Err(VulkanError::ImageTooLarge);
         }
-        let size = stride.checked_mul(height).ok_or(VulkanError::ShmOverflow)?;
+        let size = stride.checked_mul(height).ok_or(VulkanError::ShmOverflow)? as u64;
         let usage = ImageUsageFlags::TRANSFER_SRC
             | match for_download {
                 true => ImageUsageFlags::COLOR_ATTACHMENT,
@@ -335,11 +697,25 @@ impl VulkanRenderer {
                 .create_image_view(&image_view_create_info, None)
         };
         let view = view.map_err(VulkanError::CreateImageView)?;
+        let mut async_data = None;
+        if let Some(cpu) = cpu_worker {
+            async_data = Some(VulkanShmImageAsyncData {
+                busy: Cell::new(false),
+                io_job: Default::default(),
+                copy_job: Default::default(),
+                staging: Default::default(),
+                callback: Default::default(),
+                callback_id: Cell::new(0),
+                regions: Default::default(),
+                cpu: cpu.clone(),
+            });
+        }
         let shm = VulkanShmImage {
-            _size: size as u64,
+            size,
             stride,
             _allocation: allocation,
             shm_info,
+            async_data,
         };
         destroy_image.forget();
         let img = Rc::new(VulkanImage {
@@ -352,6 +728,7 @@ impl VulkanRenderer {
             render_view: None,
             image,
             is_undefined: Cell::new(true),
+            contents_are_undefined: Cell::new(true),
             ty: VulkanImageMemory::Internal(shm),
             render_ops: Default::default(),
             bridge: None,
diff --git a/src/gfx_apis/vulkan/staging.rs b/src/gfx_apis/vulkan/staging.rs
index e2d6b39f..5a3faeac 100644
--- a/src/gfx_apis/vulkan/staging.rs
+++ b/src/gfx_apis/vulkan/staging.rs
@@ -51,7 +51,6 @@ impl VulkanDevice {
         })
     }
 
-    #[expect(dead_code)]
     pub(super) fn create_shm_staging(
         self: &Rc<Self>,
         renderer: &Rc<VulkanRenderer>,

From 80310f4c0de755c6cb4795a04ba8ba35571fecbc Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sun, 8 Sep 2024 09:23:02 +0200
Subject: [PATCH 19/22] utils: add DoubleBuffered

---
 src/utils.rs                 |  1 +
 src/utils/double_buffered.rs | 40 ++++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 src/utils/double_buffered.rs

diff --git a/src/utils.rs b/src/utils.rs
index 9aed5327..a5c94ee7 100644
--- a/src/utils.rs
+++ b/src/utils.rs
@@ -12,6 +12,7 @@ pub mod clone3;
 pub mod clonecell;
 pub mod copyhashmap;
 pub mod debug_fn;
+pub mod double_buffered;
 pub mod double_click_state;
 pub mod errorfmt;
 pub mod event_listener;
diff --git a/src/utils/double_buffered.rs b/src/utils/double_buffered.rs
new file mode 100644
index 00000000..6a60f783
--- /dev/null
+++ b/src/utils/double_buffered.rs
@@ -0,0 +1,40 @@
+use std::{cell::Cell, ops::Deref};
+
+#[derive(Default)]
+pub struct DoubleBuffered<T> {
+    bufs: [T; 2],
+    front: Cell<usize>,
+}
+
+impl<T> DoubleBuffered<T> {
+    #[expect(dead_code)]
+    pub fn new(bufs: [T; 2]) -> Self {
+        Self {
+            bufs,
+            front: Cell::new(0),
+        }
+    }
+
+    #[expect(dead_code)]
+    pub fn front(&self) -> &T {
+        unsafe { self.bufs.get_unchecked(self.front.get()) }
+    }
+
+    #[expect(dead_code)]
+    pub fn back(&self) -> &T {
+        unsafe { self.bufs.get_unchecked(1 - self.front.get()) }
+    }
+
+    #[expect(dead_code)]
+    pub fn flip(&self) {
+        self.front.set(1 - self.front.get());
+    }
+}
+
+impl<T> Deref for DoubleBuffered<T> {
+    type Target = [T; 2];
+
+    fn deref(&self) -> &Self::Target {
+        &self.bufs
+    }
+}

From d40e605f6651d6d56150d3bfde92b2913569335b Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 16:52:03 +0200
Subject: [PATCH 20/22] surface: use async uploads for shm buffers

---
 src/gfx_api.rs                        |   4 -
 src/ifs/wl_buffer.rs                  |  57 +++++---
 src/ifs/wl_surface.rs                 |  53 +++++--
 src/ifs/wl_surface/commit_timeline.rs | 194 +++++++++++++++++++++++---
 src/rect.rs                           |   4 +-
 src/rect/region.rs                    |   4 -
 src/state.rs                          |   3 +-
 src/utils/double_buffered.rs          |   4 -
 8 files changed, 250 insertions(+), 73 deletions(-)

diff --git a/src/gfx_api.rs b/src/gfx_api.rs
index 23ac5bab..9e7882ec 100644
--- a/src/gfx_api.rs
+++ b/src/gfx_api.rs
@@ -551,7 +551,6 @@ pub struct PendingShmUpload {
 }
 
 pub trait AsyncShmGfxTexture: GfxTexture {
-    #[expect(dead_code)]
     fn async_upload(
         self: Rc<Self>,
         callback: Rc<dyn AsyncShmGfxTextureCallback>,
@@ -561,7 +560,6 @@ pub trait AsyncShmGfxTexture: GfxTexture {
 
     fn sync_upload(self: Rc<Self>, shm: &[Cell<u8>], damage: Region) -> Result<(), GfxError>;
 
-    #[expect(dead_code)]
     fn compatible_with(
         &self,
         format: &'static Format,
@@ -570,7 +568,6 @@ pub trait AsyncShmGfxTexture: GfxTexture {
         stride: i32,
     ) -> bool;
 
-    #[expect(dead_code)]
     fn into_texture(self: Rc<Self>) -> Rc<dyn GfxTexture>;
 }
 
@@ -598,7 +595,6 @@ pub trait GfxContext: Debug {
         damage: Option<&[Rect]>,
     ) -> Result<Rc<dyn ShmGfxTexture>, GfxError>;
 
-    #[expect(dead_code)]
     fn async_shmem_texture(
         self: Rc<Self>,
         format: &'static Format,
diff --git a/src/ifs/wl_buffer.rs b/src/ifs/wl_buffer.rs
index 268f72a8..3068d572 100644
--- a/src/ifs/wl_buffer.rs
+++ b/src/ifs/wl_buffer.rs
@@ -7,7 +7,7 @@ use {
         ifs::wl_surface::WlSurface,
         leaks::Tracker,
         object::{Object, Version},
-        rect::Rect,
+        rect::{Rect, Region},
         theme::Color,
         utils::errorfmt::ErrorFmt,
         video::dmabuf::DmaBuf,
@@ -22,7 +22,7 @@ use {
 
 pub enum WlBufferStorage {
     Shm {
-        mem: ClientMemOffset,
+        mem: Rc<ClientMemOffset>,
         stride: i32,
     },
     Dmabuf {
@@ -41,6 +41,7 @@ pub struct WlBuffer {
     pub dmabuf: Option<DmaBuf>,
     render_ctx_version: Cell<u32>,
     pub storage: RefCell<Option<WlBufferStorage>>,
+    shm: bool,
     pub color: Option<Color>,
     width: i32,
     height: i32,
@@ -52,6 +53,10 @@ impl WlBuffer {
         self.destroyed.get()
     }
 
+    pub fn is_shm(&self) -> bool {
+        self.shm
+    }
+
     pub fn new_dmabuf(
         id: WlBufferId,
         client: &Rc<Client>,
@@ -76,6 +81,7 @@ impl WlBuffer {
                 tex: None,
                 fb: None,
             })),
+            shm: false,
             tracker: Default::default(),
             color: None,
         }
@@ -100,7 +106,7 @@ impl WlBuffer {
         if required > mem.len() as u64 {
             return Err(WlBufferError::OutOfBounds);
         }
-        let mem = mem.offset(offset);
+        let mem = Rc::new(mem.offset(offset));
         let min_row_size = width as u64 * shm_info.bpp as u64;
         if (stride as u64) < min_row_size {
             return Err(WlBufferError::StrideTooSmall);
@@ -114,6 +120,7 @@ impl WlBuffer {
             dmabuf: None,
             render_ctx_version: Cell::new(client.state.render_ctx_version.get()),
             storage: RefCell::new(Some(WlBufferStorage::Shm { mem, stride })),
+            shm: true,
             width,
             height,
             tracker: Default::default(),
@@ -138,6 +145,7 @@ impl WlBuffer {
             dmabuf: None,
             render_ctx_version: Cell::new(client.state.render_ctx_version.get()),
             storage: RefCell::new(None),
+            shm: false,
             width: 1,
             height: 1,
             tracker: Default::default(),
@@ -153,7 +161,7 @@ impl WlBuffer {
         let had_texture = self.reset_gfx_objects(surface);
         if had_texture {
             if let Some(surface) = surface {
-                self.update_texture_or_log(surface, None);
+                self.update_texture_or_log(surface, true);
             }
         }
     }
@@ -166,7 +174,10 @@ impl WlBuffer {
         let had_texture = match s {
             WlBufferStorage::Shm { .. } => {
                 return match surface {
-                    Some(s) => s.shm_texture.take().is_some(),
+                    Some(s) => {
+                        s.shm_textures.back().tex.take();
+                        s.shm_textures.front().tex.take().is_some()
+                    }
                     None => false,
                 };
             }
@@ -201,24 +212,24 @@ impl WlBuffer {
         match &*self.storage.borrow() {
             None => None,
             Some(s) => match s {
-                WlBufferStorage::Shm { .. } => surface.shm_texture.get().map(|t| t.into_texture()),
+                WlBufferStorage::Shm { .. } => surface
+                    .shm_textures
+                    .front()
+                    .tex
+                    .get()
+                    .map(|t| t.into_texture()),
                 WlBufferStorage::Dmabuf { tex, .. } => tex.clone(),
             },
         }
     }
 
-    pub fn update_texture_or_log(&self, surface: &WlSurface, damage: Option<&[Rect]>) {
-        if let Err(e) = self.update_texture(surface, damage) {
+    pub fn update_texture_or_log(&self, surface: &WlSurface, sync_shm: bool) {
+        if let Err(e) = self.update_texture(surface, sync_shm) {
             log::warn!("Could not update texture: {}", ErrorFmt(e));
         }
     }
 
-    fn update_texture(
-        &self,
-        surface: &WlSurface,
-        damage: Option<&[Rect]>,
-    ) -> Result<(), WlBufferError> {
-        let old_shm_texture = surface.shm_texture.take();
+    fn update_texture(&self, surface: &WlSurface, sync_shm: bool) -> Result<(), WlBufferError> {
         let storage = &mut *self.storage.borrow_mut();
         let storage = match storage {
             Some(s) => s,
@@ -226,19 +237,19 @@ impl WlBuffer {
         };
         match storage {
             WlBufferStorage::Shm { mem, stride } => {
-                if let Some(ctx) = self.client.state.render_ctx.get() {
-                    let tex = mem.access(|mem| {
-                        ctx.shmem_texture(
-                            old_shm_texture,
-                            mem,
+                if sync_shm {
+                    if let Some(ctx) = self.client.state.render_ctx.get() {
+                        let tex = ctx.async_shmem_texture(
                             self.format,
                             self.width,
                             self.height,
                             *stride,
-                            damage,
-                        )
-                    })??;
-                    surface.shm_texture.set(Some(tex));
+                            &self.client.state.cpu_worker,
+                        )?;
+                        mem.access(|mem| tex.clone().sync_upload(mem, Region::new2(self.rect)))??;
+                        surface.shm_textures.front().tex.set(Some(tex));
+                        surface.shm_textures.front().damage.clear();
+                    }
                 }
             }
             WlBufferStorage::Dmabuf { img, tex, .. } => {
diff --git a/src/ifs/wl_surface.rs b/src/ifs/wl_surface.rs
index 48422742..63b9101b 100644
--- a/src/ifs/wl_surface.rs
+++ b/src/ifs/wl_surface.rs
@@ -23,8 +23,8 @@ use {
         drm_feedback::DrmFeedback,
         fixed::Fixed,
         gfx_api::{
-            AcquireSync, BufferResv, BufferResvUser, ReleaseSync, SampleRect, ShmGfxTexture,
-            SyncFile,
+            AcquireSync, AsyncShmGfxTexture, BufferResv, BufferResvUser, GfxError, ReleaseSync,
+            SampleRect, SyncFile,
         },
         ifs::{
             wl_buffer::WlBuffer,
@@ -60,16 +60,16 @@ use {
         },
         leaks::Tracker,
         object::{Object, Version},
-        rect::{Rect, Region},
+        rect::{DamageQueue, Rect, Region},
         renderer::Renderer,
         tree::{
             ContainerNode, FindTreeResult, FoundNode, Node, NodeId, NodeVisitor, NodeVisitorBase,
             OutputNode, OutputNodeId, PlaceholderNode, ToplevelNode,
         },
         utils::{
-            cell_ext::CellExt, clonecell::CloneCell, copyhashmap::CopyHashMap, errorfmt::ErrorFmt,
-            linkedlist::LinkedList, numcell::NumCell, smallmap::SmallMap,
-            transform_ext::TransformExt,
+            cell_ext::CellExt, clonecell::CloneCell, copyhashmap::CopyHashMap,
+            double_buffered::DoubleBuffered, errorfmt::ErrorFmt, linkedlist::LinkedList,
+            numcell::NumCell, smallmap::SmallMap, transform_ext::TransformExt,
         },
         video::{
             dmabuf::DMA_BUF_SYNC_READ,
@@ -250,6 +250,11 @@ impl BufferResv for SurfaceBuffer {
     }
 }
 
+pub struct SurfaceShmTexture {
+    pub tex: CloneCell<Option<Rc<dyn AsyncShmGfxTexture>>>,
+    pub damage: DamageQueue,
+}
+
 pub struct WlSurface {
     pub id: WlSurfaceId,
     pub node_id: SurfaceNodeId,
@@ -272,7 +277,7 @@ pub struct WlSurface {
     pub buffer: CloneCell<Option<Rc<SurfaceBuffer>>>,
     buffer_presented: Cell<bool>,
     buffer_had_frame_request: Cell<bool>,
-    pub shm_texture: CloneCell<Option<Rc<dyn ShmGfxTexture>>>,
+    pub shm_textures: DoubleBuffered<SurfaceShmTexture>,
     pub buf_x: NumCell<i32>,
     pub buf_y: NumCell<i32>,
     pub children: RefCell<Option<Box<ParentData>>>,
@@ -585,7 +590,10 @@ impl WlSurface {
             buffer: Default::default(),
             buffer_presented: Default::default(),
             buffer_had_frame_request: Default::default(),
-            shm_texture: Default::default(),
+            shm_textures: DoubleBuffered::new(DamageQueue::new().map(|damage| SurfaceShmTexture {
+                tex: Default::default(),
+                damage,
+            })),
             buf_x: Default::default(),
             buf_y: Default::default(),
             children: Default::default(),
@@ -910,7 +918,7 @@ impl WlSurfaceRequestHandler for WlSurface {
             *children = None;
         }
         self.buffer.set(None);
-        self.shm_texture.take();
+        self.reset_shm_textures();
         if let Some(xwayland_serial) = self.xwayland_serial.get() {
             self.client
                 .surfaces_by_xwayland_serial
@@ -1078,11 +1086,13 @@ impl WlSurface {
                 old_raw_size = Some(buffer.buffer.rect);
             }
             if let Some(buffer) = buffer_change {
-                let damage = match pending.damage_full || pending.surface_damage.is_not_empty() {
-                    true => None,
-                    false => Some(&pending.buffer_damage[..]),
-                };
-                buffer.update_texture_or_log(self, damage);
+                if buffer.is_shm() {
+                    self.shm_textures.flip();
+                    self.shm_textures.front().damage.clear();
+                } else {
+                    self.reset_shm_textures();
+                }
+                buffer.update_texture_or_log(self, false);
                 let (sync, release_sync) = match pending.explicit_sync {
                     false => (AcquireSync::Implicit, ReleaseSync::Implicit),
                     true => (AcquireSync::Unnecessary, ReleaseSync::Explicit),
@@ -1104,7 +1114,7 @@ impl WlSurface {
                     self.buffer_had_frame_request.set(false);
                 }
             } else {
-                self.shm_texture.take();
+                self.reset_shm_textures();
                 self.buf_x.set(0);
                 self.buf_y.set(0);
                 for (_, cursor) in &self.cursors {
@@ -1322,6 +1332,13 @@ impl WlSurface {
         Ok(())
     }
 
+    pub fn reset_shm_textures(&self) {
+        for tex in &*self.shm_textures {
+            tex.tex.take();
+            tex.damage.clear();
+        }
+    }
+
     fn apply_damage(&self, pending: &PendingState) {
         let bounds = self.toplevel.get().map(|tl| tl.node_absolute_position());
         let pos = self.buffer_abs_pos.get();
@@ -1916,6 +1933,12 @@ pub enum WlSurfaceError {
     UnexpectedSyncPoints,
     #[error("The supplied region is invalid")]
     InvalidRect,
+    #[error("There is no render context")]
+    NoRenderContext,
+    #[error("Could not create a shm texture")]
+    CreateAsyncShmTexture(#[source] GfxError),
+    #[error("Could not prepare upload to a shm texture")]
+    PrepareAsyncUpload(#[source] GfxError),
 }
 efrom!(WlSurfaceError, ClientError);
 efrom!(WlSurfaceError, XdgSurfaceError);
diff --git a/src/ifs/wl_surface/commit_timeline.rs b/src/ifs/wl_surface/commit_timeline.rs
index a150b6fa..8ec7a1b0 100644
--- a/src/ifs/wl_surface/commit_timeline.rs
+++ b/src/ifs/wl_surface/commit_timeline.rs
@@ -1,6 +1,10 @@
 use {
     crate::{
-        ifs::wl_surface::{PendingState, WlSurface, WlSurfaceError},
+        gfx_api::{AsyncShmGfxTextureCallback, GfxError, PendingShmUpload},
+        ifs::{
+            wl_buffer::WlBufferStorage,
+            wl_surface::{PendingState, WlSurface, WlSurfaceError},
+        },
         utils::{
             clonecell::CloneCell,
             copyhashmap::CopyHashMap,
@@ -14,13 +18,14 @@ use {
             DrmError,
         },
     },
-    isnt::std_1::primitive::IsntSliceExt,
+    isnt::std_1::{primitive::IsntSliceExt, vec::IsntVecExt},
     smallvec::SmallVec,
     std::{
         cell::{Cell, RefCell},
         mem,
-        ops::{Deref, DerefMut},
+        ops::DerefMut,
         rc::Rc,
+        slice,
     },
     thiserror::Error,
 };
@@ -76,6 +81,8 @@ pub enum CommitTimelineError {
     Wait(#[source] DrmError),
     #[error("The client has too many pending commits")]
     Depth,
+    #[error("Could not upload a shm texture")]
+    ShmUpload(#[source] GfxError),
 }
 
 impl CommitTimelines {
@@ -119,6 +126,7 @@ fn break_loops(list: &LinkedList<Entry>) {
         entry.link.take();
         if let EntryKind::Commit(c) = &entry.kind {
             c.wait_handles.take();
+            *c.shm_upload.borrow_mut() = ShmUploadState::None;
         }
     }
 }
@@ -145,7 +153,9 @@ impl CommitTimeline {
     ) -> Result<(), CommitTimelineError> {
         let mut points = SmallVec::new();
         consume_acquire_points(pending, &mut points);
-        if points.is_empty() && self.own_timeline.entries.is_empty() {
+        let mut pending_uploads = 0;
+        count_shm_uploads(pending, &mut pending_uploads);
+        if points.is_empty() && pending_uploads == 0 && self.own_timeline.entries.is_empty() {
             return surface
                 .apply_state(pending)
                 .map_err(CommitTimelineError::ImmediateCommit);
@@ -162,23 +172,35 @@ impl CommitTimeline {
                 pending: RefCell::new(mem::take(pending)),
                 sync_obj: NumCell::new(points.len()),
                 wait_handles: Cell::new(Default::default()),
+                pending_uploads: NumCell::new(pending_uploads),
+                shm_upload: RefCell::new(ShmUploadState::None),
             }),
         );
-        if points.is_not_empty() {
-            let mut wait_handles = SmallVec::new();
-            let noderef = Rc::new(noderef);
-            for (sync_obj, point) in points {
-                let handle = self
-                    .shared
-                    .wfs
-                    .wait(&sync_obj, point, true, noderef.clone())
-                    .map_err(CommitTimelineError::RegisterWait)?;
-                wait_handles.push(handle);
-            }
+        let mut needs_flush = false;
+        if points.is_not_empty() || pending_uploads > 0 {
+            let noderef = Rc::new(noderef.clone());
             let EntryKind::Commit(commit) = &noderef.kind else {
                 unreachable!();
             };
-            commit.wait_handles.set(wait_handles);
+            if points.is_not_empty() {
+                let mut wait_handles = SmallVec::new();
+                for (sync_obj, point) in points {
+                    let handle = self
+                        .shared
+                        .wfs
+                        .wait(&sync_obj, point, true, noderef.clone())
+                        .map_err(CommitTimelineError::RegisterWait)?;
+                    wait_handles.push(handle);
+                }
+                commit.wait_handles.set(wait_handles);
+            }
+            if pending_uploads > 0 {
+                *commit.shm_upload.borrow_mut() = ShmUploadState::Todo(noderef.clone());
+                needs_flush = true;
+            }
+        }
+        if needs_flush && noderef.prev().is_none() {
+            flush_from(noderef.clone()).map_err(CommitTimelineError::DelayedCommit)?;
         }
         Ok(())
     }
@@ -194,12 +216,33 @@ impl SyncObjWaiter for NodeRef<Entry> {
             return;
         }
         commit.sync_obj.fetch_sub(1);
-        if let Err(e) = flush_from(self.deref().clone()) {
+        flush_commit(&self, commit);
+    }
+}
+
+fn flush_commit(node_ref: &NodeRef<Entry>, commit: &Commit) {
+    if let Err(e) = flush_from(node_ref.clone()) {
+        commit
+            .surface
+            .client
+            .error(CommitTimelineError::DelayedCommit(e));
+    }
+}
+
+impl AsyncShmGfxTextureCallback for NodeRef<Entry> {
+    fn completed(self: Rc<Self>, res: Result<(), GfxError>) {
+        let EntryKind::Commit(commit) = &self.kind else {
+            unreachable!();
+        };
+        if let Err(e) = res {
             commit
                 .surface
                 .client
-                .error(CommitTimelineError::DelayedCommit(e));
+                .error(CommitTimelineError::ShmUpload(e));
+            return;
         }
+        commit.pending_uploads.fetch_sub(1);
+        flush_commit(&self, commit);
     }
 }
 
@@ -216,11 +259,19 @@ enum EntryKind {
     Gc(CommitTimelineId),
 }
 
+enum ShmUploadState {
+    None,
+    Todo(Rc<NodeRef<Entry>>),
+    Scheduled(#[expect(dead_code)] SmallVec<[PendingShmUpload; 1]>),
+}
+
 struct Commit {
     surface: Rc<WlSurface>,
     pending: RefCell<Box<PendingState>>,
     sync_obj: NumCell<usize>,
     wait_handles: Cell<SmallVec<[WaitForSyncObjHandle; 1]>>,
+    pending_uploads: NumCell<usize>,
+    shm_upload: RefCell<ShmUploadState>,
 }
 
 fn flush_from(mut point: NodeRef<Entry>) -> Result<(), WlSurfaceError> {
@@ -243,7 +294,17 @@ impl NodeRef<Entry> {
         }
         match &self.kind {
             EntryKind::Commit(c) => {
+                let mut has_unmet_dependencies = false;
                 if c.sync_obj.get() > 0 {
+                    has_unmet_dependencies = true;
+                }
+                if c.pending_uploads.get() > 0 {
+                    check_shm_uploads(c)?;
+                    if c.pending_uploads.get() > 0 {
+                        has_unmet_dependencies = true;
+                    }
+                }
+                if has_unmet_dependencies {
                     return Ok(false);
                 }
                 c.surface.apply_state(c.pending.borrow_mut().deref_mut())?;
@@ -266,6 +327,90 @@ impl NodeRef<Entry> {
     }
 }
 
+fn check_shm_uploads(c: &Commit) -> Result<(), WlSurfaceError> {
+    let state = &mut *c.shm_upload.borrow_mut();
+    if let ShmUploadState::Todo(node_ref) = state {
+        let mut pending = SmallVec::new();
+        schedule_async_uploads(node_ref, &c.surface, &c.pending.borrow(), &mut pending)?;
+        c.pending_uploads.set(pending.len());
+        *state = ShmUploadState::Scheduled(pending);
+    }
+    Ok(())
+}
+
+fn schedule_async_uploads(
+    node_ref: &Rc<NodeRef<Entry>>,
+    surface: &WlSurface,
+    pending: &PendingState,
+    uploads: &mut SmallVec<[PendingShmUpload; 1]>,
+) -> Result<(), WlSurfaceError> {
+    if let Some(pending) = schedule_async_upload(node_ref, surface, pending)? {
+        uploads.push(pending);
+    }
+    for ss in pending.subsurfaces.values() {
+        if let Some(state) = &ss.pending.state {
+            schedule_async_uploads(node_ref, &ss.subsurface.surface, state, uploads)?;
+        }
+    }
+    Ok(())
+}
+
+fn schedule_async_upload(
+    node_ref: &Rc<NodeRef<Entry>>,
+    surface: &WlSurface,
+    pending: &PendingState,
+) -> Result<Option<PendingShmUpload>, WlSurfaceError> {
+    let Some(Some(buf)) = &pending.buffer else {
+        return Ok(None);
+    };
+    let Some(WlBufferStorage::Shm { mem, stride, .. }) = &*buf.storage.borrow() else {
+        return Ok(None);
+    };
+    let back = surface.shm_textures.back();
+    let mut back_tex_opt = back.tex.get();
+    if let Some(back_tex) = &back_tex_opt {
+        if !back_tex.compatible_with(buf.format, buf.rect.width(), buf.rect.height(), *stride) {
+            back_tex_opt = None;
+        }
+    }
+    let damage_full = || {
+        back.damage.clear();
+        back.damage.damage(slice::from_ref(&buf.rect));
+    };
+    let back_tex = match back_tex_opt {
+        Some(b) => {
+            if pending.damage_full || pending.surface_damage.is_not_empty() {
+                damage_full();
+            } else {
+                back.damage.damage(&pending.buffer_damage);
+            }
+            b
+        }
+        None => {
+            damage_full();
+            let state = &surface.client.state;
+            let ctx = state
+                .render_ctx
+                .get()
+                .ok_or(WlSurfaceError::NoRenderContext)?;
+            let back_tex = ctx
+                .async_shmem_texture(
+                    buf.format,
+                    buf.rect.width(),
+                    buf.rect.height(),
+                    *stride,
+                    &state.cpu_worker,
+                )
+                .map_err(WlSurfaceError::CreateAsyncShmTexture)?;
+            back.tex.set(Some(back_tex.clone()));
+            back_tex
+        }
+    };
+    back_tex
+        .async_upload(node_ref.clone(), mem, back.damage.get())
+        .map_err(WlSurfaceError::PrepareAsyncUpload)
+}
+
 type Point = (Rc<SyncObj>, SyncObjPoint);
 
 fn consume_acquire_points(pending: &mut PendingState, points: &mut SmallVec<[Point; 1]>) {
@@ -301,3 +446,16 @@ fn set_effective_timeline(
         }
     }
 }
+
+fn count_shm_uploads(pending: &PendingState, count: &mut usize) {
+    if let Some(Some(buffer)) = &pending.buffer {
+        if buffer.is_shm() {
+            *count += 1;
+        }
+    }
+    for ss in pending.subsurfaces.values() {
+        if let Some(state) = &ss.pending.state {
+            count_shm_uploads(state, count);
+        }
+    }
+}
diff --git a/src/rect.rs b/src/rect.rs
index a5cd711c..a8bd1230 100644
--- a/src/rect.rs
+++ b/src/rect.rs
@@ -3,9 +3,7 @@ mod region;
 #[cfg(test)]
 mod tests;
 
-#[expect(unused_imports)]
-pub use region::DamageQueue;
-pub use region::RegionBuilder;
+pub use region::{DamageQueue, RegionBuilder};
 use {
     jay_algorithms::rect::RectRaw,
     smallvec::SmallVec,
diff --git a/src/rect/region.rs b/src/rect/region.rs
index ade706c1..6ea178de 100644
--- a/src/rect/region.rs
+++ b/src/rect/region.rs
@@ -197,7 +197,6 @@ pub struct DamageQueue {
 }
 
 impl DamageQueue {
-    #[expect(dead_code)]
     pub fn new<const N: usize>() -> [DamageQueue; N] {
         let datas = Rc::new(UnsafeCell::new(vec![vec!(); N]));
         array::from_fn(|this| DamageQueue {
@@ -206,7 +205,6 @@ impl DamageQueue {
         })
     }
 
-    #[expect(dead_code)]
     pub fn damage(&self, rects: &[Rect]) {
         let datas = unsafe { self.datas.get().deref_mut() };
         for data in datas {
@@ -214,13 +212,11 @@ impl DamageQueue {
         }
     }
 
-    #[expect(dead_code)]
     pub fn clear(&self) {
         let data = unsafe { &mut self.datas.get().deref_mut()[self.this] };
         data.clear();
     }
 
-    #[expect(dead_code)]
     pub fn get(&self) -> Region {
         let data = unsafe { &self.datas.get().deref()[self.this] };
         Region::from_rects2(data)
diff --git a/src/state.rs b/src/state.rs
index fb44f9f6..f3f2f5b0 100644
--- a/src/state.rs
+++ b/src/state.rs
@@ -215,7 +215,6 @@ pub struct State {
     pub enable_ei_acceptor: Cell<bool>,
     pub ei_clients: EiClients,
     pub slow_ei_clients: AsyncQueue<Rc<EiClient>>,
-    #[expect(dead_code)]
     pub cpu_worker: Rc<CpuWorker>,
 }
 
@@ -485,7 +484,7 @@ impl State {
                         updated_buffers.insert(buffer.buffer.id);
                         buffer.buffer.handle_gfx_context_change(Some(surface));
                     } else {
-                        surface.shm_texture.take();
+                        surface.reset_shm_textures();
                     }
                 }
                 for buffer in client.data.objects.buffers.lock().values() {
diff --git a/src/utils/double_buffered.rs b/src/utils/double_buffered.rs
index 6a60f783..9844377a 100644
--- a/src/utils/double_buffered.rs
+++ b/src/utils/double_buffered.rs
@@ -7,7 +7,6 @@ pub struct DoubleBuffered<T> {
 }
 
 impl<T> DoubleBuffered<T> {
-    #[expect(dead_code)]
     pub fn new(bufs: [T; 2]) -> Self {
         Self {
             bufs,
@@ -15,17 +14,14 @@ impl<T> DoubleBuffered<T> {
         }
     }
 
-    #[expect(dead_code)]
     pub fn front(&self) -> &T {
         unsafe { self.bufs.get_unchecked(self.front.get()) }
     }
 
-    #[expect(dead_code)]
     pub fn back(&self) -> &T {
         unsafe { self.bufs.get_unchecked(1 - self.front.get()) }
     }
 
-    #[expect(dead_code)]
     pub fn flip(&self) {
         self.front.set(1 - self.front.get());
     }

From ece56d91d36b0f7d1bed22e3c4d8e609a9bd7495 Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sat, 7 Sep 2024 22:54:36 +0200
Subject: [PATCH 21/22] shm: close client buffers in the cpu worker

---
 src/cli/input.rs                           |  7 +--
 src/clientmem.rs                           | 55 +++++++++++++++++++---
 src/cpu_worker.rs                          |  1 -
 src/ifs/jay_input.rs                       |  9 +++-
 src/ifs/wl_seat/zwp_virtual_keyboard_v1.rs | 12 +++--
 src/ifs/wl_shm_pool.rs                     |  9 +++-
 src/it/tests/t0040_virtual_keyboard.rs     |  2 +-
 7 files changed, 79 insertions(+), 16 deletions(-)

diff --git a/src/cli/input.rs b/src/cli/input.rs
index 46625bd5..d48b5624 100644
--- a/src/cli/input.rs
+++ b/src/cli/input.rs
@@ -353,9 +353,10 @@ impl Input {
     async fn handle_keymap(&self, input: JayInputId) -> Vec<u8> {
         let data = Rc::new(RefCell::new(Vec::new()));
         jay_input::Keymap::handle(&self.tc, input, data.clone(), |d, map| {
-            let mem =
-                Rc::new(ClientMem::new(&map.keymap, map.keymap_len as _, true, None).unwrap())
-                    .offset(0);
+            let mem = Rc::new(
+                ClientMem::new(&map.keymap, map.keymap_len as _, true, None, None).unwrap(),
+            )
+            .offset(0);
             mem.read(d.borrow_mut().deref_mut()).unwrap();
         });
         self.tc.round_trip().await;
diff --git a/src/clientmem.rs b/src/clientmem.rs
index 8b7d506f..3f435e0c 100644
--- a/src/clientmem.rs
+++ b/src/clientmem.rs
@@ -1,8 +1,12 @@
 use {
-    crate::{client::Client, utils::vec_ext::VecExt},
+    crate::{
+        client::Client,
+        cpu_worker::{AsyncCpuWork, CpuJob, CpuWork, CpuWorker},
+        utils::vec_ext::VecExt,
+    },
     std::{
         cell::Cell,
-        mem::MaybeUninit,
+        mem::{ManuallyDrop, MaybeUninit},
         ptr,
         rc::Rc,
         sync::atomic::{compiler_fence, Ordering},
@@ -25,10 +29,11 @@ pub enum ClientMemError {
 }
 
 pub struct ClientMem {
-    fd: Rc<OwnedFd>,
+    fd: ManuallyDrop<Rc<OwnedFd>>,
     failed: Cell<bool>,
     sigbus_impossible: bool,
     data: *const [Cell<u8>],
+    cpu: Option<Rc<CpuWorker>>,
 }
 
 #[derive(Clone)]
@@ -44,6 +49,7 @@ impl ClientMem {
         len: usize,
         read_only: bool,
         client: Option<&Client>,
+        cpu: Option<&Rc<CpuWorker>>,
     ) -> Result<Self, ClientMemError> {
         let mut sigbus_impossible = false;
         if let Ok(seals) = uapi::fcntl_get_seals(fd.raw()) {
@@ -78,10 +84,11 @@ impl ClientMem {
             }
         };
         Ok(Self {
-            fd: fd.clone(),
+            fd: ManuallyDrop::new(fd.clone()),
             failed: Cell::new(false),
             sigbus_impossible,
             data,
+            cpu: cpu.cloned(),
         })
     }
 
@@ -155,8 +162,17 @@ impl ClientMemOffset {
 
 impl Drop for ClientMem {
     fn drop(&mut self) {
-        unsafe {
-            c::munmap(self.data as _, self.len());
+        let fd = unsafe { ManuallyDrop::take(&mut self.fd) };
+        if let Some(cpu) = &self.cpu {
+            let pending = cpu.submit(Box::new(CloseMemWork {
+                fd: Rc::try_unwrap(fd).ok(),
+                data: self.data,
+            }));
+            pending.detach();
+        } else {
+            unsafe {
+                c::munmap(self.data as _, self.len());
+            }
         }
     }
 }
@@ -219,3 +235,30 @@ pub fn init() -> Result<(), ClientMemError> {
         }
     }
 }
+
+struct CloseMemWork {
+    fd: Option<OwnedFd>,
+    data: *const [Cell<u8>],
+}
+
+unsafe impl Send for CloseMemWork {}
+
+impl CpuJob for CloseMemWork {
+    fn work(&mut self) -> &mut dyn CpuWork {
+        self
+    }
+
+    fn completed(self: Box<Self>) {
+        // nothing
+    }
+}
+
+impl CpuWork for CloseMemWork {
+    fn run(&mut self) -> Option<Box<dyn AsyncCpuWork>> {
+        self.fd.take();
+        unsafe {
+            c::munmap(self.data as _, self.data.len());
+        }
+        None
+    }
+}
diff --git a/src/cpu_worker.rs b/src/cpu_worker.rs
index 3875401f..ec360aa0 100644
--- a/src/cpu_worker.rs
+++ b/src/cpu_worker.rs
@@ -140,7 +140,6 @@ pub enum CpuWorkerError {
 }
 
 impl PendingJob {
-    #[expect(dead_code)]
     pub fn detach(self) {
         match self.job_data.state.get() {
             PendingJobState::Waiting => {
diff --git a/src/ifs/jay_input.rs b/src/ifs/jay_input.rs
index 8d196063..2e6d26fe 100644
--- a/src/ifs/jay_input.rs
+++ b/src/ifs/jay_input.rs
@@ -169,7 +169,14 @@ impl JayInput {
     where
         F: FnOnce(&Rc<XkbKeymap>) -> Result<(), JayInputError>,
     {
-        let cm = Rc::new(ClientMem::new(keymap, len as _, true, Some(&self.client))?).offset(0);
+        let cm = Rc::new(ClientMem::new(
+            keymap,
+            len as _,
+            true,
+            Some(&self.client),
+            None,
+        )?)
+        .offset(0);
         let mut map = vec![];
         cm.read(&mut map)?;
         self.or_error(|| {
diff --git a/src/ifs/wl_seat/zwp_virtual_keyboard_v1.rs b/src/ifs/wl_seat/zwp_virtual_keyboard_v1.rs
index 914ac247..e100ce3e 100644
--- a/src/ifs/wl_seat/zwp_virtual_keyboard_v1.rs
+++ b/src/ifs/wl_seat/zwp_virtual_keyboard_v1.rs
@@ -56,9 +56,15 @@ impl ZwpVirtualKeyboardV1RequestHandler for ZwpVirtualKeyboardV1 {
         if req.size > MAX_SIZE {
             return Err(ZwpVirtualKeyboardV1Error::OversizedKeymap);
         }
-        let client_mem = ClientMem::new(&req.fd, req.size as usize - 1, true, Some(&self.client))
-            .map(Rc::new)
-            .map_err(ZwpVirtualKeyboardV1Error::MapKeymap)?;
+        let client_mem = ClientMem::new(
+            &req.fd,
+            req.size as usize - 1,
+            true,
+            Some(&self.client),
+            None,
+        )
+        .map(Rc::new)
+        .map_err(ZwpVirtualKeyboardV1Error::MapKeymap)?;
         let mut map = vec![];
         client_mem
             .offset(0)
diff --git a/src/ifs/wl_shm_pool.rs b/src/ifs/wl_shm_pool.rs
index 6ea0c670..b03dd608 100644
--- a/src/ifs/wl_shm_pool.rs
+++ b/src/ifs/wl_shm_pool.rs
@@ -34,7 +34,13 @@ impl WlShmPool {
         Ok(Self {
             id,
             client: client.clone(),
-            mem: CloneCell::new(Rc::new(ClientMem::new(&fd, len, false, Some(client))?)),
+            mem: CloneCell::new(Rc::new(ClientMem::new(
+                &fd,
+                len,
+                false,
+                Some(client),
+                Some(&client.state.cpu_worker),
+            )?)),
             fd,
             tracker: Default::default(),
             version,
@@ -86,6 +92,7 @@ impl WlShmPoolRequestHandler for WlShmPool {
             req.size as usize,
             false,
             Some(&self.client),
+            Some(&self.client.state.cpu_worker),
         )?));
         Ok(())
     }
diff --git a/src/it/tests/t0040_virtual_keyboard.rs b/src/it/tests/t0040_virtual_keyboard.rs
index bc0627c3..dc2b1d1f 100644
--- a/src/it/tests/t0040_virtual_keyboard.rs
+++ b/src/it/tests/t0040_virtual_keyboard.rs
@@ -149,7 +149,7 @@ async fn test(run: Rc<TestRun>) -> TestResult {
 }
 
 fn read_keymap(fd: &Rc<OwnedFd>, size: usize) -> String {
-    let client_mem = ClientMem::new(fd, size - 1, true, None).unwrap();
+    let client_mem = ClientMem::new(fd, size - 1, true, None, None).unwrap();
     let client_mem = Rc::new(client_mem).offset(0);
     let mut v = vec![];
     client_mem.read(&mut v).unwrap();

From 7b5be32b2144e57a739fddf0cbcd2b8a12dba80d Mon Sep 17 00:00:00 2001
From: Julian Orth <ju.orth@gmail.com>
Date: Sun, 8 Sep 2024 14:48:20 +0200
Subject: [PATCH 22/22] text: limit width to 4k to avoid performance
 degradation

---
 src/text.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/text.rs b/src/text.rs
index 104e8776..f0176800 100644
--- a/src/text.rs
+++ b/src/text.rs
@@ -170,6 +170,7 @@ fn render2(
     markup: bool,
     scale: Option<f64>,
 ) -> Result<TextTexture, TextError> {
+    let width = width.min(3840);
     let config = Config {
         x,
         y,