Restaged repo, allocator and runtime implemented, ioring-backed async fs/net/channel/timer primitives

lib/runtime/src/platform/linux_x86_64/mesh_alloc/allocator.rs

@@ -0,0 +1,864 @@
+use core::alloc::Layout;
+use core::mem::size_of;
+use core::ptr::copy_nonoverlapping;
+use core::sync::atomic::{AtomicU32, Ordering};
+
+use super::arena::Arena;
+use super::constants::{
+    MAX_ATTACHED_MINIHEAPS_PER_CLASS, MAX_SMALL_ALLOCATION, MIN_SHUFFLE_VECTOR_LENGTH,
+    MINIHEAP_REFILL_GOAL_SIZE, NUM_SIZE_CLASSES, is_below_partial_threshold,
+};
+use super::fault::{self, ActiveMeshGuard};
+use super::meshing::bitmaps_meshable;
+use super::miniheap::{MiniHeap, MiniHeapId};
+use super::page::{page_count, page_size, runtime_slots_per_span};
+use super::platform;
+use super::pool::MiniHeapPool;
+use super::raw_sys;
+use super::rng::Mwc;
+use super::shuffle::ShuffleEntry;
+use super::size_map::{byte_size_for_class, size_class_for};
+use super::stats::{MeshStats, StatsState};
+use super::sync::{FutexMutex, futex_wait_for_value, futex_wake_all};
+use super::thread_local_heap::ThreadLocalHeap;
+
+#[derive(Debug)]
+pub struct MeshAllocator {
+    arena: Arena,
+    pool: MiniHeapPool,
+    bootstrap_thread: *mut ThreadLocalHeap,
+    compaction_candidates: *mut MiniHeapId,
+    meshing_rng: Mwc,
+    mesh_epoch: AtomicU32,
+    pool_lock: FutexMutex,
+    stats: StatsState,
+}
+
+#[derive(Clone, Copy, Debug)]
+struct ResolvedPtr {
+    owner_id: MiniHeapId,
+    slot: usize,
+}
+
+impl MeshAllocator {
+    pub fn new(arena_size: usize, miniheap_capacity: u32) -> raw_sys::Result<Self> {
+        fault::ensure_fault_mediation_installed()?;
+        let bootstrap_thread = unsafe {
+            platform::map_anonymous(
+                size_of::<ThreadLocalHeap>(),
+                raw_sys::PROT_READ | raw_sys::PROT_WRITE,
+            )? as *mut ThreadLocalHeap
+        };
+        unsafe {
+            bootstrap_thread.write(ThreadLocalHeap::new()?);
+        }
+        let compaction_candidates = unsafe {
+            platform::map_anonymous(
+                miniheap_capacity as usize * size_of::<MiniHeapId>(),
+                raw_sys::PROT_READ | raw_sys::PROT_WRITE,
+            )? as *mut MiniHeapId
+        };
+
+        Ok(Self {
+            arena: Arena::with_size(arena_size)?,
+            pool: MiniHeapPool::with_capacity(miniheap_capacity)?,
+            bootstrap_thread,
+            compaction_candidates,
+            meshing_rng: Mwc::from_os_seed()?,
+            mesh_epoch: AtomicU32::new(0),
+            pool_lock: FutexMutex::new(),
+            stats: StatsState::new(),
+        })
+    }
+
+    #[inline(always)]
+    pub fn arena(&self) -> &Arena {
+        &self.arena
+    }
+
+    #[inline(always)]
+    pub fn pool(&self) -> &MiniHeapPool {
+        &self.pool
+    }
+
+    #[inline(always)]
+    pub fn live_miniheap_count(&self) -> u32 {
+        self.pool.live_len()
+    }
+
+    pub fn stats(&self) -> MeshStats {
+        let page = page_size();
+        let (reusable_span_count, reusable_pages) = self.arena.reusable_span_stats();
+        let counters = self.stats.snapshot();
+        let mut stats = MeshStats {
+            arena_size: self.arena.arena_size(),
+            reserved_bytes: self.arena.reserved_pages() as usize * page,
+            reusable_span_count,
+            reusable_span_bytes: reusable_pages as usize * page,
+            live_miniheaps: self.pool.live_len(),
+            small_allocations: counters.small_allocations,
+            small_deallocations: counters.small_deallocations,
+            large_allocations: counters.large_allocations,
+            large_deallocations: counters.large_deallocations,
+            compact_calls: counters.compact_calls,
+            meshes_performed: counters.meshes_performed,
+            meshed_pages: counters.meshed_pages,
+            meshed_bytes: counters.meshed_bytes,
+            ..MeshStats::default()
+        };
+
+        let mut candidate_heaps_by_class = [0u32; NUM_SIZE_CLASSES];
+        let mut candidate_pages_by_class = [0u32; NUM_SIZE_CLASSES];
+        let mut candidate_free_bytes_by_class = [0usize; NUM_SIZE_CLASSES];
+        let mut candidate_span_bytes_by_class = [0usize; NUM_SIZE_CLASSES];
+
+        let len = self.pool.len();
+        let mut id_value = 1u32;
+        while id_value <= len {
+            let id = MiniHeapId::new(id_value);
+            if let Some(heap) = self.pool.get(id) {
+                if heap.is_large_alloc() {
+                    stats.live_large_allocations += 1;
+                    stats.live_large_bytes += heap.span_size();
+                    stats.retained_large_span_bytes += heap.span_size();
+                    id_value += 1;
+                    continue;
+                }
+
+                stats.live_small_heaps += 1;
+                stats.live_small_bytes += heap.in_use_count() as usize * heap.object_size();
+                stats.virtual_small_span_bytes += heap.span_size();
+
+                if heap.is_meshed() {
+                    stats.meshed_small_heaps += 1;
+                } else {
+                    stats.retained_small_span_bytes += heap.span_size();
+                }
+
+                if heap.is_full() {
+                    stats.full_small_heaps += 1;
+                } else if !heap.is_empty() {
+                    stats.partial_small_heaps += 1;
+                }
+
+                if !heap.is_attached() && !heap.is_full() && !heap.is_meshed() {
+                    stats.reusable_small_heaps += 1;
+                }
+
+                if self.heap_is_compaction_candidate(heap.size_class(), heap) {
+                    let class = heap.size_class() as usize;
+                    stats.compaction.candidate_heaps += 1;
+                    stats.compaction.candidate_pages += heap.span().length;
+                    stats.compaction.candidate_free_bytes += heap.bytes_free();
+                    candidate_heaps_by_class[class] += 1;
+                    candidate_pages_by_class[class] += heap.span().length;
+                    candidate_free_bytes_by_class[class] += heap.bytes_free();
+                    candidate_span_bytes_by_class[class] = heap.span_size();
+                }
+            }
+            id_value += 1;
+        }
+
+        let mut class = 1usize;
+        while class < NUM_SIZE_CLASSES {
+            let span_bytes = candidate_span_bytes_by_class[class];
+            if let Some(pair_bound_by_free) =
+                candidate_free_bytes_by_class[class].checked_div(span_bytes)
+            {
+                let pair_bound_by_count = candidate_heaps_by_class[class] / 2;
+                let best_case_meshes = pair_bound_by_count.min(pair_bound_by_free as u32);
+                let pages_per_mesh =
+                    candidate_pages_by_class[class] / candidate_heaps_by_class[class].max(1);
+                stats.compaction.best_case_meshes += best_case_meshes;
+                stats.compaction.best_case_reclaimable_pages += best_case_meshes * pages_per_mesh;
+                stats.compaction.best_case_reclaimable_bytes +=
+                    best_case_meshes as usize * span_bytes;
+            }
+            class += 1;
+        }
+
+        stats
+    }
+
+    pub fn allocate(&mut self, size: usize) -> Option<*mut u8> {
+        let thread_heap = unsafe { &mut *self.bootstrap_thread };
+        self.allocate_with_thread(size, thread_heap)
+    }
+
+    pub fn allocate_with_thread(
+        &mut self,
+        size: usize,
+        thread_heap: &mut ThreadLocalHeap,
+    ) -> Option<*mut u8> {
+        let size = size.max(1);
+        if size <= MAX_SMALL_ALLOCATION {
+            let class = size_class_for(size)?;
+            if let Some(ptr) = self.try_allocate_small_local(thread_heap, class) {
+                return Some(ptr);
+            }
+            return self.allocate_small_with_thread(thread_heap, class);
+        }
+
+        self.allocate_large(size)
+    }
+
+    pub fn allocate_layout(&mut self, layout: Layout) -> Option<*mut u8> {
+        let thread_heap = unsafe { &mut *self.bootstrap_thread };
+        self.allocate_layout_with_thread(thread_heap, layout)
+    }
+
+    pub fn allocate_layout_with_thread(
+        &mut self,
+        thread_heap: &mut ThreadLocalHeap,
+        layout: Layout,
+    ) -> Option<*mut u8> {
+        let aligned_size = round_up_to_alignment(layout.size().max(1), layout.align())?;
+        if aligned_size <= MAX_SMALL_ALLOCATION && layout.align() <= page_size() {
+            let class = size_class_for(aligned_size)?;
+            if byte_size_for_class(class).is_multiple_of(layout.align()) {
+                if let Some(ptr) = self.try_allocate_small_local(thread_heap, class) {
+                    return Some(ptr);
+                }
+                return self.allocate_small_with_thread(thread_heap, class);
+            }
+        }
+
+        self.allocate_large_aligned(aligned_size, layout.align())
+    }
+
+    pub fn deallocate(&mut self, ptr: *mut u8) {
+        let thread_heap = unsafe { &mut *self.bootstrap_thread };
+        self.deallocate_with_thread(ptr, thread_heap);
+    }
+
+    pub fn deallocate_with_thread(&mut self, ptr: *mut u8, thread_heap: &mut ThreadLocalHeap) {
+        if ptr.is_null() {
+            return;
+        }
+
+        let Some(resolved) = self.resolve_pointer(ptr) else {
+            return;
+        };
+        let id = resolved.owner_id;
+        let Some(heap) = self.pool.get(id) else {
+            return;
+        };
+
+        if heap.is_large_alloc() {
+            let span = heap.span();
+            self.stats.record_large_deallocation();
+            let _ = heap.free_offset(0);
+            self.arena.clear_miniheap(span);
+            self.arena.release_span(span);
+            let _ = {
+                let _guard = self.pool_lock.lock();
+                self.pool.release(id)
+            };
+            return;
+        }
+
+        let slot = resolved.slot;
+        let class = heap.size_class();
+        let thread_id = thread_heap.thread_id();
+        self.stats.record_small_deallocation();
+
+        if heap.current_thread() == thread_id && heap.is_attached() && !heap.is_meshed() {
+            let state = thread_heap.class_mut(class);
+            let attached_idx = state.find_attached(id);
+            if let Some(attached_idx) = attached_idx
+                && !state.shuffle.is_full()
+            {
+                let cached = state.shuffle.count_entries_for_offset(attached_idx as u16);
+                if cached + 1 == heap.max_count() as usize {
+                    self.release_class_attached(thread_heap, class);
+                    if let Some(heap) = self.pool.get(id) {
+                        let _ = heap.free_offset(slot);
+                    }
+                    self.reclaim_empty_detached_heap(id);
+                    return;
+                }
+                if cached + 1 < heap.max_count() as usize {
+                    state
+                        .shuffle
+                        .push(ShuffleEntry::new(attached_idx as u16, slot as u16));
+                    return;
+                }
+            }
+        }
+
+        let state = thread_heap.class_mut(class);
+        let _ = heap.free_offset(slot);
+        if heap.is_attached()
+            && is_below_partial_threshold(heap.in_use_count(), heap.max_count() as u32)
+        {
+            heap.unset_attached();
+            if let Some(attached_idx) = state.find_attached(id) {
+                state.attached_ids[attached_idx as usize] = MiniHeapId::new(0);
+                state.attached_heaps[attached_idx as usize] = core::ptr::null();
+            }
+        }
+
+        if heap.is_empty() && !heap.is_meshed() {
+            self.reclaim_empty_detached_heap(id);
+        }
+    }
+
+    pub fn deallocate_layout(&mut self, ptr: *mut u8, _layout: Layout) {
+        self.deallocate(ptr);
+    }
+
+    pub fn try_deallocate_local(&self, ptr: *mut u8, thread_heap: &mut ThreadLocalHeap) -> bool {
+        if ptr.is_null() {
+            return true;
+        }
+
+        if self.mesh_epoch.load(Ordering::Acquire) & 1 != 0 {
+            return false;
+        }
+
+        let Some(resolved) = self.resolve_pointer(ptr) else {
+            return true;
+        };
+        let id = resolved.owner_id;
+        let Some(heap) = self.pool.get(id) else {
+            return true;
+        };
+        if heap.is_large_alloc()
+            || heap.current_thread() != thread_heap.thread_id()
+            || heap.is_meshed()
+            || !heap.contains_ptr(self.arena.base_ptr() as usize, ptr)
+        {
+            return false;
+        }
+
+        let class = heap.size_class();
+        let slot = resolved.slot;
+        let state = thread_heap.class_mut(class);
+        let Some(attached_idx) = state.find_attached(id) else {
+            return false;
+        };
+        if state.shuffle.is_full() {
+            return false;
+        }
+
+        let cached = state.shuffle.count_entries_for_offset(attached_idx as u16);
+        if cached + 1 >= heap.max_count() as usize {
+            return false;
+        }
+
+        state
+            .shuffle
+            .push(ShuffleEntry::new(attached_idx as u16, slot as u16));
+        self.stats.record_small_deallocation();
+        true
+    }
+
+    /// # Safety
+    ///
+    /// `ptr` must have been allocated by this allocator with `layout`, and must not be used
+    /// after this call if a new allocation is returned.
+    pub unsafe fn reallocate(
+        &mut self,
+        ptr: *mut u8,
+        layout: Layout,
+        new_size: usize,
+    ) -> Option<*mut u8> {
+        let thread_heap = unsafe { &mut *self.bootstrap_thread };
+        unsafe { self.reallocate_with_thread(ptr, layout, new_size, thread_heap) }
+    }
+
+    /// # Safety
+    ///
+    /// `ptr` must have been allocated by this allocator with `layout`, and must not be used
+    /// after this call if a new allocation is returned.
+    pub unsafe fn reallocate_with_thread(
+        &mut self,
+        ptr: *mut u8,
+        layout: Layout,
+        new_size: usize,
+        thread_heap: &mut ThreadLocalHeap,
+    ) -> Option<*mut u8> {
+        if ptr.is_null() {
+            return self.allocate_layout_with_thread(
+                thread_heap,
+                Layout::from_size_align(new_size.max(1), layout.align()).ok()?,
+            );
+        }
+        if new_size == 0 {
+            self.deallocate_with_thread(ptr, thread_heap);
+            return None;
+        }
+
+        let new_layout = Layout::from_size_align(new_size, layout.align()).ok()?;
+        let new_ptr = self.allocate_layout_with_thread(thread_heap, new_layout)?;
+        unsafe {
+            copy_nonoverlapping(ptr, new_ptr, layout.size().min(new_size));
+        }
+        self.deallocate_with_thread(ptr, thread_heap);
+        Some(new_ptr)
+    }
+
+    pub fn compact(&mut self) -> usize {
+        let thread_heap = unsafe { &mut *self.bootstrap_thread };
+        self.compact_with_thread(thread_heap)
+    }
+
+    pub fn compact_with_thread(&mut self, thread_heap: &mut ThreadLocalHeap) -> usize {
+        let _epoch_guard = MeshingEpochGuard::new(core::ptr::addr_of!(self.mesh_epoch));
+        self.stats.record_compact_call();
+        self.shutdown_thread(thread_heap);
+
+        let mut meshes = 0usize;
+        for class_idx in 1..NUM_SIZE_CLASSES {
+            meshes += self.mesh_class_candidates(class_idx as u8);
+        }
+
+        meshes
+    }
+
+    pub fn try_allocate_small_local(
+        &self,
+        thread_heap: &mut ThreadLocalHeap,
+        class: u8,
+    ) -> Option<*mut u8> {
+        if self.mesh_epoch.load(Ordering::Acquire) & 1 != 0 {
+            return None;
+        }
+        if thread_heap.class(class).shuffle.is_exhausted() && !self.local_refill(thread_heap, class)
+        {
+            return None;
+        }
+
+        let state = thread_heap.class_mut(class);
+        let entry = state.shuffle.pop()?;
+        let heap = state.heap_at(entry.miniheap_offset as usize)?;
+        self.stats.record_small_allocation();
+        Some(heap.ptr_from_offset(self.arena.base_ptr() as usize, entry.slot_index as usize))
+    }
+
+    fn allocate_small_with_thread(
+        &mut self,
+        thread_heap: &mut ThreadLocalHeap,
+        class: u8,
+    ) -> Option<*mut u8> {
+        self.global_refill(thread_heap, class)?;
+        self.try_allocate_small_local(thread_heap, class)
+    }
+
+    fn allocate_large(&mut self, size: usize) -> Option<*mut u8> {
+        self.allocate_large_aligned(size, 1)
+    }
+
+    fn allocate_large_aligned(&mut self, size: usize, align: usize) -> Option<*mut u8> {
+        let page_align = page_alignment_for(align)?;
+        let (_, span) = self.arena.allocate_bytes(size, page_align)?;
+        let (id, heap) = {
+            let _guard = self.pool_lock.lock();
+            self.pool.allocate(span, 1, size)?
+        };
+        self.arena.track_miniheap(span, id);
+        self.stats.record_large_allocation();
+        heap.malloc_at(self.arena.base_ptr() as usize, 0)
+    }
+
+    fn local_refill(&self, thread_heap: &mut ThreadLocalHeap, class: u8) -> bool {
+        let state = thread_heap.class_mut(class);
+        let count = state.attached_len as usize;
+        if count == 0 {
+            return false;
+        }
+
+        let mut scanned = 0usize;
+        while scanned < count && state.shuffle.is_exhausted() {
+            let idx = (state.attached_cursor as usize) % count;
+            state.attached_cursor = ((idx + 1) % count) as u8;
+            let heap_ptr = state.attached_heaps[idx];
+            if !heap_ptr.is_null() {
+                let heap = unsafe { &*heap_ptr };
+                if !heap.is_full() {
+                    let _ = state.shuffle.refill_from_heap(idx as u16, heap);
+                }
+            }
+            scanned += 1;
+        }
+
+        !state.shuffle.is_exhausted()
+    }
+
+    fn global_refill(&mut self, thread_heap: &mut ThreadLocalHeap, class: u8) -> Option<()> {
+        self.release_class_attached(thread_heap, class);
+
+        let object_size = byte_size_for_class(class);
+        let object_count = miniheap_object_count(object_size);
+        let page_count = page_count(object_size * object_count) as u32;
+        let thread_id = thread_heap.thread_id();
+
+        let mut bytes_free = self.attach_reusable_heaps(thread_heap, class);
+        while bytes_free < MINIHEAP_REFILL_GOAL_SIZE && !thread_heap.class(class).attached_full() {
+            let (_, span) = self.arena.page_alloc(page_count, 1)?;
+            let (id, heap) = {
+                let _guard = self.pool_lock.lock();
+                self.pool.allocate(span, object_count as u16, object_size)?
+            };
+            self.arena.track_miniheap(span, id);
+
+            let slot = thread_heap
+                .class_mut(class)
+                .push_attached(id, heap as *const MiniHeap)?;
+            heap.set_attached(thread_id);
+            heap.set_shuffle_vector_offset(slot);
+            bytes_free += heap.bytes_free();
+        }
+
+        thread_heap.class_mut(class).shuffle.clear();
+        if self.local_refill(thread_heap, class) {
+            Some(())
+        } else {
+            None
+        }
+    }
+
+    fn attach_reusable_heaps(&mut self, thread_heap: &mut ThreadLocalHeap, class: u8) -> usize {
+        let mut bytes_free = 0usize;
+        let len = self.pool.len();
+        let thread_id = thread_heap.thread_id();
+
+        let mut id_val = 1u32;
+        while id_val <= len
+            && bytes_free < MINIHEAP_REFILL_GOAL_SIZE
+            && !thread_heap.class(class).attached_full()
+        {
+            let id = MiniHeapId::new(id_val);
+            if let Some(heap) = self.pool.get(id)
+                && heap.size_class() == class
+                && !heap.is_attached()
+                && !heap.is_full()
+                && !heap.is_meshed()
+                && let Some(slot) = thread_heap
+                    .class_mut(class)
+                    .push_attached(id, heap as *const MiniHeap)
+            {
+                heap.set_attached(thread_id);
+                heap.set_shuffle_vector_offset(slot);
+                bytes_free += heap.bytes_free();
+            }
+            id_val += 1;
+        }
+
+        bytes_free
+    }
+
+    fn release_all_attached(&mut self, thread_heap: &mut ThreadLocalHeap) {
+        for class in 1..NUM_SIZE_CLASSES as u8 {
+            self.release_class_attached(thread_heap, class);
+        }
+    }
+
+    pub fn shutdown_thread(&mut self, thread_heap: &mut ThreadLocalHeap) {
+        self.release_all_attached(thread_heap);
+    }
+
+    fn reclaim_empty_detached_heap(&mut self, id: MiniHeapId) {
+        let Some(heap) = self.pool.get(id) else {
+            return;
+        };
+        if !heap.is_empty() || heap.is_meshed() || heap.is_attached() || heap.has_meshed_partner() {
+            return;
+        }
+
+        let span = heap.span();
+        self.arena.clear_miniheap(span);
+        self.arena.release_span(span);
+        let _ = {
+            let _guard = self.pool_lock.lock();
+            self.pool.release(id)
+        };
+    }
+
+    fn release_class_attached(&mut self, thread_heap: &mut ThreadLocalHeap, class: u8) {
+        let mut released_ids = [MiniHeapId::new(0); MAX_ATTACHED_MINIHEAPS_PER_CLASS];
+        let released_len;
+        {
+            let state = thread_heap.class_mut(class);
+            for entry in state.shuffle.active_entries() {
+                let attached_idx = entry.miniheap_offset as usize;
+                if attached_idx >= state.attached_len as usize {
+                    continue;
+                }
+
+                if let Some(heap) = state.heap_at(attached_idx) {
+                    let _ = heap.free_offset(entry.slot_index as usize);
+                }
+            }
+
+            released_len = state.attached_len as usize;
+            for (idx, id) in released_ids.iter_mut().enumerate().take(released_len) {
+                *id = state.attached_ids[idx];
+                if let Some(heap) = state.heap_at(idx) {
+                    heap.unset_attached();
+                }
+            }
+            state.clear_attached();
+        }
+
+        for id in released_ids.into_iter().take(released_len) {
+            if id != MiniHeapId::new(0) {
+                self.reclaim_empty_detached_heap(id);
+            }
+        }
+    }
+
+    fn heap_is_compaction_candidate(&self, class: u8, heap: &MiniHeap) -> bool {
+        heap.size_class() == class
+            && !heap.is_attached()
+            && !heap.is_full()
+            && !heap.is_meshed()
+            && heap.object_size() < page_size()
+            && is_below_partial_threshold(heap.in_use_count(), heap.max_count() as u32)
+    }
+
+    fn mesh_pair(&mut self, dst_id: MiniHeapId, src_id: MiniHeapId) -> raw_sys::Result<()> {
+        let dst = self.pool.get(dst_id).expect("valid dst id");
+        let src = self.pool.get(src_id).expect("valid src id");
+        let span_size = dst.span_size();
+        let arena_base = self.arena.base_ptr() as usize;
+        let object_size = dst.object_size();
+        let src_snapshot = src.bitmap().snapshot();
+        let keep = dst.ptr_from_offset(arena_base, 0);
+        let remove = src.ptr_from_offset(arena_base, 0);
+        let barrier = ActiveMeshGuard::begin(remove, span_size)?;
+        let scratch = match unsafe { self.arena.begin_mesh(remove, span_size) } {
+            Ok(scratch) => scratch,
+            Err(error) => {
+                barrier.finish();
+                return Err(error);
+            }
+        };
+
+        for slot in src_snapshot.iter_set_bits() {
+            let src_ptr = unsafe { scratch.add(slot * object_size) };
+            let dst_ptr = dst.ptr_from_offset(arena_base, slot);
+            unsafe {
+                copy_nonoverlapping(src_ptr, dst_ptr, object_size);
+            }
+            let _ = dst.bitmap().try_set(slot);
+            let _ = src.free_offset(slot);
+        }
+
+        let previous_family_head = dst.next_meshed();
+        src.track_meshed_span(previous_family_head);
+        dst.track_meshed_span(src_id);
+        src.set_meshed();
+
+        if let Err(error) = unsafe { self.arena.finalize_mesh(keep, remove, scratch, span_size) } {
+            let _ = unsafe { self.arena.abort_mesh(remove, scratch, span_size) };
+            barrier.finish();
+            return Err(error);
+        }
+        barrier.finish();
+        self.arena.free_phys(remove, span_size)?;
+        self.stats.record_mesh(dst.span().length, span_size);
+        Ok(())
+    }
+
+    fn mesh_class_candidates(&mut self, class: u8) -> usize {
+        let mut candidate_len = self.collect_compaction_candidates(class);
+        if candidate_len < 2 {
+            return 0;
+        }
+        self.shuffle_compaction_candidates(candidate_len);
+
+        let mut meshes = 0usize;
+        while candidate_len > 1 {
+            let left_index = candidate_len - 1;
+            let left_id = unsafe { *self.compaction_candidates.add(left_index) };
+            candidate_len -= 1;
+
+            let mut match_index = 0usize;
+            while match_index < candidate_len {
+                let right_id = unsafe { *self.compaction_candidates.add(match_index) };
+                let mesh_result = if let (Some(left), Some(right)) =
+                    (self.pool.get(left_id), self.pool.get(right_id))
+                {
+                    if self.heap_is_compaction_candidate(class, left)
+                        && self.heap_is_compaction_candidate(class, right)
+                        && bitmaps_meshable(left.bitmap(), right.bitmap())
+                    {
+                        if left.has_meshed_partner() && !right.has_meshed_partner() {
+                            self.mesh_pair(left_id, right_id)
+                        } else if right.has_meshed_partner() && !left.has_meshed_partner() {
+                            self.mesh_pair(right_id, left_id)
+                        } else if !right.has_meshed_partner() {
+                            self.mesh_pair(left_id, right_id)
+                        } else if !left.has_meshed_partner() {
+                            self.mesh_pair(right_id, left_id)
+                        } else {
+                            Err(raw_sys::Error(raw_sys::EAGAIN))
+                        }
+                    } else {
+                        Err(raw_sys::Error(raw_sys::EAGAIN))
+                    }
+                } else {
+                    Err(raw_sys::Error(raw_sys::EAGAIN))
+                };
+
+                if mesh_result.is_ok() {
+                    meshes += 1;
+                    if match_index != candidate_len - 1 {
+                        unsafe {
+                            let replacement = *self.compaction_candidates.add(candidate_len - 1);
+                            self.compaction_candidates
+                                .add(match_index)
+                                .write(replacement);
+                        }
+                    }
+                    candidate_len -= 1;
+                    break;
+                }
+                match_index += 1;
+            }
+        }
+
+        meshes
+    }
+
+    fn collect_compaction_candidates(&mut self, class: u8) -> usize {
+        let len = self.pool.len();
+        let mut candidate_len = 0usize;
+        let mut id_val = 1u32;
+        while id_val <= len {
+            let id = MiniHeapId::new(id_val);
+            if let Some(heap) = self.pool.get(id)
+                && self.heap_is_compaction_candidate(class, heap)
+            {
+                unsafe {
+                    self.compaction_candidates.add(candidate_len).write(id);
+                }
+                candidate_len += 1;
+            }
+            id_val += 1;
+        }
+        candidate_len
+    }
+
+    fn shuffle_compaction_candidates(&mut self, len: usize) {
+        if len <= 1 {
+            return;
+        }
+
+        let mut index = len - 1;
+        while index > 0 {
+            let swap_index = self.meshing_rng.in_range(0, index);
+            unsafe {
+                let left = *self.compaction_candidates.add(index);
+                let right = *self.compaction_candidates.add(swap_index);
+                self.compaction_candidates.add(index).write(right);
+                self.compaction_candidates.add(swap_index).write(left);
+            }
+            index -= 1;
+        }
+    }
+
+    fn resolve_pointer(&self, ptr: *mut u8) -> Option<ResolvedPtr> {
+        loop {
+            let start_epoch = self.mesh_epoch.load(Ordering::Acquire);
+            if start_epoch & 1 != 0 {
+                futex_wait_for_value(&self.mesh_epoch, start_epoch);
+                continue;
+            }
+
+            let owner_id = self.arena.miniheap_id_for_ptr(ptr)?;
+            let slot = self.resolve_family_slot(owner_id, ptr)?;
+            let end_epoch = self.mesh_epoch.load(Ordering::Acquire);
+            if start_epoch == end_epoch {
+                return Some(ResolvedPtr { owner_id, slot });
+            }
+        }
+    }
+
+    fn resolve_family_slot(&self, owner_id: MiniHeapId, ptr: *mut u8) -> Option<usize> {
+        let arena_base = self.arena.base_ptr() as usize;
+        let owner = self.pool.get(owner_id)?;
+        if owner.contains_ptr(arena_base, ptr) {
+            return Some(owner.slot_for_ptr(arena_base, ptr));
+        }
+
+        let mut current = owner.next_meshed();
+        while current.has_value() {
+            let heap = self.pool.get(current)?;
+            if heap.contains_ptr(arena_base, ptr) {
+                return Some(heap.slot_for_ptr(arena_base, ptr));
+            }
+            current = heap.next_meshed();
+        }
+
+        None
+    }
+}
+
+fn miniheap_object_count(object_size: usize) -> usize {
+    let bitmap_limit = runtime_slots_per_span();
+    (page_size() / object_size)
+        .max(MIN_SHUFFLE_VECTOR_LENGTH)
+        .min(bitmap_limit)
+}
+
+#[inline(always)]
+fn round_up_to_alignment(size: usize, align: usize) -> Option<usize> {
+    debug_assert!(align.is_power_of_two());
+    let mask = align - 1;
+    size.checked_add(mask).map(|value| value & !mask)
+}
+
+#[inline(always)]
+fn page_alignment_for(align: usize) -> Option<u32> {
+    let page = page_size();
+    if align <= page {
+        return Some(1);
+    }
+
+    let pages = align / page;
+    if pages * page != align {
+        return None;
+    }
+
+    u32::try_from(pages).ok()
+}
+
+struct MeshingEpochGuard {
+    epoch: *const AtomicU32,
+}
+
+impl MeshingEpochGuard {
+    fn new(epoch: *const AtomicU32) -> MeshingEpochGuard {
+        let epoch_ref = unsafe { &*epoch };
+        let previous = epoch_ref.fetch_add(1, Ordering::AcqRel);
+        debug_assert_eq!(previous & 1, 0);
+        MeshingEpochGuard { epoch }
+    }
+}
+
+impl Drop for MeshingEpochGuard {
+    fn drop(&mut self) {
+        let epoch = unsafe { &*self.epoch };
+        let previous = epoch.fetch_add(1, Ordering::AcqRel);
+        debug_assert_eq!(previous & 1, 1);
+        futex_wake_all(epoch);
+    }
+}
+
+impl Drop for MeshAllocator {
+    fn drop(&mut self) {
+        unsafe {
+            core::ptr::drop_in_place(self.bootstrap_thread);
+            let _ = platform::munmap(
+                self.bootstrap_thread.cast::<u8>(),
+                size_of::<ThreadLocalHeap>(),
+            );
+            let _ = platform::munmap(
+                self.compaction_candidates.cast::<u8>(),
+                self.pool.capacity() as usize * size_of::<MiniHeapId>(),
+            );
+        }
+    }
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/arena.rs

@@ -0,0 +1,428 @@
+use core::cell::UnsafeCell;
+use core::mem::size_of;
+use core::ptr::null_mut;
+use core::sync::atomic::{AtomicU32, Ordering};
+
+use super::constants::DEFAULT_ARENA_SIZE;
+use super::miniheap::MiniHeapId;
+use super::page::{PageConfig, page_count, round_up_to_page};
+use super::platform;
+use super::raw_sys;
+use super::span::Span;
+use super::sync::FutexMutex;
+
+const MAX_FREE_SPANS: usize = 4096;
+
+#[derive(Debug)]
+pub struct Arena {
+    config: PageConfig,
+    arena_size: usize,
+    page_count: u32,
+    fd: i32,
+    base: *mut u8,
+    owners: *mut AtomicU32,
+    next_page: AtomicU32,
+    free_spans: *mut Span,
+    free_span_count: UnsafeCell<u32>,
+    free_span_lock: FutexMutex,
+}
+
+impl Arena {
+    #[inline]
+    pub fn new() -> raw_sys::Result<Self> {
+        Self::with_size(DEFAULT_ARENA_SIZE)
+    }
+
+    #[inline]
+    pub fn with_size(arena_size: usize) -> raw_sys::Result<Self> {
+        let config = PageConfig::get();
+        assert!(arena_size > 0);
+        assert_eq!(arena_size % config.size(), 0);
+
+        let page_count = page_count(arena_size) as u32;
+        let fd = platform::memfd_create(c"rust-mesh-alloc".as_ptr().cast(), raw_sys::MFD_CLOEXEC)?;
+        platform::ftruncate(fd, arena_size as u64)?;
+
+        let base = unsafe {
+            platform::mmap(
+                null_mut(),
+                arena_size,
+                raw_sys::PROT_READ | raw_sys::PROT_WRITE,
+                raw_sys::MAP_SHARED,
+                fd,
+                0,
+            )?
+        };
+
+        let owner_bytes = page_count as usize * size_of::<AtomicU32>();
+        let owners = unsafe {
+            platform::map_anonymous(owner_bytes, raw_sys::PROT_READ | raw_sys::PROT_WRITE)?
+                as *mut AtomicU32
+        };
+        let free_span_bytes = MAX_FREE_SPANS * size_of::<Span>();
+        let free_spans = unsafe {
+            platform::map_anonymous(free_span_bytes, raw_sys::PROT_READ | raw_sys::PROT_WRITE)?
+                as *mut Span
+        };
+
+        Ok(Self {
+            config,
+            arena_size,
+            page_count,
+            fd,
+            base,
+            owners,
+            next_page: AtomicU32::new(0),
+            free_spans,
+            free_span_count: UnsafeCell::new(0),
+            free_span_lock: FutexMutex::new(),
+        })
+    }
+
+    #[inline(always)]
+    pub const fn config(&self) -> PageConfig {
+        self.config
+    }
+
+    #[inline(always)]
+    pub const fn arena_size(&self) -> usize {
+        self.arena_size
+    }
+
+    #[inline(always)]
+    pub const fn base_ptr(&self) -> *mut u8 {
+        self.base
+    }
+
+    #[inline(always)]
+    pub fn contains(&self, ptr: *const u8) -> bool {
+        let start = self.base as usize;
+        let end = start + self.arena_size;
+        let value = ptr as usize;
+        start <= value && value < end
+    }
+
+    #[inline]
+    pub fn reserve_pages(&self, page_count: u32, page_alignment: u32) -> Option<Span> {
+        assert!(page_count > 0);
+        assert!(page_alignment > 0);
+        let alignment = page_alignment.next_power_of_two();
+
+        if let Some(span) = self.take_free_span(page_count, alignment) {
+            return Some(span);
+        }
+
+        loop {
+            let current = self.next_page.load(Ordering::Acquire);
+            let aligned = align_up_u32(current, alignment);
+            let end = aligned.checked_add(page_count)?;
+            if end > self.page_count {
+                return None;
+            }
+
+            match self
+                .next_page
+                .compare_exchange(current, end, Ordering::AcqRel, Ordering::Acquire)
+            {
+                Ok(_) => return Some(Span::new(aligned, page_count)),
+                Err(_) => continue,
+            }
+        }
+    }
+
+    #[inline]
+    pub fn page_alloc(&self, page_count: u32, page_alignment: u32) -> Option<(*mut u8, Span)> {
+        let span = self.reserve_pages(page_count, page_alignment)?;
+        Some((self.ptr_from_offset(span.offset as usize), span))
+    }
+
+    #[inline]
+    pub fn allocate_bytes(&self, size: usize, page_alignment: u32) -> Option<(*mut u8, Span)> {
+        let pages = page_count(size) as u32;
+        self.page_alloc(pages, page_alignment)
+    }
+
+    #[inline(always)]
+    pub fn reserved_pages(&self) -> u32 {
+        self.next_page.load(Ordering::Acquire)
+    }
+
+    pub fn reusable_span_stats(&self) -> (u32, u32) {
+        let _guard = self.free_span_lock.lock();
+        let count = unsafe { *self.free_span_count.get() };
+        let mut pages = 0u32;
+        let mut index = 0usize;
+        while index < count as usize {
+            let span = unsafe { *self.free_spans.add(index) };
+            pages += span.length;
+            index += 1;
+        }
+        (count, pages)
+    }
+
+    #[inline]
+    pub fn track_miniheap(&self, span: Span, id: MiniHeapId) {
+        for page in 0..span.length {
+            self.owner_at_offset(span.offset + page)
+                .store(id.value(), Ordering::Release);
+        }
+    }
+
+    #[inline]
+    pub fn clear_miniheap(&self, span: Span) {
+        for page in 0..span.length {
+            self.owner_at_offset(span.offset + page)
+                .store(0, Ordering::Release);
+        }
+    }
+
+    #[inline]
+    pub fn release_span(&self, span: Span) {
+        if span.empty() {
+            return;
+        }
+
+        let _guard = self.free_span_lock.lock();
+        let count = unsafe { &mut *self.free_span_count.get() };
+        let mut merged = span;
+        let mut index = 0usize;
+        while index < *count as usize {
+            let other = unsafe { *self.free_spans.add(index) };
+            if other.offset + other.length == merged.offset {
+                merged = Span::new(other.offset, other.length + merged.length);
+                self.remove_free_span_at(index, count);
+                continue;
+            }
+            if merged.offset + merged.length == other.offset {
+                merged = Span::new(merged.offset, merged.length + other.length);
+                self.remove_free_span_at(index, count);
+                continue;
+            }
+            index += 1;
+        }
+
+        self.push_free_span(merged, count);
+    }
+
+    #[inline]
+    pub fn miniheap_id_for_ptr(&self, ptr: *const u8) -> Option<MiniHeapId> {
+        if !self.contains(ptr) {
+            return None;
+        }
+        let off = self.offset_for(ptr);
+        let value = self.owner_at_offset(off).load(Ordering::Acquire);
+        if value == 0 {
+            None
+        } else {
+            Some(MiniHeapId::new(value))
+        }
+    }
+
+    /// # Safety
+    ///
+    /// `remove..remove+size` must describe a valid, page-aligned mapping within this arena.
+    /// The returned alias is a private scratch mapping of the old source backing. The original
+    /// `remove` range is protected with `PROT_NONE` and must be restored or remapped by the
+    /// caller before any blocked mutators are allowed to resume.
+    #[inline]
+    pub unsafe fn begin_mesh(&self, remove: *mut u8, size: usize) -> raw_sys::Result<*mut u8> {
+        let rounded = round_up_to_page(size);
+        let remove_off = self.offset_for(remove);
+        unsafe {
+            platform::mprotect(remove, rounded, raw_sys::PROT_NONE)?;
+            platform::mmap(
+                core::ptr::null_mut(),
+                rounded,
+                raw_sys::PROT_READ,
+                raw_sys::MAP_SHARED,
+                self.fd,
+                (remove_off as usize * self.config.size()) as u64,
+            )
+        }
+    }
+
+    /// # Safety
+    ///
+    /// Restores the source mapping to its original backing after a failed mesh attempt.
+    #[inline]
+    pub unsafe fn abort_mesh(
+        &self,
+        remove: *mut u8,
+        scratch: *mut u8,
+        size: usize,
+    ) -> raw_sys::Result<()> {
+        let rounded = round_up_to_page(size);
+        let remove_off = self.offset_for(remove);
+        unsafe {
+            if !scratch.is_null() {
+                platform::munmap(scratch, rounded)?;
+            }
+            platform::mmap(
+                remove,
+                rounded,
+                raw_sys::PROT_READ | raw_sys::PROT_WRITE,
+                raw_sys::MAP_SHARED | raw_sys::MAP_FIXED,
+                self.fd,
+                (remove_off as usize * self.config.size()) as u64,
+            )?;
+        }
+        Ok(())
+    }
+
+    /// # Safety
+    ///
+    /// `keep` and `remove` must each point to valid page-aligned ranges of at least `size`
+    /// bytes within this arena. The caller must ensure that aliasing these ranges is valid for
+    /// the current allocator state and that any required object copying has already completed.
+    #[inline]
+    pub unsafe fn finalize_mesh(
+        &self,
+        keep: *mut u8,
+        remove: *mut u8,
+        scratch: *mut u8,
+        size: usize,
+    ) -> raw_sys::Result<()> {
+        let rounded = round_up_to_page(size);
+        let keep_off = self.offset_for(keep);
+        let remove_off = self.offset_for(remove);
+        let pages = page_count(rounded);
+
+        unsafe {
+            platform::mmap(
+                remove,
+                rounded,
+                raw_sys::PROT_READ | raw_sys::PROT_WRITE,
+                raw_sys::MAP_SHARED | raw_sys::MAP_FIXED,
+                self.fd,
+                (keep_off as usize * self.config.size()) as u64,
+            )?;
+            if !scratch.is_null() {
+                platform::munmap(scratch, rounded)?;
+            }
+        }
+
+        let keep_id = self.owner_at_offset(keep_off).load(Ordering::Acquire);
+        for page in 0..pages {
+            self.owner_at_offset(remove_off + page as u32)
+                .store(keep_id, Ordering::Release);
+        }
+
+        Ok(())
+    }
+
+    #[inline]
+    pub fn free_phys(&self, ptr: *mut u8, size: usize) -> raw_sys::Result<()> {
+        let rounded = round_up_to_page(size);
+        let offset = (ptr as usize).wrapping_sub(self.base as usize);
+        platform::fallocate(
+            self.fd,
+            raw_sys::FALLOC_FL_PUNCH_HOLE | raw_sys::FALLOC_FL_KEEP_SIZE,
+            offset as u64,
+            rounded as u64,
+        )
+    }
+
+    #[inline]
+    pub fn reset_identity_mapping(&self, span: Span) -> raw_sys::Result<()> {
+        let ptr = self.ptr_from_offset(span.offset as usize);
+        unsafe {
+            platform::mmap(
+                ptr,
+                span.byte_length_for_page_size(self.config.size()),
+                raw_sys::PROT_READ | raw_sys::PROT_WRITE,
+                raw_sys::MAP_SHARED | raw_sys::MAP_FIXED,
+                self.fd,
+                (span.offset as usize * self.config.size()) as u64,
+            )?;
+        }
+        Ok(())
+    }
+
+    #[inline(always)]
+    pub fn offset_for(&self, ptr: *const u8) -> u32 {
+        let delta = (ptr as usize).wrapping_sub(self.base as usize);
+        (delta >> self.config.shift()) as u32
+    }
+
+    #[inline(always)]
+    pub fn ptr_from_offset(&self, offset: usize) -> *mut u8 {
+        unsafe { self.base.add(offset << self.config.shift()) }
+    }
+
+    #[inline(always)]
+    fn owner_at_offset(&self, offset: u32) -> &AtomicU32 {
+        assert!(offset < self.page_count);
+        unsafe { &*self.owners.add(offset as usize) }
+    }
+}
+
+impl Drop for Arena {
+    fn drop(&mut self) {
+        let owner_bytes = self.page_count as usize * size_of::<AtomicU32>();
+        let free_span_bytes = MAX_FREE_SPANS * size_of::<Span>();
+        unsafe {
+            let _ = platform::munmap(self.free_spans as *mut u8, free_span_bytes);
+            let _ = platform::munmap(self.owners as *mut u8, owner_bytes);
+            let _ = platform::munmap(self.base, self.arena_size);
+        }
+        let _ = platform::close(self.fd);
+    }
+}
+
+#[inline(always)]
+fn align_up_u32(value: u32, alignment: u32) -> u32 {
+    debug_assert!(alignment.is_power_of_two());
+    (value + alignment - 1) & !(alignment - 1)
+}
+
+impl Arena {
+    fn take_free_span(&self, page_count: u32, alignment: u32) -> Option<Span> {
+        let _guard = self.free_span_lock.lock();
+        let count = unsafe { &mut *self.free_span_count.get() };
+        let mut index = 0usize;
+        while index < *count as usize {
+            let span = unsafe { *self.free_spans.add(index) };
+            let aligned = align_up_u32(span.offset, alignment);
+            let prefix = aligned.checked_sub(span.offset)?;
+            let total = prefix.checked_add(page_count)?;
+            if total <= span.length {
+                self.remove_free_span_at(index, count);
+
+                if prefix > 0 {
+                    self.push_free_span(Span::new(span.offset, prefix), count);
+                }
+
+                let suffix_offset = aligned + page_count;
+                let suffix_length = span.length - total;
+                if suffix_length > 0 {
+                    self.push_free_span(Span::new(suffix_offset, suffix_length), count);
+                }
+
+                return Some(Span::new(aligned, page_count));
+            }
+            index += 1;
+        }
+
+        None
+    }
+
+    fn push_free_span(&self, span: Span, count: &mut u32) {
+        assert!((*count as usize) < MAX_FREE_SPANS);
+        unsafe {
+            self.free_spans.add(*count as usize).write(span);
+        }
+        *count += 1;
+    }
+
+    fn remove_free_span_at(&self, index: usize, count: &mut u32) {
+        debug_assert!(index < *count as usize);
+        let last = *count as usize - 1;
+        if index != last {
+            let replacement = unsafe { *self.free_spans.add(last) };
+            unsafe {
+                self.free_spans.add(index).write(replacement);
+            }
+        }
+        *count -= 1;
+    }
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/bitmap.rs

@@ -0,0 +1,236 @@
+use core::sync::atomic::{AtomicUsize, Ordering};
+
+use super::constants::MAX_OBJECT_SLOTS_PER_SPAN;
+
+const USIZE_BITS: usize = usize::BITS as usize;
+const BITMAP_WORDS: usize = MAX_OBJECT_SLOTS_PER_SPAN / USIZE_BITS;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct RelaxedBitmap {
+    bit_count: u16,
+    words: [usize; BITMAP_WORDS],
+}
+
+impl RelaxedBitmap {
+    #[inline]
+    pub fn new(bit_count: usize) -> Self {
+        assert!(bit_count <= MAX_OBJECT_SLOTS_PER_SPAN);
+        Self {
+            bit_count: bit_count as u16,
+            words: [0; BITMAP_WORDS],
+        }
+    }
+
+    #[inline(always)]
+    pub const fn bit_count(&self) -> usize {
+        self.bit_count as usize
+    }
+
+    #[inline(always)]
+    pub fn words(&self) -> &[usize; BITMAP_WORDS] {
+        &self.words
+    }
+
+    #[inline(always)]
+    pub fn words_mut(&mut self) -> &mut [usize; BITMAP_WORDS] {
+        &mut self.words
+    }
+
+    #[inline]
+    pub fn clear(&mut self) {
+        self.words = [0; BITMAP_WORDS];
+    }
+
+    #[inline]
+    pub fn set_all(&mut self) {
+        self.words = [usize::MAX; BITMAP_WORDS];
+        self.mask_unused_bits();
+    }
+
+    #[inline]
+    pub fn invert_masked(&mut self) {
+        for word in &mut self.words {
+            *word = !*word;
+        }
+        self.mask_unused_bits();
+    }
+
+    #[inline(always)]
+    pub fn try_set(&mut self, index: usize) -> bool {
+        let (word, mask) = word_and_mask(index);
+        let old = self.words[word];
+        self.words[word] = old | mask;
+        old & mask == 0
+    }
+
+    #[inline(always)]
+    pub fn unset(&mut self, index: usize) -> bool {
+        let (word, mask) = word_and_mask(index);
+        let old = self.words[word];
+        self.words[word] = old & !mask;
+        old & mask != 0
+    }
+
+    #[inline(always)]
+    pub fn is_set(&self, index: usize) -> bool {
+        let (word, mask) = word_and_mask(index);
+        self.words[word] & mask != 0
+    }
+
+    #[inline]
+    pub fn in_use_count(&self) -> u32 {
+        self.words.iter().map(|word| word.count_ones()).sum()
+    }
+
+    #[inline]
+    pub fn iter_set_bits(&self) -> BitIter {
+        BitIter::new(self.words, self.bit_count())
+    }
+
+    #[inline]
+    fn mask_unused_bits(&mut self) {
+        let valid_bits = self.bit_count();
+        if valid_bits == MAX_OBJECT_SLOTS_PER_SPAN {
+            return;
+        }
+
+        let used_words = valid_bits / USIZE_BITS;
+        let remainder = valid_bits % USIZE_BITS;
+        for word in self
+            .words
+            .iter_mut()
+            .skip(used_words + usize::from(remainder != 0))
+        {
+            *word = 0;
+        }
+        if remainder != 0 {
+            self.words[used_words] &= (1usize << remainder) - 1;
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct AtomicBitmap {
+    bit_count: u16,
+    words: [AtomicUsize; BITMAP_WORDS],
+}
+
+impl AtomicBitmap {
+    #[inline]
+    pub fn new(bit_count: usize) -> Self {
+        assert!(bit_count <= MAX_OBJECT_SLOTS_PER_SPAN);
+        Self {
+            bit_count: bit_count as u16,
+            words: [const { AtomicUsize::new(0) }; BITMAP_WORDS],
+        }
+    }
+
+    #[inline(always)]
+    pub const fn bit_count(&self) -> usize {
+        self.bit_count as usize
+    }
+
+    #[inline(always)]
+    pub fn try_set(&self, index: usize) -> bool {
+        let (word, mask) = word_and_mask(index);
+        let old = self.words[word].fetch_or(mask, Ordering::AcqRel);
+        old & mask == 0
+    }
+
+    #[inline(always)]
+    pub fn unset(&self, index: usize) -> bool {
+        let (word, mask) = word_and_mask(index);
+        let old = self.words[word].fetch_and(!mask, Ordering::AcqRel);
+        old & mask != 0
+    }
+
+    #[inline(always)]
+    pub fn is_set(&self, index: usize) -> bool {
+        let (word, mask) = word_and_mask(index);
+        self.words[word].load(Ordering::Acquire) & mask != 0
+    }
+
+    #[inline]
+    pub fn in_use_count(&self) -> u32 {
+        self.words
+            .iter()
+            .map(|word| word.load(Ordering::Acquire).count_ones())
+            .sum()
+    }
+
+    #[inline]
+    pub fn swap_words(&self, new_words: &[usize; BITMAP_WORDS]) -> [usize; BITMAP_WORDS] {
+        let mut old_words = [0usize; BITMAP_WORDS];
+        let mut i = 0;
+        while i < BITMAP_WORDS {
+            old_words[i] = self.words[i].swap(new_words[i], Ordering::AcqRel);
+            i += 1;
+        }
+        old_words
+    }
+
+    #[inline]
+    pub fn snapshot(&self) -> RelaxedBitmap {
+        let mut bitmap = RelaxedBitmap::new(self.bit_count());
+        let mut i = 0;
+        while i < BITMAP_WORDS {
+            bitmap.words_mut()[i] = self.words[i].load(Ordering::Acquire);
+            i += 1;
+        }
+        bitmap
+    }
+
+    #[inline]
+    pub fn take_free_bits(&self) -> RelaxedBitmap {
+        let mut all_used = RelaxedBitmap::new(self.bit_count());
+        all_used.set_all();
+        let previous = self.swap_words(all_used.words());
+
+        let mut free_bits = RelaxedBitmap::new(self.bit_count());
+        *free_bits.words_mut() = previous;
+        free_bits.invert_masked();
+        free_bits
+    }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct BitIter {
+    words: [usize; BITMAP_WORDS],
+    bit_count: usize,
+    next_index: usize,
+}
+
+impl BitIter {
+    #[inline(always)]
+    pub fn new(words: [usize; BITMAP_WORDS], bit_count: usize) -> Self {
+        Self {
+            words,
+            bit_count,
+            next_index: 0,
+        }
+    }
+}
+
+impl Iterator for BitIter {
+    type Item = usize;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        while self.next_index < self.bit_count {
+            let current = self.next_index;
+            self.next_index += 1;
+            let (word, mask) = word_and_mask(current);
+            if self.words[word] & mask != 0 {
+                return Some(current);
+            }
+        }
+        None
+    }
+}
+
+#[inline(always)]
+fn word_and_mask(index: usize) -> (usize, usize) {
+    debug_assert!(index < MAX_OBJECT_SLOTS_PER_SPAN);
+    let word = index / USIZE_BITS;
+    let bit = index % USIZE_BITS;
+    (word, 1usize << bit)
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/constants.rs

@@ -0,0 +1,19 @@
+pub const MIN_SUPPORTED_PAGE_SIZE: usize = 4096;
+pub const MAX_SUPPORTED_PAGE_SIZE: usize = 16384;
+pub const MIN_OBJECT_SIZE: usize = 16;
+pub const MAX_SMALL_ALLOCATION: usize = 16_384;
+pub const NUM_SIZE_CLASSES: usize = 25;
+pub const OCCUPANCY_CUTOFF_NUMERATOR: u32 = 4;
+pub const OCCUPANCY_CUTOFF_DENOMINATOR: u32 = 5;
+pub const MIN_SHUFFLE_VECTOR_LENGTH: usize = 8;
+pub const MAX_ATTACHED_MINIHEAPS_PER_CLASS: usize = 48;
+pub const MAX_OBJECT_SLOTS_PER_SPAN: usize = MAX_SUPPORTED_PAGE_SIZE / MIN_OBJECT_SIZE;
+pub const MAX_SHUFFLE_VECTOR_LENGTH: usize = MAX_OBJECT_SLOTS_PER_SPAN;
+pub const DEFAULT_ARENA_SIZE: usize = 64 * 1024 * 1024 * 1024;
+pub const MINIHEAP_REFILL_GOAL_SIZE: usize = 16 * 1024;
+
+#[inline(always)]
+pub const fn is_below_partial_threshold(in_use: u32, max_count: u32) -> bool {
+    in_use.saturating_mul(OCCUPANCY_CUTOFF_DENOMINATOR)
+        < max_count.saturating_mul(OCCUPANCY_CUTOFF_NUMERATOR)
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/fault.rs

@@ -0,0 +1,132 @@
+use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering};
+
+use super::raw_sys;
+use super::sync::{futex_wait_for_value, futex_wake_all};
+
+const INSTALL_UNINITIALIZED: u32 = 0;
+const INSTALL_READY: u32 = 1;
+const INSTALL_FAILED: u32 = 2;
+
+static INSTALL_STATE: AtomicU32 = AtomicU32::new(INSTALL_UNINITIALIZED);
+static ACTIVE_MESH_SEQ: AtomicU32 = AtomicU32::new(0);
+static ACTIVE_MESH_START: AtomicUsize = AtomicUsize::new(0);
+static ACTIVE_MESH_LEN: AtomicUsize = AtomicUsize::new(0);
+
+#[derive(Debug)]
+pub struct ActiveMeshGuard {
+    active: bool,
+}
+
+impl ActiveMeshGuard {
+    pub fn begin(start: *mut u8, len: usize) -> raw_sys::Result<Self> {
+        ensure_fault_mediation_installed()?;
+        loop {
+            let seq = ACTIVE_MESH_SEQ.load(Ordering::Acquire);
+            if seq & 1 == 0 {
+                break;
+            }
+            futex_wait_for_value(&ACTIVE_MESH_SEQ, seq);
+        }
+        ACTIVE_MESH_START.store(start as usize, Ordering::Release);
+        ACTIVE_MESH_LEN.store(len, Ordering::Release);
+        let previous = ACTIVE_MESH_SEQ.fetch_add(1, Ordering::AcqRel);
+        debug_assert_eq!(previous & 1, 0);
+        Ok(Self { active: true })
+    }
+
+    pub fn finish(mut self) {
+        self.release();
+    }
+
+    fn release(&mut self) {
+        if !self.active {
+            return;
+        }
+
+        ACTIVE_MESH_START.store(0, Ordering::Release);
+        ACTIVE_MESH_LEN.store(0, Ordering::Release);
+        let previous = ACTIVE_MESH_SEQ.fetch_add(1, Ordering::AcqRel);
+        debug_assert_eq!(previous & 1, 1);
+        futex_wake_all(&ACTIVE_MESH_SEQ);
+        self.active = false;
+    }
+}
+
+impl Drop for ActiveMeshGuard {
+    fn drop(&mut self) {
+        self.release();
+    }
+}
+
+pub fn ensure_fault_mediation_installed() -> raw_sys::Result<()> {
+    match INSTALL_STATE.compare_exchange(
+        INSTALL_UNINITIALIZED,
+        INSTALL_READY,
+        Ordering::AcqRel,
+        Ordering::Acquire,
+    ) {
+        Ok(_) | Err(INSTALL_READY) => Ok(()),
+        Err(_) => {
+            INSTALL_STATE.store(INSTALL_FAILED, Ordering::Release);
+            Err(raw_sys::Error(raw_sys::EAGAIN))
+        }
+    }
+}
+
+pub fn ok_to_proceed(ptr: *const u8) -> bool {
+    let address = ptr as usize;
+    let mut waited = false;
+    loop {
+        let seq = ACTIVE_MESH_SEQ.load(Ordering::Acquire);
+        if seq & 1 == 0 {
+            return waited;
+        }
+
+        let start = ACTIVE_MESH_START.load(Ordering::Acquire);
+        let len = ACTIVE_MESH_LEN.load(Ordering::Acquire);
+        let end = start.saturating_add(len);
+        if address < start || address >= end {
+            return waited;
+        }
+
+        waited = true;
+        futex_wait_for_value(&ACTIVE_MESH_SEQ, seq);
+    }
+}
+
+pub fn retry_on_efault<T, F>(ptr: *const u8, mut op: F) -> raw_sys::Result<T>
+where
+    F: FnMut() -> raw_sys::Result<T>,
+{
+    loop {
+        match op() {
+            Ok(value) => return Ok(value),
+            Err(error) if error.errno() == raw_sys::EFAULT && ok_to_proceed(ptr) => continue,
+            Err(error) => return Err(error),
+        }
+    }
+}
+
+pub fn retry_on_efault_ptrs<T, F>(ptrs: &[*const u8], mut op: F) -> raw_sys::Result<T>
+where
+    F: FnMut() -> raw_sys::Result<T>,
+{
+    loop {
+        match op() {
+            Ok(value) => return Ok(value),
+            Err(error) if error.errno() == raw_sys::EFAULT => {
+                let mut waited = false;
+                for &ptr in ptrs {
+                    if !ptr.is_null() {
+                        waited |= ok_to_proceed(ptr);
+                    }
+                }
+                if waited {
+                    continue;
+                }
+                return Err(error);
+            }
+            Err(error) => return Err(error),
+        }
+    }
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/global.rs

@@ -0,0 +1,453 @@
+use core::alloc::{GlobalAlloc, Layout};
+use core::cell::UnsafeCell;
+use core::mem::MaybeUninit;
+use core::ptr::{addr_of_mut, drop_in_place, null, null_mut};
+use core::sync::atomic::{AtomicU32, Ordering};
+
+use super::allocator::MeshAllocator;
+use super::constants::DEFAULT_ARENA_SIZE;
+use super::stats::{
+    CompactionAdvice, CompactionSkipReason, MeshStats, RuntimeCompactionPolicy,
+    RuntimeCompactionResult,
+};
+use super::sync::{FutexMutex, futex_wait_for_value, futex_wake_all};
+use super::thread_local_heap::ThreadLocalHeap;
+
+const INIT_UNINITIALIZED: u32 = 0;
+const INIT_IN_PROGRESS: u32 = 1;
+const INIT_READY: u32 = 2;
+const INIT_FAILED: u32 = 3;
+
+pub const DEFAULT_GLOBAL_MINIHEAP_CAPACITY: u32 = 4096;
+
+const TLS_UNINITIALIZED: u32 = 0;
+const TLS_READY: u32 = 1;
+const TLS_FAILED: u32 = 2;
+const SAFEPOINT_INACTIVE: u32 = 0;
+const SAFEPOINT_ACTIVE: u32 = 1;
+
+#[thread_local]
+static mut THREAD_HEAP_STATE: u32 = TLS_UNINITIALIZED;
+#[thread_local]
+static mut THREAD_HEAP: MaybeUninit<ThreadLocalHeap> = MaybeUninit::uninit();
+#[thread_local]
+static mut THREAD_SAFEPOINT_STATE: u32 = SAFEPOINT_INACTIVE;
+#[thread_local]
+static mut THREAD_HEAP_OWNER: *const GlobalMeshAllocator = null();
+
+#[derive(Debug)]
+pub struct GlobalMeshAllocator {
+    arena_size: usize,
+    miniheap_capacity: u32,
+    init_state: AtomicU32,
+    registered_threads: AtomicU32,
+    quiescent_threads: AtomicU32,
+    lock: FutexMutex,
+    allocator: UnsafeCell<MaybeUninit<MeshAllocator>>,
+}
+
+impl GlobalMeshAllocator {
+    pub const fn new(arena_size: usize, miniheap_capacity: u32) -> Self {
+        Self {
+            arena_size,
+            miniheap_capacity,
+            init_state: AtomicU32::new(INIT_UNINITIALIZED),
+            registered_threads: AtomicU32::new(0),
+            quiescent_threads: AtomicU32::new(0),
+            lock: FutexMutex::new(),
+            allocator: UnsafeCell::new(MaybeUninit::uninit()),
+        }
+    }
+
+    pub const fn with_default_config() -> Self {
+        Self::new(DEFAULT_ARENA_SIZE, DEFAULT_GLOBAL_MINIHEAP_CAPACITY)
+    }
+
+    pub fn init_thread(&self) -> bool {
+        self.thread_heap().is_some()
+    }
+
+    pub fn shutdown_thread(&self) {
+        unsafe {
+            let state_ptr = addr_of_mut!(THREAD_HEAP_STATE);
+            let heap_ptr = addr_of_mut!(THREAD_HEAP);
+            let safepoint_ptr = addr_of_mut!(THREAD_SAFEPOINT_STATE);
+            let owner_ptr = addr_of_mut!(THREAD_HEAP_OWNER);
+            if *safepoint_ptr == SAFEPOINT_ACTIVE {
+                self.quiescent_threads.fetch_sub(1, Ordering::AcqRel);
+                *safepoint_ptr = SAFEPOINT_INACTIVE;
+            }
+            if *state_ptr == TLS_READY {
+                let _ = self.with_existing_allocator_mut(|allocator| {
+                    allocator.shutdown_thread((*heap_ptr).assume_init_mut());
+                });
+                drop_in_place((*heap_ptr).as_mut_ptr());
+                self.registered_threads.fetch_sub(1, Ordering::AcqRel);
+            }
+            *state_ptr = TLS_UNINITIALIZED;
+            *owner_ptr = null();
+        }
+    }
+
+    /// Marks the current thread quiescent for cooperative compaction.
+    ///
+    /// This is **not** true concurrent compaction: arbitrary loads/stores through existing raw
+    /// pointers remain outside allocator control, so compaction is only safe once all registered
+    /// allocator threads have voluntarily entered this state.
+    pub fn enter_quiescent_compaction_state(&self) -> bool {
+        if self.thread_heap().is_none() {
+            return false;
+        }
+
+        unsafe {
+            let safepoint_ptr = addr_of_mut!(THREAD_SAFEPOINT_STATE);
+            if *safepoint_ptr == SAFEPOINT_ACTIVE {
+                return true;
+            }
+            *safepoint_ptr = SAFEPOINT_ACTIVE;
+        }
+        self.quiescent_threads.fetch_add(1, Ordering::AcqRel);
+        true
+    }
+
+    /// Leaves the cooperative quiescent compaction state for the current thread.
+    pub fn leave_quiescent_compaction_state(&self) {
+        unsafe {
+            let safepoint_ptr = addr_of_mut!(THREAD_SAFEPOINT_STATE);
+            if *safepoint_ptr == SAFEPOINT_ACTIVE {
+                *safepoint_ptr = SAFEPOINT_INACTIVE;
+                self.quiescent_threads.fetch_sub(1, Ordering::AcqRel);
+            }
+        }
+    }
+
+    /// Returns true only when every registered allocator thread is quiescent.
+    ///
+    /// This is a cooperative global-quiescence check, not a proof that active mutators can safely
+    /// race with remap/migration.
+    pub fn quiescent_compaction_ready(&self) -> bool {
+        let registered = self.registered_threads.load(Ordering::Acquire);
+        registered != 0 && registered == self.quiescent_threads.load(Ordering::Acquire)
+    }
+
+    /// Runs cooperative quiescent compaction when the current thread and every other registered
+    /// allocator thread have voluntarily stopped allocator-visible activity.
+    ///
+    /// This API intentionally does **not** claim to provide concurrent compaction with active
+    /// mutators. Achieving that would require heavier machinery such as page-fault mediation,
+    /// signal handling, syscall retry/interposition, or equivalent runtime coordination for raw
+    /// pointer accesses and kernel I/O into moving pages.
+    pub fn compact_when_quiescent(
+        &self,
+        policy: RuntimeCompactionPolicy,
+    ) -> RuntimeCompactionResult {
+        if !self.current_thread_in_safepoint() {
+            return RuntimeCompactionResult::Skipped {
+                reason: CompactionSkipReason::NotAtSafepoint,
+                advice: self.compaction_advice(),
+            };
+        }
+        if !self.quiescent_compaction_ready() {
+            return RuntimeCompactionResult::Skipped {
+                reason: CompactionSkipReason::ThreadsActive,
+                advice: self.compaction_advice(),
+            };
+        }
+
+        let Some(thread_heap) = self.thread_heap() else {
+            return RuntimeCompactionResult::Skipped {
+                reason: CompactionSkipReason::ThreadUnavailable,
+                advice: self.compaction_advice(),
+            };
+        };
+        let Some(result) = self.with_existing_allocator_mut(|allocator| {
+            allocator.shutdown_thread(thread_heap);
+            let advice = allocator.stats().compaction_advice();
+            if !policy.should_compact(&advice) {
+                return RuntimeCompactionResult::Skipped {
+                    reason: CompactionSkipReason::Policy,
+                    advice: Some(advice),
+                };
+            }
+
+            let meshes = allocator.compact_with_thread(thread_heap);
+            RuntimeCompactionResult::Compacted { meshes, advice }
+        }) else {
+            return RuntimeCompactionResult::Skipped {
+                reason: CompactionSkipReason::AllocatorUnavailable,
+                advice: None,
+            };
+        };
+
+        result
+    }
+
+    /// Compatibility alias for `enter_quiescent_compaction_state`.
+    pub fn enter_compaction_safepoint(&self) -> bool {
+        self.enter_quiescent_compaction_state()
+    }
+
+    /// Compatibility alias for `leave_quiescent_compaction_state`.
+    pub fn leave_compaction_safepoint(&self) {
+        self.leave_quiescent_compaction_state();
+    }
+
+    /// Compatibility alias for `quiescent_compaction_ready`.
+    pub fn compaction_safepoint_ready(&self) -> bool {
+        self.quiescent_compaction_ready()
+    }
+
+    /// Compatibility alias for `compact_when_quiescent`.
+    pub fn compact_at_safepoint(&self, policy: RuntimeCompactionPolicy) -> RuntimeCompactionResult {
+        self.compact_when_quiescent(policy)
+    }
+
+    pub fn compact(&self) -> usize {
+        let Some(thread_heap) = self.thread_heap() else {
+            return 0;
+        };
+        self.with_allocator(|allocator| allocator.compact_with_thread(thread_heap))
+            .unwrap_or_default()
+    }
+
+    pub fn stats(&self) -> Option<MeshStats> {
+        self.with_existing_allocator(|allocator| allocator.stats())
+    }
+
+    pub fn compaction_advice(&self) -> Option<CompactionAdvice> {
+        self.stats().map(|stats| stats.compaction_advice())
+    }
+
+    fn with_allocator<R>(&self, f: impl FnOnce(&mut MeshAllocator) -> R) -> Option<R> {
+        if !self.ensure_initialized() {
+            return None;
+        }
+
+        let _guard = self.lock.lock();
+        let allocator = unsafe { (&mut *self.allocator.get()).assume_init_mut() };
+        Some(f(allocator))
+    }
+
+    fn with_existing_allocator<R>(&self, f: impl FnOnce(&MeshAllocator) -> R) -> Option<R> {
+        if self.init_state.load(Ordering::Acquire) != INIT_READY {
+            return None;
+        }
+
+        let _guard = self.lock.lock();
+        if self.init_state.load(Ordering::Acquire) != INIT_READY {
+            return None;
+        }
+        let allocator = unsafe { (&*self.allocator.get()).assume_init_ref() };
+        Some(f(allocator))
+    }
+
+    fn with_existing_allocator_mut<R>(&self, f: impl FnOnce(&mut MeshAllocator) -> R) -> Option<R> {
+        if self.init_state.load(Ordering::Acquire) != INIT_READY {
+            return None;
+        }
+
+        let _guard = self.lock.lock();
+        if self.init_state.load(Ordering::Acquire) != INIT_READY {
+            return None;
+        }
+        let allocator = unsafe { (&mut *self.allocator.get()).assume_init_mut() };
+        Some(f(allocator))
+    }
+
+    fn ensure_initialized(&self) -> bool {
+        loop {
+            match self.init_state.load(Ordering::Acquire) {
+                INIT_READY => return true,
+                INIT_FAILED => return false,
+                INIT_UNINITIALIZED => {
+                    if self
+                        .init_state
+                        .compare_exchange(
+                            INIT_UNINITIALIZED,
+                            INIT_IN_PROGRESS,
+                            Ordering::AcqRel,
+                            Ordering::Acquire,
+                        )
+                        .is_ok()
+                    {
+                        let result = MeshAllocator::new(self.arena_size, self.miniheap_capacity);
+                        match result {
+                            Ok(allocator) => unsafe {
+                                (*self.allocator.get()).write(allocator);
+                                self.init_state.store(INIT_READY, Ordering::Release);
+                                futex_wake_all(&self.init_state);
+                                return true;
+                            },
+                            Err(_) => {
+                                self.init_state.store(INIT_FAILED, Ordering::Release);
+                                futex_wake_all(&self.init_state);
+                                return false;
+                            }
+                        }
+                    }
+                }
+                INIT_IN_PROGRESS => futex_wait_for_value(&self.init_state, INIT_IN_PROGRESS),
+                _ => return false,
+            }
+        }
+    }
+
+    fn thread_heap(&self) -> Option<&'static mut ThreadLocalHeap> {
+        unsafe {
+            let owner_ptr = addr_of_mut!(THREAD_HEAP_OWNER);
+            let heap_ptr = addr_of_mut!(THREAD_HEAP);
+            match THREAD_HEAP_STATE {
+                TLS_READY if core::ptr::eq(*owner_ptr, self) => Some((*heap_ptr).assume_init_mut()),
+                TLS_READY => {
+                    self.reset_foreign_thread_heap();
+                    self.thread_heap()
+                }
+                TLS_FAILED => None,
+                TLS_UNINITIALIZED => match ThreadLocalHeap::new() {
+                    Ok(heap) => {
+                        (*heap_ptr).write(heap);
+                        *addr_of_mut!(THREAD_SAFEPOINT_STATE) = SAFEPOINT_INACTIVE;
+                        *owner_ptr = self as *const _;
+                        self.registered_threads.fetch_add(1, Ordering::AcqRel);
+                        THREAD_HEAP_STATE = TLS_READY;
+                        Some((*heap_ptr).assume_init_mut())
+                    }
+                    Err(_) => {
+                        THREAD_HEAP_STATE = TLS_FAILED;
+                        None
+                    }
+                },
+                _ => None,
+            }
+        }
+    }
+
+    fn current_thread_in_safepoint(&self) -> bool {
+        let _ = self;
+        unsafe {
+            core::ptr::eq(THREAD_HEAP_OWNER, self) && THREAD_SAFEPOINT_STATE == SAFEPOINT_ACTIVE
+        }
+    }
+
+    fn reset_foreign_thread_heap(&self) {
+        let _ = self;
+        unsafe {
+            if THREAD_HEAP_STATE == TLS_READY {
+                drop_in_place(addr_of_mut!(THREAD_HEAP).cast::<ThreadLocalHeap>());
+            }
+            THREAD_HEAP_STATE = TLS_UNINITIALIZED;
+            THREAD_SAFEPOINT_STATE = SAFEPOINT_INACTIVE;
+            THREAD_HEAP_OWNER = null();
+        }
+    }
+}
+
+impl Drop for GlobalMeshAllocator {
+    fn drop(&mut self) {
+        unsafe {
+            if core::ptr::eq(THREAD_HEAP_OWNER, self) {
+                self.shutdown_thread();
+            }
+        }
+
+        if self.init_state.load(Ordering::Acquire) == INIT_READY {
+            let _guard = self.lock.lock();
+            if self.init_state.load(Ordering::Acquire) == INIT_READY {
+                unsafe {
+                    drop_in_place((&mut *self.allocator.get()).as_mut_ptr());
+                }
+                self.init_state.store(INIT_FAILED, Ordering::Release);
+            }
+        }
+    }
+}
+
+unsafe impl Sync for GlobalMeshAllocator {}
+unsafe impl Send for GlobalMeshAllocator {}
+
+unsafe impl GlobalAlloc for GlobalMeshAllocator {
+    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+        if !self.ensure_initialized() {
+            return null_mut();
+        }
+        let Some(thread_heap) = self.thread_heap() else {
+            return null_mut();
+        };
+
+        let allocator_ref = unsafe { (&*self.allocator.get()).assume_init_ref() };
+        if let Some(class) = size_class_for_layout(layout)
+            && let Some(ptr) = allocator_ref.try_allocate_small_local(thread_heap, class)
+        {
+            return ptr;
+        }
+
+        self.with_allocator(|allocator| {
+            allocator
+                .allocate_layout_with_thread(thread_heap, layout)
+                .unwrap_or(null_mut())
+        })
+        .unwrap_or(null_mut())
+    }
+
+    unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) {
+        if !self.ensure_initialized() {
+            return;
+        }
+        let Some(thread_heap) = self.thread_heap() else {
+            return;
+        };
+
+        let allocator_ref = unsafe { (&*self.allocator.get()).assume_init_ref() };
+        if allocator_ref.try_deallocate_local(ptr, thread_heap) {
+            return;
+        }
+
+        let _ = self.with_allocator(|allocator| allocator.deallocate_with_thread(ptr, thread_heap));
+    }
+
+    unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
+        let ptr = unsafe { self.alloc(layout) };
+        if !ptr.is_null() {
+            unsafe {
+                ptr.write_bytes(0, layout.size());
+            }
+        }
+        ptr
+    }
+
+    unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
+        if !self.ensure_initialized() {
+            return null_mut();
+        }
+        let Some(thread_heap) = self.thread_heap() else {
+            return null_mut();
+        };
+
+        self.with_allocator(|allocator| unsafe {
+            allocator
+                .reallocate_with_thread(ptr, layout, new_size, thread_heap)
+                .unwrap_or(null_mut())
+        })
+        .unwrap_or(null_mut())
+    }
+}
+
+fn size_class_for_layout(layout: Layout) -> Option<u8> {
+    if layout.align() > super::page::page_size() {
+        return None;
+    }
+    let aligned_size = layout
+        .size()
+        .max(1)
+        .checked_add(layout.align() - 1)
+        .map(|value| value & !(layout.align() - 1))?;
+    if aligned_size > super::constants::MAX_SMALL_ALLOCATION {
+        return None;
+    }
+    let class = super::size_map::size_class_for(aligned_size)?;
+    if super::size_map::byte_size_for_class(class).is_multiple_of(layout.align()) {
+        Some(class)
+    } else {
+        None
+    }
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/meshing.rs

@@ -0,0 +1,15 @@
+use super::bitmap::AtomicBitmap;
+
+#[inline]
+pub fn bitmaps_meshable(left: &AtomicBitmap, right: &AtomicBitmap) -> bool {
+    let left_words = left.snapshot();
+    let right_words = right.snapshot();
+
+    for (lhs, rhs) in left_words.words().iter().zip(right_words.words().iter()) {
+        if lhs & rhs != 0 {
+            return false;
+        }
+    }
+
+    true
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/miniheap.rs

@@ -0,0 +1,374 @@
+use core::sync::atomic::{AtomicU32, Ordering};
+
+use super::bitmap::AtomicBitmap;
+use super::page::page_size;
+use super::size_map::{byte_size_for_class, size_class_for};
+use super::span::Span;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[repr(u8)]
+pub enum FreelistId {
+    Full = 0,
+    Partial = 1,
+    Empty = 2,
+    Attached = 3,
+}
+
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+pub struct MiniHeapId(u32);
+
+impl MiniHeapId {
+    #[inline(always)]
+    pub const fn new(id: u32) -> Self {
+        Self(id)
+    }
+
+    #[inline(always)]
+    pub const fn value(self) -> u32 {
+        self.0
+    }
+
+    #[inline(always)]
+    pub const fn has_value(self) -> bool {
+        self.0 != 0
+    }
+}
+
+#[derive(Debug)]
+pub struct MiniHeapFlags {
+    bits: AtomicU32,
+}
+
+impl MiniHeapFlags {
+    const SIZE_CLASS_SHIFT: u32 = 0;
+    const FREELIST_ID_SHIFT: u32 = 6;
+    const SHUFFLE_OFFSET_SHIFT: u32 = 8;
+    const MAX_COUNT_SHIFT: u32 = 16;
+    const PENDING_OFFSET: u32 = 27;
+    const MESHED_OFFSET: u32 = 30;
+
+    #[inline]
+    pub fn new(
+        max_count: u16,
+        size_class: u8,
+        shuffle_offset: u8,
+        freelist_id: FreelistId,
+    ) -> Self {
+        let bits = ((max_count as u32) << Self::MAX_COUNT_SHIFT)
+            | ((shuffle_offset as u32) << Self::SHUFFLE_OFFSET_SHIFT)
+            | ((freelist_id as u32) << Self::FREELIST_ID_SHIFT)
+            | ((size_class as u32) << Self::SIZE_CLASS_SHIFT);
+        Self {
+            bits: AtomicU32::new(bits),
+        }
+    }
+
+    #[inline(always)]
+    fn load(&self) -> u32 {
+        self.bits.load(Ordering::Acquire)
+    }
+
+    #[inline(always)]
+    fn update_masked(&self, mask: u32, value: u32) {
+        let mut old = self.bits.load(Ordering::Relaxed);
+        loop {
+            let new = (old & mask) | value;
+            match self
+                .bits
+                .compare_exchange_weak(old, new, Ordering::AcqRel, Ordering::Relaxed)
+            {
+                Ok(_) => return,
+                Err(next) => old = next,
+            }
+        }
+    }
+
+    #[inline(always)]
+    pub fn max_count(&self) -> u16 {
+        ((self.load() >> Self::MAX_COUNT_SHIFT) & 0x7ff) as u16
+    }
+
+    #[inline(always)]
+    pub fn size_class(&self) -> u8 {
+        ((self.load() >> Self::SIZE_CLASS_SHIFT) & 0x3f) as u8
+    }
+
+    #[inline(always)]
+    pub fn freelist_id(&self) -> FreelistId {
+        match (self.load() >> Self::FREELIST_ID_SHIFT) & 0x3 {
+            0 => FreelistId::Full,
+            1 => FreelistId::Partial,
+            2 => FreelistId::Empty,
+            _ => FreelistId::Attached,
+        }
+    }
+
+    #[inline(always)]
+    pub fn set_freelist_id(&self, id: FreelistId) {
+        let mask = !(0x3 << Self::FREELIST_ID_SHIFT);
+        self.update_masked(mask, (id as u32) << Self::FREELIST_ID_SHIFT);
+    }
+
+    #[inline(always)]
+    pub fn shuffle_vector_offset(&self) -> u8 {
+        ((self.load() >> Self::SHUFFLE_OFFSET_SHIFT) & 0xff) as u8
+    }
+
+    #[inline(always)]
+    pub fn set_shuffle_vector_offset(&self, offset: u8) {
+        let mask = !(0xff << Self::SHUFFLE_OFFSET_SHIFT);
+        self.update_masked(mask, (offset as u32) << Self::SHUFFLE_OFFSET_SHIFT);
+    }
+
+    #[inline(always)]
+    pub fn is_pending(&self) -> bool {
+        self.load() & (1 << Self::PENDING_OFFSET) != 0
+    }
+
+    #[inline(always)]
+    pub fn clear_pending(&self) {
+        self.bits
+            .fetch_and(!(1 << Self::PENDING_OFFSET), Ordering::AcqRel);
+    }
+
+    #[inline(always)]
+    pub fn try_set_pending_from_full(&self) -> bool {
+        let full = (FreelistId::Full as u32) << Self::FREELIST_ID_SHIFT;
+        let pending = 1 << Self::PENDING_OFFSET;
+        let freelist_mask = 0x3 << Self::FREELIST_ID_SHIFT;
+
+        let mut old = self.bits.load(Ordering::Relaxed);
+        loop {
+            if (old & freelist_mask) != full || (old & pending) != 0 {
+                return false;
+            }
+            let new = old | pending;
+            match self
+                .bits
+                .compare_exchange_weak(old, new, Ordering::AcqRel, Ordering::Relaxed)
+            {
+                Ok(_) => return true,
+                Err(next) => old = next,
+            }
+        }
+    }
+
+    #[inline(always)]
+    pub fn is_meshed(&self) -> bool {
+        self.load() & (1 << Self::MESHED_OFFSET) != 0
+    }
+
+    #[inline(always)]
+    pub fn set_meshed(&self) {
+        self.bits
+            .fetch_or(1 << Self::MESHED_OFFSET, Ordering::AcqRel);
+    }
+}
+
+#[derive(Debug)]
+pub struct MiniHeap {
+    span: Span,
+    current_thread: AtomicU32,
+    flags: MiniHeapFlags,
+    next_meshed: AtomicU32,
+    pending_next: AtomicU32,
+    bitmap: AtomicBitmap,
+}
+
+impl MiniHeap {
+    #[inline]
+    pub fn new(span: Span, object_count: u16, object_size: usize) -> Self {
+        let size_class = if object_count > 1 {
+            size_class_for(object_size).unwrap_or(1)
+        } else {
+            1
+        };
+
+        Self {
+            span,
+            current_thread: AtomicU32::new(0),
+            flags: MiniHeapFlags::new(object_count, size_class, 0, FreelistId::Attached),
+            next_meshed: AtomicU32::new(0),
+            pending_next: AtomicU32::new(0),
+            bitmap: AtomicBitmap::new(object_count as usize),
+        }
+    }
+
+    #[inline(always)]
+    pub const fn span(&self) -> Span {
+        self.span
+    }
+
+    #[inline(always)]
+    pub fn flags(&self) -> &MiniHeapFlags {
+        &self.flags
+    }
+
+    #[inline(always)]
+    pub fn bitmap(&self) -> &AtomicBitmap {
+        &self.bitmap
+    }
+
+    #[inline(always)]
+    pub fn max_count(&self) -> u16 {
+        self.flags.max_count()
+    }
+
+    #[inline(always)]
+    pub fn size_class(&self) -> u8 {
+        self.flags.size_class()
+    }
+
+    #[inline(always)]
+    pub fn is_large_alloc(&self) -> bool {
+        self.max_count() == 1
+    }
+
+    #[inline(always)]
+    pub fn object_size(&self) -> usize {
+        if self.is_large_alloc() {
+            self.span.byte_length_for_page_size(page_size())
+        } else {
+            byte_size_for_class(self.size_class())
+        }
+    }
+
+    #[inline(always)]
+    pub fn span_size(&self) -> usize {
+        self.span.byte_length_for_page_size(page_size())
+    }
+
+    #[inline(always)]
+    pub fn in_use_count(&self) -> u32 {
+        self.bitmap.in_use_count()
+    }
+
+    #[inline(always)]
+    pub fn is_empty(&self) -> bool {
+        self.in_use_count() == 0
+    }
+
+    #[inline(always)]
+    pub fn is_full(&self) -> bool {
+        self.in_use_count() == self.max_count() as u32
+    }
+
+    #[inline(always)]
+    pub fn bytes_free(&self) -> usize {
+        (self.max_count() as usize - self.in_use_count() as usize) * self.object_size()
+    }
+
+    #[inline(always)]
+    pub fn current_thread(&self) -> u32 {
+        self.current_thread.load(Ordering::Acquire)
+    }
+
+    #[inline(always)]
+    pub fn set_attached(&self, thread_id: u32) {
+        self.current_thread.store(thread_id, Ordering::Release);
+        self.flags.set_freelist_id(FreelistId::Attached);
+    }
+
+    #[inline(always)]
+    pub fn unset_attached(&self) {
+        self.current_thread.store(0, Ordering::Release);
+    }
+
+    #[inline(always)]
+    pub fn is_attached(&self) -> bool {
+        self.current_thread() != 0
+    }
+
+    #[inline(always)]
+    pub fn set_shuffle_vector_offset(&self, offset: u8) {
+        self.flags.set_shuffle_vector_offset(offset);
+    }
+
+    #[inline(always)]
+    pub fn shuffle_vector_offset(&self) -> u8 {
+        self.flags.shuffle_vector_offset()
+    }
+
+    #[inline(always)]
+    pub fn set_pending_next(&self, next: MiniHeapId) {
+        self.pending_next.store(next.value(), Ordering::Release);
+    }
+
+    #[inline(always)]
+    pub fn pending_next(&self) -> MiniHeapId {
+        MiniHeapId::new(self.pending_next.load(Ordering::Acquire))
+    }
+
+    #[inline(always)]
+    pub fn track_meshed_span(&self, next: MiniHeapId) {
+        self.next_meshed.store(next.value(), Ordering::Release);
+    }
+
+    #[inline(always)]
+    pub fn next_meshed(&self) -> MiniHeapId {
+        MiniHeapId::new(self.next_meshed.load(Ordering::Acquire))
+    }
+
+    #[inline(always)]
+    pub fn has_meshed_partner(&self) -> bool {
+        self.next_meshed().has_value()
+    }
+
+    #[inline(always)]
+    pub fn set_meshed(&self) {
+        self.flags.set_meshed();
+    }
+
+    #[inline(always)]
+    pub fn is_meshed(&self) -> bool {
+        self.flags.is_meshed()
+    }
+
+    #[inline(always)]
+    pub fn is_meshing_candidate(&self) -> bool {
+        !self.is_attached() && self.object_size() < page_size()
+    }
+
+    #[inline(always)]
+    pub fn fullness(&self) -> f32 {
+        self.in_use_count() as f32 / self.max_count() as f32
+    }
+
+    #[inline(always)]
+    pub fn malloc_at(&self, arena_begin: usize, slot: usize) -> Option<*mut u8> {
+        if !self.bitmap.try_set(slot) {
+            return None;
+        }
+        Some(self.ptr_from_offset(arena_begin, slot))
+    }
+
+    #[inline(always)]
+    pub fn ptr_from_offset(&self, arena_begin: usize, slot: usize) -> *mut u8 {
+        let span_start = arena_begin + (self.span.offset as usize * page_size());
+        (span_start + slot * self.object_size()) as *mut u8
+    }
+
+    #[inline(always)]
+    pub fn contains_ptr(&self, arena_begin: usize, ptr: *const u8) -> bool {
+        let span_start = arena_begin + (self.span.offset as usize * page_size());
+        let span_end = span_start + self.span_size();
+        let ptr = ptr as usize;
+        span_start <= ptr && ptr < span_end
+    }
+
+    #[inline(always)]
+    pub fn free_offset(&self, slot: usize) -> bool {
+        self.bitmap.unset(slot)
+    }
+
+    #[inline(always)]
+    pub fn clear_if_not_free(&self, slot: usize) -> bool {
+        self.bitmap.unset(slot)
+    }
+
+    #[inline(always)]
+    pub fn slot_for_ptr(&self, arena_begin: usize, ptr: *const u8) -> usize {
+        let span_start = arena_begin + (self.span.offset as usize * page_size());
+        ((ptr as usize) - span_start) / self.object_size()
+    }
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/mod.rs

@@ -0,0 +1,45 @@
+pub mod allocator;
+pub mod arena;
+pub mod bitmap;
+pub mod constants;
+pub mod fault;
+pub mod global;
+pub mod meshing;
+pub mod miniheap;
+pub mod page;
+pub mod platform;
+pub mod pool;
+pub mod raw_sys;
+pub mod rng;
+pub mod shuffle;
+pub mod size_map;
+pub mod span;
+pub mod stats;
+pub mod sync;
+pub mod thread_local_heap;
+
+pub use allocator::MeshAllocator;
+pub use arena::Arena;
+pub use bitmap::{AtomicBitmap, BitIter, RelaxedBitmap};
+pub use constants::*;
+pub use fault::{
+    ActiveMeshGuard, ensure_fault_mediation_installed, ok_to_proceed, retry_on_efault,
+    retry_on_efault_ptrs,
+};
+pub use global::{DEFAULT_GLOBAL_MINIHEAP_CAPACITY, GlobalMeshAllocator};
+pub use meshing::bitmaps_meshable;
+pub use miniheap::{FreelistId, MiniHeap, MiniHeapFlags, MiniHeapId};
+pub use page::{
+    PageConfig, page_count, page_shift, page_size, round_up_to_page, runtime_slots_per_span,
+};
+pub use platform::{PlatformHooks, PlatformInstallError, install_platform_hooks};
+pub use rng::{Mwc, Mwc64};
+pub use shuffle::{ShuffleEntry, ShuffleVector};
+pub use size_map::{CLASS_TO_SIZE, NUM_SIZE_CLASSES, byte_size_for_class, size_class_for};
+pub use span::Span;
+pub use stats::{
+    CompactionAdvice, CompactionEstimate, CompactionRecommendation, CompactionSkipReason,
+    MeshStats, RuntimeCompactionPolicy, RuntimeCompactionResult,
+};
+pub use sync::FutexMutex;
+pub use thread_local_heap::ThreadLocalHeap;

lib/runtime/src/platform/linux_x86_64/mesh_alloc/page.rs

@@ -0,0 +1,94 @@
+use core::sync::atomic::{AtomicUsize, Ordering};
+
+use super::constants::{MAX_SUPPORTED_PAGE_SIZE, MIN_OBJECT_SIZE, MIN_SUPPORTED_PAGE_SIZE};
+use super::platform;
+static PAGE_SIZE_CACHE: AtomicUsize = AtomicUsize::new(0);
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct PageConfig {
+    size: usize,
+    shift: u32,
+    slots_per_span: usize,
+}
+
+impl PageConfig {
+    #[inline(always)]
+    pub fn get() -> Self {
+        let size = page_size();
+        Self {
+            size,
+            shift: size.trailing_zeros(),
+            slots_per_span: size / MIN_OBJECT_SIZE,
+        }
+    }
+
+    #[inline(always)]
+    pub const fn size(self) -> usize {
+        self.size
+    }
+
+    #[inline(always)]
+    pub const fn shift(self) -> u32 {
+        self.shift
+    }
+
+    #[inline(always)]
+    pub const fn slots_per_span(self) -> usize {
+        self.slots_per_span
+    }
+}
+
+#[inline]
+pub fn page_size() -> usize {
+    let cached = PAGE_SIZE_CACHE.load(Ordering::Acquire);
+    if cached != 0 {
+        return cached;
+    }
+
+    let size = query_page_size();
+    match PAGE_SIZE_CACHE.compare_exchange(0, size, Ordering::AcqRel, Ordering::Acquire) {
+        Ok(_) => size,
+        Err(existing) => existing,
+    }
+}
+
+#[inline(always)]
+pub fn page_shift() -> u32 {
+    page_size().trailing_zeros()
+}
+
+#[inline(always)]
+pub fn page_count(size: usize) -> usize {
+    let page = page_size();
+    size.div_ceil(page)
+}
+
+#[inline(always)]
+pub fn round_up_to_page(size: usize) -> usize {
+    page_count(size) * page_size()
+}
+
+#[inline(always)]
+pub fn runtime_slots_per_span() -> usize {
+    page_size() / MIN_OBJECT_SIZE
+}
+
+fn query_page_size() -> usize {
+    let size = platform::page_size();
+    assert!(
+        size.is_power_of_two(),
+        "page size is not a power of two: {size}"
+    );
+    assert!(
+        (MIN_SUPPORTED_PAGE_SIZE..=MAX_SUPPORTED_PAGE_SIZE).contains(&size),
+        "unsupported page size {size}; supported range is {}..={}",
+        MIN_SUPPORTED_PAGE_SIZE,
+        MAX_SUPPORTED_PAGE_SIZE
+    );
+    assert_eq!(
+        size % MIN_OBJECT_SIZE,
+        0,
+        "page size {size} is not MIN_OBJECT_SIZE-aligned"
+    );
+    size
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/platform.rs

@@ -0,0 +1,168 @@
+use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering};
+
+use super::raw_sys;
+
+#[derive(Clone, Copy, Debug)]
+pub struct PlatformHooks {
+    pub page_size: fn() -> usize,
+    pub gettid: fn() -> raw_sys::Result<u32>,
+    pub getrandom: fn(&mut [u8], u32) -> raw_sys::Result<usize>,
+    pub memfd_create: fn(*const u8, u32) -> raw_sys::Result<i32>,
+    pub ftruncate: fn(i32, u64) -> raw_sys::Result<()>,
+    pub fallocate: fn(i32, u32, u64, u64) -> raw_sys::Result<()>,
+    pub close: fn(i32) -> raw_sys::Result<()>,
+    pub futex_wait: unsafe fn(*const u32, u32, u32) -> raw_sys::Result<()>,
+    pub futex_wake: unsafe fn(*const u32, u32, u32) -> raw_sys::Result<u32>,
+    pub mmap: unsafe fn(*mut u8, usize, u32, u32, i32, u64) -> raw_sys::Result<*mut u8>,
+    pub map_anonymous: unsafe fn(usize, u32) -> raw_sys::Result<*mut u8>,
+    pub mprotect: unsafe fn(*mut u8, usize, u32) -> raw_sys::Result<()>,
+    pub munmap: unsafe fn(*mut u8, usize) -> raw_sys::Result<()>,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum PlatformInstallError {
+    AlreadyConfigured,
+    AlreadyInUse,
+}
+
+static DEFAULT_PLATFORM_HOOKS: PlatformHooks = PlatformHooks {
+    page_size: default_page_size,
+    gettid: raw_sys::gettid,
+    getrandom: raw_sys::getrandom,
+    memfd_create: raw_sys::memfd_create,
+    ftruncate: raw_sys::ftruncate,
+    fallocate: raw_sys::fallocate,
+    close: raw_sys::close,
+    futex_wait: raw_sys::futex_wait,
+    futex_wake: raw_sys::futex_wake,
+    mmap: raw_sys::mmap,
+    map_anonymous: raw_sys::map_anonymous,
+    mprotect: raw_sys::mprotect,
+    munmap: raw_sys::munmap,
+};
+
+static PLATFORM_HOOKS: AtomicPtr<PlatformHooks> =
+    AtomicPtr::new((&DEFAULT_PLATFORM_HOOKS as *const PlatformHooks).cast_mut());
+static PLATFORM_FROZEN: AtomicBool = AtomicBool::new(false);
+
+pub fn install_platform_hooks(hooks: &'static PlatformHooks) -> Result<(), PlatformInstallError> {
+    if PLATFORM_FROZEN.load(Ordering::Acquire) {
+        return Err(PlatformInstallError::AlreadyInUse);
+    }
+
+    let default_ptr = (&DEFAULT_PLATFORM_HOOKS as *const PlatformHooks).cast_mut();
+    let hooks_ptr = (hooks as *const PlatformHooks).cast_mut();
+    match PLATFORM_HOOKS.compare_exchange(
+        default_ptr,
+        hooks_ptr,
+        Ordering::AcqRel,
+        Ordering::Acquire,
+    ) {
+        Ok(_) => Ok(()),
+        Err(existing) if existing == hooks_ptr => Ok(()),
+        Err(_) => Err(PlatformInstallError::AlreadyConfigured),
+    }
+}
+
+#[inline(always)]
+pub fn page_size() -> usize {
+    (platform_hooks().page_size)()
+}
+
+#[inline(always)]
+pub fn gettid() -> raw_sys::Result<u32> {
+    (platform_hooks().gettid)()
+}
+
+#[inline(always)]
+pub fn getrandom(buf: &mut [u8], flags: u32) -> raw_sys::Result<usize> {
+    (platform_hooks().getrandom)(buf, flags)
+}
+
+#[inline(always)]
+pub fn memfd_create(name: *const u8, flags: u32) -> raw_sys::Result<i32> {
+    (platform_hooks().memfd_create)(name, flags)
+}
+
+#[inline(always)]
+pub fn ftruncate(fd: i32, len: u64) -> raw_sys::Result<()> {
+    (platform_hooks().ftruncate)(fd, len)
+}
+
+#[inline(always)]
+pub fn fallocate(fd: i32, mode: u32, offset: u64, len: u64) -> raw_sys::Result<()> {
+    (platform_hooks().fallocate)(fd, mode, offset, len)
+}
+
+#[inline(always)]
+pub fn close(fd: i32) -> raw_sys::Result<()> {
+    (platform_hooks().close)(fd)
+}
+
+#[inline(always)]
+/// # Safety
+///
+/// `uaddr` must be valid for the kernel to access as a futex word for the duration of the call.
+pub unsafe fn futex_wait(uaddr: *const u32, op: u32, expected: u32) -> raw_sys::Result<()> {
+    unsafe { (platform_hooks().futex_wait)(uaddr, op, expected) }
+}
+
+#[inline(always)]
+/// # Safety
+///
+/// `uaddr` must be valid for the kernel to access as a futex word for the duration of the call.
+pub unsafe fn futex_wake(uaddr: *const u32, op: u32, count: u32) -> raw_sys::Result<u32> {
+    unsafe { (platform_hooks().futex_wake)(uaddr, op, count) }
+}
+
+#[inline(always)]
+/// # Safety
+///
+/// The caller must ensure the mapping arguments satisfy the platform `mmap(2)` contract.
+pub unsafe fn mmap(
+    addr: *mut u8,
+    len: usize,
+    prot: u32,
+    flags: u32,
+    fd: i32,
+    offset: u64,
+) -> raw_sys::Result<*mut u8> {
+    unsafe { (platform_hooks().mmap)(addr, len, prot, flags, fd, offset) }
+}
+
+#[inline(always)]
+/// # Safety
+///
+/// The caller must later unmap the returned memory exactly once.
+pub unsafe fn map_anonymous(len: usize, prot: u32) -> raw_sys::Result<*mut u8> {
+    unsafe { (platform_hooks().map_anonymous)(len, prot) }
+}
+
+#[inline(always)]
+/// # Safety
+///
+/// `addr..addr+len` must refer to a valid mapped region.
+pub unsafe fn mprotect(addr: *mut u8, len: usize, prot: u32) -> raw_sys::Result<()> {
+    unsafe { (platform_hooks().mprotect)(addr, len, prot) }
+}
+
+#[inline(always)]
+/// # Safety
+///
+/// `addr..addr+len` must refer to a valid mapping that may be unmapped exactly once.
+pub unsafe fn munmap(addr: *mut u8, len: usize) -> raw_sys::Result<()> {
+    unsafe { (platform_hooks().munmap)(addr, len) }
+}
+
+#[inline(always)]
+fn platform_hooks() -> &'static PlatformHooks {
+    PLATFORM_FROZEN.store(true, Ordering::Release);
+    let hooks = PLATFORM_HOOKS.load(Ordering::Acquire);
+    unsafe { &*hooks }
+}
+
+fn default_page_size() -> usize {
+    let size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
+    assert!(size > 0, "sysconf(_SC_PAGESIZE) failed");
+    size as usize
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/pool.rs

@@ -0,0 +1,152 @@
+use core::mem::size_of;
+use core::sync::atomic::{AtomicU8, AtomicU32, Ordering};
+
+use super::miniheap::{MiniHeap, MiniHeapId};
+use super::platform;
+use super::raw_sys;
+use super::span::Span;
+
+#[derive(Debug)]
+pub struct MiniHeapPool {
+    base: *mut MiniHeap,
+    live: *mut AtomicU8,
+    free_ids: *mut u32,
+    capacity: u32,
+    len: AtomicU32,
+    live_len: AtomicU32,
+    free_len: u32,
+}
+
+impl MiniHeapPool {
+    #[inline]
+    pub fn with_capacity(capacity: u32) -> raw_sys::Result<Self> {
+        assert!(capacity > 0);
+        let bytes = capacity as usize * size_of::<MiniHeap>();
+        let live = unsafe {
+            platform::map_anonymous(capacity as usize, raw_sys::PROT_READ | raw_sys::PROT_WRITE)?
+        };
+        let free_ids = unsafe {
+            platform::map_anonymous(
+                capacity as usize * size_of::<u32>(),
+                raw_sys::PROT_READ | raw_sys::PROT_WRITE,
+            )? as *mut u32
+        };
+        let base = unsafe {
+            platform::map_anonymous(bytes, raw_sys::PROT_READ | raw_sys::PROT_WRITE)?
+                as *mut MiniHeap
+        };
+
+        Ok(Self {
+            base,
+            live: live as *mut AtomicU8,
+            free_ids,
+            capacity,
+            len: AtomicU32::new(0),
+            live_len: AtomicU32::new(0),
+            free_len: 0,
+        })
+    }
+
+    #[inline(always)]
+    pub fn len(&self) -> u32 {
+        self.len.load(Ordering::Acquire)
+    }
+
+    #[inline(always)]
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    #[inline(always)]
+    pub const fn capacity(&self) -> u32 {
+        self.capacity
+    }
+
+    #[inline(always)]
+    pub fn live_len(&self) -> u32 {
+        self.live_len.load(Ordering::Acquire)
+    }
+
+    #[inline]
+    pub fn allocate(
+        &mut self,
+        span: Span,
+        object_count: u16,
+        object_size: usize,
+    ) -> Option<(MiniHeapId, &MiniHeap)> {
+        let (id, index) = if self.free_len > 0 {
+            self.free_len -= 1;
+            let id = unsafe { *self.free_ids.add(self.free_len as usize) };
+            (MiniHeapId::new(id), (id - 1) as usize)
+        } else {
+            let len = self.len.load(Ordering::Relaxed);
+            if len >= self.capacity {
+                return None;
+            }
+            let id = len + 1;
+            self.len.store(id, Ordering::Release);
+            (MiniHeapId::new(id), (id - 1) as usize)
+        };
+        self.live_len.fetch_add(1, Ordering::AcqRel);
+
+        unsafe {
+            let ptr = self.base.add(index);
+            ptr.write(MiniHeap::new(span, object_count, object_size));
+            (&*self.live.add(index)).store(1, Ordering::Release);
+            Some((id, &*ptr))
+        }
+    }
+
+    #[inline]
+    pub fn get(&self, id: MiniHeapId) -> Option<&MiniHeap> {
+        if !id.has_value() {
+            return None;
+        }
+
+        let index = id.value() - 1;
+        if index >= self.len.load(Ordering::Acquire) {
+            return None;
+        }
+        if unsafe { (&*self.live.add(index as usize)).load(Ordering::Acquire) } == 0 {
+            return None;
+        }
+
+        unsafe { Some(&*self.base.add(index as usize)) }
+    }
+
+    #[inline]
+    pub fn release(&mut self, id: MiniHeapId) -> bool {
+        if !id.has_value() {
+            return false;
+        }
+
+        let index = id.value() - 1;
+        if index >= self.len.load(Ordering::Acquire) {
+            return false;
+        }
+        if unsafe { (&*self.live.add(index as usize)).load(Ordering::Acquire) } == 0 {
+            return false;
+        }
+
+        unsafe {
+            (&*self.live.add(index as usize)).store(0, Ordering::Release);
+            self.free_ids.add(self.free_len as usize).write(id.value());
+        }
+        self.free_len += 1;
+        self.live_len.fetch_sub(1, Ordering::AcqRel);
+        true
+    }
+}
+
+impl Drop for MiniHeapPool {
+    fn drop(&mut self) {
+        let bytes = self.capacity as usize * size_of::<MiniHeap>();
+        let live_bytes = self.capacity as usize;
+        let free_bytes = self.capacity as usize * size_of::<u32>();
+        unsafe {
+            let _ = platform::munmap(self.free_ids as *mut u8, free_bytes);
+            let _ = platform::munmap(self.live as *mut u8, live_bytes);
+            let _ = platform::munmap(self.base as *mut u8, bytes);
+        }
+    }
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/raw_sys.rs

@@ -0,0 +1,218 @@
+use core::ptr::null_mut;
+
+pub type Result<T> = core::result::Result<T, Error>;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct Error(pub i32);
+
+impl Error {
+    #[inline(always)]
+    pub const fn errno(self) -> i32 {
+        self.0
+    }
+}
+
+pub const PROT_NONE: u32 = libc::PROT_NONE as u32;
+pub const PROT_READ: u32 = libc::PROT_READ as u32;
+pub const PROT_WRITE: u32 = libc::PROT_WRITE as u32;
+
+pub const MAP_SHARED: u32 = libc::MAP_SHARED as u32;
+pub const MAP_PRIVATE: u32 = libc::MAP_PRIVATE as u32;
+pub const MAP_FIXED: u32 = libc::MAP_FIXED as u32;
+pub const MAP_ANONYMOUS: u32 = libc::MAP_ANONYMOUS as u32;
+
+pub const MFD_CLOEXEC: u32 = libc::MFD_CLOEXEC;
+
+pub const FALLOC_FL_KEEP_SIZE: u32 = libc::FALLOC_FL_KEEP_SIZE as u32;
+pub const FALLOC_FL_PUNCH_HOLE: u32 = libc::FALLOC_FL_PUNCH_HOLE as u32;
+
+pub const FUTEX_WAIT: u32 = libc::FUTEX_WAIT as u32;
+pub const FUTEX_WAKE: u32 = libc::FUTEX_WAKE as u32;
+pub const FUTEX_PRIVATE_FLAG: u32 = libc::FUTEX_PRIVATE_FLAG as u32;
+pub const FUTEX_WAIT_PRIVATE: u32 = FUTEX_WAIT | FUTEX_PRIVATE_FLAG;
+pub const FUTEX_WAKE_PRIVATE: u32 = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
+
+pub const EINTR: i32 = libc::EINTR;
+pub const EAGAIN: i32 = libc::EAGAIN;
+pub const EFAULT: i32 = libc::EFAULT;
+
+#[inline(always)]
+fn last_error() -> Error {
+    Error(
+        std::io::Error::last_os_error()
+            .raw_os_error()
+            .unwrap_or(libc::EINVAL),
+    )
+}
+
+#[inline(always)]
+fn map_c_int(result: libc::c_int) -> Result<libc::c_int> {
+    if result == -1 {
+        Err(last_error())
+    } else {
+        Ok(result)
+    }
+}
+
+#[inline(always)]
+fn map_c_long(result: libc::c_long) -> Result<libc::c_long> {
+    if result == -1 {
+        Err(last_error())
+    } else {
+        Ok(result)
+    }
+}
+
+#[inline(always)]
+fn to_off_t(value: u64) -> Result<libc::off_t> {
+    value.try_into().map_err(|_| Error(libc::EINVAL))
+}
+
+#[inline(always)]
+pub fn getpid() -> Result<u32> {
+    let pid = unsafe { libc::getpid() };
+    if pid == -1 {
+        Err(last_error())
+    } else {
+        Ok(pid as u32)
+    }
+}
+
+#[inline(always)]
+pub fn gettid() -> Result<u32> {
+    unsafe { map_c_long(libc::syscall(libc::SYS_gettid)).map(|value| value as u32) }
+}
+
+#[inline(always)]
+pub fn close(fd: i32) -> Result<()> {
+    unsafe { map_c_int(libc::close(fd)).map(|_| ()) }
+}
+
+#[inline(always)]
+pub fn memfd_create(name: *const u8, flags: u32) -> Result<i32> {
+    unsafe {
+        map_c_long(libc::syscall(
+            libc::SYS_memfd_create,
+            name.cast::<libc::c_char>(),
+            flags as libc::c_uint,
+        ))
+        .map(|fd| fd as i32)
+    }
+}
+
+#[inline(always)]
+pub fn ftruncate(fd: i32, len: u64) -> Result<()> {
+    let len = to_off_t(len)?;
+    unsafe { map_c_int(libc::ftruncate(fd, len)).map(|_| ()) }
+}
+
+#[inline(always)]
+pub fn fallocate(fd: i32, mode: u32, offset: u64, len: u64) -> Result<()> {
+    let offset = to_off_t(offset)?;
+    let len = to_off_t(len)?;
+    unsafe { map_c_int(libc::fallocate(fd, mode as libc::c_int, offset, len)).map(|_| ()) }
+}
+
+#[inline(always)]
+pub fn getrandom(buf: &mut [u8], flags: u32) -> Result<usize> {
+    let result =
+        unsafe { libc::getrandom(buf.as_mut_ptr().cast(), buf.len(), flags as libc::c_uint) };
+    if result == -1 {
+        Err(last_error())
+    } else {
+        Ok(result as usize)
+    }
+}
+
+#[inline(always)]
+/// # Safety
+///
+/// `uaddr` must be valid for the kernel to read as a futex word for the duration of the call.
+pub unsafe fn futex_wait(uaddr: *const u32, op: u32, expected: u32) -> Result<()> {
+    unsafe {
+        map_c_long(libc::syscall(
+            libc::SYS_futex,
+            uaddr,
+            op as libc::c_int,
+            expected,
+            null_mut::<libc::timespec>(),
+            0,
+            0,
+        ))
+        .map(|_| ())
+    }
+}
+
+#[inline(always)]
+/// # Safety
+///
+/// `uaddr` must be valid for the kernel to access as a futex word for the duration of the call.
+pub unsafe fn futex_wake(uaddr: *const u32, op: u32, count: u32) -> Result<u32> {
+    unsafe {
+        map_c_long(libc::syscall(
+            libc::SYS_futex,
+            uaddr,
+            op as libc::c_int,
+            count,
+            null_mut::<libc::timespec>(),
+            0,
+            0,
+        ))
+        .map(|woken| woken as u32)
+    }
+}
+
+#[inline(always)]
+/// # Safety
+///
+/// The caller must uphold the platform `mmap(2)` contract for the provided arguments and manage
+/// any returned mapping according to Rust aliasing and lifetime rules.
+pub unsafe fn mmap(
+    addr: *mut u8,
+    len: usize,
+    prot: u32,
+    flags: u32,
+    fd: i32,
+    offset: u64,
+) -> Result<*mut u8> {
+    let offset = to_off_t(offset)?;
+    let result = unsafe {
+        libc::mmap(
+            addr.cast(),
+            len,
+            prot as libc::c_int,
+            flags as libc::c_int,
+            fd,
+            offset,
+        )
+    };
+    if result == libc::MAP_FAILED {
+        Err(last_error())
+    } else {
+        Ok(result.cast())
+    }
+}
+
+#[inline(always)]
+/// # Safety
+///
+/// The caller must later unmap the returned memory exactly once.
+pub unsafe fn map_anonymous(len: usize, prot: u32) -> Result<*mut u8> {
+    unsafe { mmap(null_mut(), len, prot, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) }
+}
+
+#[inline(always)]
+/// # Safety
+///
+/// `addr..addr+len` must refer to a valid mapped region.
+pub unsafe fn mprotect(addr: *mut u8, len: usize, prot: u32) -> Result<()> {
+    unsafe { map_c_int(libc::mprotect(addr.cast(), len, prot as libc::c_int)).map(|_| ()) }
+}
+
+#[inline(always)]
+/// # Safety
+///
+/// `addr..addr+len` must refer to a valid mapping that can be unmapped exactly once.
+pub unsafe fn munmap(addr: *mut u8, len: usize) -> Result<()> {
+    unsafe { map_c_int(libc::munmap(addr.cast(), len)).map(|_| ()) }
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/rng.rs

@@ -0,0 +1,96 @@
+use super::platform;
+use super::raw_sys;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct Mwc64 {
+    x: u64,
+    c: u64,
+    t: u64,
+    value: u64,
+    index: u8,
+}
+
+impl Mwc64 {
+    #[inline(always)]
+    pub const fn new(seed1: u64, seed2: u64) -> Self {
+        Self {
+            x: (seed1 << 32).wrapping_add(seed2),
+            c: 123_456_123_456_123_456,
+            t: 0,
+            value: 0,
+            index: 2,
+        }
+    }
+
+    #[inline(always)]
+    pub fn from_os_seed() -> raw_sys::Result<Self> {
+        let mut buf = [0u8; 16];
+        let mut filled = 0usize;
+        while filled < buf.len() {
+            let read = platform::getrandom(&mut buf[filled..], 0)?;
+            if read == 0 {
+                return Err(raw_sys::Error(5));
+            }
+            filled += read;
+        }
+
+        let seed1 = u64::from_ne_bytes(buf[0..8].try_into().unwrap());
+        let seed2 = u64::from_ne_bytes(buf[8..16].try_into().unwrap());
+        Ok(Self::new(seed1.max(1), seed2.max(1)))
+    }
+
+    #[inline(always)]
+    fn next_block(&mut self) -> u64 {
+        self.t = (self.x << 58).wrapping_add(self.c);
+        self.c = self.x >> 6;
+        self.x = self.x.wrapping_add(self.t);
+        self.c = self.c.wrapping_add((self.x < self.t) as u64);
+        self.x
+    }
+
+    #[inline(always)]
+    pub fn next_u32(&mut self) -> u32 {
+        if self.index == 2 {
+            self.value = self.next_block();
+            self.index = 0;
+        }
+
+        let shift = (self.index as u32) * 32;
+        let value = (self.value >> shift) as u32;
+        self.index += 1;
+        value
+    }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct Mwc {
+    inner: Mwc64,
+}
+
+impl Mwc {
+    #[inline(always)]
+    pub const fn new(seed1: u64, seed2: u64) -> Self {
+        Self {
+            inner: Mwc64::new(seed1, seed2),
+        }
+    }
+
+    #[inline(always)]
+    pub fn from_os_seed() -> raw_sys::Result<Self> {
+        Ok(Self {
+            inner: Mwc64::from_os_seed()?,
+        })
+    }
+
+    #[inline(always)]
+    pub fn next_u32(&mut self) -> u32 {
+        self.inner.next_u32()
+    }
+
+    #[inline(always)]
+    pub fn in_range(&mut self, min: usize, max: usize) -> usize {
+        debug_assert!(min <= max);
+        let range = 1 + max - min;
+        min + ((((self.next_u32() as u64) * (range as u64)) >> 32) as usize)
+    }
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/shuffle.rs

@@ -0,0 +1,204 @@
+use super::bitmap::RelaxedBitmap;
+use super::constants::{
+    MAX_OBJECT_SLOTS_PER_SPAN, MAX_SHUFFLE_VECTOR_LENGTH, MIN_OBJECT_SIZE,
+    MIN_SHUFFLE_VECTOR_LENGTH,
+};
+use super::page::page_size;
+use super::rng::Mwc;
+
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+pub struct ShuffleEntry {
+    pub miniheap_offset: u16,
+    pub slot_index: u16,
+}
+
+impl ShuffleEntry {
+    pub const EMPTY: Self = Self {
+        miniheap_offset: u16::MAX,
+        slot_index: u16::MAX,
+    };
+
+    #[inline(always)]
+    pub const fn new(miniheap_offset: u16, slot_index: u16) -> Self {
+        Self {
+            miniheap_offset,
+            slot_index,
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct ShuffleVector {
+    entries: [ShuffleEntry; MAX_SHUFFLE_VECTOR_LENGTH],
+    max_count: u16,
+    off: u16,
+    prng: Mwc,
+}
+
+impl ShuffleVector {
+    #[inline]
+    pub fn for_object_size(object_size: usize, seed1: u64, seed2: u64) -> Self {
+        Self::with_capacity(Self::capacity_for_object_size(object_size), seed1, seed2)
+    }
+
+    #[inline]
+    pub fn with_capacity(max_count: usize, seed1: u64, seed2: u64) -> Self {
+        assert!(max_count <= MAX_SHUFFLE_VECTOR_LENGTH);
+        Self {
+            entries: [ShuffleEntry::EMPTY; MAX_SHUFFLE_VECTOR_LENGTH],
+            max_count: max_count as u16,
+            off: max_count as u16,
+            prng: Mwc::new(seed1.max(1), seed2.max(1)),
+        }
+    }
+
+    #[inline(always)]
+    pub fn capacity_for_object_size(object_size: usize) -> usize {
+        let size = if object_size < MIN_OBJECT_SIZE {
+            MIN_OBJECT_SIZE
+        } else {
+            object_size
+        };
+        let per_page = page_size() / size;
+        let with_min = if per_page < MIN_SHUFFLE_VECTOR_LENGTH {
+            MIN_SHUFFLE_VECTOR_LENGTH
+        } else {
+            per_page
+        };
+        if with_min > MAX_OBJECT_SLOTS_PER_SPAN {
+            MAX_OBJECT_SLOTS_PER_SPAN
+        } else {
+            with_min
+        }
+    }
+
+    #[inline(always)]
+    pub const fn max_count(&self) -> usize {
+        self.max_count as usize
+    }
+
+    #[inline(always)]
+    pub const fn len(&self) -> usize {
+        self.max_count() - self.off as usize
+    }
+
+    #[inline(always)]
+    pub const fn is_empty(&self) -> bool {
+        self.off == self.max_count
+    }
+
+    #[inline(always)]
+    pub const fn is_full(&self) -> bool {
+        self.off == 0
+    }
+
+    #[inline(always)]
+    pub const fn is_exhausted(&self) -> bool {
+        self.off >= self.max_count
+    }
+
+    #[inline(always)]
+    pub fn clear(&mut self) {
+        self.off = self.max_count;
+    }
+
+    #[inline(always)]
+    pub fn active_entries(&self) -> &[ShuffleEntry] {
+        &self.entries[self.off as usize..self.max_count as usize]
+    }
+
+    #[inline]
+    pub fn count_entries_for_offset(&self, miniheap_offset: u16) -> usize {
+        self.active_entries()
+            .iter()
+            .filter(|entry| entry.miniheap_offset == miniheap_offset)
+            .count()
+    }
+
+    #[inline]
+    pub fn push(&mut self, entry: ShuffleEntry) {
+        assert!(self.off > 0);
+        self.off -= 1;
+        let inserted = self.off as usize;
+        self.entries[inserted] = entry;
+        let swap_index = self.prng.in_range(inserted, self.max_count() - 1);
+        self.entries.swap(inserted, swap_index);
+    }
+
+    #[inline]
+    pub fn pop(&mut self) -> Option<ShuffleEntry> {
+        if self.is_exhausted() {
+            return None;
+        }
+        let idx = self.off as usize;
+        let value = self.entries[idx];
+        self.off += 1;
+        Some(value)
+    }
+
+    #[inline]
+    pub fn refill_from_bitmap(
+        &mut self,
+        miniheap_offset: u16,
+        bitmap: &mut RelaxedBitmap,
+    ) -> usize {
+        let mut free_bits = *bitmap;
+        free_bits.invert_masked();
+        bitmap.set_all();
+
+        let mut added = 0usize;
+        for slot in free_bits.iter_set_bits() {
+            if self.is_full() {
+                let _ = bitmap.unset(slot);
+                continue;
+            }
+
+            self.off -= 1;
+            self.entries[self.off as usize] = ShuffleEntry::new(miniheap_offset, slot as u16);
+            added += 1;
+        }
+
+        if added > 1 {
+            self.shuffle_active();
+        }
+
+        added
+    }
+
+    #[inline]
+    pub fn refill_from_heap(
+        &mut self,
+        miniheap_offset: u16,
+        heap: &super::miniheap::MiniHeap,
+    ) -> usize {
+        let free_bits = heap.bitmap().take_free_bits();
+        let mut added = 0usize;
+        for slot in free_bits.iter_set_bits() {
+            if self.is_full() {
+                let _ = heap.free_offset(slot);
+                continue;
+            }
+
+            self.off -= 1;
+            self.entries[self.off as usize] = ShuffleEntry::new(miniheap_offset, slot as u16);
+            added += 1;
+        }
+
+        if added > 1 {
+            self.shuffle_active();
+        }
+
+        added
+    }
+
+    fn shuffle_active(&mut self) {
+        let start = self.off as usize;
+        let end = self.max_count();
+        let mut i = start;
+        while i < end {
+            let swap_index = self.prng.in_range(i, end - 1);
+            self.entries.swap(i, swap_index);
+            i += 1;
+        }
+    }
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/size_map.rs

@@ -0,0 +1,73 @@
+use super::constants::MAX_SMALL_ALLOCATION;
+
+pub const NUM_SIZE_CLASSES: usize = 25;
+
+pub const CLASS_TO_SIZE: [usize; NUM_SIZE_CLASSES] = [
+    16, 16, 32, 48, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, 448, 512, 640, 768, 896,
+    1024, 2048, 4096, 8192, 16384,
+];
+
+#[inline(always)]
+pub const fn byte_size_for_class(class: u8) -> usize {
+    CLASS_TO_SIZE[class as usize]
+}
+
+#[inline(always)]
+pub const fn size_class_for(size: usize) -> Option<u8> {
+    if size <= 16 {
+        Some(1)
+    } else if size <= 32 {
+        Some(2)
+    } else if size <= 48 {
+        Some(3)
+    } else if size <= 64 {
+        Some(4)
+    } else if size <= 80 {
+        Some(5)
+    } else if size <= 96 {
+        Some(6)
+    } else if size <= 112 {
+        Some(7)
+    } else if size <= 128 {
+        Some(8)
+    } else if size <= 160 {
+        Some(9)
+    } else if size <= 192 {
+        Some(10)
+    } else if size <= 224 {
+        Some(11)
+    } else if size <= 256 {
+        Some(12)
+    } else if size <= 320 {
+        Some(13)
+    } else if size <= 384 {
+        Some(14)
+    } else if size <= 448 {
+        Some(15)
+    } else if size <= 512 {
+        Some(16)
+    } else if size <= 640 {
+        Some(17)
+    } else if size <= 768 {
+        Some(18)
+    } else if size <= 896 {
+        Some(19)
+    } else if size <= 1024 {
+        Some(20)
+    } else if size <= 2048 {
+        Some(21)
+    } else if size <= 4096 {
+        Some(22)
+    } else if size <= 8192 {
+        Some(23)
+    } else if size <= 16384 {
+        Some(24)
+    } else {
+        None
+    }
+}
+
+#[inline(always)]
+pub const fn is_small_allocation(size: usize) -> bool {
+    size <= MAX_SMALL_ALLOCATION
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/span.rs

@@ -0,0 +1,45 @@
+pub const SPAN_CLASS_COUNT: u32 = 256;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct Span {
+    pub offset: u32,
+    pub length: u32,
+}
+
+impl Span {
+    #[inline(always)]
+    pub const fn new(offset: u32, length: u32) -> Self {
+        Self { offset, length }
+    }
+
+    #[inline(always)]
+    pub const fn empty(self) -> bool {
+        self.length == 0
+    }
+
+    #[inline(always)]
+    pub fn split_after(&mut self, count: u32) -> Self {
+        assert!(count <= self.length);
+        let rest = Self {
+            offset: self.offset + count,
+            length: self.length - count,
+        };
+        self.length = count;
+        rest
+    }
+
+    #[inline(always)]
+    pub const fn span_class(self) -> u32 {
+        let length = if self.length > SPAN_CLASS_COUNT {
+            SPAN_CLASS_COUNT
+        } else {
+            self.length
+        };
+        length - 1
+    }
+
+    #[inline(always)]
+    pub fn byte_length_for_page_size(self, page_size: usize) -> usize {
+        self.length as usize * page_size
+    }
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/stats.rs

@@ -0,0 +1,267 @@
+use core::sync::atomic::{AtomicU64, Ordering};
+
+use super::page::page_size;
+
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+pub struct CompactionEstimate {
+    pub candidate_heaps: u32,
+    pub candidate_pages: u32,
+    pub candidate_free_bytes: usize,
+    pub best_case_meshes: u32,
+    pub best_case_reclaimable_pages: u32,
+    pub best_case_reclaimable_bytes: usize,
+}
+
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+pub struct MeshStats {
+    pub arena_size: usize,
+    pub reserved_bytes: usize,
+    pub reusable_span_count: u32,
+    pub reusable_span_bytes: usize,
+    pub live_miniheaps: u32,
+    pub live_small_heaps: u32,
+    pub partial_small_heaps: u32,
+    pub full_small_heaps: u32,
+    pub meshed_small_heaps: u32,
+    pub reusable_small_heaps: u32,
+    pub live_large_allocations: u32,
+    pub live_small_bytes: usize,
+    pub live_large_bytes: usize,
+    pub retained_small_span_bytes: usize,
+    pub retained_large_span_bytes: usize,
+    pub virtual_small_span_bytes: usize,
+    pub small_allocations: u64,
+    pub small_deallocations: u64,
+    pub large_allocations: u64,
+    pub large_deallocations: u64,
+    pub compact_calls: u64,
+    pub meshes_performed: u64,
+    pub meshed_pages: u64,
+    pub meshed_bytes: u64,
+    pub compaction: CompactionEstimate,
+}
+
+impl MeshStats {
+    #[inline(always)]
+    pub const fn live_bytes(&self) -> usize {
+        self.live_small_bytes + self.live_large_bytes
+    }
+
+    #[inline(always)]
+    pub const fn retained_bytes(&self) -> usize {
+        self.retained_small_span_bytes + self.retained_large_span_bytes
+    }
+
+    #[inline(always)]
+    pub const fn small_fragmentation_bytes(&self) -> usize {
+        self.retained_small_span_bytes
+            .saturating_sub(self.live_small_bytes)
+    }
+
+    #[inline(always)]
+    pub const fn mesh_alias_bytes(&self) -> usize {
+        self.virtual_small_span_bytes
+            .saturating_sub(self.retained_small_span_bytes)
+    }
+
+    pub fn compaction_advice(&self) -> CompactionAdvice {
+        let fragmented = self.small_fragmentation_bytes();
+        let retained = self.retained_small_span_bytes;
+        let fragmentation_percent = fragmented
+            .saturating_mul(100)
+            .checked_div(retained)
+            .unwrap_or(0)
+            .min(100) as u8;
+
+        let reclaimable = self.compaction.best_case_reclaimable_bytes;
+        let page = page_size();
+        let recommendation = if self.compaction.best_case_meshes == 0 || reclaimable < page {
+            CompactionRecommendation::Idle
+        } else if reclaimable >= page * 4
+            && (fragmentation_percent >= 20 || reclaimable.saturating_mul(4) >= retained.max(page))
+        {
+            CompactionRecommendation::Compact
+        } else if reclaimable >= page
+            && (fragmentation_percent >= 10 || self.compaction.candidate_heaps >= 2)
+        {
+            CompactionRecommendation::Consider
+        } else {
+            CompactionRecommendation::Idle
+        };
+
+        CompactionAdvice {
+            recommendation,
+            fragmentation_bytes: fragmented,
+            fragmentation_percent,
+            candidate_heaps: self.compaction.candidate_heaps,
+            candidate_free_bytes: self.compaction.candidate_free_bytes,
+            best_case_meshes: self.compaction.best_case_meshes,
+            best_case_reclaimable_pages: self.compaction.best_case_reclaimable_pages,
+            best_case_reclaimable_bytes: reclaimable,
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+pub struct CompactionAdvice {
+    pub recommendation: CompactionRecommendation,
+    pub fragmentation_bytes: usize,
+    pub fragmentation_percent: u8,
+    pub candidate_heaps: u32,
+    pub candidate_free_bytes: usize,
+    pub best_case_meshes: u32,
+    pub best_case_reclaimable_pages: u32,
+    pub best_case_reclaimable_bytes: usize,
+}
+
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+pub enum CompactionRecommendation {
+    #[default]
+    Idle,
+    Consider,
+    Compact,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct RuntimeCompactionPolicy {
+    pub minimum_recommendation: CompactionRecommendation,
+    pub min_fragmentation_bytes: usize,
+    pub min_reclaimable_bytes: usize,
+    pub min_candidate_heaps: u32,
+}
+
+impl RuntimeCompactionPolicy {
+    pub fn should_compact(&self, advice: &CompactionAdvice) -> bool {
+        recommendation_rank(advice.recommendation)
+            >= recommendation_rank(self.minimum_recommendation)
+            && advice.fragmentation_bytes >= self.min_fragmentation_bytes
+            && advice.best_case_reclaimable_bytes >= self.min_reclaimable_bytes
+            && advice.candidate_heaps >= self.min_candidate_heaps
+    }
+}
+
+impl Default for RuntimeCompactionPolicy {
+    fn default() -> Self {
+        let page = page_size();
+        Self {
+            minimum_recommendation: CompactionRecommendation::Consider,
+            min_fragmentation_bytes: page,
+            min_reclaimable_bytes: page,
+            min_candidate_heaps: 2,
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum RuntimeCompactionResult {
+    Compacted {
+        meshes: usize,
+        advice: CompactionAdvice,
+    },
+    Skipped {
+        reason: CompactionSkipReason,
+        advice: Option<CompactionAdvice>,
+    },
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum CompactionSkipReason {
+    AllocatorUnavailable,
+    ThreadUnavailable,
+    NotAtSafepoint,
+    ThreadsActive,
+    Policy,
+}
+
+#[inline(always)]
+const fn recommendation_rank(recommendation: CompactionRecommendation) -> u8 {
+    match recommendation {
+        CompactionRecommendation::Idle => 0,
+        CompactionRecommendation::Consider => 1,
+        CompactionRecommendation::Compact => 2,
+    }
+}
+
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+pub(crate) struct CounterSnapshot {
+    pub small_allocations: u64,
+    pub small_deallocations: u64,
+    pub large_allocations: u64,
+    pub large_deallocations: u64,
+    pub compact_calls: u64,
+    pub meshes_performed: u64,
+    pub meshed_pages: u64,
+    pub meshed_bytes: u64,
+}
+
+#[derive(Debug)]
+pub(crate) struct StatsState {
+    small_allocations: AtomicU64,
+    small_deallocations: AtomicU64,
+    large_allocations: AtomicU64,
+    large_deallocations: AtomicU64,
+    compact_calls: AtomicU64,
+    meshes_performed: AtomicU64,
+    meshed_pages: AtomicU64,
+    meshed_bytes: AtomicU64,
+}
+
+impl StatsState {
+    pub const fn new() -> Self {
+        Self {
+            small_allocations: AtomicU64::new(0),
+            small_deallocations: AtomicU64::new(0),
+            large_allocations: AtomicU64::new(0),
+            large_deallocations: AtomicU64::new(0),
+            compact_calls: AtomicU64::new(0),
+            meshes_performed: AtomicU64::new(0),
+            meshed_pages: AtomicU64::new(0),
+            meshed_bytes: AtomicU64::new(0),
+        }
+    }
+
+    #[inline(always)]
+    pub fn record_small_allocation(&self) {
+        self.small_allocations.fetch_add(1, Ordering::Relaxed);
+    }
+
+    #[inline(always)]
+    pub fn record_small_deallocation(&self) {
+        self.small_deallocations.fetch_add(1, Ordering::Relaxed);
+    }
+
+    #[inline(always)]
+    pub fn record_large_allocation(&self) {
+        self.large_allocations.fetch_add(1, Ordering::Relaxed);
+    }
+
+    #[inline(always)]
+    pub fn record_large_deallocation(&self) {
+        self.large_deallocations.fetch_add(1, Ordering::Relaxed);
+    }
+
+    #[inline(always)]
+    pub fn record_compact_call(&self) {
+        self.compact_calls.fetch_add(1, Ordering::Relaxed);
+    }
+
+    #[inline(always)]
+    pub fn record_mesh(&self, pages: u32, bytes: usize) {
+        self.meshes_performed.fetch_add(1, Ordering::Relaxed);
+        self.meshed_pages.fetch_add(pages as u64, Ordering::Relaxed);
+        self.meshed_bytes.fetch_add(bytes as u64, Ordering::Relaxed);
+    }
+
+    pub fn snapshot(&self) -> CounterSnapshot {
+        CounterSnapshot {
+            small_allocations: self.small_allocations.load(Ordering::Relaxed),
+            small_deallocations: self.small_deallocations.load(Ordering::Relaxed),
+            large_allocations: self.large_allocations.load(Ordering::Relaxed),
+            large_deallocations: self.large_deallocations.load(Ordering::Relaxed),
+            compact_calls: self.compact_calls.load(Ordering::Relaxed),
+            meshes_performed: self.meshes_performed.load(Ordering::Relaxed),
+            meshed_pages: self.meshed_pages.load(Ordering::Relaxed),
+            meshed_bytes: self.meshed_bytes.load(Ordering::Relaxed),
+        }
+    }
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/sync.rs

@@ -0,0 +1,137 @@
+use core::sync::atomic::{AtomicU32, Ordering};
+
+use super::platform;
+use super::raw_sys;
+
+const UNLOCKED: u32 = 0;
+const LOCKED: u32 = 1;
+const CONTENDED: u32 = 2;
+
+#[derive(Debug)]
+pub struct FutexMutex {
+    state: AtomicU32,
+}
+
+impl FutexMutex {
+    pub const fn new() -> Self {
+        Self {
+            state: AtomicU32::new(UNLOCKED),
+        }
+    }
+
+    pub fn lock(&self) -> FutexMutexGuard<'_> {
+        if self
+            .state
+            .compare_exchange(UNLOCKED, LOCKED, Ordering::Acquire, Ordering::Relaxed)
+            .is_err()
+        {
+            self.lock_contended();
+        }
+
+        FutexMutexGuard { mutex: self }
+    }
+
+    fn lock_contended(&self) {
+        loop {
+            if self.state.swap(CONTENDED, Ordering::Acquire) == UNLOCKED {
+                return;
+            }
+
+            match unsafe {
+                platform::futex_wait(self.state_ptr(), raw_sys::FUTEX_WAIT_PRIVATE, CONTENDED)
+            } {
+                Ok(()) => {}
+                Err(error) if matches!(error.errno(), raw_sys::EAGAIN | raw_sys::EINTR) => {}
+                Err(_) => {}
+            }
+        }
+    }
+
+    fn unlock(&self) {
+        if self.state.fetch_sub(1, Ordering::Release) != LOCKED {
+            self.state.store(UNLOCKED, Ordering::Release);
+            let _ =
+                unsafe { platform::futex_wake(self.state_ptr(), raw_sys::FUTEX_WAKE_PRIVATE, 1) };
+        }
+    }
+
+    #[inline(always)]
+    fn state_ptr(&self) -> *const u32 {
+        (&self.state as *const AtomicU32).cast::<u32>()
+    }
+}
+
+impl Default for FutexMutex {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+pub struct FutexMutexGuard<'a> {
+    mutex: &'a FutexMutex,
+}
+
+impl Drop for FutexMutexGuard<'_> {
+    fn drop(&mut self) {
+        self.mutex.unlock();
+    }
+}
+
+pub(crate) fn futex_wait_for_value(state: &AtomicU32, expected: u32) {
+    match unsafe {
+        platform::futex_wait(
+            (state as *const AtomicU32).cast::<u32>(),
+            raw_sys::FUTEX_WAIT_PRIVATE,
+            expected,
+        )
+    } {
+        Ok(()) => {}
+        Err(error) if matches!(error.errno(), raw_sys::EAGAIN | raw_sys::EINTR) => {}
+        Err(_) => {}
+    }
+}
+
+pub(crate) fn futex_wake_all(state: &AtomicU32) {
+    let _ = unsafe {
+        platform::futex_wake(
+            (state as *const AtomicU32).cast::<u32>(),
+            raw_sys::FUTEX_WAKE_PRIVATE,
+            i32::MAX as u32,
+        )
+    };
+}
+
+#[cfg(test)]
+mod tests {
+    use core::sync::atomic::{AtomicUsize, Ordering};
+
+    use std::sync::Arc;
+    use std::thread;
+    use std::vec::Vec;
+
+    use super::FutexMutex;
+
+    #[test]
+    fn futex_mutex_serializes_multiple_threads() {
+        let mutex = Arc::new(FutexMutex::new());
+        let counter = Arc::new(AtomicUsize::new(0));
+        let mut threads = Vec::new();
+
+        for _ in 0..4 {
+            let mutex = Arc::clone(&mutex);
+            let counter = Arc::clone(&counter);
+            threads.push(thread::spawn(move || {
+                for _ in 0..5000 {
+                    let _guard = mutex.lock();
+                    counter.fetch_add(1, Ordering::Relaxed);
+                }
+            }));
+        }
+
+        for thread in threads {
+            thread.join().unwrap();
+        }
+
+        assert_eq!(counter.load(Ordering::Acquire), 20_000);
+    }
+}

lib/runtime/src/platform/linux_x86_64/mesh_alloc/thread_local_heap.rs

@@ -0,0 +1,124 @@
+use core::array;
+use core::ptr::null;
+
+use super::constants::{MAX_ATTACHED_MINIHEAPS_PER_CLASS, MIN_OBJECT_SIZE, NUM_SIZE_CLASSES};
+use super::miniheap::{MiniHeap, MiniHeapId};
+use super::platform;
+use super::raw_sys;
+use super::rng::Mwc;
+use super::shuffle::ShuffleVector;
+use super::size_map::byte_size_for_class;
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct ClassState {
+    pub shuffle: ShuffleVector,
+    pub attached_ids: [MiniHeapId; MAX_ATTACHED_MINIHEAPS_PER_CLASS],
+    pub attached_heaps: [*const MiniHeap; MAX_ATTACHED_MINIHEAPS_PER_CLASS],
+    pub attached_len: u8,
+    pub attached_cursor: u8,
+}
+
+impl ClassState {
+    fn new(object_size: usize, seed1: u64, seed2: u64) -> Self {
+        Self {
+            shuffle: ShuffleVector::for_object_size(object_size, seed1, seed2),
+            attached_ids: [MiniHeapId::new(0); MAX_ATTACHED_MINIHEAPS_PER_CLASS],
+            attached_heaps: [null(); MAX_ATTACHED_MINIHEAPS_PER_CLASS],
+            attached_len: 0,
+            attached_cursor: 0,
+        }
+    }
+
+    #[inline(always)]
+    pub fn clear_attached(&mut self) {
+        self.attached_len = 0;
+        self.attached_cursor = 0;
+        self.shuffle.clear();
+
+        let mut index = 0usize;
+        while index < MAX_ATTACHED_MINIHEAPS_PER_CLASS {
+            self.attached_ids[index] = MiniHeapId::new(0);
+            self.attached_heaps[index] = null();
+            index += 1;
+        }
+    }
+
+    #[inline(always)]
+    pub fn attached_full(&self) -> bool {
+        self.attached_len as usize == MAX_ATTACHED_MINIHEAPS_PER_CLASS
+    }
+
+    #[inline(always)]
+    pub fn push_attached(&mut self, id: MiniHeapId, heap: *const MiniHeap) -> Option<u8> {
+        if self.attached_full() {
+            return None;
+        }
+        let index = self.attached_len as usize;
+        self.attached_ids[index] = id;
+        self.attached_heaps[index] = heap;
+        self.attached_len += 1;
+        Some(index as u8)
+    }
+
+    #[inline(always)]
+    pub fn find_attached(&self, id: MiniHeapId) -> Option<u8> {
+        let len = self.attached_len as usize;
+        let mut i = 0usize;
+        while i < len {
+            if self.attached_ids[i] == id {
+                return Some(i as u8);
+            }
+            i += 1;
+        }
+        None
+    }
+
+    #[inline(always)]
+    pub fn heap_at(&self, index: usize) -> Option<&MiniHeap> {
+        if index >= self.attached_len as usize {
+            return None;
+        }
+        let heap = self.attached_heaps[index];
+        if heap.is_null() {
+            return None;
+        }
+
+        unsafe { Some(&*heap) }
+    }
+}
+
+#[derive(Debug)]
+pub struct ThreadLocalHeap {
+    thread_id: u32,
+    classes: [ClassState; NUM_SIZE_CLASSES],
+}
+
+impl ThreadLocalHeap {
+    pub fn new() -> raw_sys::Result<Self> {
+        let thread_id = platform::gettid()?;
+        let mut seed_rng = Mwc::from_os_seed()?;
+        let classes = array::from_fn(|class| {
+            let object_size = byte_size_for_class(class as u8).max(MIN_OBJECT_SIZE);
+            let seed1 = seed_rng.next_u32() as u64 + 1;
+            let seed2 = seed_rng.next_u32() as u64 + 1;
+            ClassState::new(object_size, seed1, seed2)
+        });
+
+        Ok(Self { thread_id, classes })
+    }
+
+    #[inline(always)]
+    pub const fn thread_id(&self) -> u32 {
+        self.thread_id
+    }
+
+    #[inline(always)]
+    pub(crate) fn class(&self, class: u8) -> &ClassState {
+        &self.classes[class as usize]
+    }
+
+    #[inline(always)]
+    pub(crate) fn class_mut(&mut self, class: u8) -> &mut ClassState {
+        &mut self.classes[class as usize]
+    }
+}

lib/runtime/src/platform/linux_x86_64/mod.rs

@@ -0,0 +1,4 @@
+pub mod mesh_alloc;
+pub mod reactor;
+pub mod runtime;
+pub(crate) mod uring;

lib/runtime/src/platform/linux_x86_64/reactor.rs

@@ -0,0 +1,340 @@
+use std::cell::Cell;
+use std::cell::RefCell;
+use std::collections::BTreeMap;
+use std::io;
+use std::os::fd::RawFd;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
+
+use super::uring::{IORING_OP_ASYNC_CANCEL, IoUring, IoUringCqe, IoUringSqe};
+
+const WAKE_TARGET_TOKEN: u64 = 1;
+const TOKEN_KIND_SHIFT: u64 = 56;
+const TOKEN_KIND_MASK: u64 = 0xff << TOKEN_KIND_SHIFT;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[repr(u8)]
+enum CompletionKind {
+    Timer = 1,
+    TimerRemove = 2,
+    NotifySend = 3,
+    Operation = 4,
+    OperationCancel = 5,
+}
+
+type CompletionHandler = Box<dyn FnOnce(IoUringCqe) + Send + 'static>;
+
+struct NotifierInner {
+    ring_fd: RawFd,
+    closed: AtomicBool,
+}
+
+impl NotifierInner {
+    fn notify(&self) -> io::Result<()> {
+        if self.closed.load(Ordering::Acquire) {
+            return Err(io::Error::new(
+                io::ErrorKind::BrokenPipe,
+                "target runtime ring is closed",
+            ));
+        }
+
+        IoUring::with_submitter(|ring| {
+            ring.submit_msg_ring(
+                self.ring_fd,
+                WAKE_TARGET_TOKEN,
+                1,
+                make_token(CompletionKind::NotifySend, 0),
+            )
+        })
+    }
+}
+
+#[derive(Clone)]
+pub struct ThreadNotifier {
+    inner: Arc<NotifierInner>,
+}
+
+impl ThreadNotifier {
+    pub fn notify(&self) -> io::Result<()> {
+        self.inner.notify()
+    }
+}
+
+#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)]
+pub struct ReadyEvents {
+    pub timer: bool,
+    pub wake: bool,
+}
+
+pub struct Reactor {
+    ring: IoUring,
+    notifier: Arc<NotifierInner>,
+    next_token: Cell<u64>,
+    active_timer_token: Cell<Option<u64>>,
+    pending_wakes: Cell<u64>,
+    pending_timers: Cell<u64>,
+    completions: RefCell<BTreeMap<u64, CompletionHandler>>,
+}
+
+pub fn create() -> io::Result<(Reactor, ThreadNotifier)> {
+    create_reactor()
+}
+
+pub fn create_reactor() -> io::Result<(Reactor, ThreadNotifier)> {
+    let ring = IoUring::new(64)?;
+    let notifier = Arc::new(NotifierInner {
+        ring_fd: ring.ring_fd(),
+        closed: AtomicBool::new(false),
+    });
+
+    Ok((
+        Reactor {
+            ring,
+            notifier: Arc::clone(&notifier),
+            next_token: Cell::new(1),
+            active_timer_token: Cell::new(None),
+            pending_wakes: Cell::new(0),
+            pending_timers: Cell::new(0),
+            completions: RefCell::new(BTreeMap::new()),
+        },
+        ThreadNotifier { inner: notifier },
+    ))
+}
+
+impl Reactor {
+    pub(crate) fn bind_current_thread(&self) {
+        self.ring.bind_current_thread();
+    }
+
+    pub(crate) fn unbind_current_thread(&self) {
+        self.ring.unbind_current_thread();
+    }
+
+    pub fn poll(&self) -> io::Result<Option<ReadyEvents>> {
+        let mut ready = ReadyEvents::default();
+        let saw_any = self
+            .ring
+            .drain_completions(|cqe| self.process_cqe(cqe, &mut ready));
+        if saw_any { Ok(Some(ready)) } else { Ok(None) }
+    }
+
+    pub fn wait(&self) -> io::Result<()> {
+        self.ring.wait_for_cqe()
+    }
+
+    pub fn rearm_timer(&self, deadline: Option<Duration>) -> io::Result<()> {
+        match (self.active_timer_token.get(), deadline) {
+            (Some(active), Some(deadline)) => {
+                self.ring.submit_timeout_update(active, deadline)?;
+            }
+            (Some(active), None) => {
+                self.active_timer_token.set(None);
+                self.ring
+                    .submit_timeout_remove(active, self.next_token(CompletionKind::TimerRemove))?;
+            }
+            (None, Some(deadline)) => {
+                let token = self.next_token(CompletionKind::Timer);
+                self.active_timer_token.set(Some(token));
+                self.ring.submit_timeout(token, deadline)?;
+            }
+            (None, None) => {}
+        }
+
+        Ok(())
+    }
+
+    pub(crate) fn submit_operation(
+        &self,
+        fill: impl FnOnce(&mut IoUringSqe),
+        on_complete: impl FnOnce(IoUringCqe) + Send + 'static,
+    ) -> io::Result<u64> {
+        let token = self.next_token(CompletionKind::Operation);
+        self.completions
+            .borrow_mut()
+            .insert(token, Box::new(on_complete));
+
+        if let Err(error) = self.ring.submit_with_token(token, fill) {
+            let _ = self.completions.borrow_mut().remove(&token);
+            return Err(error);
+        }
+
+        Ok(token)
+    }
+
+    pub(crate) fn cancel_operation(&self, token: u64) -> io::Result<()> {
+        self.ring
+            .submit_with_token(self.next_token(CompletionKind::OperationCancel), |sqe| {
+                sqe.opcode = IORING_OP_ASYNC_CANCEL;
+                sqe.fd = -1;
+                sqe.addr = token;
+            })
+    }
+
+    pub fn drain_wake(&self) -> io::Result<u64> {
+        let wakes = self.pending_wakes.replace(0);
+        if wakes == 0 {
+            Err(io::Error::new(
+                io::ErrorKind::WouldBlock,
+                "no wake completions are pending",
+            ))
+        } else {
+            Ok(wakes)
+        }
+    }
+
+    pub fn drain_timer(&self) -> io::Result<u64> {
+        let timers = self.pending_timers.replace(0);
+        if timers == 0 {
+            Err(io::Error::new(
+                io::ErrorKind::WouldBlock,
+                "no timer completions are pending",
+            ))
+        } else {
+            Ok(timers)
+        }
+    }
+
+    fn process_cqe(&self, cqe: IoUringCqe, ready: &mut ReadyEvents) {
+        if cqe.user_data == WAKE_TARGET_TOKEN {
+            ready.wake = true;
+            let wakes = cqe.res.max(1) as u64;
+            self.pending_wakes
+                .set(self.pending_wakes.get().saturating_add(wakes));
+            return;
+        }
+
+        match decode_token_kind(cqe.user_data) {
+            Some(CompletionKind::Timer) => {
+                if self.active_timer_token.get() == Some(cqe.user_data) {
+                    self.active_timer_token.set(None);
+                }
+                if cqe.res == -libc::ETIME {
+                    ready.timer = true;
+                    self.pending_timers
+                        .set(self.pending_timers.get().saturating_add(1));
+                }
+            }
+            Some(CompletionKind::Operation) => {
+                if let Some(callback) = self.completions.borrow_mut().remove(&cqe.user_data) {
+                    callback(cqe);
+                }
+            }
+            Some(CompletionKind::TimerRemove)
+            | Some(CompletionKind::NotifySend)
+            | Some(CompletionKind::OperationCancel)
+            | None => {}
+        }
+    }
+
+    fn next_token(&self, kind: CompletionKind) -> u64 {
+        let seq = self.next_token.get();
+        self.next_token.set(seq.wrapping_add(1));
+        make_token(kind, seq)
+    }
+}
+
+impl Drop for Reactor {
+    fn drop(&mut self) {
+        self.notifier.closed.store(true, Ordering::Release);
+    }
+}
+
+pub fn monotonic_now() -> io::Result<Duration> {
+    let mut now = std::mem::MaybeUninit::<libc::timespec>::uninit();
+    let result = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, now.as_mut_ptr()) };
+    if result == -1 {
+        return Err(io::Error::last_os_error());
+    }
+
+    let now = unsafe { now.assume_init() };
+    Ok(Duration::new(now.tv_sec as u64, now.tv_nsec as u32))
+}
+
+fn make_token(kind: CompletionKind, seq: u64) -> u64 {
+    ((kind as u64) << TOKEN_KIND_SHIFT) | (seq & !TOKEN_KIND_MASK)
+}
+
+fn decode_token_kind(token: u64) -> Option<CompletionKind> {
+    match ((token & TOKEN_KIND_MASK) >> TOKEN_KIND_SHIFT) as u8 {
+        1 => Some(CompletionKind::Timer),
+        2 => Some(CompletionKind::TimerRemove),
+        3 => Some(CompletionKind::NotifySend),
+        4 => Some(CompletionKind::Operation),
+        5 => Some(CompletionKind::OperationCancel),
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{create_reactor, monotonic_now};
+    use std::thread;
+    use std::time::Duration;
+
+    #[test]
+    fn notifier_wakes_target_ring() {
+        let (sender, _) = create_reactor().expect("sender reactor should initialize");
+        sender.bind_current_thread();
+
+        let (target, notifier) = create_reactor().expect("target reactor should initialize");
+        notifier.notify().expect("notify should succeed");
+
+        let ready = loop {
+            if let Some(ready) = target.poll().expect("poll should succeed") {
+                break ready;
+            }
+            thread::sleep(Duration::from_millis(1));
+        };
+
+        assert!(ready.wake);
+        assert!(!ready.timer);
+        assert_eq!(target.drain_wake().expect("wake drain should succeed"), 1);
+        sender.unbind_current_thread();
+    }
+
+    #[test]
+    fn notifier_wakes_target_ring_from_plain_thread() {
+        let (target, notifier) = create_reactor().expect("target reactor should initialize");
+
+        thread::spawn(move || {
+            notifier.notify().expect("notify should succeed");
+        })
+        .join()
+        .expect("notifier thread should exit cleanly");
+
+        let ready = loop {
+            if let Some(ready) = target.poll().expect("poll should succeed") {
+                break ready;
+            }
+            thread::sleep(Duration::from_millis(1));
+        };
+
+        assert!(ready.wake);
+        assert!(!ready.timer);
+        assert_eq!(target.drain_wake().expect("wake drain should succeed"), 1);
+    }
+
+    #[test]
+    fn timeout_reports_deadlines() {
+        let (reactor, _notifier) = create_reactor().expect("reactor should initialize");
+        let deadline = monotonic_now().expect("clock should work") + Duration::from_millis(20);
+        reactor
+            .rearm_timer(Some(deadline))
+            .expect("timer should arm");
+
+        let ready = loop {
+            if let Some(ready) = reactor.poll().expect("poll should succeed") {
+                break ready;
+            }
+            thread::sleep(Duration::from_millis(5));
+        };
+
+        assert!(ready.timer);
+        assert!(!ready.wake);
+        assert_eq!(
+            reactor.drain_timer().expect("timer drain should succeed"),
+            1
+        );
+    }
+}

lib/runtime/src/platform/linux_x86_64/uring.rs

@@ -0,0 +1,478 @@
+use std::cell::Cell;
+use std::io;
+use std::os::fd::RawFd;
+use std::ptr;
+use std::sync::atomic::{Ordering, compiler_fence};
+use std::sync::{Mutex, OnceLock};
+use std::time::Duration;
+
+const IORING_OFF_SQ_RING: libc::off_t = 0;
+const IORING_OFF_CQ_RING: libc::off_t = 0x0800_0000;
+const IORING_OFF_SQES: libc::off_t = 0x1000_0000;
+
+const IORING_ENTER_GETEVENTS: u32 = 1 << 0;
+const IORING_SETUP_CLAMP: u32 = 1 << 4;
+
+const IORING_FEAT_SINGLE_MMAP: u32 = 1 << 0;
+
+pub(crate) const IORING_OP_FSYNC: u8 = 3;
+pub(crate) const IORING_OP_TIMEOUT: u8 = 11;
+pub(crate) const IORING_OP_TIMEOUT_REMOVE: u8 = 12;
+pub(crate) const IORING_OP_ACCEPT: u8 = 13;
+pub(crate) const IORING_OP_ASYNC_CANCEL: u8 = 14;
+pub(crate) const IORING_OP_CONNECT: u8 = 16;
+pub(crate) const IORING_OP_OPENAT: u8 = 18;
+pub(crate) const IORING_OP_CLOSE: u8 = 19;
+pub(crate) const IORING_OP_STATX: u8 = 21;
+pub(crate) const IORING_OP_READ: u8 = 22;
+pub(crate) const IORING_OP_WRITE: u8 = 23;
+pub(crate) const IORING_OP_SEND: u8 = 26;
+pub(crate) const IORING_OP_RECV: u8 = 27;
+pub(crate) const IORING_OP_SHUTDOWN: u8 = 34;
+pub(crate) const IORING_OP_RENAMEAT: u8 = 35;
+pub(crate) const IORING_OP_UNLINKAT: u8 = 36;
+pub(crate) const IORING_OP_MKDIRAT: u8 = 37;
+pub(crate) const IORING_OP_MSG_RING: u8 = 40;
+pub(crate) const IORING_OP_SOCKET: u8 = 45;
+pub(crate) const IORING_OP_FTRUNCATE: u8 = 55;
+pub(crate) const IORING_OP_BIND: u8 = 56;
+pub(crate) const IORING_OP_LISTEN: u8 = 57;
+
+const IORING_MSG_DATA: u64 = 0;
+pub(crate) const IORING_FSYNC_DATASYNC: u32 = 1 << 0;
+pub(crate) const IORING_TIMEOUT_ABS: u32 = 1 << 0;
+pub(crate) const IORING_TIMEOUT_UPDATE: u32 = 1 << 1;
+pub(crate) const IOSQE_CQE_SKIP_SUCCESS: u8 = 1 << 6;
+
+thread_local! {
+    static CURRENT_SUBMITTER: Cell<*const IoUring> = const { Cell::new(ptr::null()) };
+}
+
+static GLOBAL_SUBMITTER: OnceLock<Mutex<Option<IoUring>>> = OnceLock::new();
+
+#[repr(C)]
+#[derive(Default, Clone, Copy)]
+struct IoSqringOffsets {
+    head: u32,
+    tail: u32,
+    ring_mask: u32,
+    ring_entries: u32,
+    flags: u32,
+    dropped: u32,
+    array: u32,
+    resv1: u32,
+    user_addr: u64,
+}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy)]
+struct IoCqringOffsets {
+    head: u32,
+    tail: u32,
+    ring_mask: u32,
+    ring_entries: u32,
+    overflow: u32,
+    cqes: u32,
+    flags: u32,
+    resv1: u32,
+    user_addr: u64,
+}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy)]
+struct IoUringParams {
+    sq_entries: u32,
+    cq_entries: u32,
+    flags: u32,
+    sq_thread_cpu: u32,
+    sq_thread_idle: u32,
+    features: u32,
+    wq_fd: u32,
+    resv: [u32; 3],
+    sq_off: IoSqringOffsets,
+    cq_off: IoCqringOffsets,
+}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy)]
+pub(crate) struct IoUringSqe {
+    pub(crate) opcode: u8,
+    pub(crate) flags: u8,
+    pub(crate) ioprio: u16,
+    pub(crate) fd: i32,
+    pub(crate) off: u64,
+    pub(crate) addr: u64,
+    pub(crate) len: u32,
+    pub(crate) op_flags: u32,
+    pub(crate) user_data: u64,
+    pub(crate) buf_index: u16,
+    pub(crate) personality: u16,
+    pub(crate) file_index: i32,
+    pub(crate) pad2: [u64; 2],
+}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy)]
+pub(crate) struct IoUringCqe {
+    pub(crate) user_data: u64,
+    pub(crate) res: i32,
+    pub(crate) flags: u32,
+}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy)]
+struct KernelTimespec {
+    tv_sec: i64,
+    tv_nsec: i64,
+}
+
+pub(crate) struct IoUring {
+    ring_fd: RawFd,
+    sq_ring_ptr: *mut u8,
+    cq_ring_ptr: *mut u8,
+    sqes_ptr: *mut IoUringSqe,
+    sq_ring_size: usize,
+    cq_ring_size: usize,
+    sqes_size: usize,
+    single_mmap: bool,
+    sq_head: *mut u32,
+    sq_tail: *mut u32,
+    sq_ring_mask: *mut u32,
+    sq_ring_entries: *mut u32,
+    sq_array: *mut u32,
+    cq_head: *mut u32,
+    cq_tail: *mut u32,
+    cq_ring_mask: *mut u32,
+    cqes: *mut IoUringCqe,
+}
+
+impl IoUring {
+    pub(crate) fn new(entries: u32) -> io::Result<Self> {
+        let mut params = IoUringParams {
+            flags: IORING_SETUP_CLAMP,
+            ..IoUringParams::default()
+        };
+
+        let ring_fd = cvt_long(unsafe {
+            libc::syscall(
+                libc::SYS_io_uring_setup,
+                entries as libc::c_uint,
+                &mut params as *mut IoUringParams,
+            )
+        })? as RawFd;
+
+        let sq_ring_size =
+            params.sq_off.array as usize + params.sq_entries as usize * std::mem::size_of::<u32>();
+        let cq_ring_size = params.cq_off.cqes as usize
+            + params.cq_entries as usize * std::mem::size_of::<IoUringCqe>();
+        let single_mmap = params.features & IORING_FEAT_SINGLE_MMAP != 0;
+
+        let sq_ring_ptr = mmap_ring(
+            if single_mmap {
+                sq_ring_size.max(cq_ring_size)
+            } else {
+                sq_ring_size
+            },
+            ring_fd,
+            IORING_OFF_SQ_RING,
+        )?;
+        let cq_ring_ptr = if single_mmap {
+            sq_ring_ptr
+        } else {
+            mmap_ring(cq_ring_size, ring_fd, IORING_OFF_CQ_RING)?
+        };
+        let sqes_size = params.sq_entries as usize * std::mem::size_of::<IoUringSqe>();
+        let sqes_ptr = mmap_ring(sqes_size, ring_fd, IORING_OFF_SQES)? as *mut IoUringSqe;
+
+        Ok(Self {
+            ring_fd,
+            sq_ring_ptr,
+            cq_ring_ptr,
+            sqes_ptr,
+            sq_ring_size,
+            cq_ring_size,
+            sqes_size,
+            single_mmap,
+            sq_head: offset_ptr(sq_ring_ptr, params.sq_off.head),
+            sq_tail: offset_ptr(sq_ring_ptr, params.sq_off.tail),
+            sq_ring_mask: offset_ptr(sq_ring_ptr, params.sq_off.ring_mask),
+            sq_ring_entries: offset_ptr(sq_ring_ptr, params.sq_off.ring_entries),
+            sq_array: offset_ptr(sq_ring_ptr, params.sq_off.array),
+            cq_head: offset_ptr(cq_ring_ptr, params.cq_off.head),
+            cq_tail: offset_ptr(cq_ring_ptr, params.cq_off.tail),
+            cq_ring_mask: offset_ptr(cq_ring_ptr, params.cq_off.ring_mask),
+            cqes: offset_ptr(cq_ring_ptr, params.cq_off.cqes),
+        })
+    }
+
+    pub(crate) fn ring_fd(&self) -> RawFd {
+        self.ring_fd
+    }
+
+    pub(crate) fn bind_current_thread(&self) {
+        CURRENT_SUBMITTER.with(|submitter| submitter.set(self as *const Self));
+    }
+
+    pub(crate) fn unbind_current_thread(&self) {
+        CURRENT_SUBMITTER.with(|submitter| {
+            if ptr::eq(submitter.get(), self) {
+                submitter.set(ptr::null());
+            }
+        });
+    }
+
+    pub(crate) fn with_submitter<T>(f: impl FnOnce(&IoUring) -> io::Result<T>) -> io::Result<T> {
+        CURRENT_SUBMITTER.with(|submitter| {
+            let ptr = submitter.get();
+            if !ptr.is_null() {
+                let ring = unsafe { &*ptr };
+                return f(ring);
+            }
+
+            let mut ring = global_submitter()
+                .lock()
+                .expect("global io_uring submitter should not be poisoned");
+            if ring.is_none() {
+                *ring = Some(IoUring::new(64)?);
+            }
+
+            f(ring
+                .as_ref()
+                .expect("global submitter ring should initialize"))
+        })
+    }
+
+    pub(crate) fn submit_timeout(&self, token: u64, deadline: Duration) -> io::Result<()> {
+        let timespec = duration_to_kernel_timespec(deadline);
+        self.push_sqe(|sqe| {
+            sqe.opcode = IORING_OP_TIMEOUT;
+            sqe.fd = -1;
+            sqe.off = 0;
+            sqe.user_data = token;
+            sqe.addr = (&timespec as *const KernelTimespec) as u64;
+            sqe.len = 1;
+            sqe.op_flags = IORING_TIMEOUT_ABS;
+        })?;
+        self.submit_pending().map(|_| ())
+    }
+
+    pub(crate) fn submit_timeout_remove(
+        &self,
+        token_to_remove: u64,
+        completion: u64,
+    ) -> io::Result<()> {
+        self.push_sqe(|sqe| {
+            sqe.opcode = IORING_OP_TIMEOUT_REMOVE;
+            sqe.fd = -1;
+            sqe.flags = IOSQE_CQE_SKIP_SUCCESS;
+            sqe.user_data = completion;
+            sqe.addr = token_to_remove;
+        })?;
+        self.submit_pending().map(|_| ())
+    }
+
+    pub(crate) fn submit_timeout_update(
+        &self,
+        token_to_update: u64,
+        deadline: Duration,
+    ) -> io::Result<()> {
+        let timespec = duration_to_kernel_timespec(deadline);
+        self.push_sqe(|sqe| {
+            sqe.opcode = IORING_OP_TIMEOUT_REMOVE;
+            sqe.fd = -1;
+            sqe.off = (&timespec as *const KernelTimespec) as u64;
+            sqe.addr = token_to_update;
+            sqe.op_flags = IORING_TIMEOUT_UPDATE | IORING_TIMEOUT_ABS;
+        })?;
+        self.submit_pending().map(|_| ())
+    }
+
+    pub(crate) fn submit_msg_ring(
+        &self,
+        target_ring_fd: RawFd,
+        target_user_data: u64,
+        value: u32,
+        completion: u64,
+    ) -> io::Result<()> {
+        self.push_sqe(|sqe| {
+            sqe.opcode = IORING_OP_MSG_RING;
+            sqe.flags = IOSQE_CQE_SKIP_SUCCESS;
+            sqe.fd = target_ring_fd;
+            sqe.off = target_user_data;
+            sqe.addr = IORING_MSG_DATA;
+            sqe.len = value;
+            sqe.user_data = completion;
+        })?;
+        self.submit_pending().map(|_| ())
+    }
+
+    pub(crate) fn submit_with_token(
+        &self,
+        token: u64,
+        fill: impl FnOnce(&mut IoUringSqe),
+    ) -> io::Result<()> {
+        self.push_sqe(|sqe| {
+            fill(sqe);
+            sqe.user_data = token;
+        })?;
+        self.submit_pending().map(|_| ())
+    }
+
+    pub(crate) fn drain_completions(&self, mut f: impl FnMut(IoUringCqe)) -> bool {
+        let mut head = load_u32(self.cq_head);
+        let tail = load_u32(self.cq_tail);
+        if head == tail {
+            return false;
+        }
+        let mask = load_u32(self.cq_ring_mask);
+
+        while head != tail {
+            let index = (head & mask) as usize;
+            let cqe = unsafe { ptr::read_volatile(self.cqes.add(index)) };
+            f(cqe);
+            head = head.wrapping_add(1);
+        }
+
+        store_u32(self.cq_head, head);
+        true
+    }
+
+    pub(crate) fn wait_for_cqe(&self) -> io::Result<()> {
+        loop {
+            match self.enter(0, 1, IORING_ENTER_GETEVENTS) {
+                Ok(_) => return Ok(()),
+                Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
+                Err(error) => return Err(error),
+            }
+        }
+    }
+
+    fn push_sqe(&self, fill: impl FnOnce(&mut IoUringSqe)) -> io::Result<()> {
+        let head = load_u32(self.sq_head);
+        let tail = load_u32(self.sq_tail);
+        let entries = load_u32(self.sq_ring_entries);
+        if tail.wrapping_sub(head) >= entries {
+            self.submit_pending()?;
+            let head = load_u32(self.sq_head);
+            let tail = load_u32(self.sq_tail);
+            if tail.wrapping_sub(head) >= entries {
+                return Err(io::Error::new(
+                    io::ErrorKind::WouldBlock,
+                    "io_uring submission queue is full",
+                ));
+            }
+        }
+
+        let tail = load_u32(self.sq_tail);
+        let mask = load_u32(self.sq_ring_mask);
+        let index = (tail & mask) as usize;
+        let sqe = unsafe { &mut *self.sqes_ptr.add(index) };
+        *sqe = IoUringSqe::default();
+        fill(sqe);
+        unsafe {
+            ptr::write_volatile(self.sq_array.add(index), index as u32);
+        }
+        compiler_fence(Ordering::Release);
+        store_u32(self.sq_tail, tail.wrapping_add(1));
+        Ok(())
+    }
+
+    fn submit_pending(&self) -> io::Result<u32> {
+        let head = load_u32(self.sq_head);
+        let tail = load_u32(self.sq_tail);
+        let to_submit = tail.wrapping_sub(head);
+        if to_submit == 0 {
+            return Ok(0);
+        }
+        self.enter(to_submit, 0, 0)
+    }
+
+    fn enter(&self, to_submit: u32, min_complete: u32, flags: u32) -> io::Result<u32> {
+        cvt_long(unsafe {
+            libc::syscall(
+                libc::SYS_io_uring_enter,
+                self.ring_fd,
+                to_submit as libc::c_uint,
+                min_complete as libc::c_uint,
+                flags as libc::c_uint,
+                ptr::null::<libc::c_void>(),
+                0usize,
+            )
+        })
+        .map(|value| value as u32)
+    }
+}
+
+impl Drop for IoUring {
+    fn drop(&mut self) {
+        unsafe {
+            libc::munmap(self.sqes_ptr.cast(), self.sqes_size);
+            if self.single_mmap {
+                libc::munmap(
+                    self.sq_ring_ptr.cast(),
+                    self.sq_ring_size.max(self.cq_ring_size),
+                );
+            } else {
+                libc::munmap(self.sq_ring_ptr.cast(), self.sq_ring_size);
+                libc::munmap(self.cq_ring_ptr.cast(), self.cq_ring_size);
+            }
+            libc::close(self.ring_fd);
+        }
+    }
+}
+
+unsafe impl Send for IoUring {}
+
+fn offset_ptr<T>(base: *mut u8, offset: u32) -> *mut T {
+    unsafe { base.add(offset as usize).cast::<T>() }
+}
+
+fn mmap_ring(length: usize, fd: RawFd, offset: libc::off_t) -> io::Result<*mut u8> {
+    let ptr = unsafe {
+        libc::mmap(
+            ptr::null_mut(),
+            length,
+            libc::PROT_READ | libc::PROT_WRITE,
+            libc::MAP_SHARED | libc::MAP_POPULATE,
+            fd,
+            offset,
+        )
+    };
+    if ptr == libc::MAP_FAILED {
+        Err(io::Error::last_os_error())
+    } else {
+        Ok(ptr.cast())
+    }
+}
+
+fn load_u32(ptr: *const u32) -> u32 {
+    let value = unsafe { ptr::read_volatile(ptr) };
+    compiler_fence(Ordering::Acquire);
+    value
+}
+
+fn store_u32(ptr: *mut u32, value: u32) {
+    compiler_fence(Ordering::Release);
+    unsafe {
+        ptr::write_volatile(ptr, value);
+    }
+}
+
+fn cvt_long(result: libc::c_long) -> io::Result<libc::c_long> {
+    if result == -1 {
+        Err(io::Error::last_os_error())
+    } else {
+        Ok(result)
+    }
+}
+
+fn global_submitter() -> &'static Mutex<Option<IoUring>> {
+    GLOBAL_SUBMITTER.get_or_init(|| Mutex::new(None))
+}
+
+fn duration_to_kernel_timespec(duration: Duration) -> KernelTimespec {
+    KernelTimespec {
+        tv_sec: duration.as_secs() as i64,
+        tv_nsec: duration.subsec_nanos() as i64,
+    }
+}

lib/runtime/src/platform/mod.rs

@@ -0,0 +1,2 @@
+#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
+pub mod linux_x86_64;