From 6258cfa3151caf4e67d8da9c2685ea28e33b7e76 Mon Sep 17 00:00:00 2001
From: Peter Zhu <peter@peterzhu.ca>
Date: Fri, 19 Jun 2026 14:39:51 +0900
Subject: [PATCH] Call rb_memerror when OOM

---
 gc/mmtk/mmtk.c            |  5 +++++
 gc/mmtk/src/collection.rs | 14 ++++++++++++++
 test/mmtk/test_oom.rb     | 15 +++++++++++++++
 3 files changed, 34 insertions(+)
 create mode 100644 test/mmtk/test_oom.rb

diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c
index a725432..0e49fbc 100644
--- a/gc/mmtk/mmtk.c
+++ b/gc/mmtk/mmtk.c
@@ -909,6 +909,11 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags
     VALUE *alloc_obj = (VALUE *)rb_mmtk_alloc_fast_path(objspace, ractor_cache, alloc_size, MMTk_MIN_OBJ_ALIGN);
     if (!alloc_obj) {
         alloc_obj = mmtk_alloc(ractor_cache->mutator, alloc_size, MMTk_MIN_OBJ_ALIGN, 0, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
+
+        // On heap exhaustion raise NoMemoryError.
+        if (RB_UNLIKELY(alloc_obj == NULL)) {
+            rb_memerror();
+        }
     }
 
     alloc_obj++;
diff --git a/gc/mmtk/src/collection.rs b/gc/mmtk/src/collection.rs
index 648efa4..81b3973 100644
--- a/gc/mmtk/src/collection.rs
+++ b/gc/mmtk/src/collection.rs
@@ -9,6 +9,7 @@ use crate::upcalls;
 use crate::Ruby;
 use mmtk::memory_manager;
 use mmtk::scheduler::*;
+use mmtk::util::alloc::AllocationError;
 use mmtk::util::heap::GCTriggerPolicy;
 use mmtk::util::VMMutatorThread;
 use mmtk::util::VMThread;
@@ -63,6 +64,19 @@ impl Collection<Ruby> for VMCollection {
         (upcalls().block_for_gc)(tls);
     }
 
+    fn out_of_memory(_tls: VMThread, err_kind: AllocationError) {
+        match err_kind {
+            // The heap is exhausted and could not be grown. Return normally
+            // without aborting.
+            AllocationError::HeapOutOfMemory => {}
+            // The OS refused an mmap. This is unrecoverable, so abort the
+            // process via the same panic handler used for GC-thread panics.
+            AllocationError::MmapOutOfMemory => {
+                (upcalls().mutator_thread_panic_handler)();
+            }
+        }
+    }
+
     fn spawn_gc_thread(_tls: VMThread, ctx: GCThreadContext<Ruby>) {
         let join_handle = match ctx {
             GCThreadContext::Worker(mut worker) => thread::Builder::new()
diff --git a/test/mmtk/test_oom.rb b/test/mmtk/test_oom.rb
new file mode 100644
index 0000000..ea717d2
--- /dev/null
+++ b/test/mmtk/test_oom.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require_relative "helper"
+
+module MMTk
+  class TestOOM < TestCase
+    def test_oom
+      assert_in_out_err([{ "MMTK_HEAP_MAX" => "64MiB" }], <<~RUBY, [], ["[FATAL] failed to allocate memory"])
+        10_000_000.times.map do
+          Object.new
+        end
+      RUBY
+    end
+  end
+end