Call rb_memerror when OOM

gc/mmtk/mmtk.c

@@ -909,6 +909,11 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags
     VALUE *alloc_obj = (VALUE *)rb_mmtk_alloc_fast_path(objspace, ractor_cache, alloc_size, MMTk_MIN_OBJ_ALIGN);
     if (!alloc_obj) {
         alloc_obj = mmtk_alloc(ractor_cache->mutator, alloc_size, MMTk_MIN_OBJ_ALIGN, 0, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
+
+        // On heap exhaustion raise NoMemoryError.
+        if (RB_UNLIKELY(alloc_obj == NULL)) {
+            rb_memerror();
+        }
     }
 
     alloc_obj++;

gc/mmtk/src/collection.rs

@@ -9,6 +9,7 @@ use crate::upcalls;
 use crate::Ruby;
 use mmtk::memory_manager;
 use mmtk::scheduler::*;
+use mmtk::util::alloc::AllocationError;
 use mmtk::util::heap::GCTriggerPolicy;
 use mmtk::util::VMMutatorThread;
 use mmtk::util::VMThread;
@@ -63,6 +64,19 @@ impl Collection<Ruby> for VMCollection {
         (upcalls().block_for_gc)(tls);
     }
 
+    fn out_of_memory(_tls: VMThread, err_kind: AllocationError) {
+        match err_kind {
+            // The heap is exhausted and could not be grown. Return normally
+            // without aborting.
+            AllocationError::HeapOutOfMemory => {}
+            // The OS refused an mmap. This is unrecoverable, so abort the
+            // process via the same panic handler used for GC-thread panics.
+            AllocationError::MmapOutOfMemory => {
+                (upcalls().mutator_thread_panic_handler)();
+            }
+        }
+    }
+
     fn spawn_gc_thread(_tls: VMThread, ctx: GCThreadContext<Ruby>) {
         let join_handle = match ctx {
             GCThreadContext::Worker(mut worker) => thread::Builder::new()

test/mmtk/test_oom.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require_relative "helper"
+
+module MMTk
+  class TestOOM < TestCase
+    def test_oom
+      assert_in_out_err([{ "MMTK_HEAP_MAX" => "64MiB" }], <<~RUBY, [], ["[FATAL] failed to allocate memory"])
+        10_000_000.times.map do
+          Object.new
+        end
+      RUBY
+    end
+  end
+end