gc.c (57213d44ce7b1a31fc9648e9cd5eb0c4507f4a49)

/**********************************************************************
 
  gc.c -
 
  $Author$
  created at: Tue Oct  5 09:44:46 JST 1993
 
  Copyright (C) 1993-2007 Yukihiro Matsumoto
  Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
  Copyright (C) 2000  Information-technology Promotion Agency, Japan
 
**********************************************************************/
 
#include "ruby/internal/config.h"
#ifdef _WIN32
# include "ruby/ruby.h"
#endif
 
#if defined(__wasm__) && !defined(__EMSCRIPTEN__)
# include "wasm/setjmp.h"
# include "wasm/machine.h"
#else
# include <setjmp.h>
#endif
#include <stdarg.h>
#include <stdio.h>
 
/* MALLOC_HEADERS_BEGIN */
#ifndef HAVE_MALLOC_USABLE_SIZE
# ifdef _WIN32
#  define HAVE_MALLOC_USABLE_SIZE
#  define malloc_usable_size(a) _msize(a)
# elif defined HAVE_MALLOC_SIZE
#  define HAVE_MALLOC_USABLE_SIZE
#  define malloc_usable_size(a) malloc_size(a)
# endif
#endif
 
#ifdef HAVE_MALLOC_USABLE_SIZE
# ifdef RUBY_ALTERNATIVE_MALLOC_HEADER
/* Alternative malloc header is included in ruby/missing.h */
# elif defined(HAVE_MALLOC_H)
#  include <malloc.h>
# elif defined(HAVE_MALLOC_NP_H)
#  include <malloc_np.h>
# elif defined(HAVE_MALLOC_MALLOC_H)
#  include <malloc/malloc.h>
# endif
#endif
 
/* MALLOC_HEADERS_END */
 
#ifdef HAVE_SYS_TIME_H
# include <sys/time.h>
#endif
 
#ifdef HAVE_SYS_RESOURCE_H
# include <sys/resource.h>
#endif
 
#if defined _WIN32 || defined __CYGWIN__
# include <windows.h>
#elif defined(HAVE_POSIX_MEMALIGN)
#elif defined(HAVE_MEMALIGN)
# include <malloc.h>
#endif
 
#include <sys/types.h>
 
#ifdef __EMSCRIPTEN__
#include <emscripten.h>
#endif
 
/* For ruby_annotate_mmap */
#ifdef HAVE_SYS_PRCTL_H
#include <sys/prctl.h>
#endif
 
#undef LIST_HEAD /* ccan/list conflicts with BSD-origin sys/queue.h. */
 
#include "constant.h"
#include "debug_counter.h"
#include "eval_intern.h"
#include "gc/gc.h"
#include "id_table.h"
#include "internal.h"
#include "internal/class.h"
#include "internal/compile.h"
#include "internal/complex.h"
#include "internal/concurrent_set.h"
#include "internal/cont.h"
#include "internal/error.h"
#include "internal/eval.h"
#include "internal/gc.h"
#include "internal/hash.h"
#include "internal/imemo.h"
#include "internal/io.h"
#include "internal/numeric.h"
#include "internal/object.h"
#include "internal/proc.h"
#include "internal/rational.h"
#include "internal/re.h"
#include "internal/sanitizers.h"
#include "internal/struct.h"
#include "internal/symbol.h"
#include "internal/thread.h"
#include "internal/variable.h"
#include "internal/warnings.h"
#include "probes.h"
#include "regint.h"
#include "ruby/debug.h"
#include "ruby/io.h"
#include "ruby/re.h"
#include "ruby/st.h"
#include "ruby/thread.h"
#include "ruby/util.h"
#include "ruby/vm.h"
#include "ruby_assert.h"
#include "ruby_atomic.h"
#include "symbol.h"
#include "variable.h"
#include "vm_core.h"
#include "vm_sync.h"
#include "vm_callinfo.h"
#include "ractor_core.h"
#include "internal/ractor.h"
#include "yjit.h"
#include "zjit.h"
 
#include "builtin.h"
#include "shape.h"
 
// TODO: Don't export this function in modular GC, instead MMTk should figure out
// how to combine GC thread backtrace with mutator thread backtrace.
void
rb_gc_print_backtrace(void)
{
    rb_print_backtrace(stderr);
}
 
unsigned int
rb_gc_vm_lock(const char *file, int line)
{
    unsigned int lev = 0;
    rb_vm_lock_enter(&lev, file, line);
    return lev;
}
 
void
rb_gc_vm_unlock(unsigned int lev, const char *file, int line)
{
    rb_vm_lock_leave(&lev, file, line);
}
 
unsigned int
rb_gc_vm_lock_no_barrier(const char *file, int line)
{
    unsigned int lev = 0;
    rb_vm_lock_enter_nb(&lev, file, line);
    return lev;
}
 
void
rb_gc_vm_unlock_no_barrier(unsigned int lev, const char *file, int line)
{
    rb_vm_lock_leave_nb(&lev, file, line);
}
 
void
rb_gc_vm_barrier(void)
{
    rb_vm_barrier();
}
 
void *
rb_gc_get_ractor_newobj_cache(void)
{
    return GET_RACTOR()->newobj_cache;
}
 
void
rb_gc_initialize_vm_context(struct rb_gc_vm_context *context)
{
    context->ec = GET_EC();
}
 
bool
rb_gc_event_hook_required_p(rb_event_flag_t event)
{
    return ruby_vm_event_flags & event;
}
 
void
rb_gc_event_hook(VALUE obj, rb_event_flag_t event)
{
    if (LIKELY(!rb_gc_event_hook_required_p(event))) return;
 
    /* Event hooks run user code in the context of the currently running
     * thread, so they must use the current EC.  rb_gc_get_ec() may instead
     * return the GC's snapshot (vm_context.ec, taken at marking), which can
     * belong to a different thread once lazy sweep is continued from another
     * thread's allocation; running the hook on it would set trace_arg on the
     * wrong EC and break get_trace_arg() (GET_EC()->trace_arg would be NULL). */
    rb_execution_context_t *ec = GET_EC();
 
#if USE_MODULAR_GC
    bool gc_thread_p = false;
    if (!ec) {
        /* A dedicated GC thread has no mutator EC: borrow the GC's snapshot and
         * install it as the current EC so GET_EC() stays consistent inside the
         * hook (e.g. for get_trace_arg()). */
        ec = rb_gc_get_ec();
        gc_thread_p = true;
 
# ifdef RB_THREAD_LOCAL_SPECIFIER
        rb_current_ec_set(ec);
# else
        native_tls_set(ruby_current_ec_key, ec);
# endif
    }
#endif
 
    if (RB_LIKELY(ec->cfp != NULL)) {
        EXEC_EVENT_HOOK(ec, event, ec->cfp->self, 0, 0, 0, obj);
    }
 
#if USE_MODULAR_GC
    if (gc_thread_p) {
# ifdef RB_THREAD_LOCAL_SPECIFIER
        rb_current_ec_set(NULL);
# else
        native_tls_set(ruby_current_ec_key, NULL);
# endif
    }
#endif
}
 
/* Nonzero once the cleanup walk has stashed an objspace, which is the only case where a
 * thread needs one other than its own Ractor's.  One unsynchronised load gates it,
 * keeping the per-reference marking path and the malloc paths clear of the lookups
 * below.  It is never unwound, so a stale zero only costs a resolution that predates the
 * stash. */
static rb_atomic_t gc_objspace_override_count;
 
/* VM destruct's free-at-exit walk can free the thread and Ractor structs first, so
 * resolving through the current Ractor would use freed memory; return the objspace
 * stashed before the walk started. */
 
void
rb_gc_stash_cleanup_objspace(void)
{
    GET_VM()->gc.cleanup_objspace = rb_gc_get_objspace();
    /* Never unwound, as the VM is going away: from here every resolution takes the slow
     * path, which is where the stashed objspace is returned from. */
    RUBY_ATOMIC_INC(gc_objspace_override_count);
}
 
/* Out of line to keep the rare arms out of RB_GC_MARK_OR_TRAVERSE's inline expansions. */
NOINLINE(static void *gc_current_objspace_slow(rb_ractor_t *const cr));
 
static void *
gc_current_objspace_slow(rb_ractor_t *const cr)
{
    /* Checked before anything reads cr or the EC: the cleanup walk is the case where
     * both may already be freed (see rb_gc_stash_cleanup_objspace). */
    if (RB_UNLIKELY(ruby_vm_during_cleanup) && GET_VM()->gc.cleanup_objspace) {
        return GET_VM()->gc.cleanup_objspace;
    }
    if (cr == NULL) {
        /* A thread with no current Ractor (a GVL-less native thread freeing in
         * thread_sched_reclaim, say) uses the main Ractor's objspace. */
        return GET_VM()->ractor.main_ractor->objspace;
    }
    /* A live current Ractor always has an objspace. */
    RUBY_ASSERT(cr->objspace != NULL);
    return cr->objspace;
}
 
static inline void *
gc_current_objspace_of(rb_ractor_t *const cr)
{
    if (RB_LIKELY(cr != NULL && gc_objspace_override_count == 0)) {
        RUBY_ASSERT(cr->objspace != NULL);
        return cr->objspace;
    }
    return gc_current_objspace_slow(cr);
}
 
void *
rb_gc_get_objspace(void)
{
    return gc_current_objspace_of(rb_current_ractor_raw(false));
}
 
void
rb_gc_run_obj_finalizer(VALUE objid, long count, VALUE (*callback)(long i, void *data), void *data)
{
    volatile struct {
        VALUE errinfo;
        VALUE final;
        rb_control_frame_t *cfp;
        VALUE *sp;
        long finished;
    } saved;
 
    rb_execution_context_t * volatile ec = GET_EC();
#define RESTORE_FINALIZER() (\
        ec->cfp = saved.cfp, \
        ec->cfp->sp = saved.sp, \
        ec->errinfo = saved.errinfo)
 
    saved.errinfo = ec->errinfo;
    saved.cfp = ec->cfp;
    saved.sp = ec->cfp->sp;
    saved.finished = 0;
    saved.final = Qundef;
 
    ASSERT_vm_unlocking();
    rb_ractor_ignore_belonging(true);
    EC_PUSH_TAG(ec);
    enum ruby_tag_type state = EC_EXEC_TAG();
    if (state != TAG_NONE) {
        ++saved.finished;       /* skip failed finalizer */
 
        VALUE failed_final = saved.final;
        saved.final = Qundef;
        if (!UNDEF_P(failed_final) && !NIL_P(ruby_verbose)) {
            rb_warn("Exception in finalizer %+"PRIsVALUE, failed_final);
            rb_ec_error_print(ec, ec->errinfo);
        }
    }
 
    for (long i = saved.finished; RESTORE_FINALIZER(), i < count; saved.finished = ++i) {
        saved.final = callback(i, data);
        rb_check_funcall(saved.final, idCall, 1, &objid);
    }
    EC_POP_TAG();
    rb_ractor_ignore_belonging(false);
#undef RESTORE_FINALIZER
}
 
void
rb_gc_set_pending_interrupt(void)
{
    rb_execution_context_t *ec = GET_EC();
    ec->interrupt_mask |= PENDING_INTERRUPT_MASK;
}
 
/* Schedule an objspace's deferred finalizers.  A global GC sweeps other Ractors'
 * objspaces too, so target the owning Ractor rather than the sweeping driver.  For an
 * objspace with no live owner the untargeted fallback is only a wake-up: a zombie's
 * entries move to the inheriting objspace in the absorb, which re-triggers there. */
void
rb_gc_trigger_finalize_deferred(void *objspace, rb_postponed_job_handle_t pjob)
{
    rb_ractor_t *const cr = rb_current_ractor_raw(false);
    if (cr == NULL || cr->objspace != objspace) {
        /* Only a global GC (stop-the-world) or an absorb settle (under the VM lock)
         * defers another objspace's finalizers, so ractor.set is stable here. */
        ASSERT_vm_locking();
        rb_vm_t *vm = GET_VM();
        rb_ractor_t *r;
        ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
            if (r->objspace == objspace) {
                rb_postponed_job_trigger_for_ractor(pjob, r->pub.self);
                return;
            }
        }
    }
    rb_postponed_job_trigger(pjob);
}
 
void
rb_gc_trigger_postponed_job_on_main(rb_postponed_job_handle_t pjob)
{
    rb_postponed_job_trigger_for_ractor(pjob, GET_VM()->ractor.main_ractor->pub.self);
}
 
void
rb_gc_unset_pending_interrupt(void)
{
    rb_execution_context_t *ec = GET_EC();
    ec->interrupt_mask &= ~PENDING_INTERRUPT_MASK;
}
 
bool
rb_gc_multi_ractor_p(void)
{
    return rb_multi_ractor_p();
}
 
bool
rb_gc_shutdown_call_finalizer_p(VALUE obj)
{
    switch (BUILTIN_TYPE(obj)) {
      case T_DATA:
        if (!ruby_free_at_exit_p()) {
            if (!RDATA(obj)->type) return false;
            if (!rbimpl_typeddata_embedded_p(obj) && !RTYPEDDATA(obj)->data) return false;
        }
        if (rb_obj_is_thread(obj)) return false;
        if (rb_obj_is_mutex(obj)) return false;
        if (rb_obj_is_fiber(obj)) return false;
        if (rb_ractor_p(obj)) return false;
        if (rb_obj_is_fstring_table(obj)) return false;
        if (rb_obj_is_symbol_table(obj)) return false;
 
        return true;
 
      case T_FILE:
        return true;
 
      case T_SYMBOL:
        return true;
 
      case T_NONE:
        return false;
 
      default:
        return ruby_free_at_exit_p();
    }
}
 
void
rb_gc_obj_changed_slot_size(VALUE obj, size_t slot_size)
{
    shape_id_t shape_id = rb_obj_shape_transition_capacity(obj, rb_shape_capacity_for_slot_size(slot_size));
    RBASIC_SET_FULL_SHAPE_ID(obj, shape_id);
}
 
void rb_vm_update_references(void *ptr);
 
#define rb_setjmp(env) RUBY_SETJMP(env)
#define rb_jmp_buf rb_jmpbuf_t
 
#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
#define MAP_ANONYMOUS MAP_ANON
#endif
 
#define unless_objspace(objspace) \
    void *objspace; \
    rb_vm_t *unless_objspace_vm = GET_VM(); \
    if (unless_objspace_vm) objspace = rb_gc_get_objspace(); \
    else /* return; or objspace will be warned uninitialized */
 
#define RMOVED(obj) ((struct RMoved *)(obj))
 
#define TYPED_UPDATE_IF_MOVED(_objspace, _type, _thing) do { \
    if (gc_object_moved_p_internal((_objspace), (VALUE)(_thing))) {    \
        *(_type *)&(_thing) = (_type)gc_location_internal(_objspace, (VALUE)_thing); \
    } \
} while (0)
 
#define UPDATE_IF_MOVED(_objspace, _thing) TYPED_UPDATE_IF_MOVED(_objspace, VALUE, _thing)
 
#if RUBY_MARK_FREE_DEBUG
int ruby_gc_debug_indent = 0;
#endif
 
#ifndef RGENGC_OBJ_INFO
# define RGENGC_OBJ_INFO RGENGC_CHECK_MODE
#endif
 
#ifndef CALC_EXACT_MALLOC_SIZE
# define CALC_EXACT_MALLOC_SIZE 0
#endif
 
VALUE rb_mGC;
 
static size_t malloc_offset = 0;
#if defined(HAVE_MALLOC_USABLE_SIZE)
static size_t
gc_compute_malloc_offset(void)
{
    // Different allocators use different metadata storage strategies which result in different
    // ideal sizes.
    // For instance malloc(64) will waste 8B with glibc, but waste 0B with jemalloc.
    // But malloc(56) will waste 0B with glibc, but waste 8B with jemalloc.
    // So we try allocating 64, 56 and 48 bytes and select the first offset that doesn't
    // waste memory.
    // This was tested on Linux with glibc 2.35 and jemalloc 5, and for both it result in
    // no wasted memory.
    size_t offset = 0;
    for (offset = 0; offset <= 16; offset += 8) {
        size_t allocated = (64 - offset);
        void *test_ptr = malloc(allocated);
        size_t wasted = malloc_usable_size(test_ptr) - allocated;
        free(test_ptr);
 
        if (wasted == 0) {
            return offset;
        }
    }
    return 0;
}
#else
static size_t
gc_compute_malloc_offset(void)
{
    // If we don't have malloc_usable_size, we use powers of 2.
    return 0;
}
#endif
 
size_t
rb_malloc_grow_capa(size_t current, size_t type_size)
{
    size_t current_capacity = current;
    if (current_capacity < 4) {
        current_capacity = 4;
    }
    current_capacity *= type_size;
 
    // We double the current capacity.
    size_t new_capacity = (current_capacity * 2);
 
    // And round up to the next power of 2 if it's not already one.
    if (rb_popcount64(new_capacity) != 1) {
        new_capacity = (size_t)(1 << (64 - nlz_int64(new_capacity)));
    }
 
    new_capacity -= malloc_offset;
    new_capacity /= type_size;
    if (current > new_capacity) {
        rb_bug("rb_malloc_grow_capa: current_capacity=%zu, new_capacity=%zu, malloc_offset=%zu", current, new_capacity, malloc_offset);
    }
    RUBY_ASSERT(new_capacity > current);
    return new_capacity;
}
 
static inline struct rbimpl_size_overflow_tag
size_mul_add_overflow(size_t x, size_t y, size_t z) /* x * y + z */
{
    struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(x, y);
    struct rbimpl_size_overflow_tag u = rbimpl_size_add_overflow(t.result, z);
    return (struct rbimpl_size_overflow_tag) { t.overflowed || u.overflowed, u.result };
}
 
static inline struct rbimpl_size_overflow_tag
size_mul_add_mul_overflow(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
{
    struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(x, y);
    struct rbimpl_size_overflow_tag u = rbimpl_size_mul_overflow(z, w);
    struct rbimpl_size_overflow_tag v = rbimpl_size_add_overflow(t.result, u.result);
    return (struct rbimpl_size_overflow_tag) { t.overflowed || u.overflowed || v.overflowed, v.result };
}
 
PRINTF_ARGS(NORETURN(static void gc_raise(VALUE, const char*, ...)), 2, 3);
 
static inline size_t
size_mul_or_raise(size_t x, size_t y, VALUE exc)
{
    struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(x, y);
    if (LIKELY(!t.overflowed)) {
        return t.result;
    }
    else if (rb_during_gc()) {
        rb_memerror();          /* or...? */
    }
    else {
        gc_raise(
            exc,
            "integer overflow: %"PRIuSIZE
            " * %"PRIuSIZE
            " > %"PRIuSIZE,
            x, y, (size_t)SIZE_MAX);
    }
}
 
size_t
rb_size_mul_or_raise(size_t x, size_t y, VALUE exc)
{
    return size_mul_or_raise(x, y, exc);
}
 
static inline size_t
size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc)
{
    struct rbimpl_size_overflow_tag t = size_mul_add_overflow(x, y, z);
    if (LIKELY(!t.overflowed)) {
        return t.result;
    }
    else if (rb_during_gc()) {
        rb_memerror();          /* or...? */
    }
    else {
        gc_raise(
            exc,
            "integer overflow: %"PRIuSIZE
            " * %"PRIuSIZE
            " + %"PRIuSIZE
            " > %"PRIuSIZE,
            x, y, z, (size_t)SIZE_MAX);
    }
}
 
size_t
rb_size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc)
{
    return size_mul_add_or_raise(x, y, z, exc);
}
 
static inline size_t
size_mul_add_mul_or_raise(size_t x, size_t y, size_t z, size_t w, VALUE exc)
{
    struct rbimpl_size_overflow_tag t = size_mul_add_mul_overflow(x, y, z, w);
    if (LIKELY(!t.overflowed)) {
        return t.result;
    }
    else if (rb_during_gc()) {
        rb_memerror();          /* or...? */
    }
    else {
        gc_raise(
            exc,
            "integer overflow: %"PRIdSIZE
            " * %"PRIdSIZE
            " + %"PRIdSIZE
            " * %"PRIdSIZE
            " > %"PRIdSIZE,
            x, y, z, w, (size_t)SIZE_MAX);
    }
}
 
#if defined(HAVE_RB_GC_GUARDED_PTR_VAL) && HAVE_RB_GC_GUARDED_PTR_VAL
/* trick the compiler into thinking a external signal handler uses this */
volatile VALUE rb_gc_guarded_val;
volatile VALUE *
rb_gc_guarded_ptr_val(volatile VALUE *ptr, VALUE val)
{
    rb_gc_guarded_val = val;
 
    return ptr;
}
#endif
 
static const char *obj_type_name(VALUE obj);
 
/* A forking parent can hold registered_globals.lock (every Ractor's root scan takes
 * it); inheriting it locked would make the child's first GC wait forever, so rebuild
 * it, like the generic_fields lock. */
void
rb_gc_atfork_global_locks(void)
{
    rb_vm_t *vm = GET_VM();
    rb_native_mutex_initialize(&vm->gc.registered_globals.lock);
}
 
#include "gc/default/default.c"
 
#if USE_MODULAR_GC && !defined(HAVE_DLOPEN)
# error "Modular GC requires dlopen"
#elif USE_MODULAR_GC
#include <dlfcn.h>
 
typedef struct gc_function_map {
    // Bootup
    void *(*objspace_alloc)(void);
    void (*objspace_init)(void *objspace_ptr);
    void *(*ractor_cache_alloc)(void *objspace_ptr, void *ractor);
    void (*objspace_retire_gc)(void *objspace_ptr);
    void (*set_params)(void *objspace_ptr);
    void (*init)(void);
    // Shutdown
    void (*shutdown_free_objects)(void *objspace_ptr);
    void (*objspace_free)(void *objspace_ptr);
    void (*ractor_cache_free)(void *objspace_ptr, void *cache);
    // GC
    void (*start)(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact, bool global);
    bool (*during_gc_p)(void *objspace_ptr);
    void (*prepare_heap)(void *objspace_ptr);
    void (*gc_enable)(void *objspace_ptr);
    void (*gc_disable)(void *objspace_ptr, bool finish_current_gc);
    bool (*gc_enabled_p)(void *objspace_ptr);
    bool (*user_gc_disabled_set)(void *objspace_ptr, bool disable);
    bool (*user_gc_disabled_p)(void *objspace_ptr);
    bool (*multi_objspace_p)(void);
    bool (*during_global_gc_p)(void *objspace_ptr);
    bool (*during_postmortem_p)(void *objspace_ptr);
    bool (*obj_foreign_p)(void *objspace_ptr, VALUE obj);
    bool (*shref_marked_p)(void *objspace_ptr, VALUE obj);
    size_t (*heap_page_count)(void *objspace_ptr);
    void (*objspace_absorb)(void *dst_ptr, void *src_ptr);
    void (*gc_rest)(void *objspace_ptr);
    VALUE (*config_get)(void *objpace_ptr);
    void (*config_set)(void *objspace_ptr, VALUE hash);
    void (*stress_set)(void *objspace_ptr, VALUE flag);
    VALUE (*stress_get)(void *objspace_ptr);
    struct rb_gc_vm_context *(*get_vm_context)(void *objspace_ptr);
    // Object allocation
    VALUE (*new_obj)(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size, size_t *actual_alloc_size);
    bool (*zjit_new_obj_fastpath)(void *objspace_ptr, size_t alloc_size, VALUE flags, VALUE klass, struct rb_gc_zjit_fastpath *fastpath);
    size_t (*obj_slot_size)(VALUE obj);
    size_t (*size_slot_size)(void *objspace_ptr, size_t size);
    bool (*size_allocatable_p)(size_t size);
    size_t (*max_allocation_size)(void);
    // Malloc
    void *(*malloc)(void *objspace_ptr, size_t size, bool gc_allowed);
    void *(*calloc)(void *objspace_ptr, size_t size, bool gc_allowed);
    void *(*realloc)(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed);
    void (*free)(void *objspace_ptr, void *ptr, size_t old_size);
    void (*adjust_memory_usage)(void *objspace_ptr, ssize_t diff);
    // Marking
    void (*mark)(void *objspace_ptr, VALUE obj);
    void (*mark_and_move)(void *objspace_ptr, VALUE *ptr);
    void (*mark_and_pin)(void *objspace_ptr, VALUE obj);
    void (*mark_maybe)(void *objspace_ptr, VALUE obj);
    // Weak references
    void (*declare_weak_references)(void *objspace_ptr, VALUE obj);
    bool (*handle_weak_references_alive_p)(void *objspace_ptr, VALUE obj);
    // Compaction
    void (*register_pinning_obj)(void *objspace_ptr, VALUE obj);
    bool (*object_moved_p)(void *objspace_ptr, VALUE obj);
    bool (*pinned_p)(void *objspace_ptr, VALUE obj);
    VALUE (*location)(void *objspace_ptr, VALUE value);
    // Write barriers
    void (*writebarrier)(void *objspace_ptr, VALUE a, VALUE b);
    void (*writebarrier_unprotect)(void *objspace_ptr, VALUE obj);
    void (*writebarrier_remember)(void *objspace_ptr, VALUE obj);
    void (*obj_became_shareable)(void *objspace_ptr, VALUE obj);
    // Heap walking
    void (*each_objects)(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data);
    void (*each_objects_shareable)(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data);
    void (*each_objects_foreign)(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data);
    void (*each_object)(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data);
    // Finalizers
    void (*make_zombie)(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data);
    VALUE (*define_finalizer)(void *objspace_ptr, VALUE obj, VALUE block);
    void (*undefine_finalizer)(void *objspace_ptr, VALUE obj);
    void (*copy_finalizer)(void *objspace_ptr, VALUE dest, VALUE obj);
    void (*shutdown_call_finalizer)(void *objspace_ptr);
    // Forking
    void (*before_fork)(void *objspace_ptr);
    void (*after_fork)(void *objspace_ptr, rb_pid_t pid);
    // Statistics
    void (*set_measure_total_time)(void *objspace_ptr, VALUE flag);
    bool (*get_measure_total_time)(void *objspace_ptr);
    unsigned long long (*get_total_time)(void *objspace_ptr);
    size_t (*gc_count)(void *objspace_ptr);
    VALUE (*latest_gc_info)(void *objspace_ptr, VALUE key);
    VALUE (*stat)(void *objspace_ptr, VALUE hash_or_sym);
    VALUE (*stat_heap)(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym);
    const char *(*active_gc_name)(void);
    // Miscellaneous
    struct rb_gc_object_metadata_entry *(*object_metadata)(void *objspace_ptr, VALUE obj);
    bool (*live_object_p)(void *objspace_ptr, const void *ptr);
    bool (*garbage_object_p)(void *objspace_ptr, VALUE obj);
    void (*set_event_hook)(void *objspace_ptr, const rb_event_flag_t event);
    void (*copy_attributes)(void *objspace_ptr, VALUE dest, VALUE obj);
 
    bool modular_gc_loaded_p;
} rb_gc_function_map_t;
 
static rb_gc_function_map_t rb_gc_functions;
 
# define RUBY_GC_LIBRARY "RUBY_GC_LIBRARY"
# define MODULAR_GC_DIR STRINGIZE(modular_gc_dir)
 
static void
ruby_modular_gc_init(void)
{
    // Assert that the directory path ends with a /
    RUBY_ASSERT_ALWAYS(MODULAR_GC_DIR[sizeof(MODULAR_GC_DIR) - 2] == '/');
 
    const char *gc_so_file = getenv(RUBY_GC_LIBRARY);
 
    rb_gc_function_map_t gc_functions = { 0 };
 
    char *gc_so_path = NULL;
    void *handle = NULL;
    if (gc_so_file) {
        /* Check to make sure that gc_so_file matches /[\w-_]+/ so that it does
         * not load a shared object outside of the directory. */
        for (size_t i = 0; i < strlen(gc_so_file); i++) {
            char c = gc_so_file[i];
            if (isalnum(c)) continue;
            switch (c) {
              case '-':
              case '_':
                break;
              default:
                fprintf(stderr, "Only alphanumeric, dash, and underscore is allowed in "RUBY_GC_LIBRARY"\n");
                exit(EXIT_FAILURE);
            }
        }
 
        size_t gc_so_path_size = strlen(MODULAR_GC_DIR "librubygc." DLEXT) + strlen(gc_so_file) + 1;
#ifdef LOAD_RELATIVE
        Dl_info dli;
        size_t prefix_len = 0;
        if (dladdr((void *)(uintptr_t)ruby_modular_gc_init, &dli)) {
            const char *base = strrchr(dli.dli_fname, '/');
            if (base) {
                size_t tail = 0;
# define end_with_p(lit) \
                (prefix_len >= (tail = rb_strlen_lit(lit)) && \
                 memcmp(base - tail, lit, tail) == 0)
 
                prefix_len = base - dli.dli_fname;
                if (end_with_p("/bin") || end_with_p("/lib")) {
                    prefix_len -= tail;
                }
                prefix_len += MODULAR_GC_DIR[0] != '/';
                gc_so_path_size += prefix_len;
            }
        }
#endif
        gc_so_path = alloca(gc_so_path_size);
        {
            size_t gc_so_path_idx = 0;
#define GC_SO_PATH_APPEND(str) do { \
    gc_so_path_idx += strlcpy(gc_so_path + gc_so_path_idx, str, gc_so_path_size - gc_so_path_idx); \
} while (0)
#ifdef LOAD_RELATIVE
            if (prefix_len > 0) {
                memcpy(gc_so_path, dli.dli_fname, prefix_len);
                gc_so_path_idx = prefix_len;
            }
#endif
            GC_SO_PATH_APPEND(MODULAR_GC_DIR "librubygc.");
            GC_SO_PATH_APPEND(gc_so_file);
            GC_SO_PATH_APPEND(DLEXT);
            GC_ASSERT(gc_so_path_idx == gc_so_path_size - 1);
#undef GC_SO_PATH_APPEND
        }
 
        handle = dlopen(gc_so_path, RTLD_LAZY | RTLD_GLOBAL);
        if (!handle) {
            fprintf(stderr, "ruby_modular_gc_init: Shared library %s cannot be opened: %s\n", gc_so_path, dlerror());
            exit(EXIT_FAILURE);
        }
 
        gc_functions.modular_gc_loaded_p = true;
    }
 
    unsigned int err_count = 0;
 
# define load_modular_gc_func(name) do { \
    if (handle) { \
        const char *func_name = "rb_gc_impl_" #name; \
        gc_functions.name = dlsym(handle, func_name); \
        if (!gc_functions.name) { \
            fprintf(stderr, "ruby_modular_gc_init: %s function not exported by library %s\n", func_name, gc_so_path); \
            err_count++; \
        } \
    } \
    else { \
        gc_functions.name = rb_gc_impl_##name; \
    } \
} while (0)
 
    // Bootup
    load_modular_gc_func(objspace_alloc);
    load_modular_gc_func(objspace_init);
    load_modular_gc_func(ractor_cache_alloc);
    load_modular_gc_func(objspace_retire_gc);
    load_modular_gc_func(set_params);
    load_modular_gc_func(init);
    // Shutdown
    load_modular_gc_func(shutdown_free_objects);
    load_modular_gc_func(objspace_free);
    load_modular_gc_func(ractor_cache_free);
    // GC
    load_modular_gc_func(start);
    load_modular_gc_func(during_gc_p);
    load_modular_gc_func(prepare_heap);
    load_modular_gc_func(gc_enable);
    load_modular_gc_func(gc_disable);
    load_modular_gc_func(gc_enabled_p);
    load_modular_gc_func(user_gc_disabled_set);
    load_modular_gc_func(user_gc_disabled_p);
    load_modular_gc_func(multi_objspace_p);
    load_modular_gc_func(during_global_gc_p);
    load_modular_gc_func(during_postmortem_p);
    load_modular_gc_func(obj_foreign_p);
    load_modular_gc_func(shref_marked_p);
    load_modular_gc_func(heap_page_count);
    load_modular_gc_func(objspace_absorb);
    load_modular_gc_func(gc_rest);
    load_modular_gc_func(config_set);
    load_modular_gc_func(config_get);
    load_modular_gc_func(stress_set);
    load_modular_gc_func(stress_get);
    load_modular_gc_func(get_vm_context);
    // Object allocation
    load_modular_gc_func(new_obj);
    load_modular_gc_func(zjit_new_obj_fastpath);
    load_modular_gc_func(obj_slot_size);
    load_modular_gc_func(size_slot_size);
    load_modular_gc_func(size_allocatable_p);
    load_modular_gc_func(max_allocation_size);
    // Malloc
    load_modular_gc_func(malloc);
    load_modular_gc_func(calloc);
    load_modular_gc_func(realloc);
    load_modular_gc_func(free);
    load_modular_gc_func(adjust_memory_usage);
    // Marking
    load_modular_gc_func(mark);
    load_modular_gc_func(mark_and_move);
    load_modular_gc_func(mark_and_pin);
    load_modular_gc_func(mark_maybe);
    // Weak references
    load_modular_gc_func(declare_weak_references);
    load_modular_gc_func(handle_weak_references_alive_p);
    // Compaction
    load_modular_gc_func(register_pinning_obj);
    load_modular_gc_func(object_moved_p);
    load_modular_gc_func(pinned_p);
    load_modular_gc_func(location);
    // Write barriers
    load_modular_gc_func(writebarrier);
    load_modular_gc_func(writebarrier_unprotect);
    load_modular_gc_func(writebarrier_remember);
    load_modular_gc_func(obj_became_shareable);
    // Heap walking
    load_modular_gc_func(each_objects);
    load_modular_gc_func(each_objects_shareable);
    load_modular_gc_func(each_objects_foreign);
    load_modular_gc_func(each_object);
    // Finalizers
    load_modular_gc_func(make_zombie);
    load_modular_gc_func(define_finalizer);
    load_modular_gc_func(undefine_finalizer);
    load_modular_gc_func(copy_finalizer);
    load_modular_gc_func(shutdown_call_finalizer);
    // Forking
    load_modular_gc_func(before_fork);
    load_modular_gc_func(after_fork);
    // Statistics
    load_modular_gc_func(set_measure_total_time);
    load_modular_gc_func(get_measure_total_time);
    load_modular_gc_func(get_total_time);
    load_modular_gc_func(gc_count);
    load_modular_gc_func(latest_gc_info);
    load_modular_gc_func(stat);
    load_modular_gc_func(stat_heap);
    load_modular_gc_func(active_gc_name);
    // Miscellaneous
    load_modular_gc_func(object_metadata);
    load_modular_gc_func(live_object_p);
    load_modular_gc_func(garbage_object_p);
    load_modular_gc_func(set_event_hook);
    load_modular_gc_func(copy_attributes);
 
    if (err_count > 0) {
        fprintf(stderr, "ruby_modular_gc_init: found %u missing exports in library %s\n", err_count, gc_so_path);
        exit(EXIT_FAILURE);
    }
 
# undef load_modular_gc_func
 
    rb_gc_functions = gc_functions;
}
 
// Bootup
# define rb_gc_impl_objspace_alloc rb_gc_functions.objspace_alloc
# define rb_gc_impl_objspace_init rb_gc_functions.objspace_init
# define rb_gc_impl_ractor_cache_alloc rb_gc_functions.ractor_cache_alloc
# define rb_gc_impl_objspace_retire_gc rb_gc_functions.objspace_retire_gc
# define rb_gc_impl_set_params rb_gc_functions.set_params
# define rb_gc_impl_init rb_gc_functions.init
// Shutdown
# define rb_gc_impl_shutdown_free_objects rb_gc_functions.shutdown_free_objects
# define rb_gc_impl_objspace_free rb_gc_functions.objspace_free
# define rb_gc_impl_ractor_cache_free rb_gc_functions.ractor_cache_free
// GC
# define rb_gc_impl_start rb_gc_functions.start
# define rb_gc_impl_during_gc_p rb_gc_functions.during_gc_p
# define rb_gc_impl_prepare_heap rb_gc_functions.prepare_heap
# define rb_gc_impl_gc_enable rb_gc_functions.gc_enable
# define rb_gc_impl_gc_disable rb_gc_functions.gc_disable
# define rb_gc_impl_gc_enabled_p rb_gc_functions.gc_enabled_p
# define rb_gc_impl_user_gc_disabled_set rb_gc_functions.user_gc_disabled_set
# define rb_gc_impl_user_gc_disabled_p rb_gc_functions.user_gc_disabled_p
# define rb_gc_impl_multi_objspace_p rb_gc_functions.multi_objspace_p
# define rb_gc_impl_during_global_gc_p rb_gc_functions.during_global_gc_p
# define rb_gc_impl_during_postmortem_p rb_gc_functions.during_postmortem_p
# define rb_gc_impl_obj_foreign_p rb_gc_functions.obj_foreign_p
# define rb_gc_impl_shref_marked_p rb_gc_functions.shref_marked_p
# define rb_gc_impl_heap_page_count rb_gc_functions.heap_page_count
# define rb_gc_impl_objspace_absorb rb_gc_functions.objspace_absorb
# define rb_gc_impl_gc_rest rb_gc_functions.gc_rest
# define rb_gc_impl_config_get rb_gc_functions.config_get
# define rb_gc_impl_config_set rb_gc_functions.config_set
# define rb_gc_impl_stress_set rb_gc_functions.stress_set
# define rb_gc_impl_stress_get rb_gc_functions.stress_get
# define rb_gc_impl_get_vm_context rb_gc_functions.get_vm_context
// Object allocation
# define rb_gc_impl_new_obj rb_gc_functions.new_obj
# define rb_gc_impl_zjit_new_obj_fastpath rb_gc_functions.zjit_new_obj_fastpath
# define rb_gc_impl_obj_slot_size rb_gc_functions.obj_slot_size
# define rb_gc_impl_size_slot_size rb_gc_functions.size_slot_size
# define rb_gc_impl_size_allocatable_p rb_gc_functions.size_allocatable_p
# define rb_gc_impl_max_allocation_size rb_gc_functions.max_allocation_size
// Malloc
# define rb_gc_impl_malloc rb_gc_functions.malloc
# define rb_gc_impl_calloc rb_gc_functions.calloc
# define rb_gc_impl_realloc rb_gc_functions.realloc
# define rb_gc_impl_free rb_gc_functions.free
# define rb_gc_impl_adjust_memory_usage rb_gc_functions.adjust_memory_usage
// Marking
# define rb_gc_impl_mark rb_gc_functions.mark
# define rb_gc_impl_mark_and_move rb_gc_functions.mark_and_move
# define rb_gc_impl_mark_and_pin rb_gc_functions.mark_and_pin
# define rb_gc_impl_mark_maybe rb_gc_functions.mark_maybe
// Weak references
# define rb_gc_impl_declare_weak_references rb_gc_functions.declare_weak_references
# define rb_gc_impl_handle_weak_references_alive_p rb_gc_functions.handle_weak_references_alive_p
// Compaction
# define rb_gc_impl_register_pinning_obj rb_gc_functions.register_pinning_obj
# define rb_gc_impl_object_moved_p rb_gc_functions.object_moved_p
# define rb_gc_impl_pinned_p rb_gc_functions.pinned_p
# define rb_gc_impl_location rb_gc_functions.location
// Write barriers
# define rb_gc_impl_writebarrier rb_gc_functions.writebarrier
# define rb_gc_impl_writebarrier_unprotect rb_gc_functions.writebarrier_unprotect
# define rb_gc_impl_writebarrier_remember rb_gc_functions.writebarrier_remember
# define rb_gc_impl_obj_became_shareable rb_gc_functions.obj_became_shareable
// Heap walking
# define rb_gc_impl_each_objects rb_gc_functions.each_objects
# define rb_gc_impl_each_objects_shareable rb_gc_functions.each_objects_shareable
# define rb_gc_impl_each_objects_foreign rb_gc_functions.each_objects_foreign
# define rb_gc_impl_each_object rb_gc_functions.each_object
// Finalizers
# define rb_gc_impl_make_zombie rb_gc_functions.make_zombie
# define rb_gc_impl_define_finalizer rb_gc_functions.define_finalizer
# define rb_gc_impl_undefine_finalizer rb_gc_functions.undefine_finalizer
# define rb_gc_impl_copy_finalizer rb_gc_functions.copy_finalizer
# define rb_gc_impl_shutdown_call_finalizer rb_gc_functions.shutdown_call_finalizer
// Forking
# define rb_gc_impl_before_fork rb_gc_functions.before_fork
# define rb_gc_impl_after_fork rb_gc_functions.after_fork
// Statistics
# define rb_gc_impl_set_measure_total_time rb_gc_functions.set_measure_total_time
# define rb_gc_impl_get_measure_total_time rb_gc_functions.get_measure_total_time
# define rb_gc_impl_get_total_time rb_gc_functions.get_total_time
# define rb_gc_impl_gc_count rb_gc_functions.gc_count
# define rb_gc_impl_latest_gc_info rb_gc_functions.latest_gc_info
# define rb_gc_impl_stat rb_gc_functions.stat
# define rb_gc_impl_stat_heap rb_gc_functions.stat_heap
# define rb_gc_impl_active_gc_name rb_gc_functions.active_gc_name
// Miscellaneous
# define rb_gc_impl_object_metadata rb_gc_functions.object_metadata
# define rb_gc_impl_live_object_p rb_gc_functions.live_object_p
# define rb_gc_impl_garbage_object_p rb_gc_functions.garbage_object_p
# define rb_gc_impl_set_event_hook rb_gc_functions.set_event_hook
# define rb_gc_impl_copy_attributes rb_gc_functions.copy_attributes
#endif
 
#ifdef RUBY_ASAN_ENABLED
static void
asan_death_callback(void)
{
    if (GET_VM()) {
        rb_bug_without_die("ASAN error");
    }
}
#endif
 
static VALUE initial_stress = Qfalse;
 
void
rb_gc_init_objspaces(void)
{
#if USE_MODULAR_GC
    ruby_modular_gc_init();
#endif
 
    rb_vm_t *vm = ruby_current_vm_ptr;
 
    void *objspace = rb_gc_impl_objspace_alloc();
    RUBY_ASSERT(vm->ractor.main_ractor != NULL);
    vm->ractor.main_ractor->objspace = objspace;
    rb_gc_impl_objspace_init(objspace);
    rb_gc_impl_stress_set(objspace, initial_stress);
 
#ifdef RUBY_ASAN_ENABLED
    __sanitizer_set_death_callback(asan_death_callback);
#endif
}
 
/* Stays true once the process has gone multi-Ractor (rb_multi_ractor_p goes back to
 * false when the other Ractors finish).  Used by verification that spans generation
 * state built while multiple Ractors ran. */
static bool gc_ever_multi_ractor = false;
 
bool
rb_gc_ever_multi_ractor_p(void)
{
    if (!gc_ever_multi_ractor && rb_multi_ractor_p()) gc_ever_multi_ractor = true;
    return gc_ever_multi_ractor;
}
 
/* Allocate the objspace of a new non-main Ractor.  Called on the creating Ractor's
 * thread, before the new Ractor starts running. */
void *
rb_gc_objspace_alloc(void)
{
    gc_ever_multi_ractor = true;
    if (!rb_gc_impl_multi_objspace_p()) {
        /* One objspace shared by every Ractor. */
        return rb_gc_get_objspace();
    }
    void *objspace = rb_gc_impl_objspace_alloc();
    rb_gc_impl_objspace_init(objspace);
 
    return objspace;
}
 
void
rb_objspace_free(void *objspace)
{
    rb_gc_impl_objspace_free(objspace);
}
 
size_t
rb_gc_obj_slot_size(VALUE obj)
{
    return rb_gc_impl_obj_slot_size(obj);
}
 
static inline void
gc_validate_pc(VALUE obj)
{
#if RUBY_DEBUG
    // IMEMOs and objects without a class (e.g managed id table) are not traceable
    if (RB_TYPE_P(obj, T_IMEMO) || !CLASS_OF(obj)) return;
 
    rb_execution_context_t *ec = GET_EC();
    const rb_control_frame_t *cfp = ec->cfp;
    if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && CFP_PC(cfp)) {
        const VALUE *iseq_encoded = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded;
        const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(CFP_ISEQ(cfp))->iseq_size;
        RUBY_ASSERT(CFP_PC(cfp) >= iseq_encoded, "PC not set when allocating, breaking tracing");
        RUBY_ASSERT(CFP_PC(cfp) <= iseq_encoded_end, "PC not set when allocating, breaking tracing");
    }
#endif
}
 
NOINLINE(static void gc_newobj_hook(VALUE obj));
static void
gc_newobj_hook(VALUE obj)
{
    int lev = RB_GC_VM_LOCK_NO_BARRIER();
    {
        size_t slot_size = rb_gc_obj_slot_size(obj);
        memset((char *)obj + sizeof(struct RBasic), 0, slot_size - sizeof(struct RBasic));
 
        /* We must disable GC here because the callback could call xmalloc
         * which could potentially trigger a GC, and a lot of code is unsafe
         * to trigger a GC right after an object has been allocated because
         * they perform initialization for the object and assume that the
         * GC does not trigger before then. */
        bool gc_disabled = RTEST(rb_gc_local_disable_no_rest());
        {
            rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_NEWOBJ);
        }
        if (!gc_disabled) rb_gc_local_enable();
    }
    RB_GC_VM_UNLOCK_NO_BARRIER(lev);
}
 
ALWAYS_INLINE(static VALUE newobj_body(rb_ractor_t *cr, void *objspace, VALUE klass, VALUE flags, shape_id_t shape_id, bool wb_protected, size_t size));
 
/* The allocation body shared by rb_newobj and rb_ec_newobj_of, forced inline into
 * both: left to this big translation unit's inline budget, gcc drops it from one
 * entry point or the other and that allocation path grows a call. */
static VALUE
newobj_body(rb_ractor_t *cr, void *objspace, VALUE klass, VALUE flags, shape_id_t shape_id, bool wb_protected, size_t size)
{
    GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
    size_t actual_alloc_size;
    VALUE obj = rb_gc_impl_new_obj(objspace, cr->newobj_cache, klass, flags, wb_protected, size, &actual_alloc_size);
 
    GC_ASSERT(actual_alloc_size >= size);
    shape_id = rb_shape_transition_slot_size(shape_id, actual_alloc_size);
 
    RBASIC_SET_FULL_SHAPE_ID_NO_CHECKS(obj, shape_id);
 
    gc_validate_pc(obj);
 
    if (UNLIKELY(rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_NEWOBJ))) {
        gc_newobj_hook(obj);
    }
 
    if (RUBY_DTRACE_GC_OBJ_NEW_ENABLED()) {
        RUBY_DTRACE_GC_OBJ_NEW((void*)obj, flags);
    }
 
#if RGENGC_CHECK_MODE
# ifndef GC_DEBUG_SLOT_FILL_SPECIAL_VALUE
#  define GC_DEBUG_SLOT_FILL_SPECIAL_VALUE 255
# endif
 
    memset(
        (void *)(obj + sizeof(struct RBasic)),
        GC_DEBUG_SLOT_FILL_SPECIAL_VALUE,
        rb_gc_obj_slot_size(obj) - sizeof(struct RBasic)
    );
#endif
 
    return obj;
}
 
VALUE
rb_newobj(rb_execution_context_t *ec, VALUE klass, VALUE flags, shape_id_t shape_id, bool wb_protected, size_t size)
{
    /* Read the Ractor's slot directly: rb_gc_get_objspace() would look cr up through
     * TLS on every allocation. */
    rb_ractor_t *cr = rb_ec_ractor_ptr(ec);
    return newobj_body(cr, cr->objspace, klass, flags, shape_id, wb_protected, size);
}
 
/* Build the object in a named objspace rather than the current Ractor's.  The only
 * foreign objspace allowed is the child this Ractor is building
 * (create_ractor_alloc_thread), whose wrappers must be objects the child owns.
 *
 * That target has no thread of its own yet, which is what makes this cheap: nothing
 * allocates, sweeps or collects there, so the half-built objects need no root and its
 * GC can be suppressed outright.  Aiming at a live Ractor's objspace -- to build a
 * copy where it will be used, say -- needs three things this does not have: a root the
 * target's own GC marks the objects under construction from, a newobj_cache paired
 * with that objspace (today the only multi-objspace collector has no per-Ractor cache,
 * and the ones that do are single-objspace), and write barriers aimed at the target.
 * The assertion stops the shortcut. */
static VALUE
rb_newobj_in_objspace(rb_execution_context_t *ec, void *objspace, VALUE klass, VALUE flags, shape_id_t shape_id, bool wb_protected, size_t size)
{
    rb_ractor_t *cr = rb_ec_ractor_ptr(ec);
    RUBY_ASSERT(objspace == cr->objspace || objspace == cr->creating_child_objspace);
    return newobj_body(cr, objspace, klass, flags, shape_id, wb_protected, size);
}
 
VALUE
rb_ec_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flags, size_t size)
{
    VALUE type = flags & T_MASK;
    RUBY_ASSERT(type != T_OBJECT);
    RUBY_ASSERT(type != T_DATA);
    RUBY_ASSERT(type != T_CLASS);
    RUBY_ASSERT(type != T_MODULE);
    RUBY_ASSERT(type != T_ICLASS);
    (void)type;
 
    rb_ractor_t *cr = rb_ec_ractor_ptr(ec);
    return newobj_body(cr, cr->objspace, klass, flags, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, true, size);
}
 
static VALUE
rb_newobj_of_with_shape(VALUE klass, VALUE flags, shape_id_t shape_id, size_t size)
{
    return rb_newobj(GET_EC(), klass, flags, shape_id, true, size);
}
 
VALUE
rb_newobj_of(VALUE klass, VALUE flags, size_t size)
{
    return rb_newobj(GET_EC(), klass, flags, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, true, size);
}
 
static
VALUE class_allocate_complex_instance(VALUE klass, uint32_t capacity)
{
    VALUE obj = rb_newobj_of_with_shape(klass, T_OBJECT, rb_shape_transition_extended(ROOT_COMPLEX_SHAPE_ID), sizeof(struct RObject));
    // The shape already says extended, so a GC during the allocation below
    // would mark an uninitialized as.extended.
    ROBJECT(obj)->as.extended = Qfalse;
    VALUE fields_obj = rb_imemo_fields_new_complex(obj, ROOT_COMPLEX_SHAPE_ID, capacity, false);
    ROBJECT_SET_EXTENDED(obj, fields_obj);
    return obj;
}
 
static inline size_t
robject_embedded_size(uint32_t fields_count)
{
    size_t size = rb_obj_embedded_size(fields_count);
    if (!rb_gc_size_allocatable_p(size)) {
        size = sizeof(struct RObject);
    }
    return size;
}
 
VALUE
rb_class_allocate_instance_capa(VALUE klass, attr_index_t max_iv_count)
{
    VALUE obj;
 
    // Directly start as COMPLEX if we know we're over the limit.
    RUBY_ASSERT(rb_shape_max_capacity() > 0);
    if (RB_UNLIKELY(max_iv_count > rb_shape_max_capacity())) {
        obj = class_allocate_complex_instance(klass, max_iv_count);
    }
    else {
        size_t size = robject_embedded_size(max_iv_count);
 
        // There might be a NEWOBJ tracepoint callback, and it may set fields.
        // So the shape must be passed to `NEWOBJ_OF`.
        obj = rb_newobj_of_with_shape(klass, T_OBJECT, rb_shape_transition_robject(0), size);
 
        #if RUBY_DEBUG
            VALUE *ptr = ROBJECT_FIELDS(obj);
            shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
            attr_index_t fields_count = RSHAPE_LEN(shape_id);
            attr_index_t capacity = RSHAPE_CAPACITY(shape_id);
 
            for (attr_index_t i = fields_count; i < capacity; i++) {
                ptr[i] = Qundef;
            }
        #endif
    }
 
#if RUBY_DEBUG
    if (rb_obj_class(obj) != rb_class_real(klass)) {
        rb_bug("Expected rb_class_allocate_instance to set the class correctly");
    }
#endif
 
    return obj;
}
 
VALUE
rb_class_allocate_instance(VALUE klass)
{
    return rb_class_allocate_instance_capa(klass, RCLASS_MAX_IV_COUNT(klass));
}
 
#if USE_ZJIT
bool
rb_zjit_class_allocate_instance_fastpath(VALUE klass, size_t *size_out, VALUE *flags_out)
{
    uint32_t index_tbl_num_entries = RCLASS_MAX_IV_COUNT(klass);
 
    RUBY_ASSERT(rb_shape_max_capacity() > 0);
    if (RB_UNLIKELY(index_tbl_num_entries > rb_shape_max_capacity())) {
        return false;
    }
 
    *size_out = robject_embedded_size(index_tbl_num_entries);
    *flags_out = T_OBJECT | rb_shape_transition_robject(0);
 
    return true;
}
 
bool
rb_zjit_newobj_hook_enabled_p(void)
{
    return rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_NEWOBJ);
}
#endif
 
void
rb_gc_register_pinning_obj(VALUE obj)
{
    rb_gc_impl_register_pinning_obj(rb_gc_get_objspace(), obj);
}
 
#define UNEXPECTED_NODE(func) \
    rb_bug(#func"(): GC does not handle T_NODE 0x%x(%p) 0x%"PRIxVALUE, \
           BUILTIN_TYPE(obj), (void*)(obj), RBASIC(obj)->flags)
 
static inline void
rb_data_object_check(VALUE klass)
{
    RUBY_ASSERT(!RCLASS_SINGLETON_P(klass));
    if (klass != rb_cObject && (rb_get_alloc_func(klass) == rb_class_allocate_instance)) {
        rb_undef_alloc_func(klass);
        rb_warn("undefining the allocator of T_DATA class %"PRIsVALUE, klass);
    }
}
 
#define RTYPEDDATA_EMBEDDED_P rbimpl_typeddata_embedded_p
#define RB_DATA_TYPE_EMBEDDABLE_P(type) ((type)->flags & RUBY_TYPED_EMBEDDABLE)
#define RTYPEDDATA_EMBEDDABLE_P(obj) RB_DATA_TYPE_EMBEDDABLE_P(RTYPEDDATA_TYPE(obj))
 
static VALUE
typed_data_alloc_in(void *objspace, VALUE klass, VALUE typed_flag, void *datap, const rb_data_type_t *type, size_t size)
{
    RBIMPL_NONNULL_ARG(type);
    if (klass) rb_data_object_check(klass);
    bool wb_protected = (type->flags & RUBY_FL_WB_PROTECTED) || !type->function.dmark;
    VALUE obj = rb_newobj_in_objspace(GET_EC(), objspace, klass, T_DATA, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_RDATA, wb_protected, size);
 
    rb_gc_register_pinning_obj(obj);
 
    struct RTypedData *data = (struct RTypedData *)obj;
    data->fields_obj = 0;
    *(VALUE *)&data->type = ((VALUE)type) | typed_flag;
    data->data = datap;
 
    return obj;
}
 
static VALUE
typed_data_wrap_in(void *objspace, VALUE klass, void *datap, const rb_data_type_t *type)
{
    if (UNLIKELY(RB_DATA_TYPE_EMBEDDABLE_P(type))) {
        rb_raise(rb_eTypeError, "Cannot wrap an embeddable TypedData");
    }
 
    return typed_data_alloc_in(objspace, klass, 0, datap, type, sizeof(struct RTypedData));
}
 
VALUE
rb_data_typed_object_wrap(VALUE klass, void *datap, const rb_data_type_t *type)
{
    return typed_data_wrap_in(rb_ec_ractor_ptr(GET_EC())->objspace, klass, datap, type);
}
 
VALUE
rb_data_typed_object_wrap_in_objspace(void *objspace, VALUE klass, void *datap, const rb_data_type_t *type)
{
    return typed_data_wrap_in(objspace, klass, datap, type);
}
 
static VALUE
typed_data_zalloc_in(void *objspace, VALUE klass, size_t size, const rb_data_type_t *type)
{
    if (RB_DATA_TYPE_EMBEDDABLE_P(type)) {
        if (!(type->flags & (RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_THREAD_SAFE_FREE))) {
            rb_raise(rb_eTypeError, "Embeddable TypedData must be freed immediately");
        }
 
        /* A deferred free outlives the slot: the sweep copies the type and data pointer
         * out and hands the slot straight back, which it can only do when the payload is
         * not in the slot.  Force such a type onto the heap -- an embeddable type already
         * has to cope with that, since a payload too large for a slot lands there too. */
        size_t embed_size = offsetof(struct RTypedData, data) + size;
        if (rb_gc_size_allocatable_p(embed_size) && !rb_gc_data_type_deferred_free_p(type)) {
            VALUE obj = typed_data_alloc_in(objspace, klass, TYPED_DATA_EMBEDDED, 0, type, embed_size);
            memset((char *)obj + offsetof(struct RTypedData, data), 0, size);
            return obj;
        }
    }
 
    VALUE obj = typed_data_alloc_in(objspace, klass, 0, NULL, type, sizeof(struct RTypedData));
    DATA_PTR(obj) = xcalloc(1, size);
    return obj;
}
 
VALUE
rb_data_typed_object_zalloc(VALUE klass, size_t size, const rb_data_type_t *type)
{
    return typed_data_zalloc_in(rb_ec_ractor_ptr(GET_EC())->objspace, klass, size, type);
}
 
VALUE
rb_data_typed_object_zalloc_in_objspace(void *objspace, VALUE klass, size_t size, const rb_data_type_t *type)
{
    return typed_data_zalloc_in(objspace, klass, size, type);
}
 
static size_t
ruby_xmalloc_usable_size(void *ptr)
{
#ifdef HAVE_MALLOC_USABLE_SIZE
#if CALC_EXACT_MALLOC_SIZE
    struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
    return malloc_usable_size(info) - sizeof(struct malloc_obj_info);
#else
    return malloc_usable_size(ptr);
#endif
#else
    return 0;
#endif
}
 
static size_t
rb_objspace_data_type_memsize(VALUE obj)
{
    size_t size = 0;
    const void *ptr = RTYPEDDATA_GET_DATA(obj);
 
    if (ptr) {
        if (RTYPEDDATA_EMBEDDABLE_P(obj) && !RTYPEDDATA_EMBEDDED_P(obj)) {
            size += ruby_xmalloc_usable_size((void *)ptr);
        }
 
        const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
        if (type->function.dsize) {
            size += type->function.dsize(ptr);
        }
    }
 
    return size;
}
 
const char *
rb_objspace_data_type_name(VALUE obj)
{
    return RTYPEDDATA_TYPE(obj)->wrap_struct_name;
}
 
void
rb_gc_declare_weak_references(VALUE obj)
{
    rb_gc_impl_declare_weak_references(rb_gc_get_objspace(), obj);
}
 
bool
rb_gc_handle_weak_references_alive_p(VALUE obj)
{
    if (SPECIAL_CONST_P(obj)) return true;
 
    return rb_gc_impl_handle_weak_references_alive_p(rb_gc_get_objspace(), obj);
}
 
void
rb_gc_handle_weak_references(VALUE obj)
{
    switch (BUILTIN_TYPE(obj)) {
      case T_DATA:
        {
            const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
 
            if (type->function.handle_weak_references) {
                (type->function.handle_weak_references)(RTYPEDDATA_GET_DATA(obj));
            }
            else {
                rb_bug(
                    "rb_gc_handle_weak_references: TypedData %s does not implement handle_weak_references",
                    RTYPEDDATA_TYPE(obj)->wrap_struct_name
                );
            }
        }
        break;
 
      case T_IMEMO: {
        switch (imemo_type(obj)) {
          case imemo_callcache: {
            struct rb_callcache *cc = (struct rb_callcache *)obj;
            if (cc->klass != Qundef &&
                (!rb_gc_handle_weak_references_alive_p(cc->klass) ||
                 !rb_gc_handle_weak_references_alive_p((VALUE)cc->cme_))) {
                vm_cc_invalidate(cc);
            }
            break;
          }
          case imemo_subclasses: {
            struct rb_subclasses *subs = (struct rb_subclasses *)obj;
            VALUE *entries = rb_imemo_subclasses_entries(obj);
            for (uint32_t i = 0; i < subs->count; i++) {
                if (entries[i] && !rb_gc_handle_weak_references_alive_p(entries[i])) {
                    entries[i] = 0;
                }
            }
            break;
          }
          default:
            rb_bug("rb_gc_handle_weak_references: unexpected imemo type");
        }
 
        break;
      }
      default:
        rb_bug("rb_gc_handle_weak_references: type not supported\n");
    }
}
 
static inline bool
rb_gc_imemo_needs_cleanup_p(VALUE obj)
{
    switch (imemo_type(obj)) {
      case imemo_constcache:
      case imemo_cref:
      case imemo_ifunc:
      case imemo_memo:
      case imemo_svar:
      case imemo_callcache:
      case imemo_throw_data:
      case imemo_cvar_entry:
        return false;
 
      case imemo_env:
      case imemo_ment:
      case imemo_iseq:
      case imemo_callinfo:
      case imemo_cdhash:
        return true;
 
      case imemo_subclasses:
        return FL_TEST_RAW(obj, IMEMO_SUBCLASSES_HEAP);
 
      case imemo_tmpbuf:
        return ((rb_imemo_tmpbuf_t *)obj)->ptr != NULL;
 
      case imemo_fields:
        return rb_obj_shape_complex_p(obj);
    }
    UNREACHABLE_RETURN(true);
}
 
/*
 * Returns true if the object requires a full rb_gc_obj_free() call during sweep,
 * false if it can be freed quickly without calling destructors or cleanup.
 *
 * Objects that return false are:
 * - Simple embedded objects without external allocations
 * - Objects without finalizers
 * - Objects without generic instance variables
 *
 * This is used by the GC sweep fast path to avoid function call overhead
 * for the majority of simple objects.
 */
bool
rb_gc_obj_needs_cleanup_p(VALUE obj)
{
    VALUE flags = RBASIC(obj)->flags;
 
    if (flags & FL_FINALIZE) return true;
 
    if ((flags & RUBY_T_MASK) == T_IMEMO) {
        return rb_gc_imemo_needs_cleanup_p(obj);
    }
 
    /* A host with generic fields must drop its table entry when it is freed.  The
     * process-wide table holds every Ractor's entries, so a sweep cannot bulk-wipe it;
     * this per-object cleanup carries the correctness. */
    shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
    if (rb_shape_has_fields(shape_id) && rb_shape_layout(shape_id) == SHAPE_ID_LAYOUT_OTHER) {
        return true;
    }
 
    switch (flags & RUBY_T_MASK) {
      case T_FLOAT:
      case T_RATIONAL:
      case T_COMPLEX:
      case T_OBJECT:
        return false;
 
      case T_FILE:
      case T_SYMBOL:
      case T_CLASS:
      case T_ICLASS:
      case T_MODULE:
      case T_REGEXP:
        return true;
 
      case T_IMEMO:
        UNREACHABLE_RETURN(true);
 
      case T_DATA:
        {
            uintptr_t type = (uintptr_t)RTYPEDDATA(obj)->type;
            if (type & TYPED_DATA_EMBEDDED) {
                RUBY_DATA_FUNC dfree = ((const rb_data_type_t *)(type & TYPED_DATA_PTR_MASK))->function.dfree;
                if (dfree == RUBY_NEVER_FREE || dfree == RUBY_TYPED_DEFAULT_FREE) {
                    return false;
                }
            }
        }
        return true;
 
      case T_STRING:
        return (flags & (RSTRING_NOEMBED | RSTRING_FSTR));
 
      case T_ARRAY:
        return !(flags & RARRAY_EMBED_FLAG);
 
      case T_HASH:
        return (flags & RHASH_ST_TABLE_FLAG);
 
      case T_MATCH:
        return (flags & (RMATCH_ONIG | RMATCH_OFFSETS_EXTERNAL)) || USE_DEBUG_COUNTER;
 
      case T_BIGNUM:
        return !(flags & BIGNUM_EMBED_FLAG);
 
      case T_STRUCT:
        return !(flags & RSTRUCT_EMBED_LEN_MASK);
    }
 
    UNREACHABLE_RETURN(true);
}
 
static void
io_fptr_finalize(void *fptr)
{
    rb_io_fptr_finalize((struct rb_io *)fptr);
}
 
static inline void
make_io_zombie(void *objspace, VALUE obj)
{
    rb_io_t *fptr = RFILE(obj)->fptr;
    rb_gc_impl_make_zombie(objspace, obj, io_fptr_finalize, fptr);
}
 
static bool
rb_data_free(void *objspace, VALUE obj)
{
    void *data = RTYPEDDATA_GET_DATA(obj);
    if (data) {
        const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
        void (*dfree)(void *) = type->function.dfree;
 
        if (dfree) {
            bool embedded = RTYPEDDATA_EMBEDDED_P(obj);
            int free_immediately = (type->flags & (RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_THREAD_SAFE_FREE)) != 0;
            bool free_embeddable_data = RB_DATA_TYPE_EMBEDDABLE_P(type) && !embedded;
 
            if (dfree == RUBY_DEFAULT_FREE) {
                if (!embedded) {
                    xfree(data);
                    RB_DEBUG_COUNTER_INC(obj_data_xfree);
                }
            }
            else if (free_immediately) {
                (*dfree)(data);
                if (free_embeddable_data) {
                    xfree(data);
                }
 
                RB_DEBUG_COUNTER_INC(obj_data_imm_free);
            }
            else {
                rb_gc_impl_make_zombie(objspace, obj, dfree, data);
                RB_DEBUG_COUNTER_INC(obj_data_zombie);
                return FALSE;
            }
        }
        else {
            RB_DEBUG_COUNTER_INC(obj_data_empty);
        }
    }
 
    return true;
}
 
struct classext_foreach_args {
    VALUE klass;
    rb_objspace_t *objspace; // used for update_*
};
 
static void
classext_free(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg)
{
    struct classext_foreach_args *args = (struct classext_foreach_args *)arg;
 
    rb_class_classext_free(args->klass, ext, is_prime);
}
 
static void
classext_iclass_free(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg)
{
    struct classext_foreach_args *args = (struct classext_foreach_args *)arg;
 
    rb_iclass_classext_free(args->klass, ext, is_prime);
}
 
bool
rb_gc_obj_free(void *objspace, VALUE obj)
{
    struct classext_foreach_args args;
 
    RB_DEBUG_COUNTER_INC(obj_free);
 
    enum ruby_value_type builtin_type = BUILTIN_TYPE(obj);
 
    if (RUBY_DTRACE_GC_OBJ_FREE_ENABLED()) {
        RUBY_DTRACE_GC_OBJ_FREE((void*)obj, RBASIC(obj)->flags);
    }
 
    switch (builtin_type) {
      case T_NIL:
      case T_FIXNUM:
      case T_TRUE:
      case T_FALSE:
        rb_bug("obj_free() called for broken object");
        break;
      default:
        break;
    }
 
    switch (builtin_type) {
      case T_OBJECT:
        break;
      case T_MODULE:
      case T_CLASS:
#if USE_ZJIT
        rb_zjit_klass_free(obj);
#endif
        args.klass = obj;
        rb_class_classext_foreach(obj, classext_free, (void *)&args);
        if (RCLASS_CLASSEXT_TBL(obj)) {
            st_free_table(RCLASS_CLASSEXT_TBL(obj));
        }
        (void)RB_DEBUG_COUNTER_INC_IF(obj_module_ptr, BUILTIN_TYPE(obj) == T_MODULE);
        (void)RB_DEBUG_COUNTER_INC_IF(obj_class_ptr, BUILTIN_TYPE(obj) == T_CLASS);
        break;
      case T_STRING:
        rb_str_free(obj);
        break;
      case T_ARRAY:
        rb_ary_free(obj);
        break;
      case T_HASH:
#if USE_DEBUG_COUNTER
        switch (RHASH_SIZE(obj)) {
          case 0:
            RB_DEBUG_COUNTER_INC(obj_hash_empty);
            break;
          case 1:
            RB_DEBUG_COUNTER_INC(obj_hash_1);
            break;
          case 2:
            RB_DEBUG_COUNTER_INC(obj_hash_2);
            break;
          case 3:
            RB_DEBUG_COUNTER_INC(obj_hash_3);
            break;
          case 4:
            RB_DEBUG_COUNTER_INC(obj_hash_4);
            break;
          case 5:
          case 6:
          case 7:
          case 8:
            RB_DEBUG_COUNTER_INC(obj_hash_5_8);
            break;
          default:
            GC_ASSERT(RHASH_SIZE(obj) > 8);
            RB_DEBUG_COUNTER_INC(obj_hash_g8);
        }
 
        if (RHASH_AR_TABLE_P(obj)) {
            if (RHASH_AR_TABLE(obj) == NULL) {
                RB_DEBUG_COUNTER_INC(obj_hash_null);
            }
            else {
                RB_DEBUG_COUNTER_INC(obj_hash_ar);
            }
        }
        else {
            RB_DEBUG_COUNTER_INC(obj_hash_st);
        }
#endif
 
        rb_hash_free(obj);
        break;
      case T_REGEXP:
        if (FL_TEST_RAW(obj, RREGEXP_INITIALIZED)) {
            onig_free_body(RREGEXP_PTR(obj));
            RB_DEBUG_COUNTER_INC(obj_regexp_ptr);
        }
        break;
      case T_DATA:
        if (!rb_data_free(objspace, obj)) return false;
        break;
      case T_MATCH:
        {
            struct RMatch *rm = RMATCH(obj);
#if USE_DEBUG_COUNTER
            if (rm->num_regs >= 8) {
                RB_DEBUG_COUNTER_INC(obj_match_ge8);
            }
            else if (rm->num_regs >= 4) {
                RB_DEBUG_COUNTER_INC(obj_match_ge4);
            }
            else if (rm->num_regs >= 1) {
                RB_DEBUG_COUNTER_INC(obj_match_under4);
            }
#endif
            if (FL_TEST_RAW(obj, RMATCH_ONIG)) {
                onig_region_free(&rm->as.onig, 0);
            }
            SIZED_FREE_N(rm->char_offset, rm->char_offset_num_allocated);
 
            RB_DEBUG_COUNTER_INC(obj_match_ptr);
        }
        break;
      case T_FILE:
        if (RFILE(obj)->fptr) {
            bool closed = rb_io_fptr_finalize_closed(RFILE(obj)->fptr);
            if (!closed) make_io_zombie(objspace, obj);
            RB_DEBUG_COUNTER_INC(obj_file_ptr);
            return closed;
        }
        break;
      case T_RATIONAL:
        RB_DEBUG_COUNTER_INC(obj_rational);
        break;
      case T_COMPLEX:
        RB_DEBUG_COUNTER_INC(obj_complex);
        break;
      case T_MOVED:
        break;
      case T_ICLASS:
        args.klass = obj;
 
        rb_class_classext_foreach(obj, classext_iclass_free, (void *)&args);
        if (RCLASS_CLASSEXT_TBL(obj)) {
            st_free_table(RCLASS_CLASSEXT_TBL(obj));
        }
 
        RB_DEBUG_COUNTER_INC(obj_iclass_ptr);
        break;
 
      case T_FLOAT:
        RB_DEBUG_COUNTER_INC(obj_float);
        break;
 
      case T_BIGNUM:
        if (!BIGNUM_EMBED_P(obj) && BIGNUM_DIGITS(obj)) {
            SIZED_FREE_N(BIGNUM_DIGITS(obj), BIGNUM_LEN(obj));
            RB_DEBUG_COUNTER_INC(obj_bignum_ptr);
        }
        else {
            RB_DEBUG_COUNTER_INC(obj_bignum_embed);
        }
        break;
 
      case T_NODE:
        UNEXPECTED_NODE(obj_free);
        break;
 
      case T_STRUCT:
        if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) ||
            RSTRUCT(obj)->as.heap.ptr == NULL) {
            RB_DEBUG_COUNTER_INC(obj_struct_embed);
        }
        else {
            SIZED_FREE_N(RSTRUCT(obj)->as.heap.ptr, RSTRUCT(obj)->as.heap.len);
            RB_DEBUG_COUNTER_INC(obj_struct_ptr);
        }
        break;
 
      case T_SYMBOL:
        RB_DEBUG_COUNTER_INC(obj_symbol);
        break;
 
      case T_IMEMO:
        rb_imemo_free((VALUE)obj);
        break;
 
      default:
        rb_bug("gc_sweep(): unknown data type 0x%x(%p) 0x%"PRIxVALUE,
               BUILTIN_TYPE(obj), (void*)obj, RBASIC(obj)->flags);
    }
 
    if (FL_TEST_RAW(obj, FL_FINALIZE)) {
        rb_gc_impl_make_zombie(objspace, obj, 0, 0);
        return FALSE;
    }
    else {
        return TRUE;
    }
}
 
void
rb_objspace_set_event_hook(const rb_event_flag_t event)
{
    /* Only the main objspace may enable the FREEOBJ hook: it runs user callbacks from
     * inside the sweep, which is unsafe in a non-main Ractor's lock-free local GC.
     * Extending it VM-wide is future work. */
    rb_event_flag_t e = event;
    const rb_ractor_t *const cr = rb_current_ractor_raw(false);
    if (cr != NULL && cr != GET_VM()->ractor.main_ractor) {
        e &= ~RUBY_INTERNAL_EVENT_FREEOBJ;
    }
    rb_gc_impl_set_event_hook(rb_gc_get_objspace(), e);
}
 
static int
internal_object_p(VALUE obj)
{
    void *ptr = asan_unpoison_object_temporary(obj);
 
    if (RBASIC(obj)->flags) {
        switch (BUILTIN_TYPE(obj)) {
          case T_NODE:
            UNEXPECTED_NODE(internal_object_p);
            break;
          case T_NONE:
          case T_MOVED:
          case T_IMEMO:
          case T_ICLASS:
          case T_ZOMBIE:
            break;
          case T_CLASS:
            if (obj == rb_mRubyVMFrozenCore)
                return 1;
 
            if (!RBASIC_CLASS(obj)) break;
            if (RCLASS_SINGLETON_P(obj)) {
                return rb_singleton_class_internal_p(obj);
            }
            return 0;
          default:
            if (!RBASIC(obj)->klass) break;
            return 0;
        }
    }
    if (ptr || !RBASIC(obj)->flags) {
        rb_asan_poison_object(obj);
    }
    return 1;
}
 
int
rb_objspace_internal_object_p(VALUE obj)
{
    return internal_object_p(obj);
}
 
struct os_each_struct {
    size_t num;
    VALUE of;
};
 
static int
os_obj_of_i(void *vstart, void *vend, size_t stride, void *data)
{
    struct os_each_struct *oes = (struct os_each_struct *)data;
 
    VALUE v = (VALUE)vstart;
    for (; v != (VALUE)vend; v += stride) {
        if (!internal_object_p(v)) {
            if (!oes->of || rb_obj_is_kind_of(v, oes->of)) {
                rb_yield(v);
                oes->num++;
            }
        }
    }
 
    return 0;
}
 
/* Like os_obj_of_i but collects into an array: foreign shareable objects are walked
 * under the barrier, where yielding is unsafe (see os_obj_of).  Pure C, allocates no
 * object (rb_ary_push only grows the buffer), so it reaches no safepoint. */
struct os_shareable_collect_struct {
    VALUE of;
    VALUE buffer;
};
 
static int
os_shareable_collect_i(void *vstart, void *vend, size_t stride, void *data)
{
    struct os_shareable_collect_struct *ocs = (struct os_shareable_collect_struct *)data;
 
    VALUE v = (VALUE)vstart;
    for (; v != (VALUE)vend; v += stride) {
        /* We walk a foreign Ractor's objspace, so collect only shareable objects.  The
         * walk already filters on shareable_bits; check again so an unshareable object
         * can never be exposed. */
        if (rb_ractor_shareable_p(v) && !internal_object_p(v)) {
            if (!ocs->of || rb_obj_is_kind_of(v, ocs->of)) {
                rb_ary_push(ocs->buffer, v);
            }
        }
    }
 
    return 0;
}
 
static void rb_gc_critical_disable(void);
static void rb_gc_critical_enable(void);
 
static VALUE
os_obj_of(VALUE of)
{
    struct os_each_struct oes;
 
    oes.num = 0;
    oes.of = of;
 
    /* Phase 1: our own Ractor's objspace, yielding every object directly with no
     * barrier.  The walk snapshots the page list and tolerates pages being freed
     * concurrently, so no VM lock is needed and the block may allocate, GC or block. */
    rb_gc_impl_each_objects(rb_gc_get_objspace(), os_obj_of_i, &oes);
 
    /* Phase 2 (multi-Ractor): other live Ractors' shareable objects, readable only
     * under the barrier (where a user block must not run), so collect them in pure C
     * with GC disabled and yield after the barrier is released. */
    if (rb_multi_ractor_p()) {
        struct os_shareable_collect_struct ocs;
        ocs.of = of;
        ocs.buffer = rb_ary_new();
 
        rb_gc_critical_disable();
        RB_VM_LOCKING() {
            rb_vm_barrier();
 
            void *self = rb_gc_get_objspace();
            rb_vm_t *vm = GET_VM();
            rb_ractor_t *r;
            ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
                if (r->objspace && r->objspace != self) {
                    rb_gc_impl_each_objects_shareable(r->objspace, os_shareable_collect_i, &ocs);
                }
            }
        }
        rb_gc_critical_enable();
 
        long len = RARRAY_LEN(ocs.buffer);
        for (long i = 0; i < len; i++) {
            rb_yield(RARRAY_AREF(ocs.buffer, i));
            oes.num++;
        }
        RB_GC_GUARD(ocs.buffer);
    }
 
    return SIZET2NUM(oes.num);
}
 
/*
 *  call-seq:
 *     ObjectSpace.each_object         {|obj| ... } -> integer
 *     ObjectSpace.each_object(module) {|obj| ... } -> integer
 *     ObjectSpace.each_object                      -> enumerator
 *     ObjectSpace.each_object(module)              -> enumerator
 *
 *  Calls the block once for each living, non-immediate object in this Ruby
 *  process, and returns the number of objects found.
 *
 *  If +module+ is given, calls the block only for objects that are an instance
 *  of +module+ or one of its subclasses.
 *
 *  Immediate objects (such as small integers, static symbols, +true+, +false+,
 *  and +nil+) are never yielded.
 *
 *  With no block given, returns a new Enumerator.
 *
 *     Job = Class.new
 *     jobs = [Job.new, Job.new]
 *     count = ObjectSpace.each_object(Job) {|x| p x }
 *     puts "Total count: #{count}"
 *
 *  <em>produces:</em>
 *
 *    #<Job:0x000000011d6cbbf0>
 *    #<Job:0x000000011d6cbc68>
 *    Total count: 2
 *
 *  Because every live object is visited, this method is mainly useful for
 *  debugging, profiling, and introspecting a running process.
 *
 *  In multi-Ractor mode this method yields every object of the current Ractor, plus
 *  the objects of the other Ractors that have been made Ractor-shareable.  Another
 *  Ractor's unshareable objects are never yielded: they belong to that Ractor and the
 *  current one must not touch them.
 *
 *     c = {}                            # not shareable, belongs to the main Ractor
 *     r = Ractor.new { d = {}; receive } # d belongs to r
 *     ObjectSpace.each_object {|x| x }  # yields c, but not d
 *
 */
 
static VALUE
os_each_obj(int argc, VALUE *argv, VALUE os)
{
    VALUE of;
 
    of = (!rb_check_arity(argc, 0, 1) ? 0 : argv[0]);
    RETURN_ENUMERATOR(os, 1, &of);
    return os_obj_of(of);
}
 
/*
 *  call-seq:
 *     ObjectSpace.undefine_finalizer(obj) -> obj
 *
 *  Removes all finalizers registered for +obj+ with
 *  ObjectSpace.define_finalizer, and returns +obj+.
 *
 *  Does nothing if +obj+ has no finalizers.
 */
 
static VALUE
undefine_final(VALUE os, VALUE obj)
{
    return rb_undefine_finalizer(obj);
}
 
VALUE
rb_undefine_finalizer(VALUE obj)
{
    rb_check_frozen(obj);
 
    rb_gc_impl_undefine_finalizer(rb_gc_get_objspace(), obj);
 
    return obj;
}
 
static void
should_be_callable(VALUE block)
{
    if (!rb_obj_respond_to(block, idCall, TRUE)) {
        rb_raise(rb_eArgError, "wrong type argument %"PRIsVALUE" (should be callable)",
                 rb_obj_class(block));
    }
}
 
static void
should_be_finalizable(VALUE obj)
{
    if (!FL_ABLE(obj)) {
        rb_raise(rb_eArgError, "cannot define finalizer for %s",
                 rb_obj_classname(obj));
    }
    rb_check_frozen(obj);
}
 
void
rb_gc_copy_finalizer(VALUE dest, VALUE obj)
{
    rb_gc_impl_copy_finalizer(rb_gc_get_objspace(), dest, obj);
}
 
/*
 *  call-seq:
 *     ObjectSpace.define_finalizer(obj) {|id| ... } -> array
 *     ObjectSpace.define_finalizer(obj, finalizer) -> array
 *
 *  Adds a new finalizer for +obj+ that is called when +obj+ is destroyed
 *  by the garbage collector or when Ruby shuts down (which ever comes first).
 *
 *  With a block given, uses the block as the callback. Without a block given,
 *  uses a callable object +finalizer+ as the callback. The callback is called
 *  when +obj+ is destroyed with a single argument +id+ which is the object
 *  ID of +obj+ (see Object#object_id).
 *
 *  The return value is an array <code>[0, callback]</code>, where +callback+
 *  is a Proc created from the block if one was given or +finalizer+ otherwise.
 *
 *  Note that defining a finalizer in an instance method of the object may prevent
 *  the object from being garbage collected since if the block or +finalizer+ refers
 *  to +obj+ then +obj+ will never be reclaimed by the garbage collector. For example,
 *  the following script demonstrates the issue:
 *
 *      class Foo
 *        def define_final
 *          ObjectSpace.define_finalizer(self) do |id|
 *            puts "Running finalizer for #{id}!"
 *          end
 *        end
 *      end
 *
 *      obj = Foo.new
 *      obj.define_final
 *
 *  There are two patterns to solve this issue:
 *
 *  - Create the finalizer in a non-instance method so it can safely capture
 *    the needed state:
 *
 *      class Foo
 *        def define_final
 *          ObjectSpace.define_finalizer(self, self.class.create_finalizer)
 *        end
 *
 *        def self.create_finalizer
 *          proc do |id|
 *            puts "Running finalizer for #{id}!"
 *          end
 *        end
 *      end
 *
 *  - Use a callable object:
 *
 *      class Foo
 *        class Finalizer
 *          def call(id)
 *            puts "Running finalizer for #{id}!"
 *          end
 *        end
 *
 *        def define_final
 *          ObjectSpace.define_finalizer(self, Finalizer.new)
 *        end
 *      end
 *
 *  Note that finalization can be unpredictable and is never guaranteed
 *  to be run except on exit.
 */
 
static VALUE
define_final(int argc, VALUE *argv, VALUE os)
{
    VALUE obj, block;
 
    rb_scan_args(argc, argv, "11", &obj, &block);
    if (argc == 1) {
        block = rb_block_proc();
    }
 
    if (rb_callable_receiver(block) == obj) {
        rb_warn("finalizer references object to be finalized");
    }
 
    return rb_define_finalizer(obj, block);
}
 
VALUE
rb_define_finalizer(VALUE obj, VALUE block)
{
    should_be_finalizable(obj);
    should_be_callable(block);
 
    block = rb_gc_impl_define_finalizer(rb_gc_get_objspace(), obj, block);
 
    block = rb_ary_new3(2, INT2FIX(0), block);
    OBJ_FREEZE(block);
    return block;
}
 
void
rb_objspace_call_finalizer(void)
{
    rb_gc_impl_shutdown_call_finalizer(rb_gc_get_objspace());
}
 
void
rb_objspace_free_objects(void *objspace)
{
    rb_gc_impl_shutdown_free_objects(objspace);
}
 
int
rb_objspace_garbage_object_p(VALUE obj)
{
    return !SPECIAL_CONST_P(obj) && rb_gc_impl_garbage_object_p(rb_gc_get_objspace(), obj);
}
 
int
rb_objspace_foreign_object_p(VALUE obj)
{
    return !SPECIAL_CONST_P(obj) && rb_gc_obj_foreign_p(obj);
}
 
#define OBJ_ID_INCREMENT (RUBY_IMMEDIATE_MASK + 1)
#define LAST_OBJECT_ID() (object_id_counter * OBJ_ID_INCREMENT)
 
#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
static size_t object_id_counter = 1;
#else
static unsigned long long object_id_counter = 1;
#endif
 
static inline VALUE
generate_next_object_id(void)
{
#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
    // 64bit atomics are available
    return SIZET2NUM(RUBY_ATOMIC_SIZE_FETCH_ADD(object_id_counter, 1) * OBJ_ID_INCREMENT);
#else
    unsigned int lock_lev = RB_GC_VM_LOCK();
    VALUE id = ULL2NUM(++object_id_counter * OBJ_ID_INCREMENT);
    RB_GC_VM_UNLOCK(lock_lev);
    return id;
#endif
}
 
static void gc_mark_tbl_no_pin(st_table *table);
 
static VALUE
class_object_id(VALUE klass)
{
    VALUE id = RUBY_ATOMIC_VALUE_LOAD(RCLASS(klass)->object_id);
    if (!id) {
        unsigned int lock_lev = RB_GC_VM_LOCK();
        id = generate_next_object_id();
        VALUE existing_id = RUBY_ATOMIC_VALUE_CAS(RCLASS(klass)->object_id, 0, id);
        if (existing_id) {
            id = existing_id;
        }
        RB_GC_VM_UNLOCK(lock_lev);
    }
    return id;
}
 
static inline VALUE
object_id_get(VALUE obj, shape_id_t shape_id)
{
    VALUE id;
    if (rb_shape_complex_p(shape_id)) {
        id = rb_obj_field_get(obj, ROOT_COMPLEX_WITH_OBJ_ID);
    }
    else {
        id = rb_obj_field_get(obj, rb_shape_object_id(shape_id));
    }
 
#if RUBY_DEBUG
    if (!(FIXNUM_P(id) || RB_TYPE_P(id, T_BIGNUM))) {
        rb_p(obj);
        rb_bug("Object's shape includes object_id, but it's missing %s", rb_obj_info(obj));
    }
#endif
 
    return id;
}
 
static VALUE
object_id0(VALUE obj)
{
    VALUE id = Qfalse;
    shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
 
    if (rb_shape_has_object_id(shape_id)) {
        return object_id_get(obj, shape_id);
    }
 
    shape_id_t object_id_shape_id = rb_obj_shape_transition_object_id(obj);
 
    id = generate_next_object_id();
    rb_obj_field_set(obj, object_id_shape_id, 0, id);
 
    RUBY_ASSERT(rb_obj_shape_has_id(obj));
 
    return id;
}
 
static VALUE
object_id(VALUE obj)
{
    switch (BUILTIN_TYPE(obj)) {
      case T_CLASS:
      case T_MODULE:
        // With Ruby Box, classes and modules have different fields
        // in different boxes, so we cannot store the object id
        // in fields.
        return class_object_id(obj);
      case T_IMEMO:
        RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields));
        break;
      default:
        break;
    }
 
    if (UNLIKELY(rb_gc_multi_ractor_p() && rb_ractor_shareable_p(obj))) {
        unsigned int lock_lev = RB_GC_VM_LOCK();
        VALUE id = object_id0(obj);
        RB_GC_VM_UNLOCK(lock_lev);
        return id;
    }
 
    return object_id0(obj);
}
 
void
rb_gc_obj_free_vm_weak_references(VALUE obj)
{
    ASSUME(!RB_SPECIAL_CONST_P(obj));
 
    /* Drop a generic-fields entry when its host's slot is freed.  The table is
     * process-wide, so no sweep bulk-wipes it; a stale entry would let the global GC's
     * weak pass (or a reader after the slot is reused) walk a freed page. */
    if (rb_obj_gen_fields_p(obj)) {
        rb_free_generic_ivar(obj);
    }
 
    switch (BUILTIN_TYPE(obj)) {
      case T_STRING:
        if (FL_TEST_RAW(obj, RSTRING_FSTR)) {
            rb_gc_free_fstring(obj);
        }
        break;
      case T_SYMBOL:
        rb_gc_free_dsymbol(obj);
        break;
      case T_IMEMO:
        switch (imemo_type(obj)) {
          case imemo_callinfo:
            rb_vm_ci_free((const struct rb_callinfo *)obj);
            break;
          case imemo_ment:
            rb_free_method_entry_vm_weak_references((const rb_method_entry_t *)obj);
            break;
          default:
            break;
        }
        break;
      default:
        break;
    }
}
 
static VALUE
rb_find_object_id(void *objspace, VALUE obj, VALUE (*get_heap_object_id)(VALUE))
{
    if (SPECIAL_CONST_P(obj)) {
#if SIZEOF_LONG == SIZEOF_VOIDP
        return LONG2NUM((SIGNED_VALUE)obj);
#else
        return LL2NUM((SIGNED_VALUE)obj);
#endif
    }
 
    return get_heap_object_id(obj);
}
 
static VALUE
nonspecial_obj_id(VALUE obj)
{
#if SIZEOF_LONG == SIZEOF_VOIDP
    return (VALUE)((SIGNED_VALUE)(obj)|FIXNUM_FLAG);
#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
    return LL2NUM((SIGNED_VALUE)(obj) / 2);
#else
# error not supported
#endif
}
 
VALUE
rb_memory_id(VALUE obj)
{
    return rb_find_object_id(NULL, obj, nonspecial_obj_id);
}
 
/*
 *  Document-method: __id__
 *  Document-method: object_id
 *
 *  call-seq:
 *     obj.__id__       -> integer
 *     obj.object_id    -> integer
 *
 *  Returns an integer identifier for +obj+.
 *
 *  The same number will be returned on all calls to +object_id+ for a given
 *  object, and no two active objects will share an id.
 *
 *  Note: that some objects of builtin classes are reused for optimization.
 *  This is the case for immediate values and frozen string literals.
 *
 *  BasicObject implements +__id__+, Kernel implements +object_id+.
 *
 *  Immediate values are not passed by reference but are passed by value:
 *  +nil+, +true+, +false+, Fixnums, Symbols, and some Floats.
 *
 *      Object.new.object_id  == Object.new.object_id  # => false
 *      (21 * 2).object_id    == (21 * 2).object_id    # => true
 *      "hello".object_id     == "hello".object_id     # => false
 *      "hi".freeze.object_id == "hi".freeze.object_id # => true
 */
 
VALUE
rb_obj_id(VALUE obj)
{
    /* If obj is an immediate, the object ID is obj directly converted to a Numeric.
     * Otherwise, the object ID is a Numeric that is a non-zero multiple of
     * (RUBY_IMMEDIATE_MASK + 1) which guarantees that it does not collide with
     * any immediates. */
    return rb_find_object_id(rb_gc_get_objspace(), obj, object_id);
}
 
bool
rb_obj_id_p(VALUE obj)
{
    return !RB_TYPE_P(obj, T_IMEMO) && rb_obj_shape_has_id(obj);
}
 
/*
 * GC implementations should call this function before the GC phase that updates references
 * embedded in the machine code generated by JIT compilers.  JIT compilers usually enforce the
 * "W^X" policy and protect the code memory from being modified during execution.  This function
 * makes the code memory writeable.
 */
void
rb_gc_before_updating_jit_code(void)
{
#if USE_YJIT
    rb_yjit_mark_all_writeable();
#endif
#if USE_ZJIT
    rb_zjit_mark_all_writable();
#endif
}
 
/*
 * GC implementations should call this function before the GC phase that updates references
 * embedded in the machine code generated by JIT compilers.  This function makes the code memory
 * executable again.
 */
void
rb_gc_after_updating_jit_code(void)
{
#if USE_YJIT
    rb_yjit_mark_all_executable();
#endif
#if USE_ZJIT
    rb_zjit_mark_all_executable();
#endif
}
 
static void
classext_memsize(rb_classext_t *ext, bool prime, VALUE box_value, void *arg)
{
    size_t *size = (size_t *)arg;
    size_t s = 0;
 
    if (RCLASSEXT_M_TBL(ext)) {
        s += rb_id_table_memsize(RCLASSEXT_M_TBL(ext));
    }
    if (RCLASSEXT_CONST_TBL(ext)) {
        s += rb_id_table_memsize(RCLASSEXT_CONST_TBL(ext));
    }
    if (RCLASSEXT_SUPERCLASSES_WITH_SELF(ext)) {
        s += (RCLASSEXT_SUPERCLASS_DEPTH(ext) + 1) * sizeof(VALUE);
    }
    if (!prime) {
        s += sizeof(rb_classext_t);
    }
    *size += s;
}
 
static void
classext_superclasses_memsize(rb_classext_t *ext, bool prime, VALUE box_value, void *arg)
{
    size_t *size = (size_t *)arg;
    size_t array_size;
    if (RCLASSEXT_SUPERCLASSES_WITH_SELF(ext)) {
        RUBY_ASSERT(prime);
        array_size = RCLASSEXT_SUPERCLASS_DEPTH(ext) + 1;
        *size += array_size * sizeof(VALUE);
    }
}
 
size_t
rb_obj_memsize_of(VALUE obj)
{
    size_t size = 0;
 
    if (SPECIAL_CONST_P(obj)) {
        return 0;
    }
 
    switch (BUILTIN_TYPE(obj)) {
      case T_OBJECT:
        break;
      case T_MODULE:
      case T_CLASS:
        rb_class_classext_foreach(obj, classext_memsize, (void *)&size);
        rb_class_classext_foreach(obj, classext_superclasses_memsize, (void *)&size);
        break;
      case T_ICLASS:
        if (RICLASS_OWNS_M_TBL_P(obj)) {
            if (RCLASS_M_TBL(obj)) {
                size += rb_id_table_memsize(RCLASS_M_TBL(obj));
            }
        }
        break;
      case T_STRING:
        size += rb_str_memsize(obj);
        break;
      case T_ARRAY:
        size += rb_ary_memsize(obj);
        break;
      case T_HASH:
        if (RHASH_ST_TABLE_P(obj)) {
            VM_ASSERT(RHASH_ST_TABLE(obj) != NULL);
            /* st_table is in the slot */
            size += st_memsize(RHASH_ST_TABLE(obj)) - sizeof(st_table);
        }
        break;
      case T_REGEXP:
        if (RREGEXP_PTR(obj)) {
            size += onig_memsize(RREGEXP_PTR(obj));
        }
        break;
      case T_DATA:
        size += rb_objspace_data_type_memsize(obj);
        break;
      case T_MATCH:
        {
            struct RMatch *rm = RMATCH(obj);
            if (FL_TEST_RAW(obj, RMATCH_ONIG)) {
                size += onig_region_memsize(&rm->as.onig);
            }
            size += sizeof(struct rmatch_offset) * rm->char_offset_num_allocated;
        }
        break;
      case T_FILE:
        if (RFILE(obj)->fptr) {
            size += rb_io_memsize(RFILE(obj)->fptr);
        }
        break;
      case T_RATIONAL:
      case T_COMPLEX:
        break;
      case T_IMEMO:
        size += rb_imemo_memsize(obj);
        break;
 
      case T_FLOAT:
      case T_SYMBOL:
        break;
 
      case T_BIGNUM:
        if (!(RBASIC(obj)->flags & BIGNUM_EMBED_FLAG) && BIGNUM_DIGITS(obj)) {
            size += BIGNUM_LEN(obj) * sizeof(BDIGIT);
        }
        break;
 
      case T_NODE:
        UNEXPECTED_NODE(obj_memsize_of);
        break;
 
      case T_STRUCT:
        if (RSTRUCT_EMBED_LEN(obj) == 0) {
            size += sizeof(VALUE) * RSTRUCT_LEN_RAW(obj);
        }
        break;
 
      case T_ZOMBIE:
      case T_MOVED:
        break;
 
      default:
        rb_bug("objspace/memsize_of(): unknown data type 0x%x(%p)",
               BUILTIN_TYPE(obj), (void*)obj);
    }
 
    return size + rb_gc_obj_slot_size(obj);
}
 
static int
set_zero(st_data_t key, st_data_t val, st_data_t arg)
{
    VALUE k = (VALUE)key;
    VALUE hash = (VALUE)arg;
    rb_hash_aset(hash, k, INT2FIX(0));
    return ST_CONTINUE;
}
 
struct count_objects_data {
    size_t counts[T_MASK+1];
    size_t freed;
    size_t total;
};
 
static void
count_objects_i(VALUE obj, void *d)
{
    struct count_objects_data *data = (struct count_objects_data *)d;
 
    if (RBASIC(obj)->flags) {
        data->counts[BUILTIN_TYPE(obj)]++;
    }
    else {
        data->freed++;
    }
 
    data->total++;
}
 
/*
 *  call-seq:
 *     ObjectSpace.count_objects(result_hash = {}) -> hash
 *
 *  Counts the number of objects, grouped by type.
 *
 *  It returns a hash that looks like:
 *
 *    {
 *      TOTAL: 10000,
 *      FREE: 3011,
 *      T_OBJECT: 6,
 *      T_CLASS: 404,
 *      # ...
 *    }
 *
 *  The contents of the returned hash are implementation specific and
 *  may be changed in future versions without notice.
 *
 *  The keys starting with +:T_+ are live objects of a particular type.
 *  For example, +:T_ARRAY+ is the number of arrays.
 *
 *  The key +:FREE+ is the number of object slots which are empty.
 *
 *  The key +:TOTAL+ is the total number of slots (which is the sum of
 *  all of the other values).
 *
 *  If the optional argument +result_hash+ is given,
 *  it is overwritten and returned.
 *  This is intended to avoid the probe effect.
 *
 *    h = {}
 *    ObjectSpace.count_objects(h)
 *    puts h
 *    # => { TOTAL: 10000, T_CLASS: 158280, T_MODULE: 20672, T_STRING: 527249 }
 *
 *  This method is only expected to work on C Ruby.
 *
 */
 
static VALUE
count_objects(int argc, VALUE *argv, VALUE os)
{
    struct count_objects_data data = { 0 };
    VALUE hash = Qnil;
    VALUE types[T_MASK + 1];
 
    if (rb_check_arity(argc, 0, 1) == 1) {
        hash = argv[0];
        if (!RB_TYPE_P(hash, T_HASH))
            rb_raise(rb_eTypeError, "non-hash given");
    }
 
    for (size_t i = 0; i <= T_MASK; i++) {
        // type_sym can allocate an object,
        // so we need to create all key symbols in advance
        // not to disturb the result
        types[i] = type_sym(i);
    }
 
    // Same as type_sym, we need to create all key symbols in advance
    VALUE total = ID2SYM(rb_intern("TOTAL"));
    VALUE free = ID2SYM(rb_intern("FREE"));
 
    rb_gc_impl_each_object(rb_gc_get_objspace(), count_objects_i, &data);
 
    if (NIL_P(hash)) {
        hash = rb_hash_new_capa(2 + T_MASK);
    }
    else if (!RHASH_EMPTY_P(hash)) {
        rb_hash_stlike_foreach(hash, set_zero, hash);
    }
    rb_hash_aset(hash, total, SIZET2NUM(data.total));
    rb_hash_aset(hash, free, SIZET2NUM(data.freed));
 
    for (size_t i = 0; i <= T_MASK; i++) {
        if (data.counts[i]) {
            rb_hash_aset(hash, types[i], SIZET2NUM(data.counts[i]));
        }
    }
 
    return hash;
}
 
#define SET_STACK_END SET_MACHINE_STACK_END(&ec->machine.stack_end)
 
#define STACK_START (ec->machine.stack_start)
#define STACK_END (ec->machine.stack_end)
#define STACK_LEVEL_MAX (ec->machine.stack_maxsize/sizeof(VALUE))
 
#if STACK_GROW_DIRECTION < 0
# define STACK_LENGTH  (size_t)(STACK_START - STACK_END)
#elif STACK_GROW_DIRECTION > 0
# define STACK_LENGTH  (size_t)(STACK_END - STACK_START + 1)
#else
# define STACK_LENGTH  ((STACK_END < STACK_START) ? (size_t)(STACK_START - STACK_END) \
                        : (size_t)(STACK_END - STACK_START + 1))
#endif
#if !STACK_GROW_DIRECTION
int ruby_stack_grow_direction;
int
ruby_get_stack_grow_direction(volatile VALUE *addr)
{
    VALUE *end;
    SET_MACHINE_STACK_END(&end);
 
    if (end > addr) return ruby_stack_grow_direction = 1;
    return ruby_stack_grow_direction = -1;
}
#endif
 
size_t
ruby_stack_length(VALUE **p)
{
    rb_execution_context_t *ec = GET_EC();
    SET_STACK_END;
    if (p) *p = STACK_UPPER(STACK_END, STACK_START, STACK_END);
    return STACK_LENGTH;
}
 
#define PREVENT_STACK_OVERFLOW 1
#ifndef PREVENT_STACK_OVERFLOW
#if !(defined(POSIX_SIGNAL) && defined(SIGSEGV) && defined(HAVE_SIGALTSTACK))
# define PREVENT_STACK_OVERFLOW 1
#else
# define PREVENT_STACK_OVERFLOW 0
#endif
#endif
#if PREVENT_STACK_OVERFLOW && !defined(__EMSCRIPTEN__)
static int
stack_check(rb_execution_context_t *ec, int water_mark)
{
    SET_STACK_END;
 
    size_t length = STACK_LENGTH;
    if (STACK_LEVEL_MAX == 0) return FALSE; /* unknown maxsize */
    if (STACK_LEVEL_MAX <= (size_t)water_mark) return TRUE;
    size_t maximum_length = STACK_LEVEL_MAX - water_mark;
 
    return length > maximum_length;
}
#else
#define stack_check(ec, water_mark) FALSE
#endif
 
#ifdef RUBY_ASAN_ENABLED
/* Unoptimized, instrumented VM frames can exceed the usual 16KB reserve. */
# define STACKFRAME_FOR_CALL_CFUNC (128 * 1024 / sizeof(VALUE))
#else
# define STACKFRAME_FOR_CALL_CFUNC 2048
#endif
 
int
rb_ec_stack_check(rb_execution_context_t *ec)
{
    return stack_check(ec, STACKFRAME_FOR_CALL_CFUNC);
}
 
int
ruby_stack_check(void)
{
    return stack_check(GET_EC(), STACKFRAME_FOR_CALL_CFUNC);
}
 
/* ==================== Marking ==================== */
 
/* The traversal mark redirect is per-Ractor so a real GC never observes a
 * foreign traversal's redirect (a VM-global slot would divert another Ractor's
 * concurrent GC mark into obj_traverse recursion). Only threads with no
 * current Ractor (modular GC's marking worker threads) fall back to the VM
 * slot, which no setter writes, so they always take the real mark path. */
static inline struct gc_mark_func_data_struct **
gc_mark_func_data_slotp_of(rb_ractor_t *const cr)
{
#if USE_MODULAR_GC
    return cr != NULL ? &cr->mark_func_data : &GET_VM()->gc.mark_func_data;
#else
    RUBY_ASSERT(cr != NULL);
    return &cr->mark_func_data;
#endif
}
#define GC_MARK_FUNC_DATA_SLOTP()  gc_mark_func_data_slotp_of(rb_current_ractor_raw(false))
 
/* Marking pays this block per marked reference, so the current Ractor is
 * resolved once and both the redirect slot and the objspace derive from it. */
#define RB_GC_MARK_OR_TRAVERSE(func, obj_or_ptr, obj, check_obj) do { \
    if (!RB_SPECIAL_CONST_P(obj)) { \
        rb_ractor_t *const mark_cr = rb_current_ractor_raw(false); \
        struct gc_mark_func_data_struct **mfdp = gc_mark_func_data_slotp_of(mark_cr); \
        struct gc_mark_func_data_struct *mark_func_data = *mfdp; \
        void *objspace = gc_current_objspace_of(mark_cr); \
        if (LIKELY(mark_func_data == NULL)) { \
            GC_ASSERT(rb_gc_impl_during_gc_p(objspace)); \
            (func)(objspace, (obj_or_ptr)); \
        } \
        else if (check_obj ? \
                rb_gc_impl_live_object_p(objspace, (const void *)obj) && \
                    !rb_gc_impl_garbage_object_p(objspace, obj) : \
                true) { \
            GC_ASSERT(!rb_gc_impl_during_gc_p(objspace)); \
            *mfdp = NULL; \
            mark_func_data->mark_func((obj), mark_func_data->data); \
            *mfdp = mark_func_data; \
        } \
    } \
} while (0)
 
static inline void
gc_mark_internal(VALUE obj)
{
    RB_GC_MARK_OR_TRAVERSE(rb_gc_impl_mark, obj, obj, false);
}
 
void
rb_gc_mark_movable(VALUE obj)
{
    gc_mark_internal(obj);
}
 
void
rb_gc_mark_and_move(VALUE *ptr)
{
    RB_GC_MARK_OR_TRAVERSE(rb_gc_impl_mark_and_move, ptr, *ptr, false);
}
 
static inline void
gc_mark_and_pin_internal(VALUE obj)
{
    RB_GC_MARK_OR_TRAVERSE(rb_gc_impl_mark_and_pin, obj, obj, false);
}
 
void
rb_gc_mark(VALUE obj)
{
    gc_mark_and_pin_internal(obj);
}
 
static inline void
gc_mark_maybe_internal(VALUE obj)
{
    RB_GC_MARK_OR_TRAVERSE(rb_gc_impl_mark_maybe, obj, obj, true);
}
 
void
rb_gc_mark_maybe(VALUE obj)
{
    gc_mark_maybe_internal(obj);
}
 
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void each_location(register const VALUE *x, register long n, void (*cb)(VALUE, void *), void *data));
static void
each_location(register const VALUE *x, register long n, void (*cb)(VALUE, void *), void *data)
{
    VALUE v;
    while (n--) {
        v = *x;
        cb(v, data);
        x++;
    }
}
 
static void
each_location_ptr(const VALUE *start, const VALUE *end, void (*cb)(VALUE, void *), void *data)
{
    if (end <= start) return;
    each_location(start, end - start, cb, data);
}
 
static void
gc_mark_maybe_each_location(VALUE obj, void *data)
{
    gc_mark_maybe_internal(obj);
}
 
void
rb_gc_mark_locations(const VALUE *start, const VALUE *end)
{
    each_location_ptr(start, end, gc_mark_maybe_each_location, NULL);
}
 
void
rb_gc_mark_values(long n, const VALUE *values)
{
    for (long i = 0; i < n; i++) {
        gc_mark_internal(values[i]);
    }
}
 
void
rb_gc_mark_vm_stack_values(long n, const VALUE *values)
{
    for (long i = 0; i < n; i++) {
        gc_mark_and_pin_internal(values[i]);
    }
}
 
static int
mark_key(st_data_t key, st_data_t value, st_data_t data)
{
    gc_mark_and_pin_internal((VALUE)key);
 
    return ST_CONTINUE;
}
 
void
rb_mark_set(st_table *tbl)
{
    if (!tbl) return;
 
    st_foreach(tbl, mark_key, (st_data_t)rb_gc_get_objspace());
}
 
static int
mark_keyvalue(st_data_t key, st_data_t value, st_data_t data)
{
    gc_mark_internal((VALUE)key);
    gc_mark_internal((VALUE)value);
 
    return ST_CONTINUE;
}
 
static int
pin_key_pin_value(st_data_t key, st_data_t value, st_data_t data)
{
    gc_mark_and_pin_internal((VALUE)key);
    gc_mark_and_pin_internal((VALUE)value);
 
    return ST_CONTINUE;
}
 
static int
pin_key_mark_value(st_data_t key, st_data_t value, st_data_t data)
{
    gc_mark_and_pin_internal((VALUE)key);
    gc_mark_internal((VALUE)value);
 
    return ST_CONTINUE;
}
 
static void
mark_hash(VALUE hash)
{
    if (rb_hash_compare_by_id_p(hash)) {
        rb_hash_stlike_foreach(hash, pin_key_mark_value, 0);
    }
    else {
        rb_hash_stlike_foreach(hash, mark_keyvalue, 0);
    }
 
    gc_mark_internal(RHASH(hash)->ifnone);
}
 
void
rb_mark_hash(st_table *tbl)
{
    if (!tbl) return;
 
    st_foreach(tbl, pin_key_pin_value, 0);
}
 
static enum rb_id_table_iterator_result
mark_method_entry_i(VALUE me, void *objspace)
{
    gc_mark_internal(me);
 
    return ID_TABLE_CONTINUE;
}
 
static void
mark_m_tbl(void *objspace, struct rb_id_table *tbl)
{
    if (tbl) {
        rb_id_table_foreach_values(tbl, mark_method_entry_i, objspace);
    }
}
 
static enum rb_id_table_iterator_result
mark_const_entry_i(VALUE value, void *objspace)
{
    const rb_const_entry_t *ce = (const rb_const_entry_t *)value;
 
    gc_mark_internal(ce->value);
    gc_mark_internal(ce->file); // TODO: ce->file should be shareable?
 
    return ID_TABLE_CONTINUE;
}
 
static void
mark_const_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl)
{
    if (!tbl) return;
    rb_id_table_foreach_values(tbl, mark_const_entry_i, objspace);
}
 
#if STACK_GROW_DIRECTION < 0
#define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_END, (end) = STACK_START)
#elif STACK_GROW_DIRECTION > 0
#define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_START, (end) = STACK_END+(appendix))
#else
#define GET_STACK_BOUNDS(start, end, appendix) \
    ((STACK_END < STACK_START) ? \
     ((start) = STACK_END, (end) = STACK_START) : ((start) = STACK_START, (end) = STACK_END+(appendix)))
#endif
 
static void
gc_mark_machine_stack_location_maybe(VALUE obj, void *data)
{
    gc_mark_maybe_internal(obj);
 
#ifdef RUBY_ASAN_ENABLED
    const rb_execution_context_t *ec = (const rb_execution_context_t *)data;
    void *fake_frame_start;
    void *fake_frame_end;
    bool is_fake_frame = asan_get_fake_stack_extents(
        ec->machine.asan_fake_stack_handle, obj,
        ec->machine.stack_start, ec->machine.stack_end,
        &fake_frame_start, &fake_frame_end
    );
    if (is_fake_frame) {
        each_location_ptr(fake_frame_start, fake_frame_end, gc_mark_maybe_each_location, NULL);
    }
#endif
}
 
static bool
gc_object_moved_p_internal(void *objspace, VALUE obj)
{
    if (SPECIAL_CONST_P(obj)) {
        return false;
    }
 
    return rb_gc_impl_object_moved_p(objspace, obj);
}
 
static VALUE
gc_location_internal(void *objspace, VALUE value)
{
    if (SPECIAL_CONST_P(value)) {
        return value;
    }
 
    return rb_gc_impl_location(objspace, value);
}
 
VALUE
rb_gc_location(VALUE value)
{
    return gc_location_internal(rb_gc_get_objspace(), value);
}
 
void
rb_gc_update_moved(VALUE *ptr)
{
    VALUE destination = rb_gc_location(*ptr);
    if (destination != *ptr) {
        *ptr = destination;
    }
}
 
#if defined(__wasm__)
 
 
static VALUE *rb_stack_range_tmp[2];
 
static void
rb_mark_locations(void *begin, void *end)
{
    rb_stack_range_tmp[0] = begin;
    rb_stack_range_tmp[1] = end;
}
 
void
rb_gc_save_machine_context(void)
{
    // no-op
}
 
# if defined(__EMSCRIPTEN__)
 
static void
mark_current_machine_context(const rb_execution_context_t *ec)
{
    emscripten_scan_stack(rb_mark_locations);
    each_location_ptr(rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe_each_location, NULL);
 
    emscripten_scan_registers(rb_mark_locations);
    each_location_ptr(rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe_each_location, NULL);
}
# else // use Asyncify version
 
static void
mark_current_machine_context(rb_execution_context_t *ec)
{
    VALUE *stack_start, *stack_end;
    SET_STACK_END;
    GET_STACK_BOUNDS(stack_start, stack_end, 1);
    each_location_ptr(stack_start, stack_end, gc_mark_maybe_each_location, NULL);
 
    rb_wasm_scan_locals(rb_mark_locations);
    each_location_ptr(rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe_each_location, NULL);
}
 
# endif
 
#else // !defined(__wasm__)
 
void
rb_gc_save_machine_context(void)
{
    rb_thread_t *thread = GET_THREAD();
 
    RB_VM_SAVE_MACHINE_CONTEXT(thread);
}
 
 
static void
mark_current_machine_context(const rb_execution_context_t *ec)
{
    rb_gc_mark_machine_context(ec);
}
#endif
 
void
rb_gc_mark_machine_context(const rb_execution_context_t *ec)
{
    VALUE *stack_start, *stack_end;
 
    GET_STACK_BOUNDS(stack_start, stack_end, 0);
    RUBY_DEBUG_LOG("ec->th:%u stack_start:%p stack_end:%p", rb_ec_thread_ptr(ec)->serial, stack_start, stack_end);
 
    void *data =
#ifdef RUBY_ASAN_ENABLED
        /* gc_mark_machine_stack_location_maybe() uses data as const */
        (rb_execution_context_t *)ec;
#else
        NULL;
#endif
 
    each_location_ptr(stack_start, stack_end, gc_mark_machine_stack_location_maybe, data);
    int num_regs = sizeof(ec->machine.regs)/(sizeof(VALUE));
    each_location((VALUE*)&ec->machine.regs, num_regs, gc_mark_machine_stack_location_maybe, data);
}
 
static int
rb_mark_tbl_i(st_data_t key, st_data_t value, st_data_t data)
{
    gc_mark_and_pin_internal((VALUE)value);
 
    return ST_CONTINUE;
}
 
void
rb_mark_tbl(st_table *tbl)
{
    if (!tbl || tbl->num_entries == 0) return;
 
    st_foreach(tbl, rb_mark_tbl_i, 0);
}
 
static void
gc_mark_tbl_no_pin(st_table *tbl)
{
    if (!tbl || tbl->num_entries == 0) return;
 
    st_foreach(tbl, gc_mark_tbl_no_pin_i, 0);
}
 
void
rb_mark_tbl_no_pin(st_table *tbl)
{
    gc_mark_tbl_no_pin(tbl);
}
 
void
rb_gc_mark_set_no_pin(st_table *tbl)
{
    if (!tbl || tbl->num_entries == 0) return;
 
    st_foreach(tbl, gc_mark_set_no_pin_i, 0);
}
 
static bool
gc_declarative_marking_p(const rb_data_type_t *type)
{
    return (type->flags & RUBY_TYPED_DECL_MARKING) != 0;
}
 
rb_execution_context_t *
rb_gc_get_ec(void)
{
    void *objspace = rb_gc_get_objspace();
 
    if (RB_LIKELY(rb_gc_impl_during_gc_p(objspace))) {
        return rb_gc_impl_get_vm_context(objspace)->ec;
    }
    else {
        return GET_EC();
    }
}
 
void
rb_gc_mark_roots(void *objspace, const char **categoryp)
{
    rb_execution_context_t *ec = rb_gc_get_ec();
    rb_vm_t *vm = rb_ec_vm_ptr(ec);
 
#define MARK_CHECKPOINT(category) do { \
    if (categoryp) *categoryp = category; \
} while (0)
 
    /* A single-objspace impl (mmtk) only has stop-the-world global GCs and no
     * per-mutator root scan, so always walk every Ractor's local roots here. */
    const bool global_gc = rb_gc_impl_during_global_gc_p(objspace) ||
                           !rb_gc_impl_multi_objspace_p();
 
    /* Mark the current Ractor's roots from its C structs (a local GC must not depend on
     * heap wrapper traversal).  A global GC does the same for every Ractor. */
    MARK_CHECKPOINT("ractor");
    if (global_gc) {
        rb_ractor_t *r;
        ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
            rb_ractor_mark_local_roots(r);
        }
 
        /* Early in boot (before rb_ractor_main_setup) main is not in vm->ractor.set
         * yet; do not drop its registered_marks in a single-objspace boot GC. */
        if (vm->ractor.cnt == 0 && vm->ractor.main_ractor) {
            rb_ractor_mark_local_roots(vm->ractor.main_ractor);
        }
        /* A Ractor that terminated (left vm->ractor.set) but whose struct is not freed
         * still owns rb_gc_register_mark_object pins.  Keep them alive until
         * ractor_free hands them to main; an orphan (owner == NULL) was moved above.
         * The join value is not rooted here: ractor_mark marks it from the wrapper. */
        for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) {
            rb_ractor_t *owner = vm->gc.zombie_objspaces[i].owner;
            if (owner) {
                rb_gc_mark_vm_stack_values((long)owner->registered_marks_cnt,
                                           owner->registered_marks);
            }
        }
 
        /* Single-objspace impl: keep terminated-but-not-freed Ractors'
         * rb_gc_register_mark_object entries alive without depending on wrapper
         * reachability.  With multiple objspaces zombie_objspaces covers this. */
        if (!rb_gc_impl_multi_objspace_p()) {
            rb_ractor_t *tr;
            rb_native_mutex_lock(&vm->gc.registered_globals.lock);
            ccan_list_for_each(&vm->ractor.terminated_set, tr, vmlr_node) {
                rb_gc_mark_vm_stack_values((long)tr->registered_marks_cnt,
                                           tr->registered_marks);
            }
            rb_native_mutex_unlock(&vm->gc.registered_globals.lock);
        }
    }
    else {
        rb_ractor_mark_local_roots(rb_ec_ractor_ptr(ec));
    }
 
    /* rb_gc_register_address slots live in one VM-wide list: *addr can later hold
     * another objspace's value, so every Ractor's GC scans all slots conservatively,
     * marking only its own residents. */
    MARK_CHECKPOINT("registered_globals");
    rb_native_mutex_lock(&vm->gc.registered_globals.lock);
    for (size_t i = 0; i < vm->gc.registered_globals.addrs_cnt; i++) {
        rb_gc_mark_maybe(*vm->gc.registered_globals.addrs[i]);
    }
    rb_native_mutex_unlock(&vm->gc.registered_globals.lock);
 
    /* Trap handlers live in the VM-global vm->trap_list.cmd[], a fixed array of aligned
     * VALUEs (signal.c uses ACCESS_ONCE): a racing walk reads either the old or the new
     * handler, both alive, so no lock. */
    MARK_CHECKPOINT("trap_list");
    rb_gc_mark_values(RUBY_NSIG, vm->trap_list.cmd);
 
    /* VM-global roots belong to the main Ractor's objspace, since the boot objects
     * live there.  A non-main Ractor's local GC skips them; a global GC walks all. */
    if (global_gc || objspace == vm->ractor.main_ractor->objspace) {
        /* Only the main Ractor can register at_exit/END procs (a non-main one gets an
         * IsolationError) so end_procs is a lock-free linked list */
        MARK_CHECKPOINT("end_proc");
        rb_mark_end_proc();
 
        MARK_CHECKPOINT("vm");
        /* rb_vm_mark and the JIT root marks walk VM-global weak tables and shared singleton
         * JIT state that other Ractors rewrite under the VM lock, so main's otherwise
         * lock-free local GC takes the VM lock for this stretch */
        const bool vm_mark_needs_lock = rb_multi_ractor_p() && !global_gc;
        unsigned int vm_mark_lock_lev = 0;
        if (vm_mark_needs_lock) vm_mark_lock_lev = RB_GC_VM_LOCK_NO_BARRIER();
        rb_vm_mark(vm);
 
        MARK_CHECKPOINT("global_tbl");
        rb_gc_mark_global_tbl();
 
#if USE_YJIT
        void rb_yjit_root_mark(void); // in Rust
 
        if (rb_yjit_enabled_p) {
            MARK_CHECKPOINT("YJIT");
            rb_yjit_root_mark();
        }
#endif
 
#if USE_ZJIT
        void rb_zjit_root_mark(void);
        if (rb_zjit_enabled_p) {
            MARK_CHECKPOINT("ZJIT");
            rb_zjit_root_mark();
        }
#endif
        if (vm_mark_needs_lock) RB_GC_VM_UNLOCK_NO_BARRIER(vm_mark_lock_lev);
 
        if (global_gc || rb_gc_single_objspace_p()) {
            MARK_CHECKPOINT("global_symbols");
            rb_sym_global_symbols_mark_and_move();
        }
    }
 
    /* The dying thread's final collection of its own objspace runs after its stack
     * has been torn down (thread_cleanup_func), so there is no live machine context
     * to scan -- the join value and the pins are rooted explicitly.  Scanning the
     * half-dead stack is not only useless but faults on some platforms. */
    if (!rb_gc_impl_during_postmortem_p(objspace)) {
        MARK_CHECKPOINT("machine_context");
        mark_current_machine_context(ec);
    }
 
    MARK_CHECKPOINT("finish");
 
#undef MARK_CHECKPOINT
}
 
struct gc_mark_classext_foreach_arg {
    rb_objspace_t *objspace;
    VALUE obj;
};
 
static void
gc_mark_classext_module(rb_classext_t *ext, bool prime, VALUE box_value, void *arg)
{
    struct gc_mark_classext_foreach_arg *foreach_arg = (struct gc_mark_classext_foreach_arg *)arg;
    rb_objspace_t *objspace = foreach_arg->objspace;
 
    if (RCLASSEXT_SUPER(ext)) {
        gc_mark_internal(RCLASSEXT_SUPER(ext));
    }
    mark_m_tbl(objspace, RCLASSEXT_M_TBL(ext));
 
    gc_mark_internal(RCLASSEXT_FIELDS_OBJ(ext));
    gc_mark_internal(RCLASSEXT_CVC_TBL(ext));
 
    if (!RCLASSEXT_SHARED_CONST_TBL(ext) && RCLASSEXT_CONST_TBL(ext)) {
        mark_const_tbl(objspace, RCLASSEXT_CONST_TBL(ext));
    }
    mark_m_tbl(objspace, RCLASSEXT_CALLABLE_M_TBL(ext));
    gc_mark_internal(RCLASSEXT_CC_TBL(ext));
    if (RCLASSEXT_SUBCLASSES(ext)) {
        gc_mark_internal(RCLASSEXT_SUBCLASSES(ext));
    }
    gc_mark_internal(RCLASSEXT_CLASSPATH(ext));
}
 
static void
gc_mark_classext_iclass(rb_classext_t *ext, bool prime, VALUE box_value, void *arg)
{
    struct gc_mark_classext_foreach_arg *foreach_arg = (struct gc_mark_classext_foreach_arg *)arg;
    rb_objspace_t *objspace = foreach_arg->objspace;
 
    if (RCLASSEXT_SUPER(ext)) {
        gc_mark_internal(RCLASSEXT_SUPER(ext));
    }
    if (RCLASSEXT_ICLASS_IS_ORIGIN(ext) && !RCLASSEXT_ICLASS_ORIGIN_SHARED_MTBL(ext)) {
        mark_m_tbl(objspace, RCLASSEXT_M_TBL(ext));
    }
    if (RCLASSEXT_INCLUDER(ext)) {
        gc_mark_internal(RCLASSEXT_INCLUDER(ext));
    }
    mark_m_tbl(objspace, RCLASSEXT_CALLABLE_M_TBL(ext));
    gc_mark_internal(RCLASSEXT_CC_TBL(ext));
    if (RCLASSEXT_SUBCLASSES(ext)) {
        gc_mark_internal(RCLASSEXT_SUBCLASSES(ext));
    }
}
 
#define TYPED_DATA_REFS_OFFSET_LIST(d) (size_t *)(uintptr_t)RTYPEDDATA_TYPE(d)->function.dmark
 
static inline bool
rb_obj_using_gen_fields_table_p(VALUE obj)
{
    switch (BUILTIN_TYPE(obj)) {
      case T_STRUCT:
      case T_DATA:
        return false;
 
      default:
        break;
    }
 
    return rb_obj_gen_fields_p(obj);
}
 
void
rb_gc_move_obj_during_marking(VALUE from, VALUE to)
{
    if (rb_obj_using_gen_fields_table_p(to)) {
        rb_mark_generic_ivar(from);
    }
}
 
void
rb_gc_mark_children(void *objspace, VALUE obj)
{
    struct gc_mark_classext_foreach_arg foreach_args;
 
    if (rb_obj_using_gen_fields_table_p(obj)) {
        rb_mark_generic_ivar(obj);
    }
 
    switch (BUILTIN_TYPE(obj)) {
      case T_FLOAT:
      case T_BIGNUM:
        return;
 
      case T_NIL:
      case T_FIXNUM:
        rb_bug("rb_gc_mark() called for broken object");
        break;
 
      case T_NODE:
        UNEXPECTED_NODE(rb_gc_mark);
        break;
 
      case T_IMEMO:
        rb_imemo_mark_and_move(obj, false);
        return;
 
      default:
        break;
    }
 
    gc_mark_internal(RBASIC(obj)->klass);
 
    switch (BUILTIN_TYPE(obj)) {
      case T_CLASS:
        if (FL_TEST_RAW(obj, FL_SINGLETON)) {
            gc_mark_internal(RCLASS_ATTACHED_OBJECT(obj));
        }
        // Continue to the shared T_CLASS/T_MODULE
      case T_MODULE:
        foreach_args.objspace = objspace;
        foreach_args.obj = obj;
        rb_class_classext_foreach(obj, gc_mark_classext_module, (void *)&foreach_args);
        if (BOX_USER_P(RCLASS_PRIME_BOX(obj))) {
            gc_mark_internal(RCLASS_PRIME_BOX(obj)->box_object);
        }
        break;
 
      case T_ICLASS:
        foreach_args.objspace = objspace;
        foreach_args.obj = obj;
        rb_class_classext_foreach(obj, gc_mark_classext_iclass, (void *)&foreach_args);
        if (BOX_USER_P(RCLASS_PRIME_BOX(obj))) {
            gc_mark_internal(RCLASS_PRIME_BOX(obj)->box_object);
        }
        break;
 
      case T_ARRAY:
        if (ARY_SHARED_P(obj)) {
            VALUE root = ARY_SHARED_ROOT(obj);
            if (RB_TYPE_P(root, T_ARRAY)) {
                gc_mark_internal(root);
            }
            else {
                /* Ractor#send(move: true) hollowed the root out in place.  If it was
                 * embedded our elements are still in its slot, and nothing says so any
                 * more, so it must not move (gc_ref_update_array cannot re-point us). */
                gc_mark_and_pin_internal(root);
            }
        }
        else {
            long len = RARRAY_LEN(obj);
            const VALUE *ptr = RARRAY_CONST_PTR(obj);
            for (long i = 0; i < len; i++) {
                gc_mark_internal(ptr[i]);
            }
        }
        break;
 
      case T_HASH:
        mark_hash(obj);
        break;
 
      case T_SYMBOL:
        gc_mark_internal(RSYMBOL(obj)->fstr);
        break;
 
      case T_STRING:
        if (STR_SHARED_P(obj)) {
            if (STR_EMBED_P(RSTRING(obj)->as.heap.aux.shared)) {
                /* Embedded shared strings cannot be moved because this string
                 * points into the slot of the shared string. There may be code
                 * using the RSTRING_PTR on the stack, which would pin this
                 * string but not pin the shared string, causing it to move. */
                gc_mark_and_pin_internal(RSTRING(obj)->as.heap.aux.shared);
            }
            else {
                gc_mark_internal(RSTRING(obj)->as.heap.aux.shared);
            }
        }
        break;
 
      case T_DATA: {
        void *const ptr = RTYPEDDATA_GET_DATA(obj);
 
        gc_mark_internal(RTYPEDDATA(obj)->fields_obj);
 
        if (ptr) {
            if (gc_declarative_marking_p(RTYPEDDATA_TYPE(obj))) {
                size_t *offset_list = TYPED_DATA_REFS_OFFSET_LIST(obj);
 
                for (size_t offset = *offset_list; offset != RUBY_REF_END; offset = *offset_list++) {
                    gc_mark_internal(*(VALUE *)((char *)ptr + offset));
                }
            }
            else {
                RUBY_DATA_FUNC mark_func = RTYPEDDATA_TYPE(obj)->function.dmark;
                if (mark_func) (*mark_func)(ptr);
            }
        }
 
        break;
      }
 
      case T_OBJECT: {
        shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
        if (rb_shape_embedded_p(shape_id)) {
            uint32_t len = RSHAPE_LEN(shape_id);
            const VALUE * const ptr = ROBJECT(obj)->as.ary;
 
            for (uint32_t i = 0; i < len; i++) {
                gc_mark_internal(ptr[i]);
            }
        }
        else {
            if (!rb_gc_checking_shareable()) {
                gc_mark_internal(ROBJECT(obj)->as.extended);
            }
        }
        break;
      }
 
      case T_FILE:
        if (RFILE(obj)->fptr) {
            gc_mark_internal(RFILE(obj)->fptr->self);
            gc_mark_internal(RFILE(obj)->fptr->pathv);
            gc_mark_internal(RFILE(obj)->fptr->tied_io_for_writing);
            gc_mark_internal(RFILE(obj)->fptr->writeconv_asciicompat);
            gc_mark_internal(RFILE(obj)->fptr->writeconv_pre_ecopts);
            gc_mark_internal(RFILE(obj)->fptr->encs.ecopts);
            gc_mark_internal(RFILE(obj)->fptr->write_lock);
            gc_mark_internal(RFILE(obj)->fptr->timeout);
            gc_mark_internal(RFILE(obj)->fptr->wakeup_mutex);
        }
        break;
 
      case T_REGEXP:
        gc_mark_internal(RREGEXP(obj)->src);
        break;
 
      case T_MATCH:
        gc_mark_internal(RMATCH(obj)->regexp);
        if (RMATCH(obj)->str) {
            gc_mark_internal(RMATCH(obj)->str);
        }
        break;
 
      case T_RATIONAL:
        gc_mark_internal(RRATIONAL(obj)->num);
        gc_mark_internal(RRATIONAL(obj)->den);
        break;
 
      case T_COMPLEX:
        gc_mark_internal(RCOMPLEX(obj)->real);
        gc_mark_internal(RCOMPLEX(obj)->imag);
        break;
 
      case T_STRUCT: {
        const long len = RSTRUCT_LEN(obj);
        const VALUE * const ptr = RSTRUCT_CONST_PTR(obj);
 
        for (long i = 0; i < len; i++) {
            gc_mark_internal(ptr[i]);
        }
 
        gc_mark_internal(RSTRUCT_FIELDS_OBJ(obj));
 
        break;
      }
 
      default:
        if (BUILTIN_TYPE(obj) == T_MOVED)   rb_bug("rb_gc_mark(): %p is T_MOVED", (void *)obj);
        if (BUILTIN_TYPE(obj) == T_NONE)   rb_bug("rb_gc_mark(): %p is T_NONE", (void *)obj);
        if (BUILTIN_TYPE(obj) == T_ZOMBIE) rb_bug("rb_gc_mark(): %p is T_ZOMBIE", (void *)obj);
        rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s",
               BUILTIN_TYPE(obj), (void *)obj,
               rb_gc_impl_live_object_p(objspace, (void *)obj) ? "corrupted object" : "non object");
    }
}
 
size_t
rb_gc_obj_optimal_size(VALUE obj)
{
    switch (BUILTIN_TYPE(obj)) {
      case T_ARRAY:
        {
            size_t size = rb_ary_size_as_embedded(obj);
            if (rb_gc_size_allocatable_p(size)) {
                return size;
            }
            else {
                return sizeof(struct RArray);
            }
        }
 
      case T_OBJECT:
        if (rb_obj_shape_complex_p(obj)) {
            return sizeof(struct RObject);
        }
        else {
            size_t size = rb_obj_embedded_size(RSHAPE_CAPACITY(RBASIC_SHAPE_ID(obj)));
            if (rb_gc_size_allocatable_p(size)) {
                return size;
            }
            else {
                return sizeof(struct RObject);
            }
        }
 
      case T_STRING:
        {
            size_t size = rb_str_size_as_embedded(obj);
            if (rb_gc_size_allocatable_p(size)) {
                return size;
            }
            else {
                return sizeof(struct RString);
            }
        }
 
      case T_HASH:
        {
            if (RHASH_AR_TABLE_P(obj)) {
                const unsigned bound = RHASH_AR_TABLE_BOUND(obj);
                const size_t ar_size = RHASH_AR_SLOT_SIZE(bound);
                if (ar_size > RHASH_ST_SLOT_SIZE || OBJ_FROZEN(obj)) {
                    return ar_size;
                }
            }
 
            return RHASH_ST_SLOT_SIZE;
        }
 
      default:
        return 0;
    }
}
 
void
rb_gc_writebarrier(VALUE a, VALUE b)
{
    rb_gc_impl_writebarrier(rb_gc_get_objspace(), a, b);
}
 
void
rb_gc_writebarrier_unprotect(VALUE obj)
{
    rb_gc_impl_writebarrier_unprotect(rb_gc_get_objspace(), obj);
}
 
/*
 * remember `obj' if needed.
 */
void
rb_gc_writebarrier_remember(VALUE obj)
{
    rb_gc_impl_writebarrier_remember(rb_gc_get_objspace(), obj);
}
 
/* obj became shareable after it was created (FL_SHAREABLE was set).  Tell the GC so it
 * updates the per-page shareable bitmap. */
void
rb_gc_obj_became_shareable(VALUE obj)
{
    rb_gc_impl_obj_became_shareable(rb_gc_get_objspace(), obj);
}
 
/* Pin an in-flight message payload in its owner's (the sender's) objspace, so the
 * sender's local GC keeps it alive while it sits in a queue the sender does not walk. */
void
rb_gc_copy_attributes(VALUE dest, VALUE obj)
{
    rb_gc_impl_copy_attributes(rb_gc_get_objspace(), dest, obj);
}
 
#if USE_MODULAR_GC
int
rb_gc_modular_gc_loaded_p(void)
{
    return rb_gc_functions.modular_gc_loaded_p;
}
 
const char *
rb_gc_active_gc_name(void)
{
    const char *gc_name = rb_gc_impl_active_gc_name();
 
    const size_t len = strlen(gc_name);
    if (len > RB_GC_MAX_NAME_LEN) {
        rb_bug("GC should have a name no more than %d chars long. Currently: %zu (%s)",
               RB_GC_MAX_NAME_LEN, len, gc_name);
    }
 
    return gc_name;
}
#endif
 
struct rb_gc_object_metadata_entry *
rb_gc_object_metadata(VALUE obj)
{
    return rb_gc_impl_object_metadata(rb_gc_get_objspace(), obj);
}
 
/* GC */
 
void *
rb_gc_ractor_cache_alloc(rb_ractor_t *ractor)
{
    return rb_gc_impl_ractor_cache_alloc(rb_gc_get_objspace(), ractor);
}
 
void
rb_gc_ractor_cache_free(void *cache)
{
    rb_gc_impl_ractor_cache_free(rb_gc_get_objspace(), cache);
}
 
bool
rb_gc_zjit_new_obj_fastpath(size_t alloc_size, VALUE flags, VALUE klass, struct rb_gc_zjit_fastpath *fastpath)
{
#if defined(RUBY_ASAN_ENABLED)
    (void)rb_gc_impl_zjit_new_obj_fastpath;
    return false;
#else
    return rb_gc_impl_zjit_new_obj_fastpath(rb_gc_get_objspace(), alloc_size, flags, klass, fastpath);
#endif
}
 
void
rb_gc_register_mark_object(VALUE obj)
{
    /* rb_gc_impl_live_object_p() walks objspace->heap_pages.sorted, which
     * another ractor may mutate while allocating heap pages under the VM lock,
     * so the lookup must be done under the VM lock as well. */
    RB_VM_LOCKING() {
        if (rb_gc_impl_live_object_p(rb_gc_get_objspace(), (void *)obj)) {
            rb_vm_register_global_object(obj);
        }
    }
}
 
void
rb_gc_register_address(VALUE *addr)
{
    rb_vm_t *vm = GET_VM();
 
    rb_native_mutex_lock(&vm->gc.registered_globals.lock);
    if (vm->gc.registered_globals.addrs_cnt == vm->gc.registered_globals.addrs_capa) {
        size_t nc = vm->gc.registered_globals.addrs_capa ? vm->gc.registered_globals.addrs_capa * 2 : 64;
        VALUE **p = realloc(vm->gc.registered_globals.addrs, nc * sizeof(VALUE *));
        if (!p) rb_bug("rb_gc_register_address: out of memory");
        vm->gc.registered_globals.addrs = p;
        vm->gc.registered_globals.addrs_capa = nc;
    }
    vm->gc.registered_globals.addrs[vm->gc.registered_globals.addrs_cnt++] = addr;
    rb_native_mutex_unlock(&vm->gc.registered_globals.lock);
 
    /* Some C extensions register before assigning, so protect obj from GC here. */
    RB_GC_GUARD(*addr);
}
 
void
rb_gc_unregister_address(VALUE *addr)
{
    rb_vm_t *vm = GET_VM();
 
    /* One VM-wide list, so a register and unregister from different Ractors (Init on
     * main, dfree elsewhere) still pair up.  Silently a no-op when not found: upstream
     * tolerates a double unregister too. */
    rb_native_mutex_lock(&vm->gc.registered_globals.lock);
    for (size_t i = 0; i < vm->gc.registered_globals.addrs_cnt; i++) {
        if (vm->gc.registered_globals.addrs[i] == addr) {
            MEMMOVE(&vm->gc.registered_globals.addrs[i], &vm->gc.registered_globals.addrs[i + 1],
                    VALUE *, vm->gc.registered_globals.addrs_cnt - i - 1);
            vm->gc.registered_globals.addrs_cnt--;
            break;
        }
    }
    rb_native_mutex_unlock(&vm->gc.registered_globals.lock);
}
 
void
rb_global_variable(VALUE *var)
{
    rb_gc_register_address(var);
}
 
static VALUE
gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE immediate_mark, VALUE immediate_sweep, VALUE compact, VALUE global)
{
    rb_gc_impl_start(rb_gc_get_objspace(), RTEST(full_mark), RTEST(immediate_mark), RTEST(immediate_sweep), RTEST(compact), RTEST(global));
 
    return Qnil;
}
 
struct each_objects_foreign_arg {
    void *self;
    int (*callback)(void *, void *, size_t, void *);
    void *data;
};
 
static void
each_objects_foreign_i(void *objspace, void *arg)
{
    struct each_objects_foreign_arg *a = (struct each_objects_foreign_arg *)arg;
    if (objspace == a->self) return;
    rb_gc_impl_each_objects_foreign(objspace, a->callback, a->data);
}
 
/*
 * rb_objspace_each_objects() is special C API to walk through
 * Ruby object space.  This C API is too difficult to use it.
 * To be frank, you should not use it. Or you need to read the
 * source code of this function and understand what this function does.
 *
 * 'callback' will be called several times (the number of heap page,
 * at current implementation) with:
 *   vstart: a pointer to the first living object of the heap_page.
 *   vend: a pointer to next to the valid heap_page area.
 *   stride: a distance to next VALUE.
 *
 * If callback() returns non-zero, the iteration will be stopped.
 *
 * This takes the VM barrier for the whole walk, stopping every other
 * Ractor: the set of heap pages must not change under the callback, and a
 * GC is stop-the-world.  Because of that, the callback must not wait on
 * another Ractor (e.g. send/receive) -- they are all suspended and it
 * would deadlock.
 *
 * This is a sample callback code to iterate liveness objects:
 *
 *   static int
 *   sample_callback(void *vstart, void *vend, int stride, void *data)
 *   {
 *       VALUE v = (VALUE)vstart;
 *       for (; v != (VALUE)vend; v += stride) {
 *           if (!rb_objspace_internal_object_p(v)) { // liveness check
 *               // do something with live object 'v'
 *           }
 *       }
 *       return 0; // continue to iteration
 *   }
 *
 * Note: 'vstart' is not a top of heap_page.  This point the first
 *       living object to grasp at least one object to avoid GC issue.
 *       This means that you can not walk through all Ruby object page
 *       including freed object page.
 *
 * Note: On this implementation, 'stride' is the same as sizeof(RVALUE).
 *       However, there are possibilities to pass variable values with
 *       'stride' with some reasons.  You must use stride instead of
 *       use some constant value in the iteration.
 */
void
rb_objspace_each_objects(int (*callback)(void *, void *, size_t, void *), void *data)
{
    RB_VM_LOCKING() {
        rb_vm_barrier();
 
        void *self = rb_gc_get_objspace();
        rb_gc_impl_each_objects(self, callback, data);
 
        /* Like upstream, cover every object in the process: walk the other live
         * Ractors' objspaces too, under the VM lock and barrier, with a pure-C callback.
         * A foreign objspace's stopped lazy sweep is not settled; the walk skips its
         * dead objects. Also covers zombie objspaces. */
        struct each_objects_foreign_arg arg = { self, callback, data };
        rb_gc_vm_each_objspace(each_objects_foreign_i, &arg);
    }
}
 
/* Enumerate live, creating, and zombie objspaces under the VM lifetime lock.
 * Reading foreign mutable collector state also needs the barrier; independently
 * synchronized publications may be read under their own locks without a barrier. */
void
rb_gc_vm_each_objspace(void (*func)(void *objspace, void *data), void *data)
{
    ASSERT_vm_locking();
 
    rb_vm_t *vm = GET_VM();
    rb_ractor_t *r;
    ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
        if (r->objspace) {
            func(r->objspace, data);
        }
        /* A child being created is not in the set yet but its objspace already holds
         * the Thread/Fiber wrappers; enumerate it through its creator so a global GC
         * cannot miss it and mark into an objspace it never cleared. */
        if (r->creating_child_objspace) {
            func(r->creating_child_objspace, data);
        }
    }
    for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) {
        func(vm->gc.zombie_objspaces[i].objspace, data);
    }
}
 
/* Merging an ownerless zombie objspace (its Ractor object was collected) into main
 * runs as a postponed job targeted at main, at main's next safepoint; never inside
 * the GC cycle that discovered the orphan. */
 
static void gc_orphan_merge_job(void *unused);
 
/* Grown with plain realloc: rb_gc_objspace_disown pushes from inside a global GC
 * sweep, where the accounting allocator is not allowed.  This table is VM-lifetime
 * metadata with at most a few dozen entries. */
static void
zombie_objspaces_push(rb_vm_t *vm, void *objspace, void **owner_slot, struct rb_ractor_struct *owner)
{
    ASSERT_vm_locking();
    if (vm->gc.zombie_objspaces_count == vm->gc.zombie_objspaces_capa) {
        size_t new_capa = vm->gc.zombie_objspaces_capa ? vm->gc.zombie_objspaces_capa * 2 : 16;
        struct rb_objspace_zombie *grown =
            realloc(vm->gc.zombie_objspaces, new_capa * sizeof(struct rb_objspace_zombie));
        if (grown == NULL) rb_bug("zombie_objspaces_push: out of memory");
        vm->gc.zombie_objspaces = grown;
        vm->gc.zombie_objspaces_capa = new_capa;
    }
    size_t pages = rb_gc_impl_heap_page_count(objspace);
    vm->gc.zombie_objspaces[vm->gc.zombie_objspaces_count++] = (struct rb_objspace_zombie){
        .objspace = objspace,
        .owner_slot = owner_slot,
        .owner = owner,
        .pages = pages,
    };
    vm->gc.zombie_total_pages += pages;
}
 
/* Called for a Ractor that terminated without being joined.  Its objspace loses its
 * owning thread, but its pages still hold shareable objects other Ractors can reach,
 * so keep it enumerable until inheritance merges it.  The owning r->objspace slot stays
 * until the inheriting path takes the objspace and clears it. */
/* Reserve the handle of the orphan-merge job if it is not registered yet.  Shared by
 * every retire and disown path; a second preregister is idempotent (the same func and
 * data are deduplicated). */
static void
gc_orphan_merge_pjob_ensure(void)
{
    if (GET_VM()->gc.orphan_merge_pjob == POSTPONED_JOB_HANDLE_INVALID) {
        GET_VM()->gc.orphan_merge_pjob = rb_postponed_job_preregister(0, gc_orphan_merge_job, NULL);
        if (GET_VM()->gc.orphan_merge_pjob == POSTPONED_JOB_HANDLE_INVALID) {
            rb_bug("Could not preregister postponed job for GC");
        }
    }
}
 
/* A terminating Ractor runs the last local GC of its own objspace; own thread only. */
void
rb_gc_objspace_retire_gc(void)
{
    rb_gc_impl_objspace_retire_gc(rb_gc_get_objspace());
}
 
void
rb_gc_objspace_retire(void **objspace_slot)
{
    rb_vm_t *vm = GET_VM();
 
    if (!rb_gc_impl_multi_objspace_p()) {
        /* It only aliased the shared objspace, so just drop it. */
        *objspace_slot = NULL;
        return;
    }
 
    /* Return the hold if the Ractor exits with GC disabled: otherwise nobody can
     * enable it again and GC stays off. */
    if (rb_gc_impl_user_gc_disabled_set(*objspace_slot, false)) {
        RUBY_ATOMIC_DEC(vm->gc.disable_holders);
    }
 
    RB_VM_LOCKING() {
        gc_orphan_merge_pjob_ensure();
        /* owner_slot is always &r->objspace of the retiring Ractor.  owner is recorded so a
         * root scan can still reach the dead Ractor's registered_marks pins and its join
         * value; rb_gc_objspace_disown clears it when the zombie becomes an orphan. */
        struct rb_ractor_struct *owner =
            (struct rb_ractor_struct *)((char *)objspace_slot - offsetof(rb_ractor_t, objspace));
        zombie_objspaces_push(vm, *objspace_slot, objspace_slot, owner);
    }
}
 
/* The owning Ractor object was collected, so nobody can join any more: drop the owner
 * slot in zombie_objspaces and hand the merge to main.  Called from ractor_free (inside
 * a sweep), where the accounting allocator is unavailable; the table itself is stable. */
void
rb_gc_objspace_disown(void *objspace)
{
    if (!rb_gc_impl_multi_objspace_p()) return;
    ASSERT_vm_locking();
    rb_vm_t *vm = GET_VM();
    bool found = false;
 
    for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) {
        if (vm->gc.zombie_objspaces[i].objspace == objspace) {
            vm->gc.zombie_objspaces[i].owner_slot = NULL;
            /* The Ractor struct is being freed, so drop owner too: nothing may read its
             * registered_marks or join value after this. */
            vm->gc.zombie_objspaces[i].owner = NULL;
            found = true;
            break;
        }
    }
    if (!found) {
        zombie_objspaces_push(vm, objspace, NULL, NULL);
    }
 
    /* The trigger is wait-free (an atomic bit plus an interrupt flag), so it is safe
     * inside a sweep, and it also covers a Ractor that never started. */
    gc_orphan_merge_pjob_ensure();
    rb_postponed_job_trigger_for_ractor(GET_VM()->gc.orphan_merge_pjob, vm->ractor.main_ractor->pub.self);
}
 
/* Is a global (stop-the-world) GC cycle running?  Only its driver runs during one, so
 * asking through the current objspace is exact. */
bool
rb_gc_during_global_gc_p(void)
{
    return rb_gc_impl_during_global_gc_p(rb_gc_get_objspace());
}
 
static void
rb_gc_vm_forget_zombie(void *objspace)
{
    ASSERT_vm_locking();
    rb_vm_t *vm = GET_VM();
    size_t n = vm->gc.zombie_objspaces_count;
    for (size_t i = 0; i < n; i++) {
        if (vm->gc.zombie_objspaces[i].objspace == objspace) {
            vm->gc.zombie_total_pages -= vm->gc.zombie_objspaces[i].pages;
            vm->gc.zombie_objspaces[i] = vm->gc.zombie_objspaces[n - 1];
            vm->gc.zombie_objspaces_count = n - 1;
            break;
        }
    }
}
 
/* Total zombie pages, deciding whether to start a global GC.  An upper bound between
 * global cycles (each re-measures under the barrier), so a stale value cannot
 * re-trigger; a lock-free read at worst fires one cycle early or late. */
size_t
rb_gc_vm_zombie_total_pages(void)
{
    return GET_VM()->gc.zombie_total_pages;
}
 
/* Number of live Ractors, for the heap growth heuristic (r_mul); a racy read is fine. */
unsigned int
rb_gc_vm_ractor_count(void)
{
    return GET_VM()->ractor.cnt;
}
 
/* Called by a global cycle from inside the barrier. */
void
rb_gc_vm_refresh_zombie_pages(void)
{
    rb_vm_t *vm = GET_VM();
    size_t total = 0;
    for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) {
        size_t pages = rb_gc_impl_heap_page_count(vm->gc.zombie_objspaces[i].objspace);
        vm->gc.zombie_objspaces[i].pages = pages;
        total += pages;
    }
    vm->gc.zombie_total_pages = total;
}
 
void
rb_gc_rest(void)
{
    // Lock to keep assertions happy. This runs right after single-ractor mode is
    // cancelled, but we can still free shareables like fstrings because ractor.cnt is 1.
    RB_VM_LOCKING() {
        rb_gc_impl_gc_rest(rb_gc_get_objspace());
    }
}
 
/* True while a zombie is being absorbed.  The zombie's count is decremented before the
 * merge (see absorb below), so in that window its live objects still exist even though
 * the process looks single-objspace. */
static int gc_absorbing_zombie = 0;
 
/* True once a zombie objspace was absorbed since the last global GC: until the unified
 * mark runs, a single-objspace local mark can miss absorbed shareable objects (a cc in
 * a class's cc_table, say), so stop treating the process as single until then. */
static bool gc_absorbed_since_global_gc = false;
 
void
rb_gc_reset_absorbed_since_global_gc(void)
{
    gc_absorbed_since_global_gc = false;
}
 
/* True when the process holds exactly one objspace (one live Ractor, no zombies) and
 * nothing was absorbed since the last global GC.  Only then is a local GC the whole
 * world and the multi-objspace guards can be skipped.  The child-creation window (the
 * child objspace exists while cnt is still 1) and both absorb windows, during (count
 * already decremented, merge unfinished) and after (merged, next global GC pending) --
 * count as multi: treating them as single would let a GC skip guards such as shareable
 * pinning and collect a live cc. */
/* False when the impl only supports one objspace (mmtk and friends); the VM then makes
 * its per-Ractor objspace machinery (retire, absorb, creation cover) a no-op. */
bool
rb_gc_multi_objspace_p(void)
{
    return rb_gc_impl_multi_objspace_p();
}
 
/* Does obj belong to another Ractor's objspace rather than the current one?  Always
 * false for a single-objspace impl, which cannot tell owners apart. */
bool
rb_gc_obj_foreign_p(VALUE obj)
{
    return rb_gc_impl_obj_foreign_p(rb_gc_get_objspace(), obj);
}
 
bool
rb_gc_single_objspace_p(void)
{
    if (!rb_gc_impl_multi_objspace_p()) return true;
    rb_vm_t *vm = GET_VM();
    /* One Ractor is not one objspace: a forked child re-enters single-Ractor mode while
     * the pre-fork Ractors' objspaces are still parked in zombie_objspaces. */
    return (ruby_single_main_ractor != NULL || vm->ractor.cnt == 1) &&
           vm->gc.zombie_objspaces_count == 0 && gc_absorbing_zombie == 0 &&
           !gc_absorbed_since_global_gc &&
           (vm->ractor.main_ractor == NULL ||
            vm->ractor.main_ractor->creating_child_objspace == NULL);
}
 
/* Inherit a dead Ractor's objspace into the calling Ractor.  Going through the owner
 * slot clears it and releases the objspace in one VM-lock section; the merge runs with
 * the inheritor's GC disabled (moving the finalizer st table could trigger it). */
static void
objspace_absorb_merge(void *dst, void *src)
{
    ASSERT_vm_locking();
    rb_gc_impl_objspace_absorb(dst, src);
    gc_absorbed_since_global_gc = true;
}
 
/* The dying thread's last collection of its own objspace, GVL still held; with
 * r->postmortem set, rb_ractor_mark_local_roots roots only the join value and the
 * registered_marks pins, so the scaffolding nobody needs any more dies here. */
void
rb_gc_objspace_postmortem_self(void)
{
    if (!rb_gc_impl_multi_objspace_p()) return;
 
    rb_gc_impl_objspace_retire_gc(rb_gc_get_objspace());
}
 
void
rb_gc_objspace_absorb_into_current(void **objspace_slot)
{
    if (!rb_gc_impl_multi_objspace_p()) {
        *objspace_slot = NULL;
        return;
    }
    RB_VM_LOCKING() {
        void *objspace = *objspace_slot;
        if (objspace != NULL) {
            *objspace_slot = NULL;
            gc_absorbing_zombie++;
            rb_gc_vm_forget_zombie(objspace);
            objspace_absorb_merge(rb_gc_get_objspace(), objspace);
            gc_absorbing_zombie--;
        }
    }
}
 
/* Merge every ownerless zombie objspace (no owner slot, i.e. the Ractor object was
 * collected) into the current Ractor's objspace.  Runs as a postponed job on the main
 * Ractor's thread; the VM teardown path calls it directly. */
static void
objspace_absorb_disowned_zombies(void)
{
    rb_vm_t *vm = GET_VM();
 
    RB_VM_LOCKING() {
        size_t i = 0;
        while (i < vm->gc.zombie_objspaces_count) {
            if (vm->gc.zombie_objspaces[i].owner_slot == NULL) {
                void *zombie = vm->gc.zombie_objspaces[i].objspace;
                /* Remove via forget, which also subtracts the entry's pages from
                 * zombie_total_pages; a hand-written swap-remove would leave a phantom
                 * total that keeps starting stop-the-world global cycles. */
                gc_absorbing_zombie++;
                rb_gc_vm_forget_zombie(zombie);
                objspace_absorb_merge(rb_gc_get_objspace(), zombie);
                gc_absorbing_zombie--;
            }
            else {
                i++;
            }
        }
    }
}
 
static void
gc_orphan_merge_job(void *unused)
{
    (void)unused;
    objspace_absorb_disowned_zombies();
}
 
/* Re-target a pending orphan merge after fork.  The job may target the parent's main
 * Ractor, whose per-Ractor trigger mask is not inherited unless that Ractor forked.
 * Called on the child side. */
/* Only main survives a fork, so rebuild the counter from main's own hold alone. */
void
rb_gc_disable_holders_atfork(void)
{
    RUBY_ATOMIC_SET(GET_VM()->gc.disable_holders,
                    rb_gc_impl_user_gc_disabled_p(rb_gc_get_objspace()) ? 1 : 0);
}
 
void
rb_gc_zombie_objspaces_atfork(void)
{
    rb_vm_t *vm = GET_VM();
 
    for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) {
        if (vm->gc.zombie_objspaces[i].owner_slot == NULL) {
            rb_postponed_job_trigger_for_ractor(GET_VM()->gc.orphan_merge_pjob, vm->ractor.main_ractor->pub.self);
            break;
        }
    }
}
 
/* VM teardown, right after every other Ractor was killed: merge all uninherited
 * objspaces into main so at-exit processing covers every object and dead Ractors'
 * deferred finalizers run on main.  The owner slot also covers collected wrappers. */
void
rb_gc_objspace_absorb_all_zombies(void)
{
    rb_vm_t *vm = GET_VM();
 
    /* Entries whose Ractor object is already gone, i.e. the pending merge job itself,
     * which we run synchronously here. */
    objspace_absorb_disowned_zombies();
 
    while (vm->gc.zombie_objspaces_count > 0) {
        size_t before = vm->gc.zombie_objspaces_count;
        GC_ASSERT(vm->gc.zombie_objspaces[0].owner_slot != NULL);
        /* Move the rb_gc_register_mark_object pins before the merge, so the objects
         * pinned in the owner's objspace do not lose their root in its sweep. */
        rb_ractor_t *owner = vm->gc.zombie_objspaces[0].owner;
        if (owner) {
            rb_ractor_absorb_registered_marks(GET_RACTOR(), owner);
        }
        rb_gc_objspace_absorb_into_current(vm->gc.zombie_objspaces[0].owner_slot);
        if (vm->gc.zombie_objspaces_count >= before) {
            rb_bug("rb_gc_objspace_absorb_all_zombies: zombie list did not shrink");
        }
    }
}
 
static void
gc_ref_update_array(void *objspace, VALUE v)
{
    if (ARY_SHARED_P(v)) {
        VALUE old_root = RARRAY(v)->as.heap.aux.shared_root;
 
        UPDATE_IF_MOVED(objspace, RARRAY(v)->as.heap.aux.shared_root);
 
        VALUE new_root = RARRAY(v)->as.heap.aux.shared_root;
        // A root hollowed out by a move is no longer an array, and it is pinned rather
        // than re-pointed (see the marking of a shared root).
        // If the root is embedded and its location has changed
        if (RB_TYPE_P(new_root, T_ARRAY) && ARY_EMBED_P(new_root) && new_root != old_root) {
            size_t offset = (size_t)(RARRAY(v)->as.heap.ptr - RARRAY(old_root)->as.ary);
            GC_ASSERT(RARRAY(v)->as.heap.ptr >= RARRAY(old_root)->as.ary);
            RARRAY(v)->as.heap.ptr = RARRAY(new_root)->as.ary + offset;
        }
    }
    else {
        long len = RARRAY_LEN(v);
 
        if (len > 0) {
            VALUE *ptr = (VALUE *)RARRAY_CONST_PTR(v);
            for (long i = 0; i < len; i++) {
                UPDATE_IF_MOVED(objspace, ptr[i]);
            }
        }
 
        if (rb_gc_obj_slot_size(v) >= rb_ary_size_as_embedded(v)) {
            /* Skip pinned arrays: a pinned array may be referenced from a
             * conservative root holding RARRAY_PTR across this compaction, so
             * freeing its heap buffer here would dangle that pointer. */
            if (rb_ary_embeddable_p(v) && !rb_gc_impl_pinned_p(objspace, v)) {
                rb_ary_make_embedded(v);
            }
        }
    }
}
 
static void
gc_ref_update_object(void *objspace, VALUE v)
{
    RUBY_ASSERT(rb_gc_obj_slot_size(v) == rb_obj_shape_slot_size(v));
    shape_id_t shape_id = RBASIC_SHAPE_ID(v);
 
    if (!rb_shape_embedded_p(shape_id)) {
        UPDATE_IF_MOVED(objspace, ROBJECT(v)->as.extended);
 
        if (!rb_shape_complex_p(shape_id) && rb_shape_embedded_capacity(shape_id) >= RSHAPE_LEN(shape_id)) {
            VALUE *embedded_fields = ROBJECT_EMBEDDED_FIELDS(v);
            VALUE *extended_fields = ROBJECT_FIELDS(v);
            MEMCPY(embedded_fields, extended_fields, VALUE, RSHAPE_LEN(shape_id));
            shape_id = rb_shape_transition_robject(shape_id);
            RBASIC_SET_FULL_SHAPE_ID(v, shape_id);
            rb_gc_writebarrier_remember(v);
        }
        else {
            return;
        }
    }
 
    VALUE *ptr = ROBJECT_FIELDS(v);
    attr_index_t len = RSHAPE_LEN(shape_id);
    for (attr_index_t i = 0; i < len; i++) {
        UPDATE_IF_MOVED(objspace, ptr[i]);
    }
}
 
void
rb_gc_ref_update_table_values_only(st_table *tbl)
{
    gc_ref_update_table_values_only(tbl);
}
 
/* Update MOVED references in a VALUE=>VALUE st_table */
void
rb_gc_update_tbl_refs(st_table *ptr)
{
    gc_update_table_refs(ptr);
}
 
static int
rb_gc_update_set_refs_i(st_data_t key, st_data_t value, st_data_t argp, int error)
{
    if (rb_gc_location((VALUE)key) != (VALUE)key) {
        return ST_REPLACE;
    }
 
    return ST_CONTINUE;
}
 
static int
rb_gc_update_set_refs_replace_i(st_data_t *key, st_data_t *value, st_data_t argp, int existing)
{
    rb_gc_update_moved((VALUE *)key);
 
    return ST_CONTINUE;
}
 
void
rb_gc_update_set_refs(st_table *tbl)
{
    if (!tbl || tbl->num_entries == 0) return;
 
    if (st_foreach_with_replace(tbl, rb_gc_update_set_refs_i, rb_gc_update_set_refs_replace_i, 0)) {
        rb_raise(rb_eRuntimeError, "hash modified during iteration");
    }
}
 
static void
gc_ref_update_hash(void *objspace, VALUE v)
{
    rb_hash_stlike_foreach_with_replace(v, hash_foreach_replace, hash_replace_ref, (st_data_t)objspace);
}
 
static void
gc_update_values(void *objspace, long n, VALUE *values)
{
    for (long i = 0; i < n; i++) {
        UPDATE_IF_MOVED(objspace, values[i]);
    }
}
 
void
rb_gc_update_values(long n, VALUE *values)
{
    gc_update_values(rb_gc_get_objspace(), n, values);
}
 
static enum rb_id_table_iterator_result
check_id_table_move(VALUE value, void *data)
{
    void *objspace = (void *)data;
 
    if (gc_object_moved_p_internal(objspace, (VALUE)value)) {
        return ID_TABLE_REPLACE;
    }
 
    return ID_TABLE_CONTINUE;
}
 
void
rb_gc_prepare_heap_process_object(VALUE obj)
{
    switch (BUILTIN_TYPE(obj)) {
      case T_STRING:
        // Precompute the string coderange. This both save time for when it will be
        // eventually needed, and avoid mutating heap pages after a potential fork.
        rb_enc_str_coderange(obj);
        break;
      default:
        break;
    }
}
 
void
rb_gc_prepare_heap(void)
{
    rb_gc_impl_prepare_heap(rb_gc_get_objspace());
}
 
size_t
rb_gc_size_slot_size(size_t size)
{
    return rb_gc_impl_size_slot_size(rb_gc_get_objspace(), size);
}
 
bool
rb_gc_size_allocatable_p(size_t size)
{
    return rb_gc_impl_size_allocatable_p(size);
}
 
size_t
rb_gc_max_allocation_size(void)
{
    return rb_gc_impl_max_allocation_size();
}
 
static enum rb_id_table_iterator_result
update_id_table(VALUE *value, void *data, int existing)
{
    void *objspace = (void *)data;
 
    if (gc_object_moved_p_internal(objspace, (VALUE)*value)) {
        *value = gc_location_internal(objspace, (VALUE)*value);
    }
 
    return ID_TABLE_CONTINUE;
}
 
static void
update_m_tbl(void *objspace, struct rb_id_table *tbl)
{
    if (tbl) {
        rb_id_table_foreach_values_with_replace(tbl, check_id_table_move, update_id_table, objspace);
    }
}
 
static enum rb_id_table_iterator_result
update_const_tbl_i(VALUE value, void *objspace)
{
    rb_const_entry_t *ce = (rb_const_entry_t *)value;
 
    if (gc_object_moved_p_internal(objspace, ce->value)) {
        ce->value = gc_location_internal(objspace, ce->value);
    }
 
    if (gc_object_moved_p_internal(objspace, ce->file)) {
        ce->file = gc_location_internal(objspace, ce->file);
    }
 
    return ID_TABLE_CONTINUE;
}
 
static void
update_const_tbl(void *objspace, struct rb_id_table *tbl)
{
    if (!tbl) return;
    rb_id_table_foreach_values(tbl, update_const_tbl_i, objspace);
}
 
static void
update_superclasses(rb_objspace_t *objspace, rb_classext_t *ext)
{
    if (RCLASSEXT_SUPERCLASSES_WITH_SELF(ext)) {
        size_t array_size = RCLASSEXT_SUPERCLASS_DEPTH(ext) + 1;
        for (size_t i = 0; i < array_size; i++) {
            UPDATE_IF_MOVED(objspace, RCLASSEXT_SUPERCLASSES(ext)[i]);
        }
    }
}
 
static void
update_classext_values(rb_objspace_t *objspace, rb_classext_t *ext, bool is_iclass)
{
    UPDATE_IF_MOVED(objspace, RCLASSEXT_ORIGIN(ext));
    UPDATE_IF_MOVED(objspace, RCLASSEXT_REFINED_CLASS(ext));
    UPDATE_IF_MOVED(objspace, RCLASSEXT_CLASSPATH(ext));
    if (is_iclass) {
        UPDATE_IF_MOVED(objspace, RCLASSEXT_INCLUDER(ext));
    }
}
 
static void
update_classext(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg)
{
    struct classext_foreach_args *args = (struct classext_foreach_args *)arg;
    rb_objspace_t *objspace = args->objspace;
 
    if (RCLASSEXT_SUPER(ext)) {
        UPDATE_IF_MOVED(objspace, RCLASSEXT_SUPER(ext));
    }
 
    update_m_tbl(objspace, RCLASSEXT_M_TBL(ext));
 
    UPDATE_IF_MOVED(objspace, ext->fields_obj);
    if (!RCLASSEXT_SHARED_CONST_TBL(ext)) {
        update_const_tbl(objspace, RCLASSEXT_CONST_TBL(ext));
    }
    UPDATE_IF_MOVED(objspace, RCLASSEXT_CC_TBL(ext));
    UPDATE_IF_MOVED(objspace, RCLASSEXT_CVC_TBL(ext));
    update_superclasses(objspace, ext);
    if (RCLASSEXT_SUBCLASSES(ext)) {
        UPDATE_IF_MOVED(objspace, RCLASSEXT_SUBCLASSES(ext));
    }
 
    update_classext_values(objspace, ext, false);
}
 
static void
update_iclass_classext(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg)
{
    struct classext_foreach_args *args = (struct classext_foreach_args *)arg;
    rb_objspace_t *objspace = args->objspace;
 
    if (RCLASSEXT_SUPER(ext)) {
        UPDATE_IF_MOVED(objspace, RCLASSEXT_SUPER(ext));
    }
    update_m_tbl(objspace, RCLASSEXT_M_TBL(ext));
    update_m_tbl(objspace, RCLASSEXT_CALLABLE_M_TBL(ext));
    UPDATE_IF_MOVED(objspace, RCLASSEXT_CC_TBL(ext));
    UPDATE_IF_MOVED(objspace, RCLASSEXT_CVC_TBL(ext));
    if (RCLASSEXT_SUBCLASSES(ext)) {
        UPDATE_IF_MOVED(objspace, RCLASSEXT_SUBCLASSES(ext));
    }
 
    update_classext_values(objspace, ext, true);
}
 
struct global_vm_table_foreach_data {
    vm_table_foreach_callback_func callback;
    vm_table_update_callback_func update_callback;
    void *data;
    bool weak_only;
    /* The generic_fields table being walked, so compaction can re-insert a moved key
     * into it (rb_generic_fields_tables_foreach hands the table to the callback). */
    struct st_table *gen_fields_current_tbl;
    /* Re-inserting a moved key adds an entry, which can rehash and break the running
     * iterator, so collect them and insert after the walk (raw realloc: we are in GC). */
    struct gen_fields_deferred_insert { st_data_t k, v; } *gf_deferred;
    size_t gf_deferred_cnt, gf_deferred_capa;
};
 
static int
vm_weak_table_foreach_weak_key(st_data_t key, st_data_t value, st_data_t data, int error)
{
    struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
 
    int ret = iter_data->callback((VALUE)key, iter_data->data);
 
    if (!iter_data->weak_only) {
        if (ret != ST_CONTINUE) return ret;
 
        ret = iter_data->callback((VALUE)value, iter_data->data);
    }
 
    return ret;
}
 
static int
vm_weak_table_foreach_update_weak_key(st_data_t *key, st_data_t *value, st_data_t data, int existing)
{
    struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
 
    int ret = iter_data->update_callback((VALUE *)key, iter_data->data);
 
    if (!iter_data->weak_only) {
        if (ret != ST_CONTINUE) return ret;
 
        ret = iter_data->update_callback((VALUE *)value, iter_data->data);
    }
 
    return ret;
}
 
static int
vm_weak_table_sym_set_foreach(VALUE *sym_ptr, void *data)
{
    VALUE sym = *sym_ptr;
    struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
 
    if (RB_SPECIAL_CONST_P(sym)) return ST_CONTINUE;
 
    int ret = iter_data->callback(sym, iter_data->data);
 
    if (ret == ST_REPLACE) {
        ret = iter_data->update_callback(sym_ptr, iter_data->data);
    }
 
    return ret;
}
 
struct st_table *rb_generic_fields_tbl_get(void);
 
static int
vm_weak_table_gen_fields_foreach(st_data_t key, st_data_t value, st_data_t data)
{
    struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
 
    int ret = iter_data->callback((VALUE)key, iter_data->data);
 
    VALUE new_value = (VALUE)value;
    VALUE new_key = (VALUE)key;
 
    switch (ret) {
      case ST_CONTINUE:
        break;
 
      case ST_DELETE:
        // When we're removing an object from the weak ref table, we need to
        // set the shape on it so that the GC finalizer won't try to remove
        // it again.  A "root shape" indicates to the GC that this object
        // has no fields on it, hence it won't be in the gen fields table.
        if (BUILTIN_TYPE((VALUE)key) != T_NONE) {
            RBASIC_SET_SHAPE_ID((VALUE)key, ROOT_SHAPE_ID);
        }
        return ST_DELETE;
 
      case ST_REPLACE: {
        ret = iter_data->update_callback(&new_key, iter_data->data);
        if (key != new_key) {
            ret = ST_DELETE;
        }
        break;
      }
 
      default:
        rb_bug("vm_weak_table_gen_fields_foreach: return value %d not supported", ret);
    }
 
    if (!iter_data->weak_only) {
        int ivar_ret = iter_data->callback(new_value, iter_data->data);
        switch (ivar_ret) {
          case ST_CONTINUE:
            break;
 
          case ST_REPLACE:
            iter_data->update_callback(&new_value, iter_data->data);
            break;
 
          case ST_DELETE:
            /* Leftover entry of a moved host: even if the key is alive, nobody can
             * read these fields once fields_obj is unreachable, so clean up as if the
             * key had died. */
            RBASIC_SET_SHAPE_ID((VALUE)key, ROOT_SHAPE_ID);
            return ST_DELETE;
 
          default:
            rb_bug("vm_weak_table_gen_fields_foreach: return value %d not supported", ivar_ret);
        }
    }
 
    if (key != new_key) {
        /* Inserting the new key adds an entry and may rehash, so defer it. */
        if (iter_data->gf_deferred_cnt == iter_data->gf_deferred_capa) {
            size_t nc = iter_data->gf_deferred_capa ? iter_data->gf_deferred_capa * 2 : 64;
            struct gen_fields_deferred_insert *p =
                realloc(iter_data->gf_deferred, nc * sizeof(*p));
            if (!p) rb_bug("vm_weak_table_gen_fields_foreach: out of memory");
            iter_data->gf_deferred = p;
            iter_data->gf_deferred_capa = nc;
        }
        iter_data->gf_deferred[iter_data->gf_deferred_cnt++] =
            (struct gen_fields_deferred_insert){ .k = (st_data_t)new_key, .v = (st_data_t)new_value };
    }
    else if (value != new_value) {
        DURING_GC_COULD_MALLOC_REGION_START();
        {
            /* Updating an existing key's value adds no entry and cannot rehash. */
            st_insert(iter_data->gen_fields_current_tbl, (st_data_t)new_key, new_value);
        }
        DURING_GC_COULD_MALLOC_REGION_END();
    }
 
    return ret;
}
 
static int
vm_weak_table_frozen_strings_foreach(VALUE *str, void *data)
{
    // int retval = vm_weak_table_foreach_weak_key(key, value, data, error);
    struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
    int retval = iter_data->callback(*str, iter_data->data);
 
    if (retval == ST_REPLACE) {
        retval = iter_data->update_callback(str, iter_data->data);
    }
 
    if (retval == ST_DELETE) {
        FL_UNSET(*str, RSTRING_FSTR);
    }
 
    return retval;
}
 
void rb_fstring_foreach_with_replace(int (*callback)(VALUE *str, void *data), void *data);
 
/* Callback of rb_generic_fields_tables_foreach: walk one generic_fields table with the
 * gen_fields foreach used by compaction, recording the current table in foreach_data so
 * a moved key is re-inserted into the right one. */
static void
vm_weak_table_gen_fields_tbl_cb(struct st_table *tbl, void *arg)
{
    struct global_vm_table_foreach_data *foreach_data = (struct global_vm_table_foreach_data *)arg;
    foreach_data->gen_fields_current_tbl = tbl;
    st_foreach(tbl, vm_weak_table_gen_fields_foreach, (st_data_t)foreach_data);
}
 
void
rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func callback,
                            vm_table_update_callback_func update_callback,
                            void *data,
                            bool weak_only,
                            enum rb_gc_vm_weak_tables table)
{
    rb_vm_t *vm = GET_VM();
 
    struct global_vm_table_foreach_data foreach_data = {
        .callback = callback,
        .update_callback = update_callback,
        .data = data,
        .weak_only = weak_only,
    };
 
    switch (table) {
      case RB_GC_VM_CI_TABLE: {
        st_foreach_with_replace(
            &vm->ci_table,
            vm_weak_table_foreach_weak_key,
            vm_weak_table_foreach_update_weak_key,
            (st_data_t)&foreach_data
        );
        break;
      }
      case RB_GC_VM_OVERLOADED_CME_TABLE: {
        st_foreach_with_replace(
            &vm->overloaded_cme_table,
            vm_weak_table_foreach_weak_key,
            vm_weak_table_foreach_update_weak_key,
            (st_data_t)&foreach_data
        );
        break;
      }
      case RB_GC_VM_GLOBAL_SYMBOLS_TABLE: {
        rb_sym_global_symbol_table_foreach_weak_reference(
            vm_weak_table_sym_set_foreach,
            &foreach_data
        );
        break;
      }
      case RB_GC_VM_GENERIC_FIELDS_TABLE: {
        /* There is one table.  A global GC walks it without a lock under the
         * stop-the-world barrier; a local compaction holds the barrier VM lock taken in
         * gc_enter, so foreign keys cannot move and fall through the moved check.  The
         * table's mutex (taken by shared_table_foreach) excludes mutator inserts. */
        if (rb_gc_during_global_gc_p()) {
            rb_generic_fields_tables_foreach(vm_weak_table_gen_fields_tbl_cb, (void *)&foreach_data);
        }
        else if (!weak_only) {
            rb_generic_fields_shared_table_foreach(vm_weak_table_gen_fields_tbl_cb, (void *)&foreach_data);
        }
        if (foreach_data.gf_deferred != NULL) {
            DURING_GC_COULD_MALLOC_REGION_START();
            {
                for (size_t i = 0; i < foreach_data.gf_deferred_cnt; i++) {
                    struct gen_fields_deferred_insert *const d = &foreach_data.gf_deferred[i];
                    st_insert(foreach_data.gen_fields_current_tbl, d->k, d->v);
                }
            }
            DURING_GC_COULD_MALLOC_REGION_END();
            free(foreach_data.gf_deferred);
        }
        break;
      }
      case RB_GC_VM_FROZEN_STRINGS_TABLE: {
        rb_fstring_foreach_with_replace(
            vm_weak_table_frozen_strings_foreach,
            &foreach_data
        );
        break;
      }
      case RB_GC_VM_WEAK_TABLE_COUNT:
        rb_bug("Unreachable");
      default:
        rb_bug("rb_gc_vm_weak_table_foreach: unknown table %d", table);
    }
}
 
/* The global GC's weak pass over the generic_fields table; under the barrier, so the
 * walk needs no lock. */
struct gf_mark_foreach_ctx {
    int (*cb)(VALUE key, VALUE val, void *arg);
    void *arg;
};
 
static int
gf_mark_foreach_i(st_data_t key, st_data_t val, st_data_t data)
{
    struct gf_mark_foreach_ctx *ctx = (struct gf_mark_foreach_ctx *)data;
    return ctx->cb((VALUE)key, (VALUE)val, ctx->arg);
}
 
static void
gf_mark_foreach_table_cb(struct st_table *tbl, void *arg)
{
    st_foreach(tbl, gf_mark_foreach_i, (st_data_t)arg);
}
 
void
rb_gc_vm_generic_fields_mark_foreach(int (*cb)(VALUE key, VALUE val, void *arg), void *arg)
{
    struct gf_mark_foreach_ctx ctx = { cb, arg };
    rb_generic_fields_tables_foreach(gf_mark_foreach_table_cb, &ctx);
}
 
struct gf_drain_ctx {
    bool (*is_dead)(VALUE key);
};
 
static int
gf_drain_i(st_data_t key, st_data_t val, st_data_t data)
{
    struct gf_drain_ctx *ctx = (struct gf_drain_ctx *)data;
    if (ctx->is_dead((VALUE)key)) {
        /* The weak pass only drains dead keys' entries, never touching the key itself:
         * after the global GC settled another objspace's lazy sweep the key may already
         * be freed (poisoned), and writing a shape there would be a use-after-poison. */
        return ST_DELETE;
    }
    return ST_CONTINUE;
}
 
static void
gf_drain_table_cb(struct st_table *tbl, void *arg)
{
    st_foreach(tbl, gf_drain_i, (st_data_t)arg);
}
 
void
rb_gc_vm_generic_fields_drain_dead(bool (*is_dead)(VALUE key))
{
    struct gf_drain_ctx ctx = { is_dead };
    rb_generic_fields_tables_foreach(gf_drain_table_cb, &ctx);
}
 
VALUE
rb_gc_vm_top_self(void)
{
    return rb_vm_top_self();
}
 
void
rb_gc_update_vm_references(void *objspace)
{
    rb_execution_context_t *ec = GET_EC();
    rb_vm_t *vm = rb_ec_vm_ptr(ec);
 
    rb_vm_update_references(vm);
    rb_gc_update_global_tbl();
    rb_sym_global_symbols_mark_and_move();
 
#if USE_YJIT
    void rb_yjit_root_update_references(void); // in Rust
 
    if (rb_yjit_enabled_p) {
        rb_yjit_root_update_references();
    }
#endif
 
#if USE_ZJIT
    void rb_zjit_root_update_references(void); // in Rust
 
    if (rb_zjit_enabled_p) {
        rb_zjit_root_update_references();
    }
#endif
}
 
void
rb_gc_update_object_references(void *objspace, VALUE obj)
{
    struct classext_foreach_args args;
 
    switch (BUILTIN_TYPE(obj)) {
      case T_CLASS:
        if (FL_TEST_RAW(obj, FL_SINGLETON)) {
            UPDATE_IF_MOVED(objspace, RCLASS_ATTACHED_OBJECT(obj));
        }
        // Continue to the shared T_CLASS/T_MODULE
      case T_MODULE:
        args.klass = obj;
        args.objspace = objspace;
        rb_class_classext_foreach(obj, update_classext, (void *)&args);
        break;
 
      case T_ICLASS:
        args.objspace = objspace;
        rb_class_classext_foreach(obj, update_iclass_classext, (void *)&args);
        break;
 
      case T_IMEMO:
        rb_imemo_mark_and_move(obj, true);
        return;
 
      case T_NIL:
      case T_FIXNUM:
      case T_NODE:
      case T_MOVED:
      case T_NONE:
        /* These can't move */
        return;
 
      case T_ARRAY:
        gc_ref_update_array(objspace, obj);
        break;
 
      case T_HASH:
        gc_ref_update_hash(objspace, obj);
        UPDATE_IF_MOVED(objspace, RHASH(obj)->ifnone);
        break;
 
      case T_STRING:
        {
            if (STR_SHARED_P(obj)) {
                UPDATE_IF_MOVED(objspace, RSTRING(obj)->as.heap.aux.shared);
            }
 
            /* If, after move the string is not embedded, and can fit in the
             * slot it's been placed in, then re-embed it. Skip pinned objects:
             * a local holding RSTRING_PTR across this compaction could otherwise
             * point to freed memory even if the String is marked and pinned. */
            if (rb_gc_obj_slot_size(obj) >= rb_str_size_as_embedded(obj)) {
                if (!STR_EMBED_P(obj) && rb_str_reembeddable_p(obj)
                        && !rb_gc_impl_pinned_p(objspace, obj)) {
                    rb_str_make_embedded(obj);
                }
            }
 
            break;
        }
      case T_DATA:
        /* Call the compaction callback, if it exists */
        {
            void *const ptr = RTYPEDDATA_GET_DATA(obj);
 
            UPDATE_IF_MOVED(objspace, RTYPEDDATA(obj)->fields_obj);
 
            if (ptr) {
                if (gc_declarative_marking_p(RTYPEDDATA_TYPE(obj))) {
                    size_t *offset_list = TYPED_DATA_REFS_OFFSET_LIST(obj);
 
                    for (size_t offset = *offset_list; offset != RUBY_REF_END; offset = *offset_list++) {
                        VALUE *ref = (VALUE *)((char *)ptr + offset);
                        *ref = gc_location_internal(objspace, *ref);
                    }
                }
                else {
                    RUBY_DATA_FUNC compact_func = RTYPEDDATA_TYPE(obj)->function.dcompact;
                    if (compact_func) (*compact_func)(ptr);
                }
            }
        }
        break;
 
      case T_OBJECT:
        gc_ref_update_object(objspace, obj);
        break;
 
      case T_FILE:
        if (RFILE(obj)->fptr) {
            UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->self);
            UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->pathv);
            UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->tied_io_for_writing);
            UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->writeconv_asciicompat);
            UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->writeconv_pre_ecopts);
            UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->encs.ecopts);
            UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->write_lock);
            UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->timeout);
            UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->wakeup_mutex);
        }
        break;
      case T_REGEXP:
        UPDATE_IF_MOVED(objspace, RREGEXP(obj)->src);
        break;
 
      case T_SYMBOL:
        UPDATE_IF_MOVED(objspace, RSYMBOL(obj)->fstr);
        break;
 
      case T_FLOAT:
      case T_BIGNUM:
        break;
 
      case T_MATCH:
        UPDATE_IF_MOVED(objspace, RMATCH(obj)->regexp);
 
        if (RMATCH(obj)->str) {
            UPDATE_IF_MOVED(objspace, RMATCH(obj)->str);
        }
        break;
 
      case T_RATIONAL:
        UPDATE_IF_MOVED(objspace, RRATIONAL(obj)->num);
        UPDATE_IF_MOVED(objspace, RRATIONAL(obj)->den);
        break;
 
      case T_COMPLEX:
        UPDATE_IF_MOVED(objspace, RCOMPLEX(obj)->real);
        UPDATE_IF_MOVED(objspace, RCOMPLEX(obj)->imag);
 
        break;
 
      case T_STRUCT:
        {
            long i, len = RSTRUCT_LEN(obj);
            VALUE *ptr = (VALUE *)RSTRUCT_CONST_PTR(obj);
 
            for (i = 0; i < len; i++) {
                UPDATE_IF_MOVED(objspace, ptr[i]);
            }
 
            UPDATE_IF_MOVED(objspace, RSTRUCT(obj)->fields_obj);
        }
        break;
      default:
        rb_bug("unreachable");
        break;
    }
 
    UPDATE_IF_MOVED(objspace, RBASIC(obj)->klass);
}
 
VALUE
rb_gc_start(void)
{
    rb_gc();
    return Qnil;
}
 
void
rb_gc(void)
{
    unless_objspace(objspace) { return; }
 
    rb_gc_impl_start(objspace, true, true, true, false, true);
}
 
int
rb_during_gc(void)
{
    unless_objspace(objspace) { return FALSE; }
 
    return rb_gc_impl_during_gc_p(objspace);
}
 
size_t
rb_gc_count(void)
{
    return rb_gc_impl_gc_count(rb_gc_get_objspace());
}
 
static VALUE
gc_count(rb_execution_context_t *ec, VALUE self)
{
    return SIZET2NUM(rb_gc_count());
}
 
VALUE
rb_gc_latest_gc_info(VALUE key)
{
    if (!SYMBOL_P(key) && !RB_TYPE_P(key, T_HASH)) {
        rb_raise(rb_eTypeError, "non-hash or symbol given");
    }
 
    VALUE val = rb_gc_impl_latest_gc_info(rb_gc_get_objspace(), key);
 
    if (val == Qundef) {
        rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
    }
 
    return val;
}
 
static VALUE
gc_stat(rb_execution_context_t *ec, VALUE self, VALUE arg, VALUE global_scope)
{
    if (NIL_P(arg)) {
        arg = rb_hash_new();
    }
    else if (!RB_TYPE_P(arg, T_HASH) && !SYMBOL_P(arg)) {
        rb_raise(rb_eTypeError, "non-hash or symbol given");
    }
 
    VALUE ret = rb_gc_impl_stat(RTEST(global_scope) ? NULL : rb_gc_get_objspace(), arg);
 
    if (ret == Qundef) {
        GC_ASSERT(SYMBOL_P(arg));
 
        rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(arg));
    }
 
    return ret;
}
 
size_t
rb_gc_stat(VALUE arg)
{
    if (!RB_TYPE_P(arg, T_HASH) && !SYMBOL_P(arg)) {
        rb_raise(rb_eTypeError, "non-hash or symbol given");
    }
 
    VALUE ret = rb_gc_impl_stat(rb_gc_get_objspace(), arg);
 
    if (ret == Qundef) {
        GC_ASSERT(SYMBOL_P(arg));
 
        rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(arg));
    }
 
    if (SYMBOL_P(arg)) {
        return NUM2SIZET(ret);
    }
    else {
        return 0;
    }
}
 
static VALUE
gc_stat_heap(rb_execution_context_t *ec, VALUE self, VALUE heap_name, VALUE arg)
{
    if (NIL_P(arg)) {
        arg = rb_hash_new();
    }
 
    if (NIL_P(heap_name)) {
        if (!RB_TYPE_P(arg, T_HASH)) {
            rb_raise(rb_eTypeError, "non-hash given");
        }
    }
    else if (FIXNUM_P(heap_name)) {
        if (!SYMBOL_P(arg) && !RB_TYPE_P(arg, T_HASH)) {
            rb_raise(rb_eTypeError, "non-hash or symbol given");
        }
    }
    else {
        rb_raise(rb_eTypeError, "heap_name must be nil or an Integer");
    }
 
    VALUE ret = rb_gc_impl_stat_heap(rb_gc_get_objspace(), heap_name, arg);
 
    if (ret == Qundef) {
        GC_ASSERT(SYMBOL_P(arg));
 
        rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(arg));
    }
 
    return ret;
}
 
static VALUE
gc_config_get(rb_execution_context_t *ec, VALUE self)
{
    VALUE cfg_hash = rb_gc_impl_config_get(rb_gc_get_objspace());
    rb_hash_aset(cfg_hash, sym("implementation"), rb_fstring_cstr(rb_gc_impl_active_gc_name()));
 
    return cfg_hash;
}
 
static VALUE
gc_config_set(rb_execution_context_t *ec, VALUE self, VALUE hash)
{
    void *objspace = rb_gc_get_objspace();
 
    rb_gc_impl_config_set(objspace, hash);
 
    return Qnil;
}
 
static VALUE
gc_stress_get(rb_execution_context_t *ec, VALUE self)
{
    return rb_gc_impl_stress_get(rb_gc_get_objspace());
}
 
static VALUE
gc_stress_set_m(rb_execution_context_t *ec, VALUE self, VALUE flag)
{
    rb_gc_impl_stress_set(rb_gc_get_objspace(), flag);
 
    return flag;
}
 
void
rb_gc_initial_stress_set(VALUE flag)
{
    initial_stress = flag;
}
 
/* Add or drop a GC-disable holder (vm->gc.disable_holders; see vm_core.h).  critical
 * is the anonymous holder used by internal sections that must not be interrupted by a
 * GC, such as collecting under the barrier. */
 
static void
rb_gc_critical_disable(void)
{
    rb_gc_impl_gc_rest(rb_gc_get_objspace());
    RUBY_ATOMIC_INC(GET_VM()->gc.disable_holders);
}
 
static void
rb_gc_critical_enable(void)
{
    RUBY_ATOMIC_DEC(GET_VM()->gc.disable_holders);
}
 
bool
rb_gc_gc_disabled_global_p(void)
{
    return RUBY_ATOMIC_LOAD(GET_VM()->gc.disable_holders) != 0;
}
 
/* GC.disable/enable set and clear this objspace's flag and only move the holder count
 * when the flag actually changes.  The returned previous state is this objspace's. */
static bool
gc_ractor_disable_set(bool disable)
{
    const bool was = rb_gc_impl_user_gc_disabled_set(rb_gc_get_objspace(), disable);
    if (was != disable) {
        if (disable) {
            RUBY_ATOMIC_INC(GET_VM()->gc.disable_holders);
        }
        else {
            RUBY_ATOMIC_DEC(GET_VM()->gc.disable_holders);
        }
    }
    return was;
}
 
VALUE
rb_gc_enable(void)
{
    return RBOOL(gc_ractor_disable_set(false));
}
 
VALUE
rb_gc_disable_no_rest(void)
{
    return RBOOL(gc_ractor_disable_set(true));
}
 
VALUE
rb_gc_disable(void)
{
    const bool was_disabled = gc_ractor_disable_set(true);
    if (!was_disabled) {
        rb_gc_impl_gc_rest(rb_gc_get_objspace());
    }
    return RBOOL(was_disabled);
}
 
VALUE
rb_objspace_gc_enable(void *objspace)
{
    bool disabled = !rb_gc_impl_gc_enabled_p(objspace);
    rb_gc_impl_gc_enable(objspace);
    return RBOOL(disabled);
}
 
VALUE
rb_objspace_gc_disable(void *objspace)
{
    bool disabled = !rb_gc_impl_gc_enabled_p(objspace);
    rb_gc_impl_gc_disable(objspace, true);
    return RBOOL(disabled);
}
 
VALUE
rb_gc_objspace_enable(void *objspace)
{
    return rb_objspace_gc_enable(objspace);
}
 
VALUE
rb_gc_local_enable(void)
{
    return rb_gc_objspace_enable(rb_gc_get_objspace());
}
 
 
VALUE
rb_gc_objspace_disable_no_rest(void *objspace)
{
    bool disabled = !rb_gc_impl_gc_enabled_p(objspace);
    rb_gc_impl_gc_disable(objspace, false);
    return RBOOL(disabled);
}
 
VALUE
rb_gc_local_disable_no_rest(void)
{
    return rb_gc_objspace_disable_no_rest(rb_gc_get_objspace());
}
 
static VALUE
gc_enable(rb_execution_context_t *ec, VALUE _)
{
    return rb_gc_enable();
}
 
static VALUE
gc_disable(rb_execution_context_t *ec, VALUE _)
{
    return rb_gc_disable();
}
 
// TODO: think about moving ruby_gc_set_params into Init_heap or Init_gc
void
ruby_gc_set_params(void)
{
    rb_gc_impl_set_params(rb_gc_get_objspace());
}
 
void
rb_objspace_reachable_objects_from(VALUE obj, void (func)(VALUE, void *), void *data)
{
    RB_VM_LOCKING() {
        if (rb_gc_impl_during_gc_p(rb_gc_get_objspace())) rb_bug("rb_objspace_reachable_objects_from() is not supported while during GC");
 
        if (!RB_SPECIAL_CONST_P(obj)) {
            struct gc_mark_func_data_struct **mfdp = GC_MARK_FUNC_DATA_SLOTP();
            struct gc_mark_func_data_struct *prev_mfd = *mfdp;
            struct gc_mark_func_data_struct mfd = {
                .mark_func = func,
                .data = data,
            };
 
            *mfdp = &mfd;
            rb_gc_mark_children(rb_gc_get_objspace(), obj);
            *mfdp = prev_mfd;
        }
    }
}
 
struct root_objects_data {
    const char *category;
    void (*func)(const char *category, VALUE, void *);
    void *data;
};
 
static void
root_objects_from(VALUE obj, void *ptr)
{
    const struct root_objects_data *data = (struct root_objects_data *)ptr;
    (*data->func)(data->category, obj, data->data);
}
 
void
rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *passing_data)
{
    if (rb_gc_impl_during_gc_p(rb_gc_get_objspace())) rb_bug("rb_gc_impl_objspace_reachable_objects_from_root() is not supported while during GC");
 
    struct root_objects_data data = {
        .func = func,
        .data = passing_data,
    };
 
    struct gc_mark_func_data_struct **mfdp = GC_MARK_FUNC_DATA_SLOTP();
    struct gc_mark_func_data_struct *prev_mfd = *mfdp;
    struct gc_mark_func_data_struct mfd = {
        .mark_func = root_objects_from,
        .data = &data,
    };
 
    *mfdp = &mfd;
    rb_gc_save_machine_context();
    rb_gc_mark_roots(rb_gc_get_objspace(), &data.category);
    *mfdp = prev_mfd;
}
 
/*
  ------------------------------ DEBUG ------------------------------
*/
 
static const char *
type_name(int type, VALUE obj)
{
    switch (type) {
#define TYPE_NAME(t) case (t): return #t;
            TYPE_NAME(T_NONE);
            TYPE_NAME(T_OBJECT);
            TYPE_NAME(T_CLASS);
            TYPE_NAME(T_MODULE);
            TYPE_NAME(T_FLOAT);
            TYPE_NAME(T_STRING);
            TYPE_NAME(T_REGEXP);
            TYPE_NAME(T_ARRAY);
            TYPE_NAME(T_HASH);
            TYPE_NAME(T_STRUCT);
            TYPE_NAME(T_BIGNUM);
            TYPE_NAME(T_FILE);
            TYPE_NAME(T_MATCH);
            TYPE_NAME(T_COMPLEX);
            TYPE_NAME(T_RATIONAL);
            TYPE_NAME(T_NIL);
            TYPE_NAME(T_TRUE);
            TYPE_NAME(T_FALSE);
            TYPE_NAME(T_SYMBOL);
            TYPE_NAME(T_FIXNUM);
            TYPE_NAME(T_UNDEF);
            TYPE_NAME(T_IMEMO);
            TYPE_NAME(T_ICLASS);
            TYPE_NAME(T_MOVED);
            TYPE_NAME(T_ZOMBIE);
      case T_DATA:
        if (obj && rb_objspace_data_type_name(obj)) {
            return rb_objspace_data_type_name(obj);
        }
        return "T_DATA";
#undef TYPE_NAME
    }
    return "unknown";
}
 
static const char *
obj_type_name(VALUE obj)
{
    return type_name(TYPE(obj), obj);
}
 
const char *
rb_method_type_name(rb_method_type_t type)
{
    switch (type) {
      case VM_METHOD_TYPE_ISEQ:           return "iseq";
      case VM_METHOD_TYPE_ATTRSET:        return "attrset";
      case VM_METHOD_TYPE_IVAR:           return "ivar";
      case VM_METHOD_TYPE_BMETHOD:        return "bmethod";
      case VM_METHOD_TYPE_ALIAS:          return "alias";
      case VM_METHOD_TYPE_REFINED:        return "refined";
      case VM_METHOD_TYPE_CFUNC:          return "cfunc";
      case VM_METHOD_TYPE_ZSUPER:         return "zsuper";
      case VM_METHOD_TYPE_MISSING:        return "missing";
      case VM_METHOD_TYPE_OPTIMIZED:      return "optimized";
      case VM_METHOD_TYPE_UNDEF:          return "undef";
      case VM_METHOD_TYPE_NOTIMPLEMENTED: return "notimplemented";
    }
    rb_bug("rb_method_type_name: unreachable (type: %d)", type);
}
 
static void
rb_raw_iseq_info(char *const buff, const size_t buff_size, const rb_iseq_t *iseq)
{
    if (buff_size > 0 && ISEQ_BODY(iseq) && ISEQ_BODY(iseq)->location.label && !RB_TYPE_P(ISEQ_BODY(iseq)->location.pathobj, T_MOVED)) {
        VALUE path = rb_iseq_path(iseq);
        int n = ISEQ_BODY(iseq)->location.first_lineno;
        VALUE label = ISEQ_BODY(iseq)->location.label;
        snprintf(buff, buff_size, " %.*s@%.*s:%d",
                 RSTRING_LENINT(label), RSTRING_PTR(label),
                 RSTRING_LENINT(path), RSTRING_PTR(path), n);
    }
}
 
static int
str_len_no_raise(VALUE str)
{
    long len = RSTRING_LEN(str);
    if (len < 0) return 0;
    if (len > INT_MAX) return INT_MAX;
    return (int)len;
}
 
#define BUFF_ARGS buff + pos, buff_size - pos
#define APPEND_F(...) if ((pos += snprintf(BUFF_ARGS, "" __VA_ARGS__)) >= buff_size) goto end
#define APPEND_S(s) do { \
        if ((pos + (int)rb_strlen_lit(s)) >= buff_size) { \
            goto end; \
        } \
        else { \
            memcpy(buff + pos, (s), rb_strlen_lit(s) + 1); \
        } \
    } while (0)
#define C(c, s) ((c) != 0 ? (s) : " ")
 
static size_t
rb_raw_obj_info_common(char *const buff, const size_t buff_size, const VALUE obj)
{
    size_t pos = 0;
 
    if (SPECIAL_CONST_P(obj)) {
        APPEND_F("%s", obj_type_name(obj));
 
        if (FIXNUM_P(obj)) {
            APPEND_F(" %ld", FIX2LONG(obj));
        }
        else if (SYMBOL_P(obj)) {
            APPEND_F(" %s", rb_id2name(SYM2ID(obj)));
        }
    }
    else {
        // const int age = RVALUE_AGE_GET(obj);
 
        if (rb_gc_impl_live_object_p(rb_gc_get_objspace(), (void *)obj)) {
            APPEND_F("%p %s/", (void *)obj, obj_type_name(obj));
            // TODO: fixme
            // APPEND_F("%p [%d%s%s%s%s%s%s] %s ",
            //          (void *)obj, age,
            //          C(RVALUE_UNCOLLECTIBLE_BITMAP(obj),  "L"),
            //          C(RVALUE_MARK_BITMAP(obj),           "M"),
            //          C(RVALUE_PIN_BITMAP(obj),            "P"),
            //          C(RVALUE_MARKING_BITMAP(obj),        "R"),
            //          C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"),
            //          C(rb_objspace_garbage_object_p(obj), "G"),
            //          obj_type_name(obj));
        }
        else {
            /* fake */
            // APPEND_F("%p [%dXXXX] %s",
            //          (void *)obj, age,
            //          obj_type_name(obj));
        }
 
        if (internal_object_p(obj)) {
            /* ignore */
        }
        else if (RBASIC(obj)->klass == 0) {
            APPEND_S("(temporary internal)");
        }
        else if (RTEST(RBASIC(obj)->klass)) {
            VALUE class_path = rb_mod_name(RBASIC(obj)->klass);
            if (!NIL_P(class_path)) {
                APPEND_F("%.*s ", str_len_no_raise(class_path), RSTRING_PTR(class_path));
            }
        }
    }
  end:
 
    return pos;
}
 
const char *rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj);
 
static size_t
rb_raw_obj_info_buitin_type(char *const buff, const size_t buff_size, const VALUE obj, size_t pos)
{
    if (LIKELY(pos < buff_size) && !SPECIAL_CONST_P(obj)) {
        const enum ruby_value_type type = BUILTIN_TYPE(obj);
 
        switch (type) {
          case T_NODE:
            UNEXPECTED_NODE(rb_raw_obj_info);
            break;
          case T_ARRAY:
            if (ARY_SHARED_P(obj)) {
                APPEND_S("shared -> ");
                rb_raw_obj_info(BUFF_ARGS, ARY_SHARED_ROOT(obj));
            }
            else {
                APPEND_F("[%s%s%s] ",
                         C(ARY_EMBED_P(obj), "E"),
                         C(ARY_SHARED_P(obj), "S"),
                         C(ARY_SHARED_ROOT_P(obj), "R"));
 
                if (ARY_EMBED_P(obj)) {
                    APPEND_F("len: %ld (embed)",
                             RARRAY_LEN(obj));
                }
                else {
                    APPEND_F("len: %ld, capa:%ld ptr:%p",
                             RARRAY_LEN(obj),
                             RARRAY(obj)->as.heap.aux.capa,
                             (void *)RARRAY_CONST_PTR(obj));
                }
            }
            break;
          case T_STRING: {
            APPEND_F("[%s%s] ",
                     C(FL_TEST(obj, RSTRING_FSTR), "F"),
                     C(RB_OBJ_FROZEN(obj), "R"));
 
            if (STR_SHARED_P(obj)) {
                APPEND_F(" [shared] len: %ld", RSTRING_LEN(obj));
            }
            else {
                if (STR_EMBED_P(obj)) APPEND_S(" [embed]");
 
                APPEND_F(" len: %ld, capa: %" PRIdSIZE, RSTRING_LEN(obj), rb_str_capacity(obj));
            }
            APPEND_F(" \"%.*s\"", str_len_no_raise(obj), RSTRING_PTR(obj));
            break;
          }
          case T_SYMBOL: {
            VALUE fstr = RSYMBOL(obj)->fstr;
            ID id = RSYMBOL(obj)->id;
            if (RB_TYPE_P(fstr, T_STRING)) {
                APPEND_F(":%.*s id:%d", str_len_no_raise(fstr), RSTRING_PTR(fstr), (unsigned int)id);
            }
            else {
                APPEND_F("(%p) id:%d", (void *)fstr, (unsigned int)id);
            }
            break;
          }
          case T_MOVED: {
            APPEND_F("-> %p", (void*)gc_location_internal(rb_gc_get_objspace(), obj));
            break;
          }
          case T_HASH: {
            APPEND_F("[%c] %"PRIdSIZE,
                     RHASH_AR_TABLE_P(obj) ? 'A' : 'S',
                     RHASH_SIZE(obj));
            break;
          }
          case T_CLASS:
          case T_MODULE:
            {
                VALUE class_path = rb_mod_name(obj);
                if (!NIL_P(class_path)) {
                    APPEND_F("%.*s", str_len_no_raise(class_path), RSTRING_PTR(class_path));
                }
                else {
                    APPEND_S("(anon)");
                }
                break;
            }
          case T_ICLASS:
            {
                VALUE class_path = rb_mod_name(RBASIC_CLASS(obj));
                if (!NIL_P(class_path)) {
                    APPEND_F("src:%.*s", str_len_no_raise(class_path), RSTRING_PTR(class_path));
                }
                break;
            }
          case T_OBJECT:
            {
                shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
                if (rb_shape_embedded_p(shape_id)) {
                    APPEND_F("(embed) len:%d capa:%d", RSHAPE_LEN(shape_id), RSHAPE_CAPACITY(shape_id));
                }
                else {
                    VALUE fields_obj = ROBJECT_FIELDS_OBJ(obj);
                    if (rb_shape_complex_p(shape_id)) {
                        size_t hash_len = rb_st_table_size(rb_imemo_fields_complex_tbl(fields_obj));
                        APPEND_F("(complex) len:%zu extended:%p", hash_len, (void *)fields_obj);
                    }
                    else {
                        APPEND_F("(extended) len:%d capa:%d extended:%p", RSHAPE_LEN(shape_id), RSHAPE_CAPACITY(shape_id), (void *)fields_obj);
                    }
                }
            }
            break;
          case T_DATA: {
            const struct rb_block *block;
            const rb_iseq_t *iseq;
            if (rb_obj_is_proc(obj) &&
                (block = vm_proc_block(obj)) != NULL &&
                (vm_block_type(block) == block_type_iseq) &&
                (iseq = vm_block_iseq(block)) != NULL) {
                rb_raw_iseq_info(BUFF_ARGS, iseq);
            }
            else if (rb_ractor_p(obj)) {
                rb_ractor_t *r = (void *)DATA_PTR(obj);
                if (r) {
                    APPEND_F("r:%"PRI_SERIALT_PREFIX"u", r->pub.id);
                }
            }
            break;
          }
          case T_IMEMO: {
            APPEND_F("<%s> ", rb_imemo_name(imemo_type(obj)));
 
            switch (imemo_type(obj)) {
              case imemo_fields:
                {
                    shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
                    if (rb_shape_complex_p(shape_id)) {
                        size_t hash_len = rb_st_table_size(rb_imemo_fields_complex_tbl(obj));
                        APPEND_F("(complex) len:%zu", hash_len);
                    }
                    else {
                        APPEND_F("(embed) len:%d capa:%d", RSHAPE_LEN(shape_id), RSHAPE_CAPACITY(shape_id));
                    }
 
                    APPEND_S("owner -> ");
                    rb_raw_obj_info(BUFF_ARGS, CLASS_OF(obj));
 
                    break;
                }
              case imemo_ment:
                {
                    const rb_method_entry_t *me = (const rb_method_entry_t *)obj;
 
                    APPEND_F(":%s (%s%s%s%s) type:%s aliased:%d owner:%p defined_class:%p",
                             rb_id2name(me->called_id),
                             METHOD_ENTRY_VISI(me) == METHOD_VISI_PUBLIC ?  "pub" :
                             METHOD_ENTRY_VISI(me) == METHOD_VISI_PRIVATE ? "pri" : "pro",
                             METHOD_ENTRY_COMPLEMENTED(me) ? ",cmp" : "",
                             METHOD_ENTRY_CACHED(me) ? ",cc" : "",
                             METHOD_ENTRY_INVALIDATED(me) ? ",inv" : "",
                             me->def ? rb_method_type_name(me->def->type) : "NULL",
                             me->def ? me->def->aliased : -1,
                             (void *)me->owner, // obj_info(me->owner),
                             (void *)me->defined_class); //obj_info(me->defined_class)));
 
                    if (me->def) {
                        switch (me->def->type) {
                          case VM_METHOD_TYPE_ISEQ:
                            APPEND_S(" (iseq:");
                            rb_raw_obj_info(BUFF_ARGS, (VALUE)me->def->body.iseq.iseqptr);
                            APPEND_S(")");
                            break;
                          default:
                            break;
                        }
                    }
 
                    break;
                }
              case imemo_iseq: {
                const rb_iseq_t *iseq = (const rb_iseq_t *)obj;
                rb_raw_iseq_info(BUFF_ARGS, iseq);
                break;
              }
              case imemo_callinfo:
                {
                    const struct rb_callinfo *ci = (const struct rb_callinfo *)obj;
                    APPEND_F("(mid:%s, flag:%x argc:%d, kwarg:%s)",
                             rb_id2name(vm_ci_mid(ci)),
                             vm_ci_flag(ci),
                             vm_ci_argc(ci),
                             vm_ci_kwarg(ci) ? "available" : "NULL");
                    break;
                }
              case imemo_callcache:
                {
                    const struct rb_callcache *cc = (const struct rb_callcache *)obj;
                    VALUE class_path = vm_cc_valid(cc) ? rb_mod_name(cc->klass) : Qnil;
                    const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
                    const char *class_name;
                    int class_name_len;
 
                    if (NIL_P(class_path)) {
                        class_name = vm_cc_valid(cc) ? "??" : "<NULL>";
                        class_name_len = vm_cc_valid(cc) ? 2 : 6;
                    }
                    else {
                        class_name = RSTRING_PTR(class_path);
                        class_name_len = str_len_no_raise(class_path);
                    }
 
                    APPEND_F("(klass:%.*s cme:%s%s (%p) call:%p",
                             class_name_len, class_name,
                             cme ? rb_id2name(cme->called_id) : "<NULL>",
                             cme ? (METHOD_ENTRY_INVALIDATED(cme) ? " [inv]" : "") : "",
                             (void *)cme,
                             (void *)(uintptr_t)vm_cc_call(cc));
                    break;
                }
              default:
                break;
            }
          }
          default:
            break;
        }
    }
  end:
 
    return pos;
}
 
#undef C
 
#ifdef RUBY_ASAN_ENABLED
void
rb_asan_poison_object(VALUE obj)
{
    MAYBE_UNUSED(struct RVALUE *) ptr = (void *)obj;
    asan_poison_memory_region(ptr, rb_gc_obj_slot_size(obj));
}
 
void
rb_asan_unpoison_object(VALUE obj, bool newobj_p)
{
    MAYBE_UNUSED(struct RVALUE *) ptr = (void *)obj;
    asan_unpoison_memory_region(ptr, rb_gc_obj_slot_size(obj), newobj_p);
}
 
void *
rb_asan_poisoned_object_p(VALUE obj)
{
    MAYBE_UNUSED(struct RVALUE *) ptr = (void *)obj;
    return __asan_region_is_poisoned(ptr, rb_gc_obj_slot_size(obj));
}
#endif
 
static void
raw_obj_info(char *const buff, const size_t buff_size, VALUE obj)
{
    size_t pos = rb_raw_obj_info_common(buff, buff_size, obj);
    pos = rb_raw_obj_info_buitin_type(buff, buff_size, obj, pos);
    if (pos >= buff_size) {} // truncated
}
 
const char *
rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj)
{
    void *objspace = rb_gc_get_objspace();
 
    if (SPECIAL_CONST_P(obj)) {
        raw_obj_info(buff, buff_size, obj);
    }
    else if (!rb_gc_impl_live_object_p(objspace, (const void *)obj)) {
        snprintf(buff, buff_size, "out-of-heap:%p", (void *)obj);
    }
#if 0 // maybe no need to check it?
    else if (0 && rb_gc_impl_garbage_object_p(objspace, obj)) {
        snprintf(buff, buff_size, "garbage:%p", (void *)obj);
    }
#endif
    else {
        asan_unpoisoning_object(obj) {
            raw_obj_info(buff, buff_size, obj);
        }
    }
    return buff;
}
 
#undef APPEND_S
#undef APPEND_F
#undef BUFF_ARGS
 
/* Increments *var atomically and resets *var to 0 when maxval is
 * reached. Returns the wraparound old *var value (0...maxval). */
static rb_atomic_t
atomic_inc_wraparound(rb_atomic_t *var, const rb_atomic_t maxval)
{
    rb_atomic_t oldval = RUBY_ATOMIC_FETCH_ADD(*var, 1);
    if (RB_UNLIKELY(oldval >= maxval - 1)) { // wraparound *var
        const rb_atomic_t newval = oldval + 1;
        RUBY_ATOMIC_CAS(*var, newval, newval % maxval);
        oldval %= maxval;
    }
    return oldval;
}
 
static const char *
obj_info(VALUE obj)
{
    if (RGENGC_OBJ_INFO) {
        static struct {
            rb_atomic_t index;
            char buffers[10][0x100];
        } info = {0};
 
        rb_atomic_t index = atomic_inc_wraparound(&info.index, numberof(info.buffers));
        char *const buff = info.buffers[index];
        return rb_raw_obj_info(buff, sizeof(info.buffers[0]), obj);
    }
    return obj_type_name(obj);
}
 
/*
  ------------------------ Extended allocator ------------------------
*/
 
struct gc_raise_tag {
    VALUE exc;
    const char *fmt;
    va_list *ap;
};
 
static void *
gc_vraise(void *ptr)
{
    struct gc_raise_tag *argv = ptr;
    rb_vraise(argv->exc, argv->fmt, *argv->ap);
    UNREACHABLE_RETURN(NULL);
}
 
static void
gc_raise(VALUE exc, const char *fmt, ...)
{
    va_list ap;
    va_start(ap, fmt);
    struct gc_raise_tag argv = {
        exc, fmt, &ap,
    };
 
    if (ruby_native_thread_p()) {
        rb_thread_call_with_gvl(gc_vraise, &argv);
        UNREACHABLE;
    }
    else {
        /* Not in a ruby thread */
        fprintf(stderr, "%s", "[FATAL] ");
        vfprintf(stderr, fmt, ap);
    }
 
    va_end(ap);
    abort();
}
 
NORETURN(static void negative_size_allocation_error(const char *));
static void
negative_size_allocation_error(const char *msg)
{
    gc_raise(rb_eNoMemError, "%s", msg);
}
 
static void *
ruby_memerror_body(void *dummy)
{
    rb_memerror();
    return 0;
}
 
NORETURN(static void ruby_memerror(void));
RBIMPL_ATTR_MAYBE_UNUSED()
static void
ruby_memerror(void)
{
    if (ruby_thread_has_gvl_p()) {
        rb_memerror();
    }
    else {
        if (ruby_native_thread_p()) {
            rb_thread_call_with_gvl(ruby_memerror_body, 0);
        }
        else {
            /* no ruby thread */
            fprintf(stderr, "[FATAL] failed to allocate memory\n");
        }
    }
 
    /* We have discussions whether we should die here; */
    /* We might rethink about it later. */
    exit(EXIT_FAILURE);
}
 
void
rb_memerror(void)
{
    /* the `GET_VM()->special_exceptions` below assumes that
     * the VM is reachable from the current thread.  We should
     * definitely make sure of that. */
    RUBY_ASSERT_ALWAYS(ruby_thread_has_gvl_p());
 
    rb_execution_context_t *ec = GET_EC();
    VALUE exc = GET_VM()->special_exceptions[ruby_error_nomemory];
 
    if (!exc ||
        rb_ec_raised_p(ec, RAISED_NOMEMORY) ||
        rb_ec_vm_lock_rec(ec) != ec->tag->lock_rec) {
        fprintf(stderr, "[FATAL] failed to allocate memory\n");
        exit(EXIT_FAILURE);
    }
    if (rb_ec_raised_p(ec, RAISED_NOMEMORY)) {
        rb_ec_raised_clear(ec);
    }
    else {
        rb_ec_raised_set(ec, RAISED_NOMEMORY);
        exc = ruby_vm_special_exception_copy(exc);
    }
    ec->errinfo = exc;
    EC_JUMP_TAG(ec, TAG_RAISE);
}
 
bool
rb_memerror_reentered(void)
{
    rb_execution_context_t *ec = GET_EC();
    return (ec && rb_ec_raised_p(ec, RAISED_NOMEMORY));
}
 
static void *
handle_malloc_failure(void *ptr)
{
    if (LIKELY(ptr)) {
        return ptr;
    }
    else {
        ruby_memerror();
        UNREACHABLE_RETURN(ptr);
    }
}
 
static void *ruby_xmalloc_body(size_t size);
 
void *
ruby_xmalloc(size_t size)
{
    if (RUBY_DTRACE_GC_XMALLOC_ENABLED()) {
        RUBY_DTRACE_GC_XMALLOC(1, size);
    }
 
    return handle_malloc_failure(ruby_xmalloc_body(size));
}
 
static bool
malloc_gc_allowed(void)
{
    rb_ractor_t *r = rb_current_ractor_raw(false);
 
    return r == NULL || !r->malloc_gc_disabled;
}
 
static void *
ruby_xmalloc_body(size_t size)
{
    if ((ssize_t)size < 0) {
        negative_size_allocation_error("too large allocation size");
    }
 
    return rb_gc_impl_malloc(rb_gc_get_objspace(), size, malloc_gc_allowed());
}
 
void
ruby_malloc_size_overflow(size_t count, size_t elsize)
{
    rb_raise(rb_eArgError,
             "malloc: possible integer overflow (%"PRIuSIZE"*%"PRIuSIZE")",
             count, elsize);
}
 
void
ruby_malloc_add_size_overflow(size_t x, size_t y)
{
    rb_raise(rb_eArgError,
             "malloc: possible integer overflow (%"PRIuSIZE"+%"PRIuSIZE")",
             x, y);
}
 
static void *ruby_xmalloc2_body(size_t n, size_t size);
 
void *
ruby_xmalloc2(size_t n, size_t size)
{
    if (RUBY_DTRACE_GC_XMALLOC_ENABLED()) {
        RUBY_DTRACE_GC_XMALLOC(n, size);
    }
 
    return handle_malloc_failure(ruby_xmalloc2_body(n, size));
}
 
static void *
ruby_xmalloc2_body(size_t n, size_t size)
{
    return rb_gc_impl_malloc(rb_gc_get_objspace(), xmalloc2_size(n, size), malloc_gc_allowed());
}
 
static void *ruby_xcalloc_body(size_t n, size_t size);
 
void *
ruby_xcalloc(size_t n, size_t size)
{
    if (RUBY_DTRACE_GC_XCALLOC_ENABLED()) {
        RUBY_DTRACE_GC_XCALLOC(n, size);
    }
 
    return handle_malloc_failure(ruby_xcalloc_body(n, size));
}
 
static void *
ruby_xcalloc_body(size_t n, size_t size)
{
    return rb_gc_impl_calloc(rb_gc_get_objspace(), xmalloc2_size(n, size), malloc_gc_allowed());
}
 
static void *ruby_xrealloc_sized_body(void *ptr, size_t new_size, size_t old_size);
 
#ifdef ruby_xrealloc_sized
#undef ruby_xrealloc_sized
#endif
void *
ruby_xrealloc_sized(void *ptr, size_t new_size, size_t old_size)
{
    return handle_malloc_failure(ruby_xrealloc_sized_body(ptr, new_size, old_size));
}
 
static void *
ruby_xrealloc_sized_body(void *ptr, size_t new_size, size_t old_size)
{
    if ((ssize_t)new_size < 0) {
        negative_size_allocation_error("too large allocation size");
    }
 
    return rb_gc_impl_realloc(rb_gc_get_objspace(), ptr, new_size, old_size, malloc_gc_allowed());
}
 
void *
ruby_xrealloc(void *ptr, size_t new_size)
{
    return ruby_xrealloc_sized(ptr, new_size, 0);
}
 
static void *ruby_xrealloc2_sized_body(void *ptr, size_t n, size_t size, size_t old_n);
 
#ifdef ruby_xrealloc2_sized
#undef ruby_xrealloc2_sized
#endif
void *
ruby_xrealloc2_sized(void *ptr, size_t n, size_t size, size_t old_n)
{
    return handle_malloc_failure(ruby_xrealloc2_sized_body(ptr, n, size, old_n));
}
 
static void *
ruby_xrealloc2_sized_body(void *ptr, size_t n, size_t size, size_t old_n)
{
    size_t len = xmalloc2_size(n, size);
    return rb_gc_impl_realloc(rb_gc_get_objspace(), ptr, len, old_n * size, malloc_gc_allowed());
}
 
void *
ruby_xrealloc2(void *ptr, size_t n, size_t size)
{
    return ruby_xrealloc2_sized(ptr, n, size, 0);
}
 
#ifdef ruby_xfree_sized
#undef ruby_xfree_sized
#endif
 
/*
 * This is a debugging flag for measuring the cost of `xfree`.
 * It can be enabled at compile time using `-DRUBY_NO_FREE`.
 * At run time, if the `RUBY_NO_FREE` environment variable is set to "1",
 * then `xfree` will not free any memory.
 */
#ifdef RUBY_NO_FREE
static bool g_nofree = false;
#endif
 
void
ruby_xfree_sized(void *x, size_t size)
{
#ifdef RUBY_NO_FREE
    if (g_nofree) {
        return;
    }
#endif
 
    if (RUBY_DTRACE_GC_XFREE_ENABLED()) {
        RUBY_DTRACE_GC_XFREE(x, size);
    }
 
    if (LIKELY(x)) {
        /* It's possible for a C extension's pthread destructor function set by pthread_key_create
         * to be called after ruby_vm_destruct and attempt to free memory. Fall back to mimfree in
         * that case. */
        if (LIKELY(GET_VM())) {
            rb_gc_impl_free(rb_gc_get_objspace(), x, size);
        }
        else {
            ruby_mimfree(x);
        }
    }
}
 
void
ruby_xfree(void *x)
{
    ruby_xfree_sized(x, 0);
}
 
void *
rb_xmalloc_mul_add(size_t x, size_t y, size_t z) /* x * y + z */
{
    size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
    return ruby_xmalloc(w);
}
 
void *
rb_xcalloc_mul_add(size_t x, size_t y, size_t z) /* x * y + z */
{
    size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
    return ruby_xcalloc(w, 1);
}
 
void *
rb_xrealloc_mul_add(const void *p, size_t x, size_t y, size_t z) /* x * y + z */
{
    size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
    return ruby_xrealloc((void *)p, w);
}
 
void *
rb_xmalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
{
    size_t u = size_mul_add_mul_or_raise(x, y, z, w, rb_eArgError);
    return ruby_xmalloc(u);
}
 
void *
rb_xcalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
{
    size_t u = size_mul_add_mul_or_raise(x, y, z, w, rb_eArgError);
    return ruby_xcalloc(u, 1);
}
 
/* Mimic ruby_xmalloc, but need not rb_objspace.
 * should return pointer suitable for ruby_xfree
 */
void *
ruby_mimmalloc(size_t size)
{
    void *mem;
#if CALC_EXACT_MALLOC_SIZE
    size += sizeof(struct malloc_obj_info);
#endif
    mem = malloc(size);
#if CALC_EXACT_MALLOC_SIZE
    if (!mem) {
        return NULL;
    }
    else
    /* set 0 for consistency of allocated_size/allocations */
    {
        struct malloc_obj_info *info = mem;
        info->size = 0;
        mem = info + 1;
    }
#endif
    return mem;
}
 
void *
ruby_mimcalloc(size_t num, size_t size)
{
    void *mem;
#if CALC_EXACT_MALLOC_SIZE
    struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(num, size);
    if (UNLIKELY(t.overflowed)) {
        return NULL;
    }
    size = t.result + sizeof(struct malloc_obj_info);
    mem = calloc1(size);
    if (!mem) {
        return NULL;
    }
    else
    /* set 0 for consistency of allocated_size/allocations */
    {
        struct malloc_obj_info *info = mem;
        info->size = 0;
        mem = info + 1;
    }
#else
    mem = calloc(num, size);
#endif
    return mem;
}
 
void
ruby_mimfree(void *ptr)
{
#if CALC_EXACT_MALLOC_SIZE
    struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
    ptr = info;
#endif
    free(ptr);
}
 
void
rb_gc_adjust_memory_usage(ssize_t diff)
{
    unless_objspace(objspace) { return; }
 
    rb_gc_impl_adjust_memory_usage(objspace, diff);
}
 
const char *
rb_obj_info(VALUE obj)
{
    return obj_info(obj);
}
 
void
rb_obj_info_dump(VALUE obj)
{
    char buff[0x100];
    fprintf(stderr, "rb_obj_info_dump: %s\n", rb_raw_obj_info(buff, 0x100, obj));
}
 
void
rb_obj_info_dump_loc(VALUE obj, const char *file, int line, const char *func)
{
    char buff[0x100];
    fprintf(stderr, "<OBJ_INFO:%s@%s:%d> %s\n", func, file, line, rb_raw_obj_info(buff, 0x100, obj));
}
 
void
rb_gc_before_fork(void)
{
    rb_gc_impl_before_fork(rb_gc_get_objspace());
}
 
void
rb_gc_after_fork(rb_pid_t pid)
{
    rb_gc_impl_after_fork(rb_gc_get_objspace(), pid);
}
 
bool
rb_gc_obj_shareable_p(VALUE obj)
{
    return RB_OBJ_SHAREABLE_P(obj);
}
 
void
rb_gc_rp(VALUE obj)
{
    rp(obj);
}
 
struct check_shareable_data {
    VALUE parent;
    long err_count;
};
 
static void
check_shareable_i(const VALUE child, void *ptr)
{
    struct check_shareable_data *data = (struct check_shareable_data *)ptr;
 
    if (!rb_gc_obj_shareable_p(child)) {
        /* A shareable object may reference an unshareable one only if the write barrier
         * recorded the edge in the target's shref bit (keeping it alive past its owner's
         * local GC).  Root-like exceptions (Ractor private fields, cref, JIT) are hidden
         * while checking_shareable is set. */
        if (rb_gc_impl_shref_marked_p(rb_gc_get_objspace(), child)) {
            return;
        }
 
        fprintf(stderr, "(a) ");
        rb_gc_rp(data->parent);
        fprintf(stderr, "(b) ");
        rb_gc_rp(child);
        fprintf(stderr, "check_shareable_i: shareable (a) -> unshareable (b) without a shref record\n");
 
        data->err_count++;
        rb_bug("!! violate shareable constraint !!");
    }
}
 
/* List obj's direct children one level deep through the traversal API and check the
 * shareable constraint: a shareable object's child is either shareable or an
 * unshareable one with a recorded shref.  The "verification walk in progress" marker
 * lives in the per-Ractor mark_func_data slot: a process-global flag would make the
 * lock-free local GC of an unrelated Ractor hit the mark gate too, skip marking a live
 * object's children (its fields imemo, say) and let the sweep collect them.  (Upstream
 * could use a global flag, since its GC always runs under the VM lock.)  The slot is
 * private to this Ractor and the walk is synchronous, so no lock is needed. */
void
rb_gc_verify_shareable(VALUE obj)
{
    struct check_shareable_data data = {
        .parent = obj,
        .err_count = 0,
    };
 
    if (!RB_SPECIAL_CONST_P(obj)) {
        struct gc_mark_func_data_struct **mfdp = GC_MARK_FUNC_DATA_SLOTP();
        struct gc_mark_func_data_struct *prev_mfd = *mfdp;
        struct gc_mark_func_data_struct mfd = {
            .mark_func = check_shareable_i,
            .data = &data,
            .checking_shareable = true,
        };
 
        *mfdp = &mfd;
        rb_gc_mark_children(rb_gc_get_objspace(), obj);
        *mfdp = prev_mfd;
    }
 
    if (data.err_count > 0) {
        rb_bug("rb_gc_verify_shareable");
    }
}
 
bool
rb_gc_checking_shareable(void)
{
    const struct gc_mark_func_data_struct *mfd = *GC_MARK_FUNC_DATA_SLOTP();
    return mfd && mfd->checking_shareable;
}
 
/*
 * Document-module: ObjectSpace
 *
 *  The ObjectSpace module contains a number of routines
 *  that interact with the garbage collection facility and allow you to
 *  traverse all living objects with an iterator.
 *
 *  ObjectSpace also provides support for object finalizers, procs that will be
 *  called after a specific object was destroyed by garbage collection.  See
 *  the documentation for +ObjectSpace.define_finalizer+ for important
 *  information on how to use this method correctly.
 *
 *     a = "A"
 *     b = "B"
 *
 *     ObjectSpace.define_finalizer(a, proc {|id| puts "Finalizer one on #{id}" })
 *     ObjectSpace.define_finalizer(b, proc {|id| puts "Finalizer two on #{id}" })
 *
 *     a = nil
 *     b = nil
 *
 *  _produces:_
 *
 *     Finalizer two on 537763470
 *     Finalizer one on 537763480
 */
 
#include "gc.rbinc"
 
void
Init_GC(void)
{
#ifdef RUBY_NO_FREE
    const char* nofree_str = getenv("RUBY_NO_FREE");
    if (nofree_str && strcmp(nofree_str, "1") == 0) {
        fprintf(stderr, "WARNING: Enabling no-free mode! xfree() will never free anything!\n");
        g_nofree = true;
    }
#endif
 
#undef rb_intern
    malloc_offset = gc_compute_malloc_offset();
 
    rb_mGC = rb_define_module("GC");
 
    VALUE rb_mObjSpace = rb_define_module("ObjectSpace");
 
    rb_define_module_function(rb_mObjSpace, "each_object", os_each_obj, -1);
 
    rb_define_module_function(rb_mObjSpace, "define_finalizer", define_final, -1);
    rb_define_module_function(rb_mObjSpace, "undefine_finalizer", undefine_final, 1);
 
    rb_vm_register_special_exception(ruby_error_nomemory, rb_eNoMemError, "failed to allocate memory");
 
    rb_define_method(rb_cBasicObject, "__id__", rb_obj_id, 0);
    rb_define_method(rb_mKernel, "object_id", rb_obj_id, 0);
 
    rb_define_module_function(rb_mObjSpace, "count_objects", count_objects, -1);
 
    rb_gc_impl_init();
}
 
// Set a name for the anonymous virtual memory area. `addr` is the starting
// address of the area and `size` is its length in bytes. `name` is a
// NUL-terminated human-readable string.
//
// This function is usually called after calling `mmap()`.  The human-readable
// annotation helps developers identify the call site of `mmap()` that created
// the memory mapping.
//
// This function currently only works on Linux 5.17 or higher.  After calling
// this function, we can see annotations in the form of "[anon:...]" in
// `/proc/self/maps`, where `...` is the content of `name`.  This function has
// no effect when called on other platforms.
void
ruby_annotate_mmap(const void *addr, unsigned long size, const char *name)
{
#if defined(HAVE_SYS_PRCTL_H) && defined(PR_SET_VMA) && defined(PR_SET_VMA_ANON_NAME)
    // The name length cannot exceed 80 (including the '\0').
    RUBY_ASSERT(strlen(name) < 80);
    prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (unsigned long)addr, size, name);
    // We ignore errors in prctl. prctl may set errno to EINVAL for several
    // reasons.
    // 1. The attr (PR_SET_VMA_ANON_NAME) is not a valid attribute.
    // 2. addr is an invalid address.
    // 3. The string pointed by name is too long.
    // The first error indicates PR_SET_VMA_ANON_NAME is not available, and may
    // happen if we run the compiled binary on an old kernel.  In theory, all
    // other errors should result in a failure.  But since EINVAL cannot tell
    // the first error from others, and this function is mainly used for
    // debugging, we silently ignore the error.
    errno = 0;
#endif
}