default.c

#include "ruby/internal/config.h"
 
#include <signal.h>
#include <string.h>
 
#ifndef _WIN32
# include <sys/mman.h>
# include <unistd.h>
# include <fcntl.h>
# ifdef HAVE_SYS_PRCTL_H
#  include <sys/prctl.h>
# endif
#endif
 
#if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H)
/* LIST_HEAD conflicts with sys/queue.h on macOS */
# include <sys/user.h>
#endif
 
#ifdef BUILDING_MODULAR_GC
# define nlz_int64(x) (x == 0 ? 64 : (unsigned int)__builtin_clzll((unsigned long long)x))
# define rb_popcount_intptr(x) ((unsigned int)__builtin_popcountll((unsigned long long)(x)))
#else
# include "internal/bits.h"
#endif
 
#include "ruby/ruby.h"
#include "ruby/atomic.h"
#include "ruby_atomic.h"
#include "ruby/debug.h"
#include "ruby/thread.h"
#include "ruby/util.h"
#include "ruby/vm.h"
#include "ruby/internal/encoding/string.h"
#include "ccan/list/list.h"
#include "darray.h"
#include "gc/gc.h"
#include "gc/gc_impl.h"
#include "yjit.h"
#include "zjit.h"
#include "internal/static_assert.h"
#include "internal/vm_map.h"
 
#ifdef BUILDING_MODULAR_GC
/* hrtime.h transitively includes internal/time.h -> internal/bits.h, which are
 * not available to out-of-tree modular GC builds.  We only use a monotonic
 * clock plus saturating add/sub, so provide that subset locally with the same
 * semantics as hrtime.h. */
# include <time.h>
typedef uint64_t rb_hrtime_t;
# define RB_HRTIME_PER_SEC ((rb_hrtime_t)1000000000)
 
static inline rb_hrtime_t
rb_hrtime_now(void)
{
# if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC)
    struct timespec ts;
    if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) {
        return (rb_hrtime_t)ts.tv_sec * RB_HRTIME_PER_SEC + (rb_hrtime_t)ts.tv_nsec;
    }
# endif
    return 0;
}
 
static inline rb_hrtime_t
rb_hrtime_add(rb_hrtime_t a, rb_hrtime_t b)
{
    rb_hrtime_t c = a + b;
    return c < a ? UINT64_MAX : c; /* saturate on overflow */
}
 
static inline rb_hrtime_t
rb_hrtime_sub(rb_hrtime_t a, rb_hrtime_t b)
{
    return a < b ? 0 : a - b;
}
#else
# include "hrtime.h"
#endif
 
#include "probes.h"
 
/* cl.exe's traditional preprocessor passes __VA_ARGS__ to a nested macro as
 * a single argument; the extra expansion re-scans it into separate ones. */
#define RUBY_DTRACE_GC_HOOK_EXPAND(expr) expr
#define RUBY_DTRACE_GC_HOOK(name, ...) \
    do {if (RUBY_DTRACE_GC_##name##_ENABLED()) RUBY_DTRACE_GC_HOOK_EXPAND(RUBY_DTRACE_GC_##name(__VA_ARGS__));} while (0)
 
#if USE_ZJIT
# include "gc/default/zjit_fastpath.h"
#endif
 
#ifdef BUILDING_MODULAR_GC
# define RB_DEBUG_COUNTER_INC(_name) ((void)0)
# define RB_DEBUG_COUNTER_INC_IF(_name, cond) (!!(cond))
#else
# include "debug_counter.h"
#endif
 
#ifdef BUILDING_MODULAR_GC
# define rb_asan_poison_object(obj) ((void)(obj))
# define rb_asan_unpoison_object(obj, newobj_p) ((void)(obj), (void)(newobj_p))
# define asan_unpoisoning_object(obj) if ((obj) || true)
# define asan_poison_memory_region(ptr, size) ((void)(ptr), (void)(size))
# define asan_unpoison_memory_region(ptr, size, malloc_p) ((void)(ptr), (size), (malloc_p))
# define asan_unpoisoning_memory_region(ptr, size) if ((ptr) || (size) || true)
 
# define VALGRIND_MAKE_MEM_DEFINED(ptr, size) ((void)(ptr), (void)(size))
# define VALGRIND_MAKE_MEM_UNDEFINED(ptr, size) ((void)(ptr), (void)(size))
#else
# include "internal/sanitizers.h"
#endif
 
/* MALLOC_HEADERS_BEGIN */
#ifndef HAVE_MALLOC_USABLE_SIZE
# ifdef _WIN32
#  define HAVE_MALLOC_USABLE_SIZE
#  define malloc_usable_size(a) _msize(a)
# elif defined HAVE_MALLOC_SIZE
#  define HAVE_MALLOC_USABLE_SIZE
#  define malloc_usable_size(a) malloc_size(a)
# endif
#endif
 
#ifdef HAVE_MALLOC_USABLE_SIZE
# ifdef RUBY_ALTERNATIVE_MALLOC_HEADER
/* Alternative malloc header is included in ruby/missing.h */
# elif defined(HAVE_MALLOC_H)
#  include <malloc.h>
# elif defined(HAVE_MALLOC_NP_H)
#  include <malloc_np.h>
# elif defined(HAVE_MALLOC_MALLOC_H)
#  include <malloc/malloc.h>
# endif
#endif
 
#ifdef HAVE_MALLOC_TRIM
# include <malloc.h>
 
# ifdef __EMSCRIPTEN__
/* malloc_trim is defined in emscripten/emmalloc.h on emscripten. */
#  include <emscripten/emmalloc.h>
# endif
#endif
 
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
# include <mach/task.h>
# include <mach/mach_init.h>
# include <mach/mach_port.h>
#endif
 
#ifndef RUBY_DEBUG_LOG
# define RUBY_DEBUG_LOG(...)
#endif
 
#ifndef GC_HEAP_INIT_BYTES
#define GC_HEAP_INIT_BYTES (2560 * 1024)
#endif
#ifndef GC_HEAP_FREE_SLOTS
#define GC_HEAP_FREE_SLOTS  4096
#endif
#ifndef GC_RACTOR_HEAP_INIT_BYTES
/* 0 is resolved at boot to the smallest size that works. */
#define GC_RACTOR_HEAP_INIT_BYTES 0
#endif
#ifndef GC_HEAP_GROWTH_FACTOR
#define GC_HEAP_GROWTH_FACTOR 1.8
#endif
#ifndef GC_HEAP_GROWTH_MAX_BYTES
#define GC_HEAP_GROWTH_MAX_BYTES 0 /* 0 is disable */
#endif
#ifndef GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO
# define GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO 0.01
#endif
#ifndef GC_HEAP_OLDOBJECT_LIMIT_FACTOR
#define GC_HEAP_OLDOBJECT_LIMIT_FACTOR 2.0
#endif
 
#ifndef GC_HEAP_FREE_SLOTS_MIN_RATIO
#define GC_HEAP_FREE_SLOTS_MIN_RATIO  0.20
#endif
#ifndef GC_HEAP_FREE_SLOTS_GOAL_RATIO
#define GC_HEAP_FREE_SLOTS_GOAL_RATIO 0.40
#endif
#ifndef GC_HEAP_FREE_SLOTS_MAX_RATIO
#define GC_HEAP_FREE_SLOTS_MAX_RATIO  0.65
#endif
 
#ifndef GC_MALLOC_LIMIT_MIN
#define GC_MALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
#endif
#ifndef GC_MALLOC_LIMIT_MAX
#define GC_MALLOC_LIMIT_MAX (32 * 1024 * 1024 /* 32MB */)
#endif
#ifndef GC_MALLOC_LIMIT_GROWTH_FACTOR
#define GC_MALLOC_LIMIT_GROWTH_FACTOR 1.4
#endif
 
#ifndef GC_OLDMALLOC_LIMIT_MIN
#define GC_OLDMALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
#endif
#ifndef GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
#define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR 1.2
#endif
#ifndef GC_OLDMALLOC_LIMIT_MAX
#define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 /* 128MB */)
#endif
 
#ifndef GC_MALLOC_INCREASE_LOCAL_THRESHOLD
#define GC_MALLOC_INCREASE_LOCAL_THRESHOLD (8 * 1024 /* 8KB */)
#endif
 
#ifdef RB_THREAD_LOCAL_SPECIFIER
#define USE_MALLOC_INCREASE_LOCAL 1
static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local;
#else
#define USE_MALLOC_INCREASE_LOCAL 0
#endif
 
#ifndef GC_CAN_COMPILE_COMPACTION
#if defined(__wasi__) /* WebAssembly doesn't support signals */
# define GC_CAN_COMPILE_COMPACTION 0
#else
# define GC_CAN_COMPILE_COMPACTION 1
#endif
#endif
 
#ifndef PRINT_ENTER_EXIT_TICK
# define PRINT_ENTER_EXIT_TICK 0
#endif
#ifndef PRINT_ROOT_TICKS
#define PRINT_ROOT_TICKS 0
#endif
 
#define USE_TICK_T                 (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS)
 
#ifndef HEAP_COUNT
# if SIZEOF_VALUE >= 8
#  define HEAP_COUNT 12
# else
#  define HEAP_COUNT 5
# endif
#endif
 
/* The reciprocal table and pool_slot_sizes array are both generated from this
 * single definition, so they can never get out of sync. */
#if SIZEOF_VALUE >= 8
# define EACH_POOL_SLOT_SIZE(SLOT) \
    SLOT(32) SLOT(40) SLOT(64) SLOT(80) SLOT(96) SLOT(128) \
    SLOT(160) SLOT(256) SLOT(512) SLOT(640) SLOT(768) SLOT(1024)
#else
# define EACH_POOL_SLOT_SIZE(SLOT) \
    SLOT(32) SLOT(64) SLOT(128) SLOT(256) SLOT(512)
#endif
 
typedef struct {
    size_t heap_init_bytes;
    size_t ractor_heap_init_bytes;
    size_t heap_free_slots;
    double growth_factor;
    size_t growth_max_bytes;
 
    double heap_free_slots_min_ratio;
    double heap_free_slots_goal_ratio;
    double heap_free_slots_max_ratio;
    double uncollectible_wb_unprotected_objects_limit_ratio;
    double oldobject_limit_factor;
 
    size_t malloc_limit_min;
    size_t malloc_limit_max;
    double malloc_limit_growth_factor;
 
    size_t oldmalloc_limit_min;
    size_t oldmalloc_limit_max;
    double oldmalloc_limit_growth_factor;
} ruby_gc_params_t;
 
static ruby_gc_params_t gc_params = {
    GC_HEAP_INIT_BYTES,
    GC_RACTOR_HEAP_INIT_BYTES,
    GC_HEAP_FREE_SLOTS,
    GC_HEAP_GROWTH_FACTOR,
    GC_HEAP_GROWTH_MAX_BYTES,
 
    GC_HEAP_FREE_SLOTS_MIN_RATIO,
    GC_HEAP_FREE_SLOTS_GOAL_RATIO,
    GC_HEAP_FREE_SLOTS_MAX_RATIO,
    GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO,
    GC_HEAP_OLDOBJECT_LIMIT_FACTOR,
 
    GC_MALLOC_LIMIT_MIN,
    GC_MALLOC_LIMIT_MAX,
    GC_MALLOC_LIMIT_GROWTH_FACTOR,
 
    GC_OLDMALLOC_LIMIT_MIN,
    GC_OLDMALLOC_LIMIT_MAX,
    GC_OLDMALLOC_LIMIT_GROWTH_FACTOR,
};
 
/* GC_DEBUG:
 *  enable to embed GC debugging information.
 */
#ifndef GC_DEBUG
#define GC_DEBUG 0
#endif
 
/* RGENGC_DEBUG:
 * 1: basic information
 * 2: remember set operation
 * 3: mark
 * 4:
 * 5: sweep
 */
#ifndef RGENGC_DEBUG
#ifdef RUBY_DEVEL
#define RGENGC_DEBUG       -1
#else
#define RGENGC_DEBUG       0
#endif
#endif
#if RGENGC_DEBUG < 0 && !defined(_MSC_VER)
# define RGENGC_DEBUG_ENABLED(level) (-(RGENGC_DEBUG) >= (level) && ruby_rgengc_debug >= (level))
#else
# define RGENGC_DEBUG_ENABLED(level) ((RGENGC_DEBUG) >= (level))
#endif
int ruby_rgengc_debug;
 
/* RGENGC_PROFILE
 * 0: disable RGenGC profiling
 * 1: enable profiling for basic information
 * 2: enable profiling for each types
 */
#ifndef RGENGC_PROFILE
# define RGENGC_PROFILE     0
#endif
 
/* RGENGC_ESTIMATE_OLDMALLOC
 * Enable/disable to estimate increase size of malloc'ed size by old objects.
 * If estimation exceeds threshold, then will invoke full GC.
 * 0: disable estimation.
 * 1: enable estimation.
 */
#ifndef RGENGC_ESTIMATE_OLDMALLOC
# define RGENGC_ESTIMATE_OLDMALLOC 1
#endif
 
#ifndef GC_PROFILE_MORE_DETAIL
# define GC_PROFILE_MORE_DETAIL 0
#endif
#ifndef GC_PROFILE_DETAIL_MEMORY
# define GC_PROFILE_DETAIL_MEMORY 0
#endif
#ifndef GC_ENABLE_LAZY_SWEEP
# define GC_ENABLE_LAZY_SWEEP   1
#endif
 
#ifndef VERIFY_FREE_SIZE
#if RUBY_DEBUG
#define VERIFY_FREE_SIZE 1
#else
#define VERIFY_FREE_SIZE 0
#endif
#endif
 
#if VERIFY_FREE_SIZE
#undef CALC_EXACT_MALLOC_SIZE
#define CALC_EXACT_MALLOC_SIZE 1
#endif
 
#ifndef CALC_EXACT_MALLOC_SIZE
# define CALC_EXACT_MALLOC_SIZE 0
#endif
 
#if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0
# ifndef MALLOC_ALLOCATED_SIZE
#  define MALLOC_ALLOCATED_SIZE 0
# endif
#else
# define MALLOC_ALLOCATED_SIZE 0
#endif
#ifndef MALLOC_ALLOCATED_SIZE_CHECK
# define MALLOC_ALLOCATED_SIZE_CHECK 0
#endif
 
#ifndef GC_DEBUG_STRESS_TO_CLASS
# define GC_DEBUG_STRESS_TO_CLASS RUBY_DEBUG
#endif
 
typedef enum {
    GPR_FLAG_NONE               = 0x000,
    /* major reason */
    GPR_FLAG_MAJOR_BY_NOFREE    = 0x001,
    GPR_FLAG_MAJOR_BY_OLDGEN    = 0x002,
    GPR_FLAG_MAJOR_BY_SHADY     = 0x004,
    GPR_FLAG_MAJOR_BY_FORCE     = 0x008,
#if RGENGC_ESTIMATE_OLDMALLOC
    GPR_FLAG_MAJOR_BY_OLDMALLOC = 0x020,
#endif
    GPR_FLAG_MAJOR_MASK         = 0x0ff,
 
    /* gc reason */
    GPR_FLAG_NEWOBJ             = 0x100,
    GPR_FLAG_MALLOC             = 0x200,
    GPR_FLAG_METHOD             = 0x400,
    GPR_FLAG_CAPI               = 0x800,
    GPR_FLAG_STRESS            = 0x1000,
 
    /* others */
    GPR_FLAG_IMMEDIATE_SWEEP   = 0x2000,
    GPR_FLAG_HAVE_FINALIZE     = 0x4000,
    GPR_FLAG_IMMEDIATE_MARK    = 0x8000,
    GPR_FLAG_FULL_MARK        = 0x10000,
    GPR_FLAG_COMPACT          = 0x20000,
    GPR_FLAG_GLOBAL           = 0x40000,
 
    GPR_DEFAULT_REASON =
        (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK |
         GPR_FLAG_IMMEDIATE_SWEEP | GPR_FLAG_CAPI),
} gc_profile_record_flag;
 
typedef struct gc_profile_record {
    unsigned int flags;
    size_t sequence;
 
    double gc_time;
    double gc_invoke_time;
    rb_hrtime_t gc_wall_time;
    rb_hrtime_t gc_invoke_wall_time;
    rb_hrtime_t gc_pause_time;
    rb_hrtime_t gc_stop_time;
    rb_hrtime_t gc_stw_time;
    rb_hrtime_t gc_mark_wall_time;
    rb_hrtime_t gc_sweep_wall_time;
    rb_hrtime_t gc_compact_wall_time;
 
    size_t heap_total_objects;
    size_t heap_use_size;
    size_t heap_total_size;
    size_t moved_objects;
 
#if GC_PROFILE_MORE_DETAIL
    double gc_mark_time;
    double gc_sweep_time;
 
    size_t heap_use_pages;
    size_t heap_live_objects;
    size_t heap_free_objects;
 
    size_t allocate_increase;
    size_t allocate_limit;
 
    double prepare_time;
    size_t removing_objects;
    size_t empty_objects;
#if GC_PROFILE_DETAIL_MEMORY
    long maxrss;
    long minflt;
    long majflt;
#endif
#endif
#if MALLOC_ALLOCATED_SIZE
    size_t allocated_size;
#endif
 
#if RGENGC_PROFILE > 0
    size_t old_objects;
    size_t remembered_normal_objects;
    size_t remembered_shady_objects;
#endif
} gc_profile_record;
 
struct RMoved {
    VALUE flags;
    VALUE dummy;
    VALUE destination;
};
 
#define RMOVED(obj) ((struct RMoved *)(obj))
 
typedef uintptr_t bits_t;
enum {
    BITS_SIZE = sizeof(bits_t),
    BITS_BITLENGTH = ( BITS_SIZE * CHAR_BIT )
};
 
struct heap_page_header {
    struct heap_page *page;
};
 
struct heap_page_body {
    struct heap_page_header header;
    /* char gap[];      */
    /* RVALUE values[]; */
};
 
#define STACK_CHUNK_SIZE 500
 
typedef struct stack_chunk {
    VALUE data[STACK_CHUNK_SIZE];
    struct stack_chunk *next;
} stack_chunk_t;
 
typedef struct mark_stack {
    stack_chunk_t *chunk;
    stack_chunk_t *cache;
    int index;
    int limit;
    size_t cache_size;
    size_t unused_cache_size;
} mark_stack_t;
 
typedef int (*gc_compact_compare_func)(const void *l, const void *r, void *d);
 
typedef struct rb_heap_newobj {
    uintptr_t alloc_cursor;
    uintptr_t alloc_cursor_end;
    struct free_region *alloc_next_region;
    struct heap_page *alloc_using_page;
} rb_heap_newobj_t;
 
typedef struct rb_heap_struct {
    short slot_size;
 
    /* Basic statistics */
    size_t total_allocated_pages;
    size_t force_major_gc_count;
    size_t force_incremental_marking_finish_count;
    size_t total_allocated_objects;
    size_t total_freed_objects;
    size_t final_slots_count;
 
    /* Sweeping statistics */
    size_t freed_slots;
    size_t empty_slots;
 
    /* Bump-pointer allocation state; only this objspace's owner thread writes it. */
    rb_heap_newobj_t newobj;
 
    struct heap_page *free_pages;
    struct ccan_list_head pages;
    struct heap_page *sweeping_page; /* iterator for .pages */
    struct heap_page *compact_cursor;
    uintptr_t compact_cursor_index;
    struct heap_page *pooled_pages;
    size_t total_pages;      /* total page count in a heap */
    size_t total_slots;      /* total slot count */
 
} rb_heap_t;
 
enum {
    gc_stress_no_major,
    gc_stress_no_immediate_sweep,
    gc_stress_full_mark_after_malloc,
    gc_stress_max
};
 
enum gc_mode {
    gc_mode_none,
    gc_mode_marking,
    gc_mode_sweeping,
    gc_mode_compacting,
};
 
typedef rbimpl_atomic_uint64_t gc_counter_t;
 
#if !defined(HAVE_GCC_ATOMIC_BUILTINS_64) && !defined(_WIN32) && \
    !(defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx)))
# define MALLOC_COUNTERS_NEED_LOCK 1
#endif
 
struct gc_malloc_bytes {
    gc_counter_t malloc;
    gc_counter_t free;
 
    /* Baselines the increase is measured from: malloc snapshotted at GC start
     * (gc_reset_malloc_info), free re-snapshotted at gc_sweep_finish so the
     * sweep's own frees never count. */
    gc_counter_t malloc_at_last_gc;
    gc_counter_t free_at_last_gc;
};
 
/* -- Deferred free of non-thread-safe T_DATA --
 *
 * A dead T_DATA not RUBY_TYPED_THREAD_SAFE_FREE cannot have its dfree run during a parallel
 * local sweep: the dfree is extension code that may touch process state other Ractors are using,
 * so it needs the world stopped (not merely serialization against other dfrees). The sweep
 * therefore copies out what the free needs, reclaims the slot immediately, and the dfrees are
 * called later under a VM barrier. Such a type is never embedded, so the payload always outlives
 * the slot.
 */
#define TDATA_UNSAFE_FREE_CHUNK_CAPA 32
/* Drained chunks kept for reuse; the rest are freed. */
#define TDATA_UNSAFE_FREE_CACHE_MAX 64
 
struct tdata_unsafe_free_entry {
    void (*dfree)(void *);
    void *data;
};
 
struct tdata_unsafe_free_chunk {
    struct tdata_unsafe_free_chunk *next;
    unsigned int count;
    uint32_t embed_xfree_bits;
    struct tdata_unsafe_free_entry entries[TDATA_UNSAFE_FREE_CHUNK_CAPA];
};
STATIC_ASSERT(tdata_unsafe_free_bits_cover_chunk,
              TDATA_UNSAFE_FREE_CHUNK_CAPA <= 32);
 
struct gc_process_stat_snapshot {
    uint64_t minor_gc_count;
    uint64_t major_gc_count;
    uint64_t global_gc_invocation_count;
    uint64_t marking_time_ns;
    uint64_t sweeping_time_ns;
};
 
struct gc_process_stat_total {
    uint64_t minor_gc_count;
    uint64_t major_gc_count;
    uint64_t global_gc_invocation_count;
    uint64_t marking_time_ns;
    uint64_t sweeping_time_ns;
};
 
typedef struct rb_objspace {
    struct {
        struct gc_malloc_bytes counters;
#if RGENGC_ESTIMATE_OLDMALLOC
        size_t oldmalloc_increase;
        gc_counter_t freed_in_gc;
#endif
#ifdef MALLOC_COUNTERS_NEED_LOCK
        rb_nativethread_lock_t lock;
#endif
    } malloc_counters;
 
    struct {
        size_t limit;
#if MALLOC_ALLOCATED_SIZE
        size_t allocated_size;
        size_t allocations;
#endif
    } malloc_params;
 
    struct rb_gc_config {
        bool full_mark;
    } gc_config;
 
    struct {
        unsigned int mode : 2;
        unsigned int immediate_sweep : 1;
        unsigned int dont_gc : 1;
        /* A user hold from GC.disable (kept in vm->gc.disable_holders); owner thread only. */
        unsigned int user_gc_disabled : 1;
        unsigned int dont_incremental : 1;
        unsigned int during_gc : 1;
        unsigned int during_global_gc : 1;
        unsigned int during_compacting : 1;
        unsigned int gc_lock_barrier : 1;
        unsigned int during_reference_updating : 1;
        unsigned int during_minor_gc : 1;
        unsigned int during_postmortem : 1;
        /* No Ractor will own this objspace any more and it will never allocate again. */
        unsigned int ownerless : 1;
        unsigned int measure_gc : 1;
    } flags;
 
    rb_event_flag_t hook_events;
 
    rb_heap_t heaps[HEAP_COUNT];
    size_t empty_pages_count;
    struct heap_page *empty_pages;
    /* Nonzero while gc.c parks this objspace in its zombie table: the pages it adds to
     * global_objspace->zombie_total_pages.  Written under the VM lock or the barrier. */
    size_t zombie_pages;
 
    struct {
        rb_atomic_t finalizing;
    } atomic_flags;
 
    mark_stack_t mark_stack;
    size_t marked_slots;
 
    /* Moved out of the per-Ractor newobj cache: allocation state is per objspace. */
    size_t incremental_mark_step_allocated_slots;
 
    /* Inputs of the global GC trigger, all owned by this objspace's thread.
     * shareable_objects is the live population of shareable objects; exceeding the
     * limit requests a global GC. */
    size_t shareable_objects;
    size_t shareable_objects_limit;
    /* Whether the last mark ran the pinned walk; the sweep asserts on it. */
    unsigned char last_cycle_pinned;
 
    struct {
        rb_darray(struct heap_page *) sorted;
 
        size_t allocated_pages;
        size_t freed_pages;
        uintptr_t range[2];
        size_t freeable_pages;
 
        size_t allocatable_bytes;
 
        /* final */
        VALUE deferred_final;
    } heap_pages;
 
    st_table *finalizer_table;
 
    struct {
        int run;
        unsigned int latest_gc_info;
        gc_profile_record *records;
        gc_profile_record *current_record;
        size_t next_index;
        size_t size;
        size_t record_count;
        size_t max_records;
        size_t record_sequence;
 
#if GC_PROFILE_MORE_DETAIL
        double prepare_time;
#endif
        double invoke_time;
        rb_hrtime_t invoke_wall_time;
 
        size_t minor_gc_count;
        size_t major_gc_count;
        size_t global_gc_count;
        size_t global_gc_invocation_count;
        size_t compact_count;
        size_t read_barrier_faults;
#if RGENGC_PROFILE > 0
        size_t total_generated_normal_object_count;
        size_t total_generated_shady_object_count;
        size_t total_shade_operation_count;
        size_t total_promoted_count;
        size_t total_remembered_normal_object_count;
        size_t total_remembered_shady_object_count;
 
#if RGENGC_PROFILE >= 2
        size_t generated_normal_object_count_types[RUBY_T_MASK];
        size_t generated_shady_object_count_types[RUBY_T_MASK];
        size_t shade_operation_count_types[RUBY_T_MASK];
        size_t promoted_types[RUBY_T_MASK];
        size_t remembered_normal_object_count_types[RUBY_T_MASK];
        size_t remembered_shady_object_count_types[RUBY_T_MASK];
#endif
#endif /* RGENGC_PROFILE */
 
        /* temporary profiling space */
        double gc_sweep_start_time;
        rb_hrtime_t gc_wall_start_time;
        rb_hrtime_t gc_sweep_wall_start_time;
        rb_hrtime_t gc_sweep_excluded_wall_time;
        rb_hrtime_t gc_pause_start_time;
        rb_hrtime_t gc_stw_start_time;
        rb_hrtime_t gc_stop_time;
        rb_hrtime_t gc_mark_phase_wall_start_time;
        rb_hrtime_t gc_sweep_phase_wall_start_time;
#if GC_PROFILE_MORE_DETAIL
        size_t total_allocated_objects_at_gc_start;
        size_t heap_used_at_gc_start;
#endif
 
        /* basic statistics */
        size_t count;
        unsigned long long marking_time_ns;
        struct timespec marking_start_time;
        unsigned long long sweeping_time_ns;
        struct timespec sweeping_start_time;
 
        /* Weak references */
        size_t weak_references_count;
    } profile;
 
 
    struct {
        bool parent_object_old_p;
        VALUE parent_object;
 
        int need_major_gc;
        size_t last_major_gc;
        size_t uncollectible_wb_unprotected_objects;
        size_t uncollectible_wb_unprotected_objects_limit;
        size_t old_objects;
        size_t old_objects_limit;
 
#if RGENGC_ESTIMATE_OLDMALLOC
        size_t oldmalloc_increase_limit;
#endif
 
#if RGENGC_CHECK_MODE >= 2
        struct st_table *allrefs_table;
        size_t error_count;
#endif
    } rgengc;
 
    struct {
        size_t considered_count_table[T_MASK];
        size_t moved_count_table[T_MASK];
        size_t moved_up_count_table[T_MASK];
        size_t moved_down_count_table[T_MASK];
        size_t total_moved;
 
        /* This function will be used, if set, to sort the heap prior to compaction */
        gc_compact_compare_func compare_func;
    } rcompactor;
 
    struct {
        size_t pooled_slots;
        size_t step_slots;
    } rincgc;
 
#if GC_DEBUG_STRESS_TO_CLASS
    VALUE stress_to_class;
#endif
 
    rb_darray(VALUE) weak_references;
    rb_postponed_job_handle_t finalize_deferred_pjob;
 
    /* Partially filled chunk of deferred non-thread-safe T_DATA metadata. */
    struct tdata_unsafe_free_chunk *tdata_unsafe_free_chunk;
 
    int sweeping_heap_count;
 
    int fork_vm_lock_lev;
 
    struct rb_gc_vm_context vm_context;
 
    /* Process-wide GC statistics publication.  Default GC only: other
     * implementations reject GC.stat(process: true) and never initialize
     * this lock. */
    struct {
        rb_nativethread_lock_t lock;
        struct gc_process_stat_snapshot published;
    } process_stat;
 
    /* The incremental-marking flag as an addressable field, so that ZJIT-generated
     * code can read it; see rb_gc_impl_zjit_writebarrier_required_offset.
     * Single-writer: a local GC runs on the owning Ractor's thread and a global GC
     * writes under the stop-the-world barrier.  Read by JIT code on the same thread. */
    bool zjit_during_incremental_marking;
} rb_objspace_t;
 
/* The one VM-global GC structure; for now it only holds the page pool.  Page bodies are
 * carved out of large mmap arenas and reused via a process-wide freelist (per-page
 * mmap/munmap would serialize on the kernel's mmap_lock).  Leaf lock: no alloc, no GC. */
typedef struct rb_global_objspace {
    struct {
        rb_nativethread_lock_t lock;
        struct heap_page_body *hot_list; /* ≤ PAGE_POOL_HOT_MAX un-advised bodies; link at body offset 0 */
        int hot_count;
        size_t os_page_size;             /* sysconf(_SC_PAGE_SIZE), cached at init */
        /* List of mmap'd memory regions (arenas) for page bodies. */
        struct page_arena {
            struct page_arena *next;
            char *start;                  /* usable area, HEAP_PAGE_ALIGN aligned */
            size_t size;                  /* usable bytes (a multiple of HEAP_PAGE_SIZE) */
            struct heap_page_body *cold_freelist; /* free bodies of this arena; link at body offset 0 */
            int free_count;               /* bodies of this arena currently free (hot list + cold_freelist) */
            int cold_count;               /* bodies on cold_freelist (subset of free_count) */
        } *arenas;                        /* every arena, newest first */
        char *arena_cursor;               /* first body not yet carved out of the newest arena */
        char *arena_end;                  /* end of current arena */
        struct page_arena *arena_current; /* arena that arena_cursor carves from */
        int arena_count;                  /* current mapped arenas (for GC.stat total_pages) */
        int advised_count;                /* page bodies with MADV_* applied (for GC.stat discarded pages) */
        size_t arenas_unmapped;           /* cumulative arenas munmapped (for GC.stat) */
    } page_pool;
 
    /* Pages held by zombie objspaces (sum of their zombie_pages), and what was left of
     * it after the last global cycle (roughly the live data).  Written under the VM
     * lock or the barrier; readers (gc_need_global_p) may be racy. */
    size_t zombie_total_pages;
    size_t zombie_pages_survivors;
 
    /* An objspace merge (objspace_absorb) is running: suppress the cross-objspace
     * verifier while the graph is in flux.  Written by the absorbing thread, read by
     * verification with the world stopped. */
    bool during_absorb;
 
    /* main's objspace, for gc_enter's locking policy.  main_ractor->objspace is swapped
     * during Ractor creation; this stable pointer decides the same way at both ends of a
     * GC.  Set at boot, re-pointed in a forked child. */
    rb_objspace_t *main_objspace;
 
    /* GC.stress is process-global (upstream semantics).  Written by GC.stress= in any
     * Ractor (rare) and read on every Ractor's alloc and GC path; it is diagnostic, so
     * plain store/load with last-writer-wins is fine. */
    bool gc_stressful;
    VALUE gc_stress_mode;
 
    /* Global GC driver state.  compacting is true during the move phase: reference
     * updates are deferred until all forwarding exists, so a cross-objspace reference is
     * rewritten exactly once.  objspaces is the snapshot being collected. */
    struct {
        bool compacting;
        struct rb_objspace **objspaces;
        size_t n_objspaces, objspaces_capa;
        size_t count;
    } global_gc;
 
    /* Index of every objspace's heap pages, ordered by body address.  Writers (page
     * alloc/free) serialize on page_pool.lock; the only reader is a stop-the-world global
     * GC, so reads need no lock.  A local GC uses its own heap_pages.sorted. */
    struct {
        struct heap_page **pages;
        size_t n_pages, capa;
        uintptr_t lomem, himem;
    } page_index;
 
    rb_postponed_job_handle_t tdata_deferred_free_pjob; /* atomic */
 
    /* Pending count of deferred non-thread-safe T_DATA frees across all objspaces:
     * bumped as each one is deferred, reset to 0 by the drain.  Crossing
     * TDATA_DEFERRED_FREE_THRESHOLD triggers the postponed job. */
    size_t tdata_deferred_free_count; /* atomic */
 
    /* Full chunks awaiting a drain (CAS stack), and drained chunks kept for reuse. */
    struct tdata_unsafe_free_chunk *tdata_unsafe_free_published; /* atomic */
    struct tdata_unsafe_free_chunk *tdata_unsafe_free_cache; /* atomic */
    size_t tdata_unsafe_free_cache_len; /* atomic */
 
    /* Archive of destroyed objspaces' final statistics, added once on absorption. */
    struct gc_process_stat_total process_stat_archive;
} rb_global_objspace_t;
 
static rb_global_objspace_t rb_global_objspace_instance;
static rb_global_objspace_t *global_objspace = NULL;
 
/* Relaxed: every reader only asks whether a drain is worth arranging, and the drain
 * itself stops the world before it touches a chunk. */
static inline size_t
tdata_deferred_free_count_load(void)
{
    return rbimpl_atomic_size_load(&global_objspace->tdata_deferred_free_count, RBIMPL_ATOMIC_RELAXED);
}
 
/* The floor keeps a global GC from running as soon as a few shareable objects appear;
 * the factor follows the rule used for the old-generation limit. */
#define SHAREABLE_OBJECTS_LIMIT_MIN (1 << 16)
#define SHAREABLE_OBJECTS_LIMIT_FACTOR 2.0
/* Start a global GC once terminated, uninherited Ractors hold this many heap pages.  A
 * small Ractor's objspace is about 13 pages, so discarding many of them still stays
 * below it, while a single fat zombie crosses it. */
#define ZOMBIE_PAGES_TRIGGER 256
/* Trigger the deferred T_DATA free postponed job once this many have accumulated
 * across all objspaces. */
#define TDATA_DEFERRED_FREE_THRESHOLD (1 << 15)
 
static void objspace_absorb(rb_objspace_t *dst, rb_objspace_t *src);
 
 
static struct heap_page_body *page_pool_acquire(struct page_arena **arena_out);
static void page_pool_release(struct heap_page_body *body, struct page_arena *arena);
#ifdef HAVE_MMAP
static void page_pool_release_locked(struct heap_page_body *body, struct page_arena *arena);
#endif
static void page_pool_reclaim(rb_global_objspace_t *g);
 
#if RGENGC_CHECK_MODE && !defined(_WIN32) && !defined(__wasi__) && defined(HAVE_PTHREAD_H)
# define PAGE_POOL_LOCK_ERRORCHECK 1
#endif
 
static void
page_pool_lock_initialize(rb_nativethread_lock_t *lock)
{
#ifdef PAGE_POOL_LOCK_ERRORCHECK
    pthread_mutexattr_t attr;
    pthread_mutexattr_init(&attr);
    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
    pthread_mutex_init(lock, &attr);
    pthread_mutexattr_destroy(&attr);
#else
    rb_native_mutex_initialize(lock);
#endif
}
 
static void
global_objspace_init(void)
{
    if (global_objspace == NULL) {
        rb_global_objspace_t *g = &rb_global_objspace_instance;
        page_pool_lock_initialize(&g->page_pool.lock);
        g->page_pool.hot_list = NULL;
        g->page_pool.hot_count = 0;
        g->page_pool.arenas = NULL;
        g->page_pool.arena_cursor = NULL;
        g->page_pool.arena_end = NULL;
        g->page_pool.arena_current = NULL;
        g->page_pool.arena_count = 0;
        g->page_pool.advised_count = 0;
        g->tdata_deferred_free_pjob = POSTPONED_JOB_HANDLE_INVALID;
        g->tdata_unsafe_free_published = NULL;
        g->tdata_unsafe_free_cache = NULL;
        g->tdata_unsafe_free_cache_len = 0;
        g->page_pool.arenas_unmapped = 0;
#ifdef HAVE_MMAP
        g->page_pool.os_page_size = sysconf(_SC_PAGE_SIZE);
#else
        g->page_pool.os_page_size = 0;
#endif
        global_objspace = g;
    }
}
 
#ifndef HEAP_PAGE_ALIGN_LOG
/* default tiny heap size: 64KiB */
#define HEAP_PAGE_ALIGN_LOG 16
#endif
 
#if GC_DEBUG
struct rvalue_overhead {
    const char *file;
    int line;
};
 
// Make sure that RVALUE_OVERHEAD aligns to sizeof(VALUE)
# define RVALUE_OVERHEAD (sizeof(struct { \
    union { \
        struct rvalue_overhead overhead; \
        VALUE value; \
    }; \
}))
size_t rb_gc_impl_obj_slot_size(VALUE obj);
# define GET_RVALUE_OVERHEAD(obj) ((struct rvalue_overhead *)((uintptr_t)obj + rb_gc_impl_obj_slot_size(obj)))
#else
# ifndef RVALUE_OVERHEAD
#  define RVALUE_OVERHEAD 0
# endif
#endif
 
#define RVALUE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD)
 
static const size_t pool_slot_sizes[HEAP_COUNT] = {
#define SLOT(size) ((size) + RVALUE_OVERHEAD),
    EACH_POOL_SLOT_SIZE(SLOT)
#undef SLOT
};
 
/* An init size below one slot in the largest heap never forces that heap's first
 * page, and allocating there then fails with "cannot create a new page after GC". */
static inline size_t
heap_init_bytes_min(void)
{
    return pool_slot_sizes[HEAP_COUNT - 1];
}
 
/* Precomputed reciprocals for fast slot index calculation.
 * For slot size d: reciprocal = ceil(2^48 / d).
 * Then offset / d == (uint32_t)((offset * reciprocal) >> 48)
 * for all offset < HEAP_PAGE_SIZE. */
#define SLOT_RECIPROCAL_SHIFT 48
#define SLOT_RECIPROCAL(size) (((1ULL << SLOT_RECIPROCAL_SHIFT) + (size) - 1) / (size))
 
static const uint64_t heap_slot_reciprocal_table[HEAP_COUNT] = {
#define SLOT(size) SLOT_RECIPROCAL((size) + RVALUE_OVERHEAD),
    EACH_POOL_SLOT_SIZE(SLOT)
#undef SLOT
};
 
#if SIZEOF_VALUE >= 8
static uint8_t size_to_heap_idx[1024 / 8 + 1];
#else
static uint8_t size_to_heap_idx[512 / 8 + 1];
#endif
 
#ifndef MAX
# define MAX(a, b) (((a) > (b)) ? (a) : (b))
#endif
#ifndef MIN
# define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#define roomof(x, y) (((x) + (y) - 1) / (y))
#define CEILDIV(i, mod) roomof(i, mod)
#define MIN_POOL_SLOT_SIZE 32
enum {
    HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG),
    HEAP_PAGE_ALIGN_MASK = (~(~0UL << HEAP_PAGE_ALIGN_LOG)),
    HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN,
    HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, MIN_POOL_SLOT_SIZE), BITS_BITLENGTH),
    HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),
};
#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG)
#define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN
 
#if !defined(INCREMENTAL_MARK_STEP_ALLOCATIONS)
# define INCREMENTAL_MARK_STEP_ALLOCATIONS 500
#endif
 
#undef INIT_HEAP_PAGE_ALLOC_USE_MMAP
/* Must define either HEAP_PAGE_ALLOC_USE_MMAP or
 * INIT_HEAP_PAGE_ALLOC_USE_MMAP. */
 
#ifndef HAVE_MMAP
/* We can't use mmap of course, if it is not available. */
static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
 
#elif defined(__wasm__)
/* wasmtime does not have proper support for mmap.
 * See https://github.com/bytecodealliance/wasmtime/blob/main/docs/WASI-rationale.md#why-no-mmap-and-friends
 */
static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
 
#elif HAVE_CONST_PAGE_SIZE
/* If we have the PAGE_SIZE and it is a constant, then we can directly use it. */
static const bool HEAP_PAGE_ALLOC_USE_MMAP = (PAGE_SIZE <= HEAP_PAGE_SIZE);
 
#elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE)
/* If we can use the maximum page size. */
static const bool HEAP_PAGE_ALLOC_USE_MMAP = true;
 
#elif defined(PAGE_SIZE)
/* If the PAGE_SIZE macro can be used dynamically. */
# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (PAGE_SIZE <= HEAP_PAGE_SIZE)
 
#elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
/* If we can use sysconf to determine the page size. */
# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE)
 
#else
/* Otherwise we can't determine the system page size, so don't use mmap. */
static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
#endif
 
#ifdef INIT_HEAP_PAGE_ALLOC_USE_MMAP
/* We can determine the system page size at runtime. */
# define HEAP_PAGE_ALLOC_USE_MMAP (heap_page_alloc_use_mmap != false)
 
static bool heap_page_alloc_use_mmap;
#endif
 
#define RVALUE_AGE_BIT_COUNT 2
#define RVALUE_AGE_BIT_MASK (((bits_t)1 << RVALUE_AGE_BIT_COUNT) - 1)
#define RVALUE_OLD_AGE   3
 
struct free_region {
    VALUE flags;                /* always 0 for freed obj */
    uintptr_t end;              /* exclusive end address of the run */
    struct free_region *next;   /* next free region in the page */
};
 
struct heap_page {
    /* Cache line 0: allocation fast path + SLOT_INDEX */
    struct free_region *free_region;
    uintptr_t start;
    uint64_t slot_size_reciprocal;
    unsigned short slot_size;
    unsigned short total_slots;
    unsigned short free_slots;
    unsigned short final_slots;
    unsigned short pinned_slots;
    /* Page state flags.  A bitfield is safe: the only writers are the owning Ractor
     * (GVL) and the global GC driver (stop-the-world), never together.  has_shareable /
     * has_shref hint that the page holds at least one such bit, for re-scanning. */
    struct {
        unsigned int before_sweep : 1;
        unsigned int has_remembered_objects : 1;
        unsigned int has_uncollectible_wb_unprotected_objects : 1;
        unsigned int has_shref_objects : 1;
        unsigned int has_shareable_objects : 1;
    } flags;
 
    rb_heap_t *heap;
 
    /* The objspace owning this page, so any object's owner is one load away
     * (GET_HEAP_OBJSPACE).  Rewritten only when a page changes owner (inheritance). */
    rb_objspace_t *objspace;
 
    struct heap_page *free_next;
    struct heap_page_body *body;
    struct page_arena *arena;
    struct ccan_list_node page_node;
 
    bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT];
    /* the following three bitmaps are cleared at the beginning of full GC */
    bits_t mark_bits[HEAP_PAGE_BITMAP_LIMIT];
    bits_t uncollectible_bits[HEAP_PAGE_BITMAP_LIMIT];
    bits_t marking_bits[HEAP_PAGE_BITMAP_LIMIT];
 
    bits_t remembered_bits[HEAP_PAGE_BITMAP_LIMIT];
 
    /* Two extra bits per object.  shareable_bits: what a local sweep must never free
     * (only a global GC decides a shareable object is dead); set at creation and by
     * rb_gc_impl_obj_became_shareable.  shref_bits: an unshareable object referenced
     * from a shareable one, a local GC root; the write barrier maintains it. */
    bits_t shareable_bits[HEAP_PAGE_BITMAP_LIMIT];
    bits_t shref_bits[HEAP_PAGE_BITMAP_LIMIT];
 
    /* If set, the object is not movable */
    bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT];
    bits_t age_bits[HEAP_PAGE_BITMAP_LIMIT * RVALUE_AGE_BIT_COUNT];
};
 
/*
 * When asan is enabled, this will prohibit writing to the freelist until it is unlocked
 */
static void
asan_lock_freelist(struct heap_page *page)
{
    asan_poison_memory_region(&page->free_region, sizeof(struct free_region *));
}
 
/*
 * When asan is enabled, this will enable the ability to write to the freelist
 */
static void
asan_unlock_freelist(struct heap_page *page)
{
    asan_unpoison_memory_region(&page->free_region, sizeof(struct free_region *), false);
}
 
static inline bool
heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page *page)
{
    if (page->total_slots == 0) {
        GC_ASSERT(page->start == 0);
        GC_ASSERT(page->slot_size == 0);
        GC_ASSERT(page->heap == NULL);
        GC_ASSERT(page->free_slots == 0);
        asan_unpoisoning_memory_region(&page->free_region, sizeof(&page->free_region)) {
            GC_ASSERT(page->free_region == NULL);
        }
 
        return true;
    }
    else {
        GC_ASSERT(page->start != 0);
        GC_ASSERT(page->slot_size != 0);
        GC_ASSERT(page->heap != NULL);
 
        return false;
    }
}
 
#define GET_PAGE_BODY(x)   ((struct heap_page_body *)((bits_t)(x) & ~(HEAP_PAGE_ALIGN_MASK)))
#define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
#define GET_HEAP_PAGE(x)   (GET_PAGE_HEADER(x)->page)
 
static inline size_t
slot_index_for_offset(size_t offset, uint64_t reciprocal)
{
    return (uint32_t)(((uint64_t)offset * reciprocal) >> SLOT_RECIPROCAL_SHIFT);
}
 
#define SLOT_INDEX(page, p)          slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_reciprocal)
#define SLOT_BITMAP_INDEX(page, p)   (SLOT_INDEX(page, p) / BITS_BITLENGTH)
#define SLOT_BITMAP_OFFSET(page, p)  (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1))
#define SLOT_BITMAP_BIT(page, p)     ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p))
 
#define _MARKED_IN_BITMAP(bits, page, p)  ((bits)[SLOT_BITMAP_INDEX(page, p)] & SLOT_BITMAP_BIT(page, p))
#define _MARK_IN_BITMAP(bits, page, p)    ((bits)[SLOT_BITMAP_INDEX(page, p)] |= SLOT_BITMAP_BIT(page, p))
#define _CLEAR_IN_BITMAP(bits, page, p)   ((bits)[SLOT_BITMAP_INDEX(page, p)] &= ~SLOT_BITMAP_BIT(page, p))
 
#define MARKED_IN_BITMAP(bits, p)    _MARKED_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
#define MARK_IN_BITMAP(bits, p)      _MARK_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
#define CLEAR_IN_BITMAP(bits, p)     _CLEAR_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
 
#define GET_HEAP_MARK_BITS(x)           (&GET_HEAP_PAGE(x)->mark_bits[0])
#define GET_HEAP_PINNED_BITS(x)         (&GET_HEAP_PAGE(x)->pinned_bits[0])
#define GET_HEAP_UNCOLLECTIBLE_BITS(x)  (&GET_HEAP_PAGE(x)->uncollectible_bits[0])
#define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0])
#define GET_HEAP_MARKING_BITS(x)        (&GET_HEAP_PAGE(x)->marking_bits[0])
#define GET_HEAP_SHAREABLE_BITS(x)      (&GET_HEAP_PAGE(x)->shareable_bits[0])
#define GET_HEAP_SHREF_BITS(x)          (&GET_HEAP_PAGE(x)->shref_bits[0])
#define GET_HEAP_OBJSPACE(x)            (GET_HEAP_PAGE(x)->objspace)
 
/* obj lives on a page of another objspace, not the current one (i.e. it is foreign). */
static inline bool
gc_foreign_object_p(const rb_objspace_t *objspace, VALUE obj)
{
    if (RB_LIKELY(!rb_gc_ever_multi_ractor_p())) return false;
 
    return RB_UNLIKELY(GET_HEAP_OBJSPACE(obj) != objspace);
}
 
/* Foreign and not inside a stop-the-world global GC.  While true, a local GC must not
 * touch obj's per-object GC state (mark, pin, remember bits): its owner handles that,
 * or the global GC does with everyone stopped. */
static inline bool
gc_skip_foreign_object_p(const rb_objspace_t *objspace, VALUE obj)
{
    return gc_foreign_object_p(objspace, obj) && !objspace->flags.during_global_gc;
}
 
/* Record obj as shareable on its owning page (bit, page flag and population counter).
 * Shared by born-shareable objects and make_shareable.  The writer is the owner thread,
 * so plain bit operations suffice. */
static inline void
gc_page_add_shareable(struct heap_page *page, VALUE obj)
{
    GC_ASSERT(page == GET_HEAP_PAGE(obj));
    GC_ASSERT(RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE));
    _MARK_IN_BITMAP(page->shareable_bits, page, obj);
    page->flags.has_shareable_objects = TRUE;
    page->objspace->shareable_objects++;
}
 
static int
RVALUE_AGE_GET(VALUE obj)
{
    struct heap_page *page = GET_HEAP_PAGE(obj);
    bits_t *age_bits = page->age_bits;
    size_t slot_idx = SLOT_INDEX(page, obj);
    size_t idx = (slot_idx / BITS_BITLENGTH) * 2;
    int shift = (int)(slot_idx & (BITS_BITLENGTH - 1));
    int lo = (age_bits[idx] >> shift) & 1;
    int hi = (age_bits[idx + 1] >> shift) & 1;
    return lo | (hi << 1);
}
 
static void
RVALUE_AGE_SET_BITMAP(VALUE obj, int age)
{
    RUBY_ASSERT(age <= RVALUE_OLD_AGE);
    struct heap_page *page = GET_HEAP_PAGE(obj);
    bits_t *age_bits = page->age_bits;
    size_t slot_idx = SLOT_INDEX(page, obj);
    size_t idx = (slot_idx / BITS_BITLENGTH) * 2;
    int shift = (int)(slot_idx & (BITS_BITLENGTH - 1));
    bits_t mask = (bits_t)1 << shift;
 
    age_bits[idx]     = (age_bits[idx]     & ~mask) | ((bits_t)(age & 1) << shift);
    age_bits[idx + 1] = (age_bits[idx + 1] & ~mask) | ((bits_t)((age >> 1) & 1) << shift);
}
 
static void
RVALUE_AGE_SET(VALUE obj, int age)
{
    RVALUE_AGE_SET_BITMAP(obj, age);
    if (age == RVALUE_OLD_AGE) {
        RB_FL_SET_RAW(obj, RUBY_FL_PROMOTED);
    }
    else {
        RB_FL_UNSET_RAW(obj, RUBY_FL_PROMOTED);
    }
}
 
#define malloc_limit            objspace->malloc_params.limit
#define malloc_increase         gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters)
#define malloc_allocated_size   objspace->malloc_params.allocated_size
 
#ifdef MALLOC_COUNTERS_NEED_LOCK
# define MALLOC_COUNTERS_LOCK(o)   rb_native_mutex_lock(&(o)->malloc_counters.lock)
# define MALLOC_COUNTERS_UNLOCK(o) rb_native_mutex_unlock(&(o)->malloc_counters.lock)
#else
# define MALLOC_COUNTERS_LOCK(o)   ((void)0)
# define MALLOC_COUNTERS_UNLOCK(o) ((void)0)
#endif
 
static inline void
gc_counter_add(gc_counter_t *p, size_t delta)
{
#ifdef MALLOC_COUNTERS_NEED_LOCK
    *p += (gc_counter_t)delta;
#else
    rbimpl_atomic_u64_fetch_add_relaxed(p, (uint64_t)delta);
#endif
}
 
static inline gc_counter_t
gc_counter_load_relaxed(const gc_counter_t *p)
{
#ifdef MALLOC_COUNTERS_NEED_LOCK
    return *p;
#else
    return rbimpl_atomic_u64_load_relaxed(p);
#endif
}
 
static inline gc_counter_t
gc_counter_load_acquire(const gc_counter_t *p)
{
#ifdef MALLOC_COUNTERS_NEED_LOCK
    return *p;
#else
    return rbimpl_atomic_u64_load_acquire(p);
#endif
}
 
static inline void
gc_counter_store_release(gc_counter_t *p, gc_counter_t v)
{
#ifdef MALLOC_COUNTERS_NEED_LOCK
    *p = v;
#else
    rbimpl_atomic_u64_set_release(p, v);
#endif
}
 
static inline int64_t
gc_malloc_counters_increase(rb_objspace_t *objspace, const struct gc_malloc_bytes *c)
{
    MALLOC_COUNTERS_LOCK(objspace);
    gc_counter_t malloc_at = gc_counter_load_acquire(&c->malloc_at_last_gc);
    gc_counter_t free_at   = gc_counter_load_acquire(&c->free_at_last_gc);
    gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc);
    gc_counter_t free_now   = gc_counter_load_relaxed(&c->free);
    MALLOC_COUNTERS_UNLOCK(objspace);
 
    gc_counter_t malloc_delta = malloc_now - malloc_at;
    gc_counter_t free_delta   = free_now   - free_at;
 
    if (malloc_delta >= free_delta) {
        return (int64_t)(malloc_delta - free_delta);
    }
    else {
        return -(int64_t)(free_delta - malloc_delta);
    }
}
 
static inline size_t
gc_malloc_counters_increase_unsigned(rb_objspace_t *objspace, const struct gc_malloc_bytes *c)
{
    int64_t inc = gc_malloc_counters_increase(objspace, c);
    if (inc <= 0) return 0;
#if SIZEOF_SIZE_T < 8
    if ((uint64_t)inc > SIZE_MAX) return SIZE_MAX;
#endif
    return (size_t)inc;
}
 
/* Frees done while sweeping are the GC's own work, not the mutator's: advance
 * free_at_last_gc past them so they cannot pay for the next cycle's allocation.
 * malloc_at_last_gc stays at gc_reset_malloc_info's snapshot (GC start). */
static inline void
gc_malloc_counters_snapshot_free_at_last_gc(rb_objspace_t *objspace, struct gc_malloc_bytes *c)
{
    MALLOC_COUNTERS_LOCK(objspace);
    gc_counter_store_release(&c->free_at_last_gc, gc_counter_load_relaxed(&c->free));
    MALLOC_COUNTERS_UNLOCK(objspace);
}
 
static inline void
gc_malloc_counters_snapshot(rb_objspace_t *objspace, struct gc_malloc_bytes *c)
{
    MALLOC_COUNTERS_LOCK(objspace);
    gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc);
    gc_counter_t free_now   = gc_counter_load_relaxed(&c->free);
    gc_counter_store_release(&c->malloc_at_last_gc, malloc_now);
    gc_counter_store_release(&c->free_at_last_gc,   free_now);
    MALLOC_COUNTERS_UNLOCK(objspace);
}
 
#if RGENGC_ESTIMATE_OLDMALLOC
static inline size_t
gc_oldmalloc_increase_load(rb_objspace_t *objspace)
{
    return rbimpl_atomic_size_load(&objspace->malloc_counters.oldmalloc_increase, RBIMPL_ATOMIC_RELAXED);
}
#endif
 
#define heap_pages_lomem        objspace->heap_pages.range[0]
#define heap_pages_himem        objspace->heap_pages.range[1]
#define heap_pages_freeable_pages       objspace->heap_pages.freeable_pages
#define heap_pages_deferred_final       objspace->heap_pages.deferred_final
#define heaps              objspace->heaps
#define during_gc               objspace->flags.during_gc
#define finalizing              objspace->atomic_flags.finalizing
#define finalizer_table         objspace->finalizer_table
#define ruby_gc_stressful       global_objspace->gc_stressful
#define ruby_gc_stress_mode     global_objspace->gc_stress_mode
#if GC_DEBUG_STRESS_TO_CLASS
#define stress_to_class         objspace->stress_to_class
#define set_stress_to_class(c)  (stress_to_class = (c))
#else
#define stress_to_class         ((void)objspace, 0)
#define set_stress_to_class(c)  ((void)objspace, (c))
#endif
 
#if 0
#define dont_gc_on()          (fprintf(stderr, "dont_gc_on@%s:%d\n",      __FILE__, __LINE__), objspace->flags.dont_gc = 1)
#define dont_gc_off()         (fprintf(stderr, "dont_gc_off@%s:%d\n",     __FILE__, __LINE__), objspace->flags.dont_gc = 0)
#define dont_gc_set(b)        (fprintf(stderr, "dont_gc_set(%d)@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = (int)(b))
#define dont_gc_val()         (objspace->flags.dont_gc)
#else
#define dont_gc_on()          (objspace->flags.dont_gc = 1)
#define dont_gc_off()         (objspace->flags.dont_gc = 0)
#define dont_gc_set(b)        (objspace->flags.dont_gc = (int)(b))
#define dont_gc_val()         (objspace->flags.dont_gc)
#endif
 
#define gc_config_full_mark_set(b) (objspace->gc_config.full_mark = (int)(b))
#define gc_config_full_mark_val    (objspace->gc_config.full_mark)
 
static inline enum gc_mode
gc_mode_verify(enum gc_mode mode)
{
#if RGENGC_CHECK_MODE > 0
    switch (mode) {
      case gc_mode_none:
      case gc_mode_marking:
      case gc_mode_sweeping:
      case gc_mode_compacting:
        break;
      default:
        rb_bug("gc_mode_verify: unreachable (%d)", (int)mode);
    }
#endif
    return mode;
}
 
static inline bool
has_sweeping_pages(rb_objspace_t *objspace)
{
    return objspace->sweeping_heap_count != 0;
}
 
static inline size_t
heap_eden_total_pages(rb_objspace_t *objspace)
{
    size_t count = 0;
    for (int i = 0; i < HEAP_COUNT; i++) {
        count += (&heaps[i])->total_pages;
    }
    return count;
}
 
static inline size_t
total_allocated_objects(rb_objspace_t *objspace)
{
    size_t count = 0;
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        count += heap->total_allocated_objects;
    }
    return count;
}
 
static inline size_t
total_freed_objects(rb_objspace_t *objspace)
{
    size_t count = 0;
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        count += heap->total_freed_objects;
    }
    return count;
}
 
static inline size_t
total_final_slots_count(rb_objspace_t *objspace)
{
    size_t count = 0;
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        count += heap->final_slots_count;
    }
    return count;
}
 
#define gc_mode(objspace)                gc_mode_verify((enum gc_mode)(objspace)->flags.mode)
#define gc_mode_set(objspace, m)         ((objspace)->flags.mode = (unsigned int)gc_mode_verify(m))
#define gc_needs_major_flags objspace->rgengc.need_major_gc
 
#define is_marking(objspace)             (gc_mode(objspace) == gc_mode_marking)
#define is_sweeping(objspace)            (gc_mode(objspace) == gc_mode_sweeping)
#define is_full_marking(objspace)        ((objspace)->flags.during_minor_gc == FALSE)
#define is_incremental_marking(objspace) ((objspace)->zjit_during_incremental_marking != FALSE)
#define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE)
 
static void
gc_set_incremental_marking(rb_objspace_t *objspace, unsigned int value)
{
    if (objspace->zjit_during_incremental_marking != value) {
        objspace->zjit_during_incremental_marking = value;
    }
}
/*
 * Byte budget for incremental sweep steps. Each step sweeps at most
 * this many bytes worth of slots before yielding. The effective slot
 * count per step is GC_INCREMENTAL_SWEEP_BYTES / heap->slot_size,
 * so larger slot pools (which are less heavily used) naturally get
 * fewer slots swept per step.
 *
 * Baseline: 2048 slots * RVALUE_SLOT_SIZE = 2048 * 40 = 81920 bytes,
 * preserving the historical behavior for the smallest heap.
 */
#define GC_INCREMENTAL_SWEEP_BYTES           (2048 * RVALUE_SLOT_SIZE)
#define GC_INCREMENTAL_SWEEP_POOL_BYTES      (1024 * RVALUE_SLOT_SIZE)
#define is_lazy_sweeping(objspace)           (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(objspace))
/* In lazy sweeping or the previous incremental marking finished and did not yield a free page. */
#define needs_continue_sweeping(objspace, heap) \
    ((heap)->free_pages == NULL && is_lazy_sweeping(objspace))
 
#if SIZEOF_LONG == SIZEOF_VOIDP
# define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG) /* unset FIXNUM_FLAG */
#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
# define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \
   ((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1))
#else
# error not supported
#endif
 
struct RZombie {
    VALUE flags;
    VALUE next;
    void (*dfree)(void *);
    void *data;
};
 
#define RZOMBIE(o) ((struct RZombie *)(o))
 
static bool ruby_enable_autocompact = false;
#if RGENGC_CHECK_MODE
static gc_compact_compare_func ruby_autocompact_compare_func;
#endif
 
static void init_mark_stack(mark_stack_t *stack);
static int garbage_collect(rb_objspace_t *, unsigned int reason);
 
static int  gc_start(rb_objspace_t *objspace, unsigned int reason);
static void gc_rest(rb_objspace_t *objspace);
 
/* GC cycle events (ENTER, EXIT, START, END_MARK, END_SWEEP) fire only if the objspace's
 * own Ractor enabled them, so a concurrent local GC never walks the VM-global hook list
 * while another Ractor mutates it.  NEWOBJ and FREEOBJ were already restricted. */
#define gc_event_hook(objspace, event) do { \
    if (RB_UNLIKELY((objspace)->hook_events & (event))) { \
        rb_gc_event_hook(0, (event)); \
    } \
} while (0)
 
enum gc_enter_event {
    gc_enter_event_start,
    gc_enter_event_continue,
    gc_enter_event_rest,
    gc_enter_event_finalizer,
    gc_enter_event_global,
    gc_enter_event_global_auto,
};
 
static inline bool gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev);
static inline void gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev);
static void gc_marking_enter(rb_objspace_t *objspace);
static void gc_marking_exit(rb_objspace_t *objspace);
static void gc_sweeping_enter(rb_objspace_t *objspace);
static void gc_sweeping_exit(rb_objspace_t *objspace);
static bool gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap);
 
static void gc_sweep(rb_objspace_t *objspace);
static void gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap);
static void gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap);
 
static inline void gc_mark(rb_objspace_t *objspace, VALUE ptr);
static inline void gc_pin(rb_objspace_t *objspace, VALUE ptr);
static inline void gc_mark_and_pin(rb_objspace_t *objspace, VALUE ptr);
 
static int gc_mark_stacked_objects_incremental(rb_objspace_t *, size_t count);
NO_SANITIZE("memory", static inline bool is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr));
 
static void gc_verify_internal_consistency(void *objspace_ptr);
 
static double getrusage_time(void);
static inline rb_hrtime_t elapsed_hrtime_from(rb_hrtime_t start);
static inline void gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason);
static inline void gc_prof_timer_start(rb_objspace_t *);
static inline void gc_prof_timer_stop(rb_objspace_t *);
static inline void gc_prof_mark_timer_start(rb_objspace_t *);
static inline void gc_prof_mark_timer_stop(rb_objspace_t *);
static inline void gc_prof_sweep_timer_start(rb_objspace_t *);
static inline void gc_prof_sweep_timer_stop(rb_objspace_t *);
static inline void gc_prof_set_malloc_info(rb_objspace_t *);
static inline void gc_prof_set_heap_info(rb_objspace_t *);
 
#define gc_prof_record(objspace) (objspace)->profile.current_record
#define gc_prof_enabled(objspace) ((objspace)->profile.run && (objspace)->profile.current_record)
 
#define gc_report(level, objspace, ...) \
    if (!RGENGC_DEBUG_ENABLED(level)) {} else gc_report_body(level, objspace, __VA_ARGS__)
PRINTF_ARGS(static void gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...), 3, 4);
 
static void gc_finalize_deferred(void *dmy);
static void gc_tdata_unsafe_drain_objspaces(rb_objspace_t **objspaces, size_t n);
 
#if USE_TICK_T
 
/* the following code is only for internal tuning. */
 
/* Source code to use RDTSC is quoted and modified from
 * https://www.mcs.anl.gov/~kazutomo/rdtsc.html
 * written by Kazutomo Yoshii <kazutomo@mcs.anl.gov>
 */
 
#if defined(__GNUC__) && defined(__i386__)
typedef unsigned long long tick_t;
#define PRItick "llu"
static inline tick_t
tick(void)
{
    unsigned long long int x;
    __asm__ __volatile__ ("rdtsc" : "=A" (x));
    return x;
}
 
#elif defined(__GNUC__) && defined(__x86_64__)
typedef unsigned long long tick_t;
#define PRItick "llu"
 
static __inline__ tick_t
tick(void)
{
    unsigned long hi, lo;
    __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi));
    return ((unsigned long long)lo)|( ((unsigned long long)hi)<<32);
}
 
#elif defined(__powerpc64__) && (GCC_VERSION_SINCE(4,8,0) || defined(__clang__))
typedef unsigned long long tick_t;
#define PRItick "llu"
 
static __inline__ tick_t
tick(void)
{
    unsigned long long val = __builtin_ppc_get_timebase();
    return val;
}
 
#elif defined(__aarch64__) &&  defined(__GNUC__)
typedef unsigned long tick_t;
#define PRItick "lu"
 
static __inline__ tick_t
tick(void)
{
    unsigned long val;
    __asm__ __volatile__ ("mrs %0, cntvct_el0" : "=r" (val));
    return val;
}
 
 
#elif defined(_WIN32) && defined(_MSC_VER)
#include <intrin.h>
typedef unsigned __int64 tick_t;
#define PRItick "llu"
 
static inline tick_t
tick(void)
{
    return __rdtsc();
}
 
#else /* use clock */
typedef clock_t tick_t;
#define PRItick "llu"
 
static inline tick_t
tick(void)
{
    return clock();
}
#endif /* TSC */
#else /* USE_TICK_T */
#define MEASURE_LINE(expr) expr
#endif /* USE_TICK_T */
 
static inline VALUE check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj);
 
#define RVALUE_MARKED_BITMAP(obj)         MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj))
#define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj))
#define RVALUE_MARKING_BITMAP(obj)        MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj))
#define RVALUE_UNCOLLECTIBLE_BITMAP(obj)  MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj))
#define RVALUE_PINNED_BITMAP(obj)         MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), (obj))
 
static inline int
RVALUE_MARKED(rb_objspace_t *objspace, VALUE obj)
{
    check_rvalue_consistency(objspace, obj);
    return RVALUE_MARKED_BITMAP(obj) != 0;
}
 
static inline int
RVALUE_PINNED(rb_objspace_t *objspace, VALUE obj)
{
    check_rvalue_consistency(objspace, obj);
    return RVALUE_PINNED_BITMAP(obj) != 0;
}
 
static inline int
RVALUE_WB_UNPROTECTED(rb_objspace_t *objspace, VALUE obj)
{
    check_rvalue_consistency(objspace, obj);
    return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
}
 
static inline int
RVALUE_MARKING(rb_objspace_t *objspace, VALUE obj)
{
    check_rvalue_consistency(objspace, obj);
    return RVALUE_MARKING_BITMAP(obj) != 0;
}
 
static inline int
RVALUE_REMEMBERED(rb_objspace_t *objspace, VALUE obj)
{
    check_rvalue_consistency(objspace, obj);
    return MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
}
 
static inline int
RVALUE_UNCOLLECTIBLE(rb_objspace_t *objspace, VALUE obj)
{
    check_rvalue_consistency(objspace, obj);
    return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
}
 
#define RVALUE_PAGE_WB_UNPROTECTED(page, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj))
#define RVALUE_PAGE_UNCOLLECTIBLE(page, obj)  MARKED_IN_BITMAP((page)->uncollectible_bits, (obj))
#define RVALUE_PAGE_MARKING(page, obj)        MARKED_IN_BITMAP((page)->marking_bits, (obj))
 
static void rgengc_remember(rb_objspace_t *objspace, VALUE obj);
static void gc_bitmaps_clear(rb_objspace_t *objspace, rb_heap_t *heap, bool clear_shref);
static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap);
static bool verify_pointer_in_any_heap_p(const void *ptr); /* cross-objspace ownership test */
 
static int
check_rvalue_consistency_force(rb_objspace_t *objspace, const VALUE obj, int terminate)
{
    int err = 0;
 
    /* Under a global GC the barrier stops every Ractor, so the cross-objspace walk
     * below is safe without the VM lock.  Sweeping an ownerless zombie objspace also
     * leaves GET_RACTOR() NULL, and taking the lock here would dereference it. */
    const bool world_stopped = objspace->flags.during_global_gc;
    /* The VM lock protects the cross-objspace walk while other Ractors run and
     * reallocate their heaps.  Not taken while this objspace is in GC: pages are stable
     * then, the cross-objspace walk needs the world stopped anyway, and the Ractor lock
     * may already be held (Ractor -> VM order inversion).  A global GC holds the barrier
     * and needs no lock. */
    const bool take_vm_lock = !world_stopped && !during_gc;
    unsigned int lev = 0;
    if (take_vm_lock) lev = RB_GC_VM_LOCK_NO_BARRIER();
    {
        if (SPECIAL_CONST_P(obj)) {
            fprintf(stderr, "check_rvalue_consistency: %p is a special const.\n", (void *)obj);
            err++;
        }
        else if (!is_pointer_to_heap(objspace, (void *)obj)) {
            /* obj may be a legitimate cross-objspace reference (a shareable object, an
             * in-flight shref payload); it is a non-object only if no objspace's heap
             * holds it.  A foreign object's mark/age/remembered bits belong to its
             * owner and reading them would race its local GC: skip per-object checks. */
            if (!world_stopped) {
                /* A mid-local-GC verify holds no barrier, so other Ractors reallocate
                 * heap_pages.sorted under verify_pointer_in_any_heap_p's page_index
                 * read.  Accept foreign pointers here; the global GC's world-stopped
                 * verify does the full existence check. */
            }
            else if (!verify_pointer_in_any_heap_p((void *)obj)) {
                struct heap_page *empty_page = objspace->empty_pages;
                while (empty_page) {
                    if ((uintptr_t)empty_page->body <= (uintptr_t)obj &&
                            (uintptr_t)obj < (uintptr_t)empty_page->body + HEAP_PAGE_SIZE) {
                        GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, empty_page));
                        fprintf(stderr, "check_rvalue_consistency: %p is in an empty page (%p).\n",
                                (void *)obj, (void *)empty_page);
                        err++;
                        goto skip;
                    }
                    empty_page = empty_page->free_next;
                }
                fprintf(stderr, "check_rvalue_consistency: %p is not a Ruby object.\n", (void *)obj);
                err++;
              skip:
                ;
            }
        }
        else {
            const int wb_unprotected_bit = RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
            const int uncollectible_bit = RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
            const int mark_bit = RVALUE_MARKED_BITMAP(obj) != 0;
            const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0;
            const int remembered_bit = MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
            const int age = RVALUE_AGE_GET((VALUE)obj);
 
            if (heap_page_in_global_empty_pages_pool(objspace, GET_HEAP_PAGE(obj))) {
                fprintf(stderr, "check_rvalue_consistency: %s is in tomb page.\n", rb_obj_info(obj));
                err++;
            }
            if (BUILTIN_TYPE(obj) == T_NONE) {
                fprintf(stderr, "check_rvalue_consistency: %s is T_NONE.\n", rb_obj_info(obj));
                err++;
            }
            if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
                fprintf(stderr, "check_rvalue_consistency: %s is T_ZOMBIE.\n", rb_obj_info(obj));
                err++;
            }
 
            /* Do not run the memsize probe once an inconsistency (a T_NONE, say) was
             * found: an rb_bug inside the probe would lose the real diagnosis. */
            if (err == 0 && BUILTIN_TYPE(obj) != T_DATA) {
                rb_obj_memsize_of((VALUE)obj);
            }
 
            /* check generation
             *
             * OLD == age == 3 && old-bitmap && mark-bit (except incremental marking)
             */
            if (age > 0 && wb_unprotected_bit) {
                fprintf(stderr, "check_rvalue_consistency: %s is not WB protected, but age is %d > 0.\n", rb_obj_info(obj), age);
                err++;
            }
 
            if (!is_marking(objspace) && uncollectible_bit && !mark_bit) {
                fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but is not marked while !gc.\n", rb_obj_info(obj));
                err++;
            }
 
            if (!is_full_marking(objspace)) {
                if (uncollectible_bit && age != RVALUE_OLD_AGE && !wb_unprotected_bit) {
                    fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but not old (age: %d) and not WB unprotected.\n",
                            rb_obj_info(obj), age);
                    err++;
                }
                if (remembered_bit && age != RVALUE_OLD_AGE) {
                    fprintf(stderr, "check_rvalue_consistency: %s is remembered, but not old (age: %d).\n",
                            rb_obj_info(obj), age);
                    err++;
                }
            }
 
            /*
             * check coloring
             *
             *               marking:false marking:true
             * marked:false  white         *invalid*
             * marked:true   black         grey
             */
            if (is_incremental_marking(objspace) && marking_bit) {
                if (!is_marking(objspace) && !mark_bit) {
                    fprintf(stderr, "check_rvalue_consistency: %s is marking, but not marked.\n", rb_obj_info(obj));
                    err++;
                }
            }
        }
    }
    if (take_vm_lock) RB_GC_VM_UNLOCK_NO_BARRIER(lev);
 
    if (err > 0 && terminate) {
        rb_bug("check_rvalue_consistency_force: there is %d errors.", err);
    }
    return err;
}
 
#if RGENGC_CHECK_MODE == 0
static inline VALUE
check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj)
{
    return obj;
}
#else
static VALUE
check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj)
{
    check_rvalue_consistency_force(objspace, obj, TRUE);
    return obj;
}
#endif
 
static inline bool
gc_object_moved_p(rb_objspace_t *objspace, VALUE obj)
{
 
    bool ret;
    asan_unpoisoning_object(obj) {
        ret = BUILTIN_TYPE(obj) == T_MOVED;
    }
    return ret;
}
 
static inline int
RVALUE_OLD_P(rb_objspace_t *objspace, VALUE obj)
{
    GC_ASSERT(!RB_SPECIAL_CONST_P(obj));
    check_rvalue_consistency(objspace, obj);
    // Because this will only ever be called on GC controlled objects,
    // we can use the faster _RAW function here
    return RB_OBJ_PROMOTED_RAW(obj);
}
 
static inline void
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
    MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
    /* Count a promotion in the object's own objspace: a global GC ages every objspace's
     * slots from the driver, and counting them there would skew the other objspaces'
     * old_objects and with it their major GC frequency. */
    page->objspace->rgengc.old_objects++;
 
#if RGENGC_PROFILE >= 2
    objspace->profile.total_promoted_count++;
    objspace->profile.promoted_types[BUILTIN_TYPE(obj)]++;
#endif
}
 
static inline void
RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, VALUE obj)
{
    RB_DEBUG_COUNTER_INC(obj_promote);
    RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, GET_HEAP_PAGE(obj), obj);
}
 
/* set age to age+1 */
static inline void
RVALUE_AGE_INC(rb_objspace_t *objspace, VALUE obj)
{
    int age = RVALUE_AGE_GET((VALUE)obj);
 
    if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) {
        rb_bug("RVALUE_AGE_INC: can not increment age of OLD object %s.", rb_obj_info(obj));
    }
 
    age++;
    RVALUE_AGE_SET(obj, age);
 
    if (age == RVALUE_OLD_AGE) {
        RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
    }
 
    check_rvalue_consistency(objspace, obj);
}
 
static inline void
RVALUE_AGE_SET_CANDIDATE(rb_objspace_t *objspace, VALUE obj)
{
    check_rvalue_consistency(objspace, obj);
    GC_ASSERT(!RVALUE_OLD_P(objspace, obj));
    RVALUE_AGE_SET(obj, RVALUE_OLD_AGE - 1);
    check_rvalue_consistency(objspace, obj);
}
 
static inline void
RVALUE_AGE_RESET(VALUE obj)
{
    RVALUE_AGE_SET(obj, 0);
}
 
static inline void
RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj)
{
    check_rvalue_consistency(objspace, obj);
    GC_ASSERT(RVALUE_OLD_P(objspace, obj));
 
    if (!is_incremental_marking(objspace) && RVALUE_REMEMBERED(objspace, obj)) {
        struct heap_page *page = GET_HEAP_PAGE(obj);
        _CLEAR_IN_BITMAP(page->remembered_bits, page, obj);
    }
 
    CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
    RVALUE_AGE_RESET(obj);
 
    if (RVALUE_MARKED(objspace, obj)) {
        /* symmetric with RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET */
        GET_HEAP_PAGE(obj)->objspace->rgengc.old_objects--;
    }
 
    check_rvalue_consistency(objspace, obj);
}
 
static inline int
RVALUE_BLACK_P(rb_objspace_t *objspace, VALUE obj)
{
    return RVALUE_MARKED(objspace, obj) && !RVALUE_MARKING(objspace, obj);
}
 
static inline int
RVALUE_WHITE_P(rb_objspace_t *objspace, VALUE obj)
{
    return !RVALUE_MARKED(objspace, obj);
}
 
bool
rb_gc_impl_user_gc_disabled_set(void *objspace_ptr, bool disable)
{
    rb_objspace_t *objspace = objspace_ptr;
    const bool was = objspace->flags.user_gc_disabled;
    objspace->flags.user_gc_disabled = disable;
    return was;
}
 
bool
rb_gc_impl_user_gc_disabled_p(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
    return objspace->flags.user_gc_disabled;
}
 
bool
rb_gc_impl_gc_enabled_p(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
    return !dont_gc_val();
}
 
void
rb_gc_impl_gc_enable(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    dont_gc_off();
}
 
void
rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    if (finish_current_gc) {
        gc_rest(objspace);
    }
 
    dont_gc_on();
}
 
/* Finish an incremental mark or lazy sweep in progress without changing the enabled
 * state.  gc.c uses it to settle the only objspace just before the process goes
 * multi-objspace. */
void
rb_gc_impl_gc_rest(void *objspace_ptr)
{
    gc_rest(objspace_ptr);
}
 
/*
  --------------------------- ObjectSpace -----------------------------
*/
 
static inline void *
calloc1(size_t n)
{
    return calloc(1, n);
}
 
void
rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event)
{
    rb_objspace_t *objspace = objspace_ptr;
    /* FREEOBJ is main-objspace only (rb_objspace_set_event_hook masks it elsewhere). */
    GC_ASSERT(!(event & RUBY_INTERNAL_EVENT_FREEOBJ) ||
              objspace == global_objspace->main_objspace);
    objspace->hook_events = event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK;
}
 
unsigned long long
rb_gc_impl_get_total_time(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    unsigned long long marking_time = objspace->profile.marking_time_ns;
    unsigned long long sweeping_time = objspace->profile.sweeping_time_ns;
 
    return marking_time + sweeping_time;
}
 
void
rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    objspace->flags.measure_gc = RTEST(flag) ? TRUE : FALSE;
}
 
bool
rb_gc_impl_get_measure_total_time(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    return objspace->flags.measure_gc;
}
 
static void
gc_process_stat_capture(const rb_objspace_t *objspace,
                        struct gc_process_stat_snapshot *out)
{
    out->minor_gc_count = objspace->profile.minor_gc_count;
    out->major_gc_count = objspace->profile.major_gc_count;
    out->global_gc_invocation_count = objspace->profile.global_gc_invocation_count;
    out->marking_time_ns = objspace->profile.marking_time_ns;
    out->sweeping_time_ns = objspace->profile.sweeping_time_ns;
}
 
static void
gc_process_stat_publish(rb_objspace_t *objspace)
{
    struct gc_process_stat_snapshot snap;
    GC_ASSERT(objspace->profile.count == objspace->profile.minor_gc_count +
              objspace->profile.major_gc_count + objspace->profile.global_gc_count);
    gc_process_stat_capture(objspace, &snap);
    rb_native_mutex_lock(&objspace->process_stat.lock);
    objspace->process_stat.published = snap;
    rb_native_mutex_unlock(&objspace->process_stat.lock);
}
 
static void
gc_process_stat_add(struct gc_process_stat_total *dst,
                    const struct gc_process_stat_snapshot *src)
{
    dst->minor_gc_count += src->minor_gc_count;
    dst->major_gc_count += src->major_gc_count;
    dst->global_gc_invocation_count += src->global_gc_invocation_count;
    dst->marking_time_ns += src->marking_time_ns;
    dst->sweeping_time_ns += src->sweeping_time_ns;
}
 
/* garbage objects will be collected soon. */
bool
rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    /* A foreign object is a live leaf: reading its type or mark bit would race the
     * owner's local GC, so outside a global GC's barrier never report it as garbage.
     * The fstring/symbol weak-set lookups do reach across objspaces, but those objects
     * are born shareable and only a stop-the-world global GC collects them, so "not
     * garbage" is correct. */
    if (gc_skip_foreign_object_p(objspace, ptr)) {
        return false;
    }
 
    /* Asking whether a freed (T_NONE), moved (T_MOVED), or finalized (T_ZOMBIE)
     * object is garbage gives an unreliable answer: the slot may since have been
     * reused for an unrelated object. A reference to one of these is stale and a
     * bug in the caller. */
    asan_unpoisoning_object(ptr) {
        GC_ASSERT(BUILTIN_TYPE(ptr) != T_NONE);
        GC_ASSERT(BUILTIN_TYPE(ptr) != T_MOVED);
        GC_ASSERT(BUILTIN_TYPE(ptr) != T_ZOMBIE);
    }
 
    return is_lazy_sweeping(objspace) && GET_HEAP_PAGE(ptr)->flags.before_sweep &&
        !RVALUE_MARKED(objspace, ptr);
}
 
struct rb_gc_vm_context *
rb_gc_impl_get_vm_context(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    return &objspace->vm_context;
}
 
static void free_stack_chunks(mark_stack_t *);
static void mark_stack_free_cache(mark_stack_t *);
static void heap_page_free(rb_objspace_t *objspace, struct heap_page *page);
 
static inline void
gc_check_obj_in_page(struct heap_page *page, VALUE obj)
{
    if (RGENGC_CHECK_MODE &&
        /* obj should belong to page */
        !(page->start <= (uintptr_t)obj &&
          (uintptr_t)obj   <  ((uintptr_t)page->start + (page->total_slots * page->slot_size)) &&
          obj % sizeof(VALUE) == 0)) {
        rb_bug("gc_check_obj_in_page: %p is not rvalue.", (void *)obj);
    }
}
 
static inline void
heap_page_add_free_region(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
    rb_asan_unpoison_object(obj, false);
 
    // Should have already been reset
    GC_ASSERT(RVALUE_AGE_GET(obj) == 0);
 
    gc_check_obj_in_page(page, obj);
 
    asan_unlock_freelist(page);
 
    /* Keep a freed slot from carrying its old shareable and shref bits into the next
     * object born there. */
    CLEAR_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj);
    CLEAR_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj);
 
    struct free_region *region = (struct free_region *)obj;
    region->flags = 0;
    region->end = (uintptr_t)obj + page->slot_size;
    region->next = page->free_region;
    page->free_region = region;
 
    asan_lock_freelist(page);
 
    rb_asan_poison_object(obj);
    gc_report(3, objspace, "heap_page_add_free_region: %p\n", (void *)obj);
}
 
/* The initial size is per objspace, so a Ractor's own gets a smaller one than
 * main's rather than paying main's again. */
static inline size_t
objspace_heap_init_bytes(const rb_objspace_t *objspace)
{
    return objspace == global_objspace->main_objspace
        ? gc_params.heap_init_bytes : gc_params.ractor_heap_init_bytes;
}
 
static void
heap_allocatable_bytes_expand(rb_objspace_t *objspace,
        rb_heap_t *heap, size_t free_slots, size_t total_slots, size_t slot_size)
{
    double goal_ratio = gc_params.heap_free_slots_goal_ratio;
    size_t target_total_slots;
 
    if (goal_ratio == 0.0) {
        target_total_slots = (size_t)(total_slots * gc_params.growth_factor);
    }
    else if (total_slots == 0) {
        target_total_slots = objspace_heap_init_bytes(objspace) / slot_size;
    }
    else {
        /* Find `f' where free_slots = f * total_slots * goal_ratio
         * => f = (total_slots - free_slots) / ((1 - goal_ratio) * total_slots)
         */
        double f = (double)(total_slots - free_slots) / ((1 - goal_ratio) * total_slots);
 
        if (f > gc_params.growth_factor) f = gc_params.growth_factor;
        if (f < 1.0) f = 1.1;
 
        target_total_slots = (size_t)(f * total_slots);
 
        if (0) {
            fprintf(stderr,
                    "free_slots(%8"PRIuSIZE")/total_slots(%8"PRIuSIZE")=%1.2f,"
                    " G(%1.2f), f(%1.2f),"
                    " total_slots(%8"PRIuSIZE") => target_total_slots(%8"PRIuSIZE")\n",
                    free_slots, total_slots, free_slots/(double)total_slots,
                    goal_ratio, f, total_slots, target_total_slots);
        }
    }
 
    if (gc_params.growth_max_bytes > 0) {
        size_t max_total_slots = total_slots + gc_params.growth_max_bytes / slot_size;
        if (target_total_slots > max_total_slots) target_total_slots = max_total_slots;
    }
 
    size_t extend_slot_count = target_total_slots - total_slots;
    /* Extend by at least 1 page. */
    if (extend_slot_count == 0) extend_slot_count = 1;
 
    objspace->heap_pages.allocatable_bytes += extend_slot_count * slot_size;
}
 
static inline void
heap_add_freepage(rb_heap_t *heap, struct heap_page *page)
{
    asan_unlock_freelist(page);
    GC_ASSERT(page->free_slots != 0);
    GC_ASSERT(page->free_region != NULL);
 
    page->free_next = heap->free_pages;
    heap->free_pages = page;
 
    RUBY_DEBUG_LOG("page:%p free_region:%p", (void *)page, (void *)page->free_region);
 
    asan_lock_freelist(page);
}
 
static inline void
heap_add_poolpage(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
    asan_unlock_freelist(page);
    GC_ASSERT(page->free_slots != 0);
    GC_ASSERT(page->free_region != NULL);
 
    page->free_next = heap->pooled_pages;
    heap->pooled_pages = page;
    objspace->rincgc.pooled_slots += page->free_slots;
 
    asan_lock_freelist(page);
}
 
static void
heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
    ccan_list_del(&page->page_node);
    heap->total_pages--;
    heap->total_slots -= page->total_slots;
}
 
/* Detach a page with no living objects from its heap and park it on the objspace's
 * empty pages, from where it is reused or released to the page pool.  The caller has
 * already taken it off the heap's free_pages chain, if it was on it. */
static void
heap_page_detach_empty(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
    heap_unlink_page(objspace, heap, page);
 
    page->start = 0;
    page->total_slots = 0;
    page->slot_size = 0;
    page->heap = NULL;
    page->free_slots = 0;
 
    asan_unlock_freelist(page);
    page->free_region = NULL;
    asan_lock_freelist(page);
 
    asan_poison_memory_region(page->body, HEAP_PAGE_SIZE);
 
    objspace->empty_pages_count++;
    page->free_next = objspace->empty_pages;
    objspace->empty_pages = page;
}
 
static void
gc_aligned_free(void *ptr, size_t size)
{
#if defined __MINGW32__
    __mingw_aligned_free(ptr);
#elif defined _WIN32
    _aligned_free(ptr);
#elif defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN)
    free(ptr);
#else
    free(((void**)ptr)[-1]);
#endif
}
 
static void
heap_page_body_free(struct heap_page_body *page_body, struct page_arena *arena)
{
    GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
 
    page_pool_release(page_body, arena);
}
 
#ifdef PAGE_POOL_LOCK_ERRORCHECK
# define ASSERT_PAGE_POOL_LOCKED(g) GC_ASSERT(pthread_mutex_lock(&(g)->page_pool.lock) == EDEADLK)
#else
# define ASSERT_PAGE_POOL_LOCKED(g) ((void)0)
#endif
 
/* Insert into page_index.  Writers serialize on page_pool.lock; lomem and himem are a
 * monotonically growing over-approximation used for a quick reject. */
static void
global_page_index_insert(struct heap_page *page)
{
    rb_global_objspace_t *g = global_objspace;
    uintptr_t body = (uintptr_t)page->body;
 
    rb_native_mutex_lock(&g->page_pool.lock);
    if (g->page_index.n_pages == g->page_index.capa) {
        size_t new_capa = g->page_index.capa ? g->page_index.capa * 2 : 128;
        struct heap_page **grown = realloc(g->page_index.pages, new_capa * sizeof(*grown));
        if (grown == NULL) rb_bug("global_page_index_insert: realloc failed");
        g->page_index.pages = grown;
        g->page_index.capa = new_capa;
    }
    size_t lo = 0, hi = g->page_index.n_pages;
    while (lo < hi) {
        size_t mid = (lo + hi) / 2;
        if ((uintptr_t)g->page_index.pages[mid]->body < body) lo = mid + 1;
        else hi = mid;
    }
    memmove(&g->page_index.pages[lo + 1], &g->page_index.pages[lo],
            (g->page_index.n_pages - lo) * sizeof(struct heap_page *));
    g->page_index.pages[lo] = page;
    g->page_index.n_pages++;
 
    uintptr_t start = body + sizeof(struct heap_page_header);
    uintptr_t end = body + HEAP_PAGE_SIZE;
    if (g->page_index.lomem == 0 || g->page_index.lomem > start) g->page_index.lomem = start;
    if (g->page_index.himem < end) g->page_index.himem = end;
    rb_native_mutex_unlock(&g->page_pool.lock);
}
 
static void
global_page_index_remove_locked(const struct heap_page *page)
{
    rb_global_objspace_t *g = global_objspace;
    uintptr_t body = (uintptr_t)page->body;
 
    ASSERT_PAGE_POOL_LOCKED(g);
 
    size_t lo = 0, hi = g->page_index.n_pages;
    while (lo < hi) {
        size_t mid = (lo + hi) / 2;
        if ((uintptr_t)g->page_index.pages[mid]->body < body) lo = mid + 1;
        else hi = mid;
    }
    GC_ASSERT(lo < g->page_index.n_pages && g->page_index.pages[lo] == page);
    memmove(&g->page_index.pages[lo], &g->page_index.pages[lo + 1],
            (g->page_index.n_pages - lo - 1) * sizeof(struct heap_page *));
    g->page_index.n_pages--;
}
 
static void
global_page_index_remove(const struct heap_page *page)
{
    rb_global_objspace_t *g = global_objspace;
 
    rb_native_mutex_lock(&g->page_pool.lock);
    global_page_index_remove_locked(page);
    rb_native_mutex_unlock(&g->page_pool.lock);
}
 
static void
heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
{
    global_page_index_remove(page);
    objspace->heap_pages.freed_pages++;
    heap_page_body_free(page->body, page->arena);
    free(page);
}
 
static void
heap_pages_free_batch(rb_objspace_t *objspace, struct heap_page *pages)
{
    rb_global_objspace_t *g = global_objspace;
 
    rb_native_mutex_lock(&g->page_pool.lock);
    for (struct heap_page *page = pages; page != NULL; page = page->free_next) {
        global_page_index_remove_locked(page);
        if (HEAP_PAGE_ALLOC_USE_MMAP) {
#ifdef HAVE_MMAP
            page_pool_release_locked(page->body, page->arena);
#endif
        }
    }
    rb_native_mutex_unlock(&g->page_pool.lock);
 
    if (!HEAP_PAGE_ALLOC_USE_MMAP) {
        /* gc_aligned_free does not need the pool lock. */
        for (struct heap_page *page = pages; page != NULL; page = page->free_next) {
            heap_page_body_free(page->body, page->arena);
        }
    }
 
    while (pages != NULL) {
        struct heap_page *next = pages->free_next;
        objspace->heap_pages.freed_pages++;
        free(pages);
        pages = next;
    }
}
 
static void
heap_pages_free_unused_pages(rb_objspace_t *objspace)
{
    if (objspace->empty_pages != NULL && heap_pages_freeable_pages > 0) {
        GC_ASSERT(objspace->empty_pages_count > 0);
        objspace->empty_pages = NULL;
        objspace->empty_pages_count = 0;
 
        size_t i, j;
        struct heap_page *to_free = NULL;
        for (i = j = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
            struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
 
            if (heap_page_in_global_empty_pages_pool(objspace, page) && heap_pages_freeable_pages > 0) {
                page->free_next = to_free;
                to_free = page;
                heap_pages_freeable_pages--;
            }
            else {
                if (heap_page_in_global_empty_pages_pool(objspace, page)) {
                    page->free_next = objspace->empty_pages;
                    objspace->empty_pages = page;
                    objspace->empty_pages_count++;
                }
 
                if (i != j) {
                    rb_darray_set(objspace->heap_pages.sorted, j, page);
                }
                j++;
            }
        }
 
        rb_darray_pop(objspace->heap_pages.sorted, i - j);
        GC_ASSERT(rb_darray_size(objspace->heap_pages.sorted) == j);
 
        /* A retire GC can free every page, so an empty objspace is legitimate. */
        if (j > 0) {
            struct heap_page *hipage = rb_darray_get(objspace->heap_pages.sorted, rb_darray_size(objspace->heap_pages.sorted) - 1);
            uintptr_t himem = (uintptr_t)hipage->body + HEAP_PAGE_SIZE;
            GC_ASSERT(himem <= heap_pages_himem);
            heap_pages_himem = himem;
 
            struct heap_page *lopage = rb_darray_get(objspace->heap_pages.sorted, 0);
            uintptr_t lomem = (uintptr_t)lopage->body + sizeof(struct heap_page_header);
            GC_ASSERT(lomem >= heap_pages_lomem);
            heap_pages_lomem = lomem;
        }
        else {
            heap_pages_lomem = 0;
            heap_pages_himem = 0;
        }
 
        heap_pages_free_batch(objspace, to_free);
    }
}
 
static void *
gc_aligned_malloc(size_t alignment, size_t size)
{
    /* alignment must be a power of 2 */
    GC_ASSERT(((alignment - 1) & alignment) == 0);
    GC_ASSERT(alignment % sizeof(void*) == 0);
 
    void *res;
 
#if defined __MINGW32__
    res = __mingw_aligned_malloc(size, alignment);
#elif defined _WIN32
    res = _aligned_malloc(size, alignment);
#elif defined(HAVE_POSIX_MEMALIGN)
    if (posix_memalign(&res, alignment, size) != 0) {
        return NULL;
    }
#elif defined(HAVE_MEMALIGN)
    res = memalign(alignment, size);
#else
    char* aligned;
    res = malloc(alignment + size + sizeof(void*));
    aligned = (char*)res + alignment + sizeof(void*);
    aligned -= ((VALUE)aligned & (alignment - 1));
    ((void**)aligned)[-1] = res;
    res = (void*)aligned;
#endif
 
    GC_ASSERT((uintptr_t)res % alignment == 0);
 
    return res;
}
 
/* The page pool (global_objspace->page_pool): heap page bodies are carved out of large
 * arenas and reused through the pool.  Free bodies are split into a small global hot
 * list (≤ PAGE_POOL_HOT_MAX, never madvise'd) and per-arena cold freelists (eligible for
 * OS release — see page_pool_reclaim). Both lists use an in-body link at offset 0. */
 
#define PAGE_POOL_ARENA_SIZE (HEAP_PAGE_SIZE * 32) /* 2MiB with 64KiB pages */
#define PAGE_POOL_ARENA_BODIES (PAGE_POOL_ARENA_SIZE / HEAP_PAGE_SIZE) /* 32 */
#define PAGE_POOL_HOT_MAX 0 /* disabled — empty_pages is the retention buffer */
#define PAGE_POOL_ARENA_KEEP_HALF (PAGE_POOL_ARENA_BODIES / 2) /* 16 */
 
/* Steal bit 0 of the in-body link word: set iff the body has been madvise'd (cold). */
#define PAGE_POOL_ADVISED_BIT ((uintptr_t)1)
 
/* While a body is free, the arena back-pointer is stored at offset sizeof(header) — one
 * word past the link, inside the spared first OS page.  PAGE_POOL_SCRATCH_SIZE covers
 * both the link (offset 0) and the tag for ASAN unpoison. */
#define PAGE_POOL_BODY_ARENA(body) \
    (*(struct page_arena **)((char *)(body) + sizeof(struct heap_page_header)))
#define PAGE_POOL_SCRATCH_SIZE (sizeof(struct heap_page_header) + sizeof(void *))
 
#ifdef HAVE_MMAP
/* mmap a new arena to carve from.  Called with the pool lock held, at which point the
 * previous arena is always fully carved. */
static bool
page_pool_add_arena(rb_global_objspace_t *g)
{
    GC_ASSERT(HEAP_PAGE_ALIGN % sysconf(_SC_PAGE_SIZE) == 0);
 
    size_t mmap_size = PAGE_POOL_ARENA_SIZE + HEAP_PAGE_ALIGN;
    char *ptr = mmap(NULL, mmap_size,
                     PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
    if (ptr == MAP_FAILED) {
        return false;
    }
 
    // If we are building `default.c` as part of the ruby executable, we
    // may just call `ruby_annotate_mmap`.  But if we are building
    // `default.c` as a shared library, we will not have access to private
    // symbols, and we have to either call prctl directly or make our own
    // wrapper.
#if defined(HAVE_SYS_PRCTL_H) && defined(PR_SET_VMA) && defined(PR_SET_VMA_ANON_NAME)
    prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, mmap_size, "Ruby:GC:default:page_pool_arena");
    errno = 0;
#endif
 
    /* Trim the unaligned head and tail so the usable area is HEAP_PAGE_ALIGN aligned. */
    char *aligned = ptr + HEAP_PAGE_ALIGN;
    aligned -= ((uintptr_t)aligned & (HEAP_PAGE_ALIGN - 1));
    GC_ASSERT(aligned > ptr);
    GC_ASSERT(aligned <= ptr + HEAP_PAGE_ALIGN);
 
    size_t start_out_of_range_size = aligned - ptr;
    GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
    if (start_out_of_range_size > 0) {
        if (munmap(ptr, start_out_of_range_size)) {
            rb_bug("page_pool_add_arena: munmap failed for start");
        }
    }
 
    size_t end_out_of_range_size = HEAP_PAGE_ALIGN - start_out_of_range_size;
    GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
    if (end_out_of_range_size > 0) {
        if (munmap(aligned + PAGE_POOL_ARENA_SIZE, end_out_of_range_size)) {
            rb_bug("page_pool_add_arena: munmap failed for end");
        }
    }
 
    struct page_arena *arena = calloc1(sizeof(struct page_arena));
    if (arena == NULL) {
        if (munmap(aligned, PAGE_POOL_ARENA_SIZE)) {
            rb_bug("page_pool_add_arena: munmap failed for arena");
        }
        return false;
    }
    arena->start = aligned;
    arena->size = PAGE_POOL_ARENA_SIZE;
    arena->cold_freelist = NULL;
    arena->free_count = 0;
    arena->cold_count = 0;
    arena->next = g->page_pool.arenas;
    g->page_pool.arenas = arena;
    g->page_pool.arena_count++;
    g->page_pool.arena_current = arena;
 
    g->page_pool.arena_cursor = aligned;
    g->page_pool.arena_end = aligned + PAGE_POOL_ARENA_SIZE;
 
    return true;
}
#endif
 
static struct heap_page_body *
page_pool_acquire(struct page_arena **arena_out)
{
    struct heap_page_body *body = NULL;
 
    if (HEAP_PAGE_ALLOC_USE_MMAP) {
#ifdef HAVE_MMAP
        bool need_reuse = false;
        rb_global_objspace_t *g = global_objspace;
 
        rb_native_mutex_lock(&g->page_pool.lock);
        if (g->page_pool.hot_list != NULL) {
            body = g->page_pool.hot_list;
            asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false);
            uintptr_t link = *(uintptr_t *)body;
            g->page_pool.hot_list = (struct heap_page_body *)(link & ~PAGE_POOL_ADVISED_BIT);
            g->page_pool.hot_count--;
            struct page_arena *arena = PAGE_POOL_BODY_ARENA(body);
            arena->free_count--;
            *arena_out = arena;
        }
        else {
            // find cold page body (madvised reusable)
            for (struct page_arena *a = g->page_pool.arenas; a; a = a->next) {
                if (a->cold_count > 0) {
                    body = a->cold_freelist;
                    asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false);
                    uintptr_t link = *(uintptr_t *)body;
                    a->cold_freelist = (struct heap_page_body *)(link & ~PAGE_POOL_ADVISED_BIT);
                    a->cold_count--;
                    a->free_count--;
                    *arena_out = a;
                    need_reuse = (link & PAGE_POOL_ADVISED_BIT) != 0;
                    if (need_reuse) g->page_pool.advised_count--;
                    break;
                }
            }
            if (body == NULL &&
                (g->page_pool.arena_cursor != g->page_pool.arena_end ||
                 page_pool_add_arena(g))) {
                GC_ASSERT(g->page_pool.arena_cursor + HEAP_PAGE_SIZE <= g->page_pool.arena_end);
                body = (struct heap_page_body *)g->page_pool.arena_cursor;
                g->page_pool.arena_cursor += HEAP_PAGE_SIZE;
                *arena_out = g->page_pool.arena_current;
            }
        }
        rb_native_mutex_unlock(&g->page_pool.lock);
 
        if (body != NULL) {
            if (need_reuse) {
                rb_vm_map_reuse((char *)body + g->page_pool.os_page_size,
                                 HEAP_PAGE_SIZE - g->page_pool.os_page_size);
            }
            asan_unpoison_memory_region(body, HEAP_PAGE_SIZE, false);
        }
#endif
    }
    else {
        body = gc_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
        *arena_out = NULL;
    }
 
    return body;
}
 
#ifdef HAVE_MMAP
static void
page_pool_release_locked(struct heap_page_body *body, struct page_arena *arena)
{
    rb_global_objspace_t *g = global_objspace;
 
    ASSERT_PAGE_POOL_LOCKED(g);
 
    /* A body in the empty-pages pool stays fully poisoned (see gc_sweep_page), so
     * unpoison the scratch area (link + arena tag) before writing. */
    asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false);
    arena->free_count++;
    PAGE_POOL_BODY_ARENA(body) = arena;
    if (g->page_pool.hot_count < PAGE_POOL_HOT_MAX) {
        *(uintptr_t *)body = (uintptr_t)g->page_pool.hot_list;
        g->page_pool.hot_list = body;
        g->page_pool.hot_count++;
    }
    else {
        *(uintptr_t *)body = (uintptr_t)arena->cold_freelist;
        arena->cold_freelist = body;
        arena->cold_count++;
    }
    asan_poison_memory_region(body, HEAP_PAGE_SIZE);
}
#endif
 
static void
page_pool_release(struct heap_page_body *body, struct page_arena *arena)
{
    if (HEAP_PAGE_ALLOC_USE_MMAP) {
#ifdef HAVE_MMAP
        rb_global_objspace_t *g = global_objspace;
 
        rb_native_mutex_lock(&g->page_pool.lock);
        page_pool_release_locked(body, arena);
        rb_native_mutex_unlock(&g->page_pool.lock);
#endif
    }
    else {
        gc_aligned_free(body, HEAP_PAGE_SIZE);
    }
}
 
/* Allow the OS to reclaim pool memory. Runs only at major GC in single-objspace mode
 * (see gc_sweep_finish).
 *
 * Step A: madvise cold bodies, sparing the first OS page (which holds the in-body
 * freelist link and arena tag).
 *
 * Step B: munmap arenas whose 32 bodies are all free, keeping one extra empty
 * arena as a retention buffer when the remaining free pool is < half an arena. */
static void
page_pool_reclaim(rb_global_objspace_t *g)
{
    if (!HEAP_PAGE_ALLOC_USE_MMAP) return;
#ifdef HAVE_MMAP
    size_t os_page_size = g->page_pool.os_page_size;
 
    rb_native_mutex_lock(&g->page_pool.lock);
 
    /* Advising spares the first OS page of a body (it holds the in-body freelist link
     * and the arena tag), so it needs sub-page granularity: when the OS page size is
     * >= HEAP_PAGE_SIZE (e.g. 64KiB pages on aarch64) no body is ever advised, and
     * advised_count must not be adjusted anywhere either. */
    const bool can_advise = os_page_size < HEAP_PAGE_SIZE;
 
    /* Step A — advise cold bodies (immediate release: drop RSS now if the platform allows). */
    if (can_advise) {
        for (struct page_arena *a = g->page_pool.arenas; a; a = a->next) {
            for (struct heap_page_body *body = a->cold_freelist; body; ) {
                asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false);
                uintptr_t link = *(uintptr_t *)body;
                struct heap_page_body *next =
                    (struct heap_page_body *)(link & ~PAGE_POOL_ADVISED_BIT);
                if (!(link & PAGE_POOL_ADVISED_BIT)) {
                    rb_vm_map_reusable_immediate((char *)body + os_page_size,
                                        HEAP_PAGE_SIZE - os_page_size, 0);
                    *(uintptr_t *)body = link | PAGE_POOL_ADVISED_BIT;
                    g->page_pool.advised_count++;
                }
                asan_poison_memory_region(body, PAGE_POOL_SCRATCH_SIZE);
                body = next;
            }
        }
    }
 
    /* Step B — munmap fully-free arenas (with retention buffer).
     *
     * total_free = Σ free_count; free_count already includes hot-list bodies
     * (page_pool_release increments it unconditionally), so no separate hot_count.
     * An arena is eligible when all 32 of its bodies are free AND none sit on
     * the hot list (≤5 entries, pre-scanned).  Keep one extra empty arena when
     * the rest of the free pool is < half an arena, to avoid thrash. */
    int total_free = 0;
    for (struct page_arena *a = g->page_pool.arenas; a; a = a->next) {
        total_free += a->free_count;
    }
 
    struct page_arena *hot_arenas[PAGE_POOL_HOT_MAX ? PAGE_POOL_HOT_MAX : 1];
    int n_hot_arenas = 0;
    /* Collect arenas that have a hot body (≤ PAGE_POOL_HOT_MAX entries). */
    for (struct heap_page_body *body = g->page_pool.hot_list; body; ) {
        asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false);
        uintptr_t link = *(uintptr_t *)body;
        struct heap_page_body *next =
            (struct heap_page_body *)(link & ~PAGE_POOL_ADVISED_BIT);
        struct page_arena *arena = PAGE_POOL_BODY_ARENA(body);
        bool found = false;
        for (int i = 0; i < n_hot_arenas; i++) {
            if (hot_arenas[i] == arena) { found = true; break; }
        }
        if (!found && n_hot_arenas < PAGE_POOL_HOT_MAX) {
            hot_arenas[n_hot_arenas++] = arena;
        }
        asan_poison_memory_region(body, PAGE_POOL_SCRATCH_SIZE);
        body = next;
    }
 
    bool retained_one = false;
    struct page_arena **pp = &g->page_pool.arenas;
    // munmap fully free arenas
    while (*pp) {
        struct page_arena *a = *pp;
        bool has_hot = false;
        for (int i = 0; i < n_hot_arenas; i++) {
            if (hot_arenas[i] == a) { has_hot = true; break; }
        }
        if (a->free_count != PAGE_POOL_ARENA_BODIES || has_hot) {
            pp = &a->next;
            continue;
        }
        GC_ASSERT(a->cold_count == PAGE_POOL_ARENA_BODIES);
 
        int free_elsewhere = total_free - PAGE_POOL_ARENA_BODIES;
        if (free_elsewhere < PAGE_POOL_ARENA_KEEP_HALF && !retained_one) {
            retained_one = true;
            pp = &a->next;
            continue;
        }
 
        *pp = a->next;
        if (munmap(a->start, a->size)) {
            rb_bug("page_pool_reclaim: munmap failed");
        }
        total_free -= PAGE_POOL_ARENA_BODIES;
        /* Every body of this arena is on its cold freelist, so Step A above has just
         * advised all of them -- but only if this platform can advise at all. */
        if (can_advise) {
            g->page_pool.advised_count -= PAGE_POOL_ARENA_BODIES;
            GC_ASSERT(g->page_pool.advised_count >= 0);
        }
        g->page_pool.arena_count--;
        g->page_pool.arenas_unmapped++;
        if (a == g->page_pool.arena_current) {
            // During next acquire, any remaining arenas that have cold bodies are used. This is guaranteed
            // because of the retention buffer.
            g->page_pool.arena_current = NULL;
            g->page_pool.arena_cursor = NULL;
            g->page_pool.arena_end = NULL;
        }
        free(a);
    }
 
    rb_native_mutex_unlock(&g->page_pool.lock);
#endif
}
 
static struct heap_page_body *
heap_page_body_allocate(struct page_arena **arena_out)
{
    struct heap_page_body *page_body = page_pool_acquire(arena_out);
 
    GC_ASSERT(page_body == NULL || (uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
 
    return page_body;
}
 
static struct heap_page *
heap_page_resurrect(rb_objspace_t *objspace)
{
    struct heap_page *page = NULL;
    if (objspace->empty_pages == NULL) {
        GC_ASSERT(objspace->empty_pages_count == 0);
    }
    else {
        GC_ASSERT(objspace->empty_pages_count > 0);
        objspace->empty_pages_count--;
        page = objspace->empty_pages;
        objspace->empty_pages = page->free_next;
        /* Clear the flags left over from emptying the page before reusing it, or the
         * shareable and shref scans would keep walking an empty bitmap forever. */
        page->flags.has_shareable_objects = FALSE;
        page->flags.has_shref_objects = FALSE;
    }
 
    return page;
}
 
static struct heap_page *
heap_page_allocate(rb_objspace_t *objspace)
{
    struct page_arena *arena;
    struct heap_page_body *page_body = heap_page_body_allocate(&arena);
    if (page_body == 0) {
        rb_memerror();
    }
 
    struct heap_page *page = calloc1(sizeof(struct heap_page));
    if (page == 0) {
        heap_page_body_free(page_body, arena);
        rb_memerror();
    }
 
    uintptr_t start = (uintptr_t)page_body + sizeof(struct heap_page_header);
    uintptr_t end = (uintptr_t)page_body + HEAP_PAGE_SIZE;
 
    size_t lo = 0;
    size_t hi = rb_darray_size(objspace->heap_pages.sorted);
    while (lo < hi) {
        struct heap_page *mid_page;
 
        size_t mid = (lo + hi) / 2;
        mid_page = rb_darray_get(objspace->heap_pages.sorted, mid);
        if ((uintptr_t)mid_page->start < start) {
            lo = mid + 1;
        }
        else if ((uintptr_t)mid_page->start > start) {
            hi = mid;
        }
        else {
            rb_bug("same heap page is allocated: %p at %"PRIuVALUE, (void *)page_body, (VALUE)mid);
        }
    }
 
    rb_darray_insert_without_gc(&objspace->heap_pages.sorted, hi, page);
 
    if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
    if (heap_pages_himem < end) heap_pages_himem = end;
 
    page->body = page_body;
    page->arena = arena;
    page_body->header.page = page;
    page->objspace = objspace;
 
    objspace->heap_pages.allocated_pages++;
 
    global_page_index_insert(page);
 
    return page;
}
 
static void
heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
    /* Adding to eden heap during incremental sweeping is forbidden */
    GC_ASSERT(!heap->sweeping_page);
    GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page));
 
    /* Align start to slot_size boundary */
    uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header);
    uintptr_t rem = start % heap->slot_size;
    if (rem) start += heap->slot_size - rem;
 
    int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size);
 
    page->start = start;
    page->total_slots = slot_count;
    page->slot_size = heap->slot_size;
    page->slot_size_reciprocal = heap_slot_reciprocal_table[heap - heaps];
    page->heap = heap;
 
    memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
    memset(&page->age_bits[0], 0, sizeof(page->age_bits));
 
    asan_unlock_freelist(page);
    asan_unpoison_memory_region(page->body, HEAP_PAGE_SIZE, false);
 
    uintptr_t slots_end = start + (uintptr_t)slot_count * heap->slot_size;
 
    memset((void *)start, 0, slots_end - start);
 
    struct free_region *region = (struct free_region *)start;
    region->flags = 0;
    region->end = slots_end;
    region->next = NULL;
    page->free_region = region;
 
    /* Poison every free slot; each is unpoisoned again as it is handed out. */
    for (uintptr_t p = start; p < slots_end; p += heap->slot_size) {
        rb_asan_poison_object((VALUE)p);
    }
    asan_lock_freelist(page);
 
    page->free_slots = slot_count;
 
    heap->total_allocated_pages++;
 
    ccan_list_add_tail(&heap->pages, &page->page_node);
    heap->total_pages++;
    heap->total_slots += page->total_slots;
}
 
static int
heap_page_allocate_and_initialize(rb_objspace_t *objspace, rb_heap_t *heap)
{
    gc_report(1, objspace, "heap_page_allocate_and_initialize: rb_darray_size(objspace->heap_pages.sorted): %"PRIdSIZE", "
                  "allocatable_bytes: %"PRIdSIZE", heap->total_pages: %"PRIdSIZE"\n",
                  rb_darray_size(objspace->heap_pages.sorted), objspace->heap_pages.allocatable_bytes, heap->total_pages);
 
    bool allocated = false;
    struct heap_page *page = heap_page_resurrect(objspace);
 
    if (page == NULL && objspace->heap_pages.allocatable_bytes > 0) {
        page = heap_page_allocate(objspace);
        allocated = true;
 
        GC_ASSERT(page != NULL);
    }
 
    if (page != NULL) {
        heap_add_page(objspace, heap, page);
        heap_add_freepage(heap, page);
 
        if (allocated) {
            size_t page_bytes = (size_t)page->total_slots * page->slot_size;
            if (objspace->heap_pages.allocatable_bytes > page_bytes) {
                objspace->heap_pages.allocatable_bytes -= page_bytes;
            }
            else {
                objspace->heap_pages.allocatable_bytes = 0;
            }
        }
    }
 
    return page != NULL;
}
 
static void
heap_page_allocate_and_initialize_force(rb_objspace_t *objspace, rb_heap_t *heap)
{
    size_t prev_allocatable_bytes = objspace->heap_pages.allocatable_bytes;
    objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE;
    heap_page_allocate_and_initialize(objspace, heap);
    GC_ASSERT(heap->free_pages != NULL);
    objspace->heap_pages.allocatable_bytes = prev_allocatable_bytes;
}
 
static void
gc_continue(rb_objspace_t *objspace, rb_heap_t *heap)
{
    unsigned int lock_lev;
    bool needs_gc = is_incremental_marking(objspace) || needs_continue_sweeping(objspace, heap);
    if (!needs_gc) return;
 
    gc_enter(objspace, gc_enter_event_continue, &lock_lev); // takes vm barrier, try to avoid
 
    /* Continue marking if in incremental marking. */
    if (is_incremental_marking(objspace)) {
        if (gc_marks_continue(objspace, heap)) {
            gc_sweep(objspace);
        }
    }
 
    if (needs_continue_sweeping(objspace, heap)) {
        gc_sweep_continue(objspace, heap);
    }
 
    gc_exit(objspace, gc_enter_event_continue, &lock_lev);
}
 
static void
heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap)
{
    GC_ASSERT(heap->free_pages == NULL);
 
    if (heap->total_slots < objspace_heap_init_bytes(objspace) / heap->slot_size &&
            heap->sweeping_page == NULL) {
        heap_page_allocate_and_initialize_force(objspace, heap);
        GC_ASSERT(heap->free_pages != NULL);
        return;
    }
 
    /* Continue incremental marking or lazy sweeping, if in any of those steps. */
    gc_continue(objspace, heap);
 
    if (heap->free_pages == NULL) {
        heap_page_allocate_and_initialize(objspace, heap);
    }
 
    /* If we still don't have a free page and not allowed to create a new page,
     * we should start a new GC cycle. */
    if (heap->free_pages == NULL) {
        GC_ASSERT(objspace->empty_pages_count == 0);
        GC_ASSERT(objspace->heap_pages.allocatable_bytes == 0);
 
        if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
            rb_memerror();
        }
        else {
            if (objspace->heap_pages.allocatable_bytes == 0 && !gc_config_full_mark_val) {
                heap_allocatable_bytes_expand(objspace, heap,
                        heap->freed_slots + heap->empty_slots,
                        heap->total_slots, heap->slot_size);
                GC_ASSERT(objspace->heap_pages.allocatable_bytes > 0);
            }
            /* Do steps of incremental marking or lazy sweeping if the GC run permits. */
            gc_continue(objspace, heap);
 
            /* If we're not incremental marking (e.g. a minor GC) or finished
             * sweeping and still don't have a free page, then
             * gc_sweep_finish_heap should allow us to create a new page. */
            if (heap->free_pages == NULL && !heap_page_allocate_and_initialize(objspace, heap)) {
                if (gc_needs_major_flags == GPR_FLAG_NONE) {
                    rb_bug("cannot create a new page after GC");
                }
                else { // Major GC is required, which will allow us to create new page
                    if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
                        rb_memerror();
                    }
                    else {
                        /* Do steps of incremental marking or lazy sweeping. */
                        gc_continue(objspace, heap);
 
                        if (heap->free_pages == NULL &&
                                !heap_page_allocate_and_initialize(objspace, heap)) {
                            rb_bug("cannot create a new page after major GC");
                        }
                    }
                }
            }
        }
    }
 
    GC_ASSERT(heap->free_pages != NULL);
}
 
#if GC_DEBUG
static inline const char*
rb_gc_impl_source_location_cstr(int *ptr)
{
    /* We could directly refer `rb_source_location_cstr()` before, but not any
     * longer.  We have to heavy lift using our debugging API. */
    if (! ptr) {
        return NULL;
    }
    else if (! (*ptr = rb_sourceline())) {
        return NULL;
    }
    else {
        return rb_sourcefile();
    }
}
#endif
 
static inline VALUE
newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, VALUE obj)
{
    GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
    GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
    RBASIC(obj)->flags = flags;
    *((VALUE *)&RBASIC(obj)->klass) = klass;
#if RBASIC_SHAPE_ID_FIELD
    RBASIC(obj)->shape_id = 0;
#endif
 
    if (RB_UNLIKELY(flags & RUBY_FL_SHAREABLE)) {
        /* A born-shareable object must be WB protected: the shref and remembered-set
         * rules for shareable objects assume the write barrier.  A local GC roots
         * shareable objects from this bit (pinned_roots_mark). */
        GC_ASSERT(wb_protected);
        gc_page_add_shareable(GET_HEAP_PAGE(obj), obj);
    }
 
#if RGENGC_CHECK_MODE
    int lev = RB_GC_VM_LOCK_NO_BARRIER();
    {
        check_rvalue_consistency(objspace, obj);
 
        GC_ASSERT(RVALUE_MARKED(objspace, obj) == FALSE);
        GC_ASSERT(RVALUE_MARKING(objspace, obj) == FALSE);
        GC_ASSERT(RVALUE_OLD_P(objspace, obj) == FALSE);
        GC_ASSERT(RVALUE_WB_UNPROTECTED(objspace, obj) == FALSE);
 
        if (RVALUE_REMEMBERED(objspace, obj)) rb_bug("newobj: %s is remembered.", rb_obj_info(obj));
    }
    RB_GC_VM_UNLOCK_NO_BARRIER(lev);
#endif
 
    if (RB_UNLIKELY(wb_protected == FALSE)) {
        MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
    }
 
#if RGENGC_PROFILE
    if (wb_protected) {
        objspace->profile.total_generated_normal_object_count++;
#if RGENGC_PROFILE >= 2
        objspace->profile.generated_normal_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
    }
    else {
        objspace->profile.total_generated_shady_object_count++;
#if RGENGC_PROFILE >= 2
        objspace->profile.generated_shady_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
    }
#endif
 
#if GC_DEBUG
    GET_RVALUE_OVERHEAD(obj)->file = rb_gc_impl_source_location_cstr(&GET_RVALUE_OVERHEAD(obj)->line);
    GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */
#endif
 
    gc_report(5, objspace, "newobj: %s\n", rb_obj_info(obj));
 
    // RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, rb_obj_info(obj));
    return obj;
}
 
size_t
rb_gc_impl_obj_slot_size(VALUE obj)
{
    return GET_HEAP_PAGE(obj)->slot_size - RVALUE_OVERHEAD;
}
 
bool
rb_gc_impl_pinned_p(void *objspace_ptr, VALUE obj)
{
    return RVALUE_PINNED((rb_objspace_t *)objspace_ptr, obj);
}
 
static inline size_t
heap_slot_size(unsigned char pool_id)
{
    GC_ASSERT(pool_id < HEAP_COUNT);
 
    return pool_slot_sizes[pool_id] - RVALUE_OVERHEAD;
}
 
size_t
rb_gc_impl_max_allocation_size(void)
{
    return heap_slot_size(HEAP_COUNT - 1);
}
 
bool
rb_gc_impl_size_allocatable_p(size_t size)
{
    return size <= rb_gc_impl_max_allocation_size();
}
 
static inline bool
heap_advance_region(rb_heap_t *heap)
{
    struct free_region *region = heap->newobj.alloc_next_region;
    if (region == NULL) {
        return false;
    }
 
    rb_asan_unpoison_object((VALUE)region, false);
    GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
    heap->newobj.alloc_cursor = (uintptr_t)region;
    heap->newobj.alloc_cursor_end = region->end;
    heap->newobj.alloc_next_region = region->next;
    rb_asan_poison_object((VALUE)region);
 
    return true;
}
 
/* The whole region is ours until the next refill, so charge it to the step now. */
static inline void
heap_charge_region(rb_objspace_t *objspace, const rb_heap_t *heap, size_t heap_idx)
{
    objspace->incremental_mark_step_allocated_slots +=
        (heap->newobj.alloc_cursor_end - heap->newobj.alloc_cursor) / pool_slot_sizes[heap_idx];
}
 
static inline VALUE
heap_alloc_slot(rb_objspace_t *objspace, size_t heap_idx)
{
    rb_heap_t *heap = &heaps[heap_idx];
 
    uintptr_t cursor = heap->newobj.alloc_cursor;
    if (RB_UNLIKELY(cursor >= heap->newobj.alloc_cursor_end)) {
        /* Marking owes us a step before the next region, and newobj_refill runs it. */
        if (RB_UNLIKELY(is_incremental_marking(objspace)) ||
            heap_advance_region(heap) == false) {
            return Qfalse;
        }
        cursor = heap->newobj.alloc_cursor;
    }
 
    VALUE obj = (VALUE)cursor;
    rb_asan_unpoison_object(obj, true);
    heap->newobj.alloc_cursor = cursor + pool_slot_sizes[heap_idx];
 
    /* Single writer (the owning Ractor under the GVL), so a plain increment is enough. */
    heap->total_allocated_objects++;
 
#if RGENGC_CHECK_MODE
    GC_ASSERT(rb_gc_impl_obj_slot_size(obj) == heap_slot_size(heap_idx));
    // zero clear
    MEMZERO((char *)obj, char, heap_slot_size(heap_idx));
#endif
    return obj;
}
 
static struct heap_page *
heap_next_free_page(rb_objspace_t *objspace, rb_heap_t *heap)
{
    struct heap_page *page;
 
    if (heap->free_pages == NULL) {
        heap_prepare(objspace, heap);
    }
 
    page = heap->free_pages;
    heap->free_pages = page->free_next;
 
    GC_ASSERT(page->free_slots != 0);
 
    asan_unlock_freelist(page);
 
    return page;
}
 
static inline void
heap_set_alloc_page(rb_objspace_t *objspace, size_t heap_idx, struct heap_page *page)
{
    gc_report(3, objspace, "heap_set_alloc_page: Using page %p\n", (void *)page->body);
 
    rb_heap_t *heap = &heaps[heap_idx];
 
    GC_ASSERT(heap->newobj.alloc_cursor >= heap->newobj.alloc_cursor_end);
    GC_ASSERT(heap->newobj.alloc_next_region == NULL);
    GC_ASSERT(page->free_slots != 0);
    GC_ASSERT(page->free_region != NULL);
 
    heap->newobj.alloc_using_page = page;
 
    struct free_region *region = page->free_region;
    rb_asan_unpoison_object((VALUE)region, false);
    GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
    heap->newobj.alloc_cursor = (uintptr_t)region;
    heap->newobj.alloc_cursor_end = region->end;
    heap->newobj.alloc_next_region = region->next;
    rb_asan_poison_object((VALUE)region);
 
    page->free_slots = 0;
    page->free_region = NULL;
}
 
static void
init_size_to_heap_idx(void)
{
    /* Process-wide and immutable, so build it once at boot.  A rebuild in a later
     * objspace_init would write the same values but race other threads' lock-free
     * allocation-fastpath reads. */
    static bool initialized = false;
    if (initialized) return;
    initialized = true;
 
    for (size_t i = 0; i < sizeof(size_to_heap_idx); i++) {
        size_t effective = i * 8 + RVALUE_OVERHEAD;
        uint8_t idx;
        for (idx = 0; idx < HEAP_COUNT; idx++) {
            if (effective <= pool_slot_sizes[idx]) break;
        }
        size_to_heap_idx[i] = idx;
    }
}
 
static inline size_t
heap_idx_for_size(size_t size)
{
    size_t compressed = (size + 7) >> 3;
    if (compressed < sizeof(size_to_heap_idx)) {
        size_t heap_idx = size_to_heap_idx[compressed];
        if (RB_LIKELY(heap_idx < HEAP_COUNT)) return heap_idx;
    }
 
    rb_bug("heap_idx_for_size: allocation size too large "
           "(size=%"PRIuSIZE")", size);
}
 
size_t
rb_gc_impl_size_slot_size(void *objspace_ptr, size_t size)
{
    return heap_slot_size((unsigned char)heap_idx_for_size(size));
}
 
bool
rb_gc_impl_zjit_new_obj_fastpath(void *objspace_ptr, size_t alloc_size, VALUE flags, VALUE klass,
                                 struct rb_gc_zjit_fastpath *fastpath)
{
#if USE_ZJIT
    size_t heap_idx = 0;
    size_t slot_size = 0;
    for (; heap_idx < HEAP_COUNT; heap_idx++) {
        if (alloc_size + RVALUE_OVERHEAD <= pool_slot_sizes[heap_idx]) {
            slot_size = pool_slot_sizes[heap_idx];
            break;
        }
    }
    if (slot_size == 0) return false;
 
#undef heaps
    size_t base = offsetof(rb_objspace_t, heaps)
                  + heap_idx * sizeof(rb_heap_t)
                  + offsetof(rb_heap_t, newobj);
#define heaps objspace->heaps
 
    struct rb_gc_zjit_default_new_obj_fastpath default_fastpath = {
        base + offsetof(rb_heap_newobj_t, alloc_cursor),
        base + offsetof(rb_heap_newobj_t, alloc_cursor_end),
        slot_size,
        base - offsetof(rb_heap_t, newobj) + offsetof(rb_heap_t, total_allocated_objects),
        flags,
        klass
    };
 
    memset(fastpath, 0, sizeof(*fastpath));
    fastpath->kind = RB_GC_ZJIT_FASTPATH_DEFAULT;
    memcpy(fastpath->data.words, &default_fastpath, sizeof(default_fastpath));
 
    return true;
#else
    return false;
#endif
}
 
int32_t
rb_gc_impl_zjit_writebarrier_required_offset(void)
{
    return (int32_t)offsetof(rb_objspace_t, zjit_during_incremental_marking);
}
 
NOINLINE(static VALUE newobj_refill(rb_objspace_t *objspace, size_t heap_idx));
 
static VALUE
newobj_refill(rb_objspace_t *objspace, size_t heap_idx)
{
    rb_heap_t *heap = &heaps[heap_idx];
    VALUE obj = Qfalse;
 
    /* No lock: a heap is single-writer (its owner thread, serialized by the GVL inside
     * the Ractor), the page pool has its own mutex, and a GC started from here takes
     * whatever gc_enter needs. */
    if (is_incremental_marking(objspace)) {
        /* The fast path sends us here at every region, which is far more often than the
         * step size, so step only once the regions add up to it. */
        if (objspace->incremental_mark_step_allocated_slots >= INCREMENTAL_MARK_STEP_ALLOCATIONS) {
            gc_continue(objspace, heap);
            objspace->incremental_mark_step_allocated_slots = 0;
        }
 
        // Move on to the region the fast path refused to take
        if (heap_advance_region(heap)) {
            heap_charge_region(objspace, heap, heap_idx);
            obj = heap_alloc_slot(objspace, heap_idx);
        }
    }
 
    if (obj == Qfalse) {
        // Get next free page (possibly running GC)
        struct heap_page *page = heap_next_free_page(objspace, heap);
        heap_set_alloc_page(objspace, heap_idx, page);
        heap_charge_region(objspace, heap, heap_idx);
 
        // Retry allocation after moving to new page
        obj = heap_alloc_slot(objspace, heap_idx);
    }
 
    if (RB_UNLIKELY(obj == Qfalse)) {
        rb_memerror();
    }
    return obj;
}
 
static VALUE
newobj_alloc(rb_objspace_t *objspace, size_t heap_idx)
{
    /* The objspace belongs to the current Ractor and is single-writer, so the fast path
     * needs no lock.  Stress GC runs in the caller's slow path, before newobj_alloc. */
    VALUE obj = heap_alloc_slot(objspace, heap_idx);
 
    if (RB_UNLIKELY(obj == Qfalse)) {
        obj = newobj_refill(objspace, heap_idx);
    }
 
    return obj;
}
 
ALWAYS_INLINE(static VALUE newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, int wb_protected, size_t heap_idx));
 
static inline VALUE
newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, int wb_protected, size_t heap_idx)
{
    VALUE obj;
 
    /* No lock (see newobj_refill); during_gc and the stress flag are this objspace's own state. */
    if (RB_UNLIKELY(during_gc || ruby_gc_stressful)) {
        if (during_gc) {
            dont_gc_on();
            during_gc = 0;
            if (rb_memerror_reentered()) {
                rb_memerror();
            }
            rb_bug("object allocation during garbage collection phase");
        }
 
        if (ruby_gc_stressful) {
            if (!garbage_collect(objspace, GPR_FLAG_NEWOBJ)) {
                rb_memerror();
            }
        }
    }
 
    obj = newobj_alloc(objspace, heap_idx);
    newobj_init(klass, flags, wb_protected, objspace, obj);
 
    if (RB_UNLIKELY(ruby_gc_stressful)) {
        rb_heap_t *heap = &heaps[heap_idx];
        heap->newobj.alloc_cursor_end = heap->newobj.alloc_cursor;
    }
 
    return obj;
}
 
NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags,
                                                   rb_objspace_t *objspace, size_t heap_idx));
NOINLINE(static VALUE newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags,
                                                     rb_objspace_t *objspace, size_t heap_idx));
 
static VALUE
newobj_slowpath_wb_protected(VALUE klass, VALUE flags, rb_objspace_t *objspace, size_t heap_idx)
{
    return newobj_slowpath(klass, flags, objspace, TRUE, heap_idx);
}
 
static VALUE
newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace, size_t heap_idx)
{
    return newobj_slowpath(klass, flags, objspace, FALSE, heap_idx);
}
 
VALUE
rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size, size_t *actual_alloc_size)
{
    VALUE obj;
    rb_objspace_t *objspace = objspace_ptr;
 
    /* There is no per-Ractor cache; the argument stays for ABI compatibility with other
     * GC implementations such as MMTk. */
    (void)cache_ptr;
 
    RB_DEBUG_COUNTER_INC(obj_newobj);
    (void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected);
 
    if (RB_UNLIKELY(stress_to_class)) {
        if (rb_hash_lookup2(stress_to_class, klass, Qundef) != Qundef) {
            rb_memerror();
        }
    }
 
    size_t heap_idx = heap_idx_for_size(alloc_size);
    *actual_alloc_size = heap_slot_size((unsigned char)heap_idx);
 
    if (!RB_UNLIKELY(during_gc || ruby_gc_stressful) &&
            wb_protected) {
        obj = newobj_alloc(objspace, heap_idx);
        newobj_init(klass, flags, wb_protected, objspace, obj);
    }
    else {
        RB_DEBUG_COUNTER_INC(obj_newobj_slowpath);
 
        obj = wb_protected ?
          newobj_slowpath_wb_protected(klass, flags, objspace, heap_idx) :
          newobj_slowpath_wb_unprotected(klass, flags, objspace, heap_idx);
    }
 
    return obj;
}
 
static int
ptr_in_page_body_p(const void *ptr, const void *memb)
{
    struct heap_page *page = *(struct heap_page **)memb;
    uintptr_t p_body = (uintptr_t)page->body;
 
    if ((uintptr_t)ptr >= p_body) {
        return (uintptr_t)ptr < (p_body + HEAP_PAGE_SIZE) ? 0 : 1;
    }
    else {
        return -1;
    }
}
 
PUREFUNC(static inline struct heap_page *heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr);)
static inline struct heap_page *
heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr)
{
    struct heap_page **res;
 
    if (ptr < (uintptr_t)heap_pages_lomem ||
            ptr > (uintptr_t)heap_pages_himem) {
        return NULL;
    }
 
    res = bsearch((void *)ptr, rb_darray_ref(objspace->heap_pages.sorted, 0),
                  rb_darray_size(objspace->heap_pages.sorted), sizeof(struct heap_page *),
                  ptr_in_page_body_p);
 
    if (res) {
        return *res;
    }
    else {
        return NULL;
    }
}
 
PUREFUNC(static inline bool is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr);)
static inline bool
is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr)
{
    register uintptr_t p = (uintptr_t)ptr;
    register struct heap_page *page;
 
    RB_DEBUG_COUNTER_INC(gc_isptr_trial);
 
    if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE;
    RB_DEBUG_COUNTER_INC(gc_isptr_range);
 
    if (p % sizeof(VALUE) != 0) return FALSE;
    RB_DEBUG_COUNTER_INC(gc_isptr_align);
 
    page = heap_page_for_ptr(objspace, (uintptr_t)ptr);
    if (page) {
        RB_DEBUG_COUNTER_INC(gc_isptr_maybe);
        if (heap_page_in_global_empty_pages_pool(objspace, page)) {
            return FALSE;
        }
        else {
            if (p < page->start) return FALSE;
            if (p >= page->start + (page->total_slots * page->slot_size)) return FALSE;
            if ((p - page->start) % page->slot_size != 0) return FALSE;
 
            return TRUE;
        }
    }
    return FALSE;
}
 
/*
 * Is this object live in the given objspace. For foreign objects in other object spaces,
 * it returns `false`. NOTE: this does NOT check whether or not it's unmarked during the process
 * of lazy sweeping.
 */
bool
rb_gc_impl_live_object_p(void *objspace_ptr, const void *ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    /* Whether ptr refers to a live object. is_pointer_to_heap is the
     * address-only check; T_NONE, T_MOVED, and T_ZOMBIE slots are valid heap
     * addresses but not live objects. */
    if (!is_pointer_to_heap(objspace, ptr)) return false;
 
    VALUE obj = (VALUE)ptr;
    bool live = false;
    asan_unpoisoning_object(obj) {
        switch (BUILTIN_TYPE(obj)) {
          case T_NONE:
          case T_MOVED:
          case T_ZOMBIE:
            break;
          default:
            live = true;
            break;
        }
    }
    return live;
}
 
/* Flags preserved from the original object when it becomes a zombie, and so also the
 * only ones that may legitimately be set on one. */
#define ZOMBIE_OBJ_KEPT_FLAGS (FL_FINALIZE)
 
void
rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    struct RZombie *zombie = RZOMBIE(obj);
    zombie->flags = T_ZOMBIE | (zombie->flags & ZOMBIE_OBJ_KEPT_FLAGS);
    zombie->dfree = dfree;
    zombie->data = data;
    VALUE prev, next = heap_pages_deferred_final;
    do {
        zombie->next = prev = next;
        next = RUBY_ATOMIC_VALUE_CAS(heap_pages_deferred_final, prev, obj);
    } while (next != prev);
 
    struct heap_page *page = GET_HEAP_PAGE(obj);
    page->final_slots++;
    page->heap->final_slots_count++;
}
 
static void
tdata_unsafe_free_chunk_reset(struct tdata_unsafe_free_chunk *chunk)
{
    chunk->next = NULL;
    chunk->count = 0;
    chunk->embed_xfree_bits = 0;
}
 
static struct tdata_unsafe_free_chunk *
tdata_unsafe_free_chunk_alloc(void)
{
    /* Pops race each other (several Ractors can be sweeping), but pushes happen only
     * inside the drain, which holds a VM barrier -- and a barrier cannot complete while
     * a Ractor is inside gc_sweep_page.  No push ever overlaps a pop, so the head only
     * moves forward and this CAS pop needs no ABA tagging.  A sweep performed by a
     * thread other than the objspace's owner would break that. */
    struct tdata_unsafe_free_chunk *head =
        rbimpl_atomic_ptr_load((void **)&global_objspace->tdata_unsafe_free_cache,
                               RBIMPL_ATOMIC_ACQUIRE);
    while (head) {
        struct tdata_unsafe_free_chunk *prev =
            rbimpl_atomic_ptr_cas((void **)&global_objspace->tdata_unsafe_free_cache,
                                  head, head->next,
                                  RBIMPL_ATOMIC_ACQ_REL, RBIMPL_ATOMIC_ACQUIRE);
        if (prev == head) {
            rbimpl_atomic_size_dec(&global_objspace->tdata_unsafe_free_cache_len,
                                   RBIMPL_ATOMIC_RELAXED);
            tdata_unsafe_free_chunk_reset(head);
            return head;
        }
        head = prev;
    }
 
    /* Not xmalloc: this runs mid-sweep, and the chunks are GC bookkeeping that should not
     * feed back into malloc_increase (mark stack chunks do the same). */
    struct tdata_unsafe_free_chunk *chunk = malloc(sizeof(struct tdata_unsafe_free_chunk));
    if (!chunk) rb_memerror();
    tdata_unsafe_free_chunk_reset(chunk);
    return chunk;
}
 
/* Hand this objspace's partial chunk to the global stack.  The entries were counted as
 * they were appended, so the pending count does not change here. */
static void
gc_tdata_unsafe_free_publish(rb_objspace_t *objspace)
{
    struct tdata_unsafe_free_chunk *chunk = objspace->tdata_unsafe_free_chunk;
    if (chunk == NULL) return;
    GC_ASSERT(chunk->count > 0);
    objspace->tdata_unsafe_free_chunk = NULL;
 
    struct tdata_unsafe_free_chunk *prev, *head =
        rbimpl_atomic_ptr_load((void **)&global_objspace->tdata_unsafe_free_published,
                               RBIMPL_ATOMIC_RELAXED);
    do {
        chunk->next = prev = head;
        head = rbimpl_atomic_ptr_cas((void **)&global_objspace->tdata_unsafe_free_published,
                                     prev, chunk,
                                     RBIMPL_ATOMIC_ACQ_REL, RBIMPL_ATOMIC_ACQUIRE);
    } while (head != prev);
}
 
/* Copy out what obj's deferred free needs, running no dfree. Returns true when the
 * caller may reclaim the slot and false when obj became a zombie, matching rb_gc_obj_free. */
static bool
gc_defer_thread_unsafe_free(rb_objspace_t *objspace, VALUE obj, bool *trigger)
{
    GC_ASSERT(!((uintptr_t)RTYPEDDATA(obj)->type & TYPED_DATA_EMBEDDED));
    const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
    void *data = RTYPEDDATA(obj)->data;
    GC_ASSERT(data != NULL);
 
    rb_gc_obj_free_vm_weak_references(obj);
 
    size_t count = rbimpl_atomic_size_fetch_add(&global_objspace->tdata_deferred_free_count, 1,
                                                RBIMPL_ATOMIC_RELAXED) + 1;
    if (!*trigger && count >= TDATA_DEFERRED_FREE_THRESHOLD) {
        *trigger = true;
    }
 
    struct tdata_unsafe_free_chunk *chunk = objspace->tdata_unsafe_free_chunk;
    if (chunk == NULL) {
        chunk = objspace->tdata_unsafe_free_chunk = tdata_unsafe_free_chunk_alloc();
    }
    if (type->flags & RUBY_TYPED_EMBEDDABLE) {
        chunk->embed_xfree_bits |= (uint32_t)1 << chunk->count;
    }
    struct tdata_unsafe_free_entry *entry = &chunk->entries[chunk->count++];
    entry->dfree = type->function.dfree;
    entry->data = data;
    if (chunk->count == TDATA_UNSAFE_FREE_CHUNK_CAPA) {
        gc_tdata_unsafe_free_publish(objspace);
    }
 
    if (FL_TEST_RAW(obj, FL_FINALIZE)) {
        /* The dfree is on the side list now, so this zombie carries none: it goes on the
         * regular deferred list, where the owner runs its Ruby finalizer promptly and
         * reclaims the slot, instead of waiting for the barrier. */
        rb_gc_impl_make_zombie(objspace, obj, 0, 0);
        return false;
    }
    return true;
}
 
typedef int each_obj_callback(void *, void *, size_t, void *);
typedef int each_page_callback(struct heap_page *, void *);
 
struct each_obj_data {
    rb_objspace_t *objspace;
    bool reenable_incremental;
 
    /* Visit only the pages that hold shareable objects, so a foreign Ractor's objspace
     * can be walked for its shareable objects alone, without touching the rest of its
     * isolated heap. */
    bool shareable_only;
 
    /* Set when walking a foreign objspace without settling its stopped lazy sweep
     * (settling would run the owner's obj_free and dfree on this thread).  Objects the
     * sweep is about to free are skipped: on an unswept page, unmarked means dead. */
    bool skip_unswept_dead;
 
    each_obj_callback *each_obj_callback;
    each_page_callback *each_page_callback;
    void *data;
 
    struct heap_page **pages[HEAP_COUNT];
    size_t pages_counts[HEAP_COUNT];
};
 
static VALUE
objspace_each_objects_ensure(VALUE arg)
{
    struct each_obj_data *data = (struct each_obj_data *)arg;
    rb_objspace_t *objspace = data->objspace;
 
    /* Reenable incremental GC */
    if (data->reenable_incremental) {
        objspace->flags.dont_incremental = FALSE;
    }
 
    for (int i = 0; i < HEAP_COUNT; i++) {
        struct heap_page **pages = data->pages[i];
        free(pages);
    }
 
    return Qnil;
}
 
static VALUE
objspace_each_objects_try(VALUE arg)
{
    struct each_obj_data *data = (struct each_obj_data *)arg;
    rb_objspace_t *objspace = data->objspace;
 
    /* Copy pages from all heaps to their respective buffers. */
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        size_t size = heap->total_pages * sizeof(struct heap_page *);
 
        struct heap_page **pages = malloc(size);
        if (!pages) rb_memerror();
 
        /* Set up pages buffer by iterating over all pages in the current eden
         * heap. This will be a snapshot of the state of the heap before we
         * call the callback over each page that exists in this buffer. Thus it
         * is safe for the callback to allocate objects without possibly entering
         * an infinite loop. */
        struct heap_page *page = 0;
        size_t pages_count = 0;
        ccan_list_for_each(&heap->pages, page, page_node) {
            pages[pages_count] = page;
            pages_count++;
        }
        data->pages[i] = pages;
        data->pages_counts[i] = pages_count;
        GC_ASSERT(pages_count == heap->total_pages);
    }
 
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        size_t pages_count = data->pages_counts[i];
        struct heap_page **pages = data->pages[i];
 
        struct heap_page *page = ccan_list_top(&heap->pages, struct heap_page, page_node);
        for (size_t i = 0; i < pages_count; i++) {
            /* If we have reached the end of the linked list then there are no
             * more pages, so break. */
            if (page == NULL) break;
 
            /* If this page does not match the one in the buffer, then move to
             * the next page in the buffer. */
            if (pages[i] != page) continue;
 
            uintptr_t pstart = (uintptr_t)page->start;
            uintptr_t pend = pstart + (page->total_slots * heap->slot_size);
 
            if (data->shareable_only) {
                /* Hand shareable objects to the callback one slot at a time, not the
                 * whole page: walking a foreign Ractor's objspace must never expose its
                 * unshareable objects, which the caller cannot inspect safely. */
                if (page->flags.has_shareable_objects) {
                    /* This walk runs over a foreign objspace under the barrier and
                     * must not settle the owner's stopped lazy sweep: settling would run
                     * the owner's obj_free and dfree on this thread with this Ractor's
                     * identity (wrong per-Ractor tables, a foreign T_DATA dfree).  So no
                     * gc_rest, and objects the sweep is about to free are skipped: on an
                     * unswept page unmarked means dead and its shareable bit merely has
                     * not been bulk-cleared yet.  Passing one to the callback would
                     * resurrect it, handing out a reference the owner's sweep frees as
                     * soon as the barrier lifts. */
                    const bool page_unswept = is_lazy_sweeping(objspace) && page->flags.before_sweep;
                    int planes = CEILDIV(page->total_slots, BITS_BITLENGTH);
                    uintptr_t base = pstart;
                    bool stop = false;
                    for (int j = 0; j < planes && !stop; j++) {
                        bits_t bits = page->shareable_bits[j];
                        uintptr_t slot = base;
                        while (bits) {
                            if ((bits & 1) && data->each_obj_callback &&
                                !(page_unswept && !RVALUE_MARKED(objspace, (VALUE)slot)) &&
                                (*data->each_obj_callback)((void *)slot, (void *)(slot + heap->slot_size),
                                                           heap->slot_size, data->data)) {
                                stop = true;
                                break;
                            }
                            slot += heap->slot_size;
                            bits >>= 1;
                        }
                        base += BITS_BITLENGTH * heap->slot_size;
                    }
                    if (stop) break;
                }
            }
            else if (data->skip_unswept_dead &&
                     is_lazy_sweeping(objspace) && page->flags.before_sweep) {
                /* A foreign page pending sweep: hand out the live objects one slot at a
                 * time and skip the unmarked (dead) ones the owner's sweep frees as soon
                 * as the barrier lifts. */
                bool stop = false;
                for (uintptr_t slot = pstart; slot < pend; slot += heap->slot_size) {
                    if (!RVALUE_MARKED(objspace, (VALUE)slot)) continue;
                    if (data->each_obj_callback &&
                        (*data->each_obj_callback)((void *)slot, (void *)(slot + heap->slot_size),
                                                   heap->slot_size, data->data)) {
                        stop = true;
                        break;
                    }
                }
                if (stop) break;
            }
            else {
                if (data->each_obj_callback &&
                    (*data->each_obj_callback)((void *)pstart, (void *)pend, heap->slot_size, data->data)) {
                    break;
                }
                if (data->each_page_callback &&
                    (*data->each_page_callback)(page, data->data)) {
                    break;
                }
            }
 
            page = ccan_list_next(&heap->pages, page, page_node);
        }
    }
 
    return Qnil;
}
 
static void
objspace_each_exec(bool protected, struct each_obj_data *each_obj_data)
{
    /* Disable incremental GC */
    rb_objspace_t *objspace = each_obj_data->objspace;
    bool reenable_incremental = FALSE;
    if (protected) {
        reenable_incremental = !objspace->flags.dont_incremental;
 
        gc_rest(objspace);
        objspace->flags.dont_incremental = TRUE;
    }
 
    each_obj_data->reenable_incremental = reenable_incremental;
    memset(&each_obj_data->pages, 0, sizeof(each_obj_data->pages));
    memset(&each_obj_data->pages_counts, 0, sizeof(each_obj_data->pages_counts));
    rb_ensure(objspace_each_objects_try, (VALUE)each_obj_data,
              objspace_each_objects_ensure, (VALUE)each_obj_data);
}
 
static void
objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data, bool protected)
{
    struct each_obj_data each_obj_data = {
        .objspace = objspace,
        .each_obj_callback = callback,
        .each_page_callback = NULL,
        .data = data,
    };
    objspace_each_exec(protected, &each_obj_data);
}
 
void
rb_gc_impl_each_objects(void *objspace_ptr, each_obj_callback *callback, void *data)
{
    objspace_each_objects(objspace_ptr, callback, data, TRUE);
}
 
/* Like rb_gc_impl_each_objects but visiting only pages that hold shareable objects, to
 * reach a foreign Ractor's shareable objects without walking the rest of its heap. */
void
rb_gc_impl_each_objects_shareable(void *objspace_ptr, each_obj_callback *callback, void *data)
{
    struct each_obj_data each_obj_data = {
        .objspace = objspace_ptr,
        .shareable_only = true,
        .each_obj_callback = callback,
        .each_page_callback = NULL,
        .data = data,
    };
    /* Not the protected variant: this objspace belongs to another Ractor (the caller
     * holds the barrier).  The protected path calls gc_rest, which would run the owner's
     * stopped lazy sweep (its obj_free and dfree) on the walking thread with the
     * walker's Ractor identity (wrong per-Ractor tables, a foreign T_DATA dfree).  The
     * owner is stopped and its page list is stable, and the walk itself skips dead,
     * unswept objects (the shareable_only branch of objspace_each_objects_try).  The
     * walker's own incremental GC state is untouched, since this is not its objspace. */
    objspace_each_exec(FALSE, &each_obj_data);
}
 
/* Walk every object of a foreign Ractor's objspace, unshareable ones included.  Only for
 * callers that hold the barrier and whose callback is pure C (a heap dump, memory
 * accounting).  As in the shareable walk above, the owner's stopped lazy sweep is not
 * settled and dead, unswept objects are skipped by the walk (skip_unswept_dead). */
void
rb_gc_impl_each_objects_foreign(void *objspace_ptr, each_obj_callback *callback, void *data)
{
    struct each_obj_data each_obj_data = {
        .objspace = objspace_ptr,
        .skip_unswept_dead = true,
        .each_obj_callback = callback,
        .each_page_callback = NULL,
        .data = data,
    };
    objspace_each_exec(FALSE, &each_obj_data);
}
 
#if GC_CAN_COMPILE_COMPACTION
static void
objspace_each_pages(rb_objspace_t *objspace, each_page_callback *callback, void *data, bool protected)
{
    struct each_obj_data each_obj_data = {
        .objspace = objspace,
        .each_obj_callback = NULL,
        .each_page_callback = callback,
        .data = data,
    };
    objspace_each_exec(protected, &each_obj_data);
}
#endif
 
VALUE
rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block)
{
    rb_objspace_t *objspace = objspace_ptr;
    VALUE table;
    st_data_t data;
 
    GC_ASSERT(!OBJ_FROZEN(obj));
 
    /* Registering, storing and running finalizers all belong to the object's own
     * objspace, so refuse to define one on another Ractor's object (even a shareable
     * one): it would land in a table the owner's sweep never consults. */
    if (GET_HEAP_OBJSPACE(obj) != objspace) {
        rb_raise(rb_eRactorIsolationError,
                 "can not define a finalizer for an object of another Ractor");
    }
 
    RBASIC(obj)->flags |= FL_FINALIZE;
 
    unsigned int lev = RB_GC_VM_LOCK();
 
    if (st_lookup(finalizer_table, obj, &data)) {
        table = (VALUE)data;
        VALUE dup_table = rb_ary_dup(table);
 
        RB_GC_VM_UNLOCK(lev);
        /* avoid duplicate block, table is usually small */
        {
            rb_len_t len = RARRAY_LEN(table);
            rb_len_t i;
 
            for (i = 0; i < len; i++) {
                VALUE recv = RARRAY_AREF(dup_table, i);
                if (rb_equal(recv, block)) { // can't be called with VM lock held
                    return recv;
                }
            }
        }
        lev = RB_GC_VM_LOCK();
        RB_GC_GUARD(dup_table);
 
        rb_ary_push(table, block);
    }
    else {
        table = rb_ary_new3(2, rb_obj_id(obj), block);
        rb_obj_hide(table);
        st_add_direct(finalizer_table, obj, table);
    }
 
    RB_GC_VM_UNLOCK(lev);
 
    return block;
}
 
void
rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    GC_ASSERT(!OBJ_FROZEN(obj));
 
    /* Symmetric with define. */
    if (GET_HEAP_OBJSPACE(obj) != objspace) {
        rb_raise(rb_eRactorIsolationError,
                 "can not undefine a finalizer of an object of another Ractor");
    }
 
    st_data_t data = obj;
 
    int lev = RB_GC_VM_LOCK();
    st_delete(finalizer_table, &data, 0);
    RB_GC_VM_UNLOCK(lev);
 
    FL_UNSET(obj, FL_FINALIZE);
}
 
void
rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj)
{
    /* Finalizers do not cross objspaces: a copy of another Ractor's object starts with
     * none (guards the public rb_gc_copy_finalizer C API; no in-tree caller crosses).
     * A same-objspace copy behaves as before.  Table accessed under the VM lock. */
    rb_objspace_t *objspace = objspace_ptr;
    VALUE table;
    st_data_t data;
 
    if (!FL_TEST(obj, FL_FINALIZE)) return;
    if (GET_HEAP_OBJSPACE(obj) != objspace) return;
 
    int lev = RB_GC_VM_LOCK();
    if (RB_LIKELY(st_lookup(finalizer_table, obj, &data))) {
        table = rb_ary_dup((VALUE)data);
        RARRAY_ASET(table, 0, rb_obj_id(dest));
        st_insert(finalizer_table, dest, table);
        FL_SET(dest, FL_FINALIZE);
    }
    else {
        rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj));
    }
    RB_GC_VM_UNLOCK(lev);
}
 
static VALUE
get_final(rb_len_t i, void *data)
{
    VALUE table = (VALUE)data;
 
    return RARRAY_AREF(table, i + 1);
}
 
static void
run_final(rb_objspace_t *objspace, VALUE zombie)
{
    if (RZOMBIE(zombie)->dfree) {
        RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data);
    }
 
    st_data_t key = (st_data_t)zombie;
    if (FL_TEST_RAW(zombie, FL_FINALIZE)) {
        FL_UNSET(zombie, FL_FINALIZE);
        st_data_t table;
        if (st_delete(finalizer_table, &key, &table)) {
            rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), RARRAY_LEN(table) - 1, get_final, (void *)table);
        }
        else {
            rb_bug("FL_FINALIZE flag is set, but finalizers are not found");
        }
    }
    else {
        GC_ASSERT(!st_lookup(finalizer_table, key, NULL));
    }
}
 
static void
finalize_list(rb_objspace_t *objspace, VALUE zombie)
{
    while (zombie) {
        VALUE next_zombie;
        struct heap_page *page;
        rb_asan_unpoison_object(zombie, false);
        next_zombie = RZOMBIE(zombie)->next;
        page = GET_HEAP_PAGE(zombie);
 
        run_final(objspace, zombie);
        {
            GC_ASSERT(BUILTIN_TYPE(zombie) == T_ZOMBIE);
            GC_ASSERT(page->heap->final_slots_count > 0);
            GC_ASSERT(page->final_slots > 0);
 
            page->heap->final_slots_count--;
            page->final_slots--;
            page->free_slots++;
            RVALUE_AGE_SET_BITMAP(zombie, 0);
            heap_page_add_free_region(objspace, page, zombie);
            page->heap->total_freed_objects++;
        }
 
        zombie = next_zombie;
    }
}
 
static void
finalize_zombies(rb_objspace_t *objspace)
{
    VALUE zombie;
    while ((zombie = RUBY_ATOMIC_VALUE_EXCHANGE(heap_pages_deferred_final, 0)) != 0) {
        finalize_list(objspace, zombie);
    }
}
 
static void
finalize_deferred(rb_objspace_t *objspace)
{
    rb_gc_set_pending_interrupt();
    finalize_zombies(objspace);
    rb_gc_unset_pending_interrupt();
}
 
static void
gc_finalize_deferred(void *dmy)
{
    /* One postponed job is shared by every objspace: the preregistration table only
     * holds about 32 entries and Ractors are created continuously.  A deferred finalizer
     * belongs to the objspace of the thread that ran the job, i.e. the current one. */
    rb_objspace_t *objspace = rb_gc_get_objspace();
    if (RUBY_ATOMIC_EXCHANGE(finalizing, 1)) return;
 
    finalize_deferred(objspace);
    RUBY_ATOMIC_SET(finalizing, 0);
}
 
static void
gc_finalize_deferred_register(rb_objspace_t *objspace)
{
    /* Enqueue gc_finalize_deferred on this objspace's owning Ractor.  A global GC can
     * defer a foreign objspace's finalizers, and those must run on their owner rather
     * than on the driver. */
    rb_gc_trigger_finalize_deferred(objspace, objspace->finalize_deferred_pjob);
}
 
static int pop_mark_stack(mark_stack_t *stack, VALUE *data);
 
/* Throw away an unfinished incremental mark and leave the objspace in gc_mode_none.  The
 * mark bits the partial mark set stay behind, so the caller must clear them before the
 * heap is collected again. */
static void
gc_abort_incremental_marking(rb_objspace_t *objspace)
{
    GC_ASSERT(is_incremental_marking(objspace));
 
    VALUE obj;
    while (pop_mark_stack(&objspace->mark_stack, &obj));
 
    /* gc_grey records the weak references it greys for gc_marks_finish to resolve; this
     * cycle never reaches it, and the entries would outlive their objects. */
    rb_darray_clear(objspace->weak_references);
 
    gc_set_incremental_marking(objspace, FALSE);
    gc_mode_set(objspace, gc_mode_none);
}
 
static void
gc_abort(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    if (is_incremental_marking(objspace)) {
        gc_abort_incremental_marking(objspace);
    }
 
    if (is_lazy_sweeping(objspace)) {
        objspace->sweeping_heap_count = 0;
        for (int i = 0; i < HEAP_COUNT; i++) {
            rb_heap_t *heap = &heaps[i];
 
            heap->sweeping_page = NULL;
            struct heap_page *page = NULL;
 
            ccan_list_for_each(&heap->pages, page, page_node) {
                page->flags.before_sweep = false;
            }
        }
    }
 
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        gc_bitmaps_clear(objspace, heap, false);
    }
 
    gc_mode_set(objspace, gc_mode_none);
}
 
#if VERIFY_FREE_SIZE
# ifdef RB_THREAD_LOCAL_SPECIFIER
#  define GC_FREEING_OBJ_TLS RB_THREAD_LOCAL_SPECIFIER
# else
#  define GC_FREEING_OBJ_TLS
# endif
 
static GC_FREEING_OBJ_TLS VALUE gc_freeing_obj;
 
/* Remember what we are tearing down so that a bad xfree() underneath can name
 * the object and not just the buffer.  Saved and restored because a dfree
 * callback can free another object. */
static bool
gc_obj_free(void *objspace, VALUE obj)
{
    VALUE prev = gc_freeing_obj;
    gc_freeing_obj = obj;
 
    bool freed = rb_gc_obj_free(objspace, obj);
 
    gc_freeing_obj = prev;
    return freed;
}
 
static const char *
gc_freeing_obj_info(void)
{
    /* Not thread-local: only reachable from a rb_bug() path, where a second
     * thread racing us is already unrecoverable. */
    static char buf[128];
 
    if (!gc_freeing_obj) return NULL;
 
    snprintf(buf, sizeof(buf), "%p %s", (void *)gc_freeing_obj, rb_obj_info(gc_freeing_obj));
    return buf;
}
#else
# define gc_obj_free(objspace, obj) rb_gc_obj_free((objspace), (obj))
# define gc_freeing_obj_info() NULL
#endif
 
void
rb_gc_impl_shutdown_free_objects(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
        struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
        short stride = page->slot_size;
 
        uintptr_t p = (uintptr_t)page->start;
        uintptr_t pend = p + page->total_slots * stride;
        for (; p < pend; p += stride) {
            VALUE vp = (VALUE)p;
            asan_unpoisoning_object(vp) {
                if (RB_BUILTIN_TYPE(vp) != T_NONE) {
                    rb_gc_obj_free_vm_weak_references(vp);
                    if (gc_obj_free(objspace, vp)) {
                        RBASIC(vp)->flags = 0;
                    }
                }
            }
        }
    }
}
 
static int
rb_gc_impl_shutdown_call_finalizer_i(st_data_t key, st_data_t val, st_data_t _data)
{
    VALUE obj = (VALUE)key;
    VALUE table = (VALUE)val;
 
    GC_ASSERT(RB_FL_TEST(obj, FL_FINALIZE));
    GC_ASSERT(RB_BUILTIN_TYPE(val) == T_ARRAY);
 
    rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), RARRAY_LEN(table) - 1, get_final, (void *)table);
 
    /* obj may be a poisoned zombie */
    asan_unpoisoning_object(obj) {
        FL_UNSET(obj, FL_FINALIZE);
    }
 
    return ST_DELETE;
}
 
void
rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
#if RGENGC_CHECK_MODE >= 2
    gc_verify_internal_consistency(objspace);
#endif
 
    /* prohibit incremental GC */
    objspace->flags.dont_incremental = 1;
 
    if (RUBY_ATOMIC_EXCHANGE(finalizing, 1)) {
        /* Abort incremental marking and lazy sweeping to speed up shutdown. */
        gc_abort(objspace);
        dont_gc_on();
        return;
    }
 
    while (finalizer_table->num_entries) {
        st_foreach(finalizer_table, rb_gc_impl_shutdown_call_finalizer_i, 0);
    }
 
    /* run finalizers */
    finalize_deferred(objspace);
    GC_ASSERT(heap_pages_deferred_final == 0);
 
    /* Deferred non-thread-safe frees: their objects are long gone, so the object walk
     * below will not reach them.  Reap them here. */
    gc_tdata_unsafe_drain_objspaces(&objspace, 1);
 
    /* Abort incremental marking and lazy sweeping to speed up shutdown. */
    gc_abort(objspace);
 
    /* prohibit GC because force T_DATA finalizers can break an object graph consistency */
    dont_gc_on();
 
    /* running data/file finalizers are part of garbage collection */
    unsigned int lock_lev;
    gc_enter(objspace, gc_enter_event_finalizer, &lock_lev);
 
    /* run data/file object's finalizers */
    for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
        struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
        short stride = page->slot_size;
 
        uintptr_t p = (uintptr_t)page->start;
        uintptr_t pend = p + page->total_slots * stride;
        for (; p < pend; p += stride) {
            VALUE vp = (VALUE)p;
            asan_unpoisoning_object(vp) {
                if (rb_gc_shutdown_call_finalizer_p(vp)) {
                    rb_gc_obj_free_vm_weak_references(vp);
                    if (gc_obj_free(objspace, vp)) {
                        RBASIC(vp)->flags = 0;
                    }
                }
            }
        }
    }
 
    gc_exit(objspace, gc_enter_event_finalizer, &lock_lev);
 
    finalize_zombies(objspace);
 
    st_free_table(finalizer_table);
    finalizer_table = 0;
    RUBY_ATOMIC_SET(finalizing, 0);
}
 
void
rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
        struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
        short stride = page->slot_size;
 
        uintptr_t p = (uintptr_t)page->start;
        uintptr_t pend = p + page->total_slots * stride;
        for (; p < pend; p += stride) {
            VALUE obj = (VALUE)p;
 
            asan_unpoisoning_object(obj) {
                func(obj, data);
            }
        }
    }
}
 
/*
  ------------------------ Garbage Collection ------------------------
*/
 
/* Sweeping */
 
static size_t
objspace_available_slots(rb_objspace_t *objspace)
{
    size_t total_slots = 0;
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        total_slots += heap->total_slots;
    }
    return total_slots;
}
 
static size_t
objspace_live_slots(rb_objspace_t *objspace)
{
    return total_allocated_objects(objspace) - total_freed_objects(objspace) - total_final_slots_count(objspace);
}
 
static size_t
objspace_free_slots(rb_objspace_t *objspace)
{
    return objspace_available_slots(objspace) - objspace_live_slots(objspace) - total_final_slots_count(objspace);
}
 
static void
gc_setup_mark_bits(struct heap_page *page)
{
    /* copy oldgen bitmap to mark bitmap */
    memcpy(&page->mark_bits[0], &page->uncollectible_bits[0], HEAP_PAGE_BITMAP_SIZE);
}
 
static int gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj);
static VALUE gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, struct heap_page *src_page, struct heap_page *dest_page);
 
#if defined(_WIN32)
enum {HEAP_PAGE_LOCK = PAGE_NOACCESS, HEAP_PAGE_UNLOCK = PAGE_READWRITE};
 
static BOOL
protect_page_body(struct heap_page_body *body, DWORD protect)
{
    DWORD old_protect;
    return VirtualProtect(body, HEAP_PAGE_SIZE, protect, &old_protect) != 0;
}
#elif defined(__wasi__)
// wasi-libc's mprotect emulation does not support PROT_NONE
enum {HEAP_PAGE_LOCK, HEAP_PAGE_UNLOCK};
#define protect_page_body(body, protect) 1
#else
enum {HEAP_PAGE_LOCK = PROT_NONE, HEAP_PAGE_UNLOCK = PROT_READ | PROT_WRITE};
#define protect_page_body(body, protect) !mprotect((body), HEAP_PAGE_SIZE, (protect))
#endif
 
static void
lock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
{
    if (!protect_page_body(body, HEAP_PAGE_LOCK)) {
        rb_bug("Couldn't protect page %p, errno: %s", (void *)body, strerror(errno));
    }
    else {
        gc_report(5, objspace, "Protecting page in move %p\n", (void *)body);
    }
}
 
static void
unlock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
{
    if (!protect_page_body(body, HEAP_PAGE_UNLOCK)) {
        rb_bug("Couldn't unprotect page %p, errno: %s", (void *)body, strerror(errno));
    }
    else {
        gc_report(5, objspace, "Unprotecting page in move %p\n", (void *)body);
    }
}
 
static uintptr_t
heap_page_alloc_slot_from_region(struct heap_page *free_page)
{
    asan_unlock_freelist(free_page);
    struct free_region *region = free_page->free_region;
    asan_lock_freelist(free_page);
 
    if (region == NULL) {
        return 0;
    }
 
    rb_asan_unpoison_object((VALUE)region, false);
    GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
    uintptr_t dest = (uintptr_t)region;
    uintptr_t region_end = region->end;
    struct free_region *next = region->next;
 
    uintptr_t new_start = dest + free_page->slot_size;
 
    asan_unlock_freelist(free_page);
    if (new_start < region_end) {
        VALUE next_start = (VALUE)new_start;
        rb_asan_unpoison_object(next_start, false);
        struct free_region *new_region = (struct free_region *)new_start;
        new_region->flags = 0;
        new_region->end = region_end;
        new_region->next = next;
        rb_asan_poison_object(next_start);
        free_page->free_region = new_region;
    }
    else {
        free_page->free_region = next;
    }
    asan_lock_freelist(free_page);
 
    return dest;
}
 
static bool
try_move(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *free_page, VALUE src)
{
    GC_ASSERT(gc_is_moveable_obj(objspace, src));
 
    struct heap_page *src_page = GET_HEAP_PAGE(src);
    if (!free_page) {
        return false;
    }
 
    /* We should return true if either src is successfully moved, or src is
     * unmoveable. A false return will cause the sweeping cursor to be
     * incremented to the next page, and src will attempt to move again */
    GC_ASSERT(RVALUE_MARKED(objspace, src));
 
    uintptr_t dest_slot = heap_page_alloc_slot_from_region(free_page);
    if (dest_slot == 0) {
        return false;
    }
    VALUE dest = (VALUE)dest_slot;
 
    GC_ASSERT(RB_BUILTIN_TYPE(dest) == T_NONE);
 
    if (src_page->slot_size > free_page->slot_size) {
        objspace->rcompactor.moved_down_count_table[BUILTIN_TYPE(src)]++;
    }
    else if (free_page->slot_size > src_page->slot_size) {
        objspace->rcompactor.moved_up_count_table[BUILTIN_TYPE(src)]++;
    }
    objspace->rcompactor.moved_count_table[BUILTIN_TYPE(src)]++;
    objspace->rcompactor.total_moved++;
 
    gc_move(objspace, src, dest, src_page, free_page);
    gc_pin(objspace, src);
    free_page->free_slots--;
 
    return true;
}
 
static void
gc_unprotect_pages(rb_objspace_t *objspace, rb_heap_t *heap)
{
    struct heap_page *cursor = heap->compact_cursor;
 
    while (cursor) {
        unlock_page_body(objspace, cursor->body);
        cursor = ccan_list_next(&heap->pages, cursor, page_node);
    }
}
 
static void gc_update_references(rb_objspace_t *objspace);
static void gc_update_references_heap(rb_objspace_t *objspace);
static void gc_update_references_global(rb_objspace_t *objspace);
#if GC_CAN_COMPILE_COMPACTION
static void invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page);
#endif
 
#if defined(__MINGW32__) || defined(_WIN32)
# define GC_COMPACTION_SUPPORTED 1
#else
/* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for
 * the read barrier, so we must disable compaction. */
# define GC_COMPACTION_SUPPORTED (GC_CAN_COMPILE_COMPACTION && HEAP_PAGE_ALLOC_USE_MMAP)
#endif
 
#if GC_CAN_COMPILE_COMPACTION
static void
read_barrier_handler(uintptr_t address)
{
    rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace();
 
    struct heap_page_body *page_body = GET_PAGE_BODY(address);
 
    /* If the page_body is NULL, then mprotect cannot handle it and will crash
     * with "Cannot allocate memory". */
    if (page_body == NULL) {
        rb_bug("read_barrier_handler: segmentation fault at %p", (void *)address);
    }
 
    int lev = RB_GC_VM_LOCK();
    {
        unlock_page_body(objspace, page_body);
 
        objspace->profile.read_barrier_faults++;
 
        invalidate_moved_page(objspace, GET_HEAP_PAGE(address));
    }
    RB_GC_VM_UNLOCK(lev);
}
#endif
 
#if !GC_CAN_COMPILE_COMPACTION
static void
uninstall_handlers(void)
{
    /* no-op */
}
 
static void
install_handlers(void)
{
    /* no-op */
}
#elif defined(_WIN32)
static LPTOP_LEVEL_EXCEPTION_FILTER old_handler;
typedef void (*signal_handler)(int);
static signal_handler old_sigsegv_handler;
 
static LONG WINAPI
read_barrier_signal(EXCEPTION_POINTERS *info)
{
    /* EXCEPTION_ACCESS_VIOLATION is what's raised by access to protected pages */
    if (info->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
        /* > The second array element specifies the virtual address of the inaccessible data.
         * https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record
         *
         * Use this address to invalidate the page */
        read_barrier_handler((uintptr_t)info->ExceptionRecord->ExceptionInformation[1]);
        return EXCEPTION_CONTINUE_EXECUTION;
    }
    else {
        return EXCEPTION_CONTINUE_SEARCH;
    }
}
 
static void
uninstall_handlers(void)
{
    signal(SIGSEGV, old_sigsegv_handler);
    SetUnhandledExceptionFilter(old_handler);
}
 
static void
install_handlers(void)
{
    /* Remove SEGV handler so that the Unhandled Exception Filter handles it */
    old_sigsegv_handler = signal(SIGSEGV, NULL);
    /* Unhandled Exception Filter has access to the violation address similar
     * to si_addr from sigaction */
    old_handler = SetUnhandledExceptionFilter(read_barrier_signal);
}
#else
static struct sigaction old_sigbus_handler;
static struct sigaction old_sigsegv_handler;
 
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
static exception_mask_t old_exception_masks[32];
static mach_port_t old_exception_ports[32];
static exception_behavior_t old_exception_behaviors[32];
static thread_state_flavor_t old_exception_flavors[32];
static mach_msg_type_number_t old_exception_count;
 
static void
disable_mach_bad_access_exc(void)
{
    old_exception_count = sizeof(old_exception_masks) / sizeof(old_exception_masks[0]);
    task_swap_exception_ports(
        mach_task_self(), EXC_MASK_BAD_ACCESS,
        MACH_PORT_NULL, EXCEPTION_DEFAULT, 0,
        old_exception_masks, &old_exception_count,
        old_exception_ports, old_exception_behaviors, old_exception_flavors
    );
}
 
static void
restore_mach_bad_access_exc(void)
{
    for (mach_msg_type_number_t i = 0; i < old_exception_count; i++) {
        task_set_exception_ports(
            mach_task_self(),
            old_exception_masks[i], old_exception_ports[i],
            old_exception_behaviors[i], old_exception_flavors[i]
        );
    }
}
#endif
 
#if defined(HAVE_PTHREAD_SIGMASK)
# define gc_sigmask pthread_sigmask
#else
# define gc_sigmask sigprocmask
#endif
 
static void
read_barrier_signal(int sig, siginfo_t *info, void *data)
{
    // setup SEGV/BUS handlers for errors
    struct sigaction prev_sigbus, prev_sigsegv;
    sigaction(SIGBUS, &old_sigbus_handler, &prev_sigbus);
    sigaction(SIGSEGV, &old_sigsegv_handler, &prev_sigsegv);
 
    // enable SIGBUS/SEGV
    sigset_t set, prev_set;
    sigemptyset(&set);
    sigaddset(&set, SIGBUS);
    sigaddset(&set, SIGSEGV);
    gc_sigmask(SIG_UNBLOCK, &set, &prev_set);
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
    disable_mach_bad_access_exc();
#endif
    // run handler
    read_barrier_handler((uintptr_t)info->si_addr);
 
    // reset SEGV/BUS handlers
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
    restore_mach_bad_access_exc();
#endif
    sigaction(SIGBUS, &prev_sigbus, NULL);
    sigaction(SIGSEGV, &prev_sigsegv, NULL);
    gc_sigmask(SIG_SETMASK, &prev_set, NULL);
}
 
static void
uninstall_handlers(void)
{
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
    restore_mach_bad_access_exc();
#endif
    sigaction(SIGBUS, &old_sigbus_handler, NULL);
    sigaction(SIGSEGV, &old_sigsegv_handler, NULL);
}
 
static void
install_handlers(void)
{
    struct sigaction action;
    memset(&action, 0, sizeof(struct sigaction));
    sigemptyset(&action.sa_mask);
    action.sa_sigaction = read_barrier_signal;
    action.sa_flags = SA_SIGINFO | SA_ONSTACK;
 
    sigaction(SIGBUS, &action, &old_sigbus_handler);
    sigaction(SIGSEGV, &action, &old_sigsegv_handler);
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
    disable_mach_bad_access_exc();
#endif
}
#endif
 
static void
gc_compact_finish(rb_objspace_t *objspace)
{
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        gc_unprotect_pages(objspace, heap);
    }
 
    if (!global_objspace->global_gc.compacting) uninstall_handlers();
 
    if (global_objspace->global_gc.compacting) {
        /* In a compacting global GC this updates only this objspace's heap references;
         * gc_start_global sets during_reference_updating on every objspace (the
         * move-or-mark decision reads it via rb_gc_get_objspace()) and runs the
         * non-idempotent VM-global side (gc_update_references_global) once at the end. */
        gc_update_references_heap(objspace);
    }
    else {
        gc_update_references(objspace);
    }
    objspace->profile.compact_count++;
 
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        heap->compact_cursor = NULL;
        heap->free_pages = NULL;
        heap->compact_cursor_index = 0;
    }
 
    if (gc_prof_enabled(objspace)) {
        gc_profile_record *record = gc_prof_record(objspace);
        record->moved_objects = objspace->rcompactor.total_moved - record->moved_objects;
    }
    if (!global_objspace->global_gc.compacting) objspace->flags.during_compacting = FALSE;
}
 
struct gc_sweep_context {
    struct heap_page *page;
    int final_slots;
    int freed_slots;
    int empty_slots;
    /* Hoisted out of the per-slot pinned-free assert: too expensive for the sweep loop
     * as an external call. */
    const bool check_pinned_free;
    /* This is a parallel local sweep (multi-Ractor, not a global GC), so a non-thread-safe
     * T_DATA dfree must be deferred to the global GC or the postponed job rather than run here. */
    const bool defer_thread_unsafe_local_sweep;
    bool trigger_thread_unsafe_sweep_postponed_job;
 
    struct free_region *free_region;
};
 
/* NOTE: We must free the root fiber during postmortem collection, otherwise another Ractor
 * can collect the fiber through a major GC while we're still tearing it down. Once fibers are
 * THREAD_SAFE_FREE, we no longer need the root fiber condition as it will be guaranteed to be
 * collected during this time. */
static bool
gc_obj_defer_local_free_p(rb_objspace_t *objspace, VALUE obj)
{
    if (BUILTIN_TYPE(obj) != T_DATA) return false;
 
    const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
    if (!rb_gc_data_type_deferred_free_p(type)) return false;
 
    if (RTYPEDDATA_GET_DATA(obj) == NULL) return false;
 
    if (type->flags & RUBY_TYPED_FREE_IMMEDIATELY) {
        if (objspace->flags.during_postmortem) {
            if (rb_fiber_current() == obj) {
                return false;
            }
        }
        return true;
    }
    else {
        return false;
    }
}
 
static void gc_tdata_deferred_free_job(void *unused);
static void gc_tdata_deferred_free_pjob_ensure(void);
static unsigned int gc_during_gc_get(const rb_objspace_t *objspace);
static void gc_during_gc_set(rb_objspace_t *objspace, unsigned int v);
static void gc_global_snapshot_objspaces(void);
 
static void
gc_tdata_deferred_free_pjob_ensure(void)
{
    if (global_objspace->tdata_deferred_free_pjob == POSTPONED_JOB_HANDLE_INVALID) {
        global_objspace->tdata_deferred_free_pjob =
            rb_postponed_job_preregister(0, gc_tdata_deferred_free_job, NULL);
        if (global_objspace->tdata_deferred_free_pjob == POSTPONED_JOB_HANDLE_INVALID) {
            rb_bug("Could not preregister postponed job for deferred T_DATA free");
        }
    }
}
 
/* A terminating Ractor's postmortem collection runs on an EC whose stack is already
 * torn down: it never checks interrupts again, so a job triggered there is lost and no
 * later sweep can rediscover the entries.  Hand those to the main Ractor. */
static void
gc_tdata_deferred_free_trigger(rb_objspace_t *objspace)
{
    if (objspace->flags.during_postmortem) {
        rb_gc_trigger_postponed_job_on_main(global_objspace->tdata_deferred_free_pjob);
    }
    else {
        rb_postponed_job_trigger(global_objspace->tdata_deferred_free_pjob);
    }
}
 
static void
gc_tdata_unsafe_free_entry(const struct tdata_unsafe_free_entry *entry, bool embed_xfree)
{
    entry->dfree(entry->data);
    if (embed_xfree) {
        xfree(entry->data);
    }
}
 
static void
tdata_unsafe_free_chunk_recycle(struct tdata_unsafe_free_chunk *chunk)
{
    if (global_objspace->tdata_unsafe_free_cache_len >= TDATA_UNSAFE_FREE_CACHE_MAX) {
        free(chunk);
        return;
    }
    tdata_unsafe_free_chunk_reset(chunk);
    chunk->next = global_objspace->tdata_unsafe_free_cache;
    rbimpl_atomic_ptr_store((volatile void **)&global_objspace->tdata_unsafe_free_cache, chunk,
                            RBIMPL_ATOMIC_RELEASE);
    global_objspace->tdata_unsafe_free_cache_len++;
}
 
static void
gc_tdata_unsafe_drain_chunk(struct tdata_unsafe_free_chunk *chunk)
{
    for (unsigned int i = 0; i < chunk->count; i++) {
        gc_tdata_unsafe_free_entry(&chunk->entries[i],
                                   (chunk->embed_xfree_bits >> i) & 1);
    }
    tdata_unsafe_free_chunk_recycle(chunk);
}
 
/* Run every pending deferred free: the published chunks (which belong to no objspace)
 * plus the given objspaces' partial chunks.  The caller must have stopped the world --
 * VM barrier held, or a single Ractor left in the process -- and must pass every live
 * objspace, since the pending count is zeroed here.  (Shutdown is the one exception:
 * nothing reads the count afterwards.) */
static void
gc_tdata_unsafe_drain_objspaces(rb_objspace_t **objspaces, size_t n)
{
    struct tdata_unsafe_free_chunk *chunk =
        rbimpl_atomic_ptr_exchange((void **)&global_objspace->tdata_unsafe_free_published, NULL,
                                   RBIMPL_ATOMIC_ACQ_REL);
    while (chunk) {
        struct tdata_unsafe_free_chunk *next = chunk->next;
        gc_tdata_unsafe_drain_chunk(chunk);
        chunk = next;
    }
 
    for (size_t i = 0; i < n; i++) {
        rb_objspace_t *os = objspaces[i];
        struct tdata_unsafe_free_chunk *partial = os->tdata_unsafe_free_chunk;
        if (partial) {
            os->tdata_unsafe_free_chunk = NULL;
            gc_tdata_unsafe_drain_chunk(partial);
        }
    }
 
    rbimpl_atomic_size_exchange(&global_objspace->tdata_deferred_free_count, 0,
                                RBIMPL_ATOMIC_RELAXED);
}
 
/* Stop the world and run the dfree function for all deferred T_DATAs. */
static void
gc_tdata_unsafe_drain(void)
{
    unsigned int lev = RB_GC_VM_LOCK();
 
    if (tdata_deferred_free_count_load() == 0) {
        RB_GC_VM_UNLOCK(lev);
        return;
    }
 
    rb_gc_vm_barrier();
 
    gc_global_snapshot_objspaces();
 
    /* Set during_gc=TRUE and init vm_context for the CURRENT objspace only.
     * The no-alloc guard checks only the allocating (=current) objspace's during_gc,
     * and rb_gc_get_ec() reads only the current objspace's vm_context.ec. */
    rb_objspace_t *objspace = rb_gc_get_objspace();
    unsigned int saved_during_gc = gc_during_gc_get(objspace);
    dont_gc_on();
    rb_gc_initialize_vm_context(&objspace->vm_context);
    gc_during_gc_set(objspace, TRUE);
 
    gc_tdata_unsafe_drain_objspaces(global_objspace->global_gc.objspaces,
                                    global_objspace->global_gc.n_objspaces);
 
    gc_during_gc_set(objspace, saved_during_gc);
    dont_gc_off();
 
    RB_GC_VM_UNLOCK(lev);
}
 
static void
gc_tdata_deferred_free_job(void *unused)
{
    (void)unused;
 
    size_t count = tdata_deferred_free_count_load();
    if (count == 0) return;
    if (count < TDATA_DEFERRED_FREE_THRESHOLD && !rb_gc_single_objspace_p()) return;
 
    gc_tdata_unsafe_drain();
}
 
static inline void
gc_sweep_register_free_slot(rb_objspace_t *objspace, struct heap_page *page, struct gc_sweep_context *ctx, uintptr_t p, short slot_size)
{
    rb_asan_unpoison_object(p, false);
    ((struct RBasic *)p)->flags = 0;
 
    /* Keep a freed slot from carrying its old shareable and shref bits into the next
     * object born there; the actual clear happens per bitmap word at the end of
     * gc_sweep_page rather than per slot. */
 
    struct free_region *existing_region = ctx->free_region;
    if (existing_region) rb_asan_unpoison_object((VALUE)existing_region, false);
 
    if (RB_LIKELY(existing_region && p == existing_region->end)) {
        existing_region->end = p + slot_size;
    }
    else {
        struct free_region *free_region = (struct free_region *)p;
        free_region->end = p + slot_size;
        free_region->next = existing_region;
 
        ctx->free_region = free_region;
    }
 
    if (existing_region) rb_asan_poison_object((VALUE)existing_region);
    rb_asan_poison_object(p);
}
 
static inline void
gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct gc_sweep_context *ctx)
{
    struct heap_page *sweep_page = ctx->page;
    short slot_size = sweep_page->slot_size;
 
    do {
        VALUE vp = (VALUE)p;
        GC_ASSERT(vp % sizeof(VALUE) == 0);
 
        rb_asan_unpoison_object(vp, false);
        if (bitset & 1) {
            switch (BUILTIN_TYPE(vp)) {
              case T_MOVED:
                if (objspace->flags.during_compacting) {
                    /* The sweep cursor shouldn't have made it to any
                     * T_MOVED slots while the compact flag is enabled.
                     * The sweep cursor and compact cursor move in
                     * opposite directions, and when they meet references will
                     * get updated and "during_compacting" should get disabled */
                    rb_bug("T_MOVED shouldn't be seen until compaction is finished");
                }
                gc_report(3, objspace, "page_sweep: %s is freed\n", rb_obj_info(vp));
                ctx->empty_slots++;
                gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
                break;
              case T_ZOMBIE:
                /* already counted */
                break;
              case T_NONE:
                ctx->empty_slots++; /* already freed */
                gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
                break;
 
              default:
#if RGENGC_CHECK_MODE
                /* A local GC must never free a pinned slot; a global GC may (its exact
                 * mark collects dead shareable objects).  Reading the bits here is
                 * CHECK-only and still valid: the bulk clear runs after the free loop. */
                if (ctx->check_pinned_free &&
                    (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(vp), vp) ||
                     MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(vp), vp))) {
                    rb_bug("page_sweep: freeing pinned slot %s (shareable=%d shref=%d single_now=%d)",
                           rb_obj_info(vp),
                           (int)!!MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(vp), vp),
                           (int)!!MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(vp), vp),
                           (int)rb_gc_single_objspace_p());
                }
#endif
#if RGENGC_CHECK_MODE
                if (!is_full_marking(objspace)) {
                    if (RVALUE_OLD_P(objspace, vp)) rb_bug("page_sweep: %p - old while minor GC.", (void *)p);
                    if (RVALUE_REMEMBERED(objspace, vp)) rb_bug("page_sweep: %p - remembered.", (void *)p);
                }
#endif
 
#if RGENGC_CHECK_MODE
#define CHECK(x) if (x(objspace, vp) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", rb_obj_info(vp))
                CHECK(RVALUE_WB_UNPROTECTED);
                CHECK(RVALUE_MARKED);
                CHECK(RVALUE_MARKING);
                CHECK(RVALUE_UNCOLLECTIBLE);
#undef CHECK
#endif
 
                if (!rb_gc_obj_needs_cleanup_p(vp)) {
                    (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size);
                    gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
                    gc_report(3, objspace, "page_sweep: %s (fast path) is freed\n", rb_obj_info(vp));
                    ctx->freed_slots++;
                }
                else {
                    gc_report(2, objspace, "page_sweep: free %p\n", (void *)p);
 
                    if (RB_UNLIKELY(ctx->defer_thread_unsafe_local_sweep && gc_obj_defer_local_free_p(objspace, vp))) {
                        /* Defer the dfree instead of running it here: it needs the world
                         * stopped, which a parallel local sweep cannot give it. The slot is reusable
                         * right away unless we had to create a zombie. */
                        if (gc_defer_thread_unsafe_free(objspace, vp,
                                                        &ctx->trigger_thread_unsafe_sweep_postponed_job)) {
                            (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size);
                            gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
                            ctx->freed_slots++;
                        }
                        else {
                            ctx->final_slots++;
                        }
                        break;
                    }
                    rb_gc_obj_free_vm_weak_references(vp);
                    if (gc_obj_free(objspace, vp)) {
                        (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size);
                        gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
                        gc_report(3, objspace, "page_sweep: %s is freed\n", rb_obj_info(vp));
                        ctx->freed_slots++;
                    }
                    else {
                        ctx->final_slots++;
                    }
                }
                break;
            }
        }
        p += slot_size;
        bitset >>= 1;
    } while (bitset);
}
 
static inline void
gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context *ctx)
{
    struct heap_page *sweep_page = ctx->page;
    GC_ASSERT(sweep_page->heap == heap);
 
    uintptr_t p;
    bits_t *bits, bitset;
 
    gc_report(2, objspace, "page_sweep: start.\n");
 
#if RGENGC_CHECK_MODE
    if (!objspace->flags.immediate_sweep) {
        GC_ASSERT(sweep_page->flags.before_sweep == TRUE);
    }
#endif
    sweep_page->flags.before_sweep = FALSE;
    sweep_page->free_slots = 0;
 
    asan_unlock_freelist(sweep_page);
    sweep_page->free_region = NULL;
    asan_lock_freelist(sweep_page);
    ctx->free_region = NULL;
 
    p = (uintptr_t)sweep_page->start;
    bits = sweep_page->mark_bits;
    short slot_size = sweep_page->slot_size;
    int total_slots = sweep_page->total_slots;
    int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
 
    int out_of_range_bits = total_slots % BITS_BITLENGTH;
    if (out_of_range_bits != 0) {
        bits[bitmap_plane_count - 1] |= ~(((bits_t)1 << out_of_range_bits) - 1);
    }
 
    // Clear wb_unprotected and age bits for all unmarked slots
    {
        bits_t *wb_unprotected_bits = sweep_page->wb_unprotected_bits;
        bits_t *age_bits = sweep_page->age_bits;
        for (int i = 0; i < bitmap_plane_count; i++) {
            bits_t unmarked = ~bits[i];
            wb_unprotected_bits[i] &= ~unmarked;
            age_bits[i * 2] &= ~unmarked;
            age_bits[i * 2 + 1] &= ~unmarked;
        }
    }
 
    for (int i = 0; i < bitmap_plane_count; i++) {
        bitset = ~bits[i];
        if (bitset) {
            gc_sweep_plane(objspace, heap, p, bitset, ctx);
        }
        p += BITS_BITLENGTH * slot_size;
    }
 
    /* Bulk-clear the freed slots' shareable and shref bits before the freelist is
     * published, so a reused slot is clean.  Freed slots are exactly the unmarked ones,
     * so `bits &= mark_bits` keeps live shareable objects (which must stay pinned) and
     * drops the rest.  Pages with neither bit are skipped. */
    if (sweep_page->flags.has_shareable_objects || sweep_page->flags.has_shref_objects) {
        bits_t *shareable_bits = sweep_page->shareable_bits;
        bits_t *shref_bits = sweep_page->shref_bits;
        bits_t sh = 0, sr = 0;
        for (int i = 0; i < bitmap_plane_count; i++) {
            shareable_bits[i] &= bits[i];
            shref_bits[i] &= bits[i];
            sh |= shareable_bits[i];
            sr |= shref_bits[i];
        }
        if (!sh) sweep_page->flags.has_shareable_objects = FALSE;
        if (!sr) sweep_page->flags.has_shref_objects = FALSE;
    }
 
    asan_unlock_freelist(sweep_page);
    sweep_page->free_region = ctx->free_region;
    asan_lock_freelist(sweep_page);
 
    if (!heap->compact_cursor) {
        gc_setup_mark_bits(sweep_page);
    }
 
#if GC_PROFILE_MORE_DETAIL
    if (gc_prof_enabled(objspace)) {
        gc_profile_record *record = gc_prof_record(objspace);
        record->removing_objects += ctx->final_slots + ctx->freed_slots;
        record->empty_objects += ctx->empty_slots;
    }
#endif
    if (0) fprintf(stderr, "gc_sweep_page(%"PRIdSIZE"): total_slots: %d, freed_slots: %d, empty_slots: %d, final_slots: %d\n",
                   rb_gc_count(),
                   sweep_page->total_slots,
                   ctx->freed_slots, ctx->empty_slots, ctx->final_slots);
 
    sweep_page->free_slots += ctx->freed_slots + ctx->empty_slots;
    sweep_page->heap->total_freed_objects += ctx->freed_slots;
 
    if (heap_pages_deferred_final && !finalizing) {
        gc_finalize_deferred_register(objspace);
    }
 
#if RGENGC_CHECK_MODE
    int region_slots = 0;
    asan_unlock_freelist(sweep_page);
    struct free_region *region = sweep_page->free_region;
    while (region) {
        rb_asan_unpoison_object((VALUE)region, false);
        GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
        uintptr_t region_start = (uintptr_t)region;
        uintptr_t region_end = region->end;
        struct free_region *next = region->next;
        rb_asan_poison_object((VALUE)region);
 
        GC_ASSERT(region_end > region_start);
        GC_ASSERT((region_end - region_start) % slot_size == 0);
        region_slots += (int)((region_end - region_start) / slot_size);
 
        region = next;
    }
    asan_lock_freelist(sweep_page);
    if (region_slots != sweep_page->free_slots) {
        rb_bug("inconsistent free region slots: expected %d but was %d", sweep_page->free_slots, region_slots);
    }
#endif
 
    gc_report(2, objspace, "page_sweep: end.\n");
}
 
static const char *
gc_mode_name(enum gc_mode mode)
{
    switch (mode) {
      case gc_mode_none: return "none";
      case gc_mode_marking: return "marking";
      case gc_mode_sweeping: return "sweeping";
      case gc_mode_compacting: return "compacting";
      default: rb_bug("gc_mode_name: unknown mode: %d", (int)mode);
    }
}
 
static void
gc_mode_transition(rb_objspace_t *objspace, enum gc_mode mode)
{
#if RGENGC_CHECK_MODE
    enum gc_mode prev_mode = gc_mode(objspace);
    switch (prev_mode) {
      case gc_mode_none:
        /* A global GC marks every objspace as one heap (mark_roots on the driver), so an
         * individual objspace's mode stays `none` during that mark; the sweep inside the
         * barrier then makes the legitimate none -> sweeping transition. */
        GC_ASSERT(mode == gc_mode_marking ||
                  (objspace->flags.during_global_gc && mode == gc_mode_sweeping));
        break;
      case gc_mode_marking:  GC_ASSERT(mode == gc_mode_sweeping); break;
      case gc_mode_sweeping: GC_ASSERT(mode == gc_mode_none || mode == gc_mode_compacting); break;
      case gc_mode_compacting: GC_ASSERT(mode == gc_mode_none); break;
    }
#endif
    if (0) fprintf(stderr, "gc_mode_transition: %s->%s\n", gc_mode_name(gc_mode(objspace)), gc_mode_name(mode));
    gc_mode_set(objspace, mode);
}
 
static void
heap_page_flush_alloc_regions(struct heap_page *page, rb_heap_t *heap)
{
    struct free_region *chain = heap->newobj.alloc_next_region;
 
    if (heap->newobj.alloc_cursor < heap->newobj.alloc_cursor_end) {
        VALUE start = (VALUE)heap->newobj.alloc_cursor;
        rb_asan_unpoison_object(start, false);
        struct free_region *remnant = (struct free_region *)start;
        remnant->flags = 0;
        remnant->end = heap->newobj.alloc_cursor_end;
        remnant->next = chain;
        rb_asan_poison_object(start);
        chain = remnant;
    }
 
    if (chain) {
        asan_unlock_freelist(page);
        if (page->free_region) {
            struct free_region *p = page->free_region;
            rb_asan_unpoison_object((VALUE)p, false);
            while (p->next) {
                struct free_region *prev = p;
                p = p->next;
                rb_asan_poison_object((VALUE)prev);
                rb_asan_unpoison_object((VALUE)p, false);
            }
            p->next = chain;
            rb_asan_poison_object((VALUE)p);
        }
        else {
            page->free_region = chain;
        }
        asan_lock_freelist(page);
    }
}
 
static void
gc_sweep_start_heap(rb_objspace_t *objspace, rb_heap_t *heap)
{
    heap->sweeping_page = ccan_list_top(&heap->pages, struct heap_page, page_node);
    if (heap->sweeping_page) {
        objspace->sweeping_heap_count++;
    }
    heap->free_pages = NULL;
    heap->pooled_pages = NULL;
    if (!objspace->flags.immediate_sweep) {
        struct heap_page *page = NULL;
 
        ccan_list_for_each(&heap->pages, page, page_node) {
            page->flags.before_sweep = TRUE;
        }
    }
}
 
#if GC_CAN_COMPILE_COMPACTION
static void gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func);
static int compare_pinned_slots(const void *left, const void *right, void *d);
#endif
 
/* Return the current allocation page and freelist to their pages, so the sweeper sees a
 * consistent heap. */
static void
heap_alloc_state_clear(rb_objspace_t *objspace)
{
    objspace->incremental_mark_step_allocated_slots = 0;
 
    for (size_t heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) {
        rb_heap_t *heap = &heaps[heap_idx];
 
        struct heap_page *page = heap->newobj.alloc_using_page;
        RUBY_DEBUG_LOG("heap alloc_using_page:%p cursor:%p", (void *)page, (void *)heap->newobj.alloc_cursor);
 
        if (page) {
            heap_page_flush_alloc_regions(page, heap);
        }
 
        heap->newobj.alloc_using_page = NULL;
        heap->newobj.alloc_cursor = 0;
        heap->newobj.alloc_cursor_end = 0;
        heap->newobj.alloc_next_region = NULL;
    }
}
 
static void
gc_sweep_freeobj_hooks_page(rb_objspace_t *objspace, struct heap_page *page)
{
    bits_t *bits = page->mark_bits;
    uintptr_t p = (uintptr_t)page->start;
    short slot_size = page->slot_size;
    int total_slots = page->total_slots;
    int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
 
    int out_of_range_bits = total_slots % BITS_BITLENGTH;
    bits_t last_plane_mask = (out_of_range_bits != 0)
        ? ~(((bits_t)1 << out_of_range_bits) - 1)
        : 0;
 
    for (int j = 0; j < bitmap_plane_count; j++) {
        bits_t bitset = ~bits[j];
        if (j == bitmap_plane_count - 1) {
            bitset &= ~last_plane_mask;
        }
 
        uintptr_t pp = p;
        while (bitset) {
            if (bitset & 1) {
                VALUE vp = (VALUE)pp;
                asan_unpoisoning_object(vp) {
                    switch (BUILTIN_TYPE(vp)) {
                      case T_NONE:
                      case T_ZOMBIE:
                      case T_MOVED:
                        break;
                      default:
                        rb_gc_event_hook(vp, RUBY_INTERNAL_EVENT_FREEOBJ);
                        break;
                    }
                }
            }
            pp += slot_size;
            bitset >>= 1;
        }
        p += BITS_BITLENGTH * slot_size;
    }
}
 
static void
gc_sweep_freeobj_hooks(rb_objspace_t *objspace)
{
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        struct heap_page *page = NULL;
 
        ccan_list_for_each(&heap->pages, page, page_node) {
            gc_sweep_freeobj_hooks_page(objspace, page);
        }
    }
}
 
static void
gc_sweep_start(rb_objspace_t *objspace)
{
    gc_mode_transition(objspace, gc_mode_sweeping);
    objspace->rincgc.pooled_slots = 0;
 
    if (RB_UNLIKELY(objspace->hook_events & RUBY_INTERNAL_EVENT_FREEOBJ)) {
        /* FREEOBJ is never enabled outside the main objspace
         * (rb_objspace_set_event_hook), so this hook, which runs user callbacks,
         * cannot fire during a non-main Ractor's lock-free local sweep. */
        GC_ASSERT(objspace == global_objspace->main_objspace);
        gc_sweep_freeobj_hooks(objspace);
    }
 
#if GC_CAN_COMPILE_COMPACTION
    if (objspace->flags.during_compacting) {
        gc_sort_heap_by_compare_func(
            objspace,
            objspace->rcompactor.compare_func ? objspace->rcompactor.compare_func : compare_pinned_slots
        );
    }
#endif
 
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        gc_sweep_start_heap(objspace, heap);
 
        /* We should call gc_sweep_finish_heap for size pools with no pages. */
        if (heap->sweeping_page == NULL) {
            GC_ASSERT(heap->total_pages == 0);
            GC_ASSERT(heap->total_slots == 0);
            gc_sweep_finish_heap(objspace, heap);
        }
    }
 
    heap_alloc_state_clear(objspace);
}
 
static void
gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap)
{
    /* Nothing allocates from an ownerless objspace, so it has no growth decision */
    if (objspace->flags.ownerless) return;
 
    size_t total_slots = heap->total_slots;
    size_t swept_slots = heap->freed_slots + heap->empty_slots;
 
    size_t init_slots = objspace_heap_init_bytes(objspace) / heap->slot_size;
    size_t min_free_slots = (size_t)(MAX(total_slots, init_slots) * gc_params.heap_free_slots_min_ratio);
 
    if (swept_slots < min_free_slots &&
            /* The heap is a growth heap if it freed more slots than had empty slots. */
            ((heap->empty_slots == 0 && total_slots > 0) || heap->freed_slots > heap->empty_slots)) {
        /* If we don't have enough slots and we have pages on the tomb heap, move
        * pages from the tomb heap to the eden heap. This may prevent page
        * creation thrashing (frequently allocating and deallocting pages) and
        * GC thrashing (running GC more frequently than required). */
        struct heap_page *resurrected_page;
        while (swept_slots < min_free_slots &&
                (resurrected_page = heap_page_resurrect(objspace))) {
            heap_add_page(objspace, heap, resurrected_page);
            heap_add_freepage(heap, resurrected_page);
 
            swept_slots += resurrected_page->free_slots;
        }
 
        if (swept_slots < min_free_slots) {
            /* Grow this heap if we are in a major GC or if we haven't run at least
             * RVALUE_OLD_AGE minor GC since the last major GC. */
            if (is_full_marking(objspace) ||
                    objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
                if (objspace->heap_pages.allocatable_bytes < min_free_slots * heap->slot_size) {
                    heap_allocatable_bytes_expand(objspace, heap, swept_slots, heap->total_slots, heap->slot_size);
                }
            }
            else if (swept_slots < min_free_slots * 7 / 8 &&
                     objspace->heap_pages.allocatable_bytes < (min_free_slots * 7 / 8 - swept_slots) * heap->slot_size) {
                gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
                heap->force_major_gc_count++;
            }
        }
    }
}
 
#if RGENGC_ESTIMATE_OLDMALLOC
static void
gc_check_oldmalloc(rb_objspace_t *objspace)
{
    size_t oldmalloc_increase = gc_oldmalloc_increase_load(objspace);
    MALLOC_COUNTERS_LOCK(objspace);
    gc_counter_t malloc_at = gc_counter_load_acquire(&objspace->malloc_counters.counters.malloc_at_last_gc);
    gc_counter_t free_at   = gc_counter_load_acquire(&objspace->malloc_counters.counters.free_at_last_gc);
    gc_counter_t malloc_now = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc);
    gc_counter_t free_now   = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free);
    gc_counter_t gc_freed   = gc_counter_load_relaxed(&objspace->malloc_counters.freed_in_gc);
    MALLOC_COUNTERS_UNLOCK(objspace);
 
    gc_counter_t freed = free_now - free_at;
    gc_counter_t mutator_freed = freed > gc_freed ? freed - gc_freed : 0;
    gc_counter_t allocated = malloc_now - malloc_at;
    gc_counter_t unswept = allocated > mutator_freed ? allocated - mutator_freed : 0;
 
    if ((gc_counter_t)oldmalloc_increase <= unswept) return;
    oldmalloc_increase -= (size_t)unswept;
 
    if (oldmalloc_increase > objspace->rgengc.oldmalloc_increase_limit) {
        gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDMALLOC;
        objspace->rgengc.oldmalloc_increase_limit =
          (size_t)(objspace->rgengc.oldmalloc_increase_limit * gc_params.oldmalloc_limit_growth_factor);
 
        if (objspace->rgengc.oldmalloc_increase_limit > gc_params.oldmalloc_limit_max) {
            objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_max;
        }
    }
}
#endif
 
static void
gc_sweep_finish(rb_objspace_t *objspace)
{
    gc_report(1, objspace, "gc_sweep_finish\n");
 
    gc_prof_set_heap_info(objspace);
    heap_pages_free_unused_pages(objspace);
    if (rb_gc_single_objspace_p() && is_full_marking(objspace)) {
        /* gc_marks_finish retains ~2/3 of empty pages in objspace->empty_pages for reuse,
         * only the excess reaches the pool. */
        page_pool_reclaim(global_objspace);
    }
#if RGENGC_ESTIMATE_OLDMALLOC
    if (!is_full_marking(objspace)) {
        gc_check_oldmalloc(objspace);
    }
#endif
 
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
 
        heap->freed_slots = 0;
        heap->empty_slots = 0;
 
        if (!will_be_incremental_marking(objspace)) {
            struct heap_page *end_page = heap->free_pages;
            if (end_page) {
                while (end_page->free_next) end_page = end_page->free_next;
                end_page->free_next = heap->pooled_pages;
            }
            else {
                heap->free_pages = heap->pooled_pages;
            }
            heap->pooled_pages = NULL;
            objspace->rincgc.pooled_slots = 0;
        }
    }
 
    /* Not before: while sweeping is in progress its frees must keep reducing
     * malloc_increase (objspace_malloc_increase_body sweeps and retries on it). */
    gc_malloc_counters_snapshot_free_at_last_gc(objspace, &objspace->malloc_counters.counters);
 
    /* Leftovers from an earlier multi-Ractor phase: no later sweep can rediscover them
     * (their slots are gone), and with one Ractor left the drain's barrier has nothing
     * to wait for. */
    if (tdata_deferred_free_count_load() > 0 && rb_gc_single_objspace_p()) {
        gc_tdata_deferred_free_trigger(objspace);
    }
 
    gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_SWEEP);
    gc_mode_transition(objspace, gc_mode_none);
}
 
static int
gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap)
{
    struct heap_page *sweep_page = heap->sweeping_page;
    int swept_slots = 0;
    int pooled_slots = 0;
    int sweep_budget = GC_INCREMENTAL_SWEEP_BYTES / heap->slot_size;
    int pool_budget = GC_INCREMENTAL_SWEEP_POOL_BYTES / heap->slot_size;
 
    if (sweep_page == NULL) return FALSE;
 
#if GC_ENABLE_LAZY_SWEEP
    gc_prof_sweep_timer_start(objspace);
#endif
 
    /* Per-slot pinned-free assert (gc_sweep_context): check only when this cycle's mark
     * ran the pinned walk.  The current world state would misfire: a single-world
     * cycle leaves dead shareable objects unmarked and its sweep can straddle the switch
     * to multi-objspace.  A global GC's exact mark does not pin, so it is excluded. */
    const bool check_pinned_free = objspace->last_cycle_pinned;
 
    const bool defer_thread_unsafe_local_sweep =
        !rb_gc_single_objspace_p() && !objspace->flags.during_global_gc;
    bool trigger_thread_unsafe_sweep_postponed_job = false;
 
    do {
        RUBY_DEBUG_LOG("sweep_page:%p", (void *)sweep_page);
 
        struct gc_sweep_context ctx = {
            .page = sweep_page,
            .final_slots = 0,
            .freed_slots = 0,
            .empty_slots = 0,
            .check_pinned_free = check_pinned_free,
            .defer_thread_unsafe_local_sweep = defer_thread_unsafe_local_sweep,
            .trigger_thread_unsafe_sweep_postponed_job = trigger_thread_unsafe_sweep_postponed_job,
        };
        gc_sweep_page(objspace, heap, &ctx);
        int free_slots = ctx.freed_slots + ctx.empty_slots;
        trigger_thread_unsafe_sweep_postponed_job = ctx.trigger_thread_unsafe_sweep_postponed_job;
 
        RUBY_DTRACE_GC_HOOK(SWEEP_PAGE, ctx.page->slot_size, ctx.final_slots, ctx.freed_slots, ctx.empty_slots);
 
        heap->sweeping_page = ccan_list_next(&heap->pages, sweep_page, page_node);
 
        if (free_slots == sweep_page->total_slots &&
                (heap->total_pages > 1 || objspace->flags.ownerless)) {
            /* There are no living objects, so move this page to the global empty pages.
             * The last one stays so the heap has something to grow from -- except in an
             * ownerless objspace, which will never allocate again. */
            heap_page_detach_empty(objspace, heap, sweep_page);
        }
        else if (free_slots > 0) {
            heap->freed_slots += ctx.freed_slots;
            heap->empty_slots += ctx.empty_slots;
 
            if (pooled_slots < pool_budget) {
                heap_add_poolpage(objspace, heap, sweep_page);
                pooled_slots += free_slots;
            }
            else {
                heap_add_freepage(heap, sweep_page);
                swept_slots += free_slots;
                if (swept_slots > sweep_budget) {
                    break;
                }
            }
        }
        else {
            sweep_page->free_next = NULL;
        }
    } while ((sweep_page = heap->sweeping_page));
 
    if (trigger_thread_unsafe_sweep_postponed_job) {
        gc_report(2, objspace, "thread-unsafe sweep postponed job triggered\n");
        gc_tdata_deferred_free_trigger(objspace);
    }
 
    if (!heap->sweeping_page) {
        objspace->sweeping_heap_count--;
        GC_ASSERT(objspace->sweeping_heap_count >= 0);
        gc_sweep_finish_heap(objspace, heap);
 
        if (!has_sweeping_pages(objspace)) {
            gc_sweep_finish(objspace);
        }
    }
 
#if GC_ENABLE_LAZY_SWEEP
    gc_prof_sweep_timer_stop(objspace);
#endif
 
    return heap->free_pages != NULL;
}
 
static void
gc_sweep_rest(rb_objspace_t *objspace)
{
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
 
        while (heap->sweeping_page) {
            gc_sweep_step(objspace, heap);
        }
    }
 
    /* An objspace with no live pages never runs gc_sweep_step and so never reaches
     * gc_sweep_finish, leaving mode at sweeping or compacting until the next cycle's
     * gc_sweep_start asserts.  If every heap is swept out, settle it to none here. */
    if (gc_mode(objspace) != gc_mode_none && !has_sweeping_pages(objspace)) {
        gc_sweep_finish(objspace);
    }
}
 
static void
gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *sweep_heap)
{
    GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
    if (!GC_ENABLE_LAZY_SWEEP) return;
 
    gc_sweeping_enter(objspace);
 
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        if (gc_sweep_step(objspace, heap)) {
            GC_ASSERT(heap->free_pages != NULL);
        }
        else if (heap == sweep_heap) {
            if (objspace->empty_pages_count > 0 || objspace->heap_pages.allocatable_bytes > 0) {
                /* [Bug #21548]
                 *
                 * If this heap is the heap we want to sweep, but we weren't able
                 * to free any slots, but we also either have empty pages or could
                 * allocate new pages, then we want to preemptively claim a page
                 * because it's possible that sweeping another heap will call
                 * gc_sweep_finish_heap, which may use up all of the
                 * empty/allocatable pages. If other heaps are not finished sweeping
                 * then we do not finish this GC and we will end up triggering a new
                 * GC cycle during this GC phase. */
                heap_page_allocate_and_initialize(objspace, heap);
 
                GC_ASSERT(heap->free_pages != NULL);
            }
            else {
                /* Not allowed to create a new page so finish sweeping. */
                gc_sweep_rest(objspace);
                GC_ASSERT(gc_mode(objspace) == gc_mode_none);
                break;
            }
        }
    }
 
    gc_sweeping_exit(objspace);
}
 
static void
gc_sweep_step_for_malloc(rb_objspace_t *objspace)
{
    GC_ASSERT(is_lazy_sweeping(objspace));
 
    unsigned int lock_lev;
    gc_enter(objspace, gc_enter_event_continue, &lock_lev);
 
    gc_sweeping_enter(objspace);
 
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        gc_sweep_step(objspace, heap);
    }
 
    gc_sweeping_exit(objspace);
 
    gc_exit(objspace, gc_enter_event_continue, &lock_lev);
}
 
static bool gc_global_pointer_to_heap_p(const void *ptr);
 
VALUE
rb_gc_impl_location(void *objspace_ptr, VALUE value)
{
    rb_objspace_t *objspace = objspace_ptr;
    VALUE destination;
 
    /* A local (single-objspace) compaction never moves another objspace's objects, so
     * leave foreign references alone.  A compacting global GC moves objects everywhere
     * under the barrier, so there every objspace's heap is searched for forwarding. */
    if (RB_UNLIKELY(objspace->flags.during_global_gc)
            ? !gc_global_pointer_to_heap_p((void *)value)
            : !is_pointer_to_heap(objspace_ptr, (void *)value)) {
        return value;
    }
 
    asan_unpoisoning_object(value) {
        if (BUILTIN_TYPE(value) == T_MOVED) {
            destination = (VALUE)RMOVED(value)->destination;
            GC_ASSERT(BUILTIN_TYPE(destination) != T_NONE);
        }
        else {
            destination = value;
        }
    }
 
    return destination;
}
 
#if GC_CAN_COMPILE_COMPACTION
static void
invalidate_moved_plane(rb_objspace_t *objspace, struct heap_page *page, uintptr_t p, bits_t bitset)
{
    if (bitset) {
        do {
            if (bitset & 1) {
                VALUE forwarding_object = (VALUE)p;
                VALUE object;
 
                if (BUILTIN_TYPE(forwarding_object) == T_MOVED) {
                    GC_ASSERT(RVALUE_PINNED(objspace, forwarding_object));
                    GC_ASSERT(!RVALUE_MARKED(objspace, forwarding_object));
 
                    CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object);
 
                    object = rb_gc_impl_location(objspace, forwarding_object);
                    gc_move(objspace, object, forwarding_object, GET_HEAP_PAGE(object), page);
                    /* forwarding_object is now our actual object, and "object"
                     * is the free slot for the original page */
 
                    struct heap_page *orig_page = GET_HEAP_PAGE(object);
                    orig_page->free_slots++;
                    RVALUE_AGE_SET_BITMAP(object, 0);
                    heap_page_add_free_region(objspace, orig_page, object);
 
                    GC_ASSERT(RVALUE_MARKED(objspace, forwarding_object));
                    GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_MOVED);
                    GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_NONE);
                }
            }
            p += page->slot_size;
            bitset >>= 1;
        } while (bitset);
    }
}
 
static void
invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page)
{
    int i;
    bits_t *mark_bits, *pin_bits;
    bits_t bitset;
    short slot_size = page->slot_size;
    int total_slots = page->total_slots;
    int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
 
    mark_bits = page->mark_bits;
    pin_bits = page->pinned_bits;
 
    uintptr_t p = page->start;
 
    for (i=0; i < bitmap_plane_count; i++) {
        /* Moved objects are pinned but never marked. We reuse the pin bits
         * to indicate there is a moved object in this slot. */
        bitset = pin_bits[i] & ~mark_bits[i];
        invalidate_moved_plane(objspace, page, p, bitset);
        p += BITS_BITLENGTH * slot_size;
    }
}
#endif
 
static void
gc_compact_start(rb_objspace_t *objspace)
{
    struct heap_page *page = NULL;
    gc_mode_transition(objspace, gc_mode_compacting);
 
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        ccan_list_for_each(&heap->pages, page, page_node) {
            page->flags.before_sweep = TRUE;
        }
 
        heap->compact_cursor = ccan_list_tail(&heap->pages, struct heap_page, page_node);
        heap->compact_cursor_index = 0;
    }
 
    if (gc_prof_enabled(objspace)) {
        gc_profile_record *record = gc_prof_record(objspace);
        record->moved_objects = objspace->rcompactor.total_moved;
    }
 
    memset(objspace->rcompactor.considered_count_table, 0, T_MASK * sizeof(size_t));
    memset(objspace->rcompactor.moved_count_table, 0, T_MASK * sizeof(size_t));
    memset(objspace->rcompactor.moved_up_count_table, 0, T_MASK * sizeof(size_t));
    memset(objspace->rcompactor.moved_down_count_table, 0, T_MASK * sizeof(size_t));
 
    /* Set up read barrier for pages containing MOVED objects */
    /* A compacting global GC installs the read barrier once for every objspace. */
    if (!global_objspace->global_gc.compacting) install_handlers();
}
 
static void gc_sweep_compact(rb_objspace_t *objspace);
 
static void
gc_sweep(rb_objspace_t *objspace)
{
    gc_sweeping_enter(objspace);
 
    const unsigned int immediate_sweep = objspace->flags.immediate_sweep;
 
    gc_report(1, objspace, "gc_sweep: immediate: %d\n", immediate_sweep);
 
    gc_sweep_start(objspace);
    if (objspace->flags.during_compacting) {
        rb_hrtime_t compact_start_time = gc_prof_enabled(objspace) ? rb_hrtime_now() : 0;
        gc_sweep_compact(objspace);
        if (gc_prof_enabled(objspace)) {
            rb_hrtime_t compact_wall_time = elapsed_hrtime_from(compact_start_time);
            gc_profile_record *record = gc_prof_record(objspace);
            record->gc_compact_wall_time = rb_hrtime_add(record->gc_compact_wall_time,
                    compact_wall_time);
            objspace->profile.gc_sweep_excluded_wall_time = rb_hrtime_add(
                    objspace->profile.gc_sweep_excluded_wall_time,
                    compact_wall_time);
        }
    }
 
    if (immediate_sweep) {
#if !GC_ENABLE_LAZY_SWEEP
        gc_prof_sweep_timer_start(objspace);
#endif
        gc_sweep_rest(objspace);
#if !GC_ENABLE_LAZY_SWEEP
        gc_prof_sweep_timer_stop(objspace);
#endif
    }
    else {
 
        /* Sweep every size pool. */
        for (int i = 0; i < HEAP_COUNT; i++) {
            rb_heap_t *heap = &heaps[i];
            gc_sweep_step(objspace, heap);
        }
    }
 
    gc_sweeping_exit(objspace);
}
 
/* Marking - Marking stack */
 
static stack_chunk_t *
stack_chunk_alloc(void)
{
    stack_chunk_t *res;
 
    res = malloc(sizeof(stack_chunk_t));
    if (!res)
        rb_memerror();
 
    return res;
}
 
static inline int
is_mark_stack_empty(mark_stack_t *stack)
{
    return stack->chunk == NULL;
}
 
static size_t
mark_stack_size(mark_stack_t *stack)
{
    size_t size = stack->index;
    stack_chunk_t *chunk = stack->chunk ? stack->chunk->next : NULL;
 
    while (chunk) {
        size += stack->limit;
        chunk = chunk->next;
    }
    return size;
}
 
static void
add_stack_chunk_cache(mark_stack_t *stack, stack_chunk_t *chunk)
{
    chunk->next = stack->cache;
    stack->cache = chunk;
    stack->cache_size++;
}
 
static void
shrink_stack_chunk_cache(mark_stack_t *stack)
{
    stack_chunk_t *chunk;
 
    if (stack->unused_cache_size > (stack->cache_size/2)) {
        chunk = stack->cache;
        stack->cache = stack->cache->next;
        stack->cache_size--;
        free(chunk);
    }
    stack->unused_cache_size = stack->cache_size;
}
 
static void
push_mark_stack_chunk(mark_stack_t *stack)
{
    stack_chunk_t *next;
 
    GC_ASSERT(stack->index == stack->limit);
 
    if (stack->cache_size > 0) {
        next = stack->cache;
        stack->cache = stack->cache->next;
        stack->cache_size--;
        if (stack->unused_cache_size > stack->cache_size)
            stack->unused_cache_size = stack->cache_size;
    }
    else {
        next = stack_chunk_alloc();
    }
    next->next = stack->chunk;
    stack->chunk = next;
    stack->index = 0;
}
 
static void
pop_mark_stack_chunk(mark_stack_t *stack)
{
    stack_chunk_t *prev;
 
    prev = stack->chunk->next;
    GC_ASSERT(stack->index == 0);
    add_stack_chunk_cache(stack, stack->chunk);
    stack->chunk = prev;
    stack->index = stack->limit;
}
 
static void
mark_stack_chunk_list_free(stack_chunk_t *chunk)
{
    stack_chunk_t *next = NULL;
 
    while (chunk != NULL) {
        next = chunk->next;
        free(chunk);
        chunk = next;
    }
}
 
static void
free_stack_chunks(mark_stack_t *stack)
{
    mark_stack_chunk_list_free(stack->chunk);
}
 
static void
mark_stack_free_cache(mark_stack_t *stack)
{
    mark_stack_chunk_list_free(stack->cache);
    stack->cache_size = 0;
    stack->unused_cache_size = 0;
}
 
static void
push_mark_stack(mark_stack_t *stack, VALUE obj)
{
    switch (BUILTIN_TYPE(obj)) {
      case T_OBJECT:
      case T_CLASS:
      case T_MODULE:
      case T_FLOAT:
      case T_STRING:
      case T_REGEXP:
      case T_ARRAY:
      case T_HASH:
      case T_STRUCT:
      case T_BIGNUM:
      case T_FILE:
      case T_DATA:
      case T_MATCH:
      case T_COMPLEX:
      case T_RATIONAL:
      case T_TRUE:
      case T_FALSE:
      case T_SYMBOL:
      case T_IMEMO:
      case T_ICLASS:
        if (stack->index == stack->limit) {
            push_mark_stack_chunk(stack);
        }
        stack->chunk->data[stack->index++] = obj;
        return;
 
      case T_NONE:
      case T_NIL:
      case T_FIXNUM:
      case T_MOVED:
      case T_ZOMBIE:
      case T_UNDEF:
      case T_MASK:
        rb_bug("push_mark_stack() called for broken object");
        break;
 
      case T_NODE:
        rb_bug("push_mark_stack: unexpected T_NODE object");
        break;
    }
 
    rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s",
            BUILTIN_TYPE(obj), (void *)obj,
            is_pointer_to_heap((rb_objspace_t *)rb_gc_get_objspace(), (void *)obj) ? "corrupted object" : "non object");
}
 
static int
pop_mark_stack(mark_stack_t *stack, VALUE *data)
{
    if (is_mark_stack_empty(stack)) {
        return FALSE;
    }
    if (stack->index == 1) {
        *data = stack->chunk->data[--stack->index];
        pop_mark_stack_chunk(stack);
    }
    else {
        *data = stack->chunk->data[--stack->index];
    }
    return TRUE;
}
 
static void
init_mark_stack(mark_stack_t *stack)
{
    int i;
 
    MEMZERO(stack, mark_stack_t, 1);
    stack->index = stack->limit = STACK_CHUNK_SIZE;
 
    for (i=0; i < 4; i++) {
        add_stack_chunk_cache(stack, stack_chunk_alloc());
    }
    stack->unused_cache_size = stack->cache_size;
}
 
/* Marking */
 
ALWAYS_INLINE(static int gc_mark_set(rb_objspace_t *objspace, VALUE obj));
ALWAYS_INLINE(static void gc_mark_check_t_none(rb_objspace_t *objspace, VALUE obj));
ALWAYS_INLINE(static void rgengc_check_relation(rb_objspace_t *objspace, VALUE obj));
ALWAYS_INLINE(static void gc_aging(rb_objspace_t *objspace, VALUE obj));
ALWAYS_INLINE(static void gc_grey(rb_objspace_t *objspace, VALUE obj));
static void
rgengc_check_relation(rb_objspace_t *objspace, VALUE obj)
{
    if (objspace->rgengc.parent_object_old_p) {
        if (!RVALUE_OLD_P(objspace, obj)) {
            rgengc_remember(objspace, objspace->rgengc.parent_object);
            /* It is in the rememberset now, so its remaining children have nothing left
             * to ask for: stop testing them. */
            objspace->rgengc.parent_object_old_p = false;
        }
    }
}
 
static inline int
gc_mark_set(rb_objspace_t *objspace, VALUE obj)
{
    if (RVALUE_MARKED(objspace, obj)) return 0;
    MARK_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj);
    return 1;
}
 
static void
gc_aging(rb_objspace_t *objspace, VALUE obj)
{
    /* Disable aging if Major GC's are disabled. This will prevent longish lived
     * objects filling up the heap at the expense of marking many more objects.
     *
     * We should always pre-warm our process when disabling majors, by running
     * GC manually several times so that most objects likely to become oldgen
     * are already oldgen.
     */
    if(!gc_config_full_mark_val)
        return;
 
    struct heap_page *page = GET_HEAP_PAGE(obj);
 
    GC_ASSERT(RVALUE_MARKING(objspace, obj) == FALSE);
    check_rvalue_consistency(objspace, obj);
 
    if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) {
        if (!RVALUE_OLD_P(objspace, obj)) {
            int t = BUILTIN_TYPE(obj);
            if (t == T_CLASS || t == T_MODULE || t == T_ICLASS) {
                gc_report(3, objspace, "gc_aging: YOUNG class: %s\n", rb_obj_info(obj));
                RVALUE_AGE_SET(obj, RVALUE_OLD_AGE);
                RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
            }
            else {
                gc_report(3, objspace, "gc_aging: YOUNG: %s\n", rb_obj_info(obj));
                RVALUE_AGE_INC(objspace, obj);
            }
        }
        else if (is_full_marking(objspace)) {
            GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE);
            RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, page, obj);
        }
    }
    check_rvalue_consistency(objspace, obj);
 
    page->objspace->marked_slots++;
}
 
static void
gc_grey(rb_objspace_t *objspace, VALUE obj)
{
#if RGENGC_CHECK_MODE
    if (RVALUE_MARKED(objspace, obj) == FALSE) rb_bug("gc_grey: %s is not marked.", rb_obj_info(obj));
    if (RVALUE_MARKING(objspace, obj) == TRUE) rb_bug("gc_grey: %s is marking/remembered.", rb_obj_info(obj));
#endif
 
    if (is_incremental_marking(objspace)) {
        MARK_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
    }
 
    if (RB_FL_TEST_RAW(obj, RUBY_FL_WEAK_REFERENCE)) {
        rb_darray_append_without_gc(&objspace->weak_references, obj);
    }
 
    push_mark_stack(&objspace->mark_stack, obj);
}
 
static inline void
gc_mark_check_t_none(rb_objspace_t *objspace, VALUE obj)
{
    if (RB_UNLIKELY(BUILTIN_TYPE(obj) == T_NONE)) {
        enum {info_size = 256};
        char obj_info_buf[info_size];
        rb_raw_obj_info(obj_info_buf, info_size, obj);
 
        char parent_obj_info_buf[info_size];
        rb_raw_obj_info(parent_obj_info_buf, info_size, objspace->rgengc.parent_object);
 
        rb_bug("try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf);
    }
}
 
static void
gc_mark(rb_objspace_t *objspace, VALUE obj)
{
    GC_ASSERT(during_gc);
    GC_ASSERT(!objspace->flags.during_reference_updating);
 
    /* Never step into another objspace: a foreign object is a live leaf whose liveness
     * belongs to its owner, so touching its bitmaps here would be unsound.  A global GC
     * lifts this: everyone is stopped and the bits live on the object's own page. */
    if (gc_skip_foreign_object_p(objspace, obj)) {
        return;
    }
 
    if (RB_UNLIKELY(objspace->flags.during_global_gc)) {
        /* Recompute the shref of every shareable -> unshareable edge, within and across
         * objspaces: the clear pass dropped all shref bits and the write barrier
         * maintains them from here on. */
        VALUE parent = objspace->rgengc.parent_object;
        if (!UNDEF_P(parent) && parent != Qfalse &&
            RB_FL_TEST_RAW(parent, RUBY_FL_SHAREABLE) &&
            !RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE)) {
            struct heap_page *page = GET_HEAP_PAGE(obj);
            _MARK_IN_BITMAP(page->shref_bits, page, obj);
            page->flags.has_shref_objects = TRUE;
        }
    }
 
    rgengc_check_relation(objspace, obj);
    if (!gc_mark_set(objspace, obj)) return; /* already marked */
 
    if (0) { // for debug GC marking miss
        RUBY_DEBUG_LOG("%p (%s) parent:%p (%s)",
                        (void *)obj, obj_type_name(obj),
                        (void *)objspace->rgengc.parent_object, obj_type_name(objspace->rgengc.parent_object));
    }
 
    gc_mark_check_t_none(objspace, obj);
 
    gc_aging(objspace, obj);
    gc_grey(objspace, obj);
}
 
static inline void
gc_pin(rb_objspace_t *objspace, VALUE obj)
{
    GC_ASSERT(!SPECIAL_CONST_P(obj));
 
    if (RB_UNLIKELY(objspace->flags.during_compacting)) {
        /* Never write a foreign page's pinned bit (a global GC may: everyone is stopped). */
        if (gc_skip_foreign_object_p(objspace, obj)) return;
 
        if (RB_LIKELY(during_gc)) {
            if (!RVALUE_PINNED(objspace, obj)) {
                GC_ASSERT(GET_HEAP_PAGE(obj)->pinned_slots <= GET_HEAP_PAGE(obj)->total_slots);
                GET_HEAP_PAGE(obj)->pinned_slots++;
                MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj);
            }
        }
    }
}
 
static inline void
gc_mark_and_pin(rb_objspace_t *objspace, VALUE obj)
{
    gc_pin(objspace, obj);
    gc_mark(objspace, obj);
}
 
void
rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    if (RB_UNLIKELY(objspace->flags.during_reference_updating)) {
        GC_ASSERT(objspace->flags.during_compacting);
        GC_ASSERT(during_gc);
 
        VALUE destination = rb_gc_impl_location(objspace, *ptr);
        if (destination != *ptr) {
            *ptr = destination;
        }
    }
    else {
        gc_mark(objspace, *ptr);
    }
}
 
void
rb_gc_impl_mark(void *objspace_ptr, VALUE obj)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    gc_mark(objspace, obj);
}
 
void
rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    gc_mark_and_pin(objspace, obj);
}
 
/* A word scanned conservatively by a global GC can point into any objspace, so ownership
 * is decided against the driver's snapshot of every objspace (the bits then land on the
 * owner's page through gc_mark and gc_pin). */
static bool
gc_global_pointer_to_heap_p(const void *ptr)
{
    const rb_global_objspace_t *g = global_objspace;
    uintptr_t p = (uintptr_t)ptr;
 
    if (p < g->page_index.lomem || p > g->page_index.himem) return false;
    if (p % sizeof(VALUE) != 0) return false;
 
    struct heap_page **res = bsearch(ptr, g->page_index.pages, g->page_index.n_pages,
                                     sizeof(struct heap_page *), ptr_in_page_body_p);
    if (res == NULL) return false;
 
    struct heap_page *page = *res;
    if (heap_page_in_global_empty_pages_pool(page->objspace, page)) return false;
    if (p < page->start) return false;
    if (p >= page->start + (page->total_slots * page->slot_size)) return false;
    if ((p - page->start) % page->slot_size != 0) return false;
    return true;
}
 
void
rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    (void)VALGRIND_MAKE_MEM_DEFINED(&obj, sizeof(obj));
 
    if (RB_UNLIKELY(objspace->flags.during_global_gc)
            ? gc_global_pointer_to_heap_p((void *)obj)
            : is_pointer_to_heap(objspace, (void *)obj)) {
        asan_unpoisoning_object(obj) {
            /* Garbage can live on the stack, so do not mark or pin */
            switch (BUILTIN_TYPE(obj)) {
              case T_ZOMBIE:
              case T_NONE:
                break;
              default:
                gc_mark_and_pin(objspace, obj);
                break;
            }
        }
    }
}
 
static int
pin_value(st_data_t key, st_data_t value, st_data_t data)
{
    rb_gc_impl_mark_and_pin((void *)data, (VALUE)value);
 
    return ST_CONTINUE;
}
 
static inline void
gc_mark_set_parent_raw(rb_objspace_t *objspace, VALUE obj, bool old_p)
{
    asan_unpoison_memory_region(&objspace->rgengc.parent_object, sizeof(objspace->rgengc.parent_object), false);
    asan_unpoison_memory_region(&objspace->rgengc.parent_object_old_p, sizeof(objspace->rgengc.parent_object_old_p), false);
    objspace->rgengc.parent_object = obj;
    objspace->rgengc.parent_object_old_p = old_p;
}
 
static inline void
gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj)
{
    gc_mark_set_parent_raw(objspace, obj, RVALUE_OLD_P(objspace, obj));
}
 
static inline void
gc_mark_set_parent_invalid(rb_objspace_t *objspace)
{
    asan_poison_memory_region(&objspace->rgengc.parent_object, sizeof(objspace->rgengc.parent_object));
    asan_poison_memory_region(&objspace->rgengc.parent_object_old_p, sizeof(objspace->rgengc.parent_object_old_p));
}
 
static void pinned_roots_mark(rb_objspace_t *objspace, rb_heap_t *heap);
 
static void
mark_roots(rb_objspace_t *objspace, const char **categoryp)
{
    VALUE objspace_guard = (VALUE)objspace;
#define MARK_CHECKPOINT(category) do { \
    if (categoryp) *categoryp = category; \
} while (0)
 
    /* Pinning shareable objects and shrefs runs at the end of marking (gc_marks_finish),
     * not here: after the full walk it only has to touch what ordinary marking missed,
     * which is both cheap and a useful retention metric. */
 
    MARK_CHECKPOINT("objspace");
    gc_mark_set_parent_raw(objspace, Qundef, false);
 
    if (objspace->flags.during_global_gc) {
        /* Pin the finalizer tables of every objspace, zombies included.
         * (finalizer_table is a macro over the local "objspace".) */
        rb_objspace_t *const driver = objspace;
        for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
            rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
            if (finalizer_table != NULL) {
                st_foreach(finalizer_table, pin_value, (st_data_t)driver);
            }
        }
    }
    else if (finalizer_table != NULL) {
        st_foreach(finalizer_table, pin_value, (st_data_t)objspace);
    }
 
    if (stress_to_class) rb_gc_mark(stress_to_class);
 
    rb_gc_save_machine_context();
    rb_gc_mark_roots(objspace, categoryp);
    /* Keep this frame, including its saved registers, until root marking has
     * scanned the machine stack. */
    RB_GC_GUARD(objspace_guard);
    gc_mark_set_parent_invalid(objspace);
}
 
static void
gc_mark_children(rb_objspace_t *objspace, VALUE obj)
{
    gc_mark_set_parent(objspace, obj);
    rb_gc_mark_children(objspace, obj);
    gc_mark_set_parent_invalid(objspace);
}
 
static inline int
gc_mark_stacked_objects(rb_objspace_t *objspace, int incremental, size_t count)
{
    mark_stack_t *mstack = &objspace->mark_stack;
    VALUE obj;
    size_t marked_slots_at_the_beginning = objspace->marked_slots;
    size_t popped_count = 0;
 
    while (pop_mark_stack(mstack, &obj)) {
        if (obj == Qundef) continue; /* skip */
 
        if (RGENGC_CHECK_MODE && !RVALUE_MARKED(objspace, obj)) {
            rb_bug("gc_mark_stacked_objects: %s is not marked.", rb_obj_info(obj));
        }
        gc_mark_children(objspace, obj);
 
        popped_count++;
 
        if (incremental) {
            if (RGENGC_CHECK_MODE && !RVALUE_MARKING(objspace, obj)) {
                rb_bug("gc_mark_stacked_objects: incremental, but marking bit is 0");
            }
            CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
 
            if (popped_count + (objspace->marked_slots - marked_slots_at_the_beginning) > count) {
                break;
            }
        }
        else {
            /* just ignore marking bits */
        }
    }
 
    RUBY_DTRACE_GC_HOOK(MARK_STACKED_OBJECTS, popped_count);
 
    if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace);
 
    if (is_mark_stack_empty(mstack)) {
        shrink_stack_chunk_cache(mstack);
        return TRUE;
    }
    else {
        return FALSE;
    }
}
 
static int
gc_mark_stacked_objects_incremental(rb_objspace_t *objspace, size_t count)
{
    return gc_mark_stacked_objects(objspace, TRUE, count);
}
 
static int
gc_mark_stacked_objects_all(rb_objspace_t *objspace)
{
    return gc_mark_stacked_objects(objspace, FALSE, 0);
}
 
#if RGENGC_CHECK_MODE >= 4
 
#define MAKE_ROOTSIG(obj) (((VALUE)(obj) << 1) | 0x01)
#define IS_ROOTSIG(obj)   ((VALUE)(obj) & 0x01)
#define GET_ROOTSIG(obj)  ((const char *)((VALUE)(obj) >> 1))
 
struct reflist {
    VALUE *list;
    int pos;
    int size;
};
 
static struct reflist *
reflist_create(VALUE obj)
{
    struct reflist *refs = xmalloc(sizeof(struct reflist));
    refs->size = 1;
    refs->list = ALLOC_N(VALUE, refs->size);
    refs->list[0] = obj;
    refs->pos = 1;
    return refs;
}
 
static void
reflist_destruct(struct reflist *refs)
{
    xfree(refs->list);
    xfree(refs);
}
 
static void
reflist_add(struct reflist *refs, VALUE obj)
{
    if (refs->pos == refs->size) {
        refs->size *= 2;
        SIZED_REALLOC_N(refs->list, VALUE, refs->size, refs->size/2);
    }
 
    refs->list[refs->pos++] = obj;
}
 
static void
reflist_dump(struct reflist *refs)
{
    int i;
    for (i=0; i<refs->pos; i++) {
        VALUE obj = refs->list[i];
        if (IS_ROOTSIG(obj)) { /* root */
            fprintf(stderr, "<root@%s>", GET_ROOTSIG(obj));
        }
        else {
            fprintf(stderr, "<%s>", rb_obj_info(obj));
        }
        if (i+1 < refs->pos) fprintf(stderr, ", ");
    }
}
 
static int
reflist_referred_from_machine_context(struct reflist *refs)
{
    int i;
    for (i=0; i<refs->pos; i++) {
        VALUE obj = refs->list[i];
        if (IS_ROOTSIG(obj) && strcmp(GET_ROOTSIG(obj), "machine_context") == 0) return 1;
    }
    return 0;
}
 
struct allrefs {
    rb_objspace_t *objspace;
    /* a -> obj1
     * b -> obj1
     * c -> obj1
     * c -> obj2
     * d -> obj3
     * #=> {obj1 => [a, b, c], obj2 => [c, d]}
     */
    struct st_table *references;
    const char *category;
    VALUE root_obj;
    mark_stack_t mark_stack;
};
 
static int
allrefs_add(struct allrefs *data, VALUE obj)
{
    struct reflist *refs;
    st_data_t r;
 
    if (st_lookup(data->references, obj, &r)) {
        refs = (struct reflist *)r;
        reflist_add(refs, data->root_obj);
        return 0;
    }
    else {
        refs = reflist_create(data->root_obj);
        st_insert(data->references, obj, (st_data_t)refs);
        return 1;
    }
}
 
static void
allrefs_i(VALUE obj, void *ptr)
{
    struct allrefs *data = (struct allrefs *)ptr;
 
    if (allrefs_add(data, obj)) {
        push_mark_stack(&data->mark_stack, obj);
    }
}
 
static void
allrefs_roots_i(VALUE obj, void *ptr)
{
    struct allrefs *data = (struct allrefs *)ptr;
    if (strlen(data->category) == 0) rb_bug("!!!");
    data->root_obj = MAKE_ROOTSIG(data->category);
 
    if (allrefs_add(data, obj)) {
        push_mark_stack(&data->mark_stack, obj);
    }
}
#define PUSH_MARK_FUNC_DATA(v) do { \
    struct gc_mark_func_data_struct *prev_mark_func_data = GET_VM()->gc.mark_func_data; \
    GET_VM()->gc.mark_func_data = (v);
 
#define POP_MARK_FUNC_DATA() GET_VM()->gc.mark_func_data = prev_mark_func_data;} while (0)
 
static st_table *
objspace_allrefs(rb_objspace_t *objspace)
{
    struct allrefs data;
    struct gc_mark_func_data_struct mfd;
    VALUE obj;
    int prev_dont_gc = dont_gc_val();
    dont_gc_on();
 
    data.objspace = objspace;
    data.references = st_init_numtable();
    init_mark_stack(&data.mark_stack);
 
    mfd.mark_func = allrefs_roots_i;
    mfd.data = &data;
 
    /* traverse root objects */
    PUSH_MARK_FUNC_DATA(&mfd);
    GET_VM()->gc.mark_func_data = &mfd;
    mark_roots(objspace, &data.category);
    POP_MARK_FUNC_DATA();
 
    /* traverse rest objects reachable from root objects */
    while (pop_mark_stack(&data.mark_stack, &obj)) {
        rb_objspace_reachable_objects_from_unlocked(data.root_obj = obj, allrefs_i, &data);
    }
    free_stack_chunks(&data.mark_stack);
 
    dont_gc_set(prev_dont_gc);
    return data.references;
}
 
static int
objspace_allrefs_destruct_i(st_data_t key, st_data_t value, st_data_t ptr)
{
    struct reflist *refs = (struct reflist *)value;
    reflist_destruct(refs);
    return ST_CONTINUE;
}
 
static void
objspace_allrefs_destruct(struct st_table *refs)
{
    st_foreach(refs, objspace_allrefs_destruct_i, 0);
    st_free_table(refs);
}
 
#if RGENGC_CHECK_MODE >= 5
static int
allrefs_dump_i(st_data_t k, st_data_t v, st_data_t ptr)
{
    VALUE obj = (VALUE)k;
    struct reflist *refs = (struct reflist *)v;
    fprintf(stderr, "[allrefs_dump_i] %s <- ", rb_obj_info(obj));
    reflist_dump(refs);
    fprintf(stderr, "\n");
    return ST_CONTINUE;
}
 
static void
allrefs_dump(rb_objspace_t *objspace)
{
    VALUE size = objspace->rgengc.allrefs_table->num_entries;
    fprintf(stderr, "[all refs] (size: %"PRIuVALUE")\n", size);
    st_foreach(objspace->rgengc.allrefs_table, allrefs_dump_i, 0);
}
#endif
 
static int
gc_check_after_marks_i(st_data_t k, st_data_t v, st_data_t ptr)
{
    VALUE obj = k;
    struct reflist *refs = (struct reflist *)v;
    rb_objspace_t *objspace = (rb_objspace_t *)ptr;
 
    /* object should be marked or oldgen */
    if (!RVALUE_MARKED(objspace, obj)) {
        fprintf(stderr, "gc_check_after_marks_i: %s is not marked and not oldgen.\n", rb_obj_info(obj));
        fprintf(stderr, "gc_check_after_marks_i: %p is referred from ", (void *)obj);
        reflist_dump(refs);
 
        if (reflist_referred_from_machine_context(refs)) {
            fprintf(stderr, " (marked from machine stack).\n");
            /* marked from machine context can be false positive */
        }
        else {
            objspace->rgengc.error_count++;
            fprintf(stderr, "\n");
        }
    }
    return ST_CONTINUE;
}
 
static void
gc_marks_check(rb_objspace_t *objspace, st_foreach_callback_func *checker_func, const char *checker_name)
{
    MALLOC_COUNTERS_LOCK(objspace);
    struct gc_malloc_bytes saved_malloc = {
        .malloc            = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc),
        .free              = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free),
        .malloc_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc_at_last_gc),
        .free_at_last_gc   = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free_at_last_gc),
    };
#if RGENGC_ESTIMATE_OLDMALLOC
    size_t saved_oldmalloc_increase = gc_oldmalloc_increase_load(objspace);
    gc_counter_t saved_freed_in_gc = gc_counter_load_relaxed(&objspace->malloc_counters.freed_in_gc);
#endif
    MALLOC_COUNTERS_UNLOCK(objspace);
    VALUE already_disabled = rb_objspace_gc_disable(objspace);
 
    objspace->rgengc.allrefs_table = objspace_allrefs(objspace);
 
    if (checker_func) {
        st_foreach(objspace->rgengc.allrefs_table, checker_func, (st_data_t)objspace);
    }
 
    if (objspace->rgengc.error_count > 0) {
#if RGENGC_CHECK_MODE >= 5
        allrefs_dump(objspace);
#endif
        if (checker_name) rb_bug("%s: GC has problem.", checker_name);
    }
 
    objspace_allrefs_destruct(objspace->rgengc.allrefs_table);
    objspace->rgengc.allrefs_table = 0;
 
    if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
    MALLOC_COUNTERS_LOCK(objspace);
    gc_counter_store_release(&objspace->malloc_counters.counters.malloc,            saved_malloc.malloc);
    gc_counter_store_release(&objspace->malloc_counters.counters.free,              saved_malloc.free);
    gc_counter_store_release(&objspace->malloc_counters.counters.malloc_at_last_gc, saved_malloc.malloc_at_last_gc);
    gc_counter_store_release(&objspace->malloc_counters.counters.free_at_last_gc,   saved_malloc.free_at_last_gc);
#if RGENGC_ESTIMATE_OLDMALLOC
    RUBY_ATOMIC_SIZE_EXCHANGE(objspace->malloc_counters.oldmalloc_increase, saved_oldmalloc_increase);
    gc_counter_store_release(&objspace->malloc_counters.freed_in_gc, saved_freed_in_gc);
#endif
    MALLOC_COUNTERS_UNLOCK(objspace);
}
#endif /* RGENGC_CHECK_MODE >= 4 */
 
struct verify_internal_consistency_struct {
    rb_objspace_t *objspace;
    /* True only while the world is stopped: a GC.verify holding the VM lock and barrier,
     * or a global GC.  Cross-objspace checks (walking every objspace's pages) are sound
     * only then. */
    bool world_stopped;
    int err_count;
    size_t live_object_count;
    size_t zombie_object_count;
 
    VALUE parent;
    bool parent_shareable;
    size_t old_object_count;
    size_t remembered_shady_count;
};
 
 
static void
check_generation_i(const VALUE child, void *ptr)
{
    struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
    const VALUE parent = data->parent;
 
    if (RGENGC_CHECK_MODE) GC_ASSERT(RVALUE_OLD_P(data->objspace, parent));
 
    /* A cross-objspace edge is kept alive by the shareable/shref mechanism and is not
     * tracked in this objspace's remembered set. */
    if (GET_HEAP_OBJSPACE(child) != data->objspace) return;
 
    /* Once the process goes multi-Ractor, the shareable world is managed by pinning and
     * shrefs rather than by the remembered set: the pinned walk at the end of a mark
     * re-marks every shareable object (and its shref'd children) each local cycle, and a
     * global GC rebuilds the generation state.  So the generational old->young invariant
     * does not hold when either endpoint is shareable: an old constcache, cc_table or
     * interned string pointing at a core class that is young after a global GC is the
     * typical false positive.  That state outlives the return to a single Ractor until
     * the next major (an old shareable singleton class pointing at a young
     * attached_object, say), so the test uses rb_gc_ever_multi_ractor_p(), which stays
     * true forever once multiple Ractors existed.  A program that never goes multi keeps
     * the strict check, and ASAN catches what is left. */
    if (rb_gc_ever_multi_ractor_p() &&
        (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(parent), parent) ||
         MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(child), child))) {
        return;
    }
 
    if (!RVALUE_OLD_P(data->objspace, child)) {
        /* A young shareable child is pinned and kept alive by the local GC (only a
         * global GC collects it), so it survives even when the old parent does not
         * remember it.  It is outside the generational remembered set, so exclude it
         * from the old->young check. */
        if (!RVALUE_REMEMBERED(data->objspace, parent) &&
            !RVALUE_REMEMBERED(data->objspace, child) &&
            !RVALUE_UNCOLLECTIBLE(data->objspace, child) &&
            !RB_FL_TEST_RAW(child, RUBY_FL_SHAREABLE)) {
            fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (O->Y) %s -> %s\n", rb_obj_info(parent), rb_obj_info(child));
            data->err_count++;
        }
    }
}
 
static void
check_color_i(const VALUE child, void *ptr)
{
    struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
    const VALUE parent = data->parent;
 
    /* This cycle never marks a foreign child (gc_skip_foreign_object_p) and the write
     * barrier is a no-op across objspaces, so its colour says nothing here. */
    if (GET_HEAP_OBJSPACE(child) != data->objspace) return;
 
    if (!RVALUE_WB_UNPROTECTED(data->objspace, parent) && RVALUE_WHITE_P(data->objspace, child)) {
        fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (B->W) - %s -> %s\n",
                rb_obj_info(parent), rb_obj_info(child));
        data->err_count++;
    }
}
 
static void
check_children_i(const VALUE child, void *ptr)
{
    struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
 
    /* Fast path: a child in this objspace (99.99% of all edges). */
    if (RB_LIKELY(is_pointer_to_heap(data->objspace, (void *)child))) {
        if (check_rvalue_consistency_force(data->objspace, child, FALSE) != 0) {
            fprintf(stderr, "check_children_i: %s has error (referenced from %s)\n",
                    rb_obj_info(child), rb_obj_info(data->parent));
            data->err_count++;
        }
        return;
    }
 
    /* The remaining cross-objspace check (verify_pointer_in_any_heap_p) walks every
     * objspace's pages, sound only with the world stopped: mid-local-GC other Ractors
     * change page structures concurrently.  The next world-stopped verify re-checks. */
    if (!data->world_stopped) return;
 
    /* A non-heap child reaches this callback only when a stale field was followed by a
     * plain rb_gc_mark (the dmark of a live but unreachable wrapper, say).  Report it and
     * keep going rather than aborting. */
    if (!verify_pointer_in_any_heap_p((void *)child)) {
        /* The graph is in flux mid-merge, so a transient non-heap edge is expected; it
         * is re-checked after the merge. */
        if (global_objspace->during_absorb) return;
        fprintf(stderr, "VERIFY-NOTE: non-heap child %p (from %s)\n",
                (void *)child, rb_obj_info(data->parent));
        return;
    }
 
    if (GET_HEAP_OBJSPACE(child) != data->objspace) {
        /* A legal cross-objspace edge either starts at a shareable object or is recorded
         * in the child's shref bit (an in-flight send or move payload kept alive across
         * its owner's local GC; root_scope_check_i honours the same record).  An
         * unshareable parent holding an unrecorded foreign unshareable child would be
         * invisible to both local GCs.  The exception is a box's top_self, which every
         * thread's th->top_self points at and which is VM-permanent.  Skipped during a
         * global GC: it clears every shref bit, so the shref exemption would not fire,
         * and its unified exact stop-the-world mark makes the invariant itself moot. */
        if (!data->parent_shareable &&
            child != rb_gc_vm_top_self() &&
            !MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(child), child) &&
            !MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(child), child) &&
            !rb_gc_impl_during_global_gc_p(data->objspace) &&
            !global_objspace->during_absorb) {
            fprintf(stderr, "check_children_i: containment violation: "
                    "unshareable %s (objspace %p) -> foreign unshareable %s (objspace %p)\n",
                    rb_obj_info(data->parent), (void *)data->objspace,
                    rb_obj_info(child), (void *)GET_HEAP_OBJSPACE(child));
            data->err_count++;
        }
 
        /* The remaining per-objspace sanity rules belong to the owner. */
        return;
    }
}
 
/* Whether a heap slot currently holds a live object. Returns false for empty
 * (T_NONE), moved (T_MOVED), and zombie (T_ZOMBIE) slots, and for garbage
 * objects about to be swept. */
static bool
gc_slot_live_object_p(rb_objspace_t *objspace, VALUE obj)
{
    switch (BUILTIN_TYPE(obj)) {
      case T_NONE:
      case T_MOVED:
      case T_ZOMBIE:
        return false;
      default:
        return !rb_gc_impl_garbage_object_p(objspace, obj);
    }
}
 
/* Verifier only: does ptr point at a live slot in any objspace?  The caller holds the VM
 * lock and the barrier, so page_index is stable. */
static bool
verify_pointer_in_any_heap_p(const void *ptr)
{
    return gc_global_pointer_to_heap_p(ptr);
}
 
/* An exact root of the calling Ractor may only point at a shareable object, its own
 * objspace, or an in-flight payload with a recorded shref.  Exempt: the conservative
 * machine scan (stale slots) and the VM-global containers that are cross-rooted by
 * design (every objspace scans them; the marker skips foreign entries). */
static void
root_scope_check_i(const char *category, VALUE obj, void *ptr)
{
    struct verify_internal_consistency_struct *data = ptr;
 
    if (RB_SPECIAL_CONST_P(obj)) return;
    /* This check walks every objspace (verify_pointer_in_any_heap_p), so it is sound
     * only with the world stopped; a mid-local-GC verify races with other Ractors'
     * lock-free allocation. */
    if (!data->world_stopped) return;
    /* Mid-merge the VM-global root tables still point at the unmerged source (transient
     * non-heap or foreign roots); re-checked after the merge. */
    if (global_objspace->during_absorb) return;
    if (strcmp(category, "machine_context") == 0 ||
        strcmp(category, "vm_registered_objects") == 0 ||
        strcmp(category, "end_proc") == 0 ||
        strcmp(category, "trap_list") == 0) {
        return;
    }
 
    if (!verify_pointer_in_any_heap_p((void *)obj)) {
        fprintf(stderr, "root_scope_check_i: root category \"%s\" names a non-heap pointer %p\n",
                category, (void *)obj);
        data->err_count++;
        return;
    }
 
    if (GET_HEAP_OBJSPACE(obj) == data->objspace) return;
    if (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj)) return;
    if (MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj)) return;
    if (obj == rb_gc_vm_top_self()) return;  /* VM-permanent (see check_children_i) */
 
    fprintf(stderr, "root_scope_check_i: root category \"%s\" names a foreign "
            "unshareable without a shref record: %s (owner %p, self %p)\n",
            category, rb_obj_info(obj),
            (void *)GET_HEAP_OBJSPACE(obj), (void *)data->objspace);
    data->err_count++;
}
 
static int
verify_internal_consistency_i(void *page_start, void *page_end, size_t stride,
                              struct verify_internal_consistency_struct *data)
{
    VALUE obj;
    rb_objspace_t *objspace = data->objspace;
 
    for (obj = (VALUE)page_start; obj != (VALUE)page_end; obj += stride) {
        asan_unpoisoning_object(obj) {
            bool sh_bit = MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj) != 0;
            bool sr_bit = MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj) != 0;
 
            if (gc_slot_live_object_p(objspace, obj)) {
                /* count objects */
                data->live_object_count++;
                data->parent = obj;
                data->parent_shareable = sh_bit;
 
                /* Bitmap invariants: a page's shareable bit matches FL_SHAREABLE
                 * exactly, and a shref record only ever points at an unshareable
                 * object. */
                if (sh_bit != !!RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE)) {
                    fprintf(stderr, "verify_internal_consistency_i: shareable bit %d "
                            "disagrees with FL_SHAREABLE on %s\n", (int)sh_bit, rb_obj_info(obj));
                    data->err_count++;
                }
                if (sr_bit && sh_bit) {
                    fprintf(stderr, "verify_internal_consistency_i: shref bit on a shareable: %s\n",
                            rb_obj_info(obj));
                    data->err_count++;
                }
 
                /* Normally, we don't expect T_MOVED objects to be in the heap.
                * But they can stay alive on the stack, */
                if (!gc_object_moved_p(objspace, obj)) {
                    /* moved slots don't have children */
                    rb_objspace_reachable_objects_from_unlocked(obj, check_children_i, (void *)data);
                }
 
                /* check health of children */
                if (RVALUE_OLD_P(objspace, obj)) data->old_object_count++;
                if (RVALUE_WB_UNPROTECTED(objspace, obj) && RVALUE_UNCOLLECTIBLE(objspace, obj)) data->remembered_shady_count++;
 
                if (!is_marking(objspace) && RVALUE_OLD_P(objspace, obj)) {
                    /* reachable objects from an oldgen object should be old or (young with remember) */
                    data->parent = obj;
                    rb_objspace_reachable_objects_from_unlocked(obj, check_generation_i, (void *)data);
                }
 
                if (!is_marking(objspace) && rb_gc_obj_shareable_p(obj)) {
                    rb_gc_verify_shareable(obj);
                }
 
                if (is_incremental_marking(objspace)) {
                    if (RVALUE_BLACK_P(objspace, obj)) {
                        /* reachable objects from black objects should be black or grey objects */
                        data->parent = obj;
                        rb_objspace_reachable_objects_from_unlocked(obj, check_color_i, (void *)data);
                    }
                }
            }
            else {
                /* A freed slot must not carry its old pin bit into the next object born
                 * there (a dead object not swept yet legitimately keeps it until the
                 * sweep arrives). */
                if (BUILTIN_TYPE(obj) == T_NONE && (sh_bit || sr_bit)) {
                    fprintf(stderr, "verify_internal_consistency_i: T_NONE slot carries "
                            "shareable=%d shref=%d bits\n", (int)sh_bit, (int)sr_bit);
                    data->err_count++;
                }
 
                if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
                    data->zombie_object_count++;
 
                    if ((RBASIC(obj)->flags & ~ZOMBIE_OBJ_KEPT_FLAGS) != T_ZOMBIE) {
                        fprintf(stderr, "verify_internal_consistency_i: T_ZOMBIE has extra flags set: %s\n",
                                rb_obj_info(obj));
                        data->err_count++;
                    }
 
                    if (!!FL_TEST(obj, FL_FINALIZE) != !!st_is_member(finalizer_table, obj)) {
                        fprintf(stderr, "verify_internal_consistency_i: FL_FINALIZE %s but %s finalizer_table: %s\n",
                                FL_TEST(obj, FL_FINALIZE) ? "set" : "not set", st_is_member(finalizer_table, obj) ? "in" : "not in",
                                rb_obj_info(obj));
                        data->err_count++;
                    }
                }
            }
        }
    }
 
    return 0;
}
 
static int
gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
    unsigned int has_remembered_shady = FALSE;
    unsigned int has_remembered_old = FALSE;
    int remembered_old_objects = 0;
    int free_objects = 0;
    int zombie_objects = 0;
 
    short slot_size = page->slot_size;
    uintptr_t start = (uintptr_t)page->start;
    uintptr_t end = start + page->total_slots * slot_size;
 
    for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
        VALUE val = (VALUE)ptr;
        asan_unpoisoning_object(val) {
            enum ruby_value_type type = BUILTIN_TYPE(val);
 
            if (type == T_NONE) free_objects++;
            if (type == T_ZOMBIE) zombie_objects++;
            if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) {
                has_remembered_shady = TRUE;
            }
            if (RVALUE_PAGE_MARKING(page, val)) {
                has_remembered_old = TRUE;
                remembered_old_objects++;
            }
        }
    }
 
    if (!is_incremental_marking(objspace) &&
        page->flags.has_remembered_objects == FALSE && has_remembered_old == TRUE) {
 
        for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
            VALUE val = (VALUE)ptr;
            if (RVALUE_PAGE_MARKING(page, val)) {
                fprintf(stderr, "marking -> %s\n", rb_obj_info(val));
            }
        }
        rb_bug("page %p's has_remembered_objects should be false, but there are remembered old objects (%d). %s",
               (void *)page, remembered_old_objects, obj ? rb_obj_info(obj) : "");
    }
 
    if (page->flags.has_uncollectible_wb_unprotected_objects == FALSE && has_remembered_shady == TRUE) {
        rb_bug("page %p's has_remembered_shady should be false, but there are remembered shady objects. %s",
               (void *)page, obj ? rb_obj_info(obj) : "");
    }
 
    if (0) {
        /* free_slots may not equal to free_objects */
        if (page->free_slots != free_objects) {
            rb_bug("page %p's free_slots should be %d, but %d", (void *)page, page->free_slots, free_objects);
        }
    }
    if (page->final_slots != zombie_objects) {
        rb_bug("page %p's final_slots should be %d, but %d", (void *)page, page->final_slots, zombie_objects);
    }
 
    return remembered_old_objects;
}
 
static int
gc_verify_heap_pages_(rb_objspace_t *objspace, struct ccan_list_head *head)
{
    int remembered_old_objects = 0;
    struct heap_page *page = 0;
 
    ccan_list_for_each(head, page, page_node) {
        asan_unlock_freelist(page);
        struct free_region *region = page->free_region;
        while (region) {
            VALUE vp = (VALUE)region;
            rb_asan_unpoison_object(vp, false);
            if (BUILTIN_TYPE(vp) != T_NONE) {
                fprintf(stderr, "free region head expected to be T_NONE but was: %s\n", rb_obj_info(vp));
            }
            struct free_region *next = region->next;
            rb_asan_poison_object(vp);
            region = next;
        }
        asan_lock_freelist(page);
 
        if (page->flags.has_remembered_objects == FALSE) {
            remembered_old_objects += gc_verify_heap_page(objspace, page, Qfalse);
        }
    }
 
    return remembered_old_objects;
}
 
static int
gc_verify_heap_pages(rb_objspace_t *objspace)
{
    int remembered_old_objects = 0;
    for (int i = 0; i < HEAP_COUNT; i++) {
        remembered_old_objects += gc_verify_heap_pages_(objspace, &((&heaps[i])->pages));
    }
    return remembered_old_objects;
}
 
static void
verify_registered_addr(VALUE *slot, VALUE initial_value, void *owner_objspace, void *d)
{
    struct verify_internal_consistency_struct *data = d;
    VALUE v = *slot;
 
    /* Conservative registration permits uninitialized data and pre-registration
     * values; only a store made after registration is a violation. */
    if (v == initial_value) return;
    if (SPECIAL_CONST_P(v)) return;
    if (!verify_pointer_in_any_heap_p((void *)v)) return;
 
    bool live = false;
    asan_unpoisoning_object(v) {
        live = BUILTIN_TYPE(v) != T_NONE && BUILTIN_TYPE(v) != T_ZOMBIE;
    }
    if (!live) return;
 
    rb_objspace_t *value_objspace = GET_HEAP_OBJSPACE(v);
    if (value_objspace == (rb_objspace_t *)owner_objspace) return;
    /* Join and orphan handling move a registration to the inheritor before the
     * source objspace merge; a global GC scans every registry while the zombie
     * exists, so this is a safe transient exemption. */
    if (value_objspace->zombie_pages != 0) return;
    if (value_objspace->flags.during_postmortem) return;
    if (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(v), v)) return;
    if (MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(v), v)) return;
    /* When multiple Ractors register one address, ownership by any registrant is
     * enough to root the value. */
    if (rb_gc_registered_addr_owned_by_registrant_p(slot, value_objspace)) return;
 
    fprintf(stderr, "registered address %p changed since registration to an unshareable object owned by another Ractor: %s\n",
            (void *)slot, rb_obj_info(v));
    data->err_count++;
}
 
static void
gc_verify_internal_consistency_(rb_objspace_t *objspace, bool world_stopped)
{
    struct verify_internal_consistency_struct data = {0};
 
    data.objspace = objspace;
    data.world_stopped = world_stopped;
    gc_report(5, objspace, "gc_verify_internal_consistency: start\n");
 
    /* check relations */
    for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
        struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
        short slot_size = page->slot_size;
 
        uintptr_t start = (uintptr_t)page->start;
        uintptr_t end = start + page->total_slots * slot_size;
 
        verify_internal_consistency_i((void *)start, (void *)end, slot_size, &data);
    }
 
    /* Check the calling Ractor's root scoping (only when verifying the current
     * objspace).  Skipped during a global GC, which deliberately spans every Ractor's
     * roots and legitimately reaches foreign objects: containment does not apply. */
    if (!rb_gc_single_objspace_p() && objspace == rb_gc_get_objspace() &&
        !rb_gc_impl_during_global_gc_p(objspace)) {
        rb_objspace_reachable_objects_from_root(root_scope_check_i, &data);
    }
 
    if (data.world_stopped && !global_objspace->during_absorb) {
        rb_gc_each_registered_addr(verify_registered_addr, &data);
    }
 
    if (data.err_count != 0) {
#if RGENGC_CHECK_MODE >= 5
        objspace->rgengc.error_count = data.err_count;
        gc_marks_check(objspace, NULL, NULL);
        allrefs_dump(objspace);
#endif
        rb_bug("gc_verify_internal_consistency: found internal inconsistency.");
    }
 
    /* check heap_page status */
    gc_verify_heap_pages(objspace);
 
    /* check counters */
 
    if (!is_lazy_sweeping(objspace) &&
            !finalizing &&
            !rb_gc_multi_ractor_p()) {
        if (objspace_live_slots(objspace) != data.live_object_count) {
            fprintf(stderr, "heap_pages_final_slots: %"PRIdSIZE", total_freed_objects: %"PRIdSIZE"\n",
                    total_final_slots_count(objspace), total_freed_objects(objspace));
            rb_bug("inconsistent live slot number: expect %"PRIuSIZE", but %"PRIuSIZE".",
                   objspace_live_slots(objspace), data.live_object_count);
        }
    }
 
    if (!is_marking(objspace)) {
        if (objspace->rgengc.old_objects != data.old_object_count) {
            rb_bug("inconsistent old slot number: expect %"PRIuSIZE", but %"PRIuSIZE".",
                   objspace->rgengc.old_objects, data.old_object_count);
        }
        if (objspace->rgengc.uncollectible_wb_unprotected_objects != data.remembered_shady_count) {
            rb_bug("inconsistent number of wb unprotected objects: expect %"PRIuSIZE", but %"PRIuSIZE".",
                   objspace->rgengc.uncollectible_wb_unprotected_objects, data.remembered_shady_count);
        }
    }
 
    if (!finalizing) {
        size_t list_count = 0;
 
        {
            VALUE z = heap_pages_deferred_final;
            while (z) {
                list_count++;
                z = RZOMBIE(z)->next;
            }
        }
 
        if (total_final_slots_count(objspace) != data.zombie_object_count ||
            total_final_slots_count(objspace) != list_count) {
 
            rb_bug("inconsistent finalizing object count:\n"
                    "  expect %"PRIuSIZE"\n"
                    "  but    %"PRIuSIZE" zombies\n"
                    "  heap_pages_deferred_final list has %"PRIuSIZE" items.",
                    total_final_slots_count(objspace),
                    data.zombie_object_count,
                    list_count);
        }
    }
 
    gc_report(5, objspace, "gc_verify_internal_consistency: OK\n");
}
 
/* The `during_gc` macro expands a bare identifier to `objspace->flags.during_gc`, so a
 * foreign objspace's flag cannot be written directly; these helpers reach it through the
 * `objspace` argument. */
static inline unsigned int
gc_during_gc_get(const rb_objspace_t *objspace)
{
    return during_gc;
}
 
static inline void
gc_during_gc_set(rb_objspace_t *objspace, unsigned int v)
{
    during_gc = v;
}
 
/* Run the check with during_gc cleared in both the verified objspace and the current
 * Ractor's: rb_objspace_reachable_objects_from() decides on rb_gc_get_objspace(), and
 * under a global GC the driver verifies foreign objspaces, so the driver's during_gc
 * needs clearing too (a no-op when cur == objspace). */
static void
gc_verify_internal_consistency_body(rb_objspace_t *objspace, bool world_stopped)
{
    const unsigned int prev_during_gc = during_gc;
    during_gc = FALSE; // stop gc here
 
    rb_objspace_t *const cur = rb_gc_get_objspace();
    const unsigned int prev_cur_during_gc = (cur != objspace) ? gc_during_gc_get(cur) : 0;
    if (cur != objspace) gc_during_gc_set(cur, FALSE);
    {
        gc_verify_internal_consistency_(objspace, world_stopped);
    }
    if (cur != objspace) gc_during_gc_set(cur, prev_cur_during_gc);
    during_gc = prev_during_gc;
}
 
static void
gc_verify_internal_consistency(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    /* Called mid-GC, take neither the VM lock nor the barrier: waiting would join a
     * pending global barrier mid-collection (a GC must never take the VM lock) and let
     * the global GC sweep the heap this mark is walking.  The barrier is unnecessary
     * anyway; the objspace is single-writer, this verify runs on its owner thread, and
     * the global driver that sets during_gc everywhere already holds both. */
    if (during_gc) {
        /* The world is stopped only when the global GC's driver runs this while holding
         * the barrier; a non-main Ractor's local GC does not stop other Ractors. */
        gc_verify_internal_consistency_body(objspace, rb_gc_impl_during_global_gc_p(objspace));
        return;
    }
 
    unsigned int lev = RB_GC_VM_LOCK();
    {
        rb_gc_vm_barrier(); // stop other ractors
        gc_verify_internal_consistency_body(objspace, true); // holding the barrier, so walking every objspace is sound
    }
    RB_GC_VM_UNLOCK(lev);
}
 
static void
heap_move_pooled_pages_to_free_pages(rb_heap_t *heap)
{
    if (heap->pooled_pages) {
        if (heap->free_pages) {
            struct heap_page *free_pages_tail = heap->free_pages;
            while (free_pages_tail->free_next) {
                free_pages_tail = free_pages_tail->free_next;
            }
            free_pages_tail->free_next = heap->pooled_pages;
        }
        else {
            heap->free_pages = heap->pooled_pages;
        }
 
        heap->pooled_pages = NULL;
    }
}
 
static int
gc_remember_unprotected(rb_objspace_t *objspace, VALUE obj)
{
    struct heap_page *page = GET_HEAP_PAGE(obj);
    bits_t *uncollectible_bits = &page->uncollectible_bits[0];
 
    if (!MARKED_IN_BITMAP(uncollectible_bits, obj)) {
        page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
        MARK_IN_BITMAP(uncollectible_bits, obj);
        /* Like RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET, count it in the object's own objspace. */
        page->objspace->rgengc.uncollectible_wb_unprotected_objects++;
 
#if RGENGC_PROFILE > 0
        objspace->profile.total_remembered_shady_object_count++;
#if RGENGC_PROFILE >= 2
        objspace->profile.remembered_shady_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
#endif
        return TRUE;
    }
    else {
        return FALSE;
    }
}
 
static inline void
gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bits, short slot_size)
{
    if (bits) {
        do {
            if (bits & 1) {
                gc_report(2, objspace, "gc_marks_wb_unprotected_objects: marked shady: %s\n", rb_obj_info((VALUE)p));
                GC_ASSERT(RVALUE_WB_UNPROTECTED(objspace, (VALUE)p));
                GC_ASSERT(RVALUE_MARKED(objspace, (VALUE)p));
                gc_mark_children(objspace, (VALUE)p);
            }
            p += slot_size;
            bits >>= 1;
        } while (bits);
    }
}
 
static void
gc_marks_wb_unprotected_objects(rb_objspace_t *objspace, rb_heap_t *heap)
{
    struct heap_page *page = 0;
 
    ccan_list_for_each(&heap->pages, page, page_node) {
        bits_t *mark_bits = page->mark_bits;
        bits_t *wbun_bits = page->wb_unprotected_bits;
        uintptr_t p = page->start;
        short slot_size = page->slot_size;
        int total_slots = page->total_slots;
        int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
        size_t j;
 
        for (j=0; j<(size_t)bitmap_plane_count; j++) {
            bits_t bits = mark_bits[j] & wbun_bits[j];
            gc_marks_wb_unprotected_objects_plane(objspace, p, bits, slot_size);
            p += BITS_BITLENGTH * slot_size;
        }
    }
 
    gc_mark_stacked_objects_all(objspace);
}
 
void
rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj)
{
    FL_SET_RAW(obj, RUBY_FL_WEAK_REFERENCE);
}
 
bool
rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    /* A local GC cannot decide a foreign object's liveness, so treat it as live; its
     * owner or the global GC decides (a global GC's unified mark is exact). */
    if (gc_skip_foreign_object_p(objspace, obj)) return true;
 
    bool marked = RVALUE_MARKED(objspace, obj);
 
    if (marked) {
        rgengc_check_relation(objspace, obj);
    }
 
    return marked;
}
 
static void
gc_update_weak_references(rb_objspace_t *objspace)
{
    VALUE *obj_ptr;
    rb_darray_foreach(objspace->weak_references, i, obj_ptr) {
        gc_mark_set_parent(objspace, *obj_ptr);
        rb_gc_handle_weak_references(*obj_ptr);
        gc_mark_set_parent_invalid(objspace);
    }
 
    size_t capa = rb_darray_capa(objspace->weak_references);
    size_t size = rb_darray_size(objspace->weak_references);
 
    objspace->profile.weak_references_count = size;
 
    rb_darray_clear(objspace->weak_references);
 
    /* If the darray has capacity for more than four times the amount used, we
     * shrink it down to half of that capacity. */
    if (capa > size * 4) {
        rb_darray_resize_capa_without_gc(&objspace->weak_references, size * 2);
    }
}
 
static void
gc_marks_finish(rb_objspace_t *objspace)
{
    /* finish incremental GC */
    if (is_incremental_marking(objspace)) {
        if (RGENGC_CHECK_MODE && is_mark_stack_empty(&objspace->mark_stack) == 0) {
            rb_bug("gc_marks_finish: mark stack is not empty (%"PRIdSIZE").",
                   mark_stack_size(&objspace->mark_stack));
        }
 
        mark_roots(objspace, NULL);
        while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == false);
 
#if RGENGC_CHECK_MODE >= 2
        if (gc_verify_heap_pages(objspace) != 0) {
            rb_bug("gc_marks_finish (incremental): there are remembered old objects.");
        }
#endif
 
        gc_set_incremental_marking(objspace, FALSE);
        /* check children of all marked wb-unprotected objects */
        for (int i = 0; i < HEAP_COUNT; i++) {
            gc_marks_wb_unprotected_objects(objspace, &heaps[i]);
        }
    }
 
    /* Pin the shareable objects and shrefs ordinary marking missed: a local GC must free
     * neither (another objspace may hold them).  Running after the full walk makes the
     * pin count a retention metric: an upper bound on the garbage only a global GC can
     * reclaim.  A global GC's exact mark does not pin.  (The allrefs comparison of
     * RGENGC_CHECK_MODE >= 4 does not model these pins; it reports false positives.)
     *
     * Running it here rather than with the other roots is also what lets an incremental
     * mark run while other objspaces exist.  The write barrier bails out on a
     * cross-objspace edge, so a store made between two mark steps leaves nothing behind
     * but a shareable or shref bit; scanning those bitmaps after the last step picks up
     * every bit set during the cycle, which a scan at gc_marks_start would miss. */
    objspace->last_cycle_pinned = 0;
    if (!rb_gc_single_objspace_p() && !objspace->flags.during_global_gc) {
        objspace->last_cycle_pinned = 1;
        gc_mark_set_parent_raw(objspace, Qundef, false);
        for (int i = 0; i < HEAP_COUNT; i++) {
            pinned_roots_mark(objspace, &heaps[i]);
        }
        /* And everything they keep alive. */
        gc_mark_stacked_objects_all(objspace);
    }
 
    gc_update_weak_references(objspace);
 
#if RGENGC_CHECK_MODE >= 4
    during_gc = FALSE;
    gc_marks_check(objspace, gc_check_after_marks_i, "after_marks");
    during_gc = TRUE;
#endif
 
    {
        /* Only this objspace's own Ractor allocates from it.  The main objspace
         * keeps the VM-wide count it has used since before per-Ractor GC. */
        const unsigned long ractor_cnt = objspace == global_objspace->main_objspace
            ? rb_gc_vm_ractor_count() : 1;
        const unsigned long r_mul = ractor_cnt > 8 ? 8 : ractor_cnt; // upto 8
 
        size_t total_slots = objspace_available_slots(objspace);
        size_t sweep_slots = total_slots - objspace->marked_slots; /* will be swept slots */
        size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio);
        size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio);
        if (min_free_slots < gc_params.heap_free_slots * r_mul) {
            min_free_slots = gc_params.heap_free_slots * r_mul;
        }
 
        int full_marking = is_full_marking(objspace);
 
        GC_ASSERT(objspace_available_slots(objspace) >= objspace->marked_slots);
 
        /* Setup freeable slots. */
        size_t total_init_slots = 0;
        for (int i = 0; i < HEAP_COUNT; i++) {
            total_init_slots += (objspace_heap_init_bytes(objspace) / heaps[i].slot_size) * r_mul;
        }
 
        if (max_free_slots < total_init_slots) {
            max_free_slots = total_init_slots;
        }
 
        /* Approximate freeable pages using the average slots-per-pages across all heaps */
        if (sweep_slots > max_free_slots) {
            size_t excess_slots = sweep_slots - max_free_slots;
            size_t total_heap_pages = heap_eden_total_pages(objspace);
            heap_pages_freeable_pages = total_heap_pages > 0
                ? excess_slots * total_heap_pages / total_slots
                : 0;
        }
        else {
            heap_pages_freeable_pages = 0;
        }
 
        if (objspace->heap_pages.allocatable_bytes == 0 && sweep_slots < min_free_slots) {
            if (!full_marking && sweep_slots < min_free_slots * 7 / 8) {
                if (objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
                    full_marking = TRUE;
                }
                else {
                    gc_report(1, objspace, "gc_marks_finish: next is full GC!!)\n");
                    gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
                }
            }
 
            if (full_marking) {
                heap_allocatable_bytes_expand(objspace, NULL, sweep_slots, total_slots, heaps[0].slot_size);
            }
        }
 
        if (full_marking) {
            /* See the comment about RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR */
            const double r = gc_params.oldobject_limit_factor;
            objspace->rgengc.uncollectible_wb_unprotected_objects_limit = MAX(
                (size_t)(objspace->rgengc.uncollectible_wb_unprotected_objects * r),
                (size_t)(objspace->rgengc.old_objects * gc_params.uncollectible_wb_unprotected_objects_limit_ratio)
            );
            objspace->rgengc.old_objects_limit = (size_t)(objspace->rgengc.old_objects * r);
        }
 
        if (objspace->rgengc.uncollectible_wb_unprotected_objects > objspace->rgengc.uncollectible_wb_unprotected_objects_limit) {
            gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_SHADY;
        }
        if (objspace->rgengc.old_objects > objspace->rgengc.old_objects_limit) {
            gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDGEN;
        }
 
        gc_report(1, objspace, "gc_marks_finish (marks %"PRIdSIZE" objects, "
                  "old %"PRIdSIZE" objects, total %"PRIdSIZE" slots, "
                   "sweep %"PRIdSIZE" slots, allocatable %"PRIdSIZE" bytes, next GC: %s)\n",
                  objspace->marked_slots, objspace->rgengc.old_objects, objspace_available_slots(objspace), sweep_slots, objspace->heap_pages.allocatable_bytes,
                  gc_needs_major_flags ? "major" : "minor");
    }
 
    // TODO: refactor so we don't need to call this
    rb_ractor_finish_marking(is_full_marking(objspace));
 
    gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_MARK);
}
 
static bool
gc_compact_heap_cursors_met_p(rb_heap_t *heap)
{
    return heap->sweeping_page == heap->compact_cursor;
}
 
 
static rb_heap_t *
gc_compact_destination_pool(rb_objspace_t *objspace, rb_heap_t *src_pool, VALUE obj)
{
    size_t obj_size = rb_gc_obj_optimal_size(obj);
    if (obj_size == 0) {
        return src_pool;
    }
 
    GC_ASSERT(rb_gc_impl_size_allocatable_p(obj_size));
 
    size_t idx = heap_idx_for_size(obj_size);
 
    return &heaps[idx];
}
 
static bool
gc_compact_move(rb_objspace_t *objspace, rb_heap_t *heap, VALUE src)
{
    GC_ASSERT(BUILTIN_TYPE(src) != T_MOVED);
    GC_ASSERT(gc_is_moveable_obj(objspace, src));
 
    rb_heap_t *dest_pool = gc_compact_destination_pool(objspace, heap, src);
    if (gc_compact_heap_cursors_met_p(dest_pool)) {
        return dest_pool != heap;
    }
 
    while (!try_move(objspace, dest_pool, dest_pool->free_pages, src)) {
        struct gc_sweep_context ctx = {
            .page = dest_pool->sweeping_page,
            .final_slots = 0,
            .freed_slots = 0,
            .empty_slots = 0,
        };
 
        /* The page of src could be partially compacted, so it may contain
         * T_MOVED. Sweeping a page may read objects on this page, so we
         * need to lock the page. */
        lock_page_body(objspace, GET_PAGE_BODY(src));
        gc_sweep_page(objspace, dest_pool, &ctx);
        unlock_page_body(objspace, GET_PAGE_BODY(src));
 
        if (dest_pool->sweeping_page->free_slots > 0) {
            heap_add_freepage(dest_pool, dest_pool->sweeping_page);
        }
 
        dest_pool->sweeping_page = ccan_list_next(&dest_pool->pages, dest_pool->sweeping_page, page_node);
        if (gc_compact_heap_cursors_met_p(dest_pool)) {
            return dest_pool != heap;
        }
    }
 
    return true;
}
 
static bool
gc_compact_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct heap_page *page)
{
    short slot_size = page->slot_size;
 
    do {
        VALUE vp = (VALUE)p;
        GC_ASSERT(vp % sizeof(VALUE) == 0);
 
        if (bitset & 1) {
            objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++;
 
            if (gc_is_moveable_obj(objspace, vp)) {
                if (!gc_compact_move(objspace, heap, vp)) {
                    //the cursors met. bubble up
                    return false;
                }
            }
        }
        p += slot_size;
        bitset >>= 1;
    } while (bitset);
 
    return true;
}
 
// Iterate up all the objects in page, moving them to where they want to go
static bool
gc_compact_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
    GC_ASSERT(page == heap->compact_cursor);
 
    bits_t *mark_bits, *pin_bits;
    bits_t bitset;
    uintptr_t p = page->start;
    short slot_size = page->slot_size;
    int total_slots = page->total_slots;
    int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
 
    mark_bits = page->mark_bits;
    pin_bits = page->pinned_bits;
 
    for (int j = 0; j < bitmap_plane_count; j++) {
        // objects that can be moved are marked and not pinned
        bitset = (mark_bits[j] & ~pin_bits[j]);
        if (bitset) {
            if (!gc_compact_plane(objspace, heap, (uintptr_t)p, bitset, page))
                return false;
        }
        p += BITS_BITLENGTH * slot_size;
    }
 
    return true;
}
 
static bool
gc_compact_all_compacted_p(rb_objspace_t *objspace)
{
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
 
        if (heap->total_pages > 0 &&
                !gc_compact_heap_cursors_met_p(heap)) {
            return false;
        }
    }
 
    return true;
}
 
/* Compaction's move phase: relocate this objspace's movable objects and leave T_MOVED
 * forwarding behind without updating references yet.  A global GC calls this for every
 * objspace before updating any of them (two phases), so a cross-objspace reference to a
 * moved object is rewritten exactly once, after all forwarding exists. */
static void
gc_compact_relocate(rb_objspace_t *objspace)
{
    gc_compact_start(objspace);
 
    while (!gc_compact_all_compacted_p(objspace)) {
        for (int i = 0; i < HEAP_COUNT; i++) {
            rb_heap_t *heap = &heaps[i];
 
            if (gc_compact_heap_cursors_met_p(heap)) {
                continue;
            }
 
            struct heap_page *start_page = heap->compact_cursor;
 
            if (!gc_compact_page(objspace, heap, start_page)) {
                lock_page_body(objspace, start_page->body);
 
                continue;
            }
 
            // If we get here, we've finished moving all objects on the compact_cursor page
            // So we can lock it and move the cursor on to the next one.
            lock_page_body(objspace, start_page->body);
            heap->compact_cursor = ccan_list_prev(&heap->pages, heap->compact_cursor, page_node);
        }
    }
}
 
static void
gc_sweep_compact(rb_objspace_t *objspace)
{
    gc_compact_relocate(objspace);
    /* A compacting global GC defers the finish (reference update) to the second phase,
     * after every objspace has been relocated. */
    if (!global_objspace->global_gc.compacting) {
        gc_compact_finish(objspace);
    }
}
 
static void
gc_marks_rest(rb_objspace_t *objspace)
{
    gc_report(1, objspace, "gc_marks_rest\n");
 
    for (int i = 0; i < HEAP_COUNT; i++) {
        (&heaps[i])->pooled_pages = NULL;
    }
 
    if (is_incremental_marking(objspace)) {
        while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == FALSE);
    }
    else {
        gc_mark_stacked_objects_all(objspace);
    }
 
    gc_marks_finish(objspace);
}
 
static bool
gc_marks_step(rb_objspace_t *objspace, size_t slots)
{
    bool marking_finished = false;
 
    GC_ASSERT(is_marking(objspace));
    if (gc_mark_stacked_objects_incremental(objspace, slots)) {
        gc_marks_finish(objspace);
 
        marking_finished = true;
    }
 
    return marking_finished;
}
 
static bool
gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap)
{
    GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
    bool marking_finished = true;
 
    gc_marking_enter(objspace);
 
    if (heap->free_pages) {
        gc_report(2, objspace, "gc_marks_continue: has pooled pages");
 
        marking_finished = gc_marks_step(objspace, objspace->rincgc.step_slots);
    }
    else {
        gc_report(2, objspace, "gc_marks_continue: no more pooled pages (stack depth: %"PRIdSIZE").\n",
                  mark_stack_size(&objspace->mark_stack));
        heap->force_incremental_marking_finish_count++;
        gc_marks_rest(objspace);
    }
 
    gc_marking_exit(objspace);
 
    return marking_finished;
}
 
/* Mark the following as roots of this objspace.
 * - Every shareable object: another objspace may hold the only reference, invisible to a
 *   local GC.  Marking them rather than skipping them in the sweep preserves the
 *   generational invariants (a pinned object ages and gets promoted like any live one).
 *   Only a global GC decides that a shareable object is dead.
 * - Every shref (an unshareable object referenced from a shareable one): the referring
 *   shareable object can live in another objspace or in an in-flight message queue.  The
 *   write barrier maintains them.
 * Skipped while the VM has a single Ractor: a local GC is then a whole-world GC and
 * shareable objects may die normally. */
static void
pinned_roots_mark(rb_objspace_t *objspace, rb_heap_t *heap)
{
    struct heap_page *page = NULL;
 
    /* Runs before mark_roots, so rgengc_check_relation sees a valid (absent) parent rather
     * than the poison left by the previous GC. */
    gc_mark_set_parent_raw(objspace, Qundef, false);
 
    /* A local GC never frees or traverses a shareable object, and keeps its unshareable
     * children alive through their shref bits, so:
     *   - a shareable object only gets its mark bit set (like an old object), which keeps
     *     the sweep off it, and is not traversed;
     *   - a shref is marked and traversed, like a remembered old->young target: without
     *     that, the referring shareable object is never walked and it would look
     *     unreachable.
     * Objects can become shareable between GCs, so this pass scans the bitmaps in every
     * mark (gc_marks_finish) instead of maintaining a pin set across the sweep. */
    ccan_list_for_each(&heap->pages, page, page_node) {
        if (!(page->flags.has_shareable_objects | page->flags.has_shref_objects)) continue;
 
        uintptr_t p = page->start;
        short slot_size = page->slot_size;
        int total_slots = page->total_slots;
        int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
 
        for (int j = 0; j < bitmap_plane_count; j++) {
            bits_t sr_bits = page->shref_bits[j];
            /* Only the pins ordinary marking left unmarked need work here: an already
             * marked object (reached by traversal, or pre-marked because it is old) is a
             * no-op in gc_mark_set, so skip visiting it. */
            bits_t bitset = (page->shareable_bits[j] | sr_bits) & ~page->mark_bits[j];
            uintptr_t pp = p;
            while (bitset) {
                if (bitset & 1) {
                    VALUE obj = (VALUE)pp;
                    asan_unpoisoning_object(obj) {
                        switch (BUILTIN_TYPE(obj)) {
                          case T_NONE:
                          case T_ZOMBIE:
                          case T_MOVED:
                            /* A dead slot (a zombie awaiting its finalizer) is not a root. */
                            break;
                          default:
                            gc_report(2, objspace, "pinned_roots_mark: mark %s\n", rb_obj_info(obj));
                            if (sr_bits & 1) {
                                gc_mark(objspace, obj);          /* shref: root + traverse */
                            }
                            else if (gc_mark_set(objspace, obj)) {
                                gc_aging(objspace, obj);         /* shareable: mark, no traverse */
                                /* Pin as well when compaction runs alongside: if a shareable
                                 * object moved, the C-struct slots of other Ractors (a
                                 * port in sync, say) are not updated and go stale. */
                                gc_pin(objspace, obj);
                            }
                            break;
                        }
                    }
                }
                pp += slot_size;
                bitset >>= 1;
                sr_bits >>= 1;
            }
            p += BITS_BITLENGTH * slot_size;
        }
    }
}
 
static void
gc_marks_start(rb_objspace_t *objspace, int full_mark)
{
    /* start marking */
    gc_report(1, objspace, "gc_marks_start: (%s)\n", full_mark ? "full" : "minor");
    gc_mode_transition(objspace, gc_mode_marking);
 
    if (full_mark) {
        size_t incremental_marking_steps = (objspace->rincgc.pooled_slots / INCREMENTAL_MARK_STEP_ALLOCATIONS) + 1;
        objspace->rincgc.step_slots = (objspace->marked_slots * 2) / incremental_marking_steps;
 
        if (0) fprintf(stderr, "objspace->marked_slots: %"PRIdSIZE", "
                       "objspace->rincgc.pooled_page_num: %"PRIdSIZE", "
                       "objspace->rincgc.step_slots: %"PRIdSIZE", \n",
                       objspace->marked_slots, objspace->rincgc.pooled_slots, objspace->rincgc.step_slots);
        objspace->flags.during_minor_gc = FALSE;
        if (ruby_enable_autocompact && rb_gc_single_objspace_p()) {
            objspace->flags.during_compacting |= TRUE;
        }
        objspace->profile.major_gc_count++;
        objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
        objspace->rgengc.old_objects = 0;
        objspace->rgengc.last_major_gc = objspace->profile.count;
        objspace->marked_slots = 0;
 
        for (int i = 0; i < HEAP_COUNT; i++) {
            rb_heap_t *heap = &heaps[i];
            gc_bitmaps_clear(objspace, heap, false);
            heap_move_pooled_pages_to_free_pages(heap);
 
            if (objspace->flags.during_compacting) {
                struct heap_page *page = NULL;
 
                ccan_list_for_each(&heap->pages, page, page_node) {
                    page->pinned_slots = 0;
                }
            }
        }
    }
    else {
        objspace->flags.during_minor_gc = TRUE;
        objspace->marked_slots =
          objspace->rgengc.old_objects + objspace->rgengc.uncollectible_wb_unprotected_objects; /* uncollectible objects are marked already */
        objspace->profile.minor_gc_count++;
 
        for (int i = 0; i < HEAP_COUNT; i++) {
            rgengc_rememberset_mark(objspace, &heaps[i]);
        }
    }
 
    mark_roots(objspace, NULL);
 
    gc_report(1, objspace, "gc_marks_start: (%s) end, stack in %"PRIdSIZE"\n",
              full_mark ? "full" : "minor", mark_stack_size(&objspace->mark_stack));
}
 
static bool
gc_marks(rb_objspace_t *objspace, int full_mark)
{
    gc_marking_enter(objspace);
 
    bool marking_finished = false;
 
    /* setup marking */
 
    gc_marks_start(objspace, full_mark);
    if (!is_incremental_marking(objspace)) {
        gc_marks_rest(objspace);
        marking_finished = true;
    }
 
#if RGENGC_PROFILE > 0
    if (gc_prof_record(objspace)) {
        gc_profile_record *record = gc_prof_record(objspace);
        record->old_objects = objspace->rgengc.old_objects;
    }
#endif
 
    gc_marking_exit(objspace);
 
    return marking_finished;
}
 
/* RGENGC */
 
static void
gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...)
{
    if (level <= RGENGC_DEBUG) {
        char buf[1024];
        FILE *out = stderr;
        va_list args;
        const char *status = " ";
 
        if (during_gc) {
            status = is_full_marking(objspace) ? "+" : "-";
        }
        else {
            if (is_lazy_sweeping(objspace)) {
                status = "S";
            }
            if (is_incremental_marking(objspace)) {
                status = "M";
            }
        }
 
        va_start(args, fmt);
        vsnprintf(buf, 1024, fmt, args);
        va_end(args);
 
        fprintf(out, "%s|", status);
        fputs(buf, out);
    }
}
 
/* bit operations */
 
static void
rgengc_remembersetbits_set(rb_objspace_t *objspace, VALUE obj)
{
    struct heap_page *page = GET_HEAP_PAGE(obj);
    bits_t *bits = &page->remembered_bits[0];
 
    /* remembered_bits writers are always serialized: the write barrier only remembers a
     * local a (under its Ractor's GVL) and a global GC writes from the driver alone.
     * Set the bit before the page flag so a page pending re-scan stays in
     * rememberset_mark. */
    _MARK_IN_BITMAP(bits, page, obj);
    page->flags.has_remembered_objects = TRUE;
}
 
/* wb, etc */
 
/* return FALSE if already remembered */
static void
rgengc_remember(rb_objspace_t *objspace, VALUE obj)
{
    gc_report(6, objspace, "rgengc_remember: %s %s\n", rb_obj_info(obj),
              RVALUE_REMEMBERED(objspace, obj) ? "was already remembered" : "is remembered now");
 
    check_rvalue_consistency(objspace, obj);
 
    if (RGENGC_CHECK_MODE) {
        if (RVALUE_WB_UNPROTECTED(objspace, obj)) rb_bug("rgengc_remember: %s is not wb protected.", rb_obj_info(obj));
    }
 
#if RGENGC_PROFILE > 0
    if (!RVALUE_REMEMBERED(objspace, obj)) {
        if (RVALUE_WB_UNPROTECTED(objspace, obj) == 0) {
            objspace->profile.total_remembered_normal_object_count++;
#if RGENGC_PROFILE >= 2
            objspace->profile.remembered_normal_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
        }
    }
#endif /* RGENGC_PROFILE > 0 */
 
    rgengc_remembersetbits_set(objspace, obj);
}
 
#ifndef PROFILE_REMEMBERSET_MARK
#define PROFILE_REMEMBERSET_MARK 0
#endif
 
static inline void
rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitset, short slot_size)
{
    if (bitset) {
        do {
            if (bitset & 1) {
                VALUE obj = (VALUE)p;
                gc_report(2, objspace, "rgengc_rememberset_mark: mark %s\n", rb_obj_info(obj));
                GC_ASSERT(RVALUE_UNCOLLECTIBLE(objspace, obj));
                GC_ASSERT(RVALUE_OLD_P(objspace, obj) || RVALUE_WB_UNPROTECTED(objspace, obj));
 
                gc_mark_children(objspace, obj);
 
                if (RB_FL_TEST_RAW(obj, RUBY_FL_WEAK_REFERENCE)) {
                    rb_darray_append_without_gc(&objspace->weak_references, obj);
                }
            }
            p += slot_size;
            bitset >>= 1;
        } while (bitset);
    }
}
 
static void
rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap)
{
    size_t j;
    struct heap_page *page = 0;
#if PROFILE_REMEMBERSET_MARK
    int has_old = 0, has_shady = 0, has_both = 0, skip = 0;
#endif
    gc_report(1, objspace, "rgengc_rememberset_mark: start\n");
 
    ccan_list_for_each(&heap->pages, page, page_node) {
        if (page->flags.has_remembered_objects | page->flags.has_uncollectible_wb_unprotected_objects) {
            uintptr_t p = page->start;
            short slot_size = page->slot_size;
            int total_slots = page->total_slots;
            int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
            bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT];
            bits_t *remembered_bits = page->remembered_bits;
            bits_t *uncollectible_bits = page->uncollectible_bits;
            bits_t *wb_unprotected_bits = page->wb_unprotected_bits;
#if PROFILE_REMEMBERSET_MARK
            if (page->flags.has_remembered_objects && page->flags.has_uncollectible_wb_unprotected_objects) has_both++;
            else if (page->flags.has_remembered_objects) has_old++;
            else if (page->flags.has_uncollectible_wb_unprotected_objects) has_shady++;
#endif
            /* Clear has_remembered_objects before draining the bits.  A concurrent
             * lock-free write barrier (another Ractor remembering a shareable object on
             * this page) sets the bit first and the flag second, so clearing the flag first
             * keeps the page scheduled for re-scan even if that set interleaves.  The
             * per-word drain is an atomic read-and-clear, so an interleaved set is not lost
             * (it lands in the zeroed word). */
            page->flags.has_remembered_objects = FALSE;
            for (j=0; j < (size_t)bitmap_plane_count; j++) {
                bits[j] = RUBY_ATOMIC_SIZE_EXCHANGE(*(volatile size_t *)&remembered_bits[j], 0)
                          | (uncollectible_bits[j] & wb_unprotected_bits[j]);
            }
 
            for (j=0; j < (size_t)bitmap_plane_count; j++) {
                bitset = bits[j];
                rgengc_rememberset_mark_plane(objspace, p, bitset, slot_size);
                p += BITS_BITLENGTH * slot_size;
            }
        }
#if PROFILE_REMEMBERSET_MARK
        else {
            skip++;
        }
#endif
    }
 
#if PROFILE_REMEMBERSET_MARK
    fprintf(stderr, "%d\t%d\t%d\t%d\n", has_both, has_old, has_shady, skip);
#endif
    gc_report(1, objspace, "rgengc_rememberset_mark: finished\n");
}
 
static void
gc_bitmaps_clear(rb_objspace_t *objspace, rb_heap_t *heap, bool clear_shref)
{
    struct heap_page *page = 0;
 
    ccan_list_for_each(&heap->pages, page, page_node) {
        memset(&page->mark_bits[0],       0, HEAP_PAGE_BITMAP_SIZE);
        memset(&page->uncollectible_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
        memset(&page->marking_bits[0],    0, HEAP_PAGE_BITMAP_SIZE);
        /* A plain memset can lose a concurrent remember, but only a shareable object can
         * be remembered from another Ractor's thread, and pinned_roots_mark re-marks
         * those every local cycle, and this clear precedes a major that re-scans all. */
        memset(&page->remembered_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
        memset(&page->pinned_bits[0],     0, HEAP_PAGE_BITMAP_SIZE);
        page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
        page->flags.has_remembered_objects = FALSE;
        /* A shref is a local GC's root, so only a stop-the-world global GC may clear them:
         * its unified mark re-derives them from every shareable -> unshareable edge. */
        if (clear_shref) {
            memset(&page->shref_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
            page->flags.has_shref_objects = FALSE;
        }
    }
}
 
/* RGENGC: APIs */
 
NOINLINE(static void gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace));
 
/* Precondition: `a` and `b` live in `objspace`. */
static void
gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace)
{
    if (RGENGC_CHECK_MODE) {
        if (!RVALUE_OLD_P(objspace, a)) rb_bug("gc_writebarrier_generational: %s is not an old object.", rb_obj_info(a));
        if ( RVALUE_OLD_P(objspace, b)) rb_bug("gc_writebarrier_generational: %s is an old object.", rb_obj_info(b));
        if (is_incremental_marking(objspace)) rb_bug("gc_writebarrier_generational: called while incremental marking: %s -> %s", rb_obj_info(a), rb_obj_info(b));
    }
 
    /* Mark and remember a (the default behaviour).
     * No lock: setting a remembered bit is atomic (rgengc_remembersetbits_set), and that is
     * the only place a concurrent local GC or another Ractor's write barrier can race. */
    if (!RVALUE_REMEMBERED(objspace, a)) {
        rgengc_remember(objspace, a);
 
        gc_report(1, objspace, "gc_writebarrier_generational: %s (remembered) -> %s\n", rb_obj_info(a), rb_obj_info(b));
    }
 
    check_rvalue_consistency(objspace, a);
    check_rvalue_consistency(objspace, b);
}
 
static void
gc_mark_from(rb_objspace_t *objspace, VALUE obj, VALUE parent)
{
    gc_mark_set_parent(objspace, parent);
    rgengc_check_relation(objspace, obj);
    if (gc_mark_set(objspace, obj) != FALSE) {
        gc_aging(objspace, obj);
        gc_grey(objspace, obj);
    }
    gc_mark_set_parent_invalid(objspace);
}
 
NOINLINE(static void gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace));
 
/* Precondition: `a` and `b` live in `objspace`. */
static void
gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace)
{
    gc_report(2, objspace, "gc_writebarrier_incremental: [LG] %p -> %s\n", (void *)a, rb_obj_info(b));
 
    if (RVALUE_BLACK_P(objspace, a)) {
        if (RVALUE_WHITE_P(objspace, b)) {
            if (!RVALUE_WB_UNPROTECTED(objspace, a)) {
                gc_report(2, objspace, "gc_writebarrier_incremental: [IN] %p -> %s\n", (void *)a, rb_obj_info(b));
                gc_mark_from(objspace, b, a);
            }
        }
        else if (RVALUE_OLD_P(objspace, a) && !RVALUE_OLD_P(objspace, b)) {
            rgengc_remember(objspace, a);
        }
 
        if (RB_UNLIKELY(objspace->flags.during_compacting)) {
            MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(b), b);
        }
    }
}
 
void
rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b)
{
    rb_objspace_t *objspace = objspace_ptr;
 
#if RGENGC_CHECK_MODE
    if (SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const: %"PRIxVALUE, a);
    if (SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: b is special const: %"PRIxVALUE, b);
#else
    RBIMPL_ASSERT_OR_ASSUME(!SPECIAL_CONST_P(a));
    RBIMPL_ASSERT_OR_ASSUME(!SPECIAL_CONST_P(b));
#endif
 
    GC_ASSERT(!during_gc);
    GC_ASSERT(RB_BUILTIN_TYPE(a) != T_NONE);
    GC_ASSERT(RB_BUILTIN_TYPE(a) != T_MOVED);
    GC_ASSERT(RB_BUILTIN_TYPE(a) != T_ZOMBIE);
 
    /* A shareable object now references an unshareable one: record b as a shref so its
     * owner's local GC roots it (the parent may live in another objspace, untraversed
     * there).  Only b's owner stores this, on its own page: a plain store suffices. */
    if (RB_UNLIKELY(RB_FL_TEST_RAW(a, RUBY_FL_SHAREABLE)) &&
            !RB_FL_TEST_RAW(b, RUBY_FL_SHAREABLE)) {
        struct heap_page *bpage = GET_HEAP_PAGE(b);
        if (!_MARKED_IN_BITMAP(bpage->shref_bits, bpage, b)) {
            _MARK_IN_BITMAP(bpage->shref_bits, bpage, b);
            bpage->flags.has_shref_objects = TRUE;
        }
    }
 
    if (!is_incremental_marking(objspace)) {
        /* The generational barrier covers old->young edges within one objspace only.
         * NOTE: we shouldn't even check the age of `a` or `b` if they are in another
         * objspace, so check locality first. */
        if (gc_foreign_object_p(objspace, a) || gc_foreign_object_p(objspace, b) ||
                !RVALUE_OLD_P(objspace, a) || RVALUE_OLD_P(objspace, b)) {
            // do nothing
        }
        else {
            gc_writebarrier_generational(a, b, objspace);
        }
    }
    else {
        // Shareable objects from different object spaces are kept alive by shareable bits
        if (gc_foreign_object_p(objspace, a) || gc_foreign_object_p(objspace, b)) {
            // do nothing
        }
        else {
            gc_writebarrier_incremental(a, b, objspace);
        }
    }
}
 
void
rb_gc_impl_obj_became_shareable(void *objspace_ptr, VALUE obj)
{
    /* An object becomes shareable on its owner thread, so this page update is
     * single-writer. */
    struct heap_page *page = GET_HEAP_PAGE(obj);
 
    if (_MARKED_IN_BITMAP(page->shareable_bits, page, obj)) return;
    gc_page_add_shareable(page, obj);
 
    /* The shref bits recorded while the object was unshareable are now covered by the
     * shareable pin, and a shref only points at an unshareable object.  The owner thread is
     * the only writer, so a plain clear is enough. */
    if (_MARKED_IN_BITMAP(page->shref_bits, page, obj)) {
        _CLEAR_IN_BITMAP(page->shref_bits, page, obj);
        // NOTE: page->has_shref_objects could become stale here (value is true even though logically false)
    }
}
 
void
rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    /* A shareable object is never WB-unprotected.  Keeping shrefs correct relies on every
     * store into s->u going through the write barrier, which keeps wb_unprotected_bits
     * single-writer (only the owner thread can unprotect its own unshareable objects). */
    GC_ASSERT(!RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE));
 
    if (RVALUE_WB_UNPROTECTED(objspace, obj)) {
        return;
    }
    else {
        gc_report(2, objspace, "rb_gc_writebarrier_unprotect: %s %s\n", rb_obj_info(obj),
                  RVALUE_REMEMBERED(objspace, obj) ? " (already remembered)" : "");
 
        /* No lock: per the assert obj is our own unshareable, so these bits
         * (wb_unprotected, uncollectible, age) are single-writer on an owned page, and
         * RVALUE_DEMOTE's remembered-bit clear is atomic against word-sharing writers. */
        if (RVALUE_OLD_P(objspace, obj)) {
            gc_report(1, objspace, "rb_gc_writebarrier_unprotect: %s\n", rb_obj_info(obj));
            RVALUE_DEMOTE(objspace, obj);
            gc_mark_set(objspace, obj);
            gc_remember_unprotected(objspace, obj);
 
#if RGENGC_PROFILE
            objspace->profile.total_shade_operation_count++;
#if RGENGC_PROFILE >= 2
            objspace->profile.shade_operation_count_types[BUILTIN_TYPE(obj)]++;
#endif /* RGENGC_PROFILE >= 2 */
#endif /* RGENGC_PROFILE */
        }
        else {
            RVALUE_AGE_RESET(obj);
        }
 
        RB_DEBUG_COUNTER_INC(obj_wb_unprotect);
        MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
    }
}
 
void
rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    if (RVALUE_WB_UNPROTECTED(objspace, obj)) {
        rb_gc_impl_writebarrier_unprotect(objspace, dest);
    }
    rb_gc_impl_copy_finalizer(objspace, dest, obj);
}
 
const char *
rb_gc_impl_active_gc_name(void)
{
    return "default";
}
 
/* NOTE: `obj` doesn't necessarily live in `objspace_ptr`, as `objspace_ptr` is just that
 * of the current Ractor. */
void
rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj)
{
 
    rb_objspace_t *objspace = objspace_ptr;
 
    // Shareable objects from other object spaces don't need to be put on the remembered set
    // and are only collected during global GC, so not while incremental marking.
    if (RB_LIKELY(!gc_foreign_object_p(objspace, obj))) {
        gc_report(1, objspace, "rb_gc_writebarrier_remember: %s\n", rb_obj_info(obj));
        if (is_incremental_marking(objspace)) {
            if (RVALUE_BLACK_P(objspace, obj)) {
                gc_grey(objspace, obj);
            }
        }
        else if (RVALUE_OLD_P(objspace, obj)) {
            rgengc_remember(objspace, obj);
        }
    }
}
 
struct rb_gc_object_metadata_names {
    // Must be ID only
    ID ID_wb_protected, ID_age, ID_old, ID_uncollectible, ID_marking,
        ID_marked, ID_pinned, ID_remembered, ID_object_id, ID_shareable;
};
 
#define RB_GC_OBJECT_METADATA_ENTRY_COUNT (sizeof(struct rb_gc_object_metadata_names) / sizeof(ID))
static struct rb_gc_object_metadata_entry object_metadata_entries[RB_GC_OBJECT_METADATA_ENTRY_COUNT + 1];
 
struct rb_gc_object_metadata_entry *
rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj)
{
    rb_objspace_t *objspace = objspace_ptr;
    size_t n = 0;
    static struct rb_gc_object_metadata_names names;
 
    if (!names.ID_marked) {
#define I(s) names.ID_##s = rb_intern(#s)
        I(wb_protected);
        I(age);
        I(old);
        I(uncollectible);
        I(marking);
        I(marked);
        I(pinned);
        I(remembered);
        I(object_id);
        I(shareable);
#undef I
    }
 
#define SET_ENTRY(na, v) do { \
    GC_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \
    object_metadata_entries[n].name = names.ID_##na; \
    object_metadata_entries[n].val = v; \
    n++; \
} while (0)
 
    if (!RVALUE_WB_UNPROTECTED(objspace, obj)) SET_ENTRY(wb_protected, Qtrue);
    SET_ENTRY(age, INT2FIX(RVALUE_AGE_GET(obj)));
    if (RVALUE_OLD_P(objspace, obj)) SET_ENTRY(old, Qtrue);
    if (RVALUE_UNCOLLECTIBLE(objspace, obj)) SET_ENTRY(uncollectible, Qtrue);
    if (RVALUE_MARKING(objspace, obj)) SET_ENTRY(marking, Qtrue);
    if (RVALUE_MARKED(objspace, obj)) SET_ENTRY(marked, Qtrue);
    if (RVALUE_PINNED(objspace, obj)) SET_ENTRY(pinned, Qtrue);
    if (RVALUE_REMEMBERED(objspace, obj)) SET_ENTRY(remembered, Qtrue);
    if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj));
    if (FL_TEST(obj, FL_SHAREABLE)) SET_ENTRY(shareable, Qtrue);
 
    object_metadata_entries[n].name = 0;
    object_metadata_entries[n].val = 0;
#undef SET_ENTRY
 
    return object_metadata_entries;
}
 
void *
rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor)
{
    /* No cache needed: allocation happens in a per-Ractor objspace. */
    return NULL;
}
 
void
rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache)
{
    GC_ASSERT(cache == NULL);
}
 
/* The terminating Ractor's final local GC, on its own thread: roots are minimal, so the
 * mark is tiny, and it reclaims what the joining side would otherwise inherit.  Never
 * promotes to a global GC (that would STW on every Ractor death); empty pages go
 * straight back to the page pool. */
/* Finalize the zombies whose cleanup is pure C (a dfree, no Ruby-level finalizer);
 * the caller has no Ruby execution context any more, so zombies with a Ruby
 * finalizer stay deferred and travel to the inheritor as before.  Returns whether
 * anything was finalized (those pages then need one more sweep to detach). */
static bool
finalize_deferred_dfree_only(rb_objspace_t *objspace)
{
    VALUE dfree_only = 0;
    VALUE zombie = RUBY_ATOMIC_VALUE_EXCHANGE(heap_pages_deferred_final, 0);
    while (zombie) {
        rb_asan_unpoison_object(zombie, false);
        VALUE next = RZOMBIE(zombie)->next;
        if (FL_TEST_RAW(zombie, FL_FINALIZE)) {
            /* re-defer, with the same push as rb_gc_impl_make_zombie */
            VALUE prev2, next2 = heap_pages_deferred_final;
            do {
                RZOMBIE(zombie)->next = prev2 = next2;
                next2 = RUBY_ATOMIC_VALUE_CAS(heap_pages_deferred_final, prev2, zombie);
            } while (next2 != prev2);
            rb_asan_poison_object(zombie);
        }
        else {
            RZOMBIE(zombie)->next = dfree_only;
            dfree_only = zombie;
        }
        zombie = next;
    }
    if (dfree_only) finalize_list(objspace, dfree_only);
    bool did = dfree_only != 0;
 
    gc_tdata_unsafe_free_publish(objspace);
    return did;
}
 
void
rb_gc_impl_objspace_retire_gc(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    /* The dying thread's stack is already torn down here, so the root scan must skip
     * its machine context (rb_gc_mark_roots). */
    objspace->flags.during_postmortem = 1;
    objspace->flags.ownerless = 1;
 
    gc_rest(objspace);
    gc_start(objspace, GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP);
 
    /* The sweep above turned this heap's dead IO and the like into deferred zombies
     * (the per-Ractor stdio holds a page per Ractor otherwise); finalize the C-only
     * ones here and re-sweep the nearly-empty heap so their pages detach as empty. */
    if (finalize_deferred_dfree_only(objspace)) {
        gc_start(objspace, GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP);
    }
 
    heap_pages_freeable_pages = objspace->empty_pages_count;
    heap_pages_free_unused_pages(objspace);
 
    objspace->flags.during_postmortem = 0;
}
 
bool
rb_gc_impl_during_postmortem_p(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
    return objspace->flags.during_postmortem != 0;
}
 
static void
heap_ready_to_gc(rb_objspace_t *objspace, rb_heap_t *heap)
{
    if (!heap->free_pages) {
        if (!heap_page_allocate_and_initialize(objspace, heap)) {
            objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE;
            heap_page_allocate_and_initialize(objspace, heap);
        }
    }
}
 
static int
ready_to_gc(rb_objspace_t *objspace)
{
    if ((!objspace->flags.during_postmortem && rb_gc_gc_disabled_global_p()) || dont_gc_val() || during_gc) {
        for (int i = 0; i < HEAP_COUNT; i++) {
            rb_heap_t *heap = &heaps[i];
            heap_ready_to_gc(objspace, heap);
        }
        return FALSE;
    }
    else {
        return TRUE;
    }
}
 
static void
gc_reset_malloc_info(rb_objspace_t *objspace, bool full_mark)
{
    gc_prof_set_malloc_info(objspace);
    {
        int64_t inc = gc_malloc_counters_increase(objspace, &objspace->malloc_counters.counters);
        gc_malloc_counters_snapshot(objspace, &objspace->malloc_counters.counters);
        size_t old_limit = malloc_limit;
 
        /* A net-negative `inc` (more freed than malloc'd since last GC) is
         * treated the same as "allocated less than malloc_limit".
         * This matches what we were doing pre-monotonic counters, but is it right? */
        if (inc > 0 && (size_t)inc > malloc_limit) {
            malloc_limit = (size_t)((size_t)inc * gc_params.malloc_limit_growth_factor);
            if (malloc_limit > gc_params.malloc_limit_max) {
                malloc_limit = gc_params.malloc_limit_max;
            }
        }
        else {
            malloc_limit = (size_t)(malloc_limit * 0.98); /* magic number */
            if (malloc_limit < gc_params.malloc_limit_min) {
                malloc_limit = gc_params.malloc_limit_min;
            }
        }
 
        if (0) {
            if (old_limit != malloc_limit) {
                fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: %"PRIuSIZE" -> %"PRIuSIZE"\n",
                        rb_gc_count(), old_limit, malloc_limit);
            }
            else {
                fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: not changed (%"PRIuSIZE")\n",
                        rb_gc_count(), malloc_limit);
            }
        }
    }
 
    /* reset oldmalloc info */
#if RGENGC_ESTIMATE_OLDMALLOC
    MALLOC_COUNTERS_LOCK(objspace);
    gc_counter_store_release(&objspace->malloc_counters.freed_in_gc, 0);
    MALLOC_COUNTERS_UNLOCK(objspace);
    if (full_mark) {
        RUBY_ATOMIC_SIZE_EXCHANGE(objspace->malloc_counters.oldmalloc_increase, 0);
 
        if ((objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) {
            objspace->rgengc.oldmalloc_increase_limit =
              (size_t)(objspace->rgengc.oldmalloc_increase_limit / ((gc_params.oldmalloc_limit_growth_factor - 1)/10 + 1));
            if (objspace->rgengc.oldmalloc_increase_limit < gc_params.oldmalloc_limit_min) {
                objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
            }
        }
    }
#endif
}
 
/* What a collection records about itself before it runs.  A global collection reports the
 * driver's objspace, so it comes through here too. */
static void
gc_start_record(rb_objspace_t *objspace, unsigned int reason, bool full_mark)
{
    objspace->profile.latest_gc_info = reason;
#if GC_PROFILE_MORE_DETAIL
    objspace->profile.total_allocated_objects_at_gc_start = total_allocated_objects(objspace);
    objspace->profile.heap_used_at_gc_start = rb_darray_size(objspace->heap_pages.sorted);
#endif
    objspace->profile.weak_references_count = 0;
    gc_prof_setup_new_record(objspace, reason);
    gc_reset_malloc_info(objspace, full_mark);
}
 
static bool gc_start_global(rb_objspace_t *driver, unsigned int reason, bool compact, bool allow_skip);
 
/* Decide whether this collection has to be global.  A local GC can reclaim neither
 * shareable objects nor zombie objspaces, so once those grow past their limits only a
 * global GC makes progress.  All inputs belong to this objspace. */
static bool
gc_need_global_p(rb_objspace_t *objspace)
{
    if (rb_gc_single_objspace_p()) return false;
    /* A Ractor's death must not stop the world, so the retire GC stays local. */
    if (objspace->flags.during_postmortem) return false;
    if (objspace->shareable_objects > objspace->shareable_objects_limit) return true;
    /* A zombie's garbage only a global cycle reclaims, but what survived the last one
     * is live data, so retrigger only once TRIGGER more pages accumulate on top of it.
     * Otherwise one live-heavy unjoined zombie turns every GC stop-the-world forever. */
    {
        size_t zp = global_objspace->zombie_total_pages;
        size_t base = global_objspace->zombie_pages_survivors < zp ? global_objspace->zombie_pages_survivors : zp;
        if (zp - base >= ZOMBIE_PAGES_TRIGGER) return true;
    }
    return false;
}
 
static int
garbage_collect(rb_objspace_t *objspace, unsigned int reason)
{
    int ret;
 
#if GC_PROFILE_MORE_DETAIL
    objspace->profile.prepare_time = getrusage_time();
#endif
 
    gc_rest(objspace);
 
#if GC_PROFILE_MORE_DETAIL
    objspace->profile.prepare_time = getrusage_time() - objspace->profile.prepare_time;
#endif
 
    ret = gc_start(objspace, reason);
 
    return ret;
}
 
static int
gc_start(rb_objspace_t *objspace, unsigned int reason)
{
    unsigned int do_full_mark = !!(reason & GPR_FLAG_FULL_MARK);
 
    if (!rb_darray_size(objspace->heap_pages.sorted)) return TRUE; /* heap is not ready */
    if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */
 
    /* An explicit GC.start(global: true) never gets here: rb_gc_impl_start has already decided from
     * the `global` keyword, and GPR_FLAG_METHOD keeps `global: false` from being promoted back. */
    if (!(reason & GPR_FLAG_METHOD) && gc_need_global_p(objspace)) {
        /* A global GC is always a major, so autocompact applies. */
        if (gc_start_global(objspace, reason, ruby_enable_autocompact, true)) {
            return TRUE;
        }
        /* Fall through to a local GC */
    }
 
    rb_gc_initialize_vm_context(&objspace->vm_context);
 
    GC_ASSERT(gc_mode(objspace) == gc_mode_none, "gc_mode is %s\n", gc_mode_name(gc_mode(objspace)));
    GC_ASSERT(!is_lazy_sweeping(objspace));
    GC_ASSERT(!is_incremental_marking(objspace));
 
    /* reason may be clobbered, later, so keep set immediate_sweep here */
    objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP);
 
    if (ruby_gc_stressful) {
        int flag = FIXNUM_P(ruby_gc_stress_mode) ? FIX2INT(ruby_gc_stress_mode) : 0;
 
        if ((flag & (1 << gc_stress_no_major)) == 0) {
            do_full_mark = TRUE;
        }
 
        objspace->flags.immediate_sweep = !(flag & (1<<gc_stress_no_immediate_sweep));
    }
 
    if (gc_needs_major_flags) {
        reason |= gc_needs_major_flags;
        do_full_mark = TRUE;
    }
 
    /* if major gc has been disabled, never do a full mark */
    if (!gc_config_full_mark_val) {
        do_full_mark = FALSE;
    }
    gc_needs_major_flags = GPR_FLAG_NONE;
 
    if (do_full_mark && (reason & GPR_FLAG_MAJOR_MASK) == 0) {
        reason |= GPR_FLAG_MAJOR_BY_FORCE; /* GC by CAPI, METHOD, and so on. */
    }
 
    if (objspace->flags.dont_incremental ||
            reason & GPR_FLAG_IMMEDIATE_MARK ||
            ruby_gc_stressful) {
        gc_set_incremental_marking(objspace, FALSE);
    }
    else {
        gc_set_incremental_marking(objspace, do_full_mark);
    }
 
    /* Compaction on the local GC path (autocompact) runs only with a single objspace:
     * without the stop-the-world barrier, moving objects would break cross-objspace
     * references.  With multiple objspaces GC.compact and autocompact go through the
     * compacting global GC instead (rb_gc_impl_start, or the promotion above). */
    if (do_full_mark && ruby_enable_autocompact && rb_gc_single_objspace_p()) {
        objspace->flags.during_compacting = TRUE;
#if RGENGC_CHECK_MODE
        objspace->rcompactor.compare_func = ruby_autocompact_compare_func;
#endif
    }
    else {
        objspace->flags.during_compacting = !!(reason & GPR_FLAG_COMPACT);
        if (objspace->flags.during_compacting && !rb_gc_single_objspace_p()) {
            // compaction is currently global GC only with more than 1 running Ractor
            objspace->flags.during_compacting = FALSE;
        }
    }
 
    if (!GC_ENABLE_LAZY_SWEEP || objspace->flags.dont_incremental) {
        objspace->flags.immediate_sweep = TRUE;
    }
 
    if (objspace->flags.immediate_sweep) reason |= GPR_FLAG_IMMEDIATE_SWEEP;
 
    /* Enter after during_compacting is decided */
    unsigned int lock_lev;
    gc_enter(objspace, gc_enter_event_start, &lock_lev);
 
    gc_report(1, objspace, "gc_start(reason: %x) => %u, %d, %d\n",
              reason,
              do_full_mark, !is_incremental_marking(objspace), objspace->flags.immediate_sweep);
 
    RB_DEBUG_COUNTER_INC(gc_count);
 
    if (reason & GPR_FLAG_MAJOR_MASK) {
        (void)RB_DEBUG_COUNTER_INC_IF(gc_major_nofree, reason & GPR_FLAG_MAJOR_BY_NOFREE);
        (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldgen, reason & GPR_FLAG_MAJOR_BY_OLDGEN);
        (void)RB_DEBUG_COUNTER_INC_IF(gc_major_shady,  reason & GPR_FLAG_MAJOR_BY_SHADY);
        (void)RB_DEBUG_COUNTER_INC_IF(gc_major_force,  reason & GPR_FLAG_MAJOR_BY_FORCE);
#if RGENGC_ESTIMATE_OLDMALLOC
        (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldmalloc, reason & GPR_FLAG_MAJOR_BY_OLDMALLOC);
#endif
    }
    else {
        (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_newobj, reason & GPR_FLAG_NEWOBJ);
        (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_malloc, reason & GPR_FLAG_MALLOC);
        (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_method, reason & GPR_FLAG_METHOD);
        (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_capi,   reason & GPR_FLAG_CAPI);
        (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_stress, reason & GPR_FLAG_STRESS);
    }
 
    objspace->profile.count++;
    gc_start_record(objspace, reason, do_full_mark);
 
    gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_START);
 
    GC_ASSERT(during_gc);
 
    gc_prof_timer_start(objspace);
    {
        if (gc_marks(objspace, do_full_mark)) {
            gc_sweep(objspace);
        }
    }
    gc_prof_timer_stop(objspace);
 
    gc_exit(objspace, gc_enter_event_start, &lock_lev);
 
    /* Verify after the GC, at a real safepoint with during_gc cleared: mid-GC
     * gc_verify_internal_consistency() takes the VM lock and a barrier, which would join
     * another Ractor's global GC barrier and let it collect on this half-collected heap. */
#if RGENGC_CHECK_MODE >= 2
    gc_verify_internal_consistency(objspace);
#endif
    return TRUE;
}
 
static void
gc_rest(rb_objspace_t *objspace)
{
    if (is_incremental_marking(objspace) || is_lazy_sweeping(objspace)) {
        unsigned int lock_lev;
        gc_enter(objspace, gc_enter_event_rest, &lock_lev);
 
        if (is_incremental_marking(objspace)) {
            gc_marking_enter(objspace);
            gc_marks_rest(objspace);
            gc_marking_exit(objspace);
 
            gc_sweep(objspace);
        }
 
        if (is_lazy_sweeping(objspace)) {
            gc_sweeping_enter(objspace);
            gc_sweep_rest(objspace);
            gc_sweeping_exit(objspace);
        }
 
        gc_exit(objspace, gc_enter_event_rest, &lock_lev);
 
        if (RGENGC_CHECK_MODE >= 2) gc_verify_internal_consistency(objspace); /* after GC, see gc_start */
    }
}
 
struct objspace_and_reason {
    rb_objspace_t *objspace;
    unsigned int reason;
};
 
static void
gc_current_status_fill(rb_objspace_t *objspace, char *buff)
{
    int i = 0;
    if (is_marking(objspace)) {
        buff[i++] = 'M';
        if (is_full_marking(objspace))        buff[i++] = 'F';
        if (is_incremental_marking(objspace)) buff[i++] = 'I';
    }
    else if (is_sweeping(objspace)) {
        buff[i++] = 'S';
        if (is_lazy_sweeping(objspace))      buff[i++] = 'L';
    }
    else {
        buff[i++] = 'N';
    }
    buff[i] = '\0';
}
 
static const char *
gc_current_status(rb_objspace_t *objspace)
{
    static char buff[0x10];
    gc_current_status_fill(objspace, buff);
    return buff;
}
 
#if PRINT_ENTER_EXIT_TICK
 
static tick_t last_exit_tick;
static tick_t enter_tick;
static int enter_count = 0;
static char last_gc_status[0x10];
 
static inline void
gc_record(rb_objspace_t *objspace, int direction, const char *event)
{
    if (direction == 0) { /* enter */
        enter_count++;
        enter_tick = tick();
        gc_current_status_fill(objspace, last_gc_status);
    }
    else { /* exit */
        tick_t exit_tick = tick();
        char current_gc_status[0x10];
        gc_current_status_fill(objspace, current_gc_status);
#if 1
        /* [last mutator time] [gc time] [event] */
        fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
                enter_tick - last_exit_tick,
                exit_tick - enter_tick,
                event,
                last_gc_status, current_gc_status,
                (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
        last_exit_tick = exit_tick;
#else
        /* [enter_tick] [gc time] [event] */
        fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
                enter_tick,
                exit_tick - enter_tick,
                event,
                last_gc_status, current_gc_status,
                (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
#endif
    }
}
#else /* PRINT_ENTER_EXIT_TICK */
static inline void
gc_record(rb_objspace_t *objspace, int direction, const char *event)
{
    /* null */
}
#endif /* PRINT_ENTER_EXIT_TICK */
 
static const char *
gc_enter_event_cstr(enum gc_enter_event event)
{
    switch (event) {
      case gc_enter_event_start: return "start";
      case gc_enter_event_continue: return "continue";
      case gc_enter_event_rest: return "rest";
      case gc_enter_event_finalizer: return "finalizer";
      case gc_enter_event_global: return "global";
      case gc_enter_event_global_auto: return "global_auto";
    }
    return NULL;
}
 
static void
gc_enter_count(enum gc_enter_event event)
{
    switch (event) {
      case gc_enter_event_start:          RB_DEBUG_COUNTER_INC(gc_enter_start); break;
      case gc_enter_event_continue:       RB_DEBUG_COUNTER_INC(gc_enter_continue); break;
      case gc_enter_event_rest:           RB_DEBUG_COUNTER_INC(gc_enter_rest); break;
      case gc_enter_event_finalizer:      RB_DEBUG_COUNTER_INC(gc_enter_finalizer); break;
      case gc_enter_event_global:         RB_DEBUG_COUNTER_INC(gc_enter_start); break;
      case gc_enter_event_global_auto:    RB_DEBUG_COUNTER_INC(gc_enter_start); break;
    }
}
 
static bool current_process_time(struct timespec *ts);
 
/* A gc phase must be timed on the collecting thread's own cpu.  A local gc runs
 * while the other ractors keep going, and process cpu time counts their work as
 * gc: with eight busy ractors the same ten collections were reported as 131ms
 * instead of 3ms, more than the wall clock they ran in.  The kernel also answers
 * this one without walking every thread in the process. */
static bool
current_thread_time(struct timespec *ts)
{
#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_THREAD_CPUTIME_ID)
    {
        static int try_clock_gettime = 1;
        if (try_clock_gettime) {
            if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, ts) == 0) {
                return true;
            }
            else {
                try_clock_gettime = 0;
            }
        }
    }
#endif
    return current_process_time(ts);
}
 
static void
gc_clock_start(struct timespec *ts)
{
    if (!current_thread_time(ts)) {
        ts->tv_sec = 0;
        ts->tv_nsec = 0;
    }
}
 
static unsigned long long
gc_clock_end(struct timespec *ts)
{
    struct timespec end_time;
 
    if ((ts->tv_sec > 0 || ts->tv_nsec > 0) &&
            current_thread_time(&end_time) &&
            end_time.tv_sec >= ts->tv_sec) {
        return (unsigned long long)(end_time.tv_sec - ts->tv_sec) * (1000 * 1000 * 1000) +
                    (end_time.tv_nsec - ts->tv_nsec);
    }
 
    return 0;
}
 
static void
gc_process_stat_after_fork_i(void *objspace_ptr, void *data)
{
    rb_objspace_t *objspace = objspace_ptr;
    rb_native_mutex_initialize(&objspace->process_stat.lock);
}
 
static inline bool
gc_local_gc_holds_vm_lock(void)
{
    return rb_gc_single_objspace_p();
}
 
static inline bool
gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev)
{
     /*
     * NOTE: The GC must never take the barrier VM lock from inside itself: the waiter could
     * join a pending barrier mid-collection and expose its half-collected heap to the global
     * GC. A no-barrier lock is safe. Other shared structures the GC paths touch use their own
     * native mutexes or the page-pool lock. */
    *lock_lev = 0;
 
    RUBY_DTRACE_GC_HOOK(ENTER, event);
 
    if (objspace->profile.run) {
        switch (event) {
          case gc_enter_event_start:
          case gc_enter_event_continue:
          case gc_enter_event_rest:
          case gc_enter_event_global:
          case gc_enter_event_global_auto:
            /* A global GC is the longest pause the process takes, so it is the last thing
             * the profiler may leave unmeasured.  The switch below stops the world for it,
             * which is exactly the interval gc_stop_time is meant to name, so start the
             * clock here like a local collection does. */
            objspace->profile.gc_pause_start_time = rb_hrtime_now();
            break;
          case gc_enter_event_finalizer:
            break;
        }
    }
    switch (event) {
      case gc_enter_event_global:
        *lock_lev = RB_GC_VM_LOCK();
        // stop other ractors
        rb_gc_vm_barrier();
        break;
      case gc_enter_event_global_auto:
        *lock_lev = RB_GC_VM_LOCK();
        if (!gc_need_global_p(objspace)) {
            RB_GC_VM_UNLOCK(*lock_lev);
            *lock_lev = 0;
            objspace->profile.gc_pause_start_time = 0;
            return false;
        }
        rb_gc_vm_barrier();
        break;
      case gc_enter_event_finalizer:
        /* Shutdown finalizers read VM-global tables (fstring, symbol) and free T_DATA that
         * is not thread-safe, so take the no-barrier VM lock. */
        *lock_lev = RB_GC_VM_LOCK_NO_BARRIER();
        break;
      default:
        objspace->flags.gc_lock_barrier = FALSE;
        if (objspace->flags.during_compacting) {
            /* Compaction relocates objects and rewrites every Ractor's JIT and global
             * references, so it stops the world with a barrier VM lock.  rb_gc_vm_barrier is
             * a reentrant no-op with a single Ractor, so an inner barrier request during the
             * move folds into this one and gc_exit ends it. */
            *lock_lev = RB_GC_VM_LOCK();
            rb_gc_vm_barrier();
            objspace->flags.gc_lock_barrier = TRUE;
        }
        else if (gc_local_gc_holds_vm_lock()) {
            *lock_lev = RB_GC_VM_LOCK_NO_BARRIER();
        }
        break;
    }
 
    if (objspace->profile.gc_pause_start_time) {
        objspace->profile.gc_stw_start_time = rb_hrtime_now();
        objspace->profile.gc_stop_time = rb_hrtime_sub(
                objspace->profile.gc_stw_start_time,
                objspace->profile.gc_pause_start_time);
    }
 
    gc_enter_count(event);
    if (RB_UNLIKELY(during_gc != 0)) rb_bug("during_gc != 0");
    if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace);
 
    during_gc = TRUE;
    RUBY_DEBUG_LOG("%s (%s)",gc_enter_event_cstr(event), gc_current_status(objspace));
    gc_report(1, objspace, "gc_enter: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace));
    gc_record(objspace, 0, gc_enter_event_cstr(event));
 
    gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_ENTER);
    return true;
}
 
static inline void
gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev)
{
    GC_ASSERT(during_gc != 0);
 
    RUBY_DTRACE_GC_HOOK(EXIT, event);
 
    gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_EXIT);
 
    if (objspace->profile.gc_pause_start_time) {
        if (gc_prof_enabled(objspace)) {
            rb_hrtime_t now = rb_hrtime_now();
            gc_profile_record *record = gc_prof_record(objspace);
            record->gc_pause_time = rb_hrtime_add(record->gc_pause_time,
                    rb_hrtime_sub(now, objspace->profile.gc_pause_start_time));
            record->gc_stop_time = rb_hrtime_add(record->gc_stop_time,
                    objspace->profile.gc_stop_time);
            record->gc_stw_time = rb_hrtime_add(record->gc_stw_time,
                    rb_hrtime_sub(now, objspace->profile.gc_stw_start_time));
        }
        objspace->profile.gc_pause_start_time = 0;
        objspace->profile.gc_stw_start_time = 0;
        objspace->profile.gc_stop_time = 0;
    }
 
    gc_record(objspace, 1, gc_enter_event_cstr(event));
    RUBY_DEBUG_LOG("%s (%s)", gc_enter_event_cstr(event), gc_current_status(objspace));
    gc_report(1, objspace, "gc_exit: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace));
    during_gc = FALSE;
    gc_process_stat_publish(objspace);
 
    switch (event) {
      case gc_enter_event_global:
      case gc_enter_event_global_auto:
        RB_GC_VM_UNLOCK(*lock_lev);
        break;
      case gc_enter_event_finalizer:
        RB_GC_VM_UNLOCK_NO_BARRIER(*lock_lev);
        break;
      default:
        if (*lock_lev != 0) {
            if (objspace->flags.gc_lock_barrier) {
                objspace->flags.gc_lock_barrier = FALSE;
                RB_GC_VM_UNLOCK(*lock_lev);
            }
            else {
                RB_GC_VM_UNLOCK_NO_BARRIER(*lock_lev);
            }
        }
        break;
    }
}
 
#ifndef MEASURE_GC
#define MEASURE_GC (objspace->flags.measure_gc)
#endif
 
static void
gc_marking_enter(rb_objspace_t *objspace)
{
    GC_ASSERT(during_gc != 0);
 
    gc_prof_mark_timer_start(objspace);
 
    if (gc_prof_enabled(objspace)) {
        objspace->profile.gc_mark_phase_wall_start_time = rb_hrtime_now();
    }
 
    if (MEASURE_GC) {
        gc_clock_start(&objspace->profile.marking_start_time);
    }
 
    rb_gc_initialize_vm_context(&objspace->vm_context);
}
 
static void
gc_marking_exit(rb_objspace_t *objspace)
{
    GC_ASSERT(during_gc != 0);
 
    if (MEASURE_GC) {
        objspace->profile.marking_time_ns += gc_clock_end(&objspace->profile.marking_start_time);
    }
 
    if (gc_prof_enabled(objspace)) {
        gc_profile_record *record = gc_prof_record(objspace);
        record->gc_mark_wall_time = rb_hrtime_add(record->gc_mark_wall_time,
                elapsed_hrtime_from(objspace->profile.gc_mark_phase_wall_start_time));
    }
 
    gc_prof_mark_timer_stop(objspace);
}
 
static void
gc_sweeping_cpu_enter(rb_objspace_t *objspace)
{
    if (MEASURE_GC) {
        gc_clock_start(&objspace->profile.sweeping_start_time);
    }
}
 
static void
gc_sweeping_cpu_exit(rb_objspace_t *objspace)
{
    if (MEASURE_GC) {
        objspace->profile.sweeping_time_ns += gc_clock_end(&objspace->profile.sweeping_start_time);
    }
}
 
static void
gc_sweeping_enter(rb_objspace_t *objspace)
{
    GC_ASSERT(during_gc != 0);
 
    if (gc_prof_enabled(objspace)) {
        objspace->profile.gc_sweep_phase_wall_start_time = rb_hrtime_now();
        objspace->profile.gc_sweep_excluded_wall_time = 0;
    }
 
    gc_sweeping_cpu_enter(objspace);
 
    rb_gc_initialize_vm_context(&objspace->vm_context);
}
 
static void
gc_sweeping_exit(rb_objspace_t *objspace)
{
    GC_ASSERT(during_gc != 0);
 
    gc_sweeping_cpu_exit(objspace);
 
    if (gc_prof_enabled(objspace)) {
        rb_hrtime_t sweep_wall_time = elapsed_hrtime_from(objspace->profile.gc_sweep_phase_wall_start_time);
        gc_profile_record *record = gc_prof_record(objspace);
        sweep_wall_time = rb_hrtime_sub(sweep_wall_time,
                objspace->profile.gc_sweep_excluded_wall_time);
        record->gc_sweep_wall_time = rb_hrtime_add(record->gc_sweep_wall_time,
                sweep_wall_time);
        objspace->profile.gc_sweep_excluded_wall_time = 0;
    }
}
 
static void *
gc_with_gvl(void *ptr)
{
    struct objspace_and_reason *oar = (struct objspace_and_reason *)ptr;
    return (void *)(VALUE)garbage_collect(oar->objspace, oar->reason);
}
 
int ruby_thread_has_gvl_p(void);
 
static int
garbage_collect_with_gvl(rb_objspace_t *objspace, unsigned int reason)
{
    if (rb_gc_gc_disabled_global_p() || dont_gc_val()) {
        return TRUE;
    }
    else if (!ruby_native_thread_p()) {
        return TRUE;
    }
    else if (!ruby_thread_has_gvl_p()) {
        void *ret;
        struct objspace_and_reason oar;
        oar.objspace = objspace;
        oar.reason = reason;
        ret = rb_thread_call_with_gvl(gc_with_gvl, (void *)&oar);
 
        return !!ret;
    }
    else {
        return garbage_collect(objspace, reason);
    }
}
 
static int
gc_set_candidate_object_i(void *vstart, void *vend, size_t stride, void *data)
{
    rb_objspace_t *objspace = (rb_objspace_t *)data;
 
    VALUE v = (VALUE)vstart;
    for (; v != (VALUE)vend; v += stride) {
        asan_unpoisoning_object(v) {
            switch (BUILTIN_TYPE(v)) {
              case T_NONE:
              case T_ZOMBIE:
                break;
              default:
                rb_gc_prepare_heap_process_object(v);
                if (!RVALUE_OLD_P(objspace, v) && !RVALUE_WB_UNPROTECTED(objspace, v)) {
                    RVALUE_AGE_SET_CANDIDATE(objspace, v);
                }
            }
        }
    }
 
    return 0;
}
 
bool
rb_gc_impl_multi_objspace_p(void)
{
    return true;
}
 
bool
rb_gc_impl_during_global_gc_p(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
    return objspace->flags.during_global_gc != 0;
}
 
bool
rb_gc_impl_obj_foreign_p(void *objspace_ptr, VALUE obj)
{
    return gc_foreign_object_p(objspace_ptr, obj);
}
 
 
/* Whether obj is recorded as an unshareable object referenced from a shareable one.  For
 * the verifier: a shareable -> unshareable edge is only accepted if the write barrier
 * recorded it here. */
bool
rb_gc_impl_shref_marked_p(void *objspace_ptr, VALUE obj)
{
    return MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj) != 0;
}
 
/* What a zombie objspace is worth to the zombie page trigger, floored: the retire GC
 * can empty an ownerless objspace outright, yet its struct, mark stack and heap_page
 * structs stay in malloc and it keeps the world multi-objspace until a global GC
 * merges it, so a page-less zombie must still push the trigger.  Never 0, so a
 * nonzero zombie_pages also marks the objspace as a zombie. */
static size_t
objspace_zombie_pages(rb_objspace_t *objspace)
{
    size_t pages = rb_darray_size(objspace->heap_pages.sorted);
    return pages > HEAP_COUNT ? pages : HEAP_COUNT;
}
 
/* gc.c parked the objspace in its zombie table, under the VM lock. */
void
rb_gc_impl_objspace_zombify(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
    GC_ASSERT(objspace->zombie_pages == 0);
    objspace->zombie_pages = objspace_zombie_pages(objspace);
    global_objspace->zombie_total_pages += objspace->zombie_pages;
}
 
/* Re-measure every zombie this cycle covered, by difference, so a zombie parked after
 * the snapshot keeps the pages it was parked with.  Inside the barrier. */
static void
gc_global_refresh_zombie_pages(void)
{
    for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
        rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
        if (objspace->zombie_pages == 0) continue;
        size_t pages = objspace_zombie_pages(objspace);
        global_objspace->zombie_total_pages -= objspace->zombie_pages;
        global_objspace->zombie_total_pages += pages;
        objspace->zombie_pages = pages;
    }
    global_objspace->zombie_pages_survivors = global_objspace->zombie_total_pages;
}
 
static void
gc_global_objspaces_i(void *os, void *data)
{
    if (global_objspace->global_gc.n_objspaces == global_objspace->global_gc.objspaces_capa) {
        size_t new_capa = global_objspace->global_gc.objspaces_capa ? global_objspace->global_gc.objspaces_capa * 2 : 16;
        struct rb_objspace **new_list = realloc(global_objspace->global_gc.objspaces, new_capa * sizeof(*new_list));
        if (new_list == NULL) rb_bug("gc_global_objspaces_i: realloc failed");
        global_objspace->global_gc.objspaces = new_list;
        global_objspace->global_gc.objspaces_capa = new_capa;
    }
    global_objspace->global_gc.objspaces[global_objspace->global_gc.n_objspaces++] = os;
}
 
/* Re-snapshot every objspace this cycle covers, zombies included.  The objspaces/capa
 * buffer is reused from the previous cycle. */
static void
gc_global_snapshot_objspaces(void)
{
    global_objspace->global_gc.n_objspaces = 0;
    rb_gc_vm_each_objspace(gc_global_objspaces_i, NULL);
 
#if RGENGC_CHECK_MODE
    /* Check that the incrementally maintained page_index agrees with the per-objspace
     * sorted arrays. */
    size_t total = 0, zombie_total = 0;
    for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
        total += rb_darray_size(global_objspace->global_gc.objspaces[i]->heap_pages.sorted);
        zombie_total += global_objspace->global_gc.objspaces[i]->zombie_pages;
    }
    GC_ASSERT(total == global_objspace->page_index.n_pages);
    /* Every zombie is enumerated, so the snapshot sees all of zombie_total_pages. */
    GC_ASSERT(zombie_total == global_objspace->zombie_total_pages);
#endif
}
 
/* Global GC: stop every Ractor and clear/mark/sweep all objspaces as one heap.  It is the
 * only collector that can free shareable objects and decide cross-objspace reachability
 * precisely. */
/* The global GC's generic_fields weak pass, after the unified mark fixpoint, before the
 * sweep.  Per-object rb_mark_generic_ivar is a no-op during a global GC (the driver has
 * GET_RACTOR() != owner); the whole table is swept here instead.  Weak-KEY: mark the val
 * (fields_obj, a strong child) only for a live key, drain dead keys' entries.  Marking a
 * val can make another key live, so repeat to a fixpoint. */
struct genfields_mark_arg {
    rb_objspace_t *objspace;
    bool progress;
};
 
static int
genfields_mark_i(VALUE key, VALUE val, void *arg)
{
    struct genfields_mark_arg *a = (struct genfields_mark_arg *)arg;
    if (RB_SPECIAL_CONST_P(val) || !RVALUE_MARKED_BITMAP(key)) {
        return ST_CONTINUE;
    }
    /* Record the old(key)->young(val) edge with the host (key) as parent, even when val
     * is already marked: a conservative machine-stack scan can mark a fresh fields_obj
     * parentless before this pass, and branching on the mark bit would leave the key
     * unremembered, so the next minor GC misses the young val ("WB miss (O->Y)").
     * gc_mark runs rgengc_check_relation before its already-marked return: call always. */
    bool newly = !RVALUE_MARKED_BITMAP(val);
    gc_mark_set_parent(a->objspace, key);
    gc_mark(a->objspace, val);
    if (newly) a->progress = true;
    return ST_CONTINUE;
}
 
static bool
genfields_dead_p(VALUE key)
{
    return RVALUE_MARKED_BITMAP(key) == 0;
}
 
static void
gc_global_mark_generic_fields(rb_objspace_t *driver)
{
    struct genfields_mark_arg arg = { driver, false };
    do {
        arg.progress = false;
        /* Each entry's mark sets parent=key (genfields_mark_i) so the generational WB is
         * recorded correctly.  gc_mark_stacked_objects_all sets its own per-object parent,
         * so restore the invalid parent (the poison contract) before calling it. */
        rb_gc_vm_generic_fields_mark_foreach(genfields_mark_i, &arg);
        gc_mark_set_parent_invalid(driver);
        if (arg.progress) {
            gc_mark_stacked_objects_all(driver);
        }
    } while (arg.progress);
 
    rb_gc_vm_generic_fields_drain_dead(genfields_dead_p);
}
 
/* Two Ractors choosing a global GC at once are serialized by the VM lock in gc_enter. If two
 * globals start concurrently, only one global will run and the other will run a local GC after
 * the barrier ends. */
static bool
gc_start_global(rb_objspace_t *driver, unsigned int reason, bool compact, bool allow_skip)
{
    unsigned int lock_lev;
    enum gc_enter_event event = allow_skip ? gc_enter_event_global_auto : gc_enter_event_global;
    if (!gc_enter(driver, event, &lock_lev)) {
        return false;
    }
 
    reason |= GPR_FLAG_GLOBAL;
 
    /* A global GC is a collection of the driver's objspace too, so report it like a local
     * one: it records a profile entry and reports what triggered it. */
    gc_start_record(driver, reason, true);
    gc_event_hook(driver, RUBY_INTERNAL_EVENT_GC_START);
    gc_prof_timer_start(driver);
 
    gc_global_snapshot_objspaces();
 
    /* Mark every objspace as in a global GC before step 3 settles the lazy sweeps: the
     * settle frees other objspaces' garbage on the driver thread, and
     * rb_free_generic_ivar must see "global GC in progress" to defer generic_fields
     * removal to the weak-pass drain. */
    for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
        global_objspace->global_gc.objspaces[i]->flags.during_global_gc = TRUE;
    }
 
    /* A global GC collects every objspace, so each needs the malloc-counter reset the
     * driver got in gc_start_record; without it, gc_sweep_finish advancing free_at_last_gc
     * (step 9) would leave their malloc_increase overstated by everything swept here. */
    for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
        rb_objspace_t *const os = global_objspace->global_gc.objspaces[i];
        if (os != driver) {
            os->profile.latest_gc_info = reason;
            gc_reset_malloc_info(os, true);
        }
    }
 
    /* step 3: settle every lazy sweep so the mark bits' meaning is fixed before the clear
     * below.  (during_gc is a macro over the local "objspace".)  rb_gc_get_ec() resolves
     * through objspace->vm_context during a GC, so initialize it for all: the driver
     * thread runs every objspace's phases. */
    for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
        rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
        /* The barrier can stop a Ractor between the steps of its own incremental mark, and
         * nothing else finishes another objspace's mark.  Drop the partial mark rather
         * than clear its flags in step 5 under a live gray stack: the owner would resume
         * with a mark stack into a heap this GC has since re-marked and swept.  Nothing is
         * lost, the unified mark below redoes the work. */
        if (is_incremental_marking(objspace)) {
            gc_abort_incremental_marking(objspace);
        }
        GC_ASSERT(!is_incremental_marking(objspace));
        GC_ASSERT(is_mark_stack_empty(&objspace->mark_stack));
        rb_gc_initialize_vm_context(&objspace->vm_context);
        if (objspace != driver) during_gc = TRUE;
        gc_sweep_rest(objspace);
    }
 
    /* step 5: clear every objspace's mark bits, remembered sets, generation counters and
     * shrefs (missing even one leaves a stale mark bit and a UAF).  (heaps is a macro over
     * the local "objspace".) */
    for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
        rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
        objspace->flags.during_minor_gc = FALSE;
        gc_set_incremental_marking(objspace, FALSE);
        /* The unified mark is precise and does not pin, so the per-objspace sweep below must
         * not re-check against a stale local cycle. */
        objspace->last_cycle_pinned = 0;
        objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
        objspace->rgengc.old_objects = 0;
        objspace->profile.count++;
        objspace->profile.global_gc_count++;
        objspace->rgengc.last_major_gc = objspace->profile.count;
        objspace->rgengc.need_major_gc = GPR_FLAG_NONE;
        objspace->marked_slots = 0;
        for (int h = 0; h < HEAP_COUNT; h++) {
            rb_heap_t *heap = &heaps[h];
            gc_bitmaps_clear(objspace, heap, true);
            heap_move_pooled_pages_to_free_pages(heap);
        }
    }
    driver->profile.global_gc_invocation_count++;
    global_objspace->global_gc.count++;
 
    /* Enable compaction in every objspace before the mark: the unified conservative root
     * scan then pins machine-stack referents (gc_pin only pins while during_compacting)
     * and step 9's sweep relocates the rest.  global_gc.compacting defers the
     * reference-update phase to phase 2 below (two phases, safe across objspaces). */
    global_objspace->global_gc.compacting = compact;
    if (compact) {
        for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
            rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
            objspace->flags.during_compacting = TRUE;
#if RGENGC_CHECK_MODE
            if (ruby_enable_autocompact) {
                objspace->rcompactor.compare_func = ruby_autocompact_compare_func;
            }
#endif
            /* A global GC skips gc_marks_start, which is what resets pinned_slots for a
             * compacting local GC, so reset it here.  step 5 cleared pinned_bits; the
             * conservative mark re-pins machine-stack referents. */
            for (int h = 0; h < HEAP_COUNT; h++) {
                struct heap_page *page = NULL;
                ccan_list_for_each(&heaps[h].pages, page, page_node) {
                    page->pinned_slots = 0;
                }
            }
        }
    }
 
    /* steps 6-7: every Ractor's roots (gc.c walks them all), then one unified precise
     * mark.  A global GC does not go through gc_marks, so the marking
     * phase is opened here instead; it closes after rb_ractor_finish_marking below, which is
     * where gc_marks_finish ends for a local collection. */
    gc_marking_enter(driver);
 
    mark_roots(driver, NULL);
    gc_mark_stacked_objects_all(driver);
 
    /* Run the generic_fields weak pass after the mark fixpoint: mark the vals (fields_obj)
     * of live keys and drain the entries of dead ones.  The per-object rb_mark_generic_ivar
     * is a no-op during a global GC, so this is the only path that marks generic_fields. */
    gc_global_mark_generic_fields(driver);
 
    gc_event_hook(driver, RUBY_INTERNAL_EVENT_GC_END_MARK);
 
    /* step 8 */
    gc_update_weak_references(driver);
 
    /* This cycle's root pass over every Ractor has swept the deleted ractor-local keys out of
     * each storage.  Free the key structs while still inside the barrier (a local GC never
     * can; see rb_ractor_finish_marking). */
    rb_ractor_finish_marking(true);
 
    gc_marking_exit(driver);
 
    /* step 9: sweep every objspace inside the barrier, not lazily.  Dead shareable objects
     * are reclaimed here and emptied pages go back to the pool. */
    if (!compact) {
        for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
            rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
            unsigned int prev_immediate = os->flags.immediate_sweep;
            os->flags.immediate_sweep = TRUE;
            gc_sweep(os);
            os->flags.immediate_sweep = prev_immediate;
        }
    }
    else {
        /* The move -> update-references -> free flow runs as three passes across ALL
         * objspaces, not per objspace: (a) updating references must see every objspace's
         * forwarding (a reference can point at a moved foreign object), and (b) freeing
         * source pages must wait until everyone is updated (or another objspace's update
         * reads a freed T_MOVED).  The read barrier is installed once for all passes. */
        install_handlers();
 
        /* Only the driver records a profile entry for a global GC (gc_start_record), so time
         * only the driver's compaction work.  The move/update/free below runs inside the
         * driver's sweep phase (gc_sweeping_enter/exit); attribute it to GC_COMPACT_WALL_TIME
         * and exclude it from the driver's sweep wall time so the two do not double-count,
         * mirroring the compacting branch of the local gc_sweep(). */
        const bool driver_prof = gc_prof_enabled(driver);
        rb_hrtime_t driver_compact_wall_time = 0;
 
        /* pass 1 (move): relocate every objspace and leave T_MOVED forwarding behind. */
        for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
            rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
            gc_sweeping_enter(os);
            gc_sweep_start(os);        /* mode -> sweeping, order the heap for compaction */
            rb_hrtime_t t0 = (os == driver && driver_prof) ? rb_hrtime_now() : 0;
            gc_compact_relocate(os);   /* mode -> compacting, move */
            if (os == driver && driver_prof) {
                driver_compact_wall_time = rb_hrtime_add(driver_compact_wall_time, elapsed_hrtime_from(t0));
            }
            gc_sweeping_cpu_exit(os);
        }
 
        /* pass 2 (update): all forwarding now exists, so update every objspace's
         * references (cross-objspace ones resolve too); gc_compact_finish also unprotects
         * pages and clears during_compacting.  The move-or-mark decision reads
         * rb_gc_get_objspace()'s during_reference_updating: set it on every objspace. */
        gc_sweeping_cpu_enter(driver);
        for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
            global_objspace->global_gc.objspaces[i]->flags.during_reference_updating = TRUE;
        }
 
        for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
            rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
            for (int h = 0; h < HEAP_COUNT; h++) {
                gc_unprotect_pages(objspace, &heaps[h]);
            }
        }
        rb_gc_before_updating_jit_code();
        gc_sweeping_cpu_exit(driver);
        for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
            rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
            gc_sweeping_cpu_enter(os);
            rb_hrtime_t t0 = (os == driver && driver_prof) ? rb_hrtime_now() : 0;
            gc_compact_finish(os);
            if (os == driver && driver_prof) {
                driver_compact_wall_time = rb_hrtime_add(driver_compact_wall_time, elapsed_hrtime_from(t0));
            }
            gc_sweeping_cpu_exit(os);
        }
        /* The VM-global / weak-table side of the reference update runs once (each objspace's
         * heap side already ran in gc_compact_finish above). */
        {
            gc_sweeping_cpu_enter(driver);
            rb_hrtime_t t0 = driver_prof ? rb_hrtime_now() : 0;
            gc_update_references_global(driver);
            if (driver_prof) {
                driver_compact_wall_time = rb_hrtime_add(driver_compact_wall_time, elapsed_hrtime_from(t0));
            }
            gc_sweeping_cpu_exit(driver);
        }
        gc_sweeping_cpu_enter(driver);
        rb_gc_after_updating_jit_code();
        for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
            global_objspace->global_gc.objspaces[i]->flags.during_reference_updating = FALSE;
            global_objspace->global_gc.objspaces[i]->flags.during_compacting = FALSE;
        }
        global_objspace->global_gc.compacting = false;
        uninstall_handlers();
        gc_sweeping_cpu_exit(driver);
 
        /* Record the driver's compaction time and exclude it from the driver's sweep phase.
         * gc_sweeping_exit(driver) in pass 3 subtracts gc_sweep_excluded_wall_time from the
         * sweep wall time, so this must be set before it runs.  The excluded value is a sum
         * of sub-intervals of the driver's sweep phase, so the subtraction cannot underflow. */
        if (driver_prof) {
            gc_profile_record *const record = gc_prof_record(driver);
            record->gc_compact_wall_time = rb_hrtime_add(record->gc_compact_wall_time,
                    driver_compact_wall_time);
            driver->profile.gc_sweep_excluded_wall_time = rb_hrtime_add(
                    driver->profile.gc_sweep_excluded_wall_time, driver_compact_wall_time);
        }
 
        /* pass 3 (free): page-sweep every objspace, freeing dead objects and the source pages
         * that are now empty.  during_compacting is already cleared, so the sweep treats
         * T_MOVED as usual. */
        for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
            rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
            gc_sweeping_cpu_enter(os);
            gc_sweep_rest(os);
            gc_sweeping_exit(os);
        }
    }
    global_objspace->global_gc.compacting = false;
 
    /* A global GC never calls gc_marks_finish, which budgets heap growth
     * (allocatable_bytes).  An objspace still full after the global sweep (materializing
     * a large received copy, say) has no free pages, no empty pages, budget 0, and its next
     * allocation would hit newobj_refill's "cannot create a new page after a major GC".
     * Give every objspace stuck like that the growth budget gc_marks_finish would. */
    for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
        rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
        if (objspace->heap_pages.allocatable_bytes != 0 || objspace->empty_pages_count != 0) {
            continue;
        }
        bool stuck = false;
        for (int h = 0; h < HEAP_COUNT; h++) {
            if (heaps[h].free_pages == NULL) { stuck = true; break; }
        }
        if (stuck) {
            heap_allocatable_bytes_expand(objspace, NULL, 0,
                    objspace_available_slots(objspace), heaps[0].slot_size);
        }
    }
 
    /* Recount the surviving shareable objects (the sweep already folded the dead ones out of
     * shareable_bits) and reset each trigger limit. */
    for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
        rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
        size_t survivors = 0;
        for (int h = 0; h < HEAP_COUNT; h++) {
            struct heap_page *page = NULL;
            ccan_list_for_each(&heaps[h].pages, page, page_node) {
                if (!page->flags.has_shareable_objects) continue;
                for (int j = 0; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
                    survivors += rb_popcount_intptr(page->shareable_bits[j]);
                }
            }
        }
        objspace->shareable_objects = survivors;
        size_t new_limit = (size_t)(survivors * SHAREABLE_OBJECTS_LIMIT_FACTOR);
        if (new_limit < SHAREABLE_OBJECTS_LIMIT_MIN) new_limit = SHAREABLE_OBJECTS_LIMIT_MIN;
        objspace->shareable_objects_limit = new_limit;
    }
 
    /* Deferred non-thread-safe frees: the world is already stopped here, so reap them
     * without a second barrier.  Uses the driver's snapshot rather than taking its own,
     * which step 10 below still walks. */
    gc_tdata_unsafe_drain_objspaces(global_objspace->global_gc.objspaces,
                                    global_objspace->global_gc.n_objspaces);
 
    /* step 10 */
    for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
        rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
        objspace->flags.during_global_gc = FALSE;
        if (objspace != driver) {
            during_gc = FALSE;
            gc_process_stat_publish(objspace);
        }
    }
 
    /* The unified mark re-established the reachability of absorbed shareable objects, so a
     * single objspace's local mark is trustworthy again (pinning can be skipped until the
     * next absorb). */
    rb_gc_reset_absorbed_since_global_gc();
 
    /* Re-measure the zombies now that the garbage is gone.  Without this, the page
     * trigger above keeps firing on the stale numbers left when a joinable (slotted)
     * zombie retires without any pass merging it. */
    gc_global_refresh_zombie_pages();
 
    /* If the sweep above collected an unjoined Ractor object, ractor_free disowned its
     * zombie_objspaces entry and posted the merge to main as a postponed job; the objspace
     * stays enumerable until main absorbs it at its next safepoint. */
 
    gc_prof_timer_stop(driver);
    gc_exit(driver, event, &lock_lev);
    return true;
}
 
static int
absorb_finalizer_i(st_data_t key, st_data_t val, st_data_t data)
{
    rb_objspace_t *objspace = (rb_objspace_t *)data;
    st_insert(finalizer_table, key, val);
    return ST_CONTINUE;
}
 
static void
gc_make_mid_mark_objspace_absorbable(rb_objspace_t *src)
{
    rb_objspace_t *objspace = src;
    for (int h = 0; h < HEAP_COUNT; h++) {
        struct heap_page *page = NULL;
        ccan_list_for_each(&heaps[h].pages, page, page_node) {
            short stride = page->slot_size;
            uintptr_t p = (uintptr_t)page->start;
            uintptr_t pend = p + page->total_slots * stride;
 
            for (; p < pend; p += stride) {
                VALUE vp = (VALUE)p;
                asan_unpoisoning_object(vp) {
                    switch (RB_BUILTIN_TYPE(vp)) {
                        case T_NONE:
                        case T_ZOMBIE:
                            break;
                        default:
                            RVALUE_AGE_RESET(vp);
                        break;
                    }
                }
            }
        }
        gc_bitmaps_clear(objspace, &heaps[h], false);
    }
    objspace->rgengc.old_objects = 0;
    objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
    objspace->marked_slots = 0;
}
 
/* Merge a dead Ractor's objspace into dst under the VM lock.  src has no owner thread and
 * dst is the calling thread's own objspace (join/value) or main with everyone stopped
 * (global GC), so single-writer holds throughout.  Pages move whole (their bits describe
 * objects, not the objspace), and dst's next collection is forced full to rebuild the
 * generational state. */
static void
objspace_absorb(rb_objspace_t *dst, rb_objspace_t *src)
{
    GC_ASSERT(dst != src);
 
    /* Suppress the cross-objspace verifier checks while the graph is in flux (see
     * global_objspace->during_absorb). */
    const bool prev_absorb = global_objspace->during_absorb;
    global_objspace->during_absorb = true;
 
    /* gc.c dropped src from its zombie table in this same VM-lock section, so src went
     * through zombify and its pages are part of the total. */
    GC_ASSERT(src->zombie_pages != 0);
    GC_ASSERT(global_objspace->zombie_total_pages >= src->zombie_pages);
    global_objspace->zombie_total_pages -= src->zombie_pages;
    src->zombie_pages = 0;
 
    /* Settle dst first: adding pages under a walking lazy-sweep cursor, or into a
     * half-marked incremental heap, would sweep the merged pages with src's stale mark
     * bits and free live objects. */
    gc_rest(dst);
 
    /* Settle src: no incremental marking, lazy sweep or in-progress allocation page. */
    {
        rb_objspace_t *objspace = src;
        /* A zombie parked by fork (rb_ractor_terminate_atfork retires the objspace
         * without the retire GC) can be mid-incremental-mark, and that mark cannot be
         * finished. Its owner's threads did not survive the fork, so mark_roots would
         * scan the absorbing Ractor's roots instead of src's. Drop the cycle. */
        if (is_incremental_marking(objspace)) {
            gc_abort_incremental_marking(objspace);
            gc_make_mid_mark_objspace_absorbable(objspace);
        }
        /* rb_ractor_terminate_atfork retires without the retire GC, so src may arrive
         * with the flag unset; the sweep gc_rest may finish is the last one it gets. */
        objspace->flags.ownerless = 1;
        gc_rest(src); // if mid-sweep
        heap_alloc_state_clear(objspace);
        /* gc_sweep_finish leaves swept pages "pooled" for a coming incremental mark; src
         * never runs one (it is about to be merged), so return them to its free list now,
         * restoring the pooled_pages == NULL the page merge below assumes (mirrors
         * gc_start_global step 3). */
        for (int h = 0; h < HEAP_COUNT; h++) {
            heap_move_pooled_pages_to_free_pages(&heaps[h]);
        }
 
        /* Backstop for pages a sweep that ran before `ownerless` was set left attached */
        for (int h = 0; h < HEAP_COUNT; h++) {
            rb_heap_t *heap = &heaps[h];
            struct heap_page **link = &heap->free_pages;
            while (*link) {
                struct heap_page *page = *link;
                if (page->free_slots == page->total_slots) {
                    *link = page->free_next;
                }
                else {
                    link = &page->free_next;
                }
            }
 
            struct heap_page *page, *next;
            ccan_list_for_each_safe(&heap->pages, page, next, page_node) {
                if (page->free_slots == page->total_slots) {
                    GC_ASSERT(page->final_slots == 0);
                    heap_page_detach_empty(objspace, heap, page);
                }
            }
        }
    }
 
    /* From here the merge must not run dst's GC: the finalizer st_insert below can cross
     * the malloc-accounting threshold, and a GC then would sweep src's detached finalizer
     * procs, reachable only from this C frame, into dangling VALUEs.  Page/darray moves
     * allocate nothing (_without_gc), so disabling costs nothing and makes the splice
     * atomic.  (The two settles above deliberately collect: they stay outside.) */
    const bool dst_gc_was_enabled = rb_gc_impl_gc_enabled_p(dst);
    if (dst_gc_was_enabled) rb_gc_impl_gc_disable(dst, false);
 
    /* Hand over the pages size pool by size pool.  ("heaps" is a macro over the local
     * objspace, so the arrays are taken through scoped locals.) */
    rb_heap_t *dst_heaps;
    rb_heap_t *src_heaps;
    {
        rb_objspace_t *objspace = dst;
        dst_heaps = heaps;
    }
    {
        rb_objspace_t *objspace = src;
        src_heaps = heaps;
    }
    for (int h = 0; h < HEAP_COUNT; h++) {
        rb_heap_t *dheap = &dst_heaps[h];
        rb_heap_t *sheap = &src_heaps[h];
        struct heap_page *page = NULL;
 
        GC_ASSERT(sheap->sweeping_page == NULL);
        GC_ASSERT(sheap->pooled_pages == NULL);
 
        ccan_list_for_each(&sheap->pages, page, page_node) {
            page->objspace = dst;
            page->heap = dheap;
        }
        ccan_list_append_list(&dheap->pages, &sheap->pages);
 
        /* Append the free-page chain to the tail. */
        if (sheap->free_pages) {
            struct heap_page **tail = &dheap->free_pages;
            while (*tail) tail = &(*tail)->free_next;
            *tail = sheap->free_pages;
            sheap->free_pages = NULL;
        }
 
        dheap->total_pages += sheap->total_pages;
        dheap->total_slots += sheap->total_slots;
        dheap->total_allocated_pages += sheap->total_allocated_pages;
        dheap->total_allocated_objects += sheap->total_allocated_objects;
        dheap->total_freed_objects += sheap->total_freed_objects;
        dheap->final_slots_count += sheap->final_slots_count;
    }
 
    /* The objspace-wide page bookkeeping. */
    {
        rb_objspace_t *objspace = dst; /* for the heap_pages_* macros */
        struct heap_page *page = NULL;
        size_t srcn = rb_darray_size(src->heap_pages.sorted);
        for (size_t i = 0; i < srcn; i++) {
            page = rb_darray_get(src->heap_pages.sorted, i);
            /* Residents of the empty pool (no live objects) are returned to page_pool rather
             * than inherited; dst's allocation demand is cheaply met from the shared pool's
             * free list. */
            if (heap_page_in_global_empty_pages_pool(src, page)) {
                heap_page_free(src, page);
                continue;
            }
            uintptr_t body = (uintptr_t)page->body;
            uintptr_t start = body + sizeof(struct heap_page_header);
            uintptr_t end = body + HEAP_PAGE_SIZE;
 
            /* Keep the array ordered by page BODY address: heap_page_for_ptr bsearches
             * body ranges, and a detached empty page has start == 0, so ordering by
             * page->start would miss live pages (a global GC would then fail to mark a
             * registered root and sweep it). */
            size_t lo = 0;
            size_t hi = rb_darray_size(objspace->heap_pages.sorted);
            while (lo < hi) {
                size_t mid = (lo + hi) / 2;
                struct heap_page *mid_page = rb_darray_get(objspace->heap_pages.sorted, mid);
                if ((uintptr_t)mid_page->body < body) lo = mid + 1;
                else hi = mid;
            }
            rb_darray_insert_without_gc(&objspace->heap_pages.sorted, hi, page);
 
            if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
            if (heap_pages_himem < end) heap_pages_himem = end;
        }
        objspace->heap_pages.allocated_pages += src->heap_pages.allocated_pages;
        objspace->heap_pages.freed_pages += src->heap_pages.freed_pages;
        rb_darray_free_without_gc(src->heap_pages.sorted);
        src->heap_pages.sorted = NULL;
        /* The empty_pages chain's structs were freed in the loop above. */
        src->empty_pages = NULL;
        src->empty_pages_count = 0;
    }
 
    /* Finalizers: move the table's entries, and the dead Ractor's deferred zombies are run
     * by dst's thread from now on. */
    {
        st_table *src_finalizers;
        {
            rb_objspace_t *objspace = src;
            src_finalizers = finalizer_table;
            finalizer_table = NULL;
        }
        if (src_finalizers) {
            rb_objspace_t *objspace = dst;
            if (finalizer_table == NULL) {
                finalizer_table = src_finalizers;
            }
            else {
                st_foreach(src_finalizers, absorb_finalizer_i, (st_data_t)dst);
                st_free_table(src_finalizers);
            }
        }
    }
    {
        VALUE src_deferred = RUBY_ATOMIC_VALUE_EXCHANGE(src->heap_pages.deferred_final, 0);
        if (src_deferred) {
            VALUE tail_obj = src_deferred;
            rb_asan_unpoison_object(tail_obj, false);
            while (RZOMBIE(tail_obj)->next) {
                VALUE next_obj = RZOMBIE(tail_obj)->next;
                rb_asan_poison_object(tail_obj);
                tail_obj = next_obj;
                rb_asan_unpoison_object(tail_obj, false);
            }
            VALUE prev;
            do {
                prev = dst->heap_pages.deferred_final;
                RZOMBIE(tail_obj)->next = prev;
            } while (RUBY_ATOMIC_VALUE_CAS(dst->heap_pages.deferred_final, prev, src_deferred) != prev);
            rb_asan_poison_object(tail_obj);
            /* No owner was left to run these zombies (register's owner walk misses a dead
             * Ractor).  dst runs this merge, so schedule dst's job here; otherwise they wait
             * until dst's next GC. */
            rb_postponed_job_trigger(dst->finalize_deferred_pjob);
        }
    }
    if (src->tdata_unsafe_free_chunk) {
        gc_tdata_unsafe_free_publish(src);
    }
    if (tdata_deferred_free_count_load() >= TDATA_DEFERRED_FREE_THRESHOLD) {
        gc_tdata_deferred_free_trigger(dst);
    }
 
    /* Counters inherited by dst. */
    dst->rgengc.old_objects += src->rgengc.old_objects;
    dst->rgengc.uncollectible_wb_unprotected_objects += src->rgengc.uncollectible_wb_unprotected_objects;
    dst->shareable_objects += src->shareable_objects;
 
    /* Merged pages carry src's mark/age state, so dst rebuilds its view at the next
     * collection. Note that this can be worked around by calling `GC.config(rgengc_allow_full_mark: false)`,
     * so the absorbed heap should be in a state where a minor GC would also work correctly.
     */
    dst->rgengc.need_major_gc |= GPR_FLAG_MAJOR_BY_FORCE;
 
    /* src's outstanding malloc pressure moves with the xmalloc'd buffers.  Later frees are
     * charged to dst, so without this transfer dst underestimates its own heap and delays
     * GCs.  dst is live, so take its counter lock where gc_counter_add is not atomic. */
    {
        int64_t inc = gc_malloc_counters_increase(src, &src->malloc_counters.counters);
#if RGENGC_ESTIMATE_OLDMALLOC
        size_t oldmalloc_increase = gc_oldmalloc_increase_load(src);
#endif
        MALLOC_COUNTERS_LOCK(dst);
        if (inc > 0) gc_counter_add(&dst->malloc_counters.counters.malloc, (size_t)inc);
#if RGENGC_ESTIMATE_OLDMALLOC
        if (oldmalloc_increase > 0) RUBY_ATOMIC_SIZE_ADD(dst->malloc_counters.oldmalloc_increase, oldmalloc_increase);
#endif
        MALLOC_COUNTERS_UNLOCK(dst);
    }
 
    {
        struct gc_process_stat_snapshot final_snap;
        gc_process_stat_capture(src, &final_snap);
        gc_process_stat_add(&global_objspace->process_stat_archive, &final_snap);
    }
    rb_native_mutex_destroy(&src->process_stat.lock);
 
    /* Free the shell (as rb_gc_impl_objspace_free does). */
    free(src->profile.records);
    free_stack_chunks(&src->mark_stack);
    mark_stack_free_cache(&src->mark_stack);
    GC_ASSERT(rb_darray_size(src->weak_references) == 0);
    rb_darray_free_without_gc(src->weak_references);
#ifdef MALLOC_COUNTERS_NEED_LOCK
    rb_native_mutex_destroy(&src->malloc_counters.lock);
#endif
    free(src);
 
    if (dst_gc_was_enabled) rb_gc_impl_gc_enable(dst);
 
    /* Return the empty pages inheritance piled up in dst (mostly from the dead Ractor's
     * teardown material) to the pool with no budget.  An empty page is by definition safe to
     * release, and re-acquiring one from the pool is cheap. */
    {
        rb_objspace_t *objspace = dst;
        heap_pages_freeable_pages = objspace->empty_pages_count;
        heap_pages_free_unused_pages(objspace);
    }
 
    global_objspace->during_absorb = prev_absorb;
}
 
void
rb_gc_impl_objspace_absorb(void *dst_ptr, void *src_ptr)
{
    objspace_absorb(dst_ptr, src_ptr);
}
 
void
rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact, bool global)
{
    rb_objspace_t *objspace = objspace_ptr;
    unsigned int reason = (GPR_FLAG_FULL_MARK |
                           GPR_FLAG_IMMEDIATE_MARK |
                           GPR_FLAG_IMMEDIATE_SWEEP |
                           GPR_FLAG_METHOD);
 
    int full_marking_p = gc_config_full_mark_val;
    gc_config_full_mark_set(TRUE);
 
    /* For now, compact implies full mark / sweep, so ignore other flags */
    if (compact) {
        GC_ASSERT(GC_COMPACTION_SUPPORTED);
 
        reason |= GPR_FLAG_COMPACT;
        if (!rb_gc_single_objspace_p()) {
            global = true;
        }
    }
    else {
        if (!full_mark)       reason &= ~GPR_FLAG_FULL_MARK;
        if (!immediate_mark)  reason &= ~GPR_FLAG_IMMEDIATE_MARK;
        if (!immediate_sweep) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
    }
 
    if ((reason & (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP)) !=
                  (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP)) {
        global = false;
    }
 
    if (global && !rb_gc_single_objspace_p()) {
        /* A mid-cycle driver is settled by gc_start_global itself: it aborts the partial
         * mark and finishes the lazy sweep, so the dead slots are T_NONE before the
         * unified conservative root scan. */
        gc_start_global(objspace, reason, compact || ruby_enable_autocompact, false);
    }
    else {
        garbage_collect(objspace, reason);
    }
 
    gc_finalize_deferred(objspace);
    /* An explicit GC.start is expected to reclaim immediately, so run the deferred non-thread-safe
     * frees synchronously instead of leaving them to gc_sweep_finish's postponed job. */
    if (tdata_deferred_free_count_load() > 0 && rb_gc_single_objspace_p()) {
        gc_tdata_unsafe_drain();
    }
    gc_config_full_mark_set(full_marking_p);
}
 
void
rb_gc_impl_prepare_heap(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    rb_gc_impl_each_objects(objspace, gc_set_candidate_object_i, objspace_ptr);
 
    double orig_max_free_slots = gc_params.heap_free_slots_max_ratio;
    /* Ensure that all empty pages are moved onto empty_pages. */
    gc_params.heap_free_slots_max_ratio = 0.0;
    rb_gc_impl_start(objspace, true, true, true, true, true);
    gc_params.heap_free_slots_max_ratio = orig_max_free_slots;
 
    objspace->heap_pages.allocatable_bytes = 0;
    heap_pages_freeable_pages = objspace->empty_pages_count;
    heap_pages_free_unused_pages(objspace_ptr);
    GC_ASSERT(heap_pages_freeable_pages == 0);
    GC_ASSERT(objspace->empty_pages_count == 0);
 
    // Process.warmup is meant to be called at the end of the boot sequence, which is commonly allocation
    // heavy and result in GC limits raising significantly, but it's not indicative of the limits needed
    // for runtime.
    // Recompute the allocatable_bytes limit based on `gc_params.heap_init_bytes`.
    GC_ASSERT(objspace->heap_pages.allocatable_bytes == 0);
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        heap_allocatable_bytes_expand(objspace, heap, heap->empty_slots, heap->total_slots, heap->slot_size);
    }
 
#if defined(HAVE_MALLOC_TRIM) && !defined(RUBY_ALTERNATIVE_MALLOC_HEADER)
    malloc_trim(0);
#endif
}
 
static int
gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
{
    GC_ASSERT(!SPECIAL_CONST_P(obj));
 
    switch (BUILTIN_TYPE(obj)) {
      case T_NONE:
      case T_MOVED:
      case T_ZOMBIE:
        return FALSE;
      case T_SYMBOL:
      case T_STRING:
      case T_OBJECT:
      case T_FLOAT:
      case T_IMEMO:
      case T_ARRAY:
      case T_BIGNUM:
      case T_ICLASS:
      case T_MODULE:
      case T_REGEXP:
      case T_DATA:
      case T_MATCH:
      case T_STRUCT:
      case T_HASH:
      case T_FILE:
      case T_COMPLEX:
      case T_RATIONAL:
      case T_NODE:
      case T_CLASS:
        if (FL_TEST_RAW(obj, FL_FINALIZE)) {
            /* The finalizer table is a numtable. It looks up objects by address.
             * We can't mark the keys in the finalizer table because that would
             * prevent the objects from being collected.  This check prevents
             * objects that are keys in the finalizer table from being moved
             * without directly pinning them. */
            GC_ASSERT(st_is_member(finalizer_table, obj));
 
            return FALSE;
        }
        GC_ASSERT(RVALUE_MARKED(objspace, obj));
        GC_ASSERT(!RVALUE_PINNED(objspace, obj));
 
        return TRUE;
 
      default:
        rb_bug("gc_is_moveable_obj: unreachable (%d)", (int)BUILTIN_TYPE(obj));
        break;
    }
 
    return FALSE;
}
 
void rb_mv_generic_ivar(VALUE src, VALUE dst);
 
static VALUE
gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, struct heap_page *src_page, struct heap_page *dest_page)
{
    size_t src_slot_size = src_page->slot_size;
    size_t slot_size = dest_page->slot_size;
 
    int marked;
    int wb_unprotected;
    int uncollectible;
    int age;
 
    gc_report(4, objspace, "Moving object: %p -> %p\n", (void *)src, (void *)dest);
 
    GC_ASSERT(BUILTIN_TYPE(src) != T_NONE);
    GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest));
 
    GC_ASSERT(!RVALUE_MARKING(objspace, src));
 
    /* Save off bits for current object. */
    marked = RVALUE_MARKED(objspace, src);
    wb_unprotected = RVALUE_WB_UNPROTECTED(objspace, src);
    uncollectible = RVALUE_UNCOLLECTIBLE(objspace, src);
    bool remembered = RVALUE_REMEMBERED(objspace, src);
    /* Pin bits travel with the object.  Losing one during single-objspace compaction would
     * silently unpin it once the process goes multi-objspace, letting a local GC free a method
     * entry or shref target that another Ractor references. */
    bool shareable = MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(src), src) != 0;
    bool shref = MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(src), src) != 0;
    age = RVALUE_AGE_GET(src);
 
    /* Clear bits for eventual T_MOVED */
    CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(src), src);
    CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(src), src);
    CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(src), src);
    CLEAR_IN_BITMAP(GET_HEAP_PAGE(src)->remembered_bits, src);
    CLEAR_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(src), src);
    CLEAR_IN_BITMAP(GET_HEAP_SHREF_BITS(src), src);
 
    /* Move the object */
    memcpy((void *)dest, (void *)src, MIN(src_slot_size, slot_size));
 
    if (src_slot_size != slot_size) {
        rb_gc_obj_changed_slot_size(dest, slot_size - RVALUE_OVERHEAD);
    }
 
    if (RVALUE_OVERHEAD > 0) {
        void *dest_overhead = (void *)(((uintptr_t)dest) + slot_size - RVALUE_OVERHEAD);
        void *src_overhead = (void *)(((uintptr_t)src) + src_slot_size - RVALUE_OVERHEAD);
 
        memcpy(dest_overhead, src_overhead, RVALUE_OVERHEAD);
    }
 
    memset((void *)src, 0, src_slot_size);
    RVALUE_AGE_SET_BITMAP(src, 0);
 
    /* Set bits for object in new location */
    if (remembered) {
        MARK_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
    }
    else {
        CLEAR_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
    }
 
    if (marked) {
        MARK_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
    }
    else {
        CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
    }
 
    if (wb_unprotected) {
        MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
    }
    else {
        CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
    }
 
    if (uncollectible) {
        MARK_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
    }
    else {
        CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
    }
 
    if (shareable) {
        MARK_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(dest), dest);
        GET_HEAP_PAGE(dest)->flags.has_shareable_objects = TRUE;
    }
    else {
        CLEAR_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(dest), dest);
    }
 
    if (shref) {
        MARK_IN_BITMAP(GET_HEAP_SHREF_BITS(dest), dest);
        GET_HEAP_PAGE(dest)->flags.has_shref_objects = TRUE;
    }
    else {
        CLEAR_IN_BITMAP(GET_HEAP_SHREF_BITS(dest), dest);
    }
 
    RVALUE_AGE_SET(dest, age);
 
    /* A re-embedded object (rb_gc_obj_changed_slot_size) references its
     * former fields_obj's contents directly; the write-barrier history
     * lived on the discarded fields_obj, so remember the object. */
    if (src_slot_size != slot_size && age >= RVALUE_OLD_AGE && !remembered) {
        rgengc_remember(objspace, dest);
    }
 
    /* Assign forwarding address */
    RMOVED(src)->flags = T_MOVED;
    RMOVED(src)->dummy = Qundef;
    RMOVED(src)->destination = dest;
    GC_ASSERT(BUILTIN_TYPE(dest) != T_NONE);
 
    GET_HEAP_PAGE(src)->heap->total_freed_objects++;
    GET_HEAP_PAGE(dest)->heap->total_allocated_objects++;
 
    return src;
}
 
#if GC_CAN_COMPILE_COMPACTION
static int
compare_pinned_slots(const void *left, const void *right, void *dummy)
{
    struct heap_page *left_page;
    struct heap_page *right_page;
 
    left_page = *(struct heap_page * const *)left;
    right_page = *(struct heap_page * const *)right;
 
    return left_page->pinned_slots - right_page->pinned_slots;
}
 
static int
compare_free_slots(const void *left, const void *right, void *dummy)
{
    struct heap_page *left_page;
    struct heap_page *right_page;
 
    left_page = *(struct heap_page * const *)left;
    right_page = *(struct heap_page * const *)right;
 
    return left_page->free_slots - right_page->free_slots;
}
 
static void
gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func)
{
    for (int j = 0; j < HEAP_COUNT; j++) {
        rb_heap_t *heap = &heaps[j];
 
        size_t total_pages = heap->total_pages;
        size_t size = rb_size_mul_or_raise(total_pages, sizeof(struct heap_page *), rb_eRuntimeError);
        struct heap_page *page = 0, **page_list = malloc(size);
        size_t i = 0;
 
        heap->free_pages = NULL;
        ccan_list_for_each(&heap->pages, page, page_node) {
            page_list[i++] = page;
            GC_ASSERT(page);
        }
 
        GC_ASSERT((size_t)i == total_pages);
 
        /* Sort the heap so "filled pages" are first. `heap_add_page` adds to the
         * head of the list, so empty pages will end up at the start of the heap */
        ruby_qsort(page_list, total_pages, sizeof(struct heap_page *), compare_func, NULL);
 
        /* Reset the eden heap */
        ccan_list_head_init(&heap->pages);
 
        for (i = 0; i < total_pages; i++) {
            ccan_list_add(&heap->pages, &page_list[i]->page_node);
            if (page_list[i]->free_slots != 0) {
                heap_add_freepage(heap, page_list[i]);
            }
        }
 
        free(page_list);
    }
}
#endif
 
void
rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj)
{
    /* no-op */
}
 
bool
rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj)
{
    return gc_object_moved_p(objspace_ptr, obj);
}
 
static int
gc_ref_update(void *vstart, void *vend, size_t stride, rb_objspace_t *objspace, struct heap_page *page)
{
    VALUE v = (VALUE)vstart;
 
    page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
    page->flags.has_remembered_objects = FALSE;
 
    /* For each object on the page */
    for (; v != (VALUE)vend; v += stride) {
        asan_unpoisoning_object(v) {
            switch (BUILTIN_TYPE(v)) {
              case T_NONE:
              case T_MOVED:
              case T_ZOMBIE:
                break;
              default:
                if (RVALUE_WB_UNPROTECTED(objspace, v)) {
                    page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
                }
                if (RVALUE_REMEMBERED(objspace, v)) {
                    page->flags.has_remembered_objects = TRUE;
                }
                if (page->flags.before_sweep) {
                    if (RVALUE_MARKED(objspace, v)) {
                        rb_gc_update_object_references(objspace, v);
                    }
                }
                else {
                    rb_gc_update_object_references(objspace, v);
                }
            }
        }
    }
 
    return 0;
}
 
static int
gc_update_references_weak_table_i(VALUE obj, void *data)
{
    int ret;
    asan_unpoisoning_object(obj) {
        ret = BUILTIN_TYPE(obj) == T_MOVED ? ST_REPLACE : ST_CONTINUE;
    }
    return ret;
}
 
static int
gc_update_references_weak_table_replace_i(VALUE *obj, void *data)
{
    rb_gc_update_moved(obj);
 
    return ST_CONTINUE;
}
 
/* The per-objspace side of the reference update: walk this objspace's heap objects and rewrite
 * moved references (following T_MOVED forwarding across objspaces).  A compacting global GC
 * runs this for every objspace. */
static void
gc_update_references_heap(rb_objspace_t *objspace)
{
    struct heap_page *page = NULL;
 
    for (int i = 0; i < HEAP_COUNT; i++) {
        bool should_set_mark_bits = TRUE;
        rb_heap_t *heap = &heaps[i];
 
        ccan_list_for_each(&heap->pages, page, page_node) {
            uintptr_t start = (uintptr_t)page->start;
            uintptr_t end = start + (page->total_slots * heap->slot_size);
 
            gc_ref_update((void *)start, (void *)end, heap->slot_size, objspace, page);
            if (page == heap->sweeping_page) {
                should_set_mark_bits = FALSE;
            }
            if (should_set_mark_bits) {
                gc_setup_mark_bits(page);
            }
        }
    }
}
 
/* The VM-global side of the reference update (finalizer table, every Ractor's VM roots,
 * weak tables).  Process-wide, so a compacting global GC runs it once after every heap
 * side: rb_gc_update_vm_references and the weak tables' mark_and_move are not idempotent. */
static void
gc_update_references_global(rb_objspace_t *objspace)
{
    gc_update_table_refs(finalizer_table);
 
    rb_gc_update_vm_references((void *)objspace);
 
    for (int table = 0; table < RB_GC_VM_WEAK_TABLE_COUNT; table++) {
        rb_gc_vm_weak_table_foreach(
            gc_update_references_weak_table_i,
            gc_update_references_weak_table_replace_i,
            NULL,
            false,
            table
        );
    }
}
 
static void
gc_update_references(rb_objspace_t *objspace)
{
    objspace->flags.during_reference_updating = true;
 
    rb_gc_before_updating_jit_code();
 
    gc_update_references_heap(objspace);
    gc_update_references_global(objspace);
 
    rb_gc_after_updating_jit_code();
 
    objspace->flags.during_reference_updating = false;
}
 
#if GC_CAN_COMPILE_COMPACTION
static void
root_obj_check_moved_i(const char *category, VALUE obj, void *data)
{
    rb_objspace_t *objspace = data;
 
    if (gc_object_moved_p(objspace, obj)) {
        rb_bug("ROOT %s points to MOVED: %p -> %s", category, (void *)obj, rb_obj_info(rb_gc_impl_location(objspace, obj)));
    }
}
 
static void
reachable_object_check_moved_i(VALUE ref, void *data)
{
    VALUE parent = (VALUE)data;
    if (gc_object_moved_p(rb_gc_get_objspace(), ref)) {
        rb_bug("Object %s points to MOVED: %p -> %s", rb_obj_info(parent), (void *)ref, rb_obj_info(rb_gc_impl_location(rb_gc_get_objspace(), ref)));
    }
}
 
static int
heap_check_moved_i(void *vstart, void *vend, size_t stride, void *data)
{
    rb_objspace_t *objspace = data;
 
    VALUE v = (VALUE)vstart;
    for (; v != (VALUE)vend; v += stride) {
        if (gc_object_moved_p(objspace, v)) {
            /* Moved object still on the heap, something may have a reference. */
        }
        else {
            asan_unpoisoning_object(v) {
                switch (BUILTIN_TYPE(v)) {
                  case T_NONE:
                  case T_ZOMBIE:
                    break;
                  default:
                    if (!rb_gc_impl_garbage_object_p(objspace, v)) {
                        rb_objspace_reachable_objects_from_unlocked(v, reachable_object_check_moved_i, (void *)v);
                    }
                }
            }
        }
    }
 
    return 0;
}
#endif
 
bool
rb_gc_impl_during_gc_p(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    return during_gc;
}
 
#if RGENGC_PROFILE >= 2
 
static const char*
type_name(int type, VALUE obj)
{
    switch ((enum ruby_value_type)type) {
      case RUBY_T_NONE:     return "T_NONE";
      case RUBY_T_OBJECT:   return "T_OBJECT";
      case RUBY_T_CLASS:    return "T_CLASS";
      case RUBY_T_MODULE:   return "T_MODULE";
      case RUBY_T_FLOAT:    return "T_FLOAT";
      case RUBY_T_STRING:   return "T_STRING";
      case RUBY_T_REGEXP:   return "T_REGEXP";
      case RUBY_T_ARRAY:    return "T_ARRAY";
      case RUBY_T_HASH:     return "T_HASH";
      case RUBY_T_STRUCT:   return "T_STRUCT";
      case RUBY_T_BIGNUM:   return "T_BIGNUM";
      case RUBY_T_FILE:     return "T_FILE";
      case RUBY_T_DATA:     return "T_DATA";
      case RUBY_T_MATCH:    return "T_MATCH";
      case RUBY_T_COMPLEX:  return "T_COMPLEX";
      case RUBY_T_RATIONAL: return "T_RATIONAL";
      case RUBY_T_NIL:      return "T_NIL";
      case RUBY_T_TRUE:     return "T_TRUE";
      case RUBY_T_FALSE:    return "T_FALSE";
      case RUBY_T_SYMBOL:   return "T_SYMBOL";
      case RUBY_T_FIXNUM:   return "T_FIXNUM";
      case RUBY_T_UNDEF:    return "T_UNDEF";
      case RUBY_T_IMEMO:    return "T_IMEMO";
      case RUBY_T_NODE:     return "T_NODE";
      case RUBY_T_ICLASS:   return "T_ICLASS";
      case RUBY_T_ZOMBIE:   return "T_ZOMBIE";
      case RUBY_T_MOVED:    return "T_MOVED";
      default:              return "unknown";
    }
}
 
static void
gc_count_add_each_types(VALUE hash, const char *name, const size_t *types)
{
    VALUE result = rb_hash_new_capa(T_MASK);
    int i;
    for (i=0; i<T_MASK; i++) {
        const char *type = type_name(i, 0);
        rb_hash_aset(result, ID2SYM(rb_intern(type)), SIZET2NUM(types[i]));
    }
    rb_hash_aset(hash, ID2SYM(rb_intern(name)), result);
}
#endif
 
size_t
rb_gc_impl_gc_count(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    return objspace->profile.count;
}
 
/* Filled by setup_gc_latest_gc_info_symbols() at boot, not on first use. */
static VALUE sym_major_by, sym_gc_by, sym_immediate_sweep, sym_have_finalizer, sym_state, sym_need_major_by;
static VALUE sym_nofree, sym_oldgen, sym_shady, sym_force, sym_stress;
#if RGENGC_ESTIMATE_OLDMALLOC
static VALUE sym_oldmalloc;
#endif
static VALUE sym_newobj, sym_malloc, sym_method, sym_capi;
static VALUE sym_none, sym_marking, sym_sweeping;
static VALUE sym_weak_references_count;
 
static void
setup_gc_latest_gc_info_symbols(void)
{
#define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
    S(major_by);
    S(gc_by);
    S(immediate_sweep);
    S(have_finalizer);
    S(state);
    S(need_major_by);
 
    S(stress);
    S(nofree);
    S(oldgen);
    S(shady);
    S(force);
#if RGENGC_ESTIMATE_OLDMALLOC
    S(oldmalloc);
#endif
    S(newobj);
    S(malloc);
    S(method);
    S(capi);
 
    S(none);
    S(marking);
    S(sweeping);
 
    S(weak_references_count);
#undef S
}
 
static VALUE
gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned int orig_flags)
{
    VALUE hash = Qnil, key = Qnil;
    VALUE major_by, need_major_by;
    unsigned int flags = orig_flags ? orig_flags : objspace->profile.latest_gc_info;
 
    if (SYMBOL_P(hash_or_key)) {
        key = hash_or_key;
    }
    else if (RB_TYPE_P(hash_or_key, T_HASH)) {
        hash = hash_or_key;
    }
    else {
        rb_bug("gc_info_decode: non-hash or symbol given");
    }
 
#define SET(name, attr) \
    if (key == sym_##name) \
        return (attr); \
    else if (hash != Qnil) \
        rb_hash_aset(hash, sym_##name, (attr));
 
    major_by =
      (flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
      (flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
      (flags & GPR_FLAG_MAJOR_BY_SHADY)  ? sym_shady :
      (flags & GPR_FLAG_MAJOR_BY_FORCE)  ? sym_force :
#if RGENGC_ESTIMATE_OLDMALLOC
      (flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
#endif
      Qnil;
    SET(major_by, major_by);
 
    if (orig_flags == 0) { /* set need_major_by only if flags not set explicitly */
        unsigned int need_major_flags = gc_needs_major_flags;
        need_major_by =
            (need_major_flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
            (need_major_flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
            (need_major_flags & GPR_FLAG_MAJOR_BY_SHADY)  ? sym_shady :
            (need_major_flags & GPR_FLAG_MAJOR_BY_FORCE)  ? sym_force :
#if RGENGC_ESTIMATE_OLDMALLOC
            (need_major_flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
#endif
            Qnil;
        SET(need_major_by, need_major_by);
    }
 
    SET(gc_by,
        (flags & GPR_FLAG_NEWOBJ) ? sym_newobj :
        (flags & GPR_FLAG_MALLOC) ? sym_malloc :
        (flags & GPR_FLAG_METHOD) ? sym_method :
        (flags & GPR_FLAG_CAPI)   ? sym_capi :
        (flags & GPR_FLAG_STRESS) ? sym_stress :
        Qnil
    );
 
    SET(have_finalizer, (flags & GPR_FLAG_HAVE_FINALIZE) ? Qtrue : Qfalse);
    SET(immediate_sweep, (flags & GPR_FLAG_IMMEDIATE_SWEEP) ? Qtrue : Qfalse);
 
    if (orig_flags == 0) {
        SET(state, gc_mode(objspace) == gc_mode_none ? sym_none :
                   gc_mode(objspace) == gc_mode_marking ? sym_marking : sym_sweeping);
    }
 
    SET(weak_references_count, LONG2FIX(objspace->profile.weak_references_count));
#undef SET
 
    if (!NIL_P(key)) {
        // Matched key should return above
        return Qundef;
    }
 
    return hash;
}
 
VALUE
rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE key)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    return gc_info_decode(objspace, key, 0);
}
 
 
enum gc_stat_sym {
    gc_stat_sym_count,
    gc_stat_sym_time,
    gc_stat_sym_marking_time,
    gc_stat_sym_sweeping_time,
    gc_stat_sym_heap_allocated_pages,
    gc_stat_sym_heap_empty_pages,
    gc_stat_sym_heap_allocatable_bytes,
    gc_stat_sym_heap_available_slots,
    gc_stat_sym_heap_live_slots,
    gc_stat_sym_heap_free_slots,
    gc_stat_sym_heap_final_slots,
    gc_stat_sym_heap_marked_slots,
    gc_stat_sym_heap_eden_pages,
    gc_stat_sym_total_allocated_pages,
    gc_stat_sym_total_freed_pages,
    gc_stat_sym_total_allocated_objects,
    gc_stat_sym_total_freed_objects,
    gc_stat_sym_total_malloc_bytes,
    gc_stat_sym_total_free_bytes,
    gc_stat_sym_malloc_increase_bytes,
    gc_stat_sym_malloc_increase_bytes_limit,
    gc_stat_sym_minor_gc_count,
    gc_stat_sym_major_gc_count,
    gc_stat_sym_global_gc_count,
    gc_stat_sym_global_gc_invocation_count,
    gc_stat_sym_compact_count,
    gc_stat_sym_read_barrier_faults,
    gc_stat_sym_total_moved_objects,
    gc_stat_sym_remembered_wb_unprotected_objects,
    gc_stat_sym_remembered_wb_unprotected_objects_limit,
    gc_stat_sym_old_objects,
    gc_stat_sym_old_objects_limit,
#if RGENGC_ESTIMATE_OLDMALLOC
    gc_stat_sym_oldmalloc_increase_bytes,
    gc_stat_sym_oldmalloc_increase_bytes_limit,
#endif
#if RGENGC_PROFILE
    gc_stat_sym_total_generated_normal_object_count,
    gc_stat_sym_total_generated_shady_object_count,
    gc_stat_sym_total_shade_operation_count,
    gc_stat_sym_total_promoted_count,
    gc_stat_sym_total_remembered_normal_object_count,
    gc_stat_sym_total_remembered_shady_object_count,
#endif
    gc_stat_sym_page_pool_arenas,
    gc_stat_sym_page_pool_arenas_freed,
    gc_stat_sym_page_pool_total_pages,
    gc_stat_sym_page_pool_discarded_pages,
    gc_stat_sym_last
};
 
static VALUE gc_stat_symbols[gc_stat_sym_last];
 
static void
setup_gc_stat_symbols(void)
{
#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s))
    S(count);
    S(time);
    S(marking_time),
    S(sweeping_time),
    S(heap_allocated_pages);
    S(heap_empty_pages);
    S(heap_allocatable_bytes);
    S(heap_available_slots);
    S(heap_live_slots);
    S(heap_free_slots);
    S(heap_final_slots);
    S(heap_marked_slots);
    S(heap_eden_pages);
    S(total_allocated_pages);
    S(total_freed_pages);
    S(total_allocated_objects);
    S(total_freed_objects);
    S(total_malloc_bytes);
    S(total_free_bytes);
    S(malloc_increase_bytes);
    S(malloc_increase_bytes_limit);
    S(minor_gc_count);
    S(major_gc_count);
    S(global_gc_count);
    S(global_gc_invocation_count);
    S(compact_count);
    S(read_barrier_faults);
    S(total_moved_objects);
    S(remembered_wb_unprotected_objects);
    S(remembered_wb_unprotected_objects_limit);
    S(old_objects);
    S(old_objects_limit);
#if RGENGC_ESTIMATE_OLDMALLOC
    S(oldmalloc_increase_bytes);
    S(oldmalloc_increase_bytes_limit);
#endif
#if RGENGC_PROFILE
    S(total_generated_normal_object_count);
    S(total_generated_shady_object_count);
    S(total_shade_operation_count);
    S(total_promoted_count);
    S(total_remembered_normal_object_count);
    S(total_remembered_shady_object_count);
#endif /* RGENGC_PROFILE */
    S(page_pool_arenas);
    S(page_pool_arenas_freed);
    S(page_pool_total_pages);
    S(page_pool_discarded_pages);
#undef S
}
 
static uint64_t
ns_to_ms(uint64_t ns)
{
    return ns / (1000 * 1000);
}
 
static void malloc_increase_local_flush(rb_objspace_t *objspace);
 
static void
gc_process_stat_accumulate_i(void *objspace_ptr, void *data)
{
    rb_objspace_t *objspace = objspace_ptr;
    struct gc_process_stat_total *total = (struct gc_process_stat_total *)data;
    struct gc_process_stat_snapshot snap;
    rb_native_mutex_lock(&objspace->process_stat.lock);
    snap = objspace->process_stat.published;
    rb_native_mutex_unlock(&objspace->process_stat.lock);
    gc_process_stat_add(total, &snap);
}
 
static VALUE
gc_process_stat(VALUE hash_or_sym)
{
    VALUE hash = Qnil, key = Qnil;
 
    if (RB_TYPE_P(hash_or_sym, T_HASH)) {
        hash = hash_or_sym;
    }
    else if (SYMBOL_P(hash_or_sym)) {
        key = hash_or_sym;
    }
    else {
        rb_bug("non-hash or symbol given");
    }
 
    struct gc_process_stat_total total;
    uint64_t direct_global_gc_count;
    unsigned int lev = RB_GC_VM_LOCK();
    total = global_objspace->process_stat_archive;
    rb_gc_vm_each_objspace(gc_process_stat_accumulate_i, &total);
    direct_global_gc_count = global_objspace->global_gc.count;
    RB_GC_VM_UNLOCK(lev);
 
    rb_objspace_t *const current = rb_gc_get_objspace();
    if (!gc_during_gc_get(current)) {
        GC_ASSERT(total.global_gc_invocation_count == direct_global_gc_count);
    }
 
    /* Convert to Ruby values after all collector locks are released. */
    uint64_t time_ns = total.marking_time_ns + total.sweeping_time_ns;
    uint64_t count = total.minor_gc_count + total.major_gc_count + total.global_gc_invocation_count;
    size_t page_pool_arenas = global_objspace->page_pool.arena_count;
 
#define SET64(name, attr) \
    if (key == gc_stat_symbols[gc_stat_sym_##name]) \
        return ULL2NUM(attr); \
    else if (hash != Qnil) \
        rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], ULL2NUM(attr));
 
    SET64(count, count);
    SET64(time, ns_to_ms(time_ns));
    SET64(marking_time, ns_to_ms(total.marking_time_ns));
    SET64(sweeping_time, ns_to_ms(total.sweeping_time_ns));
    SET64(minor_gc_count, total.minor_gc_count);
    SET64(major_gc_count, total.major_gc_count);
    SET64(global_gc_count, total.global_gc_invocation_count);
    SET64(page_pool_arenas, page_pool_arenas);
    SET64(page_pool_arenas_freed, global_objspace->page_pool.arenas_unmapped);
    SET64(page_pool_total_pages, (uint64_t)page_pool_arenas * PAGE_POOL_ARENA_BODIES);
    SET64(page_pool_discarded_pages, global_objspace->page_pool.advised_count);
 
#undef SET64
 
    if (!NIL_P(key)) {
        /* Matched key should return above. */
        return Qundef;
    }
 
    return hash;
}
 
VALUE
rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
{
    if (objspace_ptr == NULL) {
        return gc_process_stat(hash_or_sym);
    }
 
    rb_objspace_t *objspace = objspace_ptr;
    VALUE hash = Qnil, key = Qnil;
 
    malloc_increase_local_flush(objspace);
 
    if (RB_TYPE_P(hash_or_sym, T_HASH)) {
        hash = hash_or_sym;
    }
    else if (SYMBOL_P(hash_or_sym)) {
        key = hash_or_sym;
    }
    else {
        rb_bug("non-hash or symbol given");
    }
 
#define SET(name, attr) \
    if (key == gc_stat_symbols[gc_stat_sym_##name]) \
        return SIZET2NUM(attr); \
    else if (hash != Qnil) \
        rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
#define SET64(name, attr) \
    if (key == gc_stat_symbols[gc_stat_sym_##name]) \
        return ULL2NUM(attr); \
    else if (hash != Qnil) \
        rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], ULL2NUM(attr));
 
    SET(count, objspace->profile.count);
    SET(time, (size_t)ns_to_ms(objspace->profile.marking_time_ns + objspace->profile.sweeping_time_ns)); // TODO: UINT64T2NUM
    SET(marking_time, (size_t)ns_to_ms(objspace->profile.marking_time_ns));
    SET(sweeping_time, (size_t)ns_to_ms(objspace->profile.sweeping_time_ns));
 
    {
        uint64_t total_malloc = (uint64_t)gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc);
        uint64_t total_free   = (uint64_t)gc_counter_load_relaxed(&objspace->malloc_counters.counters.free);
        SET64(total_malloc_bytes, total_malloc);
        SET64(total_free_bytes, total_free);
    }
 
    /* implementation dependent counters (small / fixnum-safe) */
    SET(heap_allocated_pages, rb_darray_size(objspace->heap_pages.sorted));
    SET(heap_empty_pages, objspace->empty_pages_count)
    SET(heap_allocatable_bytes, objspace->heap_pages.allocatable_bytes);
    SET(heap_eden_pages, heap_eden_total_pages(objspace));
    SET(total_allocated_pages, objspace->heap_pages.allocated_pages);
    SET(total_freed_pages, objspace->heap_pages.freed_pages);
    SET(malloc_increase_bytes, gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters));
    SET(malloc_increase_bytes_limit, malloc_limit);
    SET(minor_gc_count, objspace->profile.minor_gc_count);
    SET(major_gc_count, objspace->profile.major_gc_count);
    SET(global_gc_count, objspace->profile.global_gc_count);
    SET(global_gc_invocation_count, objspace->profile.global_gc_invocation_count);
    SET(compact_count, objspace->profile.compact_count);
    SET(read_barrier_faults, objspace->profile.read_barrier_faults);
    SET(total_moved_objects, objspace->rcompactor.total_moved);
    SET(remembered_wb_unprotected_objects, objspace->rgengc.uncollectible_wb_unprotected_objects);
    SET(remembered_wb_unprotected_objects_limit, objspace->rgengc.uncollectible_wb_unprotected_objects_limit);
    SET(old_objects, objspace->rgengc.old_objects);
    SET(old_objects_limit, objspace->rgengc.old_objects_limit);
#if RGENGC_ESTIMATE_OLDMALLOC
    SET(oldmalloc_increase_bytes, gc_oldmalloc_increase_load(objspace));
    SET(oldmalloc_increase_bytes_limit, objspace->rgengc.oldmalloc_increase_limit);
#endif
 
    SET(total_allocated_objects, total_allocated_objects(objspace));
    SET(total_freed_objects, total_freed_objects(objspace));
    SET(heap_available_slots, objspace_available_slots(objspace));
    SET(heap_live_slots, objspace_live_slots(objspace));
    SET(heap_free_slots, objspace_free_slots(objspace));
    SET(heap_final_slots, total_final_slots_count(objspace));
    SET(heap_marked_slots, objspace->marked_slots);
 
#if RGENGC_PROFILE
    SET(total_generated_normal_object_count, objspace->profile.total_generated_normal_object_count);
    SET(total_generated_shady_object_count, objspace->profile.total_generated_shady_object_count);
    SET(total_shade_operation_count, objspace->profile.total_shade_operation_count);
    SET(total_promoted_count, objspace->profile.total_promoted_count);
    SET(total_remembered_normal_object_count, objspace->profile.total_remembered_normal_object_count);
    SET(total_remembered_shady_object_count, objspace->profile.total_remembered_shady_object_count);
#endif /* RGENGC_PROFILE */
#undef SET
#undef SET64
 
    if (!NIL_P(key)) {
        // Matched key should return above
        return Qundef;
    }
 
#if defined(RGENGC_PROFILE) && RGENGC_PROFILE >= 2
    if (hash != Qnil) {
        gc_count_add_each_types(hash, "generated_normal_object_count_types", objspace->profile.generated_normal_object_count_types);
        gc_count_add_each_types(hash, "generated_shady_object_count_types", objspace->profile.generated_shady_object_count_types);
        gc_count_add_each_types(hash, "shade_operation_count_types", objspace->profile.shade_operation_count_types);
        gc_count_add_each_types(hash, "promoted_types", objspace->profile.promoted_types);
        gc_count_add_each_types(hash, "remembered_normal_object_count_types", objspace->profile.remembered_normal_object_count_types);
        gc_count_add_each_types(hash, "remembered_shady_object_count_types", objspace->profile.remembered_shady_object_count_types);
    }
#endif
 
    return hash;
}
 
enum gc_stat_heap_sym {
    gc_stat_heap_sym_slot_size,
    gc_stat_heap_sym_heap_live_slots,
    gc_stat_heap_sym_heap_free_slots,
    gc_stat_heap_sym_heap_final_slots,
    gc_stat_heap_sym_heap_eden_pages,
    gc_stat_heap_sym_heap_eden_slots,
    gc_stat_heap_sym_total_allocated_pages,
    gc_stat_heap_sym_force_major_gc_count,
    gc_stat_heap_sym_force_incremental_marking_finish_count,
    gc_stat_heap_sym_heap_allocatable_slots,
    gc_stat_heap_sym_total_allocated_objects,
    gc_stat_heap_sym_total_freed_objects,
    gc_stat_heap_sym_last
};
 
static VALUE gc_stat_heap_symbols[gc_stat_heap_sym_last];
 
static void
setup_gc_stat_heap_symbols(void)
{
#define S(s) gc_stat_heap_symbols[gc_stat_heap_sym_##s] = ID2SYM(rb_intern_const(#s))
    S(slot_size);
    S(heap_live_slots);
    S(heap_free_slots);
    S(heap_final_slots);
    S(heap_eden_pages);
    S(heap_eden_slots);
    S(heap_allocatable_slots);
    S(total_allocated_pages);
    S(force_major_gc_count);
    S(force_incremental_marking_finish_count);
    S(total_allocated_objects);
    S(total_freed_objects);
#undef S
}
 
static VALUE
stat_one_heap(rb_objspace_t *objspace, rb_heap_t *heap, VALUE hash, VALUE key)
{
#define SET(name, attr) \
    if (key == gc_stat_heap_symbols[gc_stat_heap_sym_##name]) \
        return SIZET2NUM(attr); \
    else if (hash != Qnil) \
        rb_hash_aset(hash, gc_stat_heap_symbols[gc_stat_heap_sym_##name], SIZET2NUM(attr));
 
    SET(slot_size, heap->slot_size);
    SET(heap_live_slots, heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count);
    SET(heap_free_slots, heap->total_slots - (heap->total_allocated_objects - heap->total_freed_objects));
    SET(heap_final_slots, heap->final_slots_count);
    SET(heap_eden_pages, heap->total_pages);
    SET(heap_eden_slots, heap->total_slots);
    SET(heap_allocatable_slots, objspace->heap_pages.allocatable_bytes / heap->slot_size);
    SET(total_allocated_pages, heap->total_allocated_pages);
    SET(force_major_gc_count, heap->force_major_gc_count);
    SET(force_incremental_marking_finish_count, heap->force_incremental_marking_finish_count);
    SET(total_allocated_objects, heap->total_allocated_objects);
    SET(total_freed_objects, heap->total_freed_objects);
#undef SET
 
    if (!NIL_P(key)) {
        // Matched key should return above
        return Qundef;
    }
 
    return hash;
}
 
VALUE
rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    if (NIL_P(heap_name)) {
        if (!RB_TYPE_P(hash_or_sym, T_HASH)) {
            rb_bug("non-hash given");
        }
 
        for (int i = 0; i < HEAP_COUNT; i++) {
            VALUE hash = rb_hash_aref(hash_or_sym, INT2FIX(i));
            if (NIL_P(hash)) {
                hash = rb_hash_new();
                rb_hash_aset(hash_or_sym, INT2FIX(i), hash);
            }
 
            stat_one_heap(objspace, &heaps[i], hash, Qnil);
        }
    }
    else if (FIXNUM_P(heap_name)) {
        int heap_idx = FIX2INT(heap_name);
 
        if (heap_idx < 0 || heap_idx >= HEAP_COUNT) {
            rb_raise(rb_eArgError, "size pool index out of range");
        }
 
        if (SYMBOL_P(hash_or_sym)) {
            return stat_one_heap(objspace, &heaps[heap_idx], Qnil, hash_or_sym);
        }
        else if (RB_TYPE_P(hash_or_sym, T_HASH)) {
            return stat_one_heap(objspace, &heaps[heap_idx], hash_or_sym, Qnil);
        }
        else {
            rb_bug("non-hash or symbol given");
        }
    }
    else {
        rb_bug("heap_name must be nil or an Integer");
    }
 
    return hash_or_sym;
}
 
/* I could include internal.h for this, but doing so undefines some Array macros
 * necessary for initialising objects, and I don't want to include all the array
 * headers to get them back
 * TODO: Investigate why RARRAY_AREF gets undefined in internal.h
 */
#ifndef RBOOL
#define RBOOL(v) (v ? Qtrue : Qfalse)
#endif
 
VALUE
rb_gc_impl_config_get(void *objspace_ptr)
{
#define sym(name) ID2SYM(rb_intern_const(name))
    rb_objspace_t *objspace = objspace_ptr;
    VALUE hash = rb_hash_new();
 
    rb_hash_aset(hash, sym("rgengc_allow_full_mark"), RBOOL(gc_config_full_mark_val));
 
    return hash;
}
 
static int
gc_config_set_key(VALUE key, VALUE value, VALUE data)
{
    rb_objspace_t *objspace = (rb_objspace_t *)data;
    if (rb_sym2id(key) == rb_intern("rgengc_allow_full_mark")) {
        gc_rest(objspace);
        gc_config_full_mark_set(RTEST(value));
    }
    return ST_CONTINUE;
}
 
void
rb_gc_impl_config_set(void *objspace_ptr, VALUE hash)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    if (!RB_TYPE_P(hash, T_HASH)) {
        rb_raise(rb_eArgError, "expected keyword arguments");
    }
 
    rb_hash_foreach(hash, gc_config_set_key, (st_data_t)objspace);
}
 
VALUE
rb_gc_impl_stress_get(void *objspace_ptr)
{
    return ruby_gc_stress_mode;
}
 
void
rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag)
{
    global_objspace->gc_stressful = RTEST(flag);
    global_objspace->gc_stress_mode = flag;
}
 
static int
get_envparam_size(const char *name, size_t *default_value, size_t lower_bound)
{
    const char *ptr = getenv(name);
    ssize_t val;
 
    if (ptr != NULL && *ptr) {
        size_t unit = 0;
        char *end;
#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
        val = strtoll(ptr, &end, 0);
#else
        val = strtol(ptr, &end, 0);
#endif
        switch (*end) {
          case 'k': case 'K':
            unit = 1024;
            ++end;
            break;
          case 'm': case 'M':
            unit = 1024*1024;
            ++end;
            break;
          case 'g': case 'G':
            unit = 1024*1024*1024;
            ++end;
            break;
        }
        while (*end && isspace((unsigned char)*end)) end++;
        if (*end) {
            if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
            return 0;
        }
        if (unit > 0) {
            if (val < -(ssize_t)(SIZE_MAX / 2 / unit) || (ssize_t)(SIZE_MAX / 2 / unit) < val) {
                if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%s is ignored because it overflows\n", name, ptr);
                return 0;
            }
            val *= unit;
        }
        if (val > 0 && (size_t)val > lower_bound) {
            if (RTEST(ruby_verbose)) {
                fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE")\n", name, val, *default_value);
            }
            *default_value = (size_t)val;
            return 1;
        }
        else {
            if (RTEST(ruby_verbose)) {
                fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE") is ignored because it must be greater than %"PRIuSIZE".\n",
                        name, val, *default_value, lower_bound);
            }
            return 0;
        }
    }
    return 0;
}
 
static int
get_envparam_double(const char *name, double *default_value, double lower_bound, double upper_bound, int accept_zero)
{
    const char *ptr = getenv(name);
    double val;
 
    if (ptr != NULL && *ptr) {
        char *end;
        val = strtod(ptr, &end);
        if (!*ptr || *end) {
            if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
            return 0;
        }
 
        if (accept_zero && val == 0.0) {
            goto accept;
        }
        else if (val <= lower_bound) {
            if (RTEST(ruby_verbose)) {
                fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be greater than %f.\n",
                        name, val, *default_value, lower_bound);
            }
        }
        else if (upper_bound != 0.0 && /* ignore upper_bound if it is 0.0 */
                 val > upper_bound) {
            if (RTEST(ruby_verbose)) {
                fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be lower than %f.\n",
                        name, val, *default_value, upper_bound);
            }
        }
        else {
            goto accept;
        }
    }
    return 0;
 
  accept:
    if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%f (default value: %f)\n", name, val, *default_value);
    *default_value = val;
    return 1;
}
 
/*
 * GC tuning environment variables
 *
 * * RUBY_GC_HEAP_FREE_SLOTS
 *   - Prepare at least this amount of slots after GC.
 *   - Allocate slots if there are not enough slots.
 * * RUBY_GC_HEAP_GROWTH_FACTOR (new from 2.1)
 *   - Allocate slots by this factor.
 *   - (next slots number) = (current slots number) * (this factor)
 * * RUBY_GC_HEAP_GROWTH_MAX_BYTES (was RUBY_GC_HEAP_GROWTH_MAX_SLOTS)
 *   - Allocation rate is limited to this number of bytes.
 * * RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO (new from 2.4)
 *   - Allocate additional pages when the number of free slots is
 *     lower than the value (total_slots * (this ratio)).
 * * RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO (new from 2.4)
 *   - Allocate slots to satisfy this formula:
 *       free_slots = total_slots * goal_ratio
 *   - In other words, prepare (total_slots * goal_ratio) free slots.
 *   - if this value is 0.0, then use RUBY_GC_HEAP_GROWTH_FACTOR directly.
 * * RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO (new from 2.4)
 *   - Allow to free pages when the number of free slots is
 *     greater than the value (total_slots * (this ratio)).
 * * RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR (new from 2.1.1)
 *   - Do full GC when the number of old objects is more than R * N
 *     where R is this factor and
 *           N is the number of old objects just after last full GC.
 *
 *  * obsolete
 *    * RUBY_FREE_MIN       -> RUBY_GC_HEAP_FREE_SLOTS (from 2.1)
 *    * RUBY_HEAP_MIN_SLOTS -> RUBY_GC_HEAP_INIT_SLOTS (from 2.1) -> RUBY_GC_HEAP_INIT_BYTES
 *
 * * RUBY_GC_MALLOC_LIMIT
 * * RUBY_GC_MALLOC_LIMIT_MAX (new from 2.1)
 * * RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
 *
 * * RUBY_GC_OLDMALLOC_LIMIT (new from 2.1)
 * * RUBY_GC_OLDMALLOC_LIMIT_MAX (new from 2.1)
 * * RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
 */
 
void
rb_gc_impl_set_params(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
    get_envparam_size("RUBY_GC_HEAP_FREE_SLOTS", &gc_params.heap_free_slots, 0);
 
    get_envparam_size("RUBY_GC_HEAP_INIT_BYTES", &gc_params.heap_init_bytes,
                      heap_init_bytes_min() - 1);
    get_envparam_size("RUBY_GC_RACTOR_HEAP_INIT_BYTES", &gc_params.ractor_heap_init_bytes,
                      heap_init_bytes_min() - 1);
 
    get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE);
    get_envparam_size  ("RUBY_GC_HEAP_GROWTH_MAX_BYTES", &gc_params.growth_max_bytes, 0);
    get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio,
                        0.0, 1.0, FALSE);
    get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", &gc_params.heap_free_slots_max_ratio,
                        gc_params.heap_free_slots_min_ratio, 1.0, FALSE);
    get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", &gc_params.heap_free_slots_goal_ratio,
                        gc_params.heap_free_slots_min_ratio, gc_params.heap_free_slots_max_ratio, TRUE);
    get_envparam_double("RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR", &gc_params.oldobject_limit_factor, 0.0, 0.0, TRUE);
    get_envparam_double("RUBY_GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO", &gc_params.uncollectible_wb_unprotected_objects_limit_ratio, 0.0, 0.0, TRUE);
 
    if (get_envparam_size("RUBY_GC_MALLOC_LIMIT", &gc_params.malloc_limit_min, 0)) {
        malloc_limit = gc_params.malloc_limit_min;
    }
    get_envparam_size  ("RUBY_GC_MALLOC_LIMIT_MAX", &gc_params.malloc_limit_max, 0);
    if (!gc_params.malloc_limit_max) { /* ignore max-check if 0 */
        gc_params.malloc_limit_max = SIZE_MAX;
    }
    get_envparam_double("RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR", &gc_params.malloc_limit_growth_factor, 1.0, 0.0, FALSE);
 
#if RGENGC_ESTIMATE_OLDMALLOC
    if (get_envparam_size("RUBY_GC_OLDMALLOC_LIMIT", &gc_params.oldmalloc_limit_min, 0)) {
        objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
    }
    get_envparam_size  ("RUBY_GC_OLDMALLOC_LIMIT_MAX", &gc_params.oldmalloc_limit_max, 0);
    get_envparam_double("RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR", &gc_params.oldmalloc_limit_growth_factor, 1.0, 0.0, FALSE);
#endif
}
 
static inline size_t
objspace_malloc_size(rb_objspace_t *objspace, void *ptr, size_t hint)
{
#ifdef HAVE_MALLOC_USABLE_SIZE
    if (!hint) {
        hint = malloc_usable_size(ptr);
    }
#endif
    return hint;
}
 
enum memop_type {
    MEMOP_TYPE_MALLOC  = 0,
    MEMOP_TYPE_FREE,
    MEMOP_TYPE_REALLOC
};
 
static inline void
atomic_sub_nounderflow(size_t *var, size_t sub)
{
    if (sub == 0) return;
 
    while (1) {
        size_t val = rbimpl_atomic_size_load(var, RBIMPL_ATOMIC_RELAXED);
        size_t decrement = val < sub ? val : sub;
        if (RUBY_ATOMIC_SIZE_CAS(*var, val, val-decrement) == val) break;
    }
}
 
#define gc_stress_full_mark_after_malloc_p() \
    (FIXNUM_P(ruby_gc_stress_mode) && (FIX2LONG(ruby_gc_stress_mode) & (1<<gc_stress_full_mark_after_malloc)))
 
static void
objspace_malloc_gc_stress(rb_objspace_t *objspace)
{
    if (ruby_gc_stressful && ruby_native_thread_p()) {
        unsigned int reason = (GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP |
                               GPR_FLAG_STRESS | GPR_FLAG_MALLOC);
 
        if (gc_stress_full_mark_after_malloc_p()) {
            reason |= GPR_FLAG_FULL_MARK;
        }
        garbage_collect_with_gvl(objspace, reason);
    }
}
 
static void
malloc_increase_commit(rb_objspace_t *objspace, size_t new_size, size_t old_size)
{
    if (new_size > old_size) {
        size_t delta = new_size - old_size;
        MALLOC_COUNTERS_LOCK(objspace);
        gc_counter_add(&objspace->malloc_counters.counters.malloc, delta);
#if RGENGC_ESTIMATE_OLDMALLOC
        RUBY_ATOMIC_SIZE_ADD(objspace->malloc_counters.oldmalloc_increase, delta);
#endif
        MALLOC_COUNTERS_UNLOCK(objspace);
    }
    else if (old_size > new_size) {
        size_t delta = old_size - new_size;
        MALLOC_COUNTERS_LOCK(objspace);
        gc_counter_add(&objspace->malloc_counters.counters.free, delta);
#if RGENGC_ESTIMATE_OLDMALLOC
        if (during_gc) gc_counter_add(&objspace->malloc_counters.freed_in_gc, delta);
        atomic_sub_nounderflow(&objspace->malloc_counters.oldmalloc_increase, delta);
#endif
        MALLOC_COUNTERS_UNLOCK(objspace);
    }
}
 
#if USE_MALLOC_INCREASE_LOCAL
static void
malloc_increase_local_flush(rb_objspace_t *objspace)
{
    int delta = malloc_increase_local;
    if (delta == 0) return;
 
    malloc_increase_local = 0;
    if (delta > 0) {
        malloc_increase_commit(objspace, (size_t)delta, 0);
    }
    else {
        malloc_increase_commit(objspace, 0, (size_t)(-delta));
    }
}
#else
static void
malloc_increase_local_flush(rb_objspace_t *objspace)
{
}
#endif
 
static inline bool
objspace_malloc_increase_report(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type, bool gc_allowed)
{
    if (0) fprintf(stderr, "increase - ptr: %p, type: %s, new_size: %"PRIdSIZE", old_size: %"PRIdSIZE"\n",
                   mem,
                   type == MEMOP_TYPE_MALLOC  ? "malloc" :
                   type == MEMOP_TYPE_FREE    ? "free  " :
                   type == MEMOP_TYPE_REALLOC ? "realloc": "error",
                   new_size, old_size);
    return false;
}
 
static bool
objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type, bool gc_allowed)
{
#if USE_MALLOC_INCREASE_LOCAL
    if (new_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD &&
        old_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD) {
        malloc_increase_local += (int)new_size - (int)old_size;
 
        if (malloc_increase_local >= GC_MALLOC_INCREASE_LOCAL_THRESHOLD ||
            malloc_increase_local <= -GC_MALLOC_INCREASE_LOCAL_THRESHOLD) {
            malloc_increase_local_flush(objspace);
        }
    }
    else {
        malloc_increase_local_flush(objspace);
        malloc_increase_commit(objspace, new_size, old_size);
    }
#else
    malloc_increase_commit(objspace, new_size, old_size);
#endif
 
    if (type == MEMOP_TYPE_MALLOC && gc_allowed) {
      retry:
        if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val() && !rb_gc_gc_disabled_global_p()) {
            if (ruby_thread_has_gvl_p() && is_lazy_sweeping(objspace)) {
                gc_sweep_step_for_malloc(objspace); /* sweeping frees may reduce malloc_increase */
                goto retry;
            }
            garbage_collect_with_gvl(objspace, GPR_FLAG_MALLOC);
        }
    }
 
#if MALLOC_ALLOCATED_SIZE
    if (new_size >= old_size) {
        RUBY_ATOMIC_SIZE_ADD(objspace->malloc_params.allocated_size, new_size - old_size);
    }
    else {
        size_t dec_size = old_size - new_size;
 
#if MALLOC_ALLOCATED_SIZE_CHECK
        size_t allocated_size = objspace->malloc_params.allocated_size;
        if (allocated_size < dec_size) {
            rb_bug("objspace_malloc_increase: underflow malloc_params.allocated_size.");
        }
#endif
        atomic_sub_nounderflow(&objspace->malloc_params.allocated_size, dec_size);
    }
 
    switch (type) {
      case MEMOP_TYPE_MALLOC:
        RUBY_ATOMIC_SIZE_INC(objspace->malloc_params.allocations);
        break;
      case MEMOP_TYPE_FREE:
        {
            size_t allocations = objspace->malloc_params.allocations;
            if (allocations > 0) {
                atomic_sub_nounderflow(&objspace->malloc_params.allocations, 1);
            }
#if MALLOC_ALLOCATED_SIZE_CHECK
            else {
                GC_ASSERT(objspace->malloc_params.allocations > 0);
            }
#endif
        }
        break;
      case MEMOP_TYPE_REALLOC: /* ignore */ break;
    }
#endif
    return true;
}
 
#define objspace_malloc_increase(...) \
    for (bool malloc_increase_done = objspace_malloc_increase_report(__VA_ARGS__); \
         !malloc_increase_done; \
         malloc_increase_done = objspace_malloc_increase_body(__VA_ARGS__))
 
struct malloc_obj_info { /* 4 words */
    size_t size;
};
 
static inline size_t
objspace_malloc_prepare(rb_objspace_t *objspace, size_t size)
{
    if (size == 0) size = 1;
 
#if CALC_EXACT_MALLOC_SIZE
    size += sizeof(struct malloc_obj_info);
#endif
 
    return size;
}
 
static bool
malloc_during_gc_p(rb_objspace_t *objspace)
{
    /* malloc is not allowed during GC when we're not using multiple ractors
     * (since ractors can run while another thread is sweeping) and when we
     * have the GVL (since if we don't have the GVL, we'll try to acquire the
     * GVL which will block and ensure the other thread finishes GC). */
    return during_gc && !dont_gc_val() && !rb_gc_multi_ractor_p() && ruby_thread_has_gvl_p();
}
 
static inline void *
objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size, bool gc_allowed)
{
    size = objspace_malloc_size(objspace, mem, size);
    objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC, gc_allowed) {}
 
#if CALC_EXACT_MALLOC_SIZE
    {
        struct malloc_obj_info *info = mem;
        info->size = size;
        mem = info + 1;
    }
#endif
 
    return mem;
}
 
#if defined(__GNUC__) && RUBY_DEBUG
#define RB_BUG_INSTEAD_OF_RB_MEMERROR 1
#endif
 
#ifndef RB_BUG_INSTEAD_OF_RB_MEMERROR
# define RB_BUG_INSTEAD_OF_RB_MEMERROR 0
#endif
 
#define GC_MEMERROR(...) \
    ((RB_BUG_INSTEAD_OF_RB_MEMERROR+0) ? rb_bug("" __VA_ARGS__) : (void)0)
 
#define TRY_WITH_GC(siz, expr) do {                          \
        const gc_profile_record_flag gpr =                   \
            GPR_FLAG_FULL_MARK           |                   \
            GPR_FLAG_IMMEDIATE_MARK      |                   \
            GPR_FLAG_IMMEDIATE_SWEEP     |                   \
            GPR_FLAG_MALLOC;                                 \
        /* stress GC must also honor gc_allowed (malloc_gc_disabled) */ \
        if (gc_allowed) objspace_malloc_gc_stress(objspace);  \
                                                             \
        if (RB_LIKELY((expr))) {                             \
            /* Success on 1st try */                         \
        }                                                    \
        else if (gc_allowed && !garbage_collect_with_gvl(objspace, gpr)) { \
            /* @shyouhei thinks this doesn't happen */       \
            GC_MEMERROR("TRY_WITH_GC: could not GC");        \
        }                                                    \
        else if ((expr)) {                                   \
            /* Success on 2nd try */                         \
        }                                                    \
        else {                                               \
            GC_MEMERROR("TRY_WITH_GC: could not allocate:"   \
                        "%"PRIdSIZE" bytes for %s",          \
                        siz, # expr);                        \
        }                                                    \
    } while (0)
 
static void
check_malloc_not_in_gc(rb_objspace_t *objspace, const char *msg)
{
    if (RB_UNLIKELY(malloc_during_gc_p(objspace))) {
        dont_gc_on();
        during_gc = false;
        rb_bug("Cannot %s during GC", msg);
    }
}
 
void
rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    if (!ptr) {
        /*
         * ISO/IEC 9899 says "If ptr is a null pointer, no action occurs" since
         * its first version.  We would better follow.
         */
        return;
    }
#if CALC_EXACT_MALLOC_SIZE
    struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
#if VERIFY_FREE_SIZE
    if (!info->size) {
        const char *freeing = gc_freeing_obj_info();
        rb_bug("buffer %p has no recorded size%s%s. Was it allocated with ruby_mimalloc? If so it should be freed with ruby_mimfree", ptr,
               freeing ? ", while freeing " : "", freeing ? freeing : "");
    }
 
    if (old_size && (old_size + sizeof(struct malloc_obj_info)) != info->size) {
        const char *freeing = gc_freeing_obj_info();
        rb_bug("buffer %p freed with old_size=%zu, but was allocated with size=%zu%s%s", ptr, old_size, info->size - sizeof(struct malloc_obj_info),
               freeing ? ", while freeing " : "", freeing ? freeing : "");
    }
#endif
    ptr = info;
    old_size = info->size;
#endif
    old_size = objspace_malloc_size(objspace, ptr, old_size);
 
    objspace_malloc_increase(objspace, ptr, 0, old_size, MEMOP_TYPE_FREE, true) {
        free(ptr);
        ptr = NULL;
        RB_DEBUG_COUNTER_INC(heap_xfree);
    }
}
 
void *
rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed)
{
    rb_objspace_t *objspace = objspace_ptr;
    check_malloc_not_in_gc(objspace, "malloc");
 
    void *mem;
 
    size = objspace_malloc_prepare(objspace, size);
    TRY_WITH_GC(size, mem = malloc(size));
    RB_DEBUG_COUNTER_INC(heap_xmalloc);
    if (!mem) return mem;
    return objspace_malloc_fixup(objspace, mem, size, gc_allowed);
}
 
void *
rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    if (RB_UNLIKELY(malloc_during_gc_p(objspace))) {
        rb_warn("calloc during GC detected, this could cause crashes if it triggers another GC");
#if RGENGC_CHECK_MODE || RUBY_DEBUG
        rb_bug("Cannot calloc during GC");
#endif
    }
 
    void *mem;
 
    size = objspace_malloc_prepare(objspace, size);
    TRY_WITH_GC(size, mem = calloc1(size));
    if (!mem) return mem;
    return objspace_malloc_fixup(objspace, mem, size, gc_allowed);
}
 
void *
rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    check_malloc_not_in_gc(objspace, "realloc");
 
    void *mem;
 
    if (!ptr) return rb_gc_impl_malloc(objspace, new_size, gc_allowed);
 
    /*
     * The behavior of realloc(ptr, 0) is implementation defined.
     * Therefore we don't use realloc(ptr, 0) for portability reason.
     * see http://www.open-std.org/jtc1/sc22/wg14/www/docs/dr_400.htm
     */
    if (new_size == 0) {
        if ((mem = rb_gc_impl_malloc(objspace, 0, gc_allowed)) != NULL) {
            /*
             * - OpenBSD's malloc(3) man page says that when 0 is passed, it
             *   returns a non-NULL pointer to an access-protected memory page.
             *   The returned pointer cannot be read / written at all, but
             *   still be a valid argument of free().
             *
             *   https://man.openbsd.org/malloc.3
             *
             * - Linux's malloc(3) man page says that it _might_ perhaps return
             *   a non-NULL pointer when its argument is 0.  That return value
             *   is safe (and is expected) to be passed to free().
             *
             *   https://man7.org/linux/man-pages/man3/malloc.3.html
             *
             * - As I read the implementation jemalloc's malloc() returns fully
             *   normal 16 bytes memory region when its argument is 0.
             *
             * - As I read the implementation musl libc's malloc() returns
             *   fully normal 32 bytes memory region when its argument is 0.
             *
             * - Other malloc implementations can also return non-NULL.
             */
            rb_gc_impl_free(objspace, ptr, old_size);
            return mem;
        }
        else {
            /*
             * It is dangerous to return NULL here, because that could lead to
             * RCE.  Fallback to 1 byte instead of zero.
             *
             * https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11932
             */
            new_size = 1;
        }
    }
 
#if CALC_EXACT_MALLOC_SIZE
    {
        struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
        new_size += sizeof(struct malloc_obj_info);
        ptr = info;
#if VERIFY_FREE_SIZE
        if (old_size && (old_size + sizeof(struct malloc_obj_info)) != info->size) {
            const char *freeing = gc_freeing_obj_info();
            rb_bug("buffer %p realloced with old_size=%zu, but was allocated with size=%zu%s%s", ptr, old_size, info->size - sizeof(struct malloc_obj_info),
                   freeing ? ", while freeing " : "", freeing ? freeing : "");
        }
#endif
        old_size = info->size;
    }
#endif
 
    old_size = objspace_malloc_size(objspace, ptr, old_size);
    TRY_WITH_GC(new_size, mem = RB_GNUC_EXTENSION_BLOCK(realloc(ptr, new_size)));
    if (!mem) return mem;
    new_size = objspace_malloc_size(objspace, mem, new_size);
 
#if CALC_EXACT_MALLOC_SIZE
    {
        struct malloc_obj_info *info = mem;
        info->size = new_size;
        mem = info + 1;
    }
#endif
 
    objspace_malloc_increase(objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC, gc_allowed);
 
    RB_DEBUG_COUNTER_INC(heap_xrealloc);
    return mem;
}
 
void
rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    if (diff > 0) {
        objspace_malloc_increase(objspace, 0, diff, 0, MEMOP_TYPE_REALLOC, true);
    }
    else if (diff < 0) {
        objspace_malloc_increase(objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC, true);
    }
}
 
// TODO: move GC profiler stuff back into gc.c
/*
  ------------------------------ GC profiler ------------------------------
*/
 
#define GC_PROFILE_RECORD_DEFAULT_SIZE 100
#define GC_PROFILE_RECORD_DEFAULT_MAX_RECORDS 4096
#define GC_PROFILE_RECORD_UNBOUNDED 0
 
static bool
current_process_time(struct timespec *ts)
{
#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
    {
        static int try_clock_gettime = 1;
        if (try_clock_gettime) {
            if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ts) == 0) {
                return true;
            }
            else {
                try_clock_gettime = 0;
            }
        }
    }
#endif
 
#ifdef RUSAGE_SELF
    {
        struct rusage usage;
        struct timeval time;
        if (getrusage(RUSAGE_SELF, &usage) == 0) {
            time = usage.ru_utime;
            ts->tv_sec = time.tv_sec;
            ts->tv_nsec = (int32_t)time.tv_usec * 1000;
            return true;
        }
    }
#endif
 
#ifdef _WIN32
    {
        FILETIME creation_time, exit_time, kernel_time, user_time;
        ULARGE_INTEGER ui;
 
        if (GetProcessTimes(GetCurrentProcess(),
                            &creation_time, &exit_time, &kernel_time, &user_time) != 0) {
            memcpy(&ui, &user_time, sizeof(FILETIME));
#define PER100NSEC (uint64_t)(1000 * 1000 * 10)
            ts->tv_nsec = (long)(ui.QuadPart % PER100NSEC);
            ts->tv_sec  = (time_t)(ui.QuadPart / PER100NSEC);
            return true;
        }
    }
#endif
 
    return false;
}
 
static double
getrusage_time(void)
{
    struct timespec ts;
    if (current_process_time(&ts)) {
        return ts.tv_sec + ts.tv_nsec * 1e-9;
    }
    else {
        return 0.0;
    }
}
 
static inline double
hrtime_to_sec(rb_hrtime_t time)
{
    return (double)time / (double)RB_HRTIME_PER_SEC;
}
 
static inline rb_hrtime_t
elapsed_hrtime_from(rb_hrtime_t start)
{
    return rb_hrtime_sub(rb_hrtime_now(), start);
}
 
 
static inline size_t
gc_profile_record_count(rb_objspace_t *objspace)
{
    return objspace->profile.record_count;
}
 
static inline size_t
gc_profile_record_index(rb_objspace_t *objspace, size_t logical_index)
{
    if (objspace->profile.max_records != GC_PROFILE_RECORD_UNBOUNDED &&
            objspace->profile.record_count == objspace->profile.size) {
        return (objspace->profile.next_index + logical_index) % objspace->profile.size;
    }
    else {
        return logical_index;
    }
}
 
static void
gc_profile_records_free(rb_objspace_t *objspace)
{
    void *p = objspace->profile.records;
    objspace->profile.records = NULL;
    objspace->profile.size = 0;
    objspace->profile.next_index = 0;
    objspace->profile.record_count = 0;
    objspace->profile.current_record = 0;
    free(p);
}
 
static inline void
gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason)
{
    if (objspace->profile.run) {
        size_t index;
        gc_profile_record *record;
 
        if (objspace->profile.max_records == GC_PROFILE_RECORD_UNBOUNDED) {
            index = objspace->profile.record_count++;
            objspace->profile.next_index = objspace->profile.record_count;
 
            if (!objspace->profile.records) {
                objspace->profile.size = GC_PROFILE_RECORD_DEFAULT_SIZE;
                objspace->profile.records = malloc(xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
            }
            if (index >= objspace->profile.size) {
                void *ptr;
                objspace->profile.size += 1000;
                ptr = realloc(objspace->profile.records, xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
                if (!ptr) rb_memerror();
                objspace->profile.records = ptr;
            }
        }
        else {
            if (!objspace->profile.records) {
                objspace->profile.size = objspace->profile.max_records;
                objspace->profile.records = malloc(xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
            }
            index = objspace->profile.next_index;
            objspace->profile.next_index = (objspace->profile.next_index + 1) % objspace->profile.size;
            if (objspace->profile.record_count < objspace->profile.size) {
                objspace->profile.record_count++;
            }
        }
 
        if (!objspace->profile.records) {
            rb_bug("gc_profile malloc or realloc miss");
        }
        record = objspace->profile.current_record = &objspace->profile.records[index];
        MEMZERO(record, gc_profile_record, 1);
 
        /* setup before-GC parameter */
        record->flags = reason | (ruby_gc_stressful ? GPR_FLAG_STRESS : 0);
        record->sequence = objspace->profile.record_sequence++;
        record->gc_invoke_wall_time = rb_hrtime_sub(rb_hrtime_now(),
                objspace->profile.invoke_wall_time);
#if MALLOC_ALLOCATED_SIZE
        record->allocated_size = malloc_allocated_size;
#endif
#if GC_PROFILE_MORE_DETAIL && GC_PROFILE_DETAIL_MEMORY
#ifdef RUSAGE_SELF
        {
            struct rusage usage;
            if (getrusage(RUSAGE_SELF, &usage) == 0) {
                record->maxrss = usage.ru_maxrss;
                record->minflt = usage.ru_minflt;
                record->majflt = usage.ru_majflt;
            }
        }
#endif
#endif
    }
}
 
static inline void
gc_prof_timer_start(rb_objspace_t *objspace)
{
    if (gc_prof_enabled(objspace)) {
        gc_profile_record *record = gc_prof_record(objspace);
#if GC_PROFILE_MORE_DETAIL
        record->prepare_time = objspace->profile.prepare_time;
#endif
        record->gc_time = 0;
        record->gc_invoke_time = getrusage_time();
        objspace->profile.gc_wall_start_time = rb_hrtime_now();
    }
}
 
static double
elapsed_time_from(double time)
{
    double now = getrusage_time();
    if (now > time) {
        return now - time;
    }
    else {
        return 0;
    }
}
 
static inline void
gc_prof_timer_stop(rb_objspace_t *objspace)
{
    if (gc_prof_enabled(objspace)) {
        gc_profile_record *record = gc_prof_record(objspace);
        record->gc_time = elapsed_time_from(record->gc_invoke_time);
        record->gc_invoke_time -= objspace->profile.invoke_time;
        record->gc_wall_time = elapsed_hrtime_from(objspace->profile.gc_wall_start_time);
    }
}
 
static inline void
gc_prof_mark_timer_start(rb_objspace_t *objspace)
{
    RUBY_DTRACE_GC_HOOK(MARK_BEGIN);
#if GC_PROFILE_MORE_DETAIL
    if (gc_prof_enabled(objspace)) {
        gc_prof_record(objspace)->gc_mark_time = getrusage_time();
    }
#endif
}
 
static inline void
gc_prof_mark_timer_stop(rb_objspace_t *objspace)
{
    RUBY_DTRACE_GC_HOOK(MARK_END);
#if GC_PROFILE_MORE_DETAIL
    if (gc_prof_enabled(objspace)) {
        gc_profile_record *record = gc_prof_record(objspace);
        record->gc_mark_time = elapsed_time_from(record->gc_mark_time);
    }
#endif
}
 
static inline void
gc_prof_sweep_timer_start(rb_objspace_t *objspace)
{
    RUBY_DTRACE_GC_HOOK(SWEEP_BEGIN);
    if (gc_prof_enabled(objspace)) {
        gc_profile_record *record = gc_prof_record(objspace);
 
        if (record->gc_time > 0 || GC_PROFILE_MORE_DETAIL) {
            objspace->profile.gc_sweep_start_time = getrusage_time();
            objspace->profile.gc_sweep_wall_start_time = rb_hrtime_now();
        }
    }
}
 
static inline void
gc_prof_sweep_timer_stop(rb_objspace_t *objspace)
{
    RUBY_DTRACE_GC_HOOK(SWEEP_END);
 
    if (gc_prof_enabled(objspace)) {
        double sweep_time;
        gc_profile_record *record = gc_prof_record(objspace);
 
        if (record->gc_time > 0) {
            sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
            /* need to accumulate GC time for lazy sweep after gc() */
            record->gc_time += sweep_time;
            record->gc_wall_time = rb_hrtime_add(record->gc_wall_time,
                    elapsed_hrtime_from(objspace->profile.gc_sweep_wall_start_time));
        }
        else if (GC_PROFILE_MORE_DETAIL) {
            sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
        }
 
#if GC_PROFILE_MORE_DETAIL
        record->gc_sweep_time += sweep_time;
        if (heap_pages_deferred_final) record->flags |= GPR_FLAG_HAVE_FINALIZE;
#endif
        if (heap_pages_deferred_final) objspace->profile.latest_gc_info |= GPR_FLAG_HAVE_FINALIZE;
    }
}
 
static inline void
gc_prof_set_malloc_info(rb_objspace_t *objspace)
{
#if GC_PROFILE_MORE_DETAIL
    if (gc_prof_enabled(objspace)) {
        gc_profile_record *record = gc_prof_record(objspace);
        record->allocate_increase = malloc_increase;
        record->allocate_limit = malloc_limit;
    }
#endif
}
 
static inline void
gc_prof_set_heap_info(rb_objspace_t *objspace)
{
    if (gc_prof_enabled(objspace)) {
        gc_profile_record *record = gc_prof_record(objspace);
 
        /* Sum across all size pools since each has a different slot size. */
        size_t total = 0;
        size_t use_size = 0;
        size_t total_size = 0;
        for (int i = 0; i < HEAP_COUNT; i++) {
            rb_heap_t *heap = &heaps[i];
            size_t heap_live = heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count;
            total += heap->total_slots;
            use_size += heap_live * heap->slot_size;
            total_size += heap->total_slots * heap->slot_size;
        }
 
#if GC_PROFILE_MORE_DETAIL
        size_t live = objspace->profile.total_allocated_objects_at_gc_start - total_freed_objects(objspace);
        record->heap_use_pages = objspace->profile.heap_used_at_gc_start;
        record->heap_live_objects = live;
        record->heap_free_objects = total - live;
#endif
 
        record->heap_total_objects = total;
        record->heap_use_size = use_size;
        record->heap_total_size = total_size;
    }
}
 
/*
 *  call-seq:
 *    GC::Profiler.clear          -> nil
 *
 *  Clears the \GC profiler data.
 *
 */
 
static VALUE
gc_profile_clear(VALUE _)
{
    rb_objspace_t *objspace = rb_gc_get_objspace();
    gc_profile_records_free(objspace);
    return Qnil;
}
 
/*
 *  call-seq:
 *     GC::Profiler.configure(max_records: 4096) -> nil
 *
 *  Configures how many raw profile records are retained by
 *  GC::Profiler.raw_data.
 *
 *  The profiler keeps at most +max_records+ records in a bounded ring buffer.
 *  When the buffer is full, newer GC records overwrite the oldest retained
 *  records.  The default limit is 4096 records.
 *
 *  Pass +nil+ to restore the historical unbounded behavior:
 *
 *    GC::Profiler.configure(max_records: nil)
 *
 *  Changing +max_records+ clears existing raw profile data.  This method does
 *  not enable or disable the profiler; use GC::Profiler.enable and
 *  GC::Profiler.disable for that.
 */
 
static VALUE
gc_profile_configure(int argc, VALUE *argv, VALUE _)
{
    static ID keywords[1] = {0};
    VALUE options, max_records;
    rb_objspace_t *objspace = rb_gc_get_objspace();
 
    if (!keywords[0]) {
        keywords[0] = rb_intern("max_records");
    }
 
    rb_scan_args_kw(rb_keyword_given_p(), argc, argv, ":", &options);
    rb_get_kwargs(options, keywords, 0, 1, &max_records);
 
    if (max_records == Qundef) {
        return Qnil;
    }
    else if (NIL_P(max_records)) {
        objspace->profile.max_records = GC_PROFILE_RECORD_UNBOUNDED;
    }
    else {
        long value = NUM2LONG(max_records);
        if (value <= 0) {
            rb_raise(rb_eArgError, "max_records must be positive or nil");
        }
        objspace->profile.max_records = (size_t)value;
    }
 
    gc_profile_records_free(objspace);
    return Qnil;
}
 
/*
 *  call-seq:
 *     GC::Profiler.raw_data(limit: nil, since: nil) -> [Hash, ...]
 *
 *  Returns an Array of retained raw profile data Hashes ordered from earliest
 *  to latest by +:GC_INVOKE_TIME+.  +limit:+ returns at most the newest
 *  retained records.  +since:+ returns records with +:GC_SEQUENCE+ greater
 *  than the given sequence.
 *
 *  For example:
 *
 *    [
 *      {
 *         :GC_TIME=>1.3000000000000858e-05,
 *         :GC_INVOKE_TIME=>0.010634999999999999,
 *         :GC_WALL_TIME=>1.4000000000000001e-05,
 *         :GC_INVOKE_WALL_TIME=>0.010640000000000000,
 *         :GC_PAUSE_TIME=>1.5000000000000000e-05,
 *         :GC_STOP_TIME=>1.0000000000000000e-06,
 *         :GC_STW_TIME=>1.4000000000000001e-05,
 *         :GC_MARK_WALL_TIME=>9.0000000000000002e-06,
 *         :GC_SWEEP_WALL_TIME=>5.0000000000000004e-06,
 *         :GC_COMPACT_WALL_TIME=>0.0000000000000000e+00,
 *         :HEAP_USE_SIZE=>289640,
 *         :HEAP_TOTAL_SIZE=>588960,
 *         :HEAP_TOTAL_OBJECTS=>14724,
 *         :GC_IS_MARKED=>false
 *      },
 *      # ...
 *    ]
 *
 *  The keys mean:
 *
 *  +:GC_SEQUENCE+::
 *      Monotonically increasing sequence number for this profiler record.
 *  +:GC_TIME+::
 *      CPU time elapsed in seconds for this GC run.  This is process CPU time,
 *      not elapsed wall-clock time.
 *  +:GC_INVOKE_TIME+::
 *      CPU time elapsed in seconds from startup to when the GC was invoked.
 *  +:GC_WALL_TIME+::
 *      Monotonic wall-clock counterpart to +:GC_TIME+ for this GC record.
 *      This does not include time spent stopping other ractors before the VM
 *      enters GC.  Use the phase wall-clock fields below for mark, sweep, and
 *      compaction attribution.
 *  +:GC_INVOKE_WALL_TIME+::
 *      Monotonic wall-clock time elapsed in seconds from startup to when the GC
 *      was invoked.
 *  +:GC_PAUSE_TIME+::
 *      Monotonic wall-clock time elapsed in seconds while user execution was
 *      blocked by this GC entry, including time to stop other ractors.  This
 *      may include time from incremental marking or lazy sweeping continuation
 *      charged to this record.
 *  +:GC_STOP_TIME+::
 *      Monotonic wall-clock time elapsed in seconds stopping other ractors.
 *  +:GC_STW_TIME+::
 *      Monotonic wall-clock time elapsed in seconds after other ractors have
 *      stopped and before the VM exits GC.
 *  +:GC_MARK_WALL_TIME+::
 *      Monotonic wall-clock time elapsed in seconds spent marking for this GC
 *      record, accumulated across incremental marking continuations.
 *  +:GC_SWEEP_WALL_TIME+::
 *      Monotonic wall-clock time elapsed in seconds spent sweeping for this GC
 *      record, accumulated across lazy sweeping continuations.  This does not
 *      include compaction time, which is reported separately as
 *      +:GC_COMPACT_WALL_TIME+.
 *  +:GC_COMPACT_WALL_TIME+::
 *      Monotonic wall-clock time elapsed in seconds spent compacting for this GC
 *      record, or +0.0+ if this GC did not compact.
 *  +:HEAP_USE_SIZE+::
 *      Total bytes of heap used
 *  +:HEAP_TOTAL_SIZE+::
 *      Total size of heap in bytes
 *  +:HEAP_TOTAL_OBJECTS+::
 *      Total number of objects
 *  +:GC_IS_MARKED+::
 *      Returns +true+ if the GC is in mark phase
 *
 *  The wall-clock timing fields relate to each other as follows:
 *
 *    GC_PAUSE_TIME == GC_STOP_TIME + GC_STW_TIME
 *
 *  +:GC_MARK_WALL_TIME+, +:GC_SWEEP_WALL_TIME+, and +:GC_COMPACT_WALL_TIME+
 *  report separate phase timings and must not be added to +:GC_WALL_TIME+.
 *
 *  +:GC_WALL_TIME+ is the wall-clock counterpart to +:GC_TIME+ and is nested
 *  inside +:GC_STW_TIME+, so it must not be added to +:GC_STW_TIME+.  The difference
 *  +GC_STW_TIME - GC_WALL_TIME+ is VM overhead inside the stopped interval
 *  (GC event hooks, bookkeeping, consistency checks, and continuation work).
 *
 *  If ruby was built with +GC_PROFILE_MORE_DETAIL+, you will also have access
 *  to the following hash keys:
 *
 *  +:GC_MARK_TIME+::
 *  +:GC_SWEEP_TIME+::
 *  +:ALLOCATE_INCREASE+::
 *  +:ALLOCATE_LIMIT+::
 *  +:HEAP_USE_PAGES+::
 *  +:HEAP_LIVE_OBJECTS+::
 *  +:HEAP_FREE_OBJECTS+::
 *  +:HAVE_FINALIZE+::
 *
 */
 
static VALUE
gc_profile_record_get(int argc, VALUE *argv, VALUE _)
{
    static ID keywords[2] = {0};
    VALUE prof, options, limit_value, since_value;
    VALUE gc_profile = rb_ary_new();
    size_t i, count, matching = 0, skip = 0, limit = SIZE_MAX, since = 0;
    bool use_since = false;
    rb_objspace_t *objspace = rb_gc_get_objspace();
 
    if (!keywords[0]) {
        keywords[0] = rb_intern("limit");
        keywords[1] = rb_intern("since");
    }
 
    rb_scan_args_kw(rb_keyword_given_p(), argc, argv, ":", &options);
    VALUE values[2] = {Qundef, Qundef};
    rb_get_kwargs(options, keywords, 0, 2, values);
    limit_value = values[0];
    since_value = values[1];
 
    if (limit_value != Qundef && !NIL_P(limit_value)) {
        long value = NUM2LONG(limit_value);
        if (value < 0) {
            rb_raise(rb_eArgError, "limit must be non-negative");
        }
        limit = (size_t)value;
    }
    if (since_value != Qundef && !NIL_P(since_value)) {
        long value = NUM2LONG(since_value);
        if (value < 0) {
            rb_raise(rb_eArgError, "since must be non-negative");
        }
        since = (size_t)value;
        use_since = true;
    }
 
    if (!objspace->profile.run) {
        return Qnil;
    }
 
    count = gc_profile_record_count(objspace);
    for (i = 0; i < count; i++) {
        gc_profile_record *record = &objspace->profile.records[gc_profile_record_index(objspace, i)];
        if (!use_since || record->sequence > since) {
            matching++;
        }
    }
    if (limit < matching) {
        skip = matching - limit;
    }
 
    for (i = 0; i < count; i++) {
        gc_profile_record *record = &objspace->profile.records[gc_profile_record_index(objspace, i)];
        if (use_since && record->sequence <= since) {
            continue;
        }
        if (skip > 0) {
            skip--;
            continue;
        }
 
        prof = rb_hash_new();
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_FLAGS")), gc_info_decode(objspace, rb_hash_new(), record->flags));
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_SEQUENCE")), SIZET2NUM(record->sequence));
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_TIME")), DBL2NUM(record->gc_time));
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_INVOKE_TIME")), DBL2NUM(record->gc_invoke_time));
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_WALL_TIME")),
                DBL2NUM(hrtime_to_sec(record->gc_wall_time)));
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_INVOKE_WALL_TIME")),
                DBL2NUM(hrtime_to_sec(record->gc_invoke_wall_time)));
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_PAUSE_TIME")),
                DBL2NUM(hrtime_to_sec(record->gc_pause_time)));
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_STOP_TIME")),
                DBL2NUM(hrtime_to_sec(record->gc_stop_time)));
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_STW_TIME")),
                DBL2NUM(hrtime_to_sec(record->gc_stw_time)));
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_MARK_WALL_TIME")),
                DBL2NUM(hrtime_to_sec(record->gc_mark_wall_time)));
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_SWEEP_WALL_TIME")),
                DBL2NUM(hrtime_to_sec(record->gc_sweep_wall_time)));
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_COMPACT_WALL_TIME")),
                DBL2NUM(hrtime_to_sec(record->gc_compact_wall_time)));
        rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_SIZE")), SIZET2NUM(record->heap_use_size));
        rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_SIZE")), SIZET2NUM(record->heap_total_size));
        rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_OBJECTS")), SIZET2NUM(record->heap_total_objects));
        rb_hash_aset(prof, ID2SYM(rb_intern("MOVED_OBJECTS")), SIZET2NUM(record->moved_objects));
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_IS_MARKED")), Qtrue);
#if GC_PROFILE_MORE_DETAIL
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_MARK_TIME")), DBL2NUM(record->gc_mark_time));
        rb_hash_aset(prof, ID2SYM(rb_intern("GC_SWEEP_TIME")), DBL2NUM(record->gc_sweep_time));
        rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_INCREASE")), SIZET2NUM(record->allocate_increase));
        rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_LIMIT")), SIZET2NUM(record->allocate_limit));
        rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_PAGES")), SIZET2NUM(record->heap_use_pages));
        rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_LIVE_OBJECTS")), SIZET2NUM(record->heap_live_objects));
        rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_FREE_OBJECTS")), SIZET2NUM(record->heap_free_objects));
 
        rb_hash_aset(prof, ID2SYM(rb_intern("REMOVING_OBJECTS")), SIZET2NUM(record->removing_objects));
        rb_hash_aset(prof, ID2SYM(rb_intern("EMPTY_OBJECTS")), SIZET2NUM(record->empty_objects));
 
        rb_hash_aset(prof, ID2SYM(rb_intern("HAVE_FINALIZE")), (record->flags & GPR_FLAG_HAVE_FINALIZE) ? Qtrue : Qfalse);
#endif
 
#if RGENGC_PROFILE > 0
        rb_hash_aset(prof, ID2SYM(rb_intern("OLD_OBJECTS")), SIZET2NUM(record->old_objects));
        rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_NORMAL_OBJECTS")), SIZET2NUM(record->remembered_normal_objects));
        rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_SHADY_OBJECTS")), SIZET2NUM(record->remembered_shady_objects));
#endif
        rb_ary_push(gc_profile, prof);
    }
 
    return gc_profile;
}
 
#if GC_PROFILE_MORE_DETAIL
#define MAJOR_REASON_MAX 0x10
 
static char *
gc_profile_dump_major_reason(unsigned int flags, char *buff)
{
    unsigned int reason = flags & GPR_FLAG_MAJOR_MASK;
    int i = 0;
 
    if (reason == GPR_FLAG_NONE) {
        buff[0] = '-';
        buff[1] = 0;
    }
    else {
#define C(x, s) \
  if (reason & GPR_FLAG_MAJOR_BY_##x) { \
      buff[i++] = #x[0]; \
      if (i >= MAJOR_REASON_MAX) rb_bug("gc_profile_dump_major_reason: overflow"); \
      buff[i] = 0; \
  }
        C(NOFREE, N);
        C(OLDGEN, O);
        C(SHADY,  S);
#if RGENGC_ESTIMATE_OLDMALLOC
        C(OLDMALLOC, M);
#endif
#undef C
    }
    return buff;
}
#endif
 
 
 
static void
gc_profile_dump_on(VALUE out, VALUE (*append)(VALUE, VALUE))
{
    rb_objspace_t *objspace = rb_gc_get_objspace();
    size_t count = gc_profile_record_count(objspace);
#ifdef MAJOR_REASON_MAX
    char reason_str[MAJOR_REASON_MAX];
#endif
 
    if (objspace->profile.run && count /* > 1 */) {
        size_t i;
        const gc_profile_record *record;
 
        append(out, rb_sprintf("GC %"PRIuSIZE" invokes.\n", objspace->profile.count));
        append(out, rb_str_new_cstr("Index    Invoke Time(sec)       Use Size(byte)     Total Size(byte)         Total Object                    GC Time(ms)\n"));
 
        for (i = 0; i < count; i++) {
            record = &objspace->profile.records[gc_profile_record_index(objspace, i)];
            append(out, rb_sprintf("%5"PRIuSIZE" %19.3f %20"PRIuSIZE" %20"PRIuSIZE" %20"PRIuSIZE" %30.20f\n",
                                   i+1, record->gc_invoke_time, record->heap_use_size,
                                   record->heap_total_size, record->heap_total_objects, record->gc_time*1000));
        }
 
#if GC_PROFILE_MORE_DETAIL
        const char *str = "\n\n" \
                                    "More detail.\n" \
                                    "Prepare Time = Previously GC's rest sweep time\n"
                                    "Index Flags          Allocate Inc.  Allocate Limit"
#if CALC_EXACT_MALLOC_SIZE
                                    "  Allocated Size"
#endif
                                    "  Use Page     Mark Time(ms)    Sweep Time(ms)  Prepare Time(ms)  LivingObj    FreeObj RemovedObj   EmptyObj"
#if RGENGC_PROFILE
                                    " OldgenObj RemNormObj RemShadObj"
#endif
#if GC_PROFILE_DETAIL_MEMORY
                                    " MaxRSS(KB) MinorFLT MajorFLT"
#endif
                                    "\n";
        append(out, rb_str_new_cstr(str));
 
        for (i = 0; i < count; i++) {
            record = &objspace->profile.records[gc_profile_record_index(objspace, i)];
            append(out, rb_sprintf("%5"PRIuSIZE" %4s/%c/%6s%c %13"PRIuSIZE" %15"PRIuSIZE
#if CALC_EXACT_MALLOC_SIZE
                                   " %15"PRIuSIZE
#endif
                                   " %9"PRIuSIZE" %17.12f %17.12f %17.12f %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
#if RGENGC_PROFILE
                                   "%10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
#endif
#if GC_PROFILE_DETAIL_MEMORY
                                   "%11ld %8ld %8ld"
#endif
 
                                   "\n",
                                   i+1,
                                   gc_profile_dump_major_reason(record->flags, reason_str),
                                   (record->flags & GPR_FLAG_HAVE_FINALIZE) ? 'F' : '.',
                                   (record->flags & GPR_FLAG_NEWOBJ) ? "NEWOBJ" :
                                   (record->flags & GPR_FLAG_MALLOC) ? "MALLOC" :
                                   (record->flags & GPR_FLAG_METHOD) ? "METHOD" :
                                   (record->flags & GPR_FLAG_CAPI)   ? "CAPI__" : "??????",
                                   (record->flags & GPR_FLAG_STRESS) ? '!' : ' ',
                                   record->allocate_increase, record->allocate_limit,
#if CALC_EXACT_MALLOC_SIZE
                                   record->allocated_size,
#endif
                                   record->heap_use_pages,
                                   record->gc_mark_time*1000,
                                   record->gc_sweep_time*1000,
                                   record->prepare_time*1000,
 
                                   record->heap_live_objects,
                                   record->heap_free_objects,
                                   record->removing_objects,
                                   record->empty_objects
#if RGENGC_PROFILE
                                   ,
                                   record->old_objects,
                                   record->remembered_normal_objects,
                                   record->remembered_shady_objects
#endif
#if GC_PROFILE_DETAIL_MEMORY
                                   ,
                                   record->maxrss / 1024,
                                   record->minflt,
                                   record->majflt
#endif
 
                       ));
        }
#endif
    }
}
 
/*
 *  call-seq:
 *     GC::Profiler.result  -> String
 *
 *  Returns a profile data report such as:
 *
 *    GC 1 invokes.
 *    Index    Invoke Time(sec)       Use Size(byte)     Total Size(byte)         Total Object                    GC time(ms)
 *        1               0.012               159240               212940                10647         0.00000000000001530000
 */
 
static VALUE
gc_profile_result(VALUE _)
{
    VALUE str = rb_str_buf_new(0);
    gc_profile_dump_on(str, rb_str_buf_append);
    return str;
}
 
/*
 *  call-seq:
 *     GC::Profiler.report
 *     GC::Profiler.report(io)
 *
 *  Writes the GC::Profiler.result to <tt>$stdout</tt> or the given IO object.
 *
 */
 
static VALUE
gc_profile_report(int argc, VALUE *argv, VALUE self)
{
    VALUE out;
 
    out = (!rb_check_arity(argc, 0, 1) ? rb_stdout : argv[0]);
    gc_profile_dump_on(out, rb_io_write);
 
    return Qnil;
}
 
/*
 *  call-seq:
 *     GC::Profiler.total_time  -> float
 *
 *  The total time used for garbage collection in seconds
 */
 
static VALUE
gc_profile_total_time(VALUE self)
{
    double time = 0;
    rb_objspace_t *objspace = rb_gc_get_objspace();
 
    if (objspace->profile.run && gc_profile_record_count(objspace) > 0) {
        size_t i;
        size_t count = gc_profile_record_count(objspace);
 
        for (i = 0; i < count; i++) {
            time += objspace->profile.records[gc_profile_record_index(objspace, i)].gc_time;
        }
    }
    return DBL2NUM(time);
}
 
/*
 *  call-seq:
 *    GC::Profiler.enabled?     -> true or false
 *
 *  The current status of \GC profile mode.
 */
 
static VALUE
gc_profile_enable_get(VALUE self)
{
    rb_objspace_t *objspace = rb_gc_get_objspace();
    return objspace->profile.run ? Qtrue : Qfalse;
}
 
/*
 *  call-seq:
 *    GC::Profiler.enable       -> nil
 *
 *  Starts the \GC profiler.
 *
 */
 
static VALUE
gc_profile_enable(VALUE _)
{
    rb_objspace_t *objspace = rb_gc_get_objspace();
    objspace->profile.run = TRUE;
    objspace->profile.current_record = 0;
    return Qnil;
}
 
/*
 *  call-seq:
 *    GC::Profiler.disable      -> nil
 *
 *  Stops the \GC profiler.
 *
 */
 
static VALUE
gc_profile_disable(VALUE _)
{
    rb_objspace_t *objspace = rb_gc_get_objspace();
 
    objspace->profile.run = FALSE;
    objspace->profile.current_record = 0;
    return Qnil;
}
 
static void
rb_gc_verify_internal_consistency(void)
{
    gc_verify_internal_consistency(rb_gc_get_objspace());
}
 
/*
 *  call-seq:
 *     GC.verify_internal_consistency -> nil
 *
 *  Verifies internal consistency of the GC.
 *  This method should only be used for debugging.
 *
 *  This method is only expected to work on CRuby.
 */
static VALUE
gc_verify_internal_consistency_m(VALUE dummy)
{
    rb_gc_verify_internal_consistency();
    return Qnil;
}
 
#if GC_CAN_COMPILE_COMPACTION
/*
 *  call-seq:
 *     GC.auto_compact = flag
 *
 *  Updates automatic compaction mode.
 *
 *  When enabled, the compactor will execute on every major collection.
 *
 *  Enabling compaction will degrade performance on major collections.
 */
static VALUE
gc_set_auto_compact(VALUE _, VALUE v)
{
    GC_ASSERT(GC_COMPACTION_SUPPORTED);
 
    ruby_enable_autocompact = RTEST(v);
 
#if RGENGC_CHECK_MODE
    ruby_autocompact_compare_func = NULL;
 
    if (SYMBOL_P(v)) {
        ID id = RB_SYM2ID(v);
        if (id == rb_intern("empty")) {
            ruby_autocompact_compare_func = compare_free_slots;
        }
    }
#endif
 
    return v;
}
#else
#  define gc_set_auto_compact rb_f_notimplement
#endif
 
#if GC_CAN_COMPILE_COMPACTION
/*
 *  call-seq:
 *     GC.auto_compact    -> true or false
 *
 *  Returns whether or not automatic compaction has been enabled.
 */
static VALUE
gc_get_auto_compact(VALUE _)
{
    return ruby_enable_autocompact ? Qtrue : Qfalse;
}
#else
#  define gc_get_auto_compact rb_f_notimplement
#endif
 
#if GC_CAN_COMPILE_COMPACTION
/*
 *  call-seq:
 *     GC.latest_compact_info -> hash
 *
 * Returns information about object moved in the most recent \GC compaction.
 *
 * The returned +hash+ contains the following keys:
 *
 * [considered]
 *   Hash containing the type of the object as the key and the number of
 *   objects of that type that were considered for movement.
 * [moved]
 *   Hash containing the type of the object as the key and the number of
 *   objects of that type that were actually moved.
 * [moved_up]
 *   Hash containing the type of the object as the key and the number of
 *   objects of that type that were increased in size.
 * [moved_down]
 *   Hash containing the type of the object as the key and the number of
 *   objects of that type that were decreased in size.
 *
 * Some objects can't be moved (due to pinning) so these numbers can be used to
 * calculate compaction efficiency.
 */
static VALUE
gc_compact_stats(VALUE self)
{
    rb_objspace_t *objspace = rb_gc_get_objspace();
    VALUE h = rb_hash_new();
    VALUE considered = rb_hash_new();
    VALUE moved = rb_hash_new();
    VALUE moved_up = rb_hash_new();
    VALUE moved_down = rb_hash_new();
 
    for (size_t i = 0; i < T_MASK; i++) {
        if (objspace->rcompactor.considered_count_table[i]) {
            rb_hash_aset(considered, type_sym(i), SIZET2NUM(objspace->rcompactor.considered_count_table[i]));
        }
 
        if (objspace->rcompactor.moved_count_table[i]) {
            rb_hash_aset(moved, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_count_table[i]));
        }
 
        if (objspace->rcompactor.moved_up_count_table[i]) {
            rb_hash_aset(moved_up, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_up_count_table[i]));
        }
 
        if (objspace->rcompactor.moved_down_count_table[i]) {
            rb_hash_aset(moved_down, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_down_count_table[i]));
        }
    }
 
    rb_hash_aset(h, ID2SYM(rb_intern("considered")), considered);
    rb_hash_aset(h, ID2SYM(rb_intern("moved")), moved);
    rb_hash_aset(h, ID2SYM(rb_intern("moved_up")), moved_up);
    rb_hash_aset(h, ID2SYM(rb_intern("moved_down")), moved_down);
 
    return h;
}
#else
#  define gc_compact_stats rb_f_notimplement
#endif
 
#if GC_CAN_COMPILE_COMPACTION
/*
 *  call-seq:
 *     GC.compact -> hash
 *
 * This function compacts objects together in Ruby's heap. It eliminates
 * unused space (or fragmentation) in the heap by moving objects in to that
 * unused space. If there is more than 1 running Ractor, it runs a global
 * GC compaction (all object spaces).
 *
 * The returned +hash+ contains statistics about the objects that were moved;
 * see GC.latest_compact_info.
 *
 * This method is only expected to work on CRuby.
 *
 * To test whether \GC compaction is supported, use the idiom:
 *
 *   GC.respond_to?(:compact)
 */
static VALUE
gc_compact(VALUE self)
{
    rb_objspace_t *objspace = rb_gc_get_objspace();
    int full_marking_p = gc_config_full_mark_val;
    gc_config_full_mark_set(TRUE);
 
    /* Run GC with compaction enabled */
    rb_gc_impl_start(rb_gc_get_objspace(), true, true, true, true, true);
    gc_config_full_mark_set(full_marking_p);
 
    return gc_compact_stats(self);
}
#else
#  define gc_compact rb_f_notimplement
#endif
 
#if GC_CAN_COMPILE_COMPACTION
struct desired_compaction_pages_i_data {
    rb_objspace_t *objspace;
    size_t required_slots[HEAP_COUNT];
};
 
static int
desired_compaction_pages_i(struct heap_page *page, void *data)
{
    struct desired_compaction_pages_i_data *tdata = data;
    rb_objspace_t *objspace = tdata->objspace;
    VALUE vstart = (VALUE)page->start;
    VALUE vend = vstart + (VALUE)(page->total_slots * page->heap->slot_size);
 
 
    for (VALUE v = vstart; v != vend; v += page->heap->slot_size) {
        asan_unpoisoning_object(v) {
            /* skip T_NONEs; they won't be moved */
            if (BUILTIN_TYPE(v) != T_NONE) {
                rb_heap_t *dest_pool = gc_compact_destination_pool(objspace, page->heap, v);
                size_t dest_pool_idx = dest_pool - heaps;
                tdata->required_slots[dest_pool_idx]++;
            }
        }
    }
 
    return 0;
}
 
/* call-seq:
 *    GC.verify_compaction_references(toward: nil, double_heap: false) -> hash
 *
 * Verify compaction reference consistency.
 *
 * This method is implementation specific.  During compaction, objects that
 * were moved are replaced with T_MOVED objects.  No object should have a
 * reference to a T_MOVED object after compaction.
 *
 * This function expands the heap to ensure room to move all objects,
 * compacts the heap to make sure everything moves, updates all references,
 * then performs a full \GC.  If any object contains a reference to a T_MOVED
 * object, that object should be pushed on the mark stack, and will
 * make a SEGV.
 */
static VALUE
gc_verify_compaction_references(int argc, VALUE* argv, VALUE self)
{
    static ID keywords[3] = {0};
    if (!keywords[0]) {
        keywords[0] = rb_intern("toward");
        keywords[1] = rb_intern("double_heap");
        keywords[2] = rb_intern("expand_heap");
    }
 
    VALUE options;
    rb_scan_args_kw(rb_keyword_given_p(), argc, argv, ":", &options);
 
    VALUE arguments[3] = { Qnil, Qfalse, Qfalse };
    int kwarg_count = rb_get_kwargs(options, keywords, 0, 3, arguments);
    bool toward_empty = kwarg_count > 0 && SYMBOL_P(arguments[0]) && SYM2ID(arguments[0]) == rb_intern("empty");
    bool expand_heap = (kwarg_count > 1 && RTEST(arguments[1])) || (kwarg_count > 2 && RTEST(arguments[2]));
 
    rb_objspace_t *objspace = rb_gc_get_objspace();
 
    /* This verification machinery (heap expansion, toward_empty page ordering, the
     * moved-reference walk) is built for a single objspace, so with several demote it
     * to a plain full GC.  Plain GC.compact does compact them via the global GC. */
    if (!rb_gc_single_objspace_p()) {
        rb_gc_impl_start(objspace, true, true, true, false, false);
        return gc_compact_stats(self);
    }
 
    /* Clear the heap. */
    rb_gc_impl_start(objspace, true, true, true, false, false);
 
    unsigned int lev = RB_GC_VM_LOCK();
    {
        gc_rest(objspace);
 
        /* if both double_heap and expand_heap are set, expand_heap takes precedence */
        if (expand_heap) {
            struct desired_compaction_pages_i_data desired_compaction = {
                .objspace = objspace,
                .required_slots = {0},
            };
            /* Work out how many objects want to be in each size pool, taking account of moves */
            objspace_each_pages(objspace, desired_compaction_pages_i, &desired_compaction, TRUE);
 
            /* Find out which pool has the most pages */
            size_t max_existing_pages = 0;
            for (int i = 0; i < HEAP_COUNT; i++) {
                rb_heap_t *heap = &heaps[i];
                max_existing_pages = MAX(max_existing_pages, heap->total_pages);
            }
 
            /* Add pages to each size pool so that compaction is guaranteed to move every object */
            for (int i = 0; i < HEAP_COUNT; i++) {
                rb_heap_t *heap = &heaps[i];
 
                size_t pages_to_add = 0;
                /*
                 * Step 1: Make sure every pool has the same number of pages, by adding empty pages
                 * to smaller pools. This is required to make sure the compact cursor can advance
                 * through all of the pools in `gc_sweep_compact` without hitting the "sweep &
                 * compact cursors met" condition on some pools before fully compacting others
                 */
                pages_to_add += max_existing_pages - heap->total_pages;
                /*
                 * Step 2: Now add additional free pages to each size pool sufficient to hold all objects
                 * that want to be in that size pool, whether moved into it or moved within it
                 */
                objspace->heap_pages.allocatable_bytes = desired_compaction.required_slots[i] * heap->slot_size;
                while (objspace->heap_pages.allocatable_bytes > 0) {
                    heap_page_allocate_and_initialize(objspace, heap);
                }
                /*
                 * Step 3: Add two more pages so that the compact & sweep cursors will meet _after_ all objects
                 * have been moved, and not on the last iteration of the `gc_sweep_compact` loop
                 */
                pages_to_add += 2;
 
                for (; pages_to_add > 0; pages_to_add--) {
                    heap_page_allocate_and_initialize_force(objspace, heap);
                }
            }
        }
 
        if (toward_empty) {
            objspace->rcompactor.compare_func = compare_free_slots;
        }
    }
    RB_GC_VM_UNLOCK(lev);
 
    rb_gc_impl_start(rb_gc_get_objspace(), true, true, true, true, false);
 
    rb_objspace_reachable_objects_from_root(root_obj_check_moved_i, objspace);
    objspace_each_objects(objspace, heap_check_moved_i, objspace, TRUE);
 
    objspace->rcompactor.compare_func = NULL;
 
    return gc_compact_stats(self);
}
#else
# define gc_verify_compaction_references rb_f_notimplement
#endif
 
void
rb_gc_impl_objspace_free(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    if (is_lazy_sweeping(objspace))
        rb_bug("lazy sweeping underway when freeing object space");
 
    free(objspace->profile.records);
    objspace->profile.records = NULL;
 
    for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
        heap_page_free(objspace, rb_darray_get(objspace->heap_pages.sorted, i));
    }
    rb_darray_free_without_gc(objspace->heap_pages.sorted);
    heap_pages_lomem = 0;
    heap_pages_himem = 0;
 
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
        heap->total_pages = 0;
        heap->total_slots = 0;
    }
 
    free_stack_chunks(&objspace->mark_stack);
    mark_stack_free_cache(&objspace->mark_stack);
 
    rb_darray_free_without_gc(objspace->weak_references);
 
#ifdef MALLOC_COUNTERS_NEED_LOCK
    rb_native_mutex_destroy(&objspace->malloc_counters.lock);
#endif
 
    rb_native_mutex_destroy(&objspace->process_stat.lock);
 
    free(objspace);
}
 
#if MALLOC_ALLOCATED_SIZE
/*
 *  call-seq:
 *     GC.malloc_allocated_size -> Integer
 *
 *  Returns the size of memory allocated by malloc().
 *
 *  Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
 */
 
static VALUE
gc_malloc_allocated_size(VALUE self)
{
    rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace();
    return ULL2NUM(objspace->malloc_params.allocated_size);
}
 
/*
 *  call-seq:
 *     GC.malloc_allocations -> Integer
 *
 *  Returns the number of malloc() allocations.
 *
 *  Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
 */
 
static VALUE
gc_malloc_allocations(VALUE self)
{
    rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace();
    return ULL2NUM(objspace->malloc_params.allocations);
}
#endif
 
void
rb_gc_impl_before_fork(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    objspace->fork_vm_lock_lev = RB_GC_VM_LOCK();
    rb_gc_vm_barrier();
}
 
void
rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    if (pid == 0) {
        rb_gc_vm_each_objspace(gc_process_stat_after_fork_i, NULL);
    }
 
    RB_GC_VM_UNLOCK(objspace->fork_vm_lock_lev);
    objspace->fork_vm_lock_lev = 0;
 
    if (pid == 0) { /* child process */
        heap_alloc_state_clear(objspace);
        /* The forking Ractor becomes the child process's main Ractor. */
        global_objspace->main_objspace = objspace;
        page_pool_lock_initialize(&rb_global_objspace_instance.page_pool.lock);
    }
}
 
VALUE rb_ident_hash_new_capa(rb_len_t size);
 
#if GC_DEBUG_STRESS_TO_CLASS
/*
 *  call-seq:
 *    GC.add_stress_to_class(class[, ...])
 *
 *  Raises NoMemoryError when allocating an instance of the given classes.
 *
 */
static VALUE
rb_gcdebug_add_stress_to_class(int argc, VALUE *argv, VALUE self)
{
    rb_objspace_t *objspace = rb_gc_get_objspace();
 
    if (!stress_to_class) {
        set_stress_to_class(rb_ident_hash_new_capa(argc));
    }
 
    for (int i = 0; i < argc; i++) {
        VALUE klass = argv[i];
        rb_hash_aset(stress_to_class, klass, Qtrue);
    }
 
    return self;
}
 
/*
 *  call-seq:
 *    GC.remove_stress_to_class(class[, ...])
 *
 *  No longer raises NoMemoryError when allocating an instance of the
 *  given classes.
 *
 */
static VALUE
rb_gcdebug_remove_stress_to_class(int argc, VALUE *argv, VALUE self)
{
    rb_objspace_t *objspace = rb_gc_get_objspace();
 
    if (stress_to_class) {
        for (int i = 0; i < argc; ++i) {
            rb_hash_delete(stress_to_class, argv[i]);
        }
 
        if (rb_hash_size(stress_to_class) == 0) {
            stress_to_class = 0;
        }
    }
 
    return Qnil;
}
#endif
 
void *
rb_gc_impl_objspace_alloc(void)
{
    global_objspace_init();
 
    rb_objspace_t *objspace = calloc1(sizeof(rb_objspace_t));
    if (objspace) {
        rb_native_mutex_initialize(&objspace->process_stat.lock);
    }
 
    return objspace;
}
 
void
rb_gc_impl_objspace_init(void *objspace_ptr)
{
    rb_objspace_t *objspace = objspace_ptr;
 
    gc_config_full_mark_set(TRUE);
 
    malloc_limit = gc_params.malloc_limit_min;
    objspace->shareable_objects_limit = SHAREABLE_OBJECTS_LIMIT_MIN;
#ifdef MALLOC_COUNTERS_NEED_LOCK
    rb_native_mutex_initialize(&objspace->malloc_counters.lock);
#endif
    /* Shared by every objspace.  preregister deduplicates on (func, data). */
    objspace->finalize_deferred_pjob = rb_postponed_job_preregister(0, gc_finalize_deferred, NULL);
    if (objspace->finalize_deferred_pjob == POSTPONED_JOB_HANDLE_INVALID) {
        rb_bug("Could not preregister postponed job for GC");
    }
 
    gc_tdata_deferred_free_pjob_ensure();
 
    /* A standard RVALUE (RBasic + embedded VALUEs + debug overhead) must fit
     * in at least one pool.  In debug builds RVALUE_OVERHEAD can push this
     * beyond the 48-byte pool into the 64-byte pool, which is fine. */
    GC_ASSERT(rb_gc_impl_size_allocatable_p(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX])));
 
    for (int i = 0; i < HEAP_COUNT; i++) {
        rb_heap_t *heap = &heaps[i];
 
        heap->slot_size = pool_slot_sizes[i];
 
        ccan_list_head_init(&heap->pages);
    }
 
    if (global_objspace->main_objspace == NULL) {
        /* Single-threaded at boot and the first objspace is main's: compute process-wide
         * constants once here.  A later objspace_init rewriting them, even with equal
         * values, would race other threads' lock-free reads. */
        global_objspace->main_objspace = objspace;
 
        init_size_to_heap_idx();
 
#if defined(INIT_HEAP_PAGE_ALLOC_USE_MMAP)
        /* Need to determine if we can use mmap at runtime. */
        heap_page_alloc_use_mmap = INIT_HEAP_PAGE_ALLOC_USE_MMAP;
#endif
        gc_params.heap_init_bytes = GC_HEAP_INIT_BYTES;
        gc_params.ractor_heap_init_bytes = GC_RACTOR_HEAP_INIT_BYTES ? GC_RACTOR_HEAP_INIT_BYTES
                                                                     : heap_init_bytes_min();
    }
    // GC.measure_total_time= sets the caller's objspace only; a new Ractor's follows
    // its creator's, which is the objspace running this init (main starts it on).
    objspace->flags.measure_gc = global_objspace->main_objspace == objspace ? true
                                 : ((rb_objspace_t *)rb_gc_get_objspace())->flags.measure_gc;
 
    rb_darray_make_without_gc(&objspace->heap_pages.sorted, 0);
    rb_darray_make_without_gc(&objspace->weak_references, 0);
 
#if RGENGC_ESTIMATE_OLDMALLOC
    objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
#endif
 
    init_mark_stack(&objspace->mark_stack);
 
    objspace->profile.invoke_time = getrusage_time();
    objspace->profile.invoke_wall_time = rb_hrtime_now();
    objspace->profile.max_records = GC_PROFILE_RECORD_DEFAULT_MAX_RECORDS;
    finalizer_table = st_init_numtable();
 
    gc_process_stat_publish(objspace);
}
 
void
rb_gc_impl_init(void)
{
    /* Fill the symbol tables here, where no other ractor exists yet: they used to
     * be filled on first use, guarded by their own first element, so a second
     * ractor could see a half-filled table and GC.stat raised on the rest. */
    setup_gc_stat_symbols();
    setup_gc_stat_heap_symbols();
    setup_gc_latest_gc_info_symbols();
 
    VALUE gc_constants = rb_hash_new();
    rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), GC_DEBUG ? Qtrue : Qfalse);
    /* Minimum slot size that fits a standard RVALUE */
    size_t rvalue_pool = 0;
    for (size_t i = 0; i < HEAP_COUNT; i++) {
        if (pool_slot_sizes[i] >= RVALUE_SLOT_SIZE) { rvalue_pool = pool_slot_sizes[i]; break; }
    }
    rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(rvalue_pool - RVALUE_OVERHEAD));
    rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic)));
    rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD));
    rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE));
    rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_SIZE")), SIZET2NUM(HEAP_PAGE_SIZE));
    rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(HEAP_COUNT));
    rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), SIZET2NUM(rb_gc_impl_max_allocation_size()));
    rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), LONG2FIX(RVALUE_OLD_AGE));
    if (RB_BUG_INSTEAD_OF_RB_MEMERROR+0) {
        rb_hash_aset(gc_constants, ID2SYM(rb_intern("RB_BUG_INSTEAD_OF_RB_MEMERROR")), Qtrue);
    }
    OBJ_FREEZE(gc_constants);
    /* Internal constants in the garbage collector. */
    rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants);
 
    if (GC_COMPACTION_SUPPORTED) {
        rb_define_singleton_method(rb_mGC, "compact", gc_compact, 0);
        rb_define_singleton_method(rb_mGC, "auto_compact", gc_get_auto_compact, 0);
        rb_define_singleton_method(rb_mGC, "auto_compact=", gc_set_auto_compact, 1);
        rb_define_singleton_method(rb_mGC, "latest_compact_info", gc_compact_stats, 0);
        rb_define_singleton_method(rb_mGC, "verify_compaction_references", gc_verify_compaction_references, -1);
    }
    else {
        rb_define_singleton_method(rb_mGC, "compact", rb_f_notimplement, 0);
        rb_define_singleton_method(rb_mGC, "auto_compact", rb_f_notimplement, 0);
        rb_define_singleton_method(rb_mGC, "auto_compact=", rb_f_notimplement, 1);
        rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0);
        rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1);
    }
 
#if GC_DEBUG_STRESS_TO_CLASS
    rb_define_singleton_method(rb_mGC, "add_stress_to_class", rb_gcdebug_add_stress_to_class, -1);
    rb_define_singleton_method(rb_mGC, "remove_stress_to_class", rb_gcdebug_remove_stress_to_class, -1);
#endif
 
    /* internal methods */
    rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency_m, 0);
 
#if MALLOC_ALLOCATED_SIZE
    rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0);
    rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0);
#endif
 
    /*  Document-class: GC::Profiler
     *
     *  The GC profiler provides access to information on GC runs including time,
     *  length and object space size.
     *
     *  Example:
     *
     *    GC::Profiler.enable
     *
     *    require 'rdoc/rdoc'
     *
     *    GC::Profiler.report
     *
     *    pp GC::Profiler.raw_data
     *
     *    GC::Profiler.disable
     *
     *  GC::Profiler.raw_data returns one Hash per GC run, including CPU time
     *  fields such as +:GC_TIME+ and wall-clock fields such as +:GC_WALL_TIME+,
     *  +:GC_PAUSE_TIME+, +:GC_STOP_TIME+, and +:GC_STW_TIME+.  +:GC_WALL_TIME+
     *  is the wall-clock counterpart to +:GC_TIME+, while +:GC_PAUSE_TIME+
     *  measures how long user execution was blocked by the GC entry.
     *
     *  See also GC.count, GC.malloc_allocated_size and GC.malloc_allocations
     */
    VALUE rb_mProfiler = rb_define_module_under(rb_mGC, "Profiler");
    rb_define_singleton_method(rb_mProfiler, "enabled?", gc_profile_enable_get, 0);
    rb_define_singleton_method(rb_mProfiler, "enable", gc_profile_enable, 0);
    rb_define_singleton_method(rb_mProfiler, "raw_data", gc_profile_record_get, -1);
    rb_define_singleton_method(rb_mProfiler, "disable", gc_profile_disable, 0);
    rb_define_singleton_method(rb_mProfiler, "clear", gc_profile_clear, 0);
    rb_define_singleton_method(rb_mProfiler, "configure", gc_profile_configure, -1);
    rb_define_singleton_method(rb_mProfiler, "result", gc_profile_result, 0);
    rb_define_singleton_method(rb_mProfiler, "report", gc_profile_report, -1);
    rb_define_singleton_method(rb_mProfiler, "total_time", gc_profile_total_time, 0);
 
    {
        VALUE opts;
        /* \GC build options */
        rb_define_const(rb_mGC, "OPTS", opts = rb_ary_new());
#define OPT(o) if (o) rb_ary_push(opts, rb_interned_str(#o, sizeof(#o) - 1))
        OPT(GC_DEBUG);
        OPT(USE_RGENGC);
        OPT(RGENGC_DEBUG);
        OPT(RGENGC_CHECK_MODE);
        OPT(RGENGC_PROFILE);
        OPT(RGENGC_ESTIMATE_OLDMALLOC);
        OPT(GC_PROFILE_MORE_DETAIL);
        OPT(GC_ENABLE_LAZY_SWEEP);
        OPT(CALC_EXACT_MALLOC_SIZE);
        OPT(MALLOC_ALLOCATED_SIZE);
        OPT(MALLOC_ALLOCATED_SIZE_CHECK);
        OPT(GC_PROFILE_DETAIL_MEMORY);
        OPT(GC_COMPACTION_SUPPORTED);
#undef OPT
        OBJ_FREEZE(opts);
    }
}