mmtk.c

#include <pthread.h>
#include <stdbool.h>
#include <string.h>
 
#include "ruby/assert.h"
#include "ruby/atomic.h"
#include "ruby/debug.h"
 
#include "gc/gc.h"
#include "gc/gc_impl.h"
#include "gc/mmtk/mmtk.h"
 
#if USE_ZJIT
# include "gc/mmtk/zjit_fastpath.h"
#endif
 
#include "ccan/list/list.h"
#include "darray.h"
 
#ifdef __APPLE__
#include <sys/sysctl.h>
#endif
 
struct objspace {
    bool user_gc_disabled;
    bool measure_gc_time;
    bool gc_stress;
 
    size_t gc_count;
    size_t moving_gc_count;
    size_t total_gc_time;
    size_t total_allocated_objects;
 
    st_table *finalizer_table;
    struct MMTk_final_job *finalizer_jobs;
    rb_postponed_job_handle_t finalizer_postponed_job;
 
    struct ccan_list_head ractor_caches;
    unsigned long live_ractor_cache_count;
 
    pthread_mutex_t mutex;
    rb_atomic_t mutator_blocking_count;
    bool world_stopped;
    pthread_cond_t cond_world_stopped;
    pthread_cond_t cond_world_started;
    size_t start_the_world_count;
 
    pthread_mutex_t event_hook_mutex;
 
    struct {
        bool gc_thread_crashed;
        char crash_msg[256];
    } crash_context;
 
    struct rb_gc_vm_context vm_context;
 
    unsigned int fork_hook_vm_lock_lev;
 
    uintptr_t vo_bit_log_region_size;
    uintptr_t vo_bit_base_addr;
    size_t max_non_los_default_alloc_bytes;
};
 
#define OBJ_FREE_BUF_CAPACITY 128
 
struct MMTk_ractor_cache {
    struct ccan_list_node list_node;
 
    MMTk_Mutator *mutator;
    bool gc_mutator_p;
 
    MMTk_BumpPointer *bump_pointer;
 
    MMTk_ObjectReference obj_free_parallel_buf[OBJ_FREE_BUF_CAPACITY];
    size_t obj_free_parallel_count;
    MMTk_ObjectReference obj_free_non_parallel_buf[OBJ_FREE_BUF_CAPACITY];
    size_t obj_free_non_parallel_count;
};
 
struct MMTk_final_job {
    struct MMTk_final_job *next;
    enum {
        MMTK_FINAL_JOB_DFREE,
        MMTK_FINAL_JOB_FINALIZE,
    } kind;
    union {
        struct {
            void (*func)(void *);
            void *data;
        } dfree;
        struct {
            /* HACK: we store the object ID on the 0th element of this array. */
            VALUE finalizer_array;
        } finalize;
    } as;
};
 
#ifdef RB_THREAD_LOCAL_SPECIFIER
RB_THREAD_LOCAL_SPECIFIER struct MMTk_GCThreadTLS *rb_mmtk_gc_thread_tls;
 
RB_THREAD_LOCAL_SPECIFIER VALUE marking_parent_object;
#else
# error We currently need language-supported TLS
#endif
 
#ifdef MMTK_DEBUG
# define MMTK_ASSERT(expr, ...) RUBY_ASSERT_ALWAYS(expr, #expr RBIMPL_VA_OPT_ARGS(__VA_ARGS__))
#else
# define MMTK_ASSERT(expr, ...) ((void)0)
#endif
 
#include <pthread.h>
 
#define MMTK_ALLOCATION_SEMANTICS_DEFAULT 0
#define MMTK_ALLOCATION_SEMANTICS_LOS 2
 
/* Functions called from the Rust binding (upcalls).  They are not static so
 * that the linker can resolve them from libmmtk_ruby.a, but they are hidden
 * so that calls do not go through the PLT and so that LTO can inline them
 * into their Rust callers. */
#ifdef __GNUC__
# define RB_MMTK_UPCALL __attribute__((visibility("hidden")))
#else
# define RB_MMTK_UPCALL
#endif
 
static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin);
 
RB_MMTK_UPCALL void
rb_mmtk_init_gc_worker_thread(MMTk_VMWorkerThread gc_thread_tls)
{
    rb_mmtk_gc_thread_tls = gc_thread_tls;
}
 
RB_MMTK_UPCALL bool
rb_mmtk_is_mutator(void)
{
    return ruby_native_thread_p();
}
 
RB_MMTK_UPCALL void
rb_mmtk_stop_the_world(void)
{
    struct objspace *objspace = rb_gc_get_objspace();
 
    int err;
    if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) {
        rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err));
    }
 
    while (!objspace->world_stopped) {
        pthread_cond_wait(&objspace->cond_world_stopped, &objspace->mutex);
    }
 
    if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) {
        rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err));
    }
}
 
RB_MMTK_UPCALL void
rb_mmtk_resume_mutators(bool current_gc_may_move)
{
    struct objspace *objspace = rb_gc_get_objspace();
 
    int err;
    if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) {
        rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err));
    }
 
    objspace->world_stopped = false;
    objspace->gc_count++;
    if (current_gc_may_move) objspace->moving_gc_count++;
    pthread_cond_broadcast(&objspace->cond_world_started);
 
    if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) {
        rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err));
    }
}
 
static void mmtk_flush_obj_free_buffer(struct MMTk_ractor_cache *cache);
 
RB_MMTK_UPCALL void
rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator)
{
    struct objspace *objspace = rb_gc_get_objspace();
 
    size_t starting_gc_count = objspace->gc_count;
    RUBY_ATOMIC_INC(objspace->mutator_blocking_count);
    int lock_lev = RB_GC_VM_LOCK();
    RUBY_ATOMIC_DEC(objspace->mutator_blocking_count);
    int err;
    if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) {
        rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err));
    }
 
    if (objspace->gc_count == starting_gc_count) {
        rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_START);
 
        rb_gc_initialize_vm_context(&objspace->vm_context);
 
        mutator->gc_mutator_p = true;
 
        struct timespec gc_start_time;
        if (objspace->measure_gc_time) {
            clock_gettime(CLOCK_MONOTONIC, &gc_start_time);
        }
 
        rb_gc_save_machine_context();
 
        rb_gc_vm_barrier();
 
        struct MMTk_ractor_cache *rc;
        ccan_list_for_each(&objspace->ractor_caches, rc, list_node) {
            mmtk_flush_obj_free_buffer(rc);
        }
 
        objspace->world_stopped = true;
 
        pthread_cond_broadcast(&objspace->cond_world_stopped);
 
        // Wait for GC end
        while (objspace->world_stopped) {
            pthread_cond_wait(&objspace->cond_world_started, &objspace->mutex);
        }
 
        if (RB_UNLIKELY(objspace->crash_context.gc_thread_crashed)) {
            rb_bug("%s", objspace->crash_context.crash_msg);
        }
 
        if (objspace->measure_gc_time) {
            struct timespec gc_end_time;
            clock_gettime(CLOCK_MONOTONIC, &gc_end_time);
 
            objspace->total_gc_time +=
                (gc_end_time.tv_sec - gc_start_time.tv_sec) * (1000 * 1000 * 1000) +
                    (gc_end_time.tv_nsec - gc_start_time.tv_nsec);
        }
    }
 
    if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) {
        rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err));
    }
    RB_GC_VM_UNLOCK(lock_lev);
}
 
RB_MMTK_UPCALL void
rb_mmtk_before_updating_jit_code(void)
{
    rb_gc_before_updating_jit_code();
}
 
RB_MMTK_UPCALL void
rb_mmtk_after_updating_jit_code(void)
{
    rb_gc_after_updating_jit_code();
}
 
RB_MMTK_UPCALL size_t
rb_mmtk_number_of_mutators(void)
{
    struct objspace *objspace = rb_gc_get_objspace();
    return objspace->live_ractor_cache_count;
}
 
RB_MMTK_UPCALL void
rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator *mutator, void *data), void *data)
{
    struct objspace *objspace = rb_gc_get_objspace();
    struct MMTk_ractor_cache *ractor_cache;
 
    ccan_list_for_each(&objspace->ractor_caches, ractor_cache, list_node) {
        visit_mutator(ractor_cache->mutator, data);
    }
}
 
RB_MMTK_UPCALL void
rb_mmtk_scan_gc_roots(void)
{
    struct objspace *objspace = rb_gc_get_objspace();
 
    rb_gc_mark_roots(objspace, NULL);
}
 
static int
pin_value(st_data_t key, st_data_t value, st_data_t data)
{
    rb_gc_impl_mark_and_pin((void *)data, (VALUE)value);
 
    return ST_CONTINUE;
}
 
RB_MMTK_UPCALL void
rb_mmtk_scan_objspace(void)
{
    struct objspace *objspace = rb_gc_get_objspace();
 
    if (objspace->finalizer_table != NULL) {
        st_foreach(objspace->finalizer_table, pin_value, (st_data_t)objspace);
    }
 
    struct MMTk_final_job *job = objspace->finalizer_jobs;
    while (job != NULL) {
        switch (job->kind) {
          case MMTK_FINAL_JOB_DFREE:
            break;
          case MMTK_FINAL_JOB_FINALIZE:
            rb_gc_impl_mark(objspace, job->as.finalize.finalizer_array);
            break;
          default:
            rb_bug("rb_mmtk_scan_objspace: unknown final job type %d", job->kind);
        }
 
        job = job->next;
    }
}
 
RB_MMTK_UPCALL void
rb_mmtk_move_obj_during_marking(MMTk_ObjectReference from, MMTk_ObjectReference to)
{
    rb_gc_move_obj_during_marking((VALUE)from, (VALUE)to);
 
    size_t from_slot_size = rb_gc_impl_obj_slot_size((VALUE)from);
    size_t to_slot_size = rb_gc_impl_obj_slot_size((VALUE)to);
    if (from_slot_size != to_slot_size) {
        /* The object was copied to a differently sized slot; transition the
         * shape so its capacity matches the new slot size.  The re-embedding
         * of contents (e.g. arrays, strings, object fields) happens later
         * during reference updating. */
        rb_gc_obj_changed_slot_size((VALUE)to, to_slot_size);
    }
}
 
RB_MMTK_UPCALL size_t
rb_mmtk_obj_optimal_size(MMTk_ObjectReference obj)
{
    return rb_gc_obj_optimal_size((VALUE)obj);
}
 
RB_MMTK_UPCALL void
rb_mmtk_update_object_references(MMTk_ObjectReference mmtk_object)
{
    VALUE object = (VALUE)mmtk_object;
 
    if (!RB_FL_TEST(object, RUBY_FL_WEAK_REFERENCE)) {
        marking_parent_object = object;
        rb_gc_update_object_references(rb_gc_get_objspace(), object);
        marking_parent_object = 0;
    }
}
 
RB_MMTK_UPCALL void
rb_mmtk_call_gc_mark_children(MMTk_ObjectReference object)
{
    marking_parent_object = (VALUE)object;
    rb_gc_mark_children(rb_gc_get_objspace(), (VALUE)object);
    marking_parent_object = 0;
}
 
RB_MMTK_UPCALL void
rb_mmtk_handle_weak_references(MMTk_ObjectReference mmtk_object, bool moving)
{
    VALUE object = (VALUE)mmtk_object;
 
    marking_parent_object = object;
 
    rb_gc_handle_weak_references(object);
 
    if (moving) {
        rb_gc_update_object_references(rb_gc_get_objspace(), object);
    }
 
    marking_parent_object = 0;
}
 
RB_MMTK_UPCALL void
rb_mmtk_call_obj_free(MMTk_ObjectReference object)
{
    VALUE obj = (VALUE)object;
    struct objspace *objspace = rb_gc_get_objspace();
 
    if (RB_UNLIKELY(rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_FREEOBJ))) {
        pthread_mutex_lock(&objspace->event_hook_mutex);
        rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_FREEOBJ);
        pthread_mutex_unlock(&objspace->event_hook_mutex);
    }
 
    if (RB_UNLIKELY(rb_gc_obj_needs_cleanup_p(obj))) {
        rb_gc_obj_free(objspace, obj);
    }
 
#ifdef MMTK_DEBUG
    memset((void *)obj, 0, rb_gc_impl_obj_slot_size(obj));
#endif
}
 
RB_MMTK_UPCALL size_t
rb_mmtk_vm_live_bytes(void)
{
    return 0;
}
 
static void
make_final_job(struct objspace *objspace, VALUE obj, VALUE table)
{
    MMTK_ASSERT(RB_BUILTIN_TYPE(table) == T_ARRAY);
 
    struct MMTk_final_job *job = xmalloc(sizeof(struct MMTk_final_job));
    job->next = objspace->finalizer_jobs;
    job->kind = MMTK_FINAL_JOB_FINALIZE;
    job->as.finalize.finalizer_array = table;
 
    objspace->finalizer_jobs = job;
}
 
static int
rb_mmtk_update_finalizer_table_i(st_data_t key, st_data_t value, st_data_t data, int error)
{
    MMTK_ASSERT(mmtk_is_reachable((MMTk_ObjectReference)value));
    MMTK_ASSERT(RB_BUILTIN_TYPE(value) == T_ARRAY);
 
    struct objspace *objspace = (struct objspace *)data;
 
    if (mmtk_is_reachable((MMTk_ObjectReference)key)) {
        VALUE new_key_location = rb_mmtk_call_object_closure((VALUE)key, false);
 
        MMTK_ASSERT(RB_FL_TEST(new_key_location, RUBY_FL_FINALIZE));
 
        if (new_key_location != key) {
            return ST_REPLACE;
        }
    }
    else {
        make_final_job(objspace, (VALUE)key, (VALUE)value);
 
        rb_postponed_job_trigger(objspace->finalizer_postponed_job);
 
        return ST_DELETE;
    }
 
    return ST_CONTINUE;
}
 
static int
rb_mmtk_update_finalizer_table_replace_i(st_data_t *key, st_data_t *value, st_data_t data, int existing)
{
    *key = rb_mmtk_call_object_closure((VALUE)*key, false);
 
    return ST_CONTINUE;
}
 
RB_MMTK_UPCALL void
rb_mmtk_update_finalizer_table(void)
{
    struct objspace *objspace = rb_gc_get_objspace();
 
    st_foreach_with_replace(
        objspace->finalizer_table,
        rb_mmtk_update_finalizer_table_i,
        rb_mmtk_update_finalizer_table_replace_i,
        (st_data_t)objspace
    );
}
 
RB_MMTK_UPCALL int
rb_mmtk_global_tables_count(void)
{
    return RB_GC_VM_WEAK_TABLE_COUNT;
}
 
static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin);
 
static int
rb_mmtk_update_global_tables_i(VALUE val, void *data)
{
    if (!mmtk_is_reachable((MMTk_ObjectReference)val)) {
        return ST_DELETE;
    }
 
    // TODO: check only if in moving GC
    if (rb_mmtk_call_object_closure(val, false) != val) {
        return ST_REPLACE;
    }
 
    return ST_CONTINUE;
}
 
static int
rb_mmtk_update_global_tables_replace_i(VALUE *ptr, void *data)
{
    // TODO: cache the new location so we don't call rb_mmtk_call_object_closure twice
    *ptr = rb_mmtk_call_object_closure(*ptr, false);
 
    return ST_CONTINUE;
}
 
RB_MMTK_UPCALL void
rb_mmtk_update_global_tables(int table, bool moving)
{
    MMTK_ASSERT(table < RB_GC_VM_WEAK_TABLE_COUNT);
 
    rb_gc_vm_weak_table_foreach(
        rb_mmtk_update_global_tables_i,
        rb_mmtk_update_global_tables_replace_i,
        NULL,
        !moving,
        (enum rb_gc_vm_weak_tables)table
    );
}
 
RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 1, 2)
RBIMPL_ATTR_NORETURN()
static void
rb_mmtk_gc_thread_bug(const char *msg, ...)
{
    struct objspace *objspace = rb_gc_get_objspace();
 
    objspace->crash_context.gc_thread_crashed = true;
 
    va_list args;
    va_start(args, msg);
    vsnprintf(objspace->crash_context.crash_msg, sizeof(objspace->crash_context.crash_msg), msg, args);
    va_end(args);
 
    fprintf(stderr, "-- GC thread backtrace "
                    "-------------------------------------------\n");
    rb_gc_print_backtrace();
    fprintf(stderr, "\n");
 
    rb_mmtk_resume_mutators(false);
 
    sleep(5);
 
    rb_bug("rb_mmtk_gc_thread_bug");
}
 
RBIMPL_ATTR_NORETURN()
RB_MMTK_UPCALL void
rb_mmtk_gc_thread_panic_handler(void)
{
    rb_mmtk_gc_thread_bug("MMTk GC thread panicked");
}
 
RBIMPL_ATTR_NORETURN()
RB_MMTK_UPCALL void
rb_mmtk_mutator_thread_panic_handler(void)
{
    rb_bug("Ruby mutator thread panicked");
}
 
// Bootup
 
// Use max 80% of the available memory by default for MMTk
#define RB_MMTK_HEAP_LIMIT_PERC 80
#define RB_MMTK_DEFAULT_HEAP_MIN (1024 * 1024)
#define RB_MMTK_DEFAULT_HEAP_MAX (rb_mmtk_system_physical_memory() / 100 * RB_MMTK_HEAP_LIMIT_PERC)
 
enum mmtk_heap_mode {
    RB_MMTK_DYNAMIC_HEAP,
    RB_MMTK_FIXED_HEAP
};
 
MMTk_Builder *
rb_mmtk_builder_init(void)
{
    MMTk_Builder *builder = mmtk_builder_default();
    return builder;
}
 
 
void *
rb_gc_impl_objspace_alloc(void)
{
    /* One heap, binding and objspace per process (multi_objspace_p=false).  The VM
     * shares the same objspace across all Ractors, so a re-entry returns the same
     * instance. */
    static struct objspace *the_objspace = NULL;
    if (the_objspace == NULL) {
        MMTk_Builder *builder = rb_mmtk_builder_init();
        MMTk_RubyBindingOptions binding_options = {
            .suffix_size = 0,
        };
        mmtk_init_binding(builder, &binding_options);
        the_objspace = calloc(1, sizeof(struct objspace));
    }
 
    return the_objspace;
}
 
bool
rb_gc_impl_multi_objspace_p(void)
{
    return false;
}
 
static void gc_run_finalizers(void *data);
 
void
rb_gc_impl_objspace_init(void *objspace_ptr)
{
    struct objspace *objspace = objspace_ptr;
 
    /* The objspace is a singleton (see rb_gc_impl_objspace_alloc).  A re-init must
     * not clobber finalizer_table or ractor_caches. */
    if (objspace->finalizer_table != NULL) return;
 
    objspace->measure_gc_time = true;
 
    objspace->finalizer_table = st_init_numtable();
    objspace->finalizer_postponed_job = rb_postponed_job_preregister(0, gc_run_finalizers, objspace);
 
    ccan_list_head_init(&objspace->ractor_caches);
 
    objspace->mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
    objspace->cond_world_stopped = (pthread_cond_t)PTHREAD_COND_INITIALIZER;
    objspace->cond_world_started = (pthread_cond_t)PTHREAD_COND_INITIALIZER;
 
    objspace->event_hook_mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
 
    objspace->vo_bit_log_region_size = mmtk_get_vo_bit_log_region_size();
    objspace->vo_bit_base_addr = mmtk_get_vo_bit_base_addr();
    objspace->max_non_los_default_alloc_bytes = mmtk_max_non_los_default_alloc_bytes();
}
 
void
rb_gc_impl_objspace_free(void *objspace_ptr)
{
    /* The objspace is a process-lifetime singleton. */
}
 
void *
rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor)
{
    struct objspace *objspace = objspace_ptr;
    if (objspace->live_ractor_cache_count == 0) {
        mmtk_initialize_collection(ractor);
    }
    objspace->live_ractor_cache_count++;
 
    struct MMTk_ractor_cache *cache = calloc(1, sizeof(struct MMTk_ractor_cache));
    ccan_list_add(&objspace->ractor_caches, &cache->list_node);
 
    cache->mutator = mmtk_bind_mutator(cache);
    cache->bump_pointer = mmtk_get_bump_pointer_allocator(cache->mutator);
 
    return cache;
}
 
void
rb_gc_impl_objspace_retire_gc(void *objspace_ptr)
{
    /* A single objspace needs no per-Ractor GC at teardown. */
}
 
void
rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache_ptr)
{
    struct objspace *objspace = objspace_ptr;
    struct MMTk_ractor_cache *cache = cache_ptr;
 
    ccan_list_del(&cache->list_node);
 
    mmtk_flush_obj_free_buffer(cache);
 
    if (ruby_free_at_exit_p()) {
        MMTK_ASSERT(objspace->live_ractor_cache_count > 0);
    }
    else {
        MMTK_ASSERT(objspace->live_ractor_cache_count > 1);
    }
 
    objspace->live_ractor_cache_count--;
 
    mmtk_destroy_mutator(cache->mutator);
 
    free(cache);
}
 
static bool
zjit_mmtk_gc_stress_p(void *objspace_ptr)
{
    return ((struct objspace *)objspace_ptr)->gc_stress;
}
 
static bool
zjit_mmtk_newobj_tracing_p(void)
{
    return rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_NEWOBJ);
}
 
void rb_gc_impl_set_params(void *objspace_ptr) { }
 
static VALUE gc_verify_internal_consistency(VALUE self) { return Qnil; }
 
static inline size_t
rb_mmtk_align_obj_size(size_t object_size)
{
    return (object_size + MMTk_MIN_OBJ_ALIGN - 1) & ~((size_t)MMTk_MIN_OBJ_ALIGN - 1);
}
 
bool
rb_gc_impl_zjit_new_obj_fastpath(void *objspace_ptr, size_t alloc_size, VALUE flags, VALUE klass,
                                 struct rb_gc_zjit_fastpath *fastpath)
{
#if USE_ZJIT
    struct objspace *objspace = objspace_ptr;
 
    size_t total_size = rb_mmtk_align_obj_size(alloc_size + sizeof(VALUE));
    size_t object_size = total_size - sizeof(VALUE);
    size_t value_size_shift = sizeof(VALUE) == 8 ? 3 : 2;
 
    if (total_size > objspace->max_non_los_default_alloc_bytes) return false;
 
    struct rb_gc_zjit_mmtk_new_obj_fastpath mmtk_fastpath = {
        objspace,
        offsetof(struct objspace, total_allocated_objects),
        offsetof(struct MMTk_ractor_cache, mutator),
        offsetof(struct MMTk_ractor_cache, bump_pointer),
        offsetof(struct MMTk_ractor_cache, obj_free_parallel_buf),
        offsetof(struct MMTk_ractor_cache, obj_free_parallel_count),
        offsetof(MMTk_BumpPointer, cursor),
        offsetof(MMTk_BumpPointer, limit),
        MMTk_MIN_OBJ_ALIGN,
        object_size,
        total_size,
        MMTK_ALLOCATION_SEMANTICS_DEFAULT,
        (uintptr_t)zjit_mmtk_gc_stress_p,
        (uintptr_t)zjit_mmtk_newobj_tracing_p,
        (uintptr_t)mmtk_post_alloc,
        OBJ_FREE_BUF_CAPACITY - 1,
        value_size_shift,
        flags,
        klass
    };
 
    memset(fastpath, 0, sizeof(*fastpath));
    fastpath->kind = RB_GC_ZJIT_FASTPATH_MMTK;
    memcpy(fastpath->data.words, &mmtk_fastpath, sizeof(mmtk_fastpath));
 
    return true;
#else
    return false;
#endif
}
 
int32_t
rb_gc_impl_zjit_writebarrier_required_offset(void)
{
    /* MMTk's write barrier is not a no-op for young, unshareable receivers,
     * so ZJIT must always call rb_gc_writebarrier(). */
    return -1;
}
 
void
rb_gc_impl_init(void)
{
    VALUE gc_constants = rb_hash_new();
    rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX])));
    rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic)));
    rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0));
    rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")),
                 INT2FIX(strcmp((const char *)mmtk_plan(), "StickyImmix") == 0 ? 1 : 0));
    OBJ_FREEZE(gc_constants);
    rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants);
 
    // no-ops for compatibility
    rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency, 0);
 
    rb_define_singleton_method(rb_mGC, "compact", rb_f_notimplement, 0);
    rb_define_singleton_method(rb_mGC, "auto_compact", rb_f_notimplement, 0);
    rb_define_singleton_method(rb_mGC, "auto_compact=", rb_f_notimplement, 1);
    rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0);
    rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1);
}
 
int
rb_mmtk_obj_free_iter_wrapper(VALUE obj, void *data)
{
    struct objspace *objspace = data;
 
    if (!RB_TYPE_P(obj, T_NONE)) {
        rb_gc_obj_free_vm_weak_references(obj);
        rb_gc_obj_free(objspace, obj);
    }
 
    return 0;
}
 
// Shutdown
static void each_object(struct objspace *objspace, int (*func)(VALUE, void *), void *data);
 
void
rb_gc_impl_shutdown_free_objects(void *objspace_ptr)
{
    mmtk_set_gc_enabled(false);
    each_object(objspace_ptr, rb_mmtk_obj_free_iter_wrapper, objspace_ptr);
    mmtk_set_gc_enabled(true);
}
 
// GC
void
rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact, bool global)
{
    mmtk_handle_user_collection_request(rb_gc_get_ractor_newobj_cache(), true, full_mark);
}
 
bool
rb_gc_impl_during_gc_p(void *objspace_ptr)
{
    struct objspace *objspace = objspace_ptr;
    return objspace->world_stopped;
}
 
static void
rb_gc_impl_prepare_heap_i(MMTk_ObjectReference obj, void *d)
{
    rb_gc_prepare_heap_process_object((VALUE)obj);
}
 
void
rb_gc_impl_prepare_heap(void *objspace_ptr)
{
    mmtk_enumerate_objects(rb_gc_impl_prepare_heap_i, NULL);
}
 
void
rb_gc_impl_gc_enable(void *objspace_ptr)
{
    mmtk_set_gc_enabled(true);
}
 
void
rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc)
{
    mmtk_set_gc_enabled(false);
}
 
bool
rb_gc_impl_user_gc_disabled_set(void *objspace_ptr, bool disable)
{
    struct objspace *objspace = objspace_ptr;
    const bool was = objspace->user_gc_disabled;
    objspace->user_gc_disabled = disable;
 
    if (was != disable) {
        mmtk_set_gc_enabled(!disable);
        objspace->user_gc_disabled = disable;
    }
 
    return was;
}
 
bool
rb_gc_impl_user_gc_disabled_p(void *objspace_ptr)
{
    struct objspace *objspace = objspace_ptr;
    return objspace->user_gc_disabled;
}
 
bool
rb_gc_impl_gc_enabled_p(void *objspace_ptr)
{
    return mmtk_gc_enabled_p();
}
 
void
rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag)
{
    struct objspace *objspace = objspace_ptr;
 
    objspace->gc_stress = RTEST(flag);
}
 
VALUE
rb_gc_impl_stress_get(void *objspace_ptr)
{
    struct objspace *objspace = objspace_ptr;
 
    return objspace->gc_stress ? Qtrue : Qfalse;
}
 
VALUE
rb_gc_impl_config_get(void *objspace_ptr)
{
    VALUE hash = rb_hash_new();
 
    rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_worker_count")), RB_ULONG2NUM(mmtk_worker_count()));
    rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_plan")), rb_str_new_cstr((const char *)mmtk_plan()));
    rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_heap_mode")), rb_str_new_cstr((const char *)mmtk_heap_mode()));
    size_t heap_min = mmtk_heap_min();
    if (heap_min > 0) rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_heap_min")), RB_ULONG2NUM(heap_min));
    rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_heap_max")), RB_ULONG2NUM(mmtk_heap_max()));
 
    return hash;
}
 
void
rb_gc_impl_config_set(void *objspace_ptr, VALUE hash)
{
    // TODO
}
 
struct rb_gc_vm_context *
rb_gc_impl_get_vm_context(void *objspace_ptr)
{
    struct objspace *objspace = objspace_ptr;
 
    return &objspace->vm_context;
}
 
// Object allocation
 
static VALUE
rb_mmtk_alloc_fast_path(struct objspace *objspace, struct MMTk_ractor_cache *ractor_cache, size_t size, size_t align)
{
    MMTk_BumpPointer *bump_pointer = ractor_cache->bump_pointer;
    if (bump_pointer == NULL) return 0;
 
    uintptr_t cursor = bump_pointer->cursor;
 
    // Ensure cursor is aligned
    size_t mask = align - 1;
    cursor = (cursor + mask) & ~mask;
 
    cursor += size;
 
    if (cursor > bump_pointer->limit) {
        return 0;
    }
    else {
        VALUE obj = cursor - size;
        bump_pointer->cursor = cursor;
        return obj;
    }
}
 
static bool
obj_can_parallel_free_p(VALUE obj)
{
    switch (RB_BUILTIN_TYPE(obj)) {
      case T_ARRAY:
      case T_BIGNUM:
      case T_COMPLEX:
      case T_FLOAT:
      case T_HASH:
      case T_OBJECT:
      case T_RATIONAL:
      case T_REGEXP:
      case T_STRING:
      case T_STRUCT:
      case T_SYMBOL:
        return true;
      default:
        return false;
    }
}
 
static bool
obj_need_obj_free_p(VALUE obj)
{
    switch (RB_BUILTIN_TYPE(obj)) {
      case T_FLOAT:
      case T_RATIONAL:
      case T_COMPLEX:
      case T_OBJECT:
        return false;
      default:
        return true;
    }
}
 
static void
mmtk_flush_obj_free_buffer(struct MMTk_ractor_cache *cache)
{
    if (cache->obj_free_parallel_count > 0) {
        mmtk_add_obj_free_candidates(cache->obj_free_parallel_buf,
                                     cache->obj_free_parallel_count, true);
        cache->obj_free_parallel_count = 0;
    }
    if (cache->obj_free_non_parallel_count > 0) {
        mmtk_add_obj_free_candidates(cache->obj_free_non_parallel_buf,
                                     cache->obj_free_non_parallel_count, false);
        cache->obj_free_non_parallel_count = 0;
    }
}
 
static inline void
mmtk_buffer_obj_free_candidate(struct MMTk_ractor_cache *cache, VALUE obj)
{
    if (obj_can_parallel_free_p(obj)) {
        cache->obj_free_parallel_buf[cache->obj_free_parallel_count++] = (MMTk_ObjectReference)obj;
        if (cache->obj_free_parallel_count >= OBJ_FREE_BUF_CAPACITY) {
            mmtk_add_obj_free_candidates(cache->obj_free_parallel_buf,
                                         cache->obj_free_parallel_count, true);
            cache->obj_free_parallel_count = 0;
        }
    }
    else {
        cache->obj_free_non_parallel_buf[cache->obj_free_non_parallel_count++] = (MMTk_ObjectReference)obj;
        if (cache->obj_free_non_parallel_count >= OBJ_FREE_BUF_CAPACITY) {
            mmtk_add_obj_free_candidates(cache->obj_free_non_parallel_buf,
                                         cache->obj_free_non_parallel_count, false);
            cache->obj_free_non_parallel_count = 0;
        }
    }
}
 
static void
mmtk_post_alloc_fast_immix(struct objspace *objspace, struct MMTk_ractor_cache *ractor_cache, uintptr_t obj)
{
    uintptr_t region_offset = obj >> objspace->vo_bit_log_region_size;
    uintptr_t byte_offset = region_offset / 8;
    uintptr_t bit_offset = region_offset % 8;
    uintptr_t meta_byte_address = objspace->vo_bit_base_addr + byte_offset;
    uint8_t byte = 1 << bit_offset;
    uint8_t *meta_byte_ptr = (uint8_t*)meta_byte_address;
    *meta_byte_ptr |= byte;
}
 
VALUE
rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size, size_t *actual_alloc_size)
{
    struct objspace *objspace = objspace_ptr;
    struct MMTk_ractor_cache *ractor_cache = cache_ptr;
 
    if (alloc_size == 0) {
        rb_bug("rb_gc_impl_new_obj: allocation size out of range (size=%"PRIuSIZE")", alloc_size);
    }
 
    // Layout: [hidden size header (sizeof(VALUE))][payload (alloc_size)]
    size_t total_size = rb_mmtk_align_obj_size(alloc_size + sizeof(VALUE));
    size_t object_size = total_size - sizeof(VALUE);
    MMTk_AllocationSemantics semantics = total_size > objspace->max_non_los_default_alloc_bytes
        ? MMTK_ALLOCATION_SEMANTICS_LOS
        : MMTK_ALLOCATION_SEMANTICS_DEFAULT;
    *actual_alloc_size = object_size;
 
    if (objspace->gc_stress) {
        mmtk_handle_user_collection_request(ractor_cache, false, false);
    }
 
    VALUE *alloc_obj = semantics == MMTK_ALLOCATION_SEMANTICS_DEFAULT
        ? (VALUE *)rb_mmtk_alloc_fast_path(objspace, ractor_cache, total_size, MMTk_MIN_OBJ_ALIGN)
        : NULL;
    if (!alloc_obj) {
        alloc_obj = mmtk_alloc(ractor_cache->mutator, total_size, MMTk_MIN_OBJ_ALIGN, 0, semantics);
 
        // On heap exhaustion raise NoMemoryError.
        if (RB_UNLIKELY(alloc_obj == NULL)) {
            rb_memerror();
        }
    }
 
    alloc_obj++;
    alloc_obj[-1] = object_size;
    alloc_obj[0] = flags;
    alloc_obj[1] = klass;
 
    if (RB_UNLIKELY(!wb_protected)) {
        mmtk_register_wb_unprotected_object((MMTk_ObjectReference)alloc_obj);
    }
 
    if (semantics == MMTK_ALLOCATION_SEMANTICS_LOS || ractor_cache->bump_pointer == NULL) {
        mmtk_post_alloc(ractor_cache->mutator, (void*)alloc_obj, total_size, semantics);
    }
    else {
        // We can use the post alloc fast path if we're using Immix bump pointer allocator
        mmtk_post_alloc_fast_immix(objspace, ractor_cache, (uintptr_t)alloc_obj);
    }
 
    if (obj_need_obj_free_p((VALUE)alloc_obj)) {
        mmtk_buffer_obj_free_candidate(ractor_cache, (VALUE)alloc_obj);
    }
 
    objspace->total_allocated_objects++;
 
    return (VALUE)alloc_obj;
}
 
size_t
rb_gc_impl_obj_slot_size(VALUE obj)
{
    return ((VALUE *)obj)[-1];
}
 
size_t
rb_gc_impl_size_slot_size(void *objspace_ptr, size_t size)
{
    if (size == 0) {
        rb_bug("rb_gc_impl_size_slot_size: size too large (size=%"PRIuSIZE")", size);
    }
 
    return rb_mmtk_align_obj_size(size + sizeof(VALUE)) - sizeof(VALUE);
}
 
bool
rb_gc_impl_size_allocatable_p(size_t size)
{
    return true;
}
 
size_t
rb_gc_impl_max_allocation_size(void)
{
    return SIZE_MAX;
}
 
// Malloc
void *
rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed)
{
    // TODO: don't use system malloc
    return malloc(size);
}
 
void *
rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed)
{
    // TODO: don't use system calloc
    return calloc(1, size);
}
 
void *
rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed)
{
    // TODO: don't use system realloc
    return realloc(ptr, new_size);
}
 
void
rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size)
{
    // TODO: don't use system free
    free(ptr);
}
 
void rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff) { }
 
// Marking
static inline VALUE
rb_mmtk_call_object_closure(VALUE obj, bool pin)
{
    if (RB_UNLIKELY(RB_BUILTIN_TYPE(obj) == T_NONE)) {
        enum { info_size = 256 };
        char obj_info_buf[info_size];
        rb_raw_obj_info(obj_info_buf, info_size, obj);
 
        char parent_obj_info_buf[info_size];
        rb_raw_obj_info(parent_obj_info_buf, info_size, marking_parent_object);
 
        rb_mmtk_gc_thread_bug("try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf);
    }
 
    return (VALUE)rb_mmtk_gc_thread_tls->object_closure.c_function(
        rb_mmtk_gc_thread_tls->object_closure.rust_closure,
        rb_mmtk_gc_thread_tls->gc_context,
        (MMTk_ObjectReference)obj,
        pin
    );
}
 
void
rb_gc_impl_mark(void *objspace_ptr, VALUE obj)
{
    if (RB_SPECIAL_CONST_P(obj)) return;
 
    rb_mmtk_call_object_closure(obj, false);
}
 
void
rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr)
{
    if (RB_SPECIAL_CONST_P(*ptr)) return;
 
    VALUE new_obj = rb_mmtk_call_object_closure(*ptr, false);
    if (new_obj != *ptr) {
        *ptr = new_obj;
    }
}
 
void
rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj)
{
    if (RB_SPECIAL_CONST_P(obj)) return;
 
    rb_mmtk_call_object_closure(obj, true);
}
 
void
rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj)
{
    if (rb_gc_impl_live_object_p(objspace_ptr, (const void *)obj)) {
        rb_gc_impl_mark_and_pin(objspace_ptr, obj);
    }
}
 
void
rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj)
{
    RB_FL_SET(obj, RUBY_FL_WEAK_REFERENCE);
    mmtk_declare_weak_references((MMTk_ObjectReference)obj);
}
 
bool
rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj)
{
    return mmtk_weak_references_alive_p((MMTk_ObjectReference)obj);
}
 
// Compaction
void
rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj)
{
    mmtk_register_pinning_obj((MMTk_ObjectReference)obj);
}
 
bool
rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj)
{
    return rb_mmtk_call_object_closure(obj, false) != obj;
}
 
bool
rb_gc_impl_pinned_p(void *objspace_ptr, VALUE obj)
{
    return mmtk_is_pinned((MMTk_ObjectReference)obj);
}
 
VALUE
rb_gc_impl_location(void *objspace_ptr, VALUE obj)
{
    return rb_mmtk_call_object_closure(obj, false);
}
 
// Write barriers
void
rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b)
{
    struct MMTk_ractor_cache *cache = rb_gc_get_ractor_newobj_cache();
 
    if (SPECIAL_CONST_P(b)) return;
 
#ifdef MMTK_DEBUG
    if (!rb_gc_impl_live_object_p(objspace_ptr, (void *)a)) {
        char buff[256];
        rb_bug("a: %s is not an object", rb_raw_obj_info(buff, 256, a));
    }
 
    if (!rb_gc_impl_live_object_p(objspace_ptr, (void *)b)) {
        char buff[256];
        rb_bug("b: %s is not an object", rb_raw_obj_info(buff, 256, b));
    }
#endif
 
    MMTK_ASSERT(BUILTIN_TYPE(a) != T_NONE);
    MMTK_ASSERT(BUILTIN_TYPE(b) != T_NONE);
 
    mmtk_object_reference_write_post(cache->mutator, (MMTk_ObjectReference)a);
}
 
void
rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj)
{
    mmtk_register_wb_unprotected_object((MMTk_ObjectReference)obj);
}
 
void
rb_gc_impl_obj_became_shareable(void *objspace_ptr, VALUE obj)
{
    /* MMTk has no per-page shareable bits. */
}
 
void
rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj)
{
    if (!ruby_native_thread_p()) {
        /* Called from an MMTk GC worker thread (via gc_ref_update_object's
         * re-embedding path during reference updating).  There is no mutator
         * whose modbuf we could log the object into.  This is only reachable
         * during a moving GC, and moving GCs are always full-heap (major)
         * GCs, so the object and its children are all marked by the current
         * GC and no remembering is needed. */
        return;
    }
 
    struct MMTk_ractor_cache *cache = rb_gc_get_ractor_newobj_cache();
 
    mmtk_object_reference_write_post(cache->mutator, (MMTk_ObjectReference)obj);
}
 
// Heap walking
static void
each_objects_i(MMTk_ObjectReference obj, void *d)
{
    rb_darray(VALUE) *objs = d;
 
    rb_darray_append(objs, (VALUE)obj);
}
 
static void
each_object(struct objspace *objspace, int (*func)(VALUE, void *), void *data)
{
    rb_darray(VALUE) objs;
    rb_darray_make(&objs, 0);
 
    mmtk_enumerate_objects(each_objects_i, &objs);
 
    VALUE *obj_ptr;
    rb_darray_foreach(objs, i, obj_ptr) {
        if (!mmtk_is_mmtk_object((MMTk_ObjectReference)*obj_ptr)) continue;
 
        if (func(*obj_ptr, data) != 0) {
            break;
        }
    }
 
    rb_darray_free(objs);
}
 
struct rb_gc_impl_each_objects_data {
    int (*func)(void *, void *, size_t, void *);
    void *data;
};
 
static int
rb_gc_impl_each_objects_i(VALUE obj, void *d)
{
    struct rb_gc_impl_each_objects_data *data = d;
 
    size_t slot_size = rb_gc_impl_obj_slot_size(obj);
 
    return data->func((void *)obj, (void *)(obj + slot_size), slot_size, data->data);
}
 
void
rb_gc_impl_each_objects(void *objspace_ptr, int (*func)(void *, void *, size_t, void *), void *data)
{
    struct rb_gc_impl_each_objects_data each_objects_data = {
        .func = func,
        .data = data
    };
 
    each_object(objspace_ptr, rb_gc_impl_each_objects_i, &each_objects_data);
}
 
struct rb_gc_impl_each_object_data {
    void (*func)(VALUE, void *);
    void *data;
};
 
static int
rb_gc_impl_each_object_i(VALUE obj, void *d)
{
    struct rb_gc_impl_each_object_data *data = d;
 
    data->func(obj, data->data);
 
    return 0;
}
 
void
rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE, void *), void *data)
{
    struct rb_gc_impl_each_object_data each_object_data = {
        .func = func,
        .data = data
    };
 
    each_object(objspace_ptr, rb_gc_impl_each_object_i, &each_object_data);
}
 
// Finalizers
static VALUE
gc_run_finalizers_get_final(rb_len_t i, void *data)
{
    VALUE table = (VALUE)data;
 
    return RARRAY_AREF(table, i + 1);
}
 
static void
gc_run_finalizers(void *data)
{
    struct objspace *objspace = data;
 
    rb_gc_set_pending_interrupt();
 
    while (objspace->finalizer_jobs != NULL) {
        struct MMTk_final_job *job = objspace->finalizer_jobs;
        objspace->finalizer_jobs = job->next;
 
        switch (job->kind) {
          case MMTK_FINAL_JOB_DFREE:
            job->as.dfree.func(job->as.dfree.data);
            break;
          case MMTK_FINAL_JOB_FINALIZE: {
            VALUE finalizer_array = job->as.finalize.finalizer_array;
 
            rb_gc_run_obj_finalizer(
                RARRAY_AREF(finalizer_array, 0),
                RARRAY_LEN(finalizer_array) - 1,
                gc_run_finalizers_get_final,
                (void *)finalizer_array
            );
 
            RB_GC_GUARD(finalizer_array);
            break;
          }
        }
 
        xfree(job);
    }
 
    rb_gc_unset_pending_interrupt();
}
 
void
rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data)
{
    if (dfree == NULL) return;
 
    struct objspace *objspace = objspace_ptr;
 
    struct MMTk_final_job *job = xmalloc(sizeof(struct MMTk_final_job));
    job->kind = MMTK_FINAL_JOB_DFREE;
    job->as.dfree.func = dfree;
    job->as.dfree.data = data;
 
    struct MMTk_final_job *prev;
    do {
        job->next = objspace->finalizer_jobs;
        prev = RUBY_ATOMIC_PTR_CAS(objspace->finalizer_jobs, job->next, job);
    } while (prev != job->next);
 
    if (!ruby_free_at_exit_p()) {
        rb_postponed_job_trigger(objspace->finalizer_postponed_job);
    }
}
 
VALUE
rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block)
{
    struct objspace *objspace = objspace_ptr;
    VALUE table;
    st_data_t data;
 
    RBASIC(obj)->flags |= FL_FINALIZE;
 
    int lev = RB_GC_VM_LOCK();
 
    if (st_lookup(objspace->finalizer_table, obj, &data)) {
        table = (VALUE)data;
 
        /* avoid duplicate block, table is usually small */
        {
            rb_len_t len = RARRAY_LEN(table);
            rb_len_t i;
 
            for (i = 0; i < len; i++) {
                VALUE recv = RARRAY_AREF(table, i);
                if (rb_equal(recv, block)) {
                    RB_GC_VM_UNLOCK(lev);
                    return recv;
                }
            }
        }
 
        rb_ary_push(table, block);
    }
    else {
        table = rb_ary_new3(2, rb_obj_id(obj), block);
        rb_obj_hide(table);
        st_add_direct(objspace->finalizer_table, obj, table);
    }
 
    RB_GC_VM_UNLOCK(lev);
 
    return block;
}
 
void
rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj)
{
    struct objspace *objspace = objspace_ptr;
 
    st_data_t data = obj;
 
    int lev = RB_GC_VM_LOCK();
    st_delete(objspace->finalizer_table, &data, 0);
    RB_GC_VM_UNLOCK(lev);
 
    FL_UNSET(obj, FL_FINALIZE);
}
 
void
rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj)
{
    struct objspace *objspace = objspace_ptr;
    VALUE table;
    st_data_t data;
 
    if (!FL_TEST(obj, FL_FINALIZE)) return;
 
    int lev = RB_GC_VM_LOCK();
    if (RB_LIKELY(st_lookup(objspace->finalizer_table, obj, &data))) {
        table = rb_ary_dup((VALUE)data);
        RARRAY_ASET(table, 0, rb_obj_id(dest));
        st_insert(objspace->finalizer_table, dest, table);
        FL_SET(dest, FL_FINALIZE);
    }
    else {
        rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj));
    }
    RB_GC_VM_UNLOCK(lev);
}
 
static int
move_finalizer_from_table_i(st_data_t key, st_data_t val, st_data_t arg)
{
    struct objspace *objspace = (struct objspace *)arg;
 
    make_final_job(objspace, (VALUE)key, (VALUE)val);
 
    return ST_DELETE;
}
 
void
rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr)
{
    struct objspace *objspace = objspace_ptr;
 
    while (objspace->finalizer_table->num_entries) {
        st_foreach(objspace->finalizer_table, move_finalizer_from_table_i, (st_data_t)objspace);
 
        gc_run_finalizers(objspace);
    }
 
    unsigned int lev = RB_GC_VM_LOCK();
    {
        struct MMTk_ractor_cache *rc;
        ccan_list_for_each(&objspace->ractor_caches, rc, list_node) {
            mmtk_flush_obj_free_buffer(rc);
        }
 
        struct MMTk_RawVecOfObjRef registered_candidates = mmtk_get_all_obj_free_candidates();
        for (size_t i = 0; i < registered_candidates.len; i++) {
            VALUE obj = (VALUE)registered_candidates.ptr[i];
 
            if (rb_gc_shutdown_call_finalizer_p(obj)) {
                rb_gc_obj_free(objspace_ptr, obj);
                RBASIC(obj)->flags = 0;
            }
        }
        mmtk_free_raw_vec_of_obj_ref(registered_candidates);
    }
    RB_GC_VM_UNLOCK(lev);
 
    gc_run_finalizers(objspace);
}
 
// Forking
 
void
rb_gc_impl_before_fork(void *objspace_ptr)
{
    struct objspace *objspace = objspace_ptr;
 
  retry:
    objspace->fork_hook_vm_lock_lev = RB_GC_VM_LOCK();
    rb_gc_vm_barrier();
 
    /* At this point, we know that all the Ractors are paused because of the
     * rb_gc_vm_barrier above. Since rb_mmtk_block_for_gc is a barrier point,
     * one or more Ractors could be paused there. However, mmtk_before_fork is
     * not compatible with that because it assumes that the MMTk workers are idle,
     * but the workers are not idle because they are busy working on a GC.
     *
     * This essentially implements a trylock. It will optimistically lock but will
     * release the lock if it detects that any other Ractors are waiting in
     * rb_mmtk_block_for_gc.
     */
    rb_atomic_t mutator_blocking_count = RUBY_ATOMIC_LOAD(objspace->mutator_blocking_count);
    if (mutator_blocking_count != 0) {
        RB_GC_VM_UNLOCK(objspace->fork_hook_vm_lock_lev);
        goto retry;
    }
 
    mmtk_before_fork();
}
 
void
rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid)
{
    struct objspace *objspace = objspace_ptr;
 
    mmtk_after_fork(rb_gc_get_ractor_newobj_cache());
 
    RB_GC_VM_UNLOCK(objspace->fork_hook_vm_lock_lev);
}
 
// Statistics
 
void
rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag)
{
    struct objspace *objspace = objspace_ptr;
 
    objspace->measure_gc_time = RTEST(flag);
}
 
bool
rb_gc_impl_get_measure_total_time(void *objspace_ptr)
{
    struct objspace *objspace = objspace_ptr;
 
    return objspace->measure_gc_time;
}
 
unsigned long long
rb_gc_impl_get_total_time(void *objspace_ptr)
{
    struct objspace *objspace = objspace_ptr;
 
    return objspace->total_gc_time;
}
 
size_t
rb_gc_impl_gc_count(void *objspace_ptr)
{
    struct objspace *objspace = objspace_ptr;
 
    return objspace->gc_count;
}
 
VALUE
rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE hash_or_key)
{
    VALUE hash = Qnil, key = Qnil;
 
    if (SYMBOL_P(hash_or_key)) {
        key = hash_or_key;
    }
    else if (RB_TYPE_P(hash_or_key, T_HASH)) {
        hash = hash_or_key;
    }
    else {
        rb_bug("gc_info_decode: non-hash or symbol given");
    }
 
#define SET(name, attr) \
    if (key == ID2SYM(rb_intern_const(#name))) \
        return (attr); \
    else if (hash != Qnil) \
        rb_hash_aset(hash, ID2SYM(rb_intern_const(#name)), (attr));
 
    /* Hack to get StackProf working because it calls rb_gc_latest_gc_info with
     * the :state key and expects a result. This always returns the :none state. */
    SET(state, ID2SYM(rb_intern_const("none")));
#undef SET
 
    if (!NIL_P(key)) {
        // Matched key should return above
        return Qundef;
    }
 
    return hash;
}
 
enum gc_stat_sym {
    gc_stat_sym_count,
    gc_stat_sym_moving_gc_count,
    gc_stat_sym_time,
    gc_stat_sym_total_allocated_objects,
    gc_stat_sym_total_bytes,
    gc_stat_sym_used_bytes,
    gc_stat_sym_free_bytes,
    gc_stat_sym_starting_heap_address,
    gc_stat_sym_last_heap_address,
    gc_stat_sym_weak_references_count,
    gc_stat_sym_last
};
 
static VALUE gc_stat_symbols[gc_stat_sym_last];
 
static void
setup_gc_stat_symbols(void)
{
    if (gc_stat_symbols[0] == 0) {
#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s))
        S(count);
        S(moving_gc_count);
        S(time);
        S(total_allocated_objects);
        S(total_bytes);
        S(used_bytes);
        S(free_bytes);
        S(starting_heap_address);
        S(last_heap_address);
        S(weak_references_count);
    }
}
 
VALUE
rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
{
    if (objspace_ptr == NULL) {
        rb_notimplement();
    }
 
    struct objspace *objspace = objspace_ptr;
    VALUE hash = Qnil, key = Qnil;
 
    setup_gc_stat_symbols();
 
    if (RB_TYPE_P(hash_or_sym, T_HASH)) {
        hash = hash_or_sym;
    }
    else if (SYMBOL_P(hash_or_sym)) {
        key = hash_or_sym;
    }
    else {
        rb_bug("non-hash or symbol given");
    }
 
#define SET(name, attr) \
    if (key == gc_stat_symbols[gc_stat_sym_##name]) \
        return SIZET2NUM(attr); \
    else if (hash != Qnil) \
        rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
 
        SET(count, objspace->gc_count);
        SET(moving_gc_count, objspace->moving_gc_count);
        SET(time, objspace->total_gc_time / (1000 * 1000));
        SET(total_allocated_objects, objspace->total_allocated_objects);
        SET(total_bytes, mmtk_total_bytes());
        SET(used_bytes, mmtk_used_bytes());
        SET(free_bytes, mmtk_free_bytes());
        SET(starting_heap_address, (size_t)mmtk_starting_heap_address());
        SET(last_heap_address, (size_t)mmtk_last_heap_address());
        SET(weak_references_count, mmtk_weak_references_count());
#undef SET
 
    if (!NIL_P(key)) {
        // Matched key should return above
        return Qundef;
    }
 
    return hash;
}
 
VALUE
rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym)
{
    if (NIL_P(heap_name) && RB_TYPE_P(hash_or_sym, T_HASH)) {
        return hash_or_sym;
    }
 
    return Qundef;
}
 
// Miscellaneous
 
#define RB_GC_OBJECT_METADATA_ENTRY_COUNT 2
static struct rb_gc_object_metadata_entry object_metadata_entries[RB_GC_OBJECT_METADATA_ENTRY_COUNT + 1];
 
struct rb_gc_object_metadata_entry *
rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj)
{
    static ID ID_wb_protected;
    static ID ID_object_id;
 
    if (!ID_object_id) {
#define I(s) ID_##s = rb_intern(#s);
        I(wb_protected);
        I(object_id);
#undef I
    }
 
    size_t n = 0;
 
#define SET_ENTRY(na, v) do { \
    MMTK_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \
    object_metadata_entries[n].name = ID_##na; \
    object_metadata_entries[n].val = v; \
    n++; \
} while (0)
 
    if (!mmtk_object_wb_unprotected_p((MMTk_ObjectReference)obj)) SET_ENTRY(wb_protected, Qtrue);
    if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj));
 
    object_metadata_entries[n].name = 0;
    object_metadata_entries[n].val = 0;
 
    return object_metadata_entries;
}
 
bool
rb_gc_impl_live_object_p(void *objspace_ptr, const void *ptr)
{
    if (ptr == NULL) return false;
    if ((uintptr_t)ptr % sizeof(void*) != 0) return false;
    return mmtk_is_mmtk_object((MMTk_Address)ptr);
}
 
bool
rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE obj)
{
    return false;
}
 
void rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event) { }
 
void
rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj)
{
    if (mmtk_object_wb_unprotected_p((MMTk_ObjectReference)obj)) {
        rb_gc_impl_writebarrier_unprotect(objspace_ptr, dest);
    }
 
    rb_gc_impl_copy_finalizer(objspace_ptr, dest, obj);
}
 
// GC Identification
 
const char *
rb_gc_impl_active_gc_name(void)
{
    return "mmtk";
}
 
bool
rb_gc_impl_during_global_gc_p(void *objspace_ptr)
{
    /* An mmtk GC is always a stop-the-world global GC.  Helpers used during marking
     * (ractor_sync_mark and friends) read this to tell whether every mutator has
     * stopped. */
    struct objspace *objspace = objspace_ptr;
    return objspace->world_stopped;
}
 
bool
rb_gc_impl_during_postmortem_p(void *objspace_ptr)
{
    /* mmtk has a single objspace and no per-Ractor retire collection. */
    return false;
}
 
bool
rb_gc_impl_obj_foreign_p(void *objspace_ptr, VALUE obj)
{
    /* With a single objspace every object is our own. */
    return false;
}
 
bool
rb_gc_impl_shref_marked_p(void *objspace_ptr, VALUE obj)
{
    /* With a single objspace there is no cross-objspace pinning to track. */
    return true;
}
 
void
rb_gc_impl_objspace_zombify(void *objspace_ptr)
{
    /* With a single objspace zombie_objspaces is always empty. */
}
 
void
rb_gc_impl_objspace_absorb(void *dst_ptr, void *src_ptr)
{
    /* A single objspace. */
}
 
void
rb_gc_impl_gc_rest(void *objspace_ptr)
{
    /* An mmtk GC completes stop-the-world: there is no in-progress incremental
     * mark or lazy sweep state. */
}
 
struct each_objects_shareable_data {
    int (*func)(void *, void *, size_t, void *);
    void *data;
};
 
static int
each_objects_shareable_i(void *start, void *end, size_t stride, void *d)
{
    struct each_objects_shareable_data *data = d;
    for (VALUE obj = (VALUE)start; obj < (VALUE)end; obj += stride) {
        if (RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE)) {
            int ret = data->func((void *)obj, (void *)(obj + stride), stride, data->data);
            if (ret) return ret;
        }
    }
    return 0;
}
 
void
rb_gc_impl_each_objects_shareable(void *objspace_ptr, int (*func)(void *, void *, size_t, void *), void *data)
{
    struct each_objects_shareable_data d = { func, data };
    rb_gc_impl_each_objects(objspace_ptr, each_objects_shareable_i, &d);
}
 
void
rb_gc_impl_each_objects_foreign(void *objspace_ptr, int (*func)(void *, void *, size_t, void *), void *data)
{
    /* With a single objspace no object lives in a foreign objspace. */
}