gc.h

#ifndef INTERNAL_GC_H                                    /*-*-C-*-vi:se ft=c:*/
#define INTERNAL_GC_H
#include "ruby/internal/config.h"
 
#include <stddef.h>             /* for size_t */
 
#include "internal/compilers.h" /* for __has_attribute */
#include "ruby/ruby.h"          /* for rb_event_flag_t */
#include "ruby_atomic.h"        /* for RUBY_ATOMIC_VALUE_SET */
 
struct rb_gc_zjit_fastpath;
 
#ifndef USE_MODULAR_GC
# define USE_MODULAR_GC 0
#endif
 
#if defined(__x86_64__) && !defined(_ILP32) && defined(__GNUC__)
#define SET_MACHINE_STACK_END(p) __asm__ __volatile__ ("movq\t%%rsp, %0" : "=r" (*(p)))
#elif defined(__i386) && defined(__GNUC__)
#define SET_MACHINE_STACK_END(p) __asm__ __volatile__ ("movl\t%%esp, %0" : "=r" (*(p)))
#elif (defined(__powerpc__) || defined(__powerpc64__)) && defined(__GNUC__) && !defined(_AIX)
#define SET_MACHINE_STACK_END(p) __asm__ __volatile__ ("mr\t%0, %%r1" : "=r" (*(p)))
#elif (defined(__powerpc__) || defined(__powerpc64__)) && defined(__GNUC__) && defined(_AIX)
#define SET_MACHINE_STACK_END(p) __asm__ __volatile__ ("mr %0,1" : "=r" (*(p)))
#elif defined(__aarch64__) && defined(__GNUC__)
#define SET_MACHINE_STACK_END(p) __asm__ __volatile__ ("mov\t%0, sp" : "=r" (*(p)))
#else
NOINLINE(void rb_gc_set_stack_end(VALUE **stack_end_p));
#define SET_MACHINE_STACK_END(p) rb_gc_set_stack_end(p)
#define USE_CONSERVATIVE_STACK_END
#endif
 
/* for GC debug */
 
#ifndef RUBY_MARK_FREE_DEBUG
#define RUBY_MARK_FREE_DEBUG 0
#endif
 
#if RUBY_MARK_FREE_DEBUG
extern int ruby_gc_debug_indent;
 
static inline void
rb_gc_debug_indent(void)
{
    ruby_debug_printf("%*s", ruby_gc_debug_indent, "");
}
 
static inline void
rb_gc_debug_body(const char *mode, const char *msg, int st, void *ptr)
{
    if (st == 0) {
        ruby_gc_debug_indent--;
    }
    rb_gc_debug_indent();
    ruby_debug_printf("%s: %s %s (%p)\n", mode, st ? "->" : "<-", msg, ptr);
 
    if (st) {
        ruby_gc_debug_indent++;
    }
 
    fflush(stdout);
}
 
#define RUBY_MARK_ENTER(msg) rb_gc_debug_body("mark", (msg), 1, ptr)
#define RUBY_MARK_LEAVE(msg) rb_gc_debug_body("mark", (msg), 0, ptr)
#define RUBY_FREE_ENTER(msg) rb_gc_debug_body("free", (msg), 1, ptr)
#define RUBY_FREE_LEAVE(msg) rb_gc_debug_body("free", (msg), 0, ptr)
#define RUBY_GC_INFO         rb_gc_debug_indent(), ruby_debug_printf
 
#else
#define RUBY_MARK_ENTER(msg)
#define RUBY_MARK_LEAVE(msg)
#define RUBY_FREE_ENTER(msg)
#define RUBY_FREE_LEAVE(msg)
#define RUBY_GC_INFO if(0)printf
#endif
 
#if STACK_GROW_DIRECTION > 0
# define STACK_UPPER(x, a, b) (a)
#elif STACK_GROW_DIRECTION < 0
# define STACK_UPPER(x, a, b) (b)
#else
RUBY_EXTERN int ruby_stack_grow_direction;
int ruby_get_stack_grow_direction(volatile VALUE *addr);
# define stack_growup_p(x) (                    \
        (ruby_stack_grow_direction ?            \
         ruby_stack_grow_direction :            \
         ruby_get_stack_grow_direction(x)) > 0)
# define STACK_UPPER(x, a, b) (stack_growup_p(x) ? (a) : (b))
#endif
 
/*
  STACK_GROW_DIR_DETECTION is used with STACK_DIR_UPPER.
 
  On most normal systems, stacks grow from high address to lower address. In
  this case, STACK_DIR_UPPER(a, b) will return (b), but on exotic systems where
  the stack grows UP (from low address to high address), it will return (a).
*/
 
#if STACK_GROW_DIRECTION
#define STACK_GROW_DIR_DETECTION
#define STACK_DIR_UPPER(a,b) STACK_UPPER(0, (a), (b))
#else
#define STACK_GROW_DIR_DETECTION VALUE stack_grow_dir_detection
#define STACK_DIR_UPPER(a,b) STACK_UPPER(&stack_grow_dir_detection, (a), (b))
#endif
#define IS_STACK_DIR_UPPER() STACK_DIR_UPPER(1,0)
 
const char *rb_obj_info(VALUE obj);
const char *rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj);
 
struct rb_execution_context_struct; /* in vm_core.h */
struct rb_objspace; /* in vm_core.h */
struct rb_ractor_struct; /* in vm_core.h */
 
#define EC_NEWOBJ_OF(var, T, c, f, s, ec)  \
    T *(var) = (T *)rb_ec_newobj_of((ec), (c), (f), s)
#define NEWOBJ_OF(var, T, c, f, s) EC_NEWOBJ_OF(var, T, c, f, s, GET_EC())
#define UNPROTECTED_NEWOBJ_OF(var, T, c, f, s) \
    T *(var) = (T *)rb_newobj((GET_EC()), (c), (f), ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, false, s)
 
#ifndef RB_GC_OBJECT_METADATA_ENTRY_DEFINED
# define RB_GC_OBJECT_METADATA_ENTRY_DEFINED
struct rb_gc_object_metadata_entry {
    ID name;
    VALUE val;
};
#endif
 
#ifndef USE_UNALIGNED_MEMBER_ACCESS
# define UNALIGNED_MEMBER_ACCESS(expr) (expr)
#elif ! USE_UNALIGNED_MEMBER_ACCESS
# define UNALIGNED_MEMBER_ACCESS(expr) (expr)
#elif ! (__has_warning("-Waddress-of-packed-member") || GCC_VERSION_SINCE(9, 0, 0))
# define UNALIGNED_MEMBER_ACCESS(expr) (expr)
#else
# include "internal/warnings.h"
# define UNALIGNED_MEMBER_ACCESS(expr) __extension__({ \
    COMPILER_WARNING_PUSH; \
    COMPILER_WARNING_IGNORED(-Waddress-of-packed-member); \
    __typeof__(expr) unaligned_member_access_result = (expr); \
    COMPILER_WARNING_POP; \
    unaligned_member_access_result; \
})
 
# define UNALIGNED_MEMBER_PTR(ptr, mem) __extension__({ \
    COMPILER_WARNING_PUSH; \
    COMPILER_WARNING_IGNORED(-Waddress-of-packed-member); \
    const volatile void *unaligned_member_ptr_result = &(ptr)->mem; \
    COMPILER_WARNING_POP; \
    (__typeof__((ptr)->mem) *)unaligned_member_ptr_result; \
})
#endif
 
#ifndef UNALIGNED_MEMBER_PTR
# define UNALIGNED_MEMBER_PTR(ptr, mem) UNALIGNED_MEMBER_ACCESS(&(ptr)->mem)
#endif
 
#define RB_OBJ_WRITE_UNALIGNED(old, slot, young) do { \
    VALUE *_slot = UNALIGNED_MEMBER_ACCESS(slot); \
    RB_OBJ_WRITE(old, _slot, young); \
} while (0)
 
/* Used in places that could malloc during, which can cause the GC to run. We
 * need to temporarily disable the GC to allow the malloc to happen.
 * Allocating memory during GC is a bad idea, so use this only when absolutely
 * necessary. */
/* Only re-entrant GC of the current objspace needs suppressing (the malloc happens inside this
 * Ractor), so use the local disable.  The during-GC malloc guard reads the per-objspace dont_gc
 * flag rather than a process-wide one. */
#define DURING_GC_COULD_MALLOC_REGION_START() \
    assert(rb_during_gc()); \
    VALUE _already_disabled = rb_gc_local_disable_no_rest()
 
#define DURING_GC_COULD_MALLOC_REGION_END() \
    if (_already_disabled == Qfalse) rb_gc_local_enable()
 
/* gc.c */
RUBY_ATTR_MALLOC void *ruby_mimmalloc(size_t size);
RUBY_ATTR_MALLOC void *ruby_mimcalloc(size_t num, size_t size);
void ruby_mimfree(void *ptr);
void rb_gc_prepare_heap(void);
VALUE rb_wmap_new_hidden(void);
VALUE rb_wmap_lookup(VALUE wmap, VALUE key);
void rb_wmap_aset(VALUE wmap, VALUE key, VALUE val);
void rb_objspace_set_event_hook(const rb_event_flag_t event);
VALUE rb_objspace_gc_enable(void *objspace);
VALUE rb_objspace_gc_disable(void *objspace);
void ruby_gc_set_params(void);
void rb_gc_copy_attributes(VALUE dest, VALUE obj);
size_t rb_size_mul_or_raise(size_t, size_t, VALUE); /* used in compile.c */
size_t rb_size_mul_add_or_raise(size_t, size_t, size_t, VALUE); /* used in iseq.h */
size_t rb_malloc_grow_capa(size_t current_capacity, size_t type_size);
RUBY_ATTR_MALLOC void *rb_xmalloc_mul_add(size_t, size_t, size_t);
RUBY_ATTR_MALLOC void *rb_xcalloc_mul_add(size_t, size_t, size_t);
void *rb_xrealloc_mul_add(const void *, size_t, size_t, size_t);
RUBY_ATTR_MALLOC void *rb_xmalloc_mul_add_mul(size_t, size_t, size_t, size_t);
RUBY_ATTR_MALLOC void *rb_xcalloc_mul_add_mul(size_t, size_t, size_t, size_t);
void rb_gc_register_pinning_obj(VALUE obj);
struct rb_execution_context_struct *rb_gc_get_ec(void);
VALUE rb_newobj_of_unprotected(VALUE klass, VALUE flags, size_t size);
 
void *rb_gc_ractor_cache_alloc(struct rb_ractor_struct *ractor);
void rb_gc_ractor_cache_free(void *cache);
bool rb_gc_zjit_new_obj_fastpath(size_t alloc_size, VALUE flags, VALUE klass, struct rb_gc_zjit_fastpath *fastpath);
const uintptr_t *rb_gc_zjit_writebarrier_required_p_ptr(void);
 
bool rb_gc_size_allocatable_p(size_t size);
size_t rb_gc_size_slot_size(size_t size);
size_t rb_gc_max_allocation_size(void);
 
void rb_gc_mark_and_move(VALUE *ptr);
 
/* The objspace a marking walk marks into, plus the per-Ractor traversal redirect slot
 * (vm_core.h's gc_mark_func_data_struct) and the shareable-check flag cached from it.
 * Resolving the current Ractor costs an out-of-line rb_current_ec() call on arm64, so a
 * walk resolves it once into a ctx and threads that down instead of repeating it per
 * reference.  Only gc.c builds one; everyone else just passes the pointer along. */
struct gc_mark_func_data_struct;
 
struct rb_gc_mark_ctx {
    void *objspace;
    struct gc_mark_func_data_struct **mfdp;
    bool checking_shareable;
};
 
void rb_gc_mark_and_pin_ctx(const struct rb_gc_mark_ctx *ctx, VALUE obj);
void rb_gc_mark_movable_ctx(const struct rb_gc_mark_ctx *ctx, VALUE obj);
void rb_gc_mark_maybe_ctx(const struct rb_gc_mark_ctx *ctx, VALUE obj);
void rb_gc_mark_and_move_ctx(const struct rb_gc_mark_ctx *ctx, VALUE *ptr);
void rb_gc_mark_locations_ctx(const struct rb_gc_mark_ctx *ctx, const VALUE *start, const VALUE *end);
void rb_mark_tbl_no_pin_ctx(const struct rb_gc_mark_ctx *ctx, st_table *tbl);
void rb_gc_mark_set_no_pin_ctx(const struct rb_gc_mark_ctx *ctx, st_table *tbl);
VALUE rb_gc_location_ctx(const struct rb_gc_mark_ctx *ctx, VALUE value);
void rb_gc_update_moved_ctx(const struct rb_gc_mark_ctx *ctx, VALUE *ptr);
 
static inline bool
rb_gc_checking_shareable_ctx(const struct rb_gc_mark_ctx *ctx)
{
    return ctx->checking_shareable;
}
 
void rb_gc_ref_update_table_values_only(st_table *tbl);
 
void rb_gc_initial_stress_set(VALUE flag);
 
void rb_gc_before_fork(void);
void rb_gc_after_fork(rb_pid_t pid);
 
#define rb_gc_mark_and_move_ptr(ptr) do { \
    VALUE _obj = (VALUE)*(ptr); \
    rb_gc_mark_and_move(&_obj); \
    if (_obj != (VALUE)*(ptr)) *(ptr) = (void *)_obj; \
} while (0)
 
#define rb_gc_update_moved_ptr(ptr) do { \
    VALUE _obj = (VALUE)*(ptr); \
    rb_gc_update_moved(&_obj); \
    if (_obj != (VALUE)*(ptr)) *(ptr) = (void *)_obj; \
} while (0)
 
#define rb_gc_mark_and_move_ptr_ctx(ctx, ptr) do { \
    VALUE _obj = (VALUE)*(ptr); \
    rb_gc_mark_and_move_ctx((ctx), &_obj); \
    if (_obj != (VALUE)*(ptr)) *(ptr) = (void *)_obj; \
} while (0)
 
#define rb_gc_update_moved_ptr_ctx(ctx, ptr) do { \
    VALUE _obj = (VALUE)*(ptr); \
    rb_gc_update_moved_ctx((ctx), &_obj); \
    if (_obj != (VALUE)*(ptr)) *(ptr) = (void *)_obj; \
} while (0)
 
void rb_objspace_reachable_objects_from_local(VALUE obj, void (func)(VALUE, void *), void *data);
 
#if USE_MODULAR_GC
RUBY_SYMBOL_EXPORT_BEGIN
#endif
/* Called only by the default GC's verifier, which lives in the GC's shared object in a
 * modular build, so it needs exporting only there. */
void rb_objspace_reachable_objects_from_unlocked(VALUE obj, void (func)(VALUE, void *), void *data);
#if USE_MODULAR_GC
RUBY_SYMBOL_EXPORT_END
#endif
 
RUBY_SYMBOL_EXPORT_BEGIN
/* exports for objspace module */
void rb_objspace_reachable_objects_from(VALUE obj, void (func)(VALUE, void *), void *data);
void rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *data);
int rb_objspace_internal_object_p(VALUE obj);
int rb_objspace_garbage_object_p(VALUE obj);
int rb_objspace_foreign_object_p(VALUE obj);
int rb_objspace_live_object_p(VALUE obj);
void rb_gc_declare_weak_references(VALUE obj);
bool rb_gc_handle_weak_references_alive_p(VALUE obj);
 
void rb_objspace_each_objects(
    int (*callback)(void *start, void *end, size_t stride, void *data),
    void *data);
 
 
size_t rb_gc_obj_slot_size(VALUE obj);
 
VALUE rb_gc_disable_no_rest(void);
/* Local GC disable/enable covering only the current Ractor's objspace.  Unlike rb_gc_disable*,
 * which became process-wide, this suppresses GC in one's own objspace only.  Exported because
 * DURING_GC_COULD_MALLOC_REGION above expands in bundled extensions. */
VALUE rb_gc_local_enable(void);
VALUE rb_gc_local_disable_no_rest(void);
 
#define RB_GC_MAX_NAME_LEN 20
 
/* gc.c (export) */
const char *rb_objspace_data_type_name(VALUE obj);
VALUE rb_newobj(struct rb_execution_context_struct *, VALUE, VALUE, uint32_t /* shape_id_t */, bool, size_t);
VALUE rb_newobj_of(VALUE, VALUE, size_t);
VALUE rb_ec_newobj_of(struct rb_execution_context_struct *, VALUE, VALUE, size_t);
size_t rb_obj_memsize_of(VALUE);
struct rb_gc_object_metadata_entry *rb_gc_object_metadata(VALUE obj);
void rb_gc_mark_values(long n, const VALUE *values);
void rb_gc_mark_vm_stack_values(long n, const VALUE *values);
struct rb_vm_struct;
void rb_gc_mark_registered_addrs(struct rb_ractor_struct *r, bool need_lock);
void rb_gc_registered_addrs_enroll_without_gc(struct rb_vm_struct *vm, struct rb_ractor_struct *r);
void rb_gc_registered_addrs_unenroll_without_gc(struct rb_vm_struct *vm, struct rb_ractor_struct *r);
void rb_gc_update_values(long n, VALUE *values);
void rb_gc_mark_set_no_pin(st_table *);
/* Exercised by the bundled -test-/gc/writebarrier extension, so it must be visible
 * outside the ruby binary. */
void rb_gc_writebarrier_remember(VALUE obj);
void rb_gc_update_set_refs(st_table *);
 
#if USE_MODULAR_GC
const char *rb_gc_active_gc_name(void);
int rb_gc_modular_gc_loaded_p(void);
#endif
 
RUBY_SYMBOL_EXPORT_END
 
static inline VALUE
rb_obj_atomic_write(
    VALUE a, VALUE *slot, VALUE b,
    RBIMPL_ATTR_MAYBE_UNUSED()
    const char *filename,
    RBIMPL_ATTR_MAYBE_UNUSED()
    int line)
{
#ifdef RGENGC_LOGGING_WRITE
    RGENGC_LOGGING_WRITE(a, slot, b, filename, line);
#endif
 
    RUBY_ATOMIC_VALUE_SET(*slot, b);
 
    rb_obj_written(a, RUBY_Qundef /* ignore `oldv' now */, b, filename, line);
    return a;
}
#define RB_OBJ_ATOMIC_WRITE(old, slot, young) \
    RBIMPL_CAST(rb_obj_atomic_write((VALUE)(old), (VALUE *)(slot), (VALUE)(young), __FILE__, __LINE__))
 
int rb_ec_stack_check(struct rb_execution_context_struct *ec);
void rb_gc_obj_became_shareable(VALUE obj);
bool rb_gc_multi_objspace_p(void);
bool rb_gc_obj_foreign_p(VALUE obj);
void *rb_gc_objspace_alloc(void);
void rb_gc_objspace_retire_gc(void);
/* Build an object, or suppress GC, in a named objspace rather than the current
 * Ractor's.  Only create_ractor_alloc_thread() needs these. */
VALUE rb_data_typed_object_wrap_in_objspace(void *objspace, VALUE klass, void *datap, const rb_data_type_t *type);
VALUE rb_data_typed_object_zalloc_in_objspace(void *objspace, VALUE klass, size_t size, const rb_data_type_t *type);
VALUE rb_gc_objspace_disable_no_rest(void *objspace);
VALUE rb_gc_objspace_enable(void *objspace);
 
void rb_gc_objspace_retire(void **objspace_slot);
void rb_gc_objspace_postmortem_self(void);
void rb_gc_objspace_absorb_into_current(void **objspace_slot);
void rb_gc_objspace_absorb_all_zombies(void);
void rb_gc_objspace_disown(void *objspace);
void rb_gc_zombie_objspaces_atfork(void);
void rb_gc_disable_holders_atfork(void);
void rb_gc_atfork_global_locks(void);
void rb_gc_stash_cleanup_objspace(void);
void rb_gc_rest(void);
bool rb_gc_during_global_gc_p(void);
bool rb_gc_single_objspace_p(void);
const char *rb_obj_info(VALUE obj);
void ruby_annotate_mmap(const void *addr, unsigned long size, const char *name);
 
# define SIZED_REALLOC_N(v, T, m, n) \
    ((v) = (T *)ruby_xrealloc2_sized((void *)(v), (m), sizeof(T), (n)))
 
# define SIZED_FREE(v) ruby_xfree_sized((void *)(v), sizeof(*(v)))
# define SIZED_FREE_N(v, n) ruby_xfree_sized((void *)(v), sizeof(*(v)) * (n))
 
static inline void *
ruby_sized_realloc_n(void *ptr, size_t new_count, size_t element_size, size_t old_count)
{
    return ruby_xrealloc2_sized(ptr, new_count, element_size, old_count);
}
 
void rb_gc_verify_shareable(VALUE);
bool rb_gc_checking_shareable(void);
 
#endif /* INTERNAL_GC_H */