Ruby 4.1.0dev (2026-11-19 revision 1e71e987c206d5d77bf37e883aaa6262747b2458)
gc.c (1e71e987c206d5d77bf37e883aaa6262747b2458)
1/**********************************************************************
2
3 gc.c -
4
5 $Author$
6 created at: Tue Oct 5 09:44:46 JST 1993
7
8 Copyright (C) 1993-2007 Yukihiro Matsumoto
9 Copyright (C) 2000 Network Applied Communication Laboratory, Inc.
10 Copyright (C) 2000 Information-technology Promotion Agency, Japan
11
12**********************************************************************/
13
14#include "ruby/internal/config.h"
15#ifdef _WIN32
16# include "ruby/ruby.h"
17#endif
18
19#if defined(__wasm__) && !defined(__EMSCRIPTEN__)
20# include "wasm/setjmp.h"
21# include "wasm/machine.h"
22#else
23# include <setjmp.h>
24#endif
25#include <stdarg.h>
26#include <stdio.h>
27
28/* MALLOC_HEADERS_BEGIN */
29#ifndef HAVE_MALLOC_USABLE_SIZE
30# ifdef USE_MIMALLOC
31# define HAVE_MALLOC_USABLE_SIZE
32# define malloc_usable_size(a) mi_usable_size(a)
33# elif defined _WIN32
34# define HAVE_MALLOC_USABLE_SIZE
35# define malloc_usable_size(a) _msize(a)
36# elif defined HAVE_MALLOC_SIZE
37# define HAVE_MALLOC_USABLE_SIZE
38# define malloc_usable_size(a) malloc_size(a)
39# endif
40#endif
41
42#ifdef HAVE_MALLOC_USABLE_SIZE
43# ifdef RUBY_ALTERNATIVE_MALLOC_HEADER
44/* Alternative malloc header is included in ruby/missing.h */
45# elif defined(USE_MIMALLOC)
46/* config.h defines restrict, which spoils __declspec(restrict) in mimalloc.h for clang-cl */
47# pragma push_macro("restrict")
48# undef restrict
49# include <mimalloc.h>
50# pragma pop_macro("restrict")
51# elif defined(HAVE_MALLOC_H)
52# include <malloc.h>
53# elif defined(HAVE_MALLOC_NP_H)
54# include <malloc_np.h>
55# elif defined(HAVE_MALLOC_MALLOC_H)
56# include <malloc/malloc.h>
57# endif
58#endif
59
60/* MALLOC_HEADERS_END */
61
62#ifdef HAVE_SYS_TIME_H
63# include <sys/time.h>
64#endif
65
66#ifdef HAVE_SYS_RESOURCE_H
67# include <sys/resource.h>
68#endif
69
70#if defined _WIN32 || defined __CYGWIN__
71# include <windows.h>
72#elif defined(HAVE_POSIX_MEMALIGN)
73#elif defined(HAVE_MEMALIGN)
74# include <malloc.h>
75#endif
76
77#include <sys/types.h>
78
79#ifdef __EMSCRIPTEN__
80#include <emscripten.h>
81#endif
82
83/* For ruby_annotate_mmap */
84#ifdef HAVE_SYS_PRCTL_H
85#include <sys/prctl.h>
86#endif
87
88#undef LIST_HEAD /* ccan/list conflicts with BSD-origin sys/queue.h. */
89
90#include "constant.h"
91#include "debug_counter.h"
92#include "eval_intern.h"
93#include "gc/gc.h"
94#include "id_table.h"
95#include "internal.h"
96#include "internal/class.h"
97#include "internal/compile.h"
98#include "internal/complex.h"
99#include "internal/concurrent_set.h"
100#include "internal/cont.h"
101#include "internal/error.h"
102#include "internal/eval.h"
103#include "internal/gc.h"
104#include "internal/hash.h"
105#include "internal/imemo.h"
106#include "internal/io.h"
107#include "internal/numeric.h"
108#include "internal/object.h"
109#include "internal/proc.h"
110#include "internal/rational.h"
111#include "internal/re.h"
112#include "internal/sanitizers.h"
113#include "internal/struct.h"
114#include "internal/symbol.h"
115#include "internal/thread.h"
116#include "internal/variable.h"
117#include "internal/warnings.h"
118#include "probes.h"
119#include "regint.h"
120#include "ruby/debug.h"
121#include "ruby/io.h"
122#include "ruby/re.h"
123#include "ruby/st.h"
124#include "ruby/thread.h"
125#include "ruby/util.h"
126#include "ruby/vm.h"
127#include "ruby_assert.h"
128#include "ruby_atomic.h"
129#include "symbol.h"
130#include "variable.h"
131#include "vm_core.h"
132#include "vm_sync.h"
133#include "vm_callinfo.h"
134#include "ractor_core.h"
135#include "internal/ractor.h"
136#include "yjit.h"
137#include "zjit.h"
138
139#include "builtin.h"
140#include "shape.h"
141
142// TODO: Don't export this function in modular GC, instead MMTk should figure out
143// how to combine GC thread backtrace with mutator thread backtrace.
144void
145rb_gc_print_backtrace(void)
146{
147 rb_print_backtrace(stderr);
148}
149
150unsigned int
151rb_gc_vm_lock(const char *file, int line)
152{
153 unsigned int lev = 0;
154 rb_vm_lock_enter(&lev, file, line);
155 return lev;
156}
157
158void
159rb_gc_vm_unlock(unsigned int lev, const char *file, int line)
160{
161 rb_vm_lock_leave(&lev, file, line);
162}
163
164unsigned int
165rb_gc_vm_lock_no_barrier(const char *file, int line)
166{
167 unsigned int lev = 0;
168 rb_vm_lock_enter_nb(&lev, file, line);
169 return lev;
170}
171
172void
173rb_gc_vm_unlock_no_barrier(unsigned int lev, const char *file, int line)
174{
175 rb_vm_lock_leave_nb(&lev, file, line);
176}
177
178void
179rb_gc_vm_barrier(void)
180{
181 rb_vm_barrier();
182}
183
184void *
185rb_gc_get_ractor_newobj_cache(void)
186{
187 return GET_RACTOR()->newobj_cache;
188}
189
190void
191rb_gc_initialize_vm_context(struct rb_gc_vm_context *context)
192{
193 context->ec = GET_EC();
194}
195
196bool
197rb_gc_event_hook_required_p(rb_event_flag_t event)
198{
199 return ruby_vm_event_flags & event;
200}
201
202void
203rb_gc_event_hook(VALUE obj, rb_event_flag_t event)
204{
205 if (LIKELY(!rb_gc_event_hook_required_p(event))) return;
206
207 /* Event hooks run user code in the context of the currently running
208 * thread, so they must use the current EC. rb_gc_get_ec() may instead
209 * return the GC's snapshot (vm_context.ec, taken at marking), which can
210 * belong to a different thread once lazy sweep is continued from another
211 * thread's allocation; running the hook on it would set trace_arg on the
212 * wrong EC and break get_trace_arg() (GET_EC()->trace_arg would be NULL). */
213 rb_execution_context_t *ec = GET_EC();
214
215#if USE_MODULAR_GC
216 bool gc_thread_p = false;
217 if (!ec) {
218 /* A dedicated GC thread has no mutator EC: borrow the GC's snapshot and
219 * install it as the current EC so GET_EC() stays consistent inside the
220 * hook (e.g. for get_trace_arg()). */
221 ec = rb_gc_get_ec();
222 gc_thread_p = true;
223
224# ifdef RB_THREAD_LOCAL_SPECIFIER
225 rb_current_ec_set(ec);
226# else
227 native_tls_set(ruby_current_ec_key, ec);
228# endif
229 }
230#endif
231
232 if (RB_LIKELY(ec->cfp != NULL)) {
233 EXEC_EVENT_HOOK(ec, event, ec->cfp->self, 0, 0, 0, obj);
234 }
235
236#if USE_MODULAR_GC
237 if (gc_thread_p) {
238# ifdef RB_THREAD_LOCAL_SPECIFIER
239 rb_current_ec_set(NULL);
240# else
241 native_tls_set(ruby_current_ec_key, NULL);
242# endif
243 }
244#endif
245}
246
247/* Nonzero once the cleanup walk has stashed an objspace, which is the only case where a
248 * thread needs one other than its own Ractor's. One unsynchronised load gates it,
249 * keeping the per-reference marking path and the malloc paths clear of the lookups
250 * below. It is never unwound, so a stale zero only costs a resolution that predates the
251 * stash. */
252static rb_atomic_t gc_objspace_override_count;
253
254/* VM destruct's free-at-exit walk can free the thread and Ractor structs first, so
255 * resolving through the current Ractor would use freed memory; return the objspace
256 * stashed before the walk started. */
257
258void
259rb_gc_stash_cleanup_objspace(void)
260{
261 GET_VM()->gc.cleanup_objspace = rb_gc_get_objspace();
262 /* Never unwound, as the VM is going away: from here every resolution takes the slow
263 * path, which is where the stashed objspace is returned from. */
264 RUBY_ATOMIC_INC(gc_objspace_override_count);
265}
266
267/* Out of line to keep the rare arms out of RB_GC_MARK_OR_TRAVERSE's inline expansions. */
268NOINLINE(static void *gc_current_objspace_slow(rb_ractor_t *const cr));
269
270static void *
271gc_current_objspace_slow(rb_ractor_t *const cr)
272{
273 /* Checked before anything reads cr or the EC: the cleanup walk is the case where
274 * both may already be freed (see rb_gc_stash_cleanup_objspace). */
275 if (RB_UNLIKELY(ruby_vm_during_cleanup) && GET_VM()->gc.cleanup_objspace) {
276 return GET_VM()->gc.cleanup_objspace;
277 }
278 if (cr == NULL) {
279 /* A thread with no current Ractor (a GVL-less native thread freeing in
280 * thread_sched_reclaim, say) uses the main Ractor's objspace. */
281 return GET_VM()->ractor.main_ractor->objspace;
282 }
283 /* A live current Ractor always has an objspace. */
284 RUBY_ASSERT(cr->objspace != NULL);
285 return cr->objspace;
286}
287
288static inline void *
289gc_current_objspace_of(rb_ractor_t *const cr)
290{
291 if (RB_LIKELY(cr != NULL && gc_objspace_override_count == 0)) {
292 RUBY_ASSERT(cr->objspace != NULL);
293 return cr->objspace;
294 }
295 return gc_current_objspace_slow(cr);
296}
297
298void *
299rb_gc_get_objspace(void)
300{
301 return gc_current_objspace_of(rb_current_ractor_raw(false));
302}
303
304void
305rb_gc_run_obj_finalizer(VALUE objid, rb_len_t count, VALUE (*callback)(rb_len_t i, void *data), void *data)
306{
307 volatile struct {
308 VALUE errinfo;
309 VALUE final;
310 rb_control_frame_t *cfp;
311 VALUE *sp;
312 rb_len_t finished;
313 } saved;
314
315 rb_execution_context_t * volatile ec = GET_EC();
316#define RESTORE_FINALIZER() (\
317 ec->cfp = saved.cfp, \
318 ec->cfp->sp = saved.sp, \
319 ec->errinfo = saved.errinfo)
320
321 saved.errinfo = ec->errinfo;
322 saved.cfp = ec->cfp;
323 saved.sp = ec->cfp->sp;
324 saved.finished = 0;
325 saved.final = Qundef;
326
327 ASSERT_vm_unlocking();
328 rb_ractor_ignore_belonging(true);
329 EC_PUSH_TAG(ec);
330 enum ruby_tag_type state = EC_EXEC_TAG();
331 if (state != TAG_NONE) {
332 ++saved.finished; /* skip failed finalizer */
333
334 VALUE failed_final = saved.final;
335 saved.final = Qundef;
336 if (!UNDEF_P(failed_final) && !NIL_P(ruby_verbose)) {
337 rb_warn("Exception in finalizer %+"PRIsVALUE, failed_final);
338 rb_ec_error_print(ec, ec->errinfo);
339 }
340 }
341
342 for (rb_len_t i = saved.finished; RESTORE_FINALIZER(), i < count; saved.finished = ++i) {
343 saved.final = callback(i, data);
344 rb_check_funcall(saved.final, idCall, 1, &objid);
345 }
346 EC_POP_TAG();
347 rb_ractor_ignore_belonging(false);
348#undef RESTORE_FINALIZER
349}
350
351void
352rb_gc_set_pending_interrupt(void)
353{
354 rb_execution_context_t *ec = GET_EC();
355 ec->interrupt_mask |= PENDING_INTERRUPT_MASK;
356}
357
358/* Schedule an objspace's deferred finalizers. A global GC sweeps other Ractors'
359 * objspaces too, so target the owning Ractor rather than the sweeping driver. For an
360 * objspace with no live owner the untargeted fallback is only a wake-up: a zombie's
361 * entries move to the inheriting objspace in the absorb, which re-triggers there. */
362void
363rb_gc_trigger_finalize_deferred(void *objspace, rb_postponed_job_handle_t pjob)
364{
365 rb_ractor_t *const cr = rb_current_ractor_raw(false);
366 if (cr == NULL || cr->objspace != objspace) {
367 /* Only a global GC (stop-the-world) or an absorb settle (under the VM lock)
368 * defers another objspace's finalizers, so ractor.set is stable here. */
369 ASSERT_vm_locking();
370 rb_vm_t *vm = GET_VM();
371 rb_ractor_t *r;
372 ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
373 if (r->objspace == objspace) {
374 rb_postponed_job_trigger_for_ractor(pjob, r->pub.self);
375 return;
376 }
377 }
378 }
380}
381
382void
383rb_gc_trigger_postponed_job_on_main(rb_postponed_job_handle_t pjob)
384{
385 rb_postponed_job_trigger_for_ractor(pjob, GET_VM()->ractor.main_ractor->pub.self);
386}
387
388void
389rb_gc_unset_pending_interrupt(void)
390{
391 rb_execution_context_t *ec = GET_EC();
392 ec->interrupt_mask &= ~PENDING_INTERRUPT_MASK;
393}
394
395bool
396rb_gc_multi_ractor_p(void)
397{
398 return rb_multi_ractor_p();
399}
400
401bool
402rb_gc_shutdown_call_finalizer_p(VALUE obj)
403{
404 switch (BUILTIN_TYPE(obj)) {
405 case T_DATA:
406 if (!ruby_free_at_exit_p()) {
407 if (!RDATA(obj)->type) return false;
408 if (!rbimpl_typeddata_embedded_p(obj) && !RTYPEDDATA(obj)->data) return false;
409 }
410 if (rb_obj_is_thread(obj)) return false;
411 if (rb_obj_is_mutex(obj)) return false;
412 if (rb_obj_is_fiber(obj)) return false;
413 if (rb_ractor_p(obj)) return false;
414 if (rb_obj_is_fstring_table(obj)) return false;
415 if (rb_obj_is_symbol_table(obj)) return false;
416
417 return true;
418
419 case T_FILE:
420 return true;
421
422 case T_SYMBOL:
423 return true;
424
425 case T_NONE:
426 return false;
427
428 default:
429 return ruby_free_at_exit_p();
430 }
431}
432
433void
434rb_gc_obj_changed_slot_size(VALUE obj, size_t slot_size)
435{
436 shape_id_t shape_id = rb_obj_shape_transition_capacity(obj, rb_shape_capacity_for_slot_size(slot_size));
437 RBASIC_SET_FULL_SHAPE_ID(obj, shape_id);
438}
439
440void rb_vm_update_references(void *ptr);
441
442#define rb_setjmp(env) RUBY_SETJMP(env)
443#define rb_jmp_buf rb_jmpbuf_t
444
445#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
446#define MAP_ANONYMOUS MAP_ANON
447#endif
448
449#define unless_objspace(objspace) \
450 void *objspace; \
451 rb_vm_t *unless_objspace_vm = GET_VM(); \
452 if (unless_objspace_vm) objspace = rb_gc_get_objspace(); \
453 else /* return; or objspace will be warned uninitialized */
454
455#define RMOVED(obj) ((struct RMoved *)(obj))
456
457#define TYPED_UPDATE_IF_MOVED(_objspace, _type, _thing) do { \
458 if (gc_object_moved_p_internal((_objspace), (VALUE)(_thing))) { \
459 *(_type *)&(_thing) = (_type)gc_location_internal(_objspace, (VALUE)_thing); \
460 } \
461} while (0)
462
463#define UPDATE_IF_MOVED(_objspace, _thing) TYPED_UPDATE_IF_MOVED(_objspace, VALUE, _thing)
464
465#if RUBY_MARK_FREE_DEBUG
466int ruby_gc_debug_indent = 0;
467#endif
468
469#ifndef RGENGC_OBJ_INFO
470# define RGENGC_OBJ_INFO RGENGC_CHECK_MODE
471#endif
472
473#ifndef CALC_EXACT_MALLOC_SIZE
474# define CALC_EXACT_MALLOC_SIZE 0
475#endif
476
478
479static size_t malloc_offset = 0;
480/* With mimalloc, malloc() below is not what xmalloc uses, and mimalloc
481 * blocks have no header to make room for. */
482#if defined(HAVE_MALLOC_USABLE_SIZE) && !defined(USE_MIMALLOC)
483static size_t
484gc_compute_malloc_offset(void)
485{
486 // Different allocators use different metadata storage strategies which result in different
487 // ideal sizes.
488 // For instance malloc(64) will waste 8B with glibc, but waste 0B with jemalloc.
489 // But malloc(56) will waste 0B with glibc, but waste 8B with jemalloc.
490 // So we try allocating 64, 56 and 48 bytes and select the first offset that doesn't
491 // waste memory.
492 // This was tested on Linux with glibc 2.35 and jemalloc 5, and for both it result in
493 // no wasted memory.
494 size_t offset = 0;
495 for (offset = 0; offset <= 16; offset += 8) {
496 size_t allocated = (64 - offset);
497 void *test_ptr = malloc(allocated);
498 size_t wasted = malloc_usable_size(test_ptr) - allocated;
499 free(test_ptr);
500
501 if (wasted == 0) {
502 return offset;
503 }
504 }
505 return 0;
506}
507#else
508static size_t
509gc_compute_malloc_offset(void)
510{
511 // If we don't have malloc_usable_size, we use powers of 2.
512 return 0;
513}
514#endif
515
516size_t
517rb_malloc_grow_capa(size_t current, size_t type_size)
518{
519 size_t current_capacity = current;
520 if (current_capacity < 4) {
521 current_capacity = 4;
522 }
523 current_capacity *= type_size;
524
525 // We double the current capacity.
526 size_t new_capacity = (current_capacity * 2);
527
528 // And round up to the next power of 2 if it's not already one.
529 if (rb_popcount64(new_capacity) != 1) {
530 new_capacity = (size_t)(1 << (64 - nlz_int64(new_capacity)));
531 }
532
533 new_capacity -= malloc_offset;
534 new_capacity /= type_size;
535 if (current > new_capacity) {
536 rb_bug("rb_malloc_grow_capa: current_capacity=%zu, new_capacity=%zu, malloc_offset=%zu", current, new_capacity, malloc_offset);
537 }
538 RUBY_ASSERT(new_capacity > current);
539 return new_capacity;
540}
541
542static inline struct rbimpl_size_overflow_tag
543size_mul_add_overflow(size_t x, size_t y, size_t z) /* x * y + z */
544{
545 struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(x, y);
546 struct rbimpl_size_overflow_tag u = rbimpl_size_add_overflow(t.result, z);
547 return (struct rbimpl_size_overflow_tag) { t.overflowed || u.overflowed, u.result };
548}
549
550static inline struct rbimpl_size_overflow_tag
551size_mul_add_mul_overflow(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
552{
553 struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(x, y);
554 struct rbimpl_size_overflow_tag u = rbimpl_size_mul_overflow(z, w);
555 struct rbimpl_size_overflow_tag v = rbimpl_size_add_overflow(t.result, u.result);
556 return (struct rbimpl_size_overflow_tag) { t.overflowed || u.overflowed || v.overflowed, v.result };
557}
558
559PRINTF_ARGS(NORETURN(static void gc_raise(VALUE, const char*, ...)), 2, 3);
560
561static inline size_t
562size_mul_or_raise(size_t x, size_t y, VALUE exc)
563{
564 struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(x, y);
565 if (LIKELY(!t.overflowed)) {
566 return t.result;
567 }
568 else if (rb_during_gc()) {
569 rb_memerror(); /* or...? */
570 }
571 else {
572 gc_raise(
573 exc,
574 "integer overflow: %"PRIuSIZE
575 " * %"PRIuSIZE
576 " > %"PRIuSIZE,
577 x, y, (size_t)SIZE_MAX);
578 }
579}
580
581size_t
582rb_size_mul_or_raise(size_t x, size_t y, VALUE exc)
583{
584 return size_mul_or_raise(x, y, exc);
585}
586
587static inline size_t
588size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc)
589{
590 struct rbimpl_size_overflow_tag t = size_mul_add_overflow(x, y, z);
591 if (LIKELY(!t.overflowed)) {
592 return t.result;
593 }
594 else if (rb_during_gc()) {
595 rb_memerror(); /* or...? */
596 }
597 else {
598 gc_raise(
599 exc,
600 "integer overflow: %"PRIuSIZE
601 " * %"PRIuSIZE
602 " + %"PRIuSIZE
603 " > %"PRIuSIZE,
604 x, y, z, (size_t)SIZE_MAX);
605 }
606}
607
608size_t
609rb_size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc)
610{
611 return size_mul_add_or_raise(x, y, z, exc);
612}
613
614static inline size_t
615size_mul_add_mul_or_raise(size_t x, size_t y, size_t z, size_t w, VALUE exc)
616{
617 struct rbimpl_size_overflow_tag t = size_mul_add_mul_overflow(x, y, z, w);
618 if (LIKELY(!t.overflowed)) {
619 return t.result;
620 }
621 else if (rb_during_gc()) {
622 rb_memerror(); /* or...? */
623 }
624 else {
625 gc_raise(
626 exc,
627 "integer overflow: %"PRIdSIZE
628 " * %"PRIdSIZE
629 " + %"PRIdSIZE
630 " * %"PRIdSIZE
631 " > %"PRIdSIZE,
632 x, y, z, w, (size_t)SIZE_MAX);
633 }
634}
635
636#if defined(HAVE_RB_GC_GUARDED_PTR_VAL) && HAVE_RB_GC_GUARDED_PTR_VAL
637/* trick the compiler into thinking a external signal handler uses this */
638volatile VALUE rb_gc_guarded_val;
639volatile VALUE *
640rb_gc_guarded_ptr_val(volatile VALUE *ptr, VALUE val)
641{
642 rb_gc_guarded_val = val;
643
644 return ptr;
645}
646#endif
647
648static const char *obj_type_name(VALUE obj);
649
650/* A forking parent can hold registered_addrs.lock; inheriting it locked would make
651 * the child's first register wait forever, so rebuild it, like the generic_fields
652 * lock. */
653void
654rb_gc_atfork_global_locks(void)
655{
656 rb_vm_t *vm = GET_VM();
657 rb_native_mutex_initialize(&vm->gc.registered_addrs.lock);
658}
659
660#include "gc/default/default.c"
661
662#if USE_MODULAR_GC && !defined(HAVE_DLOPEN)
663# error "Modular GC requires dlopen"
664#elif USE_MODULAR_GC
665#include <dlfcn.h>
666
667typedef struct gc_function_map {
668 // Bootup
669 void *(*objspace_alloc)(void);
670 void (*objspace_init)(void *objspace_ptr);
671 void *(*ractor_cache_alloc)(void *objspace_ptr, void *ractor);
672 void (*objspace_retire_gc)(void *objspace_ptr);
673 void (*set_params)(void *objspace_ptr);
674 void (*init)(void);
675 // Shutdown
676 void (*shutdown_free_objects)(void *objspace_ptr);
677 void (*objspace_free)(void *objspace_ptr);
678 void (*ractor_cache_free)(void *objspace_ptr, void *cache);
679 // GC
680 void (*start)(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact, bool global);
681 bool (*during_gc_p)(void *objspace_ptr);
682 void (*prepare_heap)(void *objspace_ptr);
683 void (*gc_enable)(void *objspace_ptr);
684 void (*gc_disable)(void *objspace_ptr, bool finish_current_gc);
685 bool (*gc_enabled_p)(void *objspace_ptr);
686 bool (*user_gc_disabled_set)(void *objspace_ptr, bool disable);
687 bool (*user_gc_disabled_p)(void *objspace_ptr);
688 bool (*multi_objspace_p)(void);
689 bool (*during_global_gc_p)(void *objspace_ptr);
690 bool (*during_postmortem_p)(void *objspace_ptr);
691 bool (*obj_foreign_p)(void *objspace_ptr, VALUE obj);
692 void (*objspace_set_ractor)(void *objspace_ptr, void *ractor);
693 void *(*obj_ractor)(void *objspace_ptr, VALUE obj);
694 bool (*shref_marked_p)(void *objspace_ptr, VALUE obj);
695 void (*objspace_zombify)(void *objspace_ptr);
696 void (*objspace_absorb)(void *dst_ptr, void *src_ptr);
697 void (*gc_rest)(void *objspace_ptr);
698 VALUE (*config_get)(void *objpace_ptr);
699 void (*config_set)(void *objspace_ptr, VALUE hash);
700 void (*stress_set)(void *objspace_ptr, VALUE flag);
701 VALUE (*stress_get)(void *objspace_ptr);
702 struct rb_gc_vm_context *(*get_vm_context)(void *objspace_ptr);
703 // Object allocation
704 VALUE (*new_obj)(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size, size_t *actual_alloc_size);
705 bool (*zjit_new_obj_fastpath)(void *objspace_ptr, size_t alloc_size, VALUE flags, VALUE klass, struct rb_gc_zjit_fastpath *fastpath);
706 int32_t (*zjit_writebarrier_required_offset)(void);
707 size_t (*obj_slot_size)(VALUE obj);
708 size_t (*size_slot_size)(void *objspace_ptr, size_t size);
709 bool (*size_allocatable_p)(size_t size);
710 size_t (*max_allocation_size)(void);
711 // Malloc
712 void *(*malloc)(void *objspace_ptr, size_t size, bool gc_allowed);
713 void *(*calloc)(void *objspace_ptr, size_t size, bool gc_allowed);
714 void *(*realloc)(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed);
715 void (*free)(void *objspace_ptr, void *ptr, size_t old_size);
716 void (*adjust_memory_usage)(void *objspace_ptr, ssize_t diff);
717 // Marking
718 void (*mark)(void *objspace_ptr, VALUE obj);
719 void (*mark_and_move)(void *objspace_ptr, VALUE *ptr);
720 void (*mark_and_pin)(void *objspace_ptr, VALUE obj);
721 void (*mark_maybe)(void *objspace_ptr, VALUE obj);
722 // Weak references
723 void (*declare_weak_references)(void *objspace_ptr, VALUE obj);
724 bool (*handle_weak_references_alive_p)(void *objspace_ptr, VALUE obj);
725 // Compaction
726 void (*register_pinning_obj)(void *objspace_ptr, VALUE obj);
727 bool (*object_moved_p)(void *objspace_ptr, VALUE obj);
728 bool (*pinned_p)(void *objspace_ptr, VALUE obj);
729 VALUE (*location)(void *objspace_ptr, VALUE value);
730 // Write barriers
731 void (*writebarrier)(void *objspace_ptr, VALUE a, VALUE b);
732 void (*writebarrier_unprotect)(void *objspace_ptr, VALUE obj);
733 void (*writebarrier_remember)(void *objspace_ptr, VALUE obj);
734 void (*obj_became_shareable)(void *objspace_ptr, VALUE obj);
735 // Heap walking
736 void (*each_objects)(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data);
737 void (*each_objects_shareable)(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data);
738 void (*each_objects_foreign)(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data);
739 void (*each_object)(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data);
740 // Finalizers
741 void (*make_zombie)(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data);
742 VALUE (*define_finalizer)(void *objspace_ptr, VALUE obj, VALUE block);
743 void (*undefine_finalizer)(void *objspace_ptr, VALUE obj);
744 void (*copy_finalizer)(void *objspace_ptr, VALUE dest, VALUE obj);
745 void (*shutdown_call_finalizer)(void *objspace_ptr);
746 // Forking
747 void (*before_fork)(void *objspace_ptr);
748 void (*after_fork)(void *objspace_ptr, rb_pid_t pid);
749 // Statistics
750 void (*set_measure_total_time)(void *objspace_ptr, VALUE flag);
751 bool (*get_measure_total_time)(void *objspace_ptr);
752 unsigned long long (*get_total_time)(void *objspace_ptr);
753 size_t (*gc_count)(void *objspace_ptr);
754 VALUE (*latest_gc_info)(void *objspace_ptr, VALUE key);
755 VALUE (*stat)(void *objspace_ptr, VALUE hash_or_sym);
756 VALUE (*stat_heap)(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym);
757 const char *(*active_gc_name)(void);
758 // Miscellaneous
759 struct rb_gc_object_metadata_entry *(*object_metadata)(void *objspace_ptr, VALUE obj);
760 bool (*live_object_p)(void *objspace_ptr, const void *ptr);
761 bool (*garbage_object_p)(void *objspace_ptr, VALUE obj);
762 void (*set_event_hook)(void *objspace_ptr, const rb_event_flag_t event);
763 void (*copy_attributes)(void *objspace_ptr, VALUE dest, VALUE obj);
764
765 bool modular_gc_loaded_p;
766} rb_gc_function_map_t;
767
768static rb_gc_function_map_t rb_gc_functions;
769
770# define RUBY_GC_LIBRARY "RUBY_GC_LIBRARY"
771# define MODULAR_GC_DIR STRINGIZE(modular_gc_dir)
772
773static void
774ruby_modular_gc_init(void)
775{
776 // Assert that the directory path ends with a /
777 RUBY_ASSERT_ALWAYS(MODULAR_GC_DIR[sizeof(MODULAR_GC_DIR) - 2] == '/');
778
779 const char *gc_so_file = getenv(RUBY_GC_LIBRARY);
780
781 rb_gc_function_map_t gc_functions = { 0 };
782
783 char *gc_so_path = NULL;
784 void *handle = NULL;
785 if (gc_so_file) {
786 /* Check to make sure that gc_so_file matches /[\w-_]+/ so that it does
787 * not load a shared object outside of the directory. */
788 for (size_t i = 0; i < strlen(gc_so_file); i++) {
789 char c = gc_so_file[i];
790 if (isalnum(c)) continue;
791 switch (c) {
792 case '-':
793 case '_':
794 break;
795 default:
796 fprintf(stderr, "Only alphanumeric, dash, and underscore is allowed in "RUBY_GC_LIBRARY"\n");
797 exit(EXIT_FAILURE);
798 }
799 }
800
801 size_t gc_so_path_size = strlen(MODULAR_GC_DIR "librubygc." DLEXT) + strlen(gc_so_file) + 1;
802#ifdef LOAD_RELATIVE
803 Dl_info dli;
804 size_t prefix_len = 0;
805 if (dladdr((void *)(uintptr_t)ruby_modular_gc_init, &dli)) {
806 const char *base = strrchr(dli.dli_fname, '/');
807 if (base) {
808 size_t tail = 0;
809# define end_with_p(lit) \
810 (prefix_len >= (tail = rb_strlen_lit(lit)) && \
811 memcmp(base - tail, lit, tail) == 0)
812
813 prefix_len = base - dli.dli_fname;
814 if (end_with_p("/bin") || end_with_p("/lib")) {
815 prefix_len -= tail;
816 }
817 prefix_len += MODULAR_GC_DIR[0] != '/';
818 gc_so_path_size += prefix_len;
819 }
820 }
821#endif
822 gc_so_path = alloca(gc_so_path_size);
823 {
824 size_t gc_so_path_idx = 0;
825#define GC_SO_PATH_APPEND(str) do { \
826 gc_so_path_idx += strlcpy(gc_so_path + gc_so_path_idx, str, gc_so_path_size - gc_so_path_idx); \
827} while (0)
828#ifdef LOAD_RELATIVE
829 if (prefix_len > 0) {
830 memcpy(gc_so_path, dli.dli_fname, prefix_len);
831 gc_so_path_idx = prefix_len;
832 }
833#endif
834 GC_SO_PATH_APPEND(MODULAR_GC_DIR "librubygc.");
835 GC_SO_PATH_APPEND(gc_so_file);
836 GC_SO_PATH_APPEND(DLEXT);
837 GC_ASSERT(gc_so_path_idx == gc_so_path_size - 1);
838#undef GC_SO_PATH_APPEND
839 }
840
841 handle = dlopen(gc_so_path, RTLD_LAZY | RTLD_GLOBAL);
842 if (!handle) {
843 fprintf(stderr, "ruby_modular_gc_init: Shared library %s cannot be opened: %s\n", gc_so_path, dlerror());
844 exit(EXIT_FAILURE);
845 }
846
847 gc_functions.modular_gc_loaded_p = true;
848 }
849
850 unsigned int err_count = 0;
851
852# define load_modular_gc_func(name) do { \
853 if (handle) { \
854 const char *func_name = "rb_gc_impl_" #name; \
855 gc_functions.name = dlsym(handle, func_name); \
856 if (!gc_functions.name) { \
857 fprintf(stderr, "ruby_modular_gc_init: %s function not exported by library %s\n", func_name, gc_so_path); \
858 err_count++; \
859 } \
860 } \
861 else { \
862 gc_functions.name = rb_gc_impl_##name; \
863 } \
864} while (0)
865
866 // Bootup
867 load_modular_gc_func(objspace_alloc);
868 load_modular_gc_func(objspace_init);
869 load_modular_gc_func(ractor_cache_alloc);
870 load_modular_gc_func(objspace_retire_gc);
871 load_modular_gc_func(set_params);
872 load_modular_gc_func(init);
873 // Shutdown
874 load_modular_gc_func(shutdown_free_objects);
875 load_modular_gc_func(objspace_free);
876 load_modular_gc_func(ractor_cache_free);
877 // GC
878 load_modular_gc_func(start);
879 load_modular_gc_func(during_gc_p);
880 load_modular_gc_func(prepare_heap);
881 load_modular_gc_func(gc_enable);
882 load_modular_gc_func(gc_disable);
883 load_modular_gc_func(gc_enabled_p);
884 load_modular_gc_func(user_gc_disabled_set);
885 load_modular_gc_func(user_gc_disabled_p);
886 load_modular_gc_func(multi_objspace_p);
887 load_modular_gc_func(during_global_gc_p);
888 load_modular_gc_func(during_postmortem_p);
889 load_modular_gc_func(obj_foreign_p);
890 load_modular_gc_func(objspace_set_ractor);
891 load_modular_gc_func(obj_ractor);
892 load_modular_gc_func(shref_marked_p);
893 load_modular_gc_func(objspace_zombify);
894 load_modular_gc_func(objspace_absorb);
895 load_modular_gc_func(gc_rest);
896 load_modular_gc_func(config_set);
897 load_modular_gc_func(config_get);
898 load_modular_gc_func(stress_set);
899 load_modular_gc_func(stress_get);
900 load_modular_gc_func(get_vm_context);
901 // Object allocation
902 load_modular_gc_func(new_obj);
903 load_modular_gc_func(zjit_new_obj_fastpath);
904 load_modular_gc_func(zjit_writebarrier_required_offset);
905 load_modular_gc_func(obj_slot_size);
906 load_modular_gc_func(size_slot_size);
907 load_modular_gc_func(size_allocatable_p);
908 load_modular_gc_func(max_allocation_size);
909 // Malloc
910 load_modular_gc_func(malloc);
911 load_modular_gc_func(calloc);
912 load_modular_gc_func(realloc);
913 load_modular_gc_func(free);
914 load_modular_gc_func(adjust_memory_usage);
915 // Marking
916 load_modular_gc_func(mark);
917 load_modular_gc_func(mark_and_move);
918 load_modular_gc_func(mark_and_pin);
919 load_modular_gc_func(mark_maybe);
920 // Weak references
921 load_modular_gc_func(declare_weak_references);
922 load_modular_gc_func(handle_weak_references_alive_p);
923 // Compaction
924 load_modular_gc_func(register_pinning_obj);
925 load_modular_gc_func(object_moved_p);
926 load_modular_gc_func(pinned_p);
927 load_modular_gc_func(location);
928 // Write barriers
929 load_modular_gc_func(writebarrier);
930 load_modular_gc_func(writebarrier_unprotect);
931 load_modular_gc_func(writebarrier_remember);
932 load_modular_gc_func(obj_became_shareable);
933 // Heap walking
934 load_modular_gc_func(each_objects);
935 load_modular_gc_func(each_objects_shareable);
936 load_modular_gc_func(each_objects_foreign);
937 load_modular_gc_func(each_object);
938 // Finalizers
939 load_modular_gc_func(make_zombie);
940 load_modular_gc_func(define_finalizer);
941 load_modular_gc_func(undefine_finalizer);
942 load_modular_gc_func(copy_finalizer);
943 load_modular_gc_func(shutdown_call_finalizer);
944 // Forking
945 load_modular_gc_func(before_fork);
946 load_modular_gc_func(after_fork);
947 // Statistics
948 load_modular_gc_func(set_measure_total_time);
949 load_modular_gc_func(get_measure_total_time);
950 load_modular_gc_func(get_total_time);
951 load_modular_gc_func(gc_count);
952 load_modular_gc_func(latest_gc_info);
953 load_modular_gc_func(stat);
954 load_modular_gc_func(stat_heap);
955 load_modular_gc_func(active_gc_name);
956 // Miscellaneous
957 load_modular_gc_func(object_metadata);
958 load_modular_gc_func(live_object_p);
959 load_modular_gc_func(garbage_object_p);
960 load_modular_gc_func(set_event_hook);
961 load_modular_gc_func(copy_attributes);
962
963 if (err_count > 0) {
964 fprintf(stderr, "ruby_modular_gc_init: found %u missing exports in library %s\n", err_count, gc_so_path);
965 exit(EXIT_FAILURE);
966 }
967
968# undef load_modular_gc_func
969
970 rb_gc_functions = gc_functions;
971}
972
973// Bootup
974# define rb_gc_impl_objspace_alloc rb_gc_functions.objspace_alloc
975# define rb_gc_impl_objspace_init rb_gc_functions.objspace_init
976# define rb_gc_impl_ractor_cache_alloc rb_gc_functions.ractor_cache_alloc
977# define rb_gc_impl_objspace_retire_gc rb_gc_functions.objspace_retire_gc
978# define rb_gc_impl_set_params rb_gc_functions.set_params
979# define rb_gc_impl_init rb_gc_functions.init
980// Shutdown
981# define rb_gc_impl_shutdown_free_objects rb_gc_functions.shutdown_free_objects
982# define rb_gc_impl_objspace_free rb_gc_functions.objspace_free
983# define rb_gc_impl_ractor_cache_free rb_gc_functions.ractor_cache_free
984// GC
985# define rb_gc_impl_start rb_gc_functions.start
986# define rb_gc_impl_during_gc_p rb_gc_functions.during_gc_p
987# define rb_gc_impl_prepare_heap rb_gc_functions.prepare_heap
988# define rb_gc_impl_gc_enable rb_gc_functions.gc_enable
989# define rb_gc_impl_gc_disable rb_gc_functions.gc_disable
990# define rb_gc_impl_gc_enabled_p rb_gc_functions.gc_enabled_p
991# define rb_gc_impl_user_gc_disabled_set rb_gc_functions.user_gc_disabled_set
992# define rb_gc_impl_user_gc_disabled_p rb_gc_functions.user_gc_disabled_p
993# define rb_gc_impl_multi_objspace_p rb_gc_functions.multi_objspace_p
994# define rb_gc_impl_during_global_gc_p rb_gc_functions.during_global_gc_p
995# define rb_gc_impl_during_postmortem_p rb_gc_functions.during_postmortem_p
996# define rb_gc_impl_obj_foreign_p rb_gc_functions.obj_foreign_p
997# define rb_gc_impl_objspace_set_ractor rb_gc_functions.objspace_set_ractor
998# define rb_gc_impl_obj_ractor rb_gc_functions.obj_ractor
999# define rb_gc_impl_shref_marked_p rb_gc_functions.shref_marked_p
1000# define rb_gc_impl_objspace_zombify rb_gc_functions.objspace_zombify
1001# define rb_gc_impl_objspace_absorb rb_gc_functions.objspace_absorb
1002# define rb_gc_impl_gc_rest rb_gc_functions.gc_rest
1003# define rb_gc_impl_config_get rb_gc_functions.config_get
1004# define rb_gc_impl_config_set rb_gc_functions.config_set
1005# define rb_gc_impl_stress_set rb_gc_functions.stress_set
1006# define rb_gc_impl_stress_get rb_gc_functions.stress_get
1007# define rb_gc_impl_get_vm_context rb_gc_functions.get_vm_context
1008// Object allocation
1009# define rb_gc_impl_new_obj rb_gc_functions.new_obj
1010# define rb_gc_impl_zjit_new_obj_fastpath rb_gc_functions.zjit_new_obj_fastpath
1011# define rb_gc_impl_zjit_writebarrier_required_offset rb_gc_functions.zjit_writebarrier_required_offset
1012# define rb_gc_impl_obj_slot_size rb_gc_functions.obj_slot_size
1013# define rb_gc_impl_size_slot_size rb_gc_functions.size_slot_size
1014# define rb_gc_impl_size_allocatable_p rb_gc_functions.size_allocatable_p
1015# define rb_gc_impl_max_allocation_size rb_gc_functions.max_allocation_size
1016// Malloc
1017# define rb_gc_impl_malloc rb_gc_functions.malloc
1018# define rb_gc_impl_calloc rb_gc_functions.calloc
1019# define rb_gc_impl_realloc rb_gc_functions.realloc
1020# define rb_gc_impl_free rb_gc_functions.free
1021# define rb_gc_impl_adjust_memory_usage rb_gc_functions.adjust_memory_usage
1022// Marking
1023# define rb_gc_impl_mark rb_gc_functions.mark
1024# define rb_gc_impl_mark_and_move rb_gc_functions.mark_and_move
1025# define rb_gc_impl_mark_and_pin rb_gc_functions.mark_and_pin
1026# define rb_gc_impl_mark_maybe rb_gc_functions.mark_maybe
1027// Weak references
1028# define rb_gc_impl_declare_weak_references rb_gc_functions.declare_weak_references
1029# define rb_gc_impl_handle_weak_references_alive_p rb_gc_functions.handle_weak_references_alive_p
1030// Compaction
1031# define rb_gc_impl_register_pinning_obj rb_gc_functions.register_pinning_obj
1032# define rb_gc_impl_object_moved_p rb_gc_functions.object_moved_p
1033# define rb_gc_impl_pinned_p rb_gc_functions.pinned_p
1034# define rb_gc_impl_location rb_gc_functions.location
1035// Write barriers
1036# define rb_gc_impl_writebarrier rb_gc_functions.writebarrier
1037# define rb_gc_impl_writebarrier_unprotect rb_gc_functions.writebarrier_unprotect
1038# define rb_gc_impl_writebarrier_remember rb_gc_functions.writebarrier_remember
1039# define rb_gc_impl_obj_became_shareable rb_gc_functions.obj_became_shareable
1040// Heap walking
1041# define rb_gc_impl_each_objects rb_gc_functions.each_objects
1042# define rb_gc_impl_each_objects_shareable rb_gc_functions.each_objects_shareable
1043# define rb_gc_impl_each_objects_foreign rb_gc_functions.each_objects_foreign
1044# define rb_gc_impl_each_object rb_gc_functions.each_object
1045// Finalizers
1046# define rb_gc_impl_make_zombie rb_gc_functions.make_zombie
1047# define rb_gc_impl_define_finalizer rb_gc_functions.define_finalizer
1048# define rb_gc_impl_undefine_finalizer rb_gc_functions.undefine_finalizer
1049# define rb_gc_impl_copy_finalizer rb_gc_functions.copy_finalizer
1050# define rb_gc_impl_shutdown_call_finalizer rb_gc_functions.shutdown_call_finalizer
1051// Forking
1052# define rb_gc_impl_before_fork rb_gc_functions.before_fork
1053# define rb_gc_impl_after_fork rb_gc_functions.after_fork
1054// Statistics
1055# define rb_gc_impl_set_measure_total_time rb_gc_functions.set_measure_total_time
1056# define rb_gc_impl_get_measure_total_time rb_gc_functions.get_measure_total_time
1057# define rb_gc_impl_get_total_time rb_gc_functions.get_total_time
1058# define rb_gc_impl_gc_count rb_gc_functions.gc_count
1059# define rb_gc_impl_latest_gc_info rb_gc_functions.latest_gc_info
1060# define rb_gc_impl_stat rb_gc_functions.stat
1061# define rb_gc_impl_stat_heap rb_gc_functions.stat_heap
1062# define rb_gc_impl_active_gc_name rb_gc_functions.active_gc_name
1063// Miscellaneous
1064# define rb_gc_impl_object_metadata rb_gc_functions.object_metadata
1065# define rb_gc_impl_live_object_p rb_gc_functions.live_object_p
1066# define rb_gc_impl_garbage_object_p rb_gc_functions.garbage_object_p
1067# define rb_gc_impl_set_event_hook rb_gc_functions.set_event_hook
1068# define rb_gc_impl_copy_attributes rb_gc_functions.copy_attributes
1069#endif
1070
1071#ifdef RUBY_ASAN_ENABLED
1072static void
1073asan_death_callback(void)
1074{
1075 if (GET_VM()) {
1076 rb_bug_without_die("ASAN error");
1077 }
1078}
1079#endif
1080
1081static VALUE initial_stress = Qfalse;
1082
1083void
1084rb_gc_init_objspaces(void)
1085{
1086#if USE_MODULAR_GC
1087 ruby_modular_gc_init();
1088#endif
1089
1090 rb_vm_t *vm = ruby_current_vm_ptr;
1091
1092 void *objspace = rb_gc_impl_objspace_alloc();
1093 RUBY_ASSERT(vm->ractor.main_ractor != NULL);
1094 vm->ractor.main_ractor->objspace = objspace;
1095 rb_gc_impl_objspace_init(objspace);
1096 rb_gc_objspace_set_ractor(objspace, vm->ractor.main_ractor);
1097 rb_gc_impl_stress_set(objspace, initial_stress);
1098
1099#ifdef RUBY_ASAN_ENABLED
1100 __sanitizer_set_death_callback(asan_death_callback);
1101#endif
1102}
1103
1104/* Stays true once a second objspace has existed (rb_multi_ractor_p goes back to false
1105 * when the other Ractors finish, and is false for the whole window in which the creator
1106 * builds the child objspace). */
1107static bool gc_ever_multi_ractor = false;
1108
1109bool
1110rb_gc_ever_multi_ractor_p(void)
1111{
1112 return gc_ever_multi_ractor;
1113}
1114
1115/* Allocate the objspace of a new non-main Ractor. Called on the creating Ractor's
1116 * thread, before the new Ractor starts running. */
1117void *
1118rb_gc_objspace_alloc(void)
1119{
1120 /* The one store that ever runs is the first Ractor creation, which is by definition
1121 * single-Ractor and so races no reader. */
1122 if (!gc_ever_multi_ractor) gc_ever_multi_ractor = true;
1123 if (!rb_gc_impl_multi_objspace_p()) {
1124 /* One objspace shared by every Ractor. */
1125 return rb_gc_get_objspace();
1126 }
1127 void *objspace = rb_gc_impl_objspace_alloc();
1128 rb_gc_impl_objspace_init(objspace);
1129
1130 return objspace;
1131}
1132
1133void
1134rb_objspace_free(void *objspace)
1135{
1136 rb_gc_impl_objspace_free(objspace);
1137}
1138
1139size_t
1140rb_gc_obj_slot_size(VALUE obj)
1141{
1142 return rb_gc_impl_obj_slot_size(obj);
1143}
1144
1145static inline void
1146gc_validate_pc(VALUE obj)
1147{
1148#if RUBY_DEBUG
1149 // IMEMOs and objects without a class (e.g managed id table) are not traceable
1150 if (RB_TYPE_P(obj, T_IMEMO) || !CLASS_OF(obj)) return;
1151
1152 rb_execution_context_t *ec = GET_EC();
1153 const rb_control_frame_t *cfp = ec->cfp;
1154 if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && CFP_PC(cfp)) {
1155 const VALUE *iseq_encoded = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded;
1156 const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(CFP_ISEQ(cfp))->iseq_size;
1157 RUBY_ASSERT(CFP_PC(cfp) >= iseq_encoded, "PC not set when allocating, breaking tracing");
1158 RUBY_ASSERT(CFP_PC(cfp) <= iseq_encoded_end, "PC not set when allocating, breaking tracing");
1159 }
1160#endif
1161}
1162
1163NOINLINE(static void gc_newobj_hook(VALUE obj));
1164static void
1165gc_newobj_hook(VALUE obj)
1166{
1167 int lev = RB_GC_VM_LOCK_NO_BARRIER();
1168 {
1169 size_t slot_size = rb_gc_obj_slot_size(obj);
1170 memset((char *)obj + sizeof(struct RBasic), 0, slot_size - sizeof(struct RBasic));
1171
1172 /* We must disable GC here because the callback could call xmalloc
1173 * which could potentially trigger a GC, and a lot of code is unsafe
1174 * to trigger a GC right after an object has been allocated because
1175 * they perform initialization for the object and assume that the
1176 * GC does not trigger before then. */
1177 bool gc_disabled = RTEST(rb_gc_local_disable_no_rest());
1178 {
1179 rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_NEWOBJ);
1180 }
1181 if (!gc_disabled) rb_gc_local_enable();
1182 }
1183 RB_GC_VM_UNLOCK_NO_BARRIER(lev);
1184}
1185
1186ALWAYS_INLINE(static VALUE newobj_body(rb_ractor_t *cr, void *objspace, VALUE klass, VALUE flags, shape_id_t shape_id, bool wb_protected, size_t size));
1187
1188/* The allocation body shared by rb_newobj and rb_ec_newobj_of, forced inline into
1189 * both: left to this big translation unit's inline budget, gcc drops it from one
1190 * entry point or the other and that allocation path grows a call. */
1191static VALUE
1192newobj_body(rb_ractor_t *cr, void *objspace, VALUE klass, VALUE flags, shape_id_t shape_id, bool wb_protected, size_t size)
1193{
1194 GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
1195 size_t actual_alloc_size;
1196 VALUE obj = rb_gc_impl_new_obj(objspace, cr->newobj_cache, klass, flags, wb_protected, size, &actual_alloc_size);
1197
1198 GC_ASSERT(actual_alloc_size >= size);
1199 shape_id = rb_shape_transition_slot_size(shape_id, actual_alloc_size);
1200
1201 RBASIC_SET_FULL_SHAPE_ID_NO_CHECKS(obj, shape_id);
1202
1203 gc_validate_pc(obj);
1204
1205 if (UNLIKELY(rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_NEWOBJ))) {
1206 gc_newobj_hook(obj);
1207 }
1208
1209 if (RUBY_DTRACE_GC_OBJ_NEW_ENABLED()) {
1210 RUBY_DTRACE_GC_OBJ_NEW((void*)obj, flags);
1211 }
1212
1213#if RGENGC_CHECK_MODE
1214# ifndef GC_DEBUG_SLOT_FILL_SPECIAL_VALUE
1215# define GC_DEBUG_SLOT_FILL_SPECIAL_VALUE 255
1216# endif
1217
1218 memset(
1219 (void *)(obj + sizeof(struct RBasic)),
1220 GC_DEBUG_SLOT_FILL_SPECIAL_VALUE,
1221 rb_gc_obj_slot_size(obj) - sizeof(struct RBasic)
1222 );
1223#endif
1224
1225 return obj;
1226}
1227
1228VALUE
1229rb_newobj(rb_execution_context_t *ec, VALUE klass, VALUE flags, shape_id_t shape_id, bool wb_protected, size_t size)
1230{
1231 /* Read the Ractor's slot directly: rb_gc_get_objspace() would look cr up through
1232 * TLS on every allocation. */
1233 rb_ractor_t *cr = rb_ec_ractor_ptr(ec);
1234 return newobj_body(cr, cr->objspace, klass, flags, shape_id, wb_protected, size);
1235}
1236
1237/* Build the object in a named objspace rather than the current Ractor's. The only
1238 * foreign objspace allowed is the child this Ractor is building
1239 * (create_ractor_alloc_thread), whose wrappers must be objects the child owns.
1240 *
1241 * That target has no thread of its own yet, which is what makes this cheap: nothing
1242 * allocates, sweeps or collects there, so the half-built objects need no root and its
1243 * GC can be suppressed outright. Aiming at a live Ractor's objspace -- to build a
1244 * copy where it will be used, say -- needs three things this does not have: a root the
1245 * target's own GC marks the objects under construction from, a newobj_cache paired
1246 * with that objspace (today the only multi-objspace collector has no per-Ractor cache,
1247 * and the ones that do are single-objspace), and write barriers aimed at the target.
1248 * The assertion stops the shortcut. */
1249static VALUE
1250rb_newobj_in_objspace(rb_execution_context_t *ec, void *objspace, VALUE klass, VALUE flags, shape_id_t shape_id, bool wb_protected, size_t size)
1251{
1252 rb_ractor_t *cr = rb_ec_ractor_ptr(ec);
1253 RUBY_ASSERT(objspace == cr->objspace || objspace == cr->creating_child_objspace);
1254 return newobj_body(cr, objspace, klass, flags, shape_id, wb_protected, size);
1255}
1256
1257VALUE
1258rb_ec_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flags, size_t size)
1259{
1260 VALUE type = flags & T_MASK;
1266 (void)type;
1267
1268 rb_ractor_t *cr = rb_ec_ractor_ptr(ec);
1269 return newobj_body(cr, cr->objspace, klass, flags, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, true, size);
1270}
1271
1272static VALUE
1273rb_newobj_of_with_shape(VALUE klass, VALUE flags, shape_id_t shape_id, size_t size)
1274{
1275 return rb_newobj(GET_EC(), klass, flags, shape_id, true, size);
1276}
1277
1278VALUE
1279rb_newobj_of(VALUE klass, VALUE flags, size_t size)
1280{
1281 return rb_newobj(GET_EC(), klass, flags, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, true, size);
1282}
1283
1284static
1285VALUE class_allocate_complex_instance(VALUE klass, uint32_t capacity)
1286{
1287 VALUE obj = rb_newobj_of_with_shape(klass, T_OBJECT, rb_shape_transition_extended(ROOT_COMPLEX_SHAPE_ID), sizeof(struct RObject));
1288 // The shape already says extended, so a GC during the allocation below
1289 // would mark an uninitialized as.extended.
1290 ROBJECT(obj)->as.extended = Qfalse;
1291 VALUE fields_obj = rb_imemo_fields_new_complex(obj, ROOT_COMPLEX_SHAPE_ID, capacity, false);
1292 ROBJECT_SET_EXTENDED(obj, fields_obj);
1293 return obj;
1294}
1295
1296static inline size_t
1297robject_embedded_size(uint32_t fields_count)
1298{
1299 size_t size = rb_obj_embedded_size(fields_count);
1300 if (!rb_gc_size_allocatable_p(size)) {
1301 size = sizeof(struct RObject);
1302 }
1303 return size;
1304}
1305
1306VALUE
1307rb_class_allocate_instance_capa(VALUE klass, attr_index_t max_iv_count)
1308{
1309 VALUE obj;
1310
1311 // Directly start as COMPLEX if we know we're over the limit.
1312 RUBY_ASSERT(rb_shape_max_capacity() > 0);
1313 if (RB_UNLIKELY(max_iv_count > rb_shape_max_capacity())) {
1314 obj = class_allocate_complex_instance(klass, max_iv_count);
1315 }
1316 else {
1317 size_t size = robject_embedded_size(max_iv_count);
1318
1319 // There might be a NEWOBJ tracepoint callback, and it may set fields.
1320 // So the shape must be passed to `NEWOBJ_OF`.
1321 obj = rb_newobj_of_with_shape(klass, T_OBJECT, rb_shape_transition_robject(0), size);
1322
1323 #if RUBY_DEBUG
1324 VALUE *ptr = ROBJECT_FIELDS(obj);
1325 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1326 attr_index_t fields_count = RSHAPE_LEN(shape_id);
1327 attr_index_t capacity = RSHAPE_CAPACITY(shape_id);
1328
1329 for (attr_index_t i = fields_count; i < capacity; i++) {
1330 ptr[i] = Qundef;
1331 }
1332 #endif
1333 }
1334
1335#if RUBY_DEBUG
1336 if (rb_obj_class(obj) != rb_class_real(klass)) {
1337 rb_bug("Expected rb_class_allocate_instance to set the class correctly");
1338 }
1339#endif
1340
1341 return obj;
1342}
1343
1344VALUE
1345rb_class_allocate_instance(VALUE klass)
1346{
1347 return rb_class_allocate_instance_capa(klass, RCLASS_MAX_IV_COUNT(klass));
1348}
1349
1350#if USE_ZJIT
1351bool
1352rb_zjit_class_allocate_instance_fastpath(VALUE klass, size_t *size_out, VALUE *flags_out)
1353{
1354 uint32_t index_tbl_num_entries = RCLASS_MAX_IV_COUNT(klass);
1355
1356 RUBY_ASSERT(rb_shape_max_capacity() > 0);
1357 if (RB_UNLIKELY(index_tbl_num_entries > rb_shape_max_capacity())) {
1358 return false;
1359 }
1360
1361 *size_out = robject_embedded_size(index_tbl_num_entries);
1362 *flags_out = T_OBJECT | rb_shape_transition_robject(0);
1363
1364 return true;
1365}
1366
1367bool
1368rb_zjit_newobj_hook_enabled_p(void)
1369{
1370 return rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_NEWOBJ);
1371}
1372#endif
1373
1374void
1375rb_gc_register_pinning_obj(VALUE obj)
1376{
1377 rb_gc_impl_register_pinning_obj(rb_gc_get_objspace(), obj);
1378}
1379
1380#define UNEXPECTED_NODE(func) \
1381 rb_bug(#func"(): GC does not handle T_NODE 0x%x(%p) 0x%"PRIxVALUE, \
1382 BUILTIN_TYPE(obj), (void*)(obj), RBASIC(obj)->flags)
1383
1384static inline void
1385rb_data_object_check(VALUE klass)
1386{
1387 RUBY_ASSERT(!RCLASS_SINGLETON_P(klass));
1388 if (klass != rb_cObject && (rb_get_alloc_func(klass) == rb_class_allocate_instance)) {
1389 rb_undef_alloc_func(klass);
1390 rb_warn("undefining the allocator of T_DATA class %"PRIsVALUE, klass);
1391 }
1392}
1393
1394#define RTYPEDDATA_EMBEDDED_P rbimpl_typeddata_embedded_p
1395#define RB_DATA_TYPE_EMBEDDABLE_P(type) ((type)->flags & RUBY_TYPED_EMBEDDABLE)
1396#define RTYPEDDATA_EMBEDDABLE_P(obj) RB_DATA_TYPE_EMBEDDABLE_P(RTYPEDDATA_TYPE(obj))
1397
1398static VALUE
1399typed_data_alloc_in(void *objspace, VALUE klass, VALUE typed_flag, void *datap, const rb_data_type_t *type, size_t size)
1400{
1401 RBIMPL_NONNULL_ARG(type);
1402 if (klass) rb_data_object_check(klass);
1403 bool wb_protected = (type->flags & RUBY_FL_WB_PROTECTED) || !type->function.dmark;
1404 VALUE obj = rb_newobj_in_objspace(GET_EC(), objspace, klass, T_DATA, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_RDATA, wb_protected, size);
1405
1406 rb_gc_register_pinning_obj(obj);
1407
1408 struct RTypedData *data = (struct RTypedData *)obj;
1409 data->fields_obj = 0;
1410 *(VALUE *)&data->type = ((VALUE)type) | typed_flag;
1411 data->data = datap;
1412
1413 return obj;
1414}
1415
1416static VALUE
1417typed_data_wrap_in(void *objspace, VALUE klass, void *datap, const rb_data_type_t *type)
1418{
1419 if (UNLIKELY(RB_DATA_TYPE_EMBEDDABLE_P(type))) {
1420 rb_raise(rb_eTypeError, "Cannot wrap an embeddable TypedData");
1421 }
1422
1423 return typed_data_alloc_in(objspace, klass, 0, datap, type, sizeof(struct RTypedData));
1424}
1425
1426VALUE
1428{
1429 return typed_data_wrap_in(rb_ec_ractor_ptr(GET_EC())->objspace, klass, datap, type);
1430}
1431
1432VALUE
1433rb_data_typed_object_wrap_in_objspace(void *objspace, VALUE klass, void *datap, const rb_data_type_t *type)
1434{
1435 return typed_data_wrap_in(objspace, klass, datap, type);
1436}
1437
1438static VALUE
1439typed_data_zalloc_in(void *objspace, VALUE klass, size_t size, const rb_data_type_t *type)
1440{
1441 if (RB_DATA_TYPE_EMBEDDABLE_P(type)) {
1442 if (!(type->flags & (RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_THREAD_SAFE_FREE))) {
1443 rb_raise(rb_eTypeError, "Embeddable TypedData must be freed immediately");
1444 }
1445
1446 /* A deferred free outlives the slot: the sweep copies the type and data pointer
1447 * out and hands the slot straight back, which it can only do when the payload is
1448 * not in the slot. Force such a type onto the heap -- an embeddable type already
1449 * has to cope with that, since a payload too large for a slot lands there too. */
1450 size_t embed_size = offsetof(struct RTypedData, data) + size;
1451 if (rb_gc_size_allocatable_p(embed_size) && !rb_gc_data_type_deferred_free_p(type)) {
1452 VALUE obj = typed_data_alloc_in(objspace, klass, TYPED_DATA_EMBEDDED, 0, type, embed_size);
1453 memset((char *)obj + offsetof(struct RTypedData, data), 0, size);
1454 return obj;
1455 }
1456 }
1457
1458 VALUE obj = typed_data_alloc_in(objspace, klass, 0, NULL, type, sizeof(struct RTypedData));
1459 DATA_PTR(obj) = xcalloc(1, size);
1460 return obj;
1461}
1462
1463VALUE
1465{
1466 return typed_data_zalloc_in(rb_ec_ractor_ptr(GET_EC())->objspace, klass, size, type);
1467}
1468
1469VALUE
1470rb_data_typed_object_zalloc_in_objspace(void *objspace, VALUE klass, size_t size, const rb_data_type_t *type)
1471{
1472 return typed_data_zalloc_in(objspace, klass, size, type);
1473}
1474
1475static size_t
1476ruby_xmalloc_usable_size(void *ptr)
1477{
1478#ifdef HAVE_MALLOC_USABLE_SIZE
1479#if CALC_EXACT_MALLOC_SIZE
1480 struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
1481 return malloc_usable_size(info) - sizeof(struct malloc_obj_info);
1482#else
1483 return malloc_usable_size(ptr);
1484#endif
1485#else
1486 return 0;
1487#endif
1488}
1489
1490static size_t
1491rb_objspace_data_type_memsize(VALUE obj)
1492{
1493 size_t size = 0;
1494 const void *ptr = RTYPEDDATA_GET_DATA(obj);
1495
1496 if (ptr) {
1497 if (RTYPEDDATA_EMBEDDABLE_P(obj) && !RTYPEDDATA_EMBEDDED_P(obj)) {
1498 size += ruby_xmalloc_usable_size((void *)ptr);
1499 }
1500
1501 const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
1502 if (type->function.dsize) {
1503 size += type->function.dsize(ptr);
1504 }
1505 }
1506
1507 return size;
1508}
1509
1510const char *
1511rb_objspace_data_type_name(VALUE obj)
1512{
1513 return RTYPEDDATA_TYPE(obj)->wrap_struct_name;
1514}
1515
1516void
1517rb_gc_declare_weak_references(VALUE obj)
1518{
1519 rb_gc_impl_declare_weak_references(rb_gc_get_objspace(), obj);
1520}
1521
1522static inline bool
1523gc_weak_reference_alive_p(void *objspace, VALUE obj)
1524{
1525 if (SPECIAL_CONST_P(obj)) return true;
1526
1527 return rb_gc_impl_handle_weak_references_alive_p(objspace, obj);
1528}
1529
1530bool
1531rb_gc_handle_weak_references_alive_p(void *objspace, VALUE obj)
1532{
1533 return gc_weak_reference_alive_p(objspace, obj);
1534}
1535
1536void
1537rb_gc_handle_weak_references(void *objspace, VALUE obj)
1538{
1539 switch (BUILTIN_TYPE(obj)) {
1540 case T_DATA:
1541 {
1542 const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
1543
1544 if (type->function.handle_weak_references) {
1545 (type->function.handle_weak_references)(RTYPEDDATA_GET_DATA(obj), objspace);
1546 }
1547 else {
1548 rb_bug(
1549 "rb_gc_handle_weak_references: TypedData %s does not implement handle_weak_references",
1550 RTYPEDDATA_TYPE(obj)->wrap_struct_name
1551 );
1552 }
1553 }
1554 break;
1555
1556 case T_IMEMO: {
1557 switch (imemo_type(obj)) {
1558 case imemo_callcache: {
1559 struct rb_callcache *cc = (struct rb_callcache *)obj;
1560 if (cc->klass != Qundef &&
1561 (!gc_weak_reference_alive_p(objspace, cc->klass) ||
1562 !gc_weak_reference_alive_p(objspace, (VALUE)cc->cme_))) {
1563 vm_cc_invalidate(cc);
1564 }
1565 break;
1566 }
1567 case imemo_subclasses: {
1568 struct rb_subclasses *subs = (struct rb_subclasses *)obj;
1569 VALUE *entries = rb_imemo_subclasses_entries(obj);
1570 for (uint32_t i = 0; i < subs->count; i++) {
1571 if (entries[i] && !gc_weak_reference_alive_p(objspace, entries[i])) {
1572 entries[i] = 0;
1573 }
1574 }
1575 break;
1576 }
1577 default:
1578 rb_bug("rb_gc_handle_weak_references: unexpected imemo type");
1579 }
1580
1581 break;
1582 }
1583 default:
1584 rb_bug("rb_gc_handle_weak_references: type not supported\n");
1585 }
1586}
1587
1588static inline bool
1589rb_gc_imemo_needs_cleanup_p(VALUE obj)
1590{
1591 switch (imemo_type(obj)) {
1592 case imemo_constcache:
1593 case imemo_cref:
1594 case imemo_ifunc:
1595 case imemo_memo:
1596 case imemo_svar:
1597 case imemo_callcache:
1598 case imemo_throw_data:
1599 case imemo_cvar_entry:
1600 return false;
1601
1602 case imemo_env:
1603 case imemo_ment:
1604 case imemo_iseq:
1605 case imemo_callinfo:
1606 case imemo_cdhash:
1607 return true;
1608
1609 case imemo_subclasses:
1610 return FL_TEST_RAW(obj, IMEMO_SUBCLASSES_HEAP);
1611
1612 case imemo_tmpbuf:
1613 return ((rb_imemo_tmpbuf_t *)obj)->ptr != NULL;
1614
1615 case imemo_fields:
1616 return rb_obj_shape_complex_p(obj);
1617 }
1618 UNREACHABLE_RETURN(true);
1619}
1620
1621/*
1622 * Returns true if the object requires a full rb_gc_obj_free() call during sweep,
1623 * false if it can be freed quickly without calling destructors or cleanup.
1624 *
1625 * Objects that return false are:
1626 * - Simple embedded objects without external allocations
1627 * - Objects without finalizers
1628 * - Objects without generic instance variables
1629 *
1630 * This is used by the GC sweep fast path to avoid function call overhead
1631 * for the majority of simple objects.
1632 */
1633bool
1634rb_gc_obj_needs_cleanup_p(VALUE obj)
1635{
1636 VALUE flags = RBASIC(obj)->flags;
1637
1638 if (flags & FL_FINALIZE) return true;
1639
1640 if ((flags & RUBY_T_MASK) == T_IMEMO) {
1641 return rb_gc_imemo_needs_cleanup_p(obj);
1642 }
1643
1644 /* A host with generic fields must drop its table entry when it is freed. The
1645 * process-wide table holds every Ractor's entries, so a sweep cannot bulk-wipe it;
1646 * this per-object cleanup carries the correctness. */
1647 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1648 if (rb_shape_has_fields(shape_id) && rb_shape_layout(shape_id) == SHAPE_ID_LAYOUT_OTHER) {
1649 return true;
1650 }
1651
1652 switch (flags & RUBY_T_MASK) {
1653 case T_FLOAT:
1654 case T_RATIONAL:
1655 case T_COMPLEX:
1656 case T_OBJECT:
1657 return false;
1658
1659 case T_FILE:
1660 case T_SYMBOL:
1661 case T_CLASS:
1662 case T_ICLASS:
1663 case T_MODULE:
1664 case T_REGEXP:
1665 return true;
1666
1667 case T_IMEMO:
1668 UNREACHABLE_RETURN(true);
1669
1670 case T_DATA:
1671 {
1672 uintptr_t type = (uintptr_t)RTYPEDDATA(obj)->type;
1673 if (type & TYPED_DATA_EMBEDDED) {
1674 RUBY_DATA_FUNC dfree = ((const rb_data_type_t *)(type & TYPED_DATA_PTR_MASK))->function.dfree;
1675 if (dfree == RUBY_NEVER_FREE || dfree == RUBY_TYPED_DEFAULT_FREE) {
1676 return false;
1677 }
1678 }
1679 }
1680 return true;
1681
1682 case T_STRING:
1683 return (flags & (RSTRING_NOEMBED | RSTRING_FSTR));
1684
1685 case T_ARRAY:
1686 return !(flags & RARRAY_EMBED_FLAG);
1687
1688 case T_HASH:
1689 return (flags & RHASH_ST_TABLE_FLAG);
1690
1691 case T_MATCH:
1692 return (flags & (RMATCH_ONIG | RMATCH_OFFSETS_EXTERNAL)) || USE_DEBUG_COUNTER;
1693
1694 case T_BIGNUM:
1695 return !(flags & BIGNUM_EMBED_FLAG);
1696
1697 case T_STRUCT:
1698 return !(flags & RSTRUCT_EMBED_LEN_MASK);
1699 }
1700
1701 UNREACHABLE_RETURN(true);
1702}
1703
1704static void
1705io_fptr_finalize(void *fptr)
1706{
1707 rb_io_fptr_finalize((struct rb_io *)fptr);
1708}
1709
1710static inline void
1711make_io_zombie(void *objspace, VALUE obj)
1712{
1713 rb_io_t *fptr = RFILE(obj)->fptr;
1714 rb_gc_impl_make_zombie(objspace, obj, io_fptr_finalize, fptr);
1715}
1716
1717static bool
1718rb_data_free(void *objspace, VALUE obj)
1719{
1720 void *data = RTYPEDDATA_GET_DATA(obj);
1721 if (data) {
1722 const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
1723 void (*dfree)(void *) = type->function.dfree;
1724
1725 if (dfree) {
1726 bool embedded = RTYPEDDATA_EMBEDDED_P(obj);
1727 int free_immediately = (type->flags & (RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_THREAD_SAFE_FREE)) != 0;
1728 bool free_embeddable_data = RB_DATA_TYPE_EMBEDDABLE_P(type) && !embedded;
1729
1730 if (dfree == RUBY_DEFAULT_FREE) {
1731 if (!embedded) {
1732 xfree(data);
1733 RB_DEBUG_COUNTER_INC(obj_data_xfree);
1734 }
1735 }
1736 else if (free_immediately) {
1737 (*dfree)(data);
1738 if (free_embeddable_data) {
1739 xfree(data);
1740 }
1741
1742 RB_DEBUG_COUNTER_INC(obj_data_imm_free);
1743 }
1744 else {
1745 rb_gc_impl_make_zombie(objspace, obj, dfree, data);
1746 RB_DEBUG_COUNTER_INC(obj_data_zombie);
1747 return FALSE;
1748 }
1749 }
1750 else {
1751 RB_DEBUG_COUNTER_INC(obj_data_empty);
1752 }
1753 }
1754
1755 return true;
1756}
1757
1759 VALUE klass;
1760 rb_objspace_t *objspace; // used for update_*
1761};
1762
1763static void
1764classext_free(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg)
1765{
1766 struct classext_foreach_args *args = (struct classext_foreach_args *)arg;
1767
1768 rb_class_classext_free(args->klass, ext, is_prime);
1769}
1770
1771static void
1772classext_iclass_free(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg)
1773{
1774 struct classext_foreach_args *args = (struct classext_foreach_args *)arg;
1775
1776 rb_iclass_classext_free(args->klass, ext, is_prime);
1777}
1778
1779bool
1780rb_gc_obj_free(void *objspace, VALUE obj)
1781{
1782 struct classext_foreach_args args;
1783
1784 RB_DEBUG_COUNTER_INC(obj_free);
1785
1786 enum ruby_value_type builtin_type = BUILTIN_TYPE(obj);
1787
1788 if (RUBY_DTRACE_GC_OBJ_FREE_ENABLED()) {
1789 RUBY_DTRACE_GC_OBJ_FREE((void*)obj, RBASIC(obj)->flags);
1790 }
1791
1792 switch (builtin_type) {
1793 case T_NIL:
1794 case T_FIXNUM:
1795 case T_TRUE:
1796 case T_FALSE:
1797 rb_bug("obj_free() called for broken object");
1798 break;
1799 default:
1800 break;
1801 }
1802
1803 switch (builtin_type) {
1804 case T_OBJECT:
1805 break;
1806 case T_MODULE:
1807 case T_CLASS:
1808#if USE_ZJIT
1809 rb_zjit_klass_free(obj);
1810#endif
1811 args.klass = obj;
1812 rb_class_classext_foreach(obj, classext_free, (void *)&args);
1813 if (RCLASS_CLASSEXT_TBL(obj)) {
1814 st_free_table(RCLASS_CLASSEXT_TBL(obj));
1815 }
1816 (void)RB_DEBUG_COUNTER_INC_IF(obj_module_ptr, BUILTIN_TYPE(obj) == T_MODULE);
1817 (void)RB_DEBUG_COUNTER_INC_IF(obj_class_ptr, BUILTIN_TYPE(obj) == T_CLASS);
1818 break;
1819 case T_STRING:
1820 rb_str_free(obj);
1821 break;
1822 case T_ARRAY:
1823 rb_ary_free(obj);
1824 break;
1825 case T_HASH:
1826#if USE_DEBUG_COUNTER
1827 switch (RHASH_SIZE(obj)) {
1828 case 0:
1829 RB_DEBUG_COUNTER_INC(obj_hash_empty);
1830 break;
1831 case 1:
1832 RB_DEBUG_COUNTER_INC(obj_hash_1);
1833 break;
1834 case 2:
1835 RB_DEBUG_COUNTER_INC(obj_hash_2);
1836 break;
1837 case 3:
1838 RB_DEBUG_COUNTER_INC(obj_hash_3);
1839 break;
1840 case 4:
1841 RB_DEBUG_COUNTER_INC(obj_hash_4);
1842 break;
1843 case 5:
1844 case 6:
1845 case 7:
1846 case 8:
1847 RB_DEBUG_COUNTER_INC(obj_hash_5_8);
1848 break;
1849 default:
1850 GC_ASSERT(RHASH_SIZE(obj) > 8);
1851 RB_DEBUG_COUNTER_INC(obj_hash_g8);
1852 }
1853
1854 if (RHASH_AR_TABLE_P(obj)) {
1855 if (RHASH_AR_TABLE(obj) == NULL) {
1856 RB_DEBUG_COUNTER_INC(obj_hash_null);
1857 }
1858 else {
1859 RB_DEBUG_COUNTER_INC(obj_hash_ar);
1860 }
1861 }
1862 else {
1863 RB_DEBUG_COUNTER_INC(obj_hash_st);
1864 }
1865#endif
1866
1867 rb_hash_free(obj);
1868 break;
1869 case T_REGEXP:
1870 if (FL_TEST_RAW(obj, RREGEXP_INITIALIZED)) {
1871 onig_free_body(RREGEXP_PTR(obj));
1872 struct rb_regexp_variant *variant = RREGEXP(obj)->variants;
1873 while (variant) {
1874 struct rb_regexp_variant *next = variant->next;
1875 onig_free_body(&variant->reg);
1876 SIZED_FREE(variant);
1877 variant = next;
1878 }
1879 RB_DEBUG_COUNTER_INC(obj_regexp_ptr);
1880 }
1881 break;
1882 case T_DATA:
1883 if (!rb_data_free(objspace, obj)) return false;
1884 break;
1885 case T_MATCH:
1886 {
1887 struct RMatch *rm = RMATCH(obj);
1888#if USE_DEBUG_COUNTER
1889 if (rm->num_regs >= 8) {
1890 RB_DEBUG_COUNTER_INC(obj_match_ge8);
1891 }
1892 else if (rm->num_regs >= 4) {
1893 RB_DEBUG_COUNTER_INC(obj_match_ge4);
1894 }
1895 else if (rm->num_regs >= 1) {
1896 RB_DEBUG_COUNTER_INC(obj_match_under4);
1897 }
1898#endif
1899 if (FL_TEST_RAW(obj, RMATCH_ONIG)) {
1900 onig_region_free(&rm->as.onig, 0);
1901 }
1902 SIZED_FREE_N(rm->char_offset, rm->char_offset_num_allocated);
1903
1904 RB_DEBUG_COUNTER_INC(obj_match_ptr);
1905 }
1906 break;
1907 case T_FILE:
1908 if (RFILE(obj)->fptr) {
1909 bool closed = rb_io_fptr_finalize_closed(RFILE(obj)->fptr);
1910 if (!closed) make_io_zombie(objspace, obj);
1911 RB_DEBUG_COUNTER_INC(obj_file_ptr);
1912 return closed;
1913 }
1914 break;
1915 case T_RATIONAL:
1916 RB_DEBUG_COUNTER_INC(obj_rational);
1917 break;
1918 case T_COMPLEX:
1919 RB_DEBUG_COUNTER_INC(obj_complex);
1920 break;
1921 case T_MOVED:
1922 break;
1923 case T_ICLASS:
1924 args.klass = obj;
1925
1926 rb_class_classext_foreach(obj, classext_iclass_free, (void *)&args);
1927 if (RCLASS_CLASSEXT_TBL(obj)) {
1928 st_free_table(RCLASS_CLASSEXT_TBL(obj));
1929 }
1930
1931 RB_DEBUG_COUNTER_INC(obj_iclass_ptr);
1932 break;
1933
1934 case T_FLOAT:
1935 RB_DEBUG_COUNTER_INC(obj_float);
1936 break;
1937
1938 case T_BIGNUM:
1939 if (!BIGNUM_EMBED_P(obj) && BIGNUM_DIGITS(obj)) {
1940 SIZED_FREE_N(BIGNUM_DIGITS(obj), BIGNUM_LEN(obj));
1941 RB_DEBUG_COUNTER_INC(obj_bignum_ptr);
1942 }
1943 else {
1944 RB_DEBUG_COUNTER_INC(obj_bignum_embed);
1945 }
1946 break;
1947
1948 case T_NODE:
1949 UNEXPECTED_NODE(obj_free);
1950 break;
1951
1952 case T_STRUCT:
1953 if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) ||
1954 RSTRUCT(obj)->as.heap.ptr == NULL) {
1955 RB_DEBUG_COUNTER_INC(obj_struct_embed);
1956 }
1957 else {
1958 SIZED_FREE_N(RSTRUCT(obj)->as.heap.ptr, RSTRUCT(obj)->as.heap.len);
1959 RB_DEBUG_COUNTER_INC(obj_struct_ptr);
1960 }
1961 break;
1962
1963 case T_SYMBOL:
1964 RB_DEBUG_COUNTER_INC(obj_symbol);
1965 break;
1966
1967 case T_IMEMO:
1968 rb_imemo_free((VALUE)obj);
1969 break;
1970
1971 default:
1972 rb_bug("gc_sweep(): unknown data type 0x%x(%p) 0x%"PRIxVALUE,
1973 BUILTIN_TYPE(obj), (void*)obj, RBASIC(obj)->flags);
1974 }
1975
1976 if (FL_TEST_RAW(obj, FL_FINALIZE)) {
1977 rb_gc_impl_make_zombie(objspace, obj, 0, 0);
1978 return FALSE;
1979 }
1980 else {
1981 return TRUE;
1982 }
1983}
1984
1985void
1986rb_objspace_set_event_hook(const rb_event_flag_t event)
1987{
1988 /* Only the main objspace may enable the FREEOBJ hook: it runs user callbacks from
1989 * inside the sweep, which is unsafe in a non-main Ractor's lock-free local GC.
1990 * Extending it VM-wide is future work. */
1991 rb_event_flag_t e = event;
1992 const rb_ractor_t *const cr = rb_current_ractor_raw(false);
1993 if (cr != NULL && cr != GET_VM()->ractor.main_ractor) {
1995 }
1996 rb_gc_impl_set_event_hook(rb_gc_get_objspace(), e);
1997}
1998
1999static int
2000internal_object_p(VALUE obj)
2001{
2002 void *ptr = asan_unpoison_object_temporary(obj);
2003
2004 if (RBASIC(obj)->flags) {
2005 switch (BUILTIN_TYPE(obj)) {
2006 case T_NODE:
2007 UNEXPECTED_NODE(internal_object_p);
2008 break;
2009 case T_NONE:
2010 case T_MOVED:
2011 case T_IMEMO:
2012 case T_ICLASS:
2013 case T_ZOMBIE:
2014 break;
2015 case T_CLASS:
2016 if (obj == rb_mRubyVMFrozenCore)
2017 return 1;
2018
2019 if (!RBASIC_CLASS(obj)) break;
2020 if (RCLASS_SINGLETON_P(obj)) {
2021 return rb_singleton_class_internal_p(obj);
2022 }
2023 return 0;
2024 default:
2025 if (!RBASIC(obj)->klass) break;
2026 return 0;
2027 }
2028 }
2029 if (ptr || !RBASIC(obj)->flags) {
2030 rb_asan_poison_object(obj);
2031 }
2032 return 1;
2033}
2034
2035int
2036rb_objspace_internal_object_p(VALUE obj)
2037{
2038 return internal_object_p(obj);
2039}
2040
2042 size_t num;
2043 VALUE of;
2044};
2045
2046static int
2047os_obj_of_i(void *vstart, void *vend, size_t stride, void *data)
2048{
2049 struct os_each_struct *oes = (struct os_each_struct *)data;
2050
2051 VALUE v = (VALUE)vstart;
2052 for (; v != (VALUE)vend; v += stride) {
2053 if (!internal_object_p(v)) {
2054 if (!oes->of || rb_obj_is_kind_of(v, oes->of)) {
2055 rb_yield(v);
2056 oes->num++;
2057 }
2058 }
2059 }
2060
2061 return 0;
2062}
2063
2064/* Like os_obj_of_i but collects into an array: foreign shareable objects are walked
2065 * under the barrier, where yielding is unsafe (see os_obj_of). Pure C, allocates no
2066 * object (rb_ary_push only grows the buffer), so it reaches no safepoint. */
2068 VALUE of;
2069 VALUE buffer;
2070};
2071
2072static int
2073os_shareable_collect_i(void *vstart, void *vend, size_t stride, void *data)
2074{
2075 struct os_shareable_collect_struct *ocs = (struct os_shareable_collect_struct *)data;
2076
2077 VALUE v = (VALUE)vstart;
2078 for (; v != (VALUE)vend; v += stride) {
2079 /* We walk a foreign Ractor's objspace, so collect only shareable objects. The
2080 * walk already filters on shareable_bits; check again so an unshareable object
2081 * can never be exposed. */
2082 if (rb_ractor_shareable_p(v) && !internal_object_p(v)) {
2083 if (!ocs->of || rb_obj_is_kind_of(v, ocs->of)) {
2084 rb_ary_push(ocs->buffer, v);
2085 }
2086 }
2087 }
2088
2089 return 0;
2090}
2091
2092static void rb_gc_critical_disable(void);
2093static void rb_gc_critical_enable(void);
2094
2095static VALUE
2096os_obj_of(VALUE of)
2097{
2098 struct os_each_struct oes;
2099
2100 oes.num = 0;
2101 oes.of = of;
2102
2103 /* Phase 1: our own Ractor's objspace, yielding every object directly with no
2104 * barrier. The walk snapshots the page list and tolerates pages being freed
2105 * concurrently, so no VM lock is needed and the block may allocate, GC or block. */
2106 rb_gc_impl_each_objects(rb_gc_get_objspace(), os_obj_of_i, &oes);
2107
2108 /* Phase 2 (multi-Ractor): other live Ractors' shareable objects, readable only
2109 * under the barrier (where a user block must not run), so collect them in pure C
2110 * with GC disabled and yield after the barrier is released. */
2111 if (rb_multi_ractor_p()) {
2112 struct os_shareable_collect_struct ocs;
2113 ocs.of = of;
2114 ocs.buffer = rb_ary_new();
2115
2116 rb_gc_critical_disable();
2117 RB_VM_LOCKING() {
2118 rb_vm_barrier();
2119
2120 void *self = rb_gc_get_objspace();
2121 rb_vm_t *vm = GET_VM();
2122 rb_ractor_t *r;
2123 ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
2124 if (r->objspace && r->objspace != self) {
2125 rb_gc_impl_each_objects_shareable(r->objspace, os_shareable_collect_i, &ocs);
2126 }
2127 }
2128 }
2129 rb_gc_critical_enable();
2130
2131 rb_len_t len = RARRAY_LEN(ocs.buffer);
2132 for (rb_len_t i = 0; i < len; i++) {
2133 rb_yield(RARRAY_AREF(ocs.buffer, i));
2134 oes.num++;
2135 }
2136 RB_GC_GUARD(ocs.buffer);
2137 }
2138
2139 return SIZET2NUM(oes.num);
2140}
2141
2142/*
2143 * call-seq:
2144 * ObjectSpace.each_object {|obj| ... } -> integer
2145 * ObjectSpace.each_object(module) {|obj| ... } -> integer
2146 * ObjectSpace.each_object -> enumerator
2147 * ObjectSpace.each_object(module) -> enumerator
2148 *
2149 * Calls the block once for each living, non-immediate object in this Ruby
2150 * process, and returns the number of objects found.
2151 *
2152 * If +module+ is given, calls the block only for objects that are an instance
2153 * of +module+ or one of its subclasses.
2154 *
2155 * Immediate objects (such as small integers, static symbols, +true+, +false+,
2156 * and +nil+) are never yielded.
2157 *
2158 * With no block given, returns a new Enumerator.
2159 *
2160 * Job = Class.new
2161 * jobs = [Job.new, Job.new]
2162 * count = ObjectSpace.each_object(Job) {|x| p x }
2163 * puts "Total count: #{count}"
2164 *
2165 * <em>produces:</em>
2166 *
2167 * #<Job:0x000000011d6cbbf0>
2168 * #<Job:0x000000011d6cbc68>
2169 * Total count: 2
2170 *
2171 * Because every live object is visited, this method is mainly useful for
2172 * debugging, profiling, and introspecting a running process.
2173 *
2174 * In multi-Ractor mode this method yields every object of the current Ractor, plus
2175 * the objects of the other Ractors that have been made Ractor-shareable. Another
2176 * Ractor's unshareable objects are never yielded: they belong to that Ractor and the
2177 * current one must not touch them.
2178 *
2179 * c = {} # not shareable, belongs to the main Ractor
2180 * r = Ractor.new { d = {}; receive } # d belongs to r
2181 * ObjectSpace.each_object {|x| x } # yields c, but not d
2182 *
2183 */
2184
2185static VALUE
2186os_each_obj(int argc, VALUE *argv, VALUE os)
2187{
2188 VALUE of;
2189
2190 of = (!rb_check_arity(argc, 0, 1) ? 0 : argv[0]);
2191 RETURN_ENUMERATOR(os, 1, &of);
2192 return os_obj_of(of);
2193}
2194
2195/*
2196 * call-seq:
2197 * ObjectSpace.undefine_finalizer(obj) -> obj
2198 *
2199 * Removes all finalizers registered for +obj+ with
2200 * ObjectSpace.define_finalizer, and returns +obj+.
2201 *
2202 * Does nothing if +obj+ has no finalizers.
2203 */
2204
2205static VALUE
2206undefine_final(VALUE os, VALUE obj)
2207{
2208 return rb_undefine_finalizer(obj);
2209}
2210
2211VALUE
2212rb_undefine_finalizer(VALUE obj)
2213{
2214 rb_check_frozen(obj);
2215
2216 rb_gc_impl_undefine_finalizer(rb_gc_get_objspace(), obj);
2217
2218 return obj;
2219}
2220
2221static void
2222should_be_callable(VALUE block)
2223{
2224 if (!rb_obj_respond_to(block, idCall, TRUE)) {
2225 rb_raise(rb_eArgError, "wrong type argument %"PRIsVALUE" (should be callable)",
2226 rb_obj_class(block));
2227 }
2228}
2229
2230static void
2231should_be_finalizable(VALUE obj)
2232{
2233 if (!FL_ABLE(obj)) {
2234 rb_raise(rb_eArgError, "cannot define finalizer for %s",
2235 rb_obj_classname(obj));
2236 }
2237 rb_check_frozen(obj);
2238}
2239
2240void
2241rb_gc_copy_finalizer(VALUE dest, VALUE obj)
2242{
2243 rb_gc_impl_copy_finalizer(rb_gc_get_objspace(), dest, obj);
2244}
2245
2246/*
2247 * call-seq:
2248 * ObjectSpace.define_finalizer(obj) {|id| ... } -> array
2249 * ObjectSpace.define_finalizer(obj, finalizer) -> array
2250 *
2251 * Adds a new finalizer for +obj+ that is called when +obj+ is destroyed
2252 * by the garbage collector or when Ruby shuts down (which ever comes first).
2253 *
2254 * With a block given, uses the block as the callback. Without a block given,
2255 * uses a callable object +finalizer+ as the callback. The callback is called
2256 * when +obj+ is destroyed with a single argument +id+ which is the object
2257 * ID of +obj+ (see Object#object_id).
2258 *
2259 * The return value is an array <code>[0, callback]</code>, where +callback+
2260 * is a Proc created from the block if one was given or +finalizer+ otherwise.
2261 *
2262 * Note that defining a finalizer in an instance method of the object may prevent
2263 * the object from being garbage collected since if the block or +finalizer+ refers
2264 * to +obj+ then +obj+ will never be reclaimed by the garbage collector. For example,
2265 * the following script demonstrates the issue:
2266 *
2267 * class Foo
2268 * def define_final
2269 * ObjectSpace.define_finalizer(self) do |id|
2270 * puts "Running finalizer for #{id}!"
2271 * end
2272 * end
2273 * end
2274 *
2275 * obj = Foo.new
2276 * obj.define_final
2277 *
2278 * There are two patterns to solve this issue:
2279 *
2280 * - Create the finalizer in a non-instance method so it can safely capture
2281 * the needed state:
2282 *
2283 * class Foo
2284 * def define_final
2285 * ObjectSpace.define_finalizer(self, self.class.create_finalizer)
2286 * end
2287 *
2288 * def self.create_finalizer
2289 * proc do |id|
2290 * puts "Running finalizer for #{id}!"
2291 * end
2292 * end
2293 * end
2294 *
2295 * - Use a callable object:
2296 *
2297 * class Foo
2298 * class Finalizer
2299 * def call(id)
2300 * puts "Running finalizer for #{id}!"
2301 * end
2302 * end
2303 *
2304 * def define_final
2305 * ObjectSpace.define_finalizer(self, Finalizer.new)
2306 * end
2307 * end
2308 *
2309 * Note that finalization can be unpredictable and is never guaranteed
2310 * to be run except on exit.
2311 */
2312
2313static VALUE
2314define_final(int argc, VALUE *argv, VALUE os)
2315{
2316 VALUE obj, block;
2317
2318 rb_scan_args(argc, argv, "11", &obj, &block);
2319 if (argc == 1) {
2320 block = rb_block_proc();
2321 }
2322
2323 if (rb_callable_receiver(block) == obj) {
2324 rb_warn("finalizer references object to be finalized");
2325 }
2326
2327 return rb_define_finalizer(obj, block);
2328}
2329
2330VALUE
2331rb_define_finalizer(VALUE obj, VALUE block)
2332{
2333 should_be_finalizable(obj);
2334 should_be_callable(block);
2335
2336 block = rb_gc_impl_define_finalizer(rb_gc_get_objspace(), obj, block);
2337
2338 block = rb_ary_new3(2, INT2FIX(0), block);
2339 OBJ_FREEZE(block);
2340 return block;
2341}
2342
2343void
2344rb_objspace_call_finalizer(void)
2345{
2346 rb_gc_impl_shutdown_call_finalizer(rb_gc_get_objspace());
2347}
2348
2349void
2350rb_objspace_free_objects(void *objspace)
2351{
2352 rb_gc_impl_shutdown_free_objects(objspace);
2353}
2354
2355int
2356rb_objspace_garbage_object_p(VALUE obj)
2357{
2358 return !SPECIAL_CONST_P(obj) && rb_gc_impl_garbage_object_p(rb_gc_get_objspace(), obj);
2359}
2360
2361int
2362rb_objspace_foreign_object_p(VALUE obj)
2363{
2364 return !SPECIAL_CONST_P(obj) && rb_gc_obj_foreign_p(obj);
2365}
2366
2367int
2368rb_objspace_live_object_p(VALUE obj)
2369{
2370 return !SPECIAL_CONST_P(obj) && rb_gc_impl_live_object_p(rb_gc_get_objspace(), (const void *)obj);
2371}
2372
2373#define OBJ_ID_INCREMENT (RUBY_IMMEDIATE_MASK + 1)
2374#define LAST_OBJECT_ID() (object_id_counter * OBJ_ID_INCREMENT)
2375
2376#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
2377static size_t object_id_counter = 1;
2378#else
2379static unsigned long long object_id_counter = 1;
2380#endif
2381
2382static inline VALUE
2383generate_next_object_id(void)
2384{
2385#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
2386 // 64bit atomics are available
2387 return SIZET2NUM(RUBY_ATOMIC_SIZE_FETCH_ADD(object_id_counter, 1) * OBJ_ID_INCREMENT);
2388#else
2389 unsigned int lock_lev = RB_GC_VM_LOCK();
2390 VALUE id = ULL2NUM(++object_id_counter * OBJ_ID_INCREMENT);
2391 RB_GC_VM_UNLOCK(lock_lev);
2392 return id;
2393#endif
2394}
2395
2396static VALUE
2397class_object_id(VALUE klass)
2398{
2399 VALUE id = RUBY_ATOMIC_VALUE_LOAD(RCLASS(klass)->object_id);
2400 if (!id) {
2401 unsigned int lock_lev = RB_GC_VM_LOCK();
2402 id = generate_next_object_id();
2403 VALUE existing_id = RUBY_ATOMIC_VALUE_CAS(RCLASS(klass)->object_id, 0, id);
2404 if (existing_id) {
2405 id = existing_id;
2406 }
2407 RB_GC_VM_UNLOCK(lock_lev);
2408 }
2409 return id;
2410}
2411
2412static inline shape_id_t
2413object_id_field_shape(shape_id_t shape_id)
2414{
2415 return rb_shape_complex_p(shape_id) ? ROOT_COMPLEX_WITH_OBJ_ID : rb_shape_object_id(shape_id);
2416}
2417
2418static inline VALUE
2419object_id_get(VALUE obj, shape_id_t shape_id)
2420{
2421 VALUE id = rb_obj_field_get(obj, object_id_field_shape(shape_id));
2422
2423#if RUBY_DEBUG
2424 if (!(FIXNUM_P(id) || RB_TYPE_P(id, T_BIGNUM))) {
2425 rb_p(obj);
2426 rb_bug("Object's shape includes object_id, but it's missing %s", rb_obj_info(obj));
2427 }
2428#endif
2429
2430 return id;
2431}
2432
2433static VALUE
2434object_id0(VALUE obj)
2435{
2436 VALUE id = Qfalse;
2437 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
2438
2439 if (rb_shape_has_object_id(shape_id)) {
2440 return object_id_get(obj, shape_id);
2441 }
2442
2443 shape_id_t object_id_shape_id = rb_obj_shape_transition_object_id(obj);
2444
2445 id = generate_next_object_id();
2446 rb_obj_field_set(obj, object_id_shape_id, 0, id);
2447
2448 RUBY_ASSERT(rb_obj_shape_has_id(obj));
2449
2450 return id;
2451}
2452
2453// generic_field_set never mutates the published fields_obj of a shareable object,
2454// and the fields_obj carries its owner's shape, so both are read from the one snapshot.
2455static VALUE
2456shareable_data_object_id(VALUE obj)
2457{
2458 VALUE fields_obj = RUBY_ATOMIC_VALUE_LOAD(RTYPEDDATA(obj)->fields_obj);
2459 if (!fields_obj) return Qfalse;
2460
2461 shape_id_t shape_id = RBASIC_SHAPE_ID(fields_obj);
2462 if (!rb_shape_has_object_id(shape_id)) return Qfalse;
2463
2464 return rb_imemo_fields_field_get(fields_obj, object_id_field_shape(shape_id));
2465}
2466
2467static VALUE
2468object_id(VALUE obj)
2469{
2470 switch (BUILTIN_TYPE(obj)) {
2471 case T_CLASS:
2472 case T_MODULE:
2473 // With Ruby Box, classes and modules have different fields
2474 // in different boxes, so we cannot store the object id
2475 // in fields.
2476 return class_object_id(obj);
2477 case T_IMEMO:
2478 RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields));
2479 break;
2480 default:
2481 break;
2482 }
2483
2484 if (UNLIKELY(rb_gc_multi_ractor_p() && rb_ractor_shareable_p(obj))) {
2485 if (BUILTIN_TYPE(obj) == T_DATA) {
2486 VALUE existing_id = shareable_data_object_id(obj);
2487 if (existing_id) return existing_id;
2488 }
2489
2490 unsigned int lock_lev = RB_GC_VM_LOCK();
2491 VALUE id = object_id0(obj);
2492 RB_GC_VM_UNLOCK(lock_lev);
2493 return id;
2494 }
2495
2496 return object_id0(obj);
2497}
2498
2499void
2500rb_gc_obj_free_vm_weak_references(VALUE obj)
2501{
2503
2504 /* Drop a generic-fields entry when its host's slot is freed. The table is
2505 * process-wide, so no sweep bulk-wipes it; a stale entry would let the global GC's
2506 * weak pass (or a reader after the slot is reused) walk a freed page. */
2507 if (rb_obj_gen_fields_p(obj)) {
2509 }
2510
2511 switch (BUILTIN_TYPE(obj)) {
2512 case T_STRING:
2513 if (FL_TEST_RAW(obj, RSTRING_FSTR)) {
2514 rb_gc_free_fstring(obj);
2515 }
2516 break;
2517 case T_SYMBOL:
2518 rb_gc_free_dsymbol(obj);
2519 break;
2520 case T_IMEMO:
2521 switch (imemo_type(obj)) {
2522 case imemo_callinfo:
2523 rb_vm_ci_free((const struct rb_callinfo *)obj);
2524 break;
2525 case imemo_ment:
2526 rb_free_method_entry_vm_weak_references((const rb_method_entry_t *)obj);
2527 break;
2528 default:
2529 break;
2530 }
2531 break;
2532 default:
2533 break;
2534 }
2535}
2536
2537static VALUE
2538rb_find_object_id(void *objspace, VALUE obj, VALUE (*get_heap_object_id)(VALUE))
2539{
2540 if (SPECIAL_CONST_P(obj)) {
2541#if SIZEOF_LONG == SIZEOF_VOIDP
2542 return LONG2NUM((SIGNED_VALUE)obj);
2543#else
2544 return LL2NUM((SIGNED_VALUE)obj);
2545#endif
2546 }
2547
2548 return get_heap_object_id(obj);
2549}
2550
2551static VALUE
2552nonspecial_obj_id(VALUE obj)
2553{
2554#if SIZEOF_LONG == SIZEOF_VOIDP
2555 return (VALUE)((SIGNED_VALUE)(obj)|FIXNUM_FLAG);
2556#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
2557 return LL2NUM((SIGNED_VALUE)(obj) / 2);
2558#else
2559# error not supported
2560#endif
2561}
2562
2563VALUE
2564rb_memory_id(VALUE obj)
2565{
2566 return rb_find_object_id(NULL, obj, nonspecial_obj_id);
2567}
2568
2569/*
2570 * Document-method: __id__
2571 * Document-method: object_id
2572 *
2573 * call-seq:
2574 * obj.__id__ -> integer
2575 * obj.object_id -> integer
2576 *
2577 * Returns an integer identifier for +obj+.
2578 *
2579 * The same number will be returned on all calls to +object_id+ for a given
2580 * object, and no two active objects will share an id.
2581 *
2582 * Note: that some objects of builtin classes are reused for optimization.
2583 * This is the case for immediate values and frozen string literals.
2584 *
2585 * BasicObject implements +__id__+, Kernel implements +object_id+.
2586 *
2587 * Immediate values are not passed by reference but are passed by value:
2588 * +nil+, +true+, +false+, Fixnums, Symbols, and some Floats.
2589 *
2590 * Object.new.object_id == Object.new.object_id # => false
2591 * (21 * 2).object_id == (21 * 2).object_id # => true
2592 * "hello".object_id == "hello".object_id # => false
2593 * "hi".freeze.object_id == "hi".freeze.object_id # => true
2594 */
2595
2596VALUE
2597rb_obj_id(VALUE obj)
2598{
2599 /* If obj is an immediate, the object ID is obj directly converted to a Numeric.
2600 * Otherwise, the object ID is a Numeric that is a non-zero multiple of
2601 * (RUBY_IMMEDIATE_MASK + 1) which guarantees that it does not collide with
2602 * any immediates. */
2603 return rb_find_object_id(rb_gc_get_objspace(), obj, object_id);
2604}
2605
2606bool
2607rb_obj_id_p(VALUE obj)
2608{
2609 return !RB_TYPE_P(obj, T_IMEMO) && rb_obj_shape_has_id(obj);
2610}
2611
2612/*
2613 * GC implementations should call this function before the GC phase that updates references
2614 * embedded in the machine code generated by JIT compilers. JIT compilers usually enforce the
2615 * "W^X" policy and protect the code memory from being modified during execution. This function
2616 * makes the code memory writeable.
2617 */
2618void
2619rb_gc_before_updating_jit_code(void)
2620{
2621#if USE_YJIT
2622 rb_yjit_mark_all_writeable();
2623#endif
2624#if USE_ZJIT
2625 rb_zjit_mark_all_writable();
2626#endif
2627}
2628
2629/*
2630 * GC implementations should call this function before the GC phase that updates references
2631 * embedded in the machine code generated by JIT compilers. This function makes the code memory
2632 * executable again.
2633 */
2634void
2635rb_gc_after_updating_jit_code(void)
2636{
2637#if USE_YJIT
2638 rb_yjit_mark_all_executable();
2639#endif
2640#if USE_ZJIT
2641 rb_zjit_mark_all_executable();
2642#endif
2643}
2644
2645static void
2646classext_memsize(rb_classext_t *ext, bool prime, VALUE box_value, void *arg)
2647{
2648 size_t *size = (size_t *)arg;
2649 size_t s = 0;
2650
2651 if (RCLASSEXT_M_TBL(ext)) {
2652 s += rb_id_table_memsize(RCLASSEXT_M_TBL(ext));
2653 }
2654 if (RCLASSEXT_CONST_TBL(ext)) {
2655 s += rb_id_table_memsize(RCLASSEXT_CONST_TBL(ext));
2656 }
2657 if (RCLASSEXT_SUPERCLASSES_WITH_SELF(ext)) {
2658 s += (RCLASSEXT_SUPERCLASS_DEPTH(ext) + 1) * sizeof(VALUE);
2659 }
2660 if (!prime) {
2661 s += sizeof(rb_classext_t);
2662 }
2663 *size += s;
2664}
2665
2666static void
2667classext_superclasses_memsize(rb_classext_t *ext, bool prime, VALUE box_value, void *arg)
2668{
2669 size_t *size = (size_t *)arg;
2670 size_t array_size;
2671 if (RCLASSEXT_SUPERCLASSES_WITH_SELF(ext)) {
2672 RUBY_ASSERT(prime);
2673 array_size = RCLASSEXT_SUPERCLASS_DEPTH(ext) + 1;
2674 *size += array_size * sizeof(VALUE);
2675 }
2676}
2677
2678size_t
2679rb_obj_memsize_of(VALUE obj)
2680{
2681 size_t size = 0;
2682
2683 if (SPECIAL_CONST_P(obj)) {
2684 return 0;
2685 }
2686
2687 switch (BUILTIN_TYPE(obj)) {
2688 case T_OBJECT:
2689 break;
2690 case T_MODULE:
2691 case T_CLASS:
2692 rb_class_classext_foreach(obj, classext_memsize, (void *)&size);
2693 rb_class_classext_foreach(obj, classext_superclasses_memsize, (void *)&size);
2694 break;
2695 case T_ICLASS:
2696 if (RICLASS_OWNS_M_TBL_P(obj)) {
2697 if (RCLASS_M_TBL(obj)) {
2698 size += rb_id_table_memsize(RCLASS_M_TBL(obj));
2699 }
2700 }
2701 break;
2702 case T_STRING:
2703 size += rb_str_memsize(obj);
2704 break;
2705 case T_ARRAY:
2706 size += rb_ary_memsize(obj);
2707 break;
2708 case T_HASH:
2709 if (RHASH_ST_TABLE_P(obj)) {
2710 VM_ASSERT(RHASH_ST_TABLE(obj) != NULL);
2711 /* st_table is in the slot */
2712 size += st_memsize(RHASH_ST_TABLE(obj)) - sizeof(st_table);
2713 }
2714 break;
2715 case T_REGEXP:
2716 if (RREGEXP_PTR(obj)) {
2717 /* regex_t is in the slot */
2718 size += onig_memsize(RREGEXP_PTR(obj)) - sizeof(regex_t);
2719 for (struct rb_regexp_variant *variant = RUBY_ATOMIC_PTR_LOAD(RREGEXP(obj)->variants);
2720 variant; variant = variant->next) {
2721 size += sizeof(*variant) - sizeof(variant->reg) + onig_memsize(&variant->reg);
2722 }
2723 }
2724 break;
2725 case T_DATA:
2726 size += rb_objspace_data_type_memsize(obj);
2727 break;
2728 case T_MATCH:
2729 {
2730 struct RMatch *rm = RMATCH(obj);
2731 if (FL_TEST_RAW(obj, RMATCH_ONIG)) {
2732 size += onig_region_memsize(&rm->as.onig);
2733 }
2734 size += sizeof(struct rmatch_offset) * rm->char_offset_num_allocated;
2735 }
2736 break;
2737 case T_FILE:
2738 if (RFILE(obj)->fptr) {
2739 size += rb_io_memsize(RFILE(obj)->fptr);
2740 }
2741 break;
2742 case T_RATIONAL:
2743 case T_COMPLEX:
2744 break;
2745 case T_IMEMO:
2746 size += rb_imemo_memsize(obj);
2747 break;
2748
2749 case T_FLOAT:
2750 case T_SYMBOL:
2751 break;
2752
2753 case T_BIGNUM:
2754 if (!(RBASIC(obj)->flags & BIGNUM_EMBED_FLAG) && BIGNUM_DIGITS(obj)) {
2755 size += BIGNUM_LEN(obj) * sizeof(BDIGIT);
2756 }
2757 break;
2758
2759 case T_NODE:
2760 UNEXPECTED_NODE(obj_memsize_of);
2761 break;
2762
2763 case T_STRUCT:
2764 if (RSTRUCT_EMBED_LEN(obj) == 0) {
2765 size += sizeof(VALUE) * RSTRUCT_LEN_RAW(obj);
2766 }
2767 break;
2768
2769 case T_ZOMBIE:
2770 case T_MOVED:
2771 break;
2772
2773 default:
2774 rb_bug("objspace/memsize_of(): unknown data type 0x%x(%p)",
2775 BUILTIN_TYPE(obj), (void*)obj);
2776 }
2777
2778 return size + rb_gc_obj_slot_size(obj);
2779}
2780
2781static int
2782set_zero(st_data_t key, st_data_t val, st_data_t arg)
2783{
2784 VALUE k = (VALUE)key;
2785 VALUE hash = (VALUE)arg;
2786 rb_hash_aset(hash, k, INT2FIX(0));
2787 return ST_CONTINUE;
2788}
2789
2791 size_t counts[T_MASK+1];
2792 size_t freed;
2793 size_t total;
2794};
2795
2796static void
2797count_objects_i(VALUE obj, void *d)
2798{
2799 struct count_objects_data *data = (struct count_objects_data *)d;
2800
2801 if (RBASIC(obj)->flags) {
2802 data->counts[BUILTIN_TYPE(obj)]++;
2803 }
2804 else {
2805 data->freed++;
2806 }
2807
2808 data->total++;
2809}
2810
2811/*
2812 * call-seq:
2813 * ObjectSpace.count_objects(result_hash = {}) -> hash
2814 *
2815 * Counts the number of objects, grouped by type.
2816 *
2817 * It returns a hash that looks like:
2818 *
2819 * {
2820 * TOTAL: 10000,
2821 * FREE: 3011,
2822 * T_OBJECT: 6,
2823 * T_CLASS: 404,
2824 * # ...
2825 * }
2826 *
2827 * The contents of the returned hash are implementation specific and
2828 * may be changed in future versions without notice.
2829 *
2830 * The keys starting with +:T_+ are live objects of a particular type.
2831 * For example, +:T_ARRAY+ is the number of arrays.
2832 *
2833 * The key +:FREE+ is the number of object slots which are empty.
2834 *
2835 * The key +:TOTAL+ is the total number of slots (which is the sum of
2836 * all of the other values).
2837 *
2838 * If the optional argument +result_hash+ is given,
2839 * it is overwritten and returned.
2840 * This is intended to avoid the probe effect.
2841 *
2842 * h = {}
2843 * ObjectSpace.count_objects(h)
2844 * puts h
2845 * # => { TOTAL: 10000, T_CLASS: 158280, T_MODULE: 20672, T_STRING: 527249 }
2846 *
2847 * This method is only expected to work on C Ruby.
2848 *
2849 */
2850
2851static VALUE
2852count_objects(int argc, VALUE *argv, VALUE os)
2853{
2854 struct count_objects_data data = { 0 };
2855 VALUE hash = Qnil;
2856 VALUE types[T_MASK + 1];
2857
2858 if (rb_check_arity(argc, 0, 1) == 1) {
2859 hash = argv[0];
2860 if (!RB_TYPE_P(hash, T_HASH))
2861 rb_raise(rb_eTypeError, "non-hash given");
2862 }
2863
2864 for (size_t i = 0; i <= T_MASK; i++) {
2865 // type_sym can allocate an object,
2866 // so we need to create all key symbols in advance
2867 // not to disturb the result
2868 types[i] = type_sym(i);
2869 }
2870
2871 // Same as type_sym, we need to create all key symbols in advance
2872 VALUE total = ID2SYM(rb_intern("TOTAL"));
2873 VALUE free = ID2SYM(rb_intern("FREE"));
2874
2875 rb_gc_impl_each_object(rb_gc_get_objspace(), count_objects_i, &data);
2876
2877 if (NIL_P(hash)) {
2878 hash = rb_hash_new_capa(2 + T_MASK);
2879 }
2880 else if (!RHASH_EMPTY_P(hash)) {
2881 rb_hash_stlike_foreach(hash, set_zero, hash);
2882 }
2883 rb_hash_aset(hash, total, SIZET2NUM(data.total));
2884 rb_hash_aset(hash, free, SIZET2NUM(data.freed));
2885
2886 for (size_t i = 0; i <= T_MASK; i++) {
2887 if (data.counts[i]) {
2888 rb_hash_aset(hash, types[i], SIZET2NUM(data.counts[i]));
2889 }
2890 }
2891
2892 return hash;
2893}
2894
2895#define SET_STACK_END SET_MACHINE_STACK_END(&ec->machine.stack_end)
2896
2897#define STACK_START (ec->machine.stack_start)
2898#define STACK_END (ec->machine.stack_end)
2899#define STACK_LEVEL_MAX (ec->machine.stack_maxsize/sizeof(VALUE))
2900
2901#if STACK_GROW_DIRECTION < 0
2902# define STACK_LENGTH (size_t)(STACK_START - STACK_END)
2903#elif STACK_GROW_DIRECTION > 0
2904# define STACK_LENGTH (size_t)(STACK_END - STACK_START + 1)
2905#else
2906# define STACK_LENGTH ((STACK_END < STACK_START) ? (size_t)(STACK_START - STACK_END) \
2907 : (size_t)(STACK_END - STACK_START + 1))
2908#endif
2909#if !STACK_GROW_DIRECTION
2910int ruby_stack_grow_direction;
2911int
2912ruby_get_stack_grow_direction(volatile VALUE *addr)
2913{
2914 VALUE *end;
2915 SET_MACHINE_STACK_END(&end);
2916
2917 if (end > addr) return ruby_stack_grow_direction = 1;
2918 return ruby_stack_grow_direction = -1;
2919}
2920#endif
2921
2922size_t
2924{
2925 rb_execution_context_t *ec = GET_EC();
2926 SET_STACK_END;
2927 if (p) *p = STACK_UPPER(STACK_END, STACK_START, STACK_END);
2928 return STACK_LENGTH;
2929}
2930
2931#define PREVENT_STACK_OVERFLOW 1
2932#ifndef PREVENT_STACK_OVERFLOW
2933#if !(defined(POSIX_SIGNAL) && defined(SIGSEGV) && defined(HAVE_SIGALTSTACK))
2934# define PREVENT_STACK_OVERFLOW 1
2935#else
2936# define PREVENT_STACK_OVERFLOW 0
2937#endif
2938#endif
2939#if PREVENT_STACK_OVERFLOW && !defined(__EMSCRIPTEN__)
2940static int
2941stack_check(rb_execution_context_t *ec, int water_mark)
2942{
2943 SET_STACK_END;
2944
2945 size_t length = STACK_LENGTH;
2946 if (STACK_LEVEL_MAX == 0) return FALSE; /* unknown maxsize */
2947 if (STACK_LEVEL_MAX <= (size_t)water_mark) return TRUE;
2948 size_t maximum_length = STACK_LEVEL_MAX - water_mark;
2949
2950 return length > maximum_length;
2951}
2952#else
2953#define stack_check(ec, water_mark) FALSE
2954#endif
2955
2956#ifdef RUBY_ASAN_ENABLED
2957/* Unoptimized, instrumented VM frames can exceed the usual 16KB reserve. */
2958# define STACKFRAME_FOR_CALL_CFUNC (128 * 1024 / sizeof(VALUE))
2959#else
2960# define STACKFRAME_FOR_CALL_CFUNC 2048
2961#endif
2962
2963int
2964rb_ec_stack_check(rb_execution_context_t *ec)
2965{
2966 return stack_check(ec, STACKFRAME_FOR_CALL_CFUNC);
2967}
2968
2969int
2971{
2972 return stack_check(GET_EC(), STACKFRAME_FOR_CALL_CFUNC);
2973}
2974
2975/*
2976 * Returns the number of bytes of machine stack that can be used from here
2977 * before rb_ec_stack_check starts reporting an overflow, or SIZE_MAX if the size
2978 * of the stack is not known.
2979 */
2980size_t
2981rb_ec_stack_remaining(rb_execution_context_t *ec)
2982{
2983#if PREVENT_STACK_OVERFLOW && !defined(__EMSCRIPTEN__)
2984 if (STACK_LEVEL_MAX == 0) return SIZE_MAX;
2985 if (rb_ec_stack_check(ec)) return 0;
2986
2987 /* rb_ec_stack_check has just updated the end of the stack. */
2988 return (STACK_LEVEL_MAX - STACKFRAME_FOR_CALL_CFUNC - STACK_LENGTH) * sizeof(VALUE);
2989#else
2990 return SIZE_MAX;
2991#endif
2992}
2993
2994/* ==================== Marking ==================== */
2995
2996/* The traversal mark redirect is per-Ractor so a real GC never observes a
2997 * foreign traversal's redirect (a VM-global slot would divert another Ractor's
2998 * concurrent GC mark into obj_traverse recursion). Only threads with no
2999 * current Ractor (modular GC's marking worker threads) fall back to the VM
3000 * slot, which no setter writes, so they always take the real mark path. */
3001static inline struct gc_mark_func_data_struct **
3002gc_mark_func_data_slotp_of(rb_ractor_t *const cr)
3003{
3004#if USE_MODULAR_GC
3005 return cr != NULL ? &cr->mark_func_data : &GET_VM()->gc.mark_func_data;
3006#else
3007 RUBY_ASSERT(cr != NULL);
3008 return &cr->mark_func_data;
3009#endif
3010}
3011#define GC_MARK_FUNC_DATA_SLOTP() gc_mark_func_data_slotp_of(rb_current_ractor_raw(false))
3012
3013/* Resolve the current Ractor once for a whole marking walk. rb_current_ractor_raw() is an
3014 * out-of-line rb_current_ec() call, so
3015 * the walk threads this ctx down rather than repeating the lookup per reference. */
3016static inline struct rb_gc_mark_ctx
3017gc_current_mark_ctx(void)
3018{
3019 rb_ractor_t *const cr = rb_current_ractor_raw(false);
3020 struct gc_mark_func_data_struct **const mfdp = gc_mark_func_data_slotp_of(cr);
3021
3022 return (struct rb_gc_mark_ctx){
3023 .objspace = gc_current_objspace_of(cr),
3024 .mfdp = mfdp,
3025 /* A walk's mfd is installed before it starts and restored after, so the flag cannot
3026 * change under us -- unlike *mfdp, which the redirect below clears and restores
3027 * around each mark_func call. */
3028 .checking_shareable = *mfdp != NULL && (*mfdp)->checking_shareable,
3029 };
3030}
3031
3032struct rb_gc_mark_ctx
3033rb_gc_current_mark_ctx(void)
3034{
3035 return gc_current_mark_ctx();
3036}
3037
3038#define RB_GC_MARK_OR_TRAVERSE(ctx, func, obj_or_ptr, obj, check_obj) do { \
3039 if (!RB_SPECIAL_CONST_P(obj)) { \
3040 struct gc_mark_func_data_struct **mfdp = (ctx)->mfdp; \
3041 struct gc_mark_func_data_struct *mark_func_data = *mfdp; \
3042 void *objspace = (ctx)->objspace; \
3043 if (LIKELY(mark_func_data == NULL)) { \
3044 GC_ASSERT(rb_gc_impl_during_gc_p(objspace)); \
3045 (func)(objspace, (obj_or_ptr)); \
3046 } \
3047 else if (check_obj ? \
3048 rb_gc_impl_live_object_p(objspace, (const void *)obj) && \
3049 !rb_gc_impl_garbage_object_p(objspace, obj) : \
3050 true) { \
3051 GC_ASSERT(!rb_gc_impl_during_gc_p(objspace)); \
3052 *mfdp = NULL; \
3053 mark_func_data->mark_func((obj), mark_func_data->data); \
3054 *mfdp = mark_func_data; \
3055 } \
3056 } \
3057} while (0)
3058
3059static inline void
3060gc_mark_internal(const struct rb_gc_mark_ctx *ctx, VALUE obj)
3061{
3062 RB_GC_MARK_OR_TRAVERSE(ctx, rb_gc_impl_mark, obj, obj, false);
3063}
3064
3065static inline void
3066gc_mark_and_pin_internal(const struct rb_gc_mark_ctx *ctx, VALUE obj)
3067{
3068 RB_GC_MARK_OR_TRAVERSE(ctx, rb_gc_impl_mark_and_pin, obj, obj, false);
3069}
3070
3071static inline void
3072gc_mark_maybe_internal(const struct rb_gc_mark_ctx *ctx, VALUE obj)
3073{
3074 RB_GC_MARK_OR_TRAVERSE(ctx, rb_gc_impl_mark_maybe, obj, obj, true);
3075}
3076
3077static inline void
3078gc_mark_and_move_internal(const struct rb_gc_mark_ctx *ctx, VALUE *ptr)
3079{
3080 RB_GC_MARK_OR_TRAVERSE(ctx, rb_gc_impl_mark_and_move, ptr, *ptr, false);
3081}
3082
3083void
3084rb_gc_mark_movable(VALUE obj)
3085{
3086 if (RB_SPECIAL_CONST_P(obj)) return;
3087
3088 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
3089
3090 gc_mark_internal(&ctx, obj);
3091}
3092
3093void
3094rb_gc_mark_movable_ctx(const struct rb_gc_mark_ctx *ctx, VALUE obj)
3095{
3096 gc_mark_internal(ctx, obj);
3097}
3098
3099void
3100rb_gc_mark(VALUE obj)
3101{
3102 if (RB_SPECIAL_CONST_P(obj)) return;
3103
3104 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
3105
3106 gc_mark_and_pin_internal(&ctx, obj);
3107}
3108
3109void
3110rb_gc_mark_and_pin_ctx(const struct rb_gc_mark_ctx *ctx, VALUE obj)
3111{
3112 gc_mark_and_pin_internal(ctx, obj);
3113}
3114
3115void
3116rb_gc_mark_maybe(VALUE obj)
3117{
3118 if (RB_SPECIAL_CONST_P(obj)) return;
3119
3120 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
3121
3122 gc_mark_maybe_internal(&ctx, obj);
3123}
3124
3125void
3126rb_gc_mark_maybe_ctx(const struct rb_gc_mark_ctx *ctx, VALUE obj)
3127{
3128 gc_mark_maybe_internal(ctx, obj);
3129}
3130
3131void
3132rb_gc_mark_and_move(VALUE *ptr)
3133{
3134 if (RB_SPECIAL_CONST_P(*ptr)) return;
3135
3136 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
3137
3138 gc_mark_and_move_internal(&ctx, ptr);
3139}
3140
3141void
3142rb_gc_mark_and_move_ctx(const struct rb_gc_mark_ctx *ctx, VALUE *ptr)
3143{
3144 gc_mark_and_move_internal(ctx, ptr);
3145}
3146
3147typedef void each_location_func(const struct rb_gc_mark_ctx *ctx, VALUE obj, void *data);
3148
3149ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void each_location(const struct rb_gc_mark_ctx *ctx, register const VALUE *x, register long n, each_location_func *cb, void *data));
3150static void
3151each_location(const struct rb_gc_mark_ctx *ctx, register const VALUE *x, register long n, each_location_func *cb, void *data)
3152{
3153 VALUE v;
3154 while (n--) {
3155 v = *x;
3156 cb(ctx, v, data);
3157 x++;
3158 }
3159}
3160
3161static void
3162each_location_ptr(const struct rb_gc_mark_ctx *ctx, const VALUE *start, const VALUE *end, each_location_func *cb, void *data)
3163{
3164 if (end <= start) return;
3165 each_location(ctx, start, end - start, cb, data);
3166}
3167
3168static void
3169gc_mark_maybe_each_location(const struct rb_gc_mark_ctx *ctx, VALUE obj, void *data)
3170{
3171 gc_mark_maybe_internal(ctx, obj);
3172}
3173
3174void
3175rb_gc_mark_locations(const VALUE *start, const VALUE *end)
3176{
3177 if (end <= start) return;
3178
3179 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
3180
3181 each_location_ptr(&ctx, start, end, gc_mark_maybe_each_location, NULL);
3182}
3183
3184void
3185rb_gc_mark_locations_ctx(const struct rb_gc_mark_ctx *ctx, const VALUE *start, const VALUE *end)
3186{
3187 each_location_ptr(ctx, start, end, gc_mark_maybe_each_location, NULL);
3188}
3189
3190void
3191rb_gc_mark_values(long n, const VALUE *values)
3192{
3193 if (n <= 0) return;
3194
3195 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
3196
3197 for (long i = 0; i < n; i++) {
3198 gc_mark_internal(&ctx, values[i]);
3199 }
3200}
3201
3202void
3203rb_gc_mark_vm_stack_values(long n, const VALUE *values)
3204{
3205 if (n <= 0) return;
3206
3207 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
3208
3209 for (long i = 0; i < n; i++) {
3210 gc_mark_and_pin_internal(&ctx, values[i]);
3211 }
3212}
3213
3214static int
3215mark_key(st_data_t key, st_data_t value, st_data_t data)
3216{
3217 gc_mark_and_pin_internal((const struct rb_gc_mark_ctx *)data, (VALUE)key);
3218
3219 return ST_CONTINUE;
3220}
3221
3222void
3223rb_mark_set(st_table *tbl)
3224{
3225 if (!tbl) return;
3226
3227 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
3228
3229 st_foreach(tbl, mark_key, (st_data_t)&ctx);
3230}
3231
3232static int
3233mark_keyvalue(st_data_t key, st_data_t value, st_data_t data)
3234{
3235 const struct rb_gc_mark_ctx *ctx = (const struct rb_gc_mark_ctx *)data;
3236
3237 gc_mark_internal(ctx, (VALUE)key);
3238 gc_mark_internal(ctx, (VALUE)value);
3239
3240 return ST_CONTINUE;
3241}
3242
3243static int
3244pin_key_pin_value(st_data_t key, st_data_t value, st_data_t data)
3245{
3246 const struct rb_gc_mark_ctx *ctx = (const struct rb_gc_mark_ctx *)data;
3247
3248 gc_mark_and_pin_internal(ctx, (VALUE)key);
3249 gc_mark_and_pin_internal(ctx, (VALUE)value);
3250
3251 return ST_CONTINUE;
3252}
3253
3254static int
3255pin_key_mark_value(st_data_t key, st_data_t value, st_data_t data)
3256{
3257 const struct rb_gc_mark_ctx *ctx = (const struct rb_gc_mark_ctx *)data;
3258
3259 gc_mark_and_pin_internal(ctx, (VALUE)key);
3260 gc_mark_internal(ctx, (VALUE)value);
3261
3262 return ST_CONTINUE;
3263}
3264
3265static int
3266gc_mark_tbl_no_pin_i(st_data_t key, st_data_t value, st_data_t data)
3267{
3268 gc_mark_internal((const struct rb_gc_mark_ctx *)data, (VALUE)value);
3269
3270 return ST_CONTINUE;
3271}
3272
3273static int
3274gc_mark_set_no_pin_i(st_data_t key, st_data_t value, st_data_t data)
3275{
3276 gc_mark_internal((const struct rb_gc_mark_ctx *)data, (VALUE)key);
3277
3278 return ST_CONTINUE;
3279}
3280
3281static void
3282mark_hash(const struct rb_gc_mark_ctx *ctx, VALUE hash)
3283{
3284 if (rb_hash_compare_by_id_p(hash)) {
3285 rb_hash_stlike_foreach(hash, pin_key_mark_value, (st_data_t)ctx);
3286 }
3287 else {
3288 rb_hash_stlike_foreach(hash, mark_keyvalue, (st_data_t)ctx);
3289 }
3290
3291 gc_mark_internal(ctx, RHASH(hash)->ifnone);
3292}
3293
3294void
3295rb_mark_hash(st_table *tbl)
3296{
3297 if (!tbl) return;
3298
3299 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
3300
3301 st_foreach(tbl, pin_key_pin_value, (st_data_t)&ctx);
3302}
3303
3304static enum rb_id_table_iterator_result
3305mark_method_entry_i(VALUE me, void *data)
3306{
3307 gc_mark_internal((const struct rb_gc_mark_ctx *)data, me);
3308
3309 return ID_TABLE_CONTINUE;
3310}
3311
3312static void
3313mark_m_tbl(const struct rb_gc_mark_ctx *ctx, struct rb_id_table *tbl)
3314{
3315 if (tbl) {
3316 rb_id_table_foreach_values(tbl, mark_method_entry_i, (void *)ctx);
3317 }
3318}
3319
3320static enum rb_id_table_iterator_result
3321mark_const_entry_i(VALUE value, void *data)
3322{
3323 const rb_const_entry_t *ce = (const rb_const_entry_t *)value;
3324 const struct rb_gc_mark_ctx *ctx = (const struct rb_gc_mark_ctx *)data;
3325
3326 gc_mark_internal(ctx, ce->value);
3327 gc_mark_internal(ctx, ce->u.file); // TODO: ce->u.file should be shareable?
3328
3329 return ID_TABLE_CONTINUE;
3330}
3331
3332static void
3333mark_const_tbl(const struct rb_gc_mark_ctx *ctx, struct rb_id_table *tbl)
3334{
3335 if (!tbl) return;
3336 rb_id_table_foreach_values(tbl, mark_const_entry_i, (void *)ctx);
3337}
3338
3339#if STACK_GROW_DIRECTION < 0
3340#define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_END, (end) = STACK_START)
3341#elif STACK_GROW_DIRECTION > 0
3342#define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_START, (end) = STACK_END+(appendix))
3343#else
3344#define GET_STACK_BOUNDS(start, end, appendix) \
3345 ((STACK_END < STACK_START) ? \
3346 ((start) = STACK_END, (end) = STACK_START) : ((start) = STACK_START, (end) = STACK_END+(appendix)))
3347#endif
3348
3349static void
3350gc_mark_machine_stack_location_maybe(const struct rb_gc_mark_ctx *ctx, VALUE obj, void *data)
3351{
3352 gc_mark_maybe_internal(ctx, obj);
3353
3354#ifdef RUBY_ASAN_ENABLED
3355 const rb_execution_context_t *ec = (const rb_execution_context_t *)data;
3356 void *fake_frame_start;
3357 void *fake_frame_end;
3358 bool is_fake_frame = asan_get_fake_stack_extents(
3359 ec->machine.asan_fake_stack_handle, obj,
3360 ec->machine.stack_start, ec->machine.stack_end,
3361 &fake_frame_start, &fake_frame_end
3362 );
3363 if (is_fake_frame) {
3364 each_location_ptr(ctx, fake_frame_start, fake_frame_end, gc_mark_maybe_each_location, NULL);
3365 }
3366#endif
3367}
3368
3369static bool
3370gc_object_moved_p_internal(void *objspace, VALUE obj)
3371{
3372 if (SPECIAL_CONST_P(obj)) {
3373 return false;
3374 }
3375
3376 return rb_gc_impl_object_moved_p(objspace, obj);
3377}
3378
3379static VALUE
3380gc_location_internal(void *objspace, VALUE value)
3381{
3382 if (SPECIAL_CONST_P(value)) {
3383 return value;
3384 }
3385
3386 return rb_gc_impl_location(objspace, value);
3387}
3388
3389VALUE
3390rb_gc_location(VALUE value)
3391{
3392 return gc_location_internal(rb_gc_get_objspace(), value);
3393}
3394
3395VALUE
3396rb_gc_location_ctx(const struct rb_gc_mark_ctx *ctx, VALUE value)
3397{
3398 return gc_location_internal(ctx->objspace, value);
3399}
3400
3401static inline void
3402gc_update_moved_internal(void *objspace, VALUE *ptr)
3403{
3404 VALUE destination = gc_location_internal(objspace, *ptr);
3405 if (destination != *ptr) {
3406 *ptr = destination;
3407 }
3408}
3409
3410void
3411rb_gc_update_moved(VALUE *ptr)
3412{
3413 gc_update_moved_internal(rb_gc_get_objspace(), ptr);
3414}
3415
3416void
3417rb_gc_update_moved_ctx(const struct rb_gc_mark_ctx *ctx, VALUE *ptr)
3418{
3419 gc_update_moved_internal(ctx->objspace, ptr);
3420}
3421
3422static void gc_mark_machine_context(const struct rb_gc_mark_ctx *ctx, const rb_execution_context_t *ec);
3423
3424#if defined(__wasm__)
3425
3426
3427static VALUE *rb_stack_range_tmp[2];
3428
3429static void
3430rb_mark_locations(void *begin, void *end)
3431{
3432 rb_stack_range_tmp[0] = begin;
3433 rb_stack_range_tmp[1] = end;
3434}
3435
3436void
3437rb_gc_save_machine_context(void)
3438{
3439 // no-op
3440}
3441
3442# if defined(__EMSCRIPTEN__)
3443
3444static void
3445mark_current_machine_context(const struct rb_gc_mark_ctx *ctx, const rb_execution_context_t *ec)
3446{
3447 emscripten_scan_stack(rb_mark_locations);
3448 each_location_ptr(ctx, rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe_each_location, NULL);
3449
3450 emscripten_scan_registers(rb_mark_locations);
3451 each_location_ptr(ctx, rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe_each_location, NULL);
3452}
3453# else // use Asyncify version
3454
3455static void
3456mark_current_machine_context(const struct rb_gc_mark_ctx *ctx, rb_execution_context_t *ec)
3457{
3458 VALUE *stack_start, *stack_end;
3459 SET_STACK_END;
3460 GET_STACK_BOUNDS(stack_start, stack_end, 1);
3461 each_location_ptr(ctx, stack_start, stack_end, gc_mark_maybe_each_location, NULL);
3462
3463 rb_wasm_scan_locals(rb_mark_locations);
3464 each_location_ptr(ctx, rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe_each_location, NULL);
3465}
3466
3467# endif
3468
3469#else // !defined(__wasm__)
3470
3471void
3472rb_gc_save_machine_context(void)
3473{
3474 rb_thread_t *thread = GET_THREAD();
3475
3476 RB_VM_SAVE_MACHINE_CONTEXT(thread);
3477}
3478
3479
3480static void
3481mark_current_machine_context(const struct rb_gc_mark_ctx *ctx, const rb_execution_context_t *ec)
3482{
3483 gc_mark_machine_context(ctx, ec);
3484}
3485#endif
3486
3487static void
3488gc_mark_machine_context(const struct rb_gc_mark_ctx *ctx, const rb_execution_context_t *ec)
3489{
3490 VALUE *stack_start, *stack_end;
3491
3492 GET_STACK_BOUNDS(stack_start, stack_end, 0);
3493 RUBY_DEBUG_LOG("ec->th:%u stack_start:%p stack_end:%p", rb_ec_thread_ptr(ec)->serial, stack_start, stack_end);
3494
3495 void *data =
3496#ifdef RUBY_ASAN_ENABLED
3497 /* gc_mark_machine_stack_location_maybe() uses data as const */
3498 (rb_execution_context_t *)ec;
3499#else
3500 NULL;
3501#endif
3502
3503 each_location_ptr(ctx, stack_start, stack_end, gc_mark_machine_stack_location_maybe, data);
3504 int num_regs = sizeof(ec->machine.regs)/(sizeof(VALUE));
3505 each_location(ctx, (VALUE*)&ec->machine.regs, num_regs, gc_mark_machine_stack_location_maybe, data);
3506}
3507
3508void
3509rb_gc_mark_machine_context_ctx(const struct rb_gc_mark_ctx *ctx, const rb_execution_context_t *ec)
3510{
3511 gc_mark_machine_context(ctx, ec);
3512}
3513
3514static int
3515rb_mark_tbl_i(st_data_t key, st_data_t value, st_data_t data)
3516{
3517 gc_mark_and_pin_internal((const struct rb_gc_mark_ctx *)data, (VALUE)value);
3518
3519 return ST_CONTINUE;
3520}
3521
3522void
3523rb_mark_tbl(st_table *tbl)
3524{
3525 if (!tbl || tbl->num_entries == 0) return;
3526
3527 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
3528
3529 st_foreach(tbl, rb_mark_tbl_i, (st_data_t)&ctx);
3530}
3531
3532static void
3533gc_mark_tbl_no_pin(const struct rb_gc_mark_ctx *ctx, st_table *tbl)
3534{
3535 if (!tbl || tbl->num_entries == 0) return;
3536
3537 st_foreach(tbl, gc_mark_tbl_no_pin_i, (st_data_t)ctx);
3538}
3539
3540void
3541rb_mark_tbl_no_pin(st_table *tbl)
3542{
3543 if (!tbl || tbl->num_entries == 0) return;
3544
3545 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
3546
3547 gc_mark_tbl_no_pin(&ctx, tbl);
3548}
3549
3550void
3551rb_mark_tbl_no_pin_ctx(const struct rb_gc_mark_ctx *ctx, st_table *tbl)
3552{
3553 gc_mark_tbl_no_pin(ctx, tbl);
3554}
3555
3556static void
3557gc_mark_set_no_pin(const struct rb_gc_mark_ctx *ctx, st_table *tbl)
3558{
3559 if (!tbl || tbl->num_entries == 0) return;
3560
3561 st_foreach(tbl, gc_mark_set_no_pin_i, (st_data_t)ctx);
3562}
3563
3564void
3565rb_gc_mark_set_no_pin(st_table *tbl)
3566{
3567 if (!tbl || tbl->num_entries == 0) return;
3568
3569 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
3570
3571 gc_mark_set_no_pin(&ctx, tbl);
3572}
3573
3574void
3575rb_gc_mark_set_no_pin_ctx(const struct rb_gc_mark_ctx *ctx, st_table *tbl)
3576{
3577 gc_mark_set_no_pin(ctx, tbl);
3578}
3579
3580static bool
3581gc_declarative_marking_p(const rb_data_type_t *type)
3582{
3583 return (type->flags & RUBY_TYPED_DECL_MARKING) != 0;
3584}
3585
3586rb_execution_context_t *
3587rb_gc_get_ec(void)
3588{
3589 void *objspace = rb_gc_get_objspace();
3590
3591 if (RB_LIKELY(rb_gc_impl_during_gc_p(objspace))) {
3592 return rb_gc_impl_get_vm_context(objspace)->ec;
3593 }
3594 else {
3595 return GET_EC();
3596 }
3597}
3598
3599/* need_lock is false when the caller already holds vm->gc.registered_addrs.lock (the
3600 * terminated_set walk in rb_gc_mark_roots); the lock is not recursive, so both the
3601 * marking path and the traversal-API snapshot path below must honour it. */
3602void
3603rb_gc_mark_registered_addrs(rb_ractor_t *r, bool need_lock)
3604{
3605 if (r->registered_addrs_cnt == 0) return;
3606
3607 if (rb_gc_impl_during_gc_p(rb_gc_get_objspace())) {
3608 rb_vm_t *vm = GET_VM();
3609 if (need_lock) rb_native_mutex_lock(&vm->gc.registered_addrs.lock);
3610 for (size_t i = 0; i < r->registered_addrs_cnt; i++) {
3611 rb_gc_mark_maybe(*r->registered_addrs[i].addr);
3612 }
3613 if (need_lock) rb_native_mutex_unlock(&vm->gc.registered_addrs.lock);
3614 return;
3615 }
3616
3617 VALUE stack_snap[16];
3618 VALUE *snap = stack_snap;
3619 size_t capa = numberof(stack_snap);
3620
3621 rb_vm_t *vm = GET_VM();
3622 if (need_lock) rb_native_mutex_lock(&vm->gc.registered_addrs.lock);
3623 size_t cnt = r->registered_addrs_cnt;
3624 if (RB_UNLIKELY(cnt > capa)) {
3625 if (need_lock) rb_native_mutex_unlock(&vm->gc.registered_addrs.lock);
3626 rb_ractor_t *cr = rb_current_ractor_raw(false);
3627 bool saved_malloc_gc_disabled = cr ? cr->malloc_gc_disabled : false;
3628 if (cr) cr->malloc_gc_disabled = true;
3629 snap = ALLOC_N(VALUE, cnt);
3630 if (cr) cr->malloc_gc_disabled = saved_malloc_gc_disabled;
3631 capa = cnt;
3632 if (need_lock) rb_native_mutex_lock(&vm->gc.registered_addrs.lock);
3633 cnt = r->registered_addrs_cnt;
3634 VM_ASSERT(cnt <= capa);
3635 }
3636 for (size_t i = 0; i < cnt; i++) {
3637 snap[i] = *r->registered_addrs[i].addr;
3638 }
3639 if (need_lock) rb_native_mutex_unlock(&vm->gc.registered_addrs.lock);
3640
3641 for (size_t i = 0; i < cnt; i++) {
3642 rb_gc_mark_maybe(snap[i]);
3643 }
3644 if (snap != stack_snap) xfree(snap);
3645}
3646
3647static void
3648gc_mark_roots_ctx(const struct rb_gc_mark_ctx *ctx, void *objspace, const char **categoryp)
3649{
3650 rb_execution_context_t *ec = rb_gc_get_ec();
3651 rb_vm_t *vm = rb_ec_vm_ptr(ec);
3652
3653#define MARK_CHECKPOINT(category) do { \
3654 if (categoryp) *categoryp = category; \
3655} while (0)
3656
3657 /* A single-objspace impl (mmtk) only has stop-the-world global GCs and no
3658 * per-mutator root scan, so always walk every Ractor's local roots here. */
3659 const bool global_gc = rb_gc_impl_during_global_gc_p(objspace) ||
3660 !rb_gc_impl_multi_objspace_p();
3661
3662 /* Mark the current Ractor's roots from its C structs (a local GC must not depend on
3663 * heap wrapper traversal). A global GC does the same for every Ractor. */
3664 MARK_CHECKPOINT("ractor");
3665 if (global_gc) {
3666 rb_ractor_t *r;
3667 ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
3668 rb_ractor_mark_local_roots(ctx, r);
3669 MARK_CHECKPOINT("registered_addrs");
3670 rb_gc_mark_registered_addrs(r, true);
3671 MARK_CHECKPOINT("ractor");
3672 }
3673
3674 /* Early in boot (before rb_ractor_main_setup) main is not in vm->ractor.set
3675 * yet; do not drop its registered_marks in a single-objspace boot GC. */
3676 if (vm->ractor.cnt == 0 && vm->ractor.main_ractor) {
3677 rb_ractor_mark_local_roots(ctx, vm->ractor.main_ractor);
3678 MARK_CHECKPOINT("registered_addrs");
3679 rb_gc_mark_registered_addrs(vm->ractor.main_ractor, true);
3680 MARK_CHECKPOINT("ractor");
3681 }
3682 /* A Ractor that terminated (left vm->ractor.set) but whose struct is not freed
3683 * still owns rb_gc_register_mark_object pins. Keep them alive until
3684 * ractor_free hands them to main; an orphan (owner == NULL) was moved above.
3685 * The join value is not rooted here: ractor_mark marks it from the wrapper. */
3686 for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) {
3687 rb_ractor_t *owner = vm->gc.zombie_objspaces[i].owner;
3688 if (owner) {
3689 rb_gc_mark_vm_stack_values((long)owner->registered_marks_cnt,
3690 owner->registered_marks);
3691 MARK_CHECKPOINT("registered_addrs");
3692 rb_gc_mark_registered_addrs(owner, true);
3693 MARK_CHECKPOINT("ractor");
3694 }
3695 }
3696
3697 /* Single-objspace impl: keep terminated-but-not-freed Ractors'
3698 * rb_gc_register_mark_object entries alive without depending on wrapper
3699 * reachability. With multiple objspaces zombie_objspaces covers this. */
3700 if (!rb_gc_impl_multi_objspace_p()) {
3701 rb_ractor_t *tr;
3702 rb_native_mutex_lock(&vm->gc.registered_addrs.lock);
3703 ccan_list_for_each(&vm->ractor.terminated_set, tr, vmlr_node) {
3704 rb_gc_mark_vm_stack_values((long)tr->registered_marks_cnt,
3705 tr->registered_marks);
3706 MARK_CHECKPOINT("registered_addrs");
3707 rb_gc_mark_registered_addrs(tr, false);
3708 MARK_CHECKPOINT("ractor");
3709 }
3710 rb_native_mutex_unlock(&vm->gc.registered_addrs.lock);
3711 }
3712 }
3713 else {
3714 rb_ractor_t *cr = rb_ec_ractor_ptr(ec);
3715 rb_ractor_mark_local_roots(ctx, cr);
3716 MARK_CHECKPOINT("registered_addrs");
3717 rb_gc_mark_registered_addrs(cr, true);
3718 }
3719
3720 /* Trap handlers live in the VM-global vm->trap_list.cmd[], a fixed array of aligned
3721 * VALUEs (signal.c uses ACCESS_ONCE): a racing walk reads either the old or the new
3722 * handler, both alive, so no lock. */
3723 MARK_CHECKPOINT("trap_list");
3724 rb_gc_mark_values(RUBY_NSIG, vm->trap_list.cmd);
3725
3726 MARK_CHECKPOINT("shareable_p");
3727 rb_ractor_mark_shareable_p_inflight(ctx);
3728
3729 /* VM-global roots belong to the main Ractor's objspace, since the boot objects
3730 * live there. A non-main Ractor's local GC skips them; a global GC walks all. */
3731 if (global_gc || objspace == vm->ractor.main_ractor->objspace) {
3732 /* Only the main Ractor can register at_exit/END procs (a non-main one gets an
3733 * IsolationError) so end_procs is a lock-free linked list */
3734 MARK_CHECKPOINT("end_proc");
3735 rb_mark_end_proc();
3736
3737 MARK_CHECKPOINT("vm");
3738 /* rb_vm_mark and the JIT root marks walk VM-global weak tables and shared singleton
3739 * JIT state that other Ractors rewrite under the VM lock, so main's otherwise
3740 * lock-free local GC takes the VM lock for this stretch */
3741 const bool vm_mark_needs_lock = rb_multi_ractor_p() && !global_gc;
3742 unsigned int vm_mark_lock_lev = 0;
3743 if (vm_mark_needs_lock) vm_mark_lock_lev = RB_GC_VM_LOCK_NO_BARRIER();
3744 rb_vm_mark(vm);
3745
3746 MARK_CHECKPOINT("global_tbl");
3747 rb_gc_mark_global_tbl();
3748
3749#if USE_YJIT
3750 void rb_yjit_root_mark(void); // in Rust
3751
3752 if (rb_yjit_enabled_p) {
3753 MARK_CHECKPOINT("YJIT");
3754 rb_yjit_root_mark();
3755 }
3756#endif
3757
3758#if USE_ZJIT
3759 void rb_zjit_root_mark(void);
3760 if (rb_zjit_enabled_p) {
3761 MARK_CHECKPOINT("ZJIT");
3762 rb_zjit_root_mark();
3763 }
3764#endif
3765 if (vm_mark_needs_lock) RB_GC_VM_UNLOCK_NO_BARRIER(vm_mark_lock_lev);
3766
3767 if (global_gc || rb_gc_single_objspace_p()) {
3768 MARK_CHECKPOINT("global_symbols");
3769 rb_sym_global_symbols_mark_and_move();
3770 }
3771 }
3772
3773 /* The dying thread's final collection of its own objspace runs after its stack
3774 * has been torn down (thread_cleanup_func), so there is no live machine context
3775 * to scan -- the join value and the pins are rooted explicitly. Scanning the
3776 * half-dead stack is not only useless but faults on some platforms. */
3777 if (!rb_gc_impl_during_postmortem_p(objspace)) {
3778 MARK_CHECKPOINT("machine_context");
3779 mark_current_machine_context(ctx, ec);
3780 }
3781
3782 MARK_CHECKPOINT("finish");
3783
3784#undef MARK_CHECKPOINT
3785}
3786
3787void
3788rb_gc_mark_roots(void *objspace, const char **categoryp)
3789{
3790 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
3791
3792 GC_ASSERT(ctx.objspace == objspace);
3793 gc_mark_roots_ctx(&ctx, objspace, categoryp);
3794}
3795
3797 const struct rb_gc_mark_ctx *ctx;
3798 VALUE obj;
3799};
3800
3801static void
3802gc_mark_classext_module(rb_classext_t *ext, bool prime, VALUE box_value, void *arg)
3803{
3805 const struct rb_gc_mark_ctx *ctx = foreach_arg->ctx;
3806
3807 if (RCLASSEXT_SUPER(ext)) {
3808 gc_mark_internal(ctx, RCLASSEXT_SUPER(ext));
3809 }
3810 mark_m_tbl(ctx, RCLASSEXT_M_TBL(ext));
3811
3812 gc_mark_internal(ctx, RCLASSEXT_FIELDS_OBJ(ext));
3813 gc_mark_internal(ctx, RCLASSEXT_CVC_TBL(ext));
3814
3815 if (!RCLASSEXT_SHARED_CONST_TBL(ext) && RCLASSEXT_CONST_TBL(ext)) {
3816 mark_const_tbl(ctx, RCLASSEXT_CONST_TBL(ext));
3817 }
3818 mark_m_tbl(ctx, RCLASSEXT_CALLABLE_M_TBL(ext));
3819 gc_mark_internal(ctx, RCLASSEXT_CC_TBL(ext));
3820 if (RCLASSEXT_SUBCLASSES(ext)) {
3821 gc_mark_internal(ctx, RCLASSEXT_SUBCLASSES(ext));
3822 }
3823 gc_mark_internal(ctx, RCLASSEXT_CLASSPATH(ext));
3824}
3825
3826static void
3827gc_mark_classext_iclass(rb_classext_t *ext, bool prime, VALUE box_value, void *arg)
3828{
3830 const struct rb_gc_mark_ctx *ctx = foreach_arg->ctx;
3831
3832 if (RCLASSEXT_SUPER(ext)) {
3833 gc_mark_internal(ctx, RCLASSEXT_SUPER(ext));
3834 }
3835 if (RCLASSEXT_ICLASS_IS_ORIGIN(ext) && !RCLASSEXT_ICLASS_ORIGIN_SHARED_MTBL(ext)) {
3836 mark_m_tbl(ctx, RCLASSEXT_M_TBL(ext));
3837 }
3838 if (RCLASSEXT_INCLUDER(ext)) {
3839 gc_mark_internal(ctx, RCLASSEXT_INCLUDER(ext));
3840 }
3841 if (RCLASSEXT_MODULE_REFINEMENT_ICLASSES(ext)) {
3842 gc_mark_internal(ctx, RCLASSEXT_MODULE_REFINEMENT_ICLASSES(ext));
3843 }
3844 mark_m_tbl(ctx, RCLASSEXT_CALLABLE_M_TBL(ext));
3845 gc_mark_internal(ctx, RCLASSEXT_CC_TBL(ext));
3846 if (RCLASSEXT_SUBCLASSES(ext)) {
3847 gc_mark_internal(ctx, RCLASSEXT_SUBCLASSES(ext));
3848 }
3849}
3850
3851#define TYPED_DATA_REFS_OFFSET_LIST(d) (size_t *)(uintptr_t)RTYPEDDATA_TYPE(d)->function.dmark
3852
3853static inline bool
3854rb_obj_using_gen_fields_table_p(VALUE obj)
3855{
3856 switch (BUILTIN_TYPE(obj)) {
3857 case T_STRUCT:
3858 case T_DATA:
3859 return false;
3860
3861 default:
3862 break;
3863 }
3864
3865 return rb_obj_gen_fields_p(obj);
3866}
3867
3868void
3869rb_gc_move_obj_during_marking(VALUE from, VALUE to)
3870{
3871 if (rb_obj_using_gen_fields_table_p(to)) {
3872 rb_mark_generic_ivar(from);
3873 }
3874}
3875
3876static void
3877gc_mark_children_ctx(const struct rb_gc_mark_ctx *ctx, VALUE obj)
3878{
3879 struct gc_mark_classext_foreach_arg foreach_args;
3880
3881 if (rb_obj_using_gen_fields_table_p(obj)) {
3882 rb_mark_generic_ivar(obj);
3883 }
3884
3885 switch (BUILTIN_TYPE(obj)) {
3886 case T_FLOAT:
3887 case T_BIGNUM:
3888 return;
3889
3890 case T_NIL:
3891 case T_FIXNUM:
3892 rb_bug("rb_gc_mark() called for broken object");
3893 break;
3894
3895 case T_NODE:
3896 UNEXPECTED_NODE(rb_gc_mark);
3897 break;
3898
3899 case T_IMEMO:
3900 rb_imemo_mark_and_move(ctx, obj, false);
3901 return;
3902
3903 default:
3904 break;
3905 }
3906
3907 gc_mark_internal(ctx, RBASIC(obj)->klass);
3908
3909 switch (BUILTIN_TYPE(obj)) {
3910 case T_CLASS:
3911 if (FL_TEST_RAW(obj, FL_SINGLETON)) {
3912 gc_mark_internal(ctx, RCLASS_ATTACHED_OBJECT(obj));
3913 }
3914 // Continue to the shared T_CLASS/T_MODULE
3915 case T_MODULE:
3916 foreach_args.ctx = ctx;
3917 foreach_args.obj = obj;
3918 rb_class_classext_foreach(obj, gc_mark_classext_module, (void *)&foreach_args);
3919 if (BOX_USER_P(RCLASS_PRIME_BOX(obj))) {
3920 gc_mark_internal(ctx, RCLASS_PRIME_BOX(obj)->box_object);
3921 }
3922 break;
3923
3924 case T_ICLASS:
3925 foreach_args.ctx = ctx;
3926 foreach_args.obj = obj;
3927 rb_class_classext_foreach(obj, gc_mark_classext_iclass, (void *)&foreach_args);
3928 if (BOX_USER_P(RCLASS_PRIME_BOX(obj))) {
3929 gc_mark_internal(ctx, RCLASS_PRIME_BOX(obj)->box_object);
3930 }
3931 break;
3932
3933 case T_ARRAY:
3934 if (ARY_SHARED_P(obj)) {
3935 VALUE root = ARY_SHARED_ROOT(obj);
3936 if (RB_TYPE_P(root, T_ARRAY)) {
3937 gc_mark_internal(ctx, root);
3938 }
3939 else {
3940 /* Ractor#send(move: true) hollowed the root out in place. If it was
3941 * embedded our elements are still in its slot, and nothing says so any
3942 * more, so it must not move (gc_ref_update_array cannot re-point us). */
3943 gc_mark_and_pin_internal(ctx, root);
3944 }
3945 }
3946 else {
3947 rb_len_t len = RARRAY_LEN(obj);
3948 const VALUE *ptr = RARRAY_CONST_PTR(obj);
3949 for (rb_len_t i = 0; i < len; i++) {
3950 gc_mark_internal(ctx, ptr[i]);
3951 }
3952 }
3953 break;
3954
3955 case T_HASH:
3956 mark_hash(ctx, obj);
3957 break;
3958
3959 case T_SYMBOL:
3960 gc_mark_internal(ctx, RSYMBOL(obj)->fstr);
3961 break;
3962
3963 case T_STRING:
3964 if (STR_SHARED_P(obj)) {
3965 if (STR_EMBED_P(RSTRING(obj)->as.heap.aux.shared)) {
3966 /* Embedded shared strings cannot be moved because this string
3967 * points into the slot of the shared string. There may be code
3968 * using the RSTRING_PTR on the stack, which would pin this
3969 * string but not pin the shared string, causing it to move. */
3970 gc_mark_and_pin_internal(ctx, RSTRING(obj)->as.heap.aux.shared);
3971 }
3972 else {
3973 gc_mark_internal(ctx, RSTRING(obj)->as.heap.aux.shared);
3974 }
3975 }
3976 break;
3977
3978 case T_DATA: {
3979 void *const ptr = RTYPEDDATA_GET_DATA(obj);
3980
3981 gc_mark_internal(ctx, RTYPEDDATA(obj)->fields_obj);
3982
3983 if (ptr) {
3984 if (gc_declarative_marking_p(RTYPEDDATA_TYPE(obj))) {
3985 size_t *offset_list = TYPED_DATA_REFS_OFFSET_LIST(obj);
3986
3987 for (size_t offset = *offset_list; offset != RUBY_REF_END; offset = *offset_list++) {
3988 gc_mark_internal(ctx, *(VALUE *)((char *)ptr + offset));
3989 }
3990 }
3991 else {
3992 RUBY_DATA_FUNC mark_func = RTYPEDDATA_TYPE(obj)->function.dmark;
3993 if (mark_func) (*mark_func)(ptr);
3994 }
3995 }
3996
3997 break;
3998 }
3999
4000 case T_OBJECT: {
4001 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
4002 if (rb_shape_embedded_p(shape_id)) {
4003 uint32_t len = RSHAPE_LEN(shape_id);
4004 const VALUE * const ptr = ROBJECT(obj)->as.ary;
4005
4006 for (uint32_t i = 0; i < len; i++) {
4007 gc_mark_internal(ctx, ptr[i]);
4008 }
4009 }
4010 else {
4011 if (!ctx->checking_shareable) {
4012 gc_mark_internal(ctx, ROBJECT(obj)->as.extended);
4013 }
4014 }
4015 break;
4016 }
4017
4018 case T_FILE:
4019 if (RFILE(obj)->fptr) {
4020 gc_mark_internal(ctx, RFILE(obj)->fptr->self);
4021 gc_mark_internal(ctx, RFILE(obj)->fptr->pathv);
4022 gc_mark_internal(ctx, RFILE(obj)->fptr->tied_io_for_writing);
4023 gc_mark_internal(ctx, RFILE(obj)->fptr->writeconv_asciicompat);
4024 gc_mark_internal(ctx, RFILE(obj)->fptr->writeconv_pre_ecopts);
4025 gc_mark_internal(ctx, RFILE(obj)->fptr->encs.ecopts);
4026 gc_mark_internal(ctx, RFILE(obj)->fptr->write_lock);
4027 gc_mark_internal(ctx, RFILE(obj)->fptr->timeout);
4028 gc_mark_internal(ctx, RFILE(obj)->fptr->wakeup_mutex);
4029 }
4030 break;
4031
4032 case T_REGEXP:
4033 gc_mark_internal(ctx, RREGEXP(obj)->src);
4034 break;
4035
4036 case T_MATCH:
4037 gc_mark_internal(ctx, RMATCH(obj)->regexp);
4038 if (RMATCH(obj)->str) {
4039 gc_mark_internal(ctx, RMATCH(obj)->str);
4040 }
4041 break;
4042
4043 case T_RATIONAL:
4044 gc_mark_internal(ctx, RRATIONAL(obj)->num);
4045 gc_mark_internal(ctx, RRATIONAL(obj)->den);
4046 break;
4047
4048 case T_COMPLEX:
4049 gc_mark_internal(ctx, RCOMPLEX(obj)->real);
4050 gc_mark_internal(ctx, RCOMPLEX(obj)->imag);
4051 break;
4052
4053 case T_STRUCT: {
4054 const rb_len_t len = RSTRUCT_LEN(obj);
4055 const VALUE * const ptr = RSTRUCT_CONST_PTR(obj);
4056
4057 for (rb_len_t i = 0; i < len; i++) {
4058 gc_mark_internal(ctx, ptr[i]);
4059 }
4060
4061 gc_mark_internal(ctx, RSTRUCT_FIELDS_OBJ(obj));
4062
4063 break;
4064 }
4065
4066 default:
4067 if (BUILTIN_TYPE(obj) == T_MOVED) rb_bug("rb_gc_mark(): %p is T_MOVED", (void *)obj);
4068 if (BUILTIN_TYPE(obj) == T_NONE) rb_bug("rb_gc_mark(): %p is T_NONE", (void *)obj);
4069 if (BUILTIN_TYPE(obj) == T_ZOMBIE) rb_bug("rb_gc_mark(): %p is T_ZOMBIE", (void *)obj);
4070 rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s",
4071 BUILTIN_TYPE(obj), (void *)obj,
4072 rb_gc_impl_live_object_p(ctx->objspace, (void *)obj) ? "corrupted object" : "non object");
4073 }
4074}
4075
4076void
4077rb_gc_mark_children(void *objspace, VALUE obj)
4078{
4079 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
4080
4081 GC_ASSERT(ctx.objspace == objspace);
4082 gc_mark_children_ctx(&ctx, obj);
4083}
4084
4085size_t
4086rb_gc_obj_optimal_size(VALUE obj)
4087{
4088 switch (BUILTIN_TYPE(obj)) {
4089 case T_ARRAY:
4090 {
4091 size_t size = rb_ary_size_as_embedded(obj);
4092 if (rb_gc_size_allocatable_p(size)) {
4093 return size;
4094 }
4095 else {
4096 return sizeof(struct RArray);
4097 }
4098 }
4099
4100 case T_OBJECT:
4101 if (rb_obj_shape_complex_p(obj)) {
4102 return sizeof(struct RObject);
4103 }
4104 else {
4105 size_t size = rb_obj_embedded_size(RSHAPE_CAPACITY(RBASIC_SHAPE_ID(obj)));
4106 if (rb_gc_size_allocatable_p(size)) {
4107 return size;
4108 }
4109 else {
4110 return sizeof(struct RObject);
4111 }
4112 }
4113
4114 case T_STRING:
4115 {
4116 size_t size = rb_str_size_as_embedded(obj);
4117 if (rb_gc_size_allocatable_p(size)) {
4118 return size;
4119 }
4120 else {
4121 return sizeof(struct RString);
4122 }
4123 }
4124
4125 case T_HASH:
4126 {
4127 if (RHASH_AR_TABLE_P(obj)) {
4128 const unsigned bound = RHASH_AR_TABLE_BOUND(obj);
4129 const size_t ar_size = RHASH_AR_SLOT_SIZE(bound);
4130 if (ar_size > RHASH_ST_SLOT_SIZE || OBJ_FROZEN(obj)) {
4131 return ar_size;
4132 }
4133 }
4134
4135 return RHASH_ST_SLOT_SIZE;
4136 }
4137
4138 default:
4139 return 0;
4140 }
4141}
4142
4143void
4144rb_gc_writebarrier(VALUE a, VALUE b)
4145{
4146 rb_gc_impl_writebarrier(rb_gc_get_objspace(), a, b);
4147}
4148
4149void
4150rb_gc_writebarrier_unprotect(VALUE obj)
4151{
4152 rb_gc_impl_writebarrier_unprotect(rb_gc_get_objspace(), obj);
4153}
4154
4155/*
4156 * remember `obj' if needed.
4157 */
4158void
4159rb_gc_writebarrier_remember(VALUE obj)
4160{
4161 rb_gc_impl_writebarrier_remember(rb_gc_get_objspace(), obj);
4162}
4163
4164/* obj became shareable after it was created (FL_SHAREABLE was set). Tell the GC so it
4165 * updates the per-page shareable bitmap. */
4166void
4167rb_gc_obj_became_shareable(VALUE obj)
4168{
4169 rb_gc_impl_obj_became_shareable(rb_gc_get_objspace(), obj);
4170}
4171
4172/* Pin an in-flight message payload in its owner's (the sender's) objspace, so the
4173 * sender's local GC keeps it alive while it sits in a queue the sender does not walk. */
4174void
4175rb_gc_copy_attributes(VALUE dest, VALUE obj)
4176{
4177 rb_gc_impl_copy_attributes(rb_gc_get_objspace(), dest, obj);
4178}
4179
4180#if USE_MODULAR_GC
4181int
4182rb_gc_modular_gc_loaded_p(void)
4183{
4184 return rb_gc_functions.modular_gc_loaded_p;
4185}
4186
4187const char *
4188rb_gc_active_gc_name(void)
4189{
4190 const char *gc_name = rb_gc_impl_active_gc_name();
4191
4192 const size_t len = strlen(gc_name);
4193 if (len > RB_GC_MAX_NAME_LEN) {
4194 rb_bug("GC should have a name no more than %d chars long. Currently: %zu (%s)",
4195 RB_GC_MAX_NAME_LEN, len, gc_name);
4196 }
4197
4198 return gc_name;
4199}
4200#endif
4201
4203rb_gc_object_metadata(VALUE obj)
4204{
4205 return rb_gc_impl_object_metadata(rb_gc_get_objspace(), obj);
4206}
4207
4208/* GC */
4209
4210void *
4211rb_gc_ractor_cache_alloc(rb_ractor_t *ractor)
4212{
4213 return rb_gc_impl_ractor_cache_alloc(rb_gc_get_objspace(), ractor);
4214}
4215
4216void
4217rb_gc_ractor_cache_free(void *cache)
4218{
4219 rb_gc_impl_ractor_cache_free(rb_gc_get_objspace(), cache);
4220}
4221
4222bool
4223rb_gc_zjit_new_obj_fastpath(size_t alloc_size, VALUE flags, VALUE klass, struct rb_gc_zjit_fastpath *fastpath)
4224{
4225#if defined(RUBY_ASAN_ENABLED)
4226 (void)rb_gc_impl_zjit_new_obj_fastpath;
4227 return false;
4228#else
4229 return rb_gc_impl_zjit_new_obj_fastpath(rb_gc_get_objspace(), alloc_size, flags, klass, fastpath);
4230#endif
4231}
4232
4233int32_t
4234rb_gc_zjit_writebarrier_required_offset(void)
4235{
4236 return rb_gc_impl_zjit_writebarrier_required_offset();
4237}
4238
4239/*
4240 * NOTE: `obj` must be from the current objspace.
4241 */
4242void
4243rb_gc_register_mark_object(VALUE obj)
4244{
4245 if (rb_gc_impl_live_object_p(rb_gc_get_objspace(), (void *)obj)) {
4246 rb_vm_register_global_object(obj);
4247 }
4248}
4249
4250static rb_ractor_t *
4251gc_registered_addrs_owner(rb_vm_t *vm)
4252{
4253 rb_ractor_t *cr = rb_current_ractor_raw(false);
4254 if (cr) return cr;
4255 RUBY_ASSERT(vm->ractor.main_ractor != NULL);
4256 return vm->ractor.main_ractor;
4257}
4258
4259/* Keep track of every ractor that owns registered addresses, so that a
4260 * rb_gc_unregister_address() call from a non-owning ractor can find the list
4261 * holding the address, and so GC can iterate every live registration list
4262 * without walking the whole ractor set. */
4263void
4264rb_gc_registered_addrs_enroll_without_gc(rb_vm_t *vm, rb_ractor_t *r)
4265{
4266 if (r->registered_addrs_listed) return;
4267 if (vm->gc.registered_addrs.registry_cnt == vm->gc.registered_addrs.registry_capa) {
4268 size_t nc = vm->gc.registered_addrs.registry_capa ? vm->gc.registered_addrs.registry_capa * 2 : 16;
4269 struct rb_ractor_struct **p = realloc(vm->gc.registered_addrs.registry,
4270 nc * sizeof(struct rb_ractor_struct *));
4271 if (!p) rb_bug("rb_gc_registered_addrs_enroll_without_gc: out of memory");
4272 vm->gc.registered_addrs.registry = p;
4273 vm->gc.registered_addrs.registry_capa = nc;
4274 }
4275 vm->gc.registered_addrs.registry[vm->gc.registered_addrs.registry_cnt++] = r;
4276 r->registered_addrs_listed = true;
4277}
4278
4279void
4280rb_gc_registered_addrs_unenroll_without_gc(rb_vm_t *vm, rb_ractor_t *r)
4281{
4282 if (!r->registered_addrs_listed) return;
4283 for (size_t i = 0; i < vm->gc.registered_addrs.registry_cnt; i++) {
4284 if (vm->gc.registered_addrs.registry[i] == r) {
4285 vm->gc.registered_addrs.registry[i] =
4286 vm->gc.registered_addrs.registry[--vm->gc.registered_addrs.registry_cnt];
4287 break;
4288 }
4289 }
4290 r->registered_addrs_listed = false;
4291}
4292
4293void
4294rb_gc_each_registered_addr(rb_gc_registered_addr_cb func, void *data)
4295{
4296#if !RB_GC_REGISTERED_ADDR_CHECK
4297 /* Production entries carry no provenance, so the verifier has nothing to check. */
4298 (void)func;
4299 (void)data;
4300 return;
4301#else
4302 rb_vm_t *vm = GET_VM();
4303 for (size_t i = 0; i < vm->gc.registered_addrs.registry_cnt; i++) {
4304 rb_ractor_t *r = vm->gc.registered_addrs.registry[i];
4305 for (size_t j = 0; j < r->registered_addrs_cnt; j++) {
4306 struct rb_ractor_registered_addr *entry = &r->registered_addrs[j];
4307 func(entry->addr, entry->initial_value, (void *)r->objspace, data);
4308 }
4309 }
4310#endif
4311}
4312
4313/* True if the same address is registered by a Ractor whose objspace is the given one;
4314 * that registrant's local GC can root the value, so the verifier accepts it. Called
4315 * from the consistency verifier with the world stopped, so no lock is taken here. */
4316bool
4317rb_gc_registered_addr_owned_by_registrant_p(VALUE *addr, void *objspace)
4318{
4319 rb_vm_t *vm = GET_VM();
4320 for (size_t i = 0; i < vm->gc.registered_addrs.registry_cnt; i++) {
4321 rb_ractor_t *r = vm->gc.registered_addrs.registry[i];
4322 if ((void *)r->objspace != objspace) continue;
4323 for (size_t j = 0; j < r->registered_addrs_cnt; j++) {
4324 if (r->registered_addrs[j].addr == addr) return true;
4325 }
4326 }
4327 return false;
4328}
4329
4330/* obj is on a page of a zombie objspace: its Ractor terminated and nobody absorbed it
4331 * yet, so no thread owns the page. */
4332bool
4333rb_gc_obj_in_zombie_objspace_p(VALUE obj)
4334{
4335 ASSERT_vm_locking();
4336 rb_vm_t *vm = GET_VM();
4337 for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) {
4338 if (!rb_gc_impl_obj_foreign_p(vm->gc.zombie_objspaces[i].objspace, obj)) return true;
4339 }
4340 return false;
4341}
4342
4343bool
4344rb_gc_obj_live_in_zombie_objspace_p(VALUE obj)
4345{
4346 ASSERT_vm_locking();
4347 rb_vm_t *vm = GET_VM();
4348 for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) {
4349 if (rb_gc_impl_live_object_p(vm->gc.zombie_objspaces[i].objspace, (const void *)obj)) return true;
4350 }
4351 return false;
4352}
4353
4354bool
4355rb_gc_obj_in_objspace_p(void *objspace, VALUE obj)
4356{
4357 return !rb_gc_impl_obj_foreign_p(objspace, obj);
4358}
4359
4360static bool
4361gc_registered_addrs_remove(rb_ractor_t *r, VALUE *addr)
4362{
4363 for (size_t i = 0; i < r->registered_addrs_cnt; i++) {
4364 if (r->registered_addrs[i].addr == addr) {
4365 MEMMOVE(&r->registered_addrs[i], &r->registered_addrs[i + 1],
4366 struct rb_ractor_registered_addr, r->registered_addrs_cnt - i - 1);
4367 r->registered_addrs_cnt--;
4368 return true;
4369 }
4370 }
4371 return false;
4372}
4373
4374void
4375rb_gc_register_address(VALUE *addr)
4376{
4377 rb_vm_t *vm = GET_VM();
4378
4379 rb_native_mutex_lock(&vm->gc.registered_addrs.lock);
4380 rb_ractor_t *owner = gc_registered_addrs_owner(vm);
4381 if (owner->registered_addrs_cnt == owner->registered_addrs_capa) {
4382 size_t nc = owner->registered_addrs_capa ? owner->registered_addrs_capa * 2 : 64;
4383 struct rb_ractor_registered_addr *p =
4384 realloc(owner->registered_addrs, nc * sizeof(struct rb_ractor_registered_addr));
4385 if (!p) rb_bug("rb_gc_register_address: out of memory");
4386 owner->registered_addrs = p;
4387 owner->registered_addrs_capa = nc;
4388 }
4389 struct rb_ractor_registered_addr *entry = &owner->registered_addrs[owner->registered_addrs_cnt];
4390 entry->addr = addr;
4391#if RB_GC_REGISTERED_ADDR_CHECK
4392 entry->initial_value = *addr;
4393#endif
4394 owner->registered_addrs_cnt++;
4395 rb_gc_registered_addrs_enroll_without_gc(vm, owner);
4396 rb_native_mutex_unlock(&vm->gc.registered_addrs.lock);
4397
4398 /* Some C extensions register before assigning, so protect obj from GC here. */
4399 RB_GC_GUARD(*addr);
4400}
4401
4402void
4403rb_gc_unregister_address(VALUE *addr)
4404{
4405 rb_vm_t *vm = GET_VM();
4406
4407 rb_native_mutex_lock(&vm->gc.registered_addrs.lock);
4408 rb_ractor_t *cr = rb_current_ractor_raw(false);
4409 if (cr == NULL) cr = vm->ractor.main_ractor;
4410 if (cr && gc_registered_addrs_remove(cr, addr)) goto done;
4411 for (size_t i = 0; i < vm->gc.registered_addrs.registry_cnt; i++) {
4412 if (vm->gc.registered_addrs.registry[i] != cr &&
4413 gc_registered_addrs_remove(vm->gc.registered_addrs.registry[i], addr)) {
4414 break;
4415 }
4416 }
4417 done:
4418 rb_native_mutex_unlock(&vm->gc.registered_addrs.lock);
4419}
4420
4421void
4423{
4424 rb_gc_register_address(var);
4425}
4426
4427static VALUE
4428gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE immediate_mark, VALUE immediate_sweep, VALUE compact, VALUE global)
4429{
4430 rb_gc_impl_start(rb_gc_get_objspace(), RTEST(full_mark), RTEST(immediate_mark), RTEST(immediate_sweep), RTEST(compact), RTEST(global));
4431
4432 return Qnil;
4433}
4434
4436 void *self;
4437 int (*callback)(void *, void *, size_t, void *);
4438 void *data;
4439};
4440
4441static void
4442each_objects_foreign_i(void *objspace, void *arg)
4443{
4444 struct each_objects_foreign_arg *a = (struct each_objects_foreign_arg *)arg;
4445 if (objspace == a->self) return;
4446 rb_gc_impl_each_objects_foreign(objspace, a->callback, a->data);
4447}
4448
4449/*
4450 * rb_objspace_each_objects() is special C API to walk through
4451 * Ruby object space. This C API is too difficult to use it.
4452 * To be frank, you should not use it. Or you need to read the
4453 * source code of this function and understand what this function does.
4454 *
4455 * 'callback' will be called several times (the number of heap page,
4456 * at current implementation) with:
4457 * vstart: a pointer to the first living object of the heap_page.
4458 * vend: a pointer to next to the valid heap_page area.
4459 * stride: a distance to next VALUE.
4460 *
4461 * If callback() returns non-zero, the iteration will be stopped.
4462 *
4463 * This takes the VM barrier for the whole walk, stopping every other
4464 * Ractor: the set of heap pages must not change under the callback, and a
4465 * GC is stop-the-world. Because of that, the callback must not wait on
4466 * another Ractor (e.g. send/receive) -- they are all suspended and it
4467 * would deadlock.
4468 *
4469 * This is a sample callback code to iterate liveness objects:
4470 *
4471 * static int
4472 * sample_callback(void *vstart, void *vend, int stride, void *data)
4473 * {
4474 * VALUE v = (VALUE)vstart;
4475 * for (; v != (VALUE)vend; v += stride) {
4476 * if (!rb_objspace_internal_object_p(v)) { // liveness check
4477 * // do something with live object 'v'
4478 * }
4479 * }
4480 * return 0; // continue to iteration
4481 * }
4482 *
4483 * Note: 'vstart' is not a top of heap_page. This point the first
4484 * living object to grasp at least one object to avoid GC issue.
4485 * This means that you can not walk through all Ruby object page
4486 * including freed object page.
4487 *
4488 * Note: On this implementation, 'stride' is the same as sizeof(RVALUE).
4489 * However, there are possibilities to pass variable values with
4490 * 'stride' with some reasons. You must use stride instead of
4491 * use some constant value in the iteration.
4492 */
4493void
4494rb_objspace_each_objects(int (*callback)(void *, void *, size_t, void *), void *data)
4495{
4496 RB_VM_LOCKING() {
4497 rb_vm_barrier();
4498
4499 void *self = rb_gc_get_objspace();
4500 rb_gc_impl_each_objects(self, callback, data);
4501
4502 /* Like upstream, cover every object in the process: walk the other live
4503 * Ractors' objspaces too, under the VM lock and barrier, with a pure-C callback.
4504 * A foreign objspace's stopped lazy sweep is not settled; the walk skips its
4505 * dead objects. Also covers zombie objspaces. */
4506 struct each_objects_foreign_arg arg = { self, callback, data };
4507 rb_gc_vm_each_objspace(each_objects_foreign_i, &arg);
4508 }
4509}
4510
4511/* Enumerate live, creating, and zombie objspaces under the VM lifetime lock.
4512 * Reading foreign mutable collector state also needs the barrier; independently
4513 * synchronized publications may be read under their own locks without a barrier. */
4514void
4515rb_gc_vm_each_objspace(void (*func)(void *objspace, void *data), void *data)
4516{
4517 ASSERT_vm_locking();
4518
4519 rb_vm_t *vm = GET_VM();
4520 rb_ractor_t *r;
4521 ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
4522 if (r->objspace) {
4523 func(r->objspace, data);
4524 }
4525 /* A child being created is not in the set yet but its objspace already holds
4526 * the Thread/Fiber wrappers; enumerate it through its creator so a global GC
4527 * cannot miss it and mark into an objspace it never cleared. */
4528 if (r->creating_child_objspace) {
4529 func(r->creating_child_objspace, data);
4530 }
4531 }
4532 for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) {
4533 func(vm->gc.zombie_objspaces[i].objspace, data);
4534 }
4535}
4536
4537/* Merging an ownerless zombie objspace (its Ractor object was collected) into main
4538 * runs as a postponed job targeted at main, at main's next safepoint; never inside
4539 * the GC cycle that discovered the orphan. */
4540
4541static void gc_orphan_merge_job(void *unused);
4542
4543/* Grown with plain realloc: rb_gc_objspace_disown pushes from inside a global GC
4544 * sweep, where the accounting allocator is not allowed. This table is VM-lifetime
4545 * metadata with at most a few dozen entries. */
4546static void
4547zombie_objspaces_push(rb_vm_t *vm, void *objspace, void **owner_slot, struct rb_ractor_struct *owner)
4548{
4549 ASSERT_vm_locking();
4550 if (vm->gc.zombie_objspaces_count == vm->gc.zombie_objspaces_capa) {
4551 size_t new_capa = vm->gc.zombie_objspaces_capa ? vm->gc.zombie_objspaces_capa * 2 : 16;
4552 struct rb_objspace_zombie *grown =
4553 realloc(vm->gc.zombie_objspaces, new_capa * sizeof(struct rb_objspace_zombie));
4554 if (grown == NULL) rb_bug("zombie_objspaces_push: out of memory");
4555 vm->gc.zombie_objspaces = grown;
4556 vm->gc.zombie_objspaces_capa = new_capa;
4557 }
4558 vm->gc.zombie_objspaces[vm->gc.zombie_objspaces_count++] = (struct rb_objspace_zombie){
4559 .objspace = objspace,
4560 .owner_slot = owner_slot,
4561 .owner = owner,
4562 };
4563 rb_gc_impl_objspace_zombify(objspace);
4564}
4565
4566/* Called for a Ractor that terminated without being joined. Its objspace loses its
4567 * owning thread, but its pages still hold shareable objects other Ractors can reach,
4568 * so keep it enumerable until inheritance merges it. The owning r->objspace slot stays
4569 * until the inheriting path takes the objspace and clears it. */
4570/* Reserve the handle of the orphan-merge job if it is not registered yet. Shared by
4571 * every retire and disown path; a second preregister is idempotent (the same func and
4572 * data are deduplicated). */
4573static void
4574gc_orphan_merge_pjob_ensure(void)
4575{
4576 if (GET_VM()->gc.orphan_merge_pjob == POSTPONED_JOB_HANDLE_INVALID) {
4577 GET_VM()->gc.orphan_merge_pjob = rb_postponed_job_preregister(0, gc_orphan_merge_job, NULL);
4578 if (GET_VM()->gc.orphan_merge_pjob == POSTPONED_JOB_HANDLE_INVALID) {
4579 rb_bug("Could not preregister postponed job for GC");
4580 }
4581 }
4582}
4583
4584/* A terminating Ractor runs the last local GC of its own objspace; own thread only. */
4585void
4586rb_gc_objspace_retire_gc(void)
4587{
4588 rb_gc_impl_objspace_retire_gc(rb_gc_get_objspace());
4589}
4590
4591void
4592rb_gc_objspace_retire(void **objspace_slot)
4593{
4594 rb_vm_t *vm = GET_VM();
4595
4596 if (!rb_gc_impl_multi_objspace_p()) {
4597 /* It only aliased the shared objspace, so just drop it. */
4598 *objspace_slot = NULL;
4599 return;
4600 }
4601
4602 /* Return the hold if the Ractor exits with GC disabled: otherwise nobody can
4603 * enable it again and GC stays off. */
4604 if (rb_gc_impl_user_gc_disabled_set(*objspace_slot, false)) {
4605 RUBY_ATOMIC_DEC(vm->gc.disable_holders);
4606 }
4607
4608 RB_VM_LOCKING() {
4609 rb_gc_objspace_set_ractor(*objspace_slot, NULL);
4610 gc_orphan_merge_pjob_ensure();
4611 /* owner_slot is always &r->objspace of the retiring Ractor. owner is recorded so a
4612 * root scan can still reach the dead Ractor's registered_marks pins and its join
4613 * value; rb_gc_objspace_disown clears it when the zombie becomes an orphan. */
4614 struct rb_ractor_struct *owner =
4615 (struct rb_ractor_struct *)((char *)objspace_slot - offsetof(rb_ractor_t, objspace));
4616 zombie_objspaces_push(vm, *objspace_slot, objspace_slot, owner);
4617 }
4618}
4619
4620/* The owning Ractor object was collected, so nobody can join any more: drop the owner
4621 * slot in zombie_objspaces and hand the merge to main. Called from ractor_free (inside
4622 * a sweep), where the accounting allocator is unavailable; the table itself is stable. */
4623void
4624rb_gc_objspace_disown(void *objspace)
4625{
4626 if (!rb_gc_impl_multi_objspace_p()) return;
4627 ASSERT_vm_locking();
4628 rb_vm_t *vm = GET_VM();
4629 bool found = false;
4630 rb_gc_objspace_set_ractor(objspace, NULL);
4631
4632 for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) {
4633 if (vm->gc.zombie_objspaces[i].objspace == objspace) {
4634 vm->gc.zombie_objspaces[i].owner_slot = NULL;
4635 /* The Ractor struct is being freed, so drop owner too: nothing may read its
4636 * registered_marks or join value after this. */
4637 vm->gc.zombie_objspaces[i].owner = NULL;
4638 found = true;
4639 break;
4640 }
4641 }
4642 if (!found) {
4643 zombie_objspaces_push(vm, objspace, NULL, NULL);
4644 }
4645
4646 /* The trigger is wait-free (an atomic bit plus an interrupt flag), so it is safe
4647 * inside a sweep, and it also covers a Ractor that never started. */
4648 gc_orphan_merge_pjob_ensure();
4649 rb_postponed_job_trigger_for_ractor(GET_VM()->gc.orphan_merge_pjob, vm->ractor.main_ractor->pub.self);
4650}
4651
4652/* Is a global (stop-the-world) GC cycle running? Only its driver runs during one, so
4653 * asking through the current objspace is exact. */
4654bool
4655rb_gc_during_global_gc_p(void)
4656{
4657 return rb_gc_impl_during_global_gc_p(rb_gc_get_objspace());
4658}
4659
4660static void
4661rb_gc_vm_forget_zombie(void *objspace)
4662{
4663 ASSERT_vm_locking();
4664 rb_vm_t *vm = GET_VM();
4665 size_t n = vm->gc.zombie_objspaces_count;
4666 for (size_t i = 0; i < n; i++) {
4667 if (vm->gc.zombie_objspaces[i].objspace == objspace) {
4668 vm->gc.zombie_objspaces[i] = vm->gc.zombie_objspaces[n - 1];
4669 vm->gc.zombie_objspaces_count = n - 1;
4670 break;
4671 }
4672 }
4673}
4674
4675/* Number of live Ractors, for the heap growth heuristic (r_mul); a racy read is fine. */
4676unsigned int
4677rb_gc_vm_ractor_count(void)
4678{
4679 return GET_VM()->ractor.cnt;
4680}
4681
4682void
4683rb_gc_rest(void)
4684{
4685 // Lock to keep assertions happy. This runs right after single-ractor mode is
4686 // cancelled, but we can still free shareables like fstrings because ractor.cnt is 1.
4687 RB_VM_LOCKING() {
4688 rb_gc_impl_gc_rest(rb_gc_get_objspace());
4689 }
4690}
4691
4692/* True while a zombie is being absorbed. The zombie's count is decremented before the
4693 * merge (see absorb below), so in that window its live objects still exist even though
4694 * the process looks single-objspace. */
4695static int gc_absorbing_zombie = 0;
4696
4697/* True once a zombie objspace was absorbed since the last global GC: until the unified
4698 * mark runs, a single-objspace local mark can miss absorbed shareable objects (a cc in
4699 * a class's cc_table, say), so stop treating the process as single until then. */
4700static bool gc_absorbed_since_global_gc = false;
4701
4702void
4703rb_gc_reset_absorbed_since_global_gc(void)
4704{
4705 gc_absorbed_since_global_gc = false;
4706}
4707
4708/* True when the process holds exactly one objspace (one live Ractor, no zombies) and
4709 * nothing was absorbed since the last global GC. Only then is a local GC the whole
4710 * world and the multi-objspace guards can be skipped. The child-creation window (the
4711 * child objspace exists while cnt is still 1) and both absorb windows, during (count
4712 * already decremented, merge unfinished) and after (merged, next global GC pending) --
4713 * count as multi: treating them as single would let a GC skip guards such as shareable
4714 * pinning and collect a live cc. */
4715/* False when the impl only supports one objspace (mmtk and friends); the VM then makes
4716 * its per-Ractor objspace machinery (retire, absorb, creation cover) a no-op. */
4717bool
4718rb_gc_multi_objspace_p(void)
4719{
4720 return rb_gc_impl_multi_objspace_p();
4721}
4722
4723/* Does obj belong to another Ractor's objspace rather than the current one? Always
4724 * false for a single-objspace impl, which cannot tell owners apart. */
4725bool
4726rb_gc_obj_foreign_p(VALUE obj)
4727{
4728 return rb_gc_impl_obj_foreign_p(rb_gc_get_objspace(), obj);
4729}
4730
4731// r runs on objspace; NULL once objspace is a zombie. Under the VM lock, except at creation.
4732void
4733rb_gc_objspace_set_ractor(void *objspace, rb_ractor_t *r)
4734{
4735 rb_gc_impl_objspace_set_ractor(objspace, r);
4736}
4737
4738// the living Ractor whose objspace holds obj, or NULL (a zombie's, or a single objspace); under the VM lock
4739rb_ractor_t *
4740rb_gc_obj_ractor(VALUE obj)
4741{
4742 return rb_gc_impl_obj_ractor(rb_gc_get_objspace(), obj);
4743}
4744
4745bool
4746rb_gc_single_objspace_p(void)
4747{
4748 if (!rb_gc_impl_multi_objspace_p()) return true;
4749 rb_vm_t *vm = GET_VM();
4750 /* One Ractor is not one objspace: a forked child re-enters single-Ractor mode while
4751 * the pre-fork Ractors' objspaces are still parked in zombie_objspaces. */
4752 return (ruby_single_main_ractor != NULL || vm->ractor.cnt == 1) &&
4753 vm->gc.zombie_objspaces_count == 0 && gc_absorbing_zombie == 0 &&
4754 !gc_absorbed_since_global_gc &&
4755 (vm->ractor.main_ractor == NULL ||
4756 vm->ractor.main_ractor->creating_child_objspace == NULL);
4757}
4758
4759/* Inherit a dead Ractor's objspace into the calling Ractor. Going through the owner
4760 * slot clears it and releases the objspace in one VM-lock section; the merge runs with
4761 * the inheritor's GC disabled (moving the finalizer st table could trigger it). */
4762static void
4763objspace_absorb_merge(void *dst, void *src)
4764{
4765 ASSERT_vm_locking();
4766 rb_gc_impl_objspace_absorb(dst, src);
4767 gc_absorbed_since_global_gc = true;
4768}
4769
4770/* The dying thread's last collection of its own objspace, GVL still held; with
4771 * r->postmortem set, rb_ractor_mark_local_roots roots only the join value and the
4772 * registered_marks pins, so the scaffolding nobody needs any more dies here. */
4773void
4774rb_gc_objspace_postmortem_self(void)
4775{
4776 if (!rb_gc_impl_multi_objspace_p()) return;
4777
4778 rb_gc_impl_objspace_retire_gc(rb_gc_get_objspace());
4779}
4780
4781void
4782rb_gc_objspace_absorb_into_current(void **objspace_slot)
4783{
4784 if (!rb_gc_impl_multi_objspace_p()) {
4785 *objspace_slot = NULL;
4786 return;
4787 }
4788 RB_VM_LOCKING() {
4789 void *objspace = *objspace_slot;
4790 if (objspace != NULL) {
4791 *objspace_slot = NULL;
4792 gc_absorbing_zombie++;
4793 rb_gc_vm_forget_zombie(objspace);
4794 objspace_absorb_merge(rb_gc_get_objspace(), objspace);
4795 gc_absorbing_zombie--;
4796 }
4797 }
4798}
4799
4800/* Merge every ownerless zombie objspace (no owner slot, i.e. the Ractor object was
4801 * collected) into the current Ractor's objspace. Runs as a postponed job on the main
4802 * Ractor's thread; the VM teardown path calls it directly. */
4803static void
4804objspace_absorb_disowned_zombies(void)
4805{
4806 rb_vm_t *vm = GET_VM();
4807
4808 RB_VM_LOCKING() {
4809 size_t i = 0;
4810 while (i < vm->gc.zombie_objspaces_count) {
4811 if (vm->gc.zombie_objspaces[i].owner_slot == NULL) {
4812 void *zombie = vm->gc.zombie_objspaces[i].objspace;
4813 /* Drop the entry before the merge, in this same VM-lock section:
4814 * objspace_absorb expects src to be out of the table. */
4815 gc_absorbing_zombie++;
4816 rb_gc_vm_forget_zombie(zombie);
4817 objspace_absorb_merge(rb_gc_get_objspace(), zombie);
4818 gc_absorbing_zombie--;
4819 }
4820 else {
4821 i++;
4822 }
4823 }
4824 }
4825}
4826
4827static void
4828gc_orphan_merge_job(void *unused)
4829{
4830 (void)unused;
4831 objspace_absorb_disowned_zombies();
4832}
4833
4834/* Re-target a pending orphan merge after fork. The job may target the parent's main
4835 * Ractor, whose per-Ractor trigger mask is not inherited unless that Ractor forked.
4836 * Called on the child side. */
4837/* Only main survives a fork, so rebuild the counter from main's own hold alone. */
4838void
4839rb_gc_disable_holders_atfork(void)
4840{
4841 RUBY_ATOMIC_SET(GET_VM()->gc.disable_holders,
4842 rb_gc_impl_user_gc_disabled_p(rb_gc_get_objspace()) ? 1 : 0);
4843}
4844
4845void
4846rb_gc_zombie_objspaces_atfork(void)
4847{
4848 rb_vm_t *vm = GET_VM();
4849
4850 for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) {
4851 if (vm->gc.zombie_objspaces[i].owner_slot == NULL) {
4852 rb_postponed_job_trigger_for_ractor(GET_VM()->gc.orphan_merge_pjob, vm->ractor.main_ractor->pub.self);
4853 break;
4854 }
4855 }
4856}
4857
4858/* VM teardown, right after every other Ractor was killed: merge all uninherited
4859 * objspaces into main so at-exit processing covers every object and dead Ractors'
4860 * deferred finalizers run on main. The owner slot also covers collected wrappers. */
4861void
4862rb_gc_objspace_absorb_all_zombies(void)
4863{
4864 rb_vm_t *vm = GET_VM();
4865
4866 /* Entries whose Ractor object is already gone, i.e. the pending merge job itself,
4867 * which we run synchronously here. */
4868 objspace_absorb_disowned_zombies();
4869
4870 while (vm->gc.zombie_objspaces_count > 0) {
4871 size_t before = vm->gc.zombie_objspaces_count;
4872 GC_ASSERT(vm->gc.zombie_objspaces[0].owner_slot != NULL);
4873 /* Move the rb_gc_register_mark_object pins before the merge, so the objects
4874 * pinned in the owner's objspace do not lose their root in its sweep. */
4875 rb_ractor_t *owner = vm->gc.zombie_objspaces[0].owner;
4876 if (owner) {
4877 rb_ractor_absorb_registered_marks(GET_RACTOR(), owner);
4878 rb_ractor_absorb_registered_addrs_without_gc(GET_RACTOR(), owner);
4879 }
4880 rb_gc_objspace_absorb_into_current(vm->gc.zombie_objspaces[0].owner_slot);
4881 if (vm->gc.zombie_objspaces_count >= before) {
4882 rb_bug("rb_gc_objspace_absorb_all_zombies: zombie list did not shrink");
4883 }
4884 }
4885}
4886
4887static void
4888gc_ref_update_array(void *objspace, VALUE v)
4889{
4890 if (ARY_SHARED_P(v)) {
4891 VALUE old_root = RARRAY(v)->as.heap.aux.shared_root;
4892
4893 UPDATE_IF_MOVED(objspace, RARRAY(v)->as.heap.aux.shared_root);
4894
4895 VALUE new_root = RARRAY(v)->as.heap.aux.shared_root;
4896 // A root hollowed out by a move is no longer an array, and it is pinned rather
4897 // than re-pointed (see the marking of a shared root).
4898 // If the root is embedded and its location has changed
4899 if (RB_TYPE_P(new_root, T_ARRAY) && ARY_EMBED_P(new_root) && new_root != old_root) {
4900 size_t offset = (size_t)(RARRAY(v)->as.heap.ptr - RARRAY(old_root)->as.ary);
4901 GC_ASSERT(RARRAY(v)->as.heap.ptr >= RARRAY(old_root)->as.ary);
4902 RARRAY(v)->as.heap.ptr = RARRAY(new_root)->as.ary + offset;
4903 }
4904 }
4905 else {
4906 rb_len_t len = RARRAY_LEN(v);
4907
4908 if (len > 0) {
4909 VALUE *ptr = (VALUE *)RARRAY_CONST_PTR(v);
4910 for (rb_len_t i = 0; i < len; i++) {
4911 UPDATE_IF_MOVED(objspace, ptr[i]);
4912 }
4913 }
4914
4915 if (!ARY_EMBED_P(v) && rb_gc_obj_slot_size(v) >= rb_ary_size_as_embedded(v)) {
4916 /* Skip pinned arrays: a pinned array may be referenced from a
4917 * conservative root holding RARRAY_PTR across this compaction, so
4918 * freeing its heap buffer here would dangle that pointer. */
4919 if (rb_ary_embeddable_p(v) && !rb_gc_impl_pinned_p(objspace, v)) {
4920 rb_ary_make_embedded(v);
4921 }
4922 }
4923 }
4924}
4925
4926static void
4927gc_ref_update_object(void *objspace, VALUE v)
4928{
4929 RUBY_ASSERT(rb_gc_obj_slot_size(v) == rb_obj_shape_slot_size(v));
4930 shape_id_t shape_id = RBASIC_SHAPE_ID(v);
4931
4932 if (!rb_shape_embedded_p(shape_id)) {
4933 UPDATE_IF_MOVED(objspace, ROBJECT(v)->as.extended);
4934
4935 if (!rb_shape_complex_p(shape_id) && rb_shape_embedded_capacity(shape_id) >= RSHAPE_LEN(shape_id)) {
4936 VALUE *embedded_fields = ROBJECT_EMBEDDED_FIELDS(v);
4937 VALUE *extended_fields = ROBJECT_FIELDS(v);
4938 MEMCPY(embedded_fields, extended_fields, VALUE, RSHAPE_LEN(shape_id));
4939 shape_id = rb_shape_transition_robject(shape_id);
4940 RBASIC_SET_FULL_SHAPE_ID(v, shape_id);
4941 rb_gc_writebarrier_remember(v);
4942 }
4943 else {
4944 return;
4945 }
4946 }
4947
4948 VALUE *ptr = ROBJECT_FIELDS(v);
4949 attr_index_t len = RSHAPE_LEN(shape_id);
4950 for (attr_index_t i = 0; i < len; i++) {
4951 UPDATE_IF_MOVED(objspace, ptr[i]);
4952 }
4953}
4954
4955void
4956rb_gc_ref_update_table_values_only(st_table *tbl)
4957{
4958 gc_ref_update_table_values_only(tbl);
4959}
4960
4961/* Update MOVED references in a VALUE=>VALUE st_table */
4962void
4963rb_gc_update_tbl_refs(st_table *ptr)
4964{
4965 gc_update_table_refs(ptr);
4966}
4967
4968static int
4969rb_gc_update_set_refs_i(st_data_t key, st_data_t value, st_data_t argp, int error)
4970{
4971 if (rb_gc_location((VALUE)key) != (VALUE)key) {
4972 return ST_REPLACE;
4973 }
4974
4975 return ST_CONTINUE;
4976}
4977
4978static int
4979rb_gc_update_set_refs_replace_i(st_data_t *key, st_data_t *value, st_data_t argp, int existing)
4980{
4981 rb_gc_update_moved((VALUE *)key);
4982
4983 return ST_CONTINUE;
4984}
4985
4986void
4987rb_gc_update_set_refs(st_table *tbl)
4988{
4989 if (!tbl || tbl->num_entries == 0) return;
4990
4991 if (st_foreach_with_replace(tbl, rb_gc_update_set_refs_i, rb_gc_update_set_refs_replace_i, 0)) {
4992 rb_raise(rb_eRuntimeError, "hash modified during iteration");
4993 }
4994}
4995
4996static void
4997gc_ref_update_hash(void *objspace, VALUE v)
4998{
4999 rb_hash_stlike_foreach_with_replace(v, hash_foreach_replace, hash_replace_ref, (st_data_t)objspace);
5000}
5001
5002static void
5003gc_update_values(void *objspace, long n, VALUE *values)
5004{
5005 for (long i = 0; i < n; i++) {
5006 UPDATE_IF_MOVED(objspace, values[i]);
5007 }
5008}
5009
5010void
5011rb_gc_update_values(long n, VALUE *values)
5012{
5013 gc_update_values(rb_gc_get_objspace(), n, values);
5014}
5015
5016static enum rb_id_table_iterator_result
5017check_id_table_move(VALUE value, void *data)
5018{
5019 void *objspace = (void *)data;
5020
5021 if (gc_object_moved_p_internal(objspace, (VALUE)value)) {
5022 return ID_TABLE_REPLACE;
5023 }
5024
5025 return ID_TABLE_CONTINUE;
5026}
5027
5028void
5029rb_gc_prepare_heap_process_object(VALUE obj)
5030{
5031 switch (BUILTIN_TYPE(obj)) {
5032 case T_STRING:
5033 // Precompute the string coderange. This both save time for when it will be
5034 // eventually needed, and avoid mutating heap pages after a potential fork.
5035 rb_enc_str_coderange(obj);
5036 break;
5037 default:
5038 break;
5039 }
5040}
5041
5042void
5043rb_gc_prepare_heap(void)
5044{
5045 rb_gc_impl_prepare_heap(rb_gc_get_objspace());
5046}
5047
5048size_t
5049rb_gc_size_slot_size(size_t size)
5050{
5051 return rb_gc_impl_size_slot_size(rb_gc_get_objspace(), size);
5052}
5053
5054bool
5055rb_gc_size_allocatable_p(size_t size)
5056{
5057 return rb_gc_impl_size_allocatable_p(size);
5058}
5059
5060size_t
5061rb_gc_max_allocation_size(void)
5062{
5063 return rb_gc_impl_max_allocation_size();
5064}
5065
5066static enum rb_id_table_iterator_result
5067update_id_table(VALUE *value, void *data, int existing)
5068{
5069 void *objspace = (void *)data;
5070
5071 if (gc_object_moved_p_internal(objspace, (VALUE)*value)) {
5072 *value = gc_location_internal(objspace, (VALUE)*value);
5073 }
5074
5075 return ID_TABLE_CONTINUE;
5076}
5077
5078static void
5079update_m_tbl(void *objspace, struct rb_id_table *tbl)
5080{
5081 if (tbl) {
5082 rb_id_table_foreach_values_with_replace(tbl, check_id_table_move, update_id_table, objspace);
5083 }
5084}
5085
5086static enum rb_id_table_iterator_result
5087update_const_tbl_i(VALUE value, void *objspace)
5088{
5089 rb_const_entry_t *ce = (rb_const_entry_t *)value;
5090
5091 if (gc_object_moved_p_internal(objspace, ce->value)) {
5092 ce->value = gc_location_internal(objspace, ce->value);
5093 }
5094
5095 if (gc_object_moved_p_internal(objspace, ce->u.file)) {
5096 ce->u.file = gc_location_internal(objspace, ce->u.file);
5097 }
5098
5099 return ID_TABLE_CONTINUE;
5100}
5101
5102static void
5103update_const_tbl(void *objspace, struct rb_id_table *tbl)
5104{
5105 if (!tbl) return;
5106 rb_id_table_foreach_values(tbl, update_const_tbl_i, objspace);
5107}
5108
5109static void
5110update_superclasses(rb_objspace_t *objspace, rb_classext_t *ext)
5111{
5112 if (RCLASSEXT_SUPERCLASSES_WITH_SELF(ext)) {
5113 size_t array_size = RCLASSEXT_SUPERCLASS_DEPTH(ext) + 1;
5114 for (size_t i = 0; i < array_size; i++) {
5115 UPDATE_IF_MOVED(objspace, RCLASSEXT_SUPERCLASSES(ext)[i]);
5116 }
5117 }
5118}
5119
5120static void
5121update_classext_values(rb_objspace_t *objspace, rb_classext_t *ext, bool is_iclass)
5122{
5123 UPDATE_IF_MOVED(objspace, RCLASSEXT_ORIGIN(ext));
5124 UPDATE_IF_MOVED(objspace, RCLASSEXT_REFINED_CLASS(ext));
5125 UPDATE_IF_MOVED(objspace, RCLASSEXT_CLASSPATH(ext));
5126 if (is_iclass) {
5127 UPDATE_IF_MOVED(objspace, RCLASSEXT_INCLUDER(ext));
5128 }
5129}
5130
5131static void
5132update_classext(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg)
5133{
5134 struct classext_foreach_args *args = (struct classext_foreach_args *)arg;
5135 rb_objspace_t *objspace = args->objspace;
5136
5137 if (RCLASSEXT_SUPER(ext)) {
5138 UPDATE_IF_MOVED(objspace, RCLASSEXT_SUPER(ext));
5139 }
5140
5141 update_m_tbl(objspace, RCLASSEXT_M_TBL(ext));
5142
5143 UPDATE_IF_MOVED(objspace, ext->fields_obj);
5144 if (!RCLASSEXT_SHARED_CONST_TBL(ext)) {
5145 update_const_tbl(objspace, RCLASSEXT_CONST_TBL(ext));
5146 }
5147 UPDATE_IF_MOVED(objspace, RCLASSEXT_CC_TBL(ext));
5148 UPDATE_IF_MOVED(objspace, RCLASSEXT_CVC_TBL(ext));
5149 update_superclasses(objspace, ext);
5150 if (RCLASSEXT_SUBCLASSES(ext)) {
5151 UPDATE_IF_MOVED(objspace, RCLASSEXT_SUBCLASSES(ext));
5152 }
5153
5154 update_classext_values(objspace, ext, false);
5155}
5156
5157static void
5158update_iclass_classext(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg)
5159{
5160 struct classext_foreach_args *args = (struct classext_foreach_args *)arg;
5161 rb_objspace_t *objspace = args->objspace;
5162
5163 if (RCLASSEXT_SUPER(ext)) {
5164 UPDATE_IF_MOVED(objspace, RCLASSEXT_SUPER(ext));
5165 }
5166 update_m_tbl(objspace, RCLASSEXT_M_TBL(ext));
5167 update_m_tbl(objspace, RCLASSEXT_CALLABLE_M_TBL(ext));
5168 UPDATE_IF_MOVED(objspace, RCLASSEXT_CC_TBL(ext));
5169 UPDATE_IF_MOVED(objspace, RCLASSEXT_CVC_TBL(ext));
5170 if (RCLASSEXT_SUBCLASSES(ext)) {
5171 UPDATE_IF_MOVED(objspace, RCLASSEXT_SUBCLASSES(ext));
5172 }
5173
5174 update_classext_values(objspace, ext, true);
5175}
5176
5178 vm_table_foreach_callback_func callback;
5179 vm_table_update_callback_func update_callback;
5180 void *data;
5181 bool weak_only;
5182 /* The generic_fields table being walked, so compaction can re-insert a moved key
5183 * into it (rb_generic_fields_tables_foreach hands the table to the callback). */
5184 struct st_table *gen_fields_current_tbl;
5185 /* Re-inserting a moved key adds an entry, which can rehash and break the running
5186 * iterator, so collect them and insert after the walk (raw realloc: we are in GC). */
5187 struct gen_fields_deferred_insert { st_data_t k, v; } *gf_deferred;
5188 size_t gf_deferred_cnt, gf_deferred_capa;
5189};
5190
5191static int
5192vm_weak_table_foreach_weak_key(st_data_t key, st_data_t value, st_data_t data, int error)
5193{
5194 struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
5195
5196 int ret = iter_data->callback((VALUE)key, iter_data->data);
5197
5198 if (!iter_data->weak_only) {
5199 if (ret != ST_CONTINUE) return ret;
5200
5201 ret = iter_data->callback((VALUE)value, iter_data->data);
5202 }
5203
5204 return ret;
5205}
5206
5207static int
5208vm_weak_table_foreach_update_weak_key(st_data_t *key, st_data_t *value, st_data_t data, int existing)
5209{
5210 struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
5211
5212 int ret = iter_data->update_callback((VALUE *)key, iter_data->data);
5213
5214 if (!iter_data->weak_only) {
5215 if (ret != ST_CONTINUE) return ret;
5216
5217 ret = iter_data->update_callback((VALUE *)value, iter_data->data);
5218 }
5219
5220 return ret;
5221}
5222
5223static int
5224vm_weak_table_sym_set_foreach(VALUE *sym_ptr, void *data)
5225{
5226 VALUE sym = *sym_ptr;
5227 struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
5228
5229 if (RB_SPECIAL_CONST_P(sym)) return ST_CONTINUE;
5230
5231 int ret = iter_data->callback(sym, iter_data->data);
5232
5233 if (ret == ST_REPLACE) {
5234 ret = iter_data->update_callback(sym_ptr, iter_data->data);
5235 }
5236
5237 return ret;
5238}
5239
5240struct st_table *rb_generic_fields_tbl_get(void);
5241
5242static int
5243vm_weak_table_gen_fields_foreach(st_data_t key, st_data_t value, st_data_t data)
5244{
5245 struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
5246
5247 int ret = iter_data->callback((VALUE)key, iter_data->data);
5248
5249 VALUE new_value = (VALUE)value;
5250 VALUE new_key = (VALUE)key;
5251
5252 switch (ret) {
5253 case ST_CONTINUE:
5254 break;
5255
5256 case ST_DELETE:
5257 // When we're removing an object from the weak ref table, we need to
5258 // set the shape on it so that the GC finalizer won't try to remove
5259 // it again. A "root shape" indicates to the GC that this object
5260 // has no fields on it, hence it won't be in the gen fields table.
5261 if (BUILTIN_TYPE((VALUE)key) != T_NONE) {
5262 RBASIC_SET_SHAPE_ID((VALUE)key, ROOT_SHAPE_ID);
5263 }
5264 return ST_DELETE;
5265
5266 case ST_REPLACE: {
5267 ret = iter_data->update_callback(&new_key, iter_data->data);
5268 if (key != new_key) {
5269 ret = ST_DELETE;
5270 }
5271 break;
5272 }
5273
5274 default:
5275 rb_bug("vm_weak_table_gen_fields_foreach: return value %d not supported", ret);
5276 }
5277
5278 if (!iter_data->weak_only) {
5279 int ivar_ret = iter_data->callback(new_value, iter_data->data);
5280 switch (ivar_ret) {
5281 case ST_CONTINUE:
5282 break;
5283
5284 case ST_REPLACE:
5285 iter_data->update_callback(&new_value, iter_data->data);
5286 break;
5287
5288 case ST_DELETE:
5289 /* Leftover entry of a moved host: even if the key is alive, nobody can
5290 * read these fields once fields_obj is unreachable, so clean up as if the
5291 * key had died. */
5292 RBASIC_SET_SHAPE_ID((VALUE)key, ROOT_SHAPE_ID);
5293 return ST_DELETE;
5294
5295 default:
5296 rb_bug("vm_weak_table_gen_fields_foreach: return value %d not supported", ivar_ret);
5297 }
5298 }
5299
5300 if (key != new_key) {
5301 /* Inserting the new key adds an entry and may rehash, so defer it. */
5302 if (iter_data->gf_deferred_cnt == iter_data->gf_deferred_capa) {
5303 size_t nc = iter_data->gf_deferred_capa ? iter_data->gf_deferred_capa * 2 : 64;
5304 struct gen_fields_deferred_insert *p =
5305 realloc(iter_data->gf_deferred, nc * sizeof(*p));
5306 if (!p) rb_bug("vm_weak_table_gen_fields_foreach: out of memory");
5307 iter_data->gf_deferred = p;
5308 iter_data->gf_deferred_capa = nc;
5309 }
5310 iter_data->gf_deferred[iter_data->gf_deferred_cnt++] =
5311 (struct gen_fields_deferred_insert){ .k = (st_data_t)new_key, .v = (st_data_t)new_value };
5312 }
5313 else if (value != new_value) {
5314 DURING_GC_COULD_MALLOC_REGION_START();
5315 {
5316 /* Updating an existing key's value adds no entry and cannot rehash. */
5317 st_insert(iter_data->gen_fields_current_tbl, (st_data_t)new_key, new_value);
5318 }
5319 DURING_GC_COULD_MALLOC_REGION_END();
5320 }
5321
5322 return ret;
5323}
5324
5325static int
5326vm_weak_table_frozen_strings_foreach(VALUE *str, void *data)
5327{
5328 // int retval = vm_weak_table_foreach_weak_key(key, value, data, error);
5329 struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
5330 int retval = iter_data->callback(*str, iter_data->data);
5331
5332 if (retval == ST_REPLACE) {
5333 retval = iter_data->update_callback(str, iter_data->data);
5334 }
5335
5336 if (retval == ST_DELETE) {
5337 FL_UNSET(*str, RSTRING_FSTR);
5338 }
5339
5340 return retval;
5341}
5342
5343void rb_fstring_foreach_with_replace(int (*callback)(VALUE *str, void *data), void *data);
5344
5345/* Callback of rb_generic_fields_tables_foreach: walk one generic_fields table with the
5346 * gen_fields foreach used by compaction, recording the current table in foreach_data so
5347 * a moved key is re-inserted into the right one. */
5348static void
5349vm_weak_table_gen_fields_tbl_cb(struct st_table *tbl, void *arg)
5350{
5351 struct global_vm_table_foreach_data *foreach_data = (struct global_vm_table_foreach_data *)arg;
5352 foreach_data->gen_fields_current_tbl = tbl;
5353 st_foreach(tbl, vm_weak_table_gen_fields_foreach, (st_data_t)foreach_data);
5354}
5355
5356void
5357rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func callback,
5358 vm_table_update_callback_func update_callback,
5359 void *data,
5360 bool weak_only,
5361 enum rb_gc_vm_weak_tables table)
5362{
5363 rb_vm_t *vm = GET_VM();
5364
5365 struct global_vm_table_foreach_data foreach_data = {
5366 .callback = callback,
5367 .update_callback = update_callback,
5368 .data = data,
5369 .weak_only = weak_only,
5370 };
5371
5372 switch (table) {
5373 case RB_GC_VM_CI_TABLE: {
5374 st_foreach_with_replace(
5375 &vm->ci_table,
5376 vm_weak_table_foreach_weak_key,
5377 vm_weak_table_foreach_update_weak_key,
5378 (st_data_t)&foreach_data
5379 );
5380 break;
5381 }
5382 case RB_GC_VM_OVERLOADED_CME_TABLE: {
5383 st_foreach_with_replace(
5384 &vm->overloaded_cme_table,
5385 vm_weak_table_foreach_weak_key,
5386 vm_weak_table_foreach_update_weak_key,
5387 (st_data_t)&foreach_data
5388 );
5389 break;
5390 }
5391 case RB_GC_VM_GLOBAL_SYMBOLS_TABLE: {
5392 rb_sym_global_symbol_table_foreach_weak_reference(
5393 vm_weak_table_sym_set_foreach,
5394 &foreach_data
5395 );
5396 break;
5397 }
5398 case RB_GC_VM_GENERIC_FIELDS_TABLE: {
5399 /* There is one table. A global GC walks it without a lock under the
5400 * stop-the-world barrier; a local compaction holds the barrier VM lock taken in
5401 * gc_enter, so foreign keys cannot move and fall through the moved check. The
5402 * table's mutex (taken by shared_table_foreach) excludes mutator inserts. */
5403 if (rb_gc_during_global_gc_p()) {
5404 rb_generic_fields_tables_foreach(vm_weak_table_gen_fields_tbl_cb, (void *)&foreach_data);
5405 }
5406 else if (!weak_only) {
5407 rb_generic_fields_shared_table_foreach(vm_weak_table_gen_fields_tbl_cb, (void *)&foreach_data);
5408 }
5409 if (foreach_data.gf_deferred != NULL) {
5410 DURING_GC_COULD_MALLOC_REGION_START();
5411 {
5412 for (size_t i = 0; i < foreach_data.gf_deferred_cnt; i++) {
5413 struct gen_fields_deferred_insert *const d = &foreach_data.gf_deferred[i];
5414 st_insert(foreach_data.gen_fields_current_tbl, d->k, d->v);
5415 }
5416 }
5417 DURING_GC_COULD_MALLOC_REGION_END();
5418 free(foreach_data.gf_deferred);
5419 }
5420 break;
5421 }
5422 case RB_GC_VM_FROZEN_STRINGS_TABLE: {
5423 rb_fstring_foreach_with_replace(
5424 vm_weak_table_frozen_strings_foreach,
5425 &foreach_data
5426 );
5427 break;
5428 }
5429 case RB_GC_VM_WEAK_TABLE_COUNT:
5430 rb_bug("Unreachable");
5431 default:
5432 rb_bug("rb_gc_vm_weak_table_foreach: unknown table %d", table);
5433 }
5434}
5435
5436/* The global GC's weak pass over the generic_fields table; under the barrier, so the
5437 * walk needs no lock. */
5439 int (*cb)(VALUE key, VALUE val, void *arg);
5440 void *arg;
5441};
5442
5443static int
5444gf_mark_foreach_i(st_data_t key, st_data_t val, st_data_t data)
5445{
5446 struct gf_mark_foreach_ctx *ctx = (struct gf_mark_foreach_ctx *)data;
5447 return ctx->cb((VALUE)key, (VALUE)val, ctx->arg);
5448}
5449
5450static void
5451gf_mark_foreach_table_cb(struct st_table *tbl, void *arg)
5452{
5453 st_foreach(tbl, gf_mark_foreach_i, (st_data_t)arg);
5454}
5455
5456void
5457rb_gc_vm_generic_fields_mark_foreach(int (*cb)(VALUE key, VALUE val, void *arg), void *arg)
5458{
5459 struct gf_mark_foreach_ctx ctx = { cb, arg };
5460 rb_generic_fields_tables_foreach(gf_mark_foreach_table_cb, &ctx);
5461}
5462
5464 bool (*is_dead)(VALUE key);
5465};
5466
5467static int
5468gf_drain_i(st_data_t key, st_data_t val, st_data_t data)
5469{
5470 struct gf_drain_ctx *ctx = (struct gf_drain_ctx *)data;
5471 if (ctx->is_dead((VALUE)key)) {
5472 /* The weak pass only drains dead keys' entries, never touching the key itself:
5473 * after the global GC settled another objspace's lazy sweep the key may already
5474 * be freed (poisoned), and writing a shape there would be a use-after-poison. */
5475 return ST_DELETE;
5476 }
5477 return ST_CONTINUE;
5478}
5479
5480static void
5481gf_drain_table_cb(struct st_table *tbl, void *arg)
5482{
5483 st_foreach(tbl, gf_drain_i, (st_data_t)arg);
5484}
5485
5486void
5487rb_gc_vm_generic_fields_drain_dead(bool (*is_dead)(VALUE key))
5488{
5489 struct gf_drain_ctx ctx = { is_dead };
5490 rb_generic_fields_tables_foreach(gf_drain_table_cb, &ctx);
5491}
5492
5493VALUE
5494rb_gc_vm_top_self(void)
5495{
5496 return rb_vm_top_self();
5497}
5498
5499void
5500rb_gc_update_vm_references(void *objspace)
5501{
5502 rb_execution_context_t *ec = GET_EC();
5503 rb_vm_t *vm = rb_ec_vm_ptr(ec);
5504
5505 rb_vm_update_references(vm);
5506 rb_gc_update_global_tbl();
5507 rb_sym_global_symbols_mark_and_move();
5508
5509#if USE_YJIT
5510 void rb_yjit_root_update_references(void); // in Rust
5511
5512 if (rb_yjit_enabled_p) {
5513 rb_yjit_root_update_references();
5514 }
5515#endif
5516
5517#if USE_ZJIT
5518 void rb_zjit_root_update_references(void); // in Rust
5519
5520 if (rb_zjit_enabled_p) {
5521 rb_zjit_root_update_references();
5522 }
5523#endif
5524}
5525
5526void
5527rb_gc_update_object_references(void *objspace, VALUE obj)
5528{
5529 struct classext_foreach_args args;
5530
5531 switch (BUILTIN_TYPE(obj)) {
5532 case T_CLASS:
5533 if (FL_TEST_RAW(obj, FL_SINGLETON)) {
5534 UPDATE_IF_MOVED(objspace, RCLASS_ATTACHED_OBJECT(obj));
5535 }
5536 // Continue to the shared T_CLASS/T_MODULE
5537 case T_MODULE:
5538 args.klass = obj;
5539 args.objspace = objspace;
5540 rb_class_classext_foreach(obj, update_classext, (void *)&args);
5541 break;
5542
5543 case T_ICLASS:
5544 args.objspace = objspace;
5545 rb_class_classext_foreach(obj, update_iclass_classext, (void *)&args);
5546 break;
5547
5548 case T_IMEMO: {
5549 /* Not derived from `objspace`: a global compaction runs this pass over every
5550 * objspace in turn, while rb_gc_impl_mark_and_move has always resolved through the
5551 * current Ractor here (which is also what sets during_reference_updating on all of
5552 * them). Keep that resolution, just hoisted out of the per-reference path. */
5553 const struct rb_gc_mark_ctx ctx = gc_current_mark_ctx();
5554
5555 rb_imemo_mark_and_move(&ctx, obj, true);
5556 return;
5557 }
5558
5559 case T_NIL:
5560 case T_FIXNUM:
5561 case T_NODE:
5562 case T_MOVED:
5563 case T_NONE:
5564 /* These can't move */
5565 return;
5566
5567 case T_ARRAY:
5568 gc_ref_update_array(objspace, obj);
5569 break;
5570
5571 case T_HASH:
5572 gc_ref_update_hash(objspace, obj);
5573 UPDATE_IF_MOVED(objspace, RHASH(obj)->ifnone);
5574 break;
5575
5576 case T_STRING:
5577 {
5578 if (STR_SHARED_P(obj)) {
5579 UPDATE_IF_MOVED(objspace, RSTRING(obj)->as.heap.aux.shared);
5580 }
5581
5582 /* If, after move the string is not embedded, and can fit in the
5583 * slot it's been placed in, then re-embed it. Skip pinned objects:
5584 * a local holding RSTRING_PTR across this compaction could otherwise
5585 * point to freed memory even if the String is marked and pinned. */
5586 if (rb_gc_obj_slot_size(obj) >= rb_str_size_as_embedded(obj)) {
5587 if (!STR_EMBED_P(obj) && rb_str_reembeddable_p(obj)
5588 && !rb_gc_impl_pinned_p(objspace, obj)) {
5589 rb_str_make_embedded(obj);
5590 }
5591 }
5592
5593 break;
5594 }
5595 case T_DATA:
5596 /* Call the compaction callback, if it exists */
5597 {
5598 void *const ptr = RTYPEDDATA_GET_DATA(obj);
5599
5600 UPDATE_IF_MOVED(objspace, RTYPEDDATA(obj)->fields_obj);
5601
5602 if (ptr) {
5603 if (gc_declarative_marking_p(RTYPEDDATA_TYPE(obj))) {
5604 size_t *offset_list = TYPED_DATA_REFS_OFFSET_LIST(obj);
5605
5606 for (size_t offset = *offset_list; offset != RUBY_REF_END; offset = *offset_list++) {
5607 VALUE *ref = (VALUE *)((char *)ptr + offset);
5608 *ref = gc_location_internal(objspace, *ref);
5609 }
5610 }
5611 else {
5612 RUBY_DATA_FUNC compact_func = RTYPEDDATA_TYPE(obj)->function.dcompact;
5613 if (compact_func) (*compact_func)(ptr);
5614 }
5615 }
5616 }
5617 break;
5618
5619 case T_OBJECT:
5620 gc_ref_update_object(objspace, obj);
5621 break;
5622
5623 case T_FILE:
5624 if (RFILE(obj)->fptr) {
5625 UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->self);
5626 UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->pathv);
5627 UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->tied_io_for_writing);
5628 UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->writeconv_asciicompat);
5629 UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->writeconv_pre_ecopts);
5630 UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->encs.ecopts);
5631 UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->write_lock);
5632 UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->timeout);
5633 UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->wakeup_mutex);
5634 }
5635 break;
5636 case T_REGEXP:
5637 UPDATE_IF_MOVED(objspace, RREGEXP(obj)->src);
5638 break;
5639
5640 case T_SYMBOL:
5641 UPDATE_IF_MOVED(objspace, RSYMBOL(obj)->fstr);
5642 break;
5643
5644 case T_FLOAT:
5645 case T_BIGNUM:
5646 break;
5647
5648 case T_MATCH:
5649 UPDATE_IF_MOVED(objspace, RMATCH(obj)->regexp);
5650
5651 if (RMATCH(obj)->str) {
5652 UPDATE_IF_MOVED(objspace, RMATCH(obj)->str);
5653 }
5654 break;
5655
5656 case T_RATIONAL:
5657 UPDATE_IF_MOVED(objspace, RRATIONAL(obj)->num);
5658 UPDATE_IF_MOVED(objspace, RRATIONAL(obj)->den);
5659 break;
5660
5661 case T_COMPLEX:
5662 UPDATE_IF_MOVED(objspace, RCOMPLEX(obj)->real);
5663 UPDATE_IF_MOVED(objspace, RCOMPLEX(obj)->imag);
5664
5665 break;
5666
5667 case T_STRUCT:
5668 {
5669 rb_len_t i, len = RSTRUCT_LEN(obj);
5670 VALUE *ptr = (VALUE *)RSTRUCT_CONST_PTR(obj);
5671
5672 for (i = 0; i < len; i++) {
5673 UPDATE_IF_MOVED(objspace, ptr[i]);
5674 }
5675
5676 UPDATE_IF_MOVED(objspace, RSTRUCT(obj)->fields_obj);
5677 }
5678 break;
5679 default:
5680 rb_bug("unreachable");
5681 break;
5682 }
5683
5684 UPDATE_IF_MOVED(objspace, RBASIC(obj)->klass);
5685}
5686
5687VALUE
5689{
5690 rb_gc();
5691 return Qnil;
5692}
5693
5694void
5696{
5697 unless_objspace(objspace) { return; }
5698
5699 rb_gc_impl_start(objspace, true, true, true, false, true);
5700}
5701
5702int
5704{
5705 unless_objspace(objspace) { return FALSE; }
5706
5707 return rb_gc_impl_during_gc_p(objspace);
5708}
5709
5710size_t
5712{
5713 return rb_gc_impl_gc_count(rb_gc_get_objspace());
5714}
5715
5716static VALUE
5717gc_count(rb_execution_context_t *ec, VALUE self)
5718{
5719 return SIZET2NUM(rb_gc_count());
5720}
5721
5722VALUE
5723rb_gc_latest_gc_info(VALUE key)
5724{
5725 if (!SYMBOL_P(key) && !RB_TYPE_P(key, T_HASH)) {
5726 rb_raise(rb_eTypeError, "non-hash or symbol given");
5727 }
5728
5729 VALUE val = rb_gc_impl_latest_gc_info(rb_gc_get_objspace(), key);
5730
5731 if (val == Qundef) {
5732 rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
5733 }
5734
5735 return val;
5736}
5737
5738static VALUE
5739gc_stat(rb_execution_context_t *ec, VALUE self, VALUE arg, VALUE process)
5740{
5741 if (NIL_P(arg)) {
5742 arg = rb_hash_new();
5743 }
5744 else if (!RB_TYPE_P(arg, T_HASH) && !SYMBOL_P(arg)) {
5745 rb_raise(rb_eTypeError, "non-hash or symbol given");
5746 }
5747
5748 VALUE ret = rb_gc_impl_stat(RTEST(process) ? NULL : rb_gc_get_objspace(), arg);
5749
5750 if (ret == Qundef) {
5751 GC_ASSERT(SYMBOL_P(arg));
5752
5753 rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(arg));
5754 }
5755
5756 return ret;
5757}
5758
5759size_t
5760rb_gc_stat(VALUE arg)
5761{
5762 if (!RB_TYPE_P(arg, T_HASH) && !SYMBOL_P(arg)) {
5763 rb_raise(rb_eTypeError, "non-hash or symbol given");
5764 }
5765
5766 VALUE ret = rb_gc_impl_stat(rb_gc_get_objspace(), arg);
5767
5768 if (ret == Qundef) {
5769 GC_ASSERT(SYMBOL_P(arg));
5770
5771 rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(arg));
5772 }
5773
5774 if (SYMBOL_P(arg)) {
5775 return NUM2SIZET(ret);
5776 }
5777 else {
5778 return 0;
5779 }
5780}
5781
5782static VALUE
5783gc_stat_heap(rb_execution_context_t *ec, VALUE self, VALUE heap_name, VALUE arg)
5784{
5785 if (NIL_P(arg)) {
5786 arg = rb_hash_new();
5787 }
5788
5789 if (NIL_P(heap_name)) {
5790 if (!RB_TYPE_P(arg, T_HASH)) {
5791 rb_raise(rb_eTypeError, "non-hash given");
5792 }
5793 }
5794 else if (FIXNUM_P(heap_name)) {
5795 if (!SYMBOL_P(arg) && !RB_TYPE_P(arg, T_HASH)) {
5796 rb_raise(rb_eTypeError, "non-hash or symbol given");
5797 }
5798 }
5799 else {
5800 rb_raise(rb_eTypeError, "heap_name must be nil or an Integer");
5801 }
5802
5803 VALUE ret = rb_gc_impl_stat_heap(rb_gc_get_objspace(), heap_name, arg);
5804
5805 if (ret == Qundef) {
5806 GC_ASSERT(SYMBOL_P(arg));
5807
5808 rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(arg));
5809 }
5810
5811 return ret;
5812}
5813
5814static VALUE
5815gc_config_get(rb_execution_context_t *ec, VALUE self)
5816{
5817 VALUE cfg_hash = rb_gc_impl_config_get(rb_gc_get_objspace());
5818 rb_hash_aset(cfg_hash, sym("implementation"), rb_fstring_cstr(rb_gc_impl_active_gc_name()));
5819
5820 return cfg_hash;
5821}
5822
5823static VALUE
5824gc_config_set(rb_execution_context_t *ec, VALUE self, VALUE hash)
5825{
5826 void *objspace = rb_gc_get_objspace();
5827
5828 rb_gc_impl_config_set(objspace, hash);
5829
5830 return Qnil;
5831}
5832
5833static VALUE
5834gc_stress_get(rb_execution_context_t *ec, VALUE self)
5835{
5836 return rb_gc_impl_stress_get(rb_gc_get_objspace());
5837}
5838
5839static VALUE
5840gc_stress_set_m(rb_execution_context_t *ec, VALUE self, VALUE flag)
5841{
5842 rb_gc_impl_stress_set(rb_gc_get_objspace(), flag);
5843
5844 return flag;
5845}
5846
5847void
5848rb_gc_initial_stress_set(VALUE flag)
5849{
5850 initial_stress = flag;
5851}
5852
5853/* Add or drop a GC-disable holder (vm->gc.disable_holders; see vm_core.h). critical
5854 * is the anonymous holder used by internal sections that must not be interrupted by a
5855 * GC, such as collecting under the barrier. */
5856
5857static void
5858rb_gc_critical_disable(void)
5859{
5860 rb_gc_impl_gc_rest(rb_gc_get_objspace());
5861 RUBY_ATOMIC_INC(GET_VM()->gc.disable_holders);
5862}
5863
5864static void
5865rb_gc_critical_enable(void)
5866{
5867 RUBY_ATOMIC_DEC(GET_VM()->gc.disable_holders);
5868}
5869
5870bool
5871rb_gc_gc_disabled_global_p(void)
5872{
5873 return RUBY_ATOMIC_LOAD(GET_VM()->gc.disable_holders) != 0;
5874}
5875
5876/* GC.disable/enable set and clear this objspace's flag and only move the holder count
5877 * when the flag actually changes. The returned previous state is this objspace's. */
5878static bool
5879gc_ractor_disable_set(bool disable)
5880{
5881 const bool was = rb_gc_impl_user_gc_disabled_set(rb_gc_get_objspace(), disable);
5882 if (was != disable) {
5883 if (disable) {
5884 RUBY_ATOMIC_INC(GET_VM()->gc.disable_holders);
5885 }
5886 else {
5887 RUBY_ATOMIC_DEC(GET_VM()->gc.disable_holders);
5888 }
5889 }
5890 return was;
5891}
5892
5893VALUE
5895{
5896 return RBOOL(gc_ractor_disable_set(false));
5897}
5898
5899VALUE
5900rb_gc_disable_no_rest(void)
5901{
5902 return RBOOL(gc_ractor_disable_set(true));
5903}
5904
5905VALUE
5907{
5908 const bool was_disabled = gc_ractor_disable_set(true);
5909 if (!was_disabled) {
5910 rb_gc_impl_gc_rest(rb_gc_get_objspace());
5911 }
5912 return RBOOL(was_disabled);
5913}
5914
5915VALUE
5916rb_objspace_gc_enable(void *objspace)
5917{
5918 bool disabled = !rb_gc_impl_gc_enabled_p(objspace);
5919 rb_gc_impl_gc_enable(objspace);
5920 return RBOOL(disabled);
5921}
5922
5923VALUE
5924rb_objspace_gc_disable(void *objspace)
5925{
5926 bool disabled = !rb_gc_impl_gc_enabled_p(objspace);
5927 rb_gc_impl_gc_disable(objspace, true);
5928 return RBOOL(disabled);
5929}
5930
5931VALUE
5932rb_gc_objspace_enable(void *objspace)
5933{
5934 return rb_objspace_gc_enable(objspace);
5935}
5936
5937VALUE
5938rb_gc_local_enable(void)
5939{
5940 return rb_gc_objspace_enable(rb_gc_get_objspace());
5941}
5942
5943
5944VALUE
5945rb_gc_objspace_disable_no_rest(void *objspace)
5946{
5947 bool disabled = !rb_gc_impl_gc_enabled_p(objspace);
5948 rb_gc_impl_gc_disable(objspace, false);
5949 return RBOOL(disabled);
5950}
5951
5952VALUE
5953rb_gc_local_disable_no_rest(void)
5954{
5955 return rb_gc_objspace_disable_no_rest(rb_gc_get_objspace());
5956}
5957
5958static VALUE
5959gc_enable(rb_execution_context_t *ec, VALUE _)
5960{
5961 return rb_gc_enable();
5962}
5963
5964static VALUE
5965gc_disable(rb_execution_context_t *ec, VALUE _)
5966{
5967 return rb_gc_disable();
5968}
5969
5970// TODO: think about moving ruby_gc_set_params into Init_heap or Init_gc
5971void
5972ruby_gc_set_params(void)
5973{
5974 rb_gc_impl_set_params(rb_gc_get_objspace());
5975}
5976
5977/* Walk obj's outgoing references, handing each child to `func` instead of marking it.
5978 * The walk is one level deep: it never descends into a child (no mark stack). Callers
5979 * wanting the whole graph must recurse it themselves. A child living in another Ractor's
5980 * objspace (a shareable) is handed to `func` like any other. NOTE: it may return garbage
5981 * objects, so callers should check `live_object_p` or `garbage_object_p`. If `can_run_ruby`
5982 * is true, then Ruby methods can be called from `func` (they may also `raise`). Regardless
5983 * of whether `can_run_ruby` is true, you may allocate Ruby objects in the callback function.
5984 */
5985static void
5986reachable_objects_from_body(VALUE obj, void (func)(VALUE, void *), void *data, bool can_run_ruby)
5987{
5988 if (rb_gc_impl_during_gc_p(rb_gc_get_objspace()))
5989 rb_bug("rb_objspace_reachable_objects_from() is not supported while during GC");
5990
5991 if (!RB_SPECIAL_CONST_P(obj)) {
5992 struct gc_mark_func_data_struct **volatile mfdp = GC_MARK_FUNC_DATA_SLOTP();
5993 struct gc_mark_func_data_struct *volatile prev_mfd = *mfdp;
5994 struct gc_mark_func_data_struct mfd = {
5995 .mark_func = func,
5996 .data = data,
5997 };
5998
5999 *mfdp = &mfd;
6000
6001 if (can_run_ruby) {
6002 rb_execution_context_t *volatile ec = GET_EC();
6003 enum ruby_tag_type state;
6004
6005 EC_PUSH_TAG(ec);
6006 if ((state = EC_EXEC_TAG()) == TAG_NONE) {
6007 rb_gc_mark_children(rb_gc_get_objspace(), obj);
6008 }
6009 EC_POP_TAG();
6010
6011 *mfdp = prev_mfd;
6012
6013 if (state != TAG_NONE) EC_JUMP_TAG(ec, state);
6014 }
6015 else {
6016 rb_gc_mark_children(rb_gc_get_objspace(), obj);
6017 *mfdp = prev_mfd;
6018 }
6019 }
6020}
6021
6022/* The walk reads tables whose only synchronization is the VM lock -- a class's m_tbl
6023 * and const_tbl are written under it by rb_method_table_insert() and const_tbl_update().
6024 * Two things make it safe to read them without the lock, and every unlocked entry point
6025 * below rests on one of them:
6026 *
6027 * - obj is owned by the current Ractor. rb_class_owner_check() makes that Ractor the
6028 * sole writer, and it is the one walking, so it cannot be mid-insert.
6029 * - the world is stopped (a global GC's barrier), so there is no concurrent writer at
6030 * all. This is the only way a foreign obj may be walked.
6031 */
6032void
6033rb_objspace_reachable_objects_from_unlocked(VALUE obj, void (func)(VALUE, void *), void *data)
6034{
6035 reachable_objects_from_body(obj, func, data, false);
6036}
6037
6038/* Unlocked walk of an object owned by the current Ractor. Because no lock is held, func
6039 * may run Ruby code. A foreign obj means the caller got ownership wrong: walking it would
6040 * race the owner, and no caller can recover from that, so stop here rather than raise.
6041 */
6042void
6043rb_objspace_reachable_objects_from_local(VALUE obj, void (func)(VALUE, void *), void *data)
6044{
6045 ASSERT_vm_unlocking();
6046 if (rb_objspace_foreign_object_p(obj)) {
6047 rb_bug("rb_objspace_reachable_objects_from_local: %s is owned by another Ractor",
6048 rb_obj_info(obj));
6049 }
6050
6051 reachable_objects_from_body(obj, func, data, true);
6052}
6053
6054/* For a caller already holding the VM lock (a zombie's obj): func may raise, and
6055 * the walk does not take the lock again, so the raise leaves no lock level behind. */
6056void
6057rb_objspace_reachable_objects_from_locked(VALUE obj, void (func)(VALUE, void *), void *data)
6058{
6059 ASSERT_vm_locking();
6060 reachable_objects_from_body(obj, func, data, true);
6061}
6062
6063/* Walk obj's outgoing references, where obj may be owned by another Ractor (a
6064 * shareable). The VM lock is what serializes the walk against the owner's writes
6065 * for shareable classes/modules, so func must not run Ruby code or check interrupts.
6066 * The lock is taken barrier-capable on purpose: the callers left here are ObjectSpace's
6067 * dump and reachable_objects_from, whose callbacks allocate, so a barrier another Ractor
6068 * is already waiting on must not be held off for the length of the walk.
6069 */
6070void
6071rb_objspace_reachable_objects_from(VALUE obj, void (func)(VALUE, void *), void *data)
6072{
6073 RB_VM_LOCKING() {
6074 reachable_objects_from_body(obj, func, data, false);
6075 }
6076}
6077
6079 const char *category;
6080 void (*func)(const char *category, VALUE, void *);
6081 void *data;
6082};
6083
6084static void
6085root_objects_from(VALUE obj, void *ptr)
6086{
6087 const struct root_objects_data *data = (struct root_objects_data *)ptr;
6088 (*data->func)(data->category, obj, data->data);
6089}
6090
6091void
6092rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *passing_data)
6093{
6094 if (rb_gc_impl_during_gc_p(rb_gc_get_objspace())) rb_bug("rb_gc_impl_objspace_reachable_objects_from_root() is not supported while during GC");
6095
6096 struct root_objects_data data = {
6097 .func = func,
6098 .data = passing_data,
6099 };
6100
6101 struct gc_mark_func_data_struct **mfdp = GC_MARK_FUNC_DATA_SLOTP();
6102 struct gc_mark_func_data_struct *prev_mfd = *mfdp;
6103 struct gc_mark_func_data_struct mfd = {
6104 .mark_func = root_objects_from,
6105 .data = &data,
6106 };
6107
6108 *mfdp = &mfd;
6109 rb_gc_save_machine_context();
6110 rb_gc_mark_roots(rb_gc_get_objspace(), &data.category);
6111 *mfdp = prev_mfd;
6112}
6113
6114/*
6115 ------------------------------ DEBUG ------------------------------
6116*/
6117
6118static const char *
6119type_name(int type, VALUE obj)
6120{
6121 switch (type) {
6122#define TYPE_NAME(t) case (t): return #t;
6123 TYPE_NAME(T_NONE);
6124 TYPE_NAME(T_OBJECT);
6125 TYPE_NAME(T_CLASS);
6126 TYPE_NAME(T_MODULE);
6127 TYPE_NAME(T_FLOAT);
6128 TYPE_NAME(T_STRING);
6129 TYPE_NAME(T_REGEXP);
6130 TYPE_NAME(T_ARRAY);
6131 TYPE_NAME(T_HASH);
6132 TYPE_NAME(T_STRUCT);
6133 TYPE_NAME(T_BIGNUM);
6134 TYPE_NAME(T_FILE);
6135 TYPE_NAME(T_MATCH);
6136 TYPE_NAME(T_COMPLEX);
6137 TYPE_NAME(T_RATIONAL);
6138 TYPE_NAME(T_NIL);
6139 TYPE_NAME(T_TRUE);
6140 TYPE_NAME(T_FALSE);
6141 TYPE_NAME(T_SYMBOL);
6142 TYPE_NAME(T_FIXNUM);
6143 TYPE_NAME(T_UNDEF);
6144 TYPE_NAME(T_IMEMO);
6145 TYPE_NAME(T_ICLASS);
6146 TYPE_NAME(T_MOVED);
6147 TYPE_NAME(T_ZOMBIE);
6148 case T_DATA:
6149 if (obj && rb_objspace_data_type_name(obj)) {
6150 return rb_objspace_data_type_name(obj);
6151 }
6152 return "T_DATA";
6153#undef TYPE_NAME
6154 }
6155 return "unknown";
6156}
6157
6158static const char *
6159obj_type_name(VALUE obj)
6160{
6161 return type_name(TYPE(obj), obj);
6162}
6163
6164const char *
6165rb_method_type_name(rb_method_type_t type)
6166{
6167 switch (type) {
6168 case VM_METHOD_TYPE_ISEQ: return "iseq";
6169 case VM_METHOD_TYPE_ATTRSET: return "attrset";
6170 case VM_METHOD_TYPE_IVAR: return "ivar";
6171 case VM_METHOD_TYPE_BMETHOD: return "bmethod";
6172 case VM_METHOD_TYPE_ALIAS: return "alias";
6173 case VM_METHOD_TYPE_REFINED: return "refined";
6174 case VM_METHOD_TYPE_CFUNC: return "cfunc";
6175 case VM_METHOD_TYPE_ZSUPER: return "zsuper";
6176 case VM_METHOD_TYPE_MISSING: return "missing";
6177 case VM_METHOD_TYPE_OPTIMIZED: return "optimized";
6178 case VM_METHOD_TYPE_UNDEF: return "undef";
6179 case VM_METHOD_TYPE_NOTIMPLEMENTED: return "notimplemented";
6180 }
6181 rb_bug("rb_method_type_name: unreachable (type: %d)", type);
6182}
6183
6184static void
6185rb_raw_iseq_info(char *const buff, const size_t buff_size, const rb_iseq_t *iseq)
6186{
6187 if (buff_size > 0 && ISEQ_BODY(iseq) && ISEQ_BODY(iseq)->location.label && !RB_TYPE_P(ISEQ_BODY(iseq)->location.pathobj, T_MOVED)) {
6188 VALUE path = rb_iseq_path(iseq);
6189 int n = ISEQ_BODY(iseq)->location.first_lineno;
6190 VALUE label = ISEQ_BODY(iseq)->location.label;
6191 snprintf(buff, buff_size, " %.*s@%.*s:%d",
6192 RSTRING_LENINT(label), RSTRING_PTR(label),
6193 RSTRING_LENINT(path), RSTRING_PTR(path), n);
6194 }
6195}
6196
6197static int
6198str_len_no_raise(VALUE str)
6199{
6200 rb_len_t len = RSTRING_LEN(str);
6201 if (len < 0) return 0;
6202 if (len > INT_MAX) return INT_MAX;
6203 return (int)len;
6204}
6205
6206#define BUFF_ARGS buff + pos, buff_size - pos
6207#define APPEND_F(...) if ((pos += snprintf(BUFF_ARGS, "" __VA_ARGS__)) >= buff_size) goto end
6208#define APPEND_S(s) do { \
6209 if ((pos + (int)rb_strlen_lit(s)) >= buff_size) { \
6210 goto end; \
6211 } \
6212 else { \
6213 memcpy(buff + pos, (s), rb_strlen_lit(s) + 1); \
6214 } \
6215 } while (0)
6216#define C(c, s) ((c) != 0 ? (s) : " ")
6217
6218static size_t
6219rb_raw_obj_info_common(char *const buff, const size_t buff_size, const VALUE obj)
6220{
6221 size_t pos = 0;
6222
6223 if (SPECIAL_CONST_P(obj)) {
6224 APPEND_F("%s", obj_type_name(obj));
6225
6226 if (FIXNUM_P(obj)) {
6227 APPEND_F(" %ld", FIX2LONG(obj));
6228 }
6229 else if (SYMBOL_P(obj)) {
6230 APPEND_F(" %s", rb_id2name(SYM2ID(obj)));
6231 }
6232 }
6233 else {
6234 // const int age = RVALUE_AGE_GET(obj);
6235
6236 if (rb_gc_impl_live_object_p(rb_gc_get_objspace(), (void *)obj)) {
6237 APPEND_F("%p %s/", (void *)obj, obj_type_name(obj));
6238 // TODO: fixme
6239 // APPEND_F("%p [%d%s%s%s%s%s%s] %s ",
6240 // (void *)obj, age,
6241 // C(RVALUE_UNCOLLECTIBLE_BITMAP(obj), "L"),
6242 // C(RVALUE_MARK_BITMAP(obj), "M"),
6243 // C(RVALUE_PIN_BITMAP(obj), "P"),
6244 // C(RVALUE_MARKING_BITMAP(obj), "R"),
6245 // C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"),
6246 // C(rb_objspace_garbage_object_p(obj), "G"),
6247 // obj_type_name(obj));
6248 }
6249 else {
6250 /* fake */
6251 // APPEND_F("%p [%dXXXX] %s",
6252 // (void *)obj, age,
6253 // obj_type_name(obj));
6254 }
6255
6256 if (internal_object_p(obj)) {
6257 /* ignore */
6258 }
6259 else if (RBASIC(obj)->klass == 0) {
6260 APPEND_S("(temporary internal)");
6261 }
6262 else if (RTEST(RBASIC(obj)->klass)) {
6263 VALUE class_path = rb_mod_name(RBASIC(obj)->klass);
6264 if (!NIL_P(class_path)) {
6265 APPEND_F("%.*s ", str_len_no_raise(class_path), RSTRING_PTR(class_path));
6266 }
6267 }
6268 }
6269 end:
6270
6271 return pos;
6272}
6273
6274const char *rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj);
6275
6276static size_t
6277rb_raw_obj_info_buitin_type(char *const buff, const size_t buff_size, const VALUE obj, size_t pos)
6278{
6279 if (LIKELY(pos < buff_size) && !SPECIAL_CONST_P(obj)) {
6280 const enum ruby_value_type type = BUILTIN_TYPE(obj);
6281
6282 switch (type) {
6283 case T_NODE:
6284 UNEXPECTED_NODE(rb_raw_obj_info);
6285 break;
6286 case T_ARRAY:
6287 if (ARY_SHARED_P(obj)) {
6288 APPEND_S("shared -> ");
6289 rb_raw_obj_info(BUFF_ARGS, ARY_SHARED_ROOT(obj));
6290 }
6291 else {
6292 APPEND_F("[%s%s%s] ",
6293 C(ARY_EMBED_P(obj), "E"),
6294 C(ARY_SHARED_P(obj), "S"),
6295 C(ARY_SHARED_ROOT_P(obj), "R"));
6296
6297 if (ARY_EMBED_P(obj)) {
6298 APPEND_F("len: %"PRIdLEN" (embed)",
6299 RARRAY_LEN(obj));
6300 }
6301 else {
6302 APPEND_F("len: %"PRIdLEN", capa:%"PRIdLEN" ptr:%p",
6303 RARRAY_LEN(obj),
6304 RARRAY(obj)->as.heap.aux.capa,
6305 (void *)RARRAY_CONST_PTR(obj));
6306 }
6307 }
6308 break;
6309 case T_STRING: {
6310 APPEND_F("[%s%s] ",
6311 C(FL_TEST(obj, RSTRING_FSTR), "F"),
6312 C(RB_OBJ_FROZEN(obj), "R"));
6313
6314 if (STR_SHARED_P(obj)) {
6315 APPEND_F(" [shared] len: %"PRIdLEN, RSTRING_LEN(obj));
6316 }
6317 else {
6318 if (STR_EMBED_P(obj)) APPEND_S(" [embed]");
6319
6320 APPEND_F(" len: %"PRIdLEN", capa: %" PRIdSIZE, RSTRING_LEN(obj), rb_str_capacity(obj));
6321 }
6322 APPEND_F(" \"%.*s\"", str_len_no_raise(obj), RSTRING_PTR(obj));
6323 break;
6324 }
6325 case T_SYMBOL: {
6326 VALUE fstr = RSYMBOL(obj)->fstr;
6327 ID id = RSYMBOL(obj)->id;
6328 if (RB_TYPE_P(fstr, T_STRING)) {
6329 APPEND_F(":%.*s id:%d", str_len_no_raise(fstr), RSTRING_PTR(fstr), (unsigned int)id);
6330 }
6331 else {
6332 APPEND_F("(%p) id:%d", (void *)fstr, (unsigned int)id);
6333 }
6334 break;
6335 }
6336 case T_MOVED: {
6337 APPEND_F("-> %p", (void*)gc_location_internal(rb_gc_get_objspace(), obj));
6338 break;
6339 }
6340 case T_HASH: {
6341 APPEND_F("[%c] %"PRIdSIZE,
6342 RHASH_AR_TABLE_P(obj) ? 'A' : 'S',
6343 RHASH_SIZE(obj));
6344 break;
6345 }
6346 case T_CLASS:
6347 case T_MODULE:
6348 {
6349 VALUE class_path = rb_mod_name(obj);
6350 if (!NIL_P(class_path)) {
6351 APPEND_F("%.*s", str_len_no_raise(class_path), RSTRING_PTR(class_path));
6352 }
6353 else {
6354 APPEND_S("(anon)");
6355 }
6356 break;
6357 }
6358 case T_ICLASS:
6359 {
6360 VALUE class_path = rb_mod_name(RBASIC_CLASS(obj));
6361 if (!NIL_P(class_path)) {
6362 APPEND_F("src:%.*s", str_len_no_raise(class_path), RSTRING_PTR(class_path));
6363 }
6364 break;
6365 }
6366 case T_OBJECT:
6367 {
6368 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
6369 if (rb_shape_embedded_p(shape_id)) {
6370 APPEND_F("(embed) len:%d capa:%d", RSHAPE_LEN(shape_id), RSHAPE_CAPACITY(shape_id));
6371 }
6372 else {
6373 VALUE fields_obj = ROBJECT_FIELDS_OBJ(obj);
6374 if (rb_shape_complex_p(shape_id)) {
6375 size_t hash_len = rb_st_table_size(rb_imemo_fields_complex_tbl(fields_obj));
6376 APPEND_F("(complex) len:%zu extended:%p", hash_len, (void *)fields_obj);
6377 }
6378 else {
6379 APPEND_F("(extended) len:%d capa:%d extended:%p", RSHAPE_LEN(shape_id), RSHAPE_CAPACITY(shape_id), (void *)fields_obj);
6380 }
6381 }
6382 }
6383 break;
6384 case T_DATA: {
6385 const struct rb_block *block;
6386 const rb_iseq_t *iseq;
6387 if (rb_obj_is_proc(obj) &&
6388 (block = vm_proc_block(obj)) != NULL &&
6389 (vm_block_type(block) == block_type_iseq) &&
6390 (iseq = vm_block_iseq(block)) != NULL) {
6391 rb_raw_iseq_info(BUFF_ARGS, iseq);
6392 }
6393 else if (rb_ractor_p(obj)) {
6394 rb_ractor_t *r = (void *)DATA_PTR(obj);
6395 if (r) {
6396 APPEND_F("r:%"PRI_SERIALT_PREFIX"u", r->pub.id);
6397 }
6398 }
6399 break;
6400 }
6401 case T_IMEMO: {
6402 APPEND_F("<%s> ", rb_imemo_name(imemo_type(obj)));
6403
6404 switch (imemo_type(obj)) {
6405 case imemo_fields:
6406 {
6407 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
6408 if (rb_shape_complex_p(shape_id)) {
6409 size_t hash_len = rb_st_table_size(rb_imemo_fields_complex_tbl(obj));
6410 APPEND_F("(complex) len:%zu", hash_len);
6411 }
6412 else {
6413 APPEND_F("(embed) len:%d capa:%d", RSHAPE_LEN(shape_id), RSHAPE_CAPACITY(shape_id));
6414 }
6415
6416 APPEND_S("owner -> ");
6417 rb_raw_obj_info(BUFF_ARGS, CLASS_OF(obj));
6418
6419 break;
6420 }
6421 case imemo_ment:
6422 {
6423 const rb_method_entry_t *me = (const rb_method_entry_t *)obj;
6424
6425 APPEND_F(":%s (%s%s%s%s) type:%s aliased:%d owner:%p defined_class:%p",
6426 rb_id2name(me->called_id),
6427 METHOD_ENTRY_VISI(me) == METHOD_VISI_PUBLIC ? "pub" :
6428 METHOD_ENTRY_VISI(me) == METHOD_VISI_PRIVATE ? "pri" : "pro",
6429 METHOD_ENTRY_COMPLEMENTED(me) ? ",cmp" : "",
6430 METHOD_ENTRY_CACHED(me) ? ",cc" : "",
6431 METHOD_ENTRY_INVALIDATED(me) ? ",inv" : "",
6432 me->def ? rb_method_type_name(me->def->type) : "NULL",
6433 me->def ? me->def->aliased : -1,
6434 (void *)me->owner, // obj_info(me->owner),
6435 (void *)me->defined_class); //obj_info(me->defined_class)));
6436
6437 if (me->def) {
6438 switch (me->def->type) {
6439 case VM_METHOD_TYPE_ISEQ:
6440 APPEND_S(" (iseq:");
6441 rb_raw_obj_info(BUFF_ARGS, (VALUE)me->def->body.iseq.iseqptr);
6442 APPEND_S(")");
6443 break;
6444 default:
6445 break;
6446 }
6447 }
6448
6449 break;
6450 }
6451 case imemo_iseq: {
6452 const rb_iseq_t *iseq = (const rb_iseq_t *)obj;
6453 rb_raw_iseq_info(BUFF_ARGS, iseq);
6454 break;
6455 }
6456 case imemo_callinfo:
6457 {
6458 const struct rb_callinfo *ci = (const struct rb_callinfo *)obj;
6459 APPEND_F("(mid:%s, flag:%x argc:%d, kwarg:%s)",
6460 rb_id2name(vm_ci_mid(ci)),
6461 vm_ci_flag(ci),
6462 vm_ci_argc(ci),
6463 vm_ci_kwarg(ci) ? "available" : "NULL");
6464 break;
6465 }
6466 case imemo_callcache:
6467 {
6468 const struct rb_callcache *cc = (const struct rb_callcache *)obj;
6469 VALUE class_path = vm_cc_valid(cc) ? rb_mod_name(cc->klass) : Qnil;
6470 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
6471 const char *class_name;
6472 int class_name_len;
6473
6474 if (NIL_P(class_path)) {
6475 class_name = vm_cc_valid(cc) ? "??" : "<NULL>";
6476 class_name_len = vm_cc_valid(cc) ? 2 : 6;
6477 }
6478 else {
6479 class_name = RSTRING_PTR(class_path);
6480 class_name_len = str_len_no_raise(class_path);
6481 }
6482
6483 APPEND_F("(klass:%.*s cme:%s%s (%p) call:%p",
6484 class_name_len, class_name,
6485 cme ? rb_id2name(cme->called_id) : "<NULL>",
6486 cme ? (METHOD_ENTRY_INVALIDATED(cme) ? " [inv]" : "") : "",
6487 (void *)cme,
6488 (void *)(uintptr_t)vm_cc_call(cc));
6489 break;
6490 }
6491 default:
6492 break;
6493 }
6494 }
6495 default:
6496 break;
6497 }
6498 }
6499 end:
6500
6501 return pos;
6502}
6503
6504#undef C
6505
6506#ifdef RUBY_ASAN_ENABLED
6507void
6508rb_asan_poison_object(VALUE obj)
6509{
6510 MAYBE_UNUSED(struct RVALUE *) ptr = (void *)obj;
6511 asan_poison_memory_region(ptr, rb_gc_obj_slot_size(obj));
6512}
6513
6514void
6515rb_asan_unpoison_object(VALUE obj, bool newobj_p)
6516{
6517 MAYBE_UNUSED(struct RVALUE *) ptr = (void *)obj;
6518 asan_unpoison_memory_region(ptr, rb_gc_obj_slot_size(obj), newobj_p);
6519}
6520
6521void *
6522rb_asan_poisoned_object_p(VALUE obj)
6523{
6524 MAYBE_UNUSED(struct RVALUE *) ptr = (void *)obj;
6525 return __asan_region_is_poisoned(ptr, rb_gc_obj_slot_size(obj));
6526}
6527#endif
6528
6529static void
6530raw_obj_info(char *const buff, const size_t buff_size, VALUE obj)
6531{
6532 size_t pos = rb_raw_obj_info_common(buff, buff_size, obj);
6533 pos = rb_raw_obj_info_buitin_type(buff, buff_size, obj, pos);
6534 if (pos >= buff_size) {} // truncated
6535}
6536
6537const char *
6538rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj)
6539{
6540 void *objspace = rb_gc_get_objspace();
6541
6542 if (SPECIAL_CONST_P(obj)) {
6543 raw_obj_info(buff, buff_size, obj);
6544 }
6545 else if (!rb_gc_impl_live_object_p(objspace, (const void *)obj)) {
6546 snprintf(buff, buff_size, "out-of-heap:%p", (void *)obj);
6547 }
6548#if 0 // maybe no need to check it?
6549 else if (0 && rb_gc_impl_garbage_object_p(objspace, obj)) {
6550 snprintf(buff, buff_size, "garbage:%p", (void *)obj);
6551 }
6552#endif
6553 else {
6554 asan_unpoisoning_object(obj) {
6555 raw_obj_info(buff, buff_size, obj);
6556 }
6557 }
6558 return buff;
6559}
6560
6561#undef APPEND_S
6562#undef APPEND_F
6563#undef BUFF_ARGS
6564
6565/* Increments *var atomically and resets *var to 0 when maxval is
6566 * reached. Returns the wraparound old *var value (0...maxval). */
6567static rb_atomic_t
6568atomic_inc_wraparound(rb_atomic_t *var, const rb_atomic_t maxval)
6569{
6570 rb_atomic_t oldval = RUBY_ATOMIC_FETCH_ADD(*var, 1);
6571 if (RB_UNLIKELY(oldval >= maxval - 1)) { // wraparound *var
6572 const rb_atomic_t newval = oldval + 1;
6573 RUBY_ATOMIC_CAS(*var, newval, newval % maxval);
6574 oldval %= maxval;
6575 }
6576 return oldval;
6577}
6578
6579static const char *
6580obj_info(VALUE obj)
6581{
6582 if (RGENGC_OBJ_INFO) {
6583 static struct {
6584 rb_atomic_t index;
6585 char buffers[10][0x100];
6586 } info = {0};
6587
6588 rb_atomic_t index = atomic_inc_wraparound(&info.index, numberof(info.buffers));
6589 char *const buff = info.buffers[index];
6590 return rb_raw_obj_info(buff, sizeof(info.buffers[0]), obj);
6591 }
6592 return obj_type_name(obj);
6593}
6594
6595/*
6596 ------------------------ Extended allocator ------------------------
6597*/
6598
6600 VALUE exc;
6601 const char *fmt;
6602 va_list *ap;
6603};
6604
6605static void *
6606gc_vraise(void *ptr)
6607{
6608 struct gc_raise_tag *argv = ptr;
6609 rb_vraise(argv->exc, argv->fmt, *argv->ap);
6610 UNREACHABLE_RETURN(NULL);
6611}
6612
6613static void
6614gc_raise(VALUE exc, const char *fmt, ...)
6615{
6616 va_list ap;
6617 va_start(ap, fmt);
6618 struct gc_raise_tag argv = {
6619 exc, fmt, &ap,
6620 };
6621
6622 if (ruby_native_thread_p()) {
6623 rb_thread_call_with_gvl(gc_vraise, &argv);
6625 }
6626 else {
6627 /* Not in a ruby thread */
6628 fprintf(stderr, "%s", "[FATAL] ");
6629 vfprintf(stderr, fmt, ap);
6630 }
6631
6632 va_end(ap);
6633 abort();
6634}
6635
6636NORETURN(static void negative_size_allocation_error(const char *));
6637static void
6638negative_size_allocation_error(const char *msg)
6639{
6640 gc_raise(rb_eNoMemError, "%s", msg);
6641}
6642
6643static void *
6644ruby_memerror_body(void *dummy)
6645{
6646 rb_memerror();
6647 return 0;
6648}
6649
6650NORETURN(static void ruby_memerror(void));
6652static void
6653ruby_memerror(void)
6654{
6655 if (ruby_thread_has_gvl_p()) {
6656 rb_memerror();
6657 }
6658 else {
6659 if (ruby_native_thread_p()) {
6660 rb_thread_call_with_gvl(ruby_memerror_body, 0);
6661 }
6662 else {
6663 /* no ruby thread */
6664 fprintf(stderr, "[FATAL] failed to allocate memory\n");
6665 }
6666 }
6667
6668 /* We have discussions whether we should die here; */
6669 /* We might rethink about it later. */
6670 exit(EXIT_FAILURE);
6671}
6672
6673void
6675{
6676 /* the `GET_VM()->special_exceptions` below assumes that
6677 * the VM is reachable from the current thread. We should
6678 * definitely make sure of that. */
6680
6681 rb_execution_context_t *ec = GET_EC();
6682 VALUE exc = GET_VM()->special_exceptions[ruby_error_nomemory];
6683
6684 if (!exc ||
6685 rb_ec_raised_p(ec, RAISED_NOMEMORY) ||
6686 rb_ec_vm_lock_rec(ec) != ec->tag->lock_rec) {
6687 fprintf(stderr, "[FATAL] failed to allocate memory\n");
6688 exit(EXIT_FAILURE);
6689 }
6690 if (rb_ec_raised_p(ec, RAISED_NOMEMORY)) {
6691 rb_ec_raised_clear(ec);
6692 }
6693 else {
6694 rb_ec_raised_set(ec, RAISED_NOMEMORY);
6695 exc = ruby_vm_special_exception_copy(exc);
6696 }
6697 ec->errinfo = exc;
6698 EC_JUMP_TAG(ec, TAG_RAISE);
6699}
6700
6701bool
6702rb_memerror_reentered(void)
6703{
6704 rb_execution_context_t *ec = GET_EC();
6705 return (ec && rb_ec_raised_p(ec, RAISED_NOMEMORY));
6706}
6707
6708static void *
6709handle_malloc_failure(void *ptr)
6710{
6711 if (LIKELY(ptr)) {
6712 return ptr;
6713 }
6714 else {
6715 ruby_memerror();
6716 UNREACHABLE_RETURN(ptr);
6717 }
6718}
6719
6720static void *ruby_xmalloc_body(size_t size);
6721
6722void *
6723ruby_xmalloc(size_t size)
6724{
6725 if (RUBY_DTRACE_GC_XMALLOC_ENABLED()) {
6726 RUBY_DTRACE_GC_XMALLOC(1, size);
6727 }
6728
6729 return handle_malloc_failure(ruby_xmalloc_body(size));
6730}
6731
6732static bool
6733malloc_gc_allowed(void)
6734{
6735 rb_ractor_t *r = rb_current_ractor_raw(false);
6736
6737 return r == NULL || !r->malloc_gc_disabled;
6738}
6739
6740static void *
6741ruby_xmalloc_body(size_t size)
6742{
6743 if ((ssize_t)size < 0) {
6744 negative_size_allocation_error("too large allocation size");
6745 }
6746
6747 return rb_gc_impl_malloc(rb_gc_get_objspace(), size, malloc_gc_allowed());
6748}
6749
6750void
6751ruby_malloc_size_overflow(size_t count, size_t elsize)
6752{
6753 rb_raise(rb_eArgError,
6754 "malloc: possible integer overflow (%"PRIuSIZE"*%"PRIuSIZE")",
6755 count, elsize);
6756}
6757
6758void
6759ruby_malloc_add_size_overflow(size_t x, size_t y)
6760{
6761 rb_raise(rb_eArgError,
6762 "malloc: possible integer overflow (%"PRIuSIZE"+%"PRIuSIZE")",
6763 x, y);
6764}
6765
6766static void *ruby_xmalloc2_body(size_t n, size_t size);
6767
6768void *
6769ruby_xmalloc2(size_t n, size_t size)
6770{
6771 if (RUBY_DTRACE_GC_XMALLOC_ENABLED()) {
6772 RUBY_DTRACE_GC_XMALLOC(n, size);
6773 }
6774
6775 return handle_malloc_failure(ruby_xmalloc2_body(n, size));
6776}
6777
6778static void *
6779ruby_xmalloc2_body(size_t n, size_t size)
6780{
6781 return rb_gc_impl_malloc(rb_gc_get_objspace(), xmalloc2_size(n, size), malloc_gc_allowed());
6782}
6783
6784static void *ruby_xcalloc_body(size_t n, size_t size);
6785
6786void *
6787ruby_xcalloc(size_t n, size_t size)
6788{
6789 if (RUBY_DTRACE_GC_XCALLOC_ENABLED()) {
6790 RUBY_DTRACE_GC_XCALLOC(n, size);
6791 }
6792
6793 return handle_malloc_failure(ruby_xcalloc_body(n, size));
6794}
6795
6796static void *
6797ruby_xcalloc_body(size_t n, size_t size)
6798{
6799 return rb_gc_impl_calloc(rb_gc_get_objspace(), xmalloc2_size(n, size), malloc_gc_allowed());
6800}
6801
6802static void *ruby_xrealloc_sized_body(void *ptr, size_t new_size, size_t old_size);
6803
6804#ifdef ruby_xrealloc_sized
6805#undef ruby_xrealloc_sized
6806#endif
6807void *
6808ruby_xrealloc_sized(void *ptr, size_t new_size, size_t old_size)
6809{
6810 return handle_malloc_failure(ruby_xrealloc_sized_body(ptr, new_size, old_size));
6811}
6812
6813static void *
6814ruby_xrealloc_sized_body(void *ptr, size_t new_size, size_t old_size)
6815{
6816 if ((ssize_t)new_size < 0) {
6817 negative_size_allocation_error("too large allocation size");
6818 }
6819
6820 return rb_gc_impl_realloc(rb_gc_get_objspace(), ptr, new_size, old_size, malloc_gc_allowed());
6821}
6822
6823void *
6824ruby_xrealloc(void *ptr, size_t new_size)
6825{
6826 return ruby_xrealloc_sized(ptr, new_size, 0);
6827}
6828
6829static void *ruby_xrealloc2_sized_body(void *ptr, size_t n, size_t size, size_t old_n);
6830
6831#ifdef ruby_xrealloc2_sized
6832#undef ruby_xrealloc2_sized
6833#endif
6834void *
6835ruby_xrealloc2_sized(void *ptr, size_t n, size_t size, size_t old_n)
6836{
6837 return handle_malloc_failure(ruby_xrealloc2_sized_body(ptr, n, size, old_n));
6838}
6839
6840static void *
6841ruby_xrealloc2_sized_body(void *ptr, size_t n, size_t size, size_t old_n)
6842{
6843 size_t len = xmalloc2_size(n, size);
6844 return rb_gc_impl_realloc(rb_gc_get_objspace(), ptr, len, old_n * size, malloc_gc_allowed());
6845}
6846
6847void *
6848ruby_xrealloc2(void *ptr, size_t n, size_t size)
6849{
6850 return ruby_xrealloc2_sized(ptr, n, size, 0);
6851}
6852
6853#ifdef ruby_xfree_sized
6854#undef ruby_xfree_sized
6855#endif
6856
6857/*
6858 * This is a debugging flag for measuring the cost of `xfree`.
6859 * It can be enabled at compile time using `-DRUBY_NO_FREE`.
6860 * At run time, if the `RUBY_NO_FREE` environment variable is set to "1",
6861 * then `xfree` will not free any memory.
6862 */
6863#ifdef RUBY_NO_FREE
6864static bool g_nofree = false;
6865#endif
6866
6867void
6868ruby_xfree_sized(void *x, size_t size)
6869{
6870#ifdef RUBY_NO_FREE
6871 if (g_nofree) {
6872 return;
6873 }
6874#endif
6875
6876 if (RUBY_DTRACE_GC_XFREE_ENABLED()) {
6877 RUBY_DTRACE_GC_XFREE(x, size);
6878 }
6879
6880 if (LIKELY(x)) {
6881 /* It's possible for a C extension's pthread destructor function set by pthread_key_create
6882 * to be called after ruby_vm_destruct and attempt to free memory. Fall back to mimfree in
6883 * that case. */
6884 if (LIKELY(GET_VM())) {
6885 rb_gc_impl_free(rb_gc_get_objspace(), x, size);
6886 }
6887 else {
6888 ruby_mimfree(x);
6889 }
6890 }
6891}
6892
6893void
6894ruby_xfree(void *x)
6895{
6896 ruby_xfree_sized(x, 0);
6897}
6898
6899void *
6900rb_xmalloc_mul_add(size_t x, size_t y, size_t z) /* x * y + z */
6901{
6902 size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
6903 return ruby_xmalloc(w);
6904}
6905
6906void *
6907rb_xcalloc_mul_add(size_t x, size_t y, size_t z) /* x * y + z */
6908{
6909 size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
6910 return ruby_xcalloc(w, 1);
6911}
6912
6913void *
6914rb_xrealloc_mul_add(const void *p, size_t x, size_t y, size_t z) /* x * y + z */
6915{
6916 size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
6917 return ruby_xrealloc((void *)p, w);
6918}
6919
6920void *
6921rb_xmalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
6922{
6923 size_t u = size_mul_add_mul_or_raise(x, y, z, w, rb_eArgError);
6924 return ruby_xmalloc(u);
6925}
6926
6927void *
6928rb_xcalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
6929{
6930 size_t u = size_mul_add_mul_or_raise(x, y, z, w, rb_eArgError);
6931 return ruby_xcalloc(u, 1);
6932}
6933
6934#ifdef USE_MIMALLOC
6935# define malloc(n) mi_malloc(n)
6936# define calloc(n, s) mi_calloc(n, s)
6937# define free(p) mi_free(p)
6938#endif
6939
6940/* Mimic ruby_xmalloc, but need not rb_objspace.
6941 * should return pointer suitable for ruby_xfree
6942 */
6943void *
6944ruby_mimmalloc(size_t size)
6945{
6946 void *mem;
6947#if CALC_EXACT_MALLOC_SIZE
6948 size += sizeof(struct malloc_obj_info);
6949#endif
6950 mem = malloc(size);
6951#if CALC_EXACT_MALLOC_SIZE
6952 if (!mem) {
6953 return NULL;
6954 }
6955 else
6956 /* set 0 for consistency of allocated_size/allocations */
6957 {
6958 struct malloc_obj_info *info = mem;
6959 info->size = 0;
6960 mem = info + 1;
6961 }
6962#endif
6963 return mem;
6964}
6965
6966void *
6967ruby_mimcalloc(size_t num, size_t size)
6968{
6969 void *mem;
6970#if CALC_EXACT_MALLOC_SIZE
6971 struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(num, size);
6972 if (UNLIKELY(t.overflowed)) {
6973 return NULL;
6974 }
6975 size = t.result + sizeof(struct malloc_obj_info);
6976 mem = calloc1(size);
6977 if (!mem) {
6978 return NULL;
6979 }
6980 else
6981 /* set 0 for consistency of allocated_size/allocations */
6982 {
6983 struct malloc_obj_info *info = mem;
6984 info->size = 0;
6985 mem = info + 1;
6986 }
6987#else
6988 mem = calloc(num, size);
6989#endif
6990 return mem;
6991}
6992
6993void
6994ruby_mimfree(void *ptr)
6995{
6996#if CALC_EXACT_MALLOC_SIZE
6997 struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
6998 ptr = info;
6999#endif
7000 free(ptr);
7001}
7002
7003#ifdef USE_MIMALLOC
7004# undef malloc
7005# undef calloc
7006# undef free
7007#endif
7008
7009void
7010rb_gc_adjust_memory_usage(ssize_t diff)
7011{
7012 unless_objspace(objspace) { return; }
7013
7014 rb_gc_impl_adjust_memory_usage(objspace, diff);
7015}
7016
7017const char *
7018rb_obj_info(VALUE obj)
7019{
7020 return obj_info(obj);
7021}
7022
7023void
7024rb_obj_info_dump(VALUE obj)
7025{
7026 char buff[0x100];
7027 fprintf(stderr, "rb_obj_info_dump: %s\n", rb_raw_obj_info(buff, 0x100, obj));
7028}
7029
7030void
7031rb_obj_info_dump_loc(VALUE obj, const char *file, int line, const char *func)
7032{
7033 char buff[0x100];
7034 fprintf(stderr, "<OBJ_INFO:%s@%s:%d> %s\n", func, file, line, rb_raw_obj_info(buff, 0x100, obj));
7035}
7036
7037void
7038rb_gc_before_fork(void)
7039{
7040 rb_gc_impl_before_fork(rb_gc_get_objspace());
7041}
7042
7043void
7044rb_gc_after_fork(rb_pid_t pid)
7045{
7046 rb_gc_impl_after_fork(rb_gc_get_objspace(), pid);
7047}
7048
7049bool
7050rb_gc_obj_shareable_p(VALUE obj)
7051{
7052 return RB_OBJ_SHAREABLE_P(obj);
7053}
7054
7055void
7056rb_gc_rp(VALUE obj)
7057{
7058 rp(obj);
7059}
7060
7062 VALUE parent;
7063 long err_count;
7064};
7065
7066static void
7067check_shareable_i(const VALUE child, void *ptr)
7068{
7069 struct check_shareable_data *data = (struct check_shareable_data *)ptr;
7070
7071 if (!rb_gc_obj_shareable_p(child)) {
7072 /* A shareable object may reference an unshareable one only if the write barrier
7073 * recorded the edge in the target's shref bit (keeping it alive past its owner's
7074 * local GC). Root-like exceptions (Ractor private fields, cref, JIT) are hidden
7075 * while checking_shareable is set. */
7076 if (rb_gc_impl_shref_marked_p(rb_gc_get_objspace(), child)) {
7077 return;
7078 }
7079
7080 fprintf(stderr, "(a) ");
7081 rb_gc_rp(data->parent);
7082 fprintf(stderr, "(b) ");
7083 rb_gc_rp(child);
7084 fprintf(stderr, "check_shareable_i: shareable (a) -> unshareable (b) without a shref record\n");
7085
7086 data->err_count++;
7087 rb_bug("!! violate shareable constraint !!");
7088 }
7089}
7090
7091/* List obj's direct children one level deep through the traversal API and check the
7092 * shareable constraint: a shareable object's child is either shareable or an
7093 * unshareable one with a recorded shref. The "verification walk in progress" marker
7094 * lives in the per-Ractor mark_func_data slot: a process-global flag would make the
7095 * lock-free local GC of an unrelated Ractor hit the mark gate too, skip marking a live
7096 * object's children (its fields imemo, say) and let the sweep collect them. (Upstream
7097 * could use a global flag, since its GC always runs under the VM lock.) The slot is
7098 * private to this Ractor and the walk is synchronous, so no lock is needed. */
7099void
7100rb_gc_verify_shareable(VALUE obj)
7101{
7102 struct check_shareable_data data = {
7103 .parent = obj,
7104 .err_count = 0,
7105 };
7106
7107 if (!RB_SPECIAL_CONST_P(obj)) {
7108 struct gc_mark_func_data_struct **mfdp = GC_MARK_FUNC_DATA_SLOTP();
7109 struct gc_mark_func_data_struct *prev_mfd = *mfdp;
7110 struct gc_mark_func_data_struct mfd = {
7111 .mark_func = check_shareable_i,
7112 .data = &data,
7113 .checking_shareable = true,
7114 };
7115
7116 *mfdp = &mfd;
7117 rb_gc_mark_children(rb_gc_get_objspace(), obj);
7118 *mfdp = prev_mfd;
7119 }
7120
7121 if (data.err_count > 0) {
7122 rb_bug("rb_gc_verify_shareable");
7123 }
7124}
7125
7126bool
7127rb_gc_checking_shareable(void)
7128{
7129 const struct gc_mark_func_data_struct *mfd = *GC_MARK_FUNC_DATA_SLOTP();
7130 return mfd && mfd->checking_shareable;
7131}
7132
7133/*
7134 * Document-module: ObjectSpace
7135 *
7136 * The ObjectSpace module contains a number of routines
7137 * that interact with the garbage collection facility and allow you to
7138 * traverse all living objects with an iterator.
7139 *
7140 * ObjectSpace also provides support for object finalizers, procs that will be
7141 * called after a specific object was destroyed by garbage collection. See
7142 * the documentation for +ObjectSpace.define_finalizer+ for important
7143 * information on how to use this method correctly.
7144 *
7145 * a = "A"
7146 * b = "B"
7147 *
7148 * ObjectSpace.define_finalizer(a, proc {|id| puts "Finalizer one on #{id}" })
7149 * ObjectSpace.define_finalizer(b, proc {|id| puts "Finalizer two on #{id}" })
7150 *
7151 * a = nil
7152 * b = nil
7153 *
7154 * _produces:_
7155 *
7156 * Finalizer two on 537763470
7157 * Finalizer one on 537763480
7158 */
7159
7160#include "gc.rbinc"
7161
7162void
7163Init_GC(void)
7164{
7165#ifdef RUBY_NO_FREE
7166 const char* nofree_str = getenv("RUBY_NO_FREE");
7167 if (nofree_str && strcmp(nofree_str, "1") == 0) {
7168 fprintf(stderr, "WARNING: Enabling no-free mode! xfree() will never free anything!\n");
7169 g_nofree = true;
7170 }
7171#endif
7172
7173#undef rb_intern
7174 malloc_offset = gc_compute_malloc_offset();
7175
7176 rb_mGC = rb_define_module("GC");
7177
7178 VALUE rb_mObjSpace = rb_define_module("ObjectSpace");
7179
7180 rb_define_module_function(rb_mObjSpace, "each_object", os_each_obj, -1);
7181
7182 rb_define_module_function(rb_mObjSpace, "define_finalizer", define_final, -1);
7183 rb_define_module_function(rb_mObjSpace, "undefine_finalizer", undefine_final, 1);
7184
7185 rb_vm_register_special_exception(ruby_error_nomemory, rb_eNoMemError, "failed to allocate memory");
7186
7187 rb_define_method(rb_cBasicObject, "__id__", rb_obj_id, 0);
7188 rb_define_method(rb_mKernel, "object_id", rb_obj_id, 0);
7189
7190 rb_define_module_function(rb_mObjSpace, "count_objects", count_objects, -1);
7191
7192 rb_gc_impl_init();
7193}
7194
7195// Set a name for the anonymous virtual memory area. `addr` is the starting
7196// address of the area and `size` is its length in bytes. `name` is a
7197// NUL-terminated human-readable string.
7198//
7199// This function is usually called after calling `mmap()`. The human-readable
7200// annotation helps developers identify the call site of `mmap()` that created
7201// the memory mapping.
7202//
7203// This function currently only works on Linux 5.17 or higher. After calling
7204// this function, we can see annotations in the form of "[anon:...]" in
7205// `/proc/self/maps`, where `...` is the content of `name`. This function has
7206// no effect when called on other platforms.
7207void
7208ruby_annotate_mmap(const void *addr, unsigned long size, const char *name)
7209{
7210#if defined(HAVE_SYS_PRCTL_H) && defined(PR_SET_VMA) && defined(PR_SET_VMA_ANON_NAME)
7211 // The name length cannot exceed 80 (including the '\0').
7212 RUBY_ASSERT(strlen(name) < 80);
7213 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (unsigned long)addr, size, name);
7214 // We ignore errors in prctl. prctl may set errno to EINVAL for several
7215 // reasons.
7216 // 1. The attr (PR_SET_VMA_ANON_NAME) is not a valid attribute.
7217 // 2. addr is an invalid address.
7218 // 3. The string pointed by name is too long.
7219 // The first error indicates PR_SET_VMA_ANON_NAME is not available, and may
7220 // happen if we run the compiled binary on an old kernel. In theory, all
7221 // other errors should result in a failure. But since EINVAL cannot tell
7222 // the first error from others, and this function is mainly used for
7223 // debugging, we silently ignore the error.
7224 errno = 0;
7225#endif
7226}
#define RUBY_ATOMIC_VALUE_CAS(var, oldval, newval)
Identical to RUBY_ATOMIC_CAS, except it expects its arguments are VALUE.
Definition atomic.h:416
#define RUBY_ATOMIC_PTR_LOAD(var)
Identical to RUBY_ATOMIC_LOAD, except it expects its arguments are void*.
Definition atomic.h:348
#define RUBY_ATOMIC_SIZE_FETCH_ADD(var, val)
Identical to RUBY_ATOMIC_FETCH_ADD, except it expects its arguments to be size_t.
Definition atomic.h:235
#define RUBY_ATOMIC_INC(var)
Atomically increments the value pointed by var.
Definition atomic.h:214
#define RUBY_ATOMIC_CAS(var, oldval, newval)
Atomic compare-and-swap.
Definition atomic.h:165
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
Definition atomic.h:69
#define RUBY_ATOMIC_FETCH_ADD(var, val)
Atomically replaces the value pointed by var with the result of addition of val to the old value of v...
Definition atomic.h:118
#define RUBY_ATOMIC_DEC(var)
Atomically decrements the value pointed by var.
Definition atomic.h:223
#define RUBY_ATOMIC_LOAD(var)
Atomic load.
Definition atomic.h:175
#define RUBY_ATOMIC_SET(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except for the return type.
Definition atomic.h:185
#define rb_define_method(klass, mid, func, arity)
Defines klass#mid.
#define rb_define_module_function(klass, mid, func, arity)
Defines klass#mid and makes it a module function.
unsigned int rb_postponed_job_handle_t
The type of a handle returned from rb_postponed_job_preregister and passed to rb_postponed_job_trigge...
Definition debug.h:703
void rb_postponed_job_trigger(rb_postponed_job_handle_t h)
Triggers a pre-registered job registered with rb_postponed_job_preregister, scheduling it for executi...
Definition vm_trace.c:1899
rb_postponed_job_handle_t rb_postponed_job_preregister(unsigned int flags, rb_postponed_job_func_t func, void *data)
Pre-registers a func in Ruby's postponed job preregistration table, returning an opaque handle which ...
Definition vm_trace.c:1865
#define RUBY_INTERNAL_EVENT_FREEOBJ
Object swept.
Definition event.h:94
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
#define RUBY_INTERNAL_EVENT_NEWOBJ
Object allocated.
Definition event.h:93
static bool RB_OBJ_FROZEN(VALUE obj)
Checks if an object is frozen.
Definition fl_type.h:714
@ RUBY_FL_WB_PROTECTED
Definition fl_type.h:186
int rb_scan_args(int argc, const VALUE *argv, const char *fmt,...)
Retrieves argument from argc and argv to given VALUE references according to the format string.
Definition class.c:3358
#define T_COMPLEX
Old name of RUBY_T_COMPLEX.
Definition value_type.h:59
#define TYPE(_)
Old name of rb_type.
Definition value_type.h:108
#define FL_SINGLETON
Old name of RUBY_FL_SINGLETON.
Definition fl_type.h:58
#define T_FILE
Old name of RUBY_T_FILE.
Definition value_type.h:62
#define T_STRING
Old name of RUBY_T_STRING.
Definition value_type.h:78
#define xfree
Old name of ruby_xfree.
Definition xmalloc.h:58
#define T_MASK
Old name of RUBY_T_MASK.
Definition value_type.h:68
#define Qundef
Old name of RUBY_Qundef.
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
Definition fl_type.h:133
#define T_NIL
Old name of RUBY_T_NIL.
Definition value_type.h:72
#define UNREACHABLE
Old name of RBIMPL_UNREACHABLE.
Definition assume.h:28
#define T_FLOAT
Old name of RUBY_T_FLOAT.
Definition value_type.h:64
#define T_IMEMO
Old name of RUBY_T_IMEMO.
Definition value_type.h:67
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
#define T_BIGNUM
Old name of RUBY_T_BIGNUM.
Definition value_type.h:57
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
#define T_STRUCT
Old name of RUBY_T_STRUCT.
Definition value_type.h:79
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
Definition fl_type.h:131
#define T_FIXNUM
Old name of RUBY_T_FIXNUM.
Definition value_type.h:63
#define UNREACHABLE_RETURN
Old name of RBIMPL_UNREACHABLE_RETURN.
Definition assume.h:29
#define SYM2ID
Old name of RB_SYM2ID.
Definition symbol.h:45
#define T_DATA
Old name of RUBY_T_DATA.
Definition value_type.h:60
#define FIXNUM_FLAG
Old name of RUBY_FIXNUM_FLAG.
#define LL2NUM
Old name of RB_LL2NUM.
Definition long_long.h:30
#define CLASS_OF
Old name of rb_class_of.
Definition globals.h:205
#define T_NONE
Old name of RUBY_T_NONE.
Definition value_type.h:74
#define T_NODE
Old name of RUBY_T_NODE.
Definition value_type.h:73
#define SIZET2NUM
Old name of RB_SIZE2NUM.
Definition size_t.h:62
#define FL_FINALIZE
Old name of RUBY_FL_FINALIZE.
Definition fl_type.h:61
#define T_MODULE
Old name of RUBY_T_MODULE.
Definition value_type.h:70
#define ASSUME
Old name of RBIMPL_ASSUME.
Definition assume.h:27
#define T_TRUE
Old name of RUBY_T_TRUE.
Definition value_type.h:81
#define T_RATIONAL
Old name of RUBY_T_RATIONAL.
Definition value_type.h:76
#define T_ICLASS
Old name of RUBY_T_ICLASS.
Definition value_type.h:66
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
#define ALLOC_N
Old name of RB_ALLOC_N.
Definition memory.h:402
#define FL_ABLE
Old name of RB_FL_ABLE.
Definition fl_type.h:118
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
Definition fl_type.h:128
#define rb_ary_new3
Old name of rb_ary_new_from_args.
Definition array.h:658
#define LONG2NUM
Old name of RB_LONG2NUM.
Definition long.h:50
#define T_FALSE
Old name of RUBY_T_FALSE.
Definition value_type.h:61
#define ULL2NUM
Old name of RB_ULL2NUM.
Definition long_long.h:31
#define T_UNDEF
Old name of RUBY_T_UNDEF.
Definition value_type.h:82
#define T_ZOMBIE
Old name of RUBY_T_ZOMBIE.
Definition value_type.h:83
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define FIX2LONG
Old name of RB_FIX2LONG.
Definition long.h:46
#define T_ARRAY
Old name of RUBY_T_ARRAY.
Definition value_type.h:56
#define T_OBJECT
Old name of RUBY_T_OBJECT.
Definition value_type.h:75
#define NIL_P
Old name of RB_NIL_P.
#define FL_WB_PROTECTED
Old name of RUBY_FL_WB_PROTECTED.
Definition fl_type.h:59
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition value_type.h:80
#define T_MATCH
Old name of RUBY_T_MATCH.
Definition value_type.h:69
#define T_CLASS
Old name of RUBY_T_CLASS.
Definition value_type.h:58
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
Definition value_type.h:85
#define T_MOVED
Old name of RUBY_T_MOVED.
Definition value_type.h:71
#define FL_TEST
Old name of RB_FL_TEST.
Definition fl_type.h:127
#define xcalloc
Old name of ruby_xcalloc.
Definition xmalloc.h:55
#define FL_UNSET
Old name of RB_FL_UNSET.
Definition fl_type.h:129
#define FIXNUM_P
Old name of RB_FIXNUM_P.
#define NUM2SIZET
Old name of RB_NUM2SIZE.
Definition size_t.h:61
#define SYMBOL_P
Old name of RB_SYMBOL_P.
Definition value_type.h:88
#define T_REGEXP
Old name of RUBY_T_REGEXP.
Definition value_type.h:77
size_t ruby_stack_length(VALUE **p)
Queries what Ruby thinks is the machine stack.
Definition gc.c:2923
int ruby_stack_check(void)
Checks for stack overflow.
Definition gc.c:2970
VALUE rb_eNoMemError
NoMemoryError exception.
Definition error.c:1468
#define ruby_verbose
This variable controls whether the interpreter is in debug mode.
Definition error.h:476
VALUE rb_eTypeError
TypeError exception.
Definition error.c:1457
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1455
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:464
VALUE rb_mKernel
Kernel module.
Definition object.c:59
VALUE rb_cObject
Object class.
Definition object.c:60
VALUE rb_mGC
GC module.
Definition gc.c:477
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
Definition object.c:230
VALUE rb_cBasicObject
BasicObject class.
Definition object.c:58
VALUE rb_class_real(VALUE klass)
Finds a "real" class.
Definition object.c:221
VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass)
Queries if the given object is an instance (of possibly descendants) of the given class.
Definition object.c:902
#define RUBY_ASSERT_ALWAYS(expr,...)
A variant of RUBY_ASSERT that does not interface with RUBY_DEBUG.
Definition assert.h:199
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:219
void rb_memerror(void)
Triggers out-of-memory error.
Definition gc.c:6674
VALUE rb_gc_disable(void)
Disables GC.
Definition gc.c:5906
VALUE rb_gc_start(void)
Identical to rb_gc(), except the return value.
Definition gc.c:5688
VALUE rb_gc_enable(void)
(Re-) enables GC.
Definition gc.c:5894
int rb_during_gc(void)
Queries if the GC is busy.
Definition gc.c:5703
void rb_gc(void)
Triggers a GC process.
Definition gc.c:5695
size_t rb_gc_count(void)
Identical to rb_gc_stat(), with "count" parameter.
Definition gc.c:5711
void rb_ary_free(VALUE ary)
Destroys the given array for no reason.
VALUE rb_ary_new(void)
Allocates a new, empty array.
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
#define RETURN_ENUMERATOR(obj, argc, argv)
Identical to RETURN_SIZED_ENUMERATOR(), except its size is unknown.
Definition enumerator.h:242
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
Definition error.h:284
VALUE rb_hash_new(void)
Creates a new, empty hash object.
Definition hash.c:1646
VALUE rb_block_proc(void)
Constructs a Proc object from implicitly passed components.
Definition proc.c:1586
VALUE rb_obj_is_proc(VALUE recv)
Queries if the given object is a proc.
Definition proc.c:392
void rb_str_free(VALUE str)
Destroys the given string for no reason.
Definition string.c:1817
size_t rb_str_capacity(VALUE str)
Queries the capacity of the given string.
Definition string.c:1037
VALUE rb_mod_name(VALUE mod)
Queries the name of a module.
Definition variable.c:151
void rb_free_generic_ivar(VALUE obj)
Frees the list of instance variables.
Definition variable.c:1449
void rb_undef_alloc_func(VALUE klass)
Deletes the allocator function of a class.
Definition vm_method.c:1893
VALUE rb_check_funcall(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it returns RUBY_Qundef instead of raising rb_eNoMethodError.
Definition vm_eval.c:716
rb_alloc_func_t rb_get_alloc_func(VALUE klass)
Queries the allocator function of a class.
Definition vm_method.c:1902
int rb_obj_respond_to(VALUE obj, ID mid, int private_p)
Identical to rb_respond_to(), except it additionally takes the visibility parameter.
Definition vm_method.c:3736
VALUE rb_sym2str(VALUE symbol)
Obtain a frozen string representation of a symbol (not including the leading colon).
Definition symbol.c:1155
int capa
Designed capacity of the buffer.
Definition io.h:11
char * ptr
Pointer to the underlying memory region, of at least capa bytes.
Definition io.h:2
int rb_io_fptr_finalize(rb_io_t *fptr)
Destroys the given IO.
Definition io.c:5899
int len
Length of the buffer.
Definition io.h:8
static bool rb_ractor_shareable_p(VALUE obj)
Queries if multiple Ractors can share the passed object or not.
Definition ractor.h:269
#define RB_OBJ_SHAREABLE_P(obj)
Queries if the passed object has previously classified as shareable or not.
Definition ractor.h:255
int ruby_thread_has_gvl_p(void)
Whether the current thread is holding the GVL.
Definition thread.c:2364
void * rb_thread_call_with_gvl(void *(*func)(void *), void *data1)
(Re-)acquires the GVL.
Definition thread.c:2323
bool ruby_free_at_exit_p(void)
Returns whether the Ruby VM will free all memory at shutdown.
Definition vm.c:5287
VALUE rb_yield(VALUE val)
Yields the block.
Definition vm_eval.c:1396
#define RBIMPL_ATTR_MAYBE_UNUSED()
Wraps (or simulates) [[maybe_unused]].
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:375
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:170
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
Definition memory.h:387
VALUE type(ANYARGS)
ANYARGS-ed function type.
int st_foreach(st_table *q, int_type *w, st_data_t e)
Iteration over the given table.
#define RARRAY_LEN
Just another name of rb_array_len.
Definition rarray.h:50
#define RARRAY(obj)
Convenient casting macro.
Definition rarray.h:44
#define RARRAY_AREF(a, i)
References an object in an array.
Definition rarray.h:422
#define RARRAY_CONST_PTR
Just another name of rb_array_const_ptr.
Definition rarray.h:51
static VALUE RBASIC_CLASS(VALUE obj)
Queries the class of an object.
Definition rbasic.h:166
#define RBASIC(obj)
Convenient casting macro.
Definition rbasic.h:40
#define RCLASS(obj)
Convenient casting macro.
Definition rclass.h:38
#define RUBY_DEFAULT_FREE
This is a value you can set to RData::dfree.
Definition rdata.h:56
void(* RUBY_DATA_FUNC)(void *)
This is the type of callbacks registered to RData.
Definition rdata.h:69
#define RUBY_NEVER_FREE
This is a value you can set to RData::dfree.
Definition rdata.h:63
#define RFILE(obj)
Convenient casting macro.
Definition rfile.h:50
#define RHASH_SIZE(h)
Queries the size of the hash.
Definition rhash.h:57
#define RHASH_EMPTY_P(h)
Checks if the hash is empty.
Definition rhash.h:67
#define RMATCH(obj)
Convenient casting macro.
Definition rmatch.h:37
#define ROBJECT(obj)
Convenient casting macro.
Definition robject.h:43
#define RREGEXP(obj)
Convenient casting macro.
Definition rregexp.h:37
static struct re_pattern_buffer * RREGEXP_PTR(VALUE rexp)
Convenient getter function.
Definition rregexp.h:77
static int RSTRING_LENINT(VALUE str)
Identical to RSTRING_LEN(), except it differs for the return type.
Definition rstring.h:438
#define RSTRING(obj)
Convenient casting macro.
Definition rstring.h:41
static rb_len_t RSTRUCT_LEN(VALUE st)
Returns the number of struct members.
Definition rstruct.h:82
#define RUBY_TYPED_DEFAULT_FREE
This is a value you can set to rb_data_type_struct::dfree.
Definition rtypeddata.h:81
VALUE rb_data_typed_object_wrap(VALUE klass, void *datap, const rb_data_type_t *type)
This is the primitive way to wrap an existing C struct into RTypedData.
Definition gc.c:1427
VALUE rb_data_typed_object_zalloc(VALUE klass, size_t size, const rb_data_type_t *type)
Identical to rb_data_typed_object_wrap(), except it allocates a new data region internally instead of...
Definition gc.c:1464
#define RUBY_TYPED_FREE_IMMEDIATELY
Macros to see if each corresponding flag is defined.
Definition rtypeddata.h:122
#define DATA_PTR(obj)
Convenient casting macro for backward compatibility.
Definition rtypeddata.h:439
static const rb_data_type_t * RTYPEDDATA_TYPE(VALUE obj)
Queries for the type of given object.
Definition rtypeddata.h:692
#define RDATA(obj)
Convenient casting macro for backward compatibility.
Definition rtypeddata.h:431
struct rb_data_type_struct rb_data_type_t
This is the struct that holds necessary info for a struct.
Definition rtypeddata.h:239
#define RTYPEDDATA(obj)
Convenient casting macro.
Definition rtypeddata.h:96
const char * rb_obj_classname(VALUE obj)
Queries the name of the class of the passed object.
Definition variable.c:532
void rb_p(VALUE obj)
Inspects an object.
Definition io.c:9288
#define errno
Ractor-aware version of errno.
Definition ruby.h:388
int ruby_native_thread_p(void)
Queries if the thread which calls this function is a ruby's thread.
Definition thread.c:6183
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
#define RTEST
This is an old name of RB_TEST.
Defines old _.
#define _(args)
This was a transition path from K&R to ANSI.
Definition stdarg.h:35
Ruby's array.
Definition rarray.h:127
Ruby object's base components.
Definition rbasic.h:69
Regular expression execution context.
Definition rmatch.h:79
struct rmatch_offset * char_offset
Capture group offsets, in C array.
Definition rmatch.h:98
union RMatch::@357327212031205367147161324375004050200256000107 as
"Registers" of a match.
int char_offset_num_allocated
Number of rmatch_offset that rmatch::char_offset holds.
Definition rmatch.h:95
int num_regs
Number of capture-group registers.
Definition rmatch.h:101
Ruby's ordinal objects.
Definition robject.h:51
Ruby's String.
Definition rstring.h:196
"Typed" user data.
Definition rtypeddata.h:397
void * data
Pointer to the actual C level struct that you want to wrap.
Definition rtypeddata.h:417
VALUE fields_obj
Direct reference to the slots that holds instance variables, if any.
Definition rtypeddata.h:403
struct rb_data_type_struct::@006053276242002167166045007236206132153037024024 function
Function pointers.
RUBY_DATA_FUNC dcompact
This function is called when the object is relocated.
Definition rtypeddata.h:293
const char * wrap_struct_name
Name of structs of this kind.
Definition rtypeddata.h:249
RUBY_DATA_FUNC dmark
This function is called when the object is experiencing GC marks.
Definition rtypeddata.h:263
Definition gc_impl.h:34
Ruby's IO, metadata and buffers.
Definition io.h:295
const rb_iseq_t * iseqptr
iseq pointer, should be separated from iseqval
Definition method.h:143
Represents the region of a capture group.
Definition rmatch.h:65
Definition st.h:79
Definition string.c:10439
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
long rb_len_t
A signed integer type for lengths and indices of objects such as String and Array.
Definition value.h:142
intptr_t SIGNED_VALUE
A signed integer type that has the same width with VALUE.
Definition value.h:63
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:376
ruby_value_type
C-level type of an object.
Definition value_type.h:113
@ RUBY_T_MASK
Bitmask of ruby_value_type.
Definition value_type.h:145