Ruby 4.1.0dev (2026-11-17 revision cd5a4468da76f4942b4796686f51c9f6fe59b0d7)
default.c
1#include "ruby/internal/config.h"
2
3#include <signal.h>
4#include <string.h>
5
6#ifndef _WIN32
7# include <sys/mman.h>
8# include <unistd.h>
9# include <fcntl.h>
10# ifdef HAVE_SYS_PRCTL_H
11# include <sys/prctl.h>
12# endif
13#endif
14
15#if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H)
16/* LIST_HEAD conflicts with sys/queue.h on macOS */
17# include <sys/user.h>
18#endif
19
20#ifdef BUILDING_MODULAR_GC
21# define nlz_int64(x) (x == 0 ? 64 : (unsigned int)__builtin_clzll((unsigned long long)x))
22# define rb_popcount_intptr(x) ((unsigned int)__builtin_popcountll((unsigned long long)(x)))
23# define rb_popcount32(x) ((unsigned int)__builtin_popcount((unsigned int)(x)))
24# define ntz_int32(x) ((int)((x) == 0 ? 32 : __builtin_ctz((unsigned int)(x))))
25#else
26# include "internal/bits.h"
27#endif
28
29#include "ruby/ruby.h"
30#include "ruby/atomic.h"
31#include "ruby_atomic.h"
32#include "ruby/debug.h"
33#include "ruby/thread.h"
34#include "ruby/util.h"
35#include "ruby/vm.h"
37#include "ccan/list/list.h"
38#include "darray.h"
39#include "gc/gc.h"
40#include "gc/gc_impl.h"
41#include "yjit.h"
42#include "zjit.h"
43#include "internal/static_assert.h"
44#include "internal/vm_map.h"
45
46#ifdef BUILDING_MODULAR_GC
47/* hrtime.h transitively includes internal/time.h -> internal/bits.h, which are
48 * not available to out-of-tree modular GC builds. We only use a monotonic
49 * clock plus saturating add/sub, so provide that subset locally with the same
50 * semantics as hrtime.h. */
51# include <time.h>
52typedef uint64_t rb_hrtime_t;
53# define RB_HRTIME_PER_SEC ((rb_hrtime_t)1000000000)
54
55static inline rb_hrtime_t
56rb_hrtime_now(void)
57{
58# if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC)
59 struct timespec ts;
60 if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) {
61 return (rb_hrtime_t)ts.tv_sec * RB_HRTIME_PER_SEC + (rb_hrtime_t)ts.tv_nsec;
62 }
63# endif
64 return 0;
65}
66
67static inline rb_hrtime_t
68rb_hrtime_add(rb_hrtime_t a, rb_hrtime_t b)
69{
70 rb_hrtime_t c = a + b;
71 return c < a ? UINT64_MAX : c; /* saturate on overflow */
72}
73
74static inline rb_hrtime_t
75rb_hrtime_sub(rb_hrtime_t a, rb_hrtime_t b)
76{
77 return a < b ? 0 : a - b;
78}
79#else
80# include "hrtime.h"
81#endif
82
83#include "probes.h"
84
85/* cl.exe's traditional preprocessor passes __VA_ARGS__ to a nested macro as
86 * a single argument; the extra expansion re-scans it into separate ones. */
87#define RUBY_DTRACE_GC_HOOK_EXPAND(expr) expr
88#define RUBY_DTRACE_GC_HOOK(name, ...) \
89 do {if (RUBY_DTRACE_GC_##name##_ENABLED()) RUBY_DTRACE_GC_HOOK_EXPAND(RUBY_DTRACE_GC_##name(__VA_ARGS__));} while (0)
90
91#if USE_ZJIT
92# include "gc/default/zjit_fastpath.h"
93#endif
94
95#ifdef BUILDING_MODULAR_GC
96# define RB_DEBUG_COUNTER_INC(_name) ((void)0)
97# define RB_DEBUG_COUNTER_INC_IF(_name, cond) (!!(cond))
98#else
99# include "debug_counter.h"
100#endif
101
102#ifdef BUILDING_MODULAR_GC
103# define rb_asan_poison_object(obj) ((void)(obj))
104# define rb_asan_unpoison_object(obj, newobj_p) ((void)(obj), (void)(newobj_p))
105# define asan_unpoisoning_object(obj) if ((obj) || true)
106# define asan_poison_memory_region(ptr, size) ((void)(ptr), (void)(size))
107# define asan_unpoison_memory_region(ptr, size, malloc_p) ((void)(ptr), (size), (malloc_p))
108# define asan_unpoisoning_memory_region(ptr, size) if ((ptr) || (size) || true)
109
110# define VALGRIND_MAKE_MEM_DEFINED(ptr, size) ((void)(ptr), (void)(size))
111# define VALGRIND_MAKE_MEM_UNDEFINED(ptr, size) ((void)(ptr), (void)(size))
112#else
113# include "internal/sanitizers.h"
114#endif
115
116/* MALLOC_HEADERS_BEGIN */
117#ifndef HAVE_MALLOC_USABLE_SIZE
118# ifdef USE_MIMALLOC
119# define HAVE_MALLOC_USABLE_SIZE
120# define malloc_usable_size(a) mi_usable_size(a)
121# elif defined _WIN32
122# define HAVE_MALLOC_USABLE_SIZE
123# define malloc_usable_size(a) _msize(a)
124# elif defined HAVE_MALLOC_SIZE
125# define HAVE_MALLOC_USABLE_SIZE
126# define malloc_usable_size(a) malloc_size(a)
127# endif
128#endif
129
130#ifdef HAVE_MALLOC_USABLE_SIZE
131# ifdef RUBY_ALTERNATIVE_MALLOC_HEADER
132/* Alternative malloc header is included in ruby/missing.h */
133# elif defined(USE_MIMALLOC)
134# pragma push_macro("restrict")
135# undef restrict
136# include <mimalloc.h>
137# pragma pop_macro("restrict")
138# elif defined(HAVE_MALLOC_H)
139# include <malloc.h>
140# elif defined(HAVE_MALLOC_NP_H)
141# include <malloc_np.h>
142# elif defined(HAVE_MALLOC_MALLOC_H)
143# include <malloc/malloc.h>
144# endif
145#endif
146
147#ifdef HAVE_MALLOC_TRIM
148# include <malloc.h>
149
150# ifdef __EMSCRIPTEN__
151/* malloc_trim is defined in emscripten/emmalloc.h on emscripten. */
152# include <emscripten/emmalloc.h>
153# endif
154#endif
155
156#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
157# include <mach/task.h>
158# include <mach/mach_init.h>
159# include <mach/mach_port.h>
160#endif
161
162#ifdef USE_MIMALLOC
163/* The allocations of this file go to mimalloc. gc.c includes this file and
164 * keeps the CRT heap for its own, so these are undefined at the end. */
165# define malloc(n) mi_malloc(n)
166# define calloc(n, s) mi_calloc(n, s)
167# define realloc(p, n) mi_realloc(p, n)
168# define free(p) mi_free(p)
169#endif
170
171#ifndef RUBY_DEBUG_LOG
172# define RUBY_DEBUG_LOG(...)
173#endif
174
175#ifndef GC_HEAP_INIT_BYTES
176#define GC_HEAP_INIT_BYTES (2560 * 1024)
177#endif
178#ifndef GC_HEAP_FREE_SLOTS
179#define GC_HEAP_FREE_SLOTS 4096
180#endif
181#ifndef GC_RACTOR_HEAP_INIT_BYTES
182/* 0 is resolved at boot to the smallest size that works. */
183#define GC_RACTOR_HEAP_INIT_BYTES 0
184#endif
185#ifndef GC_HEAP_GROWTH_FACTOR
186#define GC_HEAP_GROWTH_FACTOR 1.8
187#endif
188#ifndef GC_HEAP_GROWTH_MAX_BYTES
189#define GC_HEAP_GROWTH_MAX_BYTES 0 /* 0 is disable */
190#endif
191#ifndef GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO
192# define GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO 0.01
193#endif
194#ifndef GC_HEAP_OLDOBJECT_LIMIT_FACTOR
195#define GC_HEAP_OLDOBJECT_LIMIT_FACTOR 2.0
196#endif
197
198#ifndef GC_HEAP_FREE_SLOTS_MIN_RATIO
199#define GC_HEAP_FREE_SLOTS_MIN_RATIO 0.20
200#endif
201#ifndef GC_HEAP_FREE_SLOTS_GOAL_RATIO
202#define GC_HEAP_FREE_SLOTS_GOAL_RATIO 0.40
203#endif
204#ifndef GC_HEAP_FREE_SLOTS_MAX_RATIO
205#define GC_HEAP_FREE_SLOTS_MAX_RATIO 0.65
206#endif
207
208#ifndef GC_MALLOC_LIMIT_MIN
209#define GC_MALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
210#endif
211#ifndef GC_MALLOC_LIMIT_MAX
212#define GC_MALLOC_LIMIT_MAX (32 * 1024 * 1024 /* 32MB */)
213#endif
214#ifndef GC_MALLOC_LIMIT_GROWTH_FACTOR
215#define GC_MALLOC_LIMIT_GROWTH_FACTOR 1.4
216#endif
217
218#ifndef GC_OLDMALLOC_LIMIT_MIN
219#define GC_OLDMALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
220#endif
221#ifndef GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
222#define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR 1.2
223#endif
224#ifndef GC_OLDMALLOC_LIMIT_MAX
225#define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 /* 128MB */)
226#endif
227
228#ifndef GC_MALLOC_INCREASE_LOCAL_THRESHOLD
229#define GC_MALLOC_INCREASE_LOCAL_THRESHOLD (8 * 1024 /* 8KB */)
230#endif
231
232#ifdef RB_THREAD_LOCAL_SPECIFIER
233#define USE_MALLOC_INCREASE_LOCAL 1
234static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local;
235#else
236#define USE_MALLOC_INCREASE_LOCAL 0
237#endif
238
239#ifndef GC_CAN_COMPILE_COMPACTION
240#if defined(__wasi__) /* WebAssembly doesn't support signals */
241# define GC_CAN_COMPILE_COMPACTION 0
242#else
243# define GC_CAN_COMPILE_COMPACTION 1
244#endif
245#endif
246
247#ifndef PRINT_ENTER_EXIT_TICK
248# define PRINT_ENTER_EXIT_TICK 0
249#endif
250#ifndef PRINT_ROOT_TICKS
251#define PRINT_ROOT_TICKS 0
252#endif
253
254#define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS)
255
256#ifndef HEAP_COUNT
257# if SIZEOF_VALUE >= 8
258# define HEAP_COUNT 12
259# else
260# define HEAP_COUNT 5
261# endif
262#endif
263
264/* The reciprocal table and pool_slot_sizes array are both generated from this
265 * single definition, so they can never get out of sync. */
266#if SIZEOF_VALUE >= 8
267# define EACH_POOL_SLOT_SIZE(SLOT) \
268 SLOT(32) SLOT(40) SLOT(64) SLOT(80) SLOT(96) SLOT(128) \
269 SLOT(160) SLOT(256) SLOT(512) SLOT(640) SLOT(768) SLOT(1024)
270#else
271# define EACH_POOL_SLOT_SIZE(SLOT) \
272 SLOT(32) SLOT(64) SLOT(128) SLOT(256) SLOT(512)
273#endif
274
275typedef struct {
276 size_t heap_init_bytes;
277 size_t ractor_heap_init_bytes;
278 size_t heap_free_slots;
279 double growth_factor;
280 size_t growth_max_bytes;
281
282 double heap_free_slots_min_ratio;
283 double heap_free_slots_goal_ratio;
284 double heap_free_slots_max_ratio;
285 double uncollectible_wb_unprotected_objects_limit_ratio;
286 double oldobject_limit_factor;
287
288 size_t malloc_limit_min;
289 size_t malloc_limit_max;
290 double malloc_limit_growth_factor;
291
292 size_t oldmalloc_limit_min;
293 size_t oldmalloc_limit_max;
294 double oldmalloc_limit_growth_factor;
296
297static ruby_gc_params_t gc_params = {
298 GC_HEAP_INIT_BYTES,
299 GC_RACTOR_HEAP_INIT_BYTES,
300 GC_HEAP_FREE_SLOTS,
301 GC_HEAP_GROWTH_FACTOR,
302 GC_HEAP_GROWTH_MAX_BYTES,
303
304 GC_HEAP_FREE_SLOTS_MIN_RATIO,
305 GC_HEAP_FREE_SLOTS_GOAL_RATIO,
306 GC_HEAP_FREE_SLOTS_MAX_RATIO,
307 GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO,
308 GC_HEAP_OLDOBJECT_LIMIT_FACTOR,
309
310 GC_MALLOC_LIMIT_MIN,
311 GC_MALLOC_LIMIT_MAX,
312 GC_MALLOC_LIMIT_GROWTH_FACTOR,
313
314 GC_OLDMALLOC_LIMIT_MIN,
315 GC_OLDMALLOC_LIMIT_MAX,
316 GC_OLDMALLOC_LIMIT_GROWTH_FACTOR,
317};
318
319/* GC_DEBUG:
320 * enable to embed GC debugging information.
321 */
322#ifndef GC_DEBUG
323#define GC_DEBUG 0
324#endif
325
326/* RGENGC_DEBUG:
327 * 1: basic information
328 * 2: remember set operation
329 * 3: mark
330 * 4:
331 * 5: sweep
332 */
333#ifndef RGENGC_DEBUG
334#ifdef RUBY_DEVEL
335#define RGENGC_DEBUG -1
336#else
337#define RGENGC_DEBUG 0
338#endif
339#endif
340#if RGENGC_DEBUG < 0 && !defined(_MSC_VER)
341# define RGENGC_DEBUG_ENABLED(level) (-(RGENGC_DEBUG) >= (level) && ruby_rgengc_debug >= (level))
342#else
343# define RGENGC_DEBUG_ENABLED(level) ((RGENGC_DEBUG) >= (level))
344#endif
345int ruby_rgengc_debug;
346
347/* RGENGC_PROFILE
348 * 0: disable RGenGC profiling
349 * 1: enable profiling for basic information
350 * 2: enable profiling for each types
351 */
352#ifndef RGENGC_PROFILE
353# define RGENGC_PROFILE 0
354#endif
355
356/* RGENGC_ESTIMATE_OLDMALLOC
357 * Enable/disable to estimate increase size of malloc'ed size by old objects.
358 * If estimation exceeds threshold, then will invoke full GC.
359 * 0: disable estimation.
360 * 1: enable estimation.
361 */
362#ifndef RGENGC_ESTIMATE_OLDMALLOC
363# define RGENGC_ESTIMATE_OLDMALLOC 1
364#endif
365
366#ifndef GC_PROFILE_MORE_DETAIL
367# define GC_PROFILE_MORE_DETAIL 0
368#endif
369#ifndef GC_PROFILE_DETAIL_MEMORY
370# define GC_PROFILE_DETAIL_MEMORY 0
371#endif
372#ifndef GC_ENABLE_LAZY_SWEEP
373# define GC_ENABLE_LAZY_SWEEP 1
374#endif
375
376#ifndef VERIFY_FREE_SIZE
377#if RUBY_DEBUG
378#define VERIFY_FREE_SIZE 1
379#else
380#define VERIFY_FREE_SIZE 0
381#endif
382#endif
383
384#if VERIFY_FREE_SIZE
385#undef CALC_EXACT_MALLOC_SIZE
386#define CALC_EXACT_MALLOC_SIZE 1
387#endif
388
389#ifndef CALC_EXACT_MALLOC_SIZE
390# define CALC_EXACT_MALLOC_SIZE 0
391#endif
392
393#if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0
394# ifndef MALLOC_ALLOCATED_SIZE
395# define MALLOC_ALLOCATED_SIZE 0
396# endif
397#else
398# define MALLOC_ALLOCATED_SIZE 0
399#endif
400#ifndef MALLOC_ALLOCATED_SIZE_CHECK
401# define MALLOC_ALLOCATED_SIZE_CHECK 0
402#endif
403
404#ifndef GC_DEBUG_STRESS_TO_CLASS
405# define GC_DEBUG_STRESS_TO_CLASS RUBY_DEBUG
406#endif
407
408typedef enum {
409 GPR_FLAG_NONE = 0x000,
410 /* major reason */
411 GPR_FLAG_MAJOR_BY_NOFREE = 0x001,
412 GPR_FLAG_MAJOR_BY_OLDGEN = 0x002,
413 GPR_FLAG_MAJOR_BY_SHADY = 0x004,
414 GPR_FLAG_MAJOR_BY_FORCE = 0x008,
415#if RGENGC_ESTIMATE_OLDMALLOC
416 GPR_FLAG_MAJOR_BY_OLDMALLOC = 0x020,
417#endif
418 GPR_FLAG_MAJOR_MASK = 0x0ff,
419
420 /* gc reason */
421 GPR_FLAG_NEWOBJ = 0x100,
422 GPR_FLAG_MALLOC = 0x200,
423 GPR_FLAG_METHOD = 0x400,
424 GPR_FLAG_CAPI = 0x800,
425 GPR_FLAG_STRESS = 0x1000,
426
427 /* others */
428 GPR_FLAG_IMMEDIATE_SWEEP = 0x2000,
429 GPR_FLAG_HAVE_FINALIZE = 0x4000,
430 GPR_FLAG_IMMEDIATE_MARK = 0x8000,
431 GPR_FLAG_FULL_MARK = 0x10000,
432 GPR_FLAG_COMPACT = 0x20000,
433 GPR_FLAG_GLOBAL = 0x40000,
434
435 GPR_DEFAULT_REASON =
436 (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK |
437 GPR_FLAG_IMMEDIATE_SWEEP | GPR_FLAG_CAPI),
438} gc_profile_record_flag;
439
440typedef struct gc_profile_record {
441 unsigned int flags;
442 size_t sequence;
443
444 double gc_time;
445 double gc_invoke_time;
446 rb_hrtime_t gc_wall_time;
447 rb_hrtime_t gc_invoke_wall_time;
448 rb_hrtime_t gc_pause_time;
449 rb_hrtime_t gc_stop_time;
450 rb_hrtime_t gc_stw_time;
451 rb_hrtime_t gc_mark_wall_time;
452 rb_hrtime_t gc_sweep_wall_time;
453 rb_hrtime_t gc_compact_wall_time;
454
455 size_t heap_total_objects;
456 size_t heap_use_size;
457 size_t heap_total_size;
458 size_t moved_objects;
459
460#if GC_PROFILE_MORE_DETAIL
461 double gc_mark_time;
462 double gc_sweep_time;
463
464 size_t heap_use_pages;
465 size_t heap_live_objects;
466 size_t heap_free_objects;
467
468 size_t allocate_increase;
469 size_t allocate_limit;
470
471 double prepare_time;
472 size_t removing_objects;
473 size_t empty_objects;
474#if GC_PROFILE_DETAIL_MEMORY
475 long maxrss;
476 long minflt;
477 long majflt;
478#endif
479#endif
480#if MALLOC_ALLOCATED_SIZE
481 size_t allocated_size;
482#endif
483
484#if RGENGC_PROFILE > 0
485 size_t old_objects;
486 size_t remembered_normal_objects;
487 size_t remembered_shady_objects;
488#endif
490
491struct RMoved {
492 VALUE flags;
493 VALUE dummy;
494 VALUE destination;
495};
496
497#define RMOVED(obj) ((struct RMoved *)(obj))
498
499typedef uintptr_t bits_t;
500enum {
501 BITS_SIZE = sizeof(bits_t),
502 BITS_BITLENGTH = ( BITS_SIZE * CHAR_BIT )
503};
504
506 struct heap_page *page;
507};
508
510 struct heap_page_header header;
511 /* char gap[]; */
512 /* RVALUE values[]; */
513};
514
515#define STACK_CHUNK_SIZE 500
516
517typedef struct stack_chunk {
518 VALUE data[STACK_CHUNK_SIZE];
519 struct stack_chunk *next;
520} stack_chunk_t;
521
522typedef struct mark_stack {
523 stack_chunk_t *chunk;
524 stack_chunk_t *cache;
525 int index;
526 int limit;
527 size_t cache_size;
528 size_t unused_cache_size;
529} mark_stack_t;
530
531typedef int (*gc_compact_compare_func)(const void *l, const void *r, void *d);
532
533typedef struct rb_heap_newobj {
534 uintptr_t alloc_cursor;
535 uintptr_t alloc_cursor_end;
536 struct free_region *alloc_next_region;
537 struct heap_page *alloc_using_page;
538} rb_heap_newobj_t;
539
540typedef struct rb_heap_struct {
541 short slot_size;
542
543 /* Basic statistics */
544 size_t total_allocated_pages;
545 size_t force_major_gc_count;
546 size_t force_incremental_marking_finish_count;
547 size_t total_allocated_objects;
548 size_t total_freed_objects;
549 size_t final_slots_count;
550
551 /* Sweeping statistics */
552 size_t freed_slots;
553 size_t empty_slots;
554
555 /* Bump-pointer allocation state; only this objspace's owner thread writes it. */
556 rb_heap_newobj_t newobj;
557
558 struct heap_page *free_pages;
559 struct ccan_list_head pages;
560 struct heap_page *sweeping_page; /* iterator for .pages */
561 struct heap_page *compact_cursor;
562 uintptr_t compact_cursor_index;
563 struct heap_page *pooled_pages;
564 size_t total_pages; /* total page count in a heap */
565 size_t total_slots; /* total slot count */
566
567} rb_heap_t;
568
569enum {
570 gc_stress_no_major,
571 gc_stress_no_immediate_sweep,
572 gc_stress_full_mark_after_malloc,
573 gc_stress_max
574};
575
576enum gc_mode {
577 gc_mode_none,
578 gc_mode_marking,
579 gc_mode_sweeping,
580 gc_mode_compacting,
581};
582
583typedef rbimpl_atomic_uint64_t gc_counter_t;
584
585#if !defined(HAVE_GCC_ATOMIC_BUILTINS_64) && !defined(_WIN32) && \
586 !(defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx)))
587# define MALLOC_COUNTERS_NEED_LOCK 1
588#endif
589
591 gc_counter_t malloc;
592 gc_counter_t free;
593
594 /* Baselines the increase is measured from: malloc snapshotted at GC start
595 * (gc_reset_malloc_info), free re-snapshotted at gc_sweep_finish so the
596 * sweep's own frees never count. */
597 gc_counter_t malloc_at_last_gc;
598 gc_counter_t free_at_last_gc;
599};
600
601/* -- Deferred free of non-thread-safe T_DATA --
602 *
603 * A dead T_DATA not RUBY_TYPED_THREAD_SAFE_FREE cannot have its dfree run during a parallel
604 * local sweep: the dfree is extension code that may touch process state other Ractors are using,
605 * so it needs the world stopped (not merely serialization against other dfrees). The sweep
606 * therefore copies out what the free needs, reclaims the slot immediately, and the dfrees are
607 * called later under a VM barrier. Such a type is never embedded, so the payload always outlives
608 * the slot.
609 */
610#define TDATA_UNSAFE_FREE_CHUNK_CAPA 32
611/* Drained chunks kept for reuse; the rest are freed. */
612#define TDATA_UNSAFE_FREE_CACHE_MAX 64
613
615 void (*dfree)(void *);
616 void *data;
617};
618
620 struct tdata_unsafe_free_chunk *next;
621 unsigned int count;
622 uint32_t embed_xfree_bits;
623 struct tdata_unsafe_free_entry entries[TDATA_UNSAFE_FREE_CHUNK_CAPA];
624};
625STATIC_ASSERT(tdata_unsafe_free_bits_cover_chunk,
626 TDATA_UNSAFE_FREE_CHUNK_CAPA <= 32);
627
629 uint64_t minor_gc_count;
630 uint64_t major_gc_count;
631 uint64_t global_gc_invocation_count;
632 uint64_t marking_time_ns;
633 uint64_t sweeping_time_ns;
634};
635
637 uint64_t minor_gc_count;
638 uint64_t major_gc_count;
639 uint64_t global_gc_invocation_count;
640 uint64_t marking_time_ns;
641 uint64_t sweeping_time_ns;
642};
643
644typedef struct rb_objspace {
645 struct {
646 struct gc_malloc_bytes counters;
647#if RGENGC_ESTIMATE_OLDMALLOC
648 size_t oldmalloc_increase;
649 gc_counter_t freed_in_gc;
650#endif
651#ifdef MALLOC_COUNTERS_NEED_LOCK
652 rb_nativethread_lock_t lock;
653#endif
654 } malloc_counters;
655
656 struct {
657 size_t limit;
658#if MALLOC_ALLOCATED_SIZE
659 size_t allocated_size;
660 size_t allocations;
661#endif
662 } malloc_params;
663
665 bool full_mark;
666 } gc_config;
667
668 struct {
669 unsigned int mode : 2;
670 unsigned int immediate_sweep : 1;
671 unsigned int dont_gc : 1;
672 /* A user hold from GC.disable (kept in vm->gc.disable_holders); owner thread only. */
673 unsigned int user_gc_disabled : 1;
674 unsigned int dont_incremental : 1;
675 unsigned int during_gc : 1;
676 unsigned int during_global_gc : 1;
677 unsigned int during_compacting : 1;
678 unsigned int gc_lock_barrier : 1;
679 unsigned int during_reference_updating : 1;
680 unsigned int during_minor_gc : 1;
681 unsigned int during_postmortem : 1;
682 /* No Ractor will own this objspace any more and it will never allocate again. */
683 unsigned int ownerless : 1;
684 unsigned int measure_gc : 1;
685 } flags;
686
687 rb_event_flag_t hook_events;
688
689 rb_heap_t heaps[HEAP_COUNT];
690 size_t empty_pages_count;
691 struct heap_page *empty_pages;
692 /* Nonzero while gc.c parks this objspace in its zombie table: the pages it adds to
693 * global_objspace->zombie_total_pages. Written under the VM lock or the barrier. */
694 size_t zombie_pages;
695
696 struct {
697 rb_atomic_t finalizing;
698 } atomic_flags;
699
700 mark_stack_t mark_stack;
701 size_t marked_slots;
702
703 /* Moved out of the per-Ractor newobj cache: allocation state is per objspace. */
704 size_t incremental_mark_step_allocated_slots;
705
706 /* Inputs of the global GC trigger, all owned by this objspace's thread.
707 * shareable_objects is the live population of shareable objects; exceeding the
708 * limit requests a global GC. */
709 size_t shareable_objects;
710 size_t shareable_objects_limit;
711 /* Whether the last mark ran the pinned walk; the sweep asserts on it. */
712 unsigned char last_cycle_pinned;
713
714 struct {
715 rb_darray(struct heap_page *) sorted;
716
717 size_t allocated_pages;
718 size_t freed_pages;
719 uintptr_t range[2];
720 size_t freeable_pages;
721
722 size_t allocatable_bytes;
723
724 /* final */
725 VALUE deferred_final;
726 } heap_pages;
727
728 st_table *finalizer_table;
729
730 struct {
731 int run;
732 unsigned int latest_gc_info;
733 gc_profile_record *records;
734 gc_profile_record *current_record;
735 size_t next_index;
736 size_t size;
737 size_t record_count;
738 size_t max_records;
739 size_t record_sequence;
740
741#if GC_PROFILE_MORE_DETAIL
742 double prepare_time;
743#endif
744 double invoke_time;
745 rb_hrtime_t invoke_wall_time;
746
747 size_t minor_gc_count;
748 size_t major_gc_count;
749 size_t global_gc_count;
750 size_t global_gc_invocation_count;
751 size_t compact_count;
752 size_t read_barrier_faults;
753#if RGENGC_PROFILE > 0
754 size_t total_generated_normal_object_count;
755 size_t total_generated_shady_object_count;
756 size_t total_shade_operation_count;
757 size_t total_promoted_count;
758 size_t total_remembered_normal_object_count;
759 size_t total_remembered_shady_object_count;
760
761#if RGENGC_PROFILE >= 2
762 size_t generated_normal_object_count_types[RUBY_T_MASK];
763 size_t generated_shady_object_count_types[RUBY_T_MASK];
764 size_t shade_operation_count_types[RUBY_T_MASK];
765 size_t promoted_types[RUBY_T_MASK];
766 size_t remembered_normal_object_count_types[RUBY_T_MASK];
767 size_t remembered_shady_object_count_types[RUBY_T_MASK];
768#endif
769#endif /* RGENGC_PROFILE */
770
771 /* temporary profiling space */
772 double gc_sweep_start_time;
773 rb_hrtime_t gc_wall_start_time;
774 rb_hrtime_t gc_sweep_wall_start_time;
775 rb_hrtime_t gc_sweep_excluded_wall_time;
776 rb_hrtime_t gc_pause_start_time;
777 rb_hrtime_t gc_stw_start_time;
778 rb_hrtime_t gc_stop_time;
779 rb_hrtime_t gc_mark_phase_wall_start_time;
780 rb_hrtime_t gc_sweep_phase_wall_start_time;
781#if GC_PROFILE_MORE_DETAIL
782 size_t total_allocated_objects_at_gc_start;
783 size_t heap_used_at_gc_start;
784#endif
785
786 /* basic statistics */
787 size_t count;
788 unsigned long long marking_time_ns;
789 struct timespec marking_start_time;
790 unsigned long long sweeping_time_ns;
791 struct timespec sweeping_start_time;
792
793 /* Weak references */
794 size_t weak_references_count;
795 } profile;
796
797
798 struct {
799 bool parent_object_old_p;
800 VALUE parent_object;
801
802 int need_major_gc;
803 size_t last_major_gc;
804 size_t uncollectible_wb_unprotected_objects;
805 size_t uncollectible_wb_unprotected_objects_limit;
806 size_t old_objects;
807 size_t old_objects_limit;
808
809#if RGENGC_ESTIMATE_OLDMALLOC
810 size_t oldmalloc_increase_limit;
811#endif
812
813#if RGENGC_CHECK_MODE >= 2
814 struct st_table *allrefs_table;
815 size_t error_count;
816#endif
817 } rgengc;
818
819 struct {
820 size_t considered_count_table[T_MASK];
821 size_t moved_count_table[T_MASK];
822 size_t moved_up_count_table[T_MASK];
823 size_t moved_down_count_table[T_MASK];
824 size_t total_moved;
825
826 /* This function will be used, if set, to sort the heap prior to compaction */
827 gc_compact_compare_func compare_func;
828 } rcompactor;
829
830 struct {
831 size_t pooled_slots;
832 size_t step_slots;
833 } rincgc;
834
835#if GC_DEBUG_STRESS_TO_CLASS
836 VALUE stress_to_class;
837#endif
838
839 rb_darray(VALUE) weak_references;
840 rb_postponed_job_handle_t finalize_deferred_pjob;
841
842 /* Partially filled chunk of deferred non-thread-safe T_DATA metadata. */
843 struct tdata_unsafe_free_chunk *tdata_unsafe_free_chunk;
844
845 int sweeping_heap_count;
846
847 int fork_vm_lock_lev;
848
849 struct rb_gc_vm_context vm_context;
850
851 /* Process-wide GC statistics publication. Default GC only: other
852 * implementations reject GC.stat(process: true) and never initialize
853 * this lock. */
854 struct {
855 rb_nativethread_lock_t lock;
856 struct gc_process_stat_snapshot published;
857 } process_stat;
858
859 /* The incremental-marking flag as an addressable field, so that ZJIT-generated
860 * code can read it; see rb_gc_impl_zjit_writebarrier_required_offset.
861 * Single-writer: a local GC runs on the owning Ractor's thread and a global GC
862 * writes under the stop-the-world barrier. Read by JIT code on the same thread. */
863 bool zjit_during_incremental_marking;
864
865 void *ractor; // the Ractor running on it, NULL once a zombie; under the VM lock
866} rb_objspace_t;
867
868/* The one VM-global GC structure; for now it only holds the page pool. Page bodies are
869 * carved out of large mmap arenas and reused through the pool (per-page mmap/munmap
870 * would serialize on the kernel's mmap_lock). Leaf lock: no alloc, no GC. */
871typedef struct rb_global_objspace {
872 struct {
873 rb_nativethread_lock_t lock;
874 bool can_advise; /* page_pool_advise_supported(), cached at init */
875 /* List of mmap'd memory regions (arenas) for page bodies. struct arena_descriptor
876 * is defined with the rest of the pool code, since its size needs HEAP_PAGE_SIZE. */
877 struct arena_descriptor *arenas; /* every arena, newest first */
878 char *arena_cursor; /* first body not yet carved out of the newest arena */
879 char *arena_end; /* end of current arena */
880 struct arena_descriptor *arena_current; /* arena that arena_cursor carves from */
881 int arena_count; /* current mapped arenas (for GC.stat total_pages) */
882 int advised_count; /* page bodies with MADV_* applied (for GC.stat discarded pages) */
883 size_t arenas_unmapped; /* cumulative arenas munmapped (for GC.stat) */
884 } page_pool;
885
886 /* Pages held by zombie objspaces (sum of their zombie_pages), and what was left of
887 * it after the last global cycle (roughly the live data). Written under the VM
888 * lock or the barrier; readers (gc_need_global_p) may be racy. */
889 size_t zombie_total_pages;
890 size_t zombie_pages_survivors;
891
892 /* An objspace merge (objspace_absorb) is running: suppress the cross-objspace
893 * verifier while the graph is in flux. Written by the absorbing thread, read by
894 * verification with the world stopped. */
895 bool during_absorb;
896
897 /* main's objspace, for gc_enter's locking policy. main_ractor->objspace is swapped
898 * during Ractor creation; this stable pointer decides the same way at both ends of a
899 * GC. Set at boot, re-pointed in a forked child. */
900 rb_objspace_t *main_objspace;
901
902 /* GC.stress is process-global (upstream semantics). Written by GC.stress= in any
903 * Ractor (rare) and read on every Ractor's alloc and GC path; it is diagnostic, so
904 * plain store/load with last-writer-wins is fine. */
905 bool gc_stressful;
906 VALUE gc_stress_mode;
907
908 /* Global GC driver state. compacting is true during the move phase: reference
909 * updates are deferred until all forwarding exists, so a cross-objspace reference is
910 * rewritten exactly once. objspaces is the snapshot being collected. */
911 struct {
912 bool compacting;
913 struct rb_objspace **objspaces;
914 size_t n_objspaces, objspaces_capa;
915 size_t count;
916 } global_gc;
917
918 /* Index of every objspace's heap pages, ordered by body address. Writers (page
919 * alloc/free) serialize on page_pool.lock; the only reader is a stop-the-world global
920 * GC, so reads need no lock. A local GC uses its own heap_pages.sorted. */
921 struct {
922 struct heap_page **pages;
923 size_t n_pages, capa;
924 uintptr_t lomem, himem;
925 } page_index;
926
927 rb_postponed_job_handle_t tdata_deferred_free_pjob; /* atomic */
928
929 /* Pending count of deferred non-thread-safe T_DATA frees across all objspaces:
930 * bumped as each one is deferred, reset to 0 by the drain. Crossing
931 * TDATA_DEFERRED_FREE_THRESHOLD triggers the postponed job. */
932 size_t tdata_deferred_free_count; /* atomic */
933
934 /* Full chunks awaiting a drain (CAS stack), and drained chunks kept for reuse. */
935 struct tdata_unsafe_free_chunk *tdata_unsafe_free_published; /* atomic */
936 struct tdata_unsafe_free_chunk *tdata_unsafe_free_cache; /* atomic */
937 size_t tdata_unsafe_free_cache_len; /* atomic */
938
939 /* Archive of destroyed objspaces' final statistics, added once on absorption. */
940 struct gc_process_stat_total process_stat_archive;
941} rb_global_objspace_t;
942
943static rb_global_objspace_t rb_global_objspace_instance;
944static rb_global_objspace_t *global_objspace = NULL;
945
946/* Relaxed: every reader only asks whether a drain is worth arranging, and the drain
947 * itself stops the world before it touches a chunk. */
948static inline size_t
949tdata_deferred_free_count_load(void)
950{
951 return rbimpl_atomic_size_load(&global_objspace->tdata_deferred_free_count, RBIMPL_ATOMIC_RELAXED);
952}
953
954/* The floor keeps a global GC from running as soon as a few shareable objects appear;
955 * the factor follows the rule used for the old-generation limit. */
956#define SHAREABLE_OBJECTS_LIMIT_MIN (1 << 16)
957#define SHAREABLE_OBJECTS_LIMIT_FACTOR 2.0
958/* Start a global GC once terminated, uninherited Ractors hold this many heap pages. A
959 * small Ractor's objspace is about 13 pages, so discarding many of them still stays
960 * below it, while a single fat zombie crosses it. */
961#define ZOMBIE_PAGES_TRIGGER 256
962/* Trigger the deferred T_DATA free postponed job once this many have accumulated
963 * across all objspaces. */
964#define TDATA_DEFERRED_FREE_THRESHOLD (1 << 15)
965
966static void objspace_absorb(rb_objspace_t *dst, rb_objspace_t *src);
967
968
969static struct heap_page_body *page_pool_acquire(struct arena_descriptor **arena_out, bool *zeroed_out);
970static void page_pool_release(struct heap_page_body *body, struct arena_descriptor *arena);
971#ifdef HAVE_MMAP
972static void page_pool_release_locked(struct heap_page_body *body, struct arena_descriptor *arena);
973#endif
974static void page_pool_reclaim(rb_global_objspace_t *g);
975static bool page_pool_advise_supported(void);
976
977#if RGENGC_CHECK_MODE && !defined(_WIN32) && !defined(__wasi__) && defined(HAVE_PTHREAD_H)
978# define PAGE_POOL_LOCK_ERRORCHECK 1
979#endif
980
981static void
982page_pool_lock_initialize(rb_nativethread_lock_t *lock)
983{
984#ifdef PAGE_POOL_LOCK_ERRORCHECK
985 pthread_mutexattr_t attr;
986 pthread_mutexattr_init(&attr);
987 pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
988 pthread_mutex_init(lock, &attr);
989 pthread_mutexattr_destroy(&attr);
990#else
992#endif
993}
994
995static void
996global_objspace_init(void)
997{
998 if (global_objspace == NULL) {
999 rb_global_objspace_t *g = &rb_global_objspace_instance;
1000 page_pool_lock_initialize(&g->page_pool.lock);
1001 g->page_pool.arenas = NULL;
1002 g->page_pool.arena_cursor = NULL;
1003 g->page_pool.arena_end = NULL;
1004 g->page_pool.arena_current = NULL;
1005 g->page_pool.arena_count = 0;
1006 g->page_pool.advised_count = 0;
1007 g->tdata_deferred_free_pjob = POSTPONED_JOB_HANDLE_INVALID;
1008 g->tdata_unsafe_free_published = NULL;
1009 g->tdata_unsafe_free_cache = NULL;
1010 g->tdata_unsafe_free_cache_len = 0;
1011 g->page_pool.arenas_unmapped = 0;
1012 g->page_pool.can_advise = page_pool_advise_supported();
1013 global_objspace = g;
1014 }
1015}
1016
1017#ifndef HEAP_PAGE_ALIGN_LOG
1018/* default tiny heap size: 64KiB */
1019#define HEAP_PAGE_ALIGN_LOG 16
1020#endif
1021
1022#if GC_DEBUG
1023struct rvalue_overhead {
1024 const char *file;
1025 int line;
1026};
1027
1028// Make sure that RVALUE_OVERHEAD aligns to sizeof(VALUE)
1029# define RVALUE_OVERHEAD (sizeof(struct { \
1030 union { \
1031 struct rvalue_overhead overhead; \
1032 VALUE value; \
1033 }; \
1034}))
1035size_t rb_gc_impl_obj_slot_size(VALUE obj);
1036# define GET_RVALUE_OVERHEAD(obj) ((struct rvalue_overhead *)((uintptr_t)obj + rb_gc_impl_obj_slot_size(obj)))
1037#else
1038# ifndef RVALUE_OVERHEAD
1039# define RVALUE_OVERHEAD 0
1040# endif
1041#endif
1042
1043#define RVALUE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD)
1044
1045static const size_t pool_slot_sizes[HEAP_COUNT] = {
1046#define SLOT(size) ((size) + RVALUE_OVERHEAD),
1047 EACH_POOL_SLOT_SIZE(SLOT)
1048#undef SLOT
1049};
1050
1051/* An init size below one slot in the largest heap never forces that heap's first
1052 * page, and allocating there then fails with "cannot create a new page after GC". */
1053static inline size_t
1054heap_init_bytes_min(void)
1055{
1056 return pool_slot_sizes[HEAP_COUNT - 1];
1057}
1058
1059/* Precomputed reciprocals for fast slot index calculation.
1060 * For slot size d: reciprocal = ceil(2^48 / d).
1061 * Then offset / d == (uint32_t)((offset * reciprocal) >> 48)
1062 * and offset % d == 0 iff ((offset * reciprocal) & (2^48 - 1)) < reciprocal
1063 * for all offset < HEAP_PAGE_SIZE. */
1064#define SLOT_RECIPROCAL_SHIFT 48
1065#define SLOT_RECIPROCAL(size) (((1ULL << SLOT_RECIPROCAL_SHIFT) + (size) - 1) / (size))
1066
1067static const uint64_t heap_slot_reciprocal_table[HEAP_COUNT] = {
1068#define SLOT(size) SLOT_RECIPROCAL((size) + RVALUE_OVERHEAD),
1069 EACH_POOL_SLOT_SIZE(SLOT)
1070#undef SLOT
1071};
1072
1073#if SIZEOF_VALUE >= 8
1074static uint8_t size_to_heap_idx[1024 / 8 + 1];
1075#else
1076static uint8_t size_to_heap_idx[512 / 8 + 1];
1077#endif
1078
1079#ifndef MAX
1080# define MAX(a, b) (((a) > (b)) ? (a) : (b))
1081#endif
1082#ifndef MIN
1083# define MIN(a, b) (((a) < (b)) ? (a) : (b))
1084#endif
1085#define roomof(x, y) (((x) + (y) - 1) / (y))
1086#define CEILDIV(i, mod) roomof(i, mod)
1087#define MIN_POOL_SLOT_SIZE 32
1088enum {
1089 HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG),
1090 HEAP_PAGE_ALIGN_MASK = (~(~0UL << HEAP_PAGE_ALIGN_LOG)),
1091 HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN,
1092 HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, MIN_POOL_SLOT_SIZE), BITS_BITLENGTH),
1093 HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),
1094};
1095#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG)
1096#define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN
1097
1098#if !defined(INCREMENTAL_MARK_STEP_ALLOCATIONS)
1099# define INCREMENTAL_MARK_STEP_ALLOCATIONS 500
1100#endif
1101
1102#undef INIT_HEAP_PAGE_ALLOC_USE_MMAP
1103/* Must define either HEAP_PAGE_ALLOC_USE_MMAP or
1104 * INIT_HEAP_PAGE_ALLOC_USE_MMAP. */
1105
1106#ifndef HAVE_MMAP
1107/* We can't use mmap of course, if it is not available. */
1108static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
1109
1110#elif defined(__wasm__)
1111/* wasmtime does not have proper support for mmap.
1112 * See https://github.com/bytecodealliance/wasmtime/blob/main/docs/WASI-rationale.md#why-no-mmap-and-friends
1113 */
1114static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
1115
1116#elif HAVE_CONST_PAGE_SIZE
1117/* If we have the PAGE_SIZE and it is a constant, then we can directly use it. */
1118static const bool HEAP_PAGE_ALLOC_USE_MMAP = (PAGE_SIZE <= HEAP_PAGE_SIZE);
1119
1120#elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE)
1121/* If we can use the maximum page size. */
1122static const bool HEAP_PAGE_ALLOC_USE_MMAP = true;
1123
1124#elif defined(PAGE_SIZE)
1125/* If the PAGE_SIZE macro can be used dynamically. */
1126# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (PAGE_SIZE <= HEAP_PAGE_SIZE)
1127
1128#elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
1129/* If we can use sysconf to determine the page size. */
1130# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE)
1131
1132#else
1133/* Otherwise we can't determine the system page size, so don't use mmap. */
1134static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
1135#endif
1136
1137#ifdef INIT_HEAP_PAGE_ALLOC_USE_MMAP
1138/* We can determine the system page size at runtime. */
1139# define HEAP_PAGE_ALLOC_USE_MMAP (heap_page_alloc_use_mmap != false)
1140
1141static bool heap_page_alloc_use_mmap;
1142#endif
1143
1144#define RVALUE_AGE_BIT_COUNT 2
1145#define RVALUE_AGE_BIT_MASK (((bits_t)1 << RVALUE_AGE_BIT_COUNT) - 1)
1146#define RVALUE_OLD_AGE 3
1147
1149 VALUE flags; /* always 0 for freed obj */
1150 uintptr_t end; /* exclusive end address of the run */
1151 struct free_region *next; /* next free region in the page */
1152};
1153
1155 /* Cache line 0: allocation fast path + SLOT_INDEX */
1156 struct free_region *free_region;
1157 uintptr_t start;
1158 uint64_t slot_size_reciprocal;
1159 unsigned short slot_size;
1160 unsigned short total_slots;
1161 unsigned short free_slots;
1162 unsigned short final_slots;
1163 unsigned short pinned_slots;
1164 /* Page state flags. A bitfield is safe: the only writers are the owning Ractor
1165 * (GVL) and the global GC driver (stop-the-world), never together. has_shareable /
1166 * has_shref hint that the page holds at least one such bit, for re-scanning. */
1167 struct {
1168 unsigned int before_sweep : 1;
1169 unsigned int has_remembered_objects : 1;
1170 unsigned int has_uncollectible_wb_unprotected_objects : 1;
1171 unsigned int has_shref_objects : 1;
1172 unsigned int has_shareable_objects : 1;
1173 unsigned int body_zeroed : 1;
1174 } flags;
1175
1176 rb_heap_t *heap;
1177
1178 /* The objspace owning this page, so any object's owner is one load away
1179 * (GET_HEAP_OBJSPACE). Rewritten only when a page changes owner (inheritance). */
1180 rb_objspace_t *objspace;
1181
1182 struct heap_page *free_next;
1183 struct heap_page_body *body;
1184 struct arena_descriptor *arena;
1185 struct ccan_list_node page_node;
1186
1187 bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT];
1188 /* the following three bitmaps are cleared at the beginning of full GC */
1189 bits_t mark_bits[HEAP_PAGE_BITMAP_LIMIT];
1190 bits_t uncollectible_bits[HEAP_PAGE_BITMAP_LIMIT];
1191 bits_t marking_bits[HEAP_PAGE_BITMAP_LIMIT];
1192
1193 bits_t remembered_bits[HEAP_PAGE_BITMAP_LIMIT];
1194
1195 /* Two extra bits per object. shareable_bits: what a local sweep must never free
1196 * (only a global GC decides a shareable object is dead); set at creation and by
1197 * rb_gc_impl_obj_became_shareable. shref_bits: an unshareable object referenced
1198 * from a shareable one, a local GC root; the write barrier maintains it. */
1199 bits_t shareable_bits[HEAP_PAGE_BITMAP_LIMIT];
1200 bits_t shref_bits[HEAP_PAGE_BITMAP_LIMIT];
1201
1202 /* If set, the object is not movable */
1203 bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT];
1204 bits_t age_bits[HEAP_PAGE_BITMAP_LIMIT * RVALUE_AGE_BIT_COUNT];
1205};
1206
1207/*
1208 * When asan is enabled, this will prohibit writing to the freelist until it is unlocked
1209 */
1210static void
1211asan_lock_freelist(struct heap_page *page)
1212{
1213 asan_poison_memory_region(&page->free_region, sizeof(struct free_region *));
1214}
1215
1216/*
1217 * When asan is enabled, this will enable the ability to write to the freelist
1218 */
1219static void
1220asan_unlock_freelist(struct heap_page *page)
1221{
1222 asan_unpoison_memory_region(&page->free_region, sizeof(struct free_region *), false);
1223}
1224
1225static inline bool
1226heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page *page)
1227{
1228 if (page->total_slots == 0) {
1229 GC_ASSERT(page->start == 0);
1230 GC_ASSERT(page->slot_size == 0);
1231 GC_ASSERT(page->heap == NULL);
1232 GC_ASSERT(page->free_slots == 0);
1233 asan_unpoisoning_memory_region(&page->free_region, sizeof(&page->free_region)) {
1234 GC_ASSERT(page->free_region == NULL);
1235 }
1236
1237 return true;
1238 }
1239 else {
1240 GC_ASSERT(page->start != 0);
1241 GC_ASSERT(page->slot_size != 0);
1242 GC_ASSERT(page->heap != NULL);
1243
1244 return false;
1245 }
1246}
1247
1248#define GET_PAGE_BODY(x) ((struct heap_page_body *)((bits_t)(x) & ~(HEAP_PAGE_ALIGN_MASK)))
1249#define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
1250#define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)
1251
1252static inline size_t
1253slot_index_for_offset(size_t offset, uint64_t reciprocal)
1254{
1255 return (uint32_t)(((uint64_t)offset * reciprocal) >> SLOT_RECIPROCAL_SHIFT);
1256}
1257
1258/* (p - page->start) % page->slot_size == 0, using the divisibility test from
1259 * "Faster Remainder by Direct Computation" (Lemire, Kaser, Kurz 2019).
1260 * p must already be known to be within the page. */
1261static inline bool
1262slot_offset_aligned_p(const struct heap_page *page, uintptr_t p)
1263{
1264 uint64_t offset = p - page->start;
1265 uint64_t reciprocal = page->slot_size_reciprocal;
1266 uint64_t lowbits = (offset * reciprocal) & ((1ULL << SLOT_RECIPROCAL_SHIFT) - 1);
1267 return lowbits < reciprocal;
1268}
1269
1270#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_reciprocal)
1271#define SLOT_BITMAP_INDEX(page, p) (SLOT_INDEX(page, p) / BITS_BITLENGTH)
1272#define SLOT_BITMAP_OFFSET(page, p) (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1))
1273#define SLOT_BITMAP_BIT(page, p) ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p))
1274
1275#define _MARKED_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] & SLOT_BITMAP_BIT(page, p))
1276#define _MARK_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] |= SLOT_BITMAP_BIT(page, p))
1277#define _CLEAR_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] &= ~SLOT_BITMAP_BIT(page, p))
1278
1279#define MARKED_IN_BITMAP(bits, p) _MARKED_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
1280#define MARK_IN_BITMAP(bits, p) _MARK_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
1281#define CLEAR_IN_BITMAP(bits, p) _CLEAR_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
1282
1283#define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0])
1284#define GET_HEAP_PINNED_BITS(x) (&GET_HEAP_PAGE(x)->pinned_bits[0])
1285#define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0])
1286#define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0])
1287#define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0])
1288#define GET_HEAP_SHAREABLE_BITS(x) (&GET_HEAP_PAGE(x)->shareable_bits[0])
1289#define GET_HEAP_SHREF_BITS(x) (&GET_HEAP_PAGE(x)->shref_bits[0])
1290#define GET_HEAP_OBJSPACE(x) (GET_HEAP_PAGE(x)->objspace)
1291
1292/* obj lives on a page of another objspace, not the current one (i.e. it is foreign). */
1293static inline bool
1294gc_foreign_object_p(const rb_objspace_t *objspace, VALUE obj)
1295{
1296 if (RB_LIKELY(!rb_gc_ever_multi_ractor_p())) return false;
1297
1298 return RB_UNLIKELY(GET_HEAP_OBJSPACE(obj) != objspace);
1299}
1300
1301/* Foreign and not inside a stop-the-world global GC. While true, a local GC must not
1302 * touch obj's per-object GC state (mark, pin, remember bits): its owner handles that,
1303 * or the global GC does with everyone stopped. */
1304static inline bool
1305gc_skip_foreign_object_p(const rb_objspace_t *objspace, VALUE obj)
1306{
1307 return gc_foreign_object_p(objspace, obj) && !objspace->flags.during_global_gc;
1308}
1309
1310/* Record obj as shareable on its owning page (bit, page flag and population counter).
1311 * Shared by born-shareable objects and make_shareable. The writer is the owner thread,
1312 * so plain bit operations suffice. */
1313static inline void
1314gc_page_add_shareable(struct heap_page *page, VALUE obj)
1315{
1316 GC_ASSERT(page == GET_HEAP_PAGE(obj));
1317 GC_ASSERT(RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE));
1318 _MARK_IN_BITMAP(page->shareable_bits, page, obj);
1319 page->flags.has_shareable_objects = TRUE;
1320 page->objspace->shareable_objects++;
1321}
1322
1323static int
1324RVALUE_AGE_GET(VALUE obj)
1325{
1326 struct heap_page *page = GET_HEAP_PAGE(obj);
1327 bits_t *age_bits = page->age_bits;
1328 size_t slot_idx = SLOT_INDEX(page, obj);
1329 size_t idx = (slot_idx / BITS_BITLENGTH) * 2;
1330 int shift = (int)(slot_idx & (BITS_BITLENGTH - 1));
1331 int lo = (age_bits[idx] >> shift) & 1;
1332 int hi = (age_bits[idx + 1] >> shift) & 1;
1333 return lo | (hi << 1);
1334}
1335
1336static void
1337RVALUE_AGE_SET_BITMAP(VALUE obj, int age)
1338{
1339 RUBY_ASSERT(age <= RVALUE_OLD_AGE);
1340 struct heap_page *page = GET_HEAP_PAGE(obj);
1341 bits_t *age_bits = page->age_bits;
1342 size_t slot_idx = SLOT_INDEX(page, obj);
1343 size_t idx = (slot_idx / BITS_BITLENGTH) * 2;
1344 int shift = (int)(slot_idx & (BITS_BITLENGTH - 1));
1345 bits_t mask = (bits_t)1 << shift;
1346
1347 age_bits[idx] = (age_bits[idx] & ~mask) | ((bits_t)(age & 1) << shift);
1348 age_bits[idx + 1] = (age_bits[idx + 1] & ~mask) | ((bits_t)((age >> 1) & 1) << shift);
1349}
1350
1351static void
1352RVALUE_AGE_SET(VALUE obj, int age)
1353{
1354 RVALUE_AGE_SET_BITMAP(obj, age);
1355 if (age == RVALUE_OLD_AGE) {
1357 }
1358 else {
1360 }
1361}
1362
1363#define malloc_limit objspace->malloc_params.limit
1364#define malloc_increase gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters)
1365#define malloc_allocated_size objspace->malloc_params.allocated_size
1366
1367#ifdef MALLOC_COUNTERS_NEED_LOCK
1368# define MALLOC_COUNTERS_LOCK(o) rb_native_mutex_lock(&(o)->malloc_counters.lock)
1369# define MALLOC_COUNTERS_UNLOCK(o) rb_native_mutex_unlock(&(o)->malloc_counters.lock)
1370#else
1371# define MALLOC_COUNTERS_LOCK(o) ((void)0)
1372# define MALLOC_COUNTERS_UNLOCK(o) ((void)0)
1373#endif
1374
1375static inline void
1376gc_counter_add(gc_counter_t *p, size_t delta)
1377{
1378#ifdef MALLOC_COUNTERS_NEED_LOCK
1379 *p += (gc_counter_t)delta;
1380#else
1381 rbimpl_atomic_u64_fetch_add_relaxed(p, (uint64_t)delta);
1382#endif
1383}
1384
1385static inline gc_counter_t
1386gc_counter_load_relaxed(const gc_counter_t *p)
1387{
1388#ifdef MALLOC_COUNTERS_NEED_LOCK
1389 return *p;
1390#else
1391 return rbimpl_atomic_u64_load_relaxed(p);
1392#endif
1393}
1394
1395static inline gc_counter_t
1396gc_counter_load_acquire(const gc_counter_t *p)
1397{
1398#ifdef MALLOC_COUNTERS_NEED_LOCK
1399 return *p;
1400#else
1401 return rbimpl_atomic_u64_load_acquire(p);
1402#endif
1403}
1404
1405static inline void
1406gc_counter_store_release(gc_counter_t *p, gc_counter_t v)
1407{
1408#ifdef MALLOC_COUNTERS_NEED_LOCK
1409 *p = v;
1410#else
1411 rbimpl_atomic_u64_set_release(p, v);
1412#endif
1413}
1414
1415static inline int64_t
1416gc_malloc_counters_increase(rb_objspace_t *objspace, const struct gc_malloc_bytes *c)
1417{
1418 MALLOC_COUNTERS_LOCK(objspace);
1419 gc_counter_t malloc_at = gc_counter_load_acquire(&c->malloc_at_last_gc);
1420 gc_counter_t free_at = gc_counter_load_acquire(&c->free_at_last_gc);
1421 gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc);
1422 gc_counter_t free_now = gc_counter_load_relaxed(&c->free);
1423 MALLOC_COUNTERS_UNLOCK(objspace);
1424
1425 gc_counter_t malloc_delta = malloc_now - malloc_at;
1426 gc_counter_t free_delta = free_now - free_at;
1427
1428 if (malloc_delta >= free_delta) {
1429 return (int64_t)(malloc_delta - free_delta);
1430 }
1431 else {
1432 return -(int64_t)(free_delta - malloc_delta);
1433 }
1434}
1435
1436static inline size_t
1437gc_malloc_counters_increase_unsigned(rb_objspace_t *objspace, const struct gc_malloc_bytes *c)
1438{
1439 int64_t inc = gc_malloc_counters_increase(objspace, c);
1440 if (inc <= 0) return 0;
1441#if SIZEOF_SIZE_T < 8
1442 if ((uint64_t)inc > SIZE_MAX) return SIZE_MAX;
1443#endif
1444 return (size_t)inc;
1445}
1446
1447/* Frees done while sweeping are the GC's own work, not the mutator's: advance
1448 * free_at_last_gc past them so they cannot pay for the next cycle's allocation.
1449 * malloc_at_last_gc stays at gc_reset_malloc_info's snapshot (GC start). */
1450static inline void
1451gc_malloc_counters_snapshot_free_at_last_gc(rb_objspace_t *objspace, struct gc_malloc_bytes *c)
1452{
1453 MALLOC_COUNTERS_LOCK(objspace);
1454 gc_counter_store_release(&c->free_at_last_gc, gc_counter_load_relaxed(&c->free));
1455 MALLOC_COUNTERS_UNLOCK(objspace);
1456}
1457
1458static inline void
1459gc_malloc_counters_snapshot(rb_objspace_t *objspace, struct gc_malloc_bytes *c)
1460{
1461 MALLOC_COUNTERS_LOCK(objspace);
1462 gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc);
1463 gc_counter_t free_now = gc_counter_load_relaxed(&c->free);
1464 gc_counter_store_release(&c->malloc_at_last_gc, malloc_now);
1465 gc_counter_store_release(&c->free_at_last_gc, free_now);
1466 MALLOC_COUNTERS_UNLOCK(objspace);
1467}
1468
1469#if RGENGC_ESTIMATE_OLDMALLOC
1470static inline size_t
1471gc_oldmalloc_increase_load(rb_objspace_t *objspace)
1472{
1473 return rbimpl_atomic_size_load(&objspace->malloc_counters.oldmalloc_increase, RBIMPL_ATOMIC_RELAXED);
1474}
1475#endif
1476
1477#define heap_pages_lomem objspace->heap_pages.range[0]
1478#define heap_pages_himem objspace->heap_pages.range[1]
1479#define heap_pages_freeable_pages objspace->heap_pages.freeable_pages
1480#define heap_pages_deferred_final objspace->heap_pages.deferred_final
1481#define heaps objspace->heaps
1482#define during_gc objspace->flags.during_gc
1483#define finalizing objspace->atomic_flags.finalizing
1484#define finalizer_table objspace->finalizer_table
1485#define ruby_gc_stressful global_objspace->gc_stressful
1486#define ruby_gc_stress_mode global_objspace->gc_stress_mode
1487#if GC_DEBUG_STRESS_TO_CLASS
1488#define stress_to_class objspace->stress_to_class
1489#define set_stress_to_class(c) (stress_to_class = (c))
1490#else
1491#define stress_to_class ((void)objspace, 0)
1492#define set_stress_to_class(c) ((void)objspace, (c))
1493#endif
1494
1495#if 0
1496#define dont_gc_on() (fprintf(stderr, "dont_gc_on@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 1)
1497#define dont_gc_off() (fprintf(stderr, "dont_gc_off@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 0)
1498#define dont_gc_set(b) (fprintf(stderr, "dont_gc_set(%d)@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = (int)(b))
1499#define dont_gc_val() (objspace->flags.dont_gc)
1500#else
1501#define dont_gc_on() (objspace->flags.dont_gc = 1)
1502#define dont_gc_off() (objspace->flags.dont_gc = 0)
1503#define dont_gc_set(b) (objspace->flags.dont_gc = (int)(b))
1504#define dont_gc_val() (objspace->flags.dont_gc)
1505#endif
1506
1507#define gc_config_full_mark_set(b) (objspace->gc_config.full_mark = (int)(b))
1508#define gc_config_full_mark_val (objspace->gc_config.full_mark)
1509
1510static inline enum gc_mode
1511gc_mode_verify(enum gc_mode mode)
1512{
1513#if RGENGC_CHECK_MODE > 0
1514 switch (mode) {
1515 case gc_mode_none:
1516 case gc_mode_marking:
1517 case gc_mode_sweeping:
1518 case gc_mode_compacting:
1519 break;
1520 default:
1521 rb_bug("gc_mode_verify: unreachable (%d)", (int)mode);
1522 }
1523#endif
1524 return mode;
1525}
1526
1527static inline bool
1528has_sweeping_pages(rb_objspace_t *objspace)
1529{
1530 return objspace->sweeping_heap_count != 0;
1531}
1532
1533static inline size_t
1534heap_eden_total_pages(rb_objspace_t *objspace)
1535{
1536 size_t count = 0;
1537 for (int i = 0; i < HEAP_COUNT; i++) {
1538 count += (&heaps[i])->total_pages;
1539 }
1540 return count;
1541}
1542
1543static inline size_t
1544total_allocated_objects(rb_objspace_t *objspace)
1545{
1546 size_t count = 0;
1547 for (int i = 0; i < HEAP_COUNT; i++) {
1548 rb_heap_t *heap = &heaps[i];
1549 count += heap->total_allocated_objects;
1550 }
1551 return count;
1552}
1553
1554static inline size_t
1555total_freed_objects(rb_objspace_t *objspace)
1556{
1557 size_t count = 0;
1558 for (int i = 0; i < HEAP_COUNT; i++) {
1559 rb_heap_t *heap = &heaps[i];
1560 count += heap->total_freed_objects;
1561 }
1562 return count;
1563}
1564
1565static inline size_t
1566total_final_slots_count(rb_objspace_t *objspace)
1567{
1568 size_t count = 0;
1569 for (int i = 0; i < HEAP_COUNT; i++) {
1570 rb_heap_t *heap = &heaps[i];
1571 count += heap->final_slots_count;
1572 }
1573 return count;
1574}
1575
1576#define gc_mode(objspace) gc_mode_verify((enum gc_mode)(objspace)->flags.mode)
1577#define gc_mode_set(objspace, m) ((objspace)->flags.mode = (unsigned int)gc_mode_verify(m))
1578#define gc_needs_major_flags objspace->rgengc.need_major_gc
1579
1580#define is_marking(objspace) (gc_mode(objspace) == gc_mode_marking)
1581#define is_sweeping(objspace) (gc_mode(objspace) == gc_mode_sweeping)
1582#define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE)
1583#define is_incremental_marking(objspace) ((objspace)->zjit_during_incremental_marking != FALSE)
1584#define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE)
1585
1586static void
1587gc_set_incremental_marking(rb_objspace_t *objspace, unsigned int value)
1588{
1589 if (objspace->zjit_during_incremental_marking != value) {
1590 objspace->zjit_during_incremental_marking = value;
1591 }
1592}
1593/*
1594 * Byte budget for incremental sweep steps. Each step sweeps at most
1595 * this many bytes worth of slots before yielding. The effective slot
1596 * count per step is GC_INCREMENTAL_SWEEP_BYTES / heap->slot_size,
1597 * so larger slot pools (which are less heavily used) naturally get
1598 * fewer slots swept per step.
1599 *
1600 * Baseline: 2048 slots * RVALUE_SLOT_SIZE = 2048 * 40 = 81920 bytes,
1601 * preserving the historical behavior for the smallest heap.
1602 */
1603#define GC_INCREMENTAL_SWEEP_BYTES (2048 * RVALUE_SLOT_SIZE)
1604#define GC_INCREMENTAL_SWEEP_POOL_BYTES (1024 * RVALUE_SLOT_SIZE)
1605#define is_lazy_sweeping(objspace) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(objspace))
1606/* In lazy sweeping or the previous incremental marking finished and did not yield a free page. */
1607#define needs_continue_sweeping(objspace, heap) \
1608 ((heap)->free_pages == NULL && is_lazy_sweeping(objspace))
1609
1610#if SIZEOF_LONG == SIZEOF_VOIDP
1611# define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG) /* unset FIXNUM_FLAG */
1612#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
1613# define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \
1614 ((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1))
1615#else
1616# error not supported
1617#endif
1618
1619struct RZombie {
1620 VALUE flags;
1621 VALUE next;
1622 void (*dfree)(void *);
1623 void *data;
1624};
1625
1626#define RZOMBIE(o) ((struct RZombie *)(o))
1627
1628static bool ruby_enable_autocompact = false;
1629#if RGENGC_CHECK_MODE
1630static gc_compact_compare_func ruby_autocompact_compare_func;
1631#endif
1632
1633static void init_mark_stack(mark_stack_t *stack);
1634static int garbage_collect(rb_objspace_t *, unsigned int reason);
1635
1636static int gc_start(rb_objspace_t *objspace, unsigned int reason);
1637static void gc_rest(rb_objspace_t *objspace);
1638
1639/* GC cycle events (ENTER, EXIT, START, END_MARK, END_SWEEP) fire only if the objspace's
1640 * own Ractor enabled them, so a concurrent local GC never walks the VM-global hook list
1641 * while another Ractor mutates it. NEWOBJ and FREEOBJ were already restricted. */
1642#define gc_event_hook(objspace, event) do { \
1643 if (RB_UNLIKELY((objspace)->hook_events & (event))) { \
1644 rb_gc_event_hook(0, (event)); \
1645 } \
1646} while (0)
1647
1648enum gc_enter_event {
1649 gc_enter_event_start,
1650 gc_enter_event_continue,
1651 gc_enter_event_rest,
1652 gc_enter_event_finalizer,
1653 gc_enter_event_global,
1654 gc_enter_event_global_auto,
1655};
1656
1657static inline bool gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev);
1658static inline void gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev);
1659static void gc_marking_enter(rb_objspace_t *objspace);
1660static void gc_marking_exit(rb_objspace_t *objspace);
1661static void gc_sweeping_enter(rb_objspace_t *objspace);
1662static void gc_sweeping_exit(rb_objspace_t *objspace);
1663static bool gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap);
1664
1665static void gc_sweep(rb_objspace_t *objspace);
1666static void gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap);
1667static void gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap);
1668
1669static inline void gc_mark(rb_objspace_t *objspace, VALUE ptr);
1670static inline void gc_pin(rb_objspace_t *objspace, VALUE ptr);
1671static inline void gc_mark_and_pin(rb_objspace_t *objspace, VALUE ptr);
1672
1673static int gc_mark_stacked_objects_incremental(rb_objspace_t *, size_t count);
1674NO_SANITIZE("memory", static inline bool is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr));
1675
1676static void gc_verify_internal_consistency(void *objspace_ptr);
1677
1678static double getrusage_time(void);
1679static inline rb_hrtime_t elapsed_hrtime_from(rb_hrtime_t start);
1680static inline void gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason);
1681static inline void gc_prof_timer_start(rb_objspace_t *);
1682static inline void gc_prof_timer_stop(rb_objspace_t *);
1683static inline void gc_prof_mark_timer_start(rb_objspace_t *);
1684static inline void gc_prof_mark_timer_stop(rb_objspace_t *);
1685static inline void gc_prof_sweep_timer_start(rb_objspace_t *);
1686static inline void gc_prof_sweep_timer_stop(rb_objspace_t *);
1687static inline void gc_prof_set_malloc_info(rb_objspace_t *);
1688static inline void gc_prof_set_heap_info(rb_objspace_t *);
1689
1690#define gc_prof_record(objspace) (objspace)->profile.current_record
1691#define gc_prof_enabled(objspace) ((objspace)->profile.run && (objspace)->profile.current_record)
1692
1693#define gc_report(level, objspace, ...) \
1694 if (!RGENGC_DEBUG_ENABLED(level)) {} else gc_report_body(level, objspace, __VA_ARGS__)
1695PRINTF_ARGS(static void gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...), 3, 4);
1696
1697static void gc_finalize_deferred(void *dmy);
1698static void gc_tdata_unsafe_drain_objspaces(rb_objspace_t **objspaces, size_t n);
1699
1700#if USE_TICK_T
1701
1702/* the following code is only for internal tuning. */
1703
1704/* Source code to use RDTSC is quoted and modified from
1705 * https://www.mcs.anl.gov/~kazutomo/rdtsc.html
1706 * written by Kazutomo Yoshii <kazutomo@mcs.anl.gov>
1707 */
1708
1709#if defined(__GNUC__) && defined(__i386__)
1710typedef unsigned long long tick_t;
1711#define PRItick "llu"
1712static inline tick_t
1713tick(void)
1714{
1715 unsigned long long int x;
1716 __asm__ __volatile__ ("rdtsc" : "=A" (x));
1717 return x;
1718}
1719
1720#elif defined(__GNUC__) && defined(__x86_64__)
1721typedef unsigned long long tick_t;
1722#define PRItick "llu"
1723
1724static __inline__ tick_t
1725tick(void)
1726{
1727 unsigned long hi, lo;
1728 __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi));
1729 return ((unsigned long long)lo)|( ((unsigned long long)hi)<<32);
1730}
1731
1732#elif defined(__powerpc64__) && (GCC_VERSION_SINCE(4,8,0) || defined(__clang__))
1733typedef unsigned long long tick_t;
1734#define PRItick "llu"
1735
1736static __inline__ tick_t
1737tick(void)
1738{
1739 unsigned long long val = __builtin_ppc_get_timebase();
1740 return val;
1741}
1742
1743#elif defined(__aarch64__) && defined(__GNUC__)
1744typedef unsigned long tick_t;
1745#define PRItick "lu"
1746
1747static __inline__ tick_t
1748tick(void)
1749{
1750 unsigned long val;
1751 __asm__ __volatile__ ("mrs %0, cntvct_el0" : "=r" (val));
1752 return val;
1753}
1754
1755
1756#elif defined(_WIN32) && defined(_MSC_VER)
1757#include <intrin.h>
1758typedef unsigned __int64 tick_t;
1759#define PRItick "llu"
1760
1761static inline tick_t
1762tick(void)
1763{
1764 return __rdtsc();
1765}
1766
1767#else /* use clock */
1768typedef clock_t tick_t;
1769#define PRItick "llu"
1770
1771static inline tick_t
1772tick(void)
1773{
1774 return clock();
1775}
1776#endif /* TSC */
1777#else /* USE_TICK_T */
1778#define MEASURE_LINE(expr) expr
1779#endif /* USE_TICK_T */
1780
1781static inline VALUE check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj);
1782
1783#define RVALUE_MARKED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj))
1784#define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj))
1785#define RVALUE_MARKING_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj))
1786#define RVALUE_UNCOLLECTIBLE_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj))
1787#define RVALUE_PINNED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), (obj))
1788
1789static inline int
1790RVALUE_MARKED(rb_objspace_t *objspace, VALUE obj)
1791{
1792 check_rvalue_consistency(objspace, obj);
1793 return RVALUE_MARKED_BITMAP(obj) != 0;
1794}
1795
1796static inline int
1797RVALUE_PINNED(rb_objspace_t *objspace, VALUE obj)
1798{
1799 check_rvalue_consistency(objspace, obj);
1800 return RVALUE_PINNED_BITMAP(obj) != 0;
1801}
1802
1803static inline int
1804RVALUE_WB_UNPROTECTED(rb_objspace_t *objspace, VALUE obj)
1805{
1806 check_rvalue_consistency(objspace, obj);
1807 return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1808}
1809
1810static inline int
1811RVALUE_MARKING(rb_objspace_t *objspace, VALUE obj)
1812{
1813 check_rvalue_consistency(objspace, obj);
1814 return RVALUE_MARKING_BITMAP(obj) != 0;
1815}
1816
1817static inline int
1818RVALUE_REMEMBERED(rb_objspace_t *objspace, VALUE obj)
1819{
1820 check_rvalue_consistency(objspace, obj);
1821 return MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
1822}
1823
1824static inline int
1825RVALUE_UNCOLLECTIBLE(rb_objspace_t *objspace, VALUE obj)
1826{
1827 check_rvalue_consistency(objspace, obj);
1828 return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1829}
1830
1831#define RVALUE_PAGE_WB_UNPROTECTED(page, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj))
1832#define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj))
1833#define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj))
1834
1835static void rgengc_remember(rb_objspace_t *objspace, VALUE obj);
1836static void gc_bitmaps_clear(rb_objspace_t *objspace, rb_heap_t *heap, bool clear_shref);
1837static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap);
1838static bool verify_pointer_in_any_heap_p(const void *ptr); /* cross-objspace ownership test */
1839
1840static int
1841check_rvalue_consistency_force(rb_objspace_t *objspace, const VALUE obj, int terminate)
1842{
1843 int err = 0;
1844
1845 /* Under a global GC the barrier stops every Ractor, so the cross-objspace walk
1846 * below is safe without the VM lock. Sweeping an ownerless zombie objspace also
1847 * leaves GET_RACTOR() NULL, and taking the lock here would dereference it. */
1848 const bool world_stopped = objspace->flags.during_global_gc;
1849 /* The VM lock protects the cross-objspace walk while other Ractors run and
1850 * reallocate their heaps. Not taken while this objspace is in GC: pages are stable
1851 * then, the cross-objspace walk needs the world stopped anyway, and the Ractor lock
1852 * may already be held (Ractor -> VM order inversion). A global GC holds the barrier
1853 * and needs no lock. */
1854 const bool take_vm_lock = !world_stopped && !during_gc;
1855 unsigned int lev = 0;
1856 if (take_vm_lock) lev = RB_GC_VM_LOCK_NO_BARRIER();
1857 {
1858 if (SPECIAL_CONST_P(obj)) {
1859 fprintf(stderr, "check_rvalue_consistency: %p is a special const.\n", (void *)obj);
1860 err++;
1861 }
1862 else if (!is_pointer_to_heap(objspace, (void *)obj)) {
1863 /* obj may be a legitimate cross-objspace reference (a shareable object, an
1864 * in-flight shref payload); it is a non-object only if no objspace's heap
1865 * holds it. A foreign object's mark/age/remembered bits belong to its
1866 * owner and reading them would race its local GC: skip per-object checks. */
1867 if (!world_stopped) {
1868 /* A mid-local-GC verify holds no barrier, so other Ractors reallocate
1869 * heap_pages.sorted under verify_pointer_in_any_heap_p's page_index
1870 * read. Accept foreign pointers here; the global GC's world-stopped
1871 * verify does the full existence check. */
1872 }
1873 else if (!verify_pointer_in_any_heap_p((void *)obj)) {
1874 struct heap_page *empty_page = objspace->empty_pages;
1875 while (empty_page) {
1876 if ((uintptr_t)empty_page->body <= (uintptr_t)obj &&
1877 (uintptr_t)obj < (uintptr_t)empty_page->body + HEAP_PAGE_SIZE) {
1878 GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, empty_page));
1879 fprintf(stderr, "check_rvalue_consistency: %p is in an empty page (%p).\n",
1880 (void *)obj, (void *)empty_page);
1881 err++;
1882 goto skip;
1883 }
1884 empty_page = empty_page->free_next;
1885 }
1886 fprintf(stderr, "check_rvalue_consistency: %p is not a Ruby object.\n", (void *)obj);
1887 err++;
1888 skip:
1889 ;
1890 }
1891 }
1892 else {
1893 const int wb_unprotected_bit = RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1894 const int uncollectible_bit = RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1895 const int mark_bit = RVALUE_MARKED_BITMAP(obj) != 0;
1896 const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0;
1897 const int remembered_bit = MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
1898 const int age = RVALUE_AGE_GET((VALUE)obj);
1899
1900 if (heap_page_in_global_empty_pages_pool(objspace, GET_HEAP_PAGE(obj))) {
1901 fprintf(stderr, "check_rvalue_consistency: %s is in tomb page.\n", rb_obj_info(obj));
1902 err++;
1903 }
1904 if (BUILTIN_TYPE(obj) == T_NONE) {
1905 fprintf(stderr, "check_rvalue_consistency: %s is T_NONE.\n", rb_obj_info(obj));
1906 err++;
1907 }
1908 if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
1909 fprintf(stderr, "check_rvalue_consistency: %s is T_ZOMBIE.\n", rb_obj_info(obj));
1910 err++;
1911 }
1912
1913 /* Do not run the memsize probe once an inconsistency (a T_NONE, say) was
1914 * found: an rb_bug inside the probe would lose the real diagnosis. */
1915 if (err == 0 && BUILTIN_TYPE(obj) != T_DATA) {
1916 rb_obj_memsize_of((VALUE)obj);
1917 }
1918
1919 /* check generation
1920 *
1921 * OLD == age == 3 && old-bitmap && mark-bit (except incremental marking)
1922 */
1923 if (age > 0 && wb_unprotected_bit) {
1924 fprintf(stderr, "check_rvalue_consistency: %s is not WB protected, but age is %d > 0.\n", rb_obj_info(obj), age);
1925 err++;
1926 }
1927
1928 if (!is_marking(objspace) && uncollectible_bit && !mark_bit) {
1929 fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but is not marked while !gc.\n", rb_obj_info(obj));
1930 err++;
1931 }
1932
1933 if (!is_full_marking(objspace)) {
1934 if (uncollectible_bit && age != RVALUE_OLD_AGE && !wb_unprotected_bit) {
1935 fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but not old (age: %d) and not WB unprotected.\n",
1936 rb_obj_info(obj), age);
1937 err++;
1938 }
1939 if (remembered_bit && age != RVALUE_OLD_AGE) {
1940 fprintf(stderr, "check_rvalue_consistency: %s is remembered, but not old (age: %d).\n",
1941 rb_obj_info(obj), age);
1942 err++;
1943 }
1944 }
1945
1946 /*
1947 * check coloring
1948 *
1949 * marking:false marking:true
1950 * marked:false white *invalid*
1951 * marked:true black grey
1952 */
1953 if (is_incremental_marking(objspace) && marking_bit) {
1954 if (!is_marking(objspace) && !mark_bit) {
1955 fprintf(stderr, "check_rvalue_consistency: %s is marking, but not marked.\n", rb_obj_info(obj));
1956 err++;
1957 }
1958 }
1959 }
1960 }
1961 if (take_vm_lock) RB_GC_VM_UNLOCK_NO_BARRIER(lev);
1962
1963 if (err > 0 && terminate) {
1964 rb_bug("check_rvalue_consistency_force: there is %d errors.", err);
1965 }
1966 return err;
1967}
1968
1969#if RGENGC_CHECK_MODE == 0
1970static inline VALUE
1971check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj)
1972{
1973 return obj;
1974}
1975#else
1976static VALUE
1977check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj)
1978{
1979 check_rvalue_consistency_force(objspace, obj, TRUE);
1980 return obj;
1981}
1982#endif
1983
1984static inline bool
1985gc_object_moved_p(rb_objspace_t *objspace, VALUE obj)
1986{
1987
1988 bool ret;
1989 asan_unpoisoning_object(obj) {
1990 ret = BUILTIN_TYPE(obj) == T_MOVED;
1991 }
1992 return ret;
1993}
1994
1995static inline int
1996RVALUE_OLD_P(rb_objspace_t *objspace, VALUE obj)
1997{
1998 GC_ASSERT(!RB_SPECIAL_CONST_P(obj));
1999 check_rvalue_consistency(objspace, obj);
2000 // Because this will only ever be called on GC controlled objects,
2001 // we can use the faster _RAW function here
2002 return RB_OBJ_PROMOTED_RAW(obj);
2003}
2004
2005static inline void
2006RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
2007{
2008 MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
2009 /* Count a promotion in the object's own objspace: a global GC ages every objspace's
2010 * slots from the driver, and counting them there would skew the other objspaces'
2011 * old_objects and with it their major GC frequency. */
2012 page->objspace->rgengc.old_objects++;
2013
2014#if RGENGC_PROFILE >= 2
2015 objspace->profile.total_promoted_count++;
2016 objspace->profile.promoted_types[BUILTIN_TYPE(obj)]++;
2017#endif
2018}
2019
2020static inline void
2021RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, VALUE obj)
2022{
2023 RB_DEBUG_COUNTER_INC(obj_promote);
2024 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, GET_HEAP_PAGE(obj), obj);
2025}
2026
2027/* set age to age+1 */
2028static inline void
2029RVALUE_AGE_INC(rb_objspace_t *objspace, VALUE obj)
2030{
2031 int age = RVALUE_AGE_GET((VALUE)obj);
2032
2033 if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) {
2034 rb_bug("RVALUE_AGE_INC: can not increment age of OLD object %s.", rb_obj_info(obj));
2035 }
2036
2037 age++;
2038 RVALUE_AGE_SET(obj, age);
2039
2040 if (age == RVALUE_OLD_AGE) {
2041 RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
2042 }
2043
2044 check_rvalue_consistency(objspace, obj);
2045}
2046
2047static inline void
2048RVALUE_AGE_SET_CANDIDATE(rb_objspace_t *objspace, VALUE obj)
2049{
2050 check_rvalue_consistency(objspace, obj);
2051 GC_ASSERT(!RVALUE_OLD_P(objspace, obj));
2052 RVALUE_AGE_SET(obj, RVALUE_OLD_AGE - 1);
2053 check_rvalue_consistency(objspace, obj);
2054}
2055
2056static inline void
2057RVALUE_AGE_RESET(VALUE obj)
2058{
2059 RVALUE_AGE_SET(obj, 0);
2060}
2061
2062static inline void
2063RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj)
2064{
2065 check_rvalue_consistency(objspace, obj);
2066 GC_ASSERT(RVALUE_OLD_P(objspace, obj));
2067
2068 if (!is_incremental_marking(objspace) && RVALUE_REMEMBERED(objspace, obj)) {
2069 struct heap_page *page = GET_HEAP_PAGE(obj);
2070 _CLEAR_IN_BITMAP(page->remembered_bits, page, obj);
2071 }
2072
2073 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
2074 RVALUE_AGE_RESET(obj);
2075
2076 if (RVALUE_MARKED(objspace, obj)) {
2077 /* symmetric with RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET */
2078 GET_HEAP_PAGE(obj)->objspace->rgengc.old_objects--;
2079 }
2080
2081 check_rvalue_consistency(objspace, obj);
2082}
2083
2084static inline int
2085RVALUE_BLACK_P(rb_objspace_t *objspace, VALUE obj)
2086{
2087 return RVALUE_MARKED(objspace, obj) && !RVALUE_MARKING(objspace, obj);
2088}
2089
2090static inline int
2091RVALUE_WHITE_P(rb_objspace_t *objspace, VALUE obj)
2092{
2093 return !RVALUE_MARKED(objspace, obj);
2094}
2095
2096bool
2097rb_gc_impl_user_gc_disabled_set(void *objspace_ptr, bool disable)
2098{
2099 rb_objspace_t *objspace = objspace_ptr;
2100 const bool was = objspace->flags.user_gc_disabled;
2101 objspace->flags.user_gc_disabled = disable;
2102 return was;
2103}
2104
2105bool
2106rb_gc_impl_user_gc_disabled_p(void *objspace_ptr)
2107{
2108 rb_objspace_t *objspace = objspace_ptr;
2109 return objspace->flags.user_gc_disabled;
2110}
2111
2112bool
2113rb_gc_impl_gc_enabled_p(void *objspace_ptr)
2114{
2115 rb_objspace_t *objspace = objspace_ptr;
2116 return !dont_gc_val();
2117}
2118
2119void
2120rb_gc_impl_gc_enable(void *objspace_ptr)
2121{
2122 rb_objspace_t *objspace = objspace_ptr;
2123
2124 dont_gc_off();
2125}
2126
2127void
2128rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc)
2129{
2130 rb_objspace_t *objspace = objspace_ptr;
2131
2132 if (finish_current_gc) {
2133 gc_rest(objspace);
2134 }
2135
2136 dont_gc_on();
2137}
2138
2139/* Finish an incremental mark or lazy sweep in progress without changing the enabled
2140 * state. gc.c uses it to settle the only objspace just before the process goes
2141 * multi-objspace. */
2142void
2143rb_gc_impl_gc_rest(void *objspace_ptr)
2144{
2145 gc_rest(objspace_ptr);
2146}
2147
2148/*
2149 --------------------------- ObjectSpace -----------------------------
2150*/
2151
2152static inline void *
2153calloc1(size_t n)
2154{
2155 return calloc(1, n);
2156}
2157
2158void
2159rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event)
2160{
2161 rb_objspace_t *objspace = objspace_ptr;
2162 /* FREEOBJ is main-objspace only (rb_objspace_set_event_hook masks it elsewhere). */
2163 GC_ASSERT(!(event & RUBY_INTERNAL_EVENT_FREEOBJ) ||
2164 objspace == global_objspace->main_objspace);
2165 objspace->hook_events = event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK;
2166}
2167
2168unsigned long long
2169rb_gc_impl_get_total_time(void *objspace_ptr)
2170{
2171 rb_objspace_t *objspace = objspace_ptr;
2172
2173 unsigned long long marking_time = objspace->profile.marking_time_ns;
2174 unsigned long long sweeping_time = objspace->profile.sweeping_time_ns;
2175
2176 return marking_time + sweeping_time;
2177}
2178
2179void
2180rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag)
2181{
2182 rb_objspace_t *objspace = objspace_ptr;
2183
2184 objspace->flags.measure_gc = RTEST(flag) ? TRUE : FALSE;
2185}
2186
2187bool
2188rb_gc_impl_get_measure_total_time(void *objspace_ptr)
2189{
2190 rb_objspace_t *objspace = objspace_ptr;
2191
2192 return objspace->flags.measure_gc;
2193}
2194
2195static void
2196gc_process_stat_capture(const rb_objspace_t *objspace,
2197 struct gc_process_stat_snapshot *out)
2198{
2199 out->minor_gc_count = objspace->profile.minor_gc_count;
2200 out->major_gc_count = objspace->profile.major_gc_count;
2201 out->global_gc_invocation_count = objspace->profile.global_gc_invocation_count;
2202 out->marking_time_ns = objspace->profile.marking_time_ns;
2203 out->sweeping_time_ns = objspace->profile.sweeping_time_ns;
2204}
2205
2206static void
2207gc_process_stat_publish(rb_objspace_t *objspace)
2208{
2209 struct gc_process_stat_snapshot snap;
2210 GC_ASSERT(objspace->profile.count == objspace->profile.minor_gc_count +
2211 objspace->profile.major_gc_count + objspace->profile.global_gc_count);
2212 gc_process_stat_capture(objspace, &snap);
2213 rb_native_mutex_lock(&objspace->process_stat.lock);
2214 objspace->process_stat.published = snap;
2215 rb_native_mutex_unlock(&objspace->process_stat.lock);
2216}
2217
2218static void
2219gc_process_stat_add(struct gc_process_stat_total *dst,
2220 const struct gc_process_stat_snapshot *src)
2221{
2222 dst->minor_gc_count += src->minor_gc_count;
2223 dst->major_gc_count += src->major_gc_count;
2224 dst->global_gc_invocation_count += src->global_gc_invocation_count;
2225 dst->marking_time_ns += src->marking_time_ns;
2226 dst->sweeping_time_ns += src->sweeping_time_ns;
2227}
2228
2229/* garbage objects will be collected soon. */
2230bool
2231rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE ptr)
2232{
2233 rb_objspace_t *objspace = objspace_ptr;
2234
2235 /* A foreign object is a live leaf: reading its type or mark bit would race the
2236 * owner's local GC, so outside a global GC's barrier never report it as garbage.
2237 * The fstring/symbol weak-set lookups do reach across objspaces, but those objects
2238 * are born shareable and only a stop-the-world global GC collects them, so "not
2239 * garbage" is correct. */
2240 if (gc_skip_foreign_object_p(objspace, ptr)) {
2241 return false;
2242 }
2243
2244 /* Asking whether a freed (T_NONE), moved (T_MOVED), or finalized (T_ZOMBIE)
2245 * object is garbage gives an unreliable answer: the slot may since have been
2246 * reused for an unrelated object. A reference to one of these is stale and a
2247 * bug in the caller. */
2248 asan_unpoisoning_object(ptr) {
2249 GC_ASSERT(BUILTIN_TYPE(ptr) != T_NONE);
2250 GC_ASSERT(BUILTIN_TYPE(ptr) != T_MOVED);
2251 GC_ASSERT(BUILTIN_TYPE(ptr) != T_ZOMBIE);
2252 }
2253
2254 return is_lazy_sweeping(objspace) && GET_HEAP_PAGE(ptr)->flags.before_sweep &&
2255 !RVALUE_MARKED(objspace, ptr);
2256}
2257
2258struct rb_gc_vm_context *
2259rb_gc_impl_get_vm_context(void *objspace_ptr)
2260{
2261 rb_objspace_t *objspace = objspace_ptr;
2262
2263 return &objspace->vm_context;
2264}
2265
2266static void free_stack_chunks(mark_stack_t *);
2267static void mark_stack_free_cache(mark_stack_t *);
2268static void heap_page_free(rb_objspace_t *objspace, struct heap_page *page);
2269
2270static inline void
2271gc_check_obj_in_page(struct heap_page *page, VALUE obj)
2272{
2273 if (RGENGC_CHECK_MODE &&
2274 /* obj should belong to page */
2275 !(page->start <= (uintptr_t)obj &&
2276 (uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) &&
2277 obj % sizeof(VALUE) == 0)) {
2278 rb_bug("gc_check_obj_in_page: %p is not rvalue.", (void *)obj);
2279 }
2280}
2281
2282static inline void
2283heap_page_add_free_region(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
2284{
2285 rb_asan_unpoison_object(obj, false);
2286
2287 // Should have already been reset
2288 GC_ASSERT(RVALUE_AGE_GET(obj) == 0);
2289
2290 gc_check_obj_in_page(page, obj);
2291
2292 asan_unlock_freelist(page);
2293
2294 /* Keep a freed slot from carrying its old shareable and shref bits into the next
2295 * object born there. */
2296 CLEAR_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj);
2297 CLEAR_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj);
2298
2299 struct free_region *region = (struct free_region *)obj;
2300 region->flags = 0;
2301 region->end = (uintptr_t)obj + page->slot_size;
2302 region->next = page->free_region;
2303 page->free_region = region;
2304
2305 asan_lock_freelist(page);
2306
2307 rb_asan_poison_object(obj);
2308 gc_report(3, objspace, "heap_page_add_free_region: %p\n", (void *)obj);
2309}
2310
2311/* The initial size is per objspace, so a Ractor's own gets a smaller one than
2312 * main's rather than paying main's again. */
2313static inline size_t
2314objspace_heap_init_bytes(const rb_objspace_t *objspace)
2315{
2316 return objspace == global_objspace->main_objspace
2317 ? gc_params.heap_init_bytes : gc_params.ractor_heap_init_bytes;
2318}
2319
2320static void
2321heap_allocatable_bytes_expand(rb_objspace_t *objspace,
2322 rb_heap_t *heap, size_t free_slots, size_t total_slots, size_t slot_size)
2323{
2324 double goal_ratio = gc_params.heap_free_slots_goal_ratio;
2325 size_t target_total_slots;
2326
2327 if (goal_ratio == 0.0) {
2328 target_total_slots = (size_t)(total_slots * gc_params.growth_factor);
2329 }
2330 else if (total_slots == 0) {
2331 target_total_slots = objspace_heap_init_bytes(objspace) / slot_size;
2332 }
2333 else {
2334 /* Find `f' where free_slots = f * total_slots * goal_ratio
2335 * => f = (total_slots - free_slots) / ((1 - goal_ratio) * total_slots)
2336 */
2337 double f = (double)(total_slots - free_slots) / ((1 - goal_ratio) * total_slots);
2338
2339 if (f > gc_params.growth_factor) f = gc_params.growth_factor;
2340 if (f < 1.0) f = 1.1;
2341
2342 target_total_slots = (size_t)(f * total_slots);
2343
2344 if (0) {
2345 fprintf(stderr,
2346 "free_slots(%8"PRIuSIZE")/total_slots(%8"PRIuSIZE")=%1.2f,"
2347 " G(%1.2f), f(%1.2f),"
2348 " total_slots(%8"PRIuSIZE") => target_total_slots(%8"PRIuSIZE")\n",
2349 free_slots, total_slots, free_slots/(double)total_slots,
2350 goal_ratio, f, total_slots, target_total_slots);
2351 }
2352 }
2353
2354 if (gc_params.growth_max_bytes > 0) {
2355 size_t max_total_slots = total_slots + gc_params.growth_max_bytes / slot_size;
2356 if (target_total_slots > max_total_slots) target_total_slots = max_total_slots;
2357 }
2358
2359 size_t extend_slot_count = target_total_slots - total_slots;
2360 /* Extend by at least 1 page. */
2361 if (extend_slot_count == 0) extend_slot_count = 1;
2362
2363 objspace->heap_pages.allocatable_bytes += extend_slot_count * slot_size;
2364}
2365
2366static inline void
2367heap_add_freepage(rb_heap_t *heap, struct heap_page *page)
2368{
2369 asan_unlock_freelist(page);
2370 GC_ASSERT(page->free_slots != 0);
2371 GC_ASSERT(page->free_region != NULL);
2372
2373 page->free_next = heap->free_pages;
2374 heap->free_pages = page;
2375
2376 RUBY_DEBUG_LOG("page:%p free_region:%p", (void *)page, (void *)page->free_region);
2377
2378 asan_lock_freelist(page);
2379}
2380
2381static inline void
2382heap_add_poolpage(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
2383{
2384 asan_unlock_freelist(page);
2385 GC_ASSERT(page->free_slots != 0);
2386 GC_ASSERT(page->free_region != NULL);
2387
2388 page->free_next = heap->pooled_pages;
2389 heap->pooled_pages = page;
2390 objspace->rincgc.pooled_slots += page->free_slots;
2391
2392 asan_lock_freelist(page);
2393}
2394
2395static void
2396heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
2397{
2398 ccan_list_del(&page->page_node);
2399 heap->total_pages--;
2400 heap->total_slots -= page->total_slots;
2401}
2402
2403/* Detach a page with no living objects from its heap and park it on the objspace's
2404 * empty pages, from where it is reused or released to the page pool. The caller has
2405 * already taken it off the heap's free_pages chain, if it was on it. */
2406static void
2407heap_page_detach_empty(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
2408{
2409 heap_unlink_page(objspace, heap, page);
2410
2411 page->start = 0;
2412 page->total_slots = 0;
2413 page->slot_size = 0;
2414 page->heap = NULL;
2415 page->free_slots = 0;
2416
2417 asan_unlock_freelist(page);
2418 page->free_region = NULL;
2419 asan_lock_freelist(page);
2420
2421 asan_poison_memory_region(page->body, HEAP_PAGE_SIZE);
2422
2423 objspace->empty_pages_count++;
2424 page->free_next = objspace->empty_pages;
2425 objspace->empty_pages = page;
2426}
2427
2428static void
2429gc_aligned_free(void *ptr, size_t size)
2430{
2431#if defined __MINGW32__
2432 __mingw_aligned_free(ptr);
2433#elif defined _WIN32
2434 _aligned_free(ptr);
2435#elif defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN)
2436 free(ptr);
2437#else
2438 free(((void**)ptr)[-1]);
2439#endif
2440}
2441
2442static void
2443heap_page_body_free(struct heap_page_body *page_body, struct arena_descriptor *arena)
2444{
2445 GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
2446
2447 page_pool_release(page_body, arena);
2448}
2449
2450#ifdef PAGE_POOL_LOCK_ERRORCHECK
2451# define ASSERT_PAGE_POOL_LOCKED(g) GC_ASSERT(pthread_mutex_lock(&(g)->page_pool.lock) == EDEADLK)
2452#else
2453# define ASSERT_PAGE_POOL_LOCKED(g) ((void)0)
2454#endif
2455
2456/* Insert into page_index. Writers serialize on page_pool.lock; lomem and himem are a
2457 * monotonically growing over-approximation used for a quick reject. We need to keep
2458 * track of this for conservative marking during global GC.
2459 *
2460 * TODO: This can use its own lock, it shouldn't block the page pool.
2461 */
2462static void
2463global_page_index_insert(struct heap_page *page)
2464{
2465 rb_global_objspace_t *g = global_objspace;
2466 uintptr_t body = (uintptr_t)page->body;
2467
2468 rb_native_mutex_lock(&g->page_pool.lock);
2469 if (g->page_index.n_pages == g->page_index.capa) {
2470 size_t new_capa = g->page_index.capa ? g->page_index.capa * 2 : 128;
2471 struct heap_page **grown = realloc(g->page_index.pages, new_capa * sizeof(*grown));
2472 if (grown == NULL) rb_bug("global_page_index_insert: realloc failed");
2473 g->page_index.pages = grown;
2474 g->page_index.capa = new_capa;
2475 }
2476 size_t lo = 0, hi = g->page_index.n_pages;
2477 while (lo < hi) {
2478 size_t mid = (lo + hi) / 2;
2479 if ((uintptr_t)g->page_index.pages[mid]->body < body) lo = mid + 1;
2480 else hi = mid;
2481 }
2482 memmove(&g->page_index.pages[lo + 1], &g->page_index.pages[lo],
2483 (g->page_index.n_pages - lo) * sizeof(struct heap_page *));
2484 g->page_index.pages[lo] = page;
2485 g->page_index.n_pages++;
2486
2487 uintptr_t start = body + sizeof(struct heap_page_header);
2488 uintptr_t end = body + HEAP_PAGE_SIZE;
2489 if (g->page_index.lomem == 0 || g->page_index.lomem > start) g->page_index.lomem = start;
2490 if (g->page_index.himem < end) g->page_index.himem = end;
2491 rb_native_mutex_unlock(&g->page_pool.lock);
2492}
2493
2494static void
2495global_page_index_remove_locked(const struct heap_page *page)
2496{
2497 rb_global_objspace_t *g = global_objspace;
2498 uintptr_t body = (uintptr_t)page->body;
2499
2500 ASSERT_PAGE_POOL_LOCKED(g);
2501
2502 size_t lo = 0, hi = g->page_index.n_pages;
2503 while (lo < hi) {
2504 size_t mid = (lo + hi) / 2;
2505 if ((uintptr_t)g->page_index.pages[mid]->body < body) lo = mid + 1;
2506 else hi = mid;
2507 }
2508 GC_ASSERT(lo < g->page_index.n_pages && g->page_index.pages[lo] == page);
2509 memmove(&g->page_index.pages[lo], &g->page_index.pages[lo + 1],
2510 (g->page_index.n_pages - lo - 1) * sizeof(struct heap_page *));
2511 g->page_index.n_pages--;
2512}
2513
2514static void
2515global_page_index_remove(const struct heap_page *page)
2516{
2517 rb_global_objspace_t *g = global_objspace;
2518
2519 rb_native_mutex_lock(&g->page_pool.lock);
2520 global_page_index_remove_locked(page);
2521 rb_native_mutex_unlock(&g->page_pool.lock);
2522}
2523
2524static void
2525heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
2526{
2527 global_page_index_remove(page);
2528 objspace->heap_pages.freed_pages++;
2529 heap_page_body_free(page->body, page->arena);
2530 free(page);
2531}
2532
2533static void
2534heap_pages_free_batch(rb_objspace_t *objspace, struct heap_page *pages)
2535{
2536 rb_global_objspace_t *g = global_objspace;
2537
2538 rb_native_mutex_lock(&g->page_pool.lock);
2539 for (struct heap_page *page = pages; page != NULL; page = page->free_next) {
2540 global_page_index_remove_locked(page);
2541 if (HEAP_PAGE_ALLOC_USE_MMAP) {
2542#ifdef HAVE_MMAP
2543 page_pool_release_locked(page->body, page->arena);
2544#endif
2545 }
2546 }
2547 rb_native_mutex_unlock(&g->page_pool.lock);
2548
2549 if (!HEAP_PAGE_ALLOC_USE_MMAP) {
2550 /* gc_aligned_free does not need the pool lock. */
2551 for (struct heap_page *page = pages; page != NULL; page = page->free_next) {
2552 heap_page_body_free(page->body, page->arena);
2553 }
2554 }
2555
2556 while (pages != NULL) {
2557 struct heap_page *next = pages->free_next;
2558 objspace->heap_pages.freed_pages++;
2559 free(pages);
2560 pages = next;
2561 }
2562}
2563
2564static void
2565heap_pages_free_unused_pages(rb_objspace_t *objspace)
2566{
2567 if (objspace->empty_pages != NULL && heap_pages_freeable_pages > 0) {
2568 GC_ASSERT(objspace->empty_pages_count > 0);
2569 objspace->empty_pages = NULL;
2570 objspace->empty_pages_count = 0;
2571
2572 size_t i, j;
2573 struct heap_page *to_free = NULL;
2574 for (i = j = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
2575 struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
2576
2577 if (heap_page_in_global_empty_pages_pool(objspace, page) && heap_pages_freeable_pages > 0) {
2578 page->free_next = to_free;
2579 to_free = page;
2580 heap_pages_freeable_pages--;
2581 }
2582 else {
2583 if (heap_page_in_global_empty_pages_pool(objspace, page)) {
2584 page->free_next = objspace->empty_pages;
2585 objspace->empty_pages = page;
2586 objspace->empty_pages_count++;
2587 }
2588
2589 if (i != j) {
2590 rb_darray_set(objspace->heap_pages.sorted, j, page);
2591 }
2592 j++;
2593 }
2594 }
2595
2596 rb_darray_pop(objspace->heap_pages.sorted, i - j);
2597 GC_ASSERT(rb_darray_size(objspace->heap_pages.sorted) == j);
2598
2599 /* A retire GC can free every page, so an empty objspace is legitimate. */
2600 if (j > 0) {
2601 struct heap_page *hipage = rb_darray_get(objspace->heap_pages.sorted, rb_darray_size(objspace->heap_pages.sorted) - 1);
2602 uintptr_t himem = (uintptr_t)hipage->body + HEAP_PAGE_SIZE;
2603 GC_ASSERT(himem <= heap_pages_himem);
2604 heap_pages_himem = himem;
2605
2606 struct heap_page *lopage = rb_darray_get(objspace->heap_pages.sorted, 0);
2607 uintptr_t lomem = (uintptr_t)lopage->body + sizeof(struct heap_page_header);
2608 GC_ASSERT(lomem >= heap_pages_lomem);
2609 heap_pages_lomem = lomem;
2610 }
2611 else {
2612 heap_pages_lomem = 0;
2613 heap_pages_himem = 0;
2614 }
2615
2616 heap_pages_free_batch(objspace, to_free);
2617 }
2618}
2619
2620static void *
2621gc_aligned_malloc(size_t alignment, size_t size)
2622{
2623 /* alignment must be a power of 2 */
2624 GC_ASSERT(((alignment - 1) & alignment) == 0);
2625 GC_ASSERT(alignment % sizeof(void*) == 0);
2626
2627 void *res;
2628
2629#if defined __MINGW32__
2630 res = __mingw_aligned_malloc(size, alignment);
2631#elif defined _WIN32
2632 res = _aligned_malloc(size, alignment);
2633#elif defined(HAVE_POSIX_MEMALIGN)
2634 if (posix_memalign(&res, alignment, size) != 0) {
2635 return NULL;
2636 }
2637#elif defined(HAVE_MEMALIGN)
2638 res = memalign(alignment, size);
2639#else
2640 char* aligned;
2641 res = malloc(alignment + size + sizeof(void*));
2642 aligned = (char*)res + alignment + sizeof(void*);
2643 aligned -= ((VALUE)aligned & (alignment - 1));
2644 ((void**)aligned)[-1] = res;
2645 res = (void*)aligned;
2646#endif
2647
2648 GC_ASSERT((uintptr_t)res % alignment == 0);
2649
2650 return res;
2651}
2652
2653/* The page pool (global_objspace->page_pool): heap page bodies are carved out of large
2654 * arenas and reused through the pool. Each arena descriptor tracks its bodies in two
2655 * bitmasks rather than threading a freelist through the bodies themselves, so a free
2656 * body holds nothing and page_pool_reclaim can advise all of it. */
2657
2658#define PAGE_POOL_ARENA_SIZE (HEAP_PAGE_SIZE * 32) /* 2MiB with 64KiB pages */
2659#define PAGE_POOL_ARENA_BODIES (PAGE_POOL_ARENA_SIZE / HEAP_PAGE_SIZE) /* 32 */
2660#define PAGE_POOL_ARENA_KEEP_HALF (PAGE_POOL_ARENA_BODIES / 2) /* 16 */
2661
2662#define PAGE_POOL_ALL_BODIES ((uint32_t)((UINT64_C(1) << PAGE_POOL_ARENA_BODIES) - 1))
2663
2665 struct arena_descriptor *next;
2666 char *start; /* usable area, HEAP_PAGE_ALIGN aligned */
2667 size_t size; /* usable bytes (a multiple of HEAP_PAGE_SIZE) */
2668 uint32_t free_mask;
2669 uint32_t advised_mask;
2670 uint32_t zeroed_mask;
2671};
2672STATIC_ASSERT(page_pool_bodies_fit_mask, PAGE_POOL_ARENA_BODIES <= 32);
2673
2674/* page_pool_reclaim advises whole bodies, so a body has to be a whole number of OS
2675 * pages: rounding a partial one up would reach into the neighbouring body, which may be
2676 * in use. A platform whose page size does not divide HEAP_PAGE_SIZE (or that cannot
2677 * report one) therefore advises nothing, and advised_count must not be adjusted
2678 * anywhere either. */
2679static bool
2680page_pool_advise_supported(void)
2681{
2682#ifdef HAVE_MMAP
2683 long os_page_size = sysconf(_SC_PAGE_SIZE);
2684 return os_page_size > 0 && HEAP_PAGE_SIZE % (size_t)os_page_size == 0;
2685#else
2686 return false;
2687#endif
2688}
2689
2690#ifdef HAVE_MMAP
2691static int
2692page_pool_body_index(const struct arena_descriptor *arena, const struct heap_page_body *body)
2693{
2694 ptrdiff_t off = (const char *)body - arena->start;
2695 GC_ASSERT(off >= 0 && off % HEAP_PAGE_SIZE == 0);
2696 int i = (int)(off / HEAP_PAGE_SIZE);
2697 GC_ASSERT(i < PAGE_POOL_ARENA_BODIES);
2698 return i;
2699}
2700
2701static struct heap_page_body *
2702page_pool_body_at(const struct arena_descriptor *arena, int i)
2703{
2704 GC_ASSERT(i >= 0 && i < PAGE_POOL_ARENA_BODIES);
2705 return (struct heap_page_body *)(arena->start + (size_t)i * HEAP_PAGE_SIZE);
2706}
2707#endif
2708
2709#ifdef HAVE_MMAP
2710/* mmap a new arena to carve from. Called with the pool lock held, at which point the
2711 * previous arena is always fully carved. */
2712static bool
2713page_pool_add_arena(rb_global_objspace_t *g)
2714{
2715 GC_ASSERT(HEAP_PAGE_ALIGN % sysconf(_SC_PAGE_SIZE) == 0);
2716
2717 size_t mmap_size = PAGE_POOL_ARENA_SIZE + HEAP_PAGE_ALIGN;
2718 char *ptr = mmap(NULL, mmap_size,
2719 PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
2720 if (ptr == MAP_FAILED) {
2721 return false;
2722 }
2723
2724 // If we are building `default.c` as part of the ruby executable, we
2725 // may just call `ruby_annotate_mmap`. But if we are building
2726 // `default.c` as a shared library, we will not have access to private
2727 // symbols, and we have to either call prctl directly or make our own
2728 // wrapper.
2729#if defined(HAVE_SYS_PRCTL_H) && defined(PR_SET_VMA) && defined(PR_SET_VMA_ANON_NAME)
2730 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, mmap_size, "Ruby:GC:default:page_pool_arena");
2731 errno = 0;
2732#endif
2733
2734 /* Trim the unaligned head and tail so the usable area is HEAP_PAGE_ALIGN aligned. */
2735 char *aligned = ptr + HEAP_PAGE_ALIGN;
2736 aligned -= ((uintptr_t)aligned & (HEAP_PAGE_ALIGN - 1));
2737 GC_ASSERT(aligned > ptr);
2738 GC_ASSERT(aligned <= ptr + HEAP_PAGE_ALIGN);
2739
2740 size_t start_out_of_range_size = aligned - ptr;
2741 GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
2742 if (start_out_of_range_size > 0) {
2743 if (munmap(ptr, start_out_of_range_size)) {
2744 rb_bug("page_pool_add_arena: munmap failed for start");
2745 }
2746 }
2747
2748 size_t end_out_of_range_size = HEAP_PAGE_ALIGN - start_out_of_range_size;
2749 GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
2750 if (end_out_of_range_size > 0) {
2751 if (munmap(aligned + PAGE_POOL_ARENA_SIZE, end_out_of_range_size)) {
2752 rb_bug("page_pool_add_arena: munmap failed for end");
2753 }
2754 }
2755
2756 struct arena_descriptor *arena = calloc1(sizeof(struct arena_descriptor));
2757 if (arena == NULL) {
2758 if (munmap(aligned, PAGE_POOL_ARENA_SIZE)) {
2759 rb_bug("page_pool_add_arena: munmap failed for arena");
2760 }
2761 return false;
2762 }
2763 arena->start = aligned;
2764 arena->size = PAGE_POOL_ARENA_SIZE;
2765 arena->free_mask = 0;
2766 arena->advised_mask = 0;
2767 arena->next = g->page_pool.arenas;
2768 g->page_pool.arenas = arena;
2769 g->page_pool.arena_count++;
2770 g->page_pool.arena_current = arena;
2771
2772 g->page_pool.arena_cursor = aligned;
2773 g->page_pool.arena_end = aligned + PAGE_POOL_ARENA_SIZE;
2774
2775 return true;
2776}
2777#endif
2778
2779/* Acquire a free page from the page pool. It can give back a warm or "cold" page that will cause
2780 * a page fault on first access. If there are no free pages, an extra arena is allocated. */
2781static struct heap_page_body *
2782page_pool_acquire(struct arena_descriptor **arena_out, bool *zeroed_out)
2783{
2784 struct heap_page_body *body = NULL;
2785 *zeroed_out = false;
2786
2787 if (HEAP_PAGE_ALLOC_USE_MMAP) {
2788#ifdef HAVE_MMAP
2789 bool need_reuse = false;
2790 rb_global_objspace_t *g = global_objspace;
2791
2792 rb_native_mutex_lock(&g->page_pool.lock);
2793 // reuse a free body (it may have been madvised)
2794 for (struct arena_descriptor *a = g->page_pool.arenas; a; a = a->next) {
2795 if (a->free_mask == 0) continue;
2796
2797 /* Prefer a body the OS still backs: that skips both the MADV_FREE_REUSE
2798 * handshake and the refault. */
2799 uint32_t warm = a->free_mask & ~a->advised_mask;
2800 int i = ntz_int32(warm ? warm : a->free_mask);
2801 uint32_t bit = (uint32_t)1 << i;
2802
2803 body = page_pool_body_at(a, i);
2804 *arena_out = a;
2805 *zeroed_out = (a->zeroed_mask & bit) != 0;
2806 a->free_mask &= ~bit;
2807 a->zeroed_mask &= ~bit;
2808 need_reuse = (a->advised_mask & bit) != 0;
2809 if (need_reuse) {
2810 a->advised_mask &= ~bit;
2811 g->page_pool.advised_count--;
2812 }
2813 break;
2814 }
2815 if (body == NULL &&
2816 (g->page_pool.arena_cursor != g->page_pool.arena_end ||
2817 page_pool_add_arena(g))) {
2818 GC_ASSERT(g->page_pool.arena_cursor + HEAP_PAGE_SIZE <= g->page_pool.arena_end);
2819 body = (struct heap_page_body *)g->page_pool.arena_cursor;
2820 g->page_pool.arena_cursor += HEAP_PAGE_SIZE;
2821 *arena_out = g->page_pool.arena_current;
2822 /* Never handed out since the arena was mapped. */
2823 *zeroed_out = true;
2824 }
2825 rb_native_mutex_unlock(&g->page_pool.lock);
2826
2827 if (body != NULL) {
2828 if (need_reuse) {
2829 rb_vm_map_reuse(body, HEAP_PAGE_SIZE);
2830 }
2831 asan_unpoison_memory_region(body, HEAP_PAGE_SIZE, false);
2832 }
2833#endif
2834 }
2835 else {
2836 body = gc_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
2837 *arena_out = NULL;
2838 }
2839
2840 return body;
2841}
2842
2843#ifdef HAVE_MMAP
2844static void
2845page_pool_release_locked(struct heap_page_body *body, struct arena_descriptor *arena)
2846{
2847 ASSERT_PAGE_POOL_LOCKED(global_objspace);
2848
2849 uint32_t bit = (uint32_t)1 << page_pool_body_index(arena, body);
2850
2851 GC_ASSERT(!(arena->free_mask & bit)); /* double release */
2852 GC_ASSERT(!(arena->advised_mask & bit)); /* a body in use is never advised */
2853
2854 arena->free_mask |= bit;
2855 arena->zeroed_mask &= ~bit;
2856 asan_poison_memory_region(body, HEAP_PAGE_SIZE);
2857}
2858#endif
2859
2860static void
2861page_pool_release(struct heap_page_body *body, struct arena_descriptor *arena)
2862{
2863 if (HEAP_PAGE_ALLOC_USE_MMAP) {
2864#ifdef HAVE_MMAP
2865 rb_global_objspace_t *g = global_objspace;
2866
2867 rb_native_mutex_lock(&g->page_pool.lock);
2868 page_pool_release_locked(body, arena);
2869 rb_native_mutex_unlock(&g->page_pool.lock);
2870#endif
2871 }
2872 else {
2873 gc_aligned_free(body, HEAP_PAGE_SIZE);
2874 }
2875}
2876
2877/* Allow the OS to reclaim pool memory.
2878 *
2879 * Step A: madvise free bodies, coalescing physically adjacent ones into a single call.
2880 * Step B: munmap arenas whose 32 bodies are all free, keeping one extra empty
2881 * arena as a retention buffer when the remaining free pool is < half an arena. */
2882static void
2883page_pool_reclaim(rb_global_objspace_t *g)
2884{
2885 if (!HEAP_PAGE_ALLOC_USE_MMAP) return;
2886#ifdef HAVE_MMAP
2887 rb_native_mutex_lock(&g->page_pool.lock);
2888
2889 /* Step A — advise free bodies (immediate release: drop RSS now if the platform
2890 * allows). The mask is in address order, so walking it issues one call per run of
2891 * physically adjacent bodies: a fully free arena costs one madvise rather than 32. */
2892 if (g->page_pool.can_advise) {
2893 for (struct arena_descriptor *a = g->page_pool.arenas; a; a = a->next) {
2894 uint32_t advise_mask = a->free_mask & ~a->advised_mask;
2895 uint32_t zeroed_mask = 0;
2896 if (advise_mask == 0) continue; /* steady state */
2897
2898 for (int i = 0; i < PAGE_POOL_ARENA_BODIES; i++) {
2899 if (!(advise_mask & ((uint32_t)1 << i))) continue;
2900
2901 int run = i;
2902 while (i + 1 < PAGE_POOL_ARENA_BODIES &&
2903 (advise_mask & ((uint32_t)1 << (i + 1)))) {
2904 i++;
2905 }
2906 int run_len = i - run + 1;
2907
2908 bool zeroed;
2909 rb_vm_map_reusable_immediate(page_pool_body_at(a, run),
2910 (size_t)run_len * HEAP_PAGE_SIZE, 0, &zeroed);
2911 if (zeroed) {
2912 for (int j = run; j <= i; j++) {
2913 zeroed_mask |= (uint32_t)1 << j;
2914 }
2915 }
2916 g->page_pool.advised_count += run_len;
2917 }
2918 a->advised_mask |= advise_mask;
2919 a->zeroed_mask |= zeroed_mask;
2920 }
2921 }
2922
2923 /* Step B — munmap fully-free arenas (with retention buffer).
2924 *
2925 * total_free = sum(popcount(free_mask)). An arena is eligible when all 32 of its
2926 * bodies are free. Keep one extra empty arena when the rest of the free pool is
2927 * < half an arena, to avoid thrash. */
2928 int total_free = 0;
2929 for (struct arena_descriptor *a = g->page_pool.arenas; a; a = a->next) {
2930 total_free += (int)rb_popcount32(a->free_mask);
2931 }
2932
2933 bool retained_one = false;
2934 struct arena_descriptor **pp = &g->page_pool.arenas;
2935 // munmap fully free arenas
2936 while (*pp) {
2937 struct arena_descriptor *a = *pp;
2938 if (a->free_mask != PAGE_POOL_ALL_BODIES) {
2939 pp = &a->next;
2940 continue;
2941 }
2942
2943 int free_elsewhere = total_free - PAGE_POOL_ARENA_BODIES;
2944 if (free_elsewhere < PAGE_POOL_ARENA_KEEP_HALF && !retained_one) {
2945 retained_one = true;
2946 pp = &a->next;
2947 continue;
2948 }
2949
2950 *pp = a->next;
2951 if (munmap(a->start, a->size)) {
2952 rb_bug("page_pool_reclaim: munmap failed");
2953 }
2954 total_free -= PAGE_POOL_ARENA_BODIES;
2955 g->page_pool.advised_count -= (int)rb_popcount32(a->advised_mask);
2956 GC_ASSERT(g->page_pool.advised_count >= 0);
2957 g->page_pool.arena_count--;
2958 g->page_pool.arenas_unmapped++;
2959 if (a == g->page_pool.arena_current) {
2960 // During next acquire, any remaining arenas that have free bodies are used. This is guaranteed
2961 // because of the retention buffer.
2962 g->page_pool.arena_current = NULL;
2963 g->page_pool.arena_cursor = NULL;
2964 g->page_pool.arena_end = NULL;
2965 }
2966 free(a);
2967 }
2968
2969 rb_native_mutex_unlock(&g->page_pool.lock);
2970#endif
2971}
2972
2973static struct heap_page_body *
2974heap_page_body_allocate(struct arena_descriptor **arena_out, bool *zeroed_out)
2975{
2976 struct heap_page_body *page_body = page_pool_acquire(arena_out, zeroed_out);
2977
2978 GC_ASSERT(page_body == NULL || (uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
2979
2980 return page_body;
2981}
2982
2983static struct heap_page *
2984heap_page_resurrect(rb_objspace_t *objspace)
2985{
2986 struct heap_page *page = NULL;
2987 if (objspace->empty_pages == NULL) {
2988 GC_ASSERT(objspace->empty_pages_count == 0);
2989 }
2990 else {
2991 GC_ASSERT(objspace->empty_pages_count > 0);
2992 objspace->empty_pages_count--;
2993 page = objspace->empty_pages;
2994 objspace->empty_pages = page->free_next;
2995 /* Clear the flags left over from emptying the page before reusing it, or the
2996 * shareable and shref scans would keep walking an empty bitmap forever. */
2997 page->flags.has_shareable_objects = FALSE;
2998 page->flags.has_shref_objects = FALSE;
2999 }
3000
3001 return page;
3002}
3003
3004static struct heap_page *
3005heap_page_allocate(rb_objspace_t *objspace)
3006{
3007 bool zeroed;
3008 struct arena_descriptor *arena;
3009 struct heap_page_body *page_body = heap_page_body_allocate(&arena, &zeroed);
3010 if (page_body == 0) {
3011 rb_memerror();
3012 }
3013
3014 struct heap_page *page = calloc1(sizeof(struct heap_page));
3015 if (page == 0) {
3016 heap_page_body_free(page_body, arena);
3017 rb_memerror();
3018 }
3019
3020 uintptr_t start = (uintptr_t)page_body + sizeof(struct heap_page_header);
3021 uintptr_t end = (uintptr_t)page_body + HEAP_PAGE_SIZE;
3022
3023 size_t lo = 0;
3024 size_t hi = rb_darray_size(objspace->heap_pages.sorted);
3025 while (lo < hi) {
3026 struct heap_page *mid_page;
3027
3028 size_t mid = (lo + hi) / 2;
3029 mid_page = rb_darray_get(objspace->heap_pages.sorted, mid);
3030 if ((uintptr_t)mid_page->start < start) {
3031 lo = mid + 1;
3032 }
3033 else if ((uintptr_t)mid_page->start > start) {
3034 hi = mid;
3035 }
3036 else {
3037 rb_bug("same heap page is allocated: %p at %"PRIuVALUE, (void *)page_body, (VALUE)mid);
3038 }
3039 }
3040
3041 rb_darray_insert_without_gc(&objspace->heap_pages.sorted, hi, page);
3042
3043 if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
3044 if (heap_pages_himem < end) heap_pages_himem = end;
3045
3046 page->body = page_body;
3047 page->arena = arena;
3048 page->flags.body_zeroed = zeroed;
3049 page_body->header.page = page;
3050 page->objspace = objspace;
3051
3052 objspace->heap_pages.allocated_pages++;
3053
3054 global_page_index_insert(page);
3055
3056 return page;
3057}
3058
3059static void
3060heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
3061{
3062 /* Adding to eden heap during incremental sweeping is forbidden */
3063 GC_ASSERT(!heap->sweeping_page);
3064 GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page));
3065
3066 /* Align start to slot_size boundary */
3067 uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header);
3068 uintptr_t rem = start % heap->slot_size;
3069 if (rem) start += heap->slot_size - rem;
3070
3071 int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size);
3072
3073 page->start = start;
3074 page->total_slots = slot_count;
3075 page->slot_size = heap->slot_size;
3076 page->slot_size_reciprocal = heap_slot_reciprocal_table[heap - heaps];
3077 page->heap = heap;
3078
3079 memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
3080 memset(&page->age_bits[0], 0, sizeof(page->age_bits));
3081
3082 asan_unlock_freelist(page);
3083 asan_unpoison_memory_region(page->body, HEAP_PAGE_SIZE, false);
3084
3085 uintptr_t slots_end = start + (uintptr_t)slot_count * heap->slot_size;
3086
3087 /* Free slots must read as T_NONE. A page the pool knows is zero already is fine
3088 * and writing it anyway would fault in the whole thing */
3089 if (!page->flags.body_zeroed) {
3090 memset((void *)start, 0, slots_end - start);
3091 }
3092 page->flags.body_zeroed = FALSE;
3093
3094 struct free_region *region = (struct free_region *)start;
3095 region->flags = 0;
3096 region->end = slots_end;
3097 region->next = NULL;
3098 page->free_region = region;
3099
3100 /* Poison every free slot; each is unpoisoned again as it is handed out. */
3101 for (uintptr_t p = start; p < slots_end; p += heap->slot_size) {
3102 rb_asan_poison_object((VALUE)p);
3103 }
3104 asan_lock_freelist(page);
3105
3106 page->free_slots = slot_count;
3107
3108 heap->total_allocated_pages++;
3109
3110 ccan_list_add_tail(&heap->pages, &page->page_node);
3111 heap->total_pages++;
3112 heap->total_slots += page->total_slots;
3113}
3114
3115static int
3116heap_page_allocate_and_initialize(rb_objspace_t *objspace, rb_heap_t *heap)
3117{
3118 gc_report(1, objspace, "heap_page_allocate_and_initialize: rb_darray_size(objspace->heap_pages.sorted): %"PRIdSIZE", "
3119 "allocatable_bytes: %"PRIdSIZE", heap->total_pages: %"PRIdSIZE"\n",
3120 rb_darray_size(objspace->heap_pages.sorted), objspace->heap_pages.allocatable_bytes, heap->total_pages);
3121
3122 bool allocated = false;
3123 struct heap_page *page = heap_page_resurrect(objspace);
3124
3125 if (page == NULL && objspace->heap_pages.allocatable_bytes > 0) {
3126 page = heap_page_allocate(objspace);
3127 allocated = true;
3128
3129 GC_ASSERT(page != NULL);
3130 }
3131
3132 if (page != NULL) {
3133 heap_add_page(objspace, heap, page);
3134 heap_add_freepage(heap, page);
3135
3136 if (allocated) {
3137 size_t page_bytes = (size_t)page->total_slots * page->slot_size;
3138 if (objspace->heap_pages.allocatable_bytes > page_bytes) {
3139 objspace->heap_pages.allocatable_bytes -= page_bytes;
3140 }
3141 else {
3142 objspace->heap_pages.allocatable_bytes = 0;
3143 }
3144 }
3145 }
3146
3147 return page != NULL;
3148}
3149
3150static void
3151heap_page_allocate_and_initialize_force(rb_objspace_t *objspace, rb_heap_t *heap)
3152{
3153 size_t prev_allocatable_bytes = objspace->heap_pages.allocatable_bytes;
3154 objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE;
3155 heap_page_allocate_and_initialize(objspace, heap);
3156 GC_ASSERT(heap->free_pages != NULL);
3157 objspace->heap_pages.allocatable_bytes = prev_allocatable_bytes;
3158}
3159
3160static void
3161gc_continue(rb_objspace_t *objspace, rb_heap_t *heap)
3162{
3163 unsigned int lock_lev;
3164 bool needs_gc = is_incremental_marking(objspace) || needs_continue_sweeping(objspace, heap);
3165 if (!needs_gc) return;
3166
3167 gc_enter(objspace, gc_enter_event_continue, &lock_lev); // takes vm barrier, try to avoid
3168
3169 /* Continue marking if in incremental marking. */
3170 if (is_incremental_marking(objspace)) {
3171 if (gc_marks_continue(objspace, heap)) {
3172 gc_sweep(objspace);
3173 }
3174 }
3175
3176 if (needs_continue_sweeping(objspace, heap)) {
3177 gc_sweep_continue(objspace, heap);
3178 }
3179
3180 gc_exit(objspace, gc_enter_event_continue, &lock_lev);
3181}
3182
3183static void
3184heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap)
3185{
3186 GC_ASSERT(heap->free_pages == NULL);
3187
3188 if (heap->total_slots < objspace_heap_init_bytes(objspace) / heap->slot_size &&
3189 heap->sweeping_page == NULL) {
3190 heap_page_allocate_and_initialize_force(objspace, heap);
3191 GC_ASSERT(heap->free_pages != NULL);
3192 return;
3193 }
3194
3195 /* Continue incremental marking or lazy sweeping, if in any of those steps. */
3196 gc_continue(objspace, heap);
3197
3198 if (heap->free_pages == NULL) {
3199 heap_page_allocate_and_initialize(objspace, heap);
3200 }
3201
3202 /* If we still don't have a free page and not allowed to create a new page,
3203 * we should start a new GC cycle. */
3204 if (heap->free_pages == NULL) {
3205 GC_ASSERT(objspace->empty_pages_count == 0);
3206 GC_ASSERT(objspace->heap_pages.allocatable_bytes == 0);
3207
3208 if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
3209 rb_memerror();
3210 }
3211 else {
3212 if (objspace->heap_pages.allocatable_bytes == 0 && !gc_config_full_mark_val) {
3213 heap_allocatable_bytes_expand(objspace, heap,
3214 heap->freed_slots + heap->empty_slots,
3215 heap->total_slots, heap->slot_size);
3216 GC_ASSERT(objspace->heap_pages.allocatable_bytes > 0);
3217 }
3218 /* Do steps of incremental marking or lazy sweeping if the GC run permits. */
3219 gc_continue(objspace, heap);
3220
3221 /* If we're not incremental marking (e.g. a minor GC) or finished
3222 * sweeping and still don't have a free page, then
3223 * gc_sweep_finish_heap should allow us to create a new page. */
3224 if (heap->free_pages == NULL && !heap_page_allocate_and_initialize(objspace, heap)) {
3225 if (gc_needs_major_flags == GPR_FLAG_NONE) {
3226 rb_bug("cannot create a new page after GC");
3227 }
3228 else { // Major GC is required, which will allow us to create new page
3229 if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
3230 rb_memerror();
3231 }
3232 else {
3233 /* Do steps of incremental marking or lazy sweeping. */
3234 gc_continue(objspace, heap);
3235
3236 if (heap->free_pages == NULL &&
3237 !heap_page_allocate_and_initialize(objspace, heap)) {
3238 rb_bug("cannot create a new page after major GC");
3239 }
3240 }
3241 }
3242 }
3243 }
3244 }
3245
3246 GC_ASSERT(heap->free_pages != NULL);
3247}
3248
3249#if GC_DEBUG
3250static inline const char*
3251rb_gc_impl_source_location_cstr(int *ptr)
3252{
3253 /* We could directly refer `rb_source_location_cstr()` before, but not any
3254 * longer. We have to heavy lift using our debugging API. */
3255 if (! ptr) {
3256 return NULL;
3257 }
3258 else if (! (*ptr = rb_sourceline())) {
3259 return NULL;
3260 }
3261 else {
3262 return rb_sourcefile();
3263 }
3264}
3265#endif
3266
3267static inline VALUE
3268newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, VALUE obj)
3269{
3270 GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
3271 GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
3272 RBASIC(obj)->flags = flags;
3273 *((VALUE *)&RBASIC(obj)->klass) = klass;
3274#if RBASIC_SHAPE_ID_FIELD
3275 RBASIC(obj)->shape_id = 0;
3276#endif
3277
3278 if (RB_UNLIKELY(flags & RUBY_FL_SHAREABLE)) {
3279 /* A born-shareable object must be WB protected: the shref and remembered-set
3280 * rules for shareable objects assume the write barrier. A local GC roots
3281 * shareable objects from this bit (pinned_roots_mark). */
3282 GC_ASSERT(wb_protected);
3283 gc_page_add_shareable(GET_HEAP_PAGE(obj), obj);
3284 }
3285
3286#if RGENGC_CHECK_MODE
3287 int lev = RB_GC_VM_LOCK_NO_BARRIER();
3288 {
3289 check_rvalue_consistency(objspace, obj);
3290
3291 GC_ASSERT(RVALUE_MARKED(objspace, obj) == FALSE);
3292 GC_ASSERT(RVALUE_MARKING(objspace, obj) == FALSE);
3293 GC_ASSERT(RVALUE_OLD_P(objspace, obj) == FALSE);
3294 GC_ASSERT(RVALUE_WB_UNPROTECTED(objspace, obj) == FALSE);
3295
3296 if (RVALUE_REMEMBERED(objspace, obj)) rb_bug("newobj: %s is remembered.", rb_obj_info(obj));
3297 }
3298 RB_GC_VM_UNLOCK_NO_BARRIER(lev);
3299#endif
3300
3301 if (RB_UNLIKELY(wb_protected == FALSE)) {
3302 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
3303 }
3304
3305#if RGENGC_PROFILE
3306 if (wb_protected) {
3307 objspace->profile.total_generated_normal_object_count++;
3308#if RGENGC_PROFILE >= 2
3309 objspace->profile.generated_normal_object_count_types[BUILTIN_TYPE(obj)]++;
3310#endif
3311 }
3312 else {
3313 objspace->profile.total_generated_shady_object_count++;
3314#if RGENGC_PROFILE >= 2
3315 objspace->profile.generated_shady_object_count_types[BUILTIN_TYPE(obj)]++;
3316#endif
3317 }
3318#endif
3319
3320#if GC_DEBUG
3321 GET_RVALUE_OVERHEAD(obj)->file = rb_gc_impl_source_location_cstr(&GET_RVALUE_OVERHEAD(obj)->line);
3322 GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */
3323#endif
3324
3325 gc_report(5, objspace, "newobj: %s\n", rb_obj_info(obj));
3326
3327 // RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, rb_obj_info(obj));
3328 return obj;
3329}
3330
3331size_t
3332rb_gc_impl_obj_slot_size(VALUE obj)
3333{
3334 return GET_HEAP_PAGE(obj)->slot_size - RVALUE_OVERHEAD;
3335}
3336
3337bool
3338rb_gc_impl_pinned_p(void *objspace_ptr, VALUE obj)
3339{
3340 return RVALUE_PINNED((rb_objspace_t *)objspace_ptr, obj);
3341}
3342
3343static inline size_t
3344heap_slot_size(unsigned char pool_id)
3345{
3346 GC_ASSERT(pool_id < HEAP_COUNT);
3347
3348 return pool_slot_sizes[pool_id] - RVALUE_OVERHEAD;
3349}
3350
3351size_t
3352rb_gc_impl_max_allocation_size(void)
3353{
3354 return heap_slot_size(HEAP_COUNT - 1);
3355}
3356
3357bool
3358rb_gc_impl_size_allocatable_p(size_t size)
3359{
3360 return size <= rb_gc_impl_max_allocation_size();
3361}
3362
3363static inline bool
3364heap_advance_region(rb_heap_t *heap)
3365{
3366 struct free_region *region = heap->newobj.alloc_next_region;
3367 if (region == NULL) {
3368 return false;
3369 }
3370
3371 rb_asan_unpoison_object((VALUE)region, false);
3372 GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
3373 heap->newobj.alloc_cursor = (uintptr_t)region;
3374 heap->newobj.alloc_cursor_end = region->end;
3375 heap->newobj.alloc_next_region = region->next;
3376 rb_asan_poison_object((VALUE)region);
3377
3378 return true;
3379}
3380
3381/* The whole region is ours until the next refill, so charge it to the step now. */
3382static inline void
3383heap_charge_region(rb_objspace_t *objspace, const rb_heap_t *heap, size_t heap_idx)
3384{
3385 objspace->incremental_mark_step_allocated_slots +=
3386 (heap->newobj.alloc_cursor_end - heap->newobj.alloc_cursor) / pool_slot_sizes[heap_idx];
3387}
3388
3389static inline VALUE
3390heap_alloc_slot(rb_objspace_t *objspace, size_t heap_idx)
3391{
3392 rb_heap_t *heap = &heaps[heap_idx];
3393
3394 uintptr_t cursor = heap->newobj.alloc_cursor;
3395 if (RB_UNLIKELY(cursor >= heap->newobj.alloc_cursor_end)) {
3396 /* Marking owes us a step before the next region, and newobj_refill runs it. */
3397 if (RB_UNLIKELY(is_incremental_marking(objspace)) ||
3398 heap_advance_region(heap) == false) {
3399 return Qfalse;
3400 }
3401 cursor = heap->newobj.alloc_cursor;
3402 }
3403
3404 VALUE obj = (VALUE)cursor;
3405 rb_asan_unpoison_object(obj, true);
3406 heap->newobj.alloc_cursor = cursor + pool_slot_sizes[heap_idx];
3407
3408 /* Single writer (the owning Ractor under the GVL), so a plain increment is enough. */
3409 heap->total_allocated_objects++;
3410
3411#if RGENGC_CHECK_MODE
3412 GC_ASSERT(rb_gc_impl_obj_slot_size(obj) == heap_slot_size(heap_idx));
3413 // zero clear
3414 MEMZERO((char *)obj, char, heap_slot_size(heap_idx));
3415#endif
3416 return obj;
3417}
3418
3419static struct heap_page *
3420heap_next_free_page(rb_objspace_t *objspace, rb_heap_t *heap)
3421{
3422 struct heap_page *page;
3423
3424 if (heap->free_pages == NULL) {
3425 heap_prepare(objspace, heap);
3426 }
3427
3428 page = heap->free_pages;
3429 heap->free_pages = page->free_next;
3430
3431 GC_ASSERT(page->free_slots != 0);
3432
3433 asan_unlock_freelist(page);
3434
3435 return page;
3436}
3437
3438static inline void
3439heap_set_alloc_page(rb_objspace_t *objspace, size_t heap_idx, struct heap_page *page)
3440{
3441 gc_report(3, objspace, "heap_set_alloc_page: Using page %p\n", (void *)page->body);
3442
3443 rb_heap_t *heap = &heaps[heap_idx];
3444
3445 GC_ASSERT(heap->newobj.alloc_cursor >= heap->newobj.alloc_cursor_end);
3446 GC_ASSERT(heap->newobj.alloc_next_region == NULL);
3447 GC_ASSERT(page->free_slots != 0);
3448 GC_ASSERT(page->free_region != NULL);
3449
3450 heap->newobj.alloc_using_page = page;
3451
3452 struct free_region *region = page->free_region;
3453 rb_asan_unpoison_object((VALUE)region, false);
3454 GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
3455 heap->newobj.alloc_cursor = (uintptr_t)region;
3456 heap->newobj.alloc_cursor_end = region->end;
3457 heap->newobj.alloc_next_region = region->next;
3458 rb_asan_poison_object((VALUE)region);
3459
3460 page->free_slots = 0;
3461 page->free_region = NULL;
3462}
3463
3464static void
3465init_size_to_heap_idx(void)
3466{
3467 /* Process-wide and immutable, so build it once at boot. A rebuild in a later
3468 * objspace_init would write the same values but race other threads' lock-free
3469 * allocation-fastpath reads. */
3470 static bool initialized = false;
3471 if (initialized) return;
3472 initialized = true;
3473
3474 for (size_t i = 0; i < sizeof(size_to_heap_idx); i++) {
3475 size_t effective = i * 8 + RVALUE_OVERHEAD;
3476 uint8_t idx;
3477 for (idx = 0; idx < HEAP_COUNT; idx++) {
3478 if (effective <= pool_slot_sizes[idx]) break;
3479 }
3480 size_to_heap_idx[i] = idx;
3481 }
3482}
3483
3484static inline size_t
3485heap_idx_for_size(size_t size)
3486{
3487 size_t compressed = (size + 7) >> 3;
3488 if (compressed < sizeof(size_to_heap_idx)) {
3489 size_t heap_idx = size_to_heap_idx[compressed];
3490 if (RB_LIKELY(heap_idx < HEAP_COUNT)) return heap_idx;
3491 }
3492
3493 rb_bug("heap_idx_for_size: allocation size too large "
3494 "(size=%"PRIuSIZE")", size);
3495}
3496
3497size_t
3498rb_gc_impl_size_slot_size(void *objspace_ptr, size_t size)
3499{
3500 return heap_slot_size((unsigned char)heap_idx_for_size(size));
3501}
3502
3503bool
3504rb_gc_impl_zjit_new_obj_fastpath(void *objspace_ptr, size_t alloc_size, VALUE flags, VALUE klass,
3505 struct rb_gc_zjit_fastpath *fastpath)
3506{
3507#if USE_ZJIT
3508 size_t heap_idx = 0;
3509 size_t slot_size = 0;
3510 for (; heap_idx < HEAP_COUNT; heap_idx++) {
3511 if (alloc_size + RVALUE_OVERHEAD <= pool_slot_sizes[heap_idx]) {
3512 slot_size = pool_slot_sizes[heap_idx];
3513 break;
3514 }
3515 }
3516 if (slot_size == 0) return false;
3517
3518#undef heaps
3519 size_t base = offsetof(rb_objspace_t, heaps)
3520 + heap_idx * sizeof(rb_heap_t)
3521 + offsetof(rb_heap_t, newobj);
3522#define heaps objspace->heaps
3523
3524 struct rb_gc_zjit_default_new_obj_fastpath default_fastpath = {
3525 base + offsetof(rb_heap_newobj_t, alloc_cursor),
3526 base + offsetof(rb_heap_newobj_t, alloc_cursor_end),
3527 slot_size,
3528 base - offsetof(rb_heap_t, newobj) + offsetof(rb_heap_t, total_allocated_objects),
3529 flags,
3530 klass
3531 };
3532
3533 memset(fastpath, 0, sizeof(*fastpath));
3534 fastpath->kind = RB_GC_ZJIT_FASTPATH_DEFAULT;
3535 memcpy(fastpath->data.words, &default_fastpath, sizeof(default_fastpath));
3536
3537 return true;
3538#else
3539 return false;
3540#endif
3541}
3542
3543int32_t
3544rb_gc_impl_zjit_writebarrier_required_offset(void)
3545{
3546 return (int32_t)offsetof(rb_objspace_t, zjit_during_incremental_marking);
3547}
3548
3549NOINLINE(static VALUE newobj_refill(rb_objspace_t *objspace, size_t heap_idx));
3550
3551static VALUE
3552newobj_refill(rb_objspace_t *objspace, size_t heap_idx)
3553{
3554 rb_heap_t *heap = &heaps[heap_idx];
3555 VALUE obj = Qfalse;
3556
3557 /* No lock: a heap is single-writer (its owner thread, serialized by the GVL inside
3558 * the Ractor), the page pool has its own mutex, and a GC started from here takes
3559 * whatever gc_enter needs. */
3560 if (is_incremental_marking(objspace)) {
3561 /* The fast path sends us here at every region, which is far more often than the
3562 * step size, so step only once the regions add up to it. */
3563 if (objspace->incremental_mark_step_allocated_slots >= INCREMENTAL_MARK_STEP_ALLOCATIONS) {
3564 gc_continue(objspace, heap);
3565 objspace->incremental_mark_step_allocated_slots = 0;
3566 }
3567
3568 // Move on to the region the fast path refused to take
3569 if (heap_advance_region(heap)) {
3570 heap_charge_region(objspace, heap, heap_idx);
3571 obj = heap_alloc_slot(objspace, heap_idx);
3572 }
3573 }
3574
3575 if (obj == Qfalse) {
3576 // Get next free page (possibly running GC)
3577 struct heap_page *page = heap_next_free_page(objspace, heap);
3578 heap_set_alloc_page(objspace, heap_idx, page);
3579 heap_charge_region(objspace, heap, heap_idx);
3580
3581 // Retry allocation after moving to new page
3582 obj = heap_alloc_slot(objspace, heap_idx);
3583 }
3584
3585 if (RB_UNLIKELY(obj == Qfalse)) {
3586 rb_memerror();
3587 }
3588 return obj;
3589}
3590
3591static VALUE
3592newobj_alloc(rb_objspace_t *objspace, size_t heap_idx)
3593{
3594 /* The objspace belongs to the current Ractor and is single-writer, so the fast path
3595 * needs no lock. Stress GC runs in the caller's slow path, before newobj_alloc. */
3596 VALUE obj = heap_alloc_slot(objspace, heap_idx);
3597
3598 if (RB_UNLIKELY(obj == Qfalse)) {
3599 obj = newobj_refill(objspace, heap_idx);
3600 }
3601
3602 return obj;
3603}
3604
3605ALWAYS_INLINE(static VALUE newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, int wb_protected, size_t heap_idx));
3606
3607static inline VALUE
3608newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, int wb_protected, size_t heap_idx)
3609{
3610 VALUE obj;
3611
3612 /* No lock (see newobj_refill); during_gc and the stress flag are this objspace's own state. */
3613 if (RB_UNLIKELY(during_gc || ruby_gc_stressful)) {
3614 if (during_gc) {
3615 dont_gc_on();
3616 during_gc = 0;
3617 if (rb_memerror_reentered()) {
3618 rb_memerror();
3619 }
3620 rb_bug("object allocation during garbage collection phase");
3621 }
3622
3623 if (ruby_gc_stressful) {
3624 if (!garbage_collect(objspace, GPR_FLAG_NEWOBJ)) {
3625 rb_memerror();
3626 }
3627 }
3628 }
3629
3630 obj = newobj_alloc(objspace, heap_idx);
3631 newobj_init(klass, flags, wb_protected, objspace, obj);
3632
3633 if (RB_UNLIKELY(ruby_gc_stressful)) {
3634 rb_heap_t *heap = &heaps[heap_idx];
3635 heap->newobj.alloc_cursor_end = heap->newobj.alloc_cursor;
3636 }
3637
3638 return obj;
3639}
3640
3641NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags,
3642 rb_objspace_t *objspace, size_t heap_idx));
3643NOINLINE(static VALUE newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags,
3644 rb_objspace_t *objspace, size_t heap_idx));
3645
3646static VALUE
3647newobj_slowpath_wb_protected(VALUE klass, VALUE flags, rb_objspace_t *objspace, size_t heap_idx)
3648{
3649 return newobj_slowpath(klass, flags, objspace, TRUE, heap_idx);
3650}
3651
3652static VALUE
3653newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace, size_t heap_idx)
3654{
3655 return newobj_slowpath(klass, flags, objspace, FALSE, heap_idx);
3656}
3657
3658VALUE
3659rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size, size_t *actual_alloc_size)
3660{
3661 VALUE obj;
3662 rb_objspace_t *objspace = objspace_ptr;
3663
3664 /* There is no per-Ractor cache; the argument stays for ABI compatibility with other
3665 * GC implementations such as MMTk. */
3666 (void)cache_ptr;
3667
3668 RB_DEBUG_COUNTER_INC(obj_newobj);
3669 (void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected);
3670
3671 if (RB_UNLIKELY(stress_to_class)) {
3672 if (rb_hash_lookup2(stress_to_class, klass, Qundef) != Qundef) {
3673 rb_memerror();
3674 }
3675 }
3676
3677 size_t heap_idx = heap_idx_for_size(alloc_size);
3678 *actual_alloc_size = heap_slot_size((unsigned char)heap_idx);
3679
3680 if (!RB_UNLIKELY(during_gc || ruby_gc_stressful) &&
3681 wb_protected) {
3682 obj = newobj_alloc(objspace, heap_idx);
3683 newobj_init(klass, flags, wb_protected, objspace, obj);
3684 }
3685 else {
3686 RB_DEBUG_COUNTER_INC(obj_newobj_slowpath);
3687
3688 obj = wb_protected ?
3689 newobj_slowpath_wb_protected(klass, flags, objspace, heap_idx) :
3690 newobj_slowpath_wb_unprotected(klass, flags, objspace, heap_idx);
3691 }
3692
3693 return obj;
3694}
3695
3696static int
3697ptr_in_page_body_p(const void *ptr, const void *memb)
3698{
3699 struct heap_page *page = *(struct heap_page **)memb;
3700 uintptr_t p_body = (uintptr_t)page->body;
3701
3702 if ((uintptr_t)ptr >= p_body) {
3703 return (uintptr_t)ptr < (p_body + HEAP_PAGE_SIZE) ? 0 : 1;
3704 }
3705 else {
3706 return -1;
3707 }
3708}
3709
3710PUREFUNC(static inline struct heap_page *heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr);)
3711static inline struct heap_page *
3712heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr)
3713{
3714 struct heap_page **res;
3715
3716 if (ptr < (uintptr_t)heap_pages_lomem ||
3717 ptr > (uintptr_t)heap_pages_himem) {
3718 return NULL;
3719 }
3720
3721 res = bsearch((void *)ptr, rb_darray_ref(objspace->heap_pages.sorted, 0),
3722 rb_darray_size(objspace->heap_pages.sorted), sizeof(struct heap_page *),
3723 ptr_in_page_body_p);
3724
3725 if (res) {
3726 return *res;
3727 }
3728 else {
3729 return NULL;
3730 }
3731}
3732
3733PUREFUNC(static inline bool is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr);)
3734static inline bool
3735is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr)
3736{
3737 register uintptr_t p = (uintptr_t)ptr;
3738 register struct heap_page *page;
3739
3740 RB_DEBUG_COUNTER_INC(gc_isptr_trial);
3741
3742 if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE;
3743 RB_DEBUG_COUNTER_INC(gc_isptr_range);
3744
3745 if (p % sizeof(VALUE) != 0) return FALSE;
3746 RB_DEBUG_COUNTER_INC(gc_isptr_align);
3747
3748 page = heap_page_for_ptr(objspace, (uintptr_t)ptr);
3749 if (page) {
3750 RB_DEBUG_COUNTER_INC(gc_isptr_maybe);
3751 if (heap_page_in_global_empty_pages_pool(objspace, page)) {
3752 return FALSE;
3753 }
3754 else {
3755 if (p < page->start) return FALSE;
3756 if (p >= page->start + (page->total_slots * page->slot_size)) return FALSE;
3757 if (!slot_offset_aligned_p(page, p)) return FALSE;
3758
3759 return TRUE;
3760 }
3761 }
3762 return FALSE;
3763}
3764
3765/*
3766 * Is this object live in the given objspace. For foreign objects in other object spaces,
3767 * it returns `false`. NOTE: this does NOT check whether or not it's unmarked during the process
3768 * of lazy sweeping.
3769 */
3770bool
3771rb_gc_impl_live_object_p(void *objspace_ptr, const void *ptr)
3772{
3773 rb_objspace_t *objspace = objspace_ptr;
3774
3775 /* Whether ptr refers to a live object. is_pointer_to_heap is the
3776 * address-only check; T_NONE, T_MOVED, and T_ZOMBIE slots are valid heap
3777 * addresses but not live objects. */
3778 if (!is_pointer_to_heap(objspace, ptr)) return false;
3779
3780 VALUE obj = (VALUE)ptr;
3781 bool live = false;
3782 asan_unpoisoning_object(obj) {
3783 switch (BUILTIN_TYPE(obj)) {
3784 case T_NONE:
3785 case T_MOVED:
3786 case T_ZOMBIE:
3787 break;
3788 default:
3789 live = true;
3790 break;
3791 }
3792 }
3793 return live;
3794}
3795
3796/* Flags preserved from the original object when it becomes a zombie, and so also the
3797 * only ones that may legitimately be set on one. */
3798#define ZOMBIE_OBJ_KEPT_FLAGS (FL_FINALIZE)
3799
3800void
3801rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data)
3802{
3803 rb_objspace_t *objspace = objspace_ptr;
3804
3805 struct RZombie *zombie = RZOMBIE(obj);
3806 zombie->flags = T_ZOMBIE | (zombie->flags & ZOMBIE_OBJ_KEPT_FLAGS);
3807 zombie->dfree = dfree;
3808 zombie->data = data;
3809 VALUE prev, next = heap_pages_deferred_final;
3810 do {
3811 zombie->next = prev = next;
3812 next = RUBY_ATOMIC_VALUE_CAS(heap_pages_deferred_final, prev, obj);
3813 } while (next != prev);
3814
3815 struct heap_page *page = GET_HEAP_PAGE(obj);
3816 page->final_slots++;
3817 page->heap->final_slots_count++;
3818}
3819
3820static void
3821tdata_unsafe_free_chunk_reset(struct tdata_unsafe_free_chunk *chunk)
3822{
3823 chunk->next = NULL;
3824 chunk->count = 0;
3825 chunk->embed_xfree_bits = 0;
3826}
3827
3828static struct tdata_unsafe_free_chunk *
3829tdata_unsafe_free_chunk_alloc(void)
3830{
3831 /* Pops race each other (several Ractors can be sweeping), but pushes happen only
3832 * inside the drain, which holds a VM barrier -- and a barrier cannot complete while
3833 * a Ractor is inside gc_sweep_page. No push ever overlaps a pop, so the head only
3834 * moves forward and this CAS pop needs no ABA tagging. A sweep performed by a
3835 * thread other than the objspace's owner would break that. */
3836 struct tdata_unsafe_free_chunk *head =
3837 rbimpl_atomic_ptr_load((void **)&global_objspace->tdata_unsafe_free_cache,
3838 RBIMPL_ATOMIC_ACQUIRE);
3839 while (head) {
3840 struct tdata_unsafe_free_chunk *prev =
3841 rbimpl_atomic_ptr_cas((void **)&global_objspace->tdata_unsafe_free_cache,
3842 head, head->next,
3843 RBIMPL_ATOMIC_ACQ_REL, RBIMPL_ATOMIC_ACQUIRE);
3844 if (prev == head) {
3845 rbimpl_atomic_size_dec(&global_objspace->tdata_unsafe_free_cache_len,
3846 RBIMPL_ATOMIC_RELAXED);
3847 tdata_unsafe_free_chunk_reset(head);
3848 return head;
3849 }
3850 head = prev;
3851 }
3852
3853 /* Not xmalloc: this runs mid-sweep, and the chunks are GC bookkeeping that should not
3854 * feed back into malloc_increase (mark stack chunks do the same). */
3855 struct tdata_unsafe_free_chunk *chunk = malloc(sizeof(struct tdata_unsafe_free_chunk));
3856 if (!chunk) rb_memerror();
3857 tdata_unsafe_free_chunk_reset(chunk);
3858 return chunk;
3859}
3860
3861/* Hand this objspace's partial chunk to the global stack. The entries were counted as
3862 * they were appended, so the pending count does not change here. */
3863static void
3864gc_tdata_unsafe_free_publish(rb_objspace_t *objspace)
3865{
3866 struct tdata_unsafe_free_chunk *chunk = objspace->tdata_unsafe_free_chunk;
3867 if (chunk == NULL) return;
3868 GC_ASSERT(chunk->count > 0);
3869 objspace->tdata_unsafe_free_chunk = NULL;
3870
3871 struct tdata_unsafe_free_chunk *prev, *head =
3872 rbimpl_atomic_ptr_load((void **)&global_objspace->tdata_unsafe_free_published,
3873 RBIMPL_ATOMIC_RELAXED);
3874 do {
3875 chunk->next = prev = head;
3876 head = rbimpl_atomic_ptr_cas((void **)&global_objspace->tdata_unsafe_free_published,
3877 prev, chunk,
3878 RBIMPL_ATOMIC_ACQ_REL, RBIMPL_ATOMIC_ACQUIRE);
3879 } while (head != prev);
3880}
3881
3882/* Copy out what obj's deferred free needs, running no dfree. Returns true when the
3883 * caller may reclaim the slot and false when obj became a zombie, matching rb_gc_obj_free. */
3884static bool
3885gc_defer_thread_unsafe_free(rb_objspace_t *objspace, VALUE obj, bool *trigger)
3886{
3887 GC_ASSERT(!((uintptr_t)RTYPEDDATA(obj)->type & TYPED_DATA_EMBEDDED));
3888 const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
3889 void *data = RTYPEDDATA(obj)->data;
3890 GC_ASSERT(data != NULL);
3891
3892 rb_gc_obj_free_vm_weak_references(obj);
3893
3894 size_t count = rbimpl_atomic_size_fetch_add(&global_objspace->tdata_deferred_free_count, 1,
3895 RBIMPL_ATOMIC_RELAXED) + 1;
3896 if (!*trigger && count >= TDATA_DEFERRED_FREE_THRESHOLD) {
3897 *trigger = true;
3898 }
3899
3900 struct tdata_unsafe_free_chunk *chunk = objspace->tdata_unsafe_free_chunk;
3901 if (chunk == NULL) {
3902 chunk = objspace->tdata_unsafe_free_chunk = tdata_unsafe_free_chunk_alloc();
3903 }
3904 if (type->flags & RUBY_TYPED_EMBEDDABLE) {
3905 chunk->embed_xfree_bits |= (uint32_t)1 << chunk->count;
3906 }
3907 struct tdata_unsafe_free_entry *entry = &chunk->entries[chunk->count++];
3908 entry->dfree = type->function.dfree;
3909 entry->data = data;
3910 if (chunk->count == TDATA_UNSAFE_FREE_CHUNK_CAPA) {
3911 gc_tdata_unsafe_free_publish(objspace);
3912 }
3913
3914 if (FL_TEST_RAW(obj, FL_FINALIZE)) {
3915 /* The dfree is on the side list now, so this zombie carries none: it goes on the
3916 * regular deferred list, where the owner runs its Ruby finalizer promptly and
3917 * reclaims the slot, instead of waiting for the barrier. */
3918 rb_gc_impl_make_zombie(objspace, obj, 0, 0);
3919 return false;
3920 }
3921 return true;
3922}
3923
3924typedef int each_obj_callback(void *, void *, size_t, void *);
3925typedef int each_page_callback(struct heap_page *, void *);
3926
3928 rb_objspace_t *objspace;
3929 bool reenable_incremental;
3930
3931 /* Visit only the pages that hold shareable objects, so a foreign Ractor's objspace
3932 * can be walked for its shareable objects alone, without touching the rest of its
3933 * isolated heap. */
3934 bool shareable_only;
3935
3936 /* Set when walking a foreign objspace without settling its stopped lazy sweep
3937 * (settling would run the owner's obj_free and dfree on this thread). Objects the
3938 * sweep is about to free are skipped: on an unswept page, unmarked means dead. */
3939 bool skip_unswept_dead;
3940
3941 each_obj_callback *each_obj_callback;
3942 each_page_callback *each_page_callback;
3943 void *data;
3944
3945 struct heap_page **pages[HEAP_COUNT];
3946 size_t pages_counts[HEAP_COUNT];
3947};
3948
3949static VALUE
3950objspace_each_objects_ensure(VALUE arg)
3951{
3952 struct each_obj_data *data = (struct each_obj_data *)arg;
3953 rb_objspace_t *objspace = data->objspace;
3954
3955 /* Reenable incremental GC */
3956 if (data->reenable_incremental) {
3957 objspace->flags.dont_incremental = FALSE;
3958 }
3959
3960 for (int i = 0; i < HEAP_COUNT; i++) {
3961 struct heap_page **pages = data->pages[i];
3962 free(pages);
3963 }
3964
3965 return Qnil;
3966}
3967
3968static VALUE
3969objspace_each_objects_try(VALUE arg)
3970{
3971 struct each_obj_data *data = (struct each_obj_data *)arg;
3972 rb_objspace_t *objspace = data->objspace;
3973
3974 /* Copy pages from all heaps to their respective buffers. */
3975 for (int i = 0; i < HEAP_COUNT; i++) {
3976 rb_heap_t *heap = &heaps[i];
3977 size_t size = heap->total_pages * sizeof(struct heap_page *);
3978
3979 struct heap_page **pages = malloc(size);
3980 if (!pages) rb_memerror();
3981
3982 /* Set up pages buffer by iterating over all pages in the current eden
3983 * heap. This will be a snapshot of the state of the heap before we
3984 * call the callback over each page that exists in this buffer. Thus it
3985 * is safe for the callback to allocate objects without possibly entering
3986 * an infinite loop. */
3987 struct heap_page *page = 0;
3988 size_t pages_count = 0;
3989 ccan_list_for_each(&heap->pages, page, page_node) {
3990 pages[pages_count] = page;
3991 pages_count++;
3992 }
3993 data->pages[i] = pages;
3994 data->pages_counts[i] = pages_count;
3995 GC_ASSERT(pages_count == heap->total_pages);
3996 }
3997
3998 for (int i = 0; i < HEAP_COUNT; i++) {
3999 rb_heap_t *heap = &heaps[i];
4000 size_t pages_count = data->pages_counts[i];
4001 struct heap_page **pages = data->pages[i];
4002
4003 struct heap_page *page = ccan_list_top(&heap->pages, struct heap_page, page_node);
4004 for (size_t i = 0; i < pages_count; i++) {
4005 /* If we have reached the end of the linked list then there are no
4006 * more pages, so break. */
4007 if (page == NULL) break;
4008
4009 /* If this page does not match the one in the buffer, then move to
4010 * the next page in the buffer. */
4011 if (pages[i] != page) continue;
4012
4013 uintptr_t pstart = (uintptr_t)page->start;
4014 uintptr_t pend = pstart + (page->total_slots * heap->slot_size);
4015
4016 if (data->shareable_only) {
4017 /* Hand shareable objects to the callback one slot at a time, not the
4018 * whole page: walking a foreign Ractor's objspace must never expose its
4019 * unshareable objects, which the caller cannot inspect safely. */
4020 if (page->flags.has_shareable_objects) {
4021 /* This walk runs over a foreign objspace under the barrier and
4022 * must not settle the owner's stopped lazy sweep: settling would run
4023 * the owner's obj_free and dfree on this thread with this Ractor's
4024 * identity (wrong per-Ractor tables, a foreign T_DATA dfree). So no
4025 * gc_rest, and objects the sweep is about to free are skipped: on an
4026 * unswept page unmarked means dead and its shareable bit merely has
4027 * not been bulk-cleared yet. Passing one to the callback would
4028 * resurrect it, handing out a reference the owner's sweep frees as
4029 * soon as the barrier lifts. */
4030 const bool page_unswept = is_lazy_sweeping(objspace) && page->flags.before_sweep;
4031 int planes = CEILDIV(page->total_slots, BITS_BITLENGTH);
4032 uintptr_t base = pstart;
4033 bool stop = false;
4034 for (int j = 0; j < planes && !stop; j++) {
4035 bits_t bits = page->shareable_bits[j];
4036 uintptr_t slot = base;
4037 while (bits) {
4038 if ((bits & 1) && data->each_obj_callback &&
4039 !(page_unswept && !RVALUE_MARKED(objspace, (VALUE)slot)) &&
4040 (*data->each_obj_callback)((void *)slot, (void *)(slot + heap->slot_size),
4041 heap->slot_size, data->data)) {
4042 stop = true;
4043 break;
4044 }
4045 slot += heap->slot_size;
4046 bits >>= 1;
4047 }
4048 base += BITS_BITLENGTH * heap->slot_size;
4049 }
4050 if (stop) break;
4051 }
4052 }
4053 else if (data->skip_unswept_dead &&
4054 is_lazy_sweeping(objspace) && page->flags.before_sweep) {
4055 /* A foreign page pending sweep: hand out the live objects one slot at a
4056 * time and skip the unmarked (dead) ones the owner's sweep frees as soon
4057 * as the barrier lifts. */
4058 bool stop = false;
4059 for (uintptr_t slot = pstart; slot < pend; slot += heap->slot_size) {
4060 if (!RVALUE_MARKED(objspace, (VALUE)slot)) continue;
4061 if (data->each_obj_callback &&
4062 (*data->each_obj_callback)((void *)slot, (void *)(slot + heap->slot_size),
4063 heap->slot_size, data->data)) {
4064 stop = true;
4065 break;
4066 }
4067 }
4068 if (stop) break;
4069 }
4070 else {
4071 if (data->each_obj_callback &&
4072 (*data->each_obj_callback)((void *)pstart, (void *)pend, heap->slot_size, data->data)) {
4073 break;
4074 }
4075 if (data->each_page_callback &&
4076 (*data->each_page_callback)(page, data->data)) {
4077 break;
4078 }
4079 }
4080
4081 page = ccan_list_next(&heap->pages, page, page_node);
4082 }
4083 }
4084
4085 return Qnil;
4086}
4087
4088static void
4089objspace_each_exec(bool protected, struct each_obj_data *each_obj_data)
4090{
4091 /* Disable incremental GC */
4092 rb_objspace_t *objspace = each_obj_data->objspace;
4093 bool reenable_incremental = FALSE;
4094 if (protected) {
4095 reenable_incremental = !objspace->flags.dont_incremental;
4096
4097 gc_rest(objspace);
4098 objspace->flags.dont_incremental = TRUE;
4099 }
4100
4101 each_obj_data->reenable_incremental = reenable_incremental;
4102 memset(&each_obj_data->pages, 0, sizeof(each_obj_data->pages));
4103 memset(&each_obj_data->pages_counts, 0, sizeof(each_obj_data->pages_counts));
4104 rb_ensure(objspace_each_objects_try, (VALUE)each_obj_data,
4105 objspace_each_objects_ensure, (VALUE)each_obj_data);
4106}
4107
4108static void
4109objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data, bool protected)
4110{
4111 struct each_obj_data each_obj_data = {
4112 .objspace = objspace,
4113 .each_obj_callback = callback,
4114 .each_page_callback = NULL,
4115 .data = data,
4116 };
4117 objspace_each_exec(protected, &each_obj_data);
4118}
4119
4120void
4121rb_gc_impl_each_objects(void *objspace_ptr, each_obj_callback *callback, void *data)
4122{
4123 objspace_each_objects(objspace_ptr, callback, data, TRUE);
4124}
4125
4126/* Like rb_gc_impl_each_objects but visiting only pages that hold shareable objects, to
4127 * reach a foreign Ractor's shareable objects without walking the rest of its heap. */
4128void
4129rb_gc_impl_each_objects_shareable(void *objspace_ptr, each_obj_callback *callback, void *data)
4130{
4131 struct each_obj_data each_obj_data = {
4132 .objspace = objspace_ptr,
4133 .shareable_only = true,
4134 .each_obj_callback = callback,
4135 .each_page_callback = NULL,
4136 .data = data,
4137 };
4138 /* Not the protected variant: this objspace belongs to another Ractor (the caller
4139 * holds the barrier). The protected path calls gc_rest, which would run the owner's
4140 * stopped lazy sweep (its obj_free and dfree) on the walking thread with the
4141 * walker's Ractor identity (wrong per-Ractor tables, a foreign T_DATA dfree). The
4142 * owner is stopped and its page list is stable, and the walk itself skips dead,
4143 * unswept objects (the shareable_only branch of objspace_each_objects_try). The
4144 * walker's own incremental GC state is untouched, since this is not its objspace. */
4145 objspace_each_exec(FALSE, &each_obj_data);
4146}
4147
4148/* Walk every object of a foreign Ractor's objspace, unshareable ones included. Only for
4149 * callers that hold the barrier and whose callback is pure C (a heap dump, memory
4150 * accounting). As in the shareable walk above, the owner's stopped lazy sweep is not
4151 * settled and dead, unswept objects are skipped by the walk (skip_unswept_dead). */
4152void
4153rb_gc_impl_each_objects_foreign(void *objspace_ptr, each_obj_callback *callback, void *data)
4154{
4155 struct each_obj_data each_obj_data = {
4156 .objspace = objspace_ptr,
4157 .skip_unswept_dead = true,
4158 .each_obj_callback = callback,
4159 .each_page_callback = NULL,
4160 .data = data,
4161 };
4162 objspace_each_exec(FALSE, &each_obj_data);
4163}
4164
4165#if GC_CAN_COMPILE_COMPACTION
4166static void
4167objspace_each_pages(rb_objspace_t *objspace, each_page_callback *callback, void *data, bool protected)
4168{
4169 struct each_obj_data each_obj_data = {
4170 .objspace = objspace,
4171 .each_obj_callback = NULL,
4172 .each_page_callback = callback,
4173 .data = data,
4174 };
4175 objspace_each_exec(protected, &each_obj_data);
4176}
4177#endif
4178
4179VALUE
4180rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block)
4181{
4182 rb_objspace_t *objspace = objspace_ptr;
4183 VALUE table;
4184 st_data_t data;
4185
4186 GC_ASSERT(!OBJ_FROZEN(obj));
4187
4188 /* Registering, storing and running finalizers all belong to the object's own
4189 * objspace, so refuse to define one on another Ractor's object (even a shareable
4190 * one): it would land in a table the owner's sweep never consults. */
4191 if (GET_HEAP_OBJSPACE(obj) != objspace) {
4192 rb_raise(rb_eRactorIsolationError,
4193 "can not define a finalizer for an object of another Ractor");
4194 }
4195
4196 RBASIC(obj)->flags |= FL_FINALIZE;
4197
4198 unsigned int lev = RB_GC_VM_LOCK();
4199
4200 if (st_lookup(finalizer_table, obj, &data)) {
4201 table = (VALUE)data;
4202 VALUE dup_table = rb_ary_dup(table);
4203
4204 RB_GC_VM_UNLOCK(lev);
4205 /* avoid duplicate block, table is usually small */
4206 {
4207 rb_len_t len = RARRAY_LEN(table);
4208 rb_len_t i;
4209
4210 for (i = 0; i < len; i++) {
4211 VALUE recv = RARRAY_AREF(dup_table, i);
4212 if (rb_equal(recv, block)) { // can't be called with VM lock held
4213 return recv;
4214 }
4215 }
4216 }
4217 lev = RB_GC_VM_LOCK();
4218 RB_GC_GUARD(dup_table);
4219
4220 rb_ary_push(table, block);
4221 }
4222 else {
4223 table = rb_ary_new3(2, rb_obj_id(obj), block);
4224 rb_obj_hide(table);
4225 st_add_direct(finalizer_table, obj, table);
4226 }
4227
4228 RB_GC_VM_UNLOCK(lev);
4229
4230 return block;
4231}
4232
4233void
4234rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj)
4235{
4236 rb_objspace_t *objspace = objspace_ptr;
4237
4238 GC_ASSERT(!OBJ_FROZEN(obj));
4239
4240 /* Symmetric with define. */
4241 if (GET_HEAP_OBJSPACE(obj) != objspace) {
4242 rb_raise(rb_eRactorIsolationError,
4243 "can not undefine a finalizer of an object of another Ractor");
4244 }
4245
4246 st_data_t data = obj;
4247
4248 int lev = RB_GC_VM_LOCK();
4249 st_delete(finalizer_table, &data, 0);
4250 RB_GC_VM_UNLOCK(lev);
4251
4252 FL_UNSET(obj, FL_FINALIZE);
4253}
4254
4255void
4256rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj)
4257{
4258 /* Finalizers do not cross objspaces: a copy of another Ractor's object starts with
4259 * none (guards the public rb_gc_copy_finalizer C API; no in-tree caller crosses).
4260 * A same-objspace copy behaves as before. Table accessed under the VM lock. */
4261 rb_objspace_t *objspace = objspace_ptr;
4262 VALUE table;
4263 st_data_t data;
4264
4265 if (!FL_TEST(obj, FL_FINALIZE)) return;
4266 if (GET_HEAP_OBJSPACE(obj) != objspace) return;
4267
4268 int lev = RB_GC_VM_LOCK();
4269 if (RB_LIKELY(st_lookup(finalizer_table, obj, &data))) {
4270 table = rb_ary_dup((VALUE)data);
4271 RARRAY_ASET(table, 0, rb_obj_id(dest));
4272 st_insert(finalizer_table, dest, table);
4273 FL_SET(dest, FL_FINALIZE);
4274 }
4275 else {
4276 rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj));
4277 }
4278 RB_GC_VM_UNLOCK(lev);
4279}
4280
4281static VALUE
4282get_final(rb_len_t i, void *data)
4283{
4284 VALUE table = (VALUE)data;
4285
4286 return RARRAY_AREF(table, i + 1);
4287}
4288
4289static void
4290run_final(rb_objspace_t *objspace, VALUE zombie)
4291{
4292 if (RZOMBIE(zombie)->dfree) {
4293 RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data);
4294 }
4295
4296 st_data_t key = (st_data_t)zombie;
4297 if (FL_TEST_RAW(zombie, FL_FINALIZE)) {
4298 FL_UNSET(zombie, FL_FINALIZE);
4299 st_data_t table;
4300 if (st_delete(finalizer_table, &key, &table)) {
4301 rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), RARRAY_LEN(table) - 1, get_final, (void *)table);
4302 }
4303 else {
4304 rb_bug("FL_FINALIZE flag is set, but finalizers are not found");
4305 }
4306 }
4307 else {
4308 GC_ASSERT(!st_lookup(finalizer_table, key, NULL));
4309 }
4310}
4311
4312static void
4313finalize_list(rb_objspace_t *objspace, VALUE zombie)
4314{
4315 while (zombie) {
4316 VALUE next_zombie;
4317 struct heap_page *page;
4318 rb_asan_unpoison_object(zombie, false);
4319 next_zombie = RZOMBIE(zombie)->next;
4320 page = GET_HEAP_PAGE(zombie);
4321
4322 run_final(objspace, zombie);
4323 {
4324 GC_ASSERT(BUILTIN_TYPE(zombie) == T_ZOMBIE);
4325 GC_ASSERT(page->heap->final_slots_count > 0);
4326 GC_ASSERT(page->final_slots > 0);
4327
4328 page->heap->final_slots_count--;
4329 page->final_slots--;
4330 page->free_slots++;
4331 RVALUE_AGE_SET_BITMAP(zombie, 0);
4332 heap_page_add_free_region(objspace, page, zombie);
4333 page->heap->total_freed_objects++;
4334 }
4335
4336 zombie = next_zombie;
4337 }
4338}
4339
4340static void
4341finalize_zombies(rb_objspace_t *objspace)
4342{
4343 VALUE zombie;
4344 while ((zombie = RUBY_ATOMIC_VALUE_EXCHANGE(heap_pages_deferred_final, 0)) != 0) {
4345 finalize_list(objspace, zombie);
4346 }
4347}
4348
4349static void
4350finalize_deferred(rb_objspace_t *objspace)
4351{
4352 rb_gc_set_pending_interrupt();
4353 finalize_zombies(objspace);
4354 rb_gc_unset_pending_interrupt();
4355}
4356
4357static void
4358gc_finalize_deferred(void *dmy)
4359{
4360 /* One postponed job is shared by every objspace: the preregistration table only
4361 * holds about 32 entries and Ractors are created continuously. A deferred finalizer
4362 * belongs to the objspace of the thread that ran the job, i.e. the current one. */
4363 rb_objspace_t *objspace = rb_gc_get_objspace();
4364 if (RUBY_ATOMIC_EXCHANGE(finalizing, 1)) return;
4365
4366 finalize_deferred(objspace);
4367 RUBY_ATOMIC_SET(finalizing, 0);
4368}
4369
4370static void
4371gc_finalize_deferred_register(rb_objspace_t *objspace)
4372{
4373 /* Enqueue gc_finalize_deferred on this objspace's owning Ractor. A global GC can
4374 * defer a foreign objspace's finalizers, and those must run on their owner rather
4375 * than on the driver. */
4376 rb_gc_trigger_finalize_deferred(objspace, objspace->finalize_deferred_pjob);
4377}
4378
4379static int pop_mark_stack(mark_stack_t *stack, VALUE *data);
4380
4381/* Throw away an unfinished incremental mark and leave the objspace in gc_mode_none. The
4382 * mark bits the partial mark set stay behind, so the caller must clear them before the
4383 * heap is collected again. */
4384static void
4385gc_abort_incremental_marking(rb_objspace_t *objspace)
4386{
4387 GC_ASSERT(is_incremental_marking(objspace));
4388
4389 VALUE obj;
4390 while (pop_mark_stack(&objspace->mark_stack, &obj));
4391
4392 /* gc_grey records the weak references it greys for gc_marks_finish to resolve; this
4393 * cycle never reaches it, and the entries would outlive their objects. */
4394 rb_darray_clear(objspace->weak_references);
4395
4396 gc_set_incremental_marking(objspace, FALSE);
4397 gc_mode_set(objspace, gc_mode_none);
4398}
4399
4400static void
4401gc_abort(void *objspace_ptr)
4402{
4403 rb_objspace_t *objspace = objspace_ptr;
4404
4405 if (is_incremental_marking(objspace)) {
4406 gc_abort_incremental_marking(objspace);
4407 }
4408
4409 if (is_lazy_sweeping(objspace)) {
4410 objspace->sweeping_heap_count = 0;
4411 for (int i = 0; i < HEAP_COUNT; i++) {
4412 rb_heap_t *heap = &heaps[i];
4413
4414 heap->sweeping_page = NULL;
4415 struct heap_page *page = NULL;
4416
4417 ccan_list_for_each(&heap->pages, page, page_node) {
4418 page->flags.before_sweep = false;
4419 }
4420 }
4421 }
4422
4423 for (int i = 0; i < HEAP_COUNT; i++) {
4424 rb_heap_t *heap = &heaps[i];
4425 gc_bitmaps_clear(objspace, heap, false);
4426 }
4427
4428 gc_mode_set(objspace, gc_mode_none);
4429}
4430
4431#if VERIFY_FREE_SIZE
4432# ifdef RB_THREAD_LOCAL_SPECIFIER
4433# define GC_FREEING_OBJ_TLS RB_THREAD_LOCAL_SPECIFIER
4434# else
4435# define GC_FREEING_OBJ_TLS
4436# endif
4437
4438static GC_FREEING_OBJ_TLS VALUE gc_freeing_obj;
4439
4440/* Remember what we are tearing down so that a bad xfree() underneath can name
4441 * the object and not just the buffer. Saved and restored because a dfree
4442 * callback can free another object. */
4443static bool
4444gc_obj_free(void *objspace, VALUE obj)
4445{
4446 VALUE prev = gc_freeing_obj;
4447 gc_freeing_obj = obj;
4448
4449 bool freed = rb_gc_obj_free(objspace, obj);
4450
4451 gc_freeing_obj = prev;
4452 return freed;
4453}
4454
4455static const char *
4456gc_freeing_obj_info(void)
4457{
4458 /* Not thread-local: only reachable from a rb_bug() path, where a second
4459 * thread racing us is already unrecoverable. */
4460 static char buf[128];
4461
4462 if (!gc_freeing_obj) return NULL;
4463
4464 snprintf(buf, sizeof(buf), "%p %s", (void *)gc_freeing_obj, rb_obj_info(gc_freeing_obj));
4465 return buf;
4466}
4467#else
4468# define gc_obj_free(objspace, obj) rb_gc_obj_free((objspace), (obj))
4469# define gc_freeing_obj_info() NULL
4470#endif
4471
4472void
4473rb_gc_impl_shutdown_free_objects(void *objspace_ptr)
4474{
4475 rb_objspace_t *objspace = objspace_ptr;
4476
4477 for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
4478 struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
4479 short stride = page->slot_size;
4480
4481 uintptr_t p = (uintptr_t)page->start;
4482 uintptr_t pend = p + page->total_slots * stride;
4483 for (; p < pend; p += stride) {
4484 VALUE vp = (VALUE)p;
4485 asan_unpoisoning_object(vp) {
4486 if (RB_BUILTIN_TYPE(vp) != T_NONE) {
4487 rb_gc_obj_free_vm_weak_references(vp);
4488 if (gc_obj_free(objspace, vp)) {
4489 RBASIC(vp)->flags = 0;
4490 }
4491 }
4492 }
4493 }
4494 }
4495}
4496
4497static int
4498rb_gc_impl_shutdown_call_finalizer_i(st_data_t key, st_data_t val, st_data_t _data)
4499{
4500 VALUE obj = (VALUE)key;
4501 VALUE table = (VALUE)val;
4502
4503 GC_ASSERT(RB_FL_TEST(obj, FL_FINALIZE));
4504 GC_ASSERT(RB_BUILTIN_TYPE(val) == T_ARRAY);
4505
4506 rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), RARRAY_LEN(table) - 1, get_final, (void *)table);
4507
4508 /* obj may be a poisoned zombie */
4509 asan_unpoisoning_object(obj) {
4510 FL_UNSET(obj, FL_FINALIZE);
4511 }
4512
4513 return ST_DELETE;
4514}
4515
4516void
4517rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr)
4518{
4519 rb_objspace_t *objspace = objspace_ptr;
4520
4521#if RGENGC_CHECK_MODE >= 2
4522 gc_verify_internal_consistency(objspace);
4523#endif
4524
4525 /* prohibit incremental GC */
4526 objspace->flags.dont_incremental = 1;
4527
4528 if (RUBY_ATOMIC_EXCHANGE(finalizing, 1)) {
4529 /* Abort incremental marking and lazy sweeping to speed up shutdown. */
4530 gc_abort(objspace);
4531 dont_gc_on();
4532 return;
4533 }
4534
4535 while (finalizer_table->num_entries) {
4536 st_foreach(finalizer_table, rb_gc_impl_shutdown_call_finalizer_i, 0);
4537 }
4538
4539 /* run finalizers */
4540 finalize_deferred(objspace);
4541 GC_ASSERT(heap_pages_deferred_final == 0);
4542
4543 /* Deferred non-thread-safe frees: their objects are long gone, so the object walk
4544 * below will not reach them. Reap them here. */
4545 gc_tdata_unsafe_drain_objspaces(&objspace, 1);
4546
4547 /* Abort incremental marking and lazy sweeping to speed up shutdown. */
4548 gc_abort(objspace);
4549
4550 /* prohibit GC because force T_DATA finalizers can break an object graph consistency */
4551 dont_gc_on();
4552
4553 /* running data/file finalizers are part of garbage collection */
4554 unsigned int lock_lev;
4555 gc_enter(objspace, gc_enter_event_finalizer, &lock_lev);
4556
4557 /* run data/file object's finalizers */
4558 for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
4559 struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
4560 short stride = page->slot_size;
4561
4562 uintptr_t p = (uintptr_t)page->start;
4563 uintptr_t pend = p + page->total_slots * stride;
4564 for (; p < pend; p += stride) {
4565 VALUE vp = (VALUE)p;
4566 asan_unpoisoning_object(vp) {
4567 if (rb_gc_shutdown_call_finalizer_p(vp)) {
4568 rb_gc_obj_free_vm_weak_references(vp);
4569 if (gc_obj_free(objspace, vp)) {
4570 RBASIC(vp)->flags = 0;
4571 }
4572 }
4573 }
4574 }
4575 }
4576
4577 gc_exit(objspace, gc_enter_event_finalizer, &lock_lev);
4578
4579 finalize_zombies(objspace);
4580
4581 st_free_table(finalizer_table);
4582 finalizer_table = 0;
4583 RUBY_ATOMIC_SET(finalizing, 0);
4584}
4585
4586void
4587rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data)
4588{
4589 rb_objspace_t *objspace = objspace_ptr;
4590
4591 for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
4592 struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
4593 short stride = page->slot_size;
4594
4595 uintptr_t p = (uintptr_t)page->start;
4596 uintptr_t pend = p + page->total_slots * stride;
4597 for (; p < pend; p += stride) {
4598 VALUE obj = (VALUE)p;
4599
4600 asan_unpoisoning_object(obj) {
4601 func(obj, data);
4602 }
4603 }
4604 }
4605}
4606
4607/*
4608 ------------------------ Garbage Collection ------------------------
4609*/
4610
4611/* Sweeping */
4612
4613static size_t
4614objspace_available_slots(rb_objspace_t *objspace)
4615{
4616 size_t total_slots = 0;
4617 for (int i = 0; i < HEAP_COUNT; i++) {
4618 rb_heap_t *heap = &heaps[i];
4619 total_slots += heap->total_slots;
4620 }
4621 return total_slots;
4622}
4623
4624static size_t
4625objspace_live_slots(rb_objspace_t *objspace)
4626{
4627 return total_allocated_objects(objspace) - total_freed_objects(objspace) - total_final_slots_count(objspace);
4628}
4629
4630static size_t
4631objspace_free_slots(rb_objspace_t *objspace)
4632{
4633 return objspace_available_slots(objspace) - objspace_live_slots(objspace) - total_final_slots_count(objspace);
4634}
4635
4636static void
4637gc_setup_mark_bits(struct heap_page *page)
4638{
4639 /* copy oldgen bitmap to mark bitmap */
4640 memcpy(&page->mark_bits[0], &page->uncollectible_bits[0], HEAP_PAGE_BITMAP_SIZE);
4641}
4642
4643static int gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj);
4644static VALUE gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, struct heap_page *src_page, struct heap_page *dest_page);
4645
4646#if defined(_WIN32)
4647enum {HEAP_PAGE_LOCK = PAGE_NOACCESS, HEAP_PAGE_UNLOCK = PAGE_READWRITE};
4648
4649static BOOL
4650protect_page_body(struct heap_page_body *body, DWORD protect)
4651{
4652 DWORD old_protect;
4653 return VirtualProtect(body, HEAP_PAGE_SIZE, protect, &old_protect) != 0;
4654}
4655#elif defined(__wasi__)
4656// wasi-libc's mprotect emulation does not support PROT_NONE
4657enum {HEAP_PAGE_LOCK, HEAP_PAGE_UNLOCK};
4658#define protect_page_body(body, protect) 1
4659#else
4660enum {HEAP_PAGE_LOCK = PROT_NONE, HEAP_PAGE_UNLOCK = PROT_READ | PROT_WRITE};
4661#define protect_page_body(body, protect) !mprotect((body), HEAP_PAGE_SIZE, (protect))
4662#endif
4663
4664static void
4665lock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
4666{
4667 if (!protect_page_body(body, HEAP_PAGE_LOCK)) {
4668 rb_bug("Couldn't protect page %p, errno: %s", (void *)body, strerror(errno));
4669 }
4670 else {
4671 gc_report(5, objspace, "Protecting page in move %p\n", (void *)body);
4672 }
4673}
4674
4675static void
4676unlock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
4677{
4678 if (!protect_page_body(body, HEAP_PAGE_UNLOCK)) {
4679 rb_bug("Couldn't unprotect page %p, errno: %s", (void *)body, strerror(errno));
4680 }
4681 else {
4682 gc_report(5, objspace, "Unprotecting page in move %p\n", (void *)body);
4683 }
4684}
4685
4686static uintptr_t
4687heap_page_alloc_slot_from_region(struct heap_page *free_page)
4688{
4689 asan_unlock_freelist(free_page);
4690 struct free_region *region = free_page->free_region;
4691 asan_lock_freelist(free_page);
4692
4693 if (region == NULL) {
4694 return 0;
4695 }
4696
4697 rb_asan_unpoison_object((VALUE)region, false);
4698 GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
4699 uintptr_t dest = (uintptr_t)region;
4700 uintptr_t region_end = region->end;
4701 struct free_region *next = region->next;
4702
4703 uintptr_t new_start = dest + free_page->slot_size;
4704
4705 asan_unlock_freelist(free_page);
4706 if (new_start < region_end) {
4707 VALUE next_start = (VALUE)new_start;
4708 rb_asan_unpoison_object(next_start, false);
4709 struct free_region *new_region = (struct free_region *)new_start;
4710 new_region->flags = 0;
4711 new_region->end = region_end;
4712 new_region->next = next;
4713 rb_asan_poison_object(next_start);
4714 free_page->free_region = new_region;
4715 }
4716 else {
4717 free_page->free_region = next;
4718 }
4719 asan_lock_freelist(free_page);
4720
4721 return dest;
4722}
4723
4724static bool
4725try_move(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *free_page, VALUE src)
4726{
4727 GC_ASSERT(gc_is_moveable_obj(objspace, src));
4728
4729 struct heap_page *src_page = GET_HEAP_PAGE(src);
4730 if (!free_page) {
4731 return false;
4732 }
4733
4734 /* We should return true if either src is successfully moved, or src is
4735 * unmoveable. A false return will cause the sweeping cursor to be
4736 * incremented to the next page, and src will attempt to move again */
4737 GC_ASSERT(RVALUE_MARKED(objspace, src));
4738
4739 uintptr_t dest_slot = heap_page_alloc_slot_from_region(free_page);
4740 if (dest_slot == 0) {
4741 return false;
4742 }
4743 VALUE dest = (VALUE)dest_slot;
4744
4745 GC_ASSERT(RB_BUILTIN_TYPE(dest) == T_NONE);
4746
4747 if (src_page->slot_size > free_page->slot_size) {
4748 objspace->rcompactor.moved_down_count_table[BUILTIN_TYPE(src)]++;
4749 }
4750 else if (free_page->slot_size > src_page->slot_size) {
4751 objspace->rcompactor.moved_up_count_table[BUILTIN_TYPE(src)]++;
4752 }
4753 objspace->rcompactor.moved_count_table[BUILTIN_TYPE(src)]++;
4754 objspace->rcompactor.total_moved++;
4755
4756 gc_move(objspace, src, dest, src_page, free_page);
4757 gc_pin(objspace, src);
4758 free_page->free_slots--;
4759
4760 return true;
4761}
4762
4763static void
4764gc_unprotect_pages(rb_objspace_t *objspace, rb_heap_t *heap)
4765{
4766 struct heap_page *cursor = heap->compact_cursor;
4767
4768 while (cursor) {
4769 unlock_page_body(objspace, cursor->body);
4770 cursor = ccan_list_next(&heap->pages, cursor, page_node);
4771 }
4772}
4773
4774static void gc_update_references(rb_objspace_t *objspace);
4775static void gc_update_references_heap(rb_objspace_t *objspace);
4776static void gc_update_references_global(rb_objspace_t *objspace);
4777#if GC_CAN_COMPILE_COMPACTION
4778static void invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page);
4779#endif
4780
4781#if defined(__MINGW32__) || defined(_WIN32)
4782# define GC_COMPACTION_SUPPORTED 1
4783#else
4784/* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for
4785 * the read barrier, so we must disable compaction. */
4786# define GC_COMPACTION_SUPPORTED (GC_CAN_COMPILE_COMPACTION && HEAP_PAGE_ALLOC_USE_MMAP)
4787#endif
4788
4789#if GC_CAN_COMPILE_COMPACTION
4790static void
4791read_barrier_handler(uintptr_t address)
4792{
4793 rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace();
4794
4795 struct heap_page_body *page_body = GET_PAGE_BODY(address);
4796
4797 /* If the page_body is NULL, then mprotect cannot handle it and will crash
4798 * with "Cannot allocate memory". */
4799 if (page_body == NULL) {
4800 rb_bug("read_barrier_handler: segmentation fault at %p", (void *)address);
4801 }
4802
4803 int lev = RB_GC_VM_LOCK();
4804 {
4805 unlock_page_body(objspace, page_body);
4806
4807 objspace->profile.read_barrier_faults++;
4808
4809 invalidate_moved_page(objspace, GET_HEAP_PAGE(address));
4810 }
4811 RB_GC_VM_UNLOCK(lev);
4812}
4813#endif
4814
4815#if !GC_CAN_COMPILE_COMPACTION
4816static void
4817uninstall_handlers(void)
4818{
4819 /* no-op */
4820}
4821
4822static void
4823install_handlers(void)
4824{
4825 /* no-op */
4826}
4827#elif defined(_WIN32)
4828static LPTOP_LEVEL_EXCEPTION_FILTER old_handler;
4829typedef void (*signal_handler)(int);
4830static signal_handler old_sigsegv_handler;
4831
4832static LONG WINAPI
4833read_barrier_signal(EXCEPTION_POINTERS *info)
4834{
4835 /* EXCEPTION_ACCESS_VIOLATION is what's raised by access to protected pages */
4836 if (info->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
4837 /* > The second array element specifies the virtual address of the inaccessible data.
4838 * https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record
4839 *
4840 * Use this address to invalidate the page */
4841 read_barrier_handler((uintptr_t)info->ExceptionRecord->ExceptionInformation[1]);
4842 return EXCEPTION_CONTINUE_EXECUTION;
4843 }
4844 else {
4845 return EXCEPTION_CONTINUE_SEARCH;
4846 }
4847}
4848
4849static void
4850uninstall_handlers(void)
4851{
4852 signal(SIGSEGV, old_sigsegv_handler);
4853 SetUnhandledExceptionFilter(old_handler);
4854}
4855
4856static void
4857install_handlers(void)
4858{
4859 /* Remove SEGV handler so that the Unhandled Exception Filter handles it */
4860 old_sigsegv_handler = signal(SIGSEGV, NULL);
4861 /* Unhandled Exception Filter has access to the violation address similar
4862 * to si_addr from sigaction */
4863 old_handler = SetUnhandledExceptionFilter(read_barrier_signal);
4864}
4865#else
4866static struct sigaction old_sigbus_handler;
4867static struct sigaction old_sigsegv_handler;
4868
4869#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
4870static exception_mask_t old_exception_masks[32];
4871static mach_port_t old_exception_ports[32];
4872static exception_behavior_t old_exception_behaviors[32];
4873static thread_state_flavor_t old_exception_flavors[32];
4874static mach_msg_type_number_t old_exception_count;
4875
4876static void
4877disable_mach_bad_access_exc(void)
4878{
4879 old_exception_count = sizeof(old_exception_masks) / sizeof(old_exception_masks[0]);
4880 task_swap_exception_ports(
4881 mach_task_self(), EXC_MASK_BAD_ACCESS,
4882 MACH_PORT_NULL, EXCEPTION_DEFAULT, 0,
4883 old_exception_masks, &old_exception_count,
4884 old_exception_ports, old_exception_behaviors, old_exception_flavors
4885 );
4886}
4887
4888static void
4889restore_mach_bad_access_exc(void)
4890{
4891 for (mach_msg_type_number_t i = 0; i < old_exception_count; i++) {
4892 task_set_exception_ports(
4893 mach_task_self(),
4894 old_exception_masks[i], old_exception_ports[i],
4895 old_exception_behaviors[i], old_exception_flavors[i]
4896 );
4897 }
4898}
4899#endif
4900
4901#if defined(HAVE_PTHREAD_SIGMASK)
4902# define gc_sigmask pthread_sigmask
4903#else
4904# define gc_sigmask sigprocmask
4905#endif
4906
4907static void
4908read_barrier_signal(int sig, siginfo_t *info, void *data)
4909{
4910 // setup SEGV/BUS handlers for errors
4911 struct sigaction prev_sigbus, prev_sigsegv;
4912 sigaction(SIGBUS, &old_sigbus_handler, &prev_sigbus);
4913 sigaction(SIGSEGV, &old_sigsegv_handler, &prev_sigsegv);
4914
4915 // enable SIGBUS/SEGV
4916 sigset_t set, prev_set;
4917 sigemptyset(&set);
4918 sigaddset(&set, SIGBUS);
4919 sigaddset(&set, SIGSEGV);
4920 gc_sigmask(SIG_UNBLOCK, &set, &prev_set);
4921#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
4922 disable_mach_bad_access_exc();
4923#endif
4924 // run handler
4925 read_barrier_handler((uintptr_t)info->si_addr);
4926
4927 // reset SEGV/BUS handlers
4928#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
4929 restore_mach_bad_access_exc();
4930#endif
4931 sigaction(SIGBUS, &prev_sigbus, NULL);
4932 sigaction(SIGSEGV, &prev_sigsegv, NULL);
4933 gc_sigmask(SIG_SETMASK, &prev_set, NULL);
4934}
4935
4936static void
4937uninstall_handlers(void)
4938{
4939#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
4940 restore_mach_bad_access_exc();
4941#endif
4942 sigaction(SIGBUS, &old_sigbus_handler, NULL);
4943 sigaction(SIGSEGV, &old_sigsegv_handler, NULL);
4944}
4945
4946static void
4947install_handlers(void)
4948{
4949 struct sigaction action;
4950 memset(&action, 0, sizeof(struct sigaction));
4951 sigemptyset(&action.sa_mask);
4952 action.sa_sigaction = read_barrier_signal;
4953 action.sa_flags = SA_SIGINFO | SA_ONSTACK;
4954
4955 sigaction(SIGBUS, &action, &old_sigbus_handler);
4956 sigaction(SIGSEGV, &action, &old_sigsegv_handler);
4957#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
4958 disable_mach_bad_access_exc();
4959#endif
4960}
4961#endif
4962
4963static void
4964gc_compact_finish(rb_objspace_t *objspace)
4965{
4966 for (int i = 0; i < HEAP_COUNT; i++) {
4967 rb_heap_t *heap = &heaps[i];
4968 gc_unprotect_pages(objspace, heap);
4969 }
4970
4971 if (!global_objspace->global_gc.compacting) uninstall_handlers();
4972
4973 if (global_objspace->global_gc.compacting) {
4974 /* In a compacting global GC this updates only this objspace's heap references;
4975 * gc_start_global sets during_reference_updating on every objspace (the
4976 * move-or-mark decision reads it via rb_gc_get_objspace()) and runs the
4977 * non-idempotent VM-global side (gc_update_references_global) once at the end. */
4978 gc_update_references_heap(objspace);
4979 }
4980 else {
4981 gc_update_references(objspace);
4982 }
4983 objspace->profile.compact_count++;
4984
4985 for (int i = 0; i < HEAP_COUNT; i++) {
4986 rb_heap_t *heap = &heaps[i];
4987 heap->compact_cursor = NULL;
4988 heap->free_pages = NULL;
4989 heap->compact_cursor_index = 0;
4990 }
4991
4992 if (gc_prof_enabled(objspace)) {
4993 gc_profile_record *record = gc_prof_record(objspace);
4994 record->moved_objects = objspace->rcompactor.total_moved - record->moved_objects;
4995 }
4996 if (!global_objspace->global_gc.compacting) objspace->flags.during_compacting = FALSE;
4997}
4998
4999#ifdef _WIN32
5000/* Allocation runs upwards within each free region, so sweeping upwards frees the slots'
5001 * malloc'ed buffers in the order they were allocated. That fragments the NT heap, and a
5002 * program that keeps allocating large strings gets slower on every GC. Sweep from the
5003 * end of the page there, to free them in the reverse order. */
5004# define GC_SWEEP_DOWNWARDS 1
5005#else
5006# define GC_SWEEP_DOWNWARDS 0
5007#endif
5008
5010 struct heap_page *page;
5011 int final_slots;
5012 int freed_slots;
5013 int empty_slots;
5014 /* Hoisted out of the per-slot pinned-free assert: too expensive for the sweep loop
5015 * as an external call. */
5016 const bool check_pinned_free;
5017 /* This is a parallel local sweep (multi-Ractor, not a global GC), so a non-thread-safe
5018 * T_DATA dfree must be deferred to the global GC or the postponed job rather than run here. */
5019 const bool defer_thread_unsafe_local_sweep;
5020 bool trigger_thread_unsafe_sweep_postponed_job;
5021
5022 struct free_region *free_region;
5023#if GC_SWEEP_DOWNWARDS
5024 /* The region still growing downwards, whose header is written when it is closed. */
5025 uintptr_t region_start;
5026 uintptr_t region_end;
5027 struct free_region *free_region_tail;
5028#endif
5029};
5030
5031/* NOTE: We must free the root fiber during postmortem collection, otherwise another Ractor
5032 * can collect the fiber through a major GC while we're still tearing it down. Once fibers are
5033 * THREAD_SAFE_FREE, we no longer need the root fiber condition as it will be guaranteed to be
5034 * collected during this time. */
5035static bool
5036gc_obj_defer_local_free_p(rb_objspace_t *objspace, VALUE obj)
5037{
5038 if (BUILTIN_TYPE(obj) != T_DATA) return false;
5039
5040 const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
5041 if (!rb_gc_data_type_deferred_free_p(type)) return false;
5042
5043 if (RTYPEDDATA_GET_DATA(obj) == NULL) return false;
5044
5045 if (type->flags & RUBY_TYPED_FREE_IMMEDIATELY) {
5046 if (objspace->flags.during_postmortem) {
5047 if (rb_fiber_current() == obj) {
5048 return false;
5049 }
5050 }
5051 return true;
5052 }
5053 else {
5054 return false;
5055 }
5056}
5057
5058static void gc_tdata_deferred_free_job(void *unused);
5059static void gc_tdata_deferred_free_pjob_ensure(void);
5060static unsigned int gc_during_gc_get(const rb_objspace_t *objspace);
5061static void gc_during_gc_set(rb_objspace_t *objspace, unsigned int v);
5062static void gc_global_snapshot_objspaces(void);
5063
5064static void
5065gc_tdata_deferred_free_pjob_ensure(void)
5066{
5067 if (global_objspace->tdata_deferred_free_pjob == POSTPONED_JOB_HANDLE_INVALID) {
5068 global_objspace->tdata_deferred_free_pjob =
5069 rb_postponed_job_preregister(0, gc_tdata_deferred_free_job, NULL);
5070 if (global_objspace->tdata_deferred_free_pjob == POSTPONED_JOB_HANDLE_INVALID) {
5071 rb_bug("Could not preregister postponed job for deferred T_DATA free");
5072 }
5073 }
5074}
5075
5076/* A terminating Ractor's postmortem collection runs on an EC whose stack is already
5077 * torn down: it never checks interrupts again, so a job triggered there is lost and no
5078 * later sweep can rediscover the entries. Hand those to the main Ractor. */
5079static void
5080gc_tdata_deferred_free_trigger(rb_objspace_t *objspace)
5081{
5082 if (objspace->flags.during_postmortem) {
5083 rb_gc_trigger_postponed_job_on_main(global_objspace->tdata_deferred_free_pjob);
5084 }
5085 else {
5086 rb_postponed_job_trigger(global_objspace->tdata_deferred_free_pjob);
5087 }
5088}
5089
5090static void
5091gc_tdata_unsafe_free_entry(const struct tdata_unsafe_free_entry *entry, bool embed_xfree)
5092{
5093 entry->dfree(entry->data);
5094 if (embed_xfree) {
5095 xfree(entry->data);
5096 }
5097}
5098
5099static void
5100tdata_unsafe_free_chunk_recycle(struct tdata_unsafe_free_chunk *chunk)
5101{
5102 if (global_objspace->tdata_unsafe_free_cache_len >= TDATA_UNSAFE_FREE_CACHE_MAX) {
5103 free(chunk);
5104 return;
5105 }
5106 tdata_unsafe_free_chunk_reset(chunk);
5107 chunk->next = global_objspace->tdata_unsafe_free_cache;
5108 rbimpl_atomic_ptr_store((volatile void **)&global_objspace->tdata_unsafe_free_cache, chunk,
5109 RBIMPL_ATOMIC_RELEASE);
5110 global_objspace->tdata_unsafe_free_cache_len++;
5111}
5112
5113static void
5114gc_tdata_unsafe_drain_chunk(struct tdata_unsafe_free_chunk *chunk)
5115{
5116 for (unsigned int i = 0; i < chunk->count; i++) {
5117 gc_tdata_unsafe_free_entry(&chunk->entries[i],
5118 (chunk->embed_xfree_bits >> i) & 1);
5119 }
5120 tdata_unsafe_free_chunk_recycle(chunk);
5121}
5122
5123/* Run every pending deferred free: the published chunks (which belong to no objspace)
5124 * plus the given objspaces' partial chunks. The caller must have stopped the world --
5125 * VM barrier held, or a single Ractor left in the process -- and must pass every live
5126 * objspace, since the pending count is zeroed here. (Shutdown is the one exception:
5127 * nothing reads the count afterwards.) */
5128static void
5129gc_tdata_unsafe_drain_objspaces(rb_objspace_t **objspaces, size_t n)
5130{
5131 struct tdata_unsafe_free_chunk *chunk =
5132 rbimpl_atomic_ptr_exchange((void **)&global_objspace->tdata_unsafe_free_published, NULL,
5133 RBIMPL_ATOMIC_ACQ_REL);
5134 while (chunk) {
5135 struct tdata_unsafe_free_chunk *next = chunk->next;
5136 gc_tdata_unsafe_drain_chunk(chunk);
5137 chunk = next;
5138 }
5139
5140 for (size_t i = 0; i < n; i++) {
5141 rb_objspace_t *os = objspaces[i];
5142 struct tdata_unsafe_free_chunk *partial = os->tdata_unsafe_free_chunk;
5143 if (partial) {
5144 os->tdata_unsafe_free_chunk = NULL;
5145 gc_tdata_unsafe_drain_chunk(partial);
5146 }
5147 }
5148
5149 rbimpl_atomic_size_exchange(&global_objspace->tdata_deferred_free_count, 0,
5150 RBIMPL_ATOMIC_RELAXED);
5151}
5152
5153/* Stop the world and run the dfree function for all deferred T_DATAs. */
5154static void
5155gc_tdata_unsafe_drain(void)
5156{
5157 unsigned int lev = RB_GC_VM_LOCK();
5158
5159 if (tdata_deferred_free_count_load() == 0) {
5160 RB_GC_VM_UNLOCK(lev);
5161 return;
5162 }
5163
5164 rb_gc_vm_barrier();
5165
5166 gc_global_snapshot_objspaces();
5167
5168 /* Set during_gc=TRUE and init vm_context for the CURRENT objspace only.
5169 * The no-alloc guard checks only the allocating (=current) objspace's during_gc,
5170 * and rb_gc_get_ec() reads only the current objspace's vm_context.ec. */
5171 rb_objspace_t *objspace = rb_gc_get_objspace();
5172 unsigned int saved_during_gc = gc_during_gc_get(objspace);
5173 dont_gc_on();
5174 rb_gc_initialize_vm_context(&objspace->vm_context);
5175 gc_during_gc_set(objspace, TRUE);
5176
5177 gc_tdata_unsafe_drain_objspaces(global_objspace->global_gc.objspaces,
5178 global_objspace->global_gc.n_objspaces);
5179
5180 gc_during_gc_set(objspace, saved_during_gc);
5181 dont_gc_off();
5182
5183 RB_GC_VM_UNLOCK(lev);
5184}
5185
5186static void
5187gc_tdata_deferred_free_job(void *unused)
5188{
5189 (void)unused;
5190
5191 size_t count = tdata_deferred_free_count_load();
5192 if (count == 0) return;
5193 if (count < TDATA_DEFERRED_FREE_THRESHOLD && !rb_gc_single_objspace_p()) return;
5194
5195 gc_tdata_unsafe_drain();
5196}
5197
5198#if GC_SWEEP_DOWNWARDS
5199static inline void
5200gc_sweep_close_free_region(struct gc_sweep_context *ctx)
5201{
5202 if (!ctx->region_start) return;
5203
5204 struct free_region *region = (struct free_region *)ctx->region_start;
5205 rb_asan_unpoison_object((VALUE)region, false);
5206 region->end = ctx->region_end;
5207 region->next = NULL;
5208 rb_asan_poison_object((VALUE)region);
5209
5210 /* Regions close from the top of the page, so appending each one keeps the highest region
5211 * first, as the upward sweep leaves the list. Only the order of the frees has to change,
5212 * and pushing would also reorder allocation, which made the NT heap fault in more pages. */
5213 if (ctx->free_region_tail) {
5214 rb_asan_unpoison_object((VALUE)ctx->free_region_tail, false);
5215 ctx->free_region_tail->next = region;
5216 rb_asan_poison_object((VALUE)ctx->free_region_tail);
5217 }
5218 else {
5219 ctx->free_region = region;
5220 }
5221 ctx->free_region_tail = region;
5222}
5223#endif
5224
5225static inline void
5226gc_sweep_register_free_slot(rb_objspace_t *objspace, struct heap_page *page, struct gc_sweep_context *ctx, uintptr_t p, short slot_size)
5227{
5228 rb_asan_unpoison_object(p, false);
5229 /* A slot that was already free reads 0 here. Skipping the store keeps the sweep from
5230 * possibly dirtying a page */
5231 if (((struct RBasic *)p)->flags) ((struct RBasic *)p)->flags = 0;
5232
5233 /* Keep a freed slot from carrying its old shareable and shref bits into the next
5234 * object born there; the actual clear happens per bitmap word at the end of
5235 * gc_sweep_page rather than per slot. */
5236
5237#if GC_SWEEP_DOWNWARDS
5238 if (RB_LIKELY(p + slot_size == ctx->region_start)) {
5239 ctx->region_start = p;
5240 }
5241 else {
5242 gc_sweep_close_free_region(ctx);
5243 ctx->region_start = p;
5244 ctx->region_end = p + slot_size;
5245 }
5246#else
5247 struct free_region *existing_region = ctx->free_region;
5248 if (existing_region) rb_asan_unpoison_object((VALUE)existing_region, false);
5249
5250 if (RB_LIKELY(existing_region && p == existing_region->end)) {
5251 existing_region->end = p + slot_size;
5252 }
5253 else {
5254 struct free_region *free_region = (struct free_region *)p;
5255 free_region->end = p + slot_size;
5256 free_region->next = existing_region;
5257
5258 ctx->free_region = free_region;
5259 }
5260
5261 if (existing_region) rb_asan_poison_object((VALUE)existing_region);
5262#endif
5263 rb_asan_poison_object(p);
5264}
5265
5266static inline void
5267gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct gc_sweep_context *ctx)
5268{
5269 struct heap_page *sweep_page = ctx->page;
5270 short slot_size = sweep_page->slot_size;
5271
5272 do {
5273 VALUE vp = (VALUE)p;
5274 GC_ASSERT(vp % sizeof(VALUE) == 0);
5275
5276#if GC_SWEEP_DOWNWARDS
5277 /* p starts at the plane's last slot, which may lie past the end of the page. */
5278 if (bitset >> (BITS_BITLENGTH - 1)) {
5279 rb_asan_unpoison_object(vp, false);
5280#else
5281 rb_asan_unpoison_object(vp, false);
5282 if (bitset & 1) {
5283#endif
5284 switch (BUILTIN_TYPE(vp)) {
5285 case T_MOVED:
5286 if (objspace->flags.during_compacting) {
5287 /* The sweep cursor shouldn't have made it to any
5288 * T_MOVED slots while the compact flag is enabled.
5289 * The sweep cursor and compact cursor move in
5290 * opposite directions, and when they meet references will
5291 * get updated and "during_compacting" should get disabled */
5292 rb_bug("T_MOVED shouldn't be seen until compaction is finished");
5293 }
5294 gc_report(3, objspace, "page_sweep: %s is freed\n", rb_obj_info(vp));
5295 ctx->empty_slots++;
5296 gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
5297 break;
5298 case T_ZOMBIE:
5299 /* already counted */
5300 break;
5301 case T_NONE:
5302 ctx->empty_slots++; /* already freed */
5303 gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
5304 break;
5305
5306 default:
5307#if RGENGC_CHECK_MODE
5308 /* A local GC must never free a pinned slot; a global GC may (its exact
5309 * mark collects dead shareable objects). Reading the bits here is
5310 * CHECK-only and still valid: the bulk clear runs after the free loop. */
5311 if (ctx->check_pinned_free &&
5312 (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(vp), vp) ||
5313 MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(vp), vp))) {
5314 rb_bug("page_sweep: freeing pinned slot %s (shareable=%d shref=%d single_now=%d)",
5315 rb_obj_info(vp),
5316 (int)!!MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(vp), vp),
5317 (int)!!MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(vp), vp),
5318 (int)rb_gc_single_objspace_p());
5319 }
5320#endif
5321#if RGENGC_CHECK_MODE
5322 if (!is_full_marking(objspace)) {
5323 if (RVALUE_OLD_P(objspace, vp)) rb_bug("page_sweep: %p - old while minor GC.", (void *)p);
5324 if (RVALUE_REMEMBERED(objspace, vp)) rb_bug("page_sweep: %p - remembered.", (void *)p);
5325 }
5326#endif
5327
5328#if RGENGC_CHECK_MODE
5329#define CHECK(x) if (x(objspace, vp) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", rb_obj_info(vp))
5330 CHECK(RVALUE_WB_UNPROTECTED);
5331 CHECK(RVALUE_MARKED);
5332 CHECK(RVALUE_MARKING);
5333 CHECK(RVALUE_UNCOLLECTIBLE);
5334#undef CHECK
5335#endif
5336
5337 if (!rb_gc_obj_needs_cleanup_p(vp)) {
5338 (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size);
5339 gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
5340 gc_report(3, objspace, "page_sweep: %s (fast path) is freed\n", rb_obj_info(vp));
5341 ctx->freed_slots++;
5342 }
5343 else {
5344 gc_report(2, objspace, "page_sweep: free %p\n", (void *)p);
5345
5346 if (RB_UNLIKELY(ctx->defer_thread_unsafe_local_sweep && gc_obj_defer_local_free_p(objspace, vp))) {
5347 /* Defer the dfree instead of running it here: it needs the world
5348 * stopped, which a parallel local sweep cannot give it. The slot is reusable
5349 * right away unless we had to create a zombie. */
5350 if (gc_defer_thread_unsafe_free(objspace, vp,
5351 &ctx->trigger_thread_unsafe_sweep_postponed_job)) {
5352 (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size);
5353 gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
5354 ctx->freed_slots++;
5355 }
5356 else {
5357 ctx->final_slots++;
5358 }
5359 break;
5360 }
5361 rb_gc_obj_free_vm_weak_references(vp);
5362 if (gc_obj_free(objspace, vp)) {
5363 (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size);
5364 gc_sweep_register_free_slot(objspace, sweep_page, ctx, p, slot_size);
5365 gc_report(3, objspace, "page_sweep: %s is freed\n", rb_obj_info(vp));
5366 ctx->freed_slots++;
5367 }
5368 else {
5369 ctx->final_slots++;
5370 }
5371 }
5372 break;
5373 }
5374 }
5375#if GC_SWEEP_DOWNWARDS
5376 p -= slot_size;
5377 bitset <<= 1;
5378#else
5379 p += slot_size;
5380 bitset >>= 1;
5381#endif
5382 } while (bitset);
5383}
5384
5385static inline void
5386gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context *ctx)
5387{
5388 struct heap_page *sweep_page = ctx->page;
5389 GC_ASSERT(sweep_page->heap == heap);
5390
5391 uintptr_t p;
5392 bits_t *bits, bitset;
5393
5394 gc_report(2, objspace, "page_sweep: start.\n");
5395
5396#if RGENGC_CHECK_MODE
5397 if (!objspace->flags.immediate_sweep) {
5398 GC_ASSERT(sweep_page->flags.before_sweep == TRUE);
5399 }
5400#endif
5401 sweep_page->flags.before_sweep = FALSE;
5402 sweep_page->free_slots = 0;
5403
5404 asan_unlock_freelist(sweep_page);
5405 sweep_page->free_region = NULL;
5406 asan_lock_freelist(sweep_page);
5407 ctx->free_region = NULL;
5408#if GC_SWEEP_DOWNWARDS
5409 ctx->region_start = 0;
5410 ctx->free_region_tail = NULL;
5411#endif
5412
5413 p = (uintptr_t)sweep_page->start;
5414 bits = sweep_page->mark_bits;
5415 short slot_size = sweep_page->slot_size;
5416 int total_slots = sweep_page->total_slots;
5417 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
5418
5419 int out_of_range_bits = total_slots % BITS_BITLENGTH;
5420 if (out_of_range_bits != 0) {
5421 bits[bitmap_plane_count - 1] |= ~(((bits_t)1 << out_of_range_bits) - 1);
5422 }
5423
5424 // Clear wb_unprotected and age bits for all unmarked slots
5425 {
5426 bits_t *wb_unprotected_bits = sweep_page->wb_unprotected_bits;
5427 bits_t *age_bits = sweep_page->age_bits;
5428 for (int i = 0; i < bitmap_plane_count; i++) {
5429 bits_t unmarked = ~bits[i];
5430 wb_unprotected_bits[i] &= ~unmarked;
5431 age_bits[i * 2] &= ~unmarked;
5432 age_bits[i * 2 + 1] &= ~unmarked;
5433 }
5434 }
5435
5436#if GC_SWEEP_DOWNWARDS
5437 p += (uintptr_t)(bitmap_plane_count * BITS_BITLENGTH - 1) * slot_size;
5438 for (int i = bitmap_plane_count - 1; i >= 0; i--) {
5439 bitset = ~bits[i];
5440 if (bitset) {
5441 gc_sweep_plane(objspace, heap, p, bitset, ctx);
5442 }
5443 p -= BITS_BITLENGTH * slot_size;
5444 }
5445 gc_sweep_close_free_region(ctx);
5446#else
5447 for (int i = 0; i < bitmap_plane_count; i++) {
5448 bitset = ~bits[i];
5449 if (bitset) {
5450 gc_sweep_plane(objspace, heap, p, bitset, ctx);
5451 }
5452 p += BITS_BITLENGTH * slot_size;
5453 }
5454#endif
5455
5456 /* Bulk-clear the freed slots' shareable and shref bits before the freelist is
5457 * published, so a reused slot is clean. Freed slots are exactly the unmarked ones,
5458 * so `bits &= mark_bits` keeps live shareable objects (which must stay pinned) and
5459 * drops the rest. Pages with neither bit are skipped. */
5460 if (sweep_page->flags.has_shareable_objects || sweep_page->flags.has_shref_objects) {
5461 bits_t *shareable_bits = sweep_page->shareable_bits;
5462 bits_t *shref_bits = sweep_page->shref_bits;
5463 bits_t sh = 0, sr = 0;
5464 for (int i = 0; i < bitmap_plane_count; i++) {
5465 shareable_bits[i] &= bits[i];
5466 shref_bits[i] &= bits[i];
5467 sh |= shareable_bits[i];
5468 sr |= shref_bits[i];
5469 }
5470 if (!sh) sweep_page->flags.has_shareable_objects = FALSE;
5471 if (!sr) sweep_page->flags.has_shref_objects = FALSE;
5472 }
5473
5474 asan_unlock_freelist(sweep_page);
5475 sweep_page->free_region = ctx->free_region;
5476 asan_lock_freelist(sweep_page);
5477
5478 if (!heap->compact_cursor) {
5479 gc_setup_mark_bits(sweep_page);
5480 }
5481
5482#if GC_PROFILE_MORE_DETAIL
5483 if (gc_prof_enabled(objspace)) {
5484 gc_profile_record *record = gc_prof_record(objspace);
5485 record->removing_objects += ctx->final_slots + ctx->freed_slots;
5486 record->empty_objects += ctx->empty_slots;
5487 }
5488#endif
5489 if (0) fprintf(stderr, "gc_sweep_page(%"PRIdSIZE"): total_slots: %d, freed_slots: %d, empty_slots: %d, final_slots: %d\n",
5490 rb_gc_count(),
5491 sweep_page->total_slots,
5492 ctx->freed_slots, ctx->empty_slots, ctx->final_slots);
5493
5494 sweep_page->free_slots += ctx->freed_slots + ctx->empty_slots;
5495 sweep_page->heap->total_freed_objects += ctx->freed_slots;
5496
5497 if (heap_pages_deferred_final && !finalizing) {
5498 gc_finalize_deferred_register(objspace);
5499 }
5500
5501#if RGENGC_CHECK_MODE
5502 int region_slots = 0;
5503 asan_unlock_freelist(sweep_page);
5504 struct free_region *region = sweep_page->free_region;
5505 while (region) {
5506 rb_asan_unpoison_object((VALUE)region, false);
5507 GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
5508 uintptr_t region_start = (uintptr_t)region;
5509 uintptr_t region_end = region->end;
5510 struct free_region *next = region->next;
5511 rb_asan_poison_object((VALUE)region);
5512
5513 GC_ASSERT(region_end > region_start);
5514 GC_ASSERT((region_end - region_start) % slot_size == 0);
5515 region_slots += (int)((region_end - region_start) / slot_size);
5516
5517 region = next;
5518 }
5519 asan_lock_freelist(sweep_page);
5520 if (region_slots != sweep_page->free_slots) {
5521 rb_bug("inconsistent free region slots: expected %d but was %d", sweep_page->free_slots, region_slots);
5522 }
5523#endif
5524
5525 gc_report(2, objspace, "page_sweep: end.\n");
5526}
5527
5528static const char *
5529gc_mode_name(enum gc_mode mode)
5530{
5531 switch (mode) {
5532 case gc_mode_none: return "none";
5533 case gc_mode_marking: return "marking";
5534 case gc_mode_sweeping: return "sweeping";
5535 case gc_mode_compacting: return "compacting";
5536 default: rb_bug("gc_mode_name: unknown mode: %d", (int)mode);
5537 }
5538}
5539
5540static void
5541gc_mode_transition(rb_objspace_t *objspace, enum gc_mode mode)
5542{
5543#if RGENGC_CHECK_MODE
5544 enum gc_mode prev_mode = gc_mode(objspace);
5545 switch (prev_mode) {
5546 case gc_mode_none:
5547 /* A global GC marks every objspace as one heap (mark_roots on the driver), so an
5548 * individual objspace's mode stays `none` during that mark; the sweep inside the
5549 * barrier then makes the legitimate none -> sweeping transition. */
5550 GC_ASSERT(mode == gc_mode_marking ||
5551 (objspace->flags.during_global_gc && mode == gc_mode_sweeping));
5552 break;
5553 case gc_mode_marking: GC_ASSERT(mode == gc_mode_sweeping); break;
5554 case gc_mode_sweeping: GC_ASSERT(mode == gc_mode_none || mode == gc_mode_compacting); break;
5555 case gc_mode_compacting: GC_ASSERT(mode == gc_mode_none); break;
5556 }
5557#endif
5558 if (0) fprintf(stderr, "gc_mode_transition: %s->%s\n", gc_mode_name(gc_mode(objspace)), gc_mode_name(mode));
5559 gc_mode_set(objspace, mode);
5560}
5561
5562static void
5563heap_page_flush_alloc_regions(struct heap_page *page, rb_heap_t *heap)
5564{
5565 struct free_region *chain = heap->newobj.alloc_next_region;
5566
5567 if (heap->newobj.alloc_cursor < heap->newobj.alloc_cursor_end) {
5568 VALUE start = (VALUE)heap->newobj.alloc_cursor;
5569 rb_asan_unpoison_object(start, false);
5570 struct free_region *remnant = (struct free_region *)start;
5571 remnant->flags = 0;
5572 remnant->end = heap->newobj.alloc_cursor_end;
5573 remnant->next = chain;
5574 rb_asan_poison_object(start);
5575 chain = remnant;
5576 }
5577
5578 if (chain) {
5579 asan_unlock_freelist(page);
5580 if (page->free_region) {
5581 struct free_region *p = page->free_region;
5582 rb_asan_unpoison_object((VALUE)p, false);
5583 while (p->next) {
5584 struct free_region *prev = p;
5585 p = p->next;
5586 rb_asan_poison_object((VALUE)prev);
5587 rb_asan_unpoison_object((VALUE)p, false);
5588 }
5589 p->next = chain;
5590 rb_asan_poison_object((VALUE)p);
5591 }
5592 else {
5593 page->free_region = chain;
5594 }
5595 asan_lock_freelist(page);
5596 }
5597}
5598
5599static void
5600gc_sweep_start_heap(rb_objspace_t *objspace, rb_heap_t *heap)
5601{
5602 heap->sweeping_page = ccan_list_top(&heap->pages, struct heap_page, page_node);
5603 if (heap->sweeping_page) {
5604 objspace->sweeping_heap_count++;
5605 }
5606 heap->free_pages = NULL;
5607 heap->pooled_pages = NULL;
5608 if (!objspace->flags.immediate_sweep) {
5609 struct heap_page *page = NULL;
5610
5611 ccan_list_for_each(&heap->pages, page, page_node) {
5612 page->flags.before_sweep = TRUE;
5613 }
5614 }
5615}
5616
5617#if GC_CAN_COMPILE_COMPACTION
5618static void gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func);
5619static int compare_pinned_slots(const void *left, const void *right, void *d);
5620#endif
5621
5622/* Return the current allocation page and freelist to their pages, so the sweeper sees a
5623 * consistent heap. */
5624static void
5625heap_alloc_state_clear(rb_objspace_t *objspace)
5626{
5627 objspace->incremental_mark_step_allocated_slots = 0;
5628
5629 for (size_t heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) {
5630 rb_heap_t *heap = &heaps[heap_idx];
5631
5632 struct heap_page *page = heap->newobj.alloc_using_page;
5633 RUBY_DEBUG_LOG("heap alloc_using_page:%p cursor:%p", (void *)page, (void *)heap->newobj.alloc_cursor);
5634
5635 if (page) {
5636 heap_page_flush_alloc_regions(page, heap);
5637 }
5638
5639 heap->newobj.alloc_using_page = NULL;
5640 heap->newobj.alloc_cursor = 0;
5641 heap->newobj.alloc_cursor_end = 0;
5642 heap->newobj.alloc_next_region = NULL;
5643 }
5644}
5645
5646static void
5647gc_sweep_freeobj_hooks_page(rb_objspace_t *objspace, struct heap_page *page)
5648{
5649 bits_t *bits = page->mark_bits;
5650 uintptr_t p = (uintptr_t)page->start;
5651 short slot_size = page->slot_size;
5652 int total_slots = page->total_slots;
5653 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
5654
5655 int out_of_range_bits = total_slots % BITS_BITLENGTH;
5656 bits_t last_plane_mask = (out_of_range_bits != 0)
5657 ? ~(((bits_t)1 << out_of_range_bits) - 1)
5658 : 0;
5659
5660 for (int j = 0; j < bitmap_plane_count; j++) {
5661 bits_t bitset = ~bits[j];
5662 if (j == bitmap_plane_count - 1) {
5663 bitset &= ~last_plane_mask;
5664 }
5665
5666 uintptr_t pp = p;
5667 while (bitset) {
5668 if (bitset & 1) {
5669 VALUE vp = (VALUE)pp;
5670 asan_unpoisoning_object(vp) {
5671 switch (BUILTIN_TYPE(vp)) {
5672 case T_NONE:
5673 case T_ZOMBIE:
5674 case T_MOVED:
5675 break;
5676 default:
5677 rb_gc_event_hook(vp, RUBY_INTERNAL_EVENT_FREEOBJ);
5678 break;
5679 }
5680 }
5681 }
5682 pp += slot_size;
5683 bitset >>= 1;
5684 }
5685 p += BITS_BITLENGTH * slot_size;
5686 }
5687}
5688
5689static void
5690gc_sweep_freeobj_hooks(rb_objspace_t *objspace)
5691{
5692 for (int i = 0; i < HEAP_COUNT; i++) {
5693 rb_heap_t *heap = &heaps[i];
5694 struct heap_page *page = NULL;
5695
5696 ccan_list_for_each(&heap->pages, page, page_node) {
5697 gc_sweep_freeobj_hooks_page(objspace, page);
5698 }
5699 }
5700}
5701
5702static void
5703gc_sweep_start(rb_objspace_t *objspace)
5704{
5705 gc_mode_transition(objspace, gc_mode_sweeping);
5706 objspace->rincgc.pooled_slots = 0;
5707
5708 if (RB_UNLIKELY(objspace->hook_events & RUBY_INTERNAL_EVENT_FREEOBJ)) {
5709 /* FREEOBJ is never enabled outside the main objspace
5710 * (rb_objspace_set_event_hook), so this hook, which runs user callbacks,
5711 * cannot fire during a non-main Ractor's lock-free local sweep. */
5712 GC_ASSERT(objspace == global_objspace->main_objspace);
5713 gc_sweep_freeobj_hooks(objspace);
5714 }
5715
5716#if GC_CAN_COMPILE_COMPACTION
5717 if (objspace->flags.during_compacting) {
5718 gc_sort_heap_by_compare_func(
5719 objspace,
5720 objspace->rcompactor.compare_func ? objspace->rcompactor.compare_func : compare_pinned_slots
5721 );
5722 }
5723#endif
5724
5725 for (int i = 0; i < HEAP_COUNT; i++) {
5726 rb_heap_t *heap = &heaps[i];
5727 gc_sweep_start_heap(objspace, heap);
5728
5729 /* We should call gc_sweep_finish_heap for size pools with no pages. */
5730 if (heap->sweeping_page == NULL) {
5731 GC_ASSERT(heap->total_pages == 0);
5732 GC_ASSERT(heap->total_slots == 0);
5733 gc_sweep_finish_heap(objspace, heap);
5734 }
5735 }
5736
5737 heap_alloc_state_clear(objspace);
5738}
5739
5740static void
5741gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap)
5742{
5743 /* Nothing allocates from an ownerless objspace, so it has no growth decision */
5744 if (objspace->flags.ownerless) return;
5745
5746 size_t total_slots = heap->total_slots;
5747 size_t swept_slots = heap->freed_slots + heap->empty_slots;
5748
5749 size_t init_slots = objspace_heap_init_bytes(objspace) / heap->slot_size;
5750 size_t min_free_slots = (size_t)(MAX(total_slots, init_slots) * gc_params.heap_free_slots_min_ratio);
5751
5752 if (swept_slots < min_free_slots &&
5753 /* The heap is a growth heap if it freed more slots than had empty slots. */
5754 ((heap->empty_slots == 0 && total_slots > 0) || heap->freed_slots > heap->empty_slots)) {
5755 /* If we don't have enough slots and we have pages on the tomb heap, move
5756 * pages from the tomb heap to the eden heap. This may prevent page
5757 * creation thrashing (frequently allocating and deallocting pages) and
5758 * GC thrashing (running GC more frequently than required). */
5759 struct heap_page *resurrected_page;
5760 while (swept_slots < min_free_slots &&
5761 (resurrected_page = heap_page_resurrect(objspace))) {
5762 heap_add_page(objspace, heap, resurrected_page);
5763 heap_add_freepage(heap, resurrected_page);
5764
5765 swept_slots += resurrected_page->free_slots;
5766 }
5767
5768 if (swept_slots < min_free_slots) {
5769 /* Grow this heap if we are in a major GC or if we haven't run at least
5770 * RVALUE_OLD_AGE minor GC since the last major GC. */
5771 if (is_full_marking(objspace) ||
5772 objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
5773 if (objspace->heap_pages.allocatable_bytes < min_free_slots * heap->slot_size) {
5774 heap_allocatable_bytes_expand(objspace, heap, swept_slots, heap->total_slots, heap->slot_size);
5775 }
5776 }
5777 else if (swept_slots < min_free_slots * 7 / 8 &&
5778 objspace->heap_pages.allocatable_bytes < (min_free_slots * 7 / 8 - swept_slots) * heap->slot_size) {
5779 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
5780 heap->force_major_gc_count++;
5781 }
5782 }
5783 }
5784}
5785
5786#if RGENGC_ESTIMATE_OLDMALLOC
5787static void
5788gc_check_oldmalloc(rb_objspace_t *objspace)
5789{
5790 size_t oldmalloc_increase = gc_oldmalloc_increase_load(objspace);
5791 MALLOC_COUNTERS_LOCK(objspace);
5792 gc_counter_t malloc_at = gc_counter_load_acquire(&objspace->malloc_counters.counters.malloc_at_last_gc);
5793 gc_counter_t free_at = gc_counter_load_acquire(&objspace->malloc_counters.counters.free_at_last_gc);
5794 gc_counter_t malloc_now = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc);
5795 gc_counter_t free_now = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free);
5796 gc_counter_t gc_freed = gc_counter_load_relaxed(&objspace->malloc_counters.freed_in_gc);
5797 MALLOC_COUNTERS_UNLOCK(objspace);
5798
5799 gc_counter_t freed = free_now - free_at;
5800 gc_counter_t mutator_freed = freed > gc_freed ? freed - gc_freed : 0;
5801 gc_counter_t allocated = malloc_now - malloc_at;
5802 gc_counter_t unswept = allocated > mutator_freed ? allocated - mutator_freed : 0;
5803
5804 if ((gc_counter_t)oldmalloc_increase <= unswept) return;
5805 oldmalloc_increase -= (size_t)unswept;
5806
5807 if (oldmalloc_increase > objspace->rgengc.oldmalloc_increase_limit) {
5808 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDMALLOC;
5809 objspace->rgengc.oldmalloc_increase_limit =
5810 (size_t)(objspace->rgengc.oldmalloc_increase_limit * gc_params.oldmalloc_limit_growth_factor);
5811
5812 if (objspace->rgengc.oldmalloc_increase_limit > gc_params.oldmalloc_limit_max) {
5813 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_max;
5814 }
5815 }
5816}
5817#endif
5818
5819static void
5820gc_sweep_finish(rb_objspace_t *objspace)
5821{
5822 gc_report(1, objspace, "gc_sweep_finish\n");
5823
5824 gc_prof_set_heap_info(objspace);
5825 heap_pages_free_unused_pages(objspace);
5826 if (rb_gc_single_objspace_p() && is_full_marking(objspace)) {
5827 /* gc_marks_finish retains ~2/3 of empty pages in objspace->empty_pages for reuse,
5828 * only the excess reaches the pool. */
5829 page_pool_reclaim(global_objspace);
5830 }
5831#if RGENGC_ESTIMATE_OLDMALLOC
5832 if (!is_full_marking(objspace)) {
5833 gc_check_oldmalloc(objspace);
5834 }
5835#endif
5836
5837 for (int i = 0; i < HEAP_COUNT; i++) {
5838 rb_heap_t *heap = &heaps[i];
5839
5840 heap->freed_slots = 0;
5841 heap->empty_slots = 0;
5842
5843 if (!will_be_incremental_marking(objspace)) {
5844 struct heap_page *end_page = heap->free_pages;
5845 if (end_page) {
5846 while (end_page->free_next) end_page = end_page->free_next;
5847 end_page->free_next = heap->pooled_pages;
5848 }
5849 else {
5850 heap->free_pages = heap->pooled_pages;
5851 }
5852 heap->pooled_pages = NULL;
5853 objspace->rincgc.pooled_slots = 0;
5854 }
5855 }
5856
5857 /* Not before: while sweeping is in progress its frees must keep reducing
5858 * malloc_increase (objspace_malloc_increase_body sweeps and retries on it). */
5859 gc_malloc_counters_snapshot_free_at_last_gc(objspace, &objspace->malloc_counters.counters);
5860
5861 /* Leftovers from an earlier multi-Ractor phase: no later sweep can rediscover them
5862 * (their slots are gone), and with one Ractor left the drain's barrier has nothing
5863 * to wait for. */
5864 if (tdata_deferred_free_count_load() > 0 && rb_gc_single_objspace_p()) {
5865 gc_tdata_deferred_free_trigger(objspace);
5866 }
5867
5869 gc_mode_transition(objspace, gc_mode_none);
5870}
5871
5872static int
5873gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap)
5874{
5875 struct heap_page *sweep_page = heap->sweeping_page;
5876 int swept_slots = 0;
5877 int pooled_slots = 0;
5878 int sweep_budget = GC_INCREMENTAL_SWEEP_BYTES / heap->slot_size;
5879 int pool_budget = GC_INCREMENTAL_SWEEP_POOL_BYTES / heap->slot_size;
5880
5881 if (sweep_page == NULL) return FALSE;
5882
5883#if GC_ENABLE_LAZY_SWEEP
5884 gc_prof_sweep_timer_start(objspace);
5885#endif
5886
5887 /* Per-slot pinned-free assert (gc_sweep_context): check only when this cycle's mark
5888 * ran the pinned walk. The current world state would misfire: a single-world
5889 * cycle leaves dead shareable objects unmarked and its sweep can straddle the switch
5890 * to multi-objspace. A global GC's exact mark does not pin, so it is excluded. */
5891 const bool check_pinned_free = objspace->last_cycle_pinned;
5892
5893 const bool defer_thread_unsafe_local_sweep =
5894 !rb_gc_single_objspace_p() && !objspace->flags.during_global_gc;
5895 bool trigger_thread_unsafe_sweep_postponed_job = false;
5896
5897 do {
5898 RUBY_DEBUG_LOG("sweep_page:%p", (void *)sweep_page);
5899
5900 struct gc_sweep_context ctx = {
5901 .page = sweep_page,
5902 .final_slots = 0,
5903 .freed_slots = 0,
5904 .empty_slots = 0,
5905 .check_pinned_free = check_pinned_free,
5906 .defer_thread_unsafe_local_sweep = defer_thread_unsafe_local_sweep,
5907 .trigger_thread_unsafe_sweep_postponed_job = trigger_thread_unsafe_sweep_postponed_job,
5908 };
5909 gc_sweep_page(objspace, heap, &ctx);
5910 int free_slots = ctx.freed_slots + ctx.empty_slots;
5911 trigger_thread_unsafe_sweep_postponed_job = ctx.trigger_thread_unsafe_sweep_postponed_job;
5912
5913 RUBY_DTRACE_GC_HOOK(SWEEP_PAGE, ctx.page->slot_size, ctx.final_slots, ctx.freed_slots, ctx.empty_slots);
5914
5915 heap->sweeping_page = ccan_list_next(&heap->pages, sweep_page, page_node);
5916
5917 if (free_slots == sweep_page->total_slots &&
5918 (heap->total_pages > 1 || objspace->flags.ownerless)) {
5919 /* There are no living objects, so move this page to the global empty pages.
5920 * The last one stays so the heap has something to grow from -- except in an
5921 * ownerless objspace, which will never allocate again. */
5922 heap_page_detach_empty(objspace, heap, sweep_page);
5923 }
5924 else if (free_slots > 0) {
5925 heap->freed_slots += ctx.freed_slots;
5926 heap->empty_slots += ctx.empty_slots;
5927
5928 if (pooled_slots < pool_budget) {
5929 heap_add_poolpage(objspace, heap, sweep_page);
5930 pooled_slots += free_slots;
5931 }
5932 else {
5933 heap_add_freepage(heap, sweep_page);
5934 swept_slots += free_slots;
5935 if (swept_slots > sweep_budget) {
5936 break;
5937 }
5938 }
5939 }
5940 else {
5941 sweep_page->free_next = NULL;
5942 }
5943 } while ((sweep_page = heap->sweeping_page));
5944
5945 if (trigger_thread_unsafe_sweep_postponed_job) {
5946 gc_report(2, objspace, "thread-unsafe sweep postponed job triggered\n");
5947 gc_tdata_deferred_free_trigger(objspace);
5948 }
5949
5950 if (!heap->sweeping_page) {
5951 objspace->sweeping_heap_count--;
5952 GC_ASSERT(objspace->sweeping_heap_count >= 0);
5953 gc_sweep_finish_heap(objspace, heap);
5954
5955 if (!has_sweeping_pages(objspace)) {
5956 gc_sweep_finish(objspace);
5957 }
5958 }
5959
5960#if GC_ENABLE_LAZY_SWEEP
5961 gc_prof_sweep_timer_stop(objspace);
5962#endif
5963
5964 return heap->free_pages != NULL;
5965}
5966
5967static void
5968gc_sweep_rest(rb_objspace_t *objspace)
5969{
5970 for (int i = 0; i < HEAP_COUNT; i++) {
5971 rb_heap_t *heap = &heaps[i];
5972
5973 while (heap->sweeping_page) {
5974 gc_sweep_step(objspace, heap);
5975 }
5976 }
5977
5978 /* An objspace with no live pages never runs gc_sweep_step and so never reaches
5979 * gc_sweep_finish, leaving mode at sweeping or compacting until the next cycle's
5980 * gc_sweep_start asserts. If every heap is swept out, settle it to none here. */
5981 if (gc_mode(objspace) != gc_mode_none && !has_sweeping_pages(objspace)) {
5982 gc_sweep_finish(objspace);
5983 }
5984}
5985
5986static void
5987gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *sweep_heap)
5988{
5989 GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
5990 if (!GC_ENABLE_LAZY_SWEEP) return;
5991
5992 gc_sweeping_enter(objspace);
5993
5994 for (int i = 0; i < HEAP_COUNT; i++) {
5995 rb_heap_t *heap = &heaps[i];
5996 if (gc_sweep_step(objspace, heap)) {
5997 GC_ASSERT(heap->free_pages != NULL);
5998 }
5999 else if (heap == sweep_heap) {
6000 if (objspace->empty_pages_count > 0 || objspace->heap_pages.allocatable_bytes > 0) {
6001 /* [Bug #21548]
6002 *
6003 * If this heap is the heap we want to sweep, but we weren't able
6004 * to free any slots, but we also either have empty pages or could
6005 * allocate new pages, then we want to preemptively claim a page
6006 * because it's possible that sweeping another heap will call
6007 * gc_sweep_finish_heap, which may use up all of the
6008 * empty/allocatable pages. If other heaps are not finished sweeping
6009 * then we do not finish this GC and we will end up triggering a new
6010 * GC cycle during this GC phase. */
6011 heap_page_allocate_and_initialize(objspace, heap);
6012
6013 GC_ASSERT(heap->free_pages != NULL);
6014 }
6015 else {
6016 /* Not allowed to create a new page so finish sweeping. */
6017 gc_sweep_rest(objspace);
6018 GC_ASSERT(gc_mode(objspace) == gc_mode_none);
6019 break;
6020 }
6021 }
6022 }
6023
6024 gc_sweeping_exit(objspace);
6025}
6026
6027static void
6028gc_sweep_step_for_malloc(rb_objspace_t *objspace)
6029{
6030 GC_ASSERT(is_lazy_sweeping(objspace));
6031
6032 unsigned int lock_lev;
6033 gc_enter(objspace, gc_enter_event_continue, &lock_lev);
6034
6035 gc_sweeping_enter(objspace);
6036
6037 for (int i = 0; i < HEAP_COUNT; i++) {
6038 rb_heap_t *heap = &heaps[i];
6039 gc_sweep_step(objspace, heap);
6040 }
6041
6042 gc_sweeping_exit(objspace);
6043
6044 gc_exit(objspace, gc_enter_event_continue, &lock_lev);
6045}
6046
6047static bool gc_global_pointer_to_heap_p(const void *ptr);
6048
6049VALUE
6050rb_gc_impl_location(void *objspace_ptr, VALUE value)
6051{
6052 rb_objspace_t *objspace = objspace_ptr;
6053 VALUE destination;
6054
6055 /* A local (single-objspace) compaction never moves another objspace's objects, so
6056 * leave foreign references alone. A compacting global GC moves objects everywhere
6057 * under the barrier, so there every objspace's heap is searched for forwarding. */
6058 if (RB_UNLIKELY(objspace->flags.during_global_gc)
6059 ? !gc_global_pointer_to_heap_p((void *)value)
6060 : !is_pointer_to_heap(objspace_ptr, (void *)value)) {
6061 return value;
6062 }
6063
6064 asan_unpoisoning_object(value) {
6065 if (BUILTIN_TYPE(value) == T_MOVED) {
6066 destination = (VALUE)RMOVED(value)->destination;
6067 GC_ASSERT(BUILTIN_TYPE(destination) != T_NONE);
6068 }
6069 else {
6070 destination = value;
6071 }
6072 }
6073
6074 return destination;
6075}
6076
6077#if GC_CAN_COMPILE_COMPACTION
6078static void
6079invalidate_moved_plane(rb_objspace_t *objspace, struct heap_page *page, uintptr_t p, bits_t bitset)
6080{
6081 if (bitset) {
6082 do {
6083 if (bitset & 1) {
6084 VALUE forwarding_object = (VALUE)p;
6085 VALUE object;
6086
6087 if (BUILTIN_TYPE(forwarding_object) == T_MOVED) {
6088 GC_ASSERT(RVALUE_PINNED(objspace, forwarding_object));
6089 GC_ASSERT(!RVALUE_MARKED(objspace, forwarding_object));
6090
6091 CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object);
6092
6093 object = rb_gc_impl_location(objspace, forwarding_object);
6094 gc_move(objspace, object, forwarding_object, GET_HEAP_PAGE(object), page);
6095 /* forwarding_object is now our actual object, and "object"
6096 * is the free slot for the original page */
6097
6098 struct heap_page *orig_page = GET_HEAP_PAGE(object);
6099 orig_page->free_slots++;
6100 RVALUE_AGE_SET_BITMAP(object, 0);
6101 heap_page_add_free_region(objspace, orig_page, object);
6102
6103 GC_ASSERT(RVALUE_MARKED(objspace, forwarding_object));
6104 GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_MOVED);
6105 GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_NONE);
6106 }
6107 }
6108 p += page->slot_size;
6109 bitset >>= 1;
6110 } while (bitset);
6111 }
6112}
6113
6114static void
6115invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page)
6116{
6117 int i;
6118 bits_t *mark_bits, *pin_bits;
6119 bits_t bitset;
6120 short slot_size = page->slot_size;
6121 int total_slots = page->total_slots;
6122 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
6123
6124 mark_bits = page->mark_bits;
6125 pin_bits = page->pinned_bits;
6126
6127 uintptr_t p = page->start;
6128
6129 for (i=0; i < bitmap_plane_count; i++) {
6130 /* Moved objects are pinned but never marked. We reuse the pin bits
6131 * to indicate there is a moved object in this slot. */
6132 bitset = pin_bits[i] & ~mark_bits[i];
6133 invalidate_moved_plane(objspace, page, p, bitset);
6134 p += BITS_BITLENGTH * slot_size;
6135 }
6136}
6137#endif
6138
6139static void
6140gc_compact_start(rb_objspace_t *objspace)
6141{
6142 struct heap_page *page = NULL;
6143 gc_mode_transition(objspace, gc_mode_compacting);
6144
6145 for (int i = 0; i < HEAP_COUNT; i++) {
6146 rb_heap_t *heap = &heaps[i];
6147 ccan_list_for_each(&heap->pages, page, page_node) {
6148 page->flags.before_sweep = TRUE;
6149 }
6150
6151 heap->compact_cursor = ccan_list_tail(&heap->pages, struct heap_page, page_node);
6152 heap->compact_cursor_index = 0;
6153 }
6154
6155 if (gc_prof_enabled(objspace)) {
6156 gc_profile_record *record = gc_prof_record(objspace);
6157 record->moved_objects = objspace->rcompactor.total_moved;
6158 }
6159
6160 memset(objspace->rcompactor.considered_count_table, 0, T_MASK * sizeof(size_t));
6161 memset(objspace->rcompactor.moved_count_table, 0, T_MASK * sizeof(size_t));
6162 memset(objspace->rcompactor.moved_up_count_table, 0, T_MASK * sizeof(size_t));
6163 memset(objspace->rcompactor.moved_down_count_table, 0, T_MASK * sizeof(size_t));
6164
6165 /* Set up read barrier for pages containing MOVED objects */
6166 /* A compacting global GC installs the read barrier once for every objspace. */
6167 if (!global_objspace->global_gc.compacting) install_handlers();
6168}
6169
6170static void gc_sweep_compact(rb_objspace_t *objspace);
6171
6172static void
6173gc_sweep(rb_objspace_t *objspace)
6174{
6175 gc_sweeping_enter(objspace);
6176
6177 const unsigned int immediate_sweep = objspace->flags.immediate_sweep;
6178
6179 gc_report(1, objspace, "gc_sweep: immediate: %d\n", immediate_sweep);
6180
6181 gc_sweep_start(objspace);
6182 if (objspace->flags.during_compacting) {
6183 rb_hrtime_t compact_start_time = gc_prof_enabled(objspace) ? rb_hrtime_now() : 0;
6184 gc_sweep_compact(objspace);
6185 if (gc_prof_enabled(objspace)) {
6186 rb_hrtime_t compact_wall_time = elapsed_hrtime_from(compact_start_time);
6187 gc_profile_record *record = gc_prof_record(objspace);
6188 record->gc_compact_wall_time = rb_hrtime_add(record->gc_compact_wall_time,
6189 compact_wall_time);
6190 objspace->profile.gc_sweep_excluded_wall_time = rb_hrtime_add(
6191 objspace->profile.gc_sweep_excluded_wall_time,
6192 compact_wall_time);
6193 }
6194 }
6195
6196 if (immediate_sweep) {
6197#if !GC_ENABLE_LAZY_SWEEP
6198 gc_prof_sweep_timer_start(objspace);
6199#endif
6200 gc_sweep_rest(objspace);
6201#if !GC_ENABLE_LAZY_SWEEP
6202 gc_prof_sweep_timer_stop(objspace);
6203#endif
6204 }
6205 else {
6206
6207 /* Sweep every size pool. */
6208 for (int i = 0; i < HEAP_COUNT; i++) {
6209 rb_heap_t *heap = &heaps[i];
6210 gc_sweep_step(objspace, heap);
6211 }
6212 }
6213
6214 gc_sweeping_exit(objspace);
6215}
6216
6217/* Marking - Marking stack */
6218
6219static stack_chunk_t *
6220stack_chunk_alloc(void)
6221{
6222 stack_chunk_t *res;
6223
6224 res = malloc(sizeof(stack_chunk_t));
6225 if (!res)
6226 rb_memerror();
6227
6228 return res;
6229}
6230
6231static inline int
6232is_mark_stack_empty(mark_stack_t *stack)
6233{
6234 return stack->chunk == NULL;
6235}
6236
6237static size_t
6238mark_stack_size(mark_stack_t *stack)
6239{
6240 size_t size = stack->index;
6241 stack_chunk_t *chunk = stack->chunk ? stack->chunk->next : NULL;
6242
6243 while (chunk) {
6244 size += stack->limit;
6245 chunk = chunk->next;
6246 }
6247 return size;
6248}
6249
6250static void
6251add_stack_chunk_cache(mark_stack_t *stack, stack_chunk_t *chunk)
6252{
6253 chunk->next = stack->cache;
6254 stack->cache = chunk;
6255 stack->cache_size++;
6256}
6257
6258static void
6259shrink_stack_chunk_cache(mark_stack_t *stack)
6260{
6261 stack_chunk_t *chunk;
6262
6263 if (stack->unused_cache_size > (stack->cache_size/2)) {
6264 chunk = stack->cache;
6265 stack->cache = stack->cache->next;
6266 stack->cache_size--;
6267 free(chunk);
6268 }
6269 stack->unused_cache_size = stack->cache_size;
6270}
6271
6272static void
6273push_mark_stack_chunk(mark_stack_t *stack)
6274{
6275 stack_chunk_t *next;
6276
6277 GC_ASSERT(stack->index == stack->limit);
6278
6279 if (stack->cache_size > 0) {
6280 next = stack->cache;
6281 stack->cache = stack->cache->next;
6282 stack->cache_size--;
6283 if (stack->unused_cache_size > stack->cache_size)
6284 stack->unused_cache_size = stack->cache_size;
6285 }
6286 else {
6287 next = stack_chunk_alloc();
6288 }
6289 next->next = stack->chunk;
6290 stack->chunk = next;
6291 stack->index = 0;
6292}
6293
6294static void
6295pop_mark_stack_chunk(mark_stack_t *stack)
6296{
6297 stack_chunk_t *prev;
6298
6299 prev = stack->chunk->next;
6300 GC_ASSERT(stack->index == 0);
6301 add_stack_chunk_cache(stack, stack->chunk);
6302 stack->chunk = prev;
6303 stack->index = stack->limit;
6304}
6305
6306static void
6307mark_stack_chunk_list_free(stack_chunk_t *chunk)
6308{
6309 stack_chunk_t *next = NULL;
6310
6311 while (chunk != NULL) {
6312 next = chunk->next;
6313 free(chunk);
6314 chunk = next;
6315 }
6316}
6317
6318static void
6319free_stack_chunks(mark_stack_t *stack)
6320{
6321 mark_stack_chunk_list_free(stack->chunk);
6322}
6323
6324static void
6325mark_stack_free_cache(mark_stack_t *stack)
6326{
6327 mark_stack_chunk_list_free(stack->cache);
6328 stack->cache_size = 0;
6329 stack->unused_cache_size = 0;
6330}
6331
6332static void
6333push_mark_stack(mark_stack_t *stack, VALUE obj)
6334{
6335 switch (BUILTIN_TYPE(obj)) {
6336 case T_OBJECT:
6337 case T_CLASS:
6338 case T_MODULE:
6339 case T_FLOAT:
6340 case T_STRING:
6341 case T_REGEXP:
6342 case T_ARRAY:
6343 case T_HASH:
6344 case T_STRUCT:
6345 case T_BIGNUM:
6346 case T_FILE:
6347 case T_DATA:
6348 case T_MATCH:
6349 case T_COMPLEX:
6350 case T_RATIONAL:
6351 case T_TRUE:
6352 case T_FALSE:
6353 case T_SYMBOL:
6354 case T_IMEMO:
6355 case T_ICLASS:
6356 if (stack->index == stack->limit) {
6357 push_mark_stack_chunk(stack);
6358 }
6359 stack->chunk->data[stack->index++] = obj;
6360 return;
6361
6362 case T_NONE:
6363 case T_NIL:
6364 case T_FIXNUM:
6365 case T_MOVED:
6366 case T_ZOMBIE:
6367 case T_UNDEF:
6368 case T_MASK:
6369 rb_bug("push_mark_stack() called for broken object");
6370 break;
6371
6372 case T_NODE:
6373 rb_bug("push_mark_stack: unexpected T_NODE object");
6374 break;
6375 }
6376
6377 rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s",
6378 BUILTIN_TYPE(obj), (void *)obj,
6379 is_pointer_to_heap((rb_objspace_t *)rb_gc_get_objspace(), (void *)obj) ? "corrupted object" : "non object");
6380}
6381
6382static int
6383pop_mark_stack(mark_stack_t *stack, VALUE *data)
6384{
6385 if (is_mark_stack_empty(stack)) {
6386 return FALSE;
6387 }
6388 if (stack->index == 1) {
6389 *data = stack->chunk->data[--stack->index];
6390 pop_mark_stack_chunk(stack);
6391 }
6392 else {
6393 *data = stack->chunk->data[--stack->index];
6394 }
6395 return TRUE;
6396}
6397
6398static void
6399init_mark_stack(mark_stack_t *stack)
6400{
6401 int i;
6402
6403 MEMZERO(stack, mark_stack_t, 1);
6404 stack->index = stack->limit = STACK_CHUNK_SIZE;
6405
6406 for (i=0; i < 4; i++) {
6407 add_stack_chunk_cache(stack, stack_chunk_alloc());
6408 }
6409 stack->unused_cache_size = stack->cache_size;
6410}
6411
6412/* Marking */
6413
6414ALWAYS_INLINE(static int gc_mark_set(rb_objspace_t *objspace, VALUE obj));
6415ALWAYS_INLINE(static void gc_mark_check_t_none(rb_objspace_t *objspace, VALUE obj));
6416ALWAYS_INLINE(static void rgengc_check_relation(rb_objspace_t *objspace, VALUE obj));
6417ALWAYS_INLINE(static void gc_aging(rb_objspace_t *objspace, VALUE obj));
6418ALWAYS_INLINE(static void gc_grey(rb_objspace_t *objspace, VALUE obj));
6419static void
6420rgengc_check_relation(rb_objspace_t *objspace, VALUE obj)
6421{
6422 if (objspace->rgengc.parent_object_old_p) {
6423 if (!RVALUE_OLD_P(objspace, obj)) {
6424 rgengc_remember(objspace, objspace->rgengc.parent_object);
6425 /* It is in the rememberset now, so its remaining children have nothing left
6426 * to ask for: stop testing them. */
6427 objspace->rgengc.parent_object_old_p = false;
6428 }
6429 }
6430}
6431
6432static inline int
6433gc_mark_set(rb_objspace_t *objspace, VALUE obj)
6434{
6435 if (RVALUE_MARKED(objspace, obj)) return 0;
6436 MARK_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj);
6437 return 1;
6438}
6439
6440static void
6441gc_aging(rb_objspace_t *objspace, VALUE obj)
6442{
6443 /* Disable aging if Major GC's are disabled. This will prevent longish lived
6444 * objects filling up the heap at the expense of marking many more objects.
6445 *
6446 * We should always pre-warm our process when disabling majors, by running
6447 * GC manually several times so that most objects likely to become oldgen
6448 * are already oldgen.
6449 */
6450 if(!gc_config_full_mark_val)
6451 return;
6452
6453 struct heap_page *page = GET_HEAP_PAGE(obj);
6454
6455 GC_ASSERT(RVALUE_MARKING(objspace, obj) == FALSE);
6456 check_rvalue_consistency(objspace, obj);
6457
6458 if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) {
6459 if (!RVALUE_OLD_P(objspace, obj)) {
6460 int t = BUILTIN_TYPE(obj);
6461 if (t == T_CLASS || t == T_MODULE || t == T_ICLASS) {
6462 gc_report(3, objspace, "gc_aging: YOUNG class: %s\n", rb_obj_info(obj));
6463 RVALUE_AGE_SET(obj, RVALUE_OLD_AGE);
6464 RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
6465 }
6466 else {
6467 gc_report(3, objspace, "gc_aging: YOUNG: %s\n", rb_obj_info(obj));
6468 RVALUE_AGE_INC(objspace, obj);
6469 }
6470 }
6471 else if (is_full_marking(objspace)) {
6472 GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE);
6473 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, page, obj);
6474 }
6475 }
6476 check_rvalue_consistency(objspace, obj);
6477
6478 page->objspace->marked_slots++;
6479}
6480
6481static void
6482gc_grey(rb_objspace_t *objspace, VALUE obj)
6483{
6484#if RGENGC_CHECK_MODE
6485 if (RVALUE_MARKED(objspace, obj) == FALSE) rb_bug("gc_grey: %s is not marked.", rb_obj_info(obj));
6486 if (RVALUE_MARKING(objspace, obj) == TRUE) rb_bug("gc_grey: %s is marking/remembered.", rb_obj_info(obj));
6487#endif
6488
6489 if (is_incremental_marking(objspace)) {
6490 MARK_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
6491 }
6492
6494 rb_darray_append_without_gc(&objspace->weak_references, obj);
6495 }
6496
6497 push_mark_stack(&objspace->mark_stack, obj);
6498}
6499
6500static inline void
6501gc_mark_check_t_none(rb_objspace_t *objspace, VALUE obj)
6502{
6503 if (RB_UNLIKELY(BUILTIN_TYPE(obj) == T_NONE)) {
6504 enum {info_size = 256};
6505 char obj_info_buf[info_size];
6506 rb_raw_obj_info(obj_info_buf, info_size, obj);
6507
6508 char parent_obj_info_buf[info_size];
6509 rb_raw_obj_info(parent_obj_info_buf, info_size, objspace->rgengc.parent_object);
6510
6511 rb_bug("try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf);
6512 }
6513}
6514
6515static void
6516gc_mark(rb_objspace_t *objspace, VALUE obj)
6517{
6518 GC_ASSERT(during_gc);
6519 GC_ASSERT(!objspace->flags.during_reference_updating);
6520
6521 /* Never step into another objspace: a foreign object is a live leaf whose liveness
6522 * belongs to its owner, so touching its bitmaps here would be unsound. A global GC
6523 * lifts this: everyone is stopped and the bits live on the object's own page. */
6524 if (gc_skip_foreign_object_p(objspace, obj)) {
6525 return;
6526 }
6527
6528 if (RB_UNLIKELY(objspace->flags.during_global_gc)) {
6529 /* Recompute the shref of every shareable -> unshareable edge, within and across
6530 * objspaces: the clear pass dropped all shref bits and the write barrier
6531 * maintains them from here on. */
6532 VALUE parent = objspace->rgengc.parent_object;
6533 if (!UNDEF_P(parent) && parent != Qfalse &&
6536 struct heap_page *page = GET_HEAP_PAGE(obj);
6537 _MARK_IN_BITMAP(page->shref_bits, page, obj);
6538 page->flags.has_shref_objects = TRUE;
6539 }
6540 }
6541
6542 rgengc_check_relation(objspace, obj);
6543 if (!gc_mark_set(objspace, obj)) return; /* already marked */
6544
6545 if (0) { // for debug GC marking miss
6546 RUBY_DEBUG_LOG("%p (%s) parent:%p (%s)",
6547 (void *)obj, obj_type_name(obj),
6548 (void *)objspace->rgengc.parent_object, obj_type_name(objspace->rgengc.parent_object));
6549 }
6550
6551 gc_mark_check_t_none(objspace, obj);
6552
6553 gc_aging(objspace, obj);
6554 gc_grey(objspace, obj);
6555}
6556
6557static inline void
6558gc_pin(rb_objspace_t *objspace, VALUE obj)
6559{
6560 GC_ASSERT(!SPECIAL_CONST_P(obj));
6561
6562 if (RB_UNLIKELY(objspace->flags.during_compacting)) {
6563 /* Never write a foreign page's pinned bit (a global GC may: everyone is stopped). */
6564 if (gc_skip_foreign_object_p(objspace, obj)) return;
6565
6566 if (RB_LIKELY(during_gc)) {
6567 if (!RVALUE_PINNED(objspace, obj)) {
6568 GC_ASSERT(GET_HEAP_PAGE(obj)->pinned_slots <= GET_HEAP_PAGE(obj)->total_slots);
6569 GET_HEAP_PAGE(obj)->pinned_slots++;
6570 MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj);
6571 }
6572 }
6573 }
6574}
6575
6576static inline void
6577gc_mark_and_pin(rb_objspace_t *objspace, VALUE obj)
6578{
6579 gc_pin(objspace, obj);
6580 gc_mark(objspace, obj);
6581}
6582
6583void
6584rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr)
6585{
6586 rb_objspace_t *objspace = objspace_ptr;
6587
6588 if (RB_UNLIKELY(objspace->flags.during_reference_updating)) {
6589 GC_ASSERT(objspace->flags.during_compacting);
6590 GC_ASSERT(during_gc);
6591
6592 VALUE destination = rb_gc_impl_location(objspace, *ptr);
6593 if (destination != *ptr) {
6594 *ptr = destination;
6595 }
6596 }
6597 else {
6598 gc_mark(objspace, *ptr);
6599 }
6600}
6601
6602void
6603rb_gc_impl_mark(void *objspace_ptr, VALUE obj)
6604{
6605 rb_objspace_t *objspace = objspace_ptr;
6606
6607 gc_mark(objspace, obj);
6608}
6609
6610void
6611rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj)
6612{
6613 rb_objspace_t *objspace = objspace_ptr;
6614
6615 gc_mark_and_pin(objspace, obj);
6616}
6617
6618/* A word scanned conservatively by a global GC can point into any objspace, so ownership
6619 * is decided against the driver's snapshot of every objspace (the bits then land on the
6620 * owner's page through gc_mark and gc_pin). */
6621static bool
6622gc_global_pointer_to_heap_p(const void *ptr)
6623{
6624 const rb_global_objspace_t *g = global_objspace;
6625 uintptr_t p = (uintptr_t)ptr;
6626
6627 if (p < g->page_index.lomem || p > g->page_index.himem) return false;
6628 if (p % sizeof(VALUE) != 0) return false;
6629
6630 struct heap_page **res = bsearch(ptr, g->page_index.pages, g->page_index.n_pages,
6631 sizeof(struct heap_page *), ptr_in_page_body_p);
6632 if (res == NULL) return false;
6633
6634 struct heap_page *page = *res;
6635 if (heap_page_in_global_empty_pages_pool(page->objspace, page)) return false;
6636 if (p < page->start) return false;
6637 if (p >= page->start + (page->total_slots * page->slot_size)) return false;
6638 if (!slot_offset_aligned_p(page, p)) return false;
6639 return true;
6640}
6641
6642void
6643rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj)
6644{
6645 rb_objspace_t *objspace = objspace_ptr;
6646
6647 (void)VALGRIND_MAKE_MEM_DEFINED(&obj, sizeof(obj));
6648
6649 if (RB_UNLIKELY(objspace->flags.during_global_gc)
6650 ? gc_global_pointer_to_heap_p((void *)obj)
6651 : is_pointer_to_heap(objspace, (void *)obj)) {
6652 asan_unpoisoning_object(obj) {
6653 /* Garbage can live on the stack, so do not mark or pin */
6654 switch (BUILTIN_TYPE(obj)) {
6655 case T_ZOMBIE:
6656 case T_NONE:
6657 break;
6658 default:
6659 gc_mark_and_pin(objspace, obj);
6660 break;
6661 }
6662 }
6663 }
6664}
6665
6666static int
6667pin_value(st_data_t key, st_data_t value, st_data_t data)
6668{
6669 rb_gc_impl_mark_and_pin((void *)data, (VALUE)value);
6670
6671 return ST_CONTINUE;
6672}
6673
6674static inline void
6675gc_mark_set_parent_raw(rb_objspace_t *objspace, VALUE obj, bool old_p)
6676{
6677 asan_unpoison_memory_region(&objspace->rgengc.parent_object, sizeof(objspace->rgengc.parent_object), false);
6678 asan_unpoison_memory_region(&objspace->rgengc.parent_object_old_p, sizeof(objspace->rgengc.parent_object_old_p), false);
6679 objspace->rgengc.parent_object = obj;
6680 objspace->rgengc.parent_object_old_p = old_p;
6681}
6682
6683static inline void
6684gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj)
6685{
6686 gc_mark_set_parent_raw(objspace, obj, RVALUE_OLD_P(objspace, obj));
6687}
6688
6689static inline void
6690gc_mark_set_parent_invalid(rb_objspace_t *objspace)
6691{
6692 asan_poison_memory_region(&objspace->rgengc.parent_object, sizeof(objspace->rgengc.parent_object));
6693 asan_poison_memory_region(&objspace->rgengc.parent_object_old_p, sizeof(objspace->rgengc.parent_object_old_p));
6694}
6695
6696static void pinned_roots_mark(rb_objspace_t *objspace, rb_heap_t *heap);
6697
6698static void
6699mark_roots(rb_objspace_t *objspace, const char **categoryp)
6700{
6701 VALUE objspace_guard = (VALUE)objspace;
6702#define MARK_CHECKPOINT(category) do { \
6703 if (categoryp) *categoryp = category; \
6704} while (0)
6705
6706 /* Pinning shareable objects and shrefs runs at the end of marking (gc_marks_finish),
6707 * not here: after the full walk it only has to touch what ordinary marking missed,
6708 * which is both cheap and a useful retention metric. */
6709
6710 MARK_CHECKPOINT("objspace");
6711 gc_mark_set_parent_raw(objspace, Qundef, false);
6712
6713 if (objspace->flags.during_global_gc) {
6714 /* Pin the finalizer tables of every objspace, zombies included.
6715 * (finalizer_table is a macro over the local "objspace".) */
6716 rb_objspace_t *const driver = objspace;
6717 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
6718 rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
6719 if (finalizer_table != NULL) {
6720 st_foreach(finalizer_table, pin_value, (st_data_t)driver);
6721 }
6722 }
6723 }
6724 else if (finalizer_table != NULL) {
6725 st_foreach(finalizer_table, pin_value, (st_data_t)objspace);
6726 }
6727
6728 if (stress_to_class) rb_gc_mark(stress_to_class);
6729
6730 rb_gc_save_machine_context();
6731 rb_gc_mark_roots(objspace, categoryp);
6732 /* Keep this frame, including its saved registers, until root marking has
6733 * scanned the machine stack. */
6734 RB_GC_GUARD(objspace_guard);
6735 gc_mark_set_parent_invalid(objspace);
6736}
6737
6738static void
6739gc_mark_children(rb_objspace_t *objspace, VALUE obj)
6740{
6741 gc_mark_set_parent(objspace, obj);
6742 rb_gc_mark_children(objspace, obj);
6743 gc_mark_set_parent_invalid(objspace);
6744}
6745
6750static inline int
6751gc_mark_stacked_objects(rb_objspace_t *objspace, int incremental, size_t count)
6752{
6753 mark_stack_t *mstack = &objspace->mark_stack;
6754 VALUE obj;
6755 size_t marked_slots_at_the_beginning = objspace->marked_slots;
6756 size_t popped_count = 0;
6757
6758 while (pop_mark_stack(mstack, &obj)) {
6759 if (obj == Qundef) continue; /* skip */
6760
6761 if (RGENGC_CHECK_MODE && !RVALUE_MARKED(objspace, obj)) {
6762 rb_bug("gc_mark_stacked_objects: %s is not marked.", rb_obj_info(obj));
6763 }
6764 gc_mark_children(objspace, obj);
6765
6766 popped_count++;
6767
6768 if (incremental) {
6769 if (RGENGC_CHECK_MODE && !RVALUE_MARKING(objspace, obj)) {
6770 rb_bug("gc_mark_stacked_objects: incremental, but marking bit is 0");
6771 }
6772 CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
6773
6774 if (popped_count + (objspace->marked_slots - marked_slots_at_the_beginning) > count) {
6775 break;
6776 }
6777 }
6778 else {
6779 /* just ignore marking bits */
6780 }
6781 }
6782
6783 RUBY_DTRACE_GC_HOOK(MARK_STACKED_OBJECTS, popped_count);
6784
6785 if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace);
6786
6787 if (is_mark_stack_empty(mstack)) {
6788 shrink_stack_chunk_cache(mstack);
6789 return TRUE;
6790 }
6791 else {
6792 return FALSE;
6793 }
6794}
6795
6796static int
6797gc_mark_stacked_objects_incremental(rb_objspace_t *objspace, size_t count)
6798{
6799 return gc_mark_stacked_objects(objspace, TRUE, count);
6800}
6801
6802static int
6803gc_mark_stacked_objects_all(rb_objspace_t *objspace)
6804{
6805 return gc_mark_stacked_objects(objspace, FALSE, 0);
6806}
6807
6808#if RGENGC_CHECK_MODE >= 4
6809
6810#define MAKE_ROOTSIG(obj) (((VALUE)(obj) << 1) | 0x01)
6811#define IS_ROOTSIG(obj) ((VALUE)(obj) & 0x01)
6812#define GET_ROOTSIG(obj) ((const char *)((VALUE)(obj) >> 1))
6813
6814struct reflist {
6815 VALUE *list;
6816 int pos;
6817 int size;
6818};
6819
6820static struct reflist *
6821reflist_create(VALUE obj)
6822{
6823 struct reflist *refs = xmalloc(sizeof(struct reflist));
6824 refs->size = 1;
6825 refs->list = ALLOC_N(VALUE, refs->size);
6826 refs->list[0] = obj;
6827 refs->pos = 1;
6828 return refs;
6829}
6830
6831static void
6832reflist_destruct(struct reflist *refs)
6833{
6834 xfree(refs->list);
6835 xfree(refs);
6836}
6837
6838static void
6839reflist_add(struct reflist *refs, VALUE obj)
6840{
6841 if (refs->pos == refs->size) {
6842 refs->size *= 2;
6843 SIZED_REALLOC_N(refs->list, VALUE, refs->size, refs->size/2);
6844 }
6845
6846 refs->list[refs->pos++] = obj;
6847}
6848
6849static void
6850reflist_dump(struct reflist *refs)
6851{
6852 int i;
6853 for (i=0; i<refs->pos; i++) {
6854 VALUE obj = refs->list[i];
6855 if (IS_ROOTSIG(obj)) { /* root */
6856 fprintf(stderr, "<root@%s>", GET_ROOTSIG(obj));
6857 }
6858 else {
6859 fprintf(stderr, "<%s>", rb_obj_info(obj));
6860 }
6861 if (i+1 < refs->pos) fprintf(stderr, ", ");
6862 }
6863}
6864
6865static int
6866reflist_referred_from_machine_context(struct reflist *refs)
6867{
6868 int i;
6869 for (i=0; i<refs->pos; i++) {
6870 VALUE obj = refs->list[i];
6871 if (IS_ROOTSIG(obj) && strcmp(GET_ROOTSIG(obj), "machine_context") == 0) return 1;
6872 }
6873 return 0;
6874}
6875
6876struct allrefs {
6877 rb_objspace_t *objspace;
6878 /* a -> obj1
6879 * b -> obj1
6880 * c -> obj1
6881 * c -> obj2
6882 * d -> obj3
6883 * #=> {obj1 => [a, b, c], obj2 => [c, d]}
6884 */
6885 struct st_table *references;
6886 const char *category;
6887 VALUE root_obj;
6888 mark_stack_t mark_stack;
6889};
6890
6891static int
6892allrefs_add(struct allrefs *data, VALUE obj)
6893{
6894 struct reflist *refs;
6895 st_data_t r;
6896
6897 if (st_lookup(data->references, obj, &r)) {
6898 refs = (struct reflist *)r;
6899 reflist_add(refs, data->root_obj);
6900 return 0;
6901 }
6902 else {
6903 refs = reflist_create(data->root_obj);
6904 st_insert(data->references, obj, (st_data_t)refs);
6905 return 1;
6906 }
6907}
6908
6909static void
6910allrefs_i(VALUE obj, void *ptr)
6911{
6912 struct allrefs *data = (struct allrefs *)ptr;
6913
6914 if (allrefs_add(data, obj)) {
6915 push_mark_stack(&data->mark_stack, obj);
6916 }
6917}
6918
6919static void
6920allrefs_roots_i(VALUE obj, void *ptr)
6921{
6922 struct allrefs *data = (struct allrefs *)ptr;
6923 if (strlen(data->category) == 0) rb_bug("!!!");
6924 data->root_obj = MAKE_ROOTSIG(data->category);
6925
6926 if (allrefs_add(data, obj)) {
6927 push_mark_stack(&data->mark_stack, obj);
6928 }
6929}
6930#define PUSH_MARK_FUNC_DATA(v) do { \
6931 struct gc_mark_func_data_struct *prev_mark_func_data = GET_VM()->gc.mark_func_data; \
6932 GET_VM()->gc.mark_func_data = (v);
6933
6934#define POP_MARK_FUNC_DATA() GET_VM()->gc.mark_func_data = prev_mark_func_data;} while (0)
6935
6936static st_table *
6937objspace_allrefs(rb_objspace_t *objspace)
6938{
6939 struct allrefs data;
6940 struct gc_mark_func_data_struct mfd;
6941 VALUE obj;
6942 int prev_dont_gc = dont_gc_val();
6943 dont_gc_on();
6944
6945 data.objspace = objspace;
6946 data.references = st_init_numtable();
6947 init_mark_stack(&data.mark_stack);
6948
6949 mfd.mark_func = allrefs_roots_i;
6950 mfd.data = &data;
6951
6952 /* traverse root objects */
6953 PUSH_MARK_FUNC_DATA(&mfd);
6954 GET_VM()->gc.mark_func_data = &mfd;
6955 mark_roots(objspace, &data.category);
6956 POP_MARK_FUNC_DATA();
6957
6958 /* traverse rest objects reachable from root objects */
6959 while (pop_mark_stack(&data.mark_stack, &obj)) {
6960 rb_objspace_reachable_objects_from_unlocked(data.root_obj = obj, allrefs_i, &data);
6961 }
6962 free_stack_chunks(&data.mark_stack);
6963
6964 dont_gc_set(prev_dont_gc);
6965 return data.references;
6966}
6967
6968static int
6969objspace_allrefs_destruct_i(st_data_t key, st_data_t value, st_data_t ptr)
6970{
6971 struct reflist *refs = (struct reflist *)value;
6972 reflist_destruct(refs);
6973 return ST_CONTINUE;
6974}
6975
6976static void
6977objspace_allrefs_destruct(struct st_table *refs)
6978{
6979 st_foreach(refs, objspace_allrefs_destruct_i, 0);
6980 st_free_table(refs);
6981}
6982
6983#if RGENGC_CHECK_MODE >= 5
6984static int
6985allrefs_dump_i(st_data_t k, st_data_t v, st_data_t ptr)
6986{
6987 VALUE obj = (VALUE)k;
6988 struct reflist *refs = (struct reflist *)v;
6989 fprintf(stderr, "[allrefs_dump_i] %s <- ", rb_obj_info(obj));
6990 reflist_dump(refs);
6991 fprintf(stderr, "\n");
6992 return ST_CONTINUE;
6993}
6994
6995static void
6996allrefs_dump(rb_objspace_t *objspace)
6997{
6998 VALUE size = objspace->rgengc.allrefs_table->num_entries;
6999 fprintf(stderr, "[all refs] (size: %"PRIuVALUE")\n", size);
7000 st_foreach(objspace->rgengc.allrefs_table, allrefs_dump_i, 0);
7001}
7002#endif
7003
7004static int
7005gc_check_after_marks_i(st_data_t k, st_data_t v, st_data_t ptr)
7006{
7007 VALUE obj = k;
7008 struct reflist *refs = (struct reflist *)v;
7009 rb_objspace_t *objspace = (rb_objspace_t *)ptr;
7010
7011 /* object should be marked or oldgen */
7012 if (!RVALUE_MARKED(objspace, obj)) {
7013 fprintf(stderr, "gc_check_after_marks_i: %s is not marked and not oldgen.\n", rb_obj_info(obj));
7014 fprintf(stderr, "gc_check_after_marks_i: %p is referred from ", (void *)obj);
7015 reflist_dump(refs);
7016
7017 if (reflist_referred_from_machine_context(refs)) {
7018 fprintf(stderr, " (marked from machine stack).\n");
7019 /* marked from machine context can be false positive */
7020 }
7021 else {
7022 objspace->rgengc.error_count++;
7023 fprintf(stderr, "\n");
7024 }
7025 }
7026 return ST_CONTINUE;
7027}
7028
7029static void
7030gc_marks_check(rb_objspace_t *objspace, st_foreach_callback_func *checker_func, const char *checker_name)
7031{
7032 MALLOC_COUNTERS_LOCK(objspace);
7033 struct gc_malloc_bytes saved_malloc = {
7034 .malloc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc),
7035 .free = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free),
7036 .malloc_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc_at_last_gc),
7037 .free_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free_at_last_gc),
7038 };
7039#if RGENGC_ESTIMATE_OLDMALLOC
7040 size_t saved_oldmalloc_increase = gc_oldmalloc_increase_load(objspace);
7041 gc_counter_t saved_freed_in_gc = gc_counter_load_relaxed(&objspace->malloc_counters.freed_in_gc);
7042#endif
7043 MALLOC_COUNTERS_UNLOCK(objspace);
7044 VALUE already_disabled = rb_objspace_gc_disable(objspace);
7045
7046 objspace->rgengc.allrefs_table = objspace_allrefs(objspace);
7047
7048 if (checker_func) {
7049 st_foreach(objspace->rgengc.allrefs_table, checker_func, (st_data_t)objspace);
7050 }
7051
7052 if (objspace->rgengc.error_count > 0) {
7053#if RGENGC_CHECK_MODE >= 5
7054 allrefs_dump(objspace);
7055#endif
7056 if (checker_name) rb_bug("%s: GC has problem.", checker_name);
7057 }
7058
7059 objspace_allrefs_destruct(objspace->rgengc.allrefs_table);
7060 objspace->rgengc.allrefs_table = 0;
7061
7062 if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
7063 MALLOC_COUNTERS_LOCK(objspace);
7064 gc_counter_store_release(&objspace->malloc_counters.counters.malloc, saved_malloc.malloc);
7065 gc_counter_store_release(&objspace->malloc_counters.counters.free, saved_malloc.free);
7066 gc_counter_store_release(&objspace->malloc_counters.counters.malloc_at_last_gc, saved_malloc.malloc_at_last_gc);
7067 gc_counter_store_release(&objspace->malloc_counters.counters.free_at_last_gc, saved_malloc.free_at_last_gc);
7068#if RGENGC_ESTIMATE_OLDMALLOC
7069 RUBY_ATOMIC_SIZE_EXCHANGE(objspace->malloc_counters.oldmalloc_increase, saved_oldmalloc_increase);
7070 gc_counter_store_release(&objspace->malloc_counters.freed_in_gc, saved_freed_in_gc);
7071#endif
7072 MALLOC_COUNTERS_UNLOCK(objspace);
7073}
7074#endif /* RGENGC_CHECK_MODE >= 4 */
7075
7077 rb_objspace_t *objspace;
7078 /* True only while the world is stopped: a GC.verify holding the VM lock and barrier,
7079 * or a global GC. Cross-objspace checks (walking every objspace's pages) are sound
7080 * only then. */
7081 bool world_stopped;
7082 int err_count;
7083 size_t live_object_count;
7084 size_t zombie_object_count;
7085
7086 VALUE parent;
7087 bool parent_shareable;
7088 size_t old_object_count;
7089 size_t remembered_shady_count;
7090};
7091
7092
7093static void
7094check_generation_i(const VALUE child, void *ptr)
7095{
7097 const VALUE parent = data->parent;
7098
7099 if (RGENGC_CHECK_MODE) GC_ASSERT(RVALUE_OLD_P(data->objspace, parent));
7100
7101 /* A cross-objspace edge is kept alive by the shareable/shref mechanism and is not
7102 * tracked in this objspace's remembered set. */
7103 if (GET_HEAP_OBJSPACE(child) != data->objspace) return;
7104
7105 /* Once the process goes multi-Ractor, the shareable world is managed by pinning and
7106 * shrefs rather than by the remembered set: the pinned walk at the end of a mark
7107 * re-marks every shareable object (and its shref'd children) each local cycle, and a
7108 * global GC rebuilds the generation state. So the generational old->young invariant
7109 * does not hold when either endpoint is shareable: an old constcache, cc_table or
7110 * interned string pointing at a core class that is young after a global GC is the
7111 * typical false positive. That state outlives the return to a single Ractor until
7112 * the next major (an old shareable singleton class pointing at a young
7113 * attached_object, say), so the test uses rb_gc_ever_multi_ractor_p(), which stays
7114 * true forever once multiple Ractors existed. A program that never goes multi keeps
7115 * the strict check, and ASAN catches what is left. */
7116 if (rb_gc_ever_multi_ractor_p() &&
7117 (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(parent), parent) ||
7118 MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(child), child))) {
7119 return;
7120 }
7121
7122 if (!RVALUE_OLD_P(data->objspace, child)) {
7123 /* A young shareable child is pinned and kept alive by the local GC (only a
7124 * global GC collects it), so it survives even when the old parent does not
7125 * remember it. It is outside the generational remembered set, so exclude it
7126 * from the old->young check. */
7127 if (!RVALUE_REMEMBERED(data->objspace, parent) &&
7128 !RVALUE_REMEMBERED(data->objspace, child) &&
7129 !RVALUE_UNCOLLECTIBLE(data->objspace, child) &&
7131 fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (O->Y) %s -> %s\n", rb_obj_info(parent), rb_obj_info(child));
7132 data->err_count++;
7133 }
7134 }
7135}
7136
7137static void
7138check_color_i(const VALUE child, void *ptr)
7139{
7141 const VALUE parent = data->parent;
7142
7143 /* This cycle never marks a foreign child (gc_skip_foreign_object_p) and the write
7144 * barrier is a no-op across objspaces, so its colour says nothing here. */
7145 if (GET_HEAP_OBJSPACE(child) != data->objspace) return;
7146
7147 if (!RVALUE_WB_UNPROTECTED(data->objspace, parent) && RVALUE_WHITE_P(data->objspace, child)) {
7148 fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (B->W) - %s -> %s\n",
7149 rb_obj_info(parent), rb_obj_info(child));
7150 data->err_count++;
7151 }
7152}
7153
7154static void
7155check_children_i(const VALUE child, void *ptr)
7156{
7158
7159 /* Fast path: a child in this objspace (99.99% of all edges). */
7160 if (RB_LIKELY(is_pointer_to_heap(data->objspace, (void *)child))) {
7161 if (check_rvalue_consistency_force(data->objspace, child, FALSE) != 0) {
7162 fprintf(stderr, "check_children_i: %s has error (referenced from %s)\n",
7163 rb_obj_info(child), rb_obj_info(data->parent));
7164 data->err_count++;
7165 }
7166 return;
7167 }
7168
7169 /* The remaining cross-objspace check (verify_pointer_in_any_heap_p) walks every
7170 * objspace's pages, sound only with the world stopped: mid-local-GC other Ractors
7171 * change page structures concurrently. The next world-stopped verify re-checks. */
7172 if (!data->world_stopped) return;
7173
7174 /* A non-heap child reaches this callback only when a stale field was followed by a
7175 * plain rb_gc_mark (the dmark of a live but unreachable wrapper, say). Report it and
7176 * keep going rather than aborting. */
7177 if (!verify_pointer_in_any_heap_p((void *)child)) {
7178 /* The graph is in flux mid-merge, so a transient non-heap edge is expected; it
7179 * is re-checked after the merge. */
7180 if (global_objspace->during_absorb) return;
7181 fprintf(stderr, "VERIFY-NOTE: non-heap child %p (from %s)\n",
7182 (void *)child, rb_obj_info(data->parent));
7183 return;
7184 }
7185
7186 if (GET_HEAP_OBJSPACE(child) != data->objspace) {
7187 /* A legal cross-objspace edge either starts at a shareable object or is recorded
7188 * in the child's shref bit (an in-flight send or move payload kept alive across
7189 * its owner's local GC; root_scope_check_i honours the same record). An
7190 * unshareable parent holding an unrecorded foreign unshareable child would be
7191 * invisible to both local GCs. The exception is a box's top_self, which every
7192 * thread's th->top_self points at and which is VM-permanent. Skipped during a
7193 * global GC: it clears every shref bit, so the shref exemption would not fire,
7194 * and its unified exact stop-the-world mark makes the invariant itself moot. */
7195 if (!data->parent_shareable &&
7196 child != rb_gc_vm_top_self() &&
7197 !MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(child), child) &&
7198 !MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(child), child) &&
7199 !rb_gc_impl_during_global_gc_p(data->objspace) &&
7200 !global_objspace->during_absorb) {
7201 fprintf(stderr, "check_children_i: containment violation: "
7202 "unshareable %s (objspace %p) -> foreign unshareable %s (objspace %p)\n",
7203 rb_obj_info(data->parent), (void *)data->objspace,
7204 rb_obj_info(child), (void *)GET_HEAP_OBJSPACE(child));
7205 data->err_count++;
7206 }
7207
7208 /* The remaining per-objspace sanity rules belong to the owner. */
7209 return;
7210 }
7211}
7212
7213/* Whether a heap slot currently holds a live object. Returns false for empty
7214 * (T_NONE), moved (T_MOVED), and zombie (T_ZOMBIE) slots, and for garbage
7215 * objects about to be swept. */
7216static bool
7217gc_slot_live_object_p(rb_objspace_t *objspace, VALUE obj)
7218{
7219 switch (BUILTIN_TYPE(obj)) {
7220 case T_NONE:
7221 case T_MOVED:
7222 case T_ZOMBIE:
7223 return false;
7224 default:
7225 return !rb_gc_impl_garbage_object_p(objspace, obj);
7226 }
7227}
7228
7229/* Verifier only: does ptr point at a live slot in any objspace? The caller holds the VM
7230 * lock and the barrier, so page_index is stable. */
7231static bool
7232verify_pointer_in_any_heap_p(const void *ptr)
7233{
7234 return gc_global_pointer_to_heap_p(ptr);
7235}
7236
7237/* An exact root of the calling Ractor may only point at a shareable object, its own
7238 * objspace, or an in-flight payload with a recorded shref. Exempt: the conservative
7239 * machine scan (stale slots) and the VM-global containers that are cross-rooted by
7240 * design (every objspace scans them; the marker skips foreign entries). */
7241static void
7242root_scope_check_i(const char *category, VALUE obj, void *ptr)
7243{
7244 struct verify_internal_consistency_struct *data = ptr;
7245
7246 if (RB_SPECIAL_CONST_P(obj)) return;
7247 /* This check walks every objspace (verify_pointer_in_any_heap_p), so it is sound
7248 * only with the world stopped; a mid-local-GC verify races with other Ractors'
7249 * lock-free allocation. */
7250 if (!data->world_stopped) return;
7251 /* Mid-merge the VM-global root tables still point at the unmerged source (transient
7252 * non-heap or foreign roots); re-checked after the merge. */
7253 if (global_objspace->during_absorb) return;
7254 if (strcmp(category, "machine_context") == 0 ||
7255 strcmp(category, "vm_registered_objects") == 0 ||
7256 strcmp(category, "end_proc") == 0 ||
7257 strcmp(category, "trap_list") == 0) {
7258 return;
7259 }
7260
7261 if (!verify_pointer_in_any_heap_p((void *)obj)) {
7262 fprintf(stderr, "root_scope_check_i: root category \"%s\" names a non-heap pointer %p\n",
7263 category, (void *)obj);
7264 data->err_count++;
7265 return;
7266 }
7267
7268 if (GET_HEAP_OBJSPACE(obj) == data->objspace) return;
7269 if (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj)) return;
7270 if (MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj)) return;
7271 if (obj == rb_gc_vm_top_self()) return; /* VM-permanent (see check_children_i) */
7272
7273 fprintf(stderr, "root_scope_check_i: root category \"%s\" names a foreign "
7274 "unshareable without a shref record: %s (owner %p, self %p)\n",
7275 category, rb_obj_info(obj),
7276 (void *)GET_HEAP_OBJSPACE(obj), (void *)data->objspace);
7277 data->err_count++;
7278}
7279
7280static int
7281verify_internal_consistency_i(void *page_start, void *page_end, size_t stride,
7283{
7284 VALUE obj;
7285 rb_objspace_t *objspace = data->objspace;
7286
7287 for (obj = (VALUE)page_start; obj != (VALUE)page_end; obj += stride) {
7288 asan_unpoisoning_object(obj) {
7289 bool sh_bit = MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj) != 0;
7290 bool sr_bit = MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj) != 0;
7291
7292 if (gc_slot_live_object_p(objspace, obj)) {
7293 /* count objects */
7294 data->live_object_count++;
7295 data->parent = obj;
7296 data->parent_shareable = sh_bit;
7297
7298 /* Bitmap invariants: a page's shareable bit matches FL_SHAREABLE
7299 * exactly, and a shref record only ever points at an unshareable
7300 * object. */
7301 if (sh_bit != !!RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE)) {
7302 fprintf(stderr, "verify_internal_consistency_i: shareable bit %d "
7303 "disagrees with FL_SHAREABLE on %s\n", (int)sh_bit, rb_obj_info(obj));
7304 data->err_count++;
7305 }
7306 if (sr_bit && sh_bit) {
7307 fprintf(stderr, "verify_internal_consistency_i: shref bit on a shareable: %s\n",
7308 rb_obj_info(obj));
7309 data->err_count++;
7310 }
7311
7312 /* Normally, we don't expect T_MOVED objects to be in the heap.
7313 * But they can stay alive on the stack, */
7314 if (!gc_object_moved_p(objspace, obj)) {
7315 /* moved slots don't have children */
7316 rb_objspace_reachable_objects_from_unlocked(obj, check_children_i, (void *)data);
7317 }
7318
7319 /* check health of children */
7320 if (RVALUE_OLD_P(objspace, obj)) data->old_object_count++;
7321 if (RVALUE_WB_UNPROTECTED(objspace, obj) && RVALUE_UNCOLLECTIBLE(objspace, obj)) data->remembered_shady_count++;
7322
7323 if (!is_marking(objspace) && RVALUE_OLD_P(objspace, obj)) {
7324 /* reachable objects from an oldgen object should be old or (young with remember) */
7325 data->parent = obj;
7326 rb_objspace_reachable_objects_from_unlocked(obj, check_generation_i, (void *)data);
7327 }
7328
7329 if (!is_marking(objspace) && rb_gc_obj_shareable_p(obj)) {
7330 rb_gc_verify_shareable(obj);
7331 }
7332
7333 if (is_incremental_marking(objspace)) {
7334 if (RVALUE_BLACK_P(objspace, obj)) {
7335 /* reachable objects from black objects should be black or grey objects */
7336 data->parent = obj;
7337 rb_objspace_reachable_objects_from_unlocked(obj, check_color_i, (void *)data);
7338 }
7339 }
7340 }
7341 else {
7342 /* A freed slot must not carry its old pin bit into the next object born
7343 * there (a dead object not swept yet legitimately keeps it until the
7344 * sweep arrives). */
7345 if (BUILTIN_TYPE(obj) == T_NONE && (sh_bit || sr_bit)) {
7346 fprintf(stderr, "verify_internal_consistency_i: T_NONE slot carries "
7347 "shareable=%d shref=%d bits\n", (int)sh_bit, (int)sr_bit);
7348 data->err_count++;
7349 }
7350
7351 if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
7352 data->zombie_object_count++;
7353
7354 if ((RBASIC(obj)->flags & ~ZOMBIE_OBJ_KEPT_FLAGS) != T_ZOMBIE) {
7355 fprintf(stderr, "verify_internal_consistency_i: T_ZOMBIE has extra flags set: %s\n",
7356 rb_obj_info(obj));
7357 data->err_count++;
7358 }
7359
7360 if (!!FL_TEST(obj, FL_FINALIZE) != !!st_is_member(finalizer_table, obj)) {
7361 fprintf(stderr, "verify_internal_consistency_i: FL_FINALIZE %s but %s finalizer_table: %s\n",
7362 FL_TEST(obj, FL_FINALIZE) ? "set" : "not set", st_is_member(finalizer_table, obj) ? "in" : "not in",
7363 rb_obj_info(obj));
7364 data->err_count++;
7365 }
7366 }
7367 }
7368 }
7369 }
7370
7371 return 0;
7372}
7373
7374static int
7375gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
7376{
7377 unsigned int has_remembered_shady = FALSE;
7378 unsigned int has_remembered_old = FALSE;
7379 int remembered_old_objects = 0;
7380 int free_objects = 0;
7381 int zombie_objects = 0;
7382
7383 short slot_size = page->slot_size;
7384 uintptr_t start = (uintptr_t)page->start;
7385 uintptr_t end = start + page->total_slots * slot_size;
7386
7387 for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
7388 VALUE val = (VALUE)ptr;
7389 asan_unpoisoning_object(val) {
7390 enum ruby_value_type type = BUILTIN_TYPE(val);
7391
7392 if (type == T_NONE) free_objects++;
7393 if (type == T_ZOMBIE) zombie_objects++;
7394 if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) {
7395 has_remembered_shady = TRUE;
7396 }
7397 if (RVALUE_PAGE_MARKING(page, val)) {
7398 has_remembered_old = TRUE;
7399 remembered_old_objects++;
7400 }
7401 }
7402 }
7403
7404 if (!is_incremental_marking(objspace) &&
7405 page->flags.has_remembered_objects == FALSE && has_remembered_old == TRUE) {
7406
7407 for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
7408 VALUE val = (VALUE)ptr;
7409 if (RVALUE_PAGE_MARKING(page, val)) {
7410 fprintf(stderr, "marking -> %s\n", rb_obj_info(val));
7411 }
7412 }
7413 rb_bug("page %p's has_remembered_objects should be false, but there are remembered old objects (%d). %s",
7414 (void *)page, remembered_old_objects, obj ? rb_obj_info(obj) : "");
7415 }
7416
7417 if (page->flags.has_uncollectible_wb_unprotected_objects == FALSE && has_remembered_shady == TRUE) {
7418 rb_bug("page %p's has_remembered_shady should be false, but there are remembered shady objects. %s",
7419 (void *)page, obj ? rb_obj_info(obj) : "");
7420 }
7421
7422 if (0) {
7423 /* free_slots may not equal to free_objects */
7424 if (page->free_slots != free_objects) {
7425 rb_bug("page %p's free_slots should be %d, but %d", (void *)page, page->free_slots, free_objects);
7426 }
7427 }
7428 if (page->final_slots != zombie_objects) {
7429 rb_bug("page %p's final_slots should be %d, but %d", (void *)page, page->final_slots, zombie_objects);
7430 }
7431
7432 return remembered_old_objects;
7433}
7434
7435static int
7436gc_verify_heap_pages_(rb_objspace_t *objspace, struct ccan_list_head *head)
7437{
7438 int remembered_old_objects = 0;
7439 struct heap_page *page = 0;
7440
7441 ccan_list_for_each(head, page, page_node) {
7442 asan_unlock_freelist(page);
7443 struct free_region *region = page->free_region;
7444 while (region) {
7445 VALUE vp = (VALUE)region;
7446 rb_asan_unpoison_object(vp, false);
7447 if (BUILTIN_TYPE(vp) != T_NONE) {
7448 fprintf(stderr, "free region head expected to be T_NONE but was: %s\n", rb_obj_info(vp));
7449 }
7450 struct free_region *next = region->next;
7451 rb_asan_poison_object(vp);
7452 region = next;
7453 }
7454 asan_lock_freelist(page);
7455
7456 if (page->flags.has_remembered_objects == FALSE) {
7457 remembered_old_objects += gc_verify_heap_page(objspace, page, Qfalse);
7458 }
7459 }
7460
7461 return remembered_old_objects;
7462}
7463
7464static int
7465gc_verify_heap_pages(rb_objspace_t *objspace)
7466{
7467 int remembered_old_objects = 0;
7468 for (int i = 0; i < HEAP_COUNT; i++) {
7469 remembered_old_objects += gc_verify_heap_pages_(objspace, &((&heaps[i])->pages));
7470 }
7471 return remembered_old_objects;
7472}
7473
7474static void
7475verify_registered_addr(VALUE *slot, VALUE initial_value, void *owner_objspace, void *d)
7476{
7477 struct verify_internal_consistency_struct *data = d;
7478 VALUE v = *slot;
7479
7480 /* Conservative registration permits uninitialized data and pre-registration
7481 * values; only a store made after registration is a violation. */
7482 if (v == initial_value) return;
7483 if (SPECIAL_CONST_P(v)) return;
7484 if (!verify_pointer_in_any_heap_p((void *)v)) return;
7485
7486 bool live = false;
7487 asan_unpoisoning_object(v) {
7488 live = BUILTIN_TYPE(v) != T_NONE && BUILTIN_TYPE(v) != T_ZOMBIE;
7489 }
7490 if (!live) return;
7491
7492 rb_objspace_t *value_objspace = GET_HEAP_OBJSPACE(v);
7493 if (value_objspace == (rb_objspace_t *)owner_objspace) return;
7494 /* Join and orphan handling move a registration to the inheritor before the
7495 * source objspace merge; a global GC scans every registry while the zombie
7496 * exists, so this is a safe transient exemption. */
7497 if (value_objspace->zombie_pages != 0) return;
7498 if (value_objspace->flags.during_postmortem) return;
7499 if (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(v), v)) return;
7500 if (MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(v), v)) return;
7501 /* When multiple Ractors register one address, ownership by any registrant is
7502 * enough to root the value. */
7503 if (rb_gc_registered_addr_owned_by_registrant_p(slot, value_objspace)) return;
7504
7505 fprintf(stderr, "registered address %p changed since registration to an unshareable object owned by another Ractor: %s\n",
7506 (void *)slot, rb_obj_info(v));
7507 data->err_count++;
7508}
7509
7510static void
7511gc_verify_internal_consistency_(rb_objspace_t *objspace, bool world_stopped)
7512{
7513 struct verify_internal_consistency_struct data = {0};
7514
7515 data.objspace = objspace;
7516 data.world_stopped = world_stopped;
7517 gc_report(5, objspace, "gc_verify_internal_consistency: start\n");
7518
7519 /* check relations */
7520 for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
7521 struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
7522 short slot_size = page->slot_size;
7523
7524 uintptr_t start = (uintptr_t)page->start;
7525 uintptr_t end = start + page->total_slots * slot_size;
7526
7527 verify_internal_consistency_i((void *)start, (void *)end, slot_size, &data);
7528 }
7529
7530 /* Check the calling Ractor's root scoping (only when verifying the current
7531 * objspace). Skipped during a global GC, which deliberately spans every Ractor's
7532 * roots and legitimately reaches foreign objects: containment does not apply. */
7533 if (!rb_gc_single_objspace_p() && objspace == rb_gc_get_objspace() &&
7534 !rb_gc_impl_during_global_gc_p(objspace)) {
7535 rb_objspace_reachable_objects_from_root(root_scope_check_i, &data);
7536 }
7537
7538 if (data.world_stopped && !global_objspace->during_absorb) {
7539 rb_gc_each_registered_addr(verify_registered_addr, &data);
7540 }
7541
7542 if (data.err_count != 0) {
7543#if RGENGC_CHECK_MODE >= 5
7544 objspace->rgengc.error_count = data.err_count;
7545 gc_marks_check(objspace, NULL, NULL);
7546 allrefs_dump(objspace);
7547#endif
7548 rb_bug("gc_verify_internal_consistency: found internal inconsistency.");
7549 }
7550
7551 /* check heap_page status */
7552 gc_verify_heap_pages(objspace);
7553
7554 /* check counters */
7555
7556 if (!is_lazy_sweeping(objspace) &&
7557 !finalizing &&
7558 !rb_gc_multi_ractor_p()) {
7559 if (objspace_live_slots(objspace) != data.live_object_count) {
7560 fprintf(stderr, "heap_pages_final_slots: %"PRIdSIZE", total_freed_objects: %"PRIdSIZE"\n",
7561 total_final_slots_count(objspace), total_freed_objects(objspace));
7562 rb_bug("inconsistent live slot number: expect %"PRIuSIZE", but %"PRIuSIZE".",
7563 objspace_live_slots(objspace), data.live_object_count);
7564 }
7565 }
7566
7567 if (!is_marking(objspace)) {
7568 if (objspace->rgengc.old_objects != data.old_object_count) {
7569 rb_bug("inconsistent old slot number: expect %"PRIuSIZE", but %"PRIuSIZE".",
7570 objspace->rgengc.old_objects, data.old_object_count);
7571 }
7572 if (objspace->rgengc.uncollectible_wb_unprotected_objects != data.remembered_shady_count) {
7573 rb_bug("inconsistent number of wb unprotected objects: expect %"PRIuSIZE", but %"PRIuSIZE".",
7574 objspace->rgengc.uncollectible_wb_unprotected_objects, data.remembered_shady_count);
7575 }
7576 }
7577
7578 if (!finalizing) {
7579 size_t list_count = 0;
7580
7581 {
7582 VALUE z = heap_pages_deferred_final;
7583 while (z) {
7584 list_count++;
7585 z = RZOMBIE(z)->next;
7586 }
7587 }
7588
7589 if (total_final_slots_count(objspace) != data.zombie_object_count ||
7590 total_final_slots_count(objspace) != list_count) {
7591
7592 rb_bug("inconsistent finalizing object count:\n"
7593 " expect %"PRIuSIZE"\n"
7594 " but %"PRIuSIZE" zombies\n"
7595 " heap_pages_deferred_final list has %"PRIuSIZE" items.",
7596 total_final_slots_count(objspace),
7597 data.zombie_object_count,
7598 list_count);
7599 }
7600 }
7601
7602 gc_report(5, objspace, "gc_verify_internal_consistency: OK\n");
7603}
7604
7605/* The `during_gc` macro expands a bare identifier to `objspace->flags.during_gc`, so a
7606 * foreign objspace's flag cannot be written directly; these helpers reach it through the
7607 * `objspace` argument. */
7608static inline unsigned int
7609gc_during_gc_get(const rb_objspace_t *objspace)
7610{
7611 return during_gc;
7612}
7613
7614static inline void
7615gc_during_gc_set(rb_objspace_t *objspace, unsigned int v)
7616{
7617 during_gc = v;
7618}
7619
7620/* Run the check with during_gc cleared in both the verified objspace and the current
7621 * Ractor's: rb_objspace_reachable_objects_from() decides on rb_gc_get_objspace(), and
7622 * under a global GC the driver verifies foreign objspaces, so the driver's during_gc
7623 * needs clearing too (a no-op when cur == objspace). */
7624static void
7625gc_verify_internal_consistency_body(rb_objspace_t *objspace, bool world_stopped)
7626{
7627 const unsigned int prev_during_gc = during_gc;
7628 during_gc = FALSE; // stop gc here
7629
7630 rb_objspace_t *const cur = rb_gc_get_objspace();
7631 const unsigned int prev_cur_during_gc = (cur != objspace) ? gc_during_gc_get(cur) : 0;
7632 if (cur != objspace) gc_during_gc_set(cur, FALSE);
7633 {
7634 gc_verify_internal_consistency_(objspace, world_stopped);
7635 }
7636 if (cur != objspace) gc_during_gc_set(cur, prev_cur_during_gc);
7637 during_gc = prev_during_gc;
7638}
7639
7640static void
7641gc_verify_internal_consistency(void *objspace_ptr)
7642{
7643 rb_objspace_t *objspace = objspace_ptr;
7644
7645 /* Called mid-GC, take neither the VM lock nor the barrier: waiting would join a
7646 * pending global barrier mid-collection (a GC must never take the VM lock) and let
7647 * the global GC sweep the heap this mark is walking. The barrier is unnecessary
7648 * anyway; the objspace is single-writer, this verify runs on its owner thread, and
7649 * the global driver that sets during_gc everywhere already holds both. */
7650 if (during_gc) {
7651 /* The world is stopped only when the global GC's driver runs this while holding
7652 * the barrier; a non-main Ractor's local GC does not stop other Ractors. */
7653 gc_verify_internal_consistency_body(objspace, rb_gc_impl_during_global_gc_p(objspace));
7654 return;
7655 }
7656
7657 unsigned int lev = RB_GC_VM_LOCK();
7658 {
7659 rb_gc_vm_barrier(); // stop other ractors
7660 gc_verify_internal_consistency_body(objspace, true); // holding the barrier, so walking every objspace is sound
7661 }
7662 RB_GC_VM_UNLOCK(lev);
7663}
7664
7665static void
7666heap_move_pooled_pages_to_free_pages(rb_heap_t *heap)
7667{
7668 if (heap->pooled_pages) {
7669 if (heap->free_pages) {
7670 struct heap_page *free_pages_tail = heap->free_pages;
7671 while (free_pages_tail->free_next) {
7672 free_pages_tail = free_pages_tail->free_next;
7673 }
7674 free_pages_tail->free_next = heap->pooled_pages;
7675 }
7676 else {
7677 heap->free_pages = heap->pooled_pages;
7678 }
7679
7680 heap->pooled_pages = NULL;
7681 }
7682}
7683
7684static int
7685gc_remember_unprotected(rb_objspace_t *objspace, VALUE obj)
7686{
7687 struct heap_page *page = GET_HEAP_PAGE(obj);
7688 bits_t *uncollectible_bits = &page->uncollectible_bits[0];
7689
7690 if (!MARKED_IN_BITMAP(uncollectible_bits, obj)) {
7691 page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
7692 MARK_IN_BITMAP(uncollectible_bits, obj);
7693 /* Like RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET, count it in the object's own objspace. */
7694 page->objspace->rgengc.uncollectible_wb_unprotected_objects++;
7695
7696#if RGENGC_PROFILE > 0
7697 objspace->profile.total_remembered_shady_object_count++;
7698#if RGENGC_PROFILE >= 2
7699 objspace->profile.remembered_shady_object_count_types[BUILTIN_TYPE(obj)]++;
7700#endif
7701#endif
7702 return TRUE;
7703 }
7704 else {
7705 return FALSE;
7706 }
7707}
7708
7709static inline void
7710gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bits, short slot_size)
7711{
7712 if (bits) {
7713 do {
7714 if (bits & 1) {
7715 gc_report(2, objspace, "gc_marks_wb_unprotected_objects: marked shady: %s\n", rb_obj_info((VALUE)p));
7716 GC_ASSERT(RVALUE_WB_UNPROTECTED(objspace, (VALUE)p));
7717 GC_ASSERT(RVALUE_MARKED(objspace, (VALUE)p));
7718 gc_mark_children(objspace, (VALUE)p);
7719 }
7720 p += slot_size;
7721 bits >>= 1;
7722 } while (bits);
7723 }
7724}
7725
7726static void
7727gc_marks_wb_unprotected_objects(rb_objspace_t *objspace, rb_heap_t *heap)
7728{
7729 struct heap_page *page = 0;
7730
7731 ccan_list_for_each(&heap->pages, page, page_node) {
7732 bits_t *mark_bits = page->mark_bits;
7733 bits_t *wbun_bits = page->wb_unprotected_bits;
7734 uintptr_t p = page->start;
7735 short slot_size = page->slot_size;
7736 int total_slots = page->total_slots;
7737 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
7738 size_t j;
7739
7740 for (j=0; j<(size_t)bitmap_plane_count; j++) {
7741 bits_t bits = mark_bits[j] & wbun_bits[j];
7742 gc_marks_wb_unprotected_objects_plane(objspace, p, bits, slot_size);
7743 p += BITS_BITLENGTH * slot_size;
7744 }
7745 }
7746
7747 gc_mark_stacked_objects_all(objspace);
7748}
7749
7750void
7751rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj)
7752{
7754}
7755
7756bool
7757rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj)
7758{
7759 rb_objspace_t *objspace = objspace_ptr;
7760
7761 /* A local GC cannot decide a foreign object's liveness, so treat it as live; its
7762 * owner or the global GC decides (a global GC's unified mark is exact). */
7763 if (gc_skip_foreign_object_p(objspace, obj)) return true;
7764
7765 bool marked = RVALUE_MARKED(objspace, obj);
7766
7767 if (marked) {
7768 rgengc_check_relation(objspace, obj);
7769 }
7770
7771 return marked;
7772}
7773
7774static void
7775gc_update_weak_references(rb_objspace_t *objspace)
7776{
7777 VALUE *obj_ptr;
7778 rb_darray_foreach(objspace->weak_references, i, obj_ptr) {
7779 gc_mark_set_parent(objspace, *obj_ptr);
7780 rb_gc_handle_weak_references(objspace, *obj_ptr);
7781 gc_mark_set_parent_invalid(objspace);
7782 }
7783
7784 size_t capa = rb_darray_capa(objspace->weak_references);
7785 size_t size = rb_darray_size(objspace->weak_references);
7786
7787 objspace->profile.weak_references_count = size;
7788
7789 rb_darray_clear(objspace->weak_references);
7790
7791 /* If the darray has capacity for more than four times the amount used, we
7792 * shrink it down to half of that capacity. */
7793 if (capa > size * 4) {
7794 rb_darray_resize_capa_without_gc(&objspace->weak_references, size * 2);
7795 }
7796}
7797
7798static void
7799gc_marks_finish(rb_objspace_t *objspace)
7800{
7801 /* finish incremental GC */
7802 if (is_incremental_marking(objspace)) {
7803 if (RGENGC_CHECK_MODE && is_mark_stack_empty(&objspace->mark_stack) == 0) {
7804 rb_bug("gc_marks_finish: mark stack is not empty (%"PRIdSIZE").",
7805 mark_stack_size(&objspace->mark_stack));
7806 }
7807
7808 mark_roots(objspace, NULL);
7809 while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == false);
7810
7811#if RGENGC_CHECK_MODE >= 2
7812 if (gc_verify_heap_pages(objspace) != 0) {
7813 rb_bug("gc_marks_finish (incremental): there are remembered old objects.");
7814 }
7815#endif
7816
7817 gc_set_incremental_marking(objspace, FALSE);
7818 /* check children of all marked wb-unprotected objects */
7819 for (int i = 0; i < HEAP_COUNT; i++) {
7820 gc_marks_wb_unprotected_objects(objspace, &heaps[i]);
7821 }
7822 }
7823
7824 /* Pin the shareable objects and shrefs ordinary marking missed: a local GC must free
7825 * neither (another objspace may hold them). Running after the full walk makes the
7826 * pin count a retention metric: an upper bound on the garbage only a global GC can
7827 * reclaim. A global GC's exact mark does not pin. (The allrefs comparison of
7828 * RGENGC_CHECK_MODE >= 4 does not model these pins; it reports false positives.)
7829 *
7830 * Running it here rather than with the other roots is also what lets an incremental
7831 * mark run while other objspaces exist. The write barrier bails out on a
7832 * cross-objspace edge, so a store made between two mark steps leaves nothing behind
7833 * but a shareable or shref bit; scanning those bitmaps after the last step picks up
7834 * every bit set during the cycle, which a scan at gc_marks_start would miss. */
7835 objspace->last_cycle_pinned = 0;
7836 if (!rb_gc_single_objspace_p() && !objspace->flags.during_global_gc) {
7837 objspace->last_cycle_pinned = 1;
7838 gc_mark_set_parent_raw(objspace, Qundef, false);
7839 for (int i = 0; i < HEAP_COUNT; i++) {
7840 pinned_roots_mark(objspace, &heaps[i]);
7841 }
7842 /* And everything they keep alive. */
7843 gc_mark_stacked_objects_all(objspace);
7844 }
7845
7846 gc_update_weak_references(objspace);
7847
7848#if RGENGC_CHECK_MODE >= 4
7849 during_gc = FALSE;
7850 gc_marks_check(objspace, gc_check_after_marks_i, "after_marks");
7851 during_gc = TRUE;
7852#endif
7853
7854 {
7855 /* Only this objspace's own Ractor allocates from it. The main objspace
7856 * keeps the VM-wide count it has used since before per-Ractor GC. */
7857 const unsigned long ractor_cnt = objspace == global_objspace->main_objspace
7858 ? rb_gc_vm_ractor_count() : 1;
7859 const unsigned long r_mul = ractor_cnt > 8 ? 8 : ractor_cnt; // upto 8
7860
7861 size_t total_slots = objspace_available_slots(objspace);
7862 size_t sweep_slots = total_slots - objspace->marked_slots; /* will be swept slots */
7863 size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio);
7864 size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio);
7865 if (min_free_slots < gc_params.heap_free_slots * r_mul) {
7866 min_free_slots = gc_params.heap_free_slots * r_mul;
7867 }
7868
7869 int full_marking = is_full_marking(objspace);
7870
7871 GC_ASSERT(objspace_available_slots(objspace) >= objspace->marked_slots);
7872
7873 /* Setup freeable slots. */
7874 size_t total_init_slots = 0;
7875 for (int i = 0; i < HEAP_COUNT; i++) {
7876 total_init_slots += (objspace_heap_init_bytes(objspace) / heaps[i].slot_size) * r_mul;
7877 }
7878
7879 if (max_free_slots < total_init_slots) {
7880 max_free_slots = total_init_slots;
7881 }
7882
7883 /* Approximate freeable pages using the average slots-per-pages across all heaps */
7884 if (sweep_slots > max_free_slots) {
7885 size_t excess_slots = sweep_slots - max_free_slots;
7886 size_t total_heap_pages = heap_eden_total_pages(objspace);
7887 heap_pages_freeable_pages = total_heap_pages > 0
7888 ? excess_slots * total_heap_pages / total_slots
7889 : 0;
7890 }
7891 else {
7892 heap_pages_freeable_pages = 0;
7893 }
7894
7895 if (objspace->heap_pages.allocatable_bytes == 0 && sweep_slots < min_free_slots) {
7896 if (!full_marking && sweep_slots < min_free_slots * 7 / 8) {
7897 if (objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
7898 full_marking = TRUE;
7899 }
7900 else {
7901 gc_report(1, objspace, "gc_marks_finish: next is full GC!!)\n");
7902 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
7903 }
7904 }
7905
7906 if (full_marking) {
7907 heap_allocatable_bytes_expand(objspace, NULL, sweep_slots, total_slots, heaps[0].slot_size);
7908 }
7909 }
7910
7911 if (full_marking) {
7912 /* See the comment about RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR */
7913 const double r = gc_params.oldobject_limit_factor;
7914 objspace->rgengc.uncollectible_wb_unprotected_objects_limit = MAX(
7915 (size_t)(objspace->rgengc.uncollectible_wb_unprotected_objects * r),
7916 (size_t)(objspace->rgengc.old_objects * gc_params.uncollectible_wb_unprotected_objects_limit_ratio)
7917 );
7918 objspace->rgengc.old_objects_limit = (size_t)(objspace->rgengc.old_objects * r);
7919 }
7920
7921 if (objspace->rgengc.uncollectible_wb_unprotected_objects > objspace->rgengc.uncollectible_wb_unprotected_objects_limit) {
7922 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_SHADY;
7923 }
7924 if (objspace->rgengc.old_objects > objspace->rgengc.old_objects_limit) {
7925 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDGEN;
7926 }
7927
7928 gc_report(1, objspace, "gc_marks_finish (marks %"PRIdSIZE" objects, "
7929 "old %"PRIdSIZE" objects, total %"PRIdSIZE" slots, "
7930 "sweep %"PRIdSIZE" slots, allocatable %"PRIdSIZE" bytes, next GC: %s)\n",
7931 objspace->marked_slots, objspace->rgengc.old_objects, objspace_available_slots(objspace), sweep_slots, objspace->heap_pages.allocatable_bytes,
7932 gc_needs_major_flags ? "major" : "minor");
7933 }
7934
7935 // TODO: refactor so we don't need to call this
7936 rb_ractor_finish_marking(is_full_marking(objspace));
7937
7939}
7940
7941static bool
7942gc_compact_heap_cursors_met_p(rb_heap_t *heap)
7943{
7944 return heap->sweeping_page == heap->compact_cursor;
7945}
7946
7947
7948static rb_heap_t *
7949gc_compact_destination_pool(rb_objspace_t *objspace, rb_heap_t *src_pool, VALUE obj)
7950{
7951 size_t obj_size = rb_gc_obj_optimal_size(obj);
7952 if (obj_size == 0) {
7953 return src_pool;
7954 }
7955
7956 GC_ASSERT(rb_gc_impl_size_allocatable_p(obj_size));
7957
7958 size_t idx = heap_idx_for_size(obj_size);
7959
7960 return &heaps[idx];
7961}
7962
7963static bool
7964gc_compact_move(rb_objspace_t *objspace, rb_heap_t *heap, VALUE src)
7965{
7966 GC_ASSERT(BUILTIN_TYPE(src) != T_MOVED);
7967 GC_ASSERT(gc_is_moveable_obj(objspace, src));
7968
7969 rb_heap_t *dest_pool = gc_compact_destination_pool(objspace, heap, src);
7970 if (gc_compact_heap_cursors_met_p(dest_pool)) {
7971 return dest_pool != heap;
7972 }
7973
7974 while (!try_move(objspace, dest_pool, dest_pool->free_pages, src)) {
7975 struct gc_sweep_context ctx = {
7976 .page = dest_pool->sweeping_page,
7977 .final_slots = 0,
7978 .freed_slots = 0,
7979 .empty_slots = 0,
7980 };
7981
7982 /* The page of src could be partially compacted, so it may contain
7983 * T_MOVED. Sweeping a page may read objects on this page, so we
7984 * need to lock the page. */
7985 lock_page_body(objspace, GET_PAGE_BODY(src));
7986 gc_sweep_page(objspace, dest_pool, &ctx);
7987 unlock_page_body(objspace, GET_PAGE_BODY(src));
7988
7989 if (dest_pool->sweeping_page->free_slots > 0) {
7990 heap_add_freepage(dest_pool, dest_pool->sweeping_page);
7991 }
7992
7993 dest_pool->sweeping_page = ccan_list_next(&dest_pool->pages, dest_pool->sweeping_page, page_node);
7994 if (gc_compact_heap_cursors_met_p(dest_pool)) {
7995 return dest_pool != heap;
7996 }
7997 }
7998
7999 return true;
8000}
8001
8002static bool
8003gc_compact_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct heap_page *page)
8004{
8005 short slot_size = page->slot_size;
8006
8007 do {
8008 VALUE vp = (VALUE)p;
8009 GC_ASSERT(vp % sizeof(VALUE) == 0);
8010
8011 if (bitset & 1) {
8012 objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++;
8013
8014 if (gc_is_moveable_obj(objspace, vp)) {
8015 if (!gc_compact_move(objspace, heap, vp)) {
8016 //the cursors met. bubble up
8017 return false;
8018 }
8019 }
8020 }
8021 p += slot_size;
8022 bitset >>= 1;
8023 } while (bitset);
8024
8025 return true;
8026}
8027
8028// Iterate up all the objects in page, moving them to where they want to go
8029static bool
8030gc_compact_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
8031{
8032 GC_ASSERT(page == heap->compact_cursor);
8033
8034 bits_t *mark_bits, *pin_bits;
8035 bits_t bitset;
8036 uintptr_t p = page->start;
8037 short slot_size = page->slot_size;
8038 int total_slots = page->total_slots;
8039 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
8040
8041 mark_bits = page->mark_bits;
8042 pin_bits = page->pinned_bits;
8043
8044 for (int j = 0; j < bitmap_plane_count; j++) {
8045 // objects that can be moved are marked and not pinned
8046 bitset = (mark_bits[j] & ~pin_bits[j]);
8047 if (bitset) {
8048 if (!gc_compact_plane(objspace, heap, (uintptr_t)p, bitset, page))
8049 return false;
8050 }
8051 p += BITS_BITLENGTH * slot_size;
8052 }
8053
8054 return true;
8055}
8056
8057static bool
8058gc_compact_all_compacted_p(rb_objspace_t *objspace)
8059{
8060 for (int i = 0; i < HEAP_COUNT; i++) {
8061 rb_heap_t *heap = &heaps[i];
8062
8063 if (heap->total_pages > 0 &&
8064 !gc_compact_heap_cursors_met_p(heap)) {
8065 return false;
8066 }
8067 }
8068
8069 return true;
8070}
8071
8072/* Compaction's move phase: relocate this objspace's movable objects and leave T_MOVED
8073 * forwarding behind without updating references yet. A global GC calls this for every
8074 * objspace before updating any of them (two phases), so a cross-objspace reference to a
8075 * moved object is rewritten exactly once, after all forwarding exists. */
8076static void
8077gc_compact_relocate(rb_objspace_t *objspace)
8078{
8079 gc_compact_start(objspace);
8080
8081 while (!gc_compact_all_compacted_p(objspace)) {
8082 for (int i = 0; i < HEAP_COUNT; i++) {
8083 rb_heap_t *heap = &heaps[i];
8084
8085 if (gc_compact_heap_cursors_met_p(heap)) {
8086 continue;
8087 }
8088
8089 struct heap_page *start_page = heap->compact_cursor;
8090
8091 if (!gc_compact_page(objspace, heap, start_page)) {
8092 lock_page_body(objspace, start_page->body);
8093
8094 continue;
8095 }
8096
8097 // If we get here, we've finished moving all objects on the compact_cursor page
8098 // So we can lock it and move the cursor on to the next one.
8099 lock_page_body(objspace, start_page->body);
8100 heap->compact_cursor = ccan_list_prev(&heap->pages, heap->compact_cursor, page_node);
8101 }
8102 }
8103}
8104
8105static void
8106gc_sweep_compact(rb_objspace_t *objspace)
8107{
8108 gc_compact_relocate(objspace);
8109 /* A compacting global GC defers the finish (reference update) to the second phase,
8110 * after every objspace has been relocated. */
8111 if (!global_objspace->global_gc.compacting) {
8112 gc_compact_finish(objspace);
8113 }
8114}
8115
8116static void
8117gc_marks_rest(rb_objspace_t *objspace)
8118{
8119 gc_report(1, objspace, "gc_marks_rest\n");
8120
8121 for (int i = 0; i < HEAP_COUNT; i++) {
8122 (&heaps[i])->pooled_pages = NULL;
8123 }
8124
8125 if (is_incremental_marking(objspace)) {
8126 while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == FALSE);
8127 }
8128 else {
8129 gc_mark_stacked_objects_all(objspace);
8130 }
8131
8132 gc_marks_finish(objspace);
8133}
8134
8135static bool
8136gc_marks_step(rb_objspace_t *objspace, size_t slots)
8137{
8138 bool marking_finished = false;
8139
8140 GC_ASSERT(is_marking(objspace));
8141 if (gc_mark_stacked_objects_incremental(objspace, slots)) {
8142 gc_marks_finish(objspace);
8143
8144 marking_finished = true;
8145 }
8146
8147 return marking_finished;
8148}
8149
8150static bool
8151gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap)
8152{
8153 GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
8154 bool marking_finished = true;
8155
8156 gc_marking_enter(objspace);
8157
8158 if (heap->free_pages) {
8159 gc_report(2, objspace, "gc_marks_continue: has pooled pages");
8160
8161 marking_finished = gc_marks_step(objspace, objspace->rincgc.step_slots);
8162 }
8163 else {
8164 gc_report(2, objspace, "gc_marks_continue: no more pooled pages (stack depth: %"PRIdSIZE").\n",
8165 mark_stack_size(&objspace->mark_stack));
8166 heap->force_incremental_marking_finish_count++;
8167 gc_marks_rest(objspace);
8168 }
8169
8170 gc_marking_exit(objspace);
8171
8172 return marking_finished;
8173}
8174
8175/* Mark the following as roots of this objspace.
8176 * - Every shareable object: another objspace may hold the only reference, invisible to a
8177 * local GC. Marking them rather than skipping them in the sweep preserves the
8178 * generational invariants (a pinned object ages and gets promoted like any live one).
8179 * Only a global GC decides that a shareable object is dead.
8180 * - Every shref (an unshareable object referenced from a shareable one): the referring
8181 * shareable object can live in another objspace or in an in-flight message queue. The
8182 * write barrier maintains them.
8183 * Skipped while the VM has a single Ractor: a local GC is then a whole-world GC and
8184 * shareable objects may die normally. */
8185static void
8186pinned_roots_mark(rb_objspace_t *objspace, rb_heap_t *heap)
8187{
8188 struct heap_page *page = NULL;
8189
8190 /* Runs before mark_roots, so rgengc_check_relation sees a valid (absent) parent rather
8191 * than the poison left by the previous GC. */
8192 gc_mark_set_parent_raw(objspace, Qundef, false);
8193
8194 /* A local GC never frees or traverses a shareable object, and keeps its unshareable
8195 * children alive through their shref bits, so:
8196 * - a shareable object only gets its mark bit set (like an old object), which keeps
8197 * the sweep off it, and is not traversed;
8198 * - a shref is marked and traversed, like a remembered old->young target: without
8199 * that, the referring shareable object is never walked and it would look
8200 * unreachable.
8201 * Objects can become shareable between GCs, so this pass scans the bitmaps in every
8202 * mark (gc_marks_finish) instead of maintaining a pin set across the sweep. */
8203 ccan_list_for_each(&heap->pages, page, page_node) {
8204 if (!(page->flags.has_shareable_objects | page->flags.has_shref_objects)) continue;
8205
8206 uintptr_t p = page->start;
8207 short slot_size = page->slot_size;
8208 int total_slots = page->total_slots;
8209 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
8210
8211 for (int j = 0; j < bitmap_plane_count; j++) {
8212 bits_t sr_bits = page->shref_bits[j];
8213 /* Only the pins ordinary marking left unmarked need work here: an already
8214 * marked object (reached by traversal, or pre-marked because it is old) is a
8215 * no-op in gc_mark_set, so skip visiting it. */
8216 bits_t bitset = (page->shareable_bits[j] | sr_bits) & ~page->mark_bits[j];
8217 uintptr_t pp = p;
8218 while (bitset) {
8219 if (bitset & 1) {
8220 VALUE obj = (VALUE)pp;
8221 asan_unpoisoning_object(obj) {
8222 switch (BUILTIN_TYPE(obj)) {
8223 case T_NONE:
8224 case T_ZOMBIE:
8225 case T_MOVED:
8226 /* A dead slot (a zombie awaiting its finalizer) is not a root. */
8227 break;
8228 default:
8229 gc_report(2, objspace, "pinned_roots_mark: mark %s\n", rb_obj_info(obj));
8230 if (sr_bits & 1) {
8231 gc_mark(objspace, obj); /* shref: root + traverse */
8232 }
8233 else if (gc_mark_set(objspace, obj)) {
8234 gc_aging(objspace, obj); /* shareable: mark, no traverse */
8235 /* Pin as well when compaction runs alongside: if a shareable
8236 * object moved, the C-struct slots of other Ractors (a
8237 * port in sync, say) are not updated and go stale. */
8238 gc_pin(objspace, obj);
8239 }
8240 break;
8241 }
8242 }
8243 }
8244 pp += slot_size;
8245 bitset >>= 1;
8246 sr_bits >>= 1;
8247 }
8248 p += BITS_BITLENGTH * slot_size;
8249 }
8250 }
8251}
8252
8253static void
8254gc_marks_start(rb_objspace_t *objspace, int full_mark)
8255{
8256 /* start marking */
8257 gc_report(1, objspace, "gc_marks_start: (%s)\n", full_mark ? "full" : "minor");
8258 gc_mode_transition(objspace, gc_mode_marking);
8259
8260 if (full_mark) {
8261 size_t incremental_marking_steps = (objspace->rincgc.pooled_slots / INCREMENTAL_MARK_STEP_ALLOCATIONS) + 1;
8262 objspace->rincgc.step_slots = (objspace->marked_slots * 2) / incremental_marking_steps;
8263
8264 if (0) fprintf(stderr, "objspace->marked_slots: %"PRIdSIZE", "
8265 "objspace->rincgc.pooled_page_num: %"PRIdSIZE", "
8266 "objspace->rincgc.step_slots: %"PRIdSIZE", \n",
8267 objspace->marked_slots, objspace->rincgc.pooled_slots, objspace->rincgc.step_slots);
8268 objspace->flags.during_minor_gc = FALSE;
8269 if (ruby_enable_autocompact && rb_gc_single_objspace_p()) {
8270 objspace->flags.during_compacting |= TRUE;
8271 }
8272 objspace->profile.major_gc_count++;
8273 objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
8274 objspace->rgengc.old_objects = 0;
8275 objspace->rgengc.last_major_gc = objspace->profile.count;
8276 objspace->marked_slots = 0;
8277
8278 for (int i = 0; i < HEAP_COUNT; i++) {
8279 rb_heap_t *heap = &heaps[i];
8280 gc_bitmaps_clear(objspace, heap, false);
8281 heap_move_pooled_pages_to_free_pages(heap);
8282
8283 if (objspace->flags.during_compacting) {
8284 struct heap_page *page = NULL;
8285
8286 ccan_list_for_each(&heap->pages, page, page_node) {
8287 page->pinned_slots = 0;
8288 }
8289 }
8290 }
8291 }
8292 else {
8293 objspace->flags.during_minor_gc = TRUE;
8294 objspace->marked_slots =
8295 objspace->rgengc.old_objects + objspace->rgengc.uncollectible_wb_unprotected_objects; /* uncollectible objects are marked already */
8296 objspace->profile.minor_gc_count++;
8297
8298 for (int i = 0; i < HEAP_COUNT; i++) {
8299 rgengc_rememberset_mark(objspace, &heaps[i]);
8300 }
8301 }
8302
8303 mark_roots(objspace, NULL);
8304
8305 gc_report(1, objspace, "gc_marks_start: (%s) end, stack in %"PRIdSIZE"\n",
8306 full_mark ? "full" : "minor", mark_stack_size(&objspace->mark_stack));
8307}
8308
8309static bool
8310gc_marks(rb_objspace_t *objspace, int full_mark)
8311{
8312 gc_marking_enter(objspace);
8313
8314 bool marking_finished = false;
8315
8316 /* setup marking */
8317
8318 gc_marks_start(objspace, full_mark);
8319 if (!is_incremental_marking(objspace)) {
8320 gc_marks_rest(objspace);
8321 marking_finished = true;
8322 }
8323
8324#if RGENGC_PROFILE > 0
8325 if (gc_prof_record(objspace)) {
8326 gc_profile_record *record = gc_prof_record(objspace);
8327 record->old_objects = objspace->rgengc.old_objects;
8328 }
8329#endif
8330
8331 gc_marking_exit(objspace);
8332
8333 return marking_finished;
8334}
8335
8336/* RGENGC */
8337
8338static void
8339gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...)
8340{
8341 if (level <= RGENGC_DEBUG) {
8342 char buf[1024];
8343 FILE *out = stderr;
8344 va_list args;
8345 const char *status = " ";
8346
8347 if (during_gc) {
8348 status = is_full_marking(objspace) ? "+" : "-";
8349 }
8350 else {
8351 if (is_lazy_sweeping(objspace)) {
8352 status = "S";
8353 }
8354 if (is_incremental_marking(objspace)) {
8355 status = "M";
8356 }
8357 }
8358
8359 va_start(args, fmt);
8360 vsnprintf(buf, 1024, fmt, args);
8361 va_end(args);
8362
8363 fprintf(out, "%s|", status);
8364 fputs(buf, out);
8365 }
8366}
8367
8368/* bit operations */
8369
8370static void
8371rgengc_remembersetbits_set(rb_objspace_t *objspace, VALUE obj)
8372{
8373 struct heap_page *page = GET_HEAP_PAGE(obj);
8374 bits_t *bits = &page->remembered_bits[0];
8375
8376 /* remembered_bits writers are always serialized: the write barrier only remembers a
8377 * local a (under its Ractor's GVL) and a global GC writes from the driver alone.
8378 * Set the bit before the page flag so a page pending re-scan stays in
8379 * rememberset_mark. */
8380 _MARK_IN_BITMAP(bits, page, obj);
8381 page->flags.has_remembered_objects = TRUE;
8382}
8383
8384/* wb, etc */
8385
8386/* return FALSE if already remembered */
8387static void
8388rgengc_remember(rb_objspace_t *objspace, VALUE obj)
8389{
8390 gc_report(6, objspace, "rgengc_remember: %s %s\n", rb_obj_info(obj),
8391 RVALUE_REMEMBERED(objspace, obj) ? "was already remembered" : "is remembered now");
8392
8393 check_rvalue_consistency(objspace, obj);
8394
8395 if (RGENGC_CHECK_MODE) {
8396 if (RVALUE_WB_UNPROTECTED(objspace, obj)) rb_bug("rgengc_remember: %s is not wb protected.", rb_obj_info(obj));
8397 }
8398
8399#if RGENGC_PROFILE > 0
8400 if (!RVALUE_REMEMBERED(objspace, obj)) {
8401 if (RVALUE_WB_UNPROTECTED(objspace, obj) == 0) {
8402 objspace->profile.total_remembered_normal_object_count++;
8403#if RGENGC_PROFILE >= 2
8404 objspace->profile.remembered_normal_object_count_types[BUILTIN_TYPE(obj)]++;
8405#endif
8406 }
8407 }
8408#endif /* RGENGC_PROFILE > 0 */
8409
8410 rgengc_remembersetbits_set(objspace, obj);
8411}
8412
8413#ifndef PROFILE_REMEMBERSET_MARK
8414#define PROFILE_REMEMBERSET_MARK 0
8415#endif
8416
8417static inline void
8418rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitset, short slot_size)
8419{
8420 if (bitset) {
8421 do {
8422 if (bitset & 1) {
8423 VALUE obj = (VALUE)p;
8424 gc_report(2, objspace, "rgengc_rememberset_mark: mark %s\n", rb_obj_info(obj));
8425 GC_ASSERT(RVALUE_UNCOLLECTIBLE(objspace, obj));
8426 GC_ASSERT(RVALUE_OLD_P(objspace, obj) || RVALUE_WB_UNPROTECTED(objspace, obj));
8427
8428 gc_mark_children(objspace, obj);
8429
8431 rb_darray_append_without_gc(&objspace->weak_references, obj);
8432 }
8433 }
8434 p += slot_size;
8435 bitset >>= 1;
8436 } while (bitset);
8437 }
8438}
8439
8440static void
8441rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap)
8442{
8443 size_t j;
8444 struct heap_page *page = 0;
8445#if PROFILE_REMEMBERSET_MARK
8446 int has_old = 0, has_shady = 0, has_both = 0, skip = 0;
8447#endif
8448 gc_report(1, objspace, "rgengc_rememberset_mark: start\n");
8449
8450 ccan_list_for_each(&heap->pages, page, page_node) {
8451 if (page->flags.has_remembered_objects | page->flags.has_uncollectible_wb_unprotected_objects) {
8452 uintptr_t p = page->start;
8453 short slot_size = page->slot_size;
8454 int total_slots = page->total_slots;
8455 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
8456 bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT];
8457 bits_t *remembered_bits = page->remembered_bits;
8458 bits_t *uncollectible_bits = page->uncollectible_bits;
8459 bits_t *wb_unprotected_bits = page->wb_unprotected_bits;
8460#if PROFILE_REMEMBERSET_MARK
8461 if (page->flags.has_remembered_objects && page->flags.has_uncollectible_wb_unprotected_objects) has_both++;
8462 else if (page->flags.has_remembered_objects) has_old++;
8463 else if (page->flags.has_uncollectible_wb_unprotected_objects) has_shady++;
8464#endif
8465 /* Clear has_remembered_objects before draining the bits. A concurrent
8466 * lock-free write barrier (another Ractor remembering a shareable object on
8467 * this page) sets the bit first and the flag second, so clearing the flag first
8468 * keeps the page scheduled for re-scan even if that set interleaves. The
8469 * per-word drain is an atomic read-and-clear, so an interleaved set is not lost
8470 * (it lands in the zeroed word). */
8471 page->flags.has_remembered_objects = FALSE;
8472 for (j=0; j < (size_t)bitmap_plane_count; j++) {
8473 bits[j] = RUBY_ATOMIC_SIZE_EXCHANGE(*(volatile size_t *)&remembered_bits[j], 0)
8474 | (uncollectible_bits[j] & wb_unprotected_bits[j]);
8475 }
8476
8477 for (j=0; j < (size_t)bitmap_plane_count; j++) {
8478 bitset = bits[j];
8479 rgengc_rememberset_mark_plane(objspace, p, bitset, slot_size);
8480 p += BITS_BITLENGTH * slot_size;
8481 }
8482 }
8483#if PROFILE_REMEMBERSET_MARK
8484 else {
8485 skip++;
8486 }
8487#endif
8488 }
8489
8490#if PROFILE_REMEMBERSET_MARK
8491 fprintf(stderr, "%d\t%d\t%d\t%d\n", has_both, has_old, has_shady, skip);
8492#endif
8493 gc_report(1, objspace, "rgengc_rememberset_mark: finished\n");
8494}
8495
8496static void
8497gc_bitmaps_clear(rb_objspace_t *objspace, rb_heap_t *heap, bool clear_shref)
8498{
8499 struct heap_page *page = 0;
8500
8501 ccan_list_for_each(&heap->pages, page, page_node) {
8502 memset(&page->mark_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8503 memset(&page->uncollectible_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8504 memset(&page->marking_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8505 /* A plain memset can lose a concurrent remember, but only a shareable object can
8506 * be remembered from another Ractor's thread, and pinned_roots_mark re-marks
8507 * those every local cycle, and this clear precedes a major that re-scans all. */
8508 memset(&page->remembered_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8509 memset(&page->pinned_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8510 page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
8511 page->flags.has_remembered_objects = FALSE;
8512 /* A shref is a local GC's root, so only a stop-the-world global GC may clear them:
8513 * its unified mark re-derives them from every shareable -> unshareable edge. */
8514 if (clear_shref) {
8515 memset(&page->shref_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8516 page->flags.has_shref_objects = FALSE;
8517 }
8518 }
8519}
8520
8521/* RGENGC: APIs */
8522
8523NOINLINE(static void gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace));
8524
8525/* Precondition: `a` and `b` live in `objspace`. */
8526static void
8527gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace)
8528{
8529 if (RGENGC_CHECK_MODE) {
8530 if (!RVALUE_OLD_P(objspace, a)) rb_bug("gc_writebarrier_generational: %s is not an old object.", rb_obj_info(a));
8531 if ( RVALUE_OLD_P(objspace, b)) rb_bug("gc_writebarrier_generational: %s is an old object.", rb_obj_info(b));
8532 if (is_incremental_marking(objspace)) rb_bug("gc_writebarrier_generational: called while incremental marking: %s -> %s", rb_obj_info(a), rb_obj_info(b));
8533 }
8534
8535 /* Mark and remember a (the default behaviour).
8536 * No lock: setting a remembered bit is atomic (rgengc_remembersetbits_set), and that is
8537 * the only place a concurrent local GC or another Ractor's write barrier can race. */
8538 if (!RVALUE_REMEMBERED(objspace, a)) {
8539 rgengc_remember(objspace, a);
8540
8541 gc_report(1, objspace, "gc_writebarrier_generational: %s (remembered) -> %s\n", rb_obj_info(a), rb_obj_info(b));
8542 }
8543
8544 check_rvalue_consistency(objspace, a);
8545 check_rvalue_consistency(objspace, b);
8546}
8547
8548static void
8549gc_mark_from(rb_objspace_t *objspace, VALUE obj, VALUE parent)
8550{
8551 gc_mark_set_parent(objspace, parent);
8552 rgengc_check_relation(objspace, obj);
8553 if (gc_mark_set(objspace, obj) != FALSE) {
8554 gc_aging(objspace, obj);
8555 gc_grey(objspace, obj);
8556 }
8557 gc_mark_set_parent_invalid(objspace);
8558}
8559
8560NOINLINE(static void gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace));
8561
8562/* Precondition: `a` and `b` live in `objspace`. */
8563static void
8564gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace)
8565{
8566 gc_report(2, objspace, "gc_writebarrier_incremental: [LG] %p -> %s\n", (void *)a, rb_obj_info(b));
8567
8568 if (RVALUE_BLACK_P(objspace, a)) {
8569 if (RVALUE_WHITE_P(objspace, b)) {
8570 if (!RVALUE_WB_UNPROTECTED(objspace, a)) {
8571 gc_report(2, objspace, "gc_writebarrier_incremental: [IN] %p -> %s\n", (void *)a, rb_obj_info(b));
8572 gc_mark_from(objspace, b, a);
8573 }
8574 }
8575 else if (RVALUE_OLD_P(objspace, a) && !RVALUE_OLD_P(objspace, b)) {
8576 rgengc_remember(objspace, a);
8577 }
8578
8579 if (RB_UNLIKELY(objspace->flags.during_compacting)) {
8580 MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(b), b);
8581 }
8582 }
8583}
8584
8585void
8586rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b)
8587{
8588 rb_objspace_t *objspace = objspace_ptr;
8589
8590#if RGENGC_CHECK_MODE
8591 if (SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const: %"PRIxVALUE, a);
8592 if (SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: b is special const: %"PRIxVALUE, b);
8593#else
8596#endif
8597
8598 GC_ASSERT(!during_gc);
8599 GC_ASSERT(RB_BUILTIN_TYPE(a) != T_NONE);
8600 GC_ASSERT(RB_BUILTIN_TYPE(a) != T_MOVED);
8601 GC_ASSERT(RB_BUILTIN_TYPE(a) != T_ZOMBIE);
8602
8603 /* A shareable object now references an unshareable one: record b as a shref so its
8604 * owner's local GC roots it (the parent may live in another objspace, untraversed
8605 * there). Only b's owner stores this, on its own page: a plain store suffices. */
8606 if (RB_UNLIKELY(RB_FL_TEST_RAW(a, RUBY_FL_SHAREABLE)) &&
8608 struct heap_page *bpage = GET_HEAP_PAGE(b);
8609 if (!_MARKED_IN_BITMAP(bpage->shref_bits, bpage, b)) {
8610 _MARK_IN_BITMAP(bpage->shref_bits, bpage, b);
8611 bpage->flags.has_shref_objects = TRUE;
8612 }
8613 }
8614
8615 if (!is_incremental_marking(objspace)) {
8616 /* The generational barrier covers old->young edges within one objspace only.
8617 * NOTE: we shouldn't even check the age of `a` or `b` if they are in another
8618 * objspace, so check locality first. */
8619 if (gc_foreign_object_p(objspace, a) || gc_foreign_object_p(objspace, b) ||
8620 !RVALUE_OLD_P(objspace, a) || RVALUE_OLD_P(objspace, b)) {
8621 // do nothing
8622 }
8623 else {
8624 gc_writebarrier_generational(a, b, objspace);
8625 }
8626 }
8627 else {
8628 // Shareable objects from different object spaces are kept alive by shareable bits
8629 if (gc_foreign_object_p(objspace, a) || gc_foreign_object_p(objspace, b)) {
8630 // do nothing
8631 }
8632 else {
8633 gc_writebarrier_incremental(a, b, objspace);
8634 }
8635 }
8636}
8637
8638void
8639rb_gc_impl_obj_became_shareable(void *objspace_ptr, VALUE obj)
8640{
8641 /* An object becomes shareable on its owner thread, so this page update is
8642 * single-writer. */
8643 struct heap_page *page = GET_HEAP_PAGE(obj);
8644
8645 if (_MARKED_IN_BITMAP(page->shareable_bits, page, obj)) return;
8646 gc_page_add_shareable(page, obj);
8647
8648 /* The shref bits recorded while the object was unshareable are now covered by the
8649 * shareable pin, and a shref only points at an unshareable object. The owner thread is
8650 * the only writer, so a plain clear is enough. */
8651 if (_MARKED_IN_BITMAP(page->shref_bits, page, obj)) {
8652 _CLEAR_IN_BITMAP(page->shref_bits, page, obj);
8653 // NOTE: page->has_shref_objects could become stale here (value is true even though logically false)
8654 }
8655}
8656
8657void
8658rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj)
8659{
8660 rb_objspace_t *objspace = objspace_ptr;
8661
8662 /* A shareable object is never WB-unprotected. Keeping shrefs correct relies on every
8663 * store into s->u going through the write barrier, which keeps wb_unprotected_bits
8664 * single-writer (only the owner thread can unprotect its own unshareable objects). */
8665 GC_ASSERT(!RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE));
8666
8667 if (RVALUE_WB_UNPROTECTED(objspace, obj)) {
8668 return;
8669 }
8670 else {
8671 gc_report(2, objspace, "rb_gc_writebarrier_unprotect: %s %s\n", rb_obj_info(obj),
8672 RVALUE_REMEMBERED(objspace, obj) ? " (already remembered)" : "");
8673
8674 /* No lock: per the assert obj is our own unshareable, so these bits
8675 * (wb_unprotected, uncollectible, age) are single-writer on an owned page, and
8676 * RVALUE_DEMOTE's remembered-bit clear is atomic against word-sharing writers. */
8677 if (RVALUE_OLD_P(objspace, obj)) {
8678 gc_report(1, objspace, "rb_gc_writebarrier_unprotect: %s\n", rb_obj_info(obj));
8679 RVALUE_DEMOTE(objspace, obj);
8680 gc_mark_set(objspace, obj);
8681 gc_remember_unprotected(objspace, obj);
8682
8683#if RGENGC_PROFILE
8684 objspace->profile.total_shade_operation_count++;
8685#if RGENGC_PROFILE >= 2
8686 objspace->profile.shade_operation_count_types[BUILTIN_TYPE(obj)]++;
8687#endif /* RGENGC_PROFILE >= 2 */
8688#endif /* RGENGC_PROFILE */
8689 }
8690 else {
8691 RVALUE_AGE_RESET(obj);
8692 }
8693
8694 RB_DEBUG_COUNTER_INC(obj_wb_unprotect);
8695 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
8696 }
8697}
8698
8699void
8700rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj)
8701{
8702 rb_objspace_t *objspace = objspace_ptr;
8703
8704 if (RVALUE_WB_UNPROTECTED(objspace, obj)) {
8705 rb_gc_impl_writebarrier_unprotect(objspace, dest);
8706 }
8707 rb_gc_impl_copy_finalizer(objspace, dest, obj);
8708}
8709
8710const char *
8711rb_gc_impl_active_gc_name(void)
8712{
8713 return "default";
8714}
8715
8716/* NOTE: `obj` doesn't necessarily live in `objspace_ptr`, as `objspace_ptr` is just that
8717 * of the current Ractor. */
8718void
8719rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj)
8720{
8721
8722 rb_objspace_t *objspace = objspace_ptr;
8723
8724 // Shareable objects from other object spaces don't need to be put on the remembered set
8725 // and are only collected during global GC, so not while incremental marking.
8726 if (RB_LIKELY(!gc_foreign_object_p(objspace, obj))) {
8727 gc_report(1, objspace, "rb_gc_writebarrier_remember: %s\n", rb_obj_info(obj));
8728 if (is_incremental_marking(objspace)) {
8729 if (RVALUE_BLACK_P(objspace, obj)) {
8730 gc_grey(objspace, obj);
8731 }
8732 }
8733 else if (RVALUE_OLD_P(objspace, obj)) {
8734 rgengc_remember(objspace, obj);
8735 }
8736 }
8737}
8738
8740 // Must be ID only
8741 ID ID_wb_protected, ID_age, ID_old, ID_uncollectible, ID_marking,
8742 ID_marked, ID_pinned, ID_remembered, ID_object_id, ID_shareable;
8743};
8744
8745#define RB_GC_OBJECT_METADATA_ENTRY_COUNT (sizeof(struct rb_gc_object_metadata_names) / sizeof(ID))
8746static struct rb_gc_object_metadata_entry object_metadata_entries[RB_GC_OBJECT_METADATA_ENTRY_COUNT + 1];
8747
8749rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj)
8750{
8751 rb_objspace_t *objspace = objspace_ptr;
8752 size_t n = 0;
8753 static struct rb_gc_object_metadata_names names;
8754
8755 if (!names.ID_marked) {
8756#define I(s) names.ID_##s = rb_intern(#s)
8757 I(wb_protected);
8758 I(age);
8759 I(old);
8760 I(uncollectible);
8761 I(marking);
8762 I(marked);
8763 I(pinned);
8764 I(remembered);
8765 I(object_id);
8766 I(shareable);
8767#undef I
8768 }
8769
8770#define SET_ENTRY(na, v) do { \
8771 GC_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \
8772 object_metadata_entries[n].name = names.ID_##na; \
8773 object_metadata_entries[n].val = v; \
8774 n++; \
8775} while (0)
8776
8777 if (!RVALUE_WB_UNPROTECTED(objspace, obj)) SET_ENTRY(wb_protected, Qtrue);
8778 SET_ENTRY(age, INT2FIX(RVALUE_AGE_GET(obj)));
8779 if (RVALUE_OLD_P(objspace, obj)) SET_ENTRY(old, Qtrue);
8780 if (RVALUE_UNCOLLECTIBLE(objspace, obj)) SET_ENTRY(uncollectible, Qtrue);
8781 if (RVALUE_MARKING(objspace, obj)) SET_ENTRY(marking, Qtrue);
8782 if (RVALUE_MARKED(objspace, obj)) SET_ENTRY(marked, Qtrue);
8783 if (RVALUE_PINNED(objspace, obj)) SET_ENTRY(pinned, Qtrue);
8784 if (RVALUE_REMEMBERED(objspace, obj)) SET_ENTRY(remembered, Qtrue);
8785 if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj));
8786 if (FL_TEST(obj, FL_SHAREABLE)) SET_ENTRY(shareable, Qtrue);
8787
8788 object_metadata_entries[n].name = 0;
8789 object_metadata_entries[n].val = 0;
8790#undef SET_ENTRY
8791
8792 return object_metadata_entries;
8793}
8794
8795void *
8796rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor)
8797{
8798 /* No cache needed: allocation happens in a per-Ractor objspace. */
8799 return NULL;
8800}
8801
8802void
8803rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache)
8804{
8805 GC_ASSERT(cache == NULL);
8806}
8807
8808/* The terminating Ractor's final local GC, on its own thread: roots are minimal, so the
8809 * mark is tiny, and it reclaims what the joining side would otherwise inherit. Never
8810 * promotes to a global GC (that would STW on every Ractor death); empty pages go
8811 * straight back to the page pool. */
8812/* Finalize the zombies whose cleanup is pure C (a dfree, no Ruby-level finalizer);
8813 * the caller has no Ruby execution context any more, so zombies with a Ruby
8814 * finalizer stay deferred and travel to the inheritor as before. Returns whether
8815 * anything was finalized (those pages then need one more sweep to detach). */
8816static bool
8817finalize_deferred_dfree_only(rb_objspace_t *objspace)
8818{
8819 VALUE dfree_only = 0;
8820 VALUE zombie = RUBY_ATOMIC_VALUE_EXCHANGE(heap_pages_deferred_final, 0);
8821 while (zombie) {
8822 rb_asan_unpoison_object(zombie, false);
8823 VALUE next = RZOMBIE(zombie)->next;
8824 if (FL_TEST_RAW(zombie, FL_FINALIZE)) {
8825 /* re-defer, with the same push as rb_gc_impl_make_zombie */
8826 VALUE prev2, next2 = heap_pages_deferred_final;
8827 do {
8828 RZOMBIE(zombie)->next = prev2 = next2;
8829 next2 = RUBY_ATOMIC_VALUE_CAS(heap_pages_deferred_final, prev2, zombie);
8830 } while (next2 != prev2);
8831 rb_asan_poison_object(zombie);
8832 }
8833 else {
8834 RZOMBIE(zombie)->next = dfree_only;
8835 dfree_only = zombie;
8836 }
8837 zombie = next;
8838 }
8839 if (dfree_only) finalize_list(objspace, dfree_only);
8840 bool did = dfree_only != 0;
8841
8842 gc_tdata_unsafe_free_publish(objspace);
8843 return did;
8844}
8845
8846void
8847rb_gc_impl_objspace_retire_gc(void *objspace_ptr)
8848{
8849 rb_objspace_t *objspace = objspace_ptr;
8850
8851 /* The dying thread's stack is already torn down here, so the root scan must skip
8852 * its machine context (rb_gc_mark_roots). */
8853 objspace->flags.during_postmortem = 1;
8854 objspace->flags.ownerless = 1;
8855
8856 gc_rest(objspace);
8857 gc_start(objspace, GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP);
8858
8859 /* The sweep above turned this heap's dead IO and the like into deferred zombies
8860 * (the per-Ractor stdio holds a page per Ractor otherwise); finalize the C-only
8861 * ones here and re-sweep the nearly-empty heap so their pages detach as empty. */
8862 if (finalize_deferred_dfree_only(objspace)) {
8863 gc_start(objspace, GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP);
8864 }
8865
8866 heap_pages_freeable_pages = objspace->empty_pages_count;
8867 heap_pages_free_unused_pages(objspace);
8868
8869 objspace->flags.during_postmortem = 0;
8870}
8871
8872bool
8873rb_gc_impl_during_postmortem_p(void *objspace_ptr)
8874{
8875 rb_objspace_t *objspace = objspace_ptr;
8876 return objspace->flags.during_postmortem != 0;
8877}
8878
8879static void
8880heap_ready_to_gc(rb_objspace_t *objspace, rb_heap_t *heap)
8881{
8882 if (!heap->free_pages) {
8883 if (!heap_page_allocate_and_initialize(objspace, heap)) {
8884 objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE;
8885 heap_page_allocate_and_initialize(objspace, heap);
8886 }
8887 }
8888}
8889
8890static int
8891ready_to_gc(rb_objspace_t *objspace)
8892{
8893 if ((!objspace->flags.during_postmortem && rb_gc_gc_disabled_global_p()) || dont_gc_val() || during_gc) {
8894 for (int i = 0; i < HEAP_COUNT; i++) {
8895 rb_heap_t *heap = &heaps[i];
8896 heap_ready_to_gc(objspace, heap);
8897 }
8898 return FALSE;
8899 }
8900 else {
8901 return TRUE;
8902 }
8903}
8904
8905static void
8906gc_reset_malloc_info(rb_objspace_t *objspace, bool full_mark)
8907{
8908 gc_prof_set_malloc_info(objspace);
8909 {
8910 int64_t inc = gc_malloc_counters_increase(objspace, &objspace->malloc_counters.counters);
8911 gc_malloc_counters_snapshot(objspace, &objspace->malloc_counters.counters);
8912 size_t old_limit = malloc_limit;
8913
8914 /* A net-negative `inc` (more freed than malloc'd since last GC) is
8915 * treated the same as "allocated less than malloc_limit".
8916 * This matches what we were doing pre-monotonic counters, but is it right? */
8917 if (inc > 0 && (size_t)inc > malloc_limit) {
8918 malloc_limit = (size_t)((size_t)inc * gc_params.malloc_limit_growth_factor);
8919 if (malloc_limit > gc_params.malloc_limit_max) {
8920 malloc_limit = gc_params.malloc_limit_max;
8921 }
8922 }
8923 else {
8924 malloc_limit = (size_t)(malloc_limit * 0.98); /* magic number */
8925 if (malloc_limit < gc_params.malloc_limit_min) {
8926 malloc_limit = gc_params.malloc_limit_min;
8927 }
8928 }
8929
8930 if (0) {
8931 if (old_limit != malloc_limit) {
8932 fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: %"PRIuSIZE" -> %"PRIuSIZE"\n",
8933 rb_gc_count(), old_limit, malloc_limit);
8934 }
8935 else {
8936 fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: not changed (%"PRIuSIZE")\n",
8937 rb_gc_count(), malloc_limit);
8938 }
8939 }
8940 }
8941
8942 /* reset oldmalloc info */
8943#if RGENGC_ESTIMATE_OLDMALLOC
8944 MALLOC_COUNTERS_LOCK(objspace);
8945 gc_counter_store_release(&objspace->malloc_counters.freed_in_gc, 0);
8946 MALLOC_COUNTERS_UNLOCK(objspace);
8947 if (full_mark) {
8948 RUBY_ATOMIC_SIZE_EXCHANGE(objspace->malloc_counters.oldmalloc_increase, 0);
8949
8950 if ((objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) {
8951 objspace->rgengc.oldmalloc_increase_limit =
8952 (size_t)(objspace->rgengc.oldmalloc_increase_limit / ((gc_params.oldmalloc_limit_growth_factor - 1)/10 + 1));
8953 if (objspace->rgengc.oldmalloc_increase_limit < gc_params.oldmalloc_limit_min) {
8954 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
8955 }
8956 }
8957 }
8958#endif
8959}
8960
8961/* What a collection records about itself before it runs. A global collection reports the
8962 * driver's objspace, so it comes through here too. */
8963static void
8964gc_start_record(rb_objspace_t *objspace, unsigned int reason, bool full_mark)
8965{
8966 objspace->profile.latest_gc_info = reason;
8967#if GC_PROFILE_MORE_DETAIL
8968 objspace->profile.total_allocated_objects_at_gc_start = total_allocated_objects(objspace);
8969 objspace->profile.heap_used_at_gc_start = rb_darray_size(objspace->heap_pages.sorted);
8970#endif
8971 objspace->profile.weak_references_count = 0;
8972 gc_prof_setup_new_record(objspace, reason);
8973 gc_reset_malloc_info(objspace, full_mark);
8974}
8975
8976static bool gc_start_global(rb_objspace_t *driver, unsigned int reason, bool compact, bool allow_skip);
8977
8978/* Decide whether this collection has to be global. A local GC can reclaim neither
8979 * shareable objects nor zombie objspaces, so once those grow past their limits only a
8980 * global GC makes progress. All inputs belong to this objspace. */
8981static bool
8982gc_need_global_p(rb_objspace_t *objspace)
8983{
8984 if (rb_gc_single_objspace_p()) return false;
8985 /* A Ractor's death must not stop the world, so the retire GC stays local. */
8986 if (objspace->flags.during_postmortem) return false;
8987 if (objspace->shareable_objects > objspace->shareable_objects_limit) return true;
8988 /* A zombie's garbage only a global cycle reclaims, but what survived the last one
8989 * is live data, so retrigger only once TRIGGER more pages accumulate on top of it.
8990 * Otherwise one live-heavy unjoined zombie turns every GC stop-the-world forever. */
8991 {
8992 size_t zp = global_objspace->zombie_total_pages;
8993 size_t base = global_objspace->zombie_pages_survivors < zp ? global_objspace->zombie_pages_survivors : zp;
8994 if (zp - base >= ZOMBIE_PAGES_TRIGGER) return true;
8995 }
8996 return false;
8997}
8998
8999static int
9000garbage_collect(rb_objspace_t *objspace, unsigned int reason)
9001{
9002 int ret;
9003
9004#if GC_PROFILE_MORE_DETAIL
9005 objspace->profile.prepare_time = getrusage_time();
9006#endif
9007
9008 gc_rest(objspace);
9009
9010#if GC_PROFILE_MORE_DETAIL
9011 objspace->profile.prepare_time = getrusage_time() - objspace->profile.prepare_time;
9012#endif
9013
9014 ret = gc_start(objspace, reason);
9015
9016 return ret;
9017}
9018
9019static int
9020gc_start(rb_objspace_t *objspace, unsigned int reason)
9021{
9022 unsigned int do_full_mark = !!(reason & GPR_FLAG_FULL_MARK);
9023
9024 if (!rb_darray_size(objspace->heap_pages.sorted)) return TRUE; /* heap is not ready */
9025 if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */
9026
9027 /* An explicit GC.start(global: true) never gets here: rb_gc_impl_start has already decided from
9028 * the `global` keyword, and GPR_FLAG_METHOD keeps `global: false` from being promoted back. */
9029 if (!(reason & GPR_FLAG_METHOD) && gc_need_global_p(objspace)) {
9030 /* A global GC is always a major, so autocompact applies. */
9031 if (gc_start_global(objspace, reason, ruby_enable_autocompact, true)) {
9032 return TRUE;
9033 }
9034 /* Fall through to a local GC */
9035 }
9036
9037 rb_gc_initialize_vm_context(&objspace->vm_context);
9038
9039 GC_ASSERT(gc_mode(objspace) == gc_mode_none, "gc_mode is %s\n", gc_mode_name(gc_mode(objspace)));
9040 GC_ASSERT(!is_lazy_sweeping(objspace));
9041 GC_ASSERT(!is_incremental_marking(objspace));
9042
9043 /* reason may be clobbered, later, so keep set immediate_sweep here */
9044 objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP);
9045
9046 if (ruby_gc_stressful) {
9047 int flag = FIXNUM_P(ruby_gc_stress_mode) ? FIX2INT(ruby_gc_stress_mode) : 0;
9048
9049 if ((flag & (1 << gc_stress_no_major)) == 0) {
9050 do_full_mark = TRUE;
9051 }
9052
9053 objspace->flags.immediate_sweep = !(flag & (1<<gc_stress_no_immediate_sweep));
9054 }
9055
9056 if (gc_needs_major_flags) {
9057 reason |= gc_needs_major_flags;
9058 do_full_mark = TRUE;
9059 }
9060
9061 /* if major gc has been disabled, never do a full mark */
9062 if (!gc_config_full_mark_val) {
9063 do_full_mark = FALSE;
9064 }
9065 gc_needs_major_flags = GPR_FLAG_NONE;
9066
9067 if (do_full_mark && (reason & GPR_FLAG_MAJOR_MASK) == 0) {
9068 reason |= GPR_FLAG_MAJOR_BY_FORCE; /* GC by CAPI, METHOD, and so on. */
9069 }
9070
9071 if (objspace->flags.dont_incremental ||
9072 reason & GPR_FLAG_IMMEDIATE_MARK ||
9073 ruby_gc_stressful) {
9074 gc_set_incremental_marking(objspace, FALSE);
9075 }
9076 else {
9077 gc_set_incremental_marking(objspace, do_full_mark);
9078 }
9079
9080 /* Compaction on the local GC path (autocompact) runs only with a single objspace:
9081 * without the stop-the-world barrier, moving objects would break cross-objspace
9082 * references. With multiple objspaces GC.compact and autocompact go through the
9083 * compacting global GC instead (rb_gc_impl_start, or the promotion above). */
9084 if (do_full_mark && ruby_enable_autocompact && rb_gc_single_objspace_p()) {
9085 objspace->flags.during_compacting = TRUE;
9086#if RGENGC_CHECK_MODE
9087 objspace->rcompactor.compare_func = ruby_autocompact_compare_func;
9088#endif
9089 }
9090 else {
9091 objspace->flags.during_compacting = !!(reason & GPR_FLAG_COMPACT);
9092 if (objspace->flags.during_compacting && !rb_gc_single_objspace_p()) {
9093 // compaction is currently global GC only with more than 1 running Ractor
9094 objspace->flags.during_compacting = FALSE;
9095 }
9096 }
9097
9098 if (!GC_ENABLE_LAZY_SWEEP || objspace->flags.dont_incremental) {
9099 objspace->flags.immediate_sweep = TRUE;
9100 }
9101
9102 if (objspace->flags.immediate_sweep) reason |= GPR_FLAG_IMMEDIATE_SWEEP;
9103
9104 /* Enter after during_compacting is decided */
9105 unsigned int lock_lev;
9106 gc_enter(objspace, gc_enter_event_start, &lock_lev);
9107
9108 gc_report(1, objspace, "gc_start(reason: %x) => %u, %d, %d\n",
9109 reason,
9110 do_full_mark, !is_incremental_marking(objspace), objspace->flags.immediate_sweep);
9111
9112 RB_DEBUG_COUNTER_INC(gc_count);
9113
9114 if (reason & GPR_FLAG_MAJOR_MASK) {
9115 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_nofree, reason & GPR_FLAG_MAJOR_BY_NOFREE);
9116 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldgen, reason & GPR_FLAG_MAJOR_BY_OLDGEN);
9117 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_shady, reason & GPR_FLAG_MAJOR_BY_SHADY);
9118 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_force, reason & GPR_FLAG_MAJOR_BY_FORCE);
9119#if RGENGC_ESTIMATE_OLDMALLOC
9120 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldmalloc, reason & GPR_FLAG_MAJOR_BY_OLDMALLOC);
9121#endif
9122 }
9123 else {
9124 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_newobj, reason & GPR_FLAG_NEWOBJ);
9125 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_malloc, reason & GPR_FLAG_MALLOC);
9126 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_method, reason & GPR_FLAG_METHOD);
9127 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_capi, reason & GPR_FLAG_CAPI);
9128 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_stress, reason & GPR_FLAG_STRESS);
9129 }
9130
9131 objspace->profile.count++;
9132 gc_start_record(objspace, reason, do_full_mark);
9133
9134 gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_START);
9135
9136 GC_ASSERT(during_gc);
9137
9138 gc_prof_timer_start(objspace);
9139 {
9140 if (gc_marks(objspace, do_full_mark)) {
9141 gc_sweep(objspace);
9142 }
9143 }
9144 gc_prof_timer_stop(objspace);
9145
9146 gc_exit(objspace, gc_enter_event_start, &lock_lev);
9147
9148 /* Verify after the GC, at a real safepoint with during_gc cleared: mid-GC
9149 * gc_verify_internal_consistency() takes the VM lock and a barrier, which would join
9150 * another Ractor's global GC barrier and let it collect on this half-collected heap. */
9151#if RGENGC_CHECK_MODE >= 2
9152 gc_verify_internal_consistency(objspace);
9153#endif
9154 return TRUE;
9155}
9156
9157static void
9158gc_rest(rb_objspace_t *objspace)
9159{
9160 if (is_incremental_marking(objspace) || is_lazy_sweeping(objspace)) {
9161 unsigned int lock_lev;
9162 gc_enter(objspace, gc_enter_event_rest, &lock_lev);
9163
9164 if (is_incremental_marking(objspace)) {
9165 gc_marking_enter(objspace);
9166 gc_marks_rest(objspace);
9167 gc_marking_exit(objspace);
9168
9169 gc_sweep(objspace);
9170 }
9171
9172 if (is_lazy_sweeping(objspace)) {
9173 gc_sweeping_enter(objspace);
9174 gc_sweep_rest(objspace);
9175 gc_sweeping_exit(objspace);
9176 }
9177
9178 gc_exit(objspace, gc_enter_event_rest, &lock_lev);
9179
9180 if (RGENGC_CHECK_MODE >= 2) gc_verify_internal_consistency(objspace); /* after GC, see gc_start */
9181 }
9182}
9183
9185 rb_objspace_t *objspace;
9186 unsigned int reason;
9187};
9188
9189static void
9190gc_current_status_fill(rb_objspace_t *objspace, char *buff)
9191{
9192 int i = 0;
9193 if (is_marking(objspace)) {
9194 buff[i++] = 'M';
9195 if (is_full_marking(objspace)) buff[i++] = 'F';
9196 if (is_incremental_marking(objspace)) buff[i++] = 'I';
9197 }
9198 else if (is_sweeping(objspace)) {
9199 buff[i++] = 'S';
9200 if (is_lazy_sweeping(objspace)) buff[i++] = 'L';
9201 }
9202 else {
9203 buff[i++] = 'N';
9204 }
9205 buff[i] = '\0';
9206}
9207
9208static const char *
9209gc_current_status(rb_objspace_t *objspace)
9210{
9211 static char buff[0x10];
9212 gc_current_status_fill(objspace, buff);
9213 return buff;
9214}
9215
9216#if PRINT_ENTER_EXIT_TICK
9217
9218static tick_t last_exit_tick;
9219static tick_t enter_tick;
9220static int enter_count = 0;
9221static char last_gc_status[0x10];
9222
9223static inline void
9224gc_record(rb_objspace_t *objspace, int direction, const char *event)
9225{
9226 if (direction == 0) { /* enter */
9227 enter_count++;
9228 enter_tick = tick();
9229 gc_current_status_fill(objspace, last_gc_status);
9230 }
9231 else { /* exit */
9232 tick_t exit_tick = tick();
9233 char current_gc_status[0x10];
9234 gc_current_status_fill(objspace, current_gc_status);
9235#if 1
9236 /* [last mutator time] [gc time] [event] */
9237 fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
9238 enter_tick - last_exit_tick,
9239 exit_tick - enter_tick,
9240 event,
9241 last_gc_status, current_gc_status,
9242 (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
9243 last_exit_tick = exit_tick;
9244#else
9245 /* [enter_tick] [gc time] [event] */
9246 fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
9247 enter_tick,
9248 exit_tick - enter_tick,
9249 event,
9250 last_gc_status, current_gc_status,
9251 (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
9252#endif
9253 }
9254}
9255#else /* PRINT_ENTER_EXIT_TICK */
9256static inline void
9257gc_record(rb_objspace_t *objspace, int direction, const char *event)
9258{
9259 /* null */
9260}
9261#endif /* PRINT_ENTER_EXIT_TICK */
9262
9263static const char *
9264gc_enter_event_cstr(enum gc_enter_event event)
9265{
9266 switch (event) {
9267 case gc_enter_event_start: return "start";
9268 case gc_enter_event_continue: return "continue";
9269 case gc_enter_event_rest: return "rest";
9270 case gc_enter_event_finalizer: return "finalizer";
9271 case gc_enter_event_global: return "global";
9272 case gc_enter_event_global_auto: return "global_auto";
9273 }
9274 return NULL;
9275}
9276
9277static void
9278gc_enter_count(enum gc_enter_event event)
9279{
9280 switch (event) {
9281 case gc_enter_event_start: RB_DEBUG_COUNTER_INC(gc_enter_start); break;
9282 case gc_enter_event_continue: RB_DEBUG_COUNTER_INC(gc_enter_continue); break;
9283 case gc_enter_event_rest: RB_DEBUG_COUNTER_INC(gc_enter_rest); break;
9284 case gc_enter_event_finalizer: RB_DEBUG_COUNTER_INC(gc_enter_finalizer); break;
9285 case gc_enter_event_global: RB_DEBUG_COUNTER_INC(gc_enter_start); break;
9286 case gc_enter_event_global_auto: RB_DEBUG_COUNTER_INC(gc_enter_start); break;
9287 }
9288}
9289
9290static bool current_process_time(struct timespec *ts);
9291
9292/* A gc phase must be timed on the collecting thread's own cpu. A local gc runs
9293 * while the other ractors keep going, and process cpu time counts their work as
9294 * gc: with eight busy ractors the same ten collections were reported as 131ms
9295 * instead of 3ms, more than the wall clock they ran in. The kernel also answers
9296 * this one without walking every thread in the process. */
9297static bool
9298current_thread_time(struct timespec *ts)
9299{
9300#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_THREAD_CPUTIME_ID)
9301 {
9302 static int try_clock_gettime = 1;
9303 if (try_clock_gettime) {
9304 if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, ts) == 0) {
9305 return true;
9306 }
9307 else {
9308 try_clock_gettime = 0;
9309 }
9310 }
9311 }
9312#endif
9313 return current_process_time(ts);
9314}
9315
9316static void
9317gc_clock_start(struct timespec *ts)
9318{
9319 if (!current_thread_time(ts)) {
9320 ts->tv_sec = 0;
9321 ts->tv_nsec = 0;
9322 }
9323}
9324
9325static unsigned long long
9326gc_clock_end(struct timespec *ts)
9327{
9328 struct timespec end_time;
9329
9330 if ((ts->tv_sec > 0 || ts->tv_nsec > 0) &&
9331 current_thread_time(&end_time) &&
9332 end_time.tv_sec >= ts->tv_sec) {
9333 return (unsigned long long)(end_time.tv_sec - ts->tv_sec) * (1000 * 1000 * 1000) +
9334 (end_time.tv_nsec - ts->tv_nsec);
9335 }
9336
9337 return 0;
9338}
9339
9340static void
9341gc_process_stat_after_fork_i(void *objspace_ptr, void *data)
9342{
9343 rb_objspace_t *objspace = objspace_ptr;
9344 rb_native_mutex_initialize(&objspace->process_stat.lock);
9345}
9346
9347static inline bool
9348gc_local_gc_holds_vm_lock(void)
9349{
9350 return rb_gc_single_objspace_p();
9351}
9352
9353static inline bool
9354gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev)
9355{
9356 /*
9357 * NOTE: The GC must never take the barrier VM lock from inside itself: the waiter could
9358 * join a pending barrier mid-collection and expose its half-collected heap to the global
9359 * GC. A no-barrier lock is safe. Other shared structures the GC paths touch use their own
9360 * native mutexes or the page-pool lock. */
9361 *lock_lev = 0;
9362
9363 RUBY_DTRACE_GC_HOOK(ENTER, event);
9364
9365 if (objspace->profile.run) {
9366 switch (event) {
9367 case gc_enter_event_start:
9368 case gc_enter_event_continue:
9369 case gc_enter_event_rest:
9370 case gc_enter_event_global:
9371 case gc_enter_event_global_auto:
9372 /* A global GC is the longest pause the process takes, so it is the last thing
9373 * the profiler may leave unmeasured. The switch below stops the world for it,
9374 * which is exactly the interval gc_stop_time is meant to name, so start the
9375 * clock here like a local collection does. */
9376 objspace->profile.gc_pause_start_time = rb_hrtime_now();
9377 break;
9378 case gc_enter_event_finalizer:
9379 break;
9380 }
9381 }
9382 switch (event) {
9383 case gc_enter_event_global:
9384 *lock_lev = RB_GC_VM_LOCK();
9385 // stop other ractors
9386 rb_gc_vm_barrier();
9387 break;
9388 case gc_enter_event_global_auto:
9389 *lock_lev = RB_GC_VM_LOCK();
9390 if (!gc_need_global_p(objspace)) {
9391 RB_GC_VM_UNLOCK(*lock_lev);
9392 *lock_lev = 0;
9393 objspace->profile.gc_pause_start_time = 0;
9394 return false;
9395 }
9396 rb_gc_vm_barrier();
9397 break;
9398 case gc_enter_event_finalizer:
9399 /* Shutdown finalizers read VM-global tables (fstring, symbol) and free T_DATA that
9400 * is not thread-safe, so take the no-barrier VM lock. */
9401 *lock_lev = RB_GC_VM_LOCK_NO_BARRIER();
9402 break;
9403 default:
9404 objspace->flags.gc_lock_barrier = FALSE;
9405 if (objspace->flags.during_compacting) {
9406 /* Compaction relocates objects and rewrites every Ractor's JIT and global
9407 * references, so it stops the world with a barrier VM lock. rb_gc_vm_barrier is
9408 * a reentrant no-op with a single Ractor, so an inner barrier request during the
9409 * move folds into this one and gc_exit ends it. */
9410 *lock_lev = RB_GC_VM_LOCK();
9411 rb_gc_vm_barrier();
9412 objspace->flags.gc_lock_barrier = TRUE;
9413 }
9414 else if (gc_local_gc_holds_vm_lock()) {
9415 *lock_lev = RB_GC_VM_LOCK_NO_BARRIER();
9416 }
9417 break;
9418 }
9419
9420 if (objspace->profile.gc_pause_start_time) {
9421 objspace->profile.gc_stw_start_time = rb_hrtime_now();
9422 objspace->profile.gc_stop_time = rb_hrtime_sub(
9423 objspace->profile.gc_stw_start_time,
9424 objspace->profile.gc_pause_start_time);
9425 }
9426
9427 gc_enter_count(event);
9428 if (RB_UNLIKELY(during_gc != 0)) rb_bug("during_gc != 0");
9429 if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace);
9430
9431 during_gc = TRUE;
9432 RUBY_DEBUG_LOG("%s (%s)",gc_enter_event_cstr(event), gc_current_status(objspace));
9433 gc_report(1, objspace, "gc_enter: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace));
9434 gc_record(objspace, 0, gc_enter_event_cstr(event));
9435
9436 gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_ENTER);
9437 return true;
9438}
9439
9440static inline void
9441gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev)
9442{
9443 GC_ASSERT(during_gc != 0);
9444
9445 RUBY_DTRACE_GC_HOOK(EXIT, event);
9446
9447 gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_EXIT);
9448
9449 if (objspace->profile.gc_pause_start_time) {
9450 if (gc_prof_enabled(objspace)) {
9451 rb_hrtime_t now = rb_hrtime_now();
9452 gc_profile_record *record = gc_prof_record(objspace);
9453 record->gc_pause_time = rb_hrtime_add(record->gc_pause_time,
9454 rb_hrtime_sub(now, objspace->profile.gc_pause_start_time));
9455 record->gc_stop_time = rb_hrtime_add(record->gc_stop_time,
9456 objspace->profile.gc_stop_time);
9457 record->gc_stw_time = rb_hrtime_add(record->gc_stw_time,
9458 rb_hrtime_sub(now, objspace->profile.gc_stw_start_time));
9459 }
9460 objspace->profile.gc_pause_start_time = 0;
9461 objspace->profile.gc_stw_start_time = 0;
9462 objspace->profile.gc_stop_time = 0;
9463 }
9464
9465 gc_record(objspace, 1, gc_enter_event_cstr(event));
9466 RUBY_DEBUG_LOG("%s (%s)", gc_enter_event_cstr(event), gc_current_status(objspace));
9467 gc_report(1, objspace, "gc_exit: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace));
9468 during_gc = FALSE;
9469 gc_process_stat_publish(objspace);
9470
9471 switch (event) {
9472 case gc_enter_event_global:
9473 case gc_enter_event_global_auto:
9474 RB_GC_VM_UNLOCK(*lock_lev);
9475 break;
9476 case gc_enter_event_finalizer:
9477 RB_GC_VM_UNLOCK_NO_BARRIER(*lock_lev);
9478 break;
9479 default:
9480 if (*lock_lev != 0) {
9481 if (objspace->flags.gc_lock_barrier) {
9482 objspace->flags.gc_lock_barrier = FALSE;
9483 RB_GC_VM_UNLOCK(*lock_lev);
9484 }
9485 else {
9486 RB_GC_VM_UNLOCK_NO_BARRIER(*lock_lev);
9487 }
9488 }
9489 break;
9490 }
9491}
9492
9493#ifndef MEASURE_GC
9494#define MEASURE_GC (objspace->flags.measure_gc)
9495#endif
9496
9497static void
9498gc_marking_enter(rb_objspace_t *objspace)
9499{
9500 GC_ASSERT(during_gc != 0);
9501
9502 gc_prof_mark_timer_start(objspace);
9503
9504 if (gc_prof_enabled(objspace)) {
9505 objspace->profile.gc_mark_phase_wall_start_time = rb_hrtime_now();
9506 }
9507
9508 if (MEASURE_GC) {
9509 gc_clock_start(&objspace->profile.marking_start_time);
9510 }
9511
9512 rb_gc_initialize_vm_context(&objspace->vm_context);
9513}
9514
9515static void
9516gc_marking_exit(rb_objspace_t *objspace)
9517{
9518 GC_ASSERT(during_gc != 0);
9519
9520 if (MEASURE_GC) {
9521 objspace->profile.marking_time_ns += gc_clock_end(&objspace->profile.marking_start_time);
9522 }
9523
9524 if (gc_prof_enabled(objspace)) {
9525 gc_profile_record *record = gc_prof_record(objspace);
9526 record->gc_mark_wall_time = rb_hrtime_add(record->gc_mark_wall_time,
9527 elapsed_hrtime_from(objspace->profile.gc_mark_phase_wall_start_time));
9528 }
9529
9530 gc_prof_mark_timer_stop(objspace);
9531}
9532
9533static void
9534gc_sweeping_cpu_enter(rb_objspace_t *objspace)
9535{
9536 if (MEASURE_GC) {
9537 gc_clock_start(&objspace->profile.sweeping_start_time);
9538 }
9539}
9540
9541static void
9542gc_sweeping_cpu_exit(rb_objspace_t *objspace)
9543{
9544 if (MEASURE_GC) {
9545 objspace->profile.sweeping_time_ns += gc_clock_end(&objspace->profile.sweeping_start_time);
9546 }
9547}
9548
9549static void
9550gc_sweeping_enter(rb_objspace_t *objspace)
9551{
9552 GC_ASSERT(during_gc != 0);
9553
9554 if (gc_prof_enabled(objspace)) {
9555 objspace->profile.gc_sweep_phase_wall_start_time = rb_hrtime_now();
9556 objspace->profile.gc_sweep_excluded_wall_time = 0;
9557 }
9558
9559 gc_sweeping_cpu_enter(objspace);
9560
9561 rb_gc_initialize_vm_context(&objspace->vm_context);
9562}
9563
9564static void
9565gc_sweeping_exit(rb_objspace_t *objspace)
9566{
9567 GC_ASSERT(during_gc != 0);
9568
9569 gc_sweeping_cpu_exit(objspace);
9570
9571 if (gc_prof_enabled(objspace)) {
9572 rb_hrtime_t sweep_wall_time = elapsed_hrtime_from(objspace->profile.gc_sweep_phase_wall_start_time);
9573 gc_profile_record *record = gc_prof_record(objspace);
9574 sweep_wall_time = rb_hrtime_sub(sweep_wall_time,
9575 objspace->profile.gc_sweep_excluded_wall_time);
9576 record->gc_sweep_wall_time = rb_hrtime_add(record->gc_sweep_wall_time,
9577 sweep_wall_time);
9578 objspace->profile.gc_sweep_excluded_wall_time = 0;
9579 }
9580}
9581
9582static void *
9583gc_with_gvl(void *ptr)
9584{
9585 struct objspace_and_reason *oar = (struct objspace_and_reason *)ptr;
9586 return (void *)(VALUE)garbage_collect(oar->objspace, oar->reason);
9587}
9588
9589int ruby_thread_has_gvl_p(void);
9590
9591static int
9592garbage_collect_with_gvl(rb_objspace_t *objspace, unsigned int reason)
9593{
9594 if (rb_gc_gc_disabled_global_p() || dont_gc_val()) {
9595 return TRUE;
9596 }
9597 else if (!ruby_native_thread_p()) {
9598 return TRUE;
9599 }
9600 else if (!ruby_thread_has_gvl_p()) {
9601 void *ret;
9602 struct objspace_and_reason oar;
9603 oar.objspace = objspace;
9604 oar.reason = reason;
9605 ret = rb_thread_call_with_gvl(gc_with_gvl, (void *)&oar);
9606
9607 return !!ret;
9608 }
9609 else {
9610 return garbage_collect(objspace, reason);
9611 }
9612}
9613
9614static int
9615gc_set_candidate_object_i(void *vstart, void *vend, size_t stride, void *data)
9616{
9617 rb_objspace_t *objspace = (rb_objspace_t *)data;
9618
9619 VALUE v = (VALUE)vstart;
9620 for (; v != (VALUE)vend; v += stride) {
9621 asan_unpoisoning_object(v) {
9622 switch (BUILTIN_TYPE(v)) {
9623 case T_NONE:
9624 case T_ZOMBIE:
9625 break;
9626 default:
9627 rb_gc_prepare_heap_process_object(v);
9628 if (!RVALUE_OLD_P(objspace, v) && !RVALUE_WB_UNPROTECTED(objspace, v)) {
9629 RVALUE_AGE_SET_CANDIDATE(objspace, v);
9630 }
9631 }
9632 }
9633 }
9634
9635 return 0;
9636}
9637
9638bool
9639rb_gc_impl_multi_objspace_p(void)
9640{
9641 return true;
9642}
9643
9644bool
9645rb_gc_impl_during_global_gc_p(void *objspace_ptr)
9646{
9647 rb_objspace_t *objspace = objspace_ptr;
9648 return objspace->flags.during_global_gc != 0;
9649}
9650
9651bool
9652rb_gc_impl_obj_foreign_p(void *objspace_ptr, VALUE obj)
9653{
9654 if (RB_LIKELY(!rb_gc_ever_multi_ractor_p())) return false;
9655 // may run without the VM lock (rb_class_field_shareable_p); paired with the store in an absorb
9656 return RUBY_ATOMIC_PTR_LOAD(GET_HEAP_PAGE(obj)->objspace) != objspace_ptr;
9657}
9658
9659void
9660rb_gc_impl_objspace_set_ractor(void *objspace_ptr, void *ractor)
9661{
9662 ((rb_objspace_t *)objspace_ptr)->ractor = ractor;
9663}
9664
9665// the absorb that moves a page to another objspace runs under the VM lock
9666void *
9667rb_gc_impl_obj_ractor(void *objspace_ptr, VALUE obj)
9668{
9669 return GET_HEAP_OBJSPACE(obj)->ractor;
9670}
9671
9672
9673/* Whether obj is recorded as an unshareable object referenced from a shareable one. For
9674 * the verifier: a shareable -> unshareable edge is only accepted if the write barrier
9675 * recorded it here. */
9676bool
9677rb_gc_impl_shref_marked_p(void *objspace_ptr, VALUE obj)
9678{
9679 return MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj) != 0;
9680}
9681
9682/* What a zombie objspace is worth to the zombie page trigger, floored: the retire GC
9683 * can empty an ownerless objspace outright, yet its struct, mark stack and heap_page
9684 * structs stay in malloc and it keeps the world multi-objspace until a global GC
9685 * merges it, so a page-less zombie must still push the trigger. Never 0, so a
9686 * nonzero zombie_pages also marks the objspace as a zombie. */
9687static size_t
9688objspace_zombie_pages(rb_objspace_t *objspace)
9689{
9690 size_t pages = rb_darray_size(objspace->heap_pages.sorted);
9691 return pages > HEAP_COUNT ? pages : HEAP_COUNT;
9692}
9693
9694/* gc.c parked the objspace in its zombie table, under the VM lock. */
9695void
9696rb_gc_impl_objspace_zombify(void *objspace_ptr)
9697{
9698 rb_objspace_t *objspace = objspace_ptr;
9699 GC_ASSERT(objspace->zombie_pages == 0);
9700 objspace->zombie_pages = objspace_zombie_pages(objspace);
9701 global_objspace->zombie_total_pages += objspace->zombie_pages;
9702}
9703
9704/* Re-measure every zombie this cycle covered, by difference, so a zombie parked after
9705 * the snapshot keeps the pages it was parked with. Inside the barrier. */
9706static void
9707gc_global_refresh_zombie_pages(void)
9708{
9709 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9710 rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
9711 if (objspace->zombie_pages == 0) continue;
9712 size_t pages = objspace_zombie_pages(objspace);
9713 global_objspace->zombie_total_pages -= objspace->zombie_pages;
9714 global_objspace->zombie_total_pages += pages;
9715 objspace->zombie_pages = pages;
9716 }
9717 global_objspace->zombie_pages_survivors = global_objspace->zombie_total_pages;
9718}
9719
9720static void
9721gc_global_objspaces_i(void *os, void *data)
9722{
9723 if (global_objspace->global_gc.n_objspaces == global_objspace->global_gc.objspaces_capa) {
9724 size_t new_capa = global_objspace->global_gc.objspaces_capa ? global_objspace->global_gc.objspaces_capa * 2 : 16;
9725 struct rb_objspace **new_list = realloc(global_objspace->global_gc.objspaces, new_capa * sizeof(*new_list));
9726 if (new_list == NULL) rb_bug("gc_global_objspaces_i: realloc failed");
9727 global_objspace->global_gc.objspaces = new_list;
9728 global_objspace->global_gc.objspaces_capa = new_capa;
9729 }
9730 global_objspace->global_gc.objspaces[global_objspace->global_gc.n_objspaces++] = os;
9731}
9732
9733/* Re-snapshot every objspace this cycle covers, zombies included. The objspaces/capa
9734 * buffer is reused from the previous cycle. */
9735static void
9736gc_global_snapshot_objspaces(void)
9737{
9738 global_objspace->global_gc.n_objspaces = 0;
9739 rb_gc_vm_each_objspace(gc_global_objspaces_i, NULL);
9740
9741#if RGENGC_CHECK_MODE
9742 /* Check that the incrementally maintained page_index agrees with the per-objspace
9743 * sorted arrays. */
9744 size_t total = 0, zombie_total = 0;
9745 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9746 total += rb_darray_size(global_objspace->global_gc.objspaces[i]->heap_pages.sorted);
9747 zombie_total += global_objspace->global_gc.objspaces[i]->zombie_pages;
9748 }
9749 GC_ASSERT(total == global_objspace->page_index.n_pages);
9750 /* Every zombie is enumerated, so the snapshot sees all of zombie_total_pages. */
9751 GC_ASSERT(zombie_total == global_objspace->zombie_total_pages);
9752#endif
9753}
9754
9755/* Global GC: stop every Ractor and clear/mark/sweep all objspaces as one heap. It is the
9756 * only collector that can free shareable objects and decide cross-objspace reachability
9757 * precisely. */
9758/* The global GC's generic_fields weak pass, after the unified mark fixpoint, before the
9759 * sweep. Per-object rb_mark_generic_ivar is a no-op during a global GC (the driver has
9760 * GET_RACTOR() != owner); the whole table is swept here instead. Weak-KEY: mark the val
9761 * (fields_obj, a strong child) only for a live key, drain dead keys' entries. Marking a
9762 * val can make another key live, so repeat to a fixpoint. */
9764 rb_objspace_t *objspace;
9765 bool progress;
9766};
9767
9768static int
9769genfields_mark_i(VALUE key, VALUE val, void *arg)
9770{
9771 struct genfields_mark_arg *a = (struct genfields_mark_arg *)arg;
9772 if (RB_SPECIAL_CONST_P(val) || !RVALUE_MARKED_BITMAP(key)) {
9773 return ST_CONTINUE;
9774 }
9775 /* Record the old(key)->young(val) edge with the host (key) as parent, even when val
9776 * is already marked: a conservative machine-stack scan can mark a fresh fields_obj
9777 * parentless before this pass, and branching on the mark bit would leave the key
9778 * unremembered, so the next minor GC misses the young val ("WB miss (O->Y)").
9779 * gc_mark runs rgengc_check_relation before its already-marked return: call always. */
9780 bool newly = !RVALUE_MARKED_BITMAP(val);
9781 gc_mark_set_parent(a->objspace, key);
9782 gc_mark(a->objspace, val);
9783 if (newly) a->progress = true;
9784 return ST_CONTINUE;
9785}
9786
9787static bool
9788genfields_dead_p(VALUE key)
9789{
9790 return RVALUE_MARKED_BITMAP(key) == 0;
9791}
9792
9793static void
9794gc_global_mark_generic_fields(rb_objspace_t *driver)
9795{
9796 struct genfields_mark_arg arg = { driver, false };
9797 do {
9798 arg.progress = false;
9799 /* Each entry's mark sets parent=key (genfields_mark_i) so the generational WB is
9800 * recorded correctly. gc_mark_stacked_objects_all sets its own per-object parent,
9801 * so restore the invalid parent (the poison contract) before calling it. */
9802 rb_gc_vm_generic_fields_mark_foreach(genfields_mark_i, &arg);
9803 gc_mark_set_parent_invalid(driver);
9804 if (arg.progress) {
9805 gc_mark_stacked_objects_all(driver);
9806 }
9807 } while (arg.progress);
9808
9809 rb_gc_vm_generic_fields_drain_dead(genfields_dead_p);
9810}
9811
9812/* Two Ractors choosing a global GC at once are serialized by the VM lock in gc_enter. If two
9813 * globals start concurrently, only one global will run and the other will run a local GC after
9814 * the barrier ends. */
9815static bool
9816gc_start_global(rb_objspace_t *driver, unsigned int reason, bool compact, bool allow_skip)
9817{
9818 unsigned int lock_lev;
9819 enum gc_enter_event event = allow_skip ? gc_enter_event_global_auto : gc_enter_event_global;
9820 if (!gc_enter(driver, event, &lock_lev)) {
9821 return false;
9822 }
9823
9824 reason |= GPR_FLAG_GLOBAL;
9825
9826 /* A global GC is a collection of the driver's objspace too, so report it like a local
9827 * one: it records a profile entry and reports what triggered it. */
9828 gc_start_record(driver, reason, true);
9829 gc_event_hook(driver, RUBY_INTERNAL_EVENT_GC_START);
9830 gc_prof_timer_start(driver);
9831
9832 gc_global_snapshot_objspaces();
9833
9834 /* Mark every objspace as in a global GC before step 3 settles the lazy sweeps: the
9835 * settle frees other objspaces' garbage on the driver thread, and
9836 * rb_free_generic_ivar must see "global GC in progress" to defer generic_fields
9837 * removal to the weak-pass drain. */
9838 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9839 global_objspace->global_gc.objspaces[i]->flags.during_global_gc = TRUE;
9840 }
9841
9842 /* A global GC collects every objspace, so each needs the malloc-counter reset the
9843 * driver got in gc_start_record; without it, gc_sweep_finish advancing free_at_last_gc
9844 * (step 9) would leave their malloc_increase overstated by everything swept here. */
9845 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9846 rb_objspace_t *const os = global_objspace->global_gc.objspaces[i];
9847 if (os != driver) {
9848 os->profile.latest_gc_info = reason;
9849 gc_reset_malloc_info(os, true);
9850 }
9851 }
9852
9853 /* step 3: settle every lazy sweep so the mark bits' meaning is fixed before the clear
9854 * below. (during_gc is a macro over the local "objspace".) rb_gc_get_ec() resolves
9855 * through objspace->vm_context during a GC, so initialize it for all: the driver
9856 * thread runs every objspace's phases. */
9857 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9858 rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
9859 /* The barrier can stop a Ractor between the steps of its own incremental mark, and
9860 * nothing else finishes another objspace's mark. Drop the partial mark rather
9861 * than clear its flags in step 5 under a live gray stack: the owner would resume
9862 * with a mark stack into a heap this GC has since re-marked and swept. Nothing is
9863 * lost, the unified mark below redoes the work. */
9864 if (is_incremental_marking(objspace)) {
9865 gc_abort_incremental_marking(objspace);
9866 }
9867 GC_ASSERT(!is_incremental_marking(objspace));
9868 GC_ASSERT(is_mark_stack_empty(&objspace->mark_stack));
9869 rb_gc_initialize_vm_context(&objspace->vm_context);
9870 if (objspace != driver) during_gc = TRUE;
9871 gc_sweep_rest(objspace);
9872 }
9873
9874 /* step 5: clear every objspace's mark bits, remembered sets, generation counters and
9875 * shrefs (missing even one leaves a stale mark bit and a UAF). (heaps is a macro over
9876 * the local "objspace".) */
9877 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9878 rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
9879 objspace->flags.during_minor_gc = FALSE;
9880 gc_set_incremental_marking(objspace, FALSE);
9881 /* The unified mark is precise and does not pin, so the per-objspace sweep below must
9882 * not re-check against a stale local cycle. */
9883 objspace->last_cycle_pinned = 0;
9884 objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
9885 objspace->rgengc.old_objects = 0;
9886 objspace->profile.count++;
9887 objspace->profile.global_gc_count++;
9888 objspace->rgengc.last_major_gc = objspace->profile.count;
9889 objspace->rgengc.need_major_gc = GPR_FLAG_NONE;
9890 objspace->marked_slots = 0;
9891 for (int h = 0; h < HEAP_COUNT; h++) {
9892 rb_heap_t *heap = &heaps[h];
9893 gc_bitmaps_clear(objspace, heap, true);
9894 heap_move_pooled_pages_to_free_pages(heap);
9895 }
9896 }
9897 driver->profile.global_gc_invocation_count++;
9898 global_objspace->global_gc.count++;
9899
9900 /* Enable compaction in every objspace before the mark: the unified conservative root
9901 * scan then pins machine-stack referents (gc_pin only pins while during_compacting)
9902 * and step 9's sweep relocates the rest. global_gc.compacting defers the
9903 * reference-update phase to phase 2 below (two phases, safe across objspaces). */
9904 global_objspace->global_gc.compacting = compact;
9905 if (compact) {
9906 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9907 rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
9908 objspace->flags.during_compacting = TRUE;
9909#if RGENGC_CHECK_MODE
9910 if (ruby_enable_autocompact) {
9911 objspace->rcompactor.compare_func = ruby_autocompact_compare_func;
9912 }
9913#endif
9914 /* A global GC skips gc_marks_start, which is what resets pinned_slots for a
9915 * compacting local GC, so reset it here. step 5 cleared pinned_bits; the
9916 * conservative mark re-pins machine-stack referents. */
9917 for (int h = 0; h < HEAP_COUNT; h++) {
9918 struct heap_page *page = NULL;
9919 ccan_list_for_each(&heaps[h].pages, page, page_node) {
9920 page->pinned_slots = 0;
9921 }
9922 }
9923 }
9924 }
9925
9926 /* steps 6-7: every Ractor's roots (gc.c walks them all), then one unified precise
9927 * mark. A global GC does not go through gc_marks, so the marking
9928 * phase is opened here instead; it closes after rb_ractor_finish_marking below, which is
9929 * where gc_marks_finish ends for a local collection. */
9930 gc_marking_enter(driver);
9931
9932 mark_roots(driver, NULL);
9933 gc_mark_stacked_objects_all(driver);
9934
9935 /* Run the generic_fields weak pass after the mark fixpoint: mark the vals (fields_obj)
9936 * of live keys and drain the entries of dead ones. The per-object rb_mark_generic_ivar
9937 * is a no-op during a global GC, so this is the only path that marks generic_fields. */
9938 gc_global_mark_generic_fields(driver);
9939
9940 gc_event_hook(driver, RUBY_INTERNAL_EVENT_GC_END_MARK);
9941
9942 /* step 8 */
9943 gc_update_weak_references(driver);
9944
9945 /* This cycle's root pass over every Ractor has swept the deleted ractor-local keys out of
9946 * each storage. Free the key structs while still inside the barrier (a local GC never
9947 * can; see rb_ractor_finish_marking). */
9948 rb_ractor_finish_marking(true);
9949
9950 gc_marking_exit(driver);
9951
9952 /* step 9: sweep every objspace inside the barrier, not lazily. Dead shareable objects
9953 * are reclaimed here and emptied pages go back to the pool. */
9954 if (!compact) {
9955 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9956 rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
9957 unsigned int prev_immediate = os->flags.immediate_sweep;
9958 os->flags.immediate_sweep = TRUE;
9959 gc_sweep(os);
9960 os->flags.immediate_sweep = prev_immediate;
9961 }
9962 }
9963 else {
9964 /* The move -> update-references -> free flow runs as three passes across ALL
9965 * objspaces, not per objspace: (a) updating references must see every objspace's
9966 * forwarding (a reference can point at a moved foreign object), and (b) freeing
9967 * source pages must wait until everyone is updated (or another objspace's update
9968 * reads a freed T_MOVED). The read barrier is installed once for all passes. */
9969 install_handlers();
9970
9971 /* Only the driver records a profile entry for a global GC (gc_start_record), so time
9972 * only the driver's compaction work. The move/update/free below runs inside the
9973 * driver's sweep phase (gc_sweeping_enter/exit); attribute it to GC_COMPACT_WALL_TIME
9974 * and exclude it from the driver's sweep wall time so the two do not double-count,
9975 * mirroring the compacting branch of the local gc_sweep(). */
9976 const bool driver_prof = gc_prof_enabled(driver);
9977 rb_hrtime_t driver_compact_wall_time = 0;
9978
9979 /* pass 1 (move): relocate every objspace and leave T_MOVED forwarding behind. */
9980 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9981 rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
9982 gc_sweeping_enter(os);
9983 gc_sweep_start(os); /* mode -> sweeping, order the heap for compaction */
9984 rb_hrtime_t t0 = (os == driver && driver_prof) ? rb_hrtime_now() : 0;
9985 gc_compact_relocate(os); /* mode -> compacting, move */
9986 if (os == driver && driver_prof) {
9987 driver_compact_wall_time = rb_hrtime_add(driver_compact_wall_time, elapsed_hrtime_from(t0));
9988 }
9989 gc_sweeping_cpu_exit(os);
9990 }
9991
9992 /* pass 2 (update): all forwarding now exists, so update every objspace's
9993 * references (cross-objspace ones resolve too); gc_compact_finish also unprotects
9994 * pages and clears during_compacting. The move-or-mark decision reads
9995 * rb_gc_get_objspace()'s during_reference_updating: set it on every objspace. */
9996 gc_sweeping_cpu_enter(driver);
9997 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9998 global_objspace->global_gc.objspaces[i]->flags.during_reference_updating = TRUE;
9999 }
10000
10001 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10002 rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
10003 for (int h = 0; h < HEAP_COUNT; h++) {
10004 gc_unprotect_pages(objspace, &heaps[h]);
10005 }
10006 }
10007 rb_gc_before_updating_jit_code();
10008 gc_sweeping_cpu_exit(driver);
10009 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10010 rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
10011 gc_sweeping_cpu_enter(os);
10012 rb_hrtime_t t0 = (os == driver && driver_prof) ? rb_hrtime_now() : 0;
10013 gc_compact_finish(os);
10014 if (os == driver && driver_prof) {
10015 driver_compact_wall_time = rb_hrtime_add(driver_compact_wall_time, elapsed_hrtime_from(t0));
10016 }
10017 gc_sweeping_cpu_exit(os);
10018 }
10019 /* The VM-global / weak-table side of the reference update runs once (each objspace's
10020 * heap side already ran in gc_compact_finish above). */
10021 {
10022 gc_sweeping_cpu_enter(driver);
10023 rb_hrtime_t t0 = driver_prof ? rb_hrtime_now() : 0;
10024 gc_update_references_global(driver);
10025 if (driver_prof) {
10026 driver_compact_wall_time = rb_hrtime_add(driver_compact_wall_time, elapsed_hrtime_from(t0));
10027 }
10028 gc_sweeping_cpu_exit(driver);
10029 }
10030 gc_sweeping_cpu_enter(driver);
10031 rb_gc_after_updating_jit_code();
10032 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10033 global_objspace->global_gc.objspaces[i]->flags.during_reference_updating = FALSE;
10034 global_objspace->global_gc.objspaces[i]->flags.during_compacting = FALSE;
10035 }
10036 global_objspace->global_gc.compacting = false;
10037 uninstall_handlers();
10038 gc_sweeping_cpu_exit(driver);
10039
10040 /* Record the driver's compaction time and exclude it from the driver's sweep phase.
10041 * gc_sweeping_exit(driver) in pass 3 subtracts gc_sweep_excluded_wall_time from the
10042 * sweep wall time, so this must be set before it runs. The excluded value is a sum
10043 * of sub-intervals of the driver's sweep phase, so the subtraction cannot underflow. */
10044 if (driver_prof) {
10045 gc_profile_record *const record = gc_prof_record(driver);
10046 record->gc_compact_wall_time = rb_hrtime_add(record->gc_compact_wall_time,
10047 driver_compact_wall_time);
10048 driver->profile.gc_sweep_excluded_wall_time = rb_hrtime_add(
10049 driver->profile.gc_sweep_excluded_wall_time, driver_compact_wall_time);
10050 }
10051
10052 /* pass 3 (free): page-sweep every objspace, freeing dead objects and the source pages
10053 * that are now empty. during_compacting is already cleared, so the sweep treats
10054 * T_MOVED as usual. */
10055 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10056 rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
10057 gc_sweeping_cpu_enter(os);
10058 gc_sweep_rest(os);
10059 gc_sweeping_exit(os);
10060 }
10061 }
10062 global_objspace->global_gc.compacting = false;
10063
10064 /* A global GC never calls gc_marks_finish, which budgets heap growth
10065 * (allocatable_bytes). An objspace still full after the global sweep (materializing
10066 * a large received copy, say) has no free pages, no empty pages, budget 0, and its next
10067 * allocation would hit newobj_refill's "cannot create a new page after a major GC".
10068 * Give every objspace stuck like that the growth budget gc_marks_finish would. */
10069 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10070 rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
10071 if (objspace->heap_pages.allocatable_bytes != 0 || objspace->empty_pages_count != 0) {
10072 continue;
10073 }
10074 bool stuck = false;
10075 for (int h = 0; h < HEAP_COUNT; h++) {
10076 if (heaps[h].free_pages == NULL) { stuck = true; break; }
10077 }
10078 if (stuck) {
10079 heap_allocatable_bytes_expand(objspace, NULL, 0,
10080 objspace_available_slots(objspace), heaps[0].slot_size);
10081 }
10082 }
10083
10084 /* Recount the surviving shareable objects (the sweep already folded the dead ones out of
10085 * shareable_bits) and reset each trigger limit. */
10086 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10087 rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
10088 size_t survivors = 0;
10089 for (int h = 0; h < HEAP_COUNT; h++) {
10090 struct heap_page *page = NULL;
10091 ccan_list_for_each(&heaps[h].pages, page, page_node) {
10092 if (!page->flags.has_shareable_objects) continue;
10093 for (int j = 0; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
10094 survivors += rb_popcount_intptr(page->shareable_bits[j]);
10095 }
10096 }
10097 }
10098 objspace->shareable_objects = survivors;
10099 size_t new_limit = (size_t)(survivors * SHAREABLE_OBJECTS_LIMIT_FACTOR);
10100 if (new_limit < SHAREABLE_OBJECTS_LIMIT_MIN) new_limit = SHAREABLE_OBJECTS_LIMIT_MIN;
10101 objspace->shareable_objects_limit = new_limit;
10102 }
10103
10104 /* Deferred non-thread-safe frees: the world is already stopped here, so reap them
10105 * without a second barrier. Uses the driver's snapshot rather than taking its own,
10106 * which step 10 below still walks. */
10107 gc_tdata_unsafe_drain_objspaces(global_objspace->global_gc.objspaces,
10108 global_objspace->global_gc.n_objspaces);
10109
10110 /* step 10 */
10111 for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10112 rb_objspace_t *objspace = global_objspace->global_gc.objspaces[i];
10113 objspace->flags.during_global_gc = FALSE;
10114 if (objspace != driver) {
10115 during_gc = FALSE;
10116 gc_process_stat_publish(objspace);
10117 }
10118 }
10119
10120 /* The unified mark re-established the reachability of absorbed shareable objects, so a
10121 * single objspace's local mark is trustworthy again (pinning can be skipped until the
10122 * next absorb). */
10123 rb_gc_reset_absorbed_since_global_gc();
10124
10125 /* Re-measure the zombies now that the garbage is gone. Without this, the page
10126 * trigger above keeps firing on the stale numbers left when a joinable (slotted)
10127 * zombie retires without any pass merging it. */
10128 gc_global_refresh_zombie_pages();
10129
10130 /* If the sweep above collected an unjoined Ractor object, ractor_free disowned its
10131 * zombie_objspaces entry and posted the merge to main as a postponed job; the objspace
10132 * stays enumerable until main absorbs it at its next safepoint. */
10133
10134 gc_prof_timer_stop(driver);
10135 gc_exit(driver, event, &lock_lev);
10136 return true;
10137}
10138
10139static int
10140absorb_finalizer_i(st_data_t key, st_data_t val, st_data_t data)
10141{
10142 rb_objspace_t *objspace = (rb_objspace_t *)data;
10143 st_insert(finalizer_table, key, val);
10144 return ST_CONTINUE;
10145}
10146
10147static void
10148gc_make_mid_mark_objspace_absorbable(rb_objspace_t *src)
10149{
10150 rb_objspace_t *objspace = src;
10151 for (int h = 0; h < HEAP_COUNT; h++) {
10152 struct heap_page *page = NULL;
10153 ccan_list_for_each(&heaps[h].pages, page, page_node) {
10154 short stride = page->slot_size;
10155 uintptr_t p = (uintptr_t)page->start;
10156 uintptr_t pend = p + page->total_slots * stride;
10157
10158 for (; p < pend; p += stride) {
10159 VALUE vp = (VALUE)p;
10160 asan_unpoisoning_object(vp) {
10161 switch (RB_BUILTIN_TYPE(vp)) {
10162 case T_NONE:
10163 case T_ZOMBIE:
10164 break;
10165 default:
10166 RVALUE_AGE_RESET(vp);
10167 break;
10168 }
10169 }
10170 }
10171 }
10172 gc_bitmaps_clear(objspace, &heaps[h], false);
10173 }
10174 objspace->rgengc.old_objects = 0;
10175 objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
10176 objspace->marked_slots = 0;
10177}
10178
10179/* Merge a dead Ractor's objspace into dst under the VM lock. src has no owner thread and
10180 * dst is the calling thread's own objspace (join/value) or main with everyone stopped
10181 * (global GC), so single-writer holds throughout. Pages move whole (their bits describe
10182 * objects, not the objspace), and dst's next collection is forced full to rebuild the
10183 * generational state. */
10184static void
10185objspace_absorb(rb_objspace_t *dst, rb_objspace_t *src)
10186{
10187 GC_ASSERT(dst != src);
10188
10189 /* Suppress the cross-objspace verifier checks while the graph is in flux (see
10190 * global_objspace->during_absorb). */
10191 const bool prev_absorb = global_objspace->during_absorb;
10192 global_objspace->during_absorb = true;
10193
10194 /* gc.c dropped src from its zombie table in this same VM-lock section, so src went
10195 * through zombify and its pages are part of the total. */
10196 GC_ASSERT(src->zombie_pages != 0);
10197 GC_ASSERT(global_objspace->zombie_total_pages >= src->zombie_pages);
10198 global_objspace->zombie_total_pages -= src->zombie_pages;
10199 src->zombie_pages = 0;
10200
10201 /* Settle dst first: adding pages under a walking lazy-sweep cursor, or into a
10202 * half-marked incremental heap, would sweep the merged pages with src's stale mark
10203 * bits and free live objects. */
10204 gc_rest(dst);
10205
10206 /* Settle src: no incremental marking, lazy sweep or in-progress allocation page. */
10207 {
10208 rb_objspace_t *objspace = src;
10209 /* A zombie parked by fork (rb_ractor_terminate_atfork retires the objspace
10210 * without the retire GC) can be mid-incremental-mark, and that mark cannot be
10211 * finished. Its owner's threads did not survive the fork, so mark_roots would
10212 * scan the absorbing Ractor's roots instead of src's. Drop the cycle. */
10213 if (is_incremental_marking(objspace)) {
10214 gc_abort_incremental_marking(objspace);
10215 gc_make_mid_mark_objspace_absorbable(objspace);
10216 }
10217 /* rb_ractor_terminate_atfork retires without the retire GC, so src may arrive
10218 * with the flag unset; the sweep gc_rest may finish is the last one it gets. */
10219 objspace->flags.ownerless = 1;
10220 gc_rest(src); // if mid-sweep
10221 heap_alloc_state_clear(objspace);
10222 /* gc_sweep_finish leaves swept pages "pooled" for a coming incremental mark; src
10223 * never runs one (it is about to be merged), so return them to its free list now,
10224 * restoring the pooled_pages == NULL the page merge below assumes (mirrors
10225 * gc_start_global step 3). */
10226 for (int h = 0; h < HEAP_COUNT; h++) {
10227 heap_move_pooled_pages_to_free_pages(&heaps[h]);
10228 }
10229
10230 /* Backstop for pages a sweep that ran before `ownerless` was set left attached */
10231 for (int h = 0; h < HEAP_COUNT; h++) {
10232 rb_heap_t *heap = &heaps[h];
10233 struct heap_page **link = &heap->free_pages;
10234 while (*link) {
10235 struct heap_page *page = *link;
10236 if (page->free_slots == page->total_slots) {
10237 *link = page->free_next;
10238 }
10239 else {
10240 link = &page->free_next;
10241 }
10242 }
10243
10244 struct heap_page *page, *next;
10245 ccan_list_for_each_safe(&heap->pages, page, next, page_node) {
10246 if (page->free_slots == page->total_slots) {
10247 GC_ASSERT(page->final_slots == 0);
10248 heap_page_detach_empty(objspace, heap, page);
10249 }
10250 }
10251 }
10252 }
10253
10254 /* From here the merge must not run dst's GC: the finalizer st_insert below can cross
10255 * the malloc-accounting threshold, and a GC then would sweep src's detached finalizer
10256 * procs, reachable only from this C frame, into dangling VALUEs. Page/darray moves
10257 * allocate nothing (_without_gc), so disabling costs nothing and makes the splice
10258 * atomic. (The two settles above deliberately collect: they stay outside.) */
10259 const bool dst_gc_was_enabled = rb_gc_impl_gc_enabled_p(dst);
10260 if (dst_gc_was_enabled) rb_gc_impl_gc_disable(dst, false);
10261
10262 /* Hand over the pages size pool by size pool. ("heaps" is a macro over the local
10263 * objspace, so the arrays are taken through scoped locals.) */
10264 rb_heap_t *dst_heaps;
10265 rb_heap_t *src_heaps;
10266 {
10267 rb_objspace_t *objspace = dst;
10268 dst_heaps = heaps;
10269 }
10270 {
10271 rb_objspace_t *objspace = src;
10272 src_heaps = heaps;
10273 }
10274 for (int h = 0; h < HEAP_COUNT; h++) {
10275 rb_heap_t *dheap = &dst_heaps[h];
10276 rb_heap_t *sheap = &src_heaps[h];
10277 struct heap_page *page = NULL;
10278
10279 GC_ASSERT(sheap->sweeping_page == NULL);
10280 GC_ASSERT(sheap->pooled_pages == NULL);
10281
10282 ccan_list_for_each(&sheap->pages, page, page_node) {
10283 RUBY_ATOMIC_PTR_SET(page->objspace, dst); // read without the VM lock: rb_gc_impl_obj_foreign_p
10284 page->heap = dheap;
10285 }
10286 ccan_list_append_list(&dheap->pages, &sheap->pages);
10287
10288 /* Append the free-page chain to the tail. */
10289 if (sheap->free_pages) {
10290 struct heap_page **tail = &dheap->free_pages;
10291 while (*tail) tail = &(*tail)->free_next;
10292 *tail = sheap->free_pages;
10293 sheap->free_pages = NULL;
10294 }
10295
10296 dheap->total_pages += sheap->total_pages;
10297 dheap->total_slots += sheap->total_slots;
10298 dheap->total_allocated_pages += sheap->total_allocated_pages;
10299 dheap->total_allocated_objects += sheap->total_allocated_objects;
10300 dheap->total_freed_objects += sheap->total_freed_objects;
10301 dheap->final_slots_count += sheap->final_slots_count;
10302 }
10303
10304 /* The objspace-wide page bookkeeping. */
10305 {
10306 rb_objspace_t *objspace = dst; /* for the heap_pages_* macros */
10307 struct heap_page *page = NULL;
10308 size_t srcn = rb_darray_size(src->heap_pages.sorted);
10309 for (size_t i = 0; i < srcn; i++) {
10310 page = rb_darray_get(src->heap_pages.sorted, i);
10311 /* Residents of the empty pool (no live objects) are returned to page_pool rather
10312 * than inherited; dst's allocation demand is cheaply met from the shared pool's
10313 * free list. */
10314 if (heap_page_in_global_empty_pages_pool(src, page)) {
10315 heap_page_free(src, page);
10316 continue;
10317 }
10318 uintptr_t body = (uintptr_t)page->body;
10319 uintptr_t start = body + sizeof(struct heap_page_header);
10320 uintptr_t end = body + HEAP_PAGE_SIZE;
10321
10322 /* Keep the array ordered by page BODY address: heap_page_for_ptr bsearches
10323 * body ranges, and a detached empty page has start == 0, so ordering by
10324 * page->start would miss live pages (a global GC would then fail to mark a
10325 * registered root and sweep it). */
10326 size_t lo = 0;
10327 size_t hi = rb_darray_size(objspace->heap_pages.sorted);
10328 while (lo < hi) {
10329 size_t mid = (lo + hi) / 2;
10330 struct heap_page *mid_page = rb_darray_get(objspace->heap_pages.sorted, mid);
10331 if ((uintptr_t)mid_page->body < body) lo = mid + 1;
10332 else hi = mid;
10333 }
10334 rb_darray_insert_without_gc(&objspace->heap_pages.sorted, hi, page);
10335
10336 if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
10337 if (heap_pages_himem < end) heap_pages_himem = end;
10338 }
10339 objspace->heap_pages.allocated_pages += src->heap_pages.allocated_pages;
10340 objspace->heap_pages.freed_pages += src->heap_pages.freed_pages;
10341 rb_darray_free_without_gc(src->heap_pages.sorted);
10342 src->heap_pages.sorted = NULL;
10343 /* The empty_pages chain's structs were freed in the loop above. */
10344 src->empty_pages = NULL;
10345 src->empty_pages_count = 0;
10346 }
10347
10348 /* Finalizers: move the table's entries, and the dead Ractor's deferred zombies are run
10349 * by dst's thread from now on. */
10350 {
10351 st_table *src_finalizers;
10352 {
10353 rb_objspace_t *objspace = src;
10354 src_finalizers = finalizer_table;
10355 finalizer_table = NULL;
10356 }
10357 if (src_finalizers) {
10358 rb_objspace_t *objspace = dst;
10359 if (finalizer_table == NULL) {
10360 finalizer_table = src_finalizers;
10361 }
10362 else {
10363 st_foreach(src_finalizers, absorb_finalizer_i, (st_data_t)dst);
10364 st_free_table(src_finalizers);
10365 }
10366 }
10367 }
10368 {
10369 VALUE src_deferred = RUBY_ATOMIC_VALUE_EXCHANGE(src->heap_pages.deferred_final, 0);
10370 if (src_deferred) {
10371 VALUE tail_obj = src_deferred;
10372 rb_asan_unpoison_object(tail_obj, false);
10373 while (RZOMBIE(tail_obj)->next) {
10374 VALUE next_obj = RZOMBIE(tail_obj)->next;
10375 rb_asan_poison_object(tail_obj);
10376 tail_obj = next_obj;
10377 rb_asan_unpoison_object(tail_obj, false);
10378 }
10379 VALUE prev;
10380 do {
10381 prev = dst->heap_pages.deferred_final;
10382 RZOMBIE(tail_obj)->next = prev;
10383 } while (RUBY_ATOMIC_VALUE_CAS(dst->heap_pages.deferred_final, prev, src_deferred) != prev);
10384 rb_asan_poison_object(tail_obj);
10385 /* No owner was left to run these zombies (register's owner walk misses a dead
10386 * Ractor). dst runs this merge, so schedule dst's job here; otherwise they wait
10387 * until dst's next GC. */
10388 rb_postponed_job_trigger(dst->finalize_deferred_pjob);
10389 }
10390 }
10391 if (src->tdata_unsafe_free_chunk) {
10392 gc_tdata_unsafe_free_publish(src);
10393 }
10394 if (tdata_deferred_free_count_load() >= TDATA_DEFERRED_FREE_THRESHOLD) {
10395 gc_tdata_deferred_free_trigger(dst);
10396 }
10397
10398 /* Counters inherited by dst. */
10399 dst->rgengc.old_objects += src->rgengc.old_objects;
10400 dst->rgengc.uncollectible_wb_unprotected_objects += src->rgengc.uncollectible_wb_unprotected_objects;
10401 dst->shareable_objects += src->shareable_objects;
10402
10403 /* Merged pages carry src's mark/age state, so dst rebuilds its view at the next
10404 * collection. Note that this can be worked around by calling `GC.config(rgengc_allow_full_mark: false)`,
10405 * so the absorbed heap should be in a state where a minor GC would also work correctly.
10406 */
10407 dst->rgengc.need_major_gc |= GPR_FLAG_MAJOR_BY_FORCE;
10408
10409 /* src's outstanding malloc pressure moves with the xmalloc'd buffers. Later frees are
10410 * charged to dst, so without this transfer dst underestimates its own heap and delays
10411 * GCs. dst is live, so take its counter lock where gc_counter_add is not atomic. */
10412 {
10413 int64_t inc = gc_malloc_counters_increase(src, &src->malloc_counters.counters);
10414#if RGENGC_ESTIMATE_OLDMALLOC
10415 size_t oldmalloc_increase = gc_oldmalloc_increase_load(src);
10416#endif
10417 MALLOC_COUNTERS_LOCK(dst);
10418 if (inc > 0) gc_counter_add(&dst->malloc_counters.counters.malloc, (size_t)inc);
10419#if RGENGC_ESTIMATE_OLDMALLOC
10420 if (oldmalloc_increase > 0) RUBY_ATOMIC_SIZE_ADD(dst->malloc_counters.oldmalloc_increase, oldmalloc_increase);
10421#endif
10422 MALLOC_COUNTERS_UNLOCK(dst);
10423 }
10424
10425 {
10426 struct gc_process_stat_snapshot final_snap;
10427 gc_process_stat_capture(src, &final_snap);
10428 gc_process_stat_add(&global_objspace->process_stat_archive, &final_snap);
10429 }
10430 rb_native_mutex_destroy(&src->process_stat.lock);
10431
10432 /* Free the shell (as rb_gc_impl_objspace_free does). */
10433 free(src->profile.records);
10434 free_stack_chunks(&src->mark_stack);
10435 mark_stack_free_cache(&src->mark_stack);
10436 GC_ASSERT(rb_darray_size(src->weak_references) == 0);
10437 rb_darray_free_without_gc(src->weak_references);
10438#ifdef MALLOC_COUNTERS_NEED_LOCK
10439 rb_native_mutex_destroy(&src->malloc_counters.lock);
10440#endif
10441 free(src);
10442
10443 if (dst_gc_was_enabled) rb_gc_impl_gc_enable(dst);
10444
10445 /* Return the empty pages inheritance piled up in dst (mostly from the dead Ractor's
10446 * teardown material) to the pool with no budget. An empty page is by definition safe to
10447 * release, and re-acquiring one from the pool is cheap. */
10448 {
10449 rb_objspace_t *objspace = dst;
10450 heap_pages_freeable_pages = objspace->empty_pages_count;
10451 heap_pages_free_unused_pages(objspace);
10452 }
10453
10454 global_objspace->during_absorb = prev_absorb;
10455}
10456
10457void
10458rb_gc_impl_objspace_absorb(void *dst_ptr, void *src_ptr)
10459{
10460 objspace_absorb(dst_ptr, src_ptr);
10461}
10462
10463void
10464rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact, bool global)
10465{
10466 rb_objspace_t *objspace = objspace_ptr;
10467 unsigned int reason = (GPR_FLAG_FULL_MARK |
10468 GPR_FLAG_IMMEDIATE_MARK |
10469 GPR_FLAG_IMMEDIATE_SWEEP |
10470 GPR_FLAG_METHOD);
10471
10472 int full_marking_p = gc_config_full_mark_val;
10473 gc_config_full_mark_set(TRUE);
10474
10475 /* For now, compact implies full mark / sweep, so ignore other flags */
10476 if (compact) {
10477 GC_ASSERT(GC_COMPACTION_SUPPORTED);
10478
10479 reason |= GPR_FLAG_COMPACT;
10480 if (!rb_gc_single_objspace_p()) {
10481 global = true;
10482 }
10483 }
10484 else {
10485 if (!full_mark) reason &= ~GPR_FLAG_FULL_MARK;
10486 if (!immediate_mark) reason &= ~GPR_FLAG_IMMEDIATE_MARK;
10487 if (!immediate_sweep) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
10488 }
10489
10490 if ((reason & (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP)) !=
10491 (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP)) {
10492 global = false;
10493 }
10494
10495 if (global && !rb_gc_single_objspace_p()) {
10496 /* A mid-cycle driver is settled by gc_start_global itself: it aborts the partial
10497 * mark and finishes the lazy sweep, so the dead slots are T_NONE before the
10498 * unified conservative root scan. */
10499 gc_start_global(objspace, reason, compact || ruby_enable_autocompact, false);
10500 }
10501 else {
10502 garbage_collect(objspace, reason);
10503 }
10504
10505 gc_finalize_deferred(objspace);
10506 /* An explicit GC.start is expected to reclaim immediately, so run the deferred non-thread-safe
10507 * frees synchronously instead of leaving them to gc_sweep_finish's postponed job. */
10508 if (tdata_deferred_free_count_load() > 0 && rb_gc_single_objspace_p()) {
10509 gc_tdata_unsafe_drain();
10510 }
10511 gc_config_full_mark_set(full_marking_p);
10512}
10513
10514void
10515rb_gc_impl_prepare_heap(void *objspace_ptr)
10516{
10517 rb_objspace_t *objspace = objspace_ptr;
10518
10519 rb_gc_impl_each_objects(objspace, gc_set_candidate_object_i, objspace_ptr);
10520
10521 double orig_max_free_slots = gc_params.heap_free_slots_max_ratio;
10522 /* Ensure that all empty pages are moved onto empty_pages. */
10523 gc_params.heap_free_slots_max_ratio = 0.0;
10524 rb_gc_impl_start(objspace, true, true, true, true, true);
10525 gc_params.heap_free_slots_max_ratio = orig_max_free_slots;
10526
10527 objspace->heap_pages.allocatable_bytes = 0;
10528 heap_pages_freeable_pages = objspace->empty_pages_count;
10529 heap_pages_free_unused_pages(objspace_ptr);
10530 GC_ASSERT(heap_pages_freeable_pages == 0);
10531 GC_ASSERT(objspace->empty_pages_count == 0);
10532
10533 // Process.warmup is meant to be called at the end of the boot sequence, which is commonly allocation
10534 // heavy and result in GC limits raising significantly, but it's not indicative of the limits needed
10535 // for runtime.
10536 // Recompute the allocatable_bytes limit based on `gc_params.heap_init_bytes`.
10537 GC_ASSERT(objspace->heap_pages.allocatable_bytes == 0);
10538 for (int i = 0; i < HEAP_COUNT; i++) {
10539 rb_heap_t *heap = &heaps[i];
10540 heap_allocatable_bytes_expand(objspace, heap, heap->empty_slots, heap->total_slots, heap->slot_size);
10541 }
10542
10543#if defined(HAVE_MALLOC_TRIM) && !defined(RUBY_ALTERNATIVE_MALLOC_HEADER)
10544 malloc_trim(0);
10545#endif
10546}
10547
10548static int
10549gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
10550{
10551 GC_ASSERT(!SPECIAL_CONST_P(obj));
10552
10553 switch (BUILTIN_TYPE(obj)) {
10554 case T_NONE:
10555 case T_MOVED:
10556 case T_ZOMBIE:
10557 return FALSE;
10558 case T_SYMBOL:
10559 case T_STRING:
10560 case T_OBJECT:
10561 case T_FLOAT:
10562 case T_IMEMO:
10563 case T_ARRAY:
10564 case T_BIGNUM:
10565 case T_ICLASS:
10566 case T_MODULE:
10567 case T_REGEXP:
10568 case T_DATA:
10569 case T_MATCH:
10570 case T_STRUCT:
10571 case T_HASH:
10572 case T_FILE:
10573 case T_COMPLEX:
10574 case T_RATIONAL:
10575 case T_NODE:
10576 case T_CLASS:
10577 if (FL_TEST_RAW(obj, FL_FINALIZE)) {
10578 /* The finalizer table is a numtable. It looks up objects by address.
10579 * We can't mark the keys in the finalizer table because that would
10580 * prevent the objects from being collected. This check prevents
10581 * objects that are keys in the finalizer table from being moved
10582 * without directly pinning them. */
10583 GC_ASSERT(st_is_member(finalizer_table, obj));
10584
10585 return FALSE;
10586 }
10587 GC_ASSERT(RVALUE_MARKED(objspace, obj));
10588 GC_ASSERT(!RVALUE_PINNED(objspace, obj));
10589
10590 return TRUE;
10591
10592 default:
10593 rb_bug("gc_is_moveable_obj: unreachable (%d)", (int)BUILTIN_TYPE(obj));
10594 break;
10595 }
10596
10597 return FALSE;
10598}
10599
10600void rb_mv_generic_ivar(VALUE src, VALUE dst);
10601
10602static VALUE
10603gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, struct heap_page *src_page, struct heap_page *dest_page)
10604{
10605 size_t src_slot_size = src_page->slot_size;
10606 size_t slot_size = dest_page->slot_size;
10607
10608 int marked;
10609 int wb_unprotected;
10610 int uncollectible;
10611 int age;
10612
10613 gc_report(4, objspace, "Moving object: %p -> %p\n", (void *)src, (void *)dest);
10614
10615 GC_ASSERT(BUILTIN_TYPE(src) != T_NONE);
10616 GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest));
10617
10618 GC_ASSERT(!RVALUE_MARKING(objspace, src));
10619
10620 /* Save off bits for current object. */
10621 marked = RVALUE_MARKED(objspace, src);
10622 wb_unprotected = RVALUE_WB_UNPROTECTED(objspace, src);
10623 uncollectible = RVALUE_UNCOLLECTIBLE(objspace, src);
10624 bool remembered = RVALUE_REMEMBERED(objspace, src);
10625 /* Pin bits travel with the object. Losing one during single-objspace compaction would
10626 * silently unpin it once the process goes multi-objspace, letting a local GC free a method
10627 * entry or shref target that another Ractor references. */
10628 bool shareable = MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(src), src) != 0;
10629 bool shref = MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(src), src) != 0;
10630 age = RVALUE_AGE_GET(src);
10631
10632 /* Clear bits for eventual T_MOVED */
10633 CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(src), src);
10634 CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(src), src);
10635 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(src), src);
10636 CLEAR_IN_BITMAP(GET_HEAP_PAGE(src)->remembered_bits, src);
10637 CLEAR_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(src), src);
10638 CLEAR_IN_BITMAP(GET_HEAP_SHREF_BITS(src), src);
10639
10640 /* Move the object */
10641 memcpy((void *)dest, (void *)src, MIN(src_slot_size, slot_size));
10642
10643 if (src_slot_size != slot_size) {
10644 rb_gc_obj_changed_slot_size(dest, slot_size - RVALUE_OVERHEAD);
10645 }
10646
10647 if (RVALUE_OVERHEAD > 0) {
10648 void *dest_overhead = (void *)(((uintptr_t)dest) + slot_size - RVALUE_OVERHEAD);
10649 void *src_overhead = (void *)(((uintptr_t)src) + src_slot_size - RVALUE_OVERHEAD);
10650
10651 memcpy(dest_overhead, src_overhead, RVALUE_OVERHEAD);
10652 }
10653
10654 memset((void *)src, 0, src_slot_size);
10655 RVALUE_AGE_SET_BITMAP(src, 0);
10656
10657 /* Set bits for object in new location */
10658 if (remembered) {
10659 MARK_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
10660 }
10661 else {
10662 CLEAR_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
10663 }
10664
10665 if (marked) {
10666 MARK_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
10667 }
10668 else {
10669 CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
10670 }
10671
10672 if (wb_unprotected) {
10673 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
10674 }
10675 else {
10676 CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
10677 }
10678
10679 if (uncollectible) {
10680 MARK_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
10681 }
10682 else {
10683 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
10684 }
10685
10686 if (shareable) {
10687 MARK_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(dest), dest);
10688 GET_HEAP_PAGE(dest)->flags.has_shareable_objects = TRUE;
10689 }
10690 else {
10691 CLEAR_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(dest), dest);
10692 }
10693
10694 if (shref) {
10695 MARK_IN_BITMAP(GET_HEAP_SHREF_BITS(dest), dest);
10696 GET_HEAP_PAGE(dest)->flags.has_shref_objects = TRUE;
10697 }
10698 else {
10699 CLEAR_IN_BITMAP(GET_HEAP_SHREF_BITS(dest), dest);
10700 }
10701
10702 RVALUE_AGE_SET(dest, age);
10703
10704 /* A re-embedded object (rb_gc_obj_changed_slot_size) references its
10705 * former fields_obj's contents directly; the write-barrier history
10706 * lived on the discarded fields_obj, so remember the object. */
10707 if (src_slot_size != slot_size && age >= RVALUE_OLD_AGE && !remembered) {
10708 rgengc_remember(objspace, dest);
10709 }
10710
10711 /* Assign forwarding address */
10712 RMOVED(src)->flags = T_MOVED;
10713 RMOVED(src)->dummy = Qundef;
10714 RMOVED(src)->destination = dest;
10715 GC_ASSERT(BUILTIN_TYPE(dest) != T_NONE);
10716
10717 GET_HEAP_PAGE(src)->heap->total_freed_objects++;
10718 GET_HEAP_PAGE(dest)->heap->total_allocated_objects++;
10719
10720 return src;
10721}
10722
10723#if GC_CAN_COMPILE_COMPACTION
10724static int
10725compare_pinned_slots(const void *left, const void *right, void *dummy)
10726{
10727 struct heap_page *left_page;
10728 struct heap_page *right_page;
10729
10730 left_page = *(struct heap_page * const *)left;
10731 right_page = *(struct heap_page * const *)right;
10732
10733 return left_page->pinned_slots - right_page->pinned_slots;
10734}
10735
10736static int
10737compare_free_slots(const void *left, const void *right, void *dummy)
10738{
10739 struct heap_page *left_page;
10740 struct heap_page *right_page;
10741
10742 left_page = *(struct heap_page * const *)left;
10743 right_page = *(struct heap_page * const *)right;
10744
10745 return left_page->free_slots - right_page->free_slots;
10746}
10747
10748static void
10749gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func)
10750{
10751 for (int j = 0; j < HEAP_COUNT; j++) {
10752 rb_heap_t *heap = &heaps[j];
10753
10754 size_t total_pages = heap->total_pages;
10755 size_t size = rb_size_mul_or_raise(total_pages, sizeof(struct heap_page *), rb_eRuntimeError);
10756 struct heap_page *page = 0, **page_list = malloc(size);
10757 size_t i = 0;
10758
10759 heap->free_pages = NULL;
10760 ccan_list_for_each(&heap->pages, page, page_node) {
10761 page_list[i++] = page;
10762 GC_ASSERT(page);
10763 }
10764
10765 GC_ASSERT((size_t)i == total_pages);
10766
10767 /* Sort the heap so "filled pages" are first. `heap_add_page` adds to the
10768 * head of the list, so empty pages will end up at the start of the heap */
10769 ruby_qsort(page_list, total_pages, sizeof(struct heap_page *), compare_func, NULL);
10770
10771 /* Reset the eden heap */
10772 ccan_list_head_init(&heap->pages);
10773
10774 for (i = 0; i < total_pages; i++) {
10775 ccan_list_add(&heap->pages, &page_list[i]->page_node);
10776 if (page_list[i]->free_slots != 0) {
10777 heap_add_freepage(heap, page_list[i]);
10778 }
10779 }
10780
10781 free(page_list);
10782 }
10783}
10784#endif
10785
10786void
10787rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj)
10788{
10789 /* no-op */
10790}
10791
10792bool
10793rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj)
10794{
10795 return gc_object_moved_p(objspace_ptr, obj);
10796}
10797
10798static int
10799gc_ref_update(void *vstart, void *vend, size_t stride, rb_objspace_t *objspace, struct heap_page *page)
10800{
10801 VALUE v = (VALUE)vstart;
10802
10803 page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
10804 page->flags.has_remembered_objects = FALSE;
10805
10806 /* For each object on the page */
10807 for (; v != (VALUE)vend; v += stride) {
10808 asan_unpoisoning_object(v) {
10809 switch (BUILTIN_TYPE(v)) {
10810 case T_NONE:
10811 case T_MOVED:
10812 case T_ZOMBIE:
10813 break;
10814 default:
10815 if (RVALUE_WB_UNPROTECTED(objspace, v)) {
10816 page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
10817 }
10818 if (RVALUE_REMEMBERED(objspace, v)) {
10819 page->flags.has_remembered_objects = TRUE;
10820 }
10821 if (page->flags.before_sweep) {
10822 if (RVALUE_MARKED(objspace, v)) {
10823 rb_gc_update_object_references(objspace, v);
10824 }
10825 }
10826 else {
10827 rb_gc_update_object_references(objspace, v);
10828 }
10829 }
10830 }
10831 }
10832
10833 return 0;
10834}
10835
10836static int
10837gc_update_references_weak_table_i(VALUE obj, void *data)
10838{
10839 int ret;
10840 asan_unpoisoning_object(obj) {
10841 ret = BUILTIN_TYPE(obj) == T_MOVED ? ST_REPLACE : ST_CONTINUE;
10842 }
10843 return ret;
10844}
10845
10846static int
10847gc_update_references_weak_table_replace_i(VALUE *obj, void *data)
10848{
10849 rb_gc_update_moved(obj);
10850
10851 return ST_CONTINUE;
10852}
10853
10854/* The per-objspace side of the reference update: walk this objspace's heap objects and rewrite
10855 * moved references (following T_MOVED forwarding across objspaces). A compacting global GC
10856 * runs this for every objspace. */
10857static void
10858gc_update_references_heap(rb_objspace_t *objspace)
10859{
10860 struct heap_page *page = NULL;
10861
10862 for (int i = 0; i < HEAP_COUNT; i++) {
10863 bool should_set_mark_bits = TRUE;
10864 rb_heap_t *heap = &heaps[i];
10865
10866 ccan_list_for_each(&heap->pages, page, page_node) {
10867 uintptr_t start = (uintptr_t)page->start;
10868 uintptr_t end = start + (page->total_slots * heap->slot_size);
10869
10870 gc_ref_update((void *)start, (void *)end, heap->slot_size, objspace, page);
10871 if (page == heap->sweeping_page) {
10872 should_set_mark_bits = FALSE;
10873 }
10874 if (should_set_mark_bits) {
10875 gc_setup_mark_bits(page);
10876 }
10877 }
10878 }
10879}
10880
10881/* The VM-global side of the reference update (finalizer table, every Ractor's VM roots,
10882 * weak tables). Process-wide, so a compacting global GC runs it once after every heap
10883 * side: rb_gc_update_vm_references and the weak tables' mark_and_move are not idempotent. */
10884static void
10885gc_update_references_global(rb_objspace_t *objspace)
10886{
10887 gc_update_table_refs(finalizer_table);
10888
10889 rb_gc_update_vm_references((void *)objspace);
10890
10891 for (int table = 0; table < RB_GC_VM_WEAK_TABLE_COUNT; table++) {
10892 rb_gc_vm_weak_table_foreach(
10893 gc_update_references_weak_table_i,
10894 gc_update_references_weak_table_replace_i,
10895 NULL,
10896 false,
10897 table
10898 );
10899 }
10900}
10901
10902static void
10903gc_update_references(rb_objspace_t *objspace)
10904{
10905 objspace->flags.during_reference_updating = true;
10906
10907 rb_gc_before_updating_jit_code();
10908
10909 gc_update_references_heap(objspace);
10910 gc_update_references_global(objspace);
10911
10912 rb_gc_after_updating_jit_code();
10913
10914 objspace->flags.during_reference_updating = false;
10915}
10916
10917#if GC_CAN_COMPILE_COMPACTION
10918static void
10919root_obj_check_moved_i(const char *category, VALUE obj, void *data)
10920{
10921 rb_objspace_t *objspace = data;
10922
10923 if (gc_object_moved_p(objspace, obj)) {
10924 rb_bug("ROOT %s points to MOVED: %p -> %s", category, (void *)obj, rb_obj_info(rb_gc_impl_location(objspace, obj)));
10925 }
10926}
10927
10928static void
10929reachable_object_check_moved_i(VALUE ref, void *data)
10930{
10931 VALUE parent = (VALUE)data;
10932 if (gc_object_moved_p(rb_gc_get_objspace(), ref)) {
10933 rb_bug("Object %s points to MOVED: %p -> %s", rb_obj_info(parent), (void *)ref, rb_obj_info(rb_gc_impl_location(rb_gc_get_objspace(), ref)));
10934 }
10935}
10936
10937static int
10938heap_check_moved_i(void *vstart, void *vend, size_t stride, void *data)
10939{
10940 rb_objspace_t *objspace = data;
10941
10942 VALUE v = (VALUE)vstart;
10943 for (; v != (VALUE)vend; v += stride) {
10944 if (gc_object_moved_p(objspace, v)) {
10945 /* Moved object still on the heap, something may have a reference. */
10946 }
10947 else {
10948 asan_unpoisoning_object(v) {
10949 switch (BUILTIN_TYPE(v)) {
10950 case T_NONE:
10951 case T_ZOMBIE:
10952 break;
10953 default:
10954 if (!rb_gc_impl_garbage_object_p(objspace, v)) {
10955 rb_objspace_reachable_objects_from_unlocked(v, reachable_object_check_moved_i, (void *)v);
10956 }
10957 }
10958 }
10959 }
10960 }
10961
10962 return 0;
10963}
10964#endif
10965
10966bool
10967rb_gc_impl_during_gc_p(void *objspace_ptr)
10968{
10969 rb_objspace_t *objspace = objspace_ptr;
10970
10971 return during_gc;
10972}
10973
10974#if RGENGC_PROFILE >= 2
10975
10976static const char*
10977type_name(int type, VALUE obj)
10978{
10979 switch ((enum ruby_value_type)type) {
10980 case RUBY_T_NONE: return "T_NONE";
10981 case RUBY_T_OBJECT: return "T_OBJECT";
10982 case RUBY_T_CLASS: return "T_CLASS";
10983 case RUBY_T_MODULE: return "T_MODULE";
10984 case RUBY_T_FLOAT: return "T_FLOAT";
10985 case RUBY_T_STRING: return "T_STRING";
10986 case RUBY_T_REGEXP: return "T_REGEXP";
10987 case RUBY_T_ARRAY: return "T_ARRAY";
10988 case RUBY_T_HASH: return "T_HASH";
10989 case RUBY_T_STRUCT: return "T_STRUCT";
10990 case RUBY_T_BIGNUM: return "T_BIGNUM";
10991 case RUBY_T_FILE: return "T_FILE";
10992 case RUBY_T_DATA: return "T_DATA";
10993 case RUBY_T_MATCH: return "T_MATCH";
10994 case RUBY_T_COMPLEX: return "T_COMPLEX";
10995 case RUBY_T_RATIONAL: return "T_RATIONAL";
10996 case RUBY_T_NIL: return "T_NIL";
10997 case RUBY_T_TRUE: return "T_TRUE";
10998 case RUBY_T_FALSE: return "T_FALSE";
10999 case RUBY_T_SYMBOL: return "T_SYMBOL";
11000 case RUBY_T_FIXNUM: return "T_FIXNUM";
11001 case RUBY_T_UNDEF: return "T_UNDEF";
11002 case RUBY_T_IMEMO: return "T_IMEMO";
11003 case RUBY_T_NODE: return "T_NODE";
11004 case RUBY_T_ICLASS: return "T_ICLASS";
11005 case RUBY_T_ZOMBIE: return "T_ZOMBIE";
11006 case RUBY_T_MOVED: return "T_MOVED";
11007 default: return "unknown";
11008 }
11009}
11010
11011static void
11012gc_count_add_each_types(VALUE hash, const char *name, const size_t *types)
11013{
11014 VALUE result = rb_hash_new_capa(T_MASK);
11015 int i;
11016 for (i=0; i<T_MASK; i++) {
11017 const char *type = type_name(i, 0);
11018 rb_hash_aset(result, ID2SYM(rb_intern(type)), SIZET2NUM(types[i]));
11019 }
11020 rb_hash_aset(hash, ID2SYM(rb_intern(name)), result);
11021}
11022#endif
11023
11024size_t
11025rb_gc_impl_gc_count(void *objspace_ptr)
11026{
11027 rb_objspace_t *objspace = objspace_ptr;
11028
11029 return objspace->profile.count;
11030}
11031
11032/* Filled by setup_gc_latest_gc_info_symbols() at boot, not on first use. */
11033static VALUE sym_major_by, sym_gc_by, sym_immediate_sweep, sym_have_finalizer, sym_state, sym_need_major_by;
11034static VALUE sym_nofree, sym_oldgen, sym_shady, sym_force, sym_stress;
11035#if RGENGC_ESTIMATE_OLDMALLOC
11036static VALUE sym_oldmalloc;
11037#endif
11038static VALUE sym_newobj, sym_malloc, sym_method, sym_capi;
11039static VALUE sym_none, sym_marking, sym_sweeping;
11040static VALUE sym_weak_references_count;
11041
11042static void
11043setup_gc_latest_gc_info_symbols(void)
11044{
11045#define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
11046 S(major_by);
11047 S(gc_by);
11048 S(immediate_sweep);
11049 S(have_finalizer);
11050 S(state);
11051 S(need_major_by);
11052
11053 S(stress);
11054 S(nofree);
11055 S(oldgen);
11056 S(shady);
11057 S(force);
11058#if RGENGC_ESTIMATE_OLDMALLOC
11059 S(oldmalloc);
11060#endif
11061 S(newobj);
11062 S(malloc);
11063 S(method);
11064 S(capi);
11065
11066 S(none);
11067 S(marking);
11068 S(sweeping);
11069
11070 S(weak_references_count);
11071#undef S
11072}
11073
11074static VALUE
11075gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned int orig_flags)
11076{
11077 VALUE hash = Qnil, key = Qnil;
11078 VALUE major_by, need_major_by;
11079 unsigned int flags = orig_flags ? orig_flags : objspace->profile.latest_gc_info;
11080
11081 if (SYMBOL_P(hash_or_key)) {
11082 key = hash_or_key;
11083 }
11084 else if (RB_TYPE_P(hash_or_key, T_HASH)) {
11085 hash = hash_or_key;
11086 }
11087 else {
11088 rb_bug("gc_info_decode: non-hash or symbol given");
11089 }
11090
11091#define SET(name, attr) \
11092 if (key == sym_##name) \
11093 return (attr); \
11094 else if (hash != Qnil) \
11095 rb_hash_aset(hash, sym_##name, (attr));
11096
11097 major_by =
11098 (flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
11099 (flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
11100 (flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
11101 (flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
11102#if RGENGC_ESTIMATE_OLDMALLOC
11103 (flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
11104#endif
11105 Qnil;
11106 SET(major_by, major_by);
11107
11108 if (orig_flags == 0) { /* set need_major_by only if flags not set explicitly */
11109 unsigned int need_major_flags = gc_needs_major_flags;
11110 need_major_by =
11111 (need_major_flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
11112 (need_major_flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
11113 (need_major_flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
11114 (need_major_flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
11115#if RGENGC_ESTIMATE_OLDMALLOC
11116 (need_major_flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
11117#endif
11118 Qnil;
11119 SET(need_major_by, need_major_by);
11120 }
11121
11122 SET(gc_by,
11123 (flags & GPR_FLAG_NEWOBJ) ? sym_newobj :
11124 (flags & GPR_FLAG_MALLOC) ? sym_malloc :
11125 (flags & GPR_FLAG_METHOD) ? sym_method :
11126 (flags & GPR_FLAG_CAPI) ? sym_capi :
11127 (flags & GPR_FLAG_STRESS) ? sym_stress :
11128 Qnil
11129 );
11130
11131 SET(have_finalizer, (flags & GPR_FLAG_HAVE_FINALIZE) ? Qtrue : Qfalse);
11132 SET(immediate_sweep, (flags & GPR_FLAG_IMMEDIATE_SWEEP) ? Qtrue : Qfalse);
11133
11134 if (orig_flags == 0) {
11135 SET(state, gc_mode(objspace) == gc_mode_none ? sym_none :
11136 gc_mode(objspace) == gc_mode_marking ? sym_marking : sym_sweeping);
11137 }
11138
11139 SET(weak_references_count, LONG2FIX(objspace->profile.weak_references_count));
11140#undef SET
11141
11142 if (!NIL_P(key)) {
11143 // Matched key should return above
11144 return Qundef;
11145 }
11146
11147 return hash;
11148}
11149
11150VALUE
11151rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE key)
11152{
11153 rb_objspace_t *objspace = objspace_ptr;
11154
11155 return gc_info_decode(objspace, key, 0);
11156}
11157
11158
11159enum gc_stat_sym {
11160 gc_stat_sym_count,
11161 gc_stat_sym_time,
11162 gc_stat_sym_marking_time,
11163 gc_stat_sym_sweeping_time,
11164 gc_stat_sym_heap_allocated_pages,
11165 gc_stat_sym_heap_empty_pages,
11166 gc_stat_sym_heap_allocatable_bytes,
11167 gc_stat_sym_heap_available_slots,
11168 gc_stat_sym_heap_live_slots,
11169 gc_stat_sym_heap_free_slots,
11170 gc_stat_sym_heap_final_slots,
11171 gc_stat_sym_heap_marked_slots,
11172 gc_stat_sym_heap_eden_pages,
11173 gc_stat_sym_total_allocated_pages,
11174 gc_stat_sym_total_freed_pages,
11175 gc_stat_sym_total_allocated_objects,
11176 gc_stat_sym_total_freed_objects,
11177 gc_stat_sym_total_malloc_bytes,
11178 gc_stat_sym_total_free_bytes,
11179 gc_stat_sym_malloc_increase_bytes,
11180 gc_stat_sym_malloc_increase_bytes_limit,
11181 gc_stat_sym_minor_gc_count,
11182 gc_stat_sym_major_gc_count,
11183 gc_stat_sym_global_gc_count,
11184 gc_stat_sym_global_gc_invocation_count,
11185 gc_stat_sym_compact_count,
11186 gc_stat_sym_read_barrier_faults,
11187 gc_stat_sym_total_moved_objects,
11188 gc_stat_sym_remembered_wb_unprotected_objects,
11189 gc_stat_sym_remembered_wb_unprotected_objects_limit,
11190 gc_stat_sym_old_objects,
11191 gc_stat_sym_old_objects_limit,
11192#if RGENGC_ESTIMATE_OLDMALLOC
11193 gc_stat_sym_oldmalloc_increase_bytes,
11194 gc_stat_sym_oldmalloc_increase_bytes_limit,
11195#endif
11196#if RGENGC_PROFILE
11197 gc_stat_sym_total_generated_normal_object_count,
11198 gc_stat_sym_total_generated_shady_object_count,
11199 gc_stat_sym_total_shade_operation_count,
11200 gc_stat_sym_total_promoted_count,
11201 gc_stat_sym_total_remembered_normal_object_count,
11202 gc_stat_sym_total_remembered_shady_object_count,
11203#endif
11204 gc_stat_sym_page_pool_arenas,
11205 gc_stat_sym_page_pool_arenas_freed,
11206 gc_stat_sym_page_pool_total_pages,
11207 gc_stat_sym_page_pool_discarded_pages,
11208 gc_stat_sym_last
11209};
11210
11211static VALUE gc_stat_symbols[gc_stat_sym_last];
11212
11213static void
11214setup_gc_stat_symbols(void)
11215{
11216#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s))
11217 S(count);
11218 S(time);
11219 S(marking_time),
11220 S(sweeping_time),
11221 S(heap_allocated_pages);
11222 S(heap_empty_pages);
11223 S(heap_allocatable_bytes);
11224 S(heap_available_slots);
11225 S(heap_live_slots);
11226 S(heap_free_slots);
11227 S(heap_final_slots);
11228 S(heap_marked_slots);
11229 S(heap_eden_pages);
11230 S(total_allocated_pages);
11231 S(total_freed_pages);
11232 S(total_allocated_objects);
11233 S(total_freed_objects);
11234 S(total_malloc_bytes);
11235 S(total_free_bytes);
11236 S(malloc_increase_bytes);
11237 S(malloc_increase_bytes_limit);
11238 S(minor_gc_count);
11239 S(major_gc_count);
11240 S(global_gc_count);
11241 S(global_gc_invocation_count);
11242 S(compact_count);
11243 S(read_barrier_faults);
11244 S(total_moved_objects);
11245 S(remembered_wb_unprotected_objects);
11246 S(remembered_wb_unprotected_objects_limit);
11247 S(old_objects);
11248 S(old_objects_limit);
11249#if RGENGC_ESTIMATE_OLDMALLOC
11250 S(oldmalloc_increase_bytes);
11251 S(oldmalloc_increase_bytes_limit);
11252#endif
11253#if RGENGC_PROFILE
11254 S(total_generated_normal_object_count);
11255 S(total_generated_shady_object_count);
11256 S(total_shade_operation_count);
11257 S(total_promoted_count);
11258 S(total_remembered_normal_object_count);
11259 S(total_remembered_shady_object_count);
11260#endif /* RGENGC_PROFILE */
11261 S(page_pool_arenas);
11262 S(page_pool_arenas_freed);
11263 S(page_pool_total_pages);
11264 S(page_pool_discarded_pages);
11265#undef S
11266}
11267
11268static uint64_t
11269ns_to_ms(uint64_t ns)
11270{
11271 return ns / (1000 * 1000);
11272}
11273
11274static void malloc_increase_local_flush(rb_objspace_t *objspace);
11275
11276static void
11277gc_process_stat_accumulate_i(void *objspace_ptr, void *data)
11278{
11279 rb_objspace_t *objspace = objspace_ptr;
11280 struct gc_process_stat_total *total = (struct gc_process_stat_total *)data;
11281 struct gc_process_stat_snapshot snap;
11282 rb_native_mutex_lock(&objspace->process_stat.lock);
11283 snap = objspace->process_stat.published;
11284 rb_native_mutex_unlock(&objspace->process_stat.lock);
11285 gc_process_stat_add(total, &snap);
11286}
11287
11288static VALUE
11289gc_process_stat(VALUE hash_or_sym)
11290{
11291 VALUE hash = Qnil, key = Qnil;
11292
11293 if (RB_TYPE_P(hash_or_sym, T_HASH)) {
11294 hash = hash_or_sym;
11295 }
11296 else if (SYMBOL_P(hash_or_sym)) {
11297 key = hash_or_sym;
11298 }
11299 else {
11300 rb_bug("non-hash or symbol given");
11301 }
11302
11303 struct gc_process_stat_total total;
11304 uint64_t direct_global_gc_count;
11305 unsigned int lev = RB_GC_VM_LOCK();
11306 total = global_objspace->process_stat_archive;
11307 rb_gc_vm_each_objspace(gc_process_stat_accumulate_i, &total);
11308 direct_global_gc_count = global_objspace->global_gc.count;
11309 RB_GC_VM_UNLOCK(lev);
11310
11311 rb_objspace_t *const current = rb_gc_get_objspace();
11312 if (!gc_during_gc_get(current)) {
11313 GC_ASSERT(total.global_gc_invocation_count == direct_global_gc_count);
11314 }
11315
11316 /* Convert to Ruby values after all collector locks are released. */
11317 uint64_t time_ns = total.marking_time_ns + total.sweeping_time_ns;
11318 uint64_t count = total.minor_gc_count + total.major_gc_count + total.global_gc_invocation_count;
11319 size_t page_pool_arenas = global_objspace->page_pool.arena_count;
11320
11321#define SET64(name, attr) \
11322 if (key == gc_stat_symbols[gc_stat_sym_##name]) \
11323 return ULL2NUM(attr); \
11324 else if (hash != Qnil) \
11325 rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], ULL2NUM(attr));
11326
11327 SET64(count, count);
11328 SET64(time, ns_to_ms(time_ns));
11329 SET64(marking_time, ns_to_ms(total.marking_time_ns));
11330 SET64(sweeping_time, ns_to_ms(total.sweeping_time_ns));
11331 SET64(minor_gc_count, total.minor_gc_count);
11332 SET64(major_gc_count, total.major_gc_count);
11333 SET64(global_gc_count, total.global_gc_invocation_count);
11334 SET64(page_pool_arenas, page_pool_arenas);
11335 SET64(page_pool_arenas_freed, global_objspace->page_pool.arenas_unmapped);
11336 SET64(page_pool_total_pages, (uint64_t)page_pool_arenas * PAGE_POOL_ARENA_BODIES);
11337 SET64(page_pool_discarded_pages, global_objspace->page_pool.advised_count);
11338
11339#undef SET64
11340
11341 if (!NIL_P(key)) {
11342 /* Matched key should return above. */
11343 return Qundef;
11344 }
11345
11346 return hash;
11347}
11348
11349VALUE
11350rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
11351{
11352 if (objspace_ptr == NULL) {
11353 return gc_process_stat(hash_or_sym);
11354 }
11355
11356 rb_objspace_t *objspace = objspace_ptr;
11357 VALUE hash = Qnil, key = Qnil;
11358
11359 malloc_increase_local_flush(objspace);
11360
11361 if (RB_TYPE_P(hash_or_sym, T_HASH)) {
11362 hash = hash_or_sym;
11363 }
11364 else if (SYMBOL_P(hash_or_sym)) {
11365 key = hash_or_sym;
11366 }
11367 else {
11368 rb_bug("non-hash or symbol given");
11369 }
11370
11371#define SET(name, attr) \
11372 if (key == gc_stat_symbols[gc_stat_sym_##name]) \
11373 return SIZET2NUM(attr); \
11374 else if (hash != Qnil) \
11375 rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
11376#define SET64(name, attr) \
11377 if (key == gc_stat_symbols[gc_stat_sym_##name]) \
11378 return ULL2NUM(attr); \
11379 else if (hash != Qnil) \
11380 rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], ULL2NUM(attr));
11381
11382 SET(count, objspace->profile.count);
11383 SET(time, (size_t)ns_to_ms(objspace->profile.marking_time_ns + objspace->profile.sweeping_time_ns)); // TODO: UINT64T2NUM
11384 SET(marking_time, (size_t)ns_to_ms(objspace->profile.marking_time_ns));
11385 SET(sweeping_time, (size_t)ns_to_ms(objspace->profile.sweeping_time_ns));
11386
11387 {
11388 uint64_t total_malloc = (uint64_t)gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc);
11389 uint64_t total_free = (uint64_t)gc_counter_load_relaxed(&objspace->malloc_counters.counters.free);
11390 SET64(total_malloc_bytes, total_malloc);
11391 SET64(total_free_bytes, total_free);
11392 }
11393
11394 /* implementation dependent counters (small / fixnum-safe) */
11395 SET(heap_allocated_pages, rb_darray_size(objspace->heap_pages.sorted));
11396 SET(heap_empty_pages, objspace->empty_pages_count)
11397 SET(heap_allocatable_bytes, objspace->heap_pages.allocatable_bytes);
11398 SET(heap_eden_pages, heap_eden_total_pages(objspace));
11399 SET(total_allocated_pages, objspace->heap_pages.allocated_pages);
11400 SET(total_freed_pages, objspace->heap_pages.freed_pages);
11401 SET(malloc_increase_bytes, gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters));
11402 SET(malloc_increase_bytes_limit, malloc_limit);
11403 SET(minor_gc_count, objspace->profile.minor_gc_count);
11404 SET(major_gc_count, objspace->profile.major_gc_count);
11405 SET(global_gc_count, objspace->profile.global_gc_count);
11406 SET(global_gc_invocation_count, objspace->profile.global_gc_invocation_count);
11407 SET(compact_count, objspace->profile.compact_count);
11408 SET(read_barrier_faults, objspace->profile.read_barrier_faults);
11409 SET(total_moved_objects, objspace->rcompactor.total_moved);
11410 SET(remembered_wb_unprotected_objects, objspace->rgengc.uncollectible_wb_unprotected_objects);
11411 SET(remembered_wb_unprotected_objects_limit, objspace->rgengc.uncollectible_wb_unprotected_objects_limit);
11412 SET(old_objects, objspace->rgengc.old_objects);
11413 SET(old_objects_limit, objspace->rgengc.old_objects_limit);
11414#if RGENGC_ESTIMATE_OLDMALLOC
11415 SET(oldmalloc_increase_bytes, gc_oldmalloc_increase_load(objspace));
11416 SET(oldmalloc_increase_bytes_limit, objspace->rgengc.oldmalloc_increase_limit);
11417#endif
11418
11419 SET(total_allocated_objects, total_allocated_objects(objspace));
11420 SET(total_freed_objects, total_freed_objects(objspace));
11421 SET(heap_available_slots, objspace_available_slots(objspace));
11422 SET(heap_live_slots, objspace_live_slots(objspace));
11423 SET(heap_free_slots, objspace_free_slots(objspace));
11424 SET(heap_final_slots, total_final_slots_count(objspace));
11425 SET(heap_marked_slots, objspace->marked_slots);
11426
11427#if RGENGC_PROFILE
11428 SET(total_generated_normal_object_count, objspace->profile.total_generated_normal_object_count);
11429 SET(total_generated_shady_object_count, objspace->profile.total_generated_shady_object_count);
11430 SET(total_shade_operation_count, objspace->profile.total_shade_operation_count);
11431 SET(total_promoted_count, objspace->profile.total_promoted_count);
11432 SET(total_remembered_normal_object_count, objspace->profile.total_remembered_normal_object_count);
11433 SET(total_remembered_shady_object_count, objspace->profile.total_remembered_shady_object_count);
11434#endif /* RGENGC_PROFILE */
11435#undef SET
11436#undef SET64
11437
11438 if (!NIL_P(key)) {
11439 // Matched key should return above
11440 return Qundef;
11441 }
11442
11443#if defined(RGENGC_PROFILE) && RGENGC_PROFILE >= 2
11444 if (hash != Qnil) {
11445 gc_count_add_each_types(hash, "generated_normal_object_count_types", objspace->profile.generated_normal_object_count_types);
11446 gc_count_add_each_types(hash, "generated_shady_object_count_types", objspace->profile.generated_shady_object_count_types);
11447 gc_count_add_each_types(hash, "shade_operation_count_types", objspace->profile.shade_operation_count_types);
11448 gc_count_add_each_types(hash, "promoted_types", objspace->profile.promoted_types);
11449 gc_count_add_each_types(hash, "remembered_normal_object_count_types", objspace->profile.remembered_normal_object_count_types);
11450 gc_count_add_each_types(hash, "remembered_shady_object_count_types", objspace->profile.remembered_shady_object_count_types);
11451 }
11452#endif
11453
11454 return hash;
11455}
11456
11457enum gc_stat_heap_sym {
11458 gc_stat_heap_sym_slot_size,
11459 gc_stat_heap_sym_heap_live_slots,
11460 gc_stat_heap_sym_heap_free_slots,
11461 gc_stat_heap_sym_heap_final_slots,
11462 gc_stat_heap_sym_heap_eden_pages,
11463 gc_stat_heap_sym_heap_eden_slots,
11464 gc_stat_heap_sym_total_allocated_pages,
11465 gc_stat_heap_sym_force_major_gc_count,
11466 gc_stat_heap_sym_force_incremental_marking_finish_count,
11467 gc_stat_heap_sym_heap_allocatable_slots,
11468 gc_stat_heap_sym_total_allocated_objects,
11469 gc_stat_heap_sym_total_freed_objects,
11470 gc_stat_heap_sym_last
11471};
11472
11473static VALUE gc_stat_heap_symbols[gc_stat_heap_sym_last];
11474
11475static void
11476setup_gc_stat_heap_symbols(void)
11477{
11478#define S(s) gc_stat_heap_symbols[gc_stat_heap_sym_##s] = ID2SYM(rb_intern_const(#s))
11479 S(slot_size);
11480 S(heap_live_slots);
11481 S(heap_free_slots);
11482 S(heap_final_slots);
11483 S(heap_eden_pages);
11484 S(heap_eden_slots);
11485 S(heap_allocatable_slots);
11486 S(total_allocated_pages);
11487 S(force_major_gc_count);
11488 S(force_incremental_marking_finish_count);
11489 S(total_allocated_objects);
11490 S(total_freed_objects);
11491#undef S
11492}
11493
11494static VALUE
11495stat_one_heap(rb_objspace_t *objspace, rb_heap_t *heap, VALUE hash, VALUE key)
11496{
11497#define SET(name, attr) \
11498 if (key == gc_stat_heap_symbols[gc_stat_heap_sym_##name]) \
11499 return SIZET2NUM(attr); \
11500 else if (hash != Qnil) \
11501 rb_hash_aset(hash, gc_stat_heap_symbols[gc_stat_heap_sym_##name], SIZET2NUM(attr));
11502
11503 SET(slot_size, heap->slot_size);
11504 SET(heap_live_slots, heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count);
11505 SET(heap_free_slots, heap->total_slots - (heap->total_allocated_objects - heap->total_freed_objects));
11506 SET(heap_final_slots, heap->final_slots_count);
11507 SET(heap_eden_pages, heap->total_pages);
11508 SET(heap_eden_slots, heap->total_slots);
11509 SET(heap_allocatable_slots, objspace->heap_pages.allocatable_bytes / heap->slot_size);
11510 SET(total_allocated_pages, heap->total_allocated_pages);
11511 SET(force_major_gc_count, heap->force_major_gc_count);
11512 SET(force_incremental_marking_finish_count, heap->force_incremental_marking_finish_count);
11513 SET(total_allocated_objects, heap->total_allocated_objects);
11514 SET(total_freed_objects, heap->total_freed_objects);
11515#undef SET
11516
11517 if (!NIL_P(key)) {
11518 // Matched key should return above
11519 return Qundef;
11520 }
11521
11522 return hash;
11523}
11524
11525VALUE
11526rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym)
11527{
11528 rb_objspace_t *objspace = objspace_ptr;
11529
11530 if (NIL_P(heap_name)) {
11531 if (!RB_TYPE_P(hash_or_sym, T_HASH)) {
11532 rb_bug("non-hash given");
11533 }
11534
11535 for (int i = 0; i < HEAP_COUNT; i++) {
11536 VALUE hash = rb_hash_aref(hash_or_sym, INT2FIX(i));
11537 if (NIL_P(hash)) {
11538 hash = rb_hash_new();
11539 rb_hash_aset(hash_or_sym, INT2FIX(i), hash);
11540 }
11541
11542 stat_one_heap(objspace, &heaps[i], hash, Qnil);
11543 }
11544 }
11545 else if (FIXNUM_P(heap_name)) {
11546 int heap_idx = FIX2INT(heap_name);
11547
11548 if (heap_idx < 0 || heap_idx >= HEAP_COUNT) {
11549 rb_raise(rb_eArgError, "size pool index out of range");
11550 }
11551
11552 if (SYMBOL_P(hash_or_sym)) {
11553 return stat_one_heap(objspace, &heaps[heap_idx], Qnil, hash_or_sym);
11554 }
11555 else if (RB_TYPE_P(hash_or_sym, T_HASH)) {
11556 return stat_one_heap(objspace, &heaps[heap_idx], hash_or_sym, Qnil);
11557 }
11558 else {
11559 rb_bug("non-hash or symbol given");
11560 }
11561 }
11562 else {
11563 rb_bug("heap_name must be nil or an Integer");
11564 }
11565
11566 return hash_or_sym;
11567}
11568
11569/* I could include internal.h for this, but doing so undefines some Array macros
11570 * necessary for initialising objects, and I don't want to include all the array
11571 * headers to get them back
11572 * TODO: Investigate why RARRAY_AREF gets undefined in internal.h
11573 */
11574#ifndef RBOOL
11575#define RBOOL(v) (v ? Qtrue : Qfalse)
11576#endif
11577
11578VALUE
11579rb_gc_impl_config_get(void *objspace_ptr)
11580{
11581#define sym(name) ID2SYM(rb_intern_const(name))
11582 rb_objspace_t *objspace = objspace_ptr;
11583 VALUE hash = rb_hash_new();
11584
11585 rb_hash_aset(hash, sym("rgengc_allow_full_mark"), RBOOL(gc_config_full_mark_val));
11586
11587 return hash;
11588}
11589
11590static int
11591gc_config_set_key(VALUE key, VALUE value, VALUE data)
11592{
11593 rb_objspace_t *objspace = (rb_objspace_t *)data;
11594 if (rb_sym2id(key) == rb_intern("rgengc_allow_full_mark")) {
11595 gc_rest(objspace);
11596 gc_config_full_mark_set(RTEST(value));
11597 }
11598 return ST_CONTINUE;
11599}
11600
11601void
11602rb_gc_impl_config_set(void *objspace_ptr, VALUE hash)
11603{
11604 rb_objspace_t *objspace = objspace_ptr;
11605
11606 if (!RB_TYPE_P(hash, T_HASH)) {
11607 rb_raise(rb_eArgError, "expected keyword arguments");
11608 }
11609
11610 rb_hash_foreach(hash, gc_config_set_key, (st_data_t)objspace);
11611}
11612
11613VALUE
11614rb_gc_impl_stress_get(void *objspace_ptr)
11615{
11616 return ruby_gc_stress_mode;
11617}
11618
11619void
11620rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag)
11621{
11622 global_objspace->gc_stressful = RTEST(flag);
11623 global_objspace->gc_stress_mode = flag;
11624}
11625
11626static int
11627get_envparam_size(const char *name, size_t *default_value, size_t lower_bound)
11628{
11629 const char *ptr = getenv(name);
11630 ssize_t val;
11631
11632 if (ptr != NULL && *ptr) {
11633 size_t unit = 0;
11634 char *end;
11635#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
11636 val = strtoll(ptr, &end, 0);
11637#else
11638 val = strtol(ptr, &end, 0);
11639#endif
11640 switch (*end) {
11641 case 'k': case 'K':
11642 unit = 1024;
11643 ++end;
11644 break;
11645 case 'm': case 'M':
11646 unit = 1024*1024;
11647 ++end;
11648 break;
11649 case 'g': case 'G':
11650 unit = 1024*1024*1024;
11651 ++end;
11652 break;
11653 }
11654 while (*end && isspace((unsigned char)*end)) end++;
11655 if (*end) {
11656 if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
11657 return 0;
11658 }
11659 if (unit > 0) {
11660 if (val < -(ssize_t)(SIZE_MAX / 2 / unit) || (ssize_t)(SIZE_MAX / 2 / unit) < val) {
11661 if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%s is ignored because it overflows\n", name, ptr);
11662 return 0;
11663 }
11664 val *= unit;
11665 }
11666 if (val > 0 && (size_t)val > lower_bound) {
11667 if (RTEST(ruby_verbose)) {
11668 fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE")\n", name, val, *default_value);
11669 }
11670 *default_value = (size_t)val;
11671 return 1;
11672 }
11673 else {
11674 if (RTEST(ruby_verbose)) {
11675 fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE") is ignored because it must be greater than %"PRIuSIZE".\n",
11676 name, val, *default_value, lower_bound);
11677 }
11678 return 0;
11679 }
11680 }
11681 return 0;
11682}
11683
11684static int
11685get_envparam_double(const char *name, double *default_value, double lower_bound, double upper_bound, int accept_zero)
11686{
11687 const char *ptr = getenv(name);
11688 double val;
11689
11690 if (ptr != NULL && *ptr) {
11691 char *end;
11692 val = strtod(ptr, &end);
11693 if (!*ptr || *end) {
11694 if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
11695 return 0;
11696 }
11697
11698 if (accept_zero && val == 0.0) {
11699 goto accept;
11700 }
11701 else if (val <= lower_bound) {
11702 if (RTEST(ruby_verbose)) {
11703 fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be greater than %f.\n",
11704 name, val, *default_value, lower_bound);
11705 }
11706 }
11707 else if (upper_bound != 0.0 && /* ignore upper_bound if it is 0.0 */
11708 val > upper_bound) {
11709 if (RTEST(ruby_verbose)) {
11710 fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be lower than %f.\n",
11711 name, val, *default_value, upper_bound);
11712 }
11713 }
11714 else {
11715 goto accept;
11716 }
11717 }
11718 return 0;
11719
11720 accept:
11721 if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%f (default value: %f)\n", name, val, *default_value);
11722 *default_value = val;
11723 return 1;
11724}
11725
11726/*
11727 * GC tuning environment variables
11728 *
11729 * * RUBY_GC_HEAP_FREE_SLOTS
11730 * - Prepare at least this amount of slots after GC.
11731 * - Allocate slots if there are not enough slots.
11732 * * RUBY_GC_HEAP_GROWTH_FACTOR (new from 2.1)
11733 * - Allocate slots by this factor.
11734 * - (next slots number) = (current slots number) * (this factor)
11735 * * RUBY_GC_HEAP_GROWTH_MAX_BYTES (was RUBY_GC_HEAP_GROWTH_MAX_SLOTS)
11736 * - Allocation rate is limited to this number of bytes.
11737 * * RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO (new from 2.4)
11738 * - Allocate additional pages when the number of free slots is
11739 * lower than the value (total_slots * (this ratio)).
11740 * * RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO (new from 2.4)
11741 * - Allocate slots to satisfy this formula:
11742 * free_slots = total_slots * goal_ratio
11743 * - In other words, prepare (total_slots * goal_ratio) free slots.
11744 * - if this value is 0.0, then use RUBY_GC_HEAP_GROWTH_FACTOR directly.
11745 * * RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO (new from 2.4)
11746 * - Allow to free pages when the number of free slots is
11747 * greater than the value (total_slots * (this ratio)).
11748 * * RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR (new from 2.1.1)
11749 * - Do full GC when the number of old objects is more than R * N
11750 * where R is this factor and
11751 * N is the number of old objects just after last full GC.
11752 *
11753 * * obsolete
11754 * * RUBY_FREE_MIN -> RUBY_GC_HEAP_FREE_SLOTS (from 2.1)
11755 * * RUBY_HEAP_MIN_SLOTS -> RUBY_GC_HEAP_INIT_SLOTS (from 2.1) -> RUBY_GC_HEAP_INIT_BYTES
11756 *
11757 * * RUBY_GC_MALLOC_LIMIT
11758 * * RUBY_GC_MALLOC_LIMIT_MAX (new from 2.1)
11759 * * RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
11760 *
11761 * * RUBY_GC_OLDMALLOC_LIMIT (new from 2.1)
11762 * * RUBY_GC_OLDMALLOC_LIMIT_MAX (new from 2.1)
11763 * * RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
11764 */
11765
11766void
11767rb_gc_impl_set_params(void *objspace_ptr)
11768{
11769 rb_objspace_t *objspace = objspace_ptr;
11770 get_envparam_size("RUBY_GC_HEAP_FREE_SLOTS", &gc_params.heap_free_slots, 0);
11771
11772 get_envparam_size("RUBY_GC_HEAP_INIT_BYTES", &gc_params.heap_init_bytes,
11773 heap_init_bytes_min() - 1);
11774 get_envparam_size("RUBY_GC_RACTOR_HEAP_INIT_BYTES", &gc_params.ractor_heap_init_bytes,
11775 heap_init_bytes_min() - 1);
11776
11777 get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE);
11778 get_envparam_size ("RUBY_GC_HEAP_GROWTH_MAX_BYTES", &gc_params.growth_max_bytes, 0);
11779 get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio,
11780 0.0, 1.0, FALSE);
11781 get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", &gc_params.heap_free_slots_max_ratio,
11782 gc_params.heap_free_slots_min_ratio, 1.0, FALSE);
11783 get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", &gc_params.heap_free_slots_goal_ratio,
11784 gc_params.heap_free_slots_min_ratio, gc_params.heap_free_slots_max_ratio, TRUE);
11785 get_envparam_double("RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR", &gc_params.oldobject_limit_factor, 0.0, 0.0, TRUE);
11786 get_envparam_double("RUBY_GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO", &gc_params.uncollectible_wb_unprotected_objects_limit_ratio, 0.0, 0.0, TRUE);
11787
11788 if (get_envparam_size("RUBY_GC_MALLOC_LIMIT", &gc_params.malloc_limit_min, 0)) {
11789 malloc_limit = gc_params.malloc_limit_min;
11790 }
11791 get_envparam_size ("RUBY_GC_MALLOC_LIMIT_MAX", &gc_params.malloc_limit_max, 0);
11792 if (!gc_params.malloc_limit_max) { /* ignore max-check if 0 */
11793 gc_params.malloc_limit_max = SIZE_MAX;
11794 }
11795 get_envparam_double("RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR", &gc_params.malloc_limit_growth_factor, 1.0, 0.0, FALSE);
11796
11797#if RGENGC_ESTIMATE_OLDMALLOC
11798 if (get_envparam_size("RUBY_GC_OLDMALLOC_LIMIT", &gc_params.oldmalloc_limit_min, 0)) {
11799 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
11800 }
11801 get_envparam_size ("RUBY_GC_OLDMALLOC_LIMIT_MAX", &gc_params.oldmalloc_limit_max, 0);
11802 get_envparam_double("RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR", &gc_params.oldmalloc_limit_growth_factor, 1.0, 0.0, FALSE);
11803#endif
11804}
11805
11806static inline size_t
11807objspace_malloc_size(rb_objspace_t *objspace, void *ptr, size_t hint)
11808{
11809#ifdef HAVE_MALLOC_USABLE_SIZE
11810 if (!hint) {
11811 hint = malloc_usable_size(ptr);
11812 }
11813#endif
11814 return hint;
11815}
11816
11817enum memop_type {
11818 MEMOP_TYPE_MALLOC = 0,
11819 MEMOP_TYPE_FREE,
11820 MEMOP_TYPE_REALLOC
11821};
11822
11823static inline void
11824atomic_sub_nounderflow(size_t *var, size_t sub)
11825{
11826 if (sub == 0) return;
11827
11828 while (1) {
11829 size_t val = rbimpl_atomic_size_load(var, RBIMPL_ATOMIC_RELAXED);
11830 size_t decrement = val < sub ? val : sub;
11831 if (RUBY_ATOMIC_SIZE_CAS(*var, val, val-decrement) == val) break;
11832 }
11833}
11834
11835#define gc_stress_full_mark_after_malloc_p() \
11836 (FIXNUM_P(ruby_gc_stress_mode) && (FIX2LONG(ruby_gc_stress_mode) & (1<<gc_stress_full_mark_after_malloc)))
11837
11838static void
11839objspace_malloc_gc_stress(rb_objspace_t *objspace)
11840{
11841 if (ruby_gc_stressful && ruby_native_thread_p()) {
11842 unsigned int reason = (GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP |
11843 GPR_FLAG_STRESS | GPR_FLAG_MALLOC);
11844
11845 if (gc_stress_full_mark_after_malloc_p()) {
11846 reason |= GPR_FLAG_FULL_MARK;
11847 }
11848 garbage_collect_with_gvl(objspace, reason);
11849 }
11850}
11851
11852static void
11853malloc_increase_commit(rb_objspace_t *objspace, size_t new_size, size_t old_size)
11854{
11855 if (new_size > old_size) {
11856 size_t delta = new_size - old_size;
11857 MALLOC_COUNTERS_LOCK(objspace);
11858 gc_counter_add(&objspace->malloc_counters.counters.malloc, delta);
11859#if RGENGC_ESTIMATE_OLDMALLOC
11860 RUBY_ATOMIC_SIZE_ADD(objspace->malloc_counters.oldmalloc_increase, delta);
11861#endif
11862 MALLOC_COUNTERS_UNLOCK(objspace);
11863 }
11864 else if (old_size > new_size) {
11865 size_t delta = old_size - new_size;
11866 MALLOC_COUNTERS_LOCK(objspace);
11867 gc_counter_add(&objspace->malloc_counters.counters.free, delta);
11868#if RGENGC_ESTIMATE_OLDMALLOC
11869 if (during_gc) gc_counter_add(&objspace->malloc_counters.freed_in_gc, delta);
11870 atomic_sub_nounderflow(&objspace->malloc_counters.oldmalloc_increase, delta);
11871#endif
11872 MALLOC_COUNTERS_UNLOCK(objspace);
11873 }
11874}
11875
11876#if USE_MALLOC_INCREASE_LOCAL
11877static void
11878malloc_increase_local_flush(rb_objspace_t *objspace)
11879{
11880 int delta = malloc_increase_local;
11881 if (delta == 0) return;
11882
11883 malloc_increase_local = 0;
11884 if (delta > 0) {
11885 malloc_increase_commit(objspace, (size_t)delta, 0);
11886 }
11887 else {
11888 malloc_increase_commit(objspace, 0, (size_t)(-delta));
11889 }
11890}
11891#else
11892static void
11893malloc_increase_local_flush(rb_objspace_t *objspace)
11894{
11895}
11896#endif
11897
11898static inline bool
11899objspace_malloc_increase_report(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type, bool gc_allowed)
11900{
11901 if (0) fprintf(stderr, "increase - ptr: %p, type: %s, new_size: %"PRIdSIZE", old_size: %"PRIdSIZE"\n",
11902 mem,
11903 type == MEMOP_TYPE_MALLOC ? "malloc" :
11904 type == MEMOP_TYPE_FREE ? "free " :
11905 type == MEMOP_TYPE_REALLOC ? "realloc": "error",
11906 new_size, old_size);
11907 return false;
11908}
11909
11910static bool
11911objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type, bool gc_allowed)
11912{
11913#if USE_MALLOC_INCREASE_LOCAL
11914 if (new_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD &&
11915 old_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD) {
11916 malloc_increase_local += (int)new_size - (int)old_size;
11917
11918 if (malloc_increase_local >= GC_MALLOC_INCREASE_LOCAL_THRESHOLD ||
11919 malloc_increase_local <= -GC_MALLOC_INCREASE_LOCAL_THRESHOLD) {
11920 malloc_increase_local_flush(objspace);
11921 }
11922 }
11923 else {
11924 malloc_increase_local_flush(objspace);
11925 malloc_increase_commit(objspace, new_size, old_size);
11926 }
11927#else
11928 malloc_increase_commit(objspace, new_size, old_size);
11929#endif
11930
11931 if (type == MEMOP_TYPE_MALLOC && gc_allowed) {
11932 retry:
11933 if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val() && !rb_gc_gc_disabled_global_p()) {
11934 if (ruby_thread_has_gvl_p() && is_lazy_sweeping(objspace)) {
11935 gc_sweep_step_for_malloc(objspace); /* sweeping frees may reduce malloc_increase */
11936 goto retry;
11937 }
11938 garbage_collect_with_gvl(objspace, GPR_FLAG_MALLOC);
11939 }
11940 }
11941
11942#if MALLOC_ALLOCATED_SIZE
11943 if (new_size >= old_size) {
11944 RUBY_ATOMIC_SIZE_ADD(objspace->malloc_params.allocated_size, new_size - old_size);
11945 }
11946 else {
11947 size_t dec_size = old_size - new_size;
11948
11949#if MALLOC_ALLOCATED_SIZE_CHECK
11950 size_t allocated_size = objspace->malloc_params.allocated_size;
11951 if (allocated_size < dec_size) {
11952 rb_bug("objspace_malloc_increase: underflow malloc_params.allocated_size.");
11953 }
11954#endif
11955 atomic_sub_nounderflow(&objspace->malloc_params.allocated_size, dec_size);
11956 }
11957
11958 switch (type) {
11959 case MEMOP_TYPE_MALLOC:
11960 RUBY_ATOMIC_SIZE_INC(objspace->malloc_params.allocations);
11961 break;
11962 case MEMOP_TYPE_FREE:
11963 {
11964 size_t allocations = objspace->malloc_params.allocations;
11965 if (allocations > 0) {
11966 atomic_sub_nounderflow(&objspace->malloc_params.allocations, 1);
11967 }
11968#if MALLOC_ALLOCATED_SIZE_CHECK
11969 else {
11970 GC_ASSERT(objspace->malloc_params.allocations > 0);
11971 }
11972#endif
11973 }
11974 break;
11975 case MEMOP_TYPE_REALLOC: /* ignore */ break;
11976 }
11977#endif
11978 return true;
11979}
11980
11981#define objspace_malloc_increase(...) \
11982 for (bool malloc_increase_done = objspace_malloc_increase_report(__VA_ARGS__); \
11983 !malloc_increase_done; \
11984 malloc_increase_done = objspace_malloc_increase_body(__VA_ARGS__))
11985
11986struct malloc_obj_info { /* 4 words */
11987 size_t size;
11988};
11989
11990static inline size_t
11991objspace_malloc_prepare(rb_objspace_t *objspace, size_t size)
11992{
11993 if (size == 0) size = 1;
11994
11995#if CALC_EXACT_MALLOC_SIZE
11996 size += sizeof(struct malloc_obj_info);
11997#endif
11998
11999 return size;
12000}
12001
12002static bool
12003malloc_during_gc_p(rb_objspace_t *objspace)
12004{
12005 /* malloc is not allowed during GC when we're not using multiple ractors
12006 * (since ractors can run while another thread is sweeping) and when we
12007 * have the GVL (since if we don't have the GVL, we'll try to acquire the
12008 * GVL which will block and ensure the other thread finishes GC). */
12009 return during_gc && !dont_gc_val() && !rb_gc_multi_ractor_p() && ruby_thread_has_gvl_p();
12010}
12011
12012static inline void *
12013objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size, bool gc_allowed)
12014{
12015 size = objspace_malloc_size(objspace, mem, size);
12016 objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC, gc_allowed) {}
12017
12018#if CALC_EXACT_MALLOC_SIZE
12019 {
12020 struct malloc_obj_info *info = mem;
12021 info->size = size;
12022 mem = info + 1;
12023 }
12024#endif
12025
12026 return mem;
12027}
12028
12029#if defined(__GNUC__) && RUBY_DEBUG
12030#define RB_BUG_INSTEAD_OF_RB_MEMERROR 1
12031#endif
12032
12033#ifndef RB_BUG_INSTEAD_OF_RB_MEMERROR
12034# define RB_BUG_INSTEAD_OF_RB_MEMERROR 0
12035#endif
12036
12037#define GC_MEMERROR(...) \
12038 ((RB_BUG_INSTEAD_OF_RB_MEMERROR+0) ? rb_bug("" __VA_ARGS__) : (void)0)
12039
12040#define TRY_WITH_GC(siz, expr) do { \
12041 const gc_profile_record_flag gpr = \
12042 GPR_FLAG_FULL_MARK | \
12043 GPR_FLAG_IMMEDIATE_MARK | \
12044 GPR_FLAG_IMMEDIATE_SWEEP | \
12045 GPR_FLAG_MALLOC; \
12046 /* stress GC must also honor gc_allowed (malloc_gc_disabled) */ \
12047 if (gc_allowed) objspace_malloc_gc_stress(objspace); \
12048 \
12049 if (RB_LIKELY((expr))) { \
12050 /* Success on 1st try */ \
12051 } \
12052 else if (gc_allowed && !garbage_collect_with_gvl(objspace, gpr)) { \
12053 /* @shyouhei thinks this doesn't happen */ \
12054 GC_MEMERROR("TRY_WITH_GC: could not GC"); \
12055 } \
12056 else if ((expr)) { \
12057 /* Success on 2nd try */ \
12058 } \
12059 else { \
12060 GC_MEMERROR("TRY_WITH_GC: could not allocate:" \
12061 "%"PRIdSIZE" bytes for %s", \
12062 siz, # expr); \
12063 } \
12064 } while (0)
12065
12066static void
12067check_malloc_not_in_gc(rb_objspace_t *objspace, const char *msg)
12068{
12069 if (RB_UNLIKELY(malloc_during_gc_p(objspace))) {
12070 dont_gc_on();
12071 during_gc = false;
12072 rb_bug("Cannot %s during GC", msg);
12073 }
12074}
12075
12076void
12077rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size)
12078{
12079 rb_objspace_t *objspace = objspace_ptr;
12080
12081 if (!ptr) {
12082 /*
12083 * ISO/IEC 9899 says "If ptr is a null pointer, no action occurs" since
12084 * its first version. We would better follow.
12085 */
12086 return;
12087 }
12088#if CALC_EXACT_MALLOC_SIZE
12089 struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
12090#if VERIFY_FREE_SIZE
12091 if (!info->size) {
12092 const char *freeing = gc_freeing_obj_info();
12093 rb_bug("buffer %p has no recorded size%s%s. Was it allocated with ruby_mimalloc? If so it should be freed with ruby_mimfree", ptr,
12094 freeing ? ", while freeing " : "", freeing ? freeing : "");
12095 }
12096
12097 if (old_size && (old_size + sizeof(struct malloc_obj_info)) != info->size) {
12098 const char *freeing = gc_freeing_obj_info();
12099 rb_bug("buffer %p freed with old_size=%zu, but was allocated with size=%zu%s%s", ptr, old_size, info->size - sizeof(struct malloc_obj_info),
12100 freeing ? ", while freeing " : "", freeing ? freeing : "");
12101 }
12102#endif
12103 ptr = info;
12104 old_size = info->size;
12105#endif
12106 old_size = objspace_malloc_size(objspace, ptr, old_size);
12107
12108 objspace_malloc_increase(objspace, ptr, 0, old_size, MEMOP_TYPE_FREE, true) {
12109 free(ptr);
12110 ptr = NULL;
12111 RB_DEBUG_COUNTER_INC(heap_xfree);
12112 }
12113}
12114
12115void *
12116rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed)
12117{
12118 rb_objspace_t *objspace = objspace_ptr;
12119 check_malloc_not_in_gc(objspace, "malloc");
12120
12121 void *mem;
12122
12123 size = objspace_malloc_prepare(objspace, size);
12124 TRY_WITH_GC(size, mem = malloc(size));
12125 RB_DEBUG_COUNTER_INC(heap_xmalloc);
12126 if (!mem) return mem;
12127 return objspace_malloc_fixup(objspace, mem, size, gc_allowed);
12128}
12129
12130void *
12131rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed)
12132{
12133 rb_objspace_t *objspace = objspace_ptr;
12134
12135 if (RB_UNLIKELY(malloc_during_gc_p(objspace))) {
12136 rb_warn("calloc during GC detected, this could cause crashes if it triggers another GC");
12137#if RGENGC_CHECK_MODE || RUBY_DEBUG
12138 rb_bug("Cannot calloc during GC");
12139#endif
12140 }
12141
12142 void *mem;
12143
12144 size = objspace_malloc_prepare(objspace, size);
12145 TRY_WITH_GC(size, mem = calloc1(size));
12146 if (!mem) return mem;
12147 return objspace_malloc_fixup(objspace, mem, size, gc_allowed);
12148}
12149
12150void *
12151rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed)
12152{
12153 rb_objspace_t *objspace = objspace_ptr;
12154
12155 check_malloc_not_in_gc(objspace, "realloc");
12156
12157 void *mem;
12158
12159 if (!ptr) return rb_gc_impl_malloc(objspace, new_size, gc_allowed);
12160
12161 /*
12162 * The behavior of realloc(ptr, 0) is implementation defined.
12163 * Therefore we don't use realloc(ptr, 0) for portability reason.
12164 * see http://www.open-std.org/jtc1/sc22/wg14/www/docs/dr_400.htm
12165 */
12166 if (new_size == 0) {
12167 if ((mem = rb_gc_impl_malloc(objspace, 0, gc_allowed)) != NULL) {
12168 /*
12169 * - OpenBSD's malloc(3) man page says that when 0 is passed, it
12170 * returns a non-NULL pointer to an access-protected memory page.
12171 * The returned pointer cannot be read / written at all, but
12172 * still be a valid argument of free().
12173 *
12174 * https://man.openbsd.org/malloc.3
12175 *
12176 * - Linux's malloc(3) man page says that it _might_ perhaps return
12177 * a non-NULL pointer when its argument is 0. That return value
12178 * is safe (and is expected) to be passed to free().
12179 *
12180 * https://man7.org/linux/man-pages/man3/malloc.3.html
12181 *
12182 * - As I read the implementation jemalloc's malloc() returns fully
12183 * normal 16 bytes memory region when its argument is 0.
12184 *
12185 * - As I read the implementation musl libc's malloc() returns
12186 * fully normal 32 bytes memory region when its argument is 0.
12187 *
12188 * - Other malloc implementations can also return non-NULL.
12189 */
12190 rb_gc_impl_free(objspace, ptr, old_size);
12191 return mem;
12192 }
12193 else {
12194 /*
12195 * It is dangerous to return NULL here, because that could lead to
12196 * RCE. Fallback to 1 byte instead of zero.
12197 *
12198 * https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11932
12199 */
12200 new_size = 1;
12201 }
12202 }
12203
12204#if CALC_EXACT_MALLOC_SIZE
12205 {
12206 struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
12207 new_size += sizeof(struct malloc_obj_info);
12208 ptr = info;
12209#if VERIFY_FREE_SIZE
12210 if (old_size && (old_size + sizeof(struct malloc_obj_info)) != info->size) {
12211 const char *freeing = gc_freeing_obj_info();
12212 rb_bug("buffer %p realloced with old_size=%zu, but was allocated with size=%zu%s%s", ptr, old_size, info->size - sizeof(struct malloc_obj_info),
12213 freeing ? ", while freeing " : "", freeing ? freeing : "");
12214 }
12215#endif
12216 old_size = info->size;
12217 }
12218#endif
12219
12220 old_size = objspace_malloc_size(objspace, ptr, old_size);
12221 TRY_WITH_GC(new_size, mem = RB_GNUC_EXTENSION_BLOCK(realloc(ptr, new_size)));
12222 if (!mem) return mem;
12223 new_size = objspace_malloc_size(objspace, mem, new_size);
12224
12225#if CALC_EXACT_MALLOC_SIZE
12226 {
12227 struct malloc_obj_info *info = mem;
12228 info->size = new_size;
12229 mem = info + 1;
12230 }
12231#endif
12232
12233 objspace_malloc_increase(objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC, gc_allowed);
12234
12235 RB_DEBUG_COUNTER_INC(heap_xrealloc);
12236 return mem;
12237}
12238
12239void
12240rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff)
12241{
12242 rb_objspace_t *objspace = objspace_ptr;
12243
12244 if (diff > 0) {
12245 objspace_malloc_increase(objspace, 0, diff, 0, MEMOP_TYPE_REALLOC, true);
12246 }
12247 else if (diff < 0) {
12248 objspace_malloc_increase(objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC, true);
12249 }
12250}
12251
12252// TODO: move GC profiler stuff back into gc.c
12253/*
12254 ------------------------------ GC profiler ------------------------------
12255*/
12256
12257#define GC_PROFILE_RECORD_DEFAULT_SIZE 100
12258#define GC_PROFILE_RECORD_DEFAULT_MAX_RECORDS 4096
12259#define GC_PROFILE_RECORD_UNBOUNDED 0
12260
12261static bool
12262current_process_time(struct timespec *ts)
12263{
12264#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
12265 {
12266 static int try_clock_gettime = 1;
12267 if (try_clock_gettime) {
12268 if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ts) == 0) {
12269 return true;
12270 }
12271 else {
12272 try_clock_gettime = 0;
12273 }
12274 }
12275 }
12276#endif
12277
12278#ifdef RUSAGE_SELF
12279 {
12280 struct rusage usage;
12281 struct timeval time;
12282 if (getrusage(RUSAGE_SELF, &usage) == 0) {
12283 time = usage.ru_utime;
12284 ts->tv_sec = time.tv_sec;
12285 ts->tv_nsec = (int32_t)time.tv_usec * 1000;
12286 return true;
12287 }
12288 }
12289#endif
12290
12291#ifdef _WIN32
12292 {
12293 FILETIME creation_time, exit_time, kernel_time, user_time;
12294 ULARGE_INTEGER ui;
12295
12296 if (GetProcessTimes(GetCurrentProcess(),
12297 &creation_time, &exit_time, &kernel_time, &user_time) != 0) {
12298 memcpy(&ui, &user_time, sizeof(FILETIME));
12299#define PER100NSEC (uint64_t)(1000 * 1000 * 10)
12300 ts->tv_nsec = (long)(ui.QuadPart % PER100NSEC);
12301 ts->tv_sec = (time_t)(ui.QuadPart / PER100NSEC);
12302 return true;
12303 }
12304 }
12305#endif
12306
12307 return false;
12308}
12309
12310static double
12311getrusage_time(void)
12312{
12313 struct timespec ts;
12314 if (current_process_time(&ts)) {
12315 return ts.tv_sec + ts.tv_nsec * 1e-9;
12316 }
12317 else {
12318 return 0.0;
12319 }
12320}
12321
12322static inline double
12323hrtime_to_sec(rb_hrtime_t time)
12324{
12325 return (double)time / (double)RB_HRTIME_PER_SEC;
12326}
12327
12328static inline rb_hrtime_t
12329elapsed_hrtime_from(rb_hrtime_t start)
12330{
12331 return rb_hrtime_sub(rb_hrtime_now(), start);
12332}
12333
12334
12335static inline size_t
12336gc_profile_record_count(rb_objspace_t *objspace)
12337{
12338 return objspace->profile.record_count;
12339}
12340
12341static inline size_t
12342gc_profile_record_index(rb_objspace_t *objspace, size_t logical_index)
12343{
12344 if (objspace->profile.max_records != GC_PROFILE_RECORD_UNBOUNDED &&
12345 objspace->profile.record_count == objspace->profile.size) {
12346 return (objspace->profile.next_index + logical_index) % objspace->profile.size;
12347 }
12348 else {
12349 return logical_index;
12350 }
12351}
12352
12353static void
12354gc_profile_records_free(rb_objspace_t *objspace)
12355{
12356 void *p = objspace->profile.records;
12357 objspace->profile.records = NULL;
12358 objspace->profile.size = 0;
12359 objspace->profile.next_index = 0;
12360 objspace->profile.record_count = 0;
12361 objspace->profile.current_record = 0;
12362 free(p);
12363}
12364
12365static inline void
12366gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason)
12367{
12368 if (objspace->profile.run) {
12369 size_t index;
12370 gc_profile_record *record;
12371
12372 if (objspace->profile.max_records == GC_PROFILE_RECORD_UNBOUNDED) {
12373 index = objspace->profile.record_count++;
12374 objspace->profile.next_index = objspace->profile.record_count;
12375
12376 if (!objspace->profile.records) {
12377 objspace->profile.size = GC_PROFILE_RECORD_DEFAULT_SIZE;
12378 objspace->profile.records = malloc(xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
12379 }
12380 if (index >= objspace->profile.size) {
12381 void *ptr;
12382 objspace->profile.size += 1000;
12383 ptr = realloc(objspace->profile.records, xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
12384 if (!ptr) rb_memerror();
12385 objspace->profile.records = ptr;
12386 }
12387 }
12388 else {
12389 if (!objspace->profile.records) {
12390 objspace->profile.size = objspace->profile.max_records;
12391 objspace->profile.records = malloc(xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
12392 }
12393 index = objspace->profile.next_index;
12394 objspace->profile.next_index = (objspace->profile.next_index + 1) % objspace->profile.size;
12395 if (objspace->profile.record_count < objspace->profile.size) {
12396 objspace->profile.record_count++;
12397 }
12398 }
12399
12400 if (!objspace->profile.records) {
12401 rb_bug("gc_profile malloc or realloc miss");
12402 }
12403 record = objspace->profile.current_record = &objspace->profile.records[index];
12404 MEMZERO(record, gc_profile_record, 1);
12405
12406 /* setup before-GC parameter */
12407 record->flags = reason | (ruby_gc_stressful ? GPR_FLAG_STRESS : 0);
12408 record->sequence = objspace->profile.record_sequence++;
12409 record->gc_invoke_wall_time = rb_hrtime_sub(rb_hrtime_now(),
12410 objspace->profile.invoke_wall_time);
12411#if MALLOC_ALLOCATED_SIZE
12412 record->allocated_size = malloc_allocated_size;
12413#endif
12414#if GC_PROFILE_MORE_DETAIL && GC_PROFILE_DETAIL_MEMORY
12415#ifdef RUSAGE_SELF
12416 {
12417 struct rusage usage;
12418 if (getrusage(RUSAGE_SELF, &usage) == 0) {
12419 record->maxrss = usage.ru_maxrss;
12420 record->minflt = usage.ru_minflt;
12421 record->majflt = usage.ru_majflt;
12422 }
12423 }
12424#endif
12425#endif
12426 }
12427}
12428
12429static inline void
12430gc_prof_timer_start(rb_objspace_t *objspace)
12431{
12432 if (gc_prof_enabled(objspace)) {
12433 gc_profile_record *record = gc_prof_record(objspace);
12434#if GC_PROFILE_MORE_DETAIL
12435 record->prepare_time = objspace->profile.prepare_time;
12436#endif
12437 record->gc_time = 0;
12438 record->gc_invoke_time = getrusage_time();
12439 objspace->profile.gc_wall_start_time = rb_hrtime_now();
12440 }
12441}
12442
12443static double
12444elapsed_time_from(double time)
12445{
12446 double now = getrusage_time();
12447 if (now > time) {
12448 return now - time;
12449 }
12450 else {
12451 return 0;
12452 }
12453}
12454
12455static inline void
12456gc_prof_timer_stop(rb_objspace_t *objspace)
12457{
12458 if (gc_prof_enabled(objspace)) {
12459 gc_profile_record *record = gc_prof_record(objspace);
12460 record->gc_time = elapsed_time_from(record->gc_invoke_time);
12461 record->gc_invoke_time -= objspace->profile.invoke_time;
12462 record->gc_wall_time = elapsed_hrtime_from(objspace->profile.gc_wall_start_time);
12463 }
12464}
12465
12466static inline void
12467gc_prof_mark_timer_start(rb_objspace_t *objspace)
12468{
12469 RUBY_DTRACE_GC_HOOK(MARK_BEGIN);
12470#if GC_PROFILE_MORE_DETAIL
12471 if (gc_prof_enabled(objspace)) {
12472 gc_prof_record(objspace)->gc_mark_time = getrusage_time();
12473 }
12474#endif
12475}
12476
12477static inline void
12478gc_prof_mark_timer_stop(rb_objspace_t *objspace)
12479{
12480 RUBY_DTRACE_GC_HOOK(MARK_END);
12481#if GC_PROFILE_MORE_DETAIL
12482 if (gc_prof_enabled(objspace)) {
12483 gc_profile_record *record = gc_prof_record(objspace);
12484 record->gc_mark_time = elapsed_time_from(record->gc_mark_time);
12485 }
12486#endif
12487}
12488
12489static inline void
12490gc_prof_sweep_timer_start(rb_objspace_t *objspace)
12491{
12492 RUBY_DTRACE_GC_HOOK(SWEEP_BEGIN);
12493 if (gc_prof_enabled(objspace)) {
12494 gc_profile_record *record = gc_prof_record(objspace);
12495
12496 if (record->gc_time > 0 || GC_PROFILE_MORE_DETAIL) {
12497 objspace->profile.gc_sweep_start_time = getrusage_time();
12498 objspace->profile.gc_sweep_wall_start_time = rb_hrtime_now();
12499 }
12500 }
12501}
12502
12503static inline void
12504gc_prof_sweep_timer_stop(rb_objspace_t *objspace)
12505{
12506 RUBY_DTRACE_GC_HOOK(SWEEP_END);
12507
12508 if (gc_prof_enabled(objspace)) {
12509 double sweep_time;
12510 gc_profile_record *record = gc_prof_record(objspace);
12511
12512 if (record->gc_time > 0) {
12513 sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
12514 /* need to accumulate GC time for lazy sweep after gc() */
12515 record->gc_time += sweep_time;
12516 record->gc_wall_time = rb_hrtime_add(record->gc_wall_time,
12517 elapsed_hrtime_from(objspace->profile.gc_sweep_wall_start_time));
12518 }
12519 else if (GC_PROFILE_MORE_DETAIL) {
12520 sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
12521 }
12522
12523#if GC_PROFILE_MORE_DETAIL
12524 record->gc_sweep_time += sweep_time;
12525 if (heap_pages_deferred_final) record->flags |= GPR_FLAG_HAVE_FINALIZE;
12526#endif
12527 if (heap_pages_deferred_final) objspace->profile.latest_gc_info |= GPR_FLAG_HAVE_FINALIZE;
12528 }
12529}
12530
12531static inline void
12532gc_prof_set_malloc_info(rb_objspace_t *objspace)
12533{
12534#if GC_PROFILE_MORE_DETAIL
12535 if (gc_prof_enabled(objspace)) {
12536 gc_profile_record *record = gc_prof_record(objspace);
12537 record->allocate_increase = malloc_increase;
12538 record->allocate_limit = malloc_limit;
12539 }
12540#endif
12541}
12542
12543static inline void
12544gc_prof_set_heap_info(rb_objspace_t *objspace)
12545{
12546 if (gc_prof_enabled(objspace)) {
12547 gc_profile_record *record = gc_prof_record(objspace);
12548
12549 /* Sum across all size pools since each has a different slot size. */
12550 size_t total = 0;
12551 size_t use_size = 0;
12552 size_t total_size = 0;
12553 for (int i = 0; i < HEAP_COUNT; i++) {
12554 rb_heap_t *heap = &heaps[i];
12555 size_t heap_live = heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count;
12556 total += heap->total_slots;
12557 use_size += heap_live * heap->slot_size;
12558 total_size += heap->total_slots * heap->slot_size;
12559 }
12560
12561#if GC_PROFILE_MORE_DETAIL
12562 size_t live = objspace->profile.total_allocated_objects_at_gc_start - total_freed_objects(objspace);
12563 record->heap_use_pages = objspace->profile.heap_used_at_gc_start;
12564 record->heap_live_objects = live;
12565 record->heap_free_objects = total - live;
12566#endif
12567
12568 record->heap_total_objects = total;
12569 record->heap_use_size = use_size;
12570 record->heap_total_size = total_size;
12571 }
12572}
12573
12574/*
12575 * call-seq:
12576 * GC::Profiler.clear -> nil
12577 *
12578 * Clears the \GC profiler data.
12579 *
12580 */
12581
12582static VALUE
12583gc_profile_clear(VALUE _)
12584{
12585 rb_objspace_t *objspace = rb_gc_get_objspace();
12586 gc_profile_records_free(objspace);
12587 return Qnil;
12588}
12589
12590/*
12591 * call-seq:
12592 * GC::Profiler.configure(max_records: 4096) -> nil
12593 *
12594 * Configures how many raw profile records are retained by
12595 * GC::Profiler.raw_data.
12596 *
12597 * The profiler keeps at most +max_records+ records in a bounded ring buffer.
12598 * When the buffer is full, newer GC records overwrite the oldest retained
12599 * records. The default limit is 4096 records.
12600 *
12601 * Pass +nil+ to restore the historical unbounded behavior:
12602 *
12603 * GC::Profiler.configure(max_records: nil)
12604 *
12605 * Changing +max_records+ clears existing raw profile data. This method does
12606 * not enable or disable the profiler; use GC::Profiler.enable and
12607 * GC::Profiler.disable for that.
12608 */
12609
12610static VALUE
12611gc_profile_configure(int argc, VALUE *argv, VALUE _)
12612{
12613 static ID keywords[1] = {0};
12614 VALUE options, max_records;
12615 rb_objspace_t *objspace = rb_gc_get_objspace();
12616
12617 if (!keywords[0]) {
12618 keywords[0] = rb_intern("max_records");
12619 }
12620
12621 rb_scan_args_kw(rb_keyword_given_p(), argc, argv, ":", &options);
12622 rb_get_kwargs(options, keywords, 0, 1, &max_records);
12623
12624 if (max_records == Qundef) {
12625 return Qnil;
12626 }
12627 else if (NIL_P(max_records)) {
12628 objspace->profile.max_records = GC_PROFILE_RECORD_UNBOUNDED;
12629 }
12630 else {
12631 long value = NUM2LONG(max_records);
12632 if (value <= 0) {
12633 rb_raise(rb_eArgError, "max_records must be positive or nil");
12634 }
12635 objspace->profile.max_records = (size_t)value;
12636 }
12637
12638 gc_profile_records_free(objspace);
12639 return Qnil;
12640}
12641
12642/*
12643 * call-seq:
12644 * GC::Profiler.raw_data(limit: nil, since: nil) -> [Hash, ...]
12645 *
12646 * Returns an Array of retained raw profile data Hashes ordered from earliest
12647 * to latest by +:GC_INVOKE_TIME+. +limit:+ returns at most the newest
12648 * retained records. +since:+ returns records with +:GC_SEQUENCE+ greater
12649 * than the given sequence.
12650 *
12651 * For example:
12652 *
12653 * [
12654 * {
12655 * :GC_TIME=>1.3000000000000858e-05,
12656 * :GC_INVOKE_TIME=>0.010634999999999999,
12657 * :GC_WALL_TIME=>1.4000000000000001e-05,
12658 * :GC_INVOKE_WALL_TIME=>0.010640000000000000,
12659 * :GC_PAUSE_TIME=>1.5000000000000000e-05,
12660 * :GC_STOP_TIME=>1.0000000000000000e-06,
12661 * :GC_STW_TIME=>1.4000000000000001e-05,
12662 * :GC_MARK_WALL_TIME=>9.0000000000000002e-06,
12663 * :GC_SWEEP_WALL_TIME=>5.0000000000000004e-06,
12664 * :GC_COMPACT_WALL_TIME=>0.0000000000000000e+00,
12665 * :HEAP_USE_SIZE=>289640,
12666 * :HEAP_TOTAL_SIZE=>588960,
12667 * :HEAP_TOTAL_OBJECTS=>14724,
12668 * :GC_IS_MARKED=>false
12669 * },
12670 * # ...
12671 * ]
12672 *
12673 * The keys mean:
12674 *
12675 * +:GC_SEQUENCE+::
12676 * Monotonically increasing sequence number for this profiler record.
12677 * +:GC_TIME+::
12678 * CPU time elapsed in seconds for this GC run. This is process CPU time,
12679 * not elapsed wall-clock time.
12680 * +:GC_INVOKE_TIME+::
12681 * CPU time elapsed in seconds from startup to when the GC was invoked.
12682 * +:GC_WALL_TIME+::
12683 * Monotonic wall-clock counterpart to +:GC_TIME+ for this GC record.
12684 * This does not include time spent stopping other ractors before the VM
12685 * enters GC. Use the phase wall-clock fields below for mark, sweep, and
12686 * compaction attribution.
12687 * +:GC_INVOKE_WALL_TIME+::
12688 * Monotonic wall-clock time elapsed in seconds from startup to when the GC
12689 * was invoked.
12690 * +:GC_PAUSE_TIME+::
12691 * Monotonic wall-clock time elapsed in seconds while user execution was
12692 * blocked by this GC entry, including time to stop other ractors. This
12693 * may include time from incremental marking or lazy sweeping continuation
12694 * charged to this record.
12695 * +:GC_STOP_TIME+::
12696 * Monotonic wall-clock time elapsed in seconds stopping other ractors.
12697 * +:GC_STW_TIME+::
12698 * Monotonic wall-clock time elapsed in seconds after other ractors have
12699 * stopped and before the VM exits GC.
12700 * +:GC_MARK_WALL_TIME+::
12701 * Monotonic wall-clock time elapsed in seconds spent marking for this GC
12702 * record, accumulated across incremental marking continuations.
12703 * +:GC_SWEEP_WALL_TIME+::
12704 * Monotonic wall-clock time elapsed in seconds spent sweeping for this GC
12705 * record, accumulated across lazy sweeping continuations. This does not
12706 * include compaction time, which is reported separately as
12707 * +:GC_COMPACT_WALL_TIME+.
12708 * +:GC_COMPACT_WALL_TIME+::
12709 * Monotonic wall-clock time elapsed in seconds spent compacting for this GC
12710 * record, or +0.0+ if this GC did not compact.
12711 * +:HEAP_USE_SIZE+::
12712 * Total bytes of heap used
12713 * +:HEAP_TOTAL_SIZE+::
12714 * Total size of heap in bytes
12715 * +:HEAP_TOTAL_OBJECTS+::
12716 * Total number of objects
12717 * +:GC_IS_MARKED+::
12718 * Returns +true+ if the GC is in mark phase
12719 *
12720 * The wall-clock timing fields relate to each other as follows:
12721 *
12722 * GC_PAUSE_TIME == GC_STOP_TIME + GC_STW_TIME
12723 *
12724 * +:GC_MARK_WALL_TIME+, +:GC_SWEEP_WALL_TIME+, and +:GC_COMPACT_WALL_TIME+
12725 * report separate phase timings and must not be added to +:GC_WALL_TIME+.
12726 *
12727 * +:GC_WALL_TIME+ is the wall-clock counterpart to +:GC_TIME+ and is nested
12728 * inside +:GC_STW_TIME+, so it must not be added to +:GC_STW_TIME+. The difference
12729 * +GC_STW_TIME - GC_WALL_TIME+ is VM overhead inside the stopped interval
12730 * (GC event hooks, bookkeeping, consistency checks, and continuation work).
12731 *
12732 * If ruby was built with +GC_PROFILE_MORE_DETAIL+, you will also have access
12733 * to the following hash keys:
12734 *
12735 * +:GC_MARK_TIME+::
12736 * +:GC_SWEEP_TIME+::
12737 * +:ALLOCATE_INCREASE+::
12738 * +:ALLOCATE_LIMIT+::
12739 * +:HEAP_USE_PAGES+::
12740 * +:HEAP_LIVE_OBJECTS+::
12741 * +:HEAP_FREE_OBJECTS+::
12742 * +:HAVE_FINALIZE+::
12743 *
12744 */
12745
12746static VALUE
12747gc_profile_record_get(int argc, VALUE *argv, VALUE _)
12748{
12749 static ID keywords[2] = {0};
12750 VALUE prof, options, limit_value, since_value;
12751 VALUE gc_profile = rb_ary_new();
12752 size_t i, count, matching = 0, skip = 0, limit = SIZE_MAX, since = 0;
12753 bool use_since = false;
12754 rb_objspace_t *objspace = rb_gc_get_objspace();
12755
12756 if (!keywords[0]) {
12757 keywords[0] = rb_intern("limit");
12758 keywords[1] = rb_intern("since");
12759 }
12760
12761 rb_scan_args_kw(rb_keyword_given_p(), argc, argv, ":", &options);
12762 VALUE values[2] = {Qundef, Qundef};
12763 rb_get_kwargs(options, keywords, 0, 2, values);
12764 limit_value = values[0];
12765 since_value = values[1];
12766
12767 if (limit_value != Qundef && !NIL_P(limit_value)) {
12768 long value = NUM2LONG(limit_value);
12769 if (value < 0) {
12770 rb_raise(rb_eArgError, "limit must be non-negative");
12771 }
12772 limit = (size_t)value;
12773 }
12774 if (since_value != Qundef && !NIL_P(since_value)) {
12775 long value = NUM2LONG(since_value);
12776 if (value < 0) {
12777 rb_raise(rb_eArgError, "since must be non-negative");
12778 }
12779 since = (size_t)value;
12780 use_since = true;
12781 }
12782
12783 if (!objspace->profile.run) {
12784 return Qnil;
12785 }
12786
12787 count = gc_profile_record_count(objspace);
12788 for (i = 0; i < count; i++) {
12789 gc_profile_record *record = &objspace->profile.records[gc_profile_record_index(objspace, i)];
12790 if (!use_since || record->sequence > since) {
12791 matching++;
12792 }
12793 }
12794 if (limit < matching) {
12795 skip = matching - limit;
12796 }
12797
12798 for (i = 0; i < count; i++) {
12799 gc_profile_record *record = &objspace->profile.records[gc_profile_record_index(objspace, i)];
12800 if (use_since && record->sequence <= since) {
12801 continue;
12802 }
12803 if (skip > 0) {
12804 skip--;
12805 continue;
12806 }
12807
12808 prof = rb_hash_new();
12809 rb_hash_aset(prof, ID2SYM(rb_intern("GC_FLAGS")), gc_info_decode(objspace, rb_hash_new(), record->flags));
12810 rb_hash_aset(prof, ID2SYM(rb_intern("GC_SEQUENCE")), SIZET2NUM(record->sequence));
12811 rb_hash_aset(prof, ID2SYM(rb_intern("GC_TIME")), DBL2NUM(record->gc_time));
12812 rb_hash_aset(prof, ID2SYM(rb_intern("GC_INVOKE_TIME")), DBL2NUM(record->gc_invoke_time));
12813 rb_hash_aset(prof, ID2SYM(rb_intern("GC_WALL_TIME")),
12814 DBL2NUM(hrtime_to_sec(record->gc_wall_time)));
12815 rb_hash_aset(prof, ID2SYM(rb_intern("GC_INVOKE_WALL_TIME")),
12816 DBL2NUM(hrtime_to_sec(record->gc_invoke_wall_time)));
12817 rb_hash_aset(prof, ID2SYM(rb_intern("GC_PAUSE_TIME")),
12818 DBL2NUM(hrtime_to_sec(record->gc_pause_time)));
12819 rb_hash_aset(prof, ID2SYM(rb_intern("GC_STOP_TIME")),
12820 DBL2NUM(hrtime_to_sec(record->gc_stop_time)));
12821 rb_hash_aset(prof, ID2SYM(rb_intern("GC_STW_TIME")),
12822 DBL2NUM(hrtime_to_sec(record->gc_stw_time)));
12823 rb_hash_aset(prof, ID2SYM(rb_intern("GC_MARK_WALL_TIME")),
12824 DBL2NUM(hrtime_to_sec(record->gc_mark_wall_time)));
12825 rb_hash_aset(prof, ID2SYM(rb_intern("GC_SWEEP_WALL_TIME")),
12826 DBL2NUM(hrtime_to_sec(record->gc_sweep_wall_time)));
12827 rb_hash_aset(prof, ID2SYM(rb_intern("GC_COMPACT_WALL_TIME")),
12828 DBL2NUM(hrtime_to_sec(record->gc_compact_wall_time)));
12829 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_SIZE")), SIZET2NUM(record->heap_use_size));
12830 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_SIZE")), SIZET2NUM(record->heap_total_size));
12831 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_OBJECTS")), SIZET2NUM(record->heap_total_objects));
12832 rb_hash_aset(prof, ID2SYM(rb_intern("MOVED_OBJECTS")), SIZET2NUM(record->moved_objects));
12833 rb_hash_aset(prof, ID2SYM(rb_intern("GC_IS_MARKED")), Qtrue);
12834#if GC_PROFILE_MORE_DETAIL
12835 rb_hash_aset(prof, ID2SYM(rb_intern("GC_MARK_TIME")), DBL2NUM(record->gc_mark_time));
12836 rb_hash_aset(prof, ID2SYM(rb_intern("GC_SWEEP_TIME")), DBL2NUM(record->gc_sweep_time));
12837 rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_INCREASE")), SIZET2NUM(record->allocate_increase));
12838 rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_LIMIT")), SIZET2NUM(record->allocate_limit));
12839 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_PAGES")), SIZET2NUM(record->heap_use_pages));
12840 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_LIVE_OBJECTS")), SIZET2NUM(record->heap_live_objects));
12841 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_FREE_OBJECTS")), SIZET2NUM(record->heap_free_objects));
12842
12843 rb_hash_aset(prof, ID2SYM(rb_intern("REMOVING_OBJECTS")), SIZET2NUM(record->removing_objects));
12844 rb_hash_aset(prof, ID2SYM(rb_intern("EMPTY_OBJECTS")), SIZET2NUM(record->empty_objects));
12845
12846 rb_hash_aset(prof, ID2SYM(rb_intern("HAVE_FINALIZE")), (record->flags & GPR_FLAG_HAVE_FINALIZE) ? Qtrue : Qfalse);
12847#endif
12848
12849#if RGENGC_PROFILE > 0
12850 rb_hash_aset(prof, ID2SYM(rb_intern("OLD_OBJECTS")), SIZET2NUM(record->old_objects));
12851 rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_NORMAL_OBJECTS")), SIZET2NUM(record->remembered_normal_objects));
12852 rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_SHADY_OBJECTS")), SIZET2NUM(record->remembered_shady_objects));
12853#endif
12854 rb_ary_push(gc_profile, prof);
12855 }
12856
12857 return gc_profile;
12858}
12859
12860#if GC_PROFILE_MORE_DETAIL
12861#define MAJOR_REASON_MAX 0x10
12862
12863static char *
12864gc_profile_dump_major_reason(unsigned int flags, char *buff)
12865{
12866 unsigned int reason = flags & GPR_FLAG_MAJOR_MASK;
12867 int i = 0;
12868
12869 if (reason == GPR_FLAG_NONE) {
12870 buff[0] = '-';
12871 buff[1] = 0;
12872 }
12873 else {
12874#define C(x, s) \
12875 if (reason & GPR_FLAG_MAJOR_BY_##x) { \
12876 buff[i++] = #x[0]; \
12877 if (i >= MAJOR_REASON_MAX) rb_bug("gc_profile_dump_major_reason: overflow"); \
12878 buff[i] = 0; \
12879 }
12880 C(NOFREE, N);
12881 C(OLDGEN, O);
12882 C(SHADY, S);
12883#if RGENGC_ESTIMATE_OLDMALLOC
12884 C(OLDMALLOC, M);
12885#endif
12886#undef C
12887 }
12888 return buff;
12889}
12890#endif
12891
12892
12893
12894static void
12895gc_profile_dump_on(VALUE out, VALUE (*append)(VALUE, VALUE))
12896{
12897 rb_objspace_t *objspace = rb_gc_get_objspace();
12898 size_t count = gc_profile_record_count(objspace);
12899#ifdef MAJOR_REASON_MAX
12900 char reason_str[MAJOR_REASON_MAX];
12901#endif
12902
12903 if (objspace->profile.run && count /* > 1 */) {
12904 size_t i;
12905 const gc_profile_record *record;
12906
12907 append(out, rb_sprintf("GC %"PRIuSIZE" invokes.\n", objspace->profile.count));
12908 append(out, rb_str_new_cstr("Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC Time(ms)\n"));
12909
12910 for (i = 0; i < count; i++) {
12911 record = &objspace->profile.records[gc_profile_record_index(objspace, i)];
12912 append(out, rb_sprintf("%5"PRIuSIZE" %19.3f %20"PRIuSIZE" %20"PRIuSIZE" %20"PRIuSIZE" %30.20f\n",
12913 i+1, record->gc_invoke_time, record->heap_use_size,
12914 record->heap_total_size, record->heap_total_objects, record->gc_time*1000));
12915 }
12916
12917#if GC_PROFILE_MORE_DETAIL
12918 const char *str = "\n\n" \
12919 "More detail.\n" \
12920 "Prepare Time = Previously GC's rest sweep time\n"
12921 "Index Flags Allocate Inc. Allocate Limit"
12922#if CALC_EXACT_MALLOC_SIZE
12923 " Allocated Size"
12924#endif
12925 " Use Page Mark Time(ms) Sweep Time(ms) Prepare Time(ms) LivingObj FreeObj RemovedObj EmptyObj"
12926#if RGENGC_PROFILE
12927 " OldgenObj RemNormObj RemShadObj"
12928#endif
12929#if GC_PROFILE_DETAIL_MEMORY
12930 " MaxRSS(KB) MinorFLT MajorFLT"
12931#endif
12932 "\n";
12933 append(out, rb_str_new_cstr(str));
12934
12935 for (i = 0; i < count; i++) {
12936 record = &objspace->profile.records[gc_profile_record_index(objspace, i)];
12937 append(out, rb_sprintf("%5"PRIuSIZE" %4s/%c/%6s%c %13"PRIuSIZE" %15"PRIuSIZE
12938#if CALC_EXACT_MALLOC_SIZE
12939 " %15"PRIuSIZE
12940#endif
12941 " %9"PRIuSIZE" %17.12f %17.12f %17.12f %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
12942#if RGENGC_PROFILE
12943 "%10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
12944#endif
12945#if GC_PROFILE_DETAIL_MEMORY
12946 "%11ld %8ld %8ld"
12947#endif
12948
12949 "\n",
12950 i+1,
12951 gc_profile_dump_major_reason(record->flags, reason_str),
12952 (record->flags & GPR_FLAG_HAVE_FINALIZE) ? 'F' : '.',
12953 (record->flags & GPR_FLAG_NEWOBJ) ? "NEWOBJ" :
12954 (record->flags & GPR_FLAG_MALLOC) ? "MALLOC" :
12955 (record->flags & GPR_FLAG_METHOD) ? "METHOD" :
12956 (record->flags & GPR_FLAG_CAPI) ? "CAPI__" : "??????",
12957 (record->flags & GPR_FLAG_STRESS) ? '!' : ' ',
12958 record->allocate_increase, record->allocate_limit,
12959#if CALC_EXACT_MALLOC_SIZE
12960 record->allocated_size,
12961#endif
12962 record->heap_use_pages,
12963 record->gc_mark_time*1000,
12964 record->gc_sweep_time*1000,
12965 record->prepare_time*1000,
12966
12967 record->heap_live_objects,
12968 record->heap_free_objects,
12969 record->removing_objects,
12970 record->empty_objects
12971#if RGENGC_PROFILE
12972 ,
12973 record->old_objects,
12974 record->remembered_normal_objects,
12975 record->remembered_shady_objects
12976#endif
12977#if GC_PROFILE_DETAIL_MEMORY
12978 ,
12979 record->maxrss / 1024,
12980 record->minflt,
12981 record->majflt
12982#endif
12983
12984 ));
12985 }
12986#endif
12987 }
12988}
12989
12990/*
12991 * call-seq:
12992 * GC::Profiler.result -> String
12993 *
12994 * Returns a profile data report such as:
12995 *
12996 * GC 1 invokes.
12997 * Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC time(ms)
12998 * 1 0.012 159240 212940 10647 0.00000000000001530000
12999 */
13000
13001static VALUE
13002gc_profile_result(VALUE _)
13003{
13004 VALUE str = rb_str_buf_new(0);
13005 gc_profile_dump_on(str, rb_str_buf_append);
13006 return str;
13007}
13008
13009/*
13010 * call-seq:
13011 * GC::Profiler.report
13012 * GC::Profiler.report(io)
13013 *
13014 * Writes the GC::Profiler.result to <tt>$stdout</tt> or the given IO object.
13015 *
13016 */
13017
13018static VALUE
13019gc_profile_report(int argc, VALUE *argv, VALUE self)
13020{
13021 VALUE out;
13022
13023 out = (!rb_check_arity(argc, 0, 1) ? rb_stdout : argv[0]);
13024 gc_profile_dump_on(out, rb_io_write);
13025
13026 return Qnil;
13027}
13028
13029/*
13030 * call-seq:
13031 * GC::Profiler.total_time -> float
13032 *
13033 * The total time used for garbage collection in seconds
13034 */
13035
13036static VALUE
13037gc_profile_total_time(VALUE self)
13038{
13039 double time = 0;
13040 rb_objspace_t *objspace = rb_gc_get_objspace();
13041
13042 if (objspace->profile.run && gc_profile_record_count(objspace) > 0) {
13043 size_t i;
13044 size_t count = gc_profile_record_count(objspace);
13045
13046 for (i = 0; i < count; i++) {
13047 time += objspace->profile.records[gc_profile_record_index(objspace, i)].gc_time;
13048 }
13049 }
13050 return DBL2NUM(time);
13051}
13052
13053/*
13054 * call-seq:
13055 * GC::Profiler.enabled? -> true or false
13056 *
13057 * The current status of \GC profile mode.
13058 */
13059
13060static VALUE
13061gc_profile_enable_get(VALUE self)
13062{
13063 rb_objspace_t *objspace = rb_gc_get_objspace();
13064 return objspace->profile.run ? Qtrue : Qfalse;
13065}
13066
13067/*
13068 * call-seq:
13069 * GC::Profiler.enable -> nil
13070 *
13071 * Starts the \GC profiler.
13072 *
13073 */
13074
13075static VALUE
13076gc_profile_enable(VALUE _)
13077{
13078 rb_objspace_t *objspace = rb_gc_get_objspace();
13079 objspace->profile.run = TRUE;
13080 objspace->profile.current_record = 0;
13081 return Qnil;
13082}
13083
13084/*
13085 * call-seq:
13086 * GC::Profiler.disable -> nil
13087 *
13088 * Stops the \GC profiler.
13089 *
13090 */
13091
13092static VALUE
13093gc_profile_disable(VALUE _)
13094{
13095 rb_objspace_t *objspace = rb_gc_get_objspace();
13096
13097 objspace->profile.run = FALSE;
13098 objspace->profile.current_record = 0;
13099 return Qnil;
13100}
13101
13102static void
13103rb_gc_verify_internal_consistency(void)
13104{
13105 gc_verify_internal_consistency(rb_gc_get_objspace());
13106}
13107
13108/*
13109 * call-seq:
13110 * GC.verify_internal_consistency -> nil
13111 *
13112 * Verifies internal consistency of the GC.
13113 * This method should only be used for debugging.
13114 *
13115 * This method is only expected to work on CRuby.
13116 */
13117static VALUE
13118gc_verify_internal_consistency_m(VALUE dummy)
13119{
13120 rb_gc_verify_internal_consistency();
13121 return Qnil;
13122}
13123
13124#if GC_CAN_COMPILE_COMPACTION
13125/*
13126 * call-seq:
13127 * GC.auto_compact = flag
13128 *
13129 * Updates automatic compaction mode.
13130 *
13131 * When enabled, the compactor will execute on every major collection.
13132 *
13133 * Enabling compaction will degrade performance on major collections.
13134 */
13135static VALUE
13136gc_set_auto_compact(VALUE _, VALUE v)
13137{
13138 GC_ASSERT(GC_COMPACTION_SUPPORTED);
13139
13140 ruby_enable_autocompact = RTEST(v);
13141
13142#if RGENGC_CHECK_MODE
13143 ruby_autocompact_compare_func = NULL;
13144
13145 if (SYMBOL_P(v)) {
13146 ID id = RB_SYM2ID(v);
13147 if (id == rb_intern("empty")) {
13148 ruby_autocompact_compare_func = compare_free_slots;
13149 }
13150 }
13151#endif
13152
13153 return v;
13154}
13155#else
13156# define gc_set_auto_compact rb_f_notimplement
13157#endif
13158
13159#if GC_CAN_COMPILE_COMPACTION
13160/*
13161 * call-seq:
13162 * GC.auto_compact -> true or false
13163 *
13164 * Returns whether or not automatic compaction has been enabled.
13165 */
13166static VALUE
13167gc_get_auto_compact(VALUE _)
13168{
13169 return ruby_enable_autocompact ? Qtrue : Qfalse;
13170}
13171#else
13172# define gc_get_auto_compact rb_f_notimplement
13173#endif
13174
13175#if GC_CAN_COMPILE_COMPACTION
13176/*
13177 * call-seq:
13178 * GC.latest_compact_info -> hash
13179 *
13180 * Returns information about object moved in the most recent \GC compaction.
13181 *
13182 * The returned +hash+ contains the following keys:
13183 *
13184 * [considered]
13185 * Hash containing the type of the object as the key and the number of
13186 * objects of that type that were considered for movement.
13187 * [moved]
13188 * Hash containing the type of the object as the key and the number of
13189 * objects of that type that were actually moved.
13190 * [moved_up]
13191 * Hash containing the type of the object as the key and the number of
13192 * objects of that type that were increased in size.
13193 * [moved_down]
13194 * Hash containing the type of the object as the key and the number of
13195 * objects of that type that were decreased in size.
13196 *
13197 * Some objects can't be moved (due to pinning) so these numbers can be used to
13198 * calculate compaction efficiency.
13199 */
13200static VALUE
13201gc_compact_stats(VALUE self)
13202{
13203 rb_objspace_t *objspace = rb_gc_get_objspace();
13204 VALUE h = rb_hash_new();
13205 VALUE considered = rb_hash_new();
13206 VALUE moved = rb_hash_new();
13207 VALUE moved_up = rb_hash_new();
13208 VALUE moved_down = rb_hash_new();
13209
13210 for (size_t i = 0; i < T_MASK; i++) {
13211 if (objspace->rcompactor.considered_count_table[i]) {
13212 rb_hash_aset(considered, type_sym(i), SIZET2NUM(objspace->rcompactor.considered_count_table[i]));
13213 }
13214
13215 if (objspace->rcompactor.moved_count_table[i]) {
13216 rb_hash_aset(moved, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_count_table[i]));
13217 }
13218
13219 if (objspace->rcompactor.moved_up_count_table[i]) {
13220 rb_hash_aset(moved_up, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_up_count_table[i]));
13221 }
13222
13223 if (objspace->rcompactor.moved_down_count_table[i]) {
13224 rb_hash_aset(moved_down, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_down_count_table[i]));
13225 }
13226 }
13227
13228 rb_hash_aset(h, ID2SYM(rb_intern("considered")), considered);
13229 rb_hash_aset(h, ID2SYM(rb_intern("moved")), moved);
13230 rb_hash_aset(h, ID2SYM(rb_intern("moved_up")), moved_up);
13231 rb_hash_aset(h, ID2SYM(rb_intern("moved_down")), moved_down);
13232
13233 return h;
13234}
13235#else
13236# define gc_compact_stats rb_f_notimplement
13237#endif
13238
13239#if GC_CAN_COMPILE_COMPACTION
13240/*
13241 * call-seq:
13242 * GC.compact -> hash
13243 *
13244 * This function compacts objects together in Ruby's heap. It eliminates
13245 * unused space (or fragmentation) in the heap by moving objects in to that
13246 * unused space. If there is more than 1 running Ractor, it runs a global
13247 * GC compaction (all object spaces).
13248 *
13249 * The returned +hash+ contains statistics about the objects that were moved;
13250 * see GC.latest_compact_info.
13251 *
13252 * This method is only expected to work on CRuby.
13253 *
13254 * To test whether \GC compaction is supported, use the idiom:
13255 *
13256 * GC.respond_to?(:compact)
13257 */
13258static VALUE
13259gc_compact(VALUE self)
13260{
13261 rb_objspace_t *objspace = rb_gc_get_objspace();
13262 int full_marking_p = gc_config_full_mark_val;
13263 gc_config_full_mark_set(TRUE);
13264
13265 /* Run GC with compaction enabled */
13266 rb_gc_impl_start(rb_gc_get_objspace(), true, true, true, true, true);
13267 gc_config_full_mark_set(full_marking_p);
13268
13269 return gc_compact_stats(self);
13270}
13271#else
13272# define gc_compact rb_f_notimplement
13273#endif
13274
13275#if GC_CAN_COMPILE_COMPACTION
13276struct desired_compaction_pages_i_data {
13277 rb_objspace_t *objspace;
13278 size_t required_slots[HEAP_COUNT];
13279};
13280
13281static int
13282desired_compaction_pages_i(struct heap_page *page, void *data)
13283{
13284 struct desired_compaction_pages_i_data *tdata = data;
13285 rb_objspace_t *objspace = tdata->objspace;
13286 VALUE vstart = (VALUE)page->start;
13287 VALUE vend = vstart + (VALUE)(page->total_slots * page->heap->slot_size);
13288
13289
13290 for (VALUE v = vstart; v != vend; v += page->heap->slot_size) {
13291 asan_unpoisoning_object(v) {
13292 /* skip T_NONEs; they won't be moved */
13293 if (BUILTIN_TYPE(v) != T_NONE) {
13294 rb_heap_t *dest_pool = gc_compact_destination_pool(objspace, page->heap, v);
13295 size_t dest_pool_idx = dest_pool - heaps;
13296 tdata->required_slots[dest_pool_idx]++;
13297 }
13298 }
13299 }
13300
13301 return 0;
13302}
13303
13304/* call-seq:
13305 * GC.verify_compaction_references(toward: nil, double_heap: false) -> hash
13306 *
13307 * Verify compaction reference consistency.
13308 *
13309 * This method is implementation specific. During compaction, objects that
13310 * were moved are replaced with T_MOVED objects. No object should have a
13311 * reference to a T_MOVED object after compaction.
13312 *
13313 * This function expands the heap to ensure room to move all objects,
13314 * compacts the heap to make sure everything moves, updates all references,
13315 * then performs a full \GC. If any object contains a reference to a T_MOVED
13316 * object, that object should be pushed on the mark stack, and will
13317 * make a SEGV.
13318 */
13319static VALUE
13320gc_verify_compaction_references(int argc, VALUE* argv, VALUE self)
13321{
13322 static ID keywords[3] = {0};
13323 if (!keywords[0]) {
13324 keywords[0] = rb_intern("toward");
13325 keywords[1] = rb_intern("double_heap");
13326 keywords[2] = rb_intern("expand_heap");
13327 }
13328
13329 VALUE options;
13330 rb_scan_args_kw(rb_keyword_given_p(), argc, argv, ":", &options);
13331
13332 VALUE arguments[3] = { Qnil, Qfalse, Qfalse };
13333 int kwarg_count = rb_get_kwargs(options, keywords, 0, 3, arguments);
13334 bool toward_empty = kwarg_count > 0 && SYMBOL_P(arguments[0]) && SYM2ID(arguments[0]) == rb_intern("empty");
13335 bool expand_heap = (kwarg_count > 1 && RTEST(arguments[1])) || (kwarg_count > 2 && RTEST(arguments[2]));
13336
13337 rb_objspace_t *objspace = rb_gc_get_objspace();
13338
13339 /* This verification machinery (heap expansion, toward_empty page ordering, the
13340 * moved-reference walk) is built for a single objspace, so with several demote it
13341 * to a plain full GC. Plain GC.compact does compact them via the global GC. */
13342 if (!rb_gc_single_objspace_p()) {
13343 rb_gc_impl_start(objspace, true, true, true, false, false);
13344 return gc_compact_stats(self);
13345 }
13346
13347 /* Clear the heap. */
13348 rb_gc_impl_start(objspace, true, true, true, false, false);
13349
13350 unsigned int lev = RB_GC_VM_LOCK();
13351 {
13352 gc_rest(objspace);
13353
13354 /* if both double_heap and expand_heap are set, expand_heap takes precedence */
13355 if (expand_heap) {
13356 struct desired_compaction_pages_i_data desired_compaction = {
13357 .objspace = objspace,
13358 .required_slots = {0},
13359 };
13360 /* Work out how many objects want to be in each size pool, taking account of moves */
13361 objspace_each_pages(objspace, desired_compaction_pages_i, &desired_compaction, TRUE);
13362
13363 /* Find out which pool has the most pages */
13364 size_t max_existing_pages = 0;
13365 for (int i = 0; i < HEAP_COUNT; i++) {
13366 rb_heap_t *heap = &heaps[i];
13367 max_existing_pages = MAX(max_existing_pages, heap->total_pages);
13368 }
13369
13370 /* Add pages to each size pool so that compaction is guaranteed to move every object */
13371 for (int i = 0; i < HEAP_COUNT; i++) {
13372 rb_heap_t *heap = &heaps[i];
13373
13374 size_t pages_to_add = 0;
13375 /*
13376 * Step 1: Make sure every pool has the same number of pages, by adding empty pages
13377 * to smaller pools. This is required to make sure the compact cursor can advance
13378 * through all of the pools in `gc_sweep_compact` without hitting the "sweep &
13379 * compact cursors met" condition on some pools before fully compacting others
13380 */
13381 pages_to_add += max_existing_pages - heap->total_pages;
13382 /*
13383 * Step 2: Now add additional free pages to each size pool sufficient to hold all objects
13384 * that want to be in that size pool, whether moved into it or moved within it
13385 */
13386 objspace->heap_pages.allocatable_bytes = desired_compaction.required_slots[i] * heap->slot_size;
13387 while (objspace->heap_pages.allocatable_bytes > 0) {
13388 heap_page_allocate_and_initialize(objspace, heap);
13389 }
13390 /*
13391 * Step 3: Add two more pages so that the compact & sweep cursors will meet _after_ all objects
13392 * have been moved, and not on the last iteration of the `gc_sweep_compact` loop
13393 */
13394 pages_to_add += 2;
13395
13396 for (; pages_to_add > 0; pages_to_add--) {
13397 heap_page_allocate_and_initialize_force(objspace, heap);
13398 }
13399 }
13400 }
13401
13402 if (toward_empty) {
13403 objspace->rcompactor.compare_func = compare_free_slots;
13404 }
13405 }
13406 RB_GC_VM_UNLOCK(lev);
13407
13408 rb_gc_impl_start(rb_gc_get_objspace(), true, true, true, true, false);
13409
13410 rb_objspace_reachable_objects_from_root(root_obj_check_moved_i, objspace);
13411 objspace_each_objects(objspace, heap_check_moved_i, objspace, TRUE);
13412
13413 objspace->rcompactor.compare_func = NULL;
13414
13415 return gc_compact_stats(self);
13416}
13417#else
13418# define gc_verify_compaction_references rb_f_notimplement
13419#endif
13420
13421void
13422rb_gc_impl_objspace_free(void *objspace_ptr)
13423{
13424 rb_objspace_t *objspace = objspace_ptr;
13425
13426 if (is_lazy_sweeping(objspace))
13427 rb_bug("lazy sweeping underway when freeing object space");
13428
13429 free(objspace->profile.records);
13430 objspace->profile.records = NULL;
13431
13432 for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
13433 heap_page_free(objspace, rb_darray_get(objspace->heap_pages.sorted, i));
13434 }
13435 rb_darray_free_without_gc(objspace->heap_pages.sorted);
13436 heap_pages_lomem = 0;
13437 heap_pages_himem = 0;
13438
13439 for (int i = 0; i < HEAP_COUNT; i++) {
13440 rb_heap_t *heap = &heaps[i];
13441 heap->total_pages = 0;
13442 heap->total_slots = 0;
13443 }
13444
13445 free_stack_chunks(&objspace->mark_stack);
13446 mark_stack_free_cache(&objspace->mark_stack);
13447
13448 rb_darray_free_without_gc(objspace->weak_references);
13449
13450#ifdef MALLOC_COUNTERS_NEED_LOCK
13451 rb_native_mutex_destroy(&objspace->malloc_counters.lock);
13452#endif
13453
13454 rb_native_mutex_destroy(&objspace->process_stat.lock);
13455
13456 free(objspace);
13457}
13458
13459#if MALLOC_ALLOCATED_SIZE
13460/*
13461 * call-seq:
13462 * GC.malloc_allocated_size -> Integer
13463 *
13464 * Returns the size of memory allocated by malloc().
13465 *
13466 * Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
13467 */
13468
13469static VALUE
13470gc_malloc_allocated_size(VALUE self)
13471{
13472 rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace();
13473 return ULL2NUM(objspace->malloc_params.allocated_size);
13474}
13475
13476/*
13477 * call-seq:
13478 * GC.malloc_allocations -> Integer
13479 *
13480 * Returns the number of malloc() allocations.
13481 *
13482 * Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
13483 */
13484
13485static VALUE
13486gc_malloc_allocations(VALUE self)
13487{
13488 rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace();
13489 return ULL2NUM(objspace->malloc_params.allocations);
13490}
13491#endif
13492
13493void
13494rb_gc_impl_before_fork(void *objspace_ptr)
13495{
13496 rb_objspace_t *objspace = objspace_ptr;
13497
13498 objspace->fork_vm_lock_lev = RB_GC_VM_LOCK();
13499 rb_gc_vm_barrier();
13500}
13501
13502void
13503rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid)
13504{
13505 rb_objspace_t *objspace = objspace_ptr;
13506
13507 if (pid == 0) {
13508 rb_gc_vm_each_objspace(gc_process_stat_after_fork_i, NULL);
13509 }
13510
13511 RB_GC_VM_UNLOCK(objspace->fork_vm_lock_lev);
13512 objspace->fork_vm_lock_lev = 0;
13513
13514 if (pid == 0) { /* child process */
13515 heap_alloc_state_clear(objspace);
13516 /* The forking Ractor becomes the child process's main Ractor. */
13517 global_objspace->main_objspace = objspace;
13518 page_pool_lock_initialize(&rb_global_objspace_instance.page_pool.lock);
13519 }
13520}
13521
13522VALUE rb_ident_hash_new_capa(rb_len_t size);
13523
13524#if GC_DEBUG_STRESS_TO_CLASS
13525/*
13526 * call-seq:
13527 * GC.add_stress_to_class(class[, ...])
13528 *
13529 * Raises NoMemoryError when allocating an instance of the given classes.
13530 *
13531 */
13532static VALUE
13533rb_gcdebug_add_stress_to_class(int argc, VALUE *argv, VALUE self)
13534{
13535 rb_objspace_t *objspace = rb_gc_get_objspace();
13536
13537 if (!stress_to_class) {
13538 set_stress_to_class(rb_ident_hash_new_capa(argc));
13539 }
13540
13541 for (int i = 0; i < argc; i++) {
13542 VALUE klass = argv[i];
13543 rb_hash_aset(stress_to_class, klass, Qtrue);
13544 }
13545
13546 return self;
13547}
13548
13549/*
13550 * call-seq:
13551 * GC.remove_stress_to_class(class[, ...])
13552 *
13553 * No longer raises NoMemoryError when allocating an instance of the
13554 * given classes.
13555 *
13556 */
13557static VALUE
13558rb_gcdebug_remove_stress_to_class(int argc, VALUE *argv, VALUE self)
13559{
13560 rb_objspace_t *objspace = rb_gc_get_objspace();
13561
13562 if (stress_to_class) {
13563 for (int i = 0; i < argc; ++i) {
13564 rb_hash_delete(stress_to_class, argv[i]);
13565 }
13566
13567 if (rb_hash_size(stress_to_class) == 0) {
13568 stress_to_class = 0;
13569 }
13570 }
13571
13572 return Qnil;
13573}
13574#endif
13575
13576void *
13577rb_gc_impl_objspace_alloc(void)
13578{
13579 global_objspace_init();
13580
13581 rb_objspace_t *objspace = calloc1(sizeof(rb_objspace_t));
13582 if (objspace) {
13583 rb_native_mutex_initialize(&objspace->process_stat.lock);
13584 }
13585
13586 return objspace;
13587}
13588
13589void
13590rb_gc_impl_objspace_init(void *objspace_ptr)
13591{
13592 rb_objspace_t *objspace = objspace_ptr;
13593
13594 gc_config_full_mark_set(TRUE);
13595
13596 malloc_limit = gc_params.malloc_limit_min;
13597 objspace->shareable_objects_limit = SHAREABLE_OBJECTS_LIMIT_MIN;
13598#ifdef MALLOC_COUNTERS_NEED_LOCK
13599 rb_native_mutex_initialize(&objspace->malloc_counters.lock);
13600#endif
13601 /* Shared by every objspace. preregister deduplicates on (func, data). */
13602 objspace->finalize_deferred_pjob = rb_postponed_job_preregister(0, gc_finalize_deferred, NULL);
13603 if (objspace->finalize_deferred_pjob == POSTPONED_JOB_HANDLE_INVALID) {
13604 rb_bug("Could not preregister postponed job for GC");
13605 }
13606
13607 gc_tdata_deferred_free_pjob_ensure();
13608
13609 /* A standard RVALUE (RBasic + embedded VALUEs + debug overhead) must fit
13610 * in at least one pool. In debug builds RVALUE_OVERHEAD can push this
13611 * beyond the 48-byte pool into the 64-byte pool, which is fine. */
13612 GC_ASSERT(rb_gc_impl_size_allocatable_p(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX])));
13613
13614 for (int i = 0; i < HEAP_COUNT; i++) {
13615 rb_heap_t *heap = &heaps[i];
13616
13617 heap->slot_size = pool_slot_sizes[i];
13618
13619 ccan_list_head_init(&heap->pages);
13620 }
13621
13622 if (global_objspace->main_objspace == NULL) {
13623 /* Single-threaded at boot and the first objspace is main's: compute process-wide
13624 * constants once here. A later objspace_init rewriting them, even with equal
13625 * values, would race other threads' lock-free reads. */
13626 global_objspace->main_objspace = objspace;
13627
13628 init_size_to_heap_idx();
13629
13630#if defined(INIT_HEAP_PAGE_ALLOC_USE_MMAP)
13631 /* Need to determine if we can use mmap at runtime. */
13632 heap_page_alloc_use_mmap = INIT_HEAP_PAGE_ALLOC_USE_MMAP;
13633#endif
13634 gc_params.heap_init_bytes = GC_HEAP_INIT_BYTES;
13635 gc_params.ractor_heap_init_bytes = GC_RACTOR_HEAP_INIT_BYTES ? GC_RACTOR_HEAP_INIT_BYTES
13636 : heap_init_bytes_min();
13637 }
13638 // GC.measure_total_time= sets the caller's objspace only; a new Ractor's follows
13639 // its creator's, which is the objspace running this init (main starts it on).
13640 objspace->flags.measure_gc = global_objspace->main_objspace == objspace ? true
13641 : ((rb_objspace_t *)rb_gc_get_objspace())->flags.measure_gc;
13642
13643 rb_darray_make_without_gc(&objspace->heap_pages.sorted, 0);
13644 rb_darray_make_without_gc(&objspace->weak_references, 0);
13645
13646#if RGENGC_ESTIMATE_OLDMALLOC
13647 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
13648#endif
13649
13650 init_mark_stack(&objspace->mark_stack);
13651
13652 objspace->profile.invoke_time = getrusage_time();
13653 objspace->profile.invoke_wall_time = rb_hrtime_now();
13654 objspace->profile.max_records = GC_PROFILE_RECORD_DEFAULT_MAX_RECORDS;
13655 finalizer_table = st_init_numtable();
13656
13657 gc_process_stat_publish(objspace);
13658}
13659
13660void
13661rb_gc_impl_init(void)
13662{
13663 /* Fill the symbol tables here, where no other ractor exists yet: they used to
13664 * be filled on first use, guarded by their own first element, so a second
13665 * ractor could see a half-filled table and GC.stat raised on the rest. */
13666 setup_gc_stat_symbols();
13667 setup_gc_stat_heap_symbols();
13668 setup_gc_latest_gc_info_symbols();
13669
13670 VALUE gc_constants = rb_hash_new();
13671 rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), GC_DEBUG ? Qtrue : Qfalse);
13672 /* Minimum slot size that fits a standard RVALUE */
13673 size_t rvalue_pool = 0;
13674 for (size_t i = 0; i < HEAP_COUNT; i++) {
13675 if (pool_slot_sizes[i] >= RVALUE_SLOT_SIZE) { rvalue_pool = pool_slot_sizes[i]; break; }
13676 }
13677 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(rvalue_pool - RVALUE_OVERHEAD));
13678 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic)));
13679 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD));
13680 rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE));
13681 rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_SIZE")), SIZET2NUM(HEAP_PAGE_SIZE));
13682 rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(HEAP_COUNT));
13683 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), SIZET2NUM(rb_gc_impl_max_allocation_size()));
13684 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), LONG2FIX(RVALUE_OLD_AGE));
13685 if (RB_BUG_INSTEAD_OF_RB_MEMERROR+0) {
13686 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RB_BUG_INSTEAD_OF_RB_MEMERROR")), Qtrue);
13687 }
13688 OBJ_FREEZE(gc_constants);
13689 /* Internal constants in the garbage collector. */
13690 rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants);
13691
13692 if (GC_COMPACTION_SUPPORTED) {
13693 rb_define_singleton_method(rb_mGC, "compact", gc_compact, 0);
13694 rb_define_singleton_method(rb_mGC, "auto_compact", gc_get_auto_compact, 0);
13695 rb_define_singleton_method(rb_mGC, "auto_compact=", gc_set_auto_compact, 1);
13696 rb_define_singleton_method(rb_mGC, "latest_compact_info", gc_compact_stats, 0);
13697 rb_define_singleton_method(rb_mGC, "verify_compaction_references", gc_verify_compaction_references, -1);
13698 }
13699 else {
13703 rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0);
13704 rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1);
13705 }
13706
13707#if GC_DEBUG_STRESS_TO_CLASS
13708 rb_define_singleton_method(rb_mGC, "add_stress_to_class", rb_gcdebug_add_stress_to_class, -1);
13709 rb_define_singleton_method(rb_mGC, "remove_stress_to_class", rb_gcdebug_remove_stress_to_class, -1);
13710#endif
13711
13712 /* internal methods */
13713 rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency_m, 0);
13714
13715#if MALLOC_ALLOCATED_SIZE
13716 rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0);
13717 rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0);
13718#endif
13719
13720 /* Document-class: GC::Profiler
13721 *
13722 * The GC profiler provides access to information on GC runs including time,
13723 * length and object space size.
13724 *
13725 * Example:
13726 *
13727 * GC::Profiler.enable
13728 *
13729 * require 'rdoc/rdoc'
13730 *
13731 * GC::Profiler.report
13732 *
13733 * pp GC::Profiler.raw_data
13734 *
13735 * GC::Profiler.disable
13736 *
13737 * GC::Profiler.raw_data returns one Hash per GC run, including CPU time
13738 * fields such as +:GC_TIME+ and wall-clock fields such as +:GC_WALL_TIME+,
13739 * +:GC_PAUSE_TIME+, +:GC_STOP_TIME+, and +:GC_STW_TIME+. +:GC_WALL_TIME+
13740 * is the wall-clock counterpart to +:GC_TIME+, while +:GC_PAUSE_TIME+
13741 * measures how long user execution was blocked by the GC entry.
13742 *
13743 * See also GC.count, GC.malloc_allocated_size and GC.malloc_allocations
13744 */
13745 VALUE rb_mProfiler = rb_define_module_under(rb_mGC, "Profiler");
13746 rb_define_singleton_method(rb_mProfiler, "enabled?", gc_profile_enable_get, 0);
13747 rb_define_singleton_method(rb_mProfiler, "enable", gc_profile_enable, 0);
13748 rb_define_singleton_method(rb_mProfiler, "raw_data", gc_profile_record_get, -1);
13749 rb_define_singleton_method(rb_mProfiler, "disable", gc_profile_disable, 0);
13750 rb_define_singleton_method(rb_mProfiler, "clear", gc_profile_clear, 0);
13751 rb_define_singleton_method(rb_mProfiler, "configure", gc_profile_configure, -1);
13752 rb_define_singleton_method(rb_mProfiler, "result", gc_profile_result, 0);
13753 rb_define_singleton_method(rb_mProfiler, "report", gc_profile_report, -1);
13754 rb_define_singleton_method(rb_mProfiler, "total_time", gc_profile_total_time, 0);
13755
13756 {
13757 VALUE opts;
13758 /* \GC build options */
13759 rb_define_const(rb_mGC, "OPTS", opts = rb_ary_new());
13760#define OPT(o) if (o) rb_ary_push(opts, rb_interned_str(#o, sizeof(#o) - 1))
13761 OPT(GC_DEBUG);
13762 OPT(USE_RGENGC);
13763 OPT(RGENGC_DEBUG);
13764 OPT(RGENGC_CHECK_MODE);
13765 OPT(RGENGC_PROFILE);
13766 OPT(RGENGC_ESTIMATE_OLDMALLOC);
13767 OPT(GC_PROFILE_MORE_DETAIL);
13768 OPT(GC_ENABLE_LAZY_SWEEP);
13769 OPT(CALC_EXACT_MALLOC_SIZE);
13770 OPT(MALLOC_ALLOCATED_SIZE);
13771 OPT(MALLOC_ALLOCATED_SIZE_CHECK);
13772 OPT(GC_PROFILE_DETAIL_MEMORY);
13773 OPT(GC_COMPACTION_SUPPORTED);
13774#undef OPT
13775 OBJ_FREEZE(opts);
13776 }
13777}
13778
13779#ifdef USE_MIMALLOC
13780# undef malloc
13781# undef calloc
13782# undef realloc
13783# undef free
13784#endif
Atomic operations.
#define RUBY_ATOMIC_VALUE_CAS(var, oldval, newval)
Identical to RUBY_ATOMIC_CAS, except it expects its arguments are VALUE.
Definition atomic.h:416
#define RUBY_ATOMIC_PTR_LOAD(var)
Identical to RUBY_ATOMIC_LOAD, except it expects its arguments are void*.
Definition atomic.h:348
#define RUBY_ATOMIC_SIZE_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are size_t.
Definition atomic.h:270
#define RUBY_ATOMIC_SIZE_INC(var)
Identical to RUBY_ATOMIC_INC, except it expects its argument is size_t.
Definition atomic.h:246
#define RUBY_ATOMIC_SIZE_CAS(var, oldval, newval)
Identical to RUBY_ATOMIC_CAS, except it expects its arguments are size_t.
Definition atomic.h:284
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
Definition atomic.h:69
#define RUBY_ATOMIC_SIZE_ADD(var, val)
Identical to RUBY_ATOMIC_ADD, except it expects its arguments are size_t.
Definition atomic.h:307
#define RUBY_ATOMIC_VALUE_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are VALUE.
Definition atomic.h:402
#define RUBY_ATOMIC_PTR_SET(var, val)
Identical to RUBY_ATOMIC_SET, except it expects its arguments are void*.
Definition atomic.h:361
#define RUBY_ATOMIC_SET(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except for the return type.
Definition atomic.h:185
#define RUBY_ATOMIC_EXCHANGE(var, val)
Atomically replaces the value pointed by var with val.
Definition atomic.h:152
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
unsigned int rb_postponed_job_handle_t
The type of a handle returned from rb_postponed_job_preregister and passed to rb_postponed_job_trigge...
Definition debug.h:703
void rb_postponed_job_trigger(rb_postponed_job_handle_t h)
Triggers a pre-registered job registered with rb_postponed_job_preregister, scheduling it for executi...
Definition vm_trace.c:1899
rb_postponed_job_handle_t rb_postponed_job_preregister(unsigned int flags, rb_postponed_job_func_t func, void *data)
Pre-registers a func in Ruby's postponed job preregistration table, returning an opaque handle which ...
Definition vm_trace.c:1865
#define RB_GNUC_EXTENSION_BLOCK(x)
This is expanded to the passed token for non-GCC compilers.
Definition defines.h:91
#define RUBY_INTERNAL_EVENT_GC_EXIT
gc_exit() is called.
Definition event.h:99
#define RUBY_INTERNAL_EVENT_GC_ENTER
gc_enter() is called.
Definition event.h:98
#define RUBY_INTERNAL_EVENT_GC_END_SWEEP
GC ended sweep phase.
Definition event.h:97
#define RUBY_INTERNAL_EVENT_GC_END_MARK
GC ended mark phase.
Definition event.h:96
#define RUBY_INTERNAL_EVENT_OBJSPACE_MASK
Bitmask of GC events.
Definition event.h:100
#define RUBY_INTERNAL_EVENT_FREEOBJ
Object swept.
Definition event.h:94
#define RUBY_INTERNAL_EVENT_GC_START
GC started.
Definition event.h:95
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
static VALUE RB_FL_TEST(VALUE obj, VALUE flags)
Tests if the given flag(s) are set or not.
Definition fl_type.h:433
static VALUE RB_FL_TEST_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_TEST().
Definition fl_type.h:407
static void RB_FL_SET_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_SET().
Definition fl_type.h:544
static void RB_FL_UNSET_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_UNSET().
Definition fl_type.h:604
@ RUBY_FL_PROMOTED
Ruby objects are "generational".
Definition fl_type.h:205
@ RUBY_FL_SHAREABLE
This flag has something to do with Ractor.
Definition fl_type.h:253
@ RUBY_FL_WEAK_REFERENCE
This object weakly refers to other objects.
Definition fl_type.h:260
int rb_scan_args_kw(int kw_flag, int argc, const VALUE *argv, const char *fmt,...)
Identical to rb_scan_args(), except it also accepts kw_splat.
Definition class.c:3371
int rb_keyword_given_p(void)
Determines if the current method is given a keyword argument.
Definition eval.c:1054
int rb_get_kwargs(VALUE keyword_hash, const ID *table, int required, int optional, VALUE *values)
Keyword argument deconstructor.
Definition class.c:3147
#define T_COMPLEX
Old name of RUBY_T_COMPLEX.
Definition value_type.h:59
#define T_FILE
Old name of RUBY_T_FILE.
Definition value_type.h:62
#define T_STRING
Old name of RUBY_T_STRING.
Definition value_type.h:78
#define xfree
Old name of ruby_xfree.
Definition xmalloc.h:58
#define T_MASK
Old name of RUBY_T_MASK.
Definition value_type.h:68
#define Qundef
Old name of RUBY_Qundef.
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
Definition fl_type.h:133
#define T_NIL
Old name of RUBY_T_NIL.
Definition value_type.h:72
#define T_FLOAT
Old name of RUBY_T_FLOAT.
Definition value_type.h:64
#define T_IMEMO
Old name of RUBY_T_IMEMO.
Definition value_type.h:67
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
#define T_BIGNUM
Old name of RUBY_T_BIGNUM.
Definition value_type.h:57
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
#define T_STRUCT
Old name of RUBY_T_STRUCT.
Definition value_type.h:79
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
Definition fl_type.h:131
#define T_FIXNUM
Old name of RUBY_T_FIXNUM.
Definition value_type.h:63
#define SYM2ID
Old name of RB_SYM2ID.
Definition symbol.h:45
#define T_DATA
Old name of RUBY_T_DATA.
Definition value_type.h:60
#define FL_SHAREABLE
Old name of RUBY_FL_SHAREABLE.
Definition fl_type.h:62
#define T_NONE
Old name of RUBY_T_NONE.
Definition value_type.h:74
#define T_NODE
Old name of RUBY_T_NODE.
Definition value_type.h:73
#define SIZET2NUM
Old name of RB_SIZE2NUM.
Definition size_t.h:62
#define xmalloc
Old name of ruby_xmalloc.
Definition xmalloc.h:53
#define LONG2FIX
Old name of RB_INT2FIX.
Definition long.h:49
#define FIX2INT
Old name of RB_FIX2INT.
Definition int.h:41
#define FL_FINALIZE
Old name of RUBY_FL_FINALIZE.
Definition fl_type.h:61
#define T_MODULE
Old name of RUBY_T_MODULE.
Definition value_type.h:70
#define T_TRUE
Old name of RUBY_T_TRUE.
Definition value_type.h:81
#define T_RATIONAL
Old name of RUBY_T_RATIONAL.
Definition value_type.h:76
#define T_ICLASS
Old name of RUBY_T_ICLASS.
Definition value_type.h:66
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
#define ALLOC_N
Old name of RB_ALLOC_N.
Definition memory.h:402
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
Definition fl_type.h:128
#define FL_SET
Old name of RB_FL_SET.
Definition fl_type.h:125
#define rb_ary_new3
Old name of rb_ary_new_from_args.
Definition array.h:658
#define T_FALSE
Old name of RUBY_T_FALSE.
Definition value_type.h:61
#define ULL2NUM
Old name of RB_ULL2NUM.
Definition long_long.h:31
#define T_UNDEF
Old name of RUBY_T_UNDEF.
Definition value_type.h:82
#define Qtrue
Old name of RUBY_Qtrue.
#define T_ZOMBIE
Old name of RUBY_T_ZOMBIE.
Definition value_type.h:83
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define T_ARRAY
Old name of RUBY_T_ARRAY.
Definition value_type.h:56
#define T_OBJECT
Old name of RUBY_T_OBJECT.
Definition value_type.h:75
#define NIL_P
Old name of RB_NIL_P.
#define FL_WB_PROTECTED
Old name of RUBY_FL_WB_PROTECTED.
Definition fl_type.h:59
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition value_type.h:80
#define DBL2NUM
Old name of rb_float_new.
Definition double.h:29
#define T_MATCH
Old name of RUBY_T_MATCH.
Definition value_type.h:69
#define T_CLASS
Old name of RUBY_T_CLASS.
Definition value_type.h:58
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
Definition value_type.h:85
#define T_MOVED
Old name of RUBY_T_MOVED.
Definition value_type.h:71
#define FL_TEST
Old name of RB_FL_TEST.
Definition fl_type.h:127
#define NUM2LONG
Old name of RB_NUM2LONG.
Definition long.h:51
#define FL_UNSET
Old name of RB_FL_UNSET.
Definition fl_type.h:129
#define FIXNUM_P
Old name of RB_FIXNUM_P.
#define FL_SET_RAW
Old name of RB_FL_SET_RAW.
Definition fl_type.h:126
#define SYMBOL_P
Old name of RB_SYMBOL_P.
Definition value_type.h:88
#define T_REGEXP
Old name of RUBY_T_REGEXP.
Definition value_type.h:77
#define ruby_verbose
This variable controls whether the interpreter is in debug mode.
Definition error.h:476
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1455
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:464
VALUE rb_obj_hide(VALUE obj)
Make the object invisible from Ruby code.
Definition object.c:94
VALUE rb_mGC
GC module.
Definition gc.c:477
VALUE rb_stdout
STDOUT constant.
Definition io.c:194
#define RBIMPL_ASSERT_OR_ASSUME(...)
This is either RUBY_ASSERT or RBIMPL_ASSUME, depending on RUBY_DEBUG.
Definition assert.h:311
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:219
Routines to manipulate encodings of strings.
void rb_memerror(void)
Triggers out-of-memory error.
Definition gc.c:6674
static bool RB_OBJ_PROMOTED_RAW(VALUE obj)
This is the implementation of RB_OBJ_PROMOTED().
Definition gc.h:594
#define USE_RGENGC
Definition gc.h:464
size_t rb_gc_count(void)
Identical to rb_gc_stat(), with "count" parameter.
Definition gc.c:5711
VALUE rb_ary_dup(VALUE ary)
Duplicates an array.
VALUE rb_ary_new(void)
Allocates a new, empty array.
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
VALUE rb_fiber_current(void)
Queries the fiber which is calling this function.
Definition cont.c:2944
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
Definition error.h:284
VALUE rb_hash_new(void)
Creates a new, empty hash object.
Definition hash.c:1646
VALUE rb_str_buf_append(VALUE dst, VALUE src)
Identical to rb_str_cat_cstr(), except it takes Ruby's string instead of C's.
Definition string.c:3904
VALUE rb_str_buf_new(rb_len_t capa)
Allocates a "string buffer".
Definition string.c:1783
#define rb_str_new_cstr(str)
Identical to rb_str_new, except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1539
const char * rb_sourcefile(void)
Resembles __FILE__.
Definition vm.c:2180
VALUE rb_f_notimplement(int argc, const VALUE *argv, VALUE obj, VALUE marker)
Raises rb_eNotImpError.
Definition vm_method.c:913
int rb_sourceline(void)
Resembles __LINE__.
Definition vm.c:2194
#define RB_SYM2ID
Just another name of rb_sym2id.
Definition symbol.h:43
ID rb_sym2id(VALUE obj)
Converts an instance of rb_cSymbol into an ID.
Definition symbol.c:1097
int capa
Designed capacity of the buffer.
Definition io.h:11
int off
Offset inside of ptr.
Definition io.h:5
int len
Length of the buffer.
Definition io.h:8
int ruby_thread_has_gvl_p(void)
Whether the current thread is holding the GVL.
Definition thread.c:2364
void * rb_thread_call_with_gvl(void *(*func)(void *), void *data1)
(Re-)acquires the GVL.
Definition thread.c:2323
#define strtod(s, e)
Just another name of ruby_strtod.
Definition util.h:223
void ruby_qsort(void *, const size_t, const size_t, int(*)(const void *, const void *, void *), void *)
Reentrant implementation of quick sort.
#define MEMZERO(p, type, n)
Handy macro to erase a region of memory.
Definition memory.h:363
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:170
VALUE type(ANYARGS)
ANYARGS-ed function type.
int st_foreach(st_table *q, int_type *w, st_data_t e)
Iteration over the given table.
void rb_hash_foreach(VALUE q, int_type *w, VALUE e)
Iteration over the given hash.
VALUE rb_ensure(type *q, VALUE w, type *e, VALUE r)
An equivalent of ensure clause.
#define RARRAY_LEN
Just another name of rb_array_len.
Definition rarray.h:50
#define RARRAY_AREF(a, i)
References an object in an array.
Definition rarray.h:421
static void RARRAY_ASET(VALUE ary, rb_len_t i, VALUE v)
Assigns an object in an array.
Definition rarray.h:385
#define RBASIC(obj)
Convenient casting macro.
Definition rbasic.h:40
#define RUBY_TYPED_FREE_IMMEDIATELY
Macros to see if each corresponding flag is defined.
Definition rtypeddata.h:122
static const rb_data_type_t * RTYPEDDATA_TYPE(VALUE obj)
Queries for the type of given object.
Definition rtypeddata.h:692
struct rb_data_type_struct rb_data_type_t
This is the struct that holds necessary info for a struct.
Definition rtypeddata.h:239
#define RTYPEDDATA(obj)
Convenient casting macro.
Definition rtypeddata.h:96
#define errno
Ractor-aware version of errno.
Definition ruby.h:388
int ruby_native_thread_p(void)
Queries if the thread which calls this function is a ruby's thread.
Definition thread.c:6183
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
#define RTEST
This is an old name of RB_TEST.
#define _(args)
This was a transition path from K&R to ANSI.
Definition stdarg.h:35
Ruby object's base components.
Definition rbasic.h:69
VALUE flags
Per-object flags.
Definition rbasic.h:81
Definition gc_impl.h:34
Definition default.c:614
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
long rb_len_t
A signed integer type for lengths and indices of objects such as String and Array.
Definition value.h:142
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
Definition value_type.h:182
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:376
ruby_value_type
C-level type of an object.
Definition value_type.h:113
@ RUBY_T_SYMBOL
Definition value_type.h:135
@ RUBY_T_MATCH
Definition value_type.h:128
@ RUBY_T_MODULE
Definition value_type.h:118
@ RUBY_T_ICLASS
Hidden classes known as IClasses.
Definition value_type.h:141
@ RUBY_T_MOVED
Definition value_type.h:143
@ RUBY_T_FIXNUM
Integers formerly known as Fixnums.
Definition value_type.h:136
@ RUBY_T_IMEMO
Definition value_type.h:139
@ RUBY_T_NODE
Definition value_type.h:140
@ RUBY_T_OBJECT
Definition value_type.h:116
@ RUBY_T_DATA
Definition value_type.h:127
@ RUBY_T_FALSE
Definition value_type.h:134
@ RUBY_T_UNDEF
Definition value_type.h:137
@ RUBY_T_COMPLEX
Definition value_type.h:129
@ RUBY_T_STRING
Definition value_type.h:120
@ RUBY_T_HASH
Definition value_type.h:123
@ RUBY_T_NIL
Definition value_type.h:132
@ RUBY_T_CLASS
Definition value_type.h:117
@ RUBY_T_ARRAY
Definition value_type.h:122
@ RUBY_T_MASK
Bitmask of ruby_value_type.
Definition value_type.h:145
@ RUBY_T_RATIONAL
Definition value_type.h:130
@ RUBY_T_ZOMBIE
Definition value_type.h:142
@ RUBY_T_BIGNUM
Definition value_type.h:125
@ RUBY_T_TRUE
Definition value_type.h:133
@ RUBY_T_FLOAT
Definition value_type.h:119
@ RUBY_T_STRUCT
Definition value_type.h:124
@ RUBY_T_NONE
Non-object (swept etc.).
Definition value_type.h:114
@ RUBY_T_REGEXP
Definition value_type.h:121
@ RUBY_T_FILE
Definition value_type.h:126