1#include "ruby/internal/config.h"
10# ifdef HAVE_SYS_PRCTL_H
11# include <sys/prctl.h>
15#if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H)
20#ifdef BUILDING_MODULAR_GC
21# define nlz_int64(x) (x == 0 ? 64 : (unsigned int)__builtin_clzll((unsigned long long)x))
22# define rb_popcount_intptr(x) ((unsigned int)__builtin_popcountll((unsigned long long)(x)))
23# define rb_popcount32(x) ((unsigned int)__builtin_popcount((unsigned int)(x)))
24# define ntz_int32(x) ((int)((x) == 0 ? 32 : __builtin_ctz((unsigned int)(x))))
26# include "internal/bits.h"
31#include "ruby_atomic.h"
37#include "ccan/list/list.h"
40#include "gc/gc_impl.h"
43#include "internal/static_assert.h"
44#include "internal/vm_map.h"
46#ifdef BUILDING_MODULAR_GC
52typedef uint64_t rb_hrtime_t;
53# define RB_HRTIME_PER_SEC ((rb_hrtime_t)1000000000)
55static inline rb_hrtime_t
58# if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC)
60 if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) {
61 return (rb_hrtime_t)ts.tv_sec * RB_HRTIME_PER_SEC + (rb_hrtime_t)ts.tv_nsec;
67static inline rb_hrtime_t
68rb_hrtime_add(rb_hrtime_t a, rb_hrtime_t b)
70 rb_hrtime_t c = a + b;
71 return c < a ? UINT64_MAX : c;
74static inline rb_hrtime_t
75rb_hrtime_sub(rb_hrtime_t a, rb_hrtime_t b)
77 return a < b ? 0 : a - b;
87#define RUBY_DTRACE_GC_HOOK_EXPAND(expr) expr
88#define RUBY_DTRACE_GC_HOOK(name, ...) \
89 do {if (RUBY_DTRACE_GC_##name##_ENABLED()) RUBY_DTRACE_GC_HOOK_EXPAND(RUBY_DTRACE_GC_##name(__VA_ARGS__));} while (0)
92# include "gc/default/zjit_fastpath.h"
95#ifdef BUILDING_MODULAR_GC
96# define RB_DEBUG_COUNTER_INC(_name) ((void)0)
97# define RB_DEBUG_COUNTER_INC_IF(_name, cond) (!!(cond))
99# include "debug_counter.h"
102#ifdef BUILDING_MODULAR_GC
103# define rb_asan_poison_object(obj) ((void)(obj))
104# define rb_asan_unpoison_object(obj, newobj_p) ((void)(obj), (void)(newobj_p))
105# define asan_unpoisoning_object(obj) if ((obj) || true)
106# define asan_poison_memory_region(ptr, size) ((void)(ptr), (void)(size))
107# define asan_unpoison_memory_region(ptr, size, malloc_p) ((void)(ptr), (size), (malloc_p))
108# define asan_unpoisoning_memory_region(ptr, size) if ((ptr) || (size) || true)
110# define VALGRIND_MAKE_MEM_DEFINED(ptr, size) ((void)(ptr), (void)(size))
111# define VALGRIND_MAKE_MEM_UNDEFINED(ptr, size) ((void)(ptr), (void)(size))
113# include "internal/sanitizers.h"
117#ifndef HAVE_MALLOC_USABLE_SIZE
119# define HAVE_MALLOC_USABLE_SIZE
120# define malloc_usable_size(a) mi_usable_size(a)
122# define HAVE_MALLOC_USABLE_SIZE
123# define malloc_usable_size(a) _msize(a)
124# elif defined HAVE_MALLOC_SIZE
125# define HAVE_MALLOC_USABLE_SIZE
126# define malloc_usable_size(a) malloc_size(a)
130#ifdef HAVE_MALLOC_USABLE_SIZE
131# ifdef RUBY_ALTERNATIVE_MALLOC_HEADER
133# elif defined(USE_MIMALLOC)
134# pragma push_macro("restrict")
136# include <mimalloc.h>
137# pragma pop_macro("restrict")
138# elif defined(HAVE_MALLOC_H)
140# elif defined(HAVE_MALLOC_NP_H)
141# include <malloc_np.h>
142# elif defined(HAVE_MALLOC_MALLOC_H)
143# include <malloc/malloc.h>
147#ifdef HAVE_MALLOC_TRIM
150# ifdef __EMSCRIPTEN__
152# include <emscripten/emmalloc.h>
156#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
157# include <mach/task.h>
158# include <mach/mach_init.h>
159# include <mach/mach_port.h>
165# define malloc(n) mi_malloc(n)
166# define calloc(n, s) mi_calloc(n, s)
167# define realloc(p, n) mi_realloc(p, n)
168# define free(p) mi_free(p)
171#ifndef RUBY_DEBUG_LOG
172# define RUBY_DEBUG_LOG(...)
175#ifndef GC_HEAP_INIT_BYTES
176#define GC_HEAP_INIT_BYTES (2560 * 1024)
178#ifndef GC_HEAP_FREE_SLOTS
179#define GC_HEAP_FREE_SLOTS 4096
181#ifndef GC_RACTOR_HEAP_INIT_BYTES
183#define GC_RACTOR_HEAP_INIT_BYTES 0
185#ifndef GC_HEAP_GROWTH_FACTOR
186#define GC_HEAP_GROWTH_FACTOR 1.8
188#ifndef GC_HEAP_GROWTH_MAX_BYTES
189#define GC_HEAP_GROWTH_MAX_BYTES 0
191#ifndef GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO
192# define GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO 0.01
194#ifndef GC_HEAP_OLDOBJECT_LIMIT_FACTOR
195#define GC_HEAP_OLDOBJECT_LIMIT_FACTOR 2.0
198#ifndef GC_HEAP_FREE_SLOTS_MIN_RATIO
199#define GC_HEAP_FREE_SLOTS_MIN_RATIO 0.20
201#ifndef GC_HEAP_FREE_SLOTS_GOAL_RATIO
202#define GC_HEAP_FREE_SLOTS_GOAL_RATIO 0.40
204#ifndef GC_HEAP_FREE_SLOTS_MAX_RATIO
205#define GC_HEAP_FREE_SLOTS_MAX_RATIO 0.65
208#ifndef GC_MALLOC_LIMIT_MIN
209#define GC_MALLOC_LIMIT_MIN (16 * 1024 * 1024 )
211#ifndef GC_MALLOC_LIMIT_MAX
212#define GC_MALLOC_LIMIT_MAX (32 * 1024 * 1024 )
214#ifndef GC_MALLOC_LIMIT_GROWTH_FACTOR
215#define GC_MALLOC_LIMIT_GROWTH_FACTOR 1.4
218#ifndef GC_OLDMALLOC_LIMIT_MIN
219#define GC_OLDMALLOC_LIMIT_MIN (16 * 1024 * 1024 )
221#ifndef GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
222#define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR 1.2
224#ifndef GC_OLDMALLOC_LIMIT_MAX
225#define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 )
228#ifndef GC_MALLOC_INCREASE_LOCAL_THRESHOLD
229#define GC_MALLOC_INCREASE_LOCAL_THRESHOLD (8 * 1024 )
232#ifdef RB_THREAD_LOCAL_SPECIFIER
233#define USE_MALLOC_INCREASE_LOCAL 1
234static RB_THREAD_LOCAL_SPECIFIER
int malloc_increase_local;
236#define USE_MALLOC_INCREASE_LOCAL 0
239#ifndef GC_CAN_COMPILE_COMPACTION
241# define GC_CAN_COMPILE_COMPACTION 0
243# define GC_CAN_COMPILE_COMPACTION 1
247#ifndef PRINT_ENTER_EXIT_TICK
248# define PRINT_ENTER_EXIT_TICK 0
250#ifndef PRINT_ROOT_TICKS
251#define PRINT_ROOT_TICKS 0
254#define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS)
257# if SIZEOF_VALUE >= 8
258# define HEAP_COUNT 12
267# define EACH_POOL_SLOT_SIZE(SLOT) \
268 SLOT(32) SLOT(40) SLOT(64) SLOT(80) SLOT(96) SLOT(128) \
269 SLOT(160) SLOT(256) SLOT(512) SLOT(640) SLOT(768) SLOT(1024)
271# define EACH_POOL_SLOT_SIZE(SLOT) \
272 SLOT(32) SLOT(64) SLOT(128) SLOT(256) SLOT(512)
276 size_t heap_init_bytes;
277 size_t ractor_heap_init_bytes;
278 size_t heap_free_slots;
279 double growth_factor;
280 size_t growth_max_bytes;
282 double heap_free_slots_min_ratio;
283 double heap_free_slots_goal_ratio;
284 double heap_free_slots_max_ratio;
285 double uncollectible_wb_unprotected_objects_limit_ratio;
286 double oldobject_limit_factor;
288 size_t malloc_limit_min;
289 size_t malloc_limit_max;
290 double malloc_limit_growth_factor;
292 size_t oldmalloc_limit_min;
293 size_t oldmalloc_limit_max;
294 double oldmalloc_limit_growth_factor;
299 GC_RACTOR_HEAP_INIT_BYTES,
301 GC_HEAP_GROWTH_FACTOR,
302 GC_HEAP_GROWTH_MAX_BYTES,
304 GC_HEAP_FREE_SLOTS_MIN_RATIO,
305 GC_HEAP_FREE_SLOTS_GOAL_RATIO,
306 GC_HEAP_FREE_SLOTS_MAX_RATIO,
307 GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO,
308 GC_HEAP_OLDOBJECT_LIMIT_FACTOR,
312 GC_MALLOC_LIMIT_GROWTH_FACTOR,
314 GC_OLDMALLOC_LIMIT_MIN,
315 GC_OLDMALLOC_LIMIT_MAX,
316 GC_OLDMALLOC_LIMIT_GROWTH_FACTOR,
335#define RGENGC_DEBUG -1
337#define RGENGC_DEBUG 0
340#if RGENGC_DEBUG < 0 && !defined(_MSC_VER)
341# define RGENGC_DEBUG_ENABLED(level) (-(RGENGC_DEBUG) >= (level) && ruby_rgengc_debug >= (level))
343# define RGENGC_DEBUG_ENABLED(level) ((RGENGC_DEBUG) >= (level))
345int ruby_rgengc_debug;
352#ifndef RGENGC_PROFILE
353# define RGENGC_PROFILE 0
362#ifndef RGENGC_ESTIMATE_OLDMALLOC
363# define RGENGC_ESTIMATE_OLDMALLOC 1
366#ifndef GC_PROFILE_MORE_DETAIL
367# define GC_PROFILE_MORE_DETAIL 0
369#ifndef GC_PROFILE_DETAIL_MEMORY
370# define GC_PROFILE_DETAIL_MEMORY 0
372#ifndef GC_ENABLE_LAZY_SWEEP
373# define GC_ENABLE_LAZY_SWEEP 1
376#ifndef VERIFY_FREE_SIZE
378#define VERIFY_FREE_SIZE 1
380#define VERIFY_FREE_SIZE 0
385#undef CALC_EXACT_MALLOC_SIZE
386#define CALC_EXACT_MALLOC_SIZE 1
389#ifndef CALC_EXACT_MALLOC_SIZE
390# define CALC_EXACT_MALLOC_SIZE 0
393#if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0
394# ifndef MALLOC_ALLOCATED_SIZE
395# define MALLOC_ALLOCATED_SIZE 0
398# define MALLOC_ALLOCATED_SIZE 0
400#ifndef MALLOC_ALLOCATED_SIZE_CHECK
401# define MALLOC_ALLOCATED_SIZE_CHECK 0
404#ifndef GC_DEBUG_STRESS_TO_CLASS
405# define GC_DEBUG_STRESS_TO_CLASS RUBY_DEBUG
409 GPR_FLAG_NONE = 0x000,
411 GPR_FLAG_MAJOR_BY_NOFREE = 0x001,
412 GPR_FLAG_MAJOR_BY_OLDGEN = 0x002,
413 GPR_FLAG_MAJOR_BY_SHADY = 0x004,
414 GPR_FLAG_MAJOR_BY_FORCE = 0x008,
415#if RGENGC_ESTIMATE_OLDMALLOC
416 GPR_FLAG_MAJOR_BY_OLDMALLOC = 0x020,
418 GPR_FLAG_MAJOR_MASK = 0x0ff,
421 GPR_FLAG_NEWOBJ = 0x100,
422 GPR_FLAG_MALLOC = 0x200,
423 GPR_FLAG_METHOD = 0x400,
424 GPR_FLAG_CAPI = 0x800,
425 GPR_FLAG_STRESS = 0x1000,
428 GPR_FLAG_IMMEDIATE_SWEEP = 0x2000,
429 GPR_FLAG_HAVE_FINALIZE = 0x4000,
430 GPR_FLAG_IMMEDIATE_MARK = 0x8000,
431 GPR_FLAG_FULL_MARK = 0x10000,
432 GPR_FLAG_COMPACT = 0x20000,
433 GPR_FLAG_GLOBAL = 0x40000,
436 (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK |
437 GPR_FLAG_IMMEDIATE_SWEEP | GPR_FLAG_CAPI),
438} gc_profile_record_flag;
445 double gc_invoke_time;
446 rb_hrtime_t gc_wall_time;
447 rb_hrtime_t gc_invoke_wall_time;
448 rb_hrtime_t gc_pause_time;
449 rb_hrtime_t gc_stop_time;
450 rb_hrtime_t gc_stw_time;
451 rb_hrtime_t gc_mark_wall_time;
452 rb_hrtime_t gc_sweep_wall_time;
453 rb_hrtime_t gc_compact_wall_time;
455 size_t heap_total_objects;
456 size_t heap_use_size;
457 size_t heap_total_size;
458 size_t moved_objects;
460#if GC_PROFILE_MORE_DETAIL
462 double gc_sweep_time;
464 size_t heap_use_pages;
465 size_t heap_live_objects;
466 size_t heap_free_objects;
468 size_t allocate_increase;
469 size_t allocate_limit;
472 size_t removing_objects;
473 size_t empty_objects;
474#if GC_PROFILE_DETAIL_MEMORY
480#if MALLOC_ALLOCATED_SIZE
481 size_t allocated_size;
484#if RGENGC_PROFILE > 0
486 size_t remembered_normal_objects;
487 size_t remembered_shady_objects;
497#define RMOVED(obj) ((struct RMoved *)(obj))
499typedef uintptr_t bits_t;
501 BITS_SIZE =
sizeof(bits_t),
502 BITS_BITLENGTH = ( BITS_SIZE * CHAR_BIT )
515#define STACK_CHUNK_SIZE 500
518 VALUE data[STACK_CHUNK_SIZE];
523 stack_chunk_t *chunk;
524 stack_chunk_t *cache;
528 size_t unused_cache_size;
531typedef int (*gc_compact_compare_func)(
const void *l,
const void *r,
void *d);
534 uintptr_t alloc_cursor;
535 uintptr_t alloc_cursor_end;
544 size_t total_allocated_pages;
545 size_t force_major_gc_count;
546 size_t force_incremental_marking_finish_count;
547 size_t total_allocated_objects;
548 size_t total_freed_objects;
549 size_t final_slots_count;
556 rb_heap_newobj_t newobj;
559 struct ccan_list_head pages;
562 uintptr_t compact_cursor_index;
571 gc_stress_no_immediate_sweep,
572 gc_stress_full_mark_after_malloc,
583typedef rbimpl_atomic_uint64_t gc_counter_t;
585#if !defined(HAVE_GCC_ATOMIC_BUILTINS_64) && !defined(_WIN32) && \
586 !(defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx)))
587# define MALLOC_COUNTERS_NEED_LOCK 1
597 gc_counter_t malloc_at_last_gc;
598 gc_counter_t free_at_last_gc;
610#define TDATA_UNSAFE_FREE_CHUNK_CAPA 32
612#define TDATA_UNSAFE_FREE_CACHE_MAX 64
615 void (*dfree)(
void *);
622 uint32_t embed_xfree_bits;
625STATIC_ASSERT(tdata_unsafe_free_bits_cover_chunk,
626 TDATA_UNSAFE_FREE_CHUNK_CAPA <= 32);
629 uint64_t minor_gc_count;
630 uint64_t major_gc_count;
631 uint64_t global_gc_invocation_count;
632 uint64_t marking_time_ns;
633 uint64_t sweeping_time_ns;
637 uint64_t minor_gc_count;
638 uint64_t major_gc_count;
639 uint64_t global_gc_invocation_count;
640 uint64_t marking_time_ns;
641 uint64_t sweeping_time_ns;
647#if RGENGC_ESTIMATE_OLDMALLOC
648 size_t oldmalloc_increase;
649 gc_counter_t freed_in_gc;
651#ifdef MALLOC_COUNTERS_NEED_LOCK
652 rb_nativethread_lock_t lock;
658#if MALLOC_ALLOCATED_SIZE
659 size_t allocated_size;
669 unsigned int mode : 2;
670 unsigned int immediate_sweep : 1;
671 unsigned int dont_gc : 1;
673 unsigned int user_gc_disabled : 1;
674 unsigned int dont_incremental : 1;
675 unsigned int during_gc : 1;
676 unsigned int during_global_gc : 1;
677 unsigned int during_compacting : 1;
678 unsigned int gc_lock_barrier : 1;
679 unsigned int during_reference_updating : 1;
680 unsigned int during_minor_gc : 1;
681 unsigned int during_postmortem : 1;
683 unsigned int ownerless : 1;
684 unsigned int measure_gc : 1;
689 rb_heap_t heaps[HEAP_COUNT];
690 size_t empty_pages_count;
700 mark_stack_t mark_stack;
704 size_t incremental_mark_step_allocated_slots;
709 size_t shareable_objects;
710 size_t shareable_objects_limit;
712 unsigned char last_cycle_pinned;
715 rb_darray(
struct heap_page *) sorted;
717 size_t allocated_pages;
720 size_t freeable_pages;
722 size_t allocatable_bytes;
725 VALUE deferred_final;
728 st_table *finalizer_table;
732 unsigned int latest_gc_info;
733 gc_profile_record *records;
734 gc_profile_record *current_record;
739 size_t record_sequence;
741#if GC_PROFILE_MORE_DETAIL
745 rb_hrtime_t invoke_wall_time;
747 size_t minor_gc_count;
748 size_t major_gc_count;
749 size_t global_gc_count;
750 size_t global_gc_invocation_count;
751 size_t compact_count;
752 size_t read_barrier_faults;
753#if RGENGC_PROFILE > 0
754 size_t total_generated_normal_object_count;
755 size_t total_generated_shady_object_count;
756 size_t total_shade_operation_count;
757 size_t total_promoted_count;
758 size_t total_remembered_normal_object_count;
759 size_t total_remembered_shady_object_count;
761#if RGENGC_PROFILE >= 2
762 size_t generated_normal_object_count_types[
RUBY_T_MASK];
763 size_t generated_shady_object_count_types[
RUBY_T_MASK];
766 size_t remembered_normal_object_count_types[
RUBY_T_MASK];
767 size_t remembered_shady_object_count_types[
RUBY_T_MASK];
772 double gc_sweep_start_time;
773 rb_hrtime_t gc_wall_start_time;
774 rb_hrtime_t gc_sweep_wall_start_time;
775 rb_hrtime_t gc_sweep_excluded_wall_time;
776 rb_hrtime_t gc_pause_start_time;
777 rb_hrtime_t gc_stw_start_time;
778 rb_hrtime_t gc_stop_time;
779 rb_hrtime_t gc_mark_phase_wall_start_time;
780 rb_hrtime_t gc_sweep_phase_wall_start_time;
781#if GC_PROFILE_MORE_DETAIL
782 size_t total_allocated_objects_at_gc_start;
783 size_t heap_used_at_gc_start;
788 unsigned long long marking_time_ns;
789 struct timespec marking_start_time;
790 unsigned long long sweeping_time_ns;
791 struct timespec sweeping_start_time;
794 size_t weak_references_count;
799 bool parent_object_old_p;
803 size_t last_major_gc;
804 size_t uncollectible_wb_unprotected_objects;
805 size_t uncollectible_wb_unprotected_objects_limit;
807 size_t old_objects_limit;
809#if RGENGC_ESTIMATE_OLDMALLOC
810 size_t oldmalloc_increase_limit;
813#if RGENGC_CHECK_MODE >= 2
814 struct st_table *allrefs_table;
820 size_t considered_count_table[
T_MASK];
821 size_t moved_count_table[
T_MASK];
822 size_t moved_up_count_table[
T_MASK];
823 size_t moved_down_count_table[
T_MASK];
827 gc_compact_compare_func compare_func;
835#if GC_DEBUG_STRESS_TO_CLASS
836 VALUE stress_to_class;
839 rb_darray(
VALUE) weak_references;
843 struct tdata_unsafe_free_chunk *tdata_unsafe_free_chunk;
845 int sweeping_heap_count;
847 int fork_vm_lock_lev;
849 struct rb_gc_vm_context vm_context;
855 rb_nativethread_lock_t lock;
856 struct gc_process_stat_snapshot published;
863 bool zjit_during_incremental_marking;
873 rb_nativethread_lock_t lock;
883 size_t arenas_unmapped;
889 size_t zombie_total_pages;
890 size_t zombie_pages_survivors;
900 rb_objspace_t *main_objspace;
906 VALUE gc_stress_mode;
914 size_t n_objspaces, objspaces_capa;
923 size_t n_pages, capa;
924 uintptr_t lomem, himem;
932 size_t tdata_deferred_free_count;
937 size_t tdata_unsafe_free_cache_len;
941} rb_global_objspace_t;
943static rb_global_objspace_t rb_global_objspace_instance;
944static rb_global_objspace_t *global_objspace = NULL;
949tdata_deferred_free_count_load(
void)
951 return rbimpl_atomic_size_load(&global_objspace->tdata_deferred_free_count, RBIMPL_ATOMIC_RELAXED);
956#define SHAREABLE_OBJECTS_LIMIT_MIN (1 << 16)
957#define SHAREABLE_OBJECTS_LIMIT_FACTOR 2.0
961#define ZOMBIE_PAGES_TRIGGER 256
964#define TDATA_DEFERRED_FREE_THRESHOLD (1 << 15)
966static void objspace_absorb(rb_objspace_t *dst, rb_objspace_t *src);
974static void page_pool_reclaim(rb_global_objspace_t *g);
975static bool page_pool_advise_supported(
void);
977#if RGENGC_CHECK_MODE && !defined(_WIN32) && !defined(__wasi__) && defined(HAVE_PTHREAD_H)
978# define PAGE_POOL_LOCK_ERRORCHECK 1
982page_pool_lock_initialize(rb_nativethread_lock_t *lock)
984#ifdef PAGE_POOL_LOCK_ERRORCHECK
985 pthread_mutexattr_t attr;
986 pthread_mutexattr_init(&attr);
987 pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
988 pthread_mutex_init(lock, &attr);
989 pthread_mutexattr_destroy(&attr);
996global_objspace_init(
void)
998 if (global_objspace == NULL) {
999 rb_global_objspace_t *g = &rb_global_objspace_instance;
1000 page_pool_lock_initialize(&g->page_pool.lock);
1001 g->page_pool.arenas = NULL;
1002 g->page_pool.arena_cursor = NULL;
1003 g->page_pool.arena_end = NULL;
1004 g->page_pool.arena_current = NULL;
1005 g->page_pool.arena_count = 0;
1006 g->page_pool.advised_count = 0;
1007 g->tdata_deferred_free_pjob = POSTPONED_JOB_HANDLE_INVALID;
1008 g->tdata_unsafe_free_published = NULL;
1009 g->tdata_unsafe_free_cache = NULL;
1010 g->tdata_unsafe_free_cache_len = 0;
1011 g->page_pool.arenas_unmapped = 0;
1012 g->page_pool.can_advise = page_pool_advise_supported();
1013 global_objspace = g;
1017#ifndef HEAP_PAGE_ALIGN_LOG
1019#define HEAP_PAGE_ALIGN_LOG 16
1023struct rvalue_overhead {
1029# define RVALUE_OVERHEAD (sizeof(struct { \
1031 struct rvalue_overhead overhead; \
1035size_t rb_gc_impl_obj_slot_size(
VALUE obj);
1036# define GET_RVALUE_OVERHEAD(obj) ((struct rvalue_overhead *)((uintptr_t)obj + rb_gc_impl_obj_slot_size(obj)))
1038# ifndef RVALUE_OVERHEAD
1039# define RVALUE_OVERHEAD 0
1043#define RVALUE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD)
1045static const size_t pool_slot_sizes[HEAP_COUNT] = {
1046#define SLOT(size) ((size) + RVALUE_OVERHEAD),
1047 EACH_POOL_SLOT_SIZE(SLOT)
1054heap_init_bytes_min(
void)
1056 return pool_slot_sizes[HEAP_COUNT - 1];
1064#define SLOT_RECIPROCAL_SHIFT 48
1065#define SLOT_RECIPROCAL(size) (((1ULL << SLOT_RECIPROCAL_SHIFT) + (size) - 1) / (size))
1067static const uint64_t heap_slot_reciprocal_table[HEAP_COUNT] = {
1068#define SLOT(size) SLOT_RECIPROCAL((size) + RVALUE_OVERHEAD),
1069 EACH_POOL_SLOT_SIZE(SLOT)
1073#if SIZEOF_VALUE >= 8
1074static uint8_t size_to_heap_idx[1024 / 8 + 1];
1076static uint8_t size_to_heap_idx[512 / 8 + 1];
1080# define MAX(a, b) (((a) > (b)) ? (a) : (b))
1083# define MIN(a, b) (((a) < (b)) ? (a) : (b))
1085#define roomof(x, y) (((x) + (y) - 1) / (y))
1086#define CEILDIV(i, mod) roomof(i, mod)
1087#define MIN_POOL_SLOT_SIZE 32
1089 HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG),
1090 HEAP_PAGE_ALIGN_MASK = (~(~0UL << HEAP_PAGE_ALIGN_LOG)),
1091 HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN,
1092 HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, MIN_POOL_SLOT_SIZE), BITS_BITLENGTH),
1093 HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),
1095#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG)
1096#define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN
1098#if !defined(INCREMENTAL_MARK_STEP_ALLOCATIONS)
1099# define INCREMENTAL_MARK_STEP_ALLOCATIONS 500
1102#undef INIT_HEAP_PAGE_ALLOC_USE_MMAP
1108static const bool HEAP_PAGE_ALLOC_USE_MMAP =
false;
1110#elif defined(__wasm__)
1114static const bool HEAP_PAGE_ALLOC_USE_MMAP =
false;
1116#elif HAVE_CONST_PAGE_SIZE
1118static const bool HEAP_PAGE_ALLOC_USE_MMAP = (PAGE_SIZE <= HEAP_PAGE_SIZE);
1120#elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE)
1122static const bool HEAP_PAGE_ALLOC_USE_MMAP =
true;
1124#elif defined(PAGE_SIZE)
1126# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (PAGE_SIZE <= HEAP_PAGE_SIZE)
1128#elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
1130# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE)
1134static const bool HEAP_PAGE_ALLOC_USE_MMAP =
false;
1137#ifdef INIT_HEAP_PAGE_ALLOC_USE_MMAP
1139# define HEAP_PAGE_ALLOC_USE_MMAP (heap_page_alloc_use_mmap != false)
1141static bool heap_page_alloc_use_mmap;
1144#define RVALUE_AGE_BIT_COUNT 2
1145#define RVALUE_AGE_BIT_MASK (((bits_t)1 << RVALUE_AGE_BIT_COUNT) - 1)
1146#define RVALUE_OLD_AGE 3
1156 struct free_region *free_region;
1158 uint64_t slot_size_reciprocal;
1159 unsigned short slot_size;
1160 unsigned short total_slots;
1161 unsigned short free_slots;
1162 unsigned short final_slots;
1163 unsigned short pinned_slots;
1168 unsigned int before_sweep : 1;
1169 unsigned int has_remembered_objects : 1;
1170 unsigned int has_uncollectible_wb_unprotected_objects : 1;
1171 unsigned int has_shref_objects : 1;
1172 unsigned int has_shareable_objects : 1;
1173 unsigned int body_zeroed : 1;
1180 rb_objspace_t *objspace;
1185 struct ccan_list_node page_node;
1187 bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT];
1189 bits_t mark_bits[HEAP_PAGE_BITMAP_LIMIT];
1190 bits_t uncollectible_bits[HEAP_PAGE_BITMAP_LIMIT];
1191 bits_t marking_bits[HEAP_PAGE_BITMAP_LIMIT];
1193 bits_t remembered_bits[HEAP_PAGE_BITMAP_LIMIT];
1199 bits_t shareable_bits[HEAP_PAGE_BITMAP_LIMIT];
1200 bits_t shref_bits[HEAP_PAGE_BITMAP_LIMIT];
1203 bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT];
1204 bits_t age_bits[HEAP_PAGE_BITMAP_LIMIT * RVALUE_AGE_BIT_COUNT];
1211asan_lock_freelist(
struct heap_page *page)
1213 asan_poison_memory_region(&page->free_region,
sizeof(
struct free_region *));
1220asan_unlock_freelist(
struct heap_page *page)
1222 asan_unpoison_memory_region(&page->free_region,
sizeof(
struct free_region *),
false);
1226heap_page_in_global_empty_pages_pool(rb_objspace_t *
objspace,
struct heap_page *page)
1228 if (page->total_slots == 0) {
1229 GC_ASSERT(page->start == 0);
1230 GC_ASSERT(page->slot_size == 0);
1231 GC_ASSERT(page->heap == NULL);
1232 GC_ASSERT(page->free_slots == 0);
1233 asan_unpoisoning_memory_region(&page->free_region,
sizeof(&page->free_region)) {
1234 GC_ASSERT(page->free_region == NULL);
1240 GC_ASSERT(page->start != 0);
1241 GC_ASSERT(page->slot_size != 0);
1242 GC_ASSERT(page->heap != NULL);
1248#define GET_PAGE_BODY(x) ((struct heap_page_body *)((bits_t)(x) & ~(HEAP_PAGE_ALIGN_MASK)))
1249#define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
1250#define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)
1253slot_index_for_offset(
size_t offset, uint64_t reciprocal)
1255 return (uint32_t)(((uint64_t)offset * reciprocal) >> SLOT_RECIPROCAL_SHIFT);
1262slot_offset_aligned_p(
const struct heap_page *page, uintptr_t p)
1264 uint64_t offset = p - page->start;
1265 uint64_t reciprocal = page->slot_size_reciprocal;
1266 uint64_t lowbits = (offset * reciprocal) & ((1ULL << SLOT_RECIPROCAL_SHIFT) - 1);
1267 return lowbits < reciprocal;
1270#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_reciprocal)
1271#define SLOT_BITMAP_INDEX(page, p) (SLOT_INDEX(page, p) / BITS_BITLENGTH)
1272#define SLOT_BITMAP_OFFSET(page, p) (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1))
1273#define SLOT_BITMAP_BIT(page, p) ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p))
1275#define _MARKED_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] & SLOT_BITMAP_BIT(page, p))
1276#define _MARK_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] |= SLOT_BITMAP_BIT(page, p))
1277#define _CLEAR_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] &= ~SLOT_BITMAP_BIT(page, p))
1279#define MARKED_IN_BITMAP(bits, p) _MARKED_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
1280#define MARK_IN_BITMAP(bits, p) _MARK_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
1281#define CLEAR_IN_BITMAP(bits, p) _CLEAR_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
1283#define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0])
1284#define GET_HEAP_PINNED_BITS(x) (&GET_HEAP_PAGE(x)->pinned_bits[0])
1285#define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0])
1286#define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0])
1287#define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0])
1288#define GET_HEAP_SHAREABLE_BITS(x) (&GET_HEAP_PAGE(x)->shareable_bits[0])
1289#define GET_HEAP_SHREF_BITS(x) (&GET_HEAP_PAGE(x)->shref_bits[0])
1290#define GET_HEAP_OBJSPACE(x) (GET_HEAP_PAGE(x)->objspace)
1294gc_foreign_object_p(
const rb_objspace_t *
objspace,
VALUE obj)
1296 if (RB_LIKELY(!rb_gc_ever_multi_ractor_p()))
return false;
1298 return RB_UNLIKELY(GET_HEAP_OBJSPACE(obj) !=
objspace);
1305gc_skip_foreign_object_p(
const rb_objspace_t *
objspace,
VALUE obj)
1307 return gc_foreign_object_p(
objspace, obj) && !
objspace->flags.during_global_gc;
1316 GC_ASSERT(page == GET_HEAP_PAGE(obj));
1318 _MARK_IN_BITMAP(page->shareable_bits, page, obj);
1319 page->flags.has_shareable_objects = TRUE;
1320 page->objspace->shareable_objects++;
1324RVALUE_AGE_GET(
VALUE obj)
1326 struct heap_page *page = GET_HEAP_PAGE(obj);
1327 bits_t *age_bits = page->age_bits;
1328 size_t slot_idx = SLOT_INDEX(page, obj);
1329 size_t idx = (slot_idx / BITS_BITLENGTH) * 2;
1330 int shift = (int)(slot_idx & (BITS_BITLENGTH - 1));
1331 int lo = (age_bits[idx] >> shift) & 1;
1332 int hi = (age_bits[idx + 1] >> shift) & 1;
1333 return lo | (hi << 1);
1337RVALUE_AGE_SET_BITMAP(
VALUE obj,
int age)
1340 struct heap_page *page = GET_HEAP_PAGE(obj);
1341 bits_t *age_bits = page->age_bits;
1342 size_t slot_idx = SLOT_INDEX(page, obj);
1343 size_t idx = (slot_idx / BITS_BITLENGTH) * 2;
1344 int shift = (int)(slot_idx & (BITS_BITLENGTH - 1));
1345 bits_t mask = (bits_t)1 << shift;
1347 age_bits[idx] = (age_bits[idx] & ~mask) | ((bits_t)(age & 1) << shift);
1348 age_bits[idx + 1] = (age_bits[idx + 1] & ~mask) | ((bits_t)((age >> 1) & 1) << shift);
1352RVALUE_AGE_SET(
VALUE obj,
int age)
1354 RVALUE_AGE_SET_BITMAP(obj, age);
1355 if (age == RVALUE_OLD_AGE) {
1363#define malloc_limit objspace->malloc_params.limit
1364#define malloc_increase gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters)
1365#define malloc_allocated_size objspace->malloc_params.allocated_size
1367#ifdef MALLOC_COUNTERS_NEED_LOCK
1368# define MALLOC_COUNTERS_LOCK(o) rb_native_mutex_lock(&(o)->malloc_counters.lock)
1369# define MALLOC_COUNTERS_UNLOCK(o) rb_native_mutex_unlock(&(o)->malloc_counters.lock)
1371# define MALLOC_COUNTERS_LOCK(o) ((void)0)
1372# define MALLOC_COUNTERS_UNLOCK(o) ((void)0)
1376gc_counter_add(gc_counter_t *p,
size_t delta)
1378#ifdef MALLOC_COUNTERS_NEED_LOCK
1379 *p += (gc_counter_t)delta;
1381 rbimpl_atomic_u64_fetch_add_relaxed(p, (uint64_t)delta);
1385static inline gc_counter_t
1386gc_counter_load_relaxed(
const gc_counter_t *p)
1388#ifdef MALLOC_COUNTERS_NEED_LOCK
1391 return rbimpl_atomic_u64_load_relaxed(p);
1395static inline gc_counter_t
1396gc_counter_load_acquire(
const gc_counter_t *p)
1398#ifdef MALLOC_COUNTERS_NEED_LOCK
1401 return rbimpl_atomic_u64_load_acquire(p);
1406gc_counter_store_release(gc_counter_t *p, gc_counter_t v)
1408#ifdef MALLOC_COUNTERS_NEED_LOCK
1411 rbimpl_atomic_u64_set_release(p, v);
1415static inline int64_t
1419 gc_counter_t malloc_at = gc_counter_load_acquire(&c->malloc_at_last_gc);
1420 gc_counter_t free_at = gc_counter_load_acquire(&c->free_at_last_gc);
1421 gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc);
1422 gc_counter_t free_now = gc_counter_load_relaxed(&c->free);
1425 gc_counter_t malloc_delta = malloc_now - malloc_at;
1426 gc_counter_t free_delta = free_now - free_at;
1428 if (malloc_delta >= free_delta) {
1429 return (int64_t)(malloc_delta - free_delta);
1432 return -(int64_t)(free_delta - malloc_delta);
1439 int64_t inc = gc_malloc_counters_increase(
objspace, c);
1440 if (inc <= 0)
return 0;
1441#if SIZEOF_SIZE_T < 8
1442 if ((uint64_t)inc > SIZE_MAX)
return SIZE_MAX;
1454 gc_counter_store_release(&c->free_at_last_gc, gc_counter_load_relaxed(&c->free));
1462 gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc);
1463 gc_counter_t free_now = gc_counter_load_relaxed(&c->free);
1464 gc_counter_store_release(&c->malloc_at_last_gc, malloc_now);
1465 gc_counter_store_release(&c->free_at_last_gc, free_now);
1469#if RGENGC_ESTIMATE_OLDMALLOC
1471gc_oldmalloc_increase_load(rb_objspace_t *
objspace)
1473 return rbimpl_atomic_size_load(&
objspace->malloc_counters.oldmalloc_increase, RBIMPL_ATOMIC_RELAXED);
1477#define heap_pages_lomem objspace->heap_pages.range[0]
1478#define heap_pages_himem objspace->heap_pages.range[1]
1479#define heap_pages_freeable_pages objspace->heap_pages.freeable_pages
1480#define heap_pages_deferred_final objspace->heap_pages.deferred_final
1481#define heaps objspace->heaps
1482#define during_gc objspace->flags.during_gc
1483#define finalizing objspace->atomic_flags.finalizing
1484#define finalizer_table objspace->finalizer_table
1485#define ruby_gc_stressful global_objspace->gc_stressful
1486#define ruby_gc_stress_mode global_objspace->gc_stress_mode
1487#if GC_DEBUG_STRESS_TO_CLASS
1488#define stress_to_class objspace->stress_to_class
1489#define set_stress_to_class(c) (stress_to_class = (c))
1491#define stress_to_class ((void)objspace, 0)
1492#define set_stress_to_class(c) ((void)objspace, (c))
1496#define dont_gc_on() (fprintf(stderr, "dont_gc_on@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 1)
1497#define dont_gc_off() (fprintf(stderr, "dont_gc_off@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 0)
1498#define dont_gc_set(b) (fprintf(stderr, "dont_gc_set(%d)@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = (int)(b))
1499#define dont_gc_val() (objspace->flags.dont_gc)
1501#define dont_gc_on() (objspace->flags.dont_gc = 1)
1502#define dont_gc_off() (objspace->flags.dont_gc = 0)
1503#define dont_gc_set(b) (objspace->flags.dont_gc = (int)(b))
1504#define dont_gc_val() (objspace->flags.dont_gc)
1507#define gc_config_full_mark_set(b) (objspace->gc_config.full_mark = (int)(b))
1508#define gc_config_full_mark_val (objspace->gc_config.full_mark)
1510static inline enum gc_mode
1511gc_mode_verify(
enum gc_mode mode)
1513#if RGENGC_CHECK_MODE > 0
1516 case gc_mode_marking:
1517 case gc_mode_sweeping:
1518 case gc_mode_compacting:
1521 rb_bug(
"gc_mode_verify: unreachable (%d)", (
int)mode);
1528has_sweeping_pages(rb_objspace_t *
objspace)
1530 return objspace->sweeping_heap_count != 0;
1534heap_eden_total_pages(rb_objspace_t *
objspace)
1537 for (
int i = 0; i < HEAP_COUNT; i++) {
1538 count += (&heaps[i])->total_pages;
1544total_allocated_objects(rb_objspace_t *
objspace)
1547 for (
int i = 0; i < HEAP_COUNT; i++) {
1548 rb_heap_t *heap = &heaps[i];
1549 count += heap->total_allocated_objects;
1555total_freed_objects(rb_objspace_t *
objspace)
1558 for (
int i = 0; i < HEAP_COUNT; i++) {
1559 rb_heap_t *heap = &heaps[i];
1560 count += heap->total_freed_objects;
1566total_final_slots_count(rb_objspace_t *
objspace)
1569 for (
int i = 0; i < HEAP_COUNT; i++) {
1570 rb_heap_t *heap = &heaps[i];
1571 count += heap->final_slots_count;
1576#define gc_mode(objspace) gc_mode_verify((enum gc_mode)(objspace)->flags.mode)
1577#define gc_mode_set(objspace, m) ((objspace)->flags.mode = (unsigned int)gc_mode_verify(m))
1578#define gc_needs_major_flags objspace->rgengc.need_major_gc
1580#define is_marking(objspace) (gc_mode(objspace) == gc_mode_marking)
1581#define is_sweeping(objspace) (gc_mode(objspace) == gc_mode_sweeping)
1582#define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE)
1583#define is_incremental_marking(objspace) ((objspace)->zjit_during_incremental_marking != FALSE)
1584#define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE)
1587gc_set_incremental_marking(rb_objspace_t *
objspace,
unsigned int value)
1589 if (
objspace->zjit_during_incremental_marking != value) {
1590 objspace->zjit_during_incremental_marking = value;
1603#define GC_INCREMENTAL_SWEEP_BYTES (2048 * RVALUE_SLOT_SIZE)
1604#define GC_INCREMENTAL_SWEEP_POOL_BYTES (1024 * RVALUE_SLOT_SIZE)
1605#define is_lazy_sweeping(objspace) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(objspace))
1607#define needs_continue_sweeping(objspace, heap) \
1608 ((heap)->free_pages == NULL && is_lazy_sweeping(objspace))
1610#if SIZEOF_LONG == SIZEOF_VOIDP
1611# define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG)
1612#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
1613# define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \
1614 ((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1))
1616# error not supported
1622 void (*dfree)(
void *);
1626#define RZOMBIE(o) ((struct RZombie *)(o))
1628static bool ruby_enable_autocompact =
false;
1629#if RGENGC_CHECK_MODE
1630static gc_compact_compare_func ruby_autocompact_compare_func;
1633static void init_mark_stack(mark_stack_t *stack);
1634static int garbage_collect(rb_objspace_t *,
unsigned int reason);
1636static int gc_start(rb_objspace_t *
objspace,
unsigned int reason);
1637static void gc_rest(rb_objspace_t *
objspace);
1642#define gc_event_hook(objspace, event) do { \
1643 if (RB_UNLIKELY((objspace)->hook_events & (event))) { \
1644 rb_gc_event_hook(0, (event)); \
1648enum gc_enter_event {
1649 gc_enter_event_start,
1650 gc_enter_event_continue,
1651 gc_enter_event_rest,
1652 gc_enter_event_finalizer,
1653 gc_enter_event_global,
1654 gc_enter_event_global_auto,
1657static inline bool gc_enter(rb_objspace_t *
objspace,
enum gc_enter_event event,
unsigned int *lock_lev);
1658static inline void gc_exit(rb_objspace_t *
objspace,
enum gc_enter_event event,
unsigned int *lock_lev);
1659static void gc_marking_enter(rb_objspace_t *
objspace);
1660static void gc_marking_exit(rb_objspace_t *
objspace);
1661static void gc_sweeping_enter(rb_objspace_t *
objspace);
1662static void gc_sweeping_exit(rb_objspace_t *
objspace);
1663static bool gc_marks_continue(rb_objspace_t *
objspace, rb_heap_t *heap);
1665static void gc_sweep(rb_objspace_t *
objspace);
1666static void gc_sweep_finish_heap(rb_objspace_t *
objspace, rb_heap_t *heap);
1667static void gc_sweep_continue(rb_objspace_t *
objspace, rb_heap_t *heap);
1669static inline void gc_mark(rb_objspace_t *
objspace,
VALUE ptr);
1670static inline void gc_pin(rb_objspace_t *
objspace,
VALUE ptr);
1671static inline void gc_mark_and_pin(rb_objspace_t *
objspace,
VALUE ptr);
1673static int gc_mark_stacked_objects_incremental(rb_objspace_t *,
size_t count);
1674NO_SANITIZE(
"memory",
static inline bool is_pointer_to_heap(rb_objspace_t *
objspace,
const void *ptr));
1676static void gc_verify_internal_consistency(
void *objspace_ptr);
1678static double getrusage_time(
void);
1679static inline rb_hrtime_t elapsed_hrtime_from(rb_hrtime_t start);
1680static inline void gc_prof_setup_new_record(rb_objspace_t *
objspace,
unsigned int reason);
1681static inline void gc_prof_timer_start(rb_objspace_t *);
1682static inline void gc_prof_timer_stop(rb_objspace_t *);
1683static inline void gc_prof_mark_timer_start(rb_objspace_t *);
1684static inline void gc_prof_mark_timer_stop(rb_objspace_t *);
1685static inline void gc_prof_sweep_timer_start(rb_objspace_t *);
1686static inline void gc_prof_sweep_timer_stop(rb_objspace_t *);
1687static inline void gc_prof_set_malloc_info(rb_objspace_t *);
1688static inline void gc_prof_set_heap_info(rb_objspace_t *);
1690#define gc_prof_record(objspace) (objspace)->profile.current_record
1691#define gc_prof_enabled(objspace) ((objspace)->profile.run && (objspace)->profile.current_record)
1693#define gc_report(level, objspace, ...) \
1694 if (!RGENGC_DEBUG_ENABLED(level)) {} else gc_report_body(level, objspace, __VA_ARGS__)
1695PRINTF_ARGS(
static void gc_report_body(
int level, rb_objspace_t *
objspace,
const char *fmt, ...), 3, 4);
1697static void gc_finalize_deferred(
void *dmy);
1698static void gc_tdata_unsafe_drain_objspaces(rb_objspace_t **objspaces,
size_t n);
1709#if defined(__GNUC__) && defined(__i386__)
1710typedef unsigned long long tick_t;
1711#define PRItick "llu"
1715 unsigned long long int x;
1716 __asm__ __volatile__ (
"rdtsc" :
"=A" (x));
1720#elif defined(__GNUC__) && defined(__x86_64__)
1721typedef unsigned long long tick_t;
1722#define PRItick "llu"
1724static __inline__ tick_t
1727 unsigned long hi, lo;
1728 __asm__ __volatile__ (
"rdtsc" :
"=a"(lo),
"=d"(hi));
1729 return ((
unsigned long long)lo)|( ((
unsigned long long)hi)<<32);
1732#elif defined(__powerpc64__) && (GCC_VERSION_SINCE(4,8,0) || defined(__clang__))
1733typedef unsigned long long tick_t;
1734#define PRItick "llu"
1736static __inline__ tick_t
1739 unsigned long long val = __builtin_ppc_get_timebase();
1743#elif defined(__aarch64__) && defined(__GNUC__)
1744typedef unsigned long tick_t;
1747static __inline__ tick_t
1751 __asm__ __volatile__ (
"mrs %0, cntvct_el0" :
"=r" (val));
1756#elif defined(_WIN32) && defined(_MSC_VER)
1758typedef unsigned __int64 tick_t;
1759#define PRItick "llu"
1768typedef clock_t tick_t;
1769#define PRItick "llu"
1778#define MEASURE_LINE(expr) expr
1781static inline VALUE check_rvalue_consistency(rb_objspace_t *
objspace,
const VALUE obj);
1783#define RVALUE_MARKED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj))
1784#define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj))
1785#define RVALUE_MARKING_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj))
1786#define RVALUE_UNCOLLECTIBLE_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj))
1787#define RVALUE_PINNED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), (obj))
1792 check_rvalue_consistency(
objspace, obj);
1793 return RVALUE_MARKED_BITMAP(obj) != 0;
1799 check_rvalue_consistency(
objspace, obj);
1800 return RVALUE_PINNED_BITMAP(obj) != 0;
1806 check_rvalue_consistency(
objspace, obj);
1807 return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1813 check_rvalue_consistency(
objspace, obj);
1814 return RVALUE_MARKING_BITMAP(obj) != 0;
1820 check_rvalue_consistency(
objspace, obj);
1821 return MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
1827 check_rvalue_consistency(
objspace, obj);
1828 return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1831#define RVALUE_PAGE_WB_UNPROTECTED(page, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj))
1832#define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj))
1833#define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj))
1835static void rgengc_remember(rb_objspace_t *
objspace,
VALUE obj);
1836static void gc_bitmaps_clear(rb_objspace_t *
objspace, rb_heap_t *heap,
bool clear_shref);
1837static void rgengc_rememberset_mark(rb_objspace_t *
objspace, rb_heap_t *heap);
1838static bool verify_pointer_in_any_heap_p(
const void *ptr);
1841check_rvalue_consistency_force(rb_objspace_t *
objspace,
const VALUE obj,
int terminate)
1848 const bool world_stopped =
objspace->flags.during_global_gc;
1854 const bool take_vm_lock = !world_stopped && !during_gc;
1855 unsigned int lev = 0;
1856 if (take_vm_lock) lev = RB_GC_VM_LOCK_NO_BARRIER();
1859 fprintf(stderr,
"check_rvalue_consistency: %p is a special const.\n", (
void *)obj);
1862 else if (!is_pointer_to_heap(
objspace, (
void *)obj)) {
1867 if (!world_stopped) {
1873 else if (!verify_pointer_in_any_heap_p((
void *)obj)) {
1875 while (empty_page) {
1876 if ((uintptr_t)empty_page->body <= (uintptr_t)obj &&
1877 (uintptr_t)obj < (uintptr_t)empty_page->body + HEAP_PAGE_SIZE) {
1878 GC_ASSERT(heap_page_in_global_empty_pages_pool(
objspace, empty_page));
1879 fprintf(stderr,
"check_rvalue_consistency: %p is in an empty page (%p).\n",
1880 (
void *)obj, (
void *)empty_page);
1884 empty_page = empty_page->free_next;
1886 fprintf(stderr,
"check_rvalue_consistency: %p is not a Ruby object.\n", (
void *)obj);
1893 const int wb_unprotected_bit = RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1894 const int uncollectible_bit = RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1895 const int mark_bit = RVALUE_MARKED_BITMAP(obj) != 0;
1896 const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0;
1897 const int remembered_bit = MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
1898 const int age = RVALUE_AGE_GET((
VALUE)obj);
1900 if (heap_page_in_global_empty_pages_pool(
objspace, GET_HEAP_PAGE(obj))) {
1901 fprintf(stderr,
"check_rvalue_consistency: %s is in tomb page.\n", rb_obj_info(obj));
1905 fprintf(stderr,
"check_rvalue_consistency: %s is T_NONE.\n", rb_obj_info(obj));
1909 fprintf(stderr,
"check_rvalue_consistency: %s is T_ZOMBIE.\n", rb_obj_info(obj));
1916 rb_obj_memsize_of((
VALUE)obj);
1923 if (age > 0 && wb_unprotected_bit) {
1924 fprintf(stderr,
"check_rvalue_consistency: %s is not WB protected, but age is %d > 0.\n", rb_obj_info(obj), age);
1928 if (!is_marking(
objspace) && uncollectible_bit && !mark_bit) {
1929 fprintf(stderr,
"check_rvalue_consistency: %s is uncollectible, but is not marked while !gc.\n", rb_obj_info(obj));
1934 if (uncollectible_bit && age != RVALUE_OLD_AGE && !wb_unprotected_bit) {
1935 fprintf(stderr,
"check_rvalue_consistency: %s is uncollectible, but not old (age: %d) and not WB unprotected.\n",
1936 rb_obj_info(obj), age);
1939 if (remembered_bit && age != RVALUE_OLD_AGE) {
1940 fprintf(stderr,
"check_rvalue_consistency: %s is remembered, but not old (age: %d).\n",
1941 rb_obj_info(obj), age);
1953 if (is_incremental_marking(
objspace) && marking_bit) {
1954 if (!is_marking(
objspace) && !mark_bit) {
1955 fprintf(stderr,
"check_rvalue_consistency: %s is marking, but not marked.\n", rb_obj_info(obj));
1961 if (take_vm_lock) RB_GC_VM_UNLOCK_NO_BARRIER(lev);
1963 if (err > 0 && terminate) {
1964 rb_bug(
"check_rvalue_consistency_force: there is %d errors.", err);
1969#if RGENGC_CHECK_MODE == 0
1971check_rvalue_consistency(rb_objspace_t *
objspace,
const VALUE obj)
1977check_rvalue_consistency(rb_objspace_t *
objspace,
const VALUE obj)
1979 check_rvalue_consistency_force(
objspace, obj, TRUE);
1989 asan_unpoisoning_object(obj) {
1999 check_rvalue_consistency(
objspace, obj);
2008 MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
2012 page->objspace->rgengc.old_objects++;
2014#if RGENGC_PROFILE >= 2
2015 objspace->profile.total_promoted_count++;
2021RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *
objspace,
VALUE obj)
2023 RB_DEBUG_COUNTER_INC(obj_promote);
2024 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(
objspace, GET_HEAP_PAGE(obj), obj);
2031 int age = RVALUE_AGE_GET((
VALUE)obj);
2033 if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) {
2034 rb_bug(
"RVALUE_AGE_INC: can not increment age of OLD object %s.", rb_obj_info(obj));
2038 RVALUE_AGE_SET(obj, age);
2040 if (age == RVALUE_OLD_AGE) {
2041 RVALUE_OLD_UNCOLLECTIBLE_SET(
objspace, obj);
2044 check_rvalue_consistency(
objspace, obj);
2050 check_rvalue_consistency(
objspace, obj);
2051 GC_ASSERT(!RVALUE_OLD_P(
objspace, obj));
2052 RVALUE_AGE_SET(obj, RVALUE_OLD_AGE - 1);
2053 check_rvalue_consistency(
objspace, obj);
2057RVALUE_AGE_RESET(
VALUE obj)
2059 RVALUE_AGE_SET(obj, 0);
2065 check_rvalue_consistency(
objspace, obj);
2066 GC_ASSERT(RVALUE_OLD_P(
objspace, obj));
2068 if (!is_incremental_marking(
objspace) && RVALUE_REMEMBERED(
objspace, obj)) {
2069 struct heap_page *page = GET_HEAP_PAGE(obj);
2070 _CLEAR_IN_BITMAP(page->remembered_bits, page, obj);
2073 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
2074 RVALUE_AGE_RESET(obj);
2076 if (RVALUE_MARKED(
objspace, obj)) {
2078 GET_HEAP_PAGE(obj)->objspace->rgengc.old_objects--;
2081 check_rvalue_consistency(
objspace, obj);
2093 return !RVALUE_MARKED(
objspace, obj);
2097rb_gc_impl_user_gc_disabled_set(
void *objspace_ptr,
bool disable)
2099 rb_objspace_t *
objspace = objspace_ptr;
2100 const bool was =
objspace->flags.user_gc_disabled;
2101 objspace->flags.user_gc_disabled = disable;
2106rb_gc_impl_user_gc_disabled_p(
void *objspace_ptr)
2108 rb_objspace_t *
objspace = objspace_ptr;
2109 return objspace->flags.user_gc_disabled;
2113rb_gc_impl_gc_enabled_p(
void *objspace_ptr)
2115 rb_objspace_t *
objspace = objspace_ptr;
2116 return !dont_gc_val();
2120rb_gc_impl_gc_enable(
void *objspace_ptr)
2122 rb_objspace_t *
objspace = objspace_ptr;
2128rb_gc_impl_gc_disable(
void *objspace_ptr,
bool finish_current_gc)
2130 rb_objspace_t *
objspace = objspace_ptr;
2132 if (finish_current_gc) {
2143rb_gc_impl_gc_rest(
void *objspace_ptr)
2145 gc_rest(objspace_ptr);
2155 return calloc(1, n);
2159rb_gc_impl_set_event_hook(
void *objspace_ptr,
const rb_event_flag_t event)
2161 rb_objspace_t *
objspace = objspace_ptr;
2164 objspace == global_objspace->main_objspace);
2169rb_gc_impl_get_total_time(
void *objspace_ptr)
2171 rb_objspace_t *
objspace = objspace_ptr;
2173 unsigned long long marking_time =
objspace->profile.marking_time_ns;
2174 unsigned long long sweeping_time =
objspace->profile.sweeping_time_ns;
2176 return marking_time + sweeping_time;
2180rb_gc_impl_set_measure_total_time(
void *objspace_ptr,
VALUE flag)
2182 rb_objspace_t *
objspace = objspace_ptr;
2188rb_gc_impl_get_measure_total_time(
void *objspace_ptr)
2190 rb_objspace_t *
objspace = objspace_ptr;
2196gc_process_stat_capture(
const rb_objspace_t *
objspace,
2199 out->minor_gc_count =
objspace->profile.minor_gc_count;
2200 out->major_gc_count =
objspace->profile.major_gc_count;
2201 out->global_gc_invocation_count =
objspace->profile.global_gc_invocation_count;
2202 out->marking_time_ns =
objspace->profile.marking_time_ns;
2203 out->sweeping_time_ns =
objspace->profile.sweeping_time_ns;
2207gc_process_stat_publish(rb_objspace_t *
objspace)
2212 gc_process_stat_capture(
objspace, &snap);
2214 objspace->process_stat.published = snap;
2222 dst->minor_gc_count += src->minor_gc_count;
2223 dst->major_gc_count += src->major_gc_count;
2224 dst->global_gc_invocation_count += src->global_gc_invocation_count;
2225 dst->marking_time_ns += src->marking_time_ns;
2226 dst->sweeping_time_ns += src->sweeping_time_ns;
2231rb_gc_impl_garbage_object_p(
void *objspace_ptr,
VALUE ptr)
2233 rb_objspace_t *
objspace = objspace_ptr;
2240 if (gc_skip_foreign_object_p(
objspace, ptr)) {
2248 asan_unpoisoning_object(ptr) {
2254 return is_lazy_sweeping(
objspace) && GET_HEAP_PAGE(ptr)->flags.before_sweep &&
2259rb_gc_impl_get_vm_context(
void *objspace_ptr)
2261 rb_objspace_t *
objspace = objspace_ptr;
2266static void free_stack_chunks(mark_stack_t *);
2267static void mark_stack_free_cache(mark_stack_t *);
2273 if (RGENGC_CHECK_MODE &&
2275 !(page->start <= (uintptr_t)obj &&
2276 (uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) &&
2277 obj %
sizeof(
VALUE) == 0)) {
2278 rb_bug(
"gc_check_obj_in_page: %p is not rvalue.", (
void *)obj);
2285 rb_asan_unpoison_object(obj,
false);
2288 GC_ASSERT(RVALUE_AGE_GET(obj) == 0);
2290 gc_check_obj_in_page(page, obj);
2292 asan_unlock_freelist(page);
2296 CLEAR_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj);
2297 CLEAR_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj);
2301 region->end = (uintptr_t)obj + page->slot_size;
2302 region->next = page->free_region;
2303 page->free_region = region;
2305 asan_lock_freelist(page);
2307 rb_asan_poison_object(obj);
2308 gc_report(3,
objspace,
"heap_page_add_free_region: %p\n", (
void *)obj);
2314objspace_heap_init_bytes(
const rb_objspace_t *
objspace)
2316 return objspace == global_objspace->main_objspace
2317 ? gc_params.heap_init_bytes : gc_params.ractor_heap_init_bytes;
2321heap_allocatable_bytes_expand(rb_objspace_t *
objspace,
2322 rb_heap_t *heap,
size_t free_slots,
size_t total_slots,
size_t slot_size)
2324 double goal_ratio = gc_params.heap_free_slots_goal_ratio;
2325 size_t target_total_slots;
2327 if (goal_ratio == 0.0) {
2328 target_total_slots = (size_t)(total_slots * gc_params.growth_factor);
2330 else if (total_slots == 0) {
2331 target_total_slots = objspace_heap_init_bytes(
objspace) / slot_size;
2337 double f = (double)(total_slots - free_slots) / ((1 - goal_ratio) * total_slots);
2339 if (f > gc_params.growth_factor) f = gc_params.growth_factor;
2340 if (f < 1.0) f = 1.1;
2342 target_total_slots = (size_t)(f * total_slots);
2346 "free_slots(%8"PRIuSIZE
")/total_slots(%8"PRIuSIZE
")=%1.2f,"
2347 " G(%1.2f), f(%1.2f),"
2348 " total_slots(%8"PRIuSIZE
") => target_total_slots(%8"PRIuSIZE
")\n",
2349 free_slots, total_slots, free_slots/(
double)total_slots,
2350 goal_ratio, f, total_slots, target_total_slots);
2354 if (gc_params.growth_max_bytes > 0) {
2355 size_t max_total_slots = total_slots + gc_params.growth_max_bytes / slot_size;
2356 if (target_total_slots > max_total_slots) target_total_slots = max_total_slots;
2359 size_t extend_slot_count = target_total_slots - total_slots;
2361 if (extend_slot_count == 0) extend_slot_count = 1;
2363 objspace->heap_pages.allocatable_bytes += extend_slot_count * slot_size;
2367heap_add_freepage(rb_heap_t *heap,
struct heap_page *page)
2369 asan_unlock_freelist(page);
2370 GC_ASSERT(page->free_slots != 0);
2371 GC_ASSERT(page->free_region != NULL);
2373 page->free_next = heap->free_pages;
2374 heap->free_pages = page;
2376 RUBY_DEBUG_LOG(
"page:%p free_region:%p", (
void *)page, (
void *)page->free_region);
2378 asan_lock_freelist(page);
2382heap_add_poolpage(rb_objspace_t *
objspace, rb_heap_t *heap,
struct heap_page *page)
2384 asan_unlock_freelist(page);
2385 GC_ASSERT(page->free_slots != 0);
2386 GC_ASSERT(page->free_region != NULL);
2388 page->free_next = heap->pooled_pages;
2389 heap->pooled_pages = page;
2390 objspace->rincgc.pooled_slots += page->free_slots;
2392 asan_lock_freelist(page);
2396heap_unlink_page(rb_objspace_t *
objspace, rb_heap_t *heap,
struct heap_page *page)
2398 ccan_list_del(&page->page_node);
2399 heap->total_pages--;
2400 heap->total_slots -= page->total_slots;
2407heap_page_detach_empty(rb_objspace_t *
objspace, rb_heap_t *heap,
struct heap_page *page)
2409 heap_unlink_page(
objspace, heap, page);
2412 page->total_slots = 0;
2413 page->slot_size = 0;
2415 page->free_slots = 0;
2417 asan_unlock_freelist(page);
2418 page->free_region = NULL;
2419 asan_lock_freelist(page);
2421 asan_poison_memory_region(page->body, HEAP_PAGE_SIZE);
2424 page->free_next =
objspace->empty_pages;
2429gc_aligned_free(
void *ptr,
size_t size)
2431#if defined __MINGW32__
2432 __mingw_aligned_free(ptr);
2435#elif defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN)
2438 free(((
void**)ptr)[-1]);
2445 GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
2447 page_pool_release(page_body, arena);
2450#ifdef PAGE_POOL_LOCK_ERRORCHECK
2451# define ASSERT_PAGE_POOL_LOCKED(g) GC_ASSERT(pthread_mutex_lock(&(g)->page_pool.lock) == EDEADLK)
2453# define ASSERT_PAGE_POOL_LOCKED(g) ((void)0)
2463global_page_index_insert(
struct heap_page *page)
2465 rb_global_objspace_t *g = global_objspace;
2466 uintptr_t body = (uintptr_t)page->body;
2469 if (g->page_index.n_pages == g->page_index.capa) {
2470 size_t new_capa = g->page_index.capa ? g->page_index.capa * 2 : 128;
2471 struct heap_page **grown = realloc(g->page_index.pages, new_capa *
sizeof(*grown));
2472 if (grown == NULL) rb_bug(
"global_page_index_insert: realloc failed");
2473 g->page_index.pages = grown;
2474 g->page_index.capa = new_capa;
2476 size_t lo = 0, hi = g->page_index.n_pages;
2478 size_t mid = (lo + hi) / 2;
2479 if ((uintptr_t)g->page_index.pages[mid]->body < body) lo = mid + 1;
2482 memmove(&g->page_index.pages[lo + 1], &g->page_index.pages[lo],
2483 (g->page_index.n_pages - lo) *
sizeof(
struct heap_page *));
2484 g->page_index.pages[lo] = page;
2485 g->page_index.n_pages++;
2488 uintptr_t end = body + HEAP_PAGE_SIZE;
2489 if (g->page_index.lomem == 0 || g->page_index.lomem > start) g->page_index.lomem = start;
2490 if (g->page_index.himem < end) g->page_index.himem = end;
2495global_page_index_remove_locked(
const struct heap_page *page)
2497 rb_global_objspace_t *g = global_objspace;
2498 uintptr_t body = (uintptr_t)page->body;
2500 ASSERT_PAGE_POOL_LOCKED(g);
2502 size_t lo = 0, hi = g->page_index.n_pages;
2504 size_t mid = (lo + hi) / 2;
2505 if ((uintptr_t)g->page_index.pages[mid]->body < body) lo = mid + 1;
2508 GC_ASSERT(lo < g->page_index.n_pages && g->page_index.pages[lo] == page);
2509 memmove(&g->page_index.pages[lo], &g->page_index.pages[lo + 1],
2510 (g->page_index.n_pages - lo - 1) *
sizeof(
struct heap_page *));
2511 g->page_index.n_pages--;
2515global_page_index_remove(
const struct heap_page *page)
2517 rb_global_objspace_t *g = global_objspace;
2520 global_page_index_remove_locked(page);
2527 global_page_index_remove(page);
2528 objspace->heap_pages.freed_pages++;
2529 heap_page_body_free(page->body, page->arena);
2536 rb_global_objspace_t *g = global_objspace;
2539 for (
struct heap_page *page = pages; page != NULL; page = page->free_next) {
2540 global_page_index_remove_locked(page);
2541 if (HEAP_PAGE_ALLOC_USE_MMAP) {
2543 page_pool_release_locked(page->body, page->arena);
2549 if (!HEAP_PAGE_ALLOC_USE_MMAP) {
2551 for (
struct heap_page *page = pages; page != NULL; page = page->free_next) {
2552 heap_page_body_free(page->body, page->arena);
2556 while (pages != NULL) {
2557 struct heap_page *next = pages->free_next;
2558 objspace->heap_pages.freed_pages++;
2565heap_pages_free_unused_pages(rb_objspace_t *
objspace)
2567 if (
objspace->empty_pages != NULL && heap_pages_freeable_pages > 0) {
2568 GC_ASSERT(
objspace->empty_pages_count > 0);
2574 for (i = j = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
2577 if (heap_page_in_global_empty_pages_pool(
objspace, page) && heap_pages_freeable_pages > 0) {
2578 page->free_next = to_free;
2580 heap_pages_freeable_pages--;
2583 if (heap_page_in_global_empty_pages_pool(
objspace, page)) {
2584 page->free_next =
objspace->empty_pages;
2590 rb_darray_set(
objspace->heap_pages.sorted, j, page);
2596 rb_darray_pop(
objspace->heap_pages.sorted, i - j);
2597 GC_ASSERT(rb_darray_size(
objspace->heap_pages.sorted) == j);
2602 uintptr_t himem = (uintptr_t)hipage->body + HEAP_PAGE_SIZE;
2603 GC_ASSERT(himem <= heap_pages_himem);
2604 heap_pages_himem = himem;
2607 uintptr_t lomem = (uintptr_t)lopage->body +
sizeof(
struct heap_page_header);
2608 GC_ASSERT(lomem >= heap_pages_lomem);
2609 heap_pages_lomem = lomem;
2612 heap_pages_lomem = 0;
2613 heap_pages_himem = 0;
2616 heap_pages_free_batch(
objspace, to_free);
2621gc_aligned_malloc(
size_t alignment,
size_t size)
2624 GC_ASSERT(((alignment - 1) & alignment) == 0);
2625 GC_ASSERT(alignment %
sizeof(
void*) == 0);
2629#if defined __MINGW32__
2630 res = __mingw_aligned_malloc(size, alignment);
2632 res = _aligned_malloc(size, alignment);
2633#elif defined(HAVE_POSIX_MEMALIGN)
2634 if (posix_memalign(&res, alignment, size) != 0) {
2637#elif defined(HAVE_MEMALIGN)
2638 res = memalign(alignment, size);
2641 res = malloc(alignment + size +
sizeof(
void*));
2642 aligned = (
char*)res + alignment +
sizeof(
void*);
2643 aligned -= ((
VALUE)aligned & (alignment - 1));
2644 ((
void**)aligned)[-1] = res;
2645 res = (
void*)aligned;
2648 GC_ASSERT((uintptr_t)res % alignment == 0);
2658#define PAGE_POOL_ARENA_SIZE (HEAP_PAGE_SIZE * 32)
2659#define PAGE_POOL_ARENA_BODIES (PAGE_POOL_ARENA_SIZE / HEAP_PAGE_SIZE)
2660#define PAGE_POOL_ARENA_KEEP_HALF (PAGE_POOL_ARENA_BODIES / 2)
2662#define PAGE_POOL_ALL_BODIES ((uint32_t)((UINT64_C(1) << PAGE_POOL_ARENA_BODIES) - 1))
2669 uint32_t advised_mask;
2670 uint32_t zeroed_mask;
2672STATIC_ASSERT(page_pool_bodies_fit_mask, PAGE_POOL_ARENA_BODIES <= 32);
2680page_pool_advise_supported(
void)
2683 long os_page_size = sysconf(_SC_PAGE_SIZE);
2684 return os_page_size > 0 && HEAP_PAGE_SIZE % (size_t)os_page_size == 0;
2694 ptrdiff_t
off = (
const char *)body - arena->start;
2695 GC_ASSERT(
off >= 0 &&
off % HEAP_PAGE_SIZE == 0);
2696 int i = (int)(
off / HEAP_PAGE_SIZE);
2697 GC_ASSERT(i < PAGE_POOL_ARENA_BODIES);
2704 GC_ASSERT(i >= 0 && i < PAGE_POOL_ARENA_BODIES);
2705 return (
struct heap_page_body *)(arena->start + (size_t)i * HEAP_PAGE_SIZE);
2713page_pool_add_arena(rb_global_objspace_t *g)
2715 GC_ASSERT(HEAP_PAGE_ALIGN % sysconf(_SC_PAGE_SIZE) == 0);
2717 size_t mmap_size = PAGE_POOL_ARENA_SIZE + HEAP_PAGE_ALIGN;
2718 char *ptr = mmap(NULL, mmap_size,
2719 PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
2720 if (ptr == MAP_FAILED) {
2729#if defined(HAVE_SYS_PRCTL_H) && defined(PR_SET_VMA) && defined(PR_SET_VMA_ANON_NAME)
2730 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, mmap_size,
"Ruby:GC:default:page_pool_arena");
2735 char *aligned = ptr + HEAP_PAGE_ALIGN;
2736 aligned -= ((uintptr_t)aligned & (HEAP_PAGE_ALIGN - 1));
2737 GC_ASSERT(aligned > ptr);
2738 GC_ASSERT(aligned <= ptr + HEAP_PAGE_ALIGN);
2740 size_t start_out_of_range_size = aligned - ptr;
2741 GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
2742 if (start_out_of_range_size > 0) {
2743 if (munmap(ptr, start_out_of_range_size)) {
2744 rb_bug(
"page_pool_add_arena: munmap failed for start");
2748 size_t end_out_of_range_size = HEAP_PAGE_ALIGN - start_out_of_range_size;
2749 GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
2750 if (end_out_of_range_size > 0) {
2751 if (munmap(aligned + PAGE_POOL_ARENA_SIZE, end_out_of_range_size)) {
2752 rb_bug(
"page_pool_add_arena: munmap failed for end");
2757 if (arena == NULL) {
2758 if (munmap(aligned, PAGE_POOL_ARENA_SIZE)) {
2759 rb_bug(
"page_pool_add_arena: munmap failed for arena");
2763 arena->start = aligned;
2764 arena->size = PAGE_POOL_ARENA_SIZE;
2765 arena->free_mask = 0;
2766 arena->advised_mask = 0;
2767 arena->next = g->page_pool.arenas;
2768 g->page_pool.arenas = arena;
2769 g->page_pool.arena_count++;
2770 g->page_pool.arena_current = arena;
2772 g->page_pool.arena_cursor = aligned;
2773 g->page_pool.arena_end = aligned + PAGE_POOL_ARENA_SIZE;
2785 *zeroed_out =
false;
2787 if (HEAP_PAGE_ALLOC_USE_MMAP) {
2789 bool need_reuse =
false;
2790 rb_global_objspace_t *g = global_objspace;
2795 if (a->free_mask == 0)
continue;
2799 uint32_t warm = a->free_mask & ~a->advised_mask;
2800 int i = ntz_int32(warm ? warm : a->free_mask);
2801 uint32_t bit = (uint32_t)1 << i;
2803 body = page_pool_body_at(a, i);
2805 *zeroed_out = (a->zeroed_mask & bit) != 0;
2806 a->free_mask &= ~bit;
2807 a->zeroed_mask &= ~bit;
2808 need_reuse = (a->advised_mask & bit) != 0;
2810 a->advised_mask &= ~bit;
2811 g->page_pool.advised_count--;
2816 (g->page_pool.arena_cursor != g->page_pool.arena_end ||
2817 page_pool_add_arena(g))) {
2818 GC_ASSERT(g->page_pool.arena_cursor + HEAP_PAGE_SIZE <= g->page_pool.arena_end);
2820 g->page_pool.arena_cursor += HEAP_PAGE_SIZE;
2821 *arena_out = g->page_pool.arena_current;
2829 rb_vm_map_reuse(body, HEAP_PAGE_SIZE);
2831 asan_unpoison_memory_region(body, HEAP_PAGE_SIZE,
false);
2836 body = gc_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
2847 ASSERT_PAGE_POOL_LOCKED(global_objspace);
2849 uint32_t bit = (uint32_t)1 << page_pool_body_index(arena, body);
2851 GC_ASSERT(!(arena->free_mask & bit));
2852 GC_ASSERT(!(arena->advised_mask & bit));
2854 arena->free_mask |= bit;
2855 arena->zeroed_mask &= ~bit;
2856 asan_poison_memory_region(body, HEAP_PAGE_SIZE);
2863 if (HEAP_PAGE_ALLOC_USE_MMAP) {
2865 rb_global_objspace_t *g = global_objspace;
2868 page_pool_release_locked(body, arena);
2873 gc_aligned_free(body, HEAP_PAGE_SIZE);
2883page_pool_reclaim(rb_global_objspace_t *g)
2885 if (!HEAP_PAGE_ALLOC_USE_MMAP)
return;
2892 if (g->page_pool.can_advise) {
2894 uint32_t advise_mask = a->free_mask & ~a->advised_mask;
2895 uint32_t zeroed_mask = 0;
2896 if (advise_mask == 0)
continue;
2898 for (
int i = 0; i < PAGE_POOL_ARENA_BODIES; i++) {
2899 if (!(advise_mask & ((uint32_t)1 << i)))
continue;
2902 while (i + 1 < PAGE_POOL_ARENA_BODIES &&
2903 (advise_mask & ((uint32_t)1 << (i + 1)))) {
2906 int run_len = i - run + 1;
2909 rb_vm_map_reusable_immediate(page_pool_body_at(a, run),
2910 (
size_t)run_len * HEAP_PAGE_SIZE, 0, &zeroed);
2912 for (
int j = run; j <= i; j++) {
2913 zeroed_mask |= (uint32_t)1 << j;
2916 g->page_pool.advised_count += run_len;
2918 a->advised_mask |= advise_mask;
2919 a->zeroed_mask |= zeroed_mask;
2930 total_free += (int)rb_popcount32(a->free_mask);
2933 bool retained_one =
false;
2938 if (a->free_mask != PAGE_POOL_ALL_BODIES) {
2943 int free_elsewhere = total_free - PAGE_POOL_ARENA_BODIES;
2944 if (free_elsewhere < PAGE_POOL_ARENA_KEEP_HALF && !retained_one) {
2945 retained_one =
true;
2951 if (munmap(a->start, a->size)) {
2952 rb_bug(
"page_pool_reclaim: munmap failed");
2954 total_free -= PAGE_POOL_ARENA_BODIES;
2955 g->page_pool.advised_count -= (int)rb_popcount32(a->advised_mask);
2956 GC_ASSERT(g->page_pool.advised_count >= 0);
2957 g->page_pool.arena_count--;
2958 g->page_pool.arenas_unmapped++;
2959 if (a == g->page_pool.arena_current) {
2962 g->page_pool.arena_current = NULL;
2963 g->page_pool.arena_cursor = NULL;
2964 g->page_pool.arena_end = NULL;
2974heap_page_body_allocate(
struct arena_descriptor **arena_out,
bool *zeroed_out)
2976 struct heap_page_body *page_body = page_pool_acquire(arena_out, zeroed_out);
2978 GC_ASSERT(page_body == NULL || (uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
2984heap_page_resurrect(rb_objspace_t *
objspace)
2987 if (
objspace->empty_pages == NULL) {
2988 GC_ASSERT(
objspace->empty_pages_count == 0);
2991 GC_ASSERT(
objspace->empty_pages_count > 0);
2994 objspace->empty_pages = page->free_next;
2997 page->flags.has_shareable_objects = FALSE;
2998 page->flags.has_shref_objects = FALSE;
3005heap_page_allocate(rb_objspace_t *
objspace)
3009 struct heap_page_body *page_body = heap_page_body_allocate(&arena, &zeroed);
3010 if (page_body == 0) {
3016 heap_page_body_free(page_body, arena);
3021 uintptr_t end = (uintptr_t)page_body + HEAP_PAGE_SIZE;
3024 size_t hi = rb_darray_size(
objspace->heap_pages.sorted);
3028 size_t mid = (lo + hi) / 2;
3029 mid_page = rb_darray_get(
objspace->heap_pages.sorted, mid);
3030 if ((uintptr_t)mid_page->start < start) {
3033 else if ((uintptr_t)mid_page->start > start) {
3037 rb_bug(
"same heap page is allocated: %p at %"PRIuVALUE, (
void *)page_body, (
VALUE)mid);
3041 rb_darray_insert_without_gc(&
objspace->heap_pages.sorted, hi, page);
3043 if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
3044 if (heap_pages_himem < end) heap_pages_himem = end;
3046 page->body = page_body;
3047 page->arena = arena;
3048 page->flags.body_zeroed = zeroed;
3049 page_body->header.page = page;
3052 objspace->heap_pages.allocated_pages++;
3054 global_page_index_insert(page);
3063 GC_ASSERT(!heap->sweeping_page);
3064 GC_ASSERT(heap_page_in_global_empty_pages_pool(
objspace, page));
3068 uintptr_t rem = start % heap->slot_size;
3069 if (rem) start += heap->slot_size - rem;
3071 int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size);
3073 page->start = start;
3074 page->total_slots = slot_count;
3075 page->slot_size = heap->slot_size;
3076 page->slot_size_reciprocal = heap_slot_reciprocal_table[heap - heaps];
3079 memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
3080 memset(&page->age_bits[0], 0,
sizeof(page->age_bits));
3082 asan_unlock_freelist(page);
3083 asan_unpoison_memory_region(page->body, HEAP_PAGE_SIZE,
false);
3085 uintptr_t slots_end = start + (uintptr_t)slot_count * heap->slot_size;
3089 if (!page->flags.body_zeroed) {
3090 memset((
void *)start, 0, slots_end - start);
3092 page->flags.body_zeroed = FALSE;
3096 region->end = slots_end;
3097 region->next = NULL;
3098 page->free_region = region;
3101 for (uintptr_t p = start; p < slots_end; p += heap->slot_size) {
3102 rb_asan_poison_object((
VALUE)p);
3104 asan_lock_freelist(page);
3106 page->free_slots = slot_count;
3108 heap->total_allocated_pages++;
3110 ccan_list_add_tail(&heap->pages, &page->page_node);
3111 heap->total_pages++;
3112 heap->total_slots += page->total_slots;
3116heap_page_allocate_and_initialize(rb_objspace_t *
objspace, rb_heap_t *heap)
3118 gc_report(1,
objspace,
"heap_page_allocate_and_initialize: rb_darray_size(objspace->heap_pages.sorted): %"PRIdSIZE
", "
3119 "allocatable_bytes: %"PRIdSIZE
", heap->total_pages: %"PRIdSIZE
"\n",
3120 rb_darray_size(
objspace->heap_pages.sorted),
objspace->heap_pages.allocatable_bytes, heap->total_pages);
3122 bool allocated =
false;
3125 if (page == NULL &&
objspace->heap_pages.allocatable_bytes > 0) {
3126 page = heap_page_allocate(
objspace);
3129 GC_ASSERT(page != NULL);
3133 heap_add_page(
objspace, heap, page);
3134 heap_add_freepage(heap, page);
3137 size_t page_bytes = (size_t)page->total_slots * page->slot_size;
3138 if (
objspace->heap_pages.allocatable_bytes > page_bytes) {
3139 objspace->heap_pages.allocatable_bytes -= page_bytes;
3142 objspace->heap_pages.allocatable_bytes = 0;
3147 return page != NULL;
3151heap_page_allocate_and_initialize_force(rb_objspace_t *
objspace, rb_heap_t *heap)
3153 size_t prev_allocatable_bytes =
objspace->heap_pages.allocatable_bytes;
3154 objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE;
3155 heap_page_allocate_and_initialize(
objspace, heap);
3156 GC_ASSERT(heap->free_pages != NULL);
3157 objspace->heap_pages.allocatable_bytes = prev_allocatable_bytes;
3161gc_continue(rb_objspace_t *
objspace, rb_heap_t *heap)
3163 unsigned int lock_lev;
3164 bool needs_gc = is_incremental_marking(
objspace) || needs_continue_sweeping(
objspace, heap);
3165 if (!needs_gc)
return;
3167 gc_enter(
objspace, gc_enter_event_continue, &lock_lev);
3170 if (is_incremental_marking(
objspace)) {
3171 if (gc_marks_continue(
objspace, heap)) {
3176 if (needs_continue_sweeping(
objspace, heap)) {
3180 gc_exit(
objspace, gc_enter_event_continue, &lock_lev);
3184heap_prepare(rb_objspace_t *
objspace, rb_heap_t *heap)
3186 GC_ASSERT(heap->free_pages == NULL);
3188 if (heap->total_slots < objspace_heap_init_bytes(
objspace) / heap->slot_size &&
3189 heap->sweeping_page == NULL) {
3190 heap_page_allocate_and_initialize_force(
objspace, heap);
3191 GC_ASSERT(heap->free_pages != NULL);
3198 if (heap->free_pages == NULL) {
3199 heap_page_allocate_and_initialize(
objspace, heap);
3204 if (heap->free_pages == NULL) {
3205 GC_ASSERT(
objspace->empty_pages_count == 0);
3206 GC_ASSERT(
objspace->heap_pages.allocatable_bytes == 0);
3208 if (gc_start(
objspace, GPR_FLAG_NEWOBJ) == FALSE) {
3212 if (
objspace->heap_pages.allocatable_bytes == 0 && !gc_config_full_mark_val) {
3213 heap_allocatable_bytes_expand(
objspace, heap,
3214 heap->freed_slots + heap->empty_slots,
3215 heap->total_slots, heap->slot_size);
3216 GC_ASSERT(
objspace->heap_pages.allocatable_bytes > 0);
3224 if (heap->free_pages == NULL && !heap_page_allocate_and_initialize(
objspace, heap)) {
3225 if (gc_needs_major_flags == GPR_FLAG_NONE) {
3226 rb_bug(
"cannot create a new page after GC");
3229 if (gc_start(
objspace, GPR_FLAG_NEWOBJ) == FALSE) {
3236 if (heap->free_pages == NULL &&
3237 !heap_page_allocate_and_initialize(
objspace, heap)) {
3238 rb_bug(
"cannot create a new page after major GC");
3246 GC_ASSERT(heap->free_pages != NULL);
3250static inline const char*
3251rb_gc_impl_source_location_cstr(
int *ptr)
3272 RBASIC(obj)->flags = flags;
3274#if RBASIC_SHAPE_ID_FIELD
3275 RBASIC(obj)->shape_id = 0;
3282 GC_ASSERT(wb_protected);
3283 gc_page_add_shareable(GET_HEAP_PAGE(obj), obj);
3286#if RGENGC_CHECK_MODE
3287 int lev = RB_GC_VM_LOCK_NO_BARRIER();
3289 check_rvalue_consistency(
objspace, obj);
3291 GC_ASSERT(RVALUE_MARKED(
objspace, obj) == FALSE);
3292 GC_ASSERT(RVALUE_MARKING(
objspace, obj) == FALSE);
3293 GC_ASSERT(RVALUE_OLD_P(
objspace, obj) == FALSE);
3294 GC_ASSERT(RVALUE_WB_UNPROTECTED(
objspace, obj) == FALSE);
3296 if (RVALUE_REMEMBERED(
objspace, obj)) rb_bug(
"newobj: %s is remembered.", rb_obj_info(obj));
3298 RB_GC_VM_UNLOCK_NO_BARRIER(lev);
3301 if (RB_UNLIKELY(wb_protected == FALSE)) {
3302 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
3307 objspace->profile.total_generated_normal_object_count++;
3308#if RGENGC_PROFILE >= 2
3313 objspace->profile.total_generated_shady_object_count++;
3314#if RGENGC_PROFILE >= 2
3321 GET_RVALUE_OVERHEAD(obj)->file = rb_gc_impl_source_location_cstr(&GET_RVALUE_OVERHEAD(obj)->line);
3325 gc_report(5,
objspace,
"newobj: %s\n", rb_obj_info(obj));
3332rb_gc_impl_obj_slot_size(
VALUE obj)
3334 return GET_HEAP_PAGE(obj)->slot_size - RVALUE_OVERHEAD;
3338rb_gc_impl_pinned_p(
void *objspace_ptr,
VALUE obj)
3340 return RVALUE_PINNED((rb_objspace_t *)objspace_ptr, obj);
3344heap_slot_size(
unsigned char pool_id)
3346 GC_ASSERT(pool_id < HEAP_COUNT);
3348 return pool_slot_sizes[pool_id] - RVALUE_OVERHEAD;
3352rb_gc_impl_max_allocation_size(
void)
3354 return heap_slot_size(HEAP_COUNT - 1);
3358rb_gc_impl_size_allocatable_p(
size_t size)
3360 return size <= rb_gc_impl_max_allocation_size();
3364heap_advance_region(rb_heap_t *heap)
3366 struct free_region *region = heap->newobj.alloc_next_region;
3367 if (region == NULL) {
3371 rb_asan_unpoison_object((
VALUE)region,
false);
3373 heap->newobj.alloc_cursor = (uintptr_t)region;
3374 heap->newobj.alloc_cursor_end = region->end;
3375 heap->newobj.alloc_next_region = region->next;
3376 rb_asan_poison_object((
VALUE)region);
3383heap_charge_region(rb_objspace_t *
objspace,
const rb_heap_t *heap,
size_t heap_idx)
3385 objspace->incremental_mark_step_allocated_slots +=
3386 (heap->newobj.alloc_cursor_end - heap->newobj.alloc_cursor) / pool_slot_sizes[heap_idx];
3390heap_alloc_slot(rb_objspace_t *
objspace,
size_t heap_idx)
3392 rb_heap_t *heap = &heaps[heap_idx];
3394 uintptr_t cursor = heap->newobj.alloc_cursor;
3395 if (RB_UNLIKELY(cursor >= heap->newobj.alloc_cursor_end)) {
3397 if (RB_UNLIKELY(is_incremental_marking(
objspace)) ||
3398 heap_advance_region(heap) ==
false) {
3401 cursor = heap->newobj.alloc_cursor;
3405 rb_asan_unpoison_object(obj,
true);
3406 heap->newobj.alloc_cursor = cursor + pool_slot_sizes[heap_idx];
3409 heap->total_allocated_objects++;
3411#if RGENGC_CHECK_MODE
3412 GC_ASSERT(rb_gc_impl_obj_slot_size(obj) == heap_slot_size(heap_idx));
3414 MEMZERO((
char *)obj,
char, heap_slot_size(heap_idx));
3420heap_next_free_page(rb_objspace_t *
objspace, rb_heap_t *heap)
3424 if (heap->free_pages == NULL) {
3428 page = heap->free_pages;
3429 heap->free_pages = page->free_next;
3431 GC_ASSERT(page->free_slots != 0);
3433 asan_unlock_freelist(page);
3439heap_set_alloc_page(rb_objspace_t *
objspace,
size_t heap_idx,
struct heap_page *page)
3441 gc_report(3,
objspace,
"heap_set_alloc_page: Using page %p\n", (
void *)page->body);
3443 rb_heap_t *heap = &heaps[heap_idx];
3445 GC_ASSERT(heap->newobj.alloc_cursor >= heap->newobj.alloc_cursor_end);
3446 GC_ASSERT(heap->newobj.alloc_next_region == NULL);
3447 GC_ASSERT(page->free_slots != 0);
3448 GC_ASSERT(page->free_region != NULL);
3450 heap->newobj.alloc_using_page = page;
3453 rb_asan_unpoison_object((
VALUE)region,
false);
3455 heap->newobj.alloc_cursor = (uintptr_t)region;
3456 heap->newobj.alloc_cursor_end = region->end;
3457 heap->newobj.alloc_next_region = region->next;
3458 rb_asan_poison_object((
VALUE)region);
3460 page->free_slots = 0;
3461 page->free_region = NULL;
3465init_size_to_heap_idx(
void)
3470 static bool initialized =
false;
3471 if (initialized)
return;
3474 for (
size_t i = 0; i <
sizeof(size_to_heap_idx); i++) {
3475 size_t effective = i * 8 + RVALUE_OVERHEAD;
3477 for (idx = 0; idx < HEAP_COUNT; idx++) {
3478 if (effective <= pool_slot_sizes[idx])
break;
3480 size_to_heap_idx[i] = idx;
3485heap_idx_for_size(
size_t size)
3487 size_t compressed = (size + 7) >> 3;
3488 if (compressed <
sizeof(size_to_heap_idx)) {
3489 size_t heap_idx = size_to_heap_idx[compressed];
3490 if (RB_LIKELY(heap_idx < HEAP_COUNT))
return heap_idx;
3493 rb_bug(
"heap_idx_for_size: allocation size too large "
3494 "(size=%"PRIuSIZE
")", size);
3498rb_gc_impl_size_slot_size(
void *objspace_ptr,
size_t size)
3500 return heap_slot_size((
unsigned char)heap_idx_for_size(size));
3504rb_gc_impl_zjit_new_obj_fastpath(
void *objspace_ptr,
size_t alloc_size,
VALUE flags,
VALUE klass,
3508 size_t heap_idx = 0;
3509 size_t slot_size = 0;
3510 for (; heap_idx < HEAP_COUNT; heap_idx++) {
3511 if (alloc_size + RVALUE_OVERHEAD <= pool_slot_sizes[heap_idx]) {
3512 slot_size = pool_slot_sizes[heap_idx];
3516 if (slot_size == 0)
return false;
3519 size_t base = offsetof(rb_objspace_t, heaps)
3520 + heap_idx *
sizeof(rb_heap_t)
3521 + offsetof(rb_heap_t, newobj);
3522#define heaps objspace->heaps
3525 base + offsetof(rb_heap_newobj_t, alloc_cursor),
3526 base + offsetof(rb_heap_newobj_t, alloc_cursor_end),
3528 base - offsetof(rb_heap_t, newobj) + offsetof(rb_heap_t, total_allocated_objects),
3533 memset(fastpath, 0,
sizeof(*fastpath));
3534 fastpath->kind = RB_GC_ZJIT_FASTPATH_DEFAULT;
3535 memcpy(fastpath->data.words, &default_fastpath,
sizeof(default_fastpath));
3544rb_gc_impl_zjit_writebarrier_required_offset(
void)
3546 return (int32_t)offsetof(rb_objspace_t, zjit_during_incremental_marking);
3549NOINLINE(
static VALUE newobj_refill(rb_objspace_t *
objspace,
size_t heap_idx));
3552newobj_refill(rb_objspace_t *
objspace,
size_t heap_idx)
3554 rb_heap_t *heap = &heaps[heap_idx];
3560 if (is_incremental_marking(
objspace)) {
3563 if (
objspace->incremental_mark_step_allocated_slots >= INCREMENTAL_MARK_STEP_ALLOCATIONS) {
3565 objspace->incremental_mark_step_allocated_slots = 0;
3569 if (heap_advance_region(heap)) {
3570 heap_charge_region(
objspace, heap, heap_idx);
3571 obj = heap_alloc_slot(
objspace, heap_idx);
3578 heap_set_alloc_page(
objspace, heap_idx, page);
3579 heap_charge_region(
objspace, heap, heap_idx);
3582 obj = heap_alloc_slot(
objspace, heap_idx);
3585 if (RB_UNLIKELY(obj ==
Qfalse)) {
3592newobj_alloc(rb_objspace_t *
objspace,
size_t heap_idx)
3598 if (RB_UNLIKELY(obj ==
Qfalse)) {
3599 obj = newobj_refill(
objspace, heap_idx);
3605ALWAYS_INLINE(
static VALUE newobj_slowpath(
VALUE klass,
VALUE flags, rb_objspace_t *
objspace,
int wb_protected,
size_t heap_idx));
3608newobj_slowpath(
VALUE klass,
VALUE flags, rb_objspace_t *
objspace,
int wb_protected,
size_t heap_idx)
3613 if (RB_UNLIKELY(during_gc || ruby_gc_stressful)) {
3617 if (rb_memerror_reentered()) {
3620 rb_bug(
"object allocation during garbage collection phase");
3623 if (ruby_gc_stressful) {
3624 if (!garbage_collect(
objspace, GPR_FLAG_NEWOBJ)) {
3630 obj = newobj_alloc(
objspace, heap_idx);
3631 newobj_init(klass, flags, wb_protected,
objspace, obj);
3633 if (RB_UNLIKELY(ruby_gc_stressful)) {
3634 rb_heap_t *heap = &heaps[heap_idx];
3635 heap->newobj.alloc_cursor_end = heap->newobj.alloc_cursor;
3641NOINLINE(
static VALUE newobj_slowpath_wb_protected(
VALUE klass,
VALUE flags,
3642 rb_objspace_t *
objspace,
size_t heap_idx));
3643NOINLINE(
static VALUE newobj_slowpath_wb_unprotected(
VALUE klass,
VALUE flags,
3644 rb_objspace_t *
objspace,
size_t heap_idx));
3647newobj_slowpath_wb_protected(
VALUE klass,
VALUE flags, rb_objspace_t *
objspace,
size_t heap_idx)
3649 return newobj_slowpath(klass, flags,
objspace, TRUE, heap_idx);
3653newobj_slowpath_wb_unprotected(
VALUE klass,
VALUE flags, rb_objspace_t *
objspace,
size_t heap_idx)
3655 return newobj_slowpath(klass, flags,
objspace, FALSE, heap_idx);
3659rb_gc_impl_new_obj(
void *objspace_ptr,
void *cache_ptr,
VALUE klass,
VALUE flags,
bool wb_protected,
size_t alloc_size,
size_t *actual_alloc_size)
3662 rb_objspace_t *
objspace = objspace_ptr;
3668 RB_DEBUG_COUNTER_INC(obj_newobj);
3669 (void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected);
3671 if (RB_UNLIKELY(stress_to_class)) {
3672 if (rb_hash_lookup2(stress_to_class, klass,
Qundef) !=
Qundef) {
3677 size_t heap_idx = heap_idx_for_size(alloc_size);
3678 *actual_alloc_size = heap_slot_size((
unsigned char)heap_idx);
3680 if (!RB_UNLIKELY(during_gc || ruby_gc_stressful) &&
3682 obj = newobj_alloc(
objspace, heap_idx);
3683 newobj_init(klass, flags, wb_protected,
objspace, obj);
3686 RB_DEBUG_COUNTER_INC(obj_newobj_slowpath);
3688 obj = wb_protected ?
3689 newobj_slowpath_wb_protected(klass, flags,
objspace, heap_idx) :
3690 newobj_slowpath_wb_unprotected(klass, flags,
objspace, heap_idx);
3697ptr_in_page_body_p(
const void *ptr,
const void *memb)
3700 uintptr_t p_body = (uintptr_t)page->body;
3702 if ((uintptr_t)ptr >= p_body) {
3703 return (uintptr_t)ptr < (p_body + HEAP_PAGE_SIZE) ? 0 : 1;
3710PUREFUNC(
static inline struct heap_page *heap_page_for_ptr(rb_objspace_t *
objspace, uintptr_t ptr);)
3712heap_page_for_ptr(rb_objspace_t *
objspace, uintptr_t ptr)
3716 if (ptr < (uintptr_t)heap_pages_lomem ||
3717 ptr > (uintptr_t)heap_pages_himem) {
3721 res = bsearch((
void *)ptr, rb_darray_ref(
objspace->heap_pages.sorted, 0),
3723 ptr_in_page_body_p);
3733PUREFUNC(
static inline bool is_pointer_to_heap(rb_objspace_t *
objspace,
const void *ptr);)
3735is_pointer_to_heap(rb_objspace_t *
objspace,
const void *ptr)
3737 register uintptr_t p = (uintptr_t)ptr;
3740 RB_DEBUG_COUNTER_INC(gc_isptr_trial);
3742 if (p < heap_pages_lomem || p > heap_pages_himem)
return FALSE;
3743 RB_DEBUG_COUNTER_INC(gc_isptr_range);
3745 if (p %
sizeof(
VALUE) != 0)
return FALSE;
3746 RB_DEBUG_COUNTER_INC(gc_isptr_align);
3748 page = heap_page_for_ptr(
objspace, (uintptr_t)ptr);
3750 RB_DEBUG_COUNTER_INC(gc_isptr_maybe);
3751 if (heap_page_in_global_empty_pages_pool(
objspace, page)) {
3755 if (p < page->start)
return FALSE;
3756 if (p >= page->start + (page->total_slots * page->slot_size))
return FALSE;
3757 if (!slot_offset_aligned_p(page, p))
return FALSE;
3771rb_gc_impl_live_object_p(
void *objspace_ptr,
const void *ptr)
3773 rb_objspace_t *
objspace = objspace_ptr;
3778 if (!is_pointer_to_heap(
objspace, ptr))
return false;
3782 asan_unpoisoning_object(obj) {
3798#define ZOMBIE_OBJ_KEPT_FLAGS (FL_FINALIZE)
3801rb_gc_impl_make_zombie(
void *objspace_ptr,
VALUE obj,
void (*dfree)(
void *),
void *data)
3803 rb_objspace_t *
objspace = objspace_ptr;
3805 struct RZombie *zombie = RZOMBIE(obj);
3806 zombie->flags =
T_ZOMBIE | (zombie->flags & ZOMBIE_OBJ_KEPT_FLAGS);
3807 zombie->dfree = dfree;
3808 zombie->data = data;
3809 VALUE prev, next = heap_pages_deferred_final;
3811 zombie->next = prev = next;
3813 }
while (next != prev);
3815 struct heap_page *page = GET_HEAP_PAGE(obj);
3816 page->final_slots++;
3817 page->heap->final_slots_count++;
3825 chunk->embed_xfree_bits = 0;
3829tdata_unsafe_free_chunk_alloc(
void)
3837 rbimpl_atomic_ptr_load((
void **)&global_objspace->tdata_unsafe_free_cache,
3838 RBIMPL_ATOMIC_ACQUIRE);
3841 rbimpl_atomic_ptr_cas((
void **)&global_objspace->tdata_unsafe_free_cache,
3843 RBIMPL_ATOMIC_ACQ_REL, RBIMPL_ATOMIC_ACQUIRE);
3845 rbimpl_atomic_size_dec(&global_objspace->tdata_unsafe_free_cache_len,
3846 RBIMPL_ATOMIC_RELAXED);
3847 tdata_unsafe_free_chunk_reset(head);
3857 tdata_unsafe_free_chunk_reset(chunk);
3864gc_tdata_unsafe_free_publish(rb_objspace_t *
objspace)
3867 if (chunk == NULL)
return;
3868 GC_ASSERT(chunk->count > 0);
3869 objspace->tdata_unsafe_free_chunk = NULL;
3872 rbimpl_atomic_ptr_load((
void **)&global_objspace->tdata_unsafe_free_published,
3873 RBIMPL_ATOMIC_RELAXED);
3875 chunk->next = prev = head;
3876 head = rbimpl_atomic_ptr_cas((
void **)&global_objspace->tdata_unsafe_free_published,
3878 RBIMPL_ATOMIC_ACQ_REL, RBIMPL_ATOMIC_ACQUIRE);
3879 }
while (head != prev);
3885gc_defer_thread_unsafe_free(rb_objspace_t *
objspace,
VALUE obj,
bool *trigger)
3887 GC_ASSERT(!((uintptr_t)
RTYPEDDATA(obj)->
type & TYPED_DATA_EMBEDDED));
3890 GC_ASSERT(data != NULL);
3892 rb_gc_obj_free_vm_weak_references(obj);
3894 size_t count = rbimpl_atomic_size_fetch_add(&global_objspace->tdata_deferred_free_count, 1,
3895 RBIMPL_ATOMIC_RELAXED) + 1;
3896 if (!*trigger && count >= TDATA_DEFERRED_FREE_THRESHOLD) {
3901 if (chunk == NULL) {
3902 chunk =
objspace->tdata_unsafe_free_chunk = tdata_unsafe_free_chunk_alloc();
3904 if (
type->flags & RUBY_TYPED_EMBEDDABLE) {
3905 chunk->embed_xfree_bits |= (uint32_t)1 << chunk->count;
3908 entry->dfree =
type->function.dfree;
3910 if (chunk->count == TDATA_UNSAFE_FREE_CHUNK_CAPA) {
3911 gc_tdata_unsafe_free_publish(
objspace);
3918 rb_gc_impl_make_zombie(
objspace, obj, 0, 0);
3924typedef int each_obj_callback(
void *,
void *,
size_t,
void *);
3925typedef int each_page_callback(
struct heap_page *,
void *);
3928 rb_objspace_t *objspace;
3929 bool reenable_incremental;
3934 bool shareable_only;
3939 bool skip_unswept_dead;
3941 each_obj_callback *each_obj_callback;
3942 each_page_callback *each_page_callback;
3946 size_t pages_counts[HEAP_COUNT];
3950objspace_each_objects_ensure(
VALUE arg)
3953 rb_objspace_t *
objspace = data->objspace;
3956 if (data->reenable_incremental) {
3957 objspace->flags.dont_incremental = FALSE;
3960 for (
int i = 0; i < HEAP_COUNT; i++) {
3961 struct heap_page **pages = data->pages[i];
3969objspace_each_objects_try(
VALUE arg)
3972 rb_objspace_t *
objspace = data->objspace;
3975 for (
int i = 0; i < HEAP_COUNT; i++) {
3976 rb_heap_t *heap = &heaps[i];
3977 size_t size = heap->total_pages *
sizeof(
struct heap_page *);
3979 struct heap_page **pages = malloc(size);
3988 size_t pages_count = 0;
3989 ccan_list_for_each(&heap->pages, page, page_node) {
3990 pages[pages_count] = page;
3993 data->pages[i] = pages;
3994 data->pages_counts[i] = pages_count;
3995 GC_ASSERT(pages_count == heap->total_pages);
3998 for (
int i = 0; i < HEAP_COUNT; i++) {
3999 rb_heap_t *heap = &heaps[i];
4000 size_t pages_count = data->pages_counts[i];
4001 struct heap_page **pages = data->pages[i];
4004 for (
size_t i = 0; i < pages_count; i++) {
4007 if (page == NULL)
break;
4011 if (pages[i] != page)
continue;
4013 uintptr_t pstart = (uintptr_t)page->start;
4014 uintptr_t pend = pstart + (page->total_slots * heap->slot_size);
4016 if (data->shareable_only) {
4020 if (page->flags.has_shareable_objects) {
4030 const bool page_unswept = is_lazy_sweeping(
objspace) && page->flags.before_sweep;
4031 int planes = CEILDIV(page->total_slots, BITS_BITLENGTH);
4032 uintptr_t base = pstart;
4034 for (
int j = 0; j < planes && !stop; j++) {
4035 bits_t bits = page->shareable_bits[j];
4036 uintptr_t slot = base;
4038 if ((bits & 1) && data->each_obj_callback &&
4040 (*data->each_obj_callback)((
void *)slot, (
void *)(slot + heap->slot_size),
4041 heap->slot_size, data->data)) {
4045 slot += heap->slot_size;
4048 base += BITS_BITLENGTH * heap->slot_size;
4053 else if (data->skip_unswept_dead &&
4054 is_lazy_sweeping(
objspace) && page->flags.before_sweep) {
4059 for (uintptr_t slot = pstart; slot < pend; slot += heap->slot_size) {
4061 if (data->each_obj_callback &&
4062 (*data->each_obj_callback)((
void *)slot, (
void *)(slot + heap->slot_size),
4063 heap->slot_size, data->data)) {
4071 if (data->each_obj_callback &&
4072 (*data->each_obj_callback)((
void *)pstart, (
void *)pend, heap->slot_size, data->data)) {
4075 if (data->each_page_callback &&
4076 (*data->each_page_callback)(page, data->data)) {
4081 page = ccan_list_next(&heap->pages, page, page_node);
4093 bool reenable_incremental = FALSE;
4095 reenable_incremental = !
objspace->flags.dont_incremental;
4098 objspace->flags.dont_incremental = TRUE;
4109objspace_each_objects(rb_objspace_t *
objspace, each_obj_callback *callback,
void *data,
bool protected)
4113 .each_obj_callback = callback,
4114 .each_page_callback = NULL,
4121rb_gc_impl_each_objects(
void *objspace_ptr, each_obj_callback *callback,
void *data)
4123 objspace_each_objects(objspace_ptr, callback, data, TRUE);
4129rb_gc_impl_each_objects_shareable(
void *objspace_ptr, each_obj_callback *callback,
void *data)
4132 .objspace = objspace_ptr,
4133 .shareable_only =
true,
4134 .each_obj_callback = callback,
4135 .each_page_callback = NULL,
4153rb_gc_impl_each_objects_foreign(
void *objspace_ptr, each_obj_callback *callback,
void *data)
4156 .objspace = objspace_ptr,
4157 .skip_unswept_dead =
true,
4158 .each_obj_callback = callback,
4159 .each_page_callback = NULL,
4165#if GC_CAN_COMPILE_COMPACTION
4167objspace_each_pages(rb_objspace_t *
objspace, each_page_callback *callback,
void *data,
bool protected)
4171 .each_obj_callback = NULL,
4172 .each_page_callback = callback,
4180rb_gc_impl_define_finalizer(
void *objspace_ptr,
VALUE obj,
VALUE block)
4182 rb_objspace_t *
objspace = objspace_ptr;
4191 if (GET_HEAP_OBJSPACE(obj) !=
objspace) {
4192 rb_raise(rb_eRactorIsolationError,
4193 "can not define a finalizer for an object of another Ractor");
4198 unsigned int lev = RB_GC_VM_LOCK();
4200 if (st_lookup(finalizer_table, obj, &data)) {
4201 table = (
VALUE)data;
4204 RB_GC_VM_UNLOCK(lev);
4210 for (i = 0; i <
len; i++) {
4212 if (rb_equal(recv, block)) {
4217 lev = RB_GC_VM_LOCK();
4225 st_add_direct(finalizer_table, obj, table);
4228 RB_GC_VM_UNLOCK(lev);
4234rb_gc_impl_undefine_finalizer(
void *objspace_ptr,
VALUE obj)
4236 rb_objspace_t *
objspace = objspace_ptr;
4241 if (GET_HEAP_OBJSPACE(obj) !=
objspace) {
4242 rb_raise(rb_eRactorIsolationError,
4243 "can not undefine a finalizer of an object of another Ractor");
4246 st_data_t data = obj;
4248 int lev = RB_GC_VM_LOCK();
4249 st_delete(finalizer_table, &data, 0);
4250 RB_GC_VM_UNLOCK(lev);
4256rb_gc_impl_copy_finalizer(
void *objspace_ptr,
VALUE dest,
VALUE obj)
4261 rb_objspace_t *
objspace = objspace_ptr;
4266 if (GET_HEAP_OBJSPACE(obj) !=
objspace)
return;
4268 int lev = RB_GC_VM_LOCK();
4269 if (RB_LIKELY(st_lookup(finalizer_table, obj, &data))) {
4272 st_insert(finalizer_table, dest, table);
4276 rb_bug(
"rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj));
4278 RB_GC_VM_UNLOCK(lev);
4292 if (RZOMBIE(zombie)->dfree) {
4293 RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data);
4296 st_data_t key = (st_data_t)zombie;
4300 if (st_delete(finalizer_table, &key, &table)) {
4304 rb_bug(
"FL_FINALIZE flag is set, but finalizers are not found");
4308 GC_ASSERT(!st_lookup(finalizer_table, key, NULL));
4318 rb_asan_unpoison_object(zombie,
false);
4319 next_zombie = RZOMBIE(zombie)->next;
4320 page = GET_HEAP_PAGE(zombie);
4325 GC_ASSERT(page->heap->final_slots_count > 0);
4326 GC_ASSERT(page->final_slots > 0);
4328 page->heap->final_slots_count--;
4329 page->final_slots--;
4331 RVALUE_AGE_SET_BITMAP(zombie, 0);
4332 heap_page_add_free_region(
objspace, page, zombie);
4333 page->heap->total_freed_objects++;
4336 zombie = next_zombie;
4341finalize_zombies(rb_objspace_t *
objspace)
4350finalize_deferred(rb_objspace_t *
objspace)
4352 rb_gc_set_pending_interrupt();
4354 rb_gc_unset_pending_interrupt();
4358gc_finalize_deferred(
void *dmy)
4363 rb_objspace_t *
objspace = rb_gc_get_objspace();
4371gc_finalize_deferred_register(rb_objspace_t *
objspace)
4376 rb_gc_trigger_finalize_deferred(
objspace,
objspace->finalize_deferred_pjob);
4379static int pop_mark_stack(mark_stack_t *stack,
VALUE *data);
4385gc_abort_incremental_marking(rb_objspace_t *
objspace)
4387 GC_ASSERT(is_incremental_marking(
objspace));
4390 while (pop_mark_stack(&
objspace->mark_stack, &obj));
4394 rb_darray_clear(
objspace->weak_references);
4396 gc_set_incremental_marking(
objspace, FALSE);
4397 gc_mode_set(
objspace, gc_mode_none);
4401gc_abort(
void *objspace_ptr)
4403 rb_objspace_t *
objspace = objspace_ptr;
4405 if (is_incremental_marking(
objspace)) {
4406 gc_abort_incremental_marking(
objspace);
4411 for (
int i = 0; i < HEAP_COUNT; i++) {
4412 rb_heap_t *heap = &heaps[i];
4414 heap->sweeping_page = NULL;
4417 ccan_list_for_each(&heap->pages, page, page_node) {
4418 page->flags.before_sweep =
false;
4423 for (
int i = 0; i < HEAP_COUNT; i++) {
4424 rb_heap_t *heap = &heaps[i];
4425 gc_bitmaps_clear(
objspace, heap,
false);
4428 gc_mode_set(
objspace, gc_mode_none);
4432# ifdef RB_THREAD_LOCAL_SPECIFIER
4433# define GC_FREEING_OBJ_TLS RB_THREAD_LOCAL_SPECIFIER
4435# define GC_FREEING_OBJ_TLS
4438static GC_FREEING_OBJ_TLS
VALUE gc_freeing_obj;
4446 VALUE prev = gc_freeing_obj;
4447 gc_freeing_obj = obj;
4449 bool freed = rb_gc_obj_free(
objspace, obj);
4451 gc_freeing_obj = prev;
4456gc_freeing_obj_info(
void)
4460 static char buf[128];
4462 if (!gc_freeing_obj)
return NULL;
4464 snprintf(buf,
sizeof(buf),
"%p %s", (
void *)gc_freeing_obj, rb_obj_info(gc_freeing_obj));
4468# define gc_obj_free(objspace, obj) rb_gc_obj_free((objspace), (obj))
4469# define gc_freeing_obj_info() NULL
4473rb_gc_impl_shutdown_free_objects(
void *objspace_ptr)
4475 rb_objspace_t *
objspace = objspace_ptr;
4477 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
4479 short stride = page->slot_size;
4481 uintptr_t p = (uintptr_t)page->start;
4482 uintptr_t pend = p + page->total_slots * stride;
4483 for (; p < pend; p += stride) {
4485 asan_unpoisoning_object(vp) {
4487 rb_gc_obj_free_vm_weak_references(vp);
4498rb_gc_impl_shutdown_call_finalizer_i(st_data_t key, st_data_t val, st_data_t _data)
4509 asan_unpoisoning_object(obj) {
4517rb_gc_impl_shutdown_call_finalizer(
void *objspace_ptr)
4519 rb_objspace_t *
objspace = objspace_ptr;
4521#if RGENGC_CHECK_MODE >= 2
4522 gc_verify_internal_consistency(
objspace);
4526 objspace->flags.dont_incremental = 1;
4535 while (finalizer_table->num_entries) {
4536 st_foreach(finalizer_table, rb_gc_impl_shutdown_call_finalizer_i, 0);
4541 GC_ASSERT(heap_pages_deferred_final == 0);
4545 gc_tdata_unsafe_drain_objspaces(&
objspace, 1);
4554 unsigned int lock_lev;
4555 gc_enter(
objspace, gc_enter_event_finalizer, &lock_lev);
4558 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
4560 short stride = page->slot_size;
4562 uintptr_t p = (uintptr_t)page->start;
4563 uintptr_t pend = p + page->total_slots * stride;
4564 for (; p < pend; p += stride) {
4566 asan_unpoisoning_object(vp) {
4567 if (rb_gc_shutdown_call_finalizer_p(vp)) {
4568 rb_gc_obj_free_vm_weak_references(vp);
4577 gc_exit(
objspace, gc_enter_event_finalizer, &lock_lev);
4581 st_free_table(finalizer_table);
4582 finalizer_table = 0;
4587rb_gc_impl_each_object(
void *objspace_ptr,
void (*func)(
VALUE obj,
void *data),
void *data)
4589 rb_objspace_t *
objspace = objspace_ptr;
4591 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
4593 short stride = page->slot_size;
4595 uintptr_t p = (uintptr_t)page->start;
4596 uintptr_t pend = p + page->total_slots * stride;
4597 for (; p < pend; p += stride) {
4600 asan_unpoisoning_object(obj) {
4614objspace_available_slots(rb_objspace_t *
objspace)
4616 size_t total_slots = 0;
4617 for (
int i = 0; i < HEAP_COUNT; i++) {
4618 rb_heap_t *heap = &heaps[i];
4619 total_slots += heap->total_slots;
4625objspace_live_slots(rb_objspace_t *
objspace)
4631objspace_free_slots(rb_objspace_t *
objspace)
4637gc_setup_mark_bits(
struct heap_page *page)
4640 memcpy(&page->mark_bits[0], &page->uncollectible_bits[0], HEAP_PAGE_BITMAP_SIZE);
4643static int gc_is_moveable_obj(rb_objspace_t *
objspace,
VALUE obj);
4647enum {HEAP_PAGE_LOCK = PAGE_NOACCESS, HEAP_PAGE_UNLOCK = PAGE_READWRITE};
4653 return VirtualProtect(body, HEAP_PAGE_SIZE, protect, &old_protect) != 0;
4655#elif defined(__wasi__)
4657enum {HEAP_PAGE_LOCK, HEAP_PAGE_UNLOCK};
4658#define protect_page_body(body, protect) 1
4660enum {HEAP_PAGE_LOCK = PROT_NONE, HEAP_PAGE_UNLOCK = PROT_READ | PROT_WRITE};
4661#define protect_page_body(body, protect) !mprotect((body), HEAP_PAGE_SIZE, (protect))
4667 if (!protect_page_body(body, HEAP_PAGE_LOCK)) {
4668 rb_bug(
"Couldn't protect page %p, errno: %s", (
void *)body, strerror(
errno));
4671 gc_report(5,
objspace,
"Protecting page in move %p\n", (
void *)body);
4678 if (!protect_page_body(body, HEAP_PAGE_UNLOCK)) {
4679 rb_bug(
"Couldn't unprotect page %p, errno: %s", (
void *)body, strerror(
errno));
4682 gc_report(5,
objspace,
"Unprotecting page in move %p\n", (
void *)body);
4687heap_page_alloc_slot_from_region(
struct heap_page *free_page)
4689 asan_unlock_freelist(free_page);
4690 struct free_region *region = free_page->free_region;
4691 asan_lock_freelist(free_page);
4693 if (region == NULL) {
4697 rb_asan_unpoison_object((
VALUE)region,
false);
4699 uintptr_t dest = (uintptr_t)region;
4700 uintptr_t region_end = region->end;
4703 uintptr_t new_start = dest + free_page->slot_size;
4705 asan_unlock_freelist(free_page);
4706 if (new_start < region_end) {
4708 rb_asan_unpoison_object(next_start,
false);
4710 new_region->flags = 0;
4711 new_region->end = region_end;
4712 new_region->next = next;
4713 rb_asan_poison_object(next_start);
4714 free_page->free_region = new_region;
4717 free_page->free_region = next;
4719 asan_lock_freelist(free_page);
4727 GC_ASSERT(gc_is_moveable_obj(
objspace, src));
4729 struct heap_page *src_page = GET_HEAP_PAGE(src);
4737 GC_ASSERT(RVALUE_MARKED(
objspace, src));
4739 uintptr_t dest_slot = heap_page_alloc_slot_from_region(free_page);
4740 if (dest_slot == 0) {
4747 if (src_page->slot_size > free_page->slot_size) {
4750 else if (free_page->slot_size > src_page->slot_size) {
4754 objspace->rcompactor.total_moved++;
4756 gc_move(
objspace, src, dest, src_page, free_page);
4758 free_page->free_slots--;
4764gc_unprotect_pages(rb_objspace_t *
objspace, rb_heap_t *heap)
4766 struct heap_page *cursor = heap->compact_cursor;
4769 unlock_page_body(
objspace, cursor->body);
4770 cursor = ccan_list_next(&heap->pages, cursor, page_node);
4774static void gc_update_references(rb_objspace_t *
objspace);
4775static void gc_update_references_heap(rb_objspace_t *
objspace);
4776static void gc_update_references_global(rb_objspace_t *
objspace);
4777#if GC_CAN_COMPILE_COMPACTION
4778static void invalidate_moved_page(rb_objspace_t *
objspace,
struct heap_page *page);
4781#if defined(__MINGW32__) || defined(_WIN32)
4782# define GC_COMPACTION_SUPPORTED 1
4786# define GC_COMPACTION_SUPPORTED (GC_CAN_COMPILE_COMPACTION && HEAP_PAGE_ALLOC_USE_MMAP)
4789#if GC_CAN_COMPILE_COMPACTION
4791read_barrier_handler(uintptr_t address)
4793 rb_objspace_t *
objspace = (rb_objspace_t *)rb_gc_get_objspace();
4799 if (page_body == NULL) {
4800 rb_bug(
"read_barrier_handler: segmentation fault at %p", (
void *)address);
4803 int lev = RB_GC_VM_LOCK();
4805 unlock_page_body(
objspace, page_body);
4807 objspace->profile.read_barrier_faults++;
4809 invalidate_moved_page(
objspace, GET_HEAP_PAGE(address));
4811 RB_GC_VM_UNLOCK(lev);
4815#if !GC_CAN_COMPILE_COMPACTION
4817uninstall_handlers(
void)
4823install_handlers(
void)
4827#elif defined(_WIN32)
4828static LPTOP_LEVEL_EXCEPTION_FILTER old_handler;
4829typedef void (*signal_handler)(int);
4830static signal_handler old_sigsegv_handler;
4833read_barrier_signal(EXCEPTION_POINTERS *info)
4836 if (info->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
4841 read_barrier_handler((uintptr_t)info->ExceptionRecord->ExceptionInformation[1]);
4842 return EXCEPTION_CONTINUE_EXECUTION;
4845 return EXCEPTION_CONTINUE_SEARCH;
4850uninstall_handlers(
void)
4852 signal(SIGSEGV, old_sigsegv_handler);
4853 SetUnhandledExceptionFilter(old_handler);
4857install_handlers(
void)
4860 old_sigsegv_handler = signal(SIGSEGV, NULL);
4863 old_handler = SetUnhandledExceptionFilter(read_barrier_signal);
4866static struct sigaction old_sigbus_handler;
4867static struct sigaction old_sigsegv_handler;
4869#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
4870static exception_mask_t old_exception_masks[32];
4871static mach_port_t old_exception_ports[32];
4872static exception_behavior_t old_exception_behaviors[32];
4873static thread_state_flavor_t old_exception_flavors[32];
4874static mach_msg_type_number_t old_exception_count;
4877disable_mach_bad_access_exc(
void)
4879 old_exception_count =
sizeof(old_exception_masks) /
sizeof(old_exception_masks[0]);
4880 task_swap_exception_ports(
4881 mach_task_self(), EXC_MASK_BAD_ACCESS,
4882 MACH_PORT_NULL, EXCEPTION_DEFAULT, 0,
4883 old_exception_masks, &old_exception_count,
4884 old_exception_ports, old_exception_behaviors, old_exception_flavors
4889restore_mach_bad_access_exc(
void)
4891 for (mach_msg_type_number_t i = 0; i < old_exception_count; i++) {
4892 task_set_exception_ports(
4894 old_exception_masks[i], old_exception_ports[i],
4895 old_exception_behaviors[i], old_exception_flavors[i]
4901#if defined(HAVE_PTHREAD_SIGMASK)
4902# define gc_sigmask pthread_sigmask
4904# define gc_sigmask sigprocmask
4908read_barrier_signal(
int sig, siginfo_t *info,
void *data)
4911 struct sigaction prev_sigbus, prev_sigsegv;
4912 sigaction(SIGBUS, &old_sigbus_handler, &prev_sigbus);
4913 sigaction(SIGSEGV, &old_sigsegv_handler, &prev_sigsegv);
4916 sigset_t set, prev_set;
4918 sigaddset(&set, SIGBUS);
4919 sigaddset(&set, SIGSEGV);
4920 gc_sigmask(SIG_UNBLOCK, &set, &prev_set);
4921#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
4922 disable_mach_bad_access_exc();
4925 read_barrier_handler((uintptr_t)info->si_addr);
4928#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
4929 restore_mach_bad_access_exc();
4931 sigaction(SIGBUS, &prev_sigbus, NULL);
4932 sigaction(SIGSEGV, &prev_sigsegv, NULL);
4933 gc_sigmask(SIG_SETMASK, &prev_set, NULL);
4937uninstall_handlers(
void)
4939#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
4940 restore_mach_bad_access_exc();
4942 sigaction(SIGBUS, &old_sigbus_handler, NULL);
4943 sigaction(SIGSEGV, &old_sigsegv_handler, NULL);
4947install_handlers(
void)
4949 struct sigaction action;
4950 memset(&action, 0,
sizeof(
struct sigaction));
4951 sigemptyset(&action.sa_mask);
4952 action.sa_sigaction = read_barrier_signal;
4953 action.sa_flags = SA_SIGINFO | SA_ONSTACK;
4955 sigaction(SIGBUS, &action, &old_sigbus_handler);
4956 sigaction(SIGSEGV, &action, &old_sigsegv_handler);
4957#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
4958 disable_mach_bad_access_exc();
4964gc_compact_finish(rb_objspace_t *
objspace)
4966 for (
int i = 0; i < HEAP_COUNT; i++) {
4967 rb_heap_t *heap = &heaps[i];
4968 gc_unprotect_pages(
objspace, heap);
4971 if (!global_objspace->global_gc.compacting) uninstall_handlers();
4973 if (global_objspace->global_gc.compacting) {
4978 gc_update_references_heap(
objspace);
4985 for (
int i = 0; i < HEAP_COUNT; i++) {
4986 rb_heap_t *heap = &heaps[i];
4987 heap->compact_cursor = NULL;
4988 heap->free_pages = NULL;
4989 heap->compact_cursor_index = 0;
4994 record->moved_objects =
objspace->rcompactor.total_moved - record->moved_objects;
4996 if (!global_objspace->global_gc.compacting)
objspace->flags.during_compacting = FALSE;
5004# define GC_SWEEP_DOWNWARDS 1
5006# define GC_SWEEP_DOWNWARDS 0
5016 const bool check_pinned_free;
5019 const bool defer_thread_unsafe_local_sweep;
5020 bool trigger_thread_unsafe_sweep_postponed_job;
5022 struct free_region *free_region;
5023#if GC_SWEEP_DOWNWARDS
5025 uintptr_t region_start;
5026 uintptr_t region_end;
5027 struct free_region *free_region_tail;
5036gc_obj_defer_local_free_p(rb_objspace_t *
objspace,
VALUE obj)
5041 if (!rb_gc_data_type_deferred_free_p(
type))
return false;
5043 if (RTYPEDDATA_GET_DATA(obj) == NULL)
return false;
5046 if (
objspace->flags.during_postmortem) {
5058static void gc_tdata_deferred_free_job(
void *unused);
5059static void gc_tdata_deferred_free_pjob_ensure(
void);
5060static unsigned int gc_during_gc_get(
const rb_objspace_t *
objspace);
5061static void gc_during_gc_set(rb_objspace_t *
objspace,
unsigned int v);
5062static void gc_global_snapshot_objspaces(
void);
5065gc_tdata_deferred_free_pjob_ensure(
void)
5067 if (global_objspace->tdata_deferred_free_pjob == POSTPONED_JOB_HANDLE_INVALID) {
5068 global_objspace->tdata_deferred_free_pjob =
5070 if (global_objspace->tdata_deferred_free_pjob == POSTPONED_JOB_HANDLE_INVALID) {
5071 rb_bug(
"Could not preregister postponed job for deferred T_DATA free");
5080gc_tdata_deferred_free_trigger(rb_objspace_t *
objspace)
5082 if (
objspace->flags.during_postmortem) {
5083 rb_gc_trigger_postponed_job_on_main(global_objspace->tdata_deferred_free_pjob);
5093 entry->dfree(entry->data);
5102 if (global_objspace->tdata_unsafe_free_cache_len >= TDATA_UNSAFE_FREE_CACHE_MAX) {
5106 tdata_unsafe_free_chunk_reset(chunk);
5107 chunk->next = global_objspace->tdata_unsafe_free_cache;
5108 rbimpl_atomic_ptr_store((
volatile void **)&global_objspace->tdata_unsafe_free_cache, chunk,
5109 RBIMPL_ATOMIC_RELEASE);
5110 global_objspace->tdata_unsafe_free_cache_len++;
5116 for (
unsigned int i = 0; i < chunk->count; i++) {
5117 gc_tdata_unsafe_free_entry(&chunk->entries[i],
5118 (chunk->embed_xfree_bits >> i) & 1);
5120 tdata_unsafe_free_chunk_recycle(chunk);
5129gc_tdata_unsafe_drain_objspaces(rb_objspace_t **objspaces,
size_t n)
5132 rbimpl_atomic_ptr_exchange((
void **)&global_objspace->tdata_unsafe_free_published, NULL,
5133 RBIMPL_ATOMIC_ACQ_REL);
5136 gc_tdata_unsafe_drain_chunk(chunk);
5140 for (
size_t i = 0; i < n; i++) {
5141 rb_objspace_t *os = objspaces[i];
5144 os->tdata_unsafe_free_chunk = NULL;
5145 gc_tdata_unsafe_drain_chunk(partial);
5149 rbimpl_atomic_size_exchange(&global_objspace->tdata_deferred_free_count, 0,
5150 RBIMPL_ATOMIC_RELAXED);
5155gc_tdata_unsafe_drain(
void)
5157 unsigned int lev = RB_GC_VM_LOCK();
5159 if (tdata_deferred_free_count_load() == 0) {
5160 RB_GC_VM_UNLOCK(lev);
5166 gc_global_snapshot_objspaces();
5171 rb_objspace_t *
objspace = rb_gc_get_objspace();
5172 unsigned int saved_during_gc = gc_during_gc_get(
objspace);
5174 rb_gc_initialize_vm_context(&
objspace->vm_context);
5177 gc_tdata_unsafe_drain_objspaces(global_objspace->global_gc.objspaces,
5178 global_objspace->global_gc.n_objspaces);
5180 gc_during_gc_set(
objspace, saved_during_gc);
5183 RB_GC_VM_UNLOCK(lev);
5187gc_tdata_deferred_free_job(
void *unused)
5191 size_t count = tdata_deferred_free_count_load();
5192 if (count == 0)
return;
5193 if (count < TDATA_DEFERRED_FREE_THRESHOLD && !rb_gc_single_objspace_p())
return;
5195 gc_tdata_unsafe_drain();
5198#if GC_SWEEP_DOWNWARDS
5202 if (!ctx->region_start)
return;
5205 rb_asan_unpoison_object((
VALUE)region,
false);
5206 region->end = ctx->region_end;
5207 region->next = NULL;
5208 rb_asan_poison_object((
VALUE)region);
5213 if (ctx->free_region_tail) {
5214 rb_asan_unpoison_object((
VALUE)ctx->free_region_tail,
false);
5215 ctx->free_region_tail->next = region;
5216 rb_asan_poison_object((
VALUE)ctx->free_region_tail);
5219 ctx->free_region = region;
5221 ctx->free_region_tail = region;
5228 rb_asan_unpoison_object(p,
false);
5237#if GC_SWEEP_DOWNWARDS
5238 if (RB_LIKELY(p + slot_size == ctx->region_start)) {
5239 ctx->region_start = p;
5242 gc_sweep_close_free_region(ctx);
5243 ctx->region_start = p;
5244 ctx->region_end = p + slot_size;
5247 struct free_region *existing_region = ctx->free_region;
5248 if (existing_region) rb_asan_unpoison_object((
VALUE)existing_region,
false);
5250 if (RB_LIKELY(existing_region && p == existing_region->end)) {
5251 existing_region->end = p + slot_size;
5261 if (existing_region) rb_asan_poison_object((
VALUE)existing_region);
5263 rb_asan_poison_object(p);
5269 struct heap_page *sweep_page = ctx->page;
5270 short slot_size = sweep_page->slot_size;
5274 GC_ASSERT(vp %
sizeof(
VALUE) == 0);
5276#if GC_SWEEP_DOWNWARDS
5278 if (bitset >> (BITS_BITLENGTH - 1)) {
5279 rb_asan_unpoison_object(vp,
false);
5281 rb_asan_unpoison_object(vp,
false);
5286 if (
objspace->flags.during_compacting) {
5292 rb_bug(
"T_MOVED shouldn't be seen until compaction is finished");
5294 gc_report(3,
objspace,
"page_sweep: %s is freed\n", rb_obj_info(vp));
5296 gc_sweep_register_free_slot(
objspace, sweep_page, ctx, p, slot_size);
5303 gc_sweep_register_free_slot(
objspace, sweep_page, ctx, p, slot_size);
5307#if RGENGC_CHECK_MODE
5311 if (ctx->check_pinned_free &&
5312 (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(vp), vp) ||
5313 MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(vp), vp))) {
5314 rb_bug(
"page_sweep: freeing pinned slot %s (shareable=%d shref=%d single_now=%d)",
5316 (
int)!!MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(vp), vp),
5317 (
int)!!MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(vp), vp),
5318 (
int)rb_gc_single_objspace_p());
5321#if RGENGC_CHECK_MODE
5323 if (RVALUE_OLD_P(
objspace, vp)) rb_bug(
"page_sweep: %p - old while minor GC.", (
void *)p);
5324 if (RVALUE_REMEMBERED(
objspace, vp)) rb_bug(
"page_sweep: %p - remembered.", (
void *)p);
5328#if RGENGC_CHECK_MODE
5329#define CHECK(x) if (x(objspace, vp) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", rb_obj_info(vp))
5330 CHECK(RVALUE_WB_UNPROTECTED);
5331 CHECK(RVALUE_MARKED);
5332 CHECK(RVALUE_MARKING);
5333 CHECK(RVALUE_UNCOLLECTIBLE);
5337 if (!rb_gc_obj_needs_cleanup_p(vp)) {
5338 (void)VALGRIND_MAKE_MEM_UNDEFINED((
void*)p, slot_size);
5339 gc_sweep_register_free_slot(
objspace, sweep_page, ctx, p, slot_size);
5340 gc_report(3,
objspace,
"page_sweep: %s (fast path) is freed\n", rb_obj_info(vp));
5344 gc_report(2,
objspace,
"page_sweep: free %p\n", (
void *)p);
5346 if (RB_UNLIKELY(ctx->defer_thread_unsafe_local_sweep && gc_obj_defer_local_free_p(
objspace, vp))) {
5350 if (gc_defer_thread_unsafe_free(
objspace, vp,
5351 &ctx->trigger_thread_unsafe_sweep_postponed_job)) {
5352 (void)VALGRIND_MAKE_MEM_UNDEFINED((
void*)p, slot_size);
5353 gc_sweep_register_free_slot(
objspace, sweep_page, ctx, p, slot_size);
5361 rb_gc_obj_free_vm_weak_references(vp);
5363 (void)VALGRIND_MAKE_MEM_UNDEFINED((
void*)p, slot_size);
5364 gc_sweep_register_free_slot(
objspace, sweep_page, ctx, p, slot_size);
5365 gc_report(3,
objspace,
"page_sweep: %s is freed\n", rb_obj_info(vp));
5375#if GC_SWEEP_DOWNWARDS
5388 struct heap_page *sweep_page = ctx->page;
5389 GC_ASSERT(sweep_page->heap == heap);
5392 bits_t *bits, bitset;
5394 gc_report(2,
objspace,
"page_sweep: start.\n");
5396#if RGENGC_CHECK_MODE
5397 if (!
objspace->flags.immediate_sweep) {
5398 GC_ASSERT(sweep_page->flags.before_sweep == TRUE);
5401 sweep_page->flags.before_sweep = FALSE;
5402 sweep_page->free_slots = 0;
5404 asan_unlock_freelist(sweep_page);
5405 sweep_page->free_region = NULL;
5406 asan_lock_freelist(sweep_page);
5407 ctx->free_region = NULL;
5408#if GC_SWEEP_DOWNWARDS
5409 ctx->region_start = 0;
5410 ctx->free_region_tail = NULL;
5413 p = (uintptr_t)sweep_page->start;
5414 bits = sweep_page->mark_bits;
5415 short slot_size = sweep_page->slot_size;
5416 int total_slots = sweep_page->total_slots;
5417 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
5419 int out_of_range_bits = total_slots % BITS_BITLENGTH;
5420 if (out_of_range_bits != 0) {
5421 bits[bitmap_plane_count - 1] |= ~(((bits_t)1 << out_of_range_bits) - 1);
5426 bits_t *wb_unprotected_bits = sweep_page->wb_unprotected_bits;
5427 bits_t *age_bits = sweep_page->age_bits;
5428 for (
int i = 0; i < bitmap_plane_count; i++) {
5429 bits_t unmarked = ~bits[i];
5430 wb_unprotected_bits[i] &= ~unmarked;
5431 age_bits[i * 2] &= ~unmarked;
5432 age_bits[i * 2 + 1] &= ~unmarked;
5436#if GC_SWEEP_DOWNWARDS
5437 p += (uintptr_t)(bitmap_plane_count * BITS_BITLENGTH - 1) * slot_size;
5438 for (
int i = bitmap_plane_count - 1; i >= 0; i--) {
5441 gc_sweep_plane(
objspace, heap, p, bitset, ctx);
5443 p -= BITS_BITLENGTH * slot_size;
5445 gc_sweep_close_free_region(ctx);
5447 for (
int i = 0; i < bitmap_plane_count; i++) {
5450 gc_sweep_plane(
objspace, heap, p, bitset, ctx);
5452 p += BITS_BITLENGTH * slot_size;
5460 if (sweep_page->flags.has_shareable_objects || sweep_page->flags.has_shref_objects) {
5461 bits_t *shareable_bits = sweep_page->shareable_bits;
5462 bits_t *shref_bits = sweep_page->shref_bits;
5463 bits_t sh = 0, sr = 0;
5464 for (
int i = 0; i < bitmap_plane_count; i++) {
5465 shareable_bits[i] &= bits[i];
5466 shref_bits[i] &= bits[i];
5467 sh |= shareable_bits[i];
5468 sr |= shref_bits[i];
5470 if (!sh) sweep_page->flags.has_shareable_objects = FALSE;
5471 if (!sr) sweep_page->flags.has_shref_objects = FALSE;
5474 asan_unlock_freelist(sweep_page);
5475 sweep_page->free_region = ctx->free_region;
5476 asan_lock_freelist(sweep_page);
5478 if (!heap->compact_cursor) {
5479 gc_setup_mark_bits(sweep_page);
5482#if GC_PROFILE_MORE_DETAIL
5485 record->removing_objects += ctx->final_slots + ctx->freed_slots;
5486 record->empty_objects += ctx->empty_slots;
5489 if (0) fprintf(stderr,
"gc_sweep_page(%"PRIdSIZE
"): total_slots: %d, freed_slots: %d, empty_slots: %d, final_slots: %d\n",
5491 sweep_page->total_slots,
5492 ctx->freed_slots, ctx->empty_slots, ctx->final_slots);
5494 sweep_page->free_slots += ctx->freed_slots + ctx->empty_slots;
5495 sweep_page->heap->total_freed_objects += ctx->freed_slots;
5497 if (heap_pages_deferred_final && !finalizing) {
5498 gc_finalize_deferred_register(
objspace);
5501#if RGENGC_CHECK_MODE
5502 int region_slots = 0;
5503 asan_unlock_freelist(sweep_page);
5504 struct free_region *region = sweep_page->free_region;
5506 rb_asan_unpoison_object((
VALUE)region,
false);
5508 uintptr_t region_start = (uintptr_t)region;
5509 uintptr_t region_end = region->end;
5511 rb_asan_poison_object((
VALUE)region);
5513 GC_ASSERT(region_end > region_start);
5514 GC_ASSERT((region_end - region_start) % slot_size == 0);
5515 region_slots += (int)((region_end - region_start) / slot_size);
5519 asan_lock_freelist(sweep_page);
5520 if (region_slots != sweep_page->free_slots) {
5521 rb_bug(
"inconsistent free region slots: expected %d but was %d", sweep_page->free_slots, region_slots);
5525 gc_report(2,
objspace,
"page_sweep: end.\n");
5529gc_mode_name(
enum gc_mode mode)
5532 case gc_mode_none:
return "none";
5533 case gc_mode_marking:
return "marking";
5534 case gc_mode_sweeping:
return "sweeping";
5535 case gc_mode_compacting:
return "compacting";
5536 default: rb_bug(
"gc_mode_name: unknown mode: %d", (
int)mode);
5541gc_mode_transition(rb_objspace_t *
objspace,
enum gc_mode mode)
5543#if RGENGC_CHECK_MODE
5544 enum gc_mode prev_mode = gc_mode(
objspace);
5545 switch (prev_mode) {
5550 GC_ASSERT(mode == gc_mode_marking ||
5551 (
objspace->flags.during_global_gc && mode == gc_mode_sweeping));
5553 case gc_mode_marking: GC_ASSERT(mode == gc_mode_sweeping);
break;
5554 case gc_mode_sweeping: GC_ASSERT(mode == gc_mode_none || mode == gc_mode_compacting);
break;
5555 case gc_mode_compacting: GC_ASSERT(mode == gc_mode_none);
break;
5558 if (0) fprintf(stderr,
"gc_mode_transition: %s->%s\n", gc_mode_name(gc_mode(
objspace)), gc_mode_name(mode));
5563heap_page_flush_alloc_regions(
struct heap_page *page, rb_heap_t *heap)
5565 struct free_region *chain = heap->newobj.alloc_next_region;
5567 if (heap->newobj.alloc_cursor < heap->newobj.alloc_cursor_end) {
5568 VALUE start = (
VALUE)heap->newobj.alloc_cursor;
5569 rb_asan_unpoison_object(start,
false);
5572 remnant->end = heap->newobj.alloc_cursor_end;
5573 remnant->next = chain;
5574 rb_asan_poison_object(start);
5579 asan_unlock_freelist(page);
5580 if (page->free_region) {
5582 rb_asan_unpoison_object((
VALUE)p,
false);
5586 rb_asan_poison_object((
VALUE)prev);
5587 rb_asan_unpoison_object((
VALUE)p,
false);
5590 rb_asan_poison_object((
VALUE)p);
5593 page->free_region = chain;
5595 asan_lock_freelist(page);
5600gc_sweep_start_heap(rb_objspace_t *
objspace, rb_heap_t *heap)
5602 heap->sweeping_page = ccan_list_top(&heap->pages,
struct heap_page, page_node);
5603 if (heap->sweeping_page) {
5606 heap->free_pages = NULL;
5607 heap->pooled_pages = NULL;
5608 if (!
objspace->flags.immediate_sweep) {
5611 ccan_list_for_each(&heap->pages, page, page_node) {
5612 page->flags.before_sweep = TRUE;
5617#if GC_CAN_COMPILE_COMPACTION
5618static void gc_sort_heap_by_compare_func(rb_objspace_t *
objspace, gc_compact_compare_func compare_func);
5619static int compare_pinned_slots(
const void *left,
const void *right,
void *d);
5625heap_alloc_state_clear(rb_objspace_t *
objspace)
5627 objspace->incremental_mark_step_allocated_slots = 0;
5629 for (
size_t heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) {
5630 rb_heap_t *heap = &heaps[heap_idx];
5632 struct heap_page *page = heap->newobj.alloc_using_page;
5633 RUBY_DEBUG_LOG(
"heap alloc_using_page:%p cursor:%p", (
void *)page, (
void *)heap->newobj.alloc_cursor);
5636 heap_page_flush_alloc_regions(page, heap);
5639 heap->newobj.alloc_using_page = NULL;
5640 heap->newobj.alloc_cursor = 0;
5641 heap->newobj.alloc_cursor_end = 0;
5642 heap->newobj.alloc_next_region = NULL;
5649 bits_t *bits = page->mark_bits;
5650 uintptr_t p = (uintptr_t)page->start;
5651 short slot_size = page->slot_size;
5652 int total_slots = page->total_slots;
5653 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
5655 int out_of_range_bits = total_slots % BITS_BITLENGTH;
5656 bits_t last_plane_mask = (out_of_range_bits != 0)
5657 ? ~(((bits_t)1 << out_of_range_bits) - 1)
5660 for (
int j = 0; j < bitmap_plane_count; j++) {
5661 bits_t bitset = ~bits[j];
5662 if (j == bitmap_plane_count - 1) {
5663 bitset &= ~last_plane_mask;
5670 asan_unpoisoning_object(vp) {
5685 p += BITS_BITLENGTH * slot_size;
5690gc_sweep_freeobj_hooks(rb_objspace_t *
objspace)
5692 for (
int i = 0; i < HEAP_COUNT; i++) {
5693 rb_heap_t *heap = &heaps[i];
5696 ccan_list_for_each(&heap->pages, page, page_node) {
5697 gc_sweep_freeobj_hooks_page(
objspace, page);
5703gc_sweep_start(rb_objspace_t *
objspace)
5705 gc_mode_transition(
objspace, gc_mode_sweeping);
5712 GC_ASSERT(
objspace == global_objspace->main_objspace);
5716#if GC_CAN_COMPILE_COMPACTION
5717 if (
objspace->flags.during_compacting) {
5718 gc_sort_heap_by_compare_func(
5720 objspace->rcompactor.compare_func ?
objspace->rcompactor.compare_func : compare_pinned_slots
5725 for (
int i = 0; i < HEAP_COUNT; i++) {
5726 rb_heap_t *heap = &heaps[i];
5727 gc_sweep_start_heap(
objspace, heap);
5730 if (heap->sweeping_page == NULL) {
5731 GC_ASSERT(heap->total_pages == 0);
5732 GC_ASSERT(heap->total_slots == 0);
5733 gc_sweep_finish_heap(
objspace, heap);
5741gc_sweep_finish_heap(rb_objspace_t *
objspace, rb_heap_t *heap)
5744 if (
objspace->flags.ownerless)
return;
5746 size_t total_slots = heap->total_slots;
5747 size_t swept_slots = heap->freed_slots + heap->empty_slots;
5749 size_t init_slots = objspace_heap_init_bytes(
objspace) / heap->slot_size;
5750 size_t min_free_slots = (size_t)(MAX(total_slots, init_slots) * gc_params.heap_free_slots_min_ratio);
5752 if (swept_slots < min_free_slots &&
5754 ((heap->empty_slots == 0 && total_slots > 0) || heap->freed_slots > heap->empty_slots)) {
5760 while (swept_slots < min_free_slots &&
5761 (resurrected_page = heap_page_resurrect(
objspace))) {
5762 heap_add_page(
objspace, heap, resurrected_page);
5763 heap_add_freepage(heap, resurrected_page);
5765 swept_slots += resurrected_page->free_slots;
5768 if (swept_slots < min_free_slots) {
5772 objspace->profile.count -
objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
5773 if (
objspace->heap_pages.allocatable_bytes < min_free_slots * heap->slot_size) {
5774 heap_allocatable_bytes_expand(
objspace, heap, swept_slots, heap->total_slots, heap->slot_size);
5777 else if (swept_slots < min_free_slots * 7 / 8 &&
5778 objspace->heap_pages.allocatable_bytes < (min_free_slots * 7 / 8 - swept_slots) * heap->slot_size) {
5779 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
5780 heap->force_major_gc_count++;
5786#if RGENGC_ESTIMATE_OLDMALLOC
5788gc_check_oldmalloc(rb_objspace_t *
objspace)
5790 size_t oldmalloc_increase = gc_oldmalloc_increase_load(
objspace);
5792 gc_counter_t malloc_at = gc_counter_load_acquire(&
objspace->malloc_counters.counters.malloc_at_last_gc);
5793 gc_counter_t free_at = gc_counter_load_acquire(&
objspace->malloc_counters.counters.free_at_last_gc);
5794 gc_counter_t malloc_now = gc_counter_load_relaxed(&
objspace->malloc_counters.counters.malloc);
5795 gc_counter_t free_now = gc_counter_load_relaxed(&
objspace->malloc_counters.counters.free);
5796 gc_counter_t gc_freed = gc_counter_load_relaxed(&
objspace->malloc_counters.freed_in_gc);
5799 gc_counter_t freed = free_now - free_at;
5800 gc_counter_t mutator_freed = freed > gc_freed ? freed - gc_freed : 0;
5801 gc_counter_t allocated = malloc_now - malloc_at;
5802 gc_counter_t unswept = allocated > mutator_freed ? allocated - mutator_freed : 0;
5804 if ((gc_counter_t)oldmalloc_increase <= unswept)
return;
5805 oldmalloc_increase -= (size_t)unswept;
5807 if (oldmalloc_increase >
objspace->rgengc.oldmalloc_increase_limit) {
5808 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDMALLOC;
5809 objspace->rgengc.oldmalloc_increase_limit =
5810 (size_t)(
objspace->rgengc.oldmalloc_increase_limit * gc_params.oldmalloc_limit_growth_factor);
5812 if (
objspace->rgengc.oldmalloc_increase_limit > gc_params.oldmalloc_limit_max) {
5813 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_max;
5820gc_sweep_finish(rb_objspace_t *
objspace)
5822 gc_report(1,
objspace,
"gc_sweep_finish\n");
5825 heap_pages_free_unused_pages(
objspace);
5826 if (rb_gc_single_objspace_p() && is_full_marking(
objspace)) {
5829 page_pool_reclaim(global_objspace);
5831#if RGENGC_ESTIMATE_OLDMALLOC
5837 for (
int i = 0; i < HEAP_COUNT; i++) {
5838 rb_heap_t *heap = &heaps[i];
5840 heap->freed_slots = 0;
5841 heap->empty_slots = 0;
5843 if (!will_be_incremental_marking(
objspace)) {
5844 struct heap_page *end_page = heap->free_pages;
5846 while (end_page->free_next) end_page = end_page->free_next;
5847 end_page->free_next = heap->pooled_pages;
5850 heap->free_pages = heap->pooled_pages;
5852 heap->pooled_pages = NULL;
5859 gc_malloc_counters_snapshot_free_at_last_gc(
objspace, &
objspace->malloc_counters.counters);
5864 if (tdata_deferred_free_count_load() > 0 && rb_gc_single_objspace_p()) {
5865 gc_tdata_deferred_free_trigger(
objspace);
5869 gc_mode_transition(
objspace, gc_mode_none);
5873gc_sweep_step(rb_objspace_t *
objspace, rb_heap_t *heap)
5875 struct heap_page *sweep_page = heap->sweeping_page;
5876 int swept_slots = 0;
5877 int pooled_slots = 0;
5878 int sweep_budget = GC_INCREMENTAL_SWEEP_BYTES / heap->slot_size;
5879 int pool_budget = GC_INCREMENTAL_SWEEP_POOL_BYTES / heap->slot_size;
5881 if (sweep_page == NULL)
return FALSE;
5883#if GC_ENABLE_LAZY_SWEEP
5884 gc_prof_sweep_timer_start(
objspace);
5891 const bool check_pinned_free =
objspace->last_cycle_pinned;
5893 const bool defer_thread_unsafe_local_sweep =
5894 !rb_gc_single_objspace_p() && !
objspace->flags.during_global_gc;
5895 bool trigger_thread_unsafe_sweep_postponed_job =
false;
5898 RUBY_DEBUG_LOG(
"sweep_page:%p", (
void *)sweep_page);
5905 .check_pinned_free = check_pinned_free,
5906 .defer_thread_unsafe_local_sweep = defer_thread_unsafe_local_sweep,
5907 .trigger_thread_unsafe_sweep_postponed_job = trigger_thread_unsafe_sweep_postponed_job,
5909 gc_sweep_page(
objspace, heap, &ctx);
5910 int free_slots = ctx.freed_slots + ctx.empty_slots;
5911 trigger_thread_unsafe_sweep_postponed_job = ctx.trigger_thread_unsafe_sweep_postponed_job;
5913 RUBY_DTRACE_GC_HOOK(SWEEP_PAGE, ctx.page->slot_size, ctx.final_slots, ctx.freed_slots, ctx.empty_slots);
5915 heap->sweeping_page = ccan_list_next(&heap->pages, sweep_page, page_node);
5917 if (free_slots == sweep_page->total_slots &&
5918 (heap->total_pages > 1 ||
objspace->flags.ownerless)) {
5922 heap_page_detach_empty(
objspace, heap, sweep_page);
5924 else if (free_slots > 0) {
5925 heap->freed_slots += ctx.freed_slots;
5926 heap->empty_slots += ctx.empty_slots;
5928 if (pooled_slots < pool_budget) {
5929 heap_add_poolpage(
objspace, heap, sweep_page);
5930 pooled_slots += free_slots;
5933 heap_add_freepage(heap, sweep_page);
5934 swept_slots += free_slots;
5935 if (swept_slots > sweep_budget) {
5941 sweep_page->free_next = NULL;
5943 }
while ((sweep_page = heap->sweeping_page));
5945 if (trigger_thread_unsafe_sweep_postponed_job) {
5946 gc_report(2,
objspace,
"thread-unsafe sweep postponed job triggered\n");
5947 gc_tdata_deferred_free_trigger(
objspace);
5950 if (!heap->sweeping_page) {
5952 GC_ASSERT(
objspace->sweeping_heap_count >= 0);
5953 gc_sweep_finish_heap(
objspace, heap);
5955 if (!has_sweeping_pages(
objspace)) {
5960#if GC_ENABLE_LAZY_SWEEP
5961 gc_prof_sweep_timer_stop(
objspace);
5964 return heap->free_pages != NULL;
5968gc_sweep_rest(rb_objspace_t *
objspace)
5970 for (
int i = 0; i < HEAP_COUNT; i++) {
5971 rb_heap_t *heap = &heaps[i];
5973 while (heap->sweeping_page) {
5987gc_sweep_continue(rb_objspace_t *
objspace, rb_heap_t *sweep_heap)
5989 GC_ASSERT(dont_gc_val() == FALSE ||
objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
5990 if (!GC_ENABLE_LAZY_SWEEP)
return;
5994 for (
int i = 0; i < HEAP_COUNT; i++) {
5995 rb_heap_t *heap = &heaps[i];
5996 if (gc_sweep_step(
objspace, heap)) {
5997 GC_ASSERT(heap->free_pages != NULL);
5999 else if (heap == sweep_heap) {
6000 if (
objspace->empty_pages_count > 0 ||
objspace->heap_pages.allocatable_bytes > 0) {
6011 heap_page_allocate_and_initialize(
objspace, heap);
6013 GC_ASSERT(heap->free_pages != NULL);
6018 GC_ASSERT(gc_mode(
objspace) == gc_mode_none);
6028gc_sweep_step_for_malloc(rb_objspace_t *
objspace)
6030 GC_ASSERT(is_lazy_sweeping(
objspace));
6032 unsigned int lock_lev;
6033 gc_enter(
objspace, gc_enter_event_continue, &lock_lev);
6037 for (
int i = 0; i < HEAP_COUNT; i++) {
6038 rb_heap_t *heap = &heaps[i];
6044 gc_exit(
objspace, gc_enter_event_continue, &lock_lev);
6047static bool gc_global_pointer_to_heap_p(
const void *ptr);
6050rb_gc_impl_location(
void *objspace_ptr,
VALUE value)
6052 rb_objspace_t *
objspace = objspace_ptr;
6058 if (RB_UNLIKELY(
objspace->flags.during_global_gc)
6059 ? !gc_global_pointer_to_heap_p((
void *)value)
6060 : !is_pointer_to_heap(objspace_ptr, (
void *)value)) {
6064 asan_unpoisoning_object(value) {
6066 destination = (
VALUE)RMOVED(value)->destination;
6070 destination = value;
6077#if GC_CAN_COMPILE_COMPACTION
6079invalidate_moved_plane(rb_objspace_t *
objspace,
struct heap_page *page, uintptr_t p, bits_t bitset)
6088 GC_ASSERT(RVALUE_PINNED(
objspace, forwarding_object));
6089 GC_ASSERT(!RVALUE_MARKED(
objspace, forwarding_object));
6091 CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object);
6093 object = rb_gc_impl_location(
objspace, forwarding_object);
6094 gc_move(
objspace,
object, forwarding_object, GET_HEAP_PAGE(
object), page);
6098 struct heap_page *orig_page = GET_HEAP_PAGE(
object);
6099 orig_page->free_slots++;
6100 RVALUE_AGE_SET_BITMAP(
object, 0);
6101 heap_page_add_free_region(
objspace, orig_page,
object);
6103 GC_ASSERT(RVALUE_MARKED(
objspace, forwarding_object));
6108 p += page->slot_size;
6118 bits_t *mark_bits, *pin_bits;
6120 short slot_size = page->slot_size;
6121 int total_slots = page->total_slots;
6122 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
6124 mark_bits = page->mark_bits;
6125 pin_bits = page->pinned_bits;
6127 uintptr_t p = page->start;
6129 for (i=0; i < bitmap_plane_count; i++) {
6132 bitset = pin_bits[i] & ~mark_bits[i];
6133 invalidate_moved_plane(
objspace, page, p, bitset);
6134 p += BITS_BITLENGTH * slot_size;
6140gc_compact_start(rb_objspace_t *
objspace)
6143 gc_mode_transition(
objspace, gc_mode_compacting);
6145 for (
int i = 0; i < HEAP_COUNT; i++) {
6146 rb_heap_t *heap = &heaps[i];
6147 ccan_list_for_each(&heap->pages, page, page_node) {
6148 page->flags.before_sweep = TRUE;
6151 heap->compact_cursor = ccan_list_tail(&heap->pages,
struct heap_page, page_node);
6152 heap->compact_cursor_index = 0;
6157 record->moved_objects =
objspace->rcompactor.total_moved;
6160 memset(
objspace->rcompactor.considered_count_table, 0,
T_MASK *
sizeof(
size_t));
6161 memset(
objspace->rcompactor.moved_count_table, 0,
T_MASK *
sizeof(
size_t));
6162 memset(
objspace->rcompactor.moved_up_count_table, 0,
T_MASK *
sizeof(
size_t));
6163 memset(
objspace->rcompactor.moved_down_count_table, 0,
T_MASK *
sizeof(
size_t));
6167 if (!global_objspace->global_gc.compacting) install_handlers();
6170static void gc_sweep_compact(rb_objspace_t *
objspace);
6177 const unsigned int immediate_sweep =
objspace->flags.immediate_sweep;
6179 gc_report(1,
objspace,
"gc_sweep: immediate: %d\n", immediate_sweep);
6182 if (
objspace->flags.during_compacting) {
6183 rb_hrtime_t compact_start_time = gc_prof_enabled(
objspace) ? rb_hrtime_now() : 0;
6186 rb_hrtime_t compact_wall_time = elapsed_hrtime_from(compact_start_time);
6188 record->gc_compact_wall_time = rb_hrtime_add(record->gc_compact_wall_time,
6190 objspace->profile.gc_sweep_excluded_wall_time = rb_hrtime_add(
6191 objspace->profile.gc_sweep_excluded_wall_time,
6196 if (immediate_sweep) {
6197#if !GC_ENABLE_LAZY_SWEEP
6198 gc_prof_sweep_timer_start(
objspace);
6201#if !GC_ENABLE_LAZY_SWEEP
6202 gc_prof_sweep_timer_stop(
objspace);
6208 for (
int i = 0; i < HEAP_COUNT; i++) {
6209 rb_heap_t *heap = &heaps[i];
6219static stack_chunk_t *
6220stack_chunk_alloc(
void)
6224 res = malloc(
sizeof(stack_chunk_t));
6232is_mark_stack_empty(mark_stack_t *stack)
6234 return stack->chunk == NULL;
6238mark_stack_size(mark_stack_t *stack)
6240 size_t size = stack->index;
6241 stack_chunk_t *chunk = stack->chunk ? stack->chunk->next : NULL;
6244 size += stack->limit;
6245 chunk = chunk->next;
6251add_stack_chunk_cache(mark_stack_t *stack, stack_chunk_t *chunk)
6253 chunk->next = stack->cache;
6254 stack->cache = chunk;
6255 stack->cache_size++;
6259shrink_stack_chunk_cache(mark_stack_t *stack)
6261 stack_chunk_t *chunk;
6263 if (stack->unused_cache_size > (stack->cache_size/2)) {
6264 chunk = stack->cache;
6265 stack->cache = stack->cache->next;
6266 stack->cache_size--;
6269 stack->unused_cache_size = stack->cache_size;
6273push_mark_stack_chunk(mark_stack_t *stack)
6275 stack_chunk_t *next;
6277 GC_ASSERT(stack->index == stack->limit);
6279 if (stack->cache_size > 0) {
6280 next = stack->cache;
6281 stack->cache = stack->cache->next;
6282 stack->cache_size--;
6283 if (stack->unused_cache_size > stack->cache_size)
6284 stack->unused_cache_size = stack->cache_size;
6287 next = stack_chunk_alloc();
6289 next->next = stack->chunk;
6290 stack->chunk = next;
6295pop_mark_stack_chunk(mark_stack_t *stack)
6297 stack_chunk_t *prev;
6299 prev = stack->chunk->next;
6300 GC_ASSERT(stack->index == 0);
6301 add_stack_chunk_cache(stack, stack->chunk);
6302 stack->chunk = prev;
6303 stack->index = stack->limit;
6307mark_stack_chunk_list_free(stack_chunk_t *chunk)
6309 stack_chunk_t *next = NULL;
6311 while (chunk != NULL) {
6319free_stack_chunks(mark_stack_t *stack)
6321 mark_stack_chunk_list_free(stack->chunk);
6325mark_stack_free_cache(mark_stack_t *stack)
6327 mark_stack_chunk_list_free(stack->cache);
6328 stack->cache_size = 0;
6329 stack->unused_cache_size = 0;
6333push_mark_stack(mark_stack_t *stack,
VALUE obj)
6356 if (stack->index == stack->limit) {
6357 push_mark_stack_chunk(stack);
6359 stack->chunk->data[stack->index++] = obj;
6369 rb_bug(
"push_mark_stack() called for broken object");
6373 rb_bug(
"push_mark_stack: unexpected T_NODE object");
6377 rb_bug(
"rb_gc_mark(): unknown data type 0x%x(%p) %s",
6379 is_pointer_to_heap((rb_objspace_t *)rb_gc_get_objspace(), (
void *)obj) ?
"corrupted object" :
"non object");
6383pop_mark_stack(mark_stack_t *stack,
VALUE *data)
6385 if (is_mark_stack_empty(stack)) {
6388 if (stack->index == 1) {
6389 *data = stack->chunk->data[--stack->index];
6390 pop_mark_stack_chunk(stack);
6393 *data = stack->chunk->data[--stack->index];
6399init_mark_stack(mark_stack_t *stack)
6403 MEMZERO(stack, mark_stack_t, 1);
6404 stack->index = stack->limit = STACK_CHUNK_SIZE;
6406 for (i=0; i < 4; i++) {
6407 add_stack_chunk_cache(stack, stack_chunk_alloc());
6409 stack->unused_cache_size = stack->cache_size;
6414ALWAYS_INLINE(
static int gc_mark_set(rb_objspace_t *
objspace,
VALUE obj));
6415ALWAYS_INLINE(
static void gc_mark_check_t_none(rb_objspace_t *
objspace,
VALUE obj));
6416ALWAYS_INLINE(
static void rgengc_check_relation(rb_objspace_t *
objspace,
VALUE obj));
6417ALWAYS_INLINE(
static void gc_aging(rb_objspace_t *
objspace,
VALUE obj));
6418ALWAYS_INLINE(
static void gc_grey(rb_objspace_t *
objspace,
VALUE obj));
6422 if (
objspace->rgengc.parent_object_old_p) {
6423 if (!RVALUE_OLD_P(
objspace, obj)) {
6427 objspace->rgengc.parent_object_old_p =
false;
6435 if (RVALUE_MARKED(
objspace, obj))
return 0;
6436 MARK_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj);
6450 if(!gc_config_full_mark_val)
6453 struct heap_page *page = GET_HEAP_PAGE(obj);
6455 GC_ASSERT(RVALUE_MARKING(
objspace, obj) == FALSE);
6456 check_rvalue_consistency(
objspace, obj);
6458 if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) {
6459 if (!RVALUE_OLD_P(
objspace, obj)) {
6462 gc_report(3,
objspace,
"gc_aging: YOUNG class: %s\n", rb_obj_info(obj));
6463 RVALUE_AGE_SET(obj, RVALUE_OLD_AGE);
6464 RVALUE_OLD_UNCOLLECTIBLE_SET(
objspace, obj);
6467 gc_report(3,
objspace,
"gc_aging: YOUNG: %s\n", rb_obj_info(obj));
6471 else if (is_full_marking(
objspace)) {
6472 GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE);
6473 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(
objspace, page, obj);
6476 check_rvalue_consistency(
objspace, obj);
6478 page->objspace->marked_slots++;
6484#if RGENGC_CHECK_MODE
6485 if (RVALUE_MARKED(
objspace, obj) == FALSE) rb_bug(
"gc_grey: %s is not marked.", rb_obj_info(obj));
6486 if (RVALUE_MARKING(
objspace, obj) == TRUE) rb_bug(
"gc_grey: %s is marking/remembered.", rb_obj_info(obj));
6489 if (is_incremental_marking(
objspace)) {
6490 MARK_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
6494 rb_darray_append_without_gc(&
objspace->weak_references, obj);
6497 push_mark_stack(&
objspace->mark_stack, obj);
6504 enum {info_size = 256};
6505 char obj_info_buf[info_size];
6506 rb_raw_obj_info(obj_info_buf, info_size, obj);
6508 char parent_obj_info_buf[info_size];
6509 rb_raw_obj_info(parent_obj_info_buf, info_size,
objspace->rgengc.parent_object);
6511 rb_bug(
"try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf);
6518 GC_ASSERT(during_gc);
6519 GC_ASSERT(!
objspace->flags.during_reference_updating);
6524 if (gc_skip_foreign_object_p(
objspace, obj)) {
6528 if (RB_UNLIKELY(
objspace->flags.during_global_gc)) {
6533 if (!UNDEF_P(parent) && parent !=
Qfalse &&
6536 struct heap_page *page = GET_HEAP_PAGE(obj);
6537 _MARK_IN_BITMAP(page->shref_bits, page, obj);
6538 page->flags.has_shref_objects = TRUE;
6542 rgengc_check_relation(
objspace, obj);
6543 if (!gc_mark_set(
objspace, obj))
return;
6546 RUBY_DEBUG_LOG(
"%p (%s) parent:%p (%s)",
6547 (
void *)obj, obj_type_name(obj),
6548 (
void *)
objspace->rgengc.parent_object, obj_type_name(
objspace->rgengc.parent_object));
6551 gc_mark_check_t_none(
objspace, obj);
6562 if (RB_UNLIKELY(
objspace->flags.during_compacting)) {
6564 if (gc_skip_foreign_object_p(
objspace, obj))
return;
6566 if (RB_LIKELY(during_gc)) {
6567 if (!RVALUE_PINNED(
objspace, obj)) {
6568 GC_ASSERT(GET_HEAP_PAGE(obj)->pinned_slots <= GET_HEAP_PAGE(obj)->total_slots);
6569 GET_HEAP_PAGE(obj)->pinned_slots++;
6570 MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj);
6584rb_gc_impl_mark_and_move(
void *objspace_ptr,
VALUE *ptr)
6586 rb_objspace_t *
objspace = objspace_ptr;
6588 if (RB_UNLIKELY(
objspace->flags.during_reference_updating)) {
6589 GC_ASSERT(
objspace->flags.during_compacting);
6590 GC_ASSERT(during_gc);
6593 if (destination != *ptr) {
6603rb_gc_impl_mark(
void *objspace_ptr,
VALUE obj)
6605 rb_objspace_t *
objspace = objspace_ptr;
6611rb_gc_impl_mark_and_pin(
void *objspace_ptr,
VALUE obj)
6613 rb_objspace_t *
objspace = objspace_ptr;
6622gc_global_pointer_to_heap_p(
const void *ptr)
6624 const rb_global_objspace_t *g = global_objspace;
6625 uintptr_t p = (uintptr_t)ptr;
6627 if (p < g->page_index.lomem || p > g->page_index.himem)
return false;
6628 if (p %
sizeof(
VALUE) != 0)
return false;
6630 struct heap_page **res = bsearch(ptr, g->page_index.pages, g->page_index.n_pages,
6631 sizeof(
struct heap_page *), ptr_in_page_body_p);
6632 if (res == NULL)
return false;
6635 if (heap_page_in_global_empty_pages_pool(page->objspace, page))
return false;
6636 if (p < page->start)
return false;
6637 if (p >= page->start + (page->total_slots * page->slot_size))
return false;
6638 if (!slot_offset_aligned_p(page, p))
return false;
6643rb_gc_impl_mark_maybe(
void *objspace_ptr,
VALUE obj)
6645 rb_objspace_t *
objspace = objspace_ptr;
6647 (void)VALGRIND_MAKE_MEM_DEFINED(&obj,
sizeof(obj));
6649 if (RB_UNLIKELY(
objspace->flags.during_global_gc)
6650 ? gc_global_pointer_to_heap_p((
void *)obj)
6651 : is_pointer_to_heap(
objspace, (
void *)obj)) {
6652 asan_unpoisoning_object(obj) {
6667pin_value(st_data_t key, st_data_t value, st_data_t data)
6669 rb_gc_impl_mark_and_pin((
void *)data, (
VALUE)value);
6675gc_mark_set_parent_raw(rb_objspace_t *
objspace,
VALUE obj,
bool old_p)
6677 asan_unpoison_memory_region(&
objspace->rgengc.parent_object,
sizeof(
objspace->rgengc.parent_object),
false);
6678 asan_unpoison_memory_region(&
objspace->rgengc.parent_object_old_p,
sizeof(
objspace->rgengc.parent_object_old_p),
false);
6679 objspace->rgengc.parent_object = obj;
6680 objspace->rgengc.parent_object_old_p = old_p;
6690gc_mark_set_parent_invalid(rb_objspace_t *
objspace)
6692 asan_poison_memory_region(&
objspace->rgengc.parent_object,
sizeof(
objspace->rgengc.parent_object));
6693 asan_poison_memory_region(&
objspace->rgengc.parent_object_old_p,
sizeof(
objspace->rgengc.parent_object_old_p));
6696static void pinned_roots_mark(rb_objspace_t *
objspace, rb_heap_t *heap);
6699mark_roots(rb_objspace_t *
objspace,
const char **categoryp)
6702#define MARK_CHECKPOINT(category) do { \
6703 if (categoryp) *categoryp = category; \
6710 MARK_CHECKPOINT(
"objspace");
6713 if (
objspace->flags.during_global_gc) {
6716 rb_objspace_t *
const driver =
objspace;
6717 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
6718 rb_objspace_t *
objspace = global_objspace->global_gc.objspaces[i];
6719 if (finalizer_table != NULL) {
6720 st_foreach(finalizer_table, pin_value, (st_data_t)driver);
6724 else if (finalizer_table != NULL) {
6728 if (stress_to_class) rb_gc_mark(stress_to_class);
6730 rb_gc_save_machine_context();
6731 rb_gc_mark_roots(
objspace, categoryp);
6735 gc_mark_set_parent_invalid(
objspace);
6742 rb_gc_mark_children(
objspace, obj);
6743 gc_mark_set_parent_invalid(
objspace);
6751gc_mark_stacked_objects(rb_objspace_t *
objspace,
int incremental,
size_t count)
6753 mark_stack_t *mstack = &
objspace->mark_stack;
6755 size_t marked_slots_at_the_beginning =
objspace->marked_slots;
6756 size_t popped_count = 0;
6758 while (pop_mark_stack(mstack, &obj)) {
6759 if (obj ==
Qundef)
continue;
6761 if (RGENGC_CHECK_MODE && !RVALUE_MARKED(
objspace, obj)) {
6762 rb_bug(
"gc_mark_stacked_objects: %s is not marked.", rb_obj_info(obj));
6769 if (RGENGC_CHECK_MODE && !RVALUE_MARKING(
objspace, obj)) {
6770 rb_bug(
"gc_mark_stacked_objects: incremental, but marking bit is 0");
6772 CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
6774 if (popped_count + (
objspace->marked_slots - marked_slots_at_the_beginning) > count) {
6783 RUBY_DTRACE_GC_HOOK(MARK_STACKED_OBJECTS, popped_count);
6785 if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(
objspace);
6787 if (is_mark_stack_empty(mstack)) {
6788 shrink_stack_chunk_cache(mstack);
6797gc_mark_stacked_objects_incremental(rb_objspace_t *
objspace,
size_t count)
6799 return gc_mark_stacked_objects(
objspace, TRUE, count);
6803gc_mark_stacked_objects_all(rb_objspace_t *
objspace)
6805 return gc_mark_stacked_objects(
objspace, FALSE, 0);
6808#if RGENGC_CHECK_MODE >= 4
6810#define MAKE_ROOTSIG(obj) (((VALUE)(obj) << 1) | 0x01)
6811#define IS_ROOTSIG(obj) ((VALUE)(obj) & 0x01)
6812#define GET_ROOTSIG(obj) ((const char *)((VALUE)(obj) >> 1))
6820static struct reflist *
6821reflist_create(
VALUE obj)
6823 struct reflist *refs =
xmalloc(
sizeof(
struct reflist));
6826 refs->list[0] = obj;
6832reflist_destruct(
struct reflist *refs)
6839reflist_add(
struct reflist *refs,
VALUE obj)
6841 if (refs->pos == refs->size) {
6843 SIZED_REALLOC_N(refs->list,
VALUE, refs->size, refs->size/2);
6846 refs->list[refs->pos++] = obj;
6850reflist_dump(
struct reflist *refs)
6853 for (i=0; i<refs->pos; i++) {
6854 VALUE obj = refs->list[i];
6855 if (IS_ROOTSIG(obj)) {
6856 fprintf(stderr,
"<root@%s>", GET_ROOTSIG(obj));
6859 fprintf(stderr,
"<%s>", rb_obj_info(obj));
6861 if (i+1 < refs->pos) fprintf(stderr,
", ");
6866reflist_referred_from_machine_context(
struct reflist *refs)
6869 for (i=0; i<refs->pos; i++) {
6870 VALUE obj = refs->list[i];
6871 if (IS_ROOTSIG(obj) && strcmp(GET_ROOTSIG(obj),
"machine_context") == 0)
return 1;
6877 rb_objspace_t *objspace;
6885 struct st_table *references;
6886 const char *category;
6888 mark_stack_t mark_stack;
6892allrefs_add(
struct allrefs *data,
VALUE obj)
6894 struct reflist *refs;
6897 if (st_lookup(data->references, obj, &r)) {
6898 refs = (
struct reflist *)r;
6899 reflist_add(refs, data->root_obj);
6903 refs = reflist_create(data->root_obj);
6904 st_insert(data->references, obj, (st_data_t)refs);
6910allrefs_i(
VALUE obj,
void *ptr)
6912 struct allrefs *data = (
struct allrefs *)ptr;
6914 if (allrefs_add(data, obj)) {
6915 push_mark_stack(&data->mark_stack, obj);
6920allrefs_roots_i(
VALUE obj,
void *ptr)
6922 struct allrefs *data = (
struct allrefs *)ptr;
6923 if (strlen(data->category) == 0) rb_bug(
"!!!");
6924 data->root_obj = MAKE_ROOTSIG(data->category);
6926 if (allrefs_add(data, obj)) {
6927 push_mark_stack(&data->mark_stack, obj);
6930#define PUSH_MARK_FUNC_DATA(v) do { \
6931 struct gc_mark_func_data_struct *prev_mark_func_data = GET_VM()->gc.mark_func_data; \
6932 GET_VM()->gc.mark_func_data = (v);
6934#define POP_MARK_FUNC_DATA() GET_VM()->gc.mark_func_data = prev_mark_func_data;} while (0)
6937objspace_allrefs(rb_objspace_t *
objspace)
6939 struct allrefs data;
6942 int prev_dont_gc = dont_gc_val();
6946 data.references = st_init_numtable();
6947 init_mark_stack(&data.mark_stack);
6949 mfd.mark_func = allrefs_roots_i;
6953 PUSH_MARK_FUNC_DATA(&mfd);
6954 GET_VM()->gc.mark_func_data = &mfd;
6955 mark_roots(
objspace, &data.category);
6956 POP_MARK_FUNC_DATA();
6959 while (pop_mark_stack(&data.mark_stack, &obj)) {
6960 rb_objspace_reachable_objects_from_unlocked(data.root_obj = obj, allrefs_i, &data);
6962 free_stack_chunks(&data.mark_stack);
6964 dont_gc_set(prev_dont_gc);
6965 return data.references;
6969objspace_allrefs_destruct_i(st_data_t key, st_data_t value, st_data_t ptr)
6971 struct reflist *refs = (
struct reflist *)value;
6972 reflist_destruct(refs);
6977objspace_allrefs_destruct(
struct st_table *refs)
6979 st_foreach(refs, objspace_allrefs_destruct_i, 0);
6980 st_free_table(refs);
6983#if RGENGC_CHECK_MODE >= 5
6985allrefs_dump_i(st_data_t k, st_data_t v, st_data_t ptr)
6988 struct reflist *refs = (
struct reflist *)v;
6989 fprintf(stderr,
"[allrefs_dump_i] %s <- ", rb_obj_info(obj));
6991 fprintf(stderr,
"\n");
6996allrefs_dump(rb_objspace_t *
objspace)
6999 fprintf(stderr,
"[all refs] (size: %"PRIuVALUE
")\n", size);
7005gc_check_after_marks_i(st_data_t k, st_data_t v, st_data_t ptr)
7008 struct reflist *refs = (
struct reflist *)v;
7009 rb_objspace_t *
objspace = (rb_objspace_t *)ptr;
7012 if (!RVALUE_MARKED(
objspace, obj)) {
7013 fprintf(stderr,
"gc_check_after_marks_i: %s is not marked and not oldgen.\n", rb_obj_info(obj));
7014 fprintf(stderr,
"gc_check_after_marks_i: %p is referred from ", (
void *)obj);
7017 if (reflist_referred_from_machine_context(refs)) {
7018 fprintf(stderr,
" (marked from machine stack).\n");
7023 fprintf(stderr,
"\n");
7030gc_marks_check(rb_objspace_t *
objspace, st_foreach_callback_func *checker_func,
const char *checker_name)
7034 .malloc = gc_counter_load_relaxed(&
objspace->malloc_counters.counters.malloc),
7035 .free = gc_counter_load_relaxed(&
objspace->malloc_counters.counters.free),
7036 .malloc_at_last_gc = gc_counter_load_relaxed(&
objspace->malloc_counters.counters.malloc_at_last_gc),
7037 .free_at_last_gc = gc_counter_load_relaxed(&
objspace->malloc_counters.counters.free_at_last_gc),
7039#if RGENGC_ESTIMATE_OLDMALLOC
7040 size_t saved_oldmalloc_increase = gc_oldmalloc_increase_load(
objspace);
7041 gc_counter_t saved_freed_in_gc = gc_counter_load_relaxed(&
objspace->malloc_counters.freed_in_gc);
7052 if (
objspace->rgengc.error_count > 0) {
7053#if RGENGC_CHECK_MODE >= 5
7056 if (checker_name) rb_bug(
"%s: GC has problem.", checker_name);
7059 objspace_allrefs_destruct(
objspace->rgengc.allrefs_table);
7060 objspace->rgengc.allrefs_table = 0;
7064 gc_counter_store_release(&
objspace->malloc_counters.counters.malloc, saved_malloc.malloc);
7065 gc_counter_store_release(&
objspace->malloc_counters.counters.free, saved_malloc.free);
7066 gc_counter_store_release(&
objspace->malloc_counters.counters.malloc_at_last_gc, saved_malloc.malloc_at_last_gc);
7067 gc_counter_store_release(&
objspace->malloc_counters.counters.free_at_last_gc, saved_malloc.free_at_last_gc);
7068#if RGENGC_ESTIMATE_OLDMALLOC
7070 gc_counter_store_release(&
objspace->malloc_counters.freed_in_gc, saved_freed_in_gc);
7077 rb_objspace_t *objspace;
7083 size_t live_object_count;
7084 size_t zombie_object_count;
7087 bool parent_shareable;
7088 size_t old_object_count;
7089 size_t remembered_shady_count;
7094check_generation_i(
const VALUE child,
void *ptr)
7097 const VALUE parent = data->parent;
7099 if (RGENGC_CHECK_MODE) GC_ASSERT(RVALUE_OLD_P(data->objspace, parent));
7103 if (GET_HEAP_OBJSPACE(child) != data->objspace)
return;
7116 if (rb_gc_ever_multi_ractor_p() &&
7117 (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(parent), parent) ||
7118 MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(child), child))) {
7122 if (!RVALUE_OLD_P(data->objspace, child)) {
7127 if (!RVALUE_REMEMBERED(data->objspace, parent) &&
7128 !RVALUE_REMEMBERED(data->objspace, child) &&
7129 !RVALUE_UNCOLLECTIBLE(data->objspace, child) &&
7131 fprintf(stderr,
"verify_internal_consistency_reachable_i: WB miss (O->Y) %s -> %s\n", rb_obj_info(parent), rb_obj_info(child));
7138check_color_i(
const VALUE child,
void *ptr)
7141 const VALUE parent = data->parent;
7145 if (GET_HEAP_OBJSPACE(child) != data->objspace)
return;
7147 if (!RVALUE_WB_UNPROTECTED(data->objspace, parent) && RVALUE_WHITE_P(data->objspace, child)) {
7148 fprintf(stderr,
"verify_internal_consistency_reachable_i: WB miss (B->W) - %s -> %s\n",
7149 rb_obj_info(parent), rb_obj_info(child));
7155check_children_i(
const VALUE child,
void *ptr)
7160 if (RB_LIKELY(is_pointer_to_heap(data->objspace, (
void *)child))) {
7161 if (check_rvalue_consistency_force(data->objspace, child, FALSE) != 0) {
7162 fprintf(stderr,
"check_children_i: %s has error (referenced from %s)\n",
7163 rb_obj_info(child), rb_obj_info(data->parent));
7172 if (!data->world_stopped)
return;
7177 if (!verify_pointer_in_any_heap_p((
void *)child)) {
7180 if (global_objspace->during_absorb)
return;
7181 fprintf(stderr,
"VERIFY-NOTE: non-heap child %p (from %s)\n",
7182 (
void *)child, rb_obj_info(data->parent));
7186 if (GET_HEAP_OBJSPACE(child) != data->objspace) {
7195 if (!data->parent_shareable &&
7196 child != rb_gc_vm_top_self() &&
7197 !MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(child), child) &&
7198 !MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(child), child) &&
7199 !rb_gc_impl_during_global_gc_p(data->objspace) &&
7200 !global_objspace->during_absorb) {
7201 fprintf(stderr,
"check_children_i: containment violation: "
7202 "unshareable %s (objspace %p) -> foreign unshareable %s (objspace %p)\n",
7203 rb_obj_info(data->parent), (
void *)data->objspace,
7204 rb_obj_info(child), (
void *)GET_HEAP_OBJSPACE(child));
7225 return !rb_gc_impl_garbage_object_p(
objspace, obj);
7232verify_pointer_in_any_heap_p(
const void *ptr)
7234 return gc_global_pointer_to_heap_p(ptr);
7242root_scope_check_i(
const char *category,
VALUE obj,
void *ptr)
7250 if (!data->world_stopped)
return;
7253 if (global_objspace->during_absorb)
return;
7254 if (strcmp(category,
"machine_context") == 0 ||
7255 strcmp(category,
"vm_registered_objects") == 0 ||
7256 strcmp(category,
"end_proc") == 0 ||
7257 strcmp(category,
"trap_list") == 0) {
7261 if (!verify_pointer_in_any_heap_p((
void *)obj)) {
7262 fprintf(stderr,
"root_scope_check_i: root category \"%s\" names a non-heap pointer %p\n",
7263 category, (
void *)obj);
7268 if (GET_HEAP_OBJSPACE(obj) == data->objspace)
return;
7269 if (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj))
return;
7270 if (MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj))
return;
7271 if (obj == rb_gc_vm_top_self())
return;
7273 fprintf(stderr,
"root_scope_check_i: root category \"%s\" names a foreign "
7274 "unshareable without a shref record: %s (owner %p, self %p)\n",
7275 category, rb_obj_info(obj),
7276 (
void *)GET_HEAP_OBJSPACE(obj), (
void *)data->objspace);
7281verify_internal_consistency_i(
void *page_start,
void *page_end,
size_t stride,
7285 rb_objspace_t *
objspace = data->objspace;
7287 for (obj = (
VALUE)page_start; obj != (
VALUE)page_end; obj += stride) {
7288 asan_unpoisoning_object(obj) {
7289 bool sh_bit = MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj) != 0;
7290 bool sr_bit = MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj) != 0;
7292 if (gc_slot_live_object_p(
objspace, obj)) {
7294 data->live_object_count++;
7296 data->parent_shareable = sh_bit;
7302 fprintf(stderr,
"verify_internal_consistency_i: shareable bit %d "
7303 "disagrees with FL_SHAREABLE on %s\n", (
int)sh_bit, rb_obj_info(obj));
7306 if (sr_bit && sh_bit) {
7307 fprintf(stderr,
"verify_internal_consistency_i: shref bit on a shareable: %s\n",
7314 if (!gc_object_moved_p(
objspace, obj)) {
7316 rb_objspace_reachable_objects_from_unlocked(obj, check_children_i, (
void *)data);
7320 if (RVALUE_OLD_P(
objspace, obj)) data->old_object_count++;
7321 if (RVALUE_WB_UNPROTECTED(
objspace, obj) && RVALUE_UNCOLLECTIBLE(
objspace, obj)) data->remembered_shady_count++;
7326 rb_objspace_reachable_objects_from_unlocked(obj, check_generation_i, (
void *)data);
7329 if (!is_marking(
objspace) && rb_gc_obj_shareable_p(obj)) {
7330 rb_gc_verify_shareable(obj);
7333 if (is_incremental_marking(
objspace)) {
7334 if (RVALUE_BLACK_P(
objspace, obj)) {
7337 rb_objspace_reachable_objects_from_unlocked(obj, check_color_i, (
void *)data);
7346 fprintf(stderr,
"verify_internal_consistency_i: T_NONE slot carries "
7347 "shareable=%d shref=%d bits\n", (
int)sh_bit, (
int)sr_bit);
7352 data->zombie_object_count++;
7355 fprintf(stderr,
"verify_internal_consistency_i: T_ZOMBIE has extra flags set: %s\n",
7361 fprintf(stderr,
"verify_internal_consistency_i: FL_FINALIZE %s but %s finalizer_table: %s\n",
7362 FL_TEST(obj,
FL_FINALIZE) ?
"set" :
"not set", st_is_member(finalizer_table, obj) ?
"in" :
"not in",
7377 unsigned int has_remembered_shady = FALSE;
7378 unsigned int has_remembered_old = FALSE;
7379 int remembered_old_objects = 0;
7380 int free_objects = 0;
7381 int zombie_objects = 0;
7383 short slot_size = page->slot_size;
7384 uintptr_t start = (uintptr_t)page->start;
7385 uintptr_t end = start + page->total_slots * slot_size;
7387 for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
7389 asan_unpoisoning_object(val) {
7394 if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) {
7395 has_remembered_shady = TRUE;
7397 if (RVALUE_PAGE_MARKING(page, val)) {
7398 has_remembered_old = TRUE;
7399 remembered_old_objects++;
7404 if (!is_incremental_marking(
objspace) &&
7405 page->flags.has_remembered_objects == FALSE && has_remembered_old == TRUE) {
7407 for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
7409 if (RVALUE_PAGE_MARKING(page, val)) {
7410 fprintf(stderr,
"marking -> %s\n", rb_obj_info(val));
7413 rb_bug(
"page %p's has_remembered_objects should be false, but there are remembered old objects (%d). %s",
7414 (
void *)page, remembered_old_objects, obj ? rb_obj_info(obj) :
"");
7417 if (page->flags.has_uncollectible_wb_unprotected_objects == FALSE && has_remembered_shady == TRUE) {
7418 rb_bug(
"page %p's has_remembered_shady should be false, but there are remembered shady objects. %s",
7419 (
void *)page, obj ? rb_obj_info(obj) :
"");
7424 if (page->free_slots != free_objects) {
7425 rb_bug(
"page %p's free_slots should be %d, but %d", (
void *)page, page->free_slots, free_objects);
7428 if (page->final_slots != zombie_objects) {
7429 rb_bug(
"page %p's final_slots should be %d, but %d", (
void *)page, page->final_slots, zombie_objects);
7432 return remembered_old_objects;
7436gc_verify_heap_pages_(rb_objspace_t *
objspace,
struct ccan_list_head *head)
7438 int remembered_old_objects = 0;
7441 ccan_list_for_each(head, page, page_node) {
7442 asan_unlock_freelist(page);
7446 rb_asan_unpoison_object(vp,
false);
7448 fprintf(stderr,
"free region head expected to be T_NONE but was: %s\n", rb_obj_info(vp));
7451 rb_asan_poison_object(vp);
7454 asan_lock_freelist(page);
7456 if (page->flags.has_remembered_objects == FALSE) {
7457 remembered_old_objects += gc_verify_heap_page(
objspace, page,
Qfalse);
7461 return remembered_old_objects;
7465gc_verify_heap_pages(rb_objspace_t *
objspace)
7467 int remembered_old_objects = 0;
7468 for (
int i = 0; i < HEAP_COUNT; i++) {
7469 remembered_old_objects += gc_verify_heap_pages_(
objspace, &((&heaps[i])->pages));
7471 return remembered_old_objects;
7475verify_registered_addr(
VALUE *slot,
VALUE initial_value,
void *owner_objspace,
void *d)
7482 if (v == initial_value)
return;
7484 if (!verify_pointer_in_any_heap_p((
void *)v))
return;
7487 asan_unpoisoning_object(v) {
7492 rb_objspace_t *value_objspace = GET_HEAP_OBJSPACE(v);
7493 if (value_objspace == (rb_objspace_t *)owner_objspace)
return;
7497 if (value_objspace->zombie_pages != 0)
return;
7498 if (value_objspace->flags.during_postmortem)
return;
7499 if (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(v), v))
return;
7500 if (MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(v), v))
return;
7503 if (rb_gc_registered_addr_owned_by_registrant_p(slot, value_objspace))
return;
7505 fprintf(stderr,
"registered address %p changed since registration to an unshareable object owned by another Ractor: %s\n",
7506 (
void *)slot, rb_obj_info(v));
7511gc_verify_internal_consistency_(rb_objspace_t *
objspace,
bool world_stopped)
7516 data.world_stopped = world_stopped;
7517 gc_report(5,
objspace,
"gc_verify_internal_consistency: start\n");
7520 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
7522 short slot_size = page->slot_size;
7524 uintptr_t start = (uintptr_t)page->start;
7525 uintptr_t end = start + page->total_slots * slot_size;
7527 verify_internal_consistency_i((
void *)start, (
void *)end, slot_size, &data);
7533 if (!rb_gc_single_objspace_p() &&
objspace == rb_gc_get_objspace() &&
7534 !rb_gc_impl_during_global_gc_p(
objspace)) {
7535 rb_objspace_reachable_objects_from_root(root_scope_check_i, &data);
7538 if (data.world_stopped && !global_objspace->during_absorb) {
7539 rb_gc_each_registered_addr(verify_registered_addr, &data);
7542 if (data.err_count != 0) {
7543#if RGENGC_CHECK_MODE >= 5
7544 objspace->rgengc.error_count = data.err_count;
7545 gc_marks_check(
objspace, NULL, NULL);
7548 rb_bug(
"gc_verify_internal_consistency: found internal inconsistency.");
7558 !rb_gc_multi_ractor_p()) {
7559 if (objspace_live_slots(
objspace) != data.live_object_count) {
7560 fprintf(stderr,
"heap_pages_final_slots: %"PRIdSIZE
", total_freed_objects: %"PRIdSIZE
"\n",
7562 rb_bug(
"inconsistent live slot number: expect %"PRIuSIZE
", but %"PRIuSIZE
".",
7563 objspace_live_slots(
objspace), data.live_object_count);
7568 if (
objspace->rgengc.old_objects != data.old_object_count) {
7569 rb_bug(
"inconsistent old slot number: expect %"PRIuSIZE
", but %"PRIuSIZE
".",
7570 objspace->rgengc.old_objects, data.old_object_count);
7572 if (
objspace->rgengc.uncollectible_wb_unprotected_objects != data.remembered_shady_count) {
7573 rb_bug(
"inconsistent number of wb unprotected objects: expect %"PRIuSIZE
", but %"PRIuSIZE
".",
7574 objspace->rgengc.uncollectible_wb_unprotected_objects, data.remembered_shady_count);
7579 size_t list_count = 0;
7582 VALUE z = heap_pages_deferred_final;
7585 z = RZOMBIE(z)->next;
7589 if (total_final_slots_count(
objspace) != data.zombie_object_count ||
7590 total_final_slots_count(
objspace) != list_count) {
7592 rb_bug(
"inconsistent finalizing object count:\n"
7593 " expect %"PRIuSIZE
"\n"
7594 " but %"PRIuSIZE
" zombies\n"
7595 " heap_pages_deferred_final list has %"PRIuSIZE
" items.",
7597 data.zombie_object_count,
7602 gc_report(5,
objspace,
"gc_verify_internal_consistency: OK\n");
7608static inline unsigned int
7609gc_during_gc_get(
const rb_objspace_t *
objspace)
7615gc_during_gc_set(rb_objspace_t *
objspace,
unsigned int v)
7625gc_verify_internal_consistency_body(rb_objspace_t *
objspace,
bool world_stopped)
7627 const unsigned int prev_during_gc = during_gc;
7630 rb_objspace_t *
const cur = rb_gc_get_objspace();
7631 const unsigned int prev_cur_during_gc = (cur !=
objspace) ? gc_during_gc_get(cur) : 0;
7632 if (cur !=
objspace) gc_during_gc_set(cur, FALSE);
7634 gc_verify_internal_consistency_(
objspace, world_stopped);
7636 if (cur !=
objspace) gc_during_gc_set(cur, prev_cur_during_gc);
7637 during_gc = prev_during_gc;
7641gc_verify_internal_consistency(
void *objspace_ptr)
7643 rb_objspace_t *
objspace = objspace_ptr;
7653 gc_verify_internal_consistency_body(
objspace, rb_gc_impl_during_global_gc_p(
objspace));
7657 unsigned int lev = RB_GC_VM_LOCK();
7660 gc_verify_internal_consistency_body(
objspace,
true);
7662 RB_GC_VM_UNLOCK(lev);
7666heap_move_pooled_pages_to_free_pages(rb_heap_t *heap)
7668 if (heap->pooled_pages) {
7669 if (heap->free_pages) {
7670 struct heap_page *free_pages_tail = heap->free_pages;
7671 while (free_pages_tail->free_next) {
7672 free_pages_tail = free_pages_tail->free_next;
7674 free_pages_tail->free_next = heap->pooled_pages;
7677 heap->free_pages = heap->pooled_pages;
7680 heap->pooled_pages = NULL;
7687 struct heap_page *page = GET_HEAP_PAGE(obj);
7688 bits_t *uncollectible_bits = &page->uncollectible_bits[0];
7690 if (!MARKED_IN_BITMAP(uncollectible_bits, obj)) {
7691 page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
7692 MARK_IN_BITMAP(uncollectible_bits, obj);
7694 page->objspace->rgengc.uncollectible_wb_unprotected_objects++;
7696#if RGENGC_PROFILE > 0
7697 objspace->profile.total_remembered_shady_object_count++;
7698#if RGENGC_PROFILE >= 2
7710gc_marks_wb_unprotected_objects_plane(rb_objspace_t *
objspace, uintptr_t p, bits_t bits,
short slot_size)
7715 gc_report(2,
objspace,
"gc_marks_wb_unprotected_objects: marked shady: %s\n", rb_obj_info((
VALUE)p));
7727gc_marks_wb_unprotected_objects(rb_objspace_t *
objspace, rb_heap_t *heap)
7731 ccan_list_for_each(&heap->pages, page, page_node) {
7732 bits_t *mark_bits = page->mark_bits;
7733 bits_t *wbun_bits = page->wb_unprotected_bits;
7734 uintptr_t p = page->start;
7735 short slot_size = page->slot_size;
7736 int total_slots = page->total_slots;
7737 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
7740 for (j=0; j<(size_t)bitmap_plane_count; j++) {
7741 bits_t bits = mark_bits[j] & wbun_bits[j];
7742 gc_marks_wb_unprotected_objects_plane(
objspace, p, bits, slot_size);
7743 p += BITS_BITLENGTH * slot_size;
7747 gc_mark_stacked_objects_all(
objspace);
7751rb_gc_impl_declare_weak_references(
void *objspace_ptr,
VALUE obj)
7757rb_gc_impl_handle_weak_references_alive_p(
void *objspace_ptr,
VALUE obj)
7759 rb_objspace_t *
objspace = objspace_ptr;
7763 if (gc_skip_foreign_object_p(
objspace, obj))
return true;
7765 bool marked = RVALUE_MARKED(
objspace, obj);
7768 rgengc_check_relation(
objspace, obj);
7775gc_update_weak_references(rb_objspace_t *
objspace)
7778 rb_darray_foreach(
objspace->weak_references, i, obj_ptr) {
7779 gc_mark_set_parent(
objspace, *obj_ptr);
7780 rb_gc_handle_weak_references(
objspace, *obj_ptr);
7781 gc_mark_set_parent_invalid(
objspace);
7784 size_t capa = rb_darray_capa(
objspace->weak_references);
7785 size_t size = rb_darray_size(
objspace->weak_references);
7787 objspace->profile.weak_references_count = size;
7789 rb_darray_clear(
objspace->weak_references);
7793 if (
capa > size * 4) {
7794 rb_darray_resize_capa_without_gc(&
objspace->weak_references, size * 2);
7799gc_marks_finish(rb_objspace_t *
objspace)
7802 if (is_incremental_marking(
objspace)) {
7803 if (RGENGC_CHECK_MODE && is_mark_stack_empty(&
objspace->mark_stack) == 0) {
7804 rb_bug(
"gc_marks_finish: mark stack is not empty (%"PRIdSIZE
").",
7805 mark_stack_size(&
objspace->mark_stack));
7809 while (gc_mark_stacked_objects_incremental(
objspace, INT_MAX) ==
false);
7811#if RGENGC_CHECK_MODE >= 2
7812 if (gc_verify_heap_pages(
objspace) != 0) {
7813 rb_bug(
"gc_marks_finish (incremental): there are remembered old objects.");
7817 gc_set_incremental_marking(
objspace, FALSE);
7819 for (
int i = 0; i < HEAP_COUNT; i++) {
7820 gc_marks_wb_unprotected_objects(
objspace, &heaps[i]);
7836 if (!rb_gc_single_objspace_p() && !
objspace->flags.during_global_gc) {
7839 for (
int i = 0; i < HEAP_COUNT; i++) {
7840 pinned_roots_mark(
objspace, &heaps[i]);
7843 gc_mark_stacked_objects_all(
objspace);
7846 gc_update_weak_references(
objspace);
7848#if RGENGC_CHECK_MODE >= 4
7850 gc_marks_check(
objspace, gc_check_after_marks_i,
"after_marks");
7857 const unsigned long ractor_cnt =
objspace == global_objspace->main_objspace
7858 ? rb_gc_vm_ractor_count() : 1;
7859 const unsigned long r_mul = ractor_cnt > 8 ? 8 : ractor_cnt;
7861 size_t total_slots = objspace_available_slots(
objspace);
7862 size_t sweep_slots = total_slots -
objspace->marked_slots;
7863 size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio);
7864 size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio);
7865 if (min_free_slots < gc_params.heap_free_slots * r_mul) {
7866 min_free_slots = gc_params.heap_free_slots * r_mul;
7869 int full_marking = is_full_marking(
objspace);
7874 size_t total_init_slots = 0;
7875 for (
int i = 0; i < HEAP_COUNT; i++) {
7876 total_init_slots += (objspace_heap_init_bytes(
objspace) / heaps[i].slot_size) * r_mul;
7879 if (max_free_slots < total_init_slots) {
7880 max_free_slots = total_init_slots;
7884 if (sweep_slots > max_free_slots) {
7885 size_t excess_slots = sweep_slots - max_free_slots;
7886 size_t total_heap_pages = heap_eden_total_pages(
objspace);
7887 heap_pages_freeable_pages = total_heap_pages > 0
7888 ? excess_slots * total_heap_pages / total_slots
7892 heap_pages_freeable_pages = 0;
7895 if (
objspace->heap_pages.allocatable_bytes == 0 && sweep_slots < min_free_slots) {
7896 if (!full_marking && sweep_slots < min_free_slots * 7 / 8) {
7897 if (
objspace->profile.count -
objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
7898 full_marking = TRUE;
7901 gc_report(1,
objspace,
"gc_marks_finish: next is full GC!!)\n");
7902 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
7907 heap_allocatable_bytes_expand(
objspace, NULL, sweep_slots, total_slots, heaps[0].slot_size);
7913 const double r = gc_params.oldobject_limit_factor;
7914 objspace->rgengc.uncollectible_wb_unprotected_objects_limit = MAX(
7915 (
size_t)(
objspace->rgengc.uncollectible_wb_unprotected_objects * r),
7916 (
size_t)(
objspace->rgengc.old_objects * gc_params.uncollectible_wb_unprotected_objects_limit_ratio)
7918 objspace->rgengc.old_objects_limit = (size_t)(
objspace->rgengc.old_objects * r);
7921 if (
objspace->rgengc.uncollectible_wb_unprotected_objects >
objspace->rgengc.uncollectible_wb_unprotected_objects_limit) {
7922 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_SHADY;
7925 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDGEN;
7928 gc_report(1,
objspace,
"gc_marks_finish (marks %"PRIdSIZE
" objects, "
7929 "old %"PRIdSIZE
" objects, total %"PRIdSIZE
" slots, "
7930 "sweep %"PRIdSIZE
" slots, allocatable %"PRIdSIZE
" bytes, next GC: %s)\n",
7932 gc_needs_major_flags ?
"major" :
"minor");
7936 rb_ractor_finish_marking(is_full_marking(
objspace));
7942gc_compact_heap_cursors_met_p(rb_heap_t *heap)
7944 return heap->sweeping_page == heap->compact_cursor;
7949gc_compact_destination_pool(rb_objspace_t *
objspace, rb_heap_t *src_pool,
VALUE obj)
7951 size_t obj_size = rb_gc_obj_optimal_size(obj);
7952 if (obj_size == 0) {
7956 GC_ASSERT(rb_gc_impl_size_allocatable_p(obj_size));
7958 size_t idx = heap_idx_for_size(obj_size);
7964gc_compact_move(rb_objspace_t *
objspace, rb_heap_t *heap,
VALUE src)
7967 GC_ASSERT(gc_is_moveable_obj(
objspace, src));
7969 rb_heap_t *dest_pool = gc_compact_destination_pool(
objspace, heap, src);
7970 if (gc_compact_heap_cursors_met_p(dest_pool)) {
7971 return dest_pool != heap;
7974 while (!try_move(
objspace, dest_pool, dest_pool->free_pages, src)) {
7976 .page = dest_pool->sweeping_page,
7985 lock_page_body(
objspace, GET_PAGE_BODY(src));
7986 gc_sweep_page(
objspace, dest_pool, &ctx);
7987 unlock_page_body(
objspace, GET_PAGE_BODY(src));
7989 if (dest_pool->sweeping_page->free_slots > 0) {
7990 heap_add_freepage(dest_pool, dest_pool->sweeping_page);
7993 dest_pool->sweeping_page = ccan_list_next(&dest_pool->pages, dest_pool->sweeping_page, page_node);
7994 if (gc_compact_heap_cursors_met_p(dest_pool)) {
7995 return dest_pool != heap;
8003gc_compact_plane(rb_objspace_t *
objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset,
struct heap_page *page)
8005 short slot_size = page->slot_size;
8009 GC_ASSERT(vp %
sizeof(
VALUE) == 0);
8014 if (gc_is_moveable_obj(
objspace, vp)) {
8015 if (!gc_compact_move(
objspace, heap, vp)) {
8030gc_compact_page(rb_objspace_t *
objspace, rb_heap_t *heap,
struct heap_page *page)
8032 GC_ASSERT(page == heap->compact_cursor);
8034 bits_t *mark_bits, *pin_bits;
8036 uintptr_t p = page->start;
8037 short slot_size = page->slot_size;
8038 int total_slots = page->total_slots;
8039 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
8041 mark_bits = page->mark_bits;
8042 pin_bits = page->pinned_bits;
8044 for (
int j = 0; j < bitmap_plane_count; j++) {
8046 bitset = (mark_bits[j] & ~pin_bits[j]);
8048 if (!gc_compact_plane(
objspace, heap, (uintptr_t)p, bitset, page))
8051 p += BITS_BITLENGTH * slot_size;
8058gc_compact_all_compacted_p(rb_objspace_t *
objspace)
8060 for (
int i = 0; i < HEAP_COUNT; i++) {
8061 rb_heap_t *heap = &heaps[i];
8063 if (heap->total_pages > 0 &&
8064 !gc_compact_heap_cursors_met_p(heap)) {
8077gc_compact_relocate(rb_objspace_t *
objspace)
8081 while (!gc_compact_all_compacted_p(
objspace)) {
8082 for (
int i = 0; i < HEAP_COUNT; i++) {
8083 rb_heap_t *heap = &heaps[i];
8085 if (gc_compact_heap_cursors_met_p(heap)) {
8089 struct heap_page *start_page = heap->compact_cursor;
8091 if (!gc_compact_page(
objspace, heap, start_page)) {
8092 lock_page_body(
objspace, start_page->body);
8099 lock_page_body(
objspace, start_page->body);
8100 heap->compact_cursor = ccan_list_prev(&heap->pages, heap->compact_cursor, page_node);
8106gc_sweep_compact(rb_objspace_t *
objspace)
8111 if (!global_objspace->global_gc.compacting) {
8117gc_marks_rest(rb_objspace_t *
objspace)
8119 gc_report(1,
objspace,
"gc_marks_rest\n");
8121 for (
int i = 0; i < HEAP_COUNT; i++) {
8122 (&heaps[i])->pooled_pages = NULL;
8125 if (is_incremental_marking(
objspace)) {
8126 while (gc_mark_stacked_objects_incremental(
objspace, INT_MAX) == FALSE);
8129 gc_mark_stacked_objects_all(
objspace);
8136gc_marks_step(rb_objspace_t *
objspace,
size_t slots)
8138 bool marking_finished =
false;
8141 if (gc_mark_stacked_objects_incremental(
objspace, slots)) {
8144 marking_finished =
true;
8147 return marking_finished;
8151gc_marks_continue(rb_objspace_t *
objspace, rb_heap_t *heap)
8153 GC_ASSERT(dont_gc_val() == FALSE ||
objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
8154 bool marking_finished =
true;
8158 if (heap->free_pages) {
8159 gc_report(2,
objspace,
"gc_marks_continue: has pooled pages");
8164 gc_report(2,
objspace,
"gc_marks_continue: no more pooled pages (stack depth: %"PRIdSIZE
").\n",
8165 mark_stack_size(&
objspace->mark_stack));
8166 heap->force_incremental_marking_finish_count++;
8172 return marking_finished;
8186pinned_roots_mark(rb_objspace_t *
objspace, rb_heap_t *heap)
8203 ccan_list_for_each(&heap->pages, page, page_node) {
8204 if (!(page->flags.has_shareable_objects | page->flags.has_shref_objects))
continue;
8206 uintptr_t p = page->start;
8207 short slot_size = page->slot_size;
8208 int total_slots = page->total_slots;
8209 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
8211 for (
int j = 0; j < bitmap_plane_count; j++) {
8212 bits_t sr_bits = page->shref_bits[j];
8216 bits_t bitset = (page->shareable_bits[j] | sr_bits) & ~page->mark_bits[j];
8221 asan_unpoisoning_object(obj) {
8229 gc_report(2,
objspace,
"pinned_roots_mark: mark %s\n", rb_obj_info(obj));
8233 else if (gc_mark_set(
objspace, obj)) {
8248 p += BITS_BITLENGTH * slot_size;
8254gc_marks_start(rb_objspace_t *
objspace,
int full_mark)
8257 gc_report(1,
objspace,
"gc_marks_start: (%s)\n", full_mark ?
"full" :
"minor");
8258 gc_mode_transition(
objspace, gc_mode_marking);
8261 size_t incremental_marking_steps = (
objspace->rincgc.pooled_slots / INCREMENTAL_MARK_STEP_ALLOCATIONS) + 1;
8262 objspace->rincgc.step_slots = (
objspace->marked_slots * 2) / incremental_marking_steps;
8264 if (0) fprintf(stderr,
"objspace->marked_slots: %"PRIdSIZE
", "
8265 "objspace->rincgc.pooled_page_num: %"PRIdSIZE
", "
8266 "objspace->rincgc.step_slots: %"PRIdSIZE
", \n",
8268 objspace->flags.during_minor_gc = FALSE;
8269 if (ruby_enable_autocompact && rb_gc_single_objspace_p()) {
8270 objspace->flags.during_compacting |= TRUE;
8272 objspace->profile.major_gc_count++;
8273 objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
8278 for (
int i = 0; i < HEAP_COUNT; i++) {
8279 rb_heap_t *heap = &heaps[i];
8280 gc_bitmaps_clear(
objspace, heap,
false);
8281 heap_move_pooled_pages_to_free_pages(heap);
8283 if (
objspace->flags.during_compacting) {
8286 ccan_list_for_each(&heap->pages, page, page_node) {
8287 page->pinned_slots = 0;
8293 objspace->flags.during_minor_gc = TRUE;
8295 objspace->rgengc.old_objects +
objspace->rgengc.uncollectible_wb_unprotected_objects;
8296 objspace->profile.minor_gc_count++;
8298 for (
int i = 0; i < HEAP_COUNT; i++) {
8299 rgengc_rememberset_mark(
objspace, &heaps[i]);
8305 gc_report(1,
objspace,
"gc_marks_start: (%s) end, stack in %"PRIdSIZE
"\n",
8306 full_mark ?
"full" :
"minor", mark_stack_size(&
objspace->mark_stack));
8310gc_marks(rb_objspace_t *
objspace,
int full_mark)
8314 bool marking_finished =
false;
8318 gc_marks_start(
objspace, full_mark);
8319 if (!is_incremental_marking(
objspace)) {
8321 marking_finished =
true;
8324#if RGENGC_PROFILE > 0
8327 record->old_objects =
objspace->rgengc.old_objects;
8333 return marking_finished;
8339gc_report_body(
int level, rb_objspace_t *
objspace,
const char *fmt, ...)
8341 if (level <= RGENGC_DEBUG) {
8345 const char *status =
" ";
8348 status = is_full_marking(
objspace) ?
"+" :
"-";
8354 if (is_incremental_marking(
objspace)) {
8359 va_start(args, fmt);
8360 vsnprintf(buf, 1024, fmt, args);
8363 fprintf(out,
"%s|", status);
8371rgengc_remembersetbits_set(rb_objspace_t *
objspace,
VALUE obj)
8373 struct heap_page *page = GET_HEAP_PAGE(obj);
8374 bits_t *bits = &page->remembered_bits[0];
8380 _MARK_IN_BITMAP(bits, page, obj);
8381 page->flags.has_remembered_objects = TRUE;
8390 gc_report(6,
objspace,
"rgengc_remember: %s %s\n", rb_obj_info(obj),
8391 RVALUE_REMEMBERED(
objspace, obj) ?
"was already remembered" :
"is remembered now");
8393 check_rvalue_consistency(
objspace, obj);
8395 if (RGENGC_CHECK_MODE) {
8396 if (RVALUE_WB_UNPROTECTED(
objspace, obj)) rb_bug(
"rgengc_remember: %s is not wb protected.", rb_obj_info(obj));
8399#if RGENGC_PROFILE > 0
8400 if (!RVALUE_REMEMBERED(
objspace, obj)) {
8401 if (RVALUE_WB_UNPROTECTED(
objspace, obj) == 0) {
8402 objspace->profile.total_remembered_normal_object_count++;
8403#if RGENGC_PROFILE >= 2
8410 rgengc_remembersetbits_set(
objspace, obj);
8413#ifndef PROFILE_REMEMBERSET_MARK
8414#define PROFILE_REMEMBERSET_MARK 0
8418rgengc_rememberset_mark_plane(rb_objspace_t *
objspace, uintptr_t p, bits_t bitset,
short slot_size)
8424 gc_report(2,
objspace,
"rgengc_rememberset_mark: mark %s\n", rb_obj_info(obj));
8425 GC_ASSERT(RVALUE_UNCOLLECTIBLE(
objspace, obj));
8426 GC_ASSERT(RVALUE_OLD_P(
objspace, obj) || RVALUE_WB_UNPROTECTED(
objspace, obj));
8431 rb_darray_append_without_gc(&
objspace->weak_references, obj);
8441rgengc_rememberset_mark(rb_objspace_t *
objspace, rb_heap_t *heap)
8445#if PROFILE_REMEMBERSET_MARK
8446 int has_old = 0, has_shady = 0, has_both = 0, skip = 0;
8448 gc_report(1,
objspace,
"rgengc_rememberset_mark: start\n");
8450 ccan_list_for_each(&heap->pages, page, page_node) {
8451 if (page->flags.has_remembered_objects | page->flags.has_uncollectible_wb_unprotected_objects) {
8452 uintptr_t p = page->start;
8453 short slot_size = page->slot_size;
8454 int total_slots = page->total_slots;
8455 int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
8456 bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT];
8457 bits_t *remembered_bits = page->remembered_bits;
8458 bits_t *uncollectible_bits = page->uncollectible_bits;
8459 bits_t *wb_unprotected_bits = page->wb_unprotected_bits;
8460#if PROFILE_REMEMBERSET_MARK
8461 if (page->flags.has_remembered_objects && page->flags.has_uncollectible_wb_unprotected_objects) has_both++;
8462 else if (page->flags.has_remembered_objects) has_old++;
8463 else if (page->flags.has_uncollectible_wb_unprotected_objects) has_shady++;
8471 page->flags.has_remembered_objects = FALSE;
8472 for (j=0; j < (size_t)bitmap_plane_count; j++) {
8474 | (uncollectible_bits[j] & wb_unprotected_bits[j]);
8477 for (j=0; j < (size_t)bitmap_plane_count; j++) {
8479 rgengc_rememberset_mark_plane(
objspace, p, bitset, slot_size);
8480 p += BITS_BITLENGTH * slot_size;
8483#if PROFILE_REMEMBERSET_MARK
8490#if PROFILE_REMEMBERSET_MARK
8491 fprintf(stderr,
"%d\t%d\t%d\t%d\n", has_both, has_old, has_shady, skip);
8493 gc_report(1,
objspace,
"rgengc_rememberset_mark: finished\n");
8497gc_bitmaps_clear(rb_objspace_t *
objspace, rb_heap_t *heap,
bool clear_shref)
8501 ccan_list_for_each(&heap->pages, page, page_node) {
8502 memset(&page->mark_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8503 memset(&page->uncollectible_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8504 memset(&page->marking_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8508 memset(&page->remembered_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8509 memset(&page->pinned_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8510 page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
8511 page->flags.has_remembered_objects = FALSE;
8515 memset(&page->shref_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
8516 page->flags.has_shref_objects = FALSE;
8523NOINLINE(
static void gc_writebarrier_generational(
VALUE a,
VALUE b, rb_objspace_t *
objspace));
8529 if (RGENGC_CHECK_MODE) {
8530 if (!RVALUE_OLD_P(
objspace, a)) rb_bug(
"gc_writebarrier_generational: %s is not an old object.", rb_obj_info(a));
8531 if ( RVALUE_OLD_P(
objspace, b)) rb_bug(
"gc_writebarrier_generational: %s is an old object.", rb_obj_info(b));
8532 if (is_incremental_marking(
objspace)) rb_bug(
"gc_writebarrier_generational: called while incremental marking: %s -> %s", rb_obj_info(a), rb_obj_info(b));
8538 if (!RVALUE_REMEMBERED(
objspace, a)) {
8541 gc_report(1,
objspace,
"gc_writebarrier_generational: %s (remembered) -> %s\n", rb_obj_info(a), rb_obj_info(b));
8544 check_rvalue_consistency(
objspace, a);
8545 check_rvalue_consistency(
objspace, b);
8551 gc_mark_set_parent(
objspace, parent);
8552 rgengc_check_relation(
objspace, obj);
8553 if (gc_mark_set(
objspace, obj) != FALSE) {
8557 gc_mark_set_parent_invalid(
objspace);
8560NOINLINE(
static void gc_writebarrier_incremental(
VALUE a,
VALUE b, rb_objspace_t *
objspace));
8566 gc_report(2,
objspace,
"gc_writebarrier_incremental: [LG] %p -> %s\n", (
void *)a, rb_obj_info(b));
8570 if (!RVALUE_WB_UNPROTECTED(
objspace, a)) {
8571 gc_report(2,
objspace,
"gc_writebarrier_incremental: [IN] %p -> %s\n", (
void *)a, rb_obj_info(b));
8579 if (RB_UNLIKELY(
objspace->flags.during_compacting)) {
8580 MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(b), b);
8586rb_gc_impl_writebarrier(
void *objspace_ptr,
VALUE a,
VALUE b)
8588 rb_objspace_t *
objspace = objspace_ptr;
8590#if RGENGC_CHECK_MODE
8591 if (
SPECIAL_CONST_P(a)) rb_bug(
"rb_gc_writebarrier: a is special const: %"PRIxVALUE, a);
8592 if (
SPECIAL_CONST_P(b)) rb_bug(
"rb_gc_writebarrier: b is special const: %"PRIxVALUE, b);
8598 GC_ASSERT(!during_gc);
8608 struct heap_page *bpage = GET_HEAP_PAGE(b);
8609 if (!_MARKED_IN_BITMAP(bpage->shref_bits, bpage, b)) {
8610 _MARK_IN_BITMAP(bpage->shref_bits, bpage, b);
8611 bpage->flags.has_shref_objects = TRUE;
8615 if (!is_incremental_marking(
objspace)) {
8619 if (gc_foreign_object_p(
objspace, a) || gc_foreign_object_p(
objspace, b) ||
8624 gc_writebarrier_generational(a, b,
objspace);
8629 if (gc_foreign_object_p(
objspace, a) || gc_foreign_object_p(
objspace, b)) {
8633 gc_writebarrier_incremental(a, b,
objspace);
8639rb_gc_impl_obj_became_shareable(
void *objspace_ptr,
VALUE obj)
8643 struct heap_page *page = GET_HEAP_PAGE(obj);
8645 if (_MARKED_IN_BITMAP(page->shareable_bits, page, obj))
return;
8646 gc_page_add_shareable(page, obj);
8651 if (_MARKED_IN_BITMAP(page->shref_bits, page, obj)) {
8652 _CLEAR_IN_BITMAP(page->shref_bits, page, obj);
8658rb_gc_impl_writebarrier_unprotect(
void *objspace_ptr,
VALUE obj)
8660 rb_objspace_t *
objspace = objspace_ptr;
8667 if (RVALUE_WB_UNPROTECTED(
objspace, obj)) {
8671 gc_report(2,
objspace,
"rb_gc_writebarrier_unprotect: %s %s\n", rb_obj_info(obj),
8672 RVALUE_REMEMBERED(
objspace, obj) ?
" (already remembered)" :
"");
8678 gc_report(1,
objspace,
"rb_gc_writebarrier_unprotect: %s\n", rb_obj_info(obj));
8681 gc_remember_unprotected(
objspace, obj);
8684 objspace->profile.total_shade_operation_count++;
8685#if RGENGC_PROFILE >= 2
8691 RVALUE_AGE_RESET(obj);
8694 RB_DEBUG_COUNTER_INC(obj_wb_unprotect);
8695 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
8700rb_gc_impl_copy_attributes(
void *objspace_ptr,
VALUE dest,
VALUE obj)
8702 rb_objspace_t *
objspace = objspace_ptr;
8704 if (RVALUE_WB_UNPROTECTED(
objspace, obj)) {
8705 rb_gc_impl_writebarrier_unprotect(
objspace, dest);
8707 rb_gc_impl_copy_finalizer(
objspace, dest, obj);
8711rb_gc_impl_active_gc_name(
void)
8719rb_gc_impl_writebarrier_remember(
void *objspace_ptr,
VALUE obj)
8722 rb_objspace_t *
objspace = objspace_ptr;
8726 if (RB_LIKELY(!gc_foreign_object_p(
objspace, obj))) {
8727 gc_report(1,
objspace,
"rb_gc_writebarrier_remember: %s\n", rb_obj_info(obj));
8728 if (is_incremental_marking(
objspace)) {
8729 if (RVALUE_BLACK_P(
objspace, obj)) {
8733 else if (RVALUE_OLD_P(
objspace, obj)) {
8741 ID ID_wb_protected, ID_age, ID_old, ID_uncollectible, ID_marking,
8742 ID_marked, ID_pinned, ID_remembered, ID_object_id, ID_shareable;
8745#define RB_GC_OBJECT_METADATA_ENTRY_COUNT (sizeof(struct rb_gc_object_metadata_names) / sizeof(ID))
8749rb_gc_impl_object_metadata(
void *objspace_ptr,
VALUE obj)
8751 rb_objspace_t *
objspace = objspace_ptr;
8755 if (!names.ID_marked) {
8756#define I(s) names.ID_##s = rb_intern(#s)
8770#define SET_ENTRY(na, v) do { \
8771 GC_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \
8772 object_metadata_entries[n].name = names.ID_##na; \
8773 object_metadata_entries[n].val = v; \
8777 if (!RVALUE_WB_UNPROTECTED(
objspace, obj)) SET_ENTRY(wb_protected,
Qtrue);
8778 SET_ENTRY(age,
INT2FIX(RVALUE_AGE_GET(obj)));
8780 if (RVALUE_UNCOLLECTIBLE(
objspace, obj)) SET_ENTRY(uncollectible,
Qtrue);
8781 if (RVALUE_MARKING(
objspace, obj)) SET_ENTRY(marking,
Qtrue);
8784 if (RVALUE_REMEMBERED(
objspace, obj)) SET_ENTRY(remembered,
Qtrue);
8785 if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj));
8788 object_metadata_entries[n].name = 0;
8789 object_metadata_entries[n].val = 0;
8792 return object_metadata_entries;
8796rb_gc_impl_ractor_cache_alloc(
void *objspace_ptr,
void *ractor)
8803rb_gc_impl_ractor_cache_free(
void *objspace_ptr,
void *cache)
8805 GC_ASSERT(cache == NULL);
8817finalize_deferred_dfree_only(rb_objspace_t *
objspace)
8819 VALUE dfree_only = 0;
8822 rb_asan_unpoison_object(zombie,
false);
8823 VALUE next = RZOMBIE(zombie)->next;
8826 VALUE prev2, next2 = heap_pages_deferred_final;
8828 RZOMBIE(zombie)->next = prev2 = next2;
8830 }
while (next2 != prev2);
8831 rb_asan_poison_object(zombie);
8834 RZOMBIE(zombie)->next = dfree_only;
8835 dfree_only = zombie;
8839 if (dfree_only) finalize_list(
objspace, dfree_only);
8840 bool did = dfree_only != 0;
8842 gc_tdata_unsafe_free_publish(
objspace);
8847rb_gc_impl_objspace_retire_gc(
void *objspace_ptr)
8849 rb_objspace_t *
objspace = objspace_ptr;
8853 objspace->flags.during_postmortem = 1;
8857 gc_start(
objspace, GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP);
8862 if (finalize_deferred_dfree_only(
objspace)) {
8863 gc_start(
objspace, GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP);
8866 heap_pages_freeable_pages =
objspace->empty_pages_count;
8867 heap_pages_free_unused_pages(
objspace);
8869 objspace->flags.during_postmortem = 0;
8873rb_gc_impl_during_postmortem_p(
void *objspace_ptr)
8875 rb_objspace_t *
objspace = objspace_ptr;
8876 return objspace->flags.during_postmortem != 0;
8880heap_ready_to_gc(rb_objspace_t *
objspace, rb_heap_t *heap)
8882 if (!heap->free_pages) {
8883 if (!heap_page_allocate_and_initialize(
objspace, heap)) {
8884 objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE;
8885 heap_page_allocate_and_initialize(
objspace, heap);
8891ready_to_gc(rb_objspace_t *
objspace)
8893 if ((!
objspace->flags.during_postmortem && rb_gc_gc_disabled_global_p()) || dont_gc_val() || during_gc) {
8894 for (
int i = 0; i < HEAP_COUNT; i++) {
8895 rb_heap_t *heap = &heaps[i];
8906gc_reset_malloc_info(rb_objspace_t *
objspace,
bool full_mark)
8910 int64_t inc = gc_malloc_counters_increase(
objspace, &
objspace->malloc_counters.counters);
8911 gc_malloc_counters_snapshot(
objspace, &
objspace->malloc_counters.counters);
8912 size_t old_limit = malloc_limit;
8917 if (inc > 0 && (
size_t)inc > malloc_limit) {
8918 malloc_limit = (size_t)((
size_t)inc * gc_params.malloc_limit_growth_factor);
8919 if (malloc_limit > gc_params.malloc_limit_max) {
8920 malloc_limit = gc_params.malloc_limit_max;
8924 malloc_limit = (size_t)(malloc_limit * 0.98);
8925 if (malloc_limit < gc_params.malloc_limit_min) {
8926 malloc_limit = gc_params.malloc_limit_min;
8931 if (old_limit != malloc_limit) {
8932 fprintf(stderr,
"[%"PRIuSIZE
"] malloc_limit: %"PRIuSIZE
" -> %"PRIuSIZE
"\n",
8936 fprintf(stderr,
"[%"PRIuSIZE
"] malloc_limit: not changed (%"PRIuSIZE
")\n",
8943#if RGENGC_ESTIMATE_OLDMALLOC
8945 gc_counter_store_release(&
objspace->malloc_counters.freed_in_gc, 0);
8950 if ((
objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) {
8951 objspace->rgengc.oldmalloc_increase_limit =
8952 (size_t)(
objspace->rgengc.oldmalloc_increase_limit / ((gc_params.oldmalloc_limit_growth_factor - 1)/10 + 1));
8953 if (
objspace->rgengc.oldmalloc_increase_limit < gc_params.oldmalloc_limit_min) {
8954 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
8964gc_start_record(rb_objspace_t *
objspace,
unsigned int reason,
bool full_mark)
8966 objspace->profile.latest_gc_info = reason;
8967#if GC_PROFILE_MORE_DETAIL
8968 objspace->profile.total_allocated_objects_at_gc_start = total_allocated_objects(
objspace);
8969 objspace->profile.heap_used_at_gc_start = rb_darray_size(
objspace->heap_pages.sorted);
8971 objspace->profile.weak_references_count = 0;
8972 gc_prof_setup_new_record(
objspace, reason);
8973 gc_reset_malloc_info(
objspace, full_mark);
8976static bool gc_start_global(rb_objspace_t *driver,
unsigned int reason,
bool compact,
bool allow_skip);
8982gc_need_global_p(rb_objspace_t *
objspace)
8984 if (rb_gc_single_objspace_p())
return false;
8986 if (
objspace->flags.during_postmortem)
return false;
8987 if (
objspace->shareable_objects >
objspace->shareable_objects_limit)
return true;
8992 size_t zp = global_objspace->zombie_total_pages;
8993 size_t base = global_objspace->zombie_pages_survivors < zp ? global_objspace->zombie_pages_survivors : zp;
8994 if (zp - base >= ZOMBIE_PAGES_TRIGGER)
return true;
9000garbage_collect(rb_objspace_t *
objspace,
unsigned int reason)
9004#if GC_PROFILE_MORE_DETAIL
9005 objspace->profile.prepare_time = getrusage_time();
9010#if GC_PROFILE_MORE_DETAIL
9011 objspace->profile.prepare_time = getrusage_time() -
objspace->profile.prepare_time;
9020gc_start(rb_objspace_t *
objspace,
unsigned int reason)
9022 unsigned int do_full_mark = !!(reason & GPR_FLAG_FULL_MARK);
9024 if (!rb_darray_size(
objspace->heap_pages.sorted))
return TRUE;
9025 if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(
objspace))
return TRUE;
9029 if (!(reason & GPR_FLAG_METHOD) && gc_need_global_p(
objspace)) {
9031 if (gc_start_global(
objspace, reason, ruby_enable_autocompact,
true)) {
9037 rb_gc_initialize_vm_context(&
objspace->vm_context);
9039 GC_ASSERT(gc_mode(
objspace) == gc_mode_none,
"gc_mode is %s\n", gc_mode_name(gc_mode(
objspace)));
9040 GC_ASSERT(!is_lazy_sweeping(
objspace));
9041 GC_ASSERT(!is_incremental_marking(
objspace));
9044 objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP);
9046 if (ruby_gc_stressful) {
9047 int flag =
FIXNUM_P(ruby_gc_stress_mode) ?
FIX2INT(ruby_gc_stress_mode) : 0;
9049 if ((flag & (1 << gc_stress_no_major)) == 0) {
9050 do_full_mark = TRUE;
9053 objspace->flags.immediate_sweep = !(flag & (1<<gc_stress_no_immediate_sweep));
9056 if (gc_needs_major_flags) {
9057 reason |= gc_needs_major_flags;
9058 do_full_mark = TRUE;
9062 if (!gc_config_full_mark_val) {
9063 do_full_mark = FALSE;
9065 gc_needs_major_flags = GPR_FLAG_NONE;
9067 if (do_full_mark && (reason & GPR_FLAG_MAJOR_MASK) == 0) {
9068 reason |= GPR_FLAG_MAJOR_BY_FORCE;
9071 if (
objspace->flags.dont_incremental ||
9072 reason & GPR_FLAG_IMMEDIATE_MARK ||
9073 ruby_gc_stressful) {
9074 gc_set_incremental_marking(
objspace, FALSE);
9077 gc_set_incremental_marking(
objspace, do_full_mark);
9084 if (do_full_mark && ruby_enable_autocompact && rb_gc_single_objspace_p()) {
9085 objspace->flags.during_compacting = TRUE;
9086#if RGENGC_CHECK_MODE
9087 objspace->rcompactor.compare_func = ruby_autocompact_compare_func;
9091 objspace->flags.during_compacting = !!(reason & GPR_FLAG_COMPACT);
9092 if (
objspace->flags.during_compacting && !rb_gc_single_objspace_p()) {
9094 objspace->flags.during_compacting = FALSE;
9098 if (!GC_ENABLE_LAZY_SWEEP ||
objspace->flags.dont_incremental) {
9099 objspace->flags.immediate_sweep = TRUE;
9102 if (
objspace->flags.immediate_sweep) reason |= GPR_FLAG_IMMEDIATE_SWEEP;
9105 unsigned int lock_lev;
9106 gc_enter(
objspace, gc_enter_event_start, &lock_lev);
9108 gc_report(1,
objspace,
"gc_start(reason: %x) => %u, %d, %d\n",
9110 do_full_mark, !is_incremental_marking(
objspace),
objspace->flags.immediate_sweep);
9112 RB_DEBUG_COUNTER_INC(gc_count);
9114 if (reason & GPR_FLAG_MAJOR_MASK) {
9115 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_nofree, reason & GPR_FLAG_MAJOR_BY_NOFREE);
9116 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldgen, reason & GPR_FLAG_MAJOR_BY_OLDGEN);
9117 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_shady, reason & GPR_FLAG_MAJOR_BY_SHADY);
9118 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_force, reason & GPR_FLAG_MAJOR_BY_FORCE);
9119#if RGENGC_ESTIMATE_OLDMALLOC
9120 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldmalloc, reason & GPR_FLAG_MAJOR_BY_OLDMALLOC);
9124 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_newobj, reason & GPR_FLAG_NEWOBJ);
9125 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_malloc, reason & GPR_FLAG_MALLOC);
9126 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_method, reason & GPR_FLAG_METHOD);
9127 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_capi, reason & GPR_FLAG_CAPI);
9128 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_stress, reason & GPR_FLAG_STRESS);
9132 gc_start_record(
objspace, reason, do_full_mark);
9136 GC_ASSERT(during_gc);
9140 if (gc_marks(
objspace, do_full_mark)) {
9146 gc_exit(
objspace, gc_enter_event_start, &lock_lev);
9151#if RGENGC_CHECK_MODE >= 2
9152 gc_verify_internal_consistency(
objspace);
9161 unsigned int lock_lev;
9162 gc_enter(
objspace, gc_enter_event_rest, &lock_lev);
9164 if (is_incremental_marking(
objspace)) {
9178 gc_exit(
objspace, gc_enter_event_rest, &lock_lev);
9180 if (RGENGC_CHECK_MODE >= 2) gc_verify_internal_consistency(
objspace);
9185 rb_objspace_t *objspace;
9186 unsigned int reason;
9190gc_current_status_fill(rb_objspace_t *
objspace,
char *buff)
9195 if (is_full_marking(
objspace)) buff[i++] =
'F';
9196 if (is_incremental_marking(
objspace)) buff[i++] =
'I';
9200 if (is_lazy_sweeping(
objspace)) buff[i++] =
'L';
9209gc_current_status(rb_objspace_t *
objspace)
9211 static char buff[0x10];
9212 gc_current_status_fill(
objspace, buff);
9216#if PRINT_ENTER_EXIT_TICK
9218static tick_t last_exit_tick;
9219static tick_t enter_tick;
9220static int enter_count = 0;
9221static char last_gc_status[0x10];
9224gc_record(rb_objspace_t *
objspace,
int direction,
const char *event)
9226 if (direction == 0) {
9228 enter_tick = tick();
9229 gc_current_status_fill(
objspace, last_gc_status);
9232 tick_t exit_tick = tick();
9233 char current_gc_status[0x10];
9234 gc_current_status_fill(
objspace, current_gc_status);
9237 fprintf(stderr,
"%"PRItick
"\t%"PRItick
"\t%s\t[%s->%s|%c]\n",
9238 enter_tick - last_exit_tick,
9239 exit_tick - enter_tick,
9241 last_gc_status, current_gc_status,
9242 (
objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ?
'+' :
'-');
9243 last_exit_tick = exit_tick;
9246 fprintf(stderr,
"%"PRItick
"\t%"PRItick
"\t%s\t[%s->%s|%c]\n",
9248 exit_tick - enter_tick,
9250 last_gc_status, current_gc_status,
9251 (
objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ?
'+' :
'-');
9257gc_record(rb_objspace_t *
objspace,
int direction,
const char *event)
9264gc_enter_event_cstr(
enum gc_enter_event event)
9267 case gc_enter_event_start:
return "start";
9268 case gc_enter_event_continue:
return "continue";
9269 case gc_enter_event_rest:
return "rest";
9270 case gc_enter_event_finalizer:
return "finalizer";
9271 case gc_enter_event_global:
return "global";
9272 case gc_enter_event_global_auto:
return "global_auto";
9278gc_enter_count(
enum gc_enter_event event)
9281 case gc_enter_event_start: RB_DEBUG_COUNTER_INC(gc_enter_start);
break;
9282 case gc_enter_event_continue: RB_DEBUG_COUNTER_INC(gc_enter_continue);
break;
9283 case gc_enter_event_rest: RB_DEBUG_COUNTER_INC(gc_enter_rest);
break;
9284 case gc_enter_event_finalizer: RB_DEBUG_COUNTER_INC(gc_enter_finalizer);
break;
9285 case gc_enter_event_global: RB_DEBUG_COUNTER_INC(gc_enter_start);
break;
9286 case gc_enter_event_global_auto: RB_DEBUG_COUNTER_INC(gc_enter_start);
break;
9290static bool current_process_time(
struct timespec *ts);
9298current_thread_time(
struct timespec *ts)
9300#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_THREAD_CPUTIME_ID)
9302 static int try_clock_gettime = 1;
9303 if (try_clock_gettime) {
9304 if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, ts) == 0) {
9308 try_clock_gettime = 0;
9313 return current_process_time(ts);
9319 if (!current_thread_time(ts)) {
9325static unsigned long long
9330 if ((ts->tv_sec > 0 || ts->tv_nsec > 0) &&
9331 current_thread_time(&end_time) &&
9332 end_time.tv_sec >= ts->tv_sec) {
9333 return (
unsigned long long)(end_time.tv_sec - ts->tv_sec) * (1000 * 1000 * 1000) +
9334 (end_time.tv_nsec - ts->tv_nsec);
9341gc_process_stat_after_fork_i(
void *objspace_ptr,
void *data)
9343 rb_objspace_t *
objspace = objspace_ptr;
9348gc_local_gc_holds_vm_lock(
void)
9350 return rb_gc_single_objspace_p();
9354gc_enter(rb_objspace_t *
objspace,
enum gc_enter_event event,
unsigned int *lock_lev)
9363 RUBY_DTRACE_GC_HOOK(ENTER, event);
9367 case gc_enter_event_start:
9368 case gc_enter_event_continue:
9369 case gc_enter_event_rest:
9370 case gc_enter_event_global:
9371 case gc_enter_event_global_auto:
9376 objspace->profile.gc_pause_start_time = rb_hrtime_now();
9378 case gc_enter_event_finalizer:
9383 case gc_enter_event_global:
9384 *lock_lev = RB_GC_VM_LOCK();
9388 case gc_enter_event_global_auto:
9389 *lock_lev = RB_GC_VM_LOCK();
9391 RB_GC_VM_UNLOCK(*lock_lev);
9393 objspace->profile.gc_pause_start_time = 0;
9398 case gc_enter_event_finalizer:
9401 *lock_lev = RB_GC_VM_LOCK_NO_BARRIER();
9404 objspace->flags.gc_lock_barrier = FALSE;
9405 if (
objspace->flags.during_compacting) {
9410 *lock_lev = RB_GC_VM_LOCK();
9412 objspace->flags.gc_lock_barrier = TRUE;
9414 else if (gc_local_gc_holds_vm_lock()) {
9415 *lock_lev = RB_GC_VM_LOCK_NO_BARRIER();
9420 if (
objspace->profile.gc_pause_start_time) {
9421 objspace->profile.gc_stw_start_time = rb_hrtime_now();
9422 objspace->profile.gc_stop_time = rb_hrtime_sub(
9423 objspace->profile.gc_stw_start_time,
9424 objspace->profile.gc_pause_start_time);
9427 gc_enter_count(event);
9428 if (RB_UNLIKELY(during_gc != 0)) rb_bug(
"during_gc != 0");
9429 if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(
objspace);
9432 RUBY_DEBUG_LOG(
"%s (%s)",gc_enter_event_cstr(event), gc_current_status(
objspace));
9433 gc_report(1,
objspace,
"gc_enter: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(
objspace));
9434 gc_record(
objspace, 0, gc_enter_event_cstr(event));
9441gc_exit(rb_objspace_t *
objspace,
enum gc_enter_event event,
unsigned int *lock_lev)
9443 GC_ASSERT(during_gc != 0);
9445 RUBY_DTRACE_GC_HOOK(EXIT, event);
9449 if (
objspace->profile.gc_pause_start_time) {
9451 rb_hrtime_t now = rb_hrtime_now();
9453 record->gc_pause_time = rb_hrtime_add(record->gc_pause_time,
9454 rb_hrtime_sub(now,
objspace->profile.gc_pause_start_time));
9455 record->gc_stop_time = rb_hrtime_add(record->gc_stop_time,
9457 record->gc_stw_time = rb_hrtime_add(record->gc_stw_time,
9458 rb_hrtime_sub(now,
objspace->profile.gc_stw_start_time));
9460 objspace->profile.gc_pause_start_time = 0;
9461 objspace->profile.gc_stw_start_time = 0;
9462 objspace->profile.gc_stop_time = 0;
9465 gc_record(
objspace, 1, gc_enter_event_cstr(event));
9466 RUBY_DEBUG_LOG(
"%s (%s)", gc_enter_event_cstr(event), gc_current_status(
objspace));
9467 gc_report(1,
objspace,
"gc_exit: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(
objspace));
9472 case gc_enter_event_global:
9473 case gc_enter_event_global_auto:
9474 RB_GC_VM_UNLOCK(*lock_lev);
9476 case gc_enter_event_finalizer:
9477 RB_GC_VM_UNLOCK_NO_BARRIER(*lock_lev);
9480 if (*lock_lev != 0) {
9481 if (
objspace->flags.gc_lock_barrier) {
9482 objspace->flags.gc_lock_barrier = FALSE;
9483 RB_GC_VM_UNLOCK(*lock_lev);
9486 RB_GC_VM_UNLOCK_NO_BARRIER(*lock_lev);
9494#define MEASURE_GC (objspace->flags.measure_gc)
9498gc_marking_enter(rb_objspace_t *
objspace)
9500 GC_ASSERT(during_gc != 0);
9502 gc_prof_mark_timer_start(
objspace);
9505 objspace->profile.gc_mark_phase_wall_start_time = rb_hrtime_now();
9509 gc_clock_start(&
objspace->profile.marking_start_time);
9512 rb_gc_initialize_vm_context(&
objspace->vm_context);
9516gc_marking_exit(rb_objspace_t *
objspace)
9518 GC_ASSERT(during_gc != 0);
9521 objspace->profile.marking_time_ns += gc_clock_end(&
objspace->profile.marking_start_time);
9526 record->gc_mark_wall_time = rb_hrtime_add(record->gc_mark_wall_time,
9527 elapsed_hrtime_from(
objspace->profile.gc_mark_phase_wall_start_time));
9534gc_sweeping_cpu_enter(rb_objspace_t *
objspace)
9537 gc_clock_start(&
objspace->profile.sweeping_start_time);
9542gc_sweeping_cpu_exit(rb_objspace_t *
objspace)
9545 objspace->profile.sweeping_time_ns += gc_clock_end(&
objspace->profile.sweeping_start_time);
9550gc_sweeping_enter(rb_objspace_t *
objspace)
9552 GC_ASSERT(during_gc != 0);
9555 objspace->profile.gc_sweep_phase_wall_start_time = rb_hrtime_now();
9556 objspace->profile.gc_sweep_excluded_wall_time = 0;
9561 rb_gc_initialize_vm_context(&
objspace->vm_context);
9565gc_sweeping_exit(rb_objspace_t *
objspace)
9567 GC_ASSERT(during_gc != 0);
9572 rb_hrtime_t sweep_wall_time = elapsed_hrtime_from(
objspace->profile.gc_sweep_phase_wall_start_time);
9574 sweep_wall_time = rb_hrtime_sub(sweep_wall_time,
9575 objspace->profile.gc_sweep_excluded_wall_time);
9576 record->gc_sweep_wall_time = rb_hrtime_add(record->gc_sweep_wall_time,
9578 objspace->profile.gc_sweep_excluded_wall_time = 0;
9583gc_with_gvl(
void *ptr)
9586 return (
void *)(
VALUE)garbage_collect(oar->objspace, oar->reason);
9592garbage_collect_with_gvl(rb_objspace_t *
objspace,
unsigned int reason)
9594 if (rb_gc_gc_disabled_global_p() || dont_gc_val()) {
9604 oar.reason = reason;
9610 return garbage_collect(
objspace, reason);
9615gc_set_candidate_object_i(
void *vstart,
void *vend,
size_t stride,
void *data)
9617 rb_objspace_t *
objspace = (rb_objspace_t *)data;
9620 for (; v != (
VALUE)vend; v += stride) {
9621 asan_unpoisoning_object(v) {
9627 rb_gc_prepare_heap_process_object(v);
9629 RVALUE_AGE_SET_CANDIDATE(
objspace, v);
9639rb_gc_impl_multi_objspace_p(
void)
9645rb_gc_impl_during_global_gc_p(
void *objspace_ptr)
9647 rb_objspace_t *
objspace = objspace_ptr;
9648 return objspace->flags.during_global_gc != 0;
9652rb_gc_impl_obj_foreign_p(
void *objspace_ptr,
VALUE obj)
9654 if (RB_LIKELY(!rb_gc_ever_multi_ractor_p()))
return false;
9660rb_gc_impl_objspace_set_ractor(
void *objspace_ptr,
void *ractor)
9662 ((rb_objspace_t *)objspace_ptr)->ractor = ractor;
9667rb_gc_impl_obj_ractor(
void *objspace_ptr,
VALUE obj)
9669 return GET_HEAP_OBJSPACE(obj)->ractor;
9677rb_gc_impl_shref_marked_p(
void *objspace_ptr,
VALUE obj)
9679 return MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj) != 0;
9688objspace_zombie_pages(rb_objspace_t *
objspace)
9690 size_t pages = rb_darray_size(
objspace->heap_pages.sorted);
9691 return pages > HEAP_COUNT ? pages : HEAP_COUNT;
9696rb_gc_impl_objspace_zombify(
void *objspace_ptr)
9698 rb_objspace_t *
objspace = objspace_ptr;
9699 GC_ASSERT(
objspace->zombie_pages == 0);
9701 global_objspace->zombie_total_pages +=
objspace->zombie_pages;
9707gc_global_refresh_zombie_pages(
void)
9709 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9710 rb_objspace_t *
objspace = global_objspace->global_gc.objspaces[i];
9711 if (
objspace->zombie_pages == 0)
continue;
9712 size_t pages = objspace_zombie_pages(
objspace);
9713 global_objspace->zombie_total_pages -=
objspace->zombie_pages;
9714 global_objspace->zombie_total_pages += pages;
9717 global_objspace->zombie_pages_survivors = global_objspace->zombie_total_pages;
9721gc_global_objspaces_i(
void *os,
void *data)
9723 if (global_objspace->global_gc.n_objspaces == global_objspace->global_gc.objspaces_capa) {
9724 size_t new_capa = global_objspace->global_gc.objspaces_capa ? global_objspace->global_gc.objspaces_capa * 2 : 16;
9725 struct rb_objspace **new_list = realloc(global_objspace->global_gc.objspaces, new_capa *
sizeof(*new_list));
9726 if (new_list == NULL) rb_bug(
"gc_global_objspaces_i: realloc failed");
9727 global_objspace->global_gc.objspaces = new_list;
9728 global_objspace->global_gc.objspaces_capa = new_capa;
9730 global_objspace->global_gc.objspaces[global_objspace->global_gc.n_objspaces++] = os;
9736gc_global_snapshot_objspaces(
void)
9738 global_objspace->global_gc.n_objspaces = 0;
9739 rb_gc_vm_each_objspace(gc_global_objspaces_i, NULL);
9741#if RGENGC_CHECK_MODE
9744 size_t total = 0, zombie_total = 0;
9745 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9746 total += rb_darray_size(global_objspace->global_gc.objspaces[i]->heap_pages.sorted);
9747 zombie_total += global_objspace->global_gc.objspaces[i]->zombie_pages;
9749 GC_ASSERT(total == global_objspace->page_index.n_pages);
9751 GC_ASSERT(zombie_total == global_objspace->zombie_total_pages);
9764 rb_objspace_t *objspace;
9769genfields_mark_i(
VALUE key,
VALUE val,
void *arg)
9780 bool newly = !RVALUE_MARKED_BITMAP(val);
9781 gc_mark_set_parent(a->objspace, key);
9782 gc_mark(a->objspace, val);
9783 if (newly) a->progress =
true;
9788genfields_dead_p(
VALUE key)
9790 return RVALUE_MARKED_BITMAP(key) == 0;
9794gc_global_mark_generic_fields(rb_objspace_t *driver)
9798 arg.progress =
false;
9802 rb_gc_vm_generic_fields_mark_foreach(genfields_mark_i, &arg);
9803 gc_mark_set_parent_invalid(driver);
9805 gc_mark_stacked_objects_all(driver);
9807 }
while (arg.progress);
9809 rb_gc_vm_generic_fields_drain_dead(genfields_dead_p);
9816gc_start_global(rb_objspace_t *driver,
unsigned int reason,
bool compact,
bool allow_skip)
9818 unsigned int lock_lev;
9819 enum gc_enter_event
event = allow_skip ? gc_enter_event_global_auto : gc_enter_event_global;
9820 if (!gc_enter(driver, event, &lock_lev)) {
9824 reason |= GPR_FLAG_GLOBAL;
9828 gc_start_record(driver, reason,
true);
9830 gc_prof_timer_start(driver);
9832 gc_global_snapshot_objspaces();
9838 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9839 global_objspace->global_gc.objspaces[i]->flags.during_global_gc = TRUE;
9845 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9846 rb_objspace_t *
const os = global_objspace->global_gc.objspaces[i];
9848 os->profile.latest_gc_info = reason;
9849 gc_reset_malloc_info(os,
true);
9857 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9858 rb_objspace_t *
objspace = global_objspace->global_gc.objspaces[i];
9864 if (is_incremental_marking(
objspace)) {
9865 gc_abort_incremental_marking(
objspace);
9867 GC_ASSERT(!is_incremental_marking(
objspace));
9868 GC_ASSERT(is_mark_stack_empty(&
objspace->mark_stack));
9869 rb_gc_initialize_vm_context(&
objspace->vm_context);
9870 if (
objspace != driver) during_gc = TRUE;
9877 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9878 rb_objspace_t *
objspace = global_objspace->global_gc.objspaces[i];
9879 objspace->flags.during_minor_gc = FALSE;
9880 gc_set_incremental_marking(
objspace, FALSE);
9884 objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
9887 objspace->profile.global_gc_count++;
9889 objspace->rgengc.need_major_gc = GPR_FLAG_NONE;
9891 for (
int h = 0; h < HEAP_COUNT; h++) {
9892 rb_heap_t *heap = &heaps[h];
9893 gc_bitmaps_clear(
objspace, heap,
true);
9894 heap_move_pooled_pages_to_free_pages(heap);
9897 driver->profile.global_gc_invocation_count++;
9898 global_objspace->global_gc.count++;
9904 global_objspace->global_gc.compacting = compact;
9906 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9907 rb_objspace_t *
objspace = global_objspace->global_gc.objspaces[i];
9908 objspace->flags.during_compacting = TRUE;
9909#if RGENGC_CHECK_MODE
9910 if (ruby_enable_autocompact) {
9911 objspace->rcompactor.compare_func = ruby_autocompact_compare_func;
9917 for (
int h = 0; h < HEAP_COUNT; h++) {
9919 ccan_list_for_each(&heaps[h].pages, page, page_node) {
9920 page->pinned_slots = 0;
9930 gc_marking_enter(driver);
9932 mark_roots(driver, NULL);
9933 gc_mark_stacked_objects_all(driver);
9938 gc_global_mark_generic_fields(driver);
9943 gc_update_weak_references(driver);
9948 rb_ractor_finish_marking(
true);
9950 gc_marking_exit(driver);
9955 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9956 rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
9957 unsigned int prev_immediate = os->flags.immediate_sweep;
9958 os->flags.immediate_sweep = TRUE;
9960 os->flags.immediate_sweep = prev_immediate;
9976 const bool driver_prof = gc_prof_enabled(driver);
9977 rb_hrtime_t driver_compact_wall_time = 0;
9980 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9981 rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
9982 gc_sweeping_enter(os);
9984 rb_hrtime_t t0 = (os == driver && driver_prof) ? rb_hrtime_now() : 0;
9985 gc_compact_relocate(os);
9986 if (os == driver && driver_prof) {
9987 driver_compact_wall_time = rb_hrtime_add(driver_compact_wall_time, elapsed_hrtime_from(t0));
9989 gc_sweeping_cpu_exit(os);
9996 gc_sweeping_cpu_enter(driver);
9997 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
9998 global_objspace->global_gc.objspaces[i]->flags.during_reference_updating = TRUE;
10001 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10002 rb_objspace_t *
objspace = global_objspace->global_gc.objspaces[i];
10003 for (
int h = 0; h < HEAP_COUNT; h++) {
10004 gc_unprotect_pages(
objspace, &heaps[h]);
10007 rb_gc_before_updating_jit_code();
10008 gc_sweeping_cpu_exit(driver);
10009 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10010 rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
10011 gc_sweeping_cpu_enter(os);
10012 rb_hrtime_t t0 = (os == driver && driver_prof) ? rb_hrtime_now() : 0;
10013 gc_compact_finish(os);
10014 if (os == driver && driver_prof) {
10015 driver_compact_wall_time = rb_hrtime_add(driver_compact_wall_time, elapsed_hrtime_from(t0));
10017 gc_sweeping_cpu_exit(os);
10022 gc_sweeping_cpu_enter(driver);
10023 rb_hrtime_t t0 = driver_prof ? rb_hrtime_now() : 0;
10024 gc_update_references_global(driver);
10026 driver_compact_wall_time = rb_hrtime_add(driver_compact_wall_time, elapsed_hrtime_from(t0));
10028 gc_sweeping_cpu_exit(driver);
10030 gc_sweeping_cpu_enter(driver);
10031 rb_gc_after_updating_jit_code();
10032 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10033 global_objspace->global_gc.objspaces[i]->flags.during_reference_updating = FALSE;
10034 global_objspace->global_gc.objspaces[i]->flags.during_compacting = FALSE;
10036 global_objspace->global_gc.compacting =
false;
10037 uninstall_handlers();
10038 gc_sweeping_cpu_exit(driver);
10046 record->gc_compact_wall_time = rb_hrtime_add(record->gc_compact_wall_time,
10047 driver_compact_wall_time);
10048 driver->profile.gc_sweep_excluded_wall_time = rb_hrtime_add(
10049 driver->profile.gc_sweep_excluded_wall_time, driver_compact_wall_time);
10055 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10056 rb_objspace_t *os = global_objspace->global_gc.objspaces[i];
10057 gc_sweeping_cpu_enter(os);
10059 gc_sweeping_exit(os);
10062 global_objspace->global_gc.compacting =
false;
10069 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10070 rb_objspace_t *
objspace = global_objspace->global_gc.objspaces[i];
10071 if (
objspace->heap_pages.allocatable_bytes != 0 ||
objspace->empty_pages_count != 0) {
10074 bool stuck =
false;
10075 for (
int h = 0; h < HEAP_COUNT; h++) {
10076 if (heaps[h].free_pages == NULL) { stuck =
true;
break; }
10079 heap_allocatable_bytes_expand(
objspace, NULL, 0,
10080 objspace_available_slots(
objspace), heaps[0].slot_size);
10086 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10087 rb_objspace_t *
objspace = global_objspace->global_gc.objspaces[i];
10088 size_t survivors = 0;
10089 for (
int h = 0; h < HEAP_COUNT; h++) {
10091 ccan_list_for_each(&heaps[h].pages, page, page_node) {
10092 if (!page->flags.has_shareable_objects)
continue;
10093 for (
int j = 0; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
10094 survivors += rb_popcount_intptr(page->shareable_bits[j]);
10098 objspace->shareable_objects = survivors;
10099 size_t new_limit = (size_t)(survivors * SHAREABLE_OBJECTS_LIMIT_FACTOR);
10100 if (new_limit < SHAREABLE_OBJECTS_LIMIT_MIN) new_limit = SHAREABLE_OBJECTS_LIMIT_MIN;
10101 objspace->shareable_objects_limit = new_limit;
10107 gc_tdata_unsafe_drain_objspaces(global_objspace->global_gc.objspaces,
10108 global_objspace->global_gc.n_objspaces);
10111 for (
size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
10112 rb_objspace_t *
objspace = global_objspace->global_gc.objspaces[i];
10113 objspace->flags.during_global_gc = FALSE;
10116 gc_process_stat_publish(
objspace);
10123 rb_gc_reset_absorbed_since_global_gc();
10128 gc_global_refresh_zombie_pages();
10134 gc_prof_timer_stop(driver);
10135 gc_exit(driver, event, &lock_lev);
10140absorb_finalizer_i(st_data_t key, st_data_t val, st_data_t data)
10142 rb_objspace_t *
objspace = (rb_objspace_t *)data;
10143 st_insert(finalizer_table, key, val);
10144 return ST_CONTINUE;
10148gc_make_mid_mark_objspace_absorbable(rb_objspace_t *src)
10151 for (
int h = 0; h < HEAP_COUNT; h++) {
10153 ccan_list_for_each(&heaps[h].pages, page, page_node) {
10154 short stride = page->slot_size;
10155 uintptr_t p = (uintptr_t)page->start;
10156 uintptr_t pend = p + page->total_slots * stride;
10158 for (; p < pend; p += stride) {
10160 asan_unpoisoning_object(vp) {
10166 RVALUE_AGE_RESET(vp);
10172 gc_bitmaps_clear(
objspace, &heaps[h],
false);
10175 objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
10185objspace_absorb(rb_objspace_t *dst, rb_objspace_t *src)
10187 GC_ASSERT(dst != src);
10191 const bool prev_absorb = global_objspace->during_absorb;
10192 global_objspace->during_absorb =
true;
10196 GC_ASSERT(src->zombie_pages != 0);
10197 GC_ASSERT(global_objspace->zombie_total_pages >= src->zombie_pages);
10198 global_objspace->zombie_total_pages -= src->zombie_pages;
10199 src->zombie_pages = 0;
10213 if (is_incremental_marking(
objspace)) {
10214 gc_abort_incremental_marking(
objspace);
10215 gc_make_mid_mark_objspace_absorbable(
objspace);
10226 for (
int h = 0; h < HEAP_COUNT; h++) {
10227 heap_move_pooled_pages_to_free_pages(&heaps[h]);
10231 for (
int h = 0; h < HEAP_COUNT; h++) {
10232 rb_heap_t *heap = &heaps[h];
10233 struct heap_page **link = &heap->free_pages;
10236 if (page->free_slots == page->total_slots) {
10237 *link = page->free_next;
10240 link = &page->free_next;
10245 ccan_list_for_each_safe(&heap->pages, page, next, page_node) {
10246 if (page->free_slots == page->total_slots) {
10247 GC_ASSERT(page->final_slots == 0);
10248 heap_page_detach_empty(
objspace, heap, page);
10259 const bool dst_gc_was_enabled = rb_gc_impl_gc_enabled_p(dst);
10260 if (dst_gc_was_enabled) rb_gc_impl_gc_disable(dst,
false);
10264 rb_heap_t *dst_heaps;
10265 rb_heap_t *src_heaps;
10274 for (
int h = 0; h < HEAP_COUNT; h++) {
10275 rb_heap_t *dheap = &dst_heaps[h];
10276 rb_heap_t *sheap = &src_heaps[h];
10279 GC_ASSERT(sheap->sweeping_page == NULL);
10280 GC_ASSERT(sheap->pooled_pages == NULL);
10282 ccan_list_for_each(&sheap->pages, page, page_node) {
10284 page->heap = dheap;
10286 ccan_list_append_list(&dheap->pages, &sheap->pages);
10289 if (sheap->free_pages) {
10290 struct heap_page **tail = &dheap->free_pages;
10291 while (*tail) tail = &(*tail)->free_next;
10292 *tail = sheap->free_pages;
10293 sheap->free_pages = NULL;
10296 dheap->total_pages += sheap->total_pages;
10297 dheap->total_slots += sheap->total_slots;
10298 dheap->total_allocated_pages += sheap->total_allocated_pages;
10299 dheap->total_allocated_objects += sheap->total_allocated_objects;
10300 dheap->total_freed_objects += sheap->total_freed_objects;
10301 dheap->final_slots_count += sheap->final_slots_count;
10308 size_t srcn = rb_darray_size(src->heap_pages.sorted);
10309 for (
size_t i = 0; i < srcn; i++) {
10310 page = rb_darray_get(src->heap_pages.sorted, i);
10314 if (heap_page_in_global_empty_pages_pool(src, page)) {
10315 heap_page_free(src, page);
10318 uintptr_t body = (uintptr_t)page->body;
10320 uintptr_t end = body + HEAP_PAGE_SIZE;
10327 size_t hi = rb_darray_size(
objspace->heap_pages.sorted);
10329 size_t mid = (lo + hi) / 2;
10331 if ((uintptr_t)mid_page->body < body) lo = mid + 1;
10334 rb_darray_insert_without_gc(&
objspace->heap_pages.sorted, hi, page);
10336 if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
10337 if (heap_pages_himem < end) heap_pages_himem = end;
10339 objspace->heap_pages.allocated_pages += src->heap_pages.allocated_pages;
10340 objspace->heap_pages.freed_pages += src->heap_pages.freed_pages;
10341 rb_darray_free_without_gc(src->heap_pages.sorted);
10342 src->heap_pages.sorted = NULL;
10344 src->empty_pages = NULL;
10345 src->empty_pages_count = 0;
10351 st_table *src_finalizers;
10354 src_finalizers = finalizer_table;
10355 finalizer_table = NULL;
10357 if (src_finalizers) {
10359 if (finalizer_table == NULL) {
10360 finalizer_table = src_finalizers;
10363 st_foreach(src_finalizers, absorb_finalizer_i, (st_data_t)dst);
10364 st_free_table(src_finalizers);
10370 if (src_deferred) {
10371 VALUE tail_obj = src_deferred;
10372 rb_asan_unpoison_object(tail_obj,
false);
10373 while (RZOMBIE(tail_obj)->next) {
10374 VALUE next_obj = RZOMBIE(tail_obj)->next;
10375 rb_asan_poison_object(tail_obj);
10376 tail_obj = next_obj;
10377 rb_asan_unpoison_object(tail_obj,
false);
10381 prev = dst->heap_pages.deferred_final;
10382 RZOMBIE(tail_obj)->next = prev;
10384 rb_asan_poison_object(tail_obj);
10391 if (src->tdata_unsafe_free_chunk) {
10392 gc_tdata_unsafe_free_publish(src);
10394 if (tdata_deferred_free_count_load() >= TDATA_DEFERRED_FREE_THRESHOLD) {
10395 gc_tdata_deferred_free_trigger(dst);
10399 dst->rgengc.old_objects += src->rgengc.old_objects;
10400 dst->rgengc.uncollectible_wb_unprotected_objects += src->rgengc.uncollectible_wb_unprotected_objects;
10401 dst->shareable_objects += src->shareable_objects;
10407 dst->rgengc.need_major_gc |= GPR_FLAG_MAJOR_BY_FORCE;
10413 int64_t inc = gc_malloc_counters_increase(src, &src->malloc_counters.counters);
10414#if RGENGC_ESTIMATE_OLDMALLOC
10415 size_t oldmalloc_increase = gc_oldmalloc_increase_load(src);
10417 MALLOC_COUNTERS_LOCK(dst);
10418 if (inc > 0) gc_counter_add(&dst->malloc_counters.counters.malloc, (
size_t)inc);
10419#if RGENGC_ESTIMATE_OLDMALLOC
10420 if (oldmalloc_increase > 0)
RUBY_ATOMIC_SIZE_ADD(dst->malloc_counters.oldmalloc_increase, oldmalloc_increase);
10422 MALLOC_COUNTERS_UNLOCK(dst);
10427 gc_process_stat_capture(src, &final_snap);
10428 gc_process_stat_add(&global_objspace->process_stat_archive, &final_snap);
10433 free(src->profile.records);
10434 free_stack_chunks(&src->mark_stack);
10435 mark_stack_free_cache(&src->mark_stack);
10436 GC_ASSERT(rb_darray_size(src->weak_references) == 0);
10437 rb_darray_free_without_gc(src->weak_references);
10438#ifdef MALLOC_COUNTERS_NEED_LOCK
10443 if (dst_gc_was_enabled) rb_gc_impl_gc_enable(dst);
10450 heap_pages_freeable_pages =
objspace->empty_pages_count;
10451 heap_pages_free_unused_pages(
objspace);
10454 global_objspace->during_absorb = prev_absorb;
10458rb_gc_impl_objspace_absorb(
void *dst_ptr,
void *src_ptr)
10460 objspace_absorb(dst_ptr, src_ptr);
10464rb_gc_impl_start(
void *objspace_ptr,
bool full_mark,
bool immediate_mark,
bool immediate_sweep,
bool compact,
bool global)
10466 rb_objspace_t *
objspace = objspace_ptr;
10467 unsigned int reason = (GPR_FLAG_FULL_MARK |
10468 GPR_FLAG_IMMEDIATE_MARK |
10469 GPR_FLAG_IMMEDIATE_SWEEP |
10472 int full_marking_p = gc_config_full_mark_val;
10473 gc_config_full_mark_set(TRUE);
10477 GC_ASSERT(GC_COMPACTION_SUPPORTED);
10479 reason |= GPR_FLAG_COMPACT;
10480 if (!rb_gc_single_objspace_p()) {
10485 if (!full_mark) reason &= ~GPR_FLAG_FULL_MARK;
10486 if (!immediate_mark) reason &= ~GPR_FLAG_IMMEDIATE_MARK;
10487 if (!immediate_sweep) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
10490 if ((reason & (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP)) !=
10491 (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP)) {
10495 if (global && !rb_gc_single_objspace_p()) {
10499 gc_start_global(
objspace, reason, compact || ruby_enable_autocompact,
false);
10502 garbage_collect(
objspace, reason);
10508 if (tdata_deferred_free_count_load() > 0 && rb_gc_single_objspace_p()) {
10509 gc_tdata_unsafe_drain();
10511 gc_config_full_mark_set(full_marking_p);
10515rb_gc_impl_prepare_heap(
void *objspace_ptr)
10517 rb_objspace_t *
objspace = objspace_ptr;
10519 rb_gc_impl_each_objects(
objspace, gc_set_candidate_object_i, objspace_ptr);
10521 double orig_max_free_slots = gc_params.heap_free_slots_max_ratio;
10523 gc_params.heap_free_slots_max_ratio = 0.0;
10524 rb_gc_impl_start(
objspace,
true,
true,
true,
true,
true);
10525 gc_params.heap_free_slots_max_ratio = orig_max_free_slots;
10527 objspace->heap_pages.allocatable_bytes = 0;
10528 heap_pages_freeable_pages =
objspace->empty_pages_count;
10529 heap_pages_free_unused_pages(objspace_ptr);
10530 GC_ASSERT(heap_pages_freeable_pages == 0);
10531 GC_ASSERT(
objspace->empty_pages_count == 0);
10537 GC_ASSERT(
objspace->heap_pages.allocatable_bytes == 0);
10538 for (
int i = 0; i < HEAP_COUNT; i++) {
10539 rb_heap_t *heap = &heaps[i];
10540 heap_allocatable_bytes_expand(
objspace, heap, heap->empty_slots, heap->total_slots, heap->slot_size);
10543#if defined(HAVE_MALLOC_TRIM) && !defined(RUBY_ALTERNATIVE_MALLOC_HEADER)
10583 GC_ASSERT(st_is_member(finalizer_table, obj));
10587 GC_ASSERT(RVALUE_MARKED(
objspace, obj));
10588 GC_ASSERT(!RVALUE_PINNED(
objspace, obj));
10593 rb_bug(
"gc_is_moveable_obj: unreachable (%d)", (
int)
BUILTIN_TYPE(obj));
10600void rb_mv_generic_ivar(
VALUE src,
VALUE dst);
10605 size_t src_slot_size = src_page->slot_size;
10606 size_t slot_size = dest_page->slot_size;
10609 int wb_unprotected;
10613 gc_report(4,
objspace,
"Moving object: %p -> %p\n", (
void *)src, (
void *)dest);
10616 GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest));
10618 GC_ASSERT(!RVALUE_MARKING(
objspace, src));
10621 marked = RVALUE_MARKED(
objspace, src);
10622 wb_unprotected = RVALUE_WB_UNPROTECTED(
objspace, src);
10623 uncollectible = RVALUE_UNCOLLECTIBLE(
objspace, src);
10624 bool remembered = RVALUE_REMEMBERED(
objspace, src);
10628 bool shareable = MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(src), src) != 0;
10629 bool shref = MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(src), src) != 0;
10630 age = RVALUE_AGE_GET(src);
10633 CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(src), src);
10634 CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(src), src);
10635 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(src), src);
10636 CLEAR_IN_BITMAP(GET_HEAP_PAGE(src)->remembered_bits, src);
10637 CLEAR_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(src), src);
10638 CLEAR_IN_BITMAP(GET_HEAP_SHREF_BITS(src), src);
10641 memcpy((
void *)dest, (
void *)src, MIN(src_slot_size, slot_size));
10643 if (src_slot_size != slot_size) {
10644 rb_gc_obj_changed_slot_size(dest, slot_size - RVALUE_OVERHEAD);
10647 if (RVALUE_OVERHEAD > 0) {
10648 void *dest_overhead = (
void *)(((uintptr_t)dest) + slot_size - RVALUE_OVERHEAD);
10649 void *src_overhead = (
void *)(((uintptr_t)src) + src_slot_size - RVALUE_OVERHEAD);
10651 memcpy(dest_overhead, src_overhead, RVALUE_OVERHEAD);
10654 memset((
void *)src, 0, src_slot_size);
10655 RVALUE_AGE_SET_BITMAP(src, 0);
10659 MARK_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
10662 CLEAR_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
10666 MARK_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
10669 CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
10672 if (wb_unprotected) {
10673 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
10676 CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
10679 if (uncollectible) {
10680 MARK_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
10683 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
10687 MARK_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(dest), dest);
10688 GET_HEAP_PAGE(dest)->flags.has_shareable_objects = TRUE;
10691 CLEAR_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(dest), dest);
10695 MARK_IN_BITMAP(GET_HEAP_SHREF_BITS(dest), dest);
10696 GET_HEAP_PAGE(dest)->flags.has_shref_objects = TRUE;
10699 CLEAR_IN_BITMAP(GET_HEAP_SHREF_BITS(dest), dest);
10702 RVALUE_AGE_SET(dest, age);
10707 if (src_slot_size != slot_size && age >= RVALUE_OLD_AGE && !remembered) {
10712 RMOVED(src)->flags =
T_MOVED;
10713 RMOVED(src)->dummy =
Qundef;
10714 RMOVED(src)->destination = dest;
10717 GET_HEAP_PAGE(src)->heap->total_freed_objects++;
10718 GET_HEAP_PAGE(dest)->heap->total_allocated_objects++;
10723#if GC_CAN_COMPILE_COMPACTION
10725compare_pinned_slots(
const void *left,
const void *right,
void *dummy)
10730 left_page = *(
struct heap_page *
const *)left;
10731 right_page = *(
struct heap_page *
const *)right;
10733 return left_page->pinned_slots - right_page->pinned_slots;
10737compare_free_slots(
const void *left,
const void *right,
void *dummy)
10742 left_page = *(
struct heap_page *
const *)left;
10743 right_page = *(
struct heap_page *
const *)right;
10745 return left_page->free_slots - right_page->free_slots;
10749gc_sort_heap_by_compare_func(rb_objspace_t *
objspace, gc_compact_compare_func compare_func)
10751 for (
int j = 0; j < HEAP_COUNT; j++) {
10752 rb_heap_t *heap = &heaps[j];
10754 size_t total_pages = heap->total_pages;
10756 struct heap_page *page = 0, **page_list = malloc(size);
10759 heap->free_pages = NULL;
10760 ccan_list_for_each(&heap->pages, page, page_node) {
10761 page_list[i++] = page;
10765 GC_ASSERT((
size_t)i == total_pages);
10772 ccan_list_head_init(&heap->pages);
10774 for (i = 0; i < total_pages; i++) {
10775 ccan_list_add(&heap->pages, &page_list[i]->page_node);
10776 if (page_list[i]->free_slots != 0) {
10777 heap_add_freepage(heap, page_list[i]);
10787rb_gc_impl_register_pinning_obj(
void *objspace_ptr,
VALUE obj)
10793rb_gc_impl_object_moved_p(
void *objspace_ptr,
VALUE obj)
10795 return gc_object_moved_p(objspace_ptr, obj);
10799gc_ref_update(
void *vstart,
void *vend,
size_t stride, rb_objspace_t *
objspace,
struct heap_page *page)
10803 page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
10804 page->flags.has_remembered_objects = FALSE;
10807 for (; v != (
VALUE)vend; v += stride) {
10808 asan_unpoisoning_object(v) {
10815 if (RVALUE_WB_UNPROTECTED(
objspace, v)) {
10816 page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
10818 if (RVALUE_REMEMBERED(
objspace, v)) {
10819 page->flags.has_remembered_objects = TRUE;
10821 if (page->flags.before_sweep) {
10823 rb_gc_update_object_references(
objspace, v);
10827 rb_gc_update_object_references(
objspace, v);
10837gc_update_references_weak_table_i(
VALUE obj,
void *data)
10840 asan_unpoisoning_object(obj) {
10847gc_update_references_weak_table_replace_i(
VALUE *obj,
void *data)
10849 rb_gc_update_moved(obj);
10851 return ST_CONTINUE;
10858gc_update_references_heap(rb_objspace_t *
objspace)
10862 for (
int i = 0; i < HEAP_COUNT; i++) {
10863 bool should_set_mark_bits = TRUE;
10864 rb_heap_t *heap = &heaps[i];
10866 ccan_list_for_each(&heap->pages, page, page_node) {
10867 uintptr_t start = (uintptr_t)page->start;
10868 uintptr_t end = start + (page->total_slots * heap->slot_size);
10870 gc_ref_update((
void *)start, (
void *)end, heap->slot_size,
objspace, page);
10871 if (page == heap->sweeping_page) {
10872 should_set_mark_bits = FALSE;
10874 if (should_set_mark_bits) {
10875 gc_setup_mark_bits(page);
10885gc_update_references_global(rb_objspace_t *
objspace)
10887 gc_update_table_refs(finalizer_table);
10889 rb_gc_update_vm_references((
void *)
objspace);
10891 for (
int table = 0; table < RB_GC_VM_WEAK_TABLE_COUNT; table++) {
10892 rb_gc_vm_weak_table_foreach(
10893 gc_update_references_weak_table_i,
10894 gc_update_references_weak_table_replace_i,
10903gc_update_references(rb_objspace_t *
objspace)
10905 objspace->flags.during_reference_updating =
true;
10907 rb_gc_before_updating_jit_code();
10909 gc_update_references_heap(
objspace);
10910 gc_update_references_global(
objspace);
10912 rb_gc_after_updating_jit_code();
10914 objspace->flags.during_reference_updating =
false;
10917#if GC_CAN_COMPILE_COMPACTION
10919root_obj_check_moved_i(
const char *category,
VALUE obj,
void *data)
10923 if (gc_object_moved_p(
objspace, obj)) {
10924 rb_bug(
"ROOT %s points to MOVED: %p -> %s", category, (
void *)obj, rb_obj_info(rb_gc_impl_location(
objspace, obj)));
10929reachable_object_check_moved_i(
VALUE ref,
void *data)
10932 if (gc_object_moved_p(rb_gc_get_objspace(), ref)) {
10933 rb_bug(
"Object %s points to MOVED: %p -> %s", rb_obj_info(parent), (
void *)ref, rb_obj_info(rb_gc_impl_location(rb_gc_get_objspace(), ref)));
10938heap_check_moved_i(
void *vstart,
void *vend,
size_t stride,
void *data)
10943 for (; v != (
VALUE)vend; v += stride) {
10944 if (gc_object_moved_p(
objspace, v)) {
10948 asan_unpoisoning_object(v) {
10954 if (!rb_gc_impl_garbage_object_p(
objspace, v)) {
10955 rb_objspace_reachable_objects_from_unlocked(v, reachable_object_check_moved_i, (
void *)v);
10967rb_gc_impl_during_gc_p(
void *objspace_ptr)
10969 rb_objspace_t *
objspace = objspace_ptr;
10974#if RGENGC_PROFILE >= 2
11007 default:
return "unknown";
11012gc_count_add_each_types(
VALUE hash,
const char *name,
const size_t *types)
11016 for (i=0; i<
T_MASK; i++) {
11017 const char *
type = type_name(i, 0);
11020 rb_hash_aset(hash,
ID2SYM(rb_intern(name)), result);
11025rb_gc_impl_gc_count(
void *objspace_ptr)
11027 rb_objspace_t *
objspace = objspace_ptr;
11033static VALUE sym_major_by, sym_gc_by, sym_immediate_sweep, sym_have_finalizer, sym_state, sym_need_major_by;
11034static VALUE sym_nofree, sym_oldgen, sym_shady, sym_force, sym_stress;
11035#if RGENGC_ESTIMATE_OLDMALLOC
11036static VALUE sym_oldmalloc;
11038static VALUE sym_newobj, sym_malloc, sym_method, sym_capi;
11039static VALUE sym_none, sym_marking, sym_sweeping;
11040static VALUE sym_weak_references_count;
11043setup_gc_latest_gc_info_symbols(
void)
11045#define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
11048 S(immediate_sweep);
11058#if RGENGC_ESTIMATE_OLDMALLOC
11070 S(weak_references_count);
11075gc_info_decode(rb_objspace_t *
objspace,
const VALUE hash_or_key,
const unsigned int orig_flags)
11078 VALUE major_by, need_major_by;
11079 unsigned int flags = orig_flags ? orig_flags :
objspace->profile.latest_gc_info;
11085 hash = hash_or_key;
11088 rb_bug(
"gc_info_decode: non-hash or symbol given");
11091#define SET(name, attr) \
11092 if (key == sym_##name) \
11094 else if (hash != Qnil) \
11095 rb_hash_aset(hash, sym_##name, (attr));
11098 (flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
11099 (flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
11100 (flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
11101 (flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
11102#if RGENGC_ESTIMATE_OLDMALLOC
11103 (flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
11106 SET(major_by, major_by);
11108 if (orig_flags == 0) {
11109 unsigned int need_major_flags = gc_needs_major_flags;
11111 (need_major_flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
11112 (need_major_flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
11113 (need_major_flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
11114 (need_major_flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
11115#if RGENGC_ESTIMATE_OLDMALLOC
11116 (need_major_flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
11119 SET(need_major_by, need_major_by);
11123 (flags & GPR_FLAG_NEWOBJ) ? sym_newobj :
11124 (flags & GPR_FLAG_MALLOC) ? sym_malloc :
11125 (flags & GPR_FLAG_METHOD) ? sym_method :
11126 (flags & GPR_FLAG_CAPI) ? sym_capi :
11127 (flags & GPR_FLAG_STRESS) ? sym_stress :
11131 SET(have_finalizer, (flags & GPR_FLAG_HAVE_FINALIZE) ?
Qtrue :
Qfalse);
11132 SET(immediate_sweep, (flags & GPR_FLAG_IMMEDIATE_SWEEP) ?
Qtrue :
Qfalse);
11134 if (orig_flags == 0) {
11135 SET(state, gc_mode(
objspace) == gc_mode_none ? sym_none :
11136 gc_mode(
objspace) == gc_mode_marking ? sym_marking : sym_sweeping);
11139 SET(weak_references_count,
LONG2FIX(
objspace->profile.weak_references_count));
11151rb_gc_impl_latest_gc_info(
void *objspace_ptr,
VALUE key)
11153 rb_objspace_t *
objspace = objspace_ptr;
11155 return gc_info_decode(
objspace, key, 0);
11162 gc_stat_sym_marking_time,
11163 gc_stat_sym_sweeping_time,
11164 gc_stat_sym_heap_allocated_pages,
11165 gc_stat_sym_heap_empty_pages,
11166 gc_stat_sym_heap_allocatable_bytes,
11167 gc_stat_sym_heap_available_slots,
11168 gc_stat_sym_heap_live_slots,
11169 gc_stat_sym_heap_free_slots,
11170 gc_stat_sym_heap_final_slots,
11171 gc_stat_sym_heap_marked_slots,
11172 gc_stat_sym_heap_eden_pages,
11173 gc_stat_sym_total_allocated_pages,
11174 gc_stat_sym_total_freed_pages,
11175 gc_stat_sym_total_allocated_objects,
11176 gc_stat_sym_total_freed_objects,
11177 gc_stat_sym_total_malloc_bytes,
11178 gc_stat_sym_total_free_bytes,
11179 gc_stat_sym_malloc_increase_bytes,
11180 gc_stat_sym_malloc_increase_bytes_limit,
11181 gc_stat_sym_minor_gc_count,
11182 gc_stat_sym_major_gc_count,
11183 gc_stat_sym_global_gc_count,
11184 gc_stat_sym_global_gc_invocation_count,
11185 gc_stat_sym_compact_count,
11186 gc_stat_sym_read_barrier_faults,
11187 gc_stat_sym_total_moved_objects,
11188 gc_stat_sym_remembered_wb_unprotected_objects,
11189 gc_stat_sym_remembered_wb_unprotected_objects_limit,
11190 gc_stat_sym_old_objects,
11191 gc_stat_sym_old_objects_limit,
11192#if RGENGC_ESTIMATE_OLDMALLOC
11193 gc_stat_sym_oldmalloc_increase_bytes,
11194 gc_stat_sym_oldmalloc_increase_bytes_limit,
11197 gc_stat_sym_total_generated_normal_object_count,
11198 gc_stat_sym_total_generated_shady_object_count,
11199 gc_stat_sym_total_shade_operation_count,
11200 gc_stat_sym_total_promoted_count,
11201 gc_stat_sym_total_remembered_normal_object_count,
11202 gc_stat_sym_total_remembered_shady_object_count,
11204 gc_stat_sym_page_pool_arenas,
11205 gc_stat_sym_page_pool_arenas_freed,
11206 gc_stat_sym_page_pool_total_pages,
11207 gc_stat_sym_page_pool_discarded_pages,
11211static VALUE gc_stat_symbols[gc_stat_sym_last];
11214setup_gc_stat_symbols(
void)
11216#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s))
11221 S(heap_allocated_pages);
11222 S(heap_empty_pages);
11223 S(heap_allocatable_bytes);
11224 S(heap_available_slots);
11225 S(heap_live_slots);
11226 S(heap_free_slots);
11227 S(heap_final_slots);
11228 S(heap_marked_slots);
11229 S(heap_eden_pages);
11230 S(total_allocated_pages);
11231 S(total_freed_pages);
11232 S(total_allocated_objects);
11233 S(total_freed_objects);
11234 S(total_malloc_bytes);
11235 S(total_free_bytes);
11236 S(malloc_increase_bytes);
11237 S(malloc_increase_bytes_limit);
11240 S(global_gc_count);
11241 S(global_gc_invocation_count);
11243 S(read_barrier_faults);
11244 S(total_moved_objects);
11245 S(remembered_wb_unprotected_objects);
11246 S(remembered_wb_unprotected_objects_limit);
11248 S(old_objects_limit);
11249#if RGENGC_ESTIMATE_OLDMALLOC
11250 S(oldmalloc_increase_bytes);
11251 S(oldmalloc_increase_bytes_limit);
11254 S(total_generated_normal_object_count);
11255 S(total_generated_shady_object_count);
11256 S(total_shade_operation_count);
11257 S(total_promoted_count);
11258 S(total_remembered_normal_object_count);
11259 S(total_remembered_shady_object_count);
11261 S(page_pool_arenas);
11262 S(page_pool_arenas_freed);
11263 S(page_pool_total_pages);
11264 S(page_pool_discarded_pages);
11269ns_to_ms(uint64_t ns)
11271 return ns / (1000 * 1000);
11274static void malloc_increase_local_flush(rb_objspace_t *
objspace);
11277gc_process_stat_accumulate_i(
void *objspace_ptr,
void *data)
11279 rb_objspace_t *
objspace = objspace_ptr;
11283 snap =
objspace->process_stat.published;
11285 gc_process_stat_add(total, &snap);
11289gc_process_stat(
VALUE hash_or_sym)
11294 hash = hash_or_sym;
11300 rb_bug(
"non-hash or symbol given");
11304 uint64_t direct_global_gc_count;
11305 unsigned int lev = RB_GC_VM_LOCK();
11306 total = global_objspace->process_stat_archive;
11307 rb_gc_vm_each_objspace(gc_process_stat_accumulate_i, &total);
11308 direct_global_gc_count = global_objspace->global_gc.count;
11309 RB_GC_VM_UNLOCK(lev);
11311 rb_objspace_t *
const current = rb_gc_get_objspace();
11312 if (!gc_during_gc_get(current)) {
11313 GC_ASSERT(total.global_gc_invocation_count == direct_global_gc_count);
11317 uint64_t time_ns = total.marking_time_ns + total.sweeping_time_ns;
11318 uint64_t count = total.minor_gc_count + total.major_gc_count + total.global_gc_invocation_count;
11319 size_t page_pool_arenas = global_objspace->page_pool.arena_count;
11321#define SET64(name, attr) \
11322 if (key == gc_stat_symbols[gc_stat_sym_##name]) \
11323 return ULL2NUM(attr); \
11324 else if (hash != Qnil) \
11325 rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], ULL2NUM(attr));
11327 SET64(count, count);
11328 SET64(time, ns_to_ms(time_ns));
11329 SET64(marking_time, ns_to_ms(total.marking_time_ns));
11330 SET64(sweeping_time, ns_to_ms(total.sweeping_time_ns));
11331 SET64(minor_gc_count, total.minor_gc_count);
11332 SET64(major_gc_count, total.major_gc_count);
11333 SET64(global_gc_count, total.global_gc_invocation_count);
11334 SET64(page_pool_arenas, page_pool_arenas);
11335 SET64(page_pool_arenas_freed, global_objspace->page_pool.arenas_unmapped);
11336 SET64(page_pool_total_pages, (uint64_t)page_pool_arenas * PAGE_POOL_ARENA_BODIES);
11337 SET64(page_pool_discarded_pages, global_objspace->page_pool.advised_count);
11350rb_gc_impl_stat(
void *objspace_ptr,
VALUE hash_or_sym)
11352 if (objspace_ptr == NULL) {
11353 return gc_process_stat(hash_or_sym);
11356 rb_objspace_t *
objspace = objspace_ptr;
11359 malloc_increase_local_flush(
objspace);
11362 hash = hash_or_sym;
11368 rb_bug(
"non-hash or symbol given");
11371#define SET(name, attr) \
11372 if (key == gc_stat_symbols[gc_stat_sym_##name]) \
11373 return SIZET2NUM(attr); \
11374 else if (hash != Qnil) \
11375 rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
11376#define SET64(name, attr) \
11377 if (key == gc_stat_symbols[gc_stat_sym_##name]) \
11378 return ULL2NUM(attr); \
11379 else if (hash != Qnil) \
11380 rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], ULL2NUM(attr));
11382 SET(count,
objspace->profile.count);
11383 SET(time, (
size_t)ns_to_ms(
objspace->profile.marking_time_ns +
objspace->profile.sweeping_time_ns));
11384 SET(marking_time, (
size_t)ns_to_ms(
objspace->profile.marking_time_ns));
11385 SET(sweeping_time, (
size_t)ns_to_ms(
objspace->profile.sweeping_time_ns));
11388 uint64_t total_malloc = (uint64_t)gc_counter_load_relaxed(&
objspace->malloc_counters.counters.malloc);
11389 uint64_t total_free = (uint64_t)gc_counter_load_relaxed(&
objspace->malloc_counters.counters.free);
11390 SET64(total_malloc_bytes, total_malloc);
11391 SET64(total_free_bytes, total_free);
11395 SET(heap_allocated_pages, rb_darray_size(
objspace->heap_pages.sorted));
11396 SET(heap_empty_pages,
objspace->empty_pages_count)
11397 SET(heap_allocatable_bytes,
objspace->heap_pages.allocatable_bytes);
11398 SET(heap_eden_pages, heap_eden_total_pages(
objspace));
11399 SET(total_allocated_pages,
objspace->heap_pages.allocated_pages);
11400 SET(total_freed_pages,
objspace->heap_pages.freed_pages);
11401 SET(malloc_increase_bytes, gc_malloc_counters_increase_unsigned(
objspace, &
objspace->malloc_counters.counters));
11402 SET(malloc_increase_bytes_limit, malloc_limit);
11403 SET(minor_gc_count,
objspace->profile.minor_gc_count);
11404 SET(major_gc_count,
objspace->profile.major_gc_count);
11405 SET(global_gc_count,
objspace->profile.global_gc_count);
11406 SET(global_gc_invocation_count,
objspace->profile.global_gc_invocation_count);
11407 SET(compact_count,
objspace->profile.compact_count);
11408 SET(read_barrier_faults,
objspace->profile.read_barrier_faults);
11409 SET(total_moved_objects,
objspace->rcompactor.total_moved);
11410 SET(remembered_wb_unprotected_objects,
objspace->rgengc.uncollectible_wb_unprotected_objects);
11411 SET(remembered_wb_unprotected_objects_limit,
objspace->rgengc.uncollectible_wb_unprotected_objects_limit);
11412 SET(old_objects,
objspace->rgengc.old_objects);
11413 SET(old_objects_limit,
objspace->rgengc.old_objects_limit);
11414#if RGENGC_ESTIMATE_OLDMALLOC
11415 SET(oldmalloc_increase_bytes, gc_oldmalloc_increase_load(
objspace));
11416 SET(oldmalloc_increase_bytes_limit,
objspace->rgengc.oldmalloc_increase_limit);
11419 SET(total_allocated_objects, total_allocated_objects(
objspace));
11420 SET(total_freed_objects, total_freed_objects(
objspace));
11421 SET(heap_available_slots, objspace_available_slots(
objspace));
11422 SET(heap_live_slots, objspace_live_slots(
objspace));
11423 SET(heap_free_slots, objspace_free_slots(
objspace));
11424 SET(heap_final_slots, total_final_slots_count(
objspace));
11425 SET(heap_marked_slots,
objspace->marked_slots);
11428 SET(total_generated_normal_object_count,
objspace->profile.total_generated_normal_object_count);
11429 SET(total_generated_shady_object_count,
objspace->profile.total_generated_shady_object_count);
11430 SET(total_shade_operation_count,
objspace->profile.total_shade_operation_count);
11431 SET(total_promoted_count,
objspace->profile.total_promoted_count);
11432 SET(total_remembered_normal_object_count,
objspace->profile.total_remembered_normal_object_count);
11433 SET(total_remembered_shady_object_count,
objspace->profile.total_remembered_shady_object_count);
11443#if defined(RGENGC_PROFILE) && RGENGC_PROFILE >= 2
11444 if (hash !=
Qnil) {
11445 gc_count_add_each_types(hash,
"generated_normal_object_count_types",
objspace->profile.generated_normal_object_count_types);
11446 gc_count_add_each_types(hash,
"generated_shady_object_count_types",
objspace->profile.generated_shady_object_count_types);
11447 gc_count_add_each_types(hash,
"shade_operation_count_types",
objspace->profile.shade_operation_count_types);
11448 gc_count_add_each_types(hash,
"promoted_types",
objspace->profile.promoted_types);
11449 gc_count_add_each_types(hash,
"remembered_normal_object_count_types",
objspace->profile.remembered_normal_object_count_types);
11450 gc_count_add_each_types(hash,
"remembered_shady_object_count_types",
objspace->profile.remembered_shady_object_count_types);
11457enum gc_stat_heap_sym {
11458 gc_stat_heap_sym_slot_size,
11459 gc_stat_heap_sym_heap_live_slots,
11460 gc_stat_heap_sym_heap_free_slots,
11461 gc_stat_heap_sym_heap_final_slots,
11462 gc_stat_heap_sym_heap_eden_pages,
11463 gc_stat_heap_sym_heap_eden_slots,
11464 gc_stat_heap_sym_total_allocated_pages,
11465 gc_stat_heap_sym_force_major_gc_count,
11466 gc_stat_heap_sym_force_incremental_marking_finish_count,
11467 gc_stat_heap_sym_heap_allocatable_slots,
11468 gc_stat_heap_sym_total_allocated_objects,
11469 gc_stat_heap_sym_total_freed_objects,
11470 gc_stat_heap_sym_last
11473static VALUE gc_stat_heap_symbols[gc_stat_heap_sym_last];
11476setup_gc_stat_heap_symbols(
void)
11478#define S(s) gc_stat_heap_symbols[gc_stat_heap_sym_##s] = ID2SYM(rb_intern_const(#s))
11480 S(heap_live_slots);
11481 S(heap_free_slots);
11482 S(heap_final_slots);
11483 S(heap_eden_pages);
11484 S(heap_eden_slots);
11485 S(heap_allocatable_slots);
11486 S(total_allocated_pages);
11487 S(force_major_gc_count);
11488 S(force_incremental_marking_finish_count);
11489 S(total_allocated_objects);
11490 S(total_freed_objects);
11497#define SET(name, attr) \
11498 if (key == gc_stat_heap_symbols[gc_stat_heap_sym_##name]) \
11499 return SIZET2NUM(attr); \
11500 else if (hash != Qnil) \
11501 rb_hash_aset(hash, gc_stat_heap_symbols[gc_stat_heap_sym_##name], SIZET2NUM(attr));
11503 SET(slot_size, heap->slot_size);
11504 SET(heap_live_slots, heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count);
11505 SET(heap_free_slots, heap->total_slots - (heap->total_allocated_objects - heap->total_freed_objects));
11506 SET(heap_final_slots, heap->final_slots_count);
11507 SET(heap_eden_pages, heap->total_pages);
11508 SET(heap_eden_slots, heap->total_slots);
11509 SET(heap_allocatable_slots,
objspace->heap_pages.allocatable_bytes / heap->slot_size);
11510 SET(total_allocated_pages, heap->total_allocated_pages);
11511 SET(force_major_gc_count, heap->force_major_gc_count);
11512 SET(force_incremental_marking_finish_count, heap->force_incremental_marking_finish_count);
11513 SET(total_allocated_objects, heap->total_allocated_objects);
11514 SET(total_freed_objects, heap->total_freed_objects);
11526rb_gc_impl_stat_heap(
void *objspace_ptr,
VALUE heap_name,
VALUE hash_or_sym)
11528 rb_objspace_t *
objspace = objspace_ptr;
11530 if (
NIL_P(heap_name)) {
11532 rb_bug(
"non-hash given");
11535 for (
int i = 0; i < HEAP_COUNT; i++) {
11539 rb_hash_aset(hash_or_sym,
INT2FIX(i), hash);
11546 int heap_idx =
FIX2INT(heap_name);
11548 if (heap_idx < 0 || heap_idx >= HEAP_COUNT) {
11549 rb_raise(rb_eArgError,
"size pool index out of range");
11553 return stat_one_heap(
objspace, &heaps[heap_idx],
Qnil, hash_or_sym);
11556 return stat_one_heap(
objspace, &heaps[heap_idx], hash_or_sym,
Qnil);
11559 rb_bug(
"non-hash or symbol given");
11563 rb_bug(
"heap_name must be nil or an Integer");
11566 return hash_or_sym;
11575#define RBOOL(v) (v ? Qtrue : Qfalse)
11579rb_gc_impl_config_get(
void *objspace_ptr)
11581#define sym(name) ID2SYM(rb_intern_const(name))
11582 rb_objspace_t *
objspace = objspace_ptr;
11585 rb_hash_aset(hash, sym(
"rgengc_allow_full_mark"), RBOOL(gc_config_full_mark_val));
11593 rb_objspace_t *
objspace = (rb_objspace_t *)data;
11594 if (
rb_sym2id(key) == rb_intern(
"rgengc_allow_full_mark")) {
11596 gc_config_full_mark_set(
RTEST(value));
11598 return ST_CONTINUE;
11602rb_gc_impl_config_set(
void *objspace_ptr,
VALUE hash)
11604 rb_objspace_t *
objspace = objspace_ptr;
11607 rb_raise(rb_eArgError,
"expected keyword arguments");
11614rb_gc_impl_stress_get(
void *objspace_ptr)
11616 return ruby_gc_stress_mode;
11620rb_gc_impl_stress_set(
void *objspace_ptr,
VALUE flag)
11622 global_objspace->gc_stressful =
RTEST(flag);
11623 global_objspace->gc_stress_mode = flag;
11627get_envparam_size(
const char *name,
size_t *default_value,
size_t lower_bound)
11629 const char *ptr = getenv(name);
11632 if (ptr != NULL && *ptr) {
11635#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
11636 val = strtoll(ptr, &end, 0);
11638 val = strtol(ptr, &end, 0);
11641 case 'k':
case 'K':
11645 case 'm':
case 'M':
11649 case 'g':
case 'G':
11650 unit = 1024*1024*1024;
11654 while (*end && isspace((
unsigned char)*end)) end++;
11656 if (
RTEST(
ruby_verbose)) fprintf(stderr,
"invalid string for %s: %s\n", name, ptr);
11660 if (val < -(ssize_t)(SIZE_MAX / 2 / unit) || (ssize_t)(SIZE_MAX / 2 / unit) < val) {
11661 if (
RTEST(
ruby_verbose)) fprintf(stderr,
"%s=%s is ignored because it overflows\n", name, ptr);
11666 if (val > 0 && (
size_t)val > lower_bound) {
11668 fprintf(stderr,
"%s=%"PRIdSIZE
" (default value: %"PRIuSIZE
")\n", name, val, *default_value);
11670 *default_value = (size_t)val;
11675 fprintf(stderr,
"%s=%"PRIdSIZE
" (default value: %"PRIuSIZE
") is ignored because it must be greater than %"PRIuSIZE
".\n",
11676 name, val, *default_value, lower_bound);
11685get_envparam_double(
const char *name,
double *default_value,
double lower_bound,
double upper_bound,
int accept_zero)
11687 const char *ptr = getenv(name);
11690 if (ptr != NULL && *ptr) {
11692 val =
strtod(ptr, &end);
11693 if (!*ptr || *end) {
11694 if (
RTEST(
ruby_verbose)) fprintf(stderr,
"invalid string for %s: %s\n", name, ptr);
11698 if (accept_zero && val == 0.0) {
11701 else if (val <= lower_bound) {
11703 fprintf(stderr,
"%s=%f (default value: %f) is ignored because it must be greater than %f.\n",
11704 name, val, *default_value, lower_bound);
11707 else if (upper_bound != 0.0 &&
11708 val > upper_bound) {
11710 fprintf(stderr,
"%s=%f (default value: %f) is ignored because it must be lower than %f.\n",
11711 name, val, *default_value, upper_bound);
11721 if (
RTEST(
ruby_verbose)) fprintf(stderr,
"%s=%f (default value: %f)\n", name, val, *default_value);
11722 *default_value = val;
11767rb_gc_impl_set_params(
void *objspace_ptr)
11769 rb_objspace_t *
objspace = objspace_ptr;
11770 get_envparam_size(
"RUBY_GC_HEAP_FREE_SLOTS", &gc_params.heap_free_slots, 0);
11772 get_envparam_size(
"RUBY_GC_HEAP_INIT_BYTES", &gc_params.heap_init_bytes,
11773 heap_init_bytes_min() - 1);
11774 get_envparam_size(
"RUBY_GC_RACTOR_HEAP_INIT_BYTES", &gc_params.ractor_heap_init_bytes,
11775 heap_init_bytes_min() - 1);
11777 get_envparam_double(
"RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE);
11778 get_envparam_size (
"RUBY_GC_HEAP_GROWTH_MAX_BYTES", &gc_params.growth_max_bytes, 0);
11779 get_envparam_double(
"RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio,
11781 get_envparam_double(
"RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", &gc_params.heap_free_slots_max_ratio,
11782 gc_params.heap_free_slots_min_ratio, 1.0, FALSE);
11783 get_envparam_double(
"RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", &gc_params.heap_free_slots_goal_ratio,
11784 gc_params.heap_free_slots_min_ratio, gc_params.heap_free_slots_max_ratio, TRUE);
11785 get_envparam_double(
"RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR", &gc_params.oldobject_limit_factor, 0.0, 0.0, TRUE);
11786 get_envparam_double(
"RUBY_GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO", &gc_params.uncollectible_wb_unprotected_objects_limit_ratio, 0.0, 0.0, TRUE);
11788 if (get_envparam_size(
"RUBY_GC_MALLOC_LIMIT", &gc_params.malloc_limit_min, 0)) {
11789 malloc_limit = gc_params.malloc_limit_min;
11791 get_envparam_size (
"RUBY_GC_MALLOC_LIMIT_MAX", &gc_params.malloc_limit_max, 0);
11792 if (!gc_params.malloc_limit_max) {
11793 gc_params.malloc_limit_max = SIZE_MAX;
11795 get_envparam_double(
"RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR", &gc_params.malloc_limit_growth_factor, 1.0, 0.0, FALSE);
11797#if RGENGC_ESTIMATE_OLDMALLOC
11798 if (get_envparam_size(
"RUBY_GC_OLDMALLOC_LIMIT", &gc_params.oldmalloc_limit_min, 0)) {
11799 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
11801 get_envparam_size (
"RUBY_GC_OLDMALLOC_LIMIT_MAX", &gc_params.oldmalloc_limit_max, 0);
11802 get_envparam_double(
"RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR", &gc_params.oldmalloc_limit_growth_factor, 1.0, 0.0, FALSE);
11806static inline size_t
11807objspace_malloc_size(rb_objspace_t *
objspace,
void *ptr,
size_t hint)
11809#ifdef HAVE_MALLOC_USABLE_SIZE
11811 hint = malloc_usable_size(ptr);
11818 MEMOP_TYPE_MALLOC = 0,
11824atomic_sub_nounderflow(
size_t *var,
size_t sub)
11826 if (sub == 0)
return;
11829 size_t val = rbimpl_atomic_size_load(var, RBIMPL_ATOMIC_RELAXED);
11830 size_t decrement = val < sub ? val : sub;
11835#define gc_stress_full_mark_after_malloc_p() \
11836 (FIXNUM_P(ruby_gc_stress_mode) && (FIX2LONG(ruby_gc_stress_mode) & (1<<gc_stress_full_mark_after_malloc)))
11839objspace_malloc_gc_stress(rb_objspace_t *
objspace)
11842 unsigned int reason = (GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP |
11843 GPR_FLAG_STRESS | GPR_FLAG_MALLOC);
11845 if (gc_stress_full_mark_after_malloc_p()) {
11846 reason |= GPR_FLAG_FULL_MARK;
11848 garbage_collect_with_gvl(
objspace, reason);
11853malloc_increase_commit(rb_objspace_t *
objspace,
size_t new_size,
size_t old_size)
11855 if (new_size > old_size) {
11856 size_t delta = new_size - old_size;
11858 gc_counter_add(&
objspace->malloc_counters.counters.malloc, delta);
11859#if RGENGC_ESTIMATE_OLDMALLOC
11864 else if (old_size > new_size) {
11865 size_t delta = old_size - new_size;
11867 gc_counter_add(&
objspace->malloc_counters.counters.free, delta);
11868#if RGENGC_ESTIMATE_OLDMALLOC
11869 if (during_gc) gc_counter_add(&
objspace->malloc_counters.freed_in_gc, delta);
11870 atomic_sub_nounderflow(&
objspace->malloc_counters.oldmalloc_increase, delta);
11876#if USE_MALLOC_INCREASE_LOCAL
11878malloc_increase_local_flush(rb_objspace_t *
objspace)
11880 int delta = malloc_increase_local;
11881 if (delta == 0)
return;
11883 malloc_increase_local = 0;
11885 malloc_increase_commit(
objspace, (
size_t)delta, 0);
11888 malloc_increase_commit(
objspace, 0, (
size_t)(-delta));
11893malloc_increase_local_flush(rb_objspace_t *
objspace)
11899objspace_malloc_increase_report(rb_objspace_t *
objspace,
void *mem,
size_t new_size,
size_t old_size,
enum memop_type
type,
bool gc_allowed)
11901 if (0) fprintf(stderr,
"increase - ptr: %p, type: %s, new_size: %"PRIdSIZE
", old_size: %"PRIdSIZE
"\n",
11903 type == MEMOP_TYPE_MALLOC ?
"malloc" :
11904 type == MEMOP_TYPE_FREE ?
"free " :
11905 type == MEMOP_TYPE_REALLOC ?
"realloc":
"error",
11906 new_size, old_size);
11911objspace_malloc_increase_body(rb_objspace_t *
objspace,
void *mem,
size_t new_size,
size_t old_size,
enum memop_type
type,
bool gc_allowed)
11913#if USE_MALLOC_INCREASE_LOCAL
11914 if (new_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD &&
11915 old_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD) {
11916 malloc_increase_local += (int)new_size - (
int)old_size;
11918 if (malloc_increase_local >= GC_MALLOC_INCREASE_LOCAL_THRESHOLD ||
11919 malloc_increase_local <= -GC_MALLOC_INCREASE_LOCAL_THRESHOLD) {
11920 malloc_increase_local_flush(
objspace);
11924 malloc_increase_local_flush(
objspace);
11925 malloc_increase_commit(
objspace, new_size, old_size);
11928 malloc_increase_commit(
objspace, new_size, old_size);
11931 if (
type == MEMOP_TYPE_MALLOC && gc_allowed) {
11933 if (malloc_increase > malloc_limit &&
ruby_native_thread_p() && !dont_gc_val() && !rb_gc_gc_disabled_global_p()) {
11935 gc_sweep_step_for_malloc(
objspace);
11938 garbage_collect_with_gvl(
objspace, GPR_FLAG_MALLOC);
11942#if MALLOC_ALLOCATED_SIZE
11943 if (new_size >= old_size) {
11947 size_t dec_size = old_size - new_size;
11949#if MALLOC_ALLOCATED_SIZE_CHECK
11950 size_t allocated_size =
objspace->malloc_params.allocated_size;
11951 if (allocated_size < dec_size) {
11952 rb_bug(
"objspace_malloc_increase: underflow malloc_params.allocated_size.");
11955 atomic_sub_nounderflow(&
objspace->malloc_params.allocated_size, dec_size);
11959 case MEMOP_TYPE_MALLOC:
11962 case MEMOP_TYPE_FREE:
11964 size_t allocations =
objspace->malloc_params.allocations;
11965 if (allocations > 0) {
11966 atomic_sub_nounderflow(&
objspace->malloc_params.allocations, 1);
11968#if MALLOC_ALLOCATED_SIZE_CHECK
11970 GC_ASSERT(
objspace->malloc_params.allocations > 0);
11975 case MEMOP_TYPE_REALLOC:
break;
11981#define objspace_malloc_increase(...) \
11982 for (bool malloc_increase_done = objspace_malloc_increase_report(__VA_ARGS__); \
11983 !malloc_increase_done; \
11984 malloc_increase_done = objspace_malloc_increase_body(__VA_ARGS__))
11990static inline size_t
11991objspace_malloc_prepare(rb_objspace_t *
objspace,
size_t size)
11993 if (size == 0) size = 1;
11995#if CALC_EXACT_MALLOC_SIZE
12003malloc_during_gc_p(rb_objspace_t *
objspace)
12012static inline void *
12013objspace_malloc_fixup(rb_objspace_t *
objspace,
void *mem,
size_t size,
bool gc_allowed)
12015 size = objspace_malloc_size(
objspace, mem, size);
12016 objspace_malloc_increase(
objspace, mem, size, 0, MEMOP_TYPE_MALLOC, gc_allowed) {}
12018#if CALC_EXACT_MALLOC_SIZE
12029#if defined(__GNUC__) && RUBY_DEBUG
12030#define RB_BUG_INSTEAD_OF_RB_MEMERROR 1
12033#ifndef RB_BUG_INSTEAD_OF_RB_MEMERROR
12034# define RB_BUG_INSTEAD_OF_RB_MEMERROR 0
12037#define GC_MEMERROR(...) \
12038 ((RB_BUG_INSTEAD_OF_RB_MEMERROR+0) ? rb_bug("" __VA_ARGS__) : (void)0)
12040#define TRY_WITH_GC(siz, expr) do { \
12041 const gc_profile_record_flag gpr = \
12042 GPR_FLAG_FULL_MARK | \
12043 GPR_FLAG_IMMEDIATE_MARK | \
12044 GPR_FLAG_IMMEDIATE_SWEEP | \
12047 if (gc_allowed) objspace_malloc_gc_stress(objspace); \
12049 if (RB_LIKELY((expr))) { \
12052 else if (gc_allowed && !garbage_collect_with_gvl(objspace, gpr)) { \
12054 GC_MEMERROR("TRY_WITH_GC: could not GC"); \
12056 else if ((expr)) { \
12060 GC_MEMERROR("TRY_WITH_GC: could not allocate:" \
12061 "%"PRIdSIZE" bytes for %s", \
12067check_malloc_not_in_gc(rb_objspace_t *
objspace,
const char *msg)
12069 if (RB_UNLIKELY(malloc_during_gc_p(
objspace))) {
12072 rb_bug(
"Cannot %s during GC", msg);
12077rb_gc_impl_free(
void *objspace_ptr,
void *ptr,
size_t old_size)
12079 rb_objspace_t *
objspace = objspace_ptr;
12088#if CALC_EXACT_MALLOC_SIZE
12090#if VERIFY_FREE_SIZE
12092 const char *freeing = gc_freeing_obj_info();
12093 rb_bug(
"buffer %p has no recorded size%s%s. Was it allocated with ruby_mimalloc? If so it should be freed with ruby_mimfree", ptr,
12094 freeing ?
", while freeing " :
"", freeing ? freeing :
"");
12097 if (old_size && (old_size +
sizeof(
struct malloc_obj_info)) != info->size) {
12098 const char *freeing = gc_freeing_obj_info();
12099 rb_bug(
"buffer %p freed with old_size=%zu, but was allocated with size=%zu%s%s", ptr, old_size, info->size -
sizeof(
struct malloc_obj_info),
12100 freeing ?
", while freeing " :
"", freeing ? freeing :
"");
12104 old_size = info->size;
12106 old_size = objspace_malloc_size(
objspace, ptr, old_size);
12108 objspace_malloc_increase(
objspace, ptr, 0, old_size, MEMOP_TYPE_FREE,
true) {
12111 RB_DEBUG_COUNTER_INC(heap_xfree);
12116rb_gc_impl_malloc(
void *objspace_ptr,
size_t size,
bool gc_allowed)
12118 rb_objspace_t *
objspace = objspace_ptr;
12119 check_malloc_not_in_gc(
objspace,
"malloc");
12123 size = objspace_malloc_prepare(
objspace, size);
12124 TRY_WITH_GC(size, mem = malloc(size));
12125 RB_DEBUG_COUNTER_INC(heap_xmalloc);
12126 if (!mem)
return mem;
12127 return objspace_malloc_fixup(
objspace, mem, size, gc_allowed);
12131rb_gc_impl_calloc(
void *objspace_ptr,
size_t size,
bool gc_allowed)
12133 rb_objspace_t *
objspace = objspace_ptr;
12135 if (RB_UNLIKELY(malloc_during_gc_p(
objspace))) {
12136 rb_warn(
"calloc during GC detected, this could cause crashes if it triggers another GC");
12137#if RGENGC_CHECK_MODE || RUBY_DEBUG
12138 rb_bug(
"Cannot calloc during GC");
12144 size = objspace_malloc_prepare(
objspace, size);
12145 TRY_WITH_GC(size, mem = calloc1(size));
12146 if (!mem)
return mem;
12147 return objspace_malloc_fixup(
objspace, mem, size, gc_allowed);
12151rb_gc_impl_realloc(
void *objspace_ptr,
void *ptr,
size_t new_size,
size_t old_size,
bool gc_allowed)
12153 rb_objspace_t *
objspace = objspace_ptr;
12155 check_malloc_not_in_gc(
objspace,
"realloc");
12159 if (!ptr)
return rb_gc_impl_malloc(
objspace, new_size, gc_allowed);
12166 if (new_size == 0) {
12167 if ((mem = rb_gc_impl_malloc(
objspace, 0, gc_allowed)) != NULL) {
12190 rb_gc_impl_free(
objspace, ptr, old_size);
12204#if CALC_EXACT_MALLOC_SIZE
12209#if VERIFY_FREE_SIZE
12210 if (old_size && (old_size +
sizeof(
struct malloc_obj_info)) != info->size) {
12211 const char *freeing = gc_freeing_obj_info();
12212 rb_bug(
"buffer %p realloced with old_size=%zu, but was allocated with size=%zu%s%s", ptr, old_size, info->size - sizeof(struct malloc_obj_info),
12213 freeing ?
", while freeing " :
"", freeing ? freeing :
"");
12216 old_size = info->size;
12220 old_size = objspace_malloc_size(
objspace, ptr, old_size);
12222 if (!mem)
return mem;
12223 new_size = objspace_malloc_size(
objspace, mem, new_size);
12225#if CALC_EXACT_MALLOC_SIZE
12228 info->size = new_size;
12233 objspace_malloc_increase(
objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC, gc_allowed);
12235 RB_DEBUG_COUNTER_INC(heap_xrealloc);
12240rb_gc_impl_adjust_memory_usage(
void *objspace_ptr, ssize_t diff)
12242 rb_objspace_t *
objspace = objspace_ptr;
12245 objspace_malloc_increase(
objspace, 0, diff, 0, MEMOP_TYPE_REALLOC,
true);
12247 else if (diff < 0) {
12248 objspace_malloc_increase(
objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC,
true);
12257#define GC_PROFILE_RECORD_DEFAULT_SIZE 100
12258#define GC_PROFILE_RECORD_DEFAULT_MAX_RECORDS 4096
12259#define GC_PROFILE_RECORD_UNBOUNDED 0
12262current_process_time(
struct timespec *ts)
12264#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
12266 static int try_clock_gettime = 1;
12267 if (try_clock_gettime) {
12268 if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ts) == 0) {
12272 try_clock_gettime = 0;
12280 struct rusage usage;
12282 if (getrusage(RUSAGE_SELF, &usage) == 0) {
12283 time = usage.ru_utime;
12284 ts->tv_sec = time.tv_sec;
12285 ts->tv_nsec = (int32_t)time.tv_usec * 1000;
12293 FILETIME creation_time, exit_time, kernel_time, user_time;
12296 if (GetProcessTimes(GetCurrentProcess(),
12297 &creation_time, &exit_time, &kernel_time, &user_time) != 0) {
12298 memcpy(&ui, &user_time,
sizeof(FILETIME));
12299#define PER100NSEC (uint64_t)(1000 * 1000 * 10)
12300 ts->tv_nsec = (long)(ui.QuadPart % PER100NSEC);
12301 ts->tv_sec = (time_t)(ui.QuadPart / PER100NSEC);
12311getrusage_time(
void)
12314 if (current_process_time(&ts)) {
12315 return ts.tv_sec + ts.tv_nsec * 1e-9;
12322static inline double
12323hrtime_to_sec(rb_hrtime_t time)
12325 return (
double)time / (double)RB_HRTIME_PER_SEC;
12328static inline rb_hrtime_t
12329elapsed_hrtime_from(rb_hrtime_t start)
12331 return rb_hrtime_sub(rb_hrtime_now(), start);
12335static inline size_t
12336gc_profile_record_count(rb_objspace_t *
objspace)
12338 return objspace->profile.record_count;
12341static inline size_t
12342gc_profile_record_index(rb_objspace_t *
objspace,
size_t logical_index)
12344 if (
objspace->profile.max_records != GC_PROFILE_RECORD_UNBOUNDED &&
12346 return (
objspace->profile.next_index + logical_index) %
objspace->profile.size;
12349 return logical_index;
12354gc_profile_records_free(rb_objspace_t *
objspace)
12356 void *p =
objspace->profile.records;
12360 objspace->profile.record_count = 0;
12361 objspace->profile.current_record = 0;
12366gc_prof_setup_new_record(rb_objspace_t *
objspace,
unsigned int reason)
12372 if (
objspace->profile.max_records == GC_PROFILE_RECORD_UNBOUNDED) {
12373 index =
objspace->profile.record_count++;
12377 objspace->profile.size = GC_PROFILE_RECORD_DEFAULT_SIZE;
12380 if (index >=
objspace->profile.size) {
12393 index =
objspace->profile.next_index;
12401 rb_bug(
"gc_profile malloc or realloc miss");
12403 record =
objspace->profile.current_record = &
objspace->profile.records[index];
12407 record->flags = reason | (ruby_gc_stressful ? GPR_FLAG_STRESS : 0);
12408 record->sequence =
objspace->profile.record_sequence++;
12409 record->gc_invoke_wall_time = rb_hrtime_sub(rb_hrtime_now(),
12410 objspace->profile.invoke_wall_time);
12411#if MALLOC_ALLOCATED_SIZE
12412 record->allocated_size = malloc_allocated_size;
12414#if GC_PROFILE_MORE_DETAIL && GC_PROFILE_DETAIL_MEMORY
12417 struct rusage usage;
12418 if (getrusage(RUSAGE_SELF, &usage) == 0) {
12419 record->maxrss = usage.ru_maxrss;
12420 record->minflt = usage.ru_minflt;
12421 record->majflt = usage.ru_majflt;
12430gc_prof_timer_start(rb_objspace_t *
objspace)
12434#if GC_PROFILE_MORE_DETAIL
12435 record->prepare_time =
objspace->profile.prepare_time;
12437 record->gc_time = 0;
12438 record->gc_invoke_time = getrusage_time();
12439 objspace->profile.gc_wall_start_time = rb_hrtime_now();
12444elapsed_time_from(
double time)
12446 double now = getrusage_time();
12456gc_prof_timer_stop(rb_objspace_t *
objspace)
12460 record->gc_time = elapsed_time_from(record->gc_invoke_time);
12461 record->gc_invoke_time -=
objspace->profile.invoke_time;
12462 record->gc_wall_time = elapsed_hrtime_from(
objspace->profile.gc_wall_start_time);
12467gc_prof_mark_timer_start(rb_objspace_t *
objspace)
12469 RUBY_DTRACE_GC_HOOK(MARK_BEGIN);
12470#if GC_PROFILE_MORE_DETAIL
12472 gc_prof_record(
objspace)->gc_mark_time = getrusage_time();
12478gc_prof_mark_timer_stop(rb_objspace_t *
objspace)
12480 RUBY_DTRACE_GC_HOOK(MARK_END);
12481#if GC_PROFILE_MORE_DETAIL
12484 record->gc_mark_time = elapsed_time_from(record->gc_mark_time);
12490gc_prof_sweep_timer_start(rb_objspace_t *
objspace)
12492 RUBY_DTRACE_GC_HOOK(SWEEP_BEGIN);
12496 if (record->gc_time > 0 || GC_PROFILE_MORE_DETAIL) {
12497 objspace->profile.gc_sweep_start_time = getrusage_time();
12498 objspace->profile.gc_sweep_wall_start_time = rb_hrtime_now();
12504gc_prof_sweep_timer_stop(rb_objspace_t *
objspace)
12506 RUBY_DTRACE_GC_HOOK(SWEEP_END);
12512 if (record->gc_time > 0) {
12513 sweep_time = elapsed_time_from(
objspace->profile.gc_sweep_start_time);
12515 record->gc_time += sweep_time;
12516 record->gc_wall_time = rb_hrtime_add(record->gc_wall_time,
12517 elapsed_hrtime_from(
objspace->profile.gc_sweep_wall_start_time));
12519 else if (GC_PROFILE_MORE_DETAIL) {
12520 sweep_time = elapsed_time_from(
objspace->profile.gc_sweep_start_time);
12523#if GC_PROFILE_MORE_DETAIL
12524 record->gc_sweep_time += sweep_time;
12525 if (heap_pages_deferred_final) record->flags |= GPR_FLAG_HAVE_FINALIZE;
12527 if (heap_pages_deferred_final)
objspace->profile.latest_gc_info |= GPR_FLAG_HAVE_FINALIZE;
12532gc_prof_set_malloc_info(rb_objspace_t *
objspace)
12534#if GC_PROFILE_MORE_DETAIL
12537 record->allocate_increase = malloc_increase;
12538 record->allocate_limit = malloc_limit;
12544gc_prof_set_heap_info(rb_objspace_t *
objspace)
12551 size_t use_size = 0;
12552 size_t total_size = 0;
12553 for (
int i = 0; i < HEAP_COUNT; i++) {
12554 rb_heap_t *heap = &heaps[i];
12555 size_t heap_live = heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count;
12556 total += heap->total_slots;
12557 use_size += heap_live * heap->slot_size;
12558 total_size += heap->total_slots * heap->slot_size;
12561#if GC_PROFILE_MORE_DETAIL
12562 size_t live =
objspace->profile.total_allocated_objects_at_gc_start - total_freed_objects(
objspace);
12563 record->heap_use_pages =
objspace->profile.heap_used_at_gc_start;
12564 record->heap_live_objects = live;
12565 record->heap_free_objects = total - live;
12568 record->heap_total_objects = total;
12569 record->heap_use_size = use_size;
12570 record->heap_total_size = total_size;
12583gc_profile_clear(
VALUE _)
12585 rb_objspace_t *
objspace = rb_gc_get_objspace();
12586 gc_profile_records_free(
objspace);
12611gc_profile_configure(
int argc,
VALUE *argv,
VALUE _)
12613 static ID keywords[1] = {0};
12614 VALUE options, max_records;
12615 rb_objspace_t *
objspace = rb_gc_get_objspace();
12617 if (!keywords[0]) {
12618 keywords[0] = rb_intern(
"max_records");
12624 if (max_records ==
Qundef) {
12627 else if (
NIL_P(max_records)) {
12628 objspace->profile.max_records = GC_PROFILE_RECORD_UNBOUNDED;
12631 long value =
NUM2LONG(max_records);
12633 rb_raise(rb_eArgError,
"max_records must be positive or nil");
12635 objspace->profile.max_records = (size_t)value;
12638 gc_profile_records_free(
objspace);
12747gc_profile_record_get(
int argc,
VALUE *argv,
VALUE _)
12749 static ID keywords[2] = {0};
12750 VALUE prof, options, limit_value, since_value;
12752 size_t i, count, matching = 0, skip = 0, limit = SIZE_MAX, since = 0;
12753 bool use_since =
false;
12754 rb_objspace_t *
objspace = rb_gc_get_objspace();
12756 if (!keywords[0]) {
12757 keywords[0] = rb_intern(
"limit");
12758 keywords[1] = rb_intern(
"since");
12764 limit_value = values[0];
12765 since_value = values[1];
12767 if (limit_value !=
Qundef && !
NIL_P(limit_value)) {
12768 long value =
NUM2LONG(limit_value);
12770 rb_raise(rb_eArgError,
"limit must be non-negative");
12772 limit = (size_t)value;
12774 if (since_value !=
Qundef && !
NIL_P(since_value)) {
12775 long value =
NUM2LONG(since_value);
12777 rb_raise(rb_eArgError,
"since must be non-negative");
12779 since = (size_t)value;
12787 count = gc_profile_record_count(
objspace);
12788 for (i = 0; i < count; i++) {
12790 if (!use_since || record->sequence > since) {
12794 if (limit < matching) {
12795 skip = matching - limit;
12798 for (i = 0; i < count; i++) {
12800 if (use_since && record->sequence <= since) {
12810 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_SEQUENCE")),
SIZET2NUM(record->sequence));
12811 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_TIME")),
DBL2NUM(record->gc_time));
12812 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_INVOKE_TIME")),
DBL2NUM(record->gc_invoke_time));
12813 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_WALL_TIME")),
12814 DBL2NUM(hrtime_to_sec(record->gc_wall_time)));
12815 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_INVOKE_WALL_TIME")),
12816 DBL2NUM(hrtime_to_sec(record->gc_invoke_wall_time)));
12817 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_PAUSE_TIME")),
12818 DBL2NUM(hrtime_to_sec(record->gc_pause_time)));
12819 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_STOP_TIME")),
12820 DBL2NUM(hrtime_to_sec(record->gc_stop_time)));
12821 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_STW_TIME")),
12822 DBL2NUM(hrtime_to_sec(record->gc_stw_time)));
12823 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_MARK_WALL_TIME")),
12824 DBL2NUM(hrtime_to_sec(record->gc_mark_wall_time)));
12825 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_SWEEP_WALL_TIME")),
12826 DBL2NUM(hrtime_to_sec(record->gc_sweep_wall_time)));
12827 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_COMPACT_WALL_TIME")),
12828 DBL2NUM(hrtime_to_sec(record->gc_compact_wall_time)));
12829 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HEAP_USE_SIZE")),
SIZET2NUM(record->heap_use_size));
12830 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HEAP_TOTAL_SIZE")),
SIZET2NUM(record->heap_total_size));
12831 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HEAP_TOTAL_OBJECTS")),
SIZET2NUM(record->heap_total_objects));
12832 rb_hash_aset(prof,
ID2SYM(rb_intern(
"MOVED_OBJECTS")),
SIZET2NUM(record->moved_objects));
12833 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_IS_MARKED")),
Qtrue);
12834#if GC_PROFILE_MORE_DETAIL
12835 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_MARK_TIME")),
DBL2NUM(record->gc_mark_time));
12836 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_SWEEP_TIME")),
DBL2NUM(record->gc_sweep_time));
12837 rb_hash_aset(prof,
ID2SYM(rb_intern(
"ALLOCATE_INCREASE")),
SIZET2NUM(record->allocate_increase));
12838 rb_hash_aset(prof,
ID2SYM(rb_intern(
"ALLOCATE_LIMIT")),
SIZET2NUM(record->allocate_limit));
12839 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HEAP_USE_PAGES")),
SIZET2NUM(record->heap_use_pages));
12840 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HEAP_LIVE_OBJECTS")),
SIZET2NUM(record->heap_live_objects));
12841 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HEAP_FREE_OBJECTS")),
SIZET2NUM(record->heap_free_objects));
12843 rb_hash_aset(prof,
ID2SYM(rb_intern(
"REMOVING_OBJECTS")),
SIZET2NUM(record->removing_objects));
12844 rb_hash_aset(prof,
ID2SYM(rb_intern(
"EMPTY_OBJECTS")),
SIZET2NUM(record->empty_objects));
12846 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HAVE_FINALIZE")), (record->flags & GPR_FLAG_HAVE_FINALIZE) ?
Qtrue :
Qfalse);
12849#if RGENGC_PROFILE > 0
12850 rb_hash_aset(prof,
ID2SYM(rb_intern(
"OLD_OBJECTS")),
SIZET2NUM(record->old_objects));
12851 rb_hash_aset(prof,
ID2SYM(rb_intern(
"REMEMBERED_NORMAL_OBJECTS")),
SIZET2NUM(record->remembered_normal_objects));
12852 rb_hash_aset(prof,
ID2SYM(rb_intern(
"REMEMBERED_SHADY_OBJECTS")),
SIZET2NUM(record->remembered_shady_objects));
12860#if GC_PROFILE_MORE_DETAIL
12861#define MAJOR_REASON_MAX 0x10
12864gc_profile_dump_major_reason(
unsigned int flags,
char *buff)
12866 unsigned int reason = flags & GPR_FLAG_MAJOR_MASK;
12869 if (reason == GPR_FLAG_NONE) {
12875 if (reason & GPR_FLAG_MAJOR_BY_##x) { \
12876 buff[i++] = #x[0]; \
12877 if (i >= MAJOR_REASON_MAX) rb_bug("gc_profile_dump_major_reason: overflow"); \
12883#if RGENGC_ESTIMATE_OLDMALLOC
12897 rb_objspace_t *
objspace = rb_gc_get_objspace();
12898 size_t count = gc_profile_record_count(
objspace);
12899#ifdef MAJOR_REASON_MAX
12900 char reason_str[MAJOR_REASON_MAX];
12903 if (
objspace->profile.run && count ) {
12907 append(out, rb_sprintf(
"GC %"PRIuSIZE
" invokes.\n",
objspace->profile.count));
12908 append(out,
rb_str_new_cstr(
"Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC Time(ms)\n"));
12910 for (i = 0; i < count; i++) {
12911 record = &
objspace->profile.records[gc_profile_record_index(
objspace, i)];
12912 append(out, rb_sprintf(
"%5"PRIuSIZE
" %19.3f %20"PRIuSIZE
" %20"PRIuSIZE
" %20"PRIuSIZE
" %30.20f\n",
12913 i+1, record->gc_invoke_time, record->heap_use_size,
12914 record->heap_total_size, record->heap_total_objects, record->gc_time*1000));
12917#if GC_PROFILE_MORE_DETAIL
12918 const char *str =
"\n\n" \
12920 "Prepare Time = Previously GC's rest sweep time\n"
12921 "Index Flags Allocate Inc. Allocate Limit"
12922#if CALC_EXACT_MALLOC_SIZE
12925 " Use Page Mark Time(ms) Sweep Time(ms) Prepare Time(ms) LivingObj FreeObj RemovedObj EmptyObj"
12927 " OldgenObj RemNormObj RemShadObj"
12929#if GC_PROFILE_DETAIL_MEMORY
12930 " MaxRSS(KB) MinorFLT MajorFLT"
12935 for (i = 0; i < count; i++) {
12936 record = &
objspace->profile.records[gc_profile_record_index(
objspace, i)];
12937 append(out, rb_sprintf(
"%5"PRIuSIZE
" %4s/%c/%6s%c %13"PRIuSIZE
" %15"PRIuSIZE
12938#
if CALC_EXACT_MALLOC_SIZE
12941 " %9"PRIuSIZE
" %17.12f %17.12f %17.12f %10"PRIuSIZE
" %10"PRIuSIZE
" %10"PRIuSIZE
" %10"PRIuSIZE
12943 "%10"PRIuSIZE
" %10"PRIuSIZE
" %10"PRIuSIZE
12945#
if GC_PROFILE_DETAIL_MEMORY
12951 gc_profile_dump_major_reason(record->flags, reason_str),
12952 (record->flags & GPR_FLAG_HAVE_FINALIZE) ?
'F' :
'.',
12953 (record->flags & GPR_FLAG_NEWOBJ) ?
"NEWOBJ" :
12954 (record->flags & GPR_FLAG_MALLOC) ?
"MALLOC" :
12955 (record->flags & GPR_FLAG_METHOD) ?
"METHOD" :
12956 (record->flags & GPR_FLAG_CAPI) ?
"CAPI__" :
"??????",
12957 (record->flags & GPR_FLAG_STRESS) ?
'!' :
' ',
12958 record->allocate_increase, record->allocate_limit,
12959#
if CALC_EXACT_MALLOC_SIZE
12960 record->allocated_size,
12962 record->heap_use_pages,
12963 record->gc_mark_time*1000,
12964 record->gc_sweep_time*1000,
12965 record->prepare_time*1000,
12967 record->heap_live_objects,
12968 record->heap_free_objects,
12969 record->removing_objects,
12970 record->empty_objects
12973 record->old_objects,
12974 record->remembered_normal_objects,
12975 record->remembered_shady_objects
12977#
if GC_PROFILE_DETAIL_MEMORY
12979 record->maxrss / 1024,
13002gc_profile_result(
VALUE _)
13019gc_profile_report(
int argc,
VALUE *argv,
VALUE self)
13024 gc_profile_dump_on(out, rb_io_write);
13037gc_profile_total_time(
VALUE self)
13040 rb_objspace_t *
objspace = rb_gc_get_objspace();
13044 size_t count = gc_profile_record_count(
objspace);
13046 for (i = 0; i < count; i++) {
13047 time +=
objspace->profile.records[gc_profile_record_index(
objspace, i)].gc_time;
13061gc_profile_enable_get(
VALUE self)
13063 rb_objspace_t *
objspace = rb_gc_get_objspace();
13076gc_profile_enable(
VALUE _)
13078 rb_objspace_t *
objspace = rb_gc_get_objspace();
13080 objspace->profile.current_record = 0;
13093gc_profile_disable(
VALUE _)
13095 rb_objspace_t *
objspace = rb_gc_get_objspace();
13098 objspace->profile.current_record = 0;
13103rb_gc_verify_internal_consistency(
void)
13105 gc_verify_internal_consistency(rb_gc_get_objspace());
13118gc_verify_internal_consistency_m(
VALUE dummy)
13120 rb_gc_verify_internal_consistency();
13124#if GC_CAN_COMPILE_COMPACTION
13138 GC_ASSERT(GC_COMPACTION_SUPPORTED);
13140 ruby_enable_autocompact =
RTEST(v);
13142#if RGENGC_CHECK_MODE
13143 ruby_autocompact_compare_func = NULL;
13147 if (
id == rb_intern(
"empty")) {
13148 ruby_autocompact_compare_func = compare_free_slots;
13156# define gc_set_auto_compact rb_f_notimplement
13159#if GC_CAN_COMPILE_COMPACTION
13167gc_get_auto_compact(
VALUE _)
13172# define gc_get_auto_compact rb_f_notimplement
13175#if GC_CAN_COMPILE_COMPACTION
13201gc_compact_stats(
VALUE self)
13203 rb_objspace_t *
objspace = rb_gc_get_objspace();
13210 for (
size_t i = 0; i <
T_MASK; i++) {
13211 if (
objspace->rcompactor.considered_count_table[i]) {
13212 rb_hash_aset(considered, type_sym(i),
SIZET2NUM(
objspace->rcompactor.considered_count_table[i]));
13215 if (
objspace->rcompactor.moved_count_table[i]) {
13216 rb_hash_aset(moved, type_sym(i),
SIZET2NUM(
objspace->rcompactor.moved_count_table[i]));
13219 if (
objspace->rcompactor.moved_up_count_table[i]) {
13220 rb_hash_aset(moved_up, type_sym(i),
SIZET2NUM(
objspace->rcompactor.moved_up_count_table[i]));
13223 if (
objspace->rcompactor.moved_down_count_table[i]) {
13224 rb_hash_aset(moved_down, type_sym(i),
SIZET2NUM(
objspace->rcompactor.moved_down_count_table[i]));
13228 rb_hash_aset(h,
ID2SYM(rb_intern(
"considered")), considered);
13229 rb_hash_aset(h,
ID2SYM(rb_intern(
"moved")), moved);
13230 rb_hash_aset(h,
ID2SYM(rb_intern(
"moved_up")), moved_up);
13231 rb_hash_aset(h,
ID2SYM(rb_intern(
"moved_down")), moved_down);
13236# define gc_compact_stats rb_f_notimplement
13239#if GC_CAN_COMPILE_COMPACTION
13259gc_compact(
VALUE self)
13261 rb_objspace_t *
objspace = rb_gc_get_objspace();
13262 int full_marking_p = gc_config_full_mark_val;
13263 gc_config_full_mark_set(TRUE);
13266 rb_gc_impl_start(rb_gc_get_objspace(),
true,
true,
true,
true,
true);
13267 gc_config_full_mark_set(full_marking_p);
13269 return gc_compact_stats(self);
13272# define gc_compact rb_f_notimplement
13275#if GC_CAN_COMPILE_COMPACTION
13276struct desired_compaction_pages_i_data {
13277 rb_objspace_t *objspace;
13278 size_t required_slots[HEAP_COUNT];
13282desired_compaction_pages_i(
struct heap_page *page,
void *data)
13284 struct desired_compaction_pages_i_data *tdata = data;
13285 rb_objspace_t *
objspace = tdata->objspace;
13287 VALUE vend = vstart + (
VALUE)(page->total_slots * page->heap->slot_size);
13290 for (
VALUE v = vstart; v != vend; v += page->heap->slot_size) {
13291 asan_unpoisoning_object(v) {
13294 rb_heap_t *dest_pool = gc_compact_destination_pool(
objspace, page->heap, v);
13295 size_t dest_pool_idx = dest_pool - heaps;
13296 tdata->required_slots[dest_pool_idx]++;
13320gc_verify_compaction_references(
int argc,
VALUE* argv,
VALUE self)
13322 static ID keywords[3] = {0};
13323 if (!keywords[0]) {
13324 keywords[0] = rb_intern(
"toward");
13325 keywords[1] = rb_intern(
"double_heap");
13326 keywords[2] = rb_intern(
"expand_heap");
13333 int kwarg_count =
rb_get_kwargs(options, keywords, 0, 3, arguments);
13334 bool toward_empty = kwarg_count > 0 &&
SYMBOL_P(arguments[0]) &&
SYM2ID(arguments[0]) == rb_intern(
"empty");
13335 bool expand_heap = (kwarg_count > 1 &&
RTEST(arguments[1])) || (kwarg_count > 2 &&
RTEST(arguments[2]));
13337 rb_objspace_t *
objspace = rb_gc_get_objspace();
13342 if (!rb_gc_single_objspace_p()) {
13343 rb_gc_impl_start(
objspace,
true,
true,
true,
false,
false);
13344 return gc_compact_stats(self);
13348 rb_gc_impl_start(
objspace,
true,
true,
true,
false,
false);
13350 unsigned int lev = RB_GC_VM_LOCK();
13356 struct desired_compaction_pages_i_data desired_compaction = {
13358 .required_slots = {0},
13361 objspace_each_pages(
objspace, desired_compaction_pages_i, &desired_compaction, TRUE);
13364 size_t max_existing_pages = 0;
13365 for (
int i = 0; i < HEAP_COUNT; i++) {
13366 rb_heap_t *heap = &heaps[i];
13367 max_existing_pages = MAX(max_existing_pages, heap->total_pages);
13371 for (
int i = 0; i < HEAP_COUNT; i++) {
13372 rb_heap_t *heap = &heaps[i];
13374 size_t pages_to_add = 0;
13381 pages_to_add += max_existing_pages - heap->total_pages;
13386 objspace->heap_pages.allocatable_bytes = desired_compaction.required_slots[i] * heap->slot_size;
13387 while (
objspace->heap_pages.allocatable_bytes > 0) {
13388 heap_page_allocate_and_initialize(
objspace, heap);
13396 for (; pages_to_add > 0; pages_to_add--) {
13397 heap_page_allocate_and_initialize_force(
objspace, heap);
13402 if (toward_empty) {
13403 objspace->rcompactor.compare_func = compare_free_slots;
13406 RB_GC_VM_UNLOCK(lev);
13408 rb_gc_impl_start(rb_gc_get_objspace(),
true,
true,
true,
true,
false);
13410 rb_objspace_reachable_objects_from_root(root_obj_check_moved_i,
objspace);
13413 objspace->rcompactor.compare_func = NULL;
13415 return gc_compact_stats(self);
13418# define gc_verify_compaction_references rb_f_notimplement
13422rb_gc_impl_objspace_free(
void *objspace_ptr)
13424 rb_objspace_t *
objspace = objspace_ptr;
13427 rb_bug(
"lazy sweeping underway when freeing object space");
13432 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
13435 rb_darray_free_without_gc(
objspace->heap_pages.sorted);
13436 heap_pages_lomem = 0;
13437 heap_pages_himem = 0;
13439 for (
int i = 0; i < HEAP_COUNT; i++) {
13440 rb_heap_t *heap = &heaps[i];
13441 heap->total_pages = 0;
13442 heap->total_slots = 0;
13445 free_stack_chunks(&
objspace->mark_stack);
13446 mark_stack_free_cache(&
objspace->mark_stack);
13448 rb_darray_free_without_gc(
objspace->weak_references);
13450#ifdef MALLOC_COUNTERS_NEED_LOCK
13459#if MALLOC_ALLOCATED_SIZE
13470gc_malloc_allocated_size(
VALUE self)
13472 rb_objspace_t *
objspace = (rb_objspace_t *)rb_gc_get_objspace();
13486gc_malloc_allocations(
VALUE self)
13488 rb_objspace_t *
objspace = (rb_objspace_t *)rb_gc_get_objspace();
13494rb_gc_impl_before_fork(
void *objspace_ptr)
13496 rb_objspace_t *
objspace = objspace_ptr;
13498 objspace->fork_vm_lock_lev = RB_GC_VM_LOCK();
13499 rb_gc_vm_barrier();
13503rb_gc_impl_after_fork(
void *objspace_ptr, rb_pid_t pid)
13505 rb_objspace_t *
objspace = objspace_ptr;
13508 rb_gc_vm_each_objspace(gc_process_stat_after_fork_i, NULL);
13511 RB_GC_VM_UNLOCK(
objspace->fork_vm_lock_lev);
13517 global_objspace->main_objspace =
objspace;
13518 page_pool_lock_initialize(&rb_global_objspace_instance.page_pool.lock);
13524#if GC_DEBUG_STRESS_TO_CLASS
13533rb_gcdebug_add_stress_to_class(
int argc,
VALUE *argv,
VALUE self)
13535 rb_objspace_t *
objspace = rb_gc_get_objspace();
13537 if (!stress_to_class) {
13538 set_stress_to_class(rb_ident_hash_new_capa(argc));
13541 for (
int i = 0; i < argc; i++) {
13542 VALUE klass = argv[i];
13543 rb_hash_aset(stress_to_class, klass,
Qtrue);
13558rb_gcdebug_remove_stress_to_class(
int argc,
VALUE *argv,
VALUE self)
13560 rb_objspace_t *
objspace = rb_gc_get_objspace();
13562 if (stress_to_class) {
13563 for (
int i = 0; i < argc; ++i) {
13564 rb_hash_delete(stress_to_class, argv[i]);
13567 if (rb_hash_size(stress_to_class) == 0) {
13568 stress_to_class = 0;
13577rb_gc_impl_objspace_alloc(
void)
13579 global_objspace_init();
13581 rb_objspace_t *
objspace = calloc1(
sizeof(rb_objspace_t));
13590rb_gc_impl_objspace_init(
void *objspace_ptr)
13592 rb_objspace_t *
objspace = objspace_ptr;
13594 gc_config_full_mark_set(TRUE);
13596 malloc_limit = gc_params.malloc_limit_min;
13597 objspace->shareable_objects_limit = SHAREABLE_OBJECTS_LIMIT_MIN;
13598#ifdef MALLOC_COUNTERS_NEED_LOCK
13603 if (
objspace->finalize_deferred_pjob == POSTPONED_JOB_HANDLE_INVALID) {
13604 rb_bug(
"Could not preregister postponed job for GC");
13607 gc_tdata_deferred_free_pjob_ensure();
13612 GC_ASSERT(rb_gc_impl_size_allocatable_p(
sizeof(
struct RBasic) +
sizeof(
VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX])));
13614 for (
int i = 0; i < HEAP_COUNT; i++) {
13615 rb_heap_t *heap = &heaps[i];
13617 heap->slot_size = pool_slot_sizes[i];
13619 ccan_list_head_init(&heap->pages);
13622 if (global_objspace->main_objspace == NULL) {
13626 global_objspace->main_objspace =
objspace;
13628 init_size_to_heap_idx();
13630#if defined(INIT_HEAP_PAGE_ALLOC_USE_MMAP)
13632 heap_page_alloc_use_mmap = INIT_HEAP_PAGE_ALLOC_USE_MMAP;
13634 gc_params.heap_init_bytes = GC_HEAP_INIT_BYTES;
13635 gc_params.ractor_heap_init_bytes = GC_RACTOR_HEAP_INIT_BYTES ? GC_RACTOR_HEAP_INIT_BYTES
13636 : heap_init_bytes_min();
13640 objspace->flags.measure_gc = global_objspace->main_objspace ==
objspace ? true
13641 : ((rb_objspace_t *)rb_gc_get_objspace())->flags.measure_gc;
13643 rb_darray_make_without_gc(&
objspace->heap_pages.sorted, 0);
13644 rb_darray_make_without_gc(&
objspace->weak_references, 0);
13646#if RGENGC_ESTIMATE_OLDMALLOC
13647 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
13650 init_mark_stack(&
objspace->mark_stack);
13652 objspace->profile.invoke_time = getrusage_time();
13653 objspace->profile.invoke_wall_time = rb_hrtime_now();
13654 objspace->profile.max_records = GC_PROFILE_RECORD_DEFAULT_MAX_RECORDS;
13655 finalizer_table = st_init_numtable();
13657 gc_process_stat_publish(
objspace);
13661rb_gc_impl_init(
void)
13666 setup_gc_stat_symbols();
13667 setup_gc_stat_heap_symbols();
13668 setup_gc_latest_gc_info_symbols();
13671 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"DEBUG")), GC_DEBUG ?
Qtrue :
Qfalse);
13673 size_t rvalue_pool = 0;
13674 for (
size_t i = 0; i < HEAP_COUNT; i++) {
13675 if (pool_slot_sizes[i] >= RVALUE_SLOT_SIZE) { rvalue_pool = pool_slot_sizes[i];
break; }
13677 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVALUE_SIZE")),
SIZET2NUM(rvalue_pool - RVALUE_OVERHEAD));
13679 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVALUE_OVERHEAD")),
SIZET2NUM(RVALUE_OVERHEAD));
13680 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"HEAP_PAGE_BITMAP_SIZE")),
SIZET2NUM(HEAP_PAGE_BITMAP_SIZE));
13681 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"HEAP_PAGE_SIZE")),
SIZET2NUM(HEAP_PAGE_SIZE));
13682 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"HEAP_COUNT")),
LONG2FIX(HEAP_COUNT));
13683 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVARGC_MAX_ALLOCATE_SIZE")),
SIZET2NUM(rb_gc_impl_max_allocation_size()));
13684 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVALUE_OLD_AGE")),
LONG2FIX(RVALUE_OLD_AGE));
13685 if (RB_BUG_INSTEAD_OF_RB_MEMERROR+0) {
13686 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RB_BUG_INSTEAD_OF_RB_MEMERROR")),
Qtrue);
13690 rb_define_const(
rb_mGC,
"INTERNAL_CONSTANTS", gc_constants);
13692 if (GC_COMPACTION_SUPPORTED) {
13707#if GC_DEBUG_STRESS_TO_CLASS
13715#if MALLOC_ALLOCATED_SIZE
13745 VALUE rb_mProfiler = rb_define_module_under(
rb_mGC,
"Profiler");
13760#define OPT(o) if (o) rb_ary_push(opts, rb_interned_str(#o, sizeof(#o) - 1))
13764 OPT(RGENGC_CHECK_MODE);
13765 OPT(RGENGC_PROFILE);
13766 OPT(RGENGC_ESTIMATE_OLDMALLOC);
13767 OPT(GC_PROFILE_MORE_DETAIL);
13768 OPT(GC_ENABLE_LAZY_SWEEP);
13769 OPT(CALC_EXACT_MALLOC_SIZE);
13770 OPT(MALLOC_ALLOCATED_SIZE);
13771 OPT(MALLOC_ALLOCATED_SIZE_CHECK);
13772 OPT(GC_PROFILE_DETAIL_MEMORY);
13773 OPT(GC_COMPACTION_SUPPORTED);
#define RUBY_ATOMIC_VALUE_CAS(var, oldval, newval)
Identical to RUBY_ATOMIC_CAS, except it expects its arguments are VALUE.
#define RUBY_ATOMIC_PTR_LOAD(var)
Identical to RUBY_ATOMIC_LOAD, except it expects its arguments are void*.
#define RUBY_ATOMIC_SIZE_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are size_t.
#define RUBY_ATOMIC_SIZE_INC(var)
Identical to RUBY_ATOMIC_INC, except it expects its argument is size_t.
#define RUBY_ATOMIC_SIZE_CAS(var, oldval, newval)
Identical to RUBY_ATOMIC_CAS, except it expects its arguments are size_t.
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
#define RUBY_ATOMIC_SIZE_ADD(var, val)
Identical to RUBY_ATOMIC_ADD, except it expects its arguments are size_t.
#define RUBY_ATOMIC_VALUE_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are VALUE.
#define RUBY_ATOMIC_PTR_SET(var, val)
Identical to RUBY_ATOMIC_SET, except it expects its arguments are void*.
#define RUBY_ATOMIC_SET(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except for the return type.
#define RUBY_ATOMIC_EXCHANGE(var, val)
Atomically replaces the value pointed by var with val.
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
unsigned int rb_postponed_job_handle_t
The type of a handle returned from rb_postponed_job_preregister and passed to rb_postponed_job_trigge...
void rb_postponed_job_trigger(rb_postponed_job_handle_t h)
Triggers a pre-registered job registered with rb_postponed_job_preregister, scheduling it for executi...
rb_postponed_job_handle_t rb_postponed_job_preregister(unsigned int flags, rb_postponed_job_func_t func, void *data)
Pre-registers a func in Ruby's postponed job preregistration table, returning an opaque handle which ...
#define RB_GNUC_EXTENSION_BLOCK(x)
This is expanded to the passed token for non-GCC compilers.
#define RUBY_INTERNAL_EVENT_GC_EXIT
gc_exit() is called.
#define RUBY_INTERNAL_EVENT_GC_ENTER
gc_enter() is called.
#define RUBY_INTERNAL_EVENT_GC_END_SWEEP
GC ended sweep phase.
#define RUBY_INTERNAL_EVENT_GC_END_MARK
GC ended mark phase.
#define RUBY_INTERNAL_EVENT_OBJSPACE_MASK
Bitmask of GC events.
#define RUBY_INTERNAL_EVENT_FREEOBJ
Object swept.
#define RUBY_INTERNAL_EVENT_GC_START
GC started.
uint32_t rb_event_flag_t
Represents event(s).
static VALUE RB_FL_TEST(VALUE obj, VALUE flags)
Tests if the given flag(s) are set or not.
static VALUE RB_FL_TEST_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_TEST().
static void RB_FL_SET_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_SET().
static void RB_FL_UNSET_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_UNSET().
@ RUBY_FL_PROMOTED
Ruby objects are "generational".
@ RUBY_FL_SHAREABLE
This flag has something to do with Ractor.
@ RUBY_FL_WEAK_REFERENCE
This object weakly refers to other objects.
int rb_scan_args_kw(int kw_flag, int argc, const VALUE *argv, const char *fmt,...)
Identical to rb_scan_args(), except it also accepts kw_splat.
int rb_keyword_given_p(void)
Determines if the current method is given a keyword argument.
int rb_get_kwargs(VALUE keyword_hash, const ID *table, int required, int optional, VALUE *values)
Keyword argument deconstructor.
#define T_COMPLEX
Old name of RUBY_T_COMPLEX.
#define T_FILE
Old name of RUBY_T_FILE.
#define T_STRING
Old name of RUBY_T_STRING.
#define xfree
Old name of ruby_xfree.
#define T_MASK
Old name of RUBY_T_MASK.
#define Qundef
Old name of RUBY_Qundef.
#define INT2FIX
Old name of RB_INT2FIX.
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
#define T_NIL
Old name of RUBY_T_NIL.
#define T_FLOAT
Old name of RUBY_T_FLOAT.
#define T_IMEMO
Old name of RUBY_T_IMEMO.
#define ID2SYM
Old name of RB_ID2SYM.
#define T_BIGNUM
Old name of RUBY_T_BIGNUM.
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
#define T_STRUCT
Old name of RUBY_T_STRUCT.
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
#define T_FIXNUM
Old name of RUBY_T_FIXNUM.
#define SYM2ID
Old name of RB_SYM2ID.
#define T_DATA
Old name of RUBY_T_DATA.
#define FL_SHAREABLE
Old name of RUBY_FL_SHAREABLE.
#define T_NONE
Old name of RUBY_T_NONE.
#define T_NODE
Old name of RUBY_T_NODE.
#define SIZET2NUM
Old name of RB_SIZE2NUM.
#define xmalloc
Old name of ruby_xmalloc.
#define LONG2FIX
Old name of RB_INT2FIX.
#define FIX2INT
Old name of RB_FIX2INT.
#define FL_FINALIZE
Old name of RUBY_FL_FINALIZE.
#define T_MODULE
Old name of RUBY_T_MODULE.
#define T_TRUE
Old name of RUBY_T_TRUE.
#define T_RATIONAL
Old name of RUBY_T_RATIONAL.
#define T_ICLASS
Old name of RUBY_T_ICLASS.
#define T_HASH
Old name of RUBY_T_HASH.
#define ALLOC_N
Old name of RB_ALLOC_N.
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
#define FL_SET
Old name of RB_FL_SET.
#define rb_ary_new3
Old name of rb_ary_new_from_args.
#define T_FALSE
Old name of RUBY_T_FALSE.
#define ULL2NUM
Old name of RB_ULL2NUM.
#define T_UNDEF
Old name of RUBY_T_UNDEF.
#define Qtrue
Old name of RUBY_Qtrue.
#define T_ZOMBIE
Old name of RUBY_T_ZOMBIE.
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define T_ARRAY
Old name of RUBY_T_ARRAY.
#define T_OBJECT
Old name of RUBY_T_OBJECT.
#define NIL_P
Old name of RB_NIL_P.
#define FL_WB_PROTECTED
Old name of RUBY_FL_WB_PROTECTED.
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
#define DBL2NUM
Old name of rb_float_new.
#define T_MATCH
Old name of RUBY_T_MATCH.
#define T_CLASS
Old name of RUBY_T_CLASS.
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
#define T_MOVED
Old name of RUBY_T_MOVED.
#define FL_TEST
Old name of RB_FL_TEST.
#define NUM2LONG
Old name of RB_NUM2LONG.
#define FL_UNSET
Old name of RB_FL_UNSET.
#define FIXNUM_P
Old name of RB_FIXNUM_P.
#define FL_SET_RAW
Old name of RB_FL_SET_RAW.
#define SYMBOL_P
Old name of RB_SYMBOL_P.
#define T_REGEXP
Old name of RUBY_T_REGEXP.
#define ruby_verbose
This variable controls whether the interpreter is in debug mode.
VALUE rb_eRuntimeError
RuntimeError exception.
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
VALUE rb_obj_hide(VALUE obj)
Make the object invisible from Ruby code.
VALUE rb_stdout
STDOUT constant.
#define RBIMPL_ASSERT_OR_ASSUME(...)
This is either RUBY_ASSERT or RBIMPL_ASSUME, depending on RUBY_DEBUG.
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Routines to manipulate encodings of strings.
void rb_memerror(void)
Triggers out-of-memory error.
static bool RB_OBJ_PROMOTED_RAW(VALUE obj)
This is the implementation of RB_OBJ_PROMOTED().
size_t rb_gc_count(void)
Identical to rb_gc_stat(), with "count" parameter.
VALUE rb_ary_dup(VALUE ary)
Duplicates an array.
VALUE rb_ary_new(void)
Allocates a new, empty array.
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
VALUE rb_fiber_current(void)
Queries the fiber which is calling this function.
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
VALUE rb_hash_new(void)
Creates a new, empty hash object.
VALUE rb_str_buf_append(VALUE dst, VALUE src)
Identical to rb_str_cat_cstr(), except it takes Ruby's string instead of C's.
VALUE rb_str_buf_new(rb_len_t capa)
Allocates a "string buffer".
#define rb_str_new_cstr(str)
Identical to rb_str_new, except it assumes the passed pointer is a pointer to a C string.
const char * rb_sourcefile(void)
Resembles __FILE__.
VALUE rb_f_notimplement(int argc, const VALUE *argv, VALUE obj, VALUE marker)
Raises rb_eNotImpError.
int rb_sourceline(void)
Resembles __LINE__.
#define RB_SYM2ID
Just another name of rb_sym2id.
ID rb_sym2id(VALUE obj)
Converts an instance of rb_cSymbol into an ID.
int capa
Designed capacity of the buffer.
int off
Offset inside of ptr.
int len
Length of the buffer.
int ruby_thread_has_gvl_p(void)
Whether the current thread is holding the GVL.
void * rb_thread_call_with_gvl(void *(*func)(void *), void *data1)
(Re-)acquires the GVL.
#define strtod(s, e)
Just another name of ruby_strtod.
void ruby_qsort(void *, const size_t, const size_t, int(*)(const void *, const void *, void *), void *)
Reentrant implementation of quick sort.
#define MEMZERO(p, type, n)
Handy macro to erase a region of memory.
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
VALUE type(ANYARGS)
ANYARGS-ed function type.
int st_foreach(st_table *q, int_type *w, st_data_t e)
Iteration over the given table.
void rb_hash_foreach(VALUE q, int_type *w, VALUE e)
Iteration over the given hash.
VALUE rb_ensure(type *q, VALUE w, type *e, VALUE r)
An equivalent of ensure clause.
#define RARRAY_LEN
Just another name of rb_array_len.
#define RARRAY_AREF(a, i)
References an object in an array.
static void RARRAY_ASET(VALUE ary, rb_len_t i, VALUE v)
Assigns an object in an array.
#define RBASIC(obj)
Convenient casting macro.
#define RUBY_TYPED_FREE_IMMEDIATELY
Macros to see if each corresponding flag is defined.
static const rb_data_type_t * RTYPEDDATA_TYPE(VALUE obj)
Queries for the type of given object.
struct rb_data_type_struct rb_data_type_t
This is the struct that holds necessary info for a struct.
#define RTYPEDDATA(obj)
Convenient casting macro.
#define errno
Ractor-aware version of errno.
int ruby_native_thread_p(void)
Queries if the thread which calls this function is a ruby's thread.
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
#define RTEST
This is an old name of RB_TEST.
#define _(args)
This was a transition path from K&R to ANSI.
Ruby object's base components.
VALUE flags
Per-object flags.
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
long rb_len_t
A signed integer type for lengths and indices of objects such as String and Array.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
ruby_value_type
C-level type of an object.
@ RUBY_T_ICLASS
Hidden classes known as IClasses.
@ RUBY_T_FIXNUM
Integers formerly known as Fixnums.
@ RUBY_T_MASK
Bitmask of ruby_value_type.
@ RUBY_T_NONE
Non-object (swept etc.).