1#include "ruby/internal/config.h"
8# ifdef HAVE_SYS_PRCTL_H
13#if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H)
18#include "internal/bits.h"
19#include "internal/hash.h"
28#include "ccan/list/list.h"
31#include "gc/gc_impl.h"
33#ifndef BUILDING_MODULAR_GC
37#include "debug_counter.h"
38#include "internal/sanitizers.h"
41#ifndef HAVE_MALLOC_USABLE_SIZE
43# define HAVE_MALLOC_USABLE_SIZE
44# define malloc_usable_size(a) _msize(a)
45# elif defined HAVE_MALLOC_SIZE
46# define HAVE_MALLOC_USABLE_SIZE
47# define malloc_usable_size(a) malloc_size(a)
51#ifdef HAVE_MALLOC_USABLE_SIZE
52# ifdef RUBY_ALTERNATIVE_MALLOC_HEADER
54# elif defined(HAVE_MALLOC_H)
56# elif defined(HAVE_MALLOC_NP_H)
57# include <malloc_np.h>
58# elif defined(HAVE_MALLOC_MALLOC_H)
59# include <malloc/malloc.h>
63#ifdef HAVE_MALLOC_TRIM
68# include <emscripten/emmalloc.h>
72#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
73# include <mach/task.h>
74# include <mach/mach_init.h>
75# include <mach/mach_port.h>
79# define VM_CHECK_MODE RUBY_DEBUG
83#ifndef RACTOR_CHECK_MODE
84# define RACTOR_CHECK_MODE (VM_CHECK_MODE || RUBY_DEBUG) && (SIZEOF_UINT64_T == SIZEOF_VALUE)
88# define RUBY_DEBUG_LOG(...)
91#ifndef GC_HEAP_INIT_SLOTS
92#define GC_HEAP_INIT_SLOTS 10000
94#ifndef GC_HEAP_FREE_SLOTS
95#define GC_HEAP_FREE_SLOTS 4096
97#ifndef GC_HEAP_GROWTH_FACTOR
98#define GC_HEAP_GROWTH_FACTOR 1.8
100#ifndef GC_HEAP_GROWTH_MAX_SLOTS
101#define GC_HEAP_GROWTH_MAX_SLOTS 0
103#ifndef GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO
104# define GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO 0.01
106#ifndef GC_HEAP_OLDOBJECT_LIMIT_FACTOR
107#define GC_HEAP_OLDOBJECT_LIMIT_FACTOR 2.0
110#ifndef GC_HEAP_FREE_SLOTS_MIN_RATIO
111#define GC_HEAP_FREE_SLOTS_MIN_RATIO 0.20
113#ifndef GC_HEAP_FREE_SLOTS_GOAL_RATIO
114#define GC_HEAP_FREE_SLOTS_GOAL_RATIO 0.40
116#ifndef GC_HEAP_FREE_SLOTS_MAX_RATIO
117#define GC_HEAP_FREE_SLOTS_MAX_RATIO 0.65
120#ifndef GC_MALLOC_LIMIT_MIN
121#define GC_MALLOC_LIMIT_MIN (16 * 1024 * 1024 )
123#ifndef GC_MALLOC_LIMIT_MAX
124#define GC_MALLOC_LIMIT_MAX (32 * 1024 * 1024 )
126#ifndef GC_MALLOC_LIMIT_GROWTH_FACTOR
127#define GC_MALLOC_LIMIT_GROWTH_FACTOR 1.4
130#ifndef GC_OLDMALLOC_LIMIT_MIN
131#define GC_OLDMALLOC_LIMIT_MIN (16 * 1024 * 1024 )
133#ifndef GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
134#define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR 1.2
136#ifndef GC_OLDMALLOC_LIMIT_MAX
137#define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 )
140#ifndef GC_CAN_COMPILE_COMPACTION
142# define GC_CAN_COMPILE_COMPACTION 0
144# define GC_CAN_COMPILE_COMPACTION 1
148#ifndef PRINT_ENTER_EXIT_TICK
149# define PRINT_ENTER_EXIT_TICK 0
151#ifndef PRINT_ROOT_TICKS
152#define PRINT_ROOT_TICKS 0
155#define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS)
167 size_t incremental_mark_step_allocated_slots;
172 size_t heap_init_slots[HEAP_COUNT];
173 size_t heap_free_slots;
174 double growth_factor;
175 size_t growth_max_slots;
177 double heap_free_slots_min_ratio;
178 double heap_free_slots_goal_ratio;
179 double heap_free_slots_max_ratio;
180 double uncollectible_wb_unprotected_objects_limit_ratio;
181 double oldobject_limit_factor;
183 size_t malloc_limit_min;
184 size_t malloc_limit_max;
185 double malloc_limit_growth_factor;
187 size_t oldmalloc_limit_min;
188 size_t oldmalloc_limit_max;
189 double oldmalloc_limit_growth_factor;
193 { GC_HEAP_INIT_SLOTS },
195 GC_HEAP_GROWTH_FACTOR,
196 GC_HEAP_GROWTH_MAX_SLOTS,
198 GC_HEAP_FREE_SLOTS_MIN_RATIO,
199 GC_HEAP_FREE_SLOTS_GOAL_RATIO,
200 GC_HEAP_FREE_SLOTS_MAX_RATIO,
201 GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO,
202 GC_HEAP_OLDOBJECT_LIMIT_FACTOR,
206 GC_MALLOC_LIMIT_GROWTH_FACTOR,
208 GC_OLDMALLOC_LIMIT_MIN,
209 GC_OLDMALLOC_LIMIT_MAX,
210 GC_OLDMALLOC_LIMIT_GROWTH_FACTOR,
229#define RGENGC_DEBUG -1
231#define RGENGC_DEBUG 0
234#if RGENGC_DEBUG < 0 && !defined(_MSC_VER)
235# define RGENGC_DEBUG_ENABLED(level) (-(RGENGC_DEBUG) >= (level) && ruby_rgengc_debug >= (level))
236#elif defined(HAVE_VA_ARGS_MACRO)
237# define RGENGC_DEBUG_ENABLED(level) ((RGENGC_DEBUG) >= (level))
239# define RGENGC_DEBUG_ENABLED(level) 0
241int ruby_rgengc_debug;
248#ifndef RGENGC_PROFILE
249# define RGENGC_PROFILE 0
258#ifndef RGENGC_ESTIMATE_OLDMALLOC
259# define RGENGC_ESTIMATE_OLDMALLOC 1
262#ifndef GC_PROFILE_MORE_DETAIL
263# define GC_PROFILE_MORE_DETAIL 0
265#ifndef GC_PROFILE_DETAIL_MEMORY
266# define GC_PROFILE_DETAIL_MEMORY 0
268#ifndef GC_ENABLE_LAZY_SWEEP
269# define GC_ENABLE_LAZY_SWEEP 1
271#ifndef CALC_EXACT_MALLOC_SIZE
272# define CALC_EXACT_MALLOC_SIZE 0
274#if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0
275# ifndef MALLOC_ALLOCATED_SIZE
276# define MALLOC_ALLOCATED_SIZE 0
279# define MALLOC_ALLOCATED_SIZE 0
281#ifndef MALLOC_ALLOCATED_SIZE_CHECK
282# define MALLOC_ALLOCATED_SIZE_CHECK 0
285#ifndef GC_DEBUG_STRESS_TO_CLASS
286# define GC_DEBUG_STRESS_TO_CLASS 1
290 GPR_FLAG_NONE = 0x000,
292 GPR_FLAG_MAJOR_BY_NOFREE = 0x001,
293 GPR_FLAG_MAJOR_BY_OLDGEN = 0x002,
294 GPR_FLAG_MAJOR_BY_SHADY = 0x004,
295 GPR_FLAG_MAJOR_BY_FORCE = 0x008,
296#if RGENGC_ESTIMATE_OLDMALLOC
297 GPR_FLAG_MAJOR_BY_OLDMALLOC = 0x020,
299 GPR_FLAG_MAJOR_MASK = 0x0ff,
302 GPR_FLAG_NEWOBJ = 0x100,
303 GPR_FLAG_MALLOC = 0x200,
304 GPR_FLAG_METHOD = 0x400,
305 GPR_FLAG_CAPI = 0x800,
306 GPR_FLAG_STRESS = 0x1000,
309 GPR_FLAG_IMMEDIATE_SWEEP = 0x2000,
310 GPR_FLAG_HAVE_FINALIZE = 0x4000,
311 GPR_FLAG_IMMEDIATE_MARK = 0x8000,
312 GPR_FLAG_FULL_MARK = 0x10000,
313 GPR_FLAG_COMPACT = 0x20000,
316 (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK |
317 GPR_FLAG_IMMEDIATE_SWEEP | GPR_FLAG_CAPI),
318} gc_profile_record_flag;
324 double gc_invoke_time;
326 size_t heap_total_objects;
327 size_t heap_use_size;
328 size_t heap_total_size;
329 size_t moved_objects;
331#if GC_PROFILE_MORE_DETAIL
333 double gc_sweep_time;
335 size_t heap_use_pages;
336 size_t heap_live_objects;
337 size_t heap_free_objects;
339 size_t allocate_increase;
340 size_t allocate_limit;
343 size_t removing_objects;
344 size_t empty_objects;
345#if GC_PROFILE_DETAIL_MEMORY
351#if MALLOC_ALLOCATED_SIZE
352 size_t allocated_size;
355#if RGENGC_PROFILE > 0
357 size_t remembered_normal_objects;
358 size_t remembered_shady_objects;
366 uint32_t original_shape_id;
369#define RMOVED(obj) ((struct RMoved *)(obj))
371typedef uintptr_t bits_t;
373 BITS_SIZE =
sizeof(bits_t),
374 BITS_BITLENGTH = ( BITS_SIZE * CHAR_BIT )
387#define STACK_CHUNK_SIZE 500
390 VALUE data[STACK_CHUNK_SIZE];
400 size_t unused_cache_size;
403typedef int (*gc_compact_compare_func)(
const void *l,
const void *r,
void *d);
409 size_t total_allocated_pages;
410 size_t force_major_gc_count;
411 size_t force_incremental_marking_finish_count;
412 size_t total_allocated_objects;
413 size_t total_freed_objects;
414 size_t final_slots_count;
421 struct ccan_list_head pages;
424 uintptr_t compact_cursor_index;
433 gc_stress_no_immediate_sweep,
434 gc_stress_full_mark_after_malloc,
449#if MALLOC_ALLOCATED_SIZE
450 size_t allocated_size;
460 unsigned int mode : 2;
461 unsigned int immediate_sweep : 1;
462 unsigned int dont_gc : 1;
463 unsigned int dont_incremental : 1;
464 unsigned int during_gc : 1;
465 unsigned int during_compacting : 1;
466 unsigned int during_reference_updating : 1;
467 unsigned int gc_stressful: 1;
468 unsigned int has_newobj_hook: 1;
469 unsigned int during_minor_gc : 1;
470 unsigned int during_incremental_marking : 1;
471 unsigned int measure_gc : 1;
475 unsigned long long next_object_id;
478 size_t empty_pages_count;
491 size_t allocated_pages;
494 size_t freeable_pages;
496 size_t allocatable_slots;
499 VALUE deferred_final;
506 unsigned int latest_gc_info;
512#if GC_PROFILE_MORE_DETAIL
517 size_t minor_gc_count;
518 size_t major_gc_count;
519 size_t compact_count;
520 size_t read_barrier_faults;
521#if RGENGC_PROFILE > 0
522 size_t total_generated_normal_object_count;
523 size_t total_generated_shady_object_count;
524 size_t total_shade_operation_count;
525 size_t total_promoted_count;
526 size_t total_remembered_normal_object_count;
527 size_t total_remembered_shady_object_count;
529#if RGENGC_PROFILE >= 2
530 size_t generated_normal_object_count_types[
RUBY_T_MASK];
531 size_t generated_shady_object_count_types[
RUBY_T_MASK];
534 size_t remembered_normal_object_count_types[
RUBY_T_MASK];
535 size_t remembered_shady_object_count_types[
RUBY_T_MASK];
540 double gc_sweep_start_time;
541 size_t total_allocated_objects_at_gc_start;
542 size_t heap_used_at_gc_start;
546 unsigned long long marking_time_ns;
548 unsigned long long sweeping_time_ns;
549 struct timespec sweeping_start_time;
552 size_t weak_references_count;
553 size_t retained_weak_references_count;
556 VALUE gc_stress_mode;
561 size_t last_major_gc;
562 size_t uncollectible_wb_unprotected_objects;
563 size_t uncollectible_wb_unprotected_objects_limit;
565 size_t old_objects_limit;
567#if RGENGC_ESTIMATE_OLDMALLOC
568 size_t oldmalloc_increase;
569 size_t oldmalloc_increase_limit;
572#if RGENGC_CHECK_MODE >= 2
579 size_t considered_count_table[
T_MASK];
580 size_t moved_count_table[
T_MASK];
581 size_t moved_up_count_table[
T_MASK];
582 size_t moved_down_count_table[
T_MASK];
586 gc_compact_compare_func compare_func;
597#if GC_DEBUG_STRESS_TO_CLASS
598 VALUE stress_to_class;
601 rb_darray(
VALUE *) weak_references;
604 unsigned long live_ractor_cache_count;
607#ifndef HEAP_PAGE_ALIGN_LOG
609#define HEAP_PAGE_ALIGN_LOG 16
612#if RACTOR_CHECK_MODE || GC_DEBUG
613struct rvalue_overhead {
614# if RACTOR_CHECK_MODE
615 uint32_t _ractor_belonging_id;
624# define RVALUE_OVERHEAD (sizeof(struct { \
626 struct rvalue_overhead overhead; \
630size_t rb_gc_impl_obj_slot_size(
VALUE obj);
631# define GET_RVALUE_OVERHEAD(obj) ((struct rvalue_overhead *)((uintptr_t)obj + rb_gc_impl_obj_slot_size(obj)))
633# define RVALUE_OVERHEAD 0
636#define BASE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD)
639# define MAX(a, b) (((a) > (b)) ? (a) : (b))
642# define MIN(a, b) (((a) < (b)) ? (a) : (b))
644#define roomof(x, y) (((x) + (y) - 1) / (y))
645#define CEILDIV(i, mod) roomof(i, mod)
647 HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG),
648 HEAP_PAGE_ALIGN_MASK = (~(~0UL << HEAP_PAGE_ALIGN_LOG)),
649 HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN,
650 HEAP_PAGE_OBJ_LIMIT = (
unsigned int)((HEAP_PAGE_SIZE -
sizeof(
struct heap_page_header)) / BASE_SLOT_SIZE),
651 HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, BASE_SLOT_SIZE), BITS_BITLENGTH),
652 HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),
654#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG)
655#define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN
657#if !defined(INCREMENTAL_MARK_STEP_ALLOCATIONS)
658# define INCREMENTAL_MARK_STEP_ALLOCATIONS 500
661#undef INIT_HEAP_PAGE_ALLOC_USE_MMAP
667static const bool HEAP_PAGE_ALLOC_USE_MMAP =
false;
669#elif defined(__wasm__)
673static const bool HEAP_PAGE_ALLOC_USE_MMAP =
false;
675#elif HAVE_CONST_PAGE_SIZE
677static const bool HEAP_PAGE_ALLOC_USE_MMAP = (PAGE_SIZE <= HEAP_PAGE_SIZE);
679#elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE)
681static const bool HEAP_PAGE_ALLOC_USE_MMAP =
true;
683#elif defined(PAGE_SIZE)
685# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (PAGE_SIZE <= HEAP_PAGE_SIZE)
687#elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
689# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE)
693static const bool HEAP_PAGE_ALLOC_USE_MMAP =
false;
696#ifdef INIT_HEAP_PAGE_ALLOC_USE_MMAP
698# define HEAP_PAGE_ALLOC_USE_MMAP (heap_page_alloc_use_mmap != false)
700static bool heap_page_alloc_use_mmap;
703#define RVALUE_AGE_BIT_COUNT 2
704#define RVALUE_AGE_BIT_MASK (((bits_t)1 << RVALUE_AGE_BIT_COUNT) - 1)
705#define RVALUE_OLD_AGE 3
713 unsigned short slot_size;
714 unsigned short total_slots;
715 unsigned short free_slots;
716 unsigned short final_slots;
717 unsigned short pinned_slots;
719 unsigned int before_sweep : 1;
720 unsigned int has_remembered_objects : 1;
721 unsigned int has_uncollectible_wb_unprotected_objects : 1;
730 struct ccan_list_node page_node;
732 bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT];
734 bits_t mark_bits[HEAP_PAGE_BITMAP_LIMIT];
735 bits_t uncollectible_bits[HEAP_PAGE_BITMAP_LIMIT];
736 bits_t marking_bits[HEAP_PAGE_BITMAP_LIMIT];
738 bits_t remembered_bits[HEAP_PAGE_BITMAP_LIMIT];
741 bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT];
742 bits_t age_bits[HEAP_PAGE_BITMAP_LIMIT * RVALUE_AGE_BIT_COUNT];
749asan_lock_freelist(
struct heap_page *page)
751 asan_poison_memory_region(&page->freelist,
sizeof(
struct free_list *));
758asan_unlock_freelist(
struct heap_page *page)
760 asan_unpoison_memory_region(&page->freelist,
sizeof(
struct free_list *),
false);
766 if (page->total_slots == 0) {
767 GC_ASSERT(page->start == 0);
768 GC_ASSERT(page->slot_size == 0);
769 GC_ASSERT(page->heap == NULL);
770 GC_ASSERT(page->free_slots == 0);
771 asan_unpoisoning_memory_region(&page->freelist,
sizeof(&page->freelist)) {
772 GC_ASSERT(page->freelist == NULL);
778 GC_ASSERT(page->start != 0);
779 GC_ASSERT(page->slot_size != 0);
780 GC_ASSERT(page->heap != NULL);
786#define GET_PAGE_BODY(x) ((struct heap_page_body *)((bits_t)(x) & ~(HEAP_PAGE_ALIGN_MASK)))
787#define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
788#define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)
790#define NUM_IN_PAGE(p) (((bits_t)(p) & HEAP_PAGE_ALIGN_MASK) / BASE_SLOT_SIZE)
791#define BITMAP_INDEX(p) (NUM_IN_PAGE(p) / BITS_BITLENGTH )
792#define BITMAP_OFFSET(p) (NUM_IN_PAGE(p) & (BITS_BITLENGTH-1))
793#define BITMAP_BIT(p) ((bits_t)1 << BITMAP_OFFSET(p))
796#define MARKED_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] & BITMAP_BIT(p))
797#define MARK_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] | BITMAP_BIT(p))
798#define CLEAR_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] & ~BITMAP_BIT(p))
801#define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0])
802#define GET_HEAP_PINNED_BITS(x) (&GET_HEAP_PAGE(x)->pinned_bits[0])
803#define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0])
804#define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0])
805#define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0])
807#define RVALUE_AGE_BITMAP_INDEX(n) (NUM_IN_PAGE(n) / (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT))
808#define RVALUE_AGE_BITMAP_OFFSET(n) ((NUM_IN_PAGE(n) % (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) * RVALUE_AGE_BIT_COUNT)
811RVALUE_AGE_GET(
VALUE obj)
813 bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits;
814 return (
int)(age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] >> RVALUE_AGE_BITMAP_OFFSET(obj)) & RVALUE_AGE_BIT_MASK;
818RVALUE_AGE_SET(
VALUE obj,
int age)
821 bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits;
823 age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] &= ~(RVALUE_AGE_BIT_MASK << (RVALUE_AGE_BITMAP_OFFSET(obj)));
825 age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] |= ((bits_t)age << RVALUE_AGE_BITMAP_OFFSET(obj));
826 if (age == RVALUE_OLD_AGE) {
834#define malloc_limit objspace->malloc_params.limit
835#define malloc_increase objspace->malloc_params.increase
836#define malloc_allocated_size objspace->malloc_params.allocated_size
837#define heap_pages_lomem objspace->heap_pages.range[0]
838#define heap_pages_himem objspace->heap_pages.range[1]
839#define heap_pages_freeable_pages objspace->heap_pages.freeable_pages
840#define heap_pages_deferred_final objspace->heap_pages.deferred_final
841#define heaps objspace->heaps
842#define during_gc objspace->flags.during_gc
843#define finalizing objspace->atomic_flags.finalizing
844#define finalizer_table objspace->finalizer_table
845#define ruby_gc_stressful objspace->flags.gc_stressful
846#define ruby_gc_stress_mode objspace->gc_stress_mode
847#if GC_DEBUG_STRESS_TO_CLASS
848#define stress_to_class objspace->stress_to_class
849#define set_stress_to_class(c) (stress_to_class = (c))
851#define stress_to_class ((void)objspace, 0)
852#define set_stress_to_class(c) ((void)objspace, (c))
856#define dont_gc_on() (fprintf(stderr, "dont_gc_on@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 1)
857#define dont_gc_off() (fprintf(stderr, "dont_gc_off@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 0)
858#define dont_gc_set(b) (fprintf(stderr, "dont_gc_set(%d)@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = (int)(b))
859#define dont_gc_val() (objspace->flags.dont_gc)
861#define dont_gc_on() (objspace->flags.dont_gc = 1)
862#define dont_gc_off() (objspace->flags.dont_gc = 0)
863#define dont_gc_set(b) (objspace->flags.dont_gc = (int)(b))
864#define dont_gc_val() (objspace->flags.dont_gc)
867#define gc_config_full_mark_set(b) (objspace->gc_config.full_mark = (int)(b))
868#define gc_config_full_mark_val (objspace->gc_config.full_mark)
870#ifndef DURING_GC_COULD_MALLOC_REGION_START
871# define DURING_GC_COULD_MALLOC_REGION_START() \
872 assert(rb_during_gc()); \
873 bool _prev_enabled = rb_gc_impl_gc_enabled_p(objspace); \
874 rb_gc_impl_gc_disable(objspace, false)
877#ifndef DURING_GC_COULD_MALLOC_REGION_END
878# define DURING_GC_COULD_MALLOC_REGION_END() \
879 if (_prev_enabled) rb_gc_impl_gc_enable(objspace)
882static inline enum gc_mode
883gc_mode_verify(
enum gc_mode mode)
885#if RGENGC_CHECK_MODE > 0
888 case gc_mode_marking:
889 case gc_mode_sweeping:
890 case gc_mode_compacting:
893 rb_bug(
"gc_mode_verify: unreachable (%d)", (
int)mode);
902 for (
int i = 0; i < HEAP_COUNT; i++) {
903 if ((&heaps[i])->sweeping_page) {
914 for (
int i = 0; i < HEAP_COUNT; i++) {
915 count += (&heaps[i])->total_pages;
924 for (
int i = 0; i < HEAP_COUNT; i++) {
926 count += heap->total_allocated_objects;
935 for (
int i = 0; i < HEAP_COUNT; i++) {
937 count += heap->total_freed_objects;
946 for (
int i = 0; i < HEAP_COUNT; i++) {
948 count += heap->final_slots_count;
953#define gc_mode(objspace) gc_mode_verify((enum gc_mode)(objspace)->flags.mode)
954#define gc_mode_set(objspace, m) ((objspace)->flags.mode = (unsigned int)gc_mode_verify(m))
955#define gc_needs_major_flags objspace->rgengc.need_major_gc
957#define is_marking(objspace) (gc_mode(objspace) == gc_mode_marking)
958#define is_sweeping(objspace) (gc_mode(objspace) == gc_mode_sweeping)
959#define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE)
960#define is_incremental_marking(objspace) ((objspace)->flags.during_incremental_marking != FALSE)
961#define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE)
962#define GC_INCREMENTAL_SWEEP_SLOT_COUNT 2048
963#define GC_INCREMENTAL_SWEEP_POOL_SLOT_COUNT 1024
964#define is_lazy_sweeping(objspace) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(objspace))
966#if SIZEOF_LONG == SIZEOF_VOIDP
967# define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG)
968#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
969# define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \
970 ((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1))
978 void (*dfree)(
void *);
982#define RZOMBIE(o) ((struct RZombie *)(o))
984static bool ruby_enable_autocompact =
false;
986static gc_compact_compare_func ruby_autocompact_compare_func;
990static int garbage_collect(
rb_objspace_t *,
unsigned int reason);
996 gc_enter_event_start,
997 gc_enter_event_continue,
999 gc_enter_event_finalizer,
1002static inline void gc_enter(
rb_objspace_t *
objspace,
enum gc_enter_event event,
unsigned int *lock_lev);
1003static inline void gc_exit(
rb_objspace_t *
objspace,
enum gc_enter_event event,
unsigned int *lock_lev);
1018static int gc_mark_stacked_objects_incremental(
rb_objspace_t *,
size_t count);
1019NO_SANITIZE(
"memory",
static inline bool is_pointer_to_heap(
rb_objspace_t *
objspace,
const void *ptr));
1021static void gc_verify_internal_consistency(
void *objspace_ptr);
1023static double getrusage_time(
void);
1027static inline void gc_prof_mark_timer_start(
rb_objspace_t *);
1029static inline void gc_prof_sweep_timer_start(
rb_objspace_t *);
1030static inline void gc_prof_sweep_timer_stop(
rb_objspace_t *);
1034#define gc_prof_record(objspace) (objspace)->profile.current_record
1035#define gc_prof_enabled(objspace) ((objspace)->profile.run && (objspace)->profile.current_record)
1037#ifdef HAVE_VA_ARGS_MACRO
1038# define gc_report(level, objspace, ...) \
1039 if (!RGENGC_DEBUG_ENABLED(level)) {} else gc_report_body(level, objspace, __VA_ARGS__)
1041# define gc_report if (!RGENGC_DEBUG_ENABLED(0)) {} else gc_report_body
1043PRINTF_ARGS(
static void gc_report_body(
int level,
rb_objspace_t *
objspace,
const char *fmt, ...), 3, 4);
1045static void gc_finalize_deferred(
void *dmy);
1056#if defined(__GNUC__) && defined(__i386__)
1057typedef unsigned long long tick_t;
1058#define PRItick "llu"
1062 unsigned long long int x;
1063 __asm__ __volatile__ (
"rdtsc" :
"=A" (x));
1067#elif defined(__GNUC__) && defined(__x86_64__)
1068typedef unsigned long long tick_t;
1069#define PRItick "llu"
1071static __inline__ tick_t
1074 unsigned long hi, lo;
1075 __asm__ __volatile__ (
"rdtsc" :
"=a"(lo),
"=d"(hi));
1076 return ((
unsigned long long)lo)|( ((
unsigned long long)hi)<<32);
1079#elif defined(__powerpc64__) && (GCC_VERSION_SINCE(4,8,0) || defined(__clang__))
1080typedef unsigned long long tick_t;
1081#define PRItick "llu"
1083static __inline__ tick_t
1086 unsigned long long val = __builtin_ppc_get_timebase();
1093#elif defined(__POWERPC__) && defined(__APPLE__)
1094typedef unsigned long long tick_t;
1095#define PRItick "llu"
1097static __inline__ tick_t
1100 unsigned long int upper, lower, tmp;
1101 # define mftbu(r) __asm__ volatile("mftbu %0" : "=r"(r))
1102 # define mftb(r) __asm__ volatile("mftb %0" : "=r"(r))
1107 }
while (tmp != upper);
1108 return ((tick_t)upper << 32) | lower;
1111#elif defined(__aarch64__) && defined(__GNUC__)
1112typedef unsigned long tick_t;
1115static __inline__ tick_t
1119 __asm__ __volatile__ (
"mrs %0, cntvct_el0" :
"=r" (val));
1124#elif defined(_WIN32) && defined(_MSC_VER)
1126typedef unsigned __int64 tick_t;
1127#define PRItick "llu"
1136typedef clock_t tick_t;
1137#define PRItick "llu"
1146#define MEASURE_LINE(expr) expr
1151#define RVALUE_MARKED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj))
1152#define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj))
1153#define RVALUE_MARKING_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj))
1154#define RVALUE_UNCOLLECTIBLE_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj))
1155#define RVALUE_PINNED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), (obj))
1160 check_rvalue_consistency(
objspace, obj);
1161 return RVALUE_MARKED_BITMAP(obj) != 0;
1167 check_rvalue_consistency(
objspace, obj);
1168 return RVALUE_PINNED_BITMAP(obj) != 0;
1174 check_rvalue_consistency(
objspace, obj);
1175 return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1181 check_rvalue_consistency(
objspace, obj);
1182 return RVALUE_MARKING_BITMAP(obj) != 0;
1188 check_rvalue_consistency(
objspace, obj);
1189 return MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
1195 check_rvalue_consistency(
objspace, obj);
1196 return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1199#define RVALUE_PAGE_WB_UNPROTECTED(page, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj))
1200#define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj))
1201#define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj))
1212 int lev = rb_gc_vm_lock_no_barrier();
1215 fprintf(stderr,
"check_rvalue_consistency: %p is a special const.\n", (
void *)obj);
1218 else if (!is_pointer_to_heap(
objspace, (
void *)obj)) {
1220 while (empty_page) {
1221 if ((uintptr_t)empty_page->body <= (uintptr_t)obj &&
1222 (uintptr_t)obj < (uintptr_t)empty_page->body + HEAP_PAGE_SIZE) {
1223 GC_ASSERT(heap_page_in_global_empty_pages_pool(
objspace, empty_page));
1224 fprintf(stderr,
"check_rvalue_consistency: %p is in an empty page (%p).\n",
1225 (
void *)obj, (
void *)empty_page);
1230 fprintf(stderr,
"check_rvalue_consistency: %p is not a Ruby object.\n", (
void *)obj);
1236 const int wb_unprotected_bit = RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1237 const int uncollectible_bit = RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1238 const int mark_bit = RVALUE_MARKED_BITMAP(obj) != 0;
1239 const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0;
1240 const int remembered_bit = MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
1241 const int age = RVALUE_AGE_GET((
VALUE)obj);
1243 if (heap_page_in_global_empty_pages_pool(
objspace, GET_HEAP_PAGE(obj))) {
1244 fprintf(stderr,
"check_rvalue_consistency: %s is in tomb page.\n", rb_obj_info(obj));
1248 fprintf(stderr,
"check_rvalue_consistency: %s is T_NONE.\n", rb_obj_info(obj));
1252 fprintf(stderr,
"check_rvalue_consistency: %s is T_ZOMBIE.\n", rb_obj_info(obj));
1257 rb_obj_memsize_of((
VALUE)obj);
1264 if (age > 0 && wb_unprotected_bit) {
1265 fprintf(stderr,
"check_rvalue_consistency: %s is not WB protected, but age is %d > 0.\n", rb_obj_info(obj), age);
1269 if (!is_marking(
objspace) && uncollectible_bit && !mark_bit) {
1270 fprintf(stderr,
"check_rvalue_consistency: %s is uncollectible, but is not marked while !gc.\n", rb_obj_info(obj));
1275 if (uncollectible_bit && age != RVALUE_OLD_AGE && !wb_unprotected_bit) {
1276 fprintf(stderr,
"check_rvalue_consistency: %s is uncollectible, but not old (age: %d) and not WB unprotected.\n",
1277 rb_obj_info(obj), age);
1280 if (remembered_bit && age != RVALUE_OLD_AGE) {
1281 fprintf(stderr,
"check_rvalue_consistency: %s is remembered, but not old (age: %d).\n",
1282 rb_obj_info(obj), age);
1294 if (is_incremental_marking(
objspace) && marking_bit) {
1295 if (!is_marking(
objspace) && !mark_bit) {
1296 fprintf(stderr,
"check_rvalue_consistency: %s is marking, but not marked.\n", rb_obj_info(obj));
1302 rb_gc_vm_unlock_no_barrier(lev);
1304 if (err > 0 && terminate) {
1305 rb_bug(
"check_rvalue_consistency_force: there is %d errors.", err);
1310#if RGENGC_CHECK_MODE == 0
1320 check_rvalue_consistency_force(
objspace, obj, TRUE);
1333 asan_unpoisoning_object(obj) {
1344 check_rvalue_consistency(
objspace, obj);
1353 MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
1356#if RGENGC_PROFILE >= 2
1357 objspace->profile.total_promoted_count++;
1365 RB_DEBUG_COUNTER_INC(obj_promote);
1366 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(
objspace, GET_HEAP_PAGE(obj), obj);
1373 int age = RVALUE_AGE_GET((
VALUE)obj);
1375 if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) {
1376 rb_bug(
"RVALUE_AGE_INC: can not increment age of OLD object %s.", rb_obj_info(obj));
1380 RVALUE_AGE_SET(obj, age);
1382 if (age == RVALUE_OLD_AGE) {
1383 RVALUE_OLD_UNCOLLECTIBLE_SET(
objspace, obj);
1386 check_rvalue_consistency(
objspace, obj);
1392 check_rvalue_consistency(
objspace, obj);
1393 GC_ASSERT(!RVALUE_OLD_P(
objspace, obj));
1394 RVALUE_AGE_SET(obj, RVALUE_OLD_AGE - 1);
1395 check_rvalue_consistency(
objspace, obj);
1399RVALUE_AGE_RESET(
VALUE obj)
1401 RVALUE_AGE_SET(obj, 0);
1407 check_rvalue_consistency(
objspace, obj);
1408 GC_ASSERT(RVALUE_OLD_P(
objspace, obj));
1410 if (!is_incremental_marking(
objspace) && RVALUE_REMEMBERED(
objspace, obj)) {
1411 CLEAR_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj);
1414 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
1415 RVALUE_AGE_RESET(obj);
1417 if (RVALUE_MARKED(
objspace, obj)) {
1421 check_rvalue_consistency(
objspace, obj);
1433 return !RVALUE_MARKED(
objspace, obj);
1437rb_gc_impl_gc_enabled_p(
void *objspace_ptr)
1440 return !dont_gc_val();
1444rb_gc_impl_gc_enable(
void *objspace_ptr)
1452rb_gc_impl_gc_disable(
void *objspace_ptr,
bool finish_current_gc)
1456 if (finish_current_gc) {
1470 return calloc(1, n);
1474rb_gc_impl_set_event_hook(
void *objspace_ptr,
const rb_event_flag_t event)
1482rb_gc_impl_get_total_time(
void *objspace_ptr)
1486 unsigned long long marking_time =
objspace->profile.marking_time_ns;
1487 unsigned long long sweeping_time =
objspace->profile.sweeping_time_ns;
1489 return marking_time + sweeping_time;
1493rb_gc_impl_set_measure_total_time(
void *objspace_ptr,
VALUE flag)
1501rb_gc_impl_get_measure_total_time(
void *objspace_ptr)
1511 size_t heap_idx = heap - heaps;
1512 return gc_params.heap_init_slots[heap_idx];
1516object_id_cmp(st_data_t x, st_data_t y)
1519 return !rb_big_eql(x, y);
1527object_id_hash(st_data_t n)
1532#define OBJ_ID_INCREMENT (RUBY_IMMEDIATE_MASK + 1)
1533#define OBJ_ID_INITIAL (OBJ_ID_INCREMENT)
1535static const struct st_hash_type object_id_hash_type = {
1542rb_gc_impl_garbage_object_p(
void *objspace_ptr,
VALUE ptr)
1548 asan_unpoisoning_object(ptr) {
1560 if (dead)
return true;
1561 return is_lazy_sweeping(
objspace) && GET_HEAP_PAGE(ptr)->flags.before_sweep &&
1566rb_gc_impl_object_id(
void *objspace_ptr,
VALUE obj)
1571 unsigned int lev = rb_gc_vm_lock();
1574 if (st_lookup(
objspace->obj_to_id_tbl, (st_data_t)obj, &val)) {
1578 rb_bug(
"rb_gc_impl_object_id: FL_SEEN_OBJ_ID flag set but not found in table");
1582 GC_ASSERT(!st_lookup(
objspace->obj_to_id_tbl, (st_data_t)obj, NULL));
1585 objspace->next_object_id += OBJ_ID_INCREMENT;
1587 st_insert(
objspace->obj_to_id_tbl, (st_data_t)obj, (st_data_t)
id);
1588 if (RB_UNLIKELY(
objspace->id_to_obj_tbl)) {
1589 st_insert(
objspace->id_to_obj_tbl, (st_data_t)
id, (st_data_t)obj);
1593 rb_gc_vm_unlock(lev);
1599build_id_to_obj_i(st_data_t key, st_data_t value, st_data_t data)
1602 st_insert(id_to_obj_tbl, value, key);
1607rb_gc_impl_object_id_to_ref(
void *objspace_ptr,
VALUE object_id)
1611 unsigned int lev = rb_gc_vm_lock();
1614 objspace->id_to_obj_tbl = st_init_table_with_size(&object_id_hash_type, st_table_size(
objspace->obj_to_id_tbl));
1615 st_foreach(
objspace->obj_to_id_tbl, build_id_to_obj_i, (st_data_t)
objspace->id_to_obj_tbl);
1619 bool found = st_lookup(
objspace->id_to_obj_tbl, object_id, &obj) && !rb_gc_impl_garbage_object_p(
objspace, obj);
1621 rb_gc_vm_unlock(lev);
1642 rb_asan_unpoison_object(obj,
false);
1644 asan_unlock_freelist(page);
1648 slot->next = page->freelist;
1649 page->freelist = slot;
1650 asan_lock_freelist(page);
1652 RVALUE_AGE_RESET(obj);
1654 if (RGENGC_CHECK_MODE &&
1656 !(page->start <= (uintptr_t)obj &&
1657 (uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) &&
1658 obj % BASE_SLOT_SIZE == 0)) {
1659 rb_bug(
"heap_page_add_freeobj: %p is not rvalue.", (
void *)obj);
1662 rb_asan_poison_object(obj);
1663 gc_report(3,
objspace,
"heap_page_add_freeobj: add %p to freelist\n", (
void *)obj);
1668 rb_heap_t *heap,
size_t free_slots,
size_t total_slots)
1670 double goal_ratio = gc_params.heap_free_slots_goal_ratio;
1671 size_t target_total_slots;
1673 if (goal_ratio == 0.0) {
1674 target_total_slots = (size_t)(total_slots * gc_params.growth_factor);
1676 else if (total_slots == 0) {
1677 target_total_slots = minimum_slots_for_heap(
objspace, heap);
1683 double f = (double)(total_slots - free_slots) / ((1 - goal_ratio) * total_slots);
1685 if (f > gc_params.growth_factor) f = gc_params.growth_factor;
1686 if (f < 1.0) f = 1.1;
1688 target_total_slots = (size_t)(f * total_slots);
1692 "free_slots(%8"PRIuSIZE
")/total_slots(%8"PRIuSIZE
")=%1.2f,"
1693 " G(%1.2f), f(%1.2f),"
1694 " total_slots(%8"PRIuSIZE
") => target_total_slots(%8"PRIuSIZE
")\n",
1695 free_slots, total_slots, free_slots/(
double)total_slots,
1696 goal_ratio, f, total_slots, target_total_slots);
1700 if (gc_params.growth_max_slots > 0) {
1701 size_t max_total_slots = (size_t)(total_slots + gc_params.growth_max_slots);
1702 if (target_total_slots > max_total_slots) target_total_slots = max_total_slots;
1705 size_t extend_slot_count = target_total_slots - total_slots;
1707 if (extend_slot_count == 0) extend_slot_count = 1;
1709 objspace->heap_pages.allocatable_slots += extend_slot_count;
1715 asan_unlock_freelist(page);
1716 GC_ASSERT(page->free_slots != 0);
1717 GC_ASSERT(page->freelist != NULL);
1719 page->free_next = heap->free_pages;
1720 heap->free_pages = page;
1722 RUBY_DEBUG_LOG(
"page:%p freelist:%p", (
void *)page, (
void *)page->freelist);
1724 asan_lock_freelist(page);
1730 asan_unlock_freelist(page);
1731 GC_ASSERT(page->free_slots != 0);
1732 GC_ASSERT(page->freelist != NULL);
1734 page->free_next = heap->pooled_pages;
1735 heap->pooled_pages = page;
1736 objspace->rincgc.pooled_slots += page->free_slots;
1738 asan_lock_freelist(page);
1744 ccan_list_del(&page->page_node);
1745 heap->total_pages--;
1746 heap->total_slots -= page->total_slots;
1750gc_aligned_free(
void *ptr,
size_t size)
1752#if defined __MINGW32__
1753 __mingw_aligned_free(ptr);
1756#elif defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN)
1759 free(((
void**)ptr)[-1]);
1766 GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
1768 if (HEAP_PAGE_ALLOC_USE_MMAP) {
1770 GC_ASSERT(HEAP_PAGE_SIZE % sysconf(_SC_PAGE_SIZE) == 0);
1771 if (munmap(page_body, HEAP_PAGE_SIZE)) {
1772 rb_bug(
"heap_page_body_free: munmap failed");
1777 gc_aligned_free(page_body, HEAP_PAGE_SIZE);
1784 objspace->heap_pages.freed_pages++;
1785 heap_page_body_free(page->body);
1792 if (
objspace->empty_pages != NULL && heap_pages_freeable_pages > 0) {
1793 GC_ASSERT(
objspace->empty_pages_count > 0);
1798 for (i = j = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
1801 if (heap_page_in_global_empty_pages_pool(
objspace, page) && heap_pages_freeable_pages > 0) {
1803 heap_pages_freeable_pages--;
1806 if (heap_page_in_global_empty_pages_pool(
objspace, page)) {
1807 page->free_next =
objspace->empty_pages;
1813 rb_darray_set(
objspace->heap_pages.sorted, j, page);
1819 rb_darray_pop(
objspace->heap_pages.sorted, i - j);
1820 GC_ASSERT(rb_darray_size(
objspace->heap_pages.sorted) == j);
1823 uintptr_t himem = (uintptr_t)hipage->body + HEAP_PAGE_SIZE;
1824 GC_ASSERT(himem <= heap_pages_himem);
1825 heap_pages_himem = himem;
1828 uintptr_t lomem = (uintptr_t)lopage->body +
sizeof(
struct heap_page_header);
1829 GC_ASSERT(lomem >= heap_pages_lomem);
1830 heap_pages_lomem = lomem;
1835gc_aligned_malloc(
size_t alignment,
size_t size)
1838 GC_ASSERT(((alignment - 1) & alignment) == 0);
1839 GC_ASSERT(alignment %
sizeof(
void*) == 0);
1843#if defined __MINGW32__
1844 res = __mingw_aligned_malloc(size, alignment);
1846 void *_aligned_malloc(
size_t,
size_t);
1847 res = _aligned_malloc(size, alignment);
1848#elif defined(HAVE_POSIX_MEMALIGN)
1849 if (posix_memalign(&res, alignment, size) != 0) {
1852#elif defined(HAVE_MEMALIGN)
1853 res = memalign(alignment, size);
1856 res = malloc(alignment + size +
sizeof(
void*));
1857 aligned = (
char*)res + alignment +
sizeof(
void*);
1858 aligned -= ((
VALUE)aligned & (alignment - 1));
1859 ((
void**)aligned)[-1] = res;
1860 res = (
void*)aligned;
1863 GC_ASSERT((uintptr_t)res % alignment == 0);
1869heap_page_body_allocate(
void)
1873 if (HEAP_PAGE_ALLOC_USE_MMAP) {
1875 GC_ASSERT(HEAP_PAGE_ALIGN % sysconf(_SC_PAGE_SIZE) == 0);
1877 size_t mmap_size = HEAP_PAGE_ALIGN + HEAP_PAGE_SIZE;
1878 char *ptr = mmap(NULL, mmap_size,
1879 PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1880 if (ptr == MAP_FAILED) {
1889#if defined(HAVE_SYS_PRCTL_H) && defined(PR_SET_VMA) && defined(PR_SET_VMA_ANON_NAME)
1890 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, mmap_size,
"Ruby:GC:default:heap_page_body_allocate");
1894 char *aligned = ptr + HEAP_PAGE_ALIGN;
1895 aligned -= ((
VALUE)aligned & (HEAP_PAGE_ALIGN - 1));
1896 GC_ASSERT(aligned > ptr);
1897 GC_ASSERT(aligned <= ptr + HEAP_PAGE_ALIGN);
1899 size_t start_out_of_range_size = aligned - ptr;
1900 GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
1901 if (start_out_of_range_size > 0) {
1902 if (munmap(ptr, start_out_of_range_size)) {
1903 rb_bug(
"heap_page_body_allocate: munmap failed for start");
1907 size_t end_out_of_range_size = HEAP_PAGE_ALIGN - start_out_of_range_size;
1908 GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
1909 if (end_out_of_range_size > 0) {
1910 if (munmap(aligned + HEAP_PAGE_SIZE, end_out_of_range_size)) {
1911 rb_bug(
"heap_page_body_allocate: munmap failed for end");
1919 page_body = gc_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
1922 GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
1931 if (
objspace->empty_pages != NULL) {
1932 GC_ASSERT(
objspace->empty_pages_count > 0);
1935 objspace->empty_pages = page->free_next;
1945 if (page_body == 0) {
1951 heap_page_body_free(page_body);
1956 uintptr_t end = (uintptr_t)page_body + HEAP_PAGE_SIZE;
1959 size_t hi = rb_darray_size(
objspace->heap_pages.sorted);
1963 size_t mid = (lo + hi) / 2;
1964 mid_page = rb_darray_get(
objspace->heap_pages.sorted, mid);
1965 if ((uintptr_t)mid_page->start < start) {
1968 else if ((uintptr_t)mid_page->start > start) {
1972 rb_bug(
"same heap page is allocated: %p at %"PRIuVALUE, (
void *)page_body, (
VALUE)mid);
1976 rb_darray_insert_without_gc(&
objspace->heap_pages.sorted, hi, page);
1978 if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
1979 if (heap_pages_himem < end) heap_pages_himem = end;
1981 page->body = page_body;
1982 page_body->header.page = page;
1984 objspace->heap_pages.allocated_pages++;
1993 GC_ASSERT(!heap->sweeping_page);
1994 GC_ASSERT(heap_page_in_global_empty_pages_pool(
objspace, page));
1998 if (start % BASE_SLOT_SIZE != 0) {
1999 int delta = BASE_SLOT_SIZE - (start % BASE_SLOT_SIZE);
2000 start = start + delta;
2001 GC_ASSERT(NUM_IN_PAGE(start) == 0 || NUM_IN_PAGE(start) == 1);
2007 if (NUM_IN_PAGE(start) == 1) {
2008 start += heap->slot_size - BASE_SLOT_SIZE;
2011 GC_ASSERT(NUM_IN_PAGE(start) * BASE_SLOT_SIZE % heap->slot_size == 0);
2014 int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size);
2016 page->start = start;
2017 page->total_slots = slot_count;
2018 page->slot_size = heap->slot_size;
2021 asan_unlock_freelist(page);
2022 page->freelist = NULL;
2023 asan_unpoison_memory_region(page->body, HEAP_PAGE_SIZE,
false);
2024 for (
VALUE p = (
VALUE)start; p < start + (slot_count * heap->slot_size); p += heap->slot_size) {
2025 heap_page_add_freeobj(
objspace, page, p);
2027 asan_lock_freelist(page);
2029 page->free_slots = slot_count;
2031 heap->total_allocated_pages++;
2033 ccan_list_add_tail(&heap->pages, &page->page_node);
2034 heap->total_pages++;
2035 heap->total_slots += page->total_slots;
2041 gc_report(1,
objspace,
"heap_page_allocate_and_initialize: rb_darray_size(objspace->heap_pages.sorted): %"PRIdSIZE
", "
2042 "allocatable_slots: %"PRIdSIZE
", heap->total_pages: %"PRIdSIZE
"\n",
2043 rb_darray_size(
objspace->heap_pages.sorted),
objspace->heap_pages.allocatable_slots, heap->total_pages);
2045 bool allocated =
false;
2048 if (page == NULL &&
objspace->heap_pages.allocatable_slots > 0) {
2049 page = heap_page_allocate(
objspace);
2054 heap_add_page(
objspace, heap, page);
2055 heap_add_freepage(heap, page);
2058 if (
objspace->heap_pages.allocatable_slots > (
size_t)page->total_slots) {
2059 objspace->heap_pages.allocatable_slots -= page->total_slots;
2062 objspace->heap_pages.allocatable_slots = 0;
2067 return page != NULL;
2073 size_t prev_allocatable_slots =
objspace->heap_pages.allocatable_slots;
2075 objspace->heap_pages.allocatable_slots = 1;
2076 heap_page_allocate_and_initialize(
objspace, heap);
2077 GC_ASSERT(heap->free_pages != NULL);
2078 objspace->heap_pages.allocatable_slots = prev_allocatable_slots;
2084 unsigned int lock_lev;
2085 gc_enter(
objspace, gc_enter_event_continue, &lock_lev);
2088 if (is_incremental_marking(
objspace)) {
2089 if (gc_marks_continue(
objspace, heap)) {
2096 if (heap->free_pages == NULL && is_lazy_sweeping(
objspace)) {
2100 gc_exit(
objspace, gc_enter_event_continue, &lock_lev);
2106 GC_ASSERT(heap->free_pages == NULL);
2108 if (heap->total_slots < gc_params.heap_init_slots[heap - heaps] &&
2109 heap->sweeping_page == NULL) {
2110 heap_page_allocate_and_initialize_force(
objspace, heap);
2111 GC_ASSERT(heap->free_pages != NULL);
2118 if (heap->free_pages == NULL) {
2119 heap_page_allocate_and_initialize(
objspace, heap);
2124 if (heap->free_pages == NULL) {
2125 if (gc_start(
objspace, GPR_FLAG_NEWOBJ) == FALSE) {
2129 if (
objspace->heap_pages.allocatable_slots == 0 && !gc_config_full_mark_val) {
2130 heap_allocatable_slots_expand(
objspace, heap,
2131 heap->freed_slots + heap->empty_slots,
2133 GC_ASSERT(
objspace->heap_pages.allocatable_slots > 0);
2141 if (heap->free_pages == NULL && !heap_page_allocate_and_initialize(
objspace, heap)) {
2142 if (gc_needs_major_flags == GPR_FLAG_NONE) {
2143 rb_bug(
"cannot create a new page after GC");
2146 if (gc_start(
objspace, GPR_FLAG_NEWOBJ) == FALSE) {
2153 if (heap->free_pages == NULL &&
2154 !heap_page_allocate_and_initialize(
objspace, heap)) {
2155 rb_bug(
"cannot create a new page after major GC");
2163 GC_ASSERT(heap->free_pages != NULL);
2177static inline const char*
2178rb_gc_impl_source_location_cstr(
int *ptr)
2197#if !__has_feature(memory_sanitizer)
2201 RBASIC(obj)->flags = flags;
2206 RVALUE_AGE_SET_CANDIDATE(
objspace, obj);
2209#if RACTOR_CHECK_MODE
2210 void rb_ractor_setup_belonging(
VALUE obj);
2211 rb_ractor_setup_belonging(obj);
2214#if RGENGC_CHECK_MODE
2215 newobj_fill(obj, 0, 0, 0);
2217 int lev = rb_gc_vm_lock_no_barrier();
2219 check_rvalue_consistency(
objspace, obj);
2221 GC_ASSERT(RVALUE_MARKED(
objspace, obj) == FALSE);
2222 GC_ASSERT(RVALUE_MARKING(
objspace, obj) == FALSE);
2223 GC_ASSERT(RVALUE_OLD_P(
objspace, obj) == FALSE);
2224 GC_ASSERT(RVALUE_WB_UNPROTECTED(
objspace, obj) == FALSE);
2226 if (RVALUE_REMEMBERED(
objspace, obj)) rb_bug(
"newobj: %s is remembered.", rb_obj_info(obj));
2228 rb_gc_vm_unlock_no_barrier(lev);
2231 if (RB_UNLIKELY(wb_protected == FALSE)) {
2232 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
2237 objspace->profile.total_generated_normal_object_count++;
2238#if RGENGC_PROFILE >= 2
2243 objspace->profile.total_generated_shady_object_count++;
2244#if RGENGC_PROFILE >= 2
2251 GET_RVALUE_OVERHEAD(obj)->file = rb_gc_impl_source_location_cstr(&GET_RVALUE_OVERHEAD(obj)->line);
2255 gc_report(5,
objspace,
"newobj: %s\n", rb_obj_info(obj));
2257 RUBY_DEBUG_LOG(
"obj:%p (%s)", (
void *)obj, rb_obj_info(obj));
2262rb_gc_impl_obj_slot_size(
VALUE obj)
2264 return GET_HEAP_PAGE(obj)->slot_size - RVALUE_OVERHEAD;
2268heap_slot_size(
unsigned char pool_id)
2270 GC_ASSERT(pool_id < HEAP_COUNT);
2272 size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE;
2274#if RGENGC_CHECK_MODE
2276 GC_ASSERT(heaps[pool_id].slot_size == (
short)slot_size);
2279 slot_size -= RVALUE_OVERHEAD;
2285rb_gc_impl_size_allocatable_p(
size_t size)
2287 return size <= heap_slot_size(HEAP_COUNT - 1);
2295 struct free_slot *p = heap_cache->freelist;
2297 if (RB_UNLIKELY(is_incremental_marking(
objspace))) {
2299 if (cache->incremental_mark_step_allocated_slots >= INCREMENTAL_MARK_STEP_ALLOCATIONS) {
2304 cache->incremental_mark_step_allocated_slots++;
2310 rb_asan_unpoison_object(obj,
true);
2311 heap_cache->freelist = p->next;
2312#if RGENGC_CHECK_MODE
2313 GC_ASSERT(rb_gc_impl_obj_slot_size(obj) == heap_slot_size(heap_idx));
2315 MEMZERO((
char *)obj,
char, heap_slot_size(heap_idx));
2329 if (heap->free_pages == NULL) {
2333 page = heap->free_pages;
2334 heap->free_pages = page->free_next;
2336 GC_ASSERT(page->free_slots != 0);
2338 asan_unlock_freelist(page);
2347 gc_report(3,
objspace,
"ractor_set_cache: Using page %p\n", (
void *)page->body);
2351 GC_ASSERT(heap_cache->freelist == NULL);
2352 GC_ASSERT(page->free_slots != 0);
2353 GC_ASSERT(page->freelist != NULL);
2355 heap_cache->using_page = page;
2356 heap_cache->freelist = page->freelist;
2357 page->free_slots = 0;
2358 page->freelist = NULL;
2360 rb_asan_unpoison_object((
VALUE)heap_cache->freelist,
false);
2362 rb_asan_poison_object((
VALUE)heap_cache->freelist);
2366heap_idx_for_size(
size_t size)
2368 size += RVALUE_OVERHEAD;
2370 size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE);
2373 size_t heap_idx = 64 - nlz_int64(slot_count - 1);
2375 if (heap_idx >= HEAP_COUNT) {
2376 rb_bug(
"heap_idx_for_size: allocation size too large "
2377 "(size=%"PRIuSIZE
"u, heap_idx=%"PRIuSIZE
"u)", size, heap_idx);
2380#if RGENGC_CHECK_MODE
2382 GC_ASSERT(size <= (
size_t)heaps[heap_idx].slot_size);
2383 if (heap_idx > 0) GC_ASSERT(size > (
size_t)heaps[heap_idx - 1].slot_size);
2390rb_gc_impl_heap_id_for_size(
void *objspace_ptr,
size_t size)
2392 return heap_idx_for_size(size);
2396static size_t heap_sizes[HEAP_COUNT + 1] = { 0 };
2399rb_gc_impl_heap_sizes(
void *objspace_ptr)
2401 if (heap_sizes[0] == 0) {
2402 for (
unsigned char i = 0; i < HEAP_COUNT; i++) {
2403 heap_sizes[i] = heap_slot_size(i);
2418 unsigned int lev = 0;
2419 bool unlock_vm =
false;
2422 lev = rb_gc_cr_lock();
2427 if (is_incremental_marking(
objspace)) {
2429 cache->incremental_mark_step_allocated_slots = 0;
2432 obj = ractor_cache_allocate_slot(
objspace, cache, heap_idx);
2438 ractor_cache_set_page(
objspace, cache, heap_idx, page);
2441 obj = ractor_cache_allocate_slot(
objspace, cache, heap_idx);
2446 rb_gc_cr_unlock(lev);
2449 if (RB_UNLIKELY(obj ==
Qfalse)) {
2458 VALUE obj = ractor_cache_allocate_slot(
objspace, cache, heap_idx);
2460 if (RB_UNLIKELY(obj ==
Qfalse)) {
2461 obj = newobj_cache_miss(
objspace, cache, heap_idx, vm_locked);
2465 heap->total_allocated_objects++;
2466 GC_ASSERT(rb_gc_multi_ractor_p() ||
2467 heap->total_slots >=
2468 (heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count));
2481 lev = rb_gc_cr_lock();
2483 if (RB_UNLIKELY(during_gc || ruby_gc_stressful)) {
2487 if (rb_memerror_reentered()) {
2490 rb_bug(
"object allocation during garbage collection phase");
2493 if (ruby_gc_stressful) {
2494 if (!garbage_collect(
objspace, GPR_FLAG_NEWOBJ)) {
2500 obj = newobj_alloc(
objspace, cache, heap_idx,
true);
2501 newobj_init(klass, flags, wb_protected,
objspace, obj);
2503 rb_gc_cr_unlock(lev);
2508NOINLINE(
static VALUE newobj_slowpath_wb_protected(
VALUE klass,
VALUE flags,
2510NOINLINE(
static VALUE newobj_slowpath_wb_unprotected(
VALUE klass,
VALUE flags,
2516 return newobj_slowpath(klass, flags,
objspace, cache, TRUE, heap_idx);
2522 return newobj_slowpath(klass, flags,
objspace, cache, FALSE, heap_idx);
2526rb_gc_impl_new_obj(
void *objspace_ptr,
void *cache_ptr,
VALUE klass,
VALUE flags,
VALUE v1,
VALUE v2,
VALUE v3,
bool wb_protected,
size_t alloc_size)
2531 RB_DEBUG_COUNTER_INC(obj_newobj);
2532 (void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected);
2534 if (RB_UNLIKELY(stress_to_class)) {
2535 if (
RTEST(rb_hash_has_key(stress_to_class, klass))) {
2540 size_t heap_idx = heap_idx_for_size(alloc_size);
2544 if (!RB_UNLIKELY(during_gc || ruby_gc_stressful) &&
2546 obj = newobj_alloc(
objspace, cache, heap_idx,
false);
2547 newobj_init(klass, flags, wb_protected,
objspace, obj);
2550 RB_DEBUG_COUNTER_INC(obj_newobj_slowpath);
2552 obj = wb_protected ?
2553 newobj_slowpath_wb_protected(klass, flags,
objspace, cache, heap_idx) :
2554 newobj_slowpath_wb_unprotected(klass, flags,
objspace, cache, heap_idx);
2557 return newobj_fill(obj, v1, v2, v3);
2561ptr_in_page_body_p(
const void *ptr,
const void *memb)
2564 uintptr_t p_body = (uintptr_t)page->body;
2566 if ((uintptr_t)ptr >= p_body) {
2567 return (uintptr_t)ptr < (p_body + HEAP_PAGE_SIZE) ? 0 : 1;
2580 if (ptr < (uintptr_t)heap_pages_lomem ||
2581 ptr > (uintptr_t)heap_pages_himem) {
2585 res = bsearch((
void *)ptr, rb_darray_ref(
objspace->heap_pages.sorted, 0),
2587 ptr_in_page_body_p);
2601 register uintptr_t p = (uintptr_t)ptr;
2604 RB_DEBUG_COUNTER_INC(gc_isptr_trial);
2606 if (p < heap_pages_lomem || p > heap_pages_himem)
return FALSE;
2607 RB_DEBUG_COUNTER_INC(gc_isptr_range);
2609 if (p % BASE_SLOT_SIZE != 0)
return FALSE;
2610 RB_DEBUG_COUNTER_INC(gc_isptr_align);
2612 page = heap_page_for_ptr(
objspace, (uintptr_t)ptr);
2614 RB_DEBUG_COUNTER_INC(gc_isptr_maybe);
2615 if (heap_page_in_global_empty_pages_pool(
objspace, page)) {
2619 if (p < page->start)
return FALSE;
2620 if (p >= page->start + (page->total_slots * page->slot_size))
return FALSE;
2621 if ((NUM_IN_PAGE(p) * BASE_SLOT_SIZE) % page->slot_size != 0)
return FALSE;
2630rb_gc_impl_pointer_to_heap_p(
void *objspace_ptr,
const void *ptr)
2632 return is_pointer_to_heap(objspace_ptr, ptr);
2635#define ZOMBIE_OBJ_KEPT_FLAGS (FL_FINALIZE)
2638rb_gc_impl_make_zombie(
void *objspace_ptr,
VALUE obj,
void (*dfree)(
void *),
void *data)
2642 struct RZombie *zombie = RZOMBIE(obj);
2644 zombie->dfree = dfree;
2645 zombie->data = data;
2646 VALUE prev, next = heap_pages_deferred_final;
2648 zombie->next = prev = next;
2650 }
while (next != prev);
2652 struct heap_page *page = GET_HEAP_PAGE(obj);
2653 page->final_slots++;
2654 page->heap->final_slots_count++;
2660 st_data_t o = (st_data_t)obj,
id;
2665 if (st_delete(
objspace->obj_to_id_tbl, &o, &
id)) {
2667 if (RB_UNLIKELY(
objspace->id_to_obj_tbl)) {
2668 st_delete(
objspace->id_to_obj_tbl, &
id, NULL);
2672 rb_bug(
"Object ID seen, but not in mapping table: %s", rb_obj_info(obj));
2676typedef int each_obj_callback(
void *,
void *,
size_t,
void *);
2677typedef int each_page_callback(
struct heap_page *,
void *);
2681 bool reenable_incremental;
2683 each_obj_callback *each_obj_callback;
2684 each_page_callback *each_page_callback;
2688 size_t pages_counts[HEAP_COUNT];
2692objspace_each_objects_ensure(
VALUE arg)
2698 if (data->reenable_incremental) {
2699 objspace->flags.dont_incremental = FALSE;
2702 for (
int i = 0; i < HEAP_COUNT; i++) {
2703 struct heap_page **pages = data->pages[i];
2711objspace_each_objects_try(
VALUE arg)
2717 for (
int i = 0; i < HEAP_COUNT; i++) {
2719 size_t size = heap->total_pages *
sizeof(
struct heap_page *);
2721 struct heap_page **pages = malloc(size);
2722 if (!pages) rb_memerror();
2730 size_t pages_count = 0;
2731 ccan_list_for_each(&heap->pages, page, page_node) {
2732 pages[pages_count] = page;
2735 data->pages[i] = pages;
2736 data->pages_counts[i] = pages_count;
2737 GC_ASSERT(pages_count == heap->total_pages);
2740 for (
int i = 0; i < HEAP_COUNT; i++) {
2742 size_t pages_count = data->pages_counts[i];
2743 struct heap_page **pages = data->pages[i];
2746 for (
size_t i = 0; i < pages_count; i++) {
2749 if (page == NULL)
break;
2753 if (pages[i] != page)
continue;
2755 uintptr_t pstart = (uintptr_t)page->start;
2756 uintptr_t pend = pstart + (page->total_slots * heap->slot_size);
2758 if (data->each_obj_callback &&
2759 (*data->each_obj_callback)((
void *)pstart, (
void *)pend, heap->slot_size, data->data)) {
2762 if (data->each_page_callback &&
2763 (*data->each_page_callback)(page, data->data)) {
2767 page = ccan_list_next(&heap->pages, page, page_node);
2779 bool reenable_incremental = FALSE;
2781 reenable_incremental = !
objspace->flags.dont_incremental;
2784 objspace->flags.dont_incremental = TRUE;
2795objspace_each_objects(
rb_objspace_t *
objspace, each_obj_callback *callback,
void *data,
bool protected)
2799 .each_obj_callback = callback,
2800 .each_page_callback = NULL,
2807rb_gc_impl_each_objects(
void *objspace_ptr, each_obj_callback *callback,
void *data)
2809 objspace_each_objects(objspace_ptr, callback, data, TRUE);
2812#if GC_CAN_COMPILE_COMPACTION
2814objspace_each_pages(
rb_objspace_t *
objspace, each_page_callback *callback,
void *data,
bool protected)
2818 .each_obj_callback = NULL,
2819 .each_page_callback = callback,
2827rb_gc_impl_define_finalizer(
void *objspace_ptr,
VALUE obj,
VALUE block)
2837 int lev = rb_gc_vm_lock();
2839 if (st_lookup(finalizer_table, obj, &data)) {
2840 table = (
VALUE)data;
2847 for (i = 0; i <
len; i++) {
2850 rb_gc_vm_unlock(lev);
2856 rb_ary_push(table, block);
2861 st_add_direct(finalizer_table, obj, table);
2864 rb_gc_vm_unlock(lev);
2870rb_gc_impl_undefine_finalizer(
void *objspace_ptr,
VALUE obj)
2876 st_data_t data = obj;
2877 st_delete(finalizer_table, &data, 0);
2882rb_gc_impl_copy_finalizer(
void *objspace_ptr,
VALUE dest,
VALUE obj)
2890 if (RB_LIKELY(st_lookup(finalizer_table, obj, &data))) {
2891 table = (
VALUE)data;
2892 st_insert(finalizer_table, dest, table);
2896 rb_bug(
"rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj));
2901get_final(
long i,
void *data)
2911 if (RZOMBIE(zombie)->dfree) {
2912 RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data);
2915 st_data_t key = (st_data_t)zombie;
2919 if (st_delete(finalizer_table, &key, &table)) {
2923 rb_bug(
"FL_FINALIZE flag is set, but finalizers are not found");
2927 GC_ASSERT(!st_lookup(finalizer_table, key, NULL));
2937 rb_asan_unpoison_object(zombie,
false);
2938 next_zombie = RZOMBIE(zombie)->next;
2939 page = GET_HEAP_PAGE(zombie);
2943 int lev = rb_gc_vm_lock();
2946 GC_ASSERT(page->heap->final_slots_count > 0);
2947 GC_ASSERT(page->final_slots > 0);
2949 page->heap->final_slots_count--;
2950 page->final_slots--;
2952 heap_page_add_freeobj(
objspace, page, zombie);
2953 page->heap->total_freed_objects++;
2955 rb_gc_vm_unlock(lev);
2957 zombie = next_zombie;
2973 rb_gc_set_pending_interrupt();
2974 finalize_deferred_heap_pages(
objspace);
2975 rb_gc_unset_pending_interrupt();
2979gc_finalize_deferred(
void *dmy)
2998gc_abort(
void *objspace_ptr)
3002 if (is_incremental_marking(
objspace)) {
3005 while (pop_mark_stack(&
objspace->mark_stack, &obj));
3007 objspace->flags.during_incremental_marking = FALSE;
3011 for (
int i = 0; i < HEAP_COUNT; i++) {
3014 heap->sweeping_page = NULL;
3017 ccan_list_for_each(&heap->pages, page, page_node) {
3018 page->flags.before_sweep =
false;
3023 for (
int i = 0; i < HEAP_COUNT; i++) {
3025 rgengc_mark_and_rememberset_clear(
objspace, heap);
3028 gc_mode_set(
objspace, gc_mode_none);
3032rb_gc_impl_shutdown_free_objects(
void *objspace_ptr)
3036 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
3038 short stride = page->slot_size;
3040 uintptr_t p = (uintptr_t)page->start;
3041 uintptr_t pend = p + page->total_slots * stride;
3042 for (; p < pend; p += stride) {
3044 asan_unpoisoning_object(vp) {
3046 rb_gc_obj_free_vm_weak_references(vp);
3047 if (rb_gc_obj_free(
objspace, vp)) {
3057rb_gc_impl_shutdown_call_finalizer_i(st_data_t key, st_data_t val, st_data_t _data)
3073rb_gc_impl_shutdown_call_finalizer(
void *objspace_ptr)
3077#if RGENGC_CHECK_MODE >= 2
3078 gc_verify_internal_consistency(
objspace);
3082 objspace->flags.dont_incremental = 1;
3091 while (finalizer_table->num_entries) {
3092 st_foreach(finalizer_table, rb_gc_impl_shutdown_call_finalizer_i, 0);
3097 GC_ASSERT(heap_pages_deferred_final == 0);
3106 unsigned int lock_lev;
3107 gc_enter(
objspace, gc_enter_event_finalizer, &lock_lev);
3110 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
3112 short stride = page->slot_size;
3114 uintptr_t p = (uintptr_t)page->start;
3115 uintptr_t pend = p + page->total_slots * stride;
3116 for (; p < pend; p += stride) {
3118 asan_unpoisoning_object(vp) {
3119 if (rb_gc_shutdown_call_finalizer_p(vp)) {
3120 rb_gc_obj_free_vm_weak_references(vp);
3121 if (rb_gc_obj_free(
objspace, vp)) {
3129 gc_exit(
objspace, gc_enter_event_finalizer, &lock_lev);
3131 finalize_deferred_heap_pages(
objspace);
3133 st_free_table(finalizer_table);
3134 finalizer_table = 0;
3139rb_gc_impl_each_object(
void *objspace_ptr,
void (*func)(
VALUE obj,
void *data),
void *data)
3143 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
3145 short stride = page->slot_size;
3147 uintptr_t p = (uintptr_t)page->start;
3148 uintptr_t pend = p + page->total_slots * stride;
3149 for (; p < pend; p += stride) {
3152 asan_unpoisoning_object(obj) {
3168 size_t total_slots = 0;
3169 for (
int i = 0; i < HEAP_COUNT; i++) {
3171 total_slots += heap->total_slots;
3189gc_setup_mark_bits(
struct heap_page *page)
3192 memcpy(&page->mark_bits[0], &page->uncollectible_bits[0], HEAP_PAGE_BITMAP_SIZE);
3199enum {HEAP_PAGE_LOCK = PAGE_NOACCESS, HEAP_PAGE_UNLOCK = PAGE_READWRITE};
3205 return VirtualProtect(body, HEAP_PAGE_SIZE, protect, &old_protect) != 0;
3207#elif defined(__wasi__)
3209enum {HEAP_PAGE_LOCK, HEAP_PAGE_UNLOCK};
3210#define protect_page_body(body, protect) 1
3212enum {HEAP_PAGE_LOCK = PROT_NONE, HEAP_PAGE_UNLOCK = PROT_READ | PROT_WRITE};
3213#define protect_page_body(body, protect) !mprotect((body), HEAP_PAGE_SIZE, (protect))
3219 if (!protect_page_body(body, HEAP_PAGE_LOCK)) {
3220 rb_bug(
"Couldn't protect page %p, errno: %s", (
void *)body, strerror(
errno));
3223 gc_report(5,
objspace,
"Protecting page in move %p\n", (
void *)body);
3230 if (!protect_page_body(body, HEAP_PAGE_UNLOCK)) {
3231 rb_bug(
"Couldn't unprotect page %p, errno: %s", (
void *)body, strerror(
errno));
3234 gc_report(5,
objspace,
"Unprotecting page in move %p\n", (
void *)body);
3241 GC_ASSERT(gc_is_moveable_obj(
objspace, src));
3243 struct heap_page *src_page = GET_HEAP_PAGE(src);
3251 GC_ASSERT(RVALUE_MARKED(
objspace, src));
3253 asan_unlock_freelist(free_page);
3255 asan_lock_freelist(free_page);
3257 rb_asan_unpoison_object(dest,
false);
3264 asan_unlock_freelist(free_page);
3265 free_page->freelist = ((
struct free_slot *)dest)->next;
3266 asan_lock_freelist(free_page);
3270 if (src_page->slot_size > free_page->slot_size) {
3273 else if (free_page->slot_size > src_page->slot_size) {
3277 objspace->rcompactor.total_moved++;
3279 gc_move(
objspace, src, dest, src_page->slot_size, free_page->slot_size);
3281 free_page->free_slots--;
3289 struct heap_page *cursor = heap->compact_cursor;
3292 unlock_page_body(
objspace, cursor->body);
3293 cursor = ccan_list_next(&heap->pages, cursor, page_node);
3298#if GC_CAN_COMPILE_COMPACTION
3302#if defined(__MINGW32__) || defined(_WIN32)
3303# define GC_COMPACTION_SUPPORTED 1
3307# define GC_COMPACTION_SUPPORTED (GC_CAN_COMPILE_COMPACTION && HEAP_PAGE_ALLOC_USE_MMAP)
3310#if GC_CAN_COMPILE_COMPACTION
3312read_barrier_handler(uintptr_t address)
3320 if (page_body == NULL) {
3321 rb_bug(
"read_barrier_handler: segmentation fault at %p", (
void *)address);
3324 int lev = rb_gc_vm_lock();
3326 unlock_page_body(
objspace, page_body);
3328 objspace->profile.read_barrier_faults++;
3330 invalidate_moved_page(
objspace, GET_HEAP_PAGE(address));
3332 rb_gc_vm_unlock(lev);
3336#if !GC_CAN_COMPILE_COMPACTION
3338uninstall_handlers(
void)
3344install_handlers(
void)
3348#elif defined(_WIN32)
3349static LPTOP_LEVEL_EXCEPTION_FILTER old_handler;
3350typedef void (*signal_handler)(int);
3351static signal_handler old_sigsegv_handler;
3354read_barrier_signal(EXCEPTION_POINTERS *info)
3357 if (info->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
3362 read_barrier_handler((uintptr_t)info->ExceptionRecord->ExceptionInformation[1]);
3363 return EXCEPTION_CONTINUE_EXECUTION;
3366 return EXCEPTION_CONTINUE_SEARCH;
3371uninstall_handlers(
void)
3373 signal(SIGSEGV, old_sigsegv_handler);
3374 SetUnhandledExceptionFilter(old_handler);
3378install_handlers(
void)
3381 old_sigsegv_handler = signal(SIGSEGV, NULL);
3384 old_handler = SetUnhandledExceptionFilter(read_barrier_signal);
3387static struct sigaction old_sigbus_handler;
3388static struct sigaction old_sigsegv_handler;
3390#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3391static exception_mask_t old_exception_masks[32];
3392static mach_port_t old_exception_ports[32];
3393static exception_behavior_t old_exception_behaviors[32];
3394static thread_state_flavor_t old_exception_flavors[32];
3395static mach_msg_type_number_t old_exception_count;
3398disable_mach_bad_access_exc(
void)
3400 old_exception_count =
sizeof(old_exception_masks) /
sizeof(old_exception_masks[0]);
3401 task_swap_exception_ports(
3402 mach_task_self(), EXC_MASK_BAD_ACCESS,
3403 MACH_PORT_NULL, EXCEPTION_DEFAULT, 0,
3404 old_exception_masks, &old_exception_count,
3405 old_exception_ports, old_exception_behaviors, old_exception_flavors
3410restore_mach_bad_access_exc(
void)
3412 for (mach_msg_type_number_t i = 0; i < old_exception_count; i++) {
3413 task_set_exception_ports(
3415 old_exception_masks[i], old_exception_ports[i],
3416 old_exception_behaviors[i], old_exception_flavors[i]
3423read_barrier_signal(
int sig, siginfo_t *info,
void *data)
3426 struct sigaction prev_sigbus, prev_sigsegv;
3427 sigaction(SIGBUS, &old_sigbus_handler, &prev_sigbus);
3428 sigaction(SIGSEGV, &old_sigsegv_handler, &prev_sigsegv);
3431 sigset_t set, prev_set;
3433 sigaddset(&set, SIGBUS);
3434 sigaddset(&set, SIGSEGV);
3435 sigprocmask(SIG_UNBLOCK, &set, &prev_set);
3436#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3437 disable_mach_bad_access_exc();
3440 read_barrier_handler((uintptr_t)info->si_addr);
3443#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3444 restore_mach_bad_access_exc();
3446 sigaction(SIGBUS, &prev_sigbus, NULL);
3447 sigaction(SIGSEGV, &prev_sigsegv, NULL);
3448 sigprocmask(SIG_SETMASK, &prev_set, NULL);
3452uninstall_handlers(
void)
3454#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3455 restore_mach_bad_access_exc();
3457 sigaction(SIGBUS, &old_sigbus_handler, NULL);
3458 sigaction(SIGSEGV, &old_sigsegv_handler, NULL);
3462install_handlers(
void)
3464 struct sigaction action;
3465 memset(&action, 0,
sizeof(
struct sigaction));
3466 sigemptyset(&action.sa_mask);
3467 action.sa_sigaction = read_barrier_signal;
3468 action.sa_flags = SA_SIGINFO | SA_ONSTACK;
3470 sigaction(SIGBUS, &action, &old_sigbus_handler);
3471 sigaction(SIGSEGV, &action, &old_sigsegv_handler);
3472#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3473 disable_mach_bad_access_exc();
3481 for (
int i = 0; i < HEAP_COUNT; i++) {
3483 gc_unprotect_pages(
objspace, heap);
3486 uninstall_handlers();
3491 for (
int i = 0; i < HEAP_COUNT; i++) {
3493 heap->compact_cursor = NULL;
3494 heap->free_pages = NULL;
3495 heap->compact_cursor_index = 0;
3500 record->moved_objects =
objspace->rcompactor.total_moved - record->moved_objects;
3502 objspace->flags.during_compacting = FALSE;
3515 struct heap_page *sweep_page = ctx->page;
3516 short slot_size = sweep_page->slot_size;
3517 short slot_bits = slot_size / BASE_SLOT_SIZE;
3518 GC_ASSERT(slot_bits > 0);
3522 GC_ASSERT(vp % BASE_SLOT_SIZE == 0);
3524 rb_asan_unpoison_object(vp,
false);
3528 gc_report(2,
objspace,
"page_sweep: free %p\n", (
void *)p);
3529#if RGENGC_CHECK_MODE
3531 if (RVALUE_OLD_P(
objspace, vp)) rb_bug(
"page_sweep: %p - old while minor GC.", (
void *)p);
3532 if (RVALUE_REMEMBERED(
objspace, vp)) rb_bug(
"page_sweep: %p - remembered.", (
void *)p);
3536 if (RVALUE_WB_UNPROTECTED(
objspace, vp)) CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(vp), vp);
3538#if RGENGC_CHECK_MODE
3539#define CHECK(x) if (x(objspace, vp) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", rb_obj_info(vp))
3540 CHECK(RVALUE_WB_UNPROTECTED);
3541 CHECK(RVALUE_MARKED);
3542 CHECK(RVALUE_MARKING);
3543 CHECK(RVALUE_UNCOLLECTIBLE);
3552 rb_gc_obj_free_vm_weak_references(vp);
3553 if (rb_gc_obj_free(
objspace, vp)) {
3556 (void)VALGRIND_MAKE_MEM_UNDEFINED((
void*)p, BASE_SLOT_SIZE);
3557 heap_page_add_freeobj(
objspace, sweep_page, vp);
3558 gc_report(3,
objspace,
"page_sweep: %s is added to freelist\n", rb_obj_info(vp));
3567 if (
objspace->flags.during_compacting) {
3573 rb_bug(
"T_MOVED shouldn't be seen until compaction is finished");
3575 gc_report(3,
objspace,
"page_sweep: %s is added to freelist\n", rb_obj_info(vp));
3577 heap_page_add_freeobj(
objspace, sweep_page, vp);
3588 bitset >>= slot_bits;
3595 struct heap_page *sweep_page = ctx->page;
3596 GC_ASSERT(sweep_page->heap == heap);
3599 bits_t *bits, bitset;
3601 gc_report(2,
objspace,
"page_sweep: start.\n");
3603#if RGENGC_CHECK_MODE
3604 if (!
objspace->flags.immediate_sweep) {
3605 GC_ASSERT(sweep_page->flags.before_sweep == TRUE);
3608 sweep_page->flags.before_sweep = FALSE;
3609 sweep_page->free_slots = 0;
3611 p = (uintptr_t)sweep_page->start;
3612 bits = sweep_page->mark_bits;
3614 int page_rvalue_count = sweep_page->total_slots * (sweep_page->slot_size / BASE_SLOT_SIZE);
3615 int out_of_range_bits = (NUM_IN_PAGE(p) + page_rvalue_count) % BITS_BITLENGTH;
3616 if (out_of_range_bits != 0) {
3617 bits[BITMAP_INDEX(p) + page_rvalue_count / BITS_BITLENGTH] |= ~(((bits_t)1 << out_of_range_bits) - 1);
3623 int bitmap_plane_count = CEILDIV(NUM_IN_PAGE(p) + page_rvalue_count, BITS_BITLENGTH);
3624 GC_ASSERT(bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT - 1 ||
3625 bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT);
3629 bitset >>= NUM_IN_PAGE(p);
3631 gc_sweep_plane(
objspace, heap, p, bitset, ctx);
3633 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
3635 for (
int i = 1; i < bitmap_plane_count; i++) {
3638 gc_sweep_plane(
objspace, heap, p, bitset, ctx);
3640 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
3643 if (!heap->compact_cursor) {
3644 gc_setup_mark_bits(sweep_page);
3647#if GC_PROFILE_MORE_DETAIL
3650 record->removing_objects += ctx->final_slots + ctx->freed_slots;
3651 record->empty_objects += ctx->empty_slots;
3654 if (0) fprintf(stderr,
"gc_sweep_page(%"PRIdSIZE
"): total_slots: %d, freed_slots: %d, empty_slots: %d, final_slots: %d\n",
3656 sweep_page->total_slots,
3657 ctx->freed_slots, ctx->empty_slots, ctx->final_slots);
3659 sweep_page->free_slots += ctx->freed_slots + ctx->empty_slots;
3660 sweep_page->heap->total_freed_objects += ctx->freed_slots;
3662 if (heap_pages_deferred_final && !finalizing) {
3663 gc_finalize_deferred_register(
objspace);
3666#if RGENGC_CHECK_MODE
3667 short freelist_len = 0;
3668 asan_unlock_freelist(sweep_page);
3669 struct free_slot *ptr = sweep_page->freelist;
3672 rb_asan_unpoison_object((
VALUE)ptr,
false);
3674 rb_asan_poison_object((
VALUE)ptr);
3677 asan_lock_freelist(sweep_page);
3678 if (freelist_len != sweep_page->free_slots) {
3679 rb_bug(
"inconsistent freelist length: expected %d but was %d", sweep_page->free_slots, freelist_len);
3683 gc_report(2,
objspace,
"page_sweep: end.\n");
3687gc_mode_name(
enum gc_mode mode)
3690 case gc_mode_none:
return "none";
3691 case gc_mode_marking:
return "marking";
3692 case gc_mode_sweeping:
return "sweeping";
3693 case gc_mode_compacting:
return "compacting";
3694 default: rb_bug(
"gc_mode_name: unknown mode: %d", (
int)mode);
3701#if RGENGC_CHECK_MODE
3702 enum gc_mode prev_mode = gc_mode(
objspace);
3703 switch (prev_mode) {
3704 case gc_mode_none: GC_ASSERT(mode == gc_mode_marking);
break;
3705 case gc_mode_marking: GC_ASSERT(mode == gc_mode_sweeping);
break;
3706 case gc_mode_sweeping: GC_ASSERT(mode == gc_mode_none || mode == gc_mode_compacting);
break;
3707 case gc_mode_compacting: GC_ASSERT(mode == gc_mode_none);
break;
3710 if (0) fprintf(stderr,
"gc_mode_transition: %s->%s\n", gc_mode_name(gc_mode(
objspace)), gc_mode_name(mode));
3718 asan_unlock_freelist(page);
3719 if (page->freelist) {
3721 rb_asan_unpoison_object((
VALUE)p,
false);
3725 rb_asan_poison_object((
VALUE)prev);
3726 rb_asan_unpoison_object((
VALUE)p,
false);
3729 rb_asan_poison_object((
VALUE)p);
3732 page->freelist = freelist;
3734 asan_lock_freelist(page);
3741 heap->sweeping_page = ccan_list_top(&heap->pages,
struct heap_page, page_node);
3742 heap->free_pages = NULL;
3743 heap->pooled_pages = NULL;
3744 if (!
objspace->flags.immediate_sweep) {
3747 ccan_list_for_each(&heap->pages, page, page_node) {
3748 page->flags.before_sweep = TRUE;
3753#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 4
3757#if GC_CAN_COMPILE_COMPACTION
3758static void gc_sort_heap_by_compare_func(
rb_objspace_t *
objspace, gc_compact_compare_func compare_func);
3759static int compare_pinned_slots(
const void *left,
const void *right,
void *d);
3763gc_ractor_newobj_cache_clear(
void *c,
void *data)
3767 newobj_cache->incremental_mark_step_allocated_slots = 0;
3769 for (
size_t heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) {
3772 struct heap_page *page = cache->using_page;
3773 struct free_slot *freelist = cache->freelist;
3774 RUBY_DEBUG_LOG(
"ractor using_page:%p freelist:%p", (
void *)page, (
void *)freelist);
3776 heap_page_freelist_append(page, freelist);
3778 cache->using_page = NULL;
3779 cache->freelist = NULL;
3786 gc_mode_transition(
objspace, gc_mode_sweeping);
3789#if GC_CAN_COMPILE_COMPACTION
3790 if (
objspace->flags.during_compacting) {
3791 gc_sort_heap_by_compare_func(
3793 objspace->rcompactor.compare_func ?
objspace->rcompactor.compare_func : compare_pinned_slots
3798 for (
int i = 0; i < HEAP_COUNT; i++) {
3800 gc_sweep_start_heap(
objspace, heap);
3803 if (heap->sweeping_page == NULL) {
3804 GC_ASSERT(heap->total_pages == 0);
3805 GC_ASSERT(heap->total_slots == 0);
3806 gc_sweep_finish_heap(
objspace, heap);
3810 rb_gc_ractor_newobj_cache_foreach(gc_ractor_newobj_cache_clear, NULL);
3816 size_t total_slots = heap->total_slots;
3817 size_t swept_slots = heap->freed_slots + heap->empty_slots;
3819 size_t init_slots = gc_params.heap_init_slots[heap - heaps];
3820 size_t min_free_slots = (size_t)(MAX(total_slots, init_slots) * gc_params.heap_free_slots_min_ratio);
3822 if (swept_slots < min_free_slots &&
3824 ((heap->empty_slots == 0 && total_slots > 0) || heap->freed_slots > heap->empty_slots)) {
3830 while (swept_slots < min_free_slots &&
3831 (resurrected_page = heap_page_resurrect(
objspace))) {
3832 heap_add_page(
objspace, heap, resurrected_page);
3833 heap_add_freepage(heap, resurrected_page);
3835 swept_slots += resurrected_page->free_slots;
3838 if (swept_slots < min_free_slots) {
3842 objspace->profile.count -
objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
3843 if (
objspace->heap_pages.allocatable_slots < min_free_slots) {
3844 heap_allocatable_slots_expand(
objspace, heap, swept_slots, heap->total_slots);
3848 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
3849 heap->force_major_gc_count++;
3858 gc_report(1,
objspace,
"gc_sweep_finish\n");
3861 heap_pages_free_unused_pages(
objspace);
3863 for (
int i = 0; i < HEAP_COUNT; i++) {
3866 heap->freed_slots = 0;
3867 heap->empty_slots = 0;
3869 if (!will_be_incremental_marking(
objspace)) {
3870 struct heap_page *end_page = heap->free_pages;
3872 while (end_page->free_next) end_page = end_page->free_next;
3873 end_page->free_next = heap->pooled_pages;
3876 heap->free_pages = heap->pooled_pages;
3878 heap->pooled_pages = NULL;
3884 gc_mode_transition(
objspace, gc_mode_none);
3886#if RGENGC_CHECK_MODE >= 2
3887 gc_verify_internal_consistency(
objspace);
3894 struct heap_page *sweep_page = heap->sweeping_page;
3895 int swept_slots = 0;
3896 int pooled_slots = 0;
3898 if (sweep_page == NULL)
return FALSE;
3900#if GC_ENABLE_LAZY_SWEEP
3901 gc_prof_sweep_timer_start(
objspace);
3905 RUBY_DEBUG_LOG(
"sweep_page:%p", (
void *)sweep_page);
3913 gc_sweep_page(
objspace, heap, &ctx);
3914 int free_slots = ctx.freed_slots + ctx.empty_slots;
3916 heap->sweeping_page = ccan_list_next(&heap->pages, sweep_page, page_node);
3918 if (free_slots == sweep_page->total_slots) {
3920 heap_unlink_page(
objspace, heap, sweep_page);
3922 sweep_page->start = 0;
3923 sweep_page->total_slots = 0;
3924 sweep_page->slot_size = 0;
3925 sweep_page->heap = NULL;
3926 sweep_page->free_slots = 0;
3928 asan_unlock_freelist(sweep_page);
3929 sweep_page->freelist = NULL;
3930 asan_lock_freelist(sweep_page);
3932 asan_poison_memory_region(sweep_page->body, HEAP_PAGE_SIZE);
3935 sweep_page->free_next =
objspace->empty_pages;
3936 objspace->empty_pages = sweep_page;
3938 else if (free_slots > 0) {
3939 heap->freed_slots += ctx.freed_slots;
3940 heap->empty_slots += ctx.empty_slots;
3942 if (pooled_slots < GC_INCREMENTAL_SWEEP_POOL_SLOT_COUNT) {
3943 heap_add_poolpage(
objspace, heap, sweep_page);
3944 pooled_slots += free_slots;
3947 heap_add_freepage(heap, sweep_page);
3948 swept_slots += free_slots;
3949 if (swept_slots > GC_INCREMENTAL_SWEEP_SLOT_COUNT) {
3955 sweep_page->free_next = NULL;
3957 }
while ((sweep_page = heap->sweeping_page));
3959 if (!heap->sweeping_page) {
3960 gc_sweep_finish_heap(
objspace, heap);
3962 if (!has_sweeping_pages(
objspace)) {
3967#if GC_ENABLE_LAZY_SWEEP
3968 gc_prof_sweep_timer_stop(
objspace);
3971 return heap->free_pages != NULL;
3977 for (
int i = 0; i < HEAP_COUNT; i++) {
3980 while (heap->sweeping_page) {
3989 GC_ASSERT(dont_gc_val() == FALSE ||
objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
3990 if (!GC_ENABLE_LAZY_SWEEP)
return;
3994 for (
int i = 0; i < HEAP_COUNT; i++) {
3996 if (!gc_sweep_step(
objspace, heap)) {
3997 if (heap == sweep_heap &&
objspace->empty_pages_count == 0 &&
objspace->heap_pages.allocatable_slots == 0) {
4009rb_gc_impl_location(
void *objspace_ptr,
VALUE value)
4013 asan_unpoisoning_object(value) {
4015 destination = (
VALUE)RMOVED(value)->destination;
4019 destination = value;
4026#if GC_CAN_COMPILE_COMPACTION
4037 GC_ASSERT(RVALUE_PINNED(
objspace, forwarding_object));
4038 GC_ASSERT(!RVALUE_MARKED(
objspace, forwarding_object));
4040 CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object);
4042 object = rb_gc_impl_location(
objspace, forwarding_object);
4044 uint32_t original_shape_id = 0;
4046 original_shape_id = RMOVED(forwarding_object)->original_shape_id;
4049 gc_move(
objspace,
object, forwarding_object, GET_HEAP_PAGE(
object)->slot_size, page->slot_size);
4053 if (original_shape_id) {
4054 rb_gc_set_shape(forwarding_object, original_shape_id);
4057 struct heap_page *orig_page = GET_HEAP_PAGE(
object);
4058 orig_page->free_slots++;
4059 heap_page_add_freeobj(
objspace, orig_page,
object);
4061 GC_ASSERT(RVALUE_MARKED(
objspace, forwarding_object));
4066 p += BASE_SLOT_SIZE;
4076 bits_t *mark_bits, *pin_bits;
4079 mark_bits = page->mark_bits;
4080 pin_bits = page->pinned_bits;
4082 uintptr_t p = page->start;
4085 bitset = pin_bits[0] & ~mark_bits[0];
4086 bitset >>= NUM_IN_PAGE(p);
4087 invalidate_moved_plane(
objspace, page, p, bitset);
4088 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
4090 for (i=1; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
4093 bitset = pin_bits[i] & ~mark_bits[i];
4095 invalidate_moved_plane(
objspace, page, p, bitset);
4096 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
4105 gc_mode_transition(
objspace, gc_mode_compacting);
4107 for (
int i = 0; i < HEAP_COUNT; i++) {
4109 ccan_list_for_each(&heap->pages, page, page_node) {
4110 page->flags.before_sweep = TRUE;
4113 heap->compact_cursor = ccan_list_tail(&heap->pages,
struct heap_page, page_node);
4114 heap->compact_cursor_index = 0;
4119 record->moved_objects =
objspace->rcompactor.total_moved;
4122 memset(
objspace->rcompactor.considered_count_table, 0,
T_MASK *
sizeof(
size_t));
4123 memset(
objspace->rcompactor.moved_count_table, 0,
T_MASK *
sizeof(
size_t));
4124 memset(
objspace->rcompactor.moved_up_count_table, 0,
T_MASK *
sizeof(
size_t));
4125 memset(
objspace->rcompactor.moved_down_count_table, 0,
T_MASK *
sizeof(
size_t));
4138 const unsigned int immediate_sweep =
objspace->flags.immediate_sweep;
4140 gc_report(1,
objspace,
"gc_sweep: immediate: %d\n", immediate_sweep);
4143 if (
objspace->flags.during_compacting) {
4147 if (immediate_sweep) {
4148#if !GC_ENABLE_LAZY_SWEEP
4149 gc_prof_sweep_timer_start(
objspace);
4152#if !GC_ENABLE_LAZY_SWEEP
4153 gc_prof_sweep_timer_stop(
objspace);
4159 for (
int i = 0; i < HEAP_COUNT; i++) {
4171stack_chunk_alloc(
void)
4185 return stack->chunk == NULL;
4191 size_t size = stack->index;
4192 stack_chunk_t *chunk = stack->chunk ? stack->chunk->next : NULL;
4195 size += stack->limit;
4196 chunk = chunk->next;
4204 chunk->next = stack->cache;
4205 stack->cache = chunk;
4206 stack->cache_size++;
4214 if (stack->unused_cache_size > (stack->cache_size/2)) {
4215 chunk = stack->cache;
4216 stack->cache = stack->cache->next;
4217 stack->cache_size--;
4220 stack->unused_cache_size = stack->cache_size;
4228 GC_ASSERT(stack->index == stack->limit);
4230 if (stack->cache_size > 0) {
4231 next = stack->cache;
4232 stack->cache = stack->cache->next;
4233 stack->cache_size--;
4234 if (stack->unused_cache_size > stack->cache_size)
4235 stack->unused_cache_size = stack->cache_size;
4238 next = stack_chunk_alloc();
4240 next->next = stack->chunk;
4241 stack->chunk = next;
4250 prev = stack->chunk->next;
4251 GC_ASSERT(stack->index == 0);
4252 add_stack_chunk_cache(stack, stack->chunk);
4253 stack->chunk = prev;
4254 stack->index = stack->limit;
4262 while (chunk != NULL) {
4272 mark_stack_chunk_list_free(stack->chunk);
4278 mark_stack_chunk_list_free(stack->cache);
4279 stack->cache_size = 0;
4280 stack->unused_cache_size = 0;
4307 if (stack->index == stack->limit) {
4308 push_mark_stack_chunk(stack);
4310 stack->chunk->data[stack->index++] = obj;
4320 rb_bug(
"push_mark_stack() called for broken object");
4324 rb_bug(
"push_mark_stack: unexpected T_NODE object");
4328 rb_bug(
"rb_gc_mark(): unknown data type 0x%x(%p) %s",
4330 is_pointer_to_heap((
rb_objspace_t *)rb_gc_get_objspace(), (
void *)obj) ?
"corrupted object" :
"non object");
4336 if (is_mark_stack_empty(stack)) {
4339 if (stack->index == 1) {
4340 *data = stack->chunk->data[--stack->index];
4341 pop_mark_stack_chunk(stack);
4344 *data = stack->chunk->data[--stack->index];
4355 stack->index = stack->limit = STACK_CHUNK_SIZE;
4357 for (i=0; i < 4; i++) {
4358 add_stack_chunk_cache(stack, stack_chunk_alloc());
4360 stack->unused_cache_size = stack->cache_size;
4371 if (RVALUE_WB_UNPROTECTED(
objspace, obj) || !RVALUE_OLD_P(
objspace, obj)) {
4372 rgengc_remember(
objspace, old_parent);
4376 GC_ASSERT(old_parent ==
objspace->rgengc.parent_object);
4382 if (RVALUE_MARKED(
objspace, obj))
return 0;
4383 MARK_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj);
4397 if(!gc_config_full_mark_val)
4400 struct heap_page *page = GET_HEAP_PAGE(obj);
4402 GC_ASSERT(RVALUE_MARKING(
objspace, obj) == FALSE);
4403 check_rvalue_consistency(
objspace, obj);
4405 if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) {
4406 if (!RVALUE_OLD_P(
objspace, obj)) {
4407 gc_report(3,
objspace,
"gc_aging: YOUNG: %s\n", rb_obj_info(obj));
4410 else if (is_full_marking(
objspace)) {
4411 GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE);
4412 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(
objspace, page, obj);
4415 check_rvalue_consistency(
objspace, obj);
4423#if RGENGC_CHECK_MODE
4424 if (RVALUE_MARKED(
objspace, obj) == FALSE) rb_bug(
"gc_grey: %s is not marked.", rb_obj_info(obj));
4425 if (RVALUE_MARKING(
objspace, obj) == TRUE) rb_bug(
"gc_grey: %s is marking/remembered.", rb_obj_info(obj));
4428 if (is_incremental_marking(
objspace)) {
4429 MARK_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
4432 push_mark_stack(&
objspace->mark_stack, obj);
4438 GC_ASSERT(during_gc);
4440 rgengc_check_relation(
objspace, obj);
4441 if (!gc_mark_set(
objspace, obj))
return;
4444 if (
objspace->rgengc.parent_object) {
4445 RUBY_DEBUG_LOG(
"%p (%s) parent:%p (%s)",
4446 (
void *)obj, obj_type_name(obj),
4447 (
void *)
objspace->rgengc.parent_object, obj_type_name(
objspace->rgengc.parent_object));
4450 RUBY_DEBUG_LOG(
"%p (%s)", (
void *)obj, obj_type_name(obj));
4455 rb_obj_info_dump(obj);
4456 rb_bug(
"try to mark T_NONE object");
4467 if (RB_UNLIKELY(
objspace->flags.during_compacting)) {
4468 if (RB_LIKELY(during_gc)) {
4469 if (!RVALUE_PINNED(
objspace, obj)) {
4470 GC_ASSERT(GET_HEAP_PAGE(obj)->pinned_slots <= GET_HEAP_PAGE(obj)->total_slots);
4471 GET_HEAP_PAGE(obj)->pinned_slots++;
4472 MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj);
4486rb_gc_impl_mark_and_move(
void *objspace_ptr,
VALUE *ptr)
4490 if (RB_UNLIKELY(
objspace->flags.during_reference_updating)) {
4491 GC_ASSERT(
objspace->flags.during_compacting);
4492 GC_ASSERT(during_gc);
4494 *ptr = rb_gc_impl_location(
objspace, *ptr);
4502rb_gc_impl_mark(
void *objspace_ptr,
VALUE obj)
4510rb_gc_impl_mark_and_pin(
void *objspace_ptr,
VALUE obj)
4518rb_gc_impl_mark_maybe(
void *objspace_ptr,
VALUE obj)
4522 (void)VALGRIND_MAKE_MEM_DEFINED(&obj,
sizeof(obj));
4524 if (is_pointer_to_heap(
objspace, (
void *)obj)) {
4525 asan_unpoisoning_object(obj) {
4540rb_gc_impl_mark_weak(
void *objspace_ptr,
VALUE *ptr)
4549 rb_obj_info_dump(obj);
4550 rb_bug(
"try to mark T_NONE object");
4557 GC_ASSERT(RVALUE_MARKED(
objspace, obj));
4558 GC_ASSERT(!
objspace->flags.during_compacting);
4563 rgengc_check_relation(
objspace, obj);
4565 rb_darray_append_without_gc(&
objspace->weak_references, ptr);
4567 objspace->profile.weak_references_count++;
4571rb_gc_impl_remove_weak(
void *objspace_ptr,
VALUE parent_obj,
VALUE *ptr)
4577 if (!is_incremental_marking(
objspace))
return;
4580 if (!RVALUE_MARKED(
objspace, parent_obj))
return;
4583 rb_darray_foreach(
objspace->weak_references, i, ptr_ptr) {
4584 if (*ptr_ptr == ptr) {
4592pin_value(st_data_t key, st_data_t value, st_data_t data)
4594 rb_gc_impl_mark_and_pin((
void *)data, (
VALUE)value);
4602#define MARK_CHECKPOINT(category) do { \
4603 if (categoryp) *categoryp = category; \
4606 MARK_CHECKPOINT(
"objspace");
4609 if (finalizer_table != NULL) {
4610 st_foreach(finalizer_table, pin_value, (st_data_t)
objspace);
4613 st_foreach(
objspace->obj_to_id_tbl, gc_mark_tbl_no_pin_i, (st_data_t)
objspace);
4615 if (stress_to_class) rb_gc_mark(stress_to_class);
4617 rb_gc_save_machine_context();
4618 rb_gc_mark_roots(
objspace, categoryp);
4625 objspace->rgengc.parent_object = obj;
4636 rb_gc_mark_children(
objspace, obj);
4648 size_t marked_slots_at_the_beginning =
objspace->marked_slots;
4649 size_t popped_count = 0;
4651 while (pop_mark_stack(mstack, &obj)) {
4652 if (obj ==
Qundef)
continue;
4654 if (RGENGC_CHECK_MODE && !RVALUE_MARKED(
objspace, obj)) {
4655 rb_bug(
"gc_mark_stacked_objects: %s is not marked.", rb_obj_info(obj));
4660 if (RGENGC_CHECK_MODE && !RVALUE_MARKING(
objspace, obj)) {
4661 rb_bug(
"gc_mark_stacked_objects: incremental, but marking bit is 0");
4663 CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
4666 if (popped_count + (
objspace->marked_slots - marked_slots_at_the_beginning) > count) {
4675 if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(
objspace);
4677 if (is_mark_stack_empty(mstack)) {
4678 shrink_stack_chunk_cache(mstack);
4689 return gc_mark_stacked_objects(
objspace, TRUE, count);
4695 return gc_mark_stacked_objects(
objspace, FALSE, 0);
4698#if RGENGC_CHECK_MODE >= 4
4700#define MAKE_ROOTSIG(obj) (((VALUE)(obj) << 1) | 0x01)
4701#define IS_ROOTSIG(obj) ((VALUE)(obj) & 0x01)
4702#define GET_ROOTSIG(obj) ((const char *)((VALUE)(obj) >> 1))
4710static struct reflist *
4711reflist_create(
VALUE obj)
4713 struct reflist *refs =
xmalloc(
sizeof(
struct reflist));
4716 refs->list[0] = obj;
4722reflist_destruct(
struct reflist *refs)
4729reflist_add(
struct reflist *refs,
VALUE obj)
4731 if (refs->pos == refs->size) {
4733 SIZED_REALLOC_N(refs->list,
VALUE, refs->size, refs->size/2);
4736 refs->list[refs->pos++] = obj;
4740reflist_dump(
struct reflist *refs)
4743 for (i=0; i<refs->pos; i++) {
4744 VALUE obj = refs->list[i];
4745 if (IS_ROOTSIG(obj)) {
4746 fprintf(stderr,
"<root@%s>", GET_ROOTSIG(obj));
4749 fprintf(stderr,
"<%s>", rb_obj_info(obj));
4751 if (i+1 < refs->pos) fprintf(stderr,
", ");
4756reflist_referred_from_machine_context(
struct reflist *refs)
4759 for (i=0; i<refs->pos; i++) {
4760 VALUE obj = refs->list[i];
4761 if (IS_ROOTSIG(obj) && strcmp(GET_ROOTSIG(obj),
"machine_context") == 0)
return 1;
4776 const char *category;
4782allrefs_add(
struct allrefs *data,
VALUE obj)
4784 struct reflist *refs;
4787 if (st_lookup(data->references, obj, &r)) {
4788 refs = (
struct reflist *)r;
4789 reflist_add(refs, data->root_obj);
4793 refs = reflist_create(data->root_obj);
4794 st_insert(data->references, obj, (st_data_t)refs);
4800allrefs_i(
VALUE obj,
void *ptr)
4802 struct allrefs *data = (
struct allrefs *)ptr;
4804 if (allrefs_add(data, obj)) {
4805 push_mark_stack(&data->mark_stack, obj);
4810allrefs_roots_i(
VALUE obj,
void *ptr)
4812 struct allrefs *data = (
struct allrefs *)ptr;
4813 if (strlen(data->category) == 0) rb_bug(
"!!!");
4814 data->root_obj = MAKE_ROOTSIG(data->category);
4816 if (allrefs_add(data, obj)) {
4817 push_mark_stack(&data->mark_stack, obj);
4820#define PUSH_MARK_FUNC_DATA(v) do { \
4821 struct gc_mark_func_data_struct *prev_mark_func_data = GET_VM()->gc.mark_func_data; \
4822 GET_VM()->gc.mark_func_data = (v);
4824#define POP_MARK_FUNC_DATA() GET_VM()->gc.mark_func_data = prev_mark_func_data;} while (0)
4829 struct allrefs data;
4830 struct gc_mark_func_data_struct mfd;
4832 int prev_dont_gc = dont_gc_val();
4836 data.references = st_init_numtable();
4837 init_mark_stack(&data.mark_stack);
4839 mfd.mark_func = allrefs_roots_i;
4843 PUSH_MARK_FUNC_DATA(&mfd);
4844 GET_VM()->gc.mark_func_data = &mfd;
4845 mark_roots(
objspace, &data.category);
4846 POP_MARK_FUNC_DATA();
4849 while (pop_mark_stack(&data.mark_stack, &obj)) {
4850 rb_objspace_reachable_objects_from(data.root_obj = obj, allrefs_i, &data);
4852 free_stack_chunks(&data.mark_stack);
4854 dont_gc_set(prev_dont_gc);
4855 return data.references;
4859objspace_allrefs_destruct_i(st_data_t key, st_data_t value, st_data_t ptr)
4861 struct reflist *refs = (
struct reflist *)value;
4862 reflist_destruct(refs);
4867objspace_allrefs_destruct(
struct st_table *refs)
4869 st_foreach(refs, objspace_allrefs_destruct_i, 0);
4870 st_free_table(refs);
4873#if RGENGC_CHECK_MODE >= 5
4875allrefs_dump_i(st_data_t k, st_data_t v, st_data_t ptr)
4878 struct reflist *refs = (
struct reflist *)v;
4879 fprintf(stderr,
"[allrefs_dump_i] %s <- ", rb_obj_info(obj));
4881 fprintf(stderr,
"\n");
4889 fprintf(stderr,
"[all refs] (size: %"PRIuVALUE
")\n", size);
4890 st_foreach(
objspace->rgengc.allrefs_table, allrefs_dump_i, 0);
4895gc_check_after_marks_i(st_data_t k, st_data_t v, st_data_t ptr)
4898 struct reflist *refs = (
struct reflist *)v;
4902 if (!RVALUE_MARKED(
objspace, obj)) {
4903 fprintf(stderr,
"gc_check_after_marks_i: %s is not marked and not oldgen.\n", rb_obj_info(obj));
4904 fprintf(stderr,
"gc_check_after_marks_i: %p is referred from ", (
void *)obj);
4907 if (reflist_referred_from_machine_context(refs)) {
4908 fprintf(stderr,
" (marked from machine stack).\n");
4913 fprintf(stderr,
"\n");
4920gc_marks_check(
rb_objspace_t *
objspace, st_foreach_callback_func *checker_func,
const char *checker_name)
4922 size_t saved_malloc_increase =
objspace->malloc_params.increase;
4923#if RGENGC_ESTIMATE_OLDMALLOC
4924 size_t saved_oldmalloc_increase =
objspace->rgengc.oldmalloc_increase;
4931 st_foreach(
objspace->rgengc.allrefs_table, checker_func, (st_data_t)
objspace);
4934 if (
objspace->rgengc.error_count > 0) {
4935#if RGENGC_CHECK_MODE >= 5
4938 if (checker_name) rb_bug(
"%s: GC has problem.", checker_name);
4941 objspace_allrefs_destruct(
objspace->rgengc.allrefs_table);
4942 objspace->rgengc.allrefs_table = 0;
4945 objspace->malloc_params.increase = saved_malloc_increase;
4946#if RGENGC_ESTIMATE_OLDMALLOC
4947 objspace->rgengc.oldmalloc_increase = saved_oldmalloc_increase;
4955 size_t live_object_count;
4956 size_t zombie_object_count;
4959 size_t old_object_count;
4960 size_t remembered_shady_count;
4964check_generation_i(
const VALUE child,
void *ptr)
4967 const VALUE parent = data->parent;
4969 if (RGENGC_CHECK_MODE) GC_ASSERT(RVALUE_OLD_P(data->objspace, parent));
4971 if (!RVALUE_OLD_P(data->objspace, child)) {
4972 if (!RVALUE_REMEMBERED(data->objspace, parent) &&
4973 !RVALUE_REMEMBERED(data->objspace, child) &&
4974 !RVALUE_UNCOLLECTIBLE(data->objspace, child)) {
4975 fprintf(stderr,
"verify_internal_consistency_reachable_i: WB miss (O->Y) %s -> %s\n", rb_obj_info(parent), rb_obj_info(child));
4982check_color_i(
const VALUE child,
void *ptr)
4985 const VALUE parent = data->parent;
4987 if (!RVALUE_WB_UNPROTECTED(data->objspace, parent) && RVALUE_WHITE_P(data->objspace, child)) {
4988 fprintf(stderr,
"verify_internal_consistency_reachable_i: WB miss (B->W) - %s -> %s\n",
4989 rb_obj_info(parent), rb_obj_info(child));
4995check_children_i(
const VALUE child,
void *ptr)
4998 if (check_rvalue_consistency_force(data->objspace, child, FALSE) != 0) {
4999 fprintf(stderr,
"check_children_i: %s has error (referenced from %s)",
5000 rb_obj_info(child), rb_obj_info(data->parent));
5007verify_internal_consistency_i(
void *page_start,
void *page_end,
size_t stride,
5013 for (obj = (
VALUE)page_start; obj != (
VALUE)page_end; obj += stride) {
5014 asan_unpoisoning_object(obj) {
5015 if (!rb_gc_impl_garbage_object_p(
objspace, obj)) {
5017 data->live_object_count++;
5022 if (!gc_object_moved_p(
objspace, obj)) {
5024 rb_objspace_reachable_objects_from(obj, check_children_i, (
void *)data);
5028 if (RVALUE_OLD_P(
objspace, obj)) data->old_object_count++;
5029 if (RVALUE_WB_UNPROTECTED(
objspace, obj) && RVALUE_UNCOLLECTIBLE(
objspace, obj)) data->remembered_shady_count++;
5034 rb_objspace_reachable_objects_from(obj, check_generation_i, (
void *)data);
5037 if (is_incremental_marking(
objspace)) {
5038 if (RVALUE_BLACK_P(
objspace, obj)) {
5041 rb_objspace_reachable_objects_from(obj, check_color_i, (
void *)data);
5047 data->zombie_object_count++;
5050 fprintf(stderr,
"verify_internal_consistency_i: T_ZOMBIE has extra flags set: %s\n",
5056 fprintf(stderr,
"verify_internal_consistency_i: FL_FINALIZE %s but %s finalizer_table: %s\n",
5057 FL_TEST(obj,
FL_FINALIZE) ?
"set" :
"not set", st_is_member(finalizer_table, obj) ?
"in" :
"not in",
5072 unsigned int has_remembered_shady = FALSE;
5073 unsigned int has_remembered_old = FALSE;
5074 int remembered_old_objects = 0;
5075 int free_objects = 0;
5076 int zombie_objects = 0;
5078 short slot_size = page->slot_size;
5079 uintptr_t start = (uintptr_t)page->start;
5080 uintptr_t end = start + page->total_slots * slot_size;
5082 for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
5084 asan_unpoisoning_object(val) {
5089 if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) {
5090 has_remembered_shady = TRUE;
5092 if (RVALUE_PAGE_MARKING(page, val)) {
5093 has_remembered_old = TRUE;
5094 remembered_old_objects++;
5099 if (!is_incremental_marking(
objspace) &&
5100 page->flags.has_remembered_objects == FALSE && has_remembered_old == TRUE) {
5102 for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
5104 if (RVALUE_PAGE_MARKING(page, val)) {
5105 fprintf(stderr,
"marking -> %s\n", rb_obj_info(val));
5108 rb_bug(
"page %p's has_remembered_objects should be false, but there are remembered old objects (%d). %s",
5109 (
void *)page, remembered_old_objects, obj ? rb_obj_info(obj) :
"");
5112 if (page->flags.has_uncollectible_wb_unprotected_objects == FALSE && has_remembered_shady == TRUE) {
5113 rb_bug(
"page %p's has_remembered_shady should be false, but there are remembered shady objects. %s",
5114 (
void *)page, obj ? rb_obj_info(obj) :
"");
5119 if (page->free_slots != free_objects) {
5120 rb_bug(
"page %p's free_slots should be %d, but %d", (
void *)page, page->free_slots, free_objects);
5123 if (page->final_slots != zombie_objects) {
5124 rb_bug(
"page %p's final_slots should be %d, but %d", (
void *)page, page->final_slots, zombie_objects);
5127 return remembered_old_objects;
5133 int remembered_old_objects = 0;
5136 ccan_list_for_each(head, page, page_node) {
5137 asan_unlock_freelist(page);
5142 rb_asan_unpoison_object(vp,
false);
5144 fprintf(stderr,
"freelist slot expected to be T_NONE but was: %s\n", rb_obj_info(vp));
5147 rb_asan_poison_object(prev);
5149 asan_lock_freelist(page);
5151 if (page->flags.has_remembered_objects == FALSE) {
5152 remembered_old_objects += gc_verify_heap_page(
objspace, page,
Qfalse);
5156 return remembered_old_objects;
5162 int remembered_old_objects = 0;
5163 for (
int i = 0; i < HEAP_COUNT; i++) {
5164 remembered_old_objects += gc_verify_heap_pages_(
objspace, &((&heaps[i])->pages));
5166 return remembered_old_objects;
5175 gc_report(5,
objspace,
"gc_verify_internal_consistency: start\n");
5178 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
5180 short slot_size = page->slot_size;
5182 uintptr_t start = (uintptr_t)page->start;
5183 uintptr_t end = start + page->total_slots * slot_size;
5185 verify_internal_consistency_i((
void *)start, (
void *)end, slot_size, &data);
5188 if (data.err_count != 0) {
5189#if RGENGC_CHECK_MODE >= 5
5190 objspace->rgengc.error_count = data.err_count;
5191 gc_marks_check(
objspace, NULL, NULL);
5194 rb_bug(
"gc_verify_internal_consistency: found internal inconsistency.");
5204 !rb_gc_multi_ractor_p()) {
5205 if (objspace_live_slots(
objspace) != data.live_object_count) {
5206 fprintf(stderr,
"heap_pages_final_slots: %"PRIdSIZE
", total_freed_objects: %"PRIdSIZE
"\n",
5208 rb_bug(
"inconsistent live slot number: expect %"PRIuSIZE
", but %"PRIuSIZE
".",
5209 objspace_live_slots(
objspace), data.live_object_count);
5214 if (
objspace->rgengc.old_objects != data.old_object_count) {
5215 rb_bug(
"inconsistent old slot number: expect %"PRIuSIZE
", but %"PRIuSIZE
".",
5216 objspace->rgengc.old_objects, data.old_object_count);
5218 if (
objspace->rgengc.uncollectible_wb_unprotected_objects != data.remembered_shady_count) {
5219 rb_bug(
"inconsistent number of wb unprotected objects: expect %"PRIuSIZE
", but %"PRIuSIZE
".",
5220 objspace->rgengc.uncollectible_wb_unprotected_objects, data.remembered_shady_count);
5225 size_t list_count = 0;
5228 VALUE z = heap_pages_deferred_final;
5231 z = RZOMBIE(z)->next;
5235 if (total_final_slots_count(
objspace) != data.zombie_object_count ||
5236 total_final_slots_count(
objspace) != list_count) {
5238 rb_bug(
"inconsistent finalizing object count:\n"
5239 " expect %"PRIuSIZE
"\n"
5240 " but %"PRIuSIZE
" zombies\n"
5241 " heap_pages_deferred_final list has %"PRIuSIZE
" items.",
5243 data.zombie_object_count,
5248 gc_report(5,
objspace,
"gc_verify_internal_consistency: OK\n");
5252gc_verify_internal_consistency(
void *objspace_ptr)
5256 unsigned int lev = rb_gc_vm_lock();
5260 unsigned int prev_during_gc = during_gc;
5263 gc_verify_internal_consistency_(
objspace);
5265 during_gc = prev_during_gc;
5267 rb_gc_vm_unlock(lev);
5271heap_move_pooled_pages_to_free_pages(
rb_heap_t *heap)
5273 if (heap->pooled_pages) {
5274 if (heap->free_pages) {
5275 struct heap_page *free_pages_tail = heap->free_pages;
5276 while (free_pages_tail->free_next) {
5277 free_pages_tail = free_pages_tail->free_next;
5279 free_pages_tail->free_next = heap->pooled_pages;
5282 heap->free_pages = heap->pooled_pages;
5285 heap->pooled_pages = NULL;
5292 struct heap_page *page = GET_HEAP_PAGE(obj);
5293 bits_t *uncollectible_bits = &page->uncollectible_bits[0];
5295 if (!MARKED_IN_BITMAP(uncollectible_bits, obj)) {
5296 page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
5297 MARK_IN_BITMAP(uncollectible_bits, obj);
5298 objspace->rgengc.uncollectible_wb_unprotected_objects++;
5300#if RGENGC_PROFILE > 0
5301 objspace->profile.total_remembered_shady_object_count++;
5302#if RGENGC_PROFILE >= 2
5319 gc_report(2,
objspace,
"gc_marks_wb_unprotected_objects: marked shady: %s\n", rb_obj_info((
VALUE)p));
5324 p += BASE_SLOT_SIZE;
5335 ccan_list_for_each(&heap->pages, page, page_node) {
5336 bits_t *mark_bits = page->mark_bits;
5337 bits_t *wbun_bits = page->wb_unprotected_bits;
5338 uintptr_t p = page->start;
5341 bits_t bits = mark_bits[0] & wbun_bits[0];
5342 bits >>= NUM_IN_PAGE(p);
5343 gc_marks_wb_unprotected_objects_plane(
objspace, p, bits);
5344 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
5346 for (j=1; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
5347 bits_t bits = mark_bits[j] & wbun_bits[j];
5349 gc_marks_wb_unprotected_objects_plane(
objspace, p, bits);
5350 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
5354 gc_mark_stacked_objects_all(
objspace);
5360 size_t retained_weak_references_count = 0;
5362 rb_darray_foreach(
objspace->weak_references, i, ptr_ptr) {
5363 if (!*ptr_ptr)
continue;
5365 VALUE obj = **ptr_ptr;
5369 if (!RVALUE_MARKED(
objspace, obj)) {
5373 retained_weak_references_count++;
5377 objspace->profile.retained_weak_references_count = retained_weak_references_count;
5379 rb_darray_clear(
objspace->weak_references);
5380 rb_darray_resize_capa_without_gc(&
objspace->weak_references, retained_weak_references_count);
5387 if (is_incremental_marking(
objspace)) {
5388 if (RGENGC_CHECK_MODE && is_mark_stack_empty(&
objspace->mark_stack) == 0) {
5389 rb_bug(
"gc_marks_finish: mark stack is not empty (%"PRIdSIZE
").",
5390 mark_stack_size(&
objspace->mark_stack));
5394 while (gc_mark_stacked_objects_incremental(
objspace, INT_MAX) ==
false);
5396#if RGENGC_CHECK_MODE >= 2
5397 if (gc_verify_heap_pages(
objspace) != 0) {
5398 rb_bug(
"gc_marks_finish (incremental): there are remembered old objects.");
5402 objspace->flags.during_incremental_marking = FALSE;
5404 for (
int i = 0; i < HEAP_COUNT; i++) {
5405 gc_marks_wb_unprotected_objects(
objspace, &heaps[i]);
5409 gc_update_weak_references(
objspace);
5411#if RGENGC_CHECK_MODE >= 2
5412 gc_verify_internal_consistency(
objspace);
5415#if RGENGC_CHECK_MODE >= 4
5417 gc_marks_check(
objspace, gc_check_after_marks_i,
"after_marks");
5422 const unsigned long r_mul =
objspace->live_ractor_cache_count > 8 ? 8 :
objspace->live_ractor_cache_count;
5424 size_t total_slots = objspace_available_slots(
objspace);
5425 size_t sweep_slots = total_slots -
objspace->marked_slots;
5426 size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio);
5427 size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio);
5428 if (min_free_slots < gc_params.heap_free_slots * r_mul) {
5429 min_free_slots = gc_params.heap_free_slots * r_mul;
5432 int full_marking = is_full_marking(
objspace);
5437 size_t total_init_slots = 0;
5438 for (
int i = 0; i < HEAP_COUNT; i++) {
5439 total_init_slots += gc_params.heap_init_slots[i] * r_mul;
5442 if (max_free_slots < total_init_slots) {
5443 max_free_slots = total_init_slots;
5446 if (sweep_slots > max_free_slots) {
5447 heap_pages_freeable_pages = (sweep_slots - max_free_slots) / HEAP_PAGE_OBJ_LIMIT;
5450 heap_pages_freeable_pages = 0;
5453 if (
objspace->heap_pages.allocatable_slots == 0 && sweep_slots < min_free_slots) {
5454 if (!full_marking) {
5455 if (
objspace->profile.count -
objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
5456 full_marking = TRUE;
5459 gc_report(1,
objspace,
"gc_marks_finish: next is full GC!!)\n");
5460 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
5465 heap_allocatable_slots_expand(
objspace, NULL, sweep_slots, total_slots);
5471 const double r = gc_params.oldobject_limit_factor;
5472 objspace->rgengc.uncollectible_wb_unprotected_objects_limit = MAX(
5473 (
size_t)(
objspace->rgengc.uncollectible_wb_unprotected_objects * r),
5474 (
size_t)(
objspace->rgengc.old_objects * gc_params.uncollectible_wb_unprotected_objects_limit_ratio)
5476 objspace->rgengc.old_objects_limit = (size_t)(
objspace->rgengc.old_objects * r);
5479 if (
objspace->rgengc.uncollectible_wb_unprotected_objects >
objspace->rgengc.uncollectible_wb_unprotected_objects_limit) {
5480 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_SHADY;
5483 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDGEN;
5486 gc_report(1,
objspace,
"gc_marks_finish (marks %"PRIdSIZE
" objects, "
5487 "old %"PRIdSIZE
" objects, total %"PRIdSIZE
" slots, "
5488 "sweep %"PRIdSIZE
" slots, allocatable %"PRIdSIZE
" slots, next GC: %s)\n",
5490 gc_needs_major_flags ?
"major" :
"minor");
5494 rb_ractor_finish_marking();
5500gc_compact_heap_cursors_met_p(
rb_heap_t *heap)
5502 return heap->sweeping_page == heap->compact_cursor;
5509 size_t obj_size = rb_gc_obj_optimal_size(obj);
5510 if (obj_size == 0) {
5515 if (rb_gc_impl_size_allocatable_p(obj_size)) {
5516 idx = heap_idx_for_size(obj_size);
5526 GC_ASSERT(gc_is_moveable_obj(
objspace, src));
5529 uint32_t orig_shape = 0;
5530 uint32_t new_shape = 0;
5532 if (gc_compact_heap_cursors_met_p(dest_pool)) {
5533 return dest_pool != heap;
5537 orig_shape = rb_gc_get_shape(src);
5539 if (dest_pool != heap) {
5540 new_shape = rb_gc_rebuild_shape(src, dest_pool - heaps);
5542 if (new_shape == 0) {
5548 while (!try_move(
objspace, dest_pool, dest_pool->free_pages, src)) {
5550 .page = dest_pool->sweeping_page,
5559 lock_page_body(
objspace, GET_PAGE_BODY(src));
5560 gc_sweep_page(
objspace, dest_pool, &ctx);
5561 unlock_page_body(
objspace, GET_PAGE_BODY(src));
5563 if (dest_pool->sweeping_page->free_slots > 0) {
5564 heap_add_freepage(dest_pool, dest_pool->sweeping_page);
5567 dest_pool->sweeping_page = ccan_list_next(&dest_pool->pages, dest_pool->sweeping_page, page_node);
5568 if (gc_compact_heap_cursors_met_p(dest_pool)) {
5569 return dest_pool != heap;
5573 if (orig_shape != 0) {
5574 if (new_shape != 0) {
5576 rb_gc_set_shape(dest, new_shape);
5578 RMOVED(src)->original_shape_id = orig_shape;
5587 short slot_size = page->slot_size;
5588 short slot_bits = slot_size / BASE_SLOT_SIZE;
5589 GC_ASSERT(slot_bits > 0);
5593 GC_ASSERT(vp % BASE_SLOT_SIZE == 0);
5598 if (gc_is_moveable_obj(
objspace, vp)) {
5599 if (!gc_compact_move(
objspace, heap, vp)) {
5606 bitset >>= slot_bits;
5616 GC_ASSERT(page == heap->compact_cursor);
5618 bits_t *mark_bits, *pin_bits;
5620 uintptr_t p = page->start;
5622 mark_bits = page->mark_bits;
5623 pin_bits = page->pinned_bits;
5626 bitset = (mark_bits[0] & ~pin_bits[0]);
5627 bitset >>= NUM_IN_PAGE(p);
5629 if (!gc_compact_plane(
objspace, heap, (uintptr_t)p, bitset, page))
5632 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
5634 for (
int j = 1; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
5635 bitset = (mark_bits[j] & ~pin_bits[j]);
5637 if (!gc_compact_plane(
objspace, heap, (uintptr_t)p, bitset, page))
5640 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
5649 for (
int i = 0; i < HEAP_COUNT; i++) {
5652 if (heap->total_pages > 0 &&
5653 !gc_compact_heap_cursors_met_p(heap)) {
5665#if RGENGC_CHECK_MODE >= 2
5666 gc_verify_internal_consistency(
objspace);
5669 while (!gc_compact_all_compacted_p(
objspace)) {
5670 for (
int i = 0; i < HEAP_COUNT; i++) {
5673 if (gc_compact_heap_cursors_met_p(heap)) {
5677 struct heap_page *start_page = heap->compact_cursor;
5679 if (!gc_compact_page(
objspace, heap, start_page)) {
5680 lock_page_body(
objspace, start_page->body);
5687 lock_page_body(
objspace, start_page->body);
5688 heap->compact_cursor = ccan_list_prev(&heap->pages, heap->compact_cursor, page_node);
5694#if RGENGC_CHECK_MODE >= 2
5695 gc_verify_internal_consistency(
objspace);
5702 gc_report(1,
objspace,
"gc_marks_rest\n");
5704 for (
int i = 0; i < HEAP_COUNT; i++) {
5705 (&heaps[i])->pooled_pages = NULL;
5708 if (is_incremental_marking(
objspace)) {
5709 while (gc_mark_stacked_objects_incremental(
objspace, INT_MAX) == FALSE);
5712 gc_mark_stacked_objects_all(
objspace);
5721 bool marking_finished =
false;
5724 if (gc_mark_stacked_objects_incremental(
objspace, slots)) {
5727 marking_finished =
true;
5730 return marking_finished;
5736 GC_ASSERT(dont_gc_val() == FALSE ||
objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
5737 bool marking_finished =
true;
5741 if (heap->free_pages) {
5742 gc_report(2,
objspace,
"gc_marks_continue: has pooled pages");
5747 gc_report(2,
objspace,
"gc_marks_continue: no more pooled pages (stack depth: %"PRIdSIZE
").\n",
5748 mark_stack_size(&
objspace->mark_stack));
5749 heap->force_incremental_marking_finish_count++;
5755 return marking_finished;
5762 gc_report(1,
objspace,
"gc_marks_start: (%s)\n", full_mark ?
"full" :
"minor");
5763 gc_mode_transition(
objspace, gc_mode_marking);
5766 size_t incremental_marking_steps = (
objspace->rincgc.pooled_slots / INCREMENTAL_MARK_STEP_ALLOCATIONS) + 1;
5767 objspace->rincgc.step_slots = (
objspace->marked_slots * 2) / incremental_marking_steps;
5769 if (0) fprintf(stderr,
"objspace->marked_slots: %"PRIdSIZE
", "
5770 "objspace->rincgc.pooled_page_num: %"PRIdSIZE
", "
5771 "objspace->rincgc.step_slots: %"PRIdSIZE
", \n",
5773 objspace->flags.during_minor_gc = FALSE;
5774 if (ruby_enable_autocompact) {
5775 objspace->flags.during_compacting |= TRUE;
5777 objspace->profile.major_gc_count++;
5778 objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
5783 for (
int i = 0; i < HEAP_COUNT; i++) {
5785 rgengc_mark_and_rememberset_clear(
objspace, heap);
5786 heap_move_pooled_pages_to_free_pages(heap);
5788 if (
objspace->flags.during_compacting) {
5791 ccan_list_for_each(&heap->pages, page, page_node) {
5792 page->pinned_slots = 0;
5798 objspace->flags.during_minor_gc = TRUE;
5800 objspace->rgengc.old_objects +
objspace->rgengc.uncollectible_wb_unprotected_objects;
5801 objspace->profile.minor_gc_count++;
5803 for (
int i = 0; i < HEAP_COUNT; i++) {
5804 rgengc_rememberset_mark(
objspace, &heaps[i]);
5810 gc_report(1,
objspace,
"gc_marks_start: (%s) end, stack in %"PRIdSIZE
"\n",
5817 gc_prof_mark_timer_start(
objspace);
5820 bool marking_finished =
false;
5824 gc_marks_start(
objspace, full_mark);
5825 if (!is_incremental_marking(
objspace)) {
5827 marking_finished =
true;
5830#if RGENGC_PROFILE > 0
5833 record->old_objects =
objspace->rgengc.old_objects;
5840 return marking_finished;
5848 if (level <= RGENGC_DEBUG) {
5852 const char *status =
" ";
5855 status = is_full_marking(
objspace) ?
"+" :
"-";
5861 if (is_incremental_marking(
objspace)) {
5866 va_start(args, fmt);
5867 vsnprintf(buf, 1024, fmt, args);
5870 fprintf(out,
"%s|", status);
5880 struct heap_page *page = GET_HEAP_PAGE(obj);
5881 bits_t *bits = &page->remembered_bits[0];
5883 if (MARKED_IN_BITMAP(bits, obj)) {
5887 page->flags.has_remembered_objects = TRUE;
5888 MARK_IN_BITMAP(bits, obj);
5899 gc_report(6,
objspace,
"rgengc_remember: %s %s\n", rb_obj_info(obj),
5900 RVALUE_REMEMBERED(
objspace, obj) ?
"was already remembered" :
"is remembered now");
5902 check_rvalue_consistency(
objspace, obj);
5904 if (RGENGC_CHECK_MODE) {
5905 if (RVALUE_WB_UNPROTECTED(
objspace, obj)) rb_bug(
"rgengc_remember: %s is not wb protected.", rb_obj_info(obj));
5908#if RGENGC_PROFILE > 0
5909 if (!RVALUE_REMEMBERED(
objspace, obj)) {
5910 if (RVALUE_WB_UNPROTECTED(
objspace, obj) == 0) {
5911 objspace->profile.total_remembered_normal_object_count++;
5912#if RGENGC_PROFILE >= 2
5919 return rgengc_remembersetbits_set(
objspace, obj);
5922#ifndef PROFILE_REMEMBERSET_MARK
5923#define PROFILE_REMEMBERSET_MARK 0
5933 gc_report(2,
objspace,
"rgengc_rememberset_mark: mark %s\n", rb_obj_info(obj));
5934 GC_ASSERT(RVALUE_UNCOLLECTIBLE(
objspace, obj));
5935 GC_ASSERT(RVALUE_OLD_P(
objspace, obj) || RVALUE_WB_UNPROTECTED(
objspace, obj));
5939 p += BASE_SLOT_SIZE;
5950#if PROFILE_REMEMBERSET_MARK
5951 int has_old = 0, has_shady = 0, has_both = 0, skip = 0;
5953 gc_report(1,
objspace,
"rgengc_rememberset_mark: start\n");
5955 ccan_list_for_each(&heap->pages, page, page_node) {
5956 if (page->flags.has_remembered_objects | page->flags.has_uncollectible_wb_unprotected_objects) {
5957 uintptr_t p = page->start;
5958 bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT];
5959 bits_t *remembered_bits = page->remembered_bits;
5960 bits_t *uncollectible_bits = page->uncollectible_bits;
5961 bits_t *wb_unprotected_bits = page->wb_unprotected_bits;
5962#if PROFILE_REMEMBERSET_MARK
5963 if (page->flags.has_remembered_objects && page->flags.has_uncollectible_wb_unprotected_objects) has_both++;
5964 else if (page->flags.has_remembered_objects) has_old++;
5965 else if (page->flags.has_uncollectible_wb_unprotected_objects) has_shady++;
5967 for (j=0; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
5968 bits[j] = remembered_bits[j] | (uncollectible_bits[j] & wb_unprotected_bits[j]);
5969 remembered_bits[j] = 0;
5971 page->flags.has_remembered_objects = FALSE;
5974 bitset >>= NUM_IN_PAGE(p);
5975 rgengc_rememberset_mark_plane(
objspace, p, bitset);
5976 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
5978 for (j=1; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
5980 rgengc_rememberset_mark_plane(
objspace, p, bitset);
5981 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
5984#if PROFILE_REMEMBERSET_MARK
5991#if PROFILE_REMEMBERSET_MARK
5992 fprintf(stderr,
"%d\t%d\t%d\t%d\n", has_both, has_old, has_shady, skip);
5994 gc_report(1,
objspace,
"rgengc_rememberset_mark: finished\n");
6002 ccan_list_for_each(&heap->pages, page, page_node) {
6003 memset(&page->mark_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6004 memset(&page->uncollectible_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6005 memset(&page->marking_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6006 memset(&page->remembered_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6007 memset(&page->pinned_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6008 page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
6009 page->flags.has_remembered_objects = FALSE;
6020 if (RGENGC_CHECK_MODE) {
6021 if (!RVALUE_OLD_P(
objspace, a)) rb_bug(
"gc_writebarrier_generational: %s is not an old object.", rb_obj_info(a));
6022 if ( RVALUE_OLD_P(
objspace, b)) rb_bug(
"gc_writebarrier_generational: %s is an old object.", rb_obj_info(b));
6023 if (is_incremental_marking(
objspace)) rb_bug(
"gc_writebarrier_generational: called while incremental marking: %s -> %s", rb_obj_info(a), rb_obj_info(b));
6027 if (!RVALUE_REMEMBERED(
objspace, a)) {
6028 int lev = rb_gc_vm_lock_no_barrier();
6032 rb_gc_vm_unlock_no_barrier(lev);
6034 gc_report(1,
objspace,
"gc_writebarrier_generational: %s (remembered) -> %s\n", rb_obj_info(a), rb_obj_info(b));
6037 check_rvalue_consistency(
objspace, a);
6038 check_rvalue_consistency(
objspace, b);
6044 gc_mark_set_parent(
objspace, parent);
6045 rgengc_check_relation(
objspace, obj);
6046 if (gc_mark_set(
objspace, obj) == FALSE)
return;
6056 gc_report(2,
objspace,
"gc_writebarrier_incremental: [LG] %p -> %s\n", (
void *)a, rb_obj_info(b));
6060 if (!RVALUE_WB_UNPROTECTED(
objspace, a)) {
6061 gc_report(2,
objspace,
"gc_writebarrier_incremental: [IN] %p -> %s\n", (
void *)a, rb_obj_info(b));
6069 if (RB_UNLIKELY(
objspace->flags.during_compacting)) {
6070 MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(b), b);
6076rb_gc_impl_writebarrier(
void *objspace_ptr,
VALUE a,
VALUE b)
6080 if (RGENGC_CHECK_MODE) {
6081 if (
SPECIAL_CONST_P(a)) rb_bug(
"rb_gc_writebarrier: a is special const: %"PRIxVALUE, a);
6082 if (
SPECIAL_CONST_P(b)) rb_bug(
"rb_gc_writebarrier: b is special const: %"PRIxVALUE, b);
6093 if (!is_incremental_marking(
objspace)) {
6098 gc_writebarrier_generational(a, b,
objspace);
6104 int lev = rb_gc_vm_lock_no_barrier();
6106 if (is_incremental_marking(
objspace)) {
6107 gc_writebarrier_incremental(a, b,
objspace);
6113 rb_gc_vm_unlock_no_barrier(lev);
6115 if (retry)
goto retry;
6121rb_gc_impl_writebarrier_unprotect(
void *objspace_ptr,
VALUE obj)
6125 if (RVALUE_WB_UNPROTECTED(
objspace, obj)) {
6129 gc_report(2,
objspace,
"rb_gc_writebarrier_unprotect: %s %s\n", rb_obj_info(obj),
6130 RVALUE_REMEMBERED(
objspace, obj) ?
" (already remembered)" :
"");
6132 unsigned int lev = rb_gc_vm_lock_no_barrier();
6135 gc_report(1,
objspace,
"rb_gc_writebarrier_unprotect: %s\n", rb_obj_info(obj));
6138 gc_remember_unprotected(
objspace, obj);
6141 objspace->profile.total_shade_operation_count++;
6142#if RGENGC_PROFILE >= 2
6148 RVALUE_AGE_RESET(obj);
6151 RB_DEBUG_COUNTER_INC(obj_wb_unprotect);
6152 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
6154 rb_gc_vm_unlock_no_barrier(lev);
6159rb_gc_impl_copy_attributes(
void *objspace_ptr,
VALUE dest,
VALUE obj)
6163 if (RVALUE_WB_UNPROTECTED(
objspace, obj)) {
6164 rb_gc_impl_writebarrier_unprotect(
objspace, dest);
6166 rb_gc_impl_copy_finalizer(
objspace, dest, obj);
6170rb_gc_impl_active_gc_name(
void)
6176rb_gc_impl_writebarrier_remember(
void *objspace_ptr,
VALUE obj)
6180 gc_report(1,
objspace,
"rb_gc_writebarrier_remember: %s\n", rb_obj_info(obj));
6182 if (is_incremental_marking(
objspace)) {
6183 if (RVALUE_BLACK_P(
objspace, obj)) {
6196 ID ID_wb_protected, ID_age, ID_old, ID_uncollectible, ID_marking,
6197 ID_marked, ID_pinned, ID_object_id, ID_shareable;
6200#define RB_GC_OBJECT_METADATA_ENTRY_COUNT (sizeof(struct rb_gc_object_metadata_names) / sizeof(ID))
6204rb_gc_impl_object_metadata(
void *objspace_ptr,
VALUE obj)
6210 if (!names.ID_marked) {
6211#define I(s) names.ID_##s = rb_intern(#s)
6224#define SET_ENTRY(na, v) do { \
6225 GC_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \
6226 object_metadata_entries[n].name = names.ID_##na; \
6227 object_metadata_entries[n].val = v; \
6231 if (!RVALUE_WB_UNPROTECTED(
objspace, obj)) SET_ENTRY(wb_protected,
Qtrue);
6232 SET_ENTRY(age,
INT2FIX(RVALUE_AGE_GET(obj)));
6234 if (RVALUE_UNCOLLECTIBLE(
objspace, obj)) SET_ENTRY(uncollectible,
Qtrue);
6235 if (RVALUE_MARKING(
objspace, obj)) SET_ENTRY(marking,
Qtrue);
6241 object_metadata_entries[n].name = 0;
6242 object_metadata_entries[n].val = 0;
6245 return object_metadata_entries;
6249rb_gc_impl_ractor_cache_alloc(
void *objspace_ptr,
void *ractor)
6253 objspace->live_ractor_cache_count++;
6259rb_gc_impl_ractor_cache_free(
void *objspace_ptr,
void *cache)
6263 objspace->live_ractor_cache_count--;
6265 gc_ractor_newobj_cache_clear(cache, NULL);
6272 if (!heap->free_pages) {
6273 if (!heap_page_allocate_and_initialize(
objspace, heap)) {
6274 objspace->heap_pages.allocatable_slots = 1;
6275 heap_page_allocate_and_initialize(
objspace, heap);
6283 if (dont_gc_val() || during_gc) {
6284 for (
int i = 0; i < HEAP_COUNT; i++) {
6301 size_t old_limit = malloc_limit;
6303 if (inc > malloc_limit) {
6304 malloc_limit = (size_t)(inc * gc_params.malloc_limit_growth_factor);
6305 if (malloc_limit > gc_params.malloc_limit_max) {
6306 malloc_limit = gc_params.malloc_limit_max;
6310 malloc_limit = (size_t)(malloc_limit * 0.98);
6311 if (malloc_limit < gc_params.malloc_limit_min) {
6312 malloc_limit = gc_params.malloc_limit_min;
6317 if (old_limit != malloc_limit) {
6318 fprintf(stderr,
"[%"PRIuSIZE
"] malloc_limit: %"PRIuSIZE
" -> %"PRIuSIZE
"\n",
6319 rb_gc_count(), old_limit, malloc_limit);
6322 fprintf(stderr,
"[%"PRIuSIZE
"] malloc_limit: not changed (%"PRIuSIZE
")\n",
6323 rb_gc_count(), malloc_limit);
6329#if RGENGC_ESTIMATE_OLDMALLOC
6331 if (
objspace->rgengc.oldmalloc_increase >
objspace->rgengc.oldmalloc_increase_limit) {
6332 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDMALLOC;
6333 objspace->rgengc.oldmalloc_increase_limit =
6334 (size_t)(
objspace->rgengc.oldmalloc_increase_limit * gc_params.oldmalloc_limit_growth_factor);
6336 if (
objspace->rgengc.oldmalloc_increase_limit > gc_params.oldmalloc_limit_max) {
6337 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_max;
6341 if (0) fprintf(stderr,
"%"PRIdSIZE
"\t%d\t%"PRIuSIZE
"\t%"PRIuSIZE
"\t%"PRIdSIZE
"\n",
6343 gc_needs_major_flags,
6344 objspace->rgengc.oldmalloc_increase,
6345 objspace->rgengc.oldmalloc_increase_limit,
6346 gc_params.oldmalloc_limit_max);
6350 objspace->rgengc.oldmalloc_increase = 0;
6352 if ((
objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) {
6353 objspace->rgengc.oldmalloc_increase_limit =
6354 (size_t)(
objspace->rgengc.oldmalloc_increase_limit / ((gc_params.oldmalloc_limit_growth_factor - 1)/10 + 1));
6355 if (
objspace->rgengc.oldmalloc_increase_limit < gc_params.oldmalloc_limit_min) {
6356 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
6368 int lev = rb_gc_vm_lock();
6370#if GC_PROFILE_MORE_DETAIL
6371 objspace->profile.prepare_time = getrusage_time();
6376#if GC_PROFILE_MORE_DETAIL
6377 objspace->profile.prepare_time = getrusage_time() -
objspace->profile.prepare_time;
6382 rb_gc_vm_unlock(lev);
6390 unsigned int do_full_mark = !!(reason & GPR_FLAG_FULL_MARK);
6393 objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP);
6395 if (!rb_darray_size(
objspace->heap_pages.sorted))
return TRUE;
6396 if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(
objspace))
return TRUE;
6398 GC_ASSERT(gc_mode(
objspace) == gc_mode_none);
6399 GC_ASSERT(!is_lazy_sweeping(
objspace));
6400 GC_ASSERT(!is_incremental_marking(
objspace));
6402 unsigned int lock_lev;
6403 gc_enter(
objspace, gc_enter_event_start, &lock_lev);
6405#if RGENGC_CHECK_MODE >= 2
6406 gc_verify_internal_consistency(
objspace);
6409 if (ruby_gc_stressful) {
6410 int flag =
FIXNUM_P(ruby_gc_stress_mode) ?
FIX2INT(ruby_gc_stress_mode) : 0;
6412 if ((flag & (1 << gc_stress_no_major)) == 0) {
6413 do_full_mark = TRUE;
6416 objspace->flags.immediate_sweep = !(flag & (1<<gc_stress_no_immediate_sweep));
6419 if (gc_needs_major_flags) {
6420 reason |= gc_needs_major_flags;
6421 do_full_mark = TRUE;
6425 if (!gc_config_full_mark_val) {
6426 do_full_mark = FALSE;
6428 gc_needs_major_flags = GPR_FLAG_NONE;
6430 if (do_full_mark && (reason & GPR_FLAG_MAJOR_MASK) == 0) {
6431 reason |= GPR_FLAG_MAJOR_BY_FORCE;
6434 if (
objspace->flags.dont_incremental ||
6435 reason & GPR_FLAG_IMMEDIATE_MARK ||
6436 ruby_gc_stressful) {
6437 objspace->flags.during_incremental_marking = FALSE;
6440 objspace->flags.during_incremental_marking = do_full_mark;
6444 if (do_full_mark && ruby_enable_autocompact) {
6445 objspace->flags.during_compacting = TRUE;
6446#if RGENGC_CHECK_MODE
6447 objspace->rcompactor.compare_func = ruby_autocompact_compare_func;
6451 objspace->flags.during_compacting = !!(reason & GPR_FLAG_COMPACT);
6454 if (!GC_ENABLE_LAZY_SWEEP ||
objspace->flags.dont_incremental) {
6455 objspace->flags.immediate_sweep = TRUE;
6458 if (
objspace->flags.immediate_sweep) reason |= GPR_FLAG_IMMEDIATE_SWEEP;
6460 gc_report(1,
objspace,
"gc_start(reason: %x) => %u, %d, %d\n",
6462 do_full_mark, !is_incremental_marking(
objspace),
objspace->flags.immediate_sweep);
6464#if USE_DEBUG_COUNTER
6465 RB_DEBUG_COUNTER_INC(gc_count);
6467 if (reason & GPR_FLAG_MAJOR_MASK) {
6468 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_nofree, reason & GPR_FLAG_MAJOR_BY_NOFREE);
6469 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldgen, reason & GPR_FLAG_MAJOR_BY_OLDGEN);
6470 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_shady, reason & GPR_FLAG_MAJOR_BY_SHADY);
6471 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_force, reason & GPR_FLAG_MAJOR_BY_FORCE);
6472#if RGENGC_ESTIMATE_OLDMALLOC
6473 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldmalloc, reason & GPR_FLAG_MAJOR_BY_OLDMALLOC);
6477 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_newobj, reason & GPR_FLAG_NEWOBJ);
6478 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_malloc, reason & GPR_FLAG_MALLOC);
6479 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_method, reason & GPR_FLAG_METHOD);
6480 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_capi, reason & GPR_FLAG_CAPI);
6481 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_stress, reason & GPR_FLAG_STRESS);
6486 objspace->profile.latest_gc_info = reason;
6487 objspace->profile.total_allocated_objects_at_gc_start = total_allocated_objects(
objspace);
6488 objspace->profile.heap_used_at_gc_start = rb_darray_size(
objspace->heap_pages.sorted);
6489 objspace->profile.weak_references_count = 0;
6490 objspace->profile.retained_weak_references_count = 0;
6491 gc_prof_setup_new_record(
objspace, reason);
6492 gc_reset_malloc_info(
objspace, do_full_mark);
6496 GC_ASSERT(during_gc);
6500 if (gc_marks(
objspace, do_full_mark)) {
6506 gc_exit(
objspace, gc_enter_event_start, &lock_lev);
6514 unsigned int lock_lev;
6515 gc_enter(
objspace, gc_enter_event_rest, &lock_lev);
6517 if (RGENGC_CHECK_MODE >= 2) gc_verify_internal_consistency(
objspace);
6519 if (is_incremental_marking(
objspace)) {
6533 gc_exit(
objspace, gc_enter_event_rest, &lock_lev);
6539 unsigned int reason;
6548 if (is_full_marking(
objspace)) buff[i++] =
'F';
6549 if (is_incremental_marking(
objspace)) buff[i++] =
'I';
6553 if (is_lazy_sweeping(
objspace)) buff[i++] =
'L';
6564 static char buff[0x10];
6565 gc_current_status_fill(
objspace, buff);
6569#if PRINT_ENTER_EXIT_TICK
6571static tick_t last_exit_tick;
6572static tick_t enter_tick;
6573static int enter_count = 0;
6574static char last_gc_status[0x10];
6579 if (direction == 0) {
6581 enter_tick = tick();
6582 gc_current_status_fill(
objspace, last_gc_status);
6585 tick_t exit_tick = tick();
6586 char current_gc_status[0x10];
6587 gc_current_status_fill(
objspace, current_gc_status);
6590 fprintf(stderr,
"%"PRItick
"\t%"PRItick
"\t%s\t[%s->%s|%c]\n",
6591 enter_tick - last_exit_tick,
6592 exit_tick - enter_tick,
6594 last_gc_status, current_gc_status,
6595 (
objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ?
'+' :
'-');
6596 last_exit_tick = exit_tick;
6599 fprintf(stderr,
"%"PRItick
"\t%"PRItick
"\t%s\t[%s->%s|%c]\n",
6601 exit_tick - enter_tick,
6603 last_gc_status, current_gc_status,
6604 (
objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ?
'+' :
'-');
6617gc_enter_event_cstr(
enum gc_enter_event event)
6620 case gc_enter_event_start:
return "start";
6621 case gc_enter_event_continue:
return "continue";
6622 case gc_enter_event_rest:
return "rest";
6623 case gc_enter_event_finalizer:
return "finalizer";
6629gc_enter_count(
enum gc_enter_event event)
6632 case gc_enter_event_start: RB_DEBUG_COUNTER_INC(gc_enter_start);
break;
6633 case gc_enter_event_continue: RB_DEBUG_COUNTER_INC(gc_enter_continue);
break;
6634 case gc_enter_event_rest: RB_DEBUG_COUNTER_INC(gc_enter_rest);
break;
6635 case gc_enter_event_finalizer: RB_DEBUG_COUNTER_INC(gc_enter_finalizer);
break;
6639static bool current_process_time(
struct timespec *ts);
6644 if (!current_process_time(ts)) {
6650static unsigned long long
6655 if ((ts->tv_sec > 0 || ts->tv_nsec > 0) &&
6656 current_process_time(&end_time) &&
6657 end_time.tv_sec >= ts->tv_sec) {
6658 return (
unsigned long long)(end_time.tv_sec - ts->tv_sec) * (1000 * 1000 * 1000) +
6659 (end_time.tv_nsec - ts->tv_nsec);
6668 *lock_lev = rb_gc_vm_lock();
6671 case gc_enter_event_rest:
6674 case gc_enter_event_start:
6675 case gc_enter_event_continue:
6683 gc_enter_count(event);
6684 if (RB_UNLIKELY(during_gc != 0)) rb_bug(
"during_gc != 0");
6685 if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(
objspace);
6688 RUBY_DEBUG_LOG(
"%s (%s)",gc_enter_event_cstr(event), gc_current_status(
objspace));
6689 gc_report(1,
objspace,
"gc_enter: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(
objspace));
6690 gc_record(
objspace, 0, gc_enter_event_cstr(event));
6698 GC_ASSERT(during_gc != 0);
6702 gc_record(
objspace, 1, gc_enter_event_cstr(event));
6703 RUBY_DEBUG_LOG(
"%s (%s)", gc_enter_event_cstr(event), gc_current_status(
objspace));
6704 gc_report(1,
objspace,
"gc_exit: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(
objspace));
6707 rb_gc_vm_unlock(*lock_lev);
6711#define MEASURE_GC (objspace->flags.measure_gc)
6717 GC_ASSERT(during_gc != 0);
6720 gc_clock_start(&
objspace->profile.marking_start_time);
6727 GC_ASSERT(during_gc != 0);
6730 objspace->profile.marking_time_ns += gc_clock_end(&
objspace->profile.marking_start_time);
6737 GC_ASSERT(during_gc != 0);
6740 gc_clock_start(&
objspace->profile.sweeping_start_time);
6747 GC_ASSERT(during_gc != 0);
6750 objspace->profile.sweeping_time_ns += gc_clock_end(&
objspace->profile.sweeping_start_time);
6755gc_with_gvl(
void *ptr)
6758 return (
void *)(
VALUE)garbage_collect(oar->objspace, oar->reason);
6761int ruby_thread_has_gvl_p(
void);
6766 if (dont_gc_val()) {
6772 else if (!ruby_thread_has_gvl_p()) {
6776 oar.reason = reason;
6782 return garbage_collect(
objspace, reason);
6787gc_set_candidate_object_i(
void *vstart,
void *vend,
size_t stride,
void *data)
6792 for (; v != (
VALUE)vend; v += stride) {
6793 asan_unpoisoning_object(v) {
6799 rb_gc_prepare_heap_process_object(v);
6801 RVALUE_AGE_SET_CANDIDATE(
objspace, v);
6811rb_gc_impl_start(
void *objspace_ptr,
bool full_mark,
bool immediate_mark,
bool immediate_sweep,
bool compact)
6814 unsigned int reason = (GPR_FLAG_FULL_MARK |
6815 GPR_FLAG_IMMEDIATE_MARK |
6816 GPR_FLAG_IMMEDIATE_SWEEP |
6819 int full_marking_p = gc_config_full_mark_val;
6820 gc_config_full_mark_set(TRUE);
6824 GC_ASSERT(GC_COMPACTION_SUPPORTED);
6826 reason |= GPR_FLAG_COMPACT;
6829 if (!full_mark) reason &= ~GPR_FLAG_FULL_MARK;
6830 if (!immediate_mark) reason &= ~GPR_FLAG_IMMEDIATE_MARK;
6831 if (!immediate_sweep) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
6837 gc_config_full_mark_set(full_marking_p);
6841rb_gc_impl_prepare_heap(
void *objspace_ptr)
6845 size_t orig_total_slots = objspace_available_slots(
objspace);
6846 size_t orig_allocatable_slots =
objspace->heap_pages.allocatable_slots;
6848 rb_gc_impl_each_objects(
objspace, gc_set_candidate_object_i, objspace_ptr);
6850 double orig_max_free_slots = gc_params.heap_free_slots_max_ratio;
6852 gc_params.heap_free_slots_max_ratio = 0.0;
6853 rb_gc_impl_start(
objspace,
true,
true,
true,
true);
6854 gc_params.heap_free_slots_max_ratio = orig_max_free_slots;
6856 objspace->heap_pages.allocatable_slots = 0;
6857 heap_pages_freeable_pages =
objspace->empty_pages_count;
6858 heap_pages_free_unused_pages(objspace_ptr);
6859 GC_ASSERT(heap_pages_freeable_pages == 0);
6860 GC_ASSERT(
objspace->empty_pages_count == 0);
6861 objspace->heap_pages.allocatable_slots = orig_allocatable_slots;
6863 size_t total_slots = objspace_available_slots(
objspace);
6864 if (orig_total_slots > total_slots) {
6865 objspace->heap_pages.allocatable_slots += orig_total_slots - total_slots;
6868#if defined(HAVE_MALLOC_TRIM) && !defined(RUBY_ALTERNATIVE_MALLOC_HEADER)
6914 GC_ASSERT(st_is_member(finalizer_table, obj));
6918 GC_ASSERT(RVALUE_MARKED(
objspace, obj));
6919 GC_ASSERT(!RVALUE_PINNED(
objspace, obj));
6924 rb_bug(
"gc_is_moveable_obj: unreachable (%d)", (
int)
BUILTIN_TYPE(obj));
6931void rb_mv_generic_ivar(
VALUE src,
VALUE dst);
6941 gc_report(4,
objspace,
"Moving object: %p -> %p\n", (
void *)src, (
void *)dest);
6944 GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest));
6946 GC_ASSERT(!RVALUE_MARKING(
objspace, src));
6949 marked = RVALUE_MARKED(
objspace, src);
6950 wb_unprotected = RVALUE_WB_UNPROTECTED(
objspace, src);
6951 uncollectible = RVALUE_UNCOLLECTIBLE(
objspace, src);
6952 bool remembered = RVALUE_REMEMBERED(
objspace, src);
6953 age = RVALUE_AGE_GET(src);
6956 CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(src), src);
6957 CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(src), src);
6958 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(src), src);
6959 CLEAR_IN_BITMAP(GET_HEAP_PAGE(src)->remembered_bits, src);
6964 st_data_t srcid = (st_data_t)src,
id;
6966 gc_report(4,
objspace,
"Moving object with seen id: %p -> %p\n", (
void *)src, (
void *)dest);
6968 DURING_GC_COULD_MALLOC_REGION_START();
6970 if (!st_delete(
objspace->obj_to_id_tbl, &srcid, &
id)) {
6971 rb_bug(
"gc_move: object ID seen, but not in mapping table: %s", rb_obj_info((
VALUE)src));
6974 st_insert(
objspace->obj_to_id_tbl, (st_data_t)dest,
id);
6976 DURING_GC_COULD_MALLOC_REGION_END();
6979 GC_ASSERT(!st_lookup(
objspace->obj_to_id_tbl, (st_data_t)src, NULL));
6983 memcpy((
void *)dest, (
void *)src, MIN(src_slot_size, slot_size));
6985 if (RVALUE_OVERHEAD > 0) {
6986 void *dest_overhead = (
void *)(((uintptr_t)dest) + slot_size - RVALUE_OVERHEAD);
6987 void *src_overhead = (
void *)(((uintptr_t)src) + src_slot_size - RVALUE_OVERHEAD);
6989 memcpy(dest_overhead, src_overhead, RVALUE_OVERHEAD);
6992 memset((
void *)src, 0, src_slot_size);
6993 RVALUE_AGE_RESET(src);
6997 MARK_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
7000 CLEAR_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
7004 MARK_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
7007 CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
7010 if (wb_unprotected) {
7011 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
7014 CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
7017 if (uncollectible) {
7018 MARK_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
7021 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
7024 RVALUE_AGE_SET(dest, age);
7027 RMOVED(src)->dummy =
Qundef;
7028 RMOVED(src)->destination = dest;
7031 GET_HEAP_PAGE(src)->heap->total_freed_objects++;
7032 GET_HEAP_PAGE(dest)->heap->total_allocated_objects++;
7037#if GC_CAN_COMPILE_COMPACTION
7039compare_pinned_slots(
const void *left,
const void *right,
void *dummy)
7044 left_page = *(
struct heap_page *
const *)left;
7045 right_page = *(
struct heap_page *
const *)right;
7047 return left_page->pinned_slots - right_page->pinned_slots;
7051compare_free_slots(
const void *left,
const void *right,
void *dummy)
7056 left_page = *(
struct heap_page *
const *)left;
7057 right_page = *(
struct heap_page *
const *)right;
7059 return left_page->free_slots - right_page->free_slots;
7065 for (
int j = 0; j < HEAP_COUNT; j++) {
7068 size_t total_pages = heap->total_pages;
7070 struct heap_page *page = 0, **page_list = malloc(size);
7073 heap->free_pages = NULL;
7074 ccan_list_for_each(&heap->pages, page, page_node) {
7075 page_list[i++] = page;
7079 GC_ASSERT((
size_t)i == total_pages);
7086 ccan_list_head_init(&heap->pages);
7088 for (i = 0; i < total_pages; i++) {
7089 ccan_list_add(&heap->pages, &page_list[i]->page_node);
7090 if (page_list[i]->free_slots != 0) {
7091 heap_add_freepage(heap, page_list[i]);
7101rb_gc_impl_object_moved_p(
void *objspace_ptr,
VALUE obj)
7103 return gc_object_moved_p(objspace_ptr, obj);
7111 page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
7112 page->flags.has_remembered_objects = FALSE;
7115 for (; v != (
VALUE)vend; v += stride) {
7116 asan_unpoisoning_object(v) {
7123 if (RVALUE_WB_UNPROTECTED(
objspace, v)) {
7124 page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
7126 if (RVALUE_REMEMBERED(
objspace, v)) {
7127 page->flags.has_remembered_objects = TRUE;
7129 if (page->flags.before_sweep) {
7131 rb_gc_update_object_references(
objspace, v);
7135 rb_gc_update_object_references(
objspace, v);
7145gc_update_references_weak_table_i(
VALUE obj,
void *data)
7148 asan_unpoisoning_object(obj) {
7155gc_update_references_weak_table_replace_i(
VALUE *obj,
void *data)
7157 *obj = rb_gc_location(*obj);
7165 objspace->flags.during_reference_updating =
true;
7169 for (
int i = 0; i < HEAP_COUNT; i++) {
7170 bool should_set_mark_bits = TRUE;
7173 ccan_list_for_each(&heap->pages, page, page_node) {
7174 uintptr_t start = (uintptr_t)page->start;
7175 uintptr_t end = start + (page->total_slots * heap->slot_size);
7177 gc_ref_update((
void *)start, (
void *)end, heap->slot_size,
objspace, page);
7178 if (page == heap->sweeping_page) {
7179 should_set_mark_bits = FALSE;
7181 if (should_set_mark_bits) {
7182 gc_setup_mark_bits(page);
7186 gc_ref_update_table_values_only(
objspace->obj_to_id_tbl);
7187 if (RB_UNLIKELY(
objspace->id_to_obj_tbl)) {
7188 gc_update_table_refs(
objspace->id_to_obj_tbl);
7190 gc_update_table_refs(finalizer_table);
7192 rb_gc_update_vm_references((
void *)
objspace);
7194 for (
int table = 0; table < RB_GC_VM_WEAK_TABLE_COUNT; table++) {
7195 rb_gc_vm_weak_table_foreach(
7196 gc_update_references_weak_table_i,
7197 gc_update_references_weak_table_replace_i,
7204 objspace->flags.during_reference_updating =
false;
7207#if GC_CAN_COMPILE_COMPACTION
7209root_obj_check_moved_i(
const char *category,
VALUE obj,
void *data)
7213 if (gc_object_moved_p(
objspace, obj)) {
7214 rb_bug(
"ROOT %s points to MOVED: %p -> %s", category, (
void *)obj, rb_obj_info(rb_gc_impl_location(
objspace, obj)));
7219reachable_object_check_moved_i(
VALUE ref,
void *data)
7222 if (gc_object_moved_p(rb_gc_get_objspace(), ref)) {
7223 rb_bug(
"Object %s points to MOVED: %p -> %s", rb_obj_info(parent), (
void *)ref, rb_obj_info(rb_gc_impl_location(rb_gc_get_objspace(), ref)));
7228heap_check_moved_i(
void *vstart,
void *vend,
size_t stride,
void *data)
7233 for (; v != (
VALUE)vend; v += stride) {
7234 if (gc_object_moved_p(
objspace, v)) {
7238 asan_unpoisoning_object(v) {
7244 if (!rb_gc_impl_garbage_object_p(
objspace, v)) {
7245 rb_objspace_reachable_objects_from(v, reachable_object_check_moved_i, (
void *)v);
7257rb_gc_impl_during_gc_p(
void *objspace_ptr)
7264#if RGENGC_PROFILE >= 2
7297 default:
return "unknown";
7302gc_count_add_each_types(
VALUE hash,
const char *name,
const size_t *types)
7306 for (i=0; i<
T_MASK; i++) {
7307 const char *
type = type_name(i, 0);
7310 rb_hash_aset(hash,
ID2SYM(rb_intern(name)), result);
7315rb_gc_impl_gc_count(
void *objspace_ptr)
7325 static VALUE sym_major_by =
Qnil, sym_gc_by, sym_immediate_sweep, sym_have_finalizer, sym_state, sym_need_major_by;
7326 static VALUE sym_nofree, sym_oldgen, sym_shady, sym_force, sym_stress;
7327#if RGENGC_ESTIMATE_OLDMALLOC
7328 static VALUE sym_oldmalloc;
7330 static VALUE sym_newobj, sym_malloc, sym_method, sym_capi;
7331 static VALUE sym_none, sym_marking, sym_sweeping;
7332 static VALUE sym_weak_references_count, sym_retained_weak_references_count;
7334 VALUE major_by, need_major_by;
7335 unsigned int flags = orig_flags ? orig_flags :
objspace->profile.latest_gc_info;
7344 rb_bug(
"gc_info_decode: non-hash or symbol given");
7347 if (
NIL_P(sym_major_by)) {
7348#define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
7361#if RGENGC_ESTIMATE_OLDMALLOC
7373 S(weak_references_count);
7374 S(retained_weak_references_count);
7378#define SET(name, attr) \
7379 if (key == sym_##name) \
7381 else if (hash != Qnil) \
7382 rb_hash_aset(hash, sym_##name, (attr));
7385 (flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
7386 (flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
7387 (flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
7388 (flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
7389#if RGENGC_ESTIMATE_OLDMALLOC
7390 (flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
7393 SET(major_by, major_by);
7395 if (orig_flags == 0) {
7396 unsigned int need_major_flags = gc_needs_major_flags;
7398 (need_major_flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
7399 (need_major_flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
7400 (need_major_flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
7401 (need_major_flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
7402#if RGENGC_ESTIMATE_OLDMALLOC
7403 (need_major_flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
7406 SET(need_major_by, need_major_by);
7410 (flags & GPR_FLAG_NEWOBJ) ? sym_newobj :
7411 (flags & GPR_FLAG_MALLOC) ? sym_malloc :
7412 (flags & GPR_FLAG_METHOD) ? sym_method :
7413 (flags & GPR_FLAG_CAPI) ? sym_capi :
7414 (flags & GPR_FLAG_STRESS) ? sym_stress :
7418 SET(have_finalizer, (flags & GPR_FLAG_HAVE_FINALIZE) ?
Qtrue :
Qfalse);
7419 SET(immediate_sweep, (flags & GPR_FLAG_IMMEDIATE_SWEEP) ?
Qtrue :
Qfalse);
7421 if (orig_flags == 0) {
7422 SET(state, gc_mode(
objspace) == gc_mode_none ? sym_none :
7423 gc_mode(
objspace) == gc_mode_marking ? sym_marking : sym_sweeping);
7426 SET(weak_references_count,
LONG2FIX(
objspace->profile.weak_references_count));
7427 SET(retained_weak_references_count,
LONG2FIX(
objspace->profile.retained_weak_references_count));
7439rb_gc_impl_latest_gc_info(
void *objspace_ptr,
VALUE key)
7443 return gc_info_decode(
objspace, key, 0);
7450 gc_stat_sym_marking_time,
7451 gc_stat_sym_sweeping_time,
7452 gc_stat_sym_heap_allocated_pages,
7453 gc_stat_sym_heap_empty_pages,
7454 gc_stat_sym_heap_allocatable_slots,
7455 gc_stat_sym_heap_available_slots,
7456 gc_stat_sym_heap_live_slots,
7457 gc_stat_sym_heap_free_slots,
7458 gc_stat_sym_heap_final_slots,
7459 gc_stat_sym_heap_marked_slots,
7460 gc_stat_sym_heap_eden_pages,
7461 gc_stat_sym_total_allocated_pages,
7462 gc_stat_sym_total_freed_pages,
7463 gc_stat_sym_total_allocated_objects,
7464 gc_stat_sym_total_freed_objects,
7465 gc_stat_sym_malloc_increase_bytes,
7466 gc_stat_sym_malloc_increase_bytes_limit,
7467 gc_stat_sym_minor_gc_count,
7468 gc_stat_sym_major_gc_count,
7469 gc_stat_sym_compact_count,
7470 gc_stat_sym_read_barrier_faults,
7471 gc_stat_sym_total_moved_objects,
7472 gc_stat_sym_remembered_wb_unprotected_objects,
7473 gc_stat_sym_remembered_wb_unprotected_objects_limit,
7474 gc_stat_sym_old_objects,
7475 gc_stat_sym_old_objects_limit,
7476#if RGENGC_ESTIMATE_OLDMALLOC
7477 gc_stat_sym_oldmalloc_increase_bytes,
7478 gc_stat_sym_oldmalloc_increase_bytes_limit,
7480 gc_stat_sym_weak_references_count,
7482 gc_stat_sym_total_generated_normal_object_count,
7483 gc_stat_sym_total_generated_shady_object_count,
7484 gc_stat_sym_total_shade_operation_count,
7485 gc_stat_sym_total_promoted_count,
7486 gc_stat_sym_total_remembered_normal_object_count,
7487 gc_stat_sym_total_remembered_shady_object_count,
7492static VALUE gc_stat_symbols[gc_stat_sym_last];
7495setup_gc_stat_symbols(
void)
7497 if (gc_stat_symbols[0] == 0) {
7498#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s))
7503 S(heap_allocated_pages);
7504 S(heap_empty_pages);
7505 S(heap_allocatable_slots);
7506 S(heap_available_slots);
7509 S(heap_final_slots);
7510 S(heap_marked_slots);
7512 S(total_allocated_pages);
7513 S(total_freed_pages);
7514 S(total_allocated_objects);
7515 S(total_freed_objects);
7516 S(malloc_increase_bytes);
7517 S(malloc_increase_bytes_limit);
7521 S(read_barrier_faults);
7522 S(total_moved_objects);
7523 S(remembered_wb_unprotected_objects);
7524 S(remembered_wb_unprotected_objects_limit);
7526 S(old_objects_limit);
7527#if RGENGC_ESTIMATE_OLDMALLOC
7528 S(oldmalloc_increase_bytes);
7529 S(oldmalloc_increase_bytes_limit);
7531 S(weak_references_count);
7533 S(total_generated_normal_object_count);
7534 S(total_generated_shady_object_count);
7535 S(total_shade_operation_count);
7536 S(total_promoted_count);
7537 S(total_remembered_normal_object_count);
7538 S(total_remembered_shady_object_count);
7545ns_to_ms(uint64_t ns)
7547 return ns / (1000 * 1000);
7551rb_gc_impl_stat(
void *objspace_ptr,
VALUE hash_or_sym)
7556 setup_gc_stat_symbols();
7565 rb_bug(
"non-hash or symbol given");
7568#define SET(name, attr) \
7569 if (key == gc_stat_symbols[gc_stat_sym_##name]) \
7570 return SIZET2NUM(attr); \
7571 else if (hash != Qnil) \
7572 rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
7574 SET(count,
objspace->profile.count);
7575 SET(time, (
size_t)ns_to_ms(
objspace->profile.marking_time_ns +
objspace->profile.sweeping_time_ns));
7576 SET(marking_time, (
size_t)ns_to_ms(
objspace->profile.marking_time_ns));
7577 SET(sweeping_time, (
size_t)ns_to_ms(
objspace->profile.sweeping_time_ns));
7580 SET(heap_allocated_pages, rb_darray_size(
objspace->heap_pages.sorted));
7581 SET(heap_empty_pages,
objspace->empty_pages_count)
7582 SET(heap_allocatable_slots,
objspace->heap_pages.allocatable_slots);
7583 SET(heap_available_slots, objspace_available_slots(
objspace));
7584 SET(heap_live_slots, objspace_live_slots(
objspace));
7585 SET(heap_free_slots, objspace_free_slots(
objspace));
7586 SET(heap_final_slots, total_final_slots_count(
objspace));
7587 SET(heap_marked_slots,
objspace->marked_slots);
7588 SET(heap_eden_pages, heap_eden_total_pages(
objspace));
7589 SET(total_allocated_pages,
objspace->heap_pages.allocated_pages);
7590 SET(total_freed_pages,
objspace->heap_pages.freed_pages);
7591 SET(total_allocated_objects, total_allocated_objects(
objspace));
7592 SET(total_freed_objects, total_freed_objects(
objspace));
7593 SET(malloc_increase_bytes, malloc_increase);
7594 SET(malloc_increase_bytes_limit, malloc_limit);
7595 SET(minor_gc_count,
objspace->profile.minor_gc_count);
7596 SET(major_gc_count,
objspace->profile.major_gc_count);
7597 SET(compact_count,
objspace->profile.compact_count);
7598 SET(read_barrier_faults,
objspace->profile.read_barrier_faults);
7599 SET(total_moved_objects,
objspace->rcompactor.total_moved);
7600 SET(remembered_wb_unprotected_objects,
objspace->rgengc.uncollectible_wb_unprotected_objects);
7601 SET(remembered_wb_unprotected_objects_limit,
objspace->rgengc.uncollectible_wb_unprotected_objects_limit);
7602 SET(old_objects,
objspace->rgengc.old_objects);
7603 SET(old_objects_limit,
objspace->rgengc.old_objects_limit);
7604#if RGENGC_ESTIMATE_OLDMALLOC
7605 SET(oldmalloc_increase_bytes,
objspace->rgengc.oldmalloc_increase);
7606 SET(oldmalloc_increase_bytes_limit,
objspace->rgengc.oldmalloc_increase_limit);
7610 SET(total_generated_normal_object_count,
objspace->profile.total_generated_normal_object_count);
7611 SET(total_generated_shady_object_count,
objspace->profile.total_generated_shady_object_count);
7612 SET(total_shade_operation_count,
objspace->profile.total_shade_operation_count);
7613 SET(total_promoted_count,
objspace->profile.total_promoted_count);
7614 SET(total_remembered_normal_object_count,
objspace->profile.total_remembered_normal_object_count);
7615 SET(total_remembered_shady_object_count,
objspace->profile.total_remembered_shady_object_count);
7624#if defined(RGENGC_PROFILE) && RGENGC_PROFILE >= 2
7626 gc_count_add_each_types(hash,
"generated_normal_object_count_types",
objspace->profile.generated_normal_object_count_types);
7627 gc_count_add_each_types(hash,
"generated_shady_object_count_types",
objspace->profile.generated_shady_object_count_types);
7628 gc_count_add_each_types(hash,
"shade_operation_count_types",
objspace->profile.shade_operation_count_types);
7629 gc_count_add_each_types(hash,
"promoted_types",
objspace->profile.promoted_types);
7630 gc_count_add_each_types(hash,
"remembered_normal_object_count_types",
objspace->profile.remembered_normal_object_count_types);
7631 gc_count_add_each_types(hash,
"remembered_shady_object_count_types",
objspace->profile.remembered_shady_object_count_types);
7638enum gc_stat_heap_sym {
7639 gc_stat_heap_sym_slot_size,
7640 gc_stat_heap_sym_heap_eden_pages,
7641 gc_stat_heap_sym_heap_eden_slots,
7642 gc_stat_heap_sym_total_allocated_pages,
7643 gc_stat_heap_sym_force_major_gc_count,
7644 gc_stat_heap_sym_force_incremental_marking_finish_count,
7645 gc_stat_heap_sym_total_allocated_objects,
7646 gc_stat_heap_sym_total_freed_objects,
7647 gc_stat_heap_sym_last
7650static VALUE gc_stat_heap_symbols[gc_stat_heap_sym_last];
7653setup_gc_stat_heap_symbols(
void)
7655 if (gc_stat_heap_symbols[0] == 0) {
7656#define S(s) gc_stat_heap_symbols[gc_stat_heap_sym_##s] = ID2SYM(rb_intern_const(#s))
7660 S(total_allocated_pages);
7661 S(force_major_gc_count);
7662 S(force_incremental_marking_finish_count);
7663 S(total_allocated_objects);
7664 S(total_freed_objects);
7672#define SET(name, attr) \
7673 if (key == gc_stat_heap_symbols[gc_stat_heap_sym_##name]) \
7674 return SIZET2NUM(attr); \
7675 else if (hash != Qnil) \
7676 rb_hash_aset(hash, gc_stat_heap_symbols[gc_stat_heap_sym_##name], SIZET2NUM(attr));
7678 SET(slot_size, heap->slot_size);
7679 SET(heap_eden_pages, heap->total_pages);
7680 SET(heap_eden_slots, heap->total_slots);
7681 SET(total_allocated_pages, heap->total_allocated_pages);
7682 SET(force_major_gc_count, heap->force_major_gc_count);
7683 SET(force_incremental_marking_finish_count, heap->force_incremental_marking_finish_count);
7684 SET(total_allocated_objects, heap->total_allocated_objects);
7685 SET(total_freed_objects, heap->total_freed_objects);
7697rb_gc_impl_stat_heap(
void *objspace_ptr,
VALUE heap_name,
VALUE hash_or_sym)
7701 setup_gc_stat_heap_symbols();
7703 if (
NIL_P(heap_name)) {
7705 rb_bug(
"non-hash given");
7708 for (
int i = 0; i < HEAP_COUNT; i++) {
7711 hash = rb_hash_new();
7712 rb_hash_aset(hash_or_sym,
INT2FIX(i), hash);
7715 stat_one_heap(&heaps[i], hash,
Qnil);
7719 int heap_idx =
FIX2INT(heap_name);
7721 if (heap_idx < 0 || heap_idx >= HEAP_COUNT) {
7722 rb_raise(rb_eArgError,
"size pool index out of range");
7726 return stat_one_heap(&heaps[heap_idx],
Qnil, hash_or_sym);
7729 return stat_one_heap(&heaps[heap_idx], hash_or_sym,
Qnil);
7732 rb_bug(
"non-hash or symbol given");
7736 rb_bug(
"heap_name must be nil or an Integer");
7748#define RBOOL(v) (v ? Qtrue : Qfalse)
7752rb_gc_impl_config_get(
void *objspace_ptr)
7754#define sym(name) ID2SYM(rb_intern_const(name))
7756 VALUE hash = rb_hash_new();
7758 rb_hash_aset(hash, sym(
"rgengc_allow_full_mark"), RBOOL(gc_config_full_mark_val));
7764gc_config_set_key(st_data_t key, st_data_t value, st_data_t data)
7767 if (
rb_sym2id(key) == rb_intern(
"rgengc_allow_full_mark")) {
7769 gc_config_full_mark_set(
RTEST(value));
7775rb_gc_impl_config_set(
void *objspace_ptr,
VALUE hash)
7780 rb_raise(rb_eArgError,
"expected keyword arguments");
7783 rb_hash_stlike_foreach(hash, gc_config_set_key, (st_data_t)
objspace);
7787rb_gc_impl_stress_get(
void *objspace_ptr)
7790 return ruby_gc_stress_mode;
7794rb_gc_impl_stress_set(
void *objspace_ptr,
VALUE flag)
7803get_envparam_size(
const char *name,
size_t *default_value,
size_t lower_bound)
7805 const char *ptr = getenv(name);
7808 if (ptr != NULL && *ptr) {
7811#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
7812 val = strtoll(ptr, &end, 0);
7814 val = strtol(ptr, &end, 0);
7826 unit = 1024*1024*1024;
7830 while (*end && isspace((
unsigned char)*end)) end++;
7832 if (
RTEST(
ruby_verbose)) fprintf(stderr,
"invalid string for %s: %s\n", name, ptr);
7836 if (val < -(ssize_t)(SIZE_MAX / 2 / unit) || (ssize_t)(SIZE_MAX / 2 / unit) < val) {
7837 if (
RTEST(
ruby_verbose)) fprintf(stderr,
"%s=%s is ignored because it overflows\n", name, ptr);
7842 if (val > 0 && (
size_t)val > lower_bound) {
7844 fprintf(stderr,
"%s=%"PRIdSIZE
" (default value: %"PRIuSIZE
")\n", name, val, *default_value);
7846 *default_value = (size_t)val;
7851 fprintf(stderr,
"%s=%"PRIdSIZE
" (default value: %"PRIuSIZE
") is ignored because it must be greater than %"PRIuSIZE
".\n",
7852 name, val, *default_value, lower_bound);
7861get_envparam_double(
const char *name,
double *default_value,
double lower_bound,
double upper_bound,
int accept_zero)
7863 const char *ptr = getenv(name);
7866 if (ptr != NULL && *ptr) {
7869 if (!*ptr || *end) {
7870 if (
RTEST(
ruby_verbose)) fprintf(stderr,
"invalid string for %s: %s\n", name, ptr);
7874 if (accept_zero && val == 0.0) {
7877 else if (val <= lower_bound) {
7879 fprintf(stderr,
"%s=%f (default value: %f) is ignored because it must be greater than %f.\n",
7880 name, val, *default_value, lower_bound);
7883 else if (upper_bound != 0.0 &&
7884 val > upper_bound) {
7886 fprintf(stderr,
"%s=%f (default value: %f) is ignored because it must be lower than %f.\n",
7887 name, val, *default_value, upper_bound);
7897 if (
RTEST(
ruby_verbose)) fprintf(stderr,
"%s=%f (default value: %f)\n", name, val, *default_value);
7898 *default_value = val;
7943rb_gc_impl_set_params(
void *objspace_ptr)
7947 if (get_envparam_size(
"RUBY_GC_HEAP_FREE_SLOTS", &gc_params.heap_free_slots, 0)) {
7951 for (
int i = 0; i < HEAP_COUNT; i++) {
7952 char env_key[
sizeof(
"RUBY_GC_HEAP_" "_INIT_SLOTS") +
DECIMAL_SIZE_OF_BITS(
sizeof(
int) * CHAR_BIT)];
7953 snprintf(env_key,
sizeof(env_key),
"RUBY_GC_HEAP_%d_INIT_SLOTS", i);
7955 get_envparam_size(env_key, &gc_params.heap_init_slots[i], 0);
7958 get_envparam_double(
"RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE);
7959 get_envparam_size (
"RUBY_GC_HEAP_GROWTH_MAX_SLOTS", &gc_params.growth_max_slots, 0);
7960 get_envparam_double(
"RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio,
7962 get_envparam_double(
"RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", &gc_params.heap_free_slots_max_ratio,
7963 gc_params.heap_free_slots_min_ratio, 1.0, FALSE);
7964 get_envparam_double(
"RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", &gc_params.heap_free_slots_goal_ratio,
7965 gc_params.heap_free_slots_min_ratio, gc_params.heap_free_slots_max_ratio, TRUE);
7966 get_envparam_double(
"RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR", &gc_params.oldobject_limit_factor, 0.0, 0.0, TRUE);
7967 get_envparam_double(
"RUBY_GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO", &gc_params.uncollectible_wb_unprotected_objects_limit_ratio, 0.0, 0.0, TRUE);
7969 if (get_envparam_size(
"RUBY_GC_MALLOC_LIMIT", &gc_params.malloc_limit_min, 0)) {
7970 malloc_limit = gc_params.malloc_limit_min;
7972 get_envparam_size (
"RUBY_GC_MALLOC_LIMIT_MAX", &gc_params.malloc_limit_max, 0);
7973 if (!gc_params.malloc_limit_max) {
7974 gc_params.malloc_limit_max = SIZE_MAX;
7976 get_envparam_double(
"RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR", &gc_params.malloc_limit_growth_factor, 1.0, 0.0, FALSE);
7978#if RGENGC_ESTIMATE_OLDMALLOC
7979 if (get_envparam_size(
"RUBY_GC_OLDMALLOC_LIMIT", &gc_params.oldmalloc_limit_min, 0)) {
7980 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
7982 get_envparam_size (
"RUBY_GC_OLDMALLOC_LIMIT_MAX", &gc_params.oldmalloc_limit_max, 0);
7983 get_envparam_double(
"RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR", &gc_params.oldmalloc_limit_growth_factor, 1.0, 0.0, FALSE);
7990#ifdef HAVE_MALLOC_USABLE_SIZE
7992 hint = malloc_usable_size(ptr);
7999 MEMOP_TYPE_MALLOC = 0,
8005atomic_sub_nounderflow(
size_t *var,
size_t sub)
8007 if (sub == 0)
return;
8011 if (val < sub) sub = val;
8016#define gc_stress_full_mark_after_malloc_p() \
8017 (FIXNUM_P(ruby_gc_stress_mode) && (FIX2LONG(ruby_gc_stress_mode) & (1<<gc_stress_full_mark_after_malloc)))
8023 unsigned int reason = (GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP |
8024 GPR_FLAG_STRESS | GPR_FLAG_MALLOC);
8026 if (gc_stress_full_mark_after_malloc_p()) {
8027 reason |= GPR_FLAG_FULL_MARK;
8029 garbage_collect_with_gvl(
objspace, reason);
8034objspace_malloc_increase_report(
rb_objspace_t *
objspace,
void *mem,
size_t new_size,
size_t old_size,
enum memop_type
type)
8036 if (0) fprintf(stderr,
"increase - ptr: %p, type: %s, new_size: %"PRIdSIZE
", old_size: %"PRIdSIZE
"\n",
8038 type == MEMOP_TYPE_MALLOC ?
"malloc" :
8039 type == MEMOP_TYPE_FREE ?
"free " :
8040 type == MEMOP_TYPE_REALLOC ?
"realloc":
"error",
8041 new_size, old_size);
8046objspace_malloc_increase_body(
rb_objspace_t *
objspace,
void *mem,
size_t new_size,
size_t old_size,
enum memop_type
type)
8048 if (new_size > old_size) {
8050#if RGENGC_ESTIMATE_OLDMALLOC
8055 atomic_sub_nounderflow(&malloc_increase, old_size - new_size);
8056#if RGENGC_ESTIMATE_OLDMALLOC
8057 atomic_sub_nounderflow(&
objspace->rgengc.oldmalloc_increase, old_size - new_size);
8061 if (
type == MEMOP_TYPE_MALLOC) {
8064 if (ruby_thread_has_gvl_p() && is_lazy_sweeping(
objspace)) {
8068 garbage_collect_with_gvl(
objspace, GPR_FLAG_MALLOC);
8072#if MALLOC_ALLOCATED_SIZE
8073 if (new_size >= old_size) {
8077 size_t dec_size = old_size - new_size;
8079#if MALLOC_ALLOCATED_SIZE_CHECK
8080 size_t allocated_size =
objspace->malloc_params.allocated_size;
8081 if (allocated_size < dec_size) {
8082 rb_bug(
"objspace_malloc_increase: underflow malloc_params.allocated_size.");
8085 atomic_sub_nounderflow(&
objspace->malloc_params.allocated_size, dec_size);
8089 case MEMOP_TYPE_MALLOC:
8092 case MEMOP_TYPE_FREE:
8094 size_t allocations =
objspace->malloc_params.allocations;
8095 if (allocations > 0) {
8096 atomic_sub_nounderflow(&
objspace->malloc_params.allocations, 1);
8098#if MALLOC_ALLOCATED_SIZE_CHECK
8100 GC_ASSERT(
objspace->malloc_params.allocations > 0);
8105 case MEMOP_TYPE_REALLOC:
break;
8111#define objspace_malloc_increase(...) \
8112 for (bool malloc_increase_done = objspace_malloc_increase_report(__VA_ARGS__); \
8113 !malloc_increase_done; \
8114 malloc_increase_done = objspace_malloc_increase_body(__VA_ARGS__))
8123 if (size == 0) size = 1;
8125#if CALC_EXACT_MALLOC_SIZE
8139 return during_gc && !dont_gc_val() && !rb_gc_multi_ractor_p() && ruby_thread_has_gvl_p();
8145 size = objspace_malloc_size(
objspace, mem, size);
8146 objspace_malloc_increase(
objspace, mem, size, 0, MEMOP_TYPE_MALLOC) {}
8148#if CALC_EXACT_MALLOC_SIZE
8159#if defined(__GNUC__) && RUBY_DEBUG
8160#define RB_BUG_INSTEAD_OF_RB_MEMERROR 1
8163#ifndef RB_BUG_INSTEAD_OF_RB_MEMERROR
8164# define RB_BUG_INSTEAD_OF_RB_MEMERROR 0
8167#define GC_MEMERROR(...) \
8168 ((RB_BUG_INSTEAD_OF_RB_MEMERROR+0) ? rb_bug("" __VA_ARGS__) : (void)0)
8170#define TRY_WITH_GC(siz, expr) do { \
8171 const gc_profile_record_flag gpr = \
8172 GPR_FLAG_FULL_MARK | \
8173 GPR_FLAG_IMMEDIATE_MARK | \
8174 GPR_FLAG_IMMEDIATE_SWEEP | \
8176 objspace_malloc_gc_stress(objspace); \
8178 if (RB_LIKELY((expr))) { \
8181 else if (!garbage_collect_with_gvl(objspace, gpr)) { \
8183 GC_MEMERROR("TRY_WITH_GC: could not GC"); \
8185 else if ((expr)) { \
8189 GC_MEMERROR("TRY_WITH_GC: could not allocate:" \
8190 "%"PRIdSIZE" bytes for %s", \
8198 if (RB_UNLIKELY(malloc_during_gc_p(
objspace))) {
8201 rb_bug(
"Cannot %s during GC", msg);
8206rb_gc_impl_free(
void *objspace_ptr,
void *ptr,
size_t old_size)
8217#if CALC_EXACT_MALLOC_SIZE
8220 old_size = info->size;
8222 old_size = objspace_malloc_size(
objspace, ptr, old_size);
8224 objspace_malloc_increase(
objspace, ptr, 0, old_size, MEMOP_TYPE_FREE) {
8227 RB_DEBUG_COUNTER_INC(heap_xfree);
8232rb_gc_impl_malloc(
void *objspace_ptr,
size_t size)
8235 check_malloc_not_in_gc(
objspace,
"malloc");
8239 size = objspace_malloc_prepare(
objspace, size);
8240 TRY_WITH_GC(size, mem = malloc(size));
8241 RB_DEBUG_COUNTER_INC(heap_xmalloc);
8242 if (!mem)
return mem;
8243 return objspace_malloc_fixup(
objspace, mem, size);
8247rb_gc_impl_calloc(
void *objspace_ptr,
size_t size)
8251 if (RB_UNLIKELY(malloc_during_gc_p(
objspace))) {
8252 rb_warn(
"calloc during GC detected, this could cause crashes if it triggers another GC");
8253#if RGENGC_CHECK_MODE || RUBY_DEBUG
8254 rb_bug(
"Cannot calloc during GC");
8260 size = objspace_malloc_prepare(
objspace, size);
8261 TRY_WITH_GC(size, mem = calloc1(size));
8262 if (!mem)
return mem;
8263 return objspace_malloc_fixup(
objspace, mem, size);
8267rb_gc_impl_realloc(
void *objspace_ptr,
void *ptr,
size_t new_size,
size_t old_size)
8271 check_malloc_not_in_gc(
objspace,
"realloc");
8275 if (!ptr)
return rb_gc_impl_malloc(
objspace, new_size);
8282 if (new_size == 0) {
8283 if ((mem = rb_gc_impl_malloc(
objspace, 0)) != NULL) {
8306 rb_gc_impl_free(
objspace, ptr, old_size);
8320#if CALC_EXACT_MALLOC_SIZE
8325 old_size = info->size;
8329 old_size = objspace_malloc_size(
objspace, ptr, old_size);
8331 if (!mem)
return mem;
8332 new_size = objspace_malloc_size(
objspace, mem, new_size);
8334#if CALC_EXACT_MALLOC_SIZE
8337 info->size = new_size;
8342 objspace_malloc_increase(
objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC);
8344 RB_DEBUG_COUNTER_INC(heap_xrealloc);
8349rb_gc_impl_adjust_memory_usage(
void *objspace_ptr, ssize_t diff)
8354 objspace_malloc_increase(
objspace, 0, diff, 0, MEMOP_TYPE_REALLOC);
8356 else if (diff < 0) {
8357 objspace_malloc_increase(
objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC);
8366#define GC_PROFILE_RECORD_DEFAULT_SIZE 100
8369current_process_time(
struct timespec *ts)
8371#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
8373 static int try_clock_gettime = 1;
8374 if (try_clock_gettime && clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ts) == 0) {
8378 try_clock_gettime = 0;
8385 struct rusage usage;
8387 if (getrusage(RUSAGE_SELF, &usage) == 0) {
8388 time = usage.ru_utime;
8389 ts->tv_sec = time.tv_sec;
8390 ts->tv_nsec = (int32_t)time.tv_usec * 1000;
8398 FILETIME creation_time, exit_time, kernel_time, user_time;
8401 if (GetProcessTimes(GetCurrentProcess(),
8402 &creation_time, &exit_time, &kernel_time, &user_time) != 0) {
8403 memcpy(&ui, &user_time,
sizeof(FILETIME));
8404#define PER100NSEC (uint64_t)(1000 * 1000 * 10)
8405 ts->tv_nsec = (long)(ui.QuadPart % PER100NSEC);
8406 ts->tv_sec = (time_t)(ui.QuadPart / PER100NSEC);
8419 if (current_process_time(&ts)) {
8420 return ts.tv_sec + ts.tv_nsec * 1e-9;
8432 size_t index =
objspace->profile.next_index;
8439 objspace->profile.size = GC_PROFILE_RECORD_DEFAULT_SIZE;
8442 if (index >=
objspace->profile.size) {
8446 if (!ptr) rb_memerror();
8450 rb_bug(
"gc_profile malloc or realloc miss");
8456 record->flags = reason | (ruby_gc_stressful ? GPR_FLAG_STRESS : 0);
8457#if MALLOC_ALLOCATED_SIZE
8458 record->allocated_size = malloc_allocated_size;
8460#if GC_PROFILE_MORE_DETAIL && GC_PROFILE_DETAIL_MEMORY
8463 struct rusage usage;
8464 if (getrusage(RUSAGE_SELF, &usage) == 0) {
8465 record->maxrss = usage.ru_maxrss;
8466 record->minflt = usage.ru_minflt;
8467 record->majflt = usage.ru_majflt;
8480#if GC_PROFILE_MORE_DETAIL
8481 record->prepare_time =
objspace->profile.prepare_time;
8483 record->gc_time = 0;
8484 record->gc_invoke_time = getrusage_time();
8489elapsed_time_from(
double time)
8491 double now = getrusage_time();
8505 record->gc_time = elapsed_time_from(record->gc_invoke_time);
8506 record->gc_invoke_time -=
objspace->profile.invoke_time;
8510#ifdef BUILDING_MODULAR_GC
8511# define RUBY_DTRACE_GC_HOOK(name)
8513# define RUBY_DTRACE_GC_HOOK(name) \
8514 do {if (RUBY_DTRACE_GC_##name##_ENABLED()) RUBY_DTRACE_GC_##name();} while (0)
8520 RUBY_DTRACE_GC_HOOK(MARK_BEGIN);
8521#if GC_PROFILE_MORE_DETAIL
8523 gc_prof_record(
objspace)->gc_mark_time = getrusage_time();
8531 RUBY_DTRACE_GC_HOOK(MARK_END);
8532#if GC_PROFILE_MORE_DETAIL
8535 record->gc_mark_time = elapsed_time_from(record->gc_mark_time);
8543 RUBY_DTRACE_GC_HOOK(SWEEP_BEGIN);
8547 if (record->gc_time > 0 || GC_PROFILE_MORE_DETAIL) {
8548 objspace->profile.gc_sweep_start_time = getrusage_time();
8556 RUBY_DTRACE_GC_HOOK(SWEEP_END);
8562 if (record->gc_time > 0) {
8563 sweep_time = elapsed_time_from(
objspace->profile.gc_sweep_start_time);
8565 record->gc_time += sweep_time;
8567 else if (GC_PROFILE_MORE_DETAIL) {
8568 sweep_time = elapsed_time_from(
objspace->profile.gc_sweep_start_time);
8571#if GC_PROFILE_MORE_DETAIL
8572 record->gc_sweep_time += sweep_time;
8573 if (heap_pages_deferred_final) record->flags |= GPR_FLAG_HAVE_FINALIZE;
8575 if (heap_pages_deferred_final)
objspace->profile.latest_gc_info |= GPR_FLAG_HAVE_FINALIZE;
8582#if GC_PROFILE_MORE_DETAIL
8585 record->allocate_increase = malloc_increase;
8586 record->allocate_limit = malloc_limit;
8596 size_t live =
objspace->profile.total_allocated_objects_at_gc_start - total_freed_objects(
objspace);
8597 size_t total =
objspace->profile.heap_used_at_gc_start * HEAP_PAGE_OBJ_LIMIT;
8599#if GC_PROFILE_MORE_DETAIL
8600 record->heap_use_pages =
objspace->profile.heap_used_at_gc_start;
8601 record->heap_live_objects = live;
8602 record->heap_free_objects = total - live;
8605 record->heap_total_objects = total;
8606 record->heap_use_size = live * BASE_SLOT_SIZE;
8607 record->heap_total_size = total * BASE_SLOT_SIZE;
8623 void *p =
objspace->profile.records;
8627 objspace->profile.current_record = 0;
8683gc_profile_record_get(
VALUE _)
8686 VALUE gc_profile = rb_ary_new();
8694 for (i =0; i <
objspace->profile.next_index; i++) {
8697 prof = rb_hash_new();
8698 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_FLAGS")), gc_info_decode(
objspace, rb_hash_new(), record->flags));
8699 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_TIME")),
DBL2NUM(record->gc_time));
8700 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_INVOKE_TIME")),
DBL2NUM(record->gc_invoke_time));
8701 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HEAP_USE_SIZE")),
SIZET2NUM(record->heap_use_size));
8702 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HEAP_TOTAL_SIZE")),
SIZET2NUM(record->heap_total_size));
8703 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HEAP_TOTAL_OBJECTS")),
SIZET2NUM(record->heap_total_objects));
8704 rb_hash_aset(prof,
ID2SYM(rb_intern(
"MOVED_OBJECTS")),
SIZET2NUM(record->moved_objects));
8705 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_IS_MARKED")),
Qtrue);
8706#if GC_PROFILE_MORE_DETAIL
8707 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_MARK_TIME")),
DBL2NUM(record->gc_mark_time));
8708 rb_hash_aset(prof,
ID2SYM(rb_intern(
"GC_SWEEP_TIME")),
DBL2NUM(record->gc_sweep_time));
8709 rb_hash_aset(prof,
ID2SYM(rb_intern(
"ALLOCATE_INCREASE")),
SIZET2NUM(record->allocate_increase));
8710 rb_hash_aset(prof,
ID2SYM(rb_intern(
"ALLOCATE_LIMIT")),
SIZET2NUM(record->allocate_limit));
8711 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HEAP_USE_PAGES")),
SIZET2NUM(record->heap_use_pages));
8712 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HEAP_LIVE_OBJECTS")),
SIZET2NUM(record->heap_live_objects));
8713 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HEAP_FREE_OBJECTS")),
SIZET2NUM(record->heap_free_objects));
8715 rb_hash_aset(prof,
ID2SYM(rb_intern(
"REMOVING_OBJECTS")),
SIZET2NUM(record->removing_objects));
8716 rb_hash_aset(prof,
ID2SYM(rb_intern(
"EMPTY_OBJECTS")),
SIZET2NUM(record->empty_objects));
8718 rb_hash_aset(prof,
ID2SYM(rb_intern(
"HAVE_FINALIZE")), (record->flags & GPR_FLAG_HAVE_FINALIZE) ?
Qtrue :
Qfalse);
8721#if RGENGC_PROFILE > 0
8722 rb_hash_aset(prof,
ID2SYM(rb_intern(
"OLD_OBJECTS")),
SIZET2NUM(record->old_objects));
8723 rb_hash_aset(prof,
ID2SYM(rb_intern(
"REMEMBERED_NORMAL_OBJECTS")),
SIZET2NUM(record->remembered_normal_objects));
8724 rb_hash_aset(prof,
ID2SYM(rb_intern(
"REMEMBERED_SHADY_OBJECTS")),
SIZET2NUM(record->remembered_shady_objects));
8726 rb_ary_push(gc_profile, prof);
8732#if GC_PROFILE_MORE_DETAIL
8733#define MAJOR_REASON_MAX 0x10
8736gc_profile_dump_major_reason(
unsigned int flags,
char *buff)
8738 unsigned int reason = flags & GPR_FLAG_MAJOR_MASK;
8741 if (reason == GPR_FLAG_NONE) {
8747 if (reason & GPR_FLAG_MAJOR_BY_##x) { \
8748 buff[i++] = #x[0]; \
8749 if (i >= MAJOR_REASON_MAX) rb_bug("gc_profile_dump_major_reason: overflow"); \
8755#if RGENGC_ESTIMATE_OLDMALLOC
8770 size_t count =
objspace->profile.next_index;
8771#ifdef MAJOR_REASON_MAX
8772 char reason_str[MAJOR_REASON_MAX];
8775 if (
objspace->profile.run && count ) {
8779 append(out, rb_sprintf(
"GC %"PRIuSIZE
" invokes.\n",
objspace->profile.count));
8780 append(out,
rb_str_new_cstr(
"Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC Time(ms)\n"));
8782 for (i = 0; i < count; i++) {
8783 record = &
objspace->profile.records[i];
8784 append(out, rb_sprintf(
"%5"PRIuSIZE
" %19.3f %20"PRIuSIZE
" %20"PRIuSIZE
" %20"PRIuSIZE
" %30.20f\n",
8785 i+1, record->gc_invoke_time, record->heap_use_size,
8786 record->heap_total_size, record->heap_total_objects, record->gc_time*1000));
8789#if GC_PROFILE_MORE_DETAIL
8790 const char *str =
"\n\n" \
8792 "Prepare Time = Previously GC's rest sweep time\n"
8793 "Index Flags Allocate Inc. Allocate Limit"
8794#if CALC_EXACT_MALLOC_SIZE
8797 " Use Page Mark Time(ms) Sweep Time(ms) Prepare Time(ms) LivingObj FreeObj RemovedObj EmptyObj"
8799 " OldgenObj RemNormObj RemShadObj"
8801#if GC_PROFILE_DETAIL_MEMORY
8802 " MaxRSS(KB) MinorFLT MajorFLT"
8807 for (i = 0; i < count; i++) {
8808 record = &
objspace->profile.records[i];
8809 append(out, rb_sprintf(
"%5"PRIuSIZE
" %4s/%c/%6s%c %13"PRIuSIZE
" %15"PRIuSIZE
8810#
if CALC_EXACT_MALLOC_SIZE
8813 " %9"PRIuSIZE
" %17.12f %17.12f %17.12f %10"PRIuSIZE
" %10"PRIuSIZE
" %10"PRIuSIZE
" %10"PRIuSIZE
8815 "%10"PRIuSIZE
" %10"PRIuSIZE
" %10"PRIuSIZE
8817#
if GC_PROFILE_DETAIL_MEMORY
8823 gc_profile_dump_major_reason(record->flags, reason_str),
8824 (record->flags & GPR_FLAG_HAVE_FINALIZE) ?
'F' :
'.',
8825 (record->flags & GPR_FLAG_NEWOBJ) ?
"NEWOBJ" :
8826 (record->flags & GPR_FLAG_MALLOC) ?
"MALLOC" :
8827 (record->flags & GPR_FLAG_METHOD) ?
"METHOD" :
8828 (record->flags & GPR_FLAG_CAPI) ?
"CAPI__" :
"??????",
8829 (record->flags & GPR_FLAG_STRESS) ?
'!' :
' ',
8830 record->allocate_increase, record->allocate_limit,
8831#if CALC_EXACT_MALLOC_SIZE
8832 record->allocated_size,
8834 record->heap_use_pages,
8835 record->gc_mark_time*1000,
8836 record->gc_sweep_time*1000,
8837 record->prepare_time*1000,
8839 record->heap_live_objects,
8840 record->heap_free_objects,
8841 record->removing_objects,
8842 record->empty_objects
8845 record->old_objects,
8846 record->remembered_normal_objects,
8847 record->remembered_shady_objects
8849#if GC_PROFILE_DETAIL_MEMORY
8851 record->maxrss / 1024,
8874gc_profile_result(
VALUE _)
8891gc_profile_report(
int argc,
VALUE *argv,
VALUE self)
8896 gc_profile_dump_on(out, rb_io_write);
8909gc_profile_total_time(
VALUE self)
8916 size_t count =
objspace->profile.next_index;
8918 for (i = 0; i < count; i++) {
8919 time +=
objspace->profile.records[i].gc_time;
8933gc_profile_enable_get(
VALUE self)
8948gc_profile_enable(
VALUE _)
8952 objspace->profile.current_record = 0;
8965gc_profile_disable(
VALUE _)
8970 objspace->profile.current_record = 0;
8985gc_verify_internal_consistency_m(
VALUE dummy)
8987 gc_verify_internal_consistency(rb_gc_get_objspace());
8991#if GC_CAN_COMPILE_COMPACTION
9005 GC_ASSERT(GC_COMPACTION_SUPPORTED);
9007 ruby_enable_autocompact =
RTEST(v);
9009#if RGENGC_CHECK_MODE
9010 ruby_autocompact_compare_func = NULL;
9014 if (
id == rb_intern(
"empty")) {
9015 ruby_autocompact_compare_func = compare_free_slots;
9023# define gc_set_auto_compact rb_f_notimplement
9026#if GC_CAN_COMPILE_COMPACTION
9034gc_get_auto_compact(
VALUE _)
9039# define gc_get_auto_compact rb_f_notimplement
9042#if GC_CAN_COMPILE_COMPACTION
9068gc_compact_stats(
VALUE self)
9071 VALUE h = rb_hash_new();
9072 VALUE considered = rb_hash_new();
9073 VALUE moved = rb_hash_new();
9074 VALUE moved_up = rb_hash_new();
9075 VALUE moved_down = rb_hash_new();
9077 for (
size_t i = 0; i <
T_MASK; i++) {
9078 if (
objspace->rcompactor.considered_count_table[i]) {
9079 rb_hash_aset(considered, type_sym(i),
SIZET2NUM(
objspace->rcompactor.considered_count_table[i]));
9082 if (
objspace->rcompactor.moved_count_table[i]) {
9083 rb_hash_aset(moved, type_sym(i),
SIZET2NUM(
objspace->rcompactor.moved_count_table[i]));
9086 if (
objspace->rcompactor.moved_up_count_table[i]) {
9087 rb_hash_aset(moved_up, type_sym(i),
SIZET2NUM(
objspace->rcompactor.moved_up_count_table[i]));
9090 if (
objspace->rcompactor.moved_down_count_table[i]) {
9091 rb_hash_aset(moved_down, type_sym(i),
SIZET2NUM(
objspace->rcompactor.moved_down_count_table[i]));
9095 rb_hash_aset(h,
ID2SYM(rb_intern(
"considered")), considered);
9096 rb_hash_aset(h,
ID2SYM(rb_intern(
"moved")), moved);
9097 rb_hash_aset(h,
ID2SYM(rb_intern(
"moved_up")), moved_up);
9098 rb_hash_aset(h,
ID2SYM(rb_intern(
"moved_down")), moved_down);
9103# define gc_compact_stats rb_f_notimplement
9106#if GC_CAN_COMPILE_COMPACTION
9125gc_compact(
VALUE self)
9128 int full_marking_p = gc_config_full_mark_val;
9129 gc_config_full_mark_set(TRUE);
9132 rb_gc_impl_start(rb_gc_get_objspace(),
true,
true,
true,
true);
9133 gc_config_full_mark_set(full_marking_p);
9135 return gc_compact_stats(self);
9138# define gc_compact rb_f_notimplement
9141#if GC_CAN_COMPILE_COMPACTION
9142struct desired_compaction_pages_i_data {
9144 size_t required_slots[HEAP_COUNT];
9148desired_compaction_pages_i(
struct heap_page *page,
void *data)
9150 struct desired_compaction_pages_i_data *tdata = data;
9153 VALUE vend = vstart + (
VALUE)(page->total_slots * page->heap->slot_size);
9156 for (
VALUE v = vstart; v != vend; v += page->heap->slot_size) {
9157 asan_unpoisoning_object(v) {
9161 size_t dest_pool_idx = dest_pool - heaps;
9162 tdata->required_slots[dest_pool_idx]++;
9186gc_verify_compaction_references(
int argc,
VALUE* argv,
VALUE self)
9188 static ID keywords[3] = {0};
9190 keywords[0] = rb_intern(
"toward");
9191 keywords[1] = rb_intern(
"double_heap");
9192 keywords[2] = rb_intern(
"expand_heap");
9199 int kwarg_count =
rb_get_kwargs(options, keywords, 0, 3, arguments);
9200 bool toward_empty = kwarg_count > 0 &&
SYMBOL_P(arguments[0]) &&
SYM2ID(arguments[0]) == rb_intern(
"empty");
9201 bool expand_heap = (kwarg_count > 1 &&
RTEST(arguments[1])) || (kwarg_count > 2 &&
RTEST(arguments[2]));
9206 rb_gc_impl_start(
objspace,
true,
true,
true,
false);
9208 unsigned int lev = rb_gc_vm_lock();
9214 struct desired_compaction_pages_i_data desired_compaction = {
9216 .required_slots = {0},
9219 objspace_each_pages(
objspace, desired_compaction_pages_i, &desired_compaction, TRUE);
9222 size_t max_existing_pages = 0;
9223 for (
int i = 0; i < HEAP_COUNT; i++) {
9225 max_existing_pages = MAX(max_existing_pages, heap->total_pages);
9229 for (
int i = 0; i < HEAP_COUNT; i++) {
9232 size_t pages_to_add = 0;
9239 pages_to_add += max_existing_pages - heap->total_pages;
9244 objspace->heap_pages.allocatable_slots = desired_compaction.required_slots[i];
9245 while (
objspace->heap_pages.allocatable_slots > 0) {
9246 heap_page_allocate_and_initialize(
objspace, heap);
9254 for (; pages_to_add > 0; pages_to_add--) {
9255 heap_page_allocate_and_initialize_force(
objspace, heap);
9261 objspace->rcompactor.compare_func = compare_free_slots;
9264 rb_gc_vm_unlock(lev);
9266 rb_gc_impl_start(rb_gc_get_objspace(),
true,
true,
true,
true);
9268 rb_objspace_reachable_objects_from_root(root_obj_check_moved_i,
objspace);
9271 objspace->rcompactor.compare_func = NULL;
9273 return gc_compact_stats(self);
9276# define gc_verify_compaction_references rb_f_notimplement
9280rb_gc_impl_objspace_free(
void *objspace_ptr)
9285 rb_bug(
"lazy sweeping underway when freeing object space");
9290 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
9293 rb_darray_free_without_gc(
objspace->heap_pages.sorted);
9294 heap_pages_lomem = 0;
9295 heap_pages_himem = 0;
9297 for (
int i = 0; i < HEAP_COUNT; i++) {
9299 heap->total_pages = 0;
9300 heap->total_slots = 0;
9305 st_free_table(
objspace->id_to_obj_tbl);
9307 st_free_table(
objspace->obj_to_id_tbl);
9309 free_stack_chunks(&
objspace->mark_stack);
9310 mark_stack_free_cache(&
objspace->mark_stack);
9312 rb_darray_free_without_gc(
objspace->weak_references);
9317#if MALLOC_ALLOCATED_SIZE
9328gc_malloc_allocated_size(
VALUE self)
9344gc_malloc_allocations(
VALUE self)
9351void rb_gc_impl_before_fork(
void *objspace_ptr) { }
9352void rb_gc_impl_after_fork(
void *objspace_ptr, rb_pid_t pid) { }
9362rb_gcdebug_add_stress_to_class(
int argc,
VALUE *argv,
VALUE self)
9366 if (!stress_to_class) {
9367 set_stress_to_class(rb_ident_hash_new_with_size(argc));
9370 for (
int i = 0; i < argc; i++) {
9371 VALUE klass = argv[i];
9372 rb_hash_aset(stress_to_class, klass,
Qtrue);
9387rb_gcdebug_remove_stress_to_class(
int argc,
VALUE *argv,
VALUE self)
9391 if (stress_to_class) {
9392 for (
int i = 0; i < argc; ++i) {
9393 rb_hash_delete(stress_to_class, argv[i]);
9396 if (rb_hash_size(stress_to_class) == 0) {
9397 stress_to_class = 0;
9405rb_gc_impl_objspace_alloc(
void)
9413rb_gc_impl_objspace_init(
void *objspace_ptr)
9417 gc_config_full_mark_set(TRUE);
9420 malloc_limit = gc_params.malloc_limit_min;
9422 if (
objspace->finalize_deferred_pjob == POSTPONED_JOB_HANDLE_INVALID) {
9423 rb_bug(
"Could not preregister postponed job for GC");
9426 for (
int i = 0; i < HEAP_COUNT; i++) {
9429 heap->slot_size = (1 << i) * BASE_SLOT_SIZE;
9431 ccan_list_head_init(&heap->pages);
9434 rb_darray_make_without_gc(&
objspace->heap_pages.sorted, 0);
9435 rb_darray_make_without_gc(&
objspace->weak_references, 0);
9442#if defined(INIT_HEAP_PAGE_ALLOC_USE_MMAP)
9444 heap_page_alloc_use_mmap = INIT_HEAP_PAGE_ALLOC_USE_MMAP;
9446 objspace->next_object_id = OBJ_ID_INITIAL;
9448 objspace->obj_to_id_tbl = st_init_numtable();
9449#if RGENGC_ESTIMATE_OLDMALLOC
9450 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
9453 for (
int i = 0; i < HEAP_COUNT; i++) {
9455 gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS;
9458 init_mark_stack(&
objspace->mark_stack);
9460 objspace->profile.invoke_time = getrusage_time();
9461 finalizer_table = st_init_numtable();
9465rb_gc_impl_init(
void)
9467 VALUE gc_constants = rb_hash_new();
9468 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"DEBUG")), GC_DEBUG ?
Qtrue :
Qfalse);
9469 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"BASE_SLOT_SIZE")),
SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD));
9470 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVALUE_OVERHEAD")),
SIZET2NUM(RVALUE_OVERHEAD));
9471 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"HEAP_PAGE_OBJ_LIMIT")),
SIZET2NUM(HEAP_PAGE_OBJ_LIMIT));
9472 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"HEAP_PAGE_BITMAP_SIZE")),
SIZET2NUM(HEAP_PAGE_BITMAP_SIZE));
9473 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"HEAP_PAGE_SIZE")),
SIZET2NUM(HEAP_PAGE_SIZE));
9474 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"HEAP_COUNT")),
LONG2FIX(HEAP_COUNT));
9475 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVARGC_MAX_ALLOCATE_SIZE")),
LONG2FIX(heap_slot_size(HEAP_COUNT - 1)));
9476 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVALUE_OLD_AGE")),
LONG2FIX(RVALUE_OLD_AGE));
9477 if (RB_BUG_INSTEAD_OF_RB_MEMERROR+0) {
9478 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RB_BUG_INSTEAD_OF_RB_MEMERROR")),
Qtrue);
9482 rb_define_const(
rb_mGC,
"INTERNAL_CONSTANTS", gc_constants);
9484 if (GC_COMPACTION_SUPPORTED) {
9499 if (GC_DEBUG_STRESS_TO_CLASS) {
9507#if MALLOC_ALLOCATED_SIZE
9525 rb_define_const(
rb_mGC,
"OPTS", opts = rb_ary_new());
9526#define OPT(o) if (o) rb_ary_push(opts, rb_interned_str(#o, sizeof(#o) - 1))
9530 OPT(RGENGC_CHECK_MODE);
9531 OPT(RGENGC_PROFILE);
9532 OPT(RGENGC_ESTIMATE_OLDMALLOC);
9533 OPT(GC_PROFILE_MORE_DETAIL);
9534 OPT(GC_ENABLE_LAZY_SWEEP);
9535 OPT(CALC_EXACT_MALLOC_SIZE);
9536 OPT(MALLOC_ALLOCATED_SIZE);
9537 OPT(MALLOC_ALLOCATED_SIZE_CHECK);
9538 OPT(GC_PROFILE_DETAIL_MEMORY);
9539 OPT(GC_COMPACTION_SUPPORTED);
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
#define RUBY_ATOMIC_VALUE_CAS(var, oldval, newval)
Identical to RUBY_ATOMIC_CAS, except it expects its arguments are VALUE.
#define RUBY_ATOMIC_SIZE_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are size_t.
#define RUBY_ATOMIC_SIZE_INC(var)
Identical to RUBY_ATOMIC_INC, except it expects its argument is size_t.
#define RUBY_ATOMIC_SIZE_CAS(var, oldval, newval)
Identical to RUBY_ATOMIC_CAS, except it expects its arguments are size_t.
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
#define RUBY_ATOMIC_SIZE_ADD(var, val)
Identical to RUBY_ATOMIC_ADD, except it expects its arguments are size_t.
#define RUBY_ATOMIC_VALUE_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are VALUE.
#define RUBY_ATOMIC_SET(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except for the return type.
#define RUBY_ATOMIC_EXCHANGE(var, val)
Atomically replaces the value pointed by var with val.
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
unsigned int rb_postponed_job_handle_t
The type of a handle returned from rb_postponed_job_preregister and passed to rb_postponed_job_trigge...
void rb_postponed_job_trigger(rb_postponed_job_handle_t h)
Triggers a pre-registered job registered with rb_postponed_job_preregister, scheduling it for executi...
rb_postponed_job_handle_t rb_postponed_job_preregister(unsigned int flags, rb_postponed_job_func_t func, void *data)
Pre-registers a func in Ruby's postponed job preregistration table, returning an opaque handle which ...
#define RUBY_INTERNAL_EVENT_GC_EXIT
gc_exit() is called.
#define RUBY_INTERNAL_EVENT_GC_ENTER
gc_enter() is called.
#define RUBY_INTERNAL_EVENT_GC_END_SWEEP
GC ended sweep phase.
#define RUBY_INTERNAL_EVENT_GC_END_MARK
GC ended mark phase.
#define RUBY_INTERNAL_EVENT_OBJSPACE_MASK
Bitmask of GC events.
#define RUBY_INTERNAL_EVENT_FREEOBJ
Object swept.
#define RUBY_INTERNAL_EVENT_GC_START
GC started.
uint32_t rb_event_flag_t
Represents event(s).
#define RUBY_INTERNAL_EVENT_NEWOBJ
Object allocated.
static VALUE RB_FL_TEST(VALUE obj, VALUE flags)
Tests if the given flag(s) are set or not.
static void RB_FL_SET_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_SET().
static void RB_FL_UNSET_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_UNSET().
@ RUBY_FL_PROMOTED
Ruby objects are "generational".
VALUE rb_define_module_under(VALUE outer, const char *name)
Defines a module under the namespace of outer.
int rb_scan_args_kw(int kw_flag, int argc, const VALUE *argv, const char *fmt,...)
Identical to rb_scan_args(), except it also accepts kw_splat.
int rb_keyword_given_p(void)
Determines if the current method is given a keyword argument.
int rb_get_kwargs(VALUE keyword_hash, const ID *table, int required, int optional, VALUE *values)
Keyword argument deconstructor.
#define T_COMPLEX
Old name of RUBY_T_COMPLEX.
#define T_FILE
Old name of RUBY_T_FILE.
#define T_STRING
Old name of RUBY_T_STRING.
#define xfree
Old name of ruby_xfree.
#define T_MASK
Old name of RUBY_T_MASK.
#define Qundef
Old name of RUBY_Qundef.
#define INT2FIX
Old name of RB_INT2FIX.
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
#define T_NIL
Old name of RUBY_T_NIL.
#define T_FLOAT
Old name of RUBY_T_FLOAT.
#define T_IMEMO
Old name of RUBY_T_IMEMO.
#define ID2SYM
Old name of RB_ID2SYM.
#define T_BIGNUM
Old name of RUBY_T_BIGNUM.
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
#define T_STRUCT
Old name of RUBY_T_STRUCT.
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
#define T_FIXNUM
Old name of RUBY_T_FIXNUM.
#define SYM2ID
Old name of RB_SYM2ID.
#define T_DATA
Old name of RUBY_T_DATA.
#define FL_SEEN_OBJ_ID
Old name of RUBY_FL_SEEN_OBJ_ID.
#define FL_SHAREABLE
Old name of RUBY_FL_SHAREABLE.
#define T_NONE
Old name of RUBY_T_NONE.
#define T_NODE
Old name of RUBY_T_NODE.
#define SIZET2NUM
Old name of RB_SIZE2NUM.
#define xmalloc
Old name of ruby_xmalloc.
#define LONG2FIX
Old name of RB_INT2FIX.
#define FIX2INT
Old name of RB_FIX2INT.
#define FL_FINALIZE
Old name of RUBY_FL_FINALIZE.
#define T_MODULE
Old name of RUBY_T_MODULE.
#define T_TRUE
Old name of RUBY_T_TRUE.
#define T_RATIONAL
Old name of RUBY_T_RATIONAL.
#define T_ICLASS
Old name of RUBY_T_ICLASS.
#define T_HASH
Old name of RUBY_T_HASH.
#define ALLOC_N
Old name of RB_ALLOC_N.
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
#define FL_SET
Old name of RB_FL_SET.
#define rb_ary_new3
Old name of rb_ary_new_from_args.
#define T_FALSE
Old name of RUBY_T_FALSE.
#define ULL2NUM
Old name of RB_ULL2NUM.
#define T_UNDEF
Old name of RUBY_T_UNDEF.
#define Qtrue
Old name of RUBY_Qtrue.
#define T_ZOMBIE
Old name of RUBY_T_ZOMBIE.
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define FIX2LONG
Old name of RB_FIX2LONG.
#define T_ARRAY
Old name of RUBY_T_ARRAY.
#define T_OBJECT
Old name of RUBY_T_OBJECT.
#define NIL_P
Old name of RB_NIL_P.
#define FL_WB_PROTECTED
Old name of RUBY_FL_WB_PROTECTED.
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
#define DBL2NUM
Old name of rb_float_new.
#define T_MATCH
Old name of RUBY_T_MATCH.
#define T_CLASS
Old name of RUBY_T_CLASS.
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
#define T_MOVED
Old name of RUBY_T_MOVED.
#define FL_TEST
Old name of RB_FL_TEST.
#define FL_UNSET
Old name of RB_FL_UNSET.
#define FIXNUM_P
Old name of RB_FIXNUM_P.
#define SYMBOL_P
Old name of RB_SYMBOL_P.
#define T_REGEXP
Old name of RUBY_T_REGEXP.
VALUE rb_eRangeError
RangeError exception.
#define ruby_verbose
This variable controls whether the interpreter is in debug mode.
VALUE rb_eRuntimeError
RuntimeError exception.
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
VALUE rb_obj_hide(VALUE obj)
Make the object invisible from Ruby code.
VALUE rb_equal(VALUE lhs, VALUE rhs)
This function is an optimised version of calling #==.
VALUE rb_stdout
STDOUT constant.
#define RB_GNUC_EXTENSION_BLOCK(x)
This is expanded to the passed token for non-GCC compilers.
Routines to manipulate encodings of strings.
VALUE rb_funcall(VALUE recv, ID mid, int n,...)
Calls a method.
static bool RB_OBJ_PROMOTED_RAW(VALUE obj)
This is the implementation of RB_OBJ_PROMOTED().
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
VALUE rb_str_buf_append(VALUE dst, VALUE src)
Identical to rb_str_cat_cstr(), except it takes Ruby's string instead of C's.
VALUE rb_str_buf_new(long capa)
Allocates a "string buffer".
#define rb_str_new_cstr(str)
Identical to rb_str_new, except it assumes the passed pointer is a pointer to a C string.
const char * rb_sourcefile(void)
Resembles __FILE__.
VALUE rb_f_notimplement(int argc, const VALUE *argv, VALUE obj, VALUE marker)
Raises rb_eNotImpError.
int rb_sourceline(void)
Resembles __LINE__.
#define RB_SYM2ID
Just another name of rb_sym2id.
ID rb_sym2id(VALUE obj)
Converts an instance of rb_cSymbol into an ID.
int len
Length of the buffer.
void * rb_thread_call_with_gvl(void *(*func)(void *), void *data1)
(Re-)acquires the GVL.
#define strtod(s, e)
Just another name of ruby_strtod.
void ruby_qsort(void *, const size_t, const size_t, int(*)(const void *, const void *, void *), void *)
Reentrant implementation of quick sort.
#define DECIMAL_SIZE_OF_BITS(n)
an approximation of ceil(n * log10(2)), up to 1,048,576 (1<<20) without overflow within 32-bit calcul...
#define MEMZERO(p, type, n)
Handy macro to erase a region of memory.
VALUE type(ANYARGS)
ANYARGS-ed function type.
VALUE rb_ensure(type *q, VALUE w, type *e, VALUE r)
An equivalent of ensure clause.
#define RARRAY_LEN
Just another name of rb_array_len.
#define RARRAY_AREF(a, i)
#define RBASIC(obj)
Convenient casting macro.
#define errno
Ractor-aware version of errno.
int ruby_native_thread_p(void)
Queries if the thread which calls this function is a ruby's thread.
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
#define RTEST
This is an old name of RB_TEST.
#define _(args)
This was a transition path from K&R to ANSI.
Ruby object's base components.
VALUE flags
Per-object flags.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
ruby_value_type
C-level type of an object.
@ RUBY_T_ICLASS
Hidden classes known as IClasses.
@ RUBY_T_FIXNUM
Integers formerly known as Fixnums.
@ RUBY_T_MASK
Bitmask of ruby_value_type.
@ RUBY_T_NONE
Non-object (swept etc.)