1 #include "ruby/internal/config.h"
8 # ifdef HAVE_SYS_PRCTL_H
9 # include <sys/prctl.h>
13 #if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H)
15 # include <sys/user.h>
18 #include "internal/hash.h"
27 #include "ccan/list/list.h"
30 #include "gc/gc_impl.h"
32 #ifndef BUILDING_SHARED_GC
36 #include "debug_counter.h"
37 #include "internal/sanitizers.h"
40 #ifndef HAVE_MALLOC_USABLE_SIZE
42 # define HAVE_MALLOC_USABLE_SIZE
43 # define malloc_usable_size(a) _msize(a)
44 # elif defined HAVE_MALLOC_SIZE
45 # define HAVE_MALLOC_USABLE_SIZE
46 # define malloc_usable_size(a) malloc_size(a)
50 #ifdef HAVE_MALLOC_USABLE_SIZE
51 # ifdef RUBY_ALTERNATIVE_MALLOC_HEADER
53 # elif defined(HAVE_MALLOC_H)
55 # elif defined(HAVE_MALLOC_NP_H)
56 # include <malloc_np.h>
57 # elif defined(HAVE_MALLOC_MALLOC_H)
58 # include <malloc/malloc.h>
62 #ifdef HAVE_MALLOC_TRIM
65 # ifdef __EMSCRIPTEN__
67 # include <emscripten/emmalloc.h>
71 #ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
72 # include <mach/task.h>
73 # include <mach/mach_init.h>
74 # include <mach/mach_port.h>
78 # define VM_CHECK_MODE RUBY_DEBUG
82 #ifndef RACTOR_CHECK_MODE
83 # define RACTOR_CHECK_MODE (VM_CHECK_MODE || RUBY_DEBUG) && (SIZEOF_UINT64_T == SIZEOF_VALUE)
86 #ifndef RUBY_DEBUG_LOG
87 # define RUBY_DEBUG_LOG(...)
90 #ifndef GC_HEAP_INIT_SLOTS
91 #define GC_HEAP_INIT_SLOTS 10000
93 #ifndef GC_HEAP_FREE_SLOTS
94 #define GC_HEAP_FREE_SLOTS 4096
96 #ifndef GC_HEAP_GROWTH_FACTOR
97 #define GC_HEAP_GROWTH_FACTOR 1.8
99 #ifndef GC_HEAP_GROWTH_MAX_SLOTS
100 #define GC_HEAP_GROWTH_MAX_SLOTS 0
102 #ifndef GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO
103 # define GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO 0.01
105 #ifndef GC_HEAP_OLDOBJECT_LIMIT_FACTOR
106 #define GC_HEAP_OLDOBJECT_LIMIT_FACTOR 2.0
109 #ifndef GC_HEAP_FREE_SLOTS_MIN_RATIO
110 #define GC_HEAP_FREE_SLOTS_MIN_RATIO 0.20
112 #ifndef GC_HEAP_FREE_SLOTS_GOAL_RATIO
113 #define GC_HEAP_FREE_SLOTS_GOAL_RATIO 0.40
115 #ifndef GC_HEAP_FREE_SLOTS_MAX_RATIO
116 #define GC_HEAP_FREE_SLOTS_MAX_RATIO 0.65
119 #ifndef GC_MALLOC_LIMIT_MIN
120 #define GC_MALLOC_LIMIT_MIN (16 * 1024 * 1024 )
122 #ifndef GC_MALLOC_LIMIT_MAX
123 #define GC_MALLOC_LIMIT_MAX (32 * 1024 * 1024 )
125 #ifndef GC_MALLOC_LIMIT_GROWTH_FACTOR
126 #define GC_MALLOC_LIMIT_GROWTH_FACTOR 1.4
129 #ifndef GC_OLDMALLOC_LIMIT_MIN
130 #define GC_OLDMALLOC_LIMIT_MIN (16 * 1024 * 1024 )
132 #ifndef GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
133 #define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR 1.2
135 #ifndef GC_OLDMALLOC_LIMIT_MAX
136 #define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 )
139 #ifndef GC_CAN_COMPILE_COMPACTION
140 #if defined(__wasi__)
141 # define GC_CAN_COMPILE_COMPACTION 0
143 # define GC_CAN_COMPILE_COMPACTION 1
147 #ifndef PRINT_ENTER_EXIT_TICK
148 # define PRINT_ENTER_EXIT_TICK 0
150 #ifndef PRINT_ROOT_TICKS
151 #define PRINT_ROOT_TICKS 0
154 #define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS)
157 # define HEAP_COUNT 5
166 size_t incremental_mark_step_allocated_slots;
171 size_t heap_init_slots[HEAP_COUNT];
172 size_t heap_free_slots;
173 double growth_factor;
174 size_t growth_max_slots;
176 double heap_free_slots_min_ratio;
177 double heap_free_slots_goal_ratio;
178 double heap_free_slots_max_ratio;
179 double uncollectible_wb_unprotected_objects_limit_ratio;
180 double oldobject_limit_factor;
182 size_t malloc_limit_min;
183 size_t malloc_limit_max;
184 double malloc_limit_growth_factor;
186 size_t oldmalloc_limit_min;
187 size_t oldmalloc_limit_max;
188 double oldmalloc_limit_growth_factor;
192 { GC_HEAP_INIT_SLOTS },
194 GC_HEAP_GROWTH_FACTOR,
195 GC_HEAP_GROWTH_MAX_SLOTS,
197 GC_HEAP_FREE_SLOTS_MIN_RATIO,
198 GC_HEAP_FREE_SLOTS_GOAL_RATIO,
199 GC_HEAP_FREE_SLOTS_MAX_RATIO,
200 GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO,
201 GC_HEAP_OLDOBJECT_LIMIT_FACTOR,
205 GC_MALLOC_LIMIT_GROWTH_FACTOR,
207 GC_OLDMALLOC_LIMIT_MIN,
208 GC_OLDMALLOC_LIMIT_MAX,
209 GC_OLDMALLOC_LIMIT_GROWTH_FACTOR,
228 #define RGENGC_DEBUG -1
230 #define RGENGC_DEBUG 0
233 #if RGENGC_DEBUG < 0 && !defined(_MSC_VER)
234 # define RGENGC_DEBUG_ENABLED(level) (-(RGENGC_DEBUG) >= (level) && ruby_rgengc_debug >= (level))
235 #elif defined(HAVE_VA_ARGS_MACRO)
236 # define RGENGC_DEBUG_ENABLED(level) ((RGENGC_DEBUG) >= (level))
238 # define RGENGC_DEBUG_ENABLED(level) 0
240 int ruby_rgengc_debug;
247 #ifndef RGENGC_PROFILE
248 # define RGENGC_PROFILE 0
257 #ifndef RGENGC_ESTIMATE_OLDMALLOC
258 # define RGENGC_ESTIMATE_OLDMALLOC 1
261 #ifndef GC_PROFILE_MORE_DETAIL
262 # define GC_PROFILE_MORE_DETAIL 0
264 #ifndef GC_PROFILE_DETAIL_MEMORY
265 # define GC_PROFILE_DETAIL_MEMORY 0
267 #ifndef GC_ENABLE_LAZY_SWEEP
268 # define GC_ENABLE_LAZY_SWEEP 1
270 #ifndef CALC_EXACT_MALLOC_SIZE
271 # define CALC_EXACT_MALLOC_SIZE 0
273 #if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0
274 # ifndef MALLOC_ALLOCATED_SIZE
275 # define MALLOC_ALLOCATED_SIZE 0
278 # define MALLOC_ALLOCATED_SIZE 0
280 #ifndef MALLOC_ALLOCATED_SIZE_CHECK
281 # define MALLOC_ALLOCATED_SIZE_CHECK 0
284 #ifndef GC_DEBUG_STRESS_TO_CLASS
285 # define GC_DEBUG_STRESS_TO_CLASS RUBY_DEBUG
289 GPR_FLAG_NONE = 0x000,
291 GPR_FLAG_MAJOR_BY_NOFREE = 0x001,
292 GPR_FLAG_MAJOR_BY_OLDGEN = 0x002,
293 GPR_FLAG_MAJOR_BY_SHADY = 0x004,
294 GPR_FLAG_MAJOR_BY_FORCE = 0x008,
295 #if RGENGC_ESTIMATE_OLDMALLOC
296 GPR_FLAG_MAJOR_BY_OLDMALLOC = 0x020,
298 GPR_FLAG_MAJOR_MASK = 0x0ff,
301 GPR_FLAG_NEWOBJ = 0x100,
302 GPR_FLAG_MALLOC = 0x200,
303 GPR_FLAG_METHOD = 0x400,
304 GPR_FLAG_CAPI = 0x800,
305 GPR_FLAG_STRESS = 0x1000,
308 GPR_FLAG_IMMEDIATE_SWEEP = 0x2000,
309 GPR_FLAG_HAVE_FINALIZE = 0x4000,
310 GPR_FLAG_IMMEDIATE_MARK = 0x8000,
311 GPR_FLAG_FULL_MARK = 0x10000,
312 GPR_FLAG_COMPACT = 0x20000,
315 (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK |
316 GPR_FLAG_IMMEDIATE_SWEEP | GPR_FLAG_CAPI),
317 } gc_profile_record_flag;
323 double gc_invoke_time;
325 size_t heap_total_objects;
326 size_t heap_use_size;
327 size_t heap_total_size;
328 size_t moved_objects;
330 #if GC_PROFILE_MORE_DETAIL
332 double gc_sweep_time;
334 size_t heap_use_pages;
335 size_t heap_live_objects;
336 size_t heap_free_objects;
338 size_t allocate_increase;
339 size_t allocate_limit;
342 size_t removing_objects;
343 size_t empty_objects;
344 #if GC_PROFILE_DETAIL_MEMORY
350 #if MALLOC_ALLOCATED_SIZE
351 size_t allocated_size;
354 #if RGENGC_PROFILE > 0
356 size_t remembered_normal_objects;
357 size_t remembered_shady_objects;
365 uint32_t original_shape_id;
368 #define RMOVED(obj) ((struct RMoved *)(obj))
370 typedef uintptr_t bits_t;
372 BITS_SIZE =
sizeof(bits_t),
373 BITS_BITLENGTH = ( BITS_SIZE * CHAR_BIT )
386 #define STACK_CHUNK_SIZE 500
389 VALUE data[STACK_CHUNK_SIZE];
399 size_t unused_cache_size;
402 typedef int (*gc_compact_compare_func)(
const void *l,
const void *r,
void *d);
408 size_t total_allocated_pages;
409 size_t force_major_gc_count;
410 size_t force_incremental_marking_finish_count;
411 size_t total_allocated_objects;
412 size_t total_freed_objects;
413 size_t final_slots_count;
420 struct ccan_list_head pages;
423 uintptr_t compact_cursor_index;
432 gc_stress_no_immediate_sweep,
433 gc_stress_full_mark_after_malloc,
448 #if MALLOC_ALLOCATED_SIZE
449 size_t allocated_size;
459 unsigned int mode : 2;
460 unsigned int immediate_sweep : 1;
461 unsigned int dont_gc : 1;
462 unsigned int dont_incremental : 1;
463 unsigned int during_gc : 1;
464 unsigned int during_compacting : 1;
465 unsigned int during_reference_updating : 1;
466 unsigned int gc_stressful: 1;
467 unsigned int has_newobj_hook: 1;
468 unsigned int during_minor_gc : 1;
469 unsigned int during_incremental_marking : 1;
470 unsigned int measure_gc : 1;
474 unsigned long long next_object_id;
477 size_t empty_pages_count;
490 size_t allocated_pages;
493 size_t freeable_pages;
495 size_t allocatable_slots;
498 VALUE deferred_final;
505 unsigned int latest_gc_info;
511 #if GC_PROFILE_MORE_DETAIL
516 size_t minor_gc_count;
517 size_t major_gc_count;
518 size_t compact_count;
519 size_t read_barrier_faults;
520 #if RGENGC_PROFILE > 0
521 size_t total_generated_normal_object_count;
522 size_t total_generated_shady_object_count;
523 size_t total_shade_operation_count;
524 size_t total_promoted_count;
525 size_t total_remembered_normal_object_count;
526 size_t total_remembered_shady_object_count;
528 #if RGENGC_PROFILE >= 2
529 size_t generated_normal_object_count_types[
RUBY_T_MASK];
530 size_t generated_shady_object_count_types[
RUBY_T_MASK];
533 size_t remembered_normal_object_count_types[
RUBY_T_MASK];
534 size_t remembered_shady_object_count_types[
RUBY_T_MASK];
539 double gc_sweep_start_time;
540 size_t total_allocated_objects_at_gc_start;
541 size_t heap_used_at_gc_start;
545 unsigned long long marking_time_ns;
547 unsigned long long sweeping_time_ns;
548 struct timespec sweeping_start_time;
551 size_t weak_references_count;
552 size_t retained_weak_references_count;
555 VALUE gc_stress_mode;
560 size_t last_major_gc;
561 size_t uncollectible_wb_unprotected_objects;
562 size_t uncollectible_wb_unprotected_objects_limit;
564 size_t old_objects_limit;
566 #if RGENGC_ESTIMATE_OLDMALLOC
567 size_t oldmalloc_increase;
568 size_t oldmalloc_increase_limit;
571 #if RGENGC_CHECK_MODE >= 2
578 size_t considered_count_table[
T_MASK];
579 size_t moved_count_table[
T_MASK];
580 size_t moved_up_count_table[
T_MASK];
581 size_t moved_down_count_table[
T_MASK];
585 gc_compact_compare_func compare_func;
596 #if GC_DEBUG_STRESS_TO_CLASS
597 VALUE stress_to_class;
600 rb_darray(
VALUE *) weak_references;
603 unsigned long live_ractor_cache_count;
606 #ifndef HEAP_PAGE_ALIGN_LOG
608 #define HEAP_PAGE_ALIGN_LOG 16
611 #if RACTOR_CHECK_MODE || GC_DEBUG
612 struct rvalue_overhead {
613 # if RACTOR_CHECK_MODE
614 uint32_t _ractor_belonging_id;
623 # define RVALUE_OVERHEAD (sizeof(struct { \
625 struct rvalue_overhead overhead; \
629 size_t rb_gc_impl_obj_slot_size(
VALUE obj);
630 # define GET_RVALUE_OVERHEAD(obj) ((struct rvalue_overhead *)((uintptr_t)obj + rb_gc_impl_obj_slot_size(obj)))
632 # define RVALUE_OVERHEAD 0
635 #define BASE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD)
638 # define MAX(a, b) (((a) > (b)) ? (a) : (b))
641 # define MIN(a, b) (((a) < (b)) ? (a) : (b))
643 #define roomof(x, y) (((x) + (y) - 1) / (y))
644 #define CEILDIV(i, mod) roomof(i, mod)
646 HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG),
647 HEAP_PAGE_ALIGN_MASK = (~(~0UL << HEAP_PAGE_ALIGN_LOG)),
648 HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN,
649 HEAP_PAGE_OBJ_LIMIT = (
unsigned int)((HEAP_PAGE_SIZE -
sizeof(
struct heap_page_header)) / BASE_SLOT_SIZE),
650 HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, BASE_SLOT_SIZE), BITS_BITLENGTH),
651 HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),
653 #define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG)
654 #define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN
656 #if !defined(INCREMENTAL_MARK_STEP_ALLOCATIONS)
657 # define INCREMENTAL_MARK_STEP_ALLOCATIONS 500
660 #undef INIT_HEAP_PAGE_ALLOC_USE_MMAP
666 static const bool HEAP_PAGE_ALLOC_USE_MMAP =
false;
668 #elif defined(__wasm__)
672 static const bool HEAP_PAGE_ALLOC_USE_MMAP =
false;
674 #elif HAVE_CONST_PAGE_SIZE
676 static const bool HEAP_PAGE_ALLOC_USE_MMAP = (PAGE_SIZE <= HEAP_PAGE_SIZE);
678 #elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE)
680 static const bool HEAP_PAGE_ALLOC_USE_MMAP =
true;
682 #elif defined(PAGE_SIZE)
684 # define INIT_HEAP_PAGE_ALLOC_USE_MMAP (PAGE_SIZE <= HEAP_PAGE_SIZE)
686 #elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
688 # define INIT_HEAP_PAGE_ALLOC_USE_MMAP (sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE)
692 static const bool HEAP_PAGE_ALLOC_USE_MMAP =
false;
695 #ifdef INIT_HEAP_PAGE_ALLOC_USE_MMAP
697 # define HEAP_PAGE_ALLOC_USE_MMAP (heap_page_alloc_use_mmap != false)
699 static bool heap_page_alloc_use_mmap;
702 #define RVALUE_AGE_BIT_COUNT 2
703 #define RVALUE_AGE_BIT_MASK (((bits_t)1 << RVALUE_AGE_BIT_COUNT) - 1)
704 #define RVALUE_OLD_AGE 3
712 unsigned short slot_size;
713 unsigned short total_slots;
714 unsigned short free_slots;
715 unsigned short final_slots;
716 unsigned short pinned_slots;
718 unsigned int before_sweep : 1;
719 unsigned int has_remembered_objects : 1;
720 unsigned int has_uncollectible_wb_unprotected_objects : 1;
729 struct ccan_list_node page_node;
731 bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT];
733 bits_t mark_bits[HEAP_PAGE_BITMAP_LIMIT];
734 bits_t uncollectible_bits[HEAP_PAGE_BITMAP_LIMIT];
735 bits_t marking_bits[HEAP_PAGE_BITMAP_LIMIT];
737 bits_t remembered_bits[HEAP_PAGE_BITMAP_LIMIT];
740 bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT];
741 bits_t age_bits[HEAP_PAGE_BITMAP_LIMIT * RVALUE_AGE_BIT_COUNT];
748 asan_lock_freelist(
struct heap_page *page)
750 asan_poison_memory_region(&page->freelist,
sizeof(
struct free_list *));
757 asan_unlock_freelist(
struct heap_page *page)
759 asan_unpoison_memory_region(&page->freelist,
sizeof(
struct free_list *),
false);
765 if (page->total_slots == 0) {
766 GC_ASSERT(page->start == 0);
767 GC_ASSERT(page->slot_size == 0);
768 GC_ASSERT(page->heap == NULL);
769 GC_ASSERT(page->free_slots == 0);
770 asan_unpoisoning_memory_region(&page->freelist,
sizeof(&page->freelist)) {
771 GC_ASSERT(page->freelist == NULL);
777 GC_ASSERT(page->start != 0);
778 GC_ASSERT(page->slot_size != 0);
779 GC_ASSERT(page->heap != NULL);
785 #define GET_PAGE_BODY(x) ((struct heap_page_body *)((bits_t)(x) & ~(HEAP_PAGE_ALIGN_MASK)))
786 #define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
787 #define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)
789 #define NUM_IN_PAGE(p) (((bits_t)(p) & HEAP_PAGE_ALIGN_MASK) / BASE_SLOT_SIZE)
790 #define BITMAP_INDEX(p) (NUM_IN_PAGE(p) / BITS_BITLENGTH )
791 #define BITMAP_OFFSET(p) (NUM_IN_PAGE(p) & (BITS_BITLENGTH-1))
792 #define BITMAP_BIT(p) ((bits_t)1 << BITMAP_OFFSET(p))
795 #define MARKED_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] & BITMAP_BIT(p))
796 #define MARK_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] | BITMAP_BIT(p))
797 #define CLEAR_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] & ~BITMAP_BIT(p))
800 #define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0])
801 #define GET_HEAP_PINNED_BITS(x) (&GET_HEAP_PAGE(x)->pinned_bits[0])
802 #define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0])
803 #define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0])
804 #define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0])
806 #define GC_SWEEP_PAGES_FREEABLE_PER_STEP 3
808 #define RVALUE_AGE_BITMAP_INDEX(n) (NUM_IN_PAGE(n) / (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT))
809 #define RVALUE_AGE_BITMAP_OFFSET(n) ((NUM_IN_PAGE(n) % (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) * RVALUE_AGE_BIT_COUNT)
812 RVALUE_AGE_GET(
VALUE obj)
814 bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits;
815 return (
int)(age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] >> RVALUE_AGE_BITMAP_OFFSET(obj)) & RVALUE_AGE_BIT_MASK;
819 RVALUE_AGE_SET(
VALUE obj,
int age)
822 bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits;
824 age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] &= ~(RVALUE_AGE_BIT_MASK << (RVALUE_AGE_BITMAP_OFFSET(obj)));
826 age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] |= ((bits_t)age << RVALUE_AGE_BITMAP_OFFSET(obj));
827 if (age == RVALUE_OLD_AGE) {
835 #define malloc_limit objspace->malloc_params.limit
836 #define malloc_increase objspace->malloc_params.increase
837 #define malloc_allocated_size objspace->malloc_params.allocated_size
838 #define heap_pages_lomem objspace->heap_pages.range[0]
839 #define heap_pages_himem objspace->heap_pages.range[1]
840 #define heap_pages_freeable_pages objspace->heap_pages.freeable_pages
841 #define heap_pages_deferred_final objspace->heap_pages.deferred_final
842 #define heaps objspace->heaps
843 #define during_gc objspace->flags.during_gc
844 #define finalizing objspace->atomic_flags.finalizing
845 #define finalizer_table objspace->finalizer_table
846 #define ruby_gc_stressful objspace->flags.gc_stressful
847 #define ruby_gc_stress_mode objspace->gc_stress_mode
848 #if GC_DEBUG_STRESS_TO_CLASS
849 #define stress_to_class objspace->stress_to_class
850 #define set_stress_to_class(c) (stress_to_class = (c))
852 #define stress_to_class (objspace, 0)
853 #define set_stress_to_class(c) (objspace, (c))
857 #define dont_gc_on() (fprintf(stderr, "dont_gc_on@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 1)
858 #define dont_gc_off() (fprintf(stderr, "dont_gc_off@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 0)
859 #define dont_gc_set(b) (fprintf(stderr, "dont_gc_set(%d)@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = (int)(b))
860 #define dont_gc_val() (objspace->flags.dont_gc)
862 #define dont_gc_on() (objspace->flags.dont_gc = 1)
863 #define dont_gc_off() (objspace->flags.dont_gc = 0)
864 #define dont_gc_set(b) (objspace->flags.dont_gc = (int)(b))
865 #define dont_gc_val() (objspace->flags.dont_gc)
868 #define gc_config_full_mark_set(b) (objspace->gc_config.full_mark = (int)(b))
869 #define gc_config_full_mark_val (objspace->gc_config.full_mark)
871 #ifndef DURING_GC_COULD_MALLOC_REGION_START
872 # define DURING_GC_COULD_MALLOC_REGION_START() \
873 assert(rb_during_gc()); \
874 bool _prev_enabled = rb_gc_impl_gc_enabled_p(objspace); \
875 rb_gc_impl_gc_disable(objspace, false)
878 #ifndef DURING_GC_COULD_MALLOC_REGION_END
879 # define DURING_GC_COULD_MALLOC_REGION_END() \
880 if (_prev_enabled) rb_gc_impl_gc_enable(objspace)
883 static inline enum gc_mode
884 gc_mode_verify(
enum gc_mode mode)
886 #if RGENGC_CHECK_MODE > 0
889 case gc_mode_marking:
890 case gc_mode_sweeping:
891 case gc_mode_compacting:
894 rb_bug(
"gc_mode_verify: unreachable (%d)", (
int)mode);
903 for (
int i = 0; i < HEAP_COUNT; i++) {
904 if ((&heaps[i])->sweeping_page) {
915 for (
int i = 0; i < HEAP_COUNT; i++) {
916 count += (&heaps[i])->total_pages;
925 for (
int i = 0; i < HEAP_COUNT; i++) {
927 count += heap->total_allocated_objects;
936 for (
int i = 0; i < HEAP_COUNT; i++) {
938 count += heap->total_freed_objects;
947 for (
int i = 0; i < HEAP_COUNT; i++) {
949 count += heap->final_slots_count;
954 #define gc_mode(objspace) gc_mode_verify((enum gc_mode)(objspace)->flags.mode)
955 #define gc_mode_set(objspace, m) ((objspace)->flags.mode = (unsigned int)gc_mode_verify(m))
956 #define gc_needs_major_flags objspace->rgengc.need_major_gc
958 #define is_marking(objspace) (gc_mode(objspace) == gc_mode_marking)
959 #define is_sweeping(objspace) (gc_mode(objspace) == gc_mode_sweeping)
960 #define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE)
961 #define is_incremental_marking(objspace) ((objspace)->flags.during_incremental_marking != FALSE)
962 #define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE)
963 #define GC_INCREMENTAL_SWEEP_SLOT_COUNT 2048
964 #define GC_INCREMENTAL_SWEEP_POOL_SLOT_COUNT 1024
965 #define is_lazy_sweeping(objspace) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(objspace))
967 #if SIZEOF_LONG == SIZEOF_VOIDP
968 # define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG)
969 #elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
970 # define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \
971 ((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1))
973 # error not supported
979 void (*dfree)(
void *);
983 #define RZOMBIE(o) ((struct RZombie *)(o))
985 int ruby_disable_gc = 0;
986 int ruby_enable_autocompact = 0;
987 #if RGENGC_CHECK_MODE
988 gc_compact_compare_func ruby_autocompact_compare_func;
992 static int garbage_collect(
rb_objspace_t *,
unsigned int reason);
997 enum gc_enter_event {
998 gc_enter_event_start,
999 gc_enter_event_continue,
1000 gc_enter_event_rest,
1001 gc_enter_event_finalizer,
1004 static inline void gc_enter(
rb_objspace_t *
objspace,
enum gc_enter_event event,
unsigned int *lock_lev);
1005 static inline void gc_exit(
rb_objspace_t *
objspace,
enum gc_enter_event event,
unsigned int *lock_lev);
1020 static int gc_mark_stacked_objects_incremental(
rb_objspace_t *,
size_t count);
1023 static void gc_verify_internal_consistency(
void *objspace_ptr);
1025 static double getrusage_time(
void);
1029 static inline void gc_prof_mark_timer_start(
rb_objspace_t *);
1030 static inline void gc_prof_mark_timer_stop(
rb_objspace_t *);
1031 static inline void gc_prof_sweep_timer_start(
rb_objspace_t *);
1032 static inline void gc_prof_sweep_timer_stop(
rb_objspace_t *);
1033 static inline void gc_prof_set_malloc_info(
rb_objspace_t *);
1036 #define gc_prof_record(objspace) (objspace)->profile.current_record
1037 #define gc_prof_enabled(objspace) ((objspace)->profile.run && (objspace)->profile.current_record)
1039 #ifdef HAVE_VA_ARGS_MACRO
1040 # define gc_report(level, objspace, ...) \
1041 if (!RGENGC_DEBUG_ENABLED(level)) {} else gc_report_body(level, objspace, __VA_ARGS__)
1043 # define gc_report if (!RGENGC_DEBUG_ENABLED(0)) {} else gc_report_body
1045 PRINTF_ARGS(
static void gc_report_body(
int level,
rb_objspace_t *
objspace,
const char *fmt, ...), 3, 4);
1047 static void gc_finalize_deferred(
void *dmy);
1058 #if defined(__GNUC__) && defined(__i386__)
1059 typedef unsigned long long tick_t;
1060 #define PRItick "llu"
1061 static inline tick_t
1064 unsigned long long int x;
1065 __asm__ __volatile__ (
"rdtsc" :
"=A" (x));
1069 #elif defined(__GNUC__) && defined(__x86_64__)
1070 typedef unsigned long long tick_t;
1071 #define PRItick "llu"
1073 static __inline__ tick_t
1076 unsigned long hi, lo;
1077 __asm__ __volatile__ (
"rdtsc" :
"=a"(lo),
"=d"(hi));
1078 return ((
unsigned long long)lo)|( ((
unsigned long long)hi)<<32);
1081 #elif defined(__powerpc64__) && (GCC_VERSION_SINCE(4,8,0) || defined(__clang__))
1082 typedef unsigned long long tick_t;
1083 #define PRItick "llu"
1085 static __inline__ tick_t
1088 unsigned long long val = __builtin_ppc_get_timebase();
1095 #elif defined(__POWERPC__) && defined(__APPLE__)
1096 typedef unsigned long long tick_t;
1097 #define PRItick "llu"
1099 static __inline__ tick_t
1102 unsigned long int upper, lower, tmp;
1103 # define mftbu(r) __asm__ volatile("mftbu %0" : "=r"(r))
1104 # define mftb(r) __asm__ volatile("mftb %0" : "=r"(r))
1109 }
while (tmp != upper);
1110 return ((tick_t)upper << 32) | lower;
1113 #elif defined(__aarch64__) && defined(__GNUC__)
1114 typedef unsigned long tick_t;
1115 #define PRItick "lu"
1117 static __inline__ tick_t
1121 __asm__ __volatile__ (
"mrs %0, cntvct_el0" :
"=r" (val));
1126 #elif defined(_WIN32) && defined(_MSC_VER)
1128 typedef unsigned __int64 tick_t;
1129 #define PRItick "llu"
1131 static inline tick_t
1138 typedef clock_t tick_t;
1139 #define PRItick "llu"
1141 static inline tick_t
1148 #define MEASURE_LINE(expr) expr
1153 #define RVALUE_MARKED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj))
1154 #define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj))
1155 #define RVALUE_MARKING_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj))
1156 #define RVALUE_UNCOLLECTIBLE_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj))
1157 #define RVALUE_PINNED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), (obj))
1162 check_rvalue_consistency(
objspace, obj);
1163 return RVALUE_MARKED_BITMAP(obj) != 0;
1169 check_rvalue_consistency(
objspace, obj);
1170 return RVALUE_PINNED_BITMAP(obj) != 0;
1176 check_rvalue_consistency(
objspace, obj);
1177 return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1183 check_rvalue_consistency(
objspace, obj);
1184 return RVALUE_MARKING_BITMAP(obj) != 0;
1190 check_rvalue_consistency(
objspace, obj);
1191 return MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
1197 check_rvalue_consistency(
objspace, obj);
1198 return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1201 #define RVALUE_PAGE_WB_UNPROTECTED(page, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj))
1202 #define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj))
1203 #define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj))
1214 int lev = rb_gc_vm_lock_no_barrier();
1217 fprintf(stderr,
"check_rvalue_consistency: %p is a special const.\n", (
void *)obj);
1220 else if (!is_pointer_to_heap(
objspace, (
void *)obj)) {
1222 while (empty_page) {
1223 if ((uintptr_t)empty_page->body <= (uintptr_t)obj &&
1224 (uintptr_t)obj < (uintptr_t)empty_page->body + HEAP_PAGE_SIZE) {
1225 GC_ASSERT(heap_page_in_global_empty_pages_pool(
objspace, empty_page));
1226 fprintf(stderr,
"check_rvalue_consistency: %p is in an empty page (%p).\n",
1227 (
void *)obj, (
void *)empty_page);
1232 fprintf(stderr,
"check_rvalue_consistency: %p is not a Ruby object.\n", (
void *)obj);
1238 const int wb_unprotected_bit = RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1239 const int uncollectible_bit = RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1240 const int mark_bit = RVALUE_MARKED_BITMAP(obj) != 0;
1241 const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0;
1242 const int remembered_bit = MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
1243 const int age = RVALUE_AGE_GET((
VALUE)obj);
1245 if (heap_page_in_global_empty_pages_pool(
objspace, GET_HEAP_PAGE(obj))) {
1246 fprintf(stderr,
"check_rvalue_consistency: %s is in tomb page.\n", rb_obj_info(obj));
1250 fprintf(stderr,
"check_rvalue_consistency: %s is T_NONE.\n", rb_obj_info(obj));
1254 fprintf(stderr,
"check_rvalue_consistency: %s is T_ZOMBIE.\n", rb_obj_info(obj));
1259 rb_obj_memsize_of((
VALUE)obj);
1266 if (age > 0 && wb_unprotected_bit) {
1267 fprintf(stderr,
"check_rvalue_consistency: %s is not WB protected, but age is %d > 0.\n", rb_obj_info(obj), age);
1271 if (!is_marking(
objspace) && uncollectible_bit && !mark_bit) {
1272 fprintf(stderr,
"check_rvalue_consistency: %s is uncollectible, but is not marked while !gc.\n", rb_obj_info(obj));
1277 if (uncollectible_bit && age != RVALUE_OLD_AGE && !wb_unprotected_bit) {
1278 fprintf(stderr,
"check_rvalue_consistency: %s is uncollectible, but not old (age: %d) and not WB unprotected.\n",
1279 rb_obj_info(obj), age);
1282 if (remembered_bit && age != RVALUE_OLD_AGE) {
1283 fprintf(stderr,
"check_rvalue_consistency: %s is remembered, but not old (age: %d).\n",
1284 rb_obj_info(obj), age);
1296 if (is_incremental_marking(
objspace) && marking_bit) {
1297 if (!is_marking(
objspace) && !mark_bit) {
1298 fprintf(stderr,
"check_rvalue_consistency: %s is marking, but not marked.\n", rb_obj_info(obj));
1304 rb_gc_vm_unlock_no_barrier(lev);
1306 if (err > 0 && terminate) {
1307 rb_bug(
"check_rvalue_consistency_force: there is %d errors.", err);
1312 #if RGENGC_CHECK_MODE == 0
1322 check_rvalue_consistency_force(
objspace, obj, TRUE);
1335 asan_unpoisoning_object(obj) {
1346 check_rvalue_consistency(
objspace, obj);
1355 MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
1358 #if RGENGC_PROFILE >= 2
1359 objspace->profile.total_promoted_count++;
1367 RB_DEBUG_COUNTER_INC(obj_promote);
1368 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(
objspace, GET_HEAP_PAGE(obj), obj);
1375 int age = RVALUE_AGE_GET((
VALUE)obj);
1377 if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) {
1378 rb_bug(
"RVALUE_AGE_INC: can not increment age of OLD object %s.", rb_obj_info(obj));
1382 RVALUE_AGE_SET(obj, age);
1384 if (age == RVALUE_OLD_AGE) {
1385 RVALUE_OLD_UNCOLLECTIBLE_SET(
objspace, obj);
1388 check_rvalue_consistency(
objspace, obj);
1394 check_rvalue_consistency(
objspace, obj);
1395 GC_ASSERT(!RVALUE_OLD_P(
objspace, obj));
1396 RVALUE_AGE_SET(obj, RVALUE_OLD_AGE - 1);
1397 check_rvalue_consistency(
objspace, obj);
1401 RVALUE_AGE_RESET(
VALUE obj)
1403 RVALUE_AGE_SET(obj, 0);
1409 check_rvalue_consistency(
objspace, obj);
1410 GC_ASSERT(RVALUE_OLD_P(
objspace, obj));
1412 if (!is_incremental_marking(
objspace) && RVALUE_REMEMBERED(
objspace, obj)) {
1413 CLEAR_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj);
1416 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
1417 RVALUE_AGE_RESET(obj);
1419 if (RVALUE_MARKED(
objspace, obj)) {
1423 check_rvalue_consistency(
objspace, obj);
1435 return !RVALUE_MARKED(
objspace, obj);
1439 rb_gc_impl_gc_enabled_p(
void *objspace_ptr)
1442 return !dont_gc_val();
1446 rb_gc_impl_gc_enable(
void *objspace_ptr)
1454 rb_gc_impl_gc_disable(
void *objspace_ptr,
bool finish_current_gc)
1458 if (finish_current_gc) {
1469 static inline void *
1472 return calloc(1, n);
1476 rb_gc_impl_set_event_hook(
void *objspace_ptr,
const rb_event_flag_t event)
1484 rb_gc_impl_get_total_time(
void *objspace_ptr)
1488 unsigned long long marking_time =
objspace->profile.marking_time_ns;
1489 unsigned long long sweeping_time =
objspace->profile.sweeping_time_ns;
1491 return marking_time + sweeping_time;
1495 rb_gc_impl_set_measure_total_time(
void *objspace_ptr,
VALUE flag)
1503 rb_gc_impl_get_measure_total_time(
void *objspace_ptr)
1513 size_t heap_idx = heap - heaps;
1514 return gc_params.heap_init_slots[heap_idx];
1518 object_id_cmp(st_data_t x, st_data_t y)
1529 object_id_hash(st_data_t n)
1534 #define OBJ_ID_INCREMENT (RUBY_IMMEDIATE_MASK + 1)
1535 #define OBJ_ID_INITIAL (OBJ_ID_INCREMENT)
1537 static const struct st_hash_type object_id_hash_type = {
1544 rb_gc_impl_garbage_object_p(
void *objspace_ptr,
VALUE ptr)
1550 asan_unpoisoning_object(
ptr) {
1562 if (dead)
return true;
1563 return is_lazy_sweeping(
objspace) && GET_HEAP_PAGE(
ptr)->flags.before_sweep &&
1568 rb_gc_impl_object_id_to_ref(
void *objspace_ptr,
VALUE object_id)
1573 if (st_lookup(
objspace->id_to_obj_tbl, object_id, &obj) &&
1574 !rb_gc_impl_garbage_object_p(
objspace, obj)) {
1587 rb_gc_impl_object_id(
void *objspace_ptr,
VALUE obj)
1592 unsigned int lev = rb_gc_vm_lock();
1595 if (st_lookup(
objspace->obj_to_id_tbl, (st_data_t)obj, &val)) {
1599 rb_bug(
"rb_gc_impl_object_id: FL_SEEN_OBJ_ID flag set but not found in table");
1603 GC_ASSERT(!st_lookup(
objspace->obj_to_id_tbl, (st_data_t)obj, NULL));
1606 objspace->next_object_id += OBJ_ID_INCREMENT;
1608 st_insert(
objspace->obj_to_id_tbl, (st_data_t)obj, (st_data_t)
id);
1609 st_insert(
objspace->id_to_obj_tbl, (st_data_t)
id, (st_data_t)obj);
1612 rb_gc_vm_unlock(lev);
1624 asan_unpoison_object(obj,
false);
1626 asan_unlock_freelist(page);
1630 slot->next = page->freelist;
1631 page->freelist = slot;
1632 asan_lock_freelist(page);
1634 RVALUE_AGE_RESET(obj);
1636 if (RGENGC_CHECK_MODE &&
1638 !(page->start <= (uintptr_t)obj &&
1639 (uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) &&
1640 obj % BASE_SLOT_SIZE == 0)) {
1641 rb_bug(
"heap_page_add_freeobj: %p is not rvalue.", (
void *)obj);
1644 asan_poison_object(obj);
1645 gc_report(3,
objspace,
"heap_page_add_freeobj: add %p to freelist\n", (
void *)obj);
1650 rb_heap_t *heap,
size_t free_slots,
size_t total_slots)
1652 double goal_ratio = gc_params.heap_free_slots_goal_ratio;
1653 size_t target_total_slots;
1655 if (goal_ratio == 0.0) {
1656 target_total_slots = (size_t)(total_slots * gc_params.growth_factor);
1658 else if (total_slots == 0) {
1659 target_total_slots = minimum_slots_for_heap(
objspace, heap);
1665 double f = (double)(total_slots - free_slots) / ((1 - goal_ratio) * total_slots);
1667 if (f > gc_params.growth_factor) f = gc_params.growth_factor;
1668 if (f < 1.0) f = 1.1;
1670 target_total_slots = (size_t)(f * total_slots);
1674 "free_slots(%8"PRIuSIZE
")/total_slots(%8"PRIuSIZE
")=%1.2f,"
1675 " G(%1.2f), f(%1.2f),"
1676 " total_slots(%8"PRIuSIZE
") => target_total_slots(%8"PRIuSIZE
")\n",
1677 free_slots, total_slots, free_slots/(
double)total_slots,
1678 goal_ratio, f, total_slots, target_total_slots);
1682 if (gc_params.growth_max_slots > 0) {
1683 size_t max_total_slots = (size_t)(total_slots + gc_params.growth_max_slots);
1684 if (target_total_slots > max_total_slots) target_total_slots = max_total_slots;
1687 size_t extend_slot_count = target_total_slots - total_slots;
1689 if (extend_slot_count == 0) extend_slot_count = 1;
1691 objspace->heap_pages.allocatable_slots += extend_slot_count;
1697 asan_unlock_freelist(page);
1698 GC_ASSERT(page->free_slots != 0);
1699 GC_ASSERT(page->freelist != NULL);
1701 page->free_next = heap->free_pages;
1702 heap->free_pages = page;
1704 RUBY_DEBUG_LOG(
"page:%p freelist:%p", (
void *)page, (
void *)page->freelist);
1706 asan_lock_freelist(page);
1712 asan_unlock_freelist(page);
1713 GC_ASSERT(page->free_slots != 0);
1714 GC_ASSERT(page->freelist != NULL);
1716 page->free_next = heap->pooled_pages;
1717 heap->pooled_pages = page;
1718 objspace->rincgc.pooled_slots += page->free_slots;
1720 asan_lock_freelist(page);
1726 ccan_list_del(&page->page_node);
1727 heap->total_pages--;
1728 heap->total_slots -= page->total_slots;
1732 gc_aligned_free(
void *
ptr,
size_t size)
1734 #if defined __MINGW32__
1735 __mingw_aligned_free(
ptr);
1736 #elif defined _WIN32
1738 #elif defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN)
1741 free(((
void**)
ptr)[-1]);
1748 GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
1750 if (HEAP_PAGE_ALLOC_USE_MMAP) {
1752 GC_ASSERT(HEAP_PAGE_SIZE % sysconf(_SC_PAGE_SIZE) == 0);
1753 if (munmap(page_body, HEAP_PAGE_SIZE)) {
1754 rb_bug(
"heap_page_body_free: munmap failed");
1759 gc_aligned_free(page_body, HEAP_PAGE_SIZE);
1766 objspace->heap_pages.freed_pages++;
1767 heap_page_body_free(page->body);
1774 size_t pages_to_keep_count =
1776 CEILDIV(
objspace->heap_pages.allocatable_slots, HEAP_PAGE_OBJ_LIMIT) *
1778 (1 << (HEAP_COUNT / 2));
1780 if (
objspace->empty_pages != NULL &&
objspace->empty_pages_count > pages_to_keep_count) {
1781 GC_ASSERT(
objspace->empty_pages_count > 0);
1786 for (i = j = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
1789 if (heap_page_in_global_empty_pages_pool(
objspace, page) && pages_to_keep_count == 0) {
1793 if (heap_page_in_global_empty_pages_pool(
objspace, page) && pages_to_keep_count > 0) {
1794 page->free_next =
objspace->empty_pages;
1797 pages_to_keep_count--;
1801 rb_darray_set(
objspace->heap_pages.sorted, j, page);
1807 rb_darray_pop(
objspace->heap_pages.sorted, i - j);
1808 GC_ASSERT(rb_darray_size(
objspace->heap_pages.sorted) == j);
1811 uintptr_t himem = (uintptr_t)hipage->body + HEAP_PAGE_SIZE;
1812 GC_ASSERT(himem <= heap_pages_himem);
1813 heap_pages_himem = himem;
1816 uintptr_t lomem = (uintptr_t)lopage->body +
sizeof(
struct heap_page_header);
1817 GC_ASSERT(lomem >= heap_pages_lomem);
1818 heap_pages_lomem = lomem;
1823 gc_aligned_malloc(
size_t alignment,
size_t size)
1826 GC_ASSERT(((alignment - 1) & alignment) == 0);
1827 GC_ASSERT(alignment %
sizeof(
void*) == 0);
1831 #if defined __MINGW32__
1832 res = __mingw_aligned_malloc(size, alignment);
1833 #elif defined _WIN32
1834 void *_aligned_malloc(
size_t,
size_t);
1835 res = _aligned_malloc(size, alignment);
1836 #elif defined(HAVE_POSIX_MEMALIGN)
1837 if (posix_memalign(&res, alignment, size) != 0) {
1840 #elif defined(HAVE_MEMALIGN)
1841 res = memalign(alignment, size);
1844 res = malloc(alignment + size +
sizeof(
void*));
1845 aligned = (
char*)res + alignment +
sizeof(
void*);
1846 aligned -= ((
VALUE)aligned & (alignment - 1));
1847 ((
void**)aligned)[-1] = res;
1848 res = (
void*)aligned;
1851 GC_ASSERT((uintptr_t)res % alignment == 0);
1857 heap_page_body_allocate(
void)
1861 if (HEAP_PAGE_ALLOC_USE_MMAP) {
1863 GC_ASSERT(HEAP_PAGE_ALIGN % sysconf(_SC_PAGE_SIZE) == 0);
1865 size_t mmap_size = HEAP_PAGE_ALIGN + HEAP_PAGE_SIZE;
1866 char *
ptr = mmap(NULL, mmap_size,
1867 PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1868 if (
ptr == MAP_FAILED) {
1877 #if defined(HAVE_SYS_PRCTL_H) && defined(PR_SET_VMA) && defined(PR_SET_VMA_ANON_NAME)
1878 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME,
ptr, mmap_size,
"Ruby:GC:default:heap_page_body_allocate");
1882 char *aligned =
ptr + HEAP_PAGE_ALIGN;
1883 aligned -= ((
VALUE)aligned & (HEAP_PAGE_ALIGN - 1));
1884 GC_ASSERT(aligned >
ptr);
1885 GC_ASSERT(aligned <=
ptr + HEAP_PAGE_ALIGN);
1887 size_t start_out_of_range_size = aligned -
ptr;
1888 GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
1889 if (start_out_of_range_size > 0) {
1890 if (munmap(
ptr, start_out_of_range_size)) {
1891 rb_bug(
"heap_page_body_allocate: munmap failed for start");
1895 size_t end_out_of_range_size = HEAP_PAGE_ALIGN - start_out_of_range_size;
1896 GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
1897 if (end_out_of_range_size > 0) {
1898 if (munmap(aligned + HEAP_PAGE_SIZE, end_out_of_range_size)) {
1899 rb_bug(
"heap_page_body_allocate: munmap failed for end");
1907 page_body = gc_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
1910 GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
1919 if (
objspace->empty_pages != NULL) {
1920 GC_ASSERT(
objspace->empty_pages_count > 0);
1923 objspace->empty_pages = page->free_next;
1933 if (page_body == 0) {
1939 heap_page_body_free(page_body);
1944 uintptr_t end = (uintptr_t)page_body + HEAP_PAGE_SIZE;
1947 size_t hi = rb_darray_size(
objspace->heap_pages.sorted);
1951 size_t mid = (lo + hi) / 2;
1952 mid_page = rb_darray_get(
objspace->heap_pages.sorted, mid);
1953 if ((uintptr_t)mid_page->start < start) {
1956 else if ((uintptr_t)mid_page->start > start) {
1960 rb_bug(
"same heap page is allocated: %p at %"PRIuVALUE, (
void *)page_body, (
VALUE)mid);
1964 rb_darray_insert(&
objspace->heap_pages.sorted, hi, page);
1966 if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
1967 if (heap_pages_himem < end) heap_pages_himem = end;
1969 page->body = page_body;
1970 page_body->header.page = page;
1972 objspace->heap_pages.allocated_pages++;
1981 GC_ASSERT(!heap->sweeping_page);
1982 GC_ASSERT(heap_page_in_global_empty_pages_pool(
objspace, page));
1986 if (start % BASE_SLOT_SIZE != 0) {
1987 int delta = BASE_SLOT_SIZE - (start % BASE_SLOT_SIZE);
1988 start = start + delta;
1989 GC_ASSERT(NUM_IN_PAGE(start) == 0 || NUM_IN_PAGE(start) == 1);
1995 if (NUM_IN_PAGE(start) == 1) {
1996 start += heap->slot_size - BASE_SLOT_SIZE;
1999 GC_ASSERT(NUM_IN_PAGE(start) * BASE_SLOT_SIZE % heap->slot_size == 0);
2002 int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size);
2004 page->start = start;
2005 page->total_slots = slot_count;
2006 page->slot_size = heap->slot_size;
2009 asan_unlock_freelist(page);
2010 page->freelist = NULL;
2011 asan_unpoison_memory_region(page->body, HEAP_PAGE_SIZE,
false);
2012 for (
VALUE p = (
VALUE)start; p < start + (slot_count * heap->slot_size); p += heap->slot_size) {
2013 heap_page_add_freeobj(
objspace, page, p);
2015 asan_lock_freelist(page);
2017 page->free_slots = slot_count;
2019 heap->total_allocated_pages++;
2021 ccan_list_add_tail(&heap->pages, &page->page_node);
2022 heap->total_pages++;
2023 heap->total_slots += page->total_slots;
2029 if (
objspace->heap_pages.allocatable_slots > 0) {
2030 gc_report(1,
objspace,
"heap_page_allocate_and_initialize: rb_darray_size(objspace->heap_pages.sorted): %"PRIdSIZE
", "
2031 "allocatable_slots: %"PRIdSIZE
", heap->total_pages: %"PRIdSIZE
"\n",
2032 rb_darray_size(
objspace->heap_pages.sorted),
objspace->heap_pages.allocatable_slots, heap->total_pages);
2036 page = heap_page_allocate(
objspace);
2038 heap_add_page(
objspace, heap, page);
2039 heap_add_freepage(heap, page);
2041 if (
objspace->heap_pages.allocatable_slots > (
size_t)page->total_slots) {
2042 objspace->heap_pages.allocatable_slots -= page->total_slots;
2045 objspace->heap_pages.allocatable_slots = 0;
2057 size_t prev_allocatable_slots =
objspace->heap_pages.allocatable_slots;
2059 objspace->heap_pages.allocatable_slots = 1;
2060 heap_page_allocate_and_initialize(
objspace, heap);
2061 GC_ASSERT(heap->free_pages != NULL);
2062 objspace->heap_pages.allocatable_slots = prev_allocatable_slots;
2068 unsigned int lock_lev;
2069 gc_enter(
objspace, gc_enter_event_continue, &lock_lev);
2072 if (is_incremental_marking(
objspace)) {
2073 if (gc_marks_continue(
objspace, heap)) {
2080 if (heap->free_pages == NULL && is_lazy_sweeping(
objspace)) {
2084 gc_exit(
objspace, gc_enter_event_continue, &lock_lev);
2090 GC_ASSERT(heap->free_pages == NULL);
2092 if (heap->total_slots < gc_params.heap_init_slots[heap - heaps] &&
2093 heap->sweeping_page == NULL) {
2094 heap_page_allocate_and_initialize_force(
objspace, heap);
2095 GC_ASSERT(heap->free_pages != NULL);
2102 if (heap->free_pages == NULL) {
2103 heap_page_allocate_and_initialize(
objspace, heap);
2108 if (heap->free_pages == NULL) {
2109 if (gc_start(
objspace, GPR_FLAG_NEWOBJ) == FALSE) {
2113 if (
objspace->heap_pages.allocatable_slots == 0 && !gc_config_full_mark_val) {
2114 heap_allocatable_slots_expand(
objspace, heap,
2115 heap->freed_slots + heap->empty_slots,
2117 GC_ASSERT(
objspace->heap_pages.allocatable_slots > 0);
2125 if (heap->free_pages == NULL && !heap_page_allocate_and_initialize(
objspace, heap)) {
2126 if (gc_needs_major_flags == GPR_FLAG_NONE) {
2127 rb_bug(
"cannot create a new page after GC");
2130 if (gc_start(
objspace, GPR_FLAG_NEWOBJ) == FALSE) {
2137 if (heap->free_pages == NULL &&
2138 !heap_page_allocate_and_initialize(
objspace, heap)) {
2139 rb_bug(
"cannot create a new page after major GC");
2147 GC_ASSERT(heap->free_pages != NULL);
2161 static inline const char*
2162 rb_gc_impl_source_location_cstr(
int *
ptr)
2181 #if !__has_feature(memory_sanitizer)
2185 RBASIC(obj)->flags = flags;
2190 RVALUE_AGE_SET_CANDIDATE(
objspace, obj);
2193 #if RACTOR_CHECK_MODE
2194 void rb_ractor_setup_belonging(
VALUE obj);
2195 rb_ractor_setup_belonging(obj);
2198 #if RGENGC_CHECK_MODE
2199 newobj_fill(obj, 0, 0, 0);
2201 int lev = rb_gc_vm_lock_no_barrier();
2203 check_rvalue_consistency(
objspace, obj);
2205 GC_ASSERT(RVALUE_MARKED(
objspace, obj) == FALSE);
2206 GC_ASSERT(RVALUE_MARKING(
objspace, obj) == FALSE);
2207 GC_ASSERT(RVALUE_OLD_P(
objspace, obj) == FALSE);
2208 GC_ASSERT(RVALUE_WB_UNPROTECTED(
objspace, obj) == FALSE);
2210 if (RVALUE_REMEMBERED(
objspace, obj))
rb_bug(
"newobj: %s is remembered.", rb_obj_info(obj));
2212 rb_gc_vm_unlock_no_barrier(lev);
2216 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
2221 objspace->profile.total_generated_normal_object_count++;
2222 #if RGENGC_PROFILE >= 2
2227 objspace->profile.total_generated_shady_object_count++;
2228 #if RGENGC_PROFILE >= 2
2235 GET_RVALUE_OVERHEAD(obj)->file = rb_gc_impl_source_location_cstr(&GET_RVALUE_OVERHEAD(obj)->line);
2239 gc_report(5,
objspace,
"newobj: %s\n", rb_obj_info(obj));
2241 RUBY_DEBUG_LOG(
"obj:%p (%s)", (
void *)obj, rb_obj_info(obj));
2246 rb_gc_impl_obj_slot_size(
VALUE obj)
2248 return GET_HEAP_PAGE(obj)->slot_size - RVALUE_OVERHEAD;
2251 static inline size_t
2252 heap_slot_size(
unsigned char pool_id)
2254 GC_ASSERT(pool_id < HEAP_COUNT);
2256 size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE;
2258 #if RGENGC_CHECK_MODE
2260 GC_ASSERT(heaps[pool_id].slot_size == (
short)slot_size);
2263 slot_size -= RVALUE_OVERHEAD;
2269 rb_gc_impl_size_allocatable_p(
size_t size)
2271 return size <= heap_slot_size(HEAP_COUNT - 1);
2279 struct free_slot *p = heap_cache->freelist;
2283 if (cache->incremental_mark_step_allocated_slots >= INCREMENTAL_MARK_STEP_ALLOCATIONS) {
2288 cache->incremental_mark_step_allocated_slots++;
2294 MAYBE_UNUSED(
const size_t) stride = heap_slot_size(heap_idx);
2295 heap_cache->freelist = p->next;
2296 asan_unpoison_memory_region(p, stride,
true);
2297 #if RGENGC_CHECK_MODE
2298 GC_ASSERT(rb_gc_impl_obj_slot_size(obj) == stride);
2300 MEMZERO((
char *)obj,
char, stride);
2314 if (heap->free_pages == NULL) {
2318 page = heap->free_pages;
2319 heap->free_pages = page->free_next;
2321 GC_ASSERT(page->free_slots != 0);
2323 asan_unlock_freelist(page);
2332 gc_report(3,
objspace,
"ractor_set_cache: Using page %p\n", (
void *)page->body);
2336 GC_ASSERT(heap_cache->freelist == NULL);
2337 GC_ASSERT(page->free_slots != 0);
2338 GC_ASSERT(page->freelist != NULL);
2340 heap_cache->using_page = page;
2341 heap_cache->freelist = page->freelist;
2342 page->free_slots = 0;
2343 page->freelist = NULL;
2345 asan_unpoison_object((
VALUE)heap_cache->freelist,
false);
2347 asan_poison_object((
VALUE)heap_cache->freelist);
2350 static inline size_t
2351 heap_idx_for_size(
size_t size)
2353 size += RVALUE_OVERHEAD;
2355 size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE);
2358 size_t heap_idx = 64 - nlz_int64(slot_count - 1);
2360 if (heap_idx >= HEAP_COUNT) {
2361 rb_bug(
"heap_idx_for_size: allocation size too large "
2362 "(size=%"PRIuSIZE
"u, heap_idx=%"PRIuSIZE
"u)", size, heap_idx);
2365 #if RGENGC_CHECK_MODE
2367 GC_ASSERT(size <= (
size_t)heaps[heap_idx].slot_size);
2368 if (heap_idx > 0) GC_ASSERT(size > (
size_t)heaps[heap_idx - 1].slot_size);
2375 rb_gc_impl_heap_id_for_size(
void *objspace_ptr,
size_t size)
2377 return heap_idx_for_size(size);
2381 static size_t heap_sizes[HEAP_COUNT + 1] = { 0 };
2384 rb_gc_impl_heap_sizes(
void *objspace_ptr)
2386 if (heap_sizes[0] == 0) {
2387 for (
unsigned char i = 0; i < HEAP_COUNT; i++) {
2388 heap_sizes[i] = heap_slot_size(i);
2403 unsigned int lev = 0;
2404 bool unlock_vm =
false;
2407 lev = rb_gc_cr_lock();
2413 if (is_incremental_marking(
objspace)) {
2415 cache->incremental_mark_step_allocated_slots = 0;
2418 obj = ractor_cache_allocate_slot(
objspace, cache, heap_idx);
2424 ractor_cache_set_page(
objspace, cache, heap_idx, page);
2427 obj = ractor_cache_allocate_slot(
objspace, cache, heap_idx);
2432 rb_gc_cr_unlock(lev);
2444 VALUE obj = ractor_cache_allocate_slot(
objspace, cache, heap_idx);
2447 obj = newobj_cache_miss(
objspace, cache, heap_idx, vm_locked);
2451 heap->total_allocated_objects++;
2452 GC_ASSERT(rb_gc_multi_ractor_p() ||
2453 heap->total_slots >=
2454 (heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count));
2467 lev = rb_gc_cr_lock();
2469 if (
RB_UNLIKELY(during_gc || ruby_gc_stressful)) {
2473 rb_bug(
"object allocation during garbage collection phase");
2476 if (ruby_gc_stressful) {
2477 if (!garbage_collect(
objspace, GPR_FLAG_NEWOBJ)) {
2483 obj = newobj_alloc(
objspace, cache, heap_idx,
true);
2484 newobj_init(klass, flags, wb_protected,
objspace, obj);
2486 rb_gc_cr_unlock(lev);
2491 NOINLINE(
static VALUE newobj_slowpath_wb_protected(
VALUE klass,
VALUE flags,
2493 NOINLINE(
static VALUE newobj_slowpath_wb_unprotected(
VALUE klass,
VALUE flags,
2499 return newobj_slowpath(klass, flags,
objspace, cache, TRUE, heap_idx);
2505 return newobj_slowpath(klass, flags,
objspace, cache, FALSE, heap_idx);
2509 rb_gc_impl_new_obj(
void *objspace_ptr,
void *cache_ptr,
VALUE klass,
VALUE flags,
VALUE v1,
VALUE v2,
VALUE v3,
bool wb_protected,
size_t alloc_size)
2514 RB_DEBUG_COUNTER_INC(obj_newobj);
2515 (void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected);
2519 for (
long i = 0; i < cnt; i++) {
2524 size_t heap_idx = heap_idx_for_size(alloc_size);
2528 if (!
RB_UNLIKELY(during_gc || ruby_gc_stressful) &&
2530 obj = newobj_alloc(
objspace, cache, heap_idx,
false);
2531 newobj_init(klass, flags, wb_protected,
objspace, obj);
2534 RB_DEBUG_COUNTER_INC(obj_newobj_slowpath);
2536 obj = wb_protected ?
2537 newobj_slowpath_wb_protected(klass, flags,
objspace, cache, heap_idx) :
2538 newobj_slowpath_wb_unprotected(klass, flags,
objspace, cache, heap_idx);
2541 return newobj_fill(obj, v1, v2, v3);
2545 ptr_in_page_body_p(
const void *
ptr,
const void *memb)
2548 uintptr_t p_body = (uintptr_t)page->body;
2550 if ((uintptr_t)
ptr >= p_body) {
2551 return (uintptr_t)
ptr < (p_body + HEAP_PAGE_SIZE) ? 0 : 1;
2564 if (
ptr < (uintptr_t)heap_pages_lomem ||
2565 ptr > (uintptr_t)heap_pages_himem) {
2569 res = bsearch((
void *)
ptr, rb_darray_ref(
objspace->heap_pages.sorted, 0),
2571 ptr_in_page_body_p);
2585 register uintptr_t p = (uintptr_t)
ptr;
2588 RB_DEBUG_COUNTER_INC(gc_isptr_trial);
2590 if (p < heap_pages_lomem || p > heap_pages_himem)
return FALSE;
2591 RB_DEBUG_COUNTER_INC(gc_isptr_range);
2593 if (p % BASE_SLOT_SIZE != 0)
return FALSE;
2594 RB_DEBUG_COUNTER_INC(gc_isptr_align);
2596 page = heap_page_for_ptr(
objspace, (uintptr_t)
ptr);
2598 RB_DEBUG_COUNTER_INC(gc_isptr_maybe);
2599 if (heap_page_in_global_empty_pages_pool(
objspace, page)) {
2603 if (p < page->start)
return FALSE;
2604 if (p >= page->start + (page->total_slots * page->slot_size))
return FALSE;
2605 if ((NUM_IN_PAGE(p) * BASE_SLOT_SIZE) % page->slot_size != 0)
return FALSE;
2614 rb_gc_impl_pointer_to_heap_p(
void *objspace_ptr,
const void *
ptr)
2616 return is_pointer_to_heap(objspace_ptr,
ptr);
2619 #define ZOMBIE_OBJ_KEPT_FLAGS (FL_SEEN_OBJ_ID | FL_FINALIZE)
2622 rb_gc_impl_make_zombie(
void *objspace_ptr,
VALUE obj,
void (*dfree)(
void *),
void *data)
2626 struct RZombie *zombie = RZOMBIE(obj);
2628 zombie->dfree = dfree;
2629 zombie->data = data;
2630 VALUE prev, next = heap_pages_deferred_final;
2632 zombie->next = prev = next;
2634 }
while (next != prev);
2636 struct heap_page *page = GET_HEAP_PAGE(obj);
2637 page->final_slots++;
2638 page->heap->final_slots_count++;
2644 st_data_t o = (st_data_t)obj,
id;
2649 if (st_delete(
objspace->obj_to_id_tbl, &o, &
id)) {
2651 st_delete(
objspace->id_to_obj_tbl, &
id, NULL);
2654 rb_bug(
"Object ID seen, but not in mapping table: %s", rb_obj_info(obj));
2658 typedef int each_obj_callback(
void *,
void *,
size_t,
void *);
2659 typedef int each_page_callback(
struct heap_page *,
void *);
2663 bool reenable_incremental;
2665 each_obj_callback *each_obj_callback;
2666 each_page_callback *each_page_callback;
2670 size_t pages_counts[HEAP_COUNT];
2674 objspace_each_objects_ensure(
VALUE arg)
2680 if (data->reenable_incremental) {
2681 objspace->flags.dont_incremental = FALSE;
2684 for (
int i = 0; i < HEAP_COUNT; i++) {
2685 struct heap_page **pages = data->pages[i];
2693 objspace_each_objects_try(
VALUE arg)
2699 for (
int i = 0; i < HEAP_COUNT; i++) {
2701 size_t size = heap->total_pages *
sizeof(
struct heap_page *);
2703 struct heap_page **pages = malloc(size);
2712 size_t pages_count = 0;
2713 ccan_list_for_each(&heap->pages, page, page_node) {
2714 pages[pages_count] = page;
2717 data->pages[i] = pages;
2718 data->pages_counts[i] = pages_count;
2719 GC_ASSERT(pages_count == heap->total_pages);
2722 for (
int i = 0; i < HEAP_COUNT; i++) {
2724 size_t pages_count = data->pages_counts[i];
2725 struct heap_page **pages = data->pages[i];
2728 for (
size_t i = 0; i < pages_count; i++) {
2731 if (page == NULL)
break;
2735 if (pages[i] != page)
continue;
2737 uintptr_t pstart = (uintptr_t)page->start;
2738 uintptr_t pend = pstart + (page->total_slots * heap->slot_size);
2740 if (data->each_obj_callback &&
2741 (*data->each_obj_callback)((
void *)pstart, (
void *)pend, heap->slot_size, data->data)) {
2744 if (data->each_page_callback &&
2745 (*data->each_page_callback)(page, data->data)) {
2749 page = ccan_list_next(&heap->pages, page, page_node);
2761 bool reenable_incremental = FALSE;
2763 reenable_incremental = !
objspace->flags.dont_incremental;
2766 objspace->flags.dont_incremental = TRUE;
2777 objspace_each_objects(
rb_objspace_t *
objspace, each_obj_callback *callback,
void *data,
bool protected)
2781 .each_obj_callback = callback,
2782 .each_page_callback = NULL,
2789 rb_gc_impl_each_objects(
void *objspace_ptr, each_obj_callback *callback,
void *data)
2791 objspace_each_objects(objspace_ptr, callback, data, TRUE);
2794 #if GC_CAN_COMPILE_COMPACTION
2796 objspace_each_pages(
rb_objspace_t *
objspace, each_page_callback *callback,
void *data,
bool protected)
2800 .each_obj_callback = NULL,
2801 .each_page_callback = callback,
2809 rb_gc_impl_define_finalizer(
void *objspace_ptr,
VALUE obj,
VALUE block)
2819 if (st_lookup(finalizer_table, obj, &data)) {
2820 table = (
VALUE)data;
2827 for (i = 0; i <
len; i++) {
2840 st_add_direct(finalizer_table, obj, table);
2847 rb_gc_impl_undefine_finalizer(
void *objspace_ptr,
VALUE obj)
2853 st_data_t data = obj;
2854 st_delete(finalizer_table, &data, 0);
2859 rb_gc_impl_copy_finalizer(
void *objspace_ptr,
VALUE dest,
VALUE obj)
2867 if (
RB_LIKELY(st_lookup(finalizer_table, obj, &data))) {
2868 table = (
VALUE)data;
2869 st_insert(finalizer_table, dest, table);
2873 rb_bug(
"rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj));
2881 return rb_gc_impl_object_id(
objspace, obj);
2885 objspace->next_object_id += OBJ_ID_INCREMENT;
2891 get_final(
long i,
void *data)
2901 if (RZOMBIE(zombie)->dfree) {
2902 RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data);
2905 st_data_t key = (st_data_t)zombie;
2909 if (st_delete(finalizer_table, &key, &table)) {
2910 rb_gc_run_obj_finalizer(get_object_id_in_finalizer(
objspace, zombie),
RARRAY_LEN(table), get_final, (
void *)table);
2913 rb_bug(
"FL_FINALIZE flag is set, but finalizers are not found");
2917 GC_ASSERT(!st_lookup(finalizer_table, key, NULL));
2927 asan_unpoison_object(zombie,
false);
2928 next_zombie = RZOMBIE(zombie)->next;
2929 page = GET_HEAP_PAGE(zombie);
2933 int lev = rb_gc_vm_lock();
2937 obj_free_object_id(
objspace, zombie);
2940 GC_ASSERT(page->heap->final_slots_count > 0);
2941 GC_ASSERT(page->final_slots > 0);
2943 page->heap->final_slots_count--;
2944 page->final_slots--;
2946 heap_page_add_freeobj(
objspace, page, zombie);
2947 page->heap->total_freed_objects++;
2949 rb_gc_vm_unlock(lev);
2951 zombie = next_zombie;
2967 rb_gc_set_pending_interrupt();
2968 finalize_deferred_heap_pages(
objspace);
2969 rb_gc_unset_pending_interrupt();
2973 gc_finalize_deferred(
void *dmy)
2992 gc_abort(
void *objspace_ptr)
2996 if (is_incremental_marking(
objspace)) {
2999 while (pop_mark_stack(&
objspace->mark_stack, &obj));
3001 objspace->flags.during_incremental_marking = FALSE;
3005 for (
int i = 0; i < HEAP_COUNT; i++) {
3008 heap->sweeping_page = NULL;
3011 ccan_list_for_each(&heap->pages, page, page_node) {
3012 page->flags.before_sweep =
false;
3017 for (
int i = 0; i < HEAP_COUNT; i++) {
3019 rgengc_mark_and_rememberset_clear(
objspace, heap);
3022 gc_mode_set(
objspace, gc_mode_none);
3026 rb_gc_impl_shutdown_free_objects(
void *objspace_ptr)
3030 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
3032 short stride = page->slot_size;
3034 uintptr_t p = (uintptr_t)page->start;
3035 uintptr_t pend = p + page->total_slots * stride;
3036 for (; p < pend; p += stride) {
3038 asan_unpoisoning_object(vp) {
3040 rb_gc_obj_free_vm_weak_references(vp);
3041 if (rb_gc_obj_free(
objspace, vp)) {
3051 rb_gc_impl_shutdown_call_finalizer_i(st_data_t key, st_data_t val, st_data_t data)
3060 rb_gc_run_obj_finalizer(rb_gc_impl_object_id(
objspace, obj),
RARRAY_LEN(table), get_final, (
void *)table);
3068 rb_gc_impl_shutdown_call_finalizer(
void *objspace_ptr)
3072 #if RGENGC_CHECK_MODE >= 2
3073 gc_verify_internal_consistency(
objspace);
3077 objspace->flags.dont_incremental = 1;
3086 while (finalizer_table->num_entries) {
3087 st_foreach(finalizer_table, rb_gc_impl_shutdown_call_finalizer_i, (st_data_t)
objspace);
3092 GC_ASSERT(heap_pages_deferred_final == 0);
3101 unsigned int lock_lev;
3102 gc_enter(
objspace, gc_enter_event_finalizer, &lock_lev);
3105 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
3107 short stride = page->slot_size;
3109 uintptr_t p = (uintptr_t)page->start;
3110 uintptr_t pend = p + page->total_slots * stride;
3111 for (; p < pend; p += stride) {
3113 asan_unpoisoning_object(vp) {
3114 if (rb_gc_shutdown_call_finalizer_p(vp)) {
3115 rb_gc_obj_free_vm_weak_references(vp);
3116 if (rb_gc_obj_free(
objspace, vp)) {
3124 gc_exit(
objspace, gc_enter_event_finalizer, &lock_lev);
3126 finalize_deferred_heap_pages(
objspace);
3128 st_free_table(finalizer_table);
3129 finalizer_table = 0;
3134 rb_gc_impl_each_object(
void *objspace_ptr,
void (*func)(
VALUE obj,
void *data),
void *data)
3138 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
3140 short stride = page->slot_size;
3142 uintptr_t p = (uintptr_t)page->start;
3143 uintptr_t pend = p + page->total_slots * stride;
3144 for (; p < pend; p += stride) {
3147 asan_unpoisoning_object(obj) {
3163 size_t total_slots = 0;
3164 for (
int i = 0; i < HEAP_COUNT; i++) {
3166 total_slots += heap->total_slots;
3184 gc_setup_mark_bits(
struct heap_page *page)
3187 memcpy(&page->mark_bits[0], &page->uncollectible_bits[0], HEAP_PAGE_BITMAP_SIZE);
3194 enum {HEAP_PAGE_LOCK = PAGE_NOACCESS, HEAP_PAGE_UNLOCK = PAGE_READWRITE};
3200 return VirtualProtect(body, HEAP_PAGE_SIZE, protect, &old_protect) != 0;
3203 enum {HEAP_PAGE_LOCK = PROT_NONE, HEAP_PAGE_UNLOCK = PROT_READ | PROT_WRITE};
3204 #define protect_page_body(body, protect) !mprotect((body), HEAP_PAGE_SIZE, (protect))
3210 if (!protect_page_body(body, HEAP_PAGE_LOCK)) {
3211 rb_bug(
"Couldn't protect page %p, errno: %s", (
void *)body, strerror(
errno));
3214 gc_report(5,
objspace,
"Protecting page in move %p\n", (
void *)body);
3221 if (!protect_page_body(body, HEAP_PAGE_UNLOCK)) {
3222 rb_bug(
"Couldn't unprotect page %p, errno: %s", (
void *)body, strerror(
errno));
3225 gc_report(5,
objspace,
"Unprotecting page in move %p\n", (
void *)body);
3232 GC_ASSERT(gc_is_moveable_obj(
objspace, src));
3234 struct heap_page *src_page = GET_HEAP_PAGE(src);
3242 GC_ASSERT(RVALUE_MARKED(
objspace, src));
3244 asan_unlock_freelist(free_page);
3246 asan_lock_freelist(free_page);
3247 asan_unpoison_object(dest,
false);
3253 asan_unlock_freelist(free_page);
3254 free_page->freelist = ((
struct free_slot *)dest)->next;
3255 asan_lock_freelist(free_page);
3259 if (src_page->slot_size > free_page->slot_size) {
3262 else if (free_page->slot_size > src_page->slot_size) {
3266 objspace->rcompactor.total_moved++;
3268 gc_move(
objspace, src, dest, src_page->slot_size, free_page->slot_size);
3270 free_page->free_slots--;
3278 struct heap_page *cursor = heap->compact_cursor;
3281 unlock_page_body(
objspace, cursor->body);
3282 cursor = ccan_list_next(&heap->pages, cursor, page_node);
3287 #if GC_CAN_COMPILE_COMPACTION
3291 #if defined(__MINGW32__) || defined(_WIN32)
3292 # define GC_COMPACTION_SUPPORTED 1
3296 # define GC_COMPACTION_SUPPORTED (GC_CAN_COMPILE_COMPACTION && HEAP_PAGE_ALLOC_USE_MMAP)
3299 #if GC_CAN_COMPILE_COMPACTION
3301 read_barrier_handler(uintptr_t original_address)
3307 uintptr_t address = original_address - (original_address % BASE_SLOT_SIZE);
3309 obj = (
VALUE)address;
3315 if (page_body == NULL) {
3316 rb_bug(
"read_barrier_handler: segmentation fault at %p", (
void *)original_address);
3319 int lev = rb_gc_vm_lock();
3321 unlock_page_body(
objspace, page_body);
3323 objspace->profile.read_barrier_faults++;
3325 invalidate_moved_page(
objspace, GET_HEAP_PAGE(obj));
3327 rb_gc_vm_unlock(lev);
3331 #if !GC_CAN_COMPILE_COMPACTION
3333 uninstall_handlers(
void)
3339 install_handlers(
void)
3343 #elif defined(_WIN32)
3344 static LPTOP_LEVEL_EXCEPTION_FILTER old_handler;
3345 typedef void (*signal_handler)(int);
3346 static signal_handler old_sigsegv_handler;
3349 read_barrier_signal(EXCEPTION_POINTERS *info)
3352 if (info->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
3357 read_barrier_handler((uintptr_t)info->ExceptionRecord->ExceptionInformation[1]);
3358 return EXCEPTION_CONTINUE_EXECUTION;
3361 return EXCEPTION_CONTINUE_SEARCH;
3366 uninstall_handlers(
void)
3368 signal(SIGSEGV, old_sigsegv_handler);
3369 SetUnhandledExceptionFilter(old_handler);
3373 install_handlers(
void)
3376 old_sigsegv_handler = signal(SIGSEGV, NULL);
3379 old_handler = SetUnhandledExceptionFilter(read_barrier_signal);
3382 static struct sigaction old_sigbus_handler;
3383 static struct sigaction old_sigsegv_handler;
3385 #ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3386 static exception_mask_t old_exception_masks[32];
3387 static mach_port_t old_exception_ports[32];
3388 static exception_behavior_t old_exception_behaviors[32];
3389 static thread_state_flavor_t old_exception_flavors[32];
3390 static mach_msg_type_number_t old_exception_count;
3393 disable_mach_bad_access_exc(
void)
3395 old_exception_count =
sizeof(old_exception_masks) /
sizeof(old_exception_masks[0]);
3396 task_swap_exception_ports(
3397 mach_task_self(), EXC_MASK_BAD_ACCESS,
3398 MACH_PORT_NULL, EXCEPTION_DEFAULT, 0,
3399 old_exception_masks, &old_exception_count,
3400 old_exception_ports, old_exception_behaviors, old_exception_flavors
3405 restore_mach_bad_access_exc(
void)
3407 for (mach_msg_type_number_t i = 0; i < old_exception_count; i++) {
3408 task_set_exception_ports(
3410 old_exception_masks[i], old_exception_ports[i],
3411 old_exception_behaviors[i], old_exception_flavors[i]
3418 read_barrier_signal(
int sig, siginfo_t *info,
void *data)
3421 struct sigaction prev_sigbus, prev_sigsegv;
3422 sigaction(SIGBUS, &old_sigbus_handler, &prev_sigbus);
3423 sigaction(SIGSEGV, &old_sigsegv_handler, &prev_sigsegv);
3426 sigset_t set, prev_set;
3428 sigaddset(&set, SIGBUS);
3429 sigaddset(&set, SIGSEGV);
3430 sigprocmask(SIG_UNBLOCK, &set, &prev_set);
3431 #ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3432 disable_mach_bad_access_exc();
3435 read_barrier_handler((uintptr_t)info->si_addr);
3438 #ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3439 restore_mach_bad_access_exc();
3441 sigaction(SIGBUS, &prev_sigbus, NULL);
3442 sigaction(SIGSEGV, &prev_sigsegv, NULL);
3443 sigprocmask(SIG_SETMASK, &prev_set, NULL);
3447 uninstall_handlers(
void)
3449 #ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3450 restore_mach_bad_access_exc();
3452 sigaction(SIGBUS, &old_sigbus_handler, NULL);
3453 sigaction(SIGSEGV, &old_sigsegv_handler, NULL);
3457 install_handlers(
void)
3459 struct sigaction action;
3460 memset(&action, 0,
sizeof(
struct sigaction));
3461 sigemptyset(&action.sa_mask);
3462 action.sa_sigaction = read_barrier_signal;
3463 action.sa_flags = SA_SIGINFO | SA_ONSTACK;
3465 sigaction(SIGBUS, &action, &old_sigbus_handler);
3466 sigaction(SIGSEGV, &action, &old_sigsegv_handler);
3467 #ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3468 disable_mach_bad_access_exc();
3476 for (
int i = 0; i < HEAP_COUNT; i++) {
3478 gc_unprotect_pages(
objspace, heap);
3481 uninstall_handlers();
3486 for (
int i = 0; i < HEAP_COUNT; i++) {
3488 heap->compact_cursor = NULL;
3489 heap->free_pages = NULL;
3490 heap->compact_cursor_index = 0;
3495 record->moved_objects =
objspace->rcompactor.total_moved - record->moved_objects;
3497 objspace->flags.during_compacting = FALSE;
3510 struct heap_page *sweep_page = ctx->page;
3511 short slot_size = sweep_page->slot_size;
3512 short slot_bits = slot_size / BASE_SLOT_SIZE;
3513 GC_ASSERT(slot_bits > 0);
3517 GC_ASSERT(vp % BASE_SLOT_SIZE == 0);
3519 asan_unpoison_object(vp,
false);
3523 gc_report(2,
objspace,
"page_sweep: free %p\n", (
void *)p);
3524 #if RGENGC_CHECK_MODE
3526 if (RVALUE_OLD_P(
objspace, vp))
rb_bug(
"page_sweep: %p - old while minor GC.", (
void *)p);
3527 if (RVALUE_REMEMBERED(
objspace, vp))
rb_bug(
"page_sweep: %p - remembered.", (
void *)p);
3531 if (RVALUE_WB_UNPROTECTED(
objspace, vp)) CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(vp), vp);
3533 #if RGENGC_CHECK_MODE
3534 #define CHECK(x) if (x(objspace, vp) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", rb_obj_info(vp))
3535 CHECK(RVALUE_WB_UNPROTECTED);
3536 CHECK(RVALUE_MARKED);
3537 CHECK(RVALUE_MARKING);
3538 CHECK(RVALUE_UNCOLLECTIBLE);
3545 rb_gc_obj_free_vm_weak_references(vp);
3546 if (rb_gc_obj_free(
objspace, vp)) {
3547 if (has_object_id) {
3552 (void)VALGRIND_MAKE_MEM_UNDEFINED((
void*)p, BASE_SLOT_SIZE);
3553 heap_page_add_freeobj(
objspace, sweep_page, vp);
3554 gc_report(3,
objspace,
"page_sweep: %s is added to freelist\n", rb_obj_info(vp));
3563 if (
objspace->flags.during_compacting) {
3569 rb_bug(
"T_MOVED shouldn't be seen until compaction is finished");
3571 gc_report(3,
objspace,
"page_sweep: %s is added to freelist\n", rb_obj_info(vp));
3573 heap_page_add_freeobj(
objspace, sweep_page, vp);
3584 bitset >>= slot_bits;
3591 struct heap_page *sweep_page = ctx->page;
3592 GC_ASSERT(sweep_page->heap == heap);
3595 bits_t *bits, bitset;
3597 gc_report(2,
objspace,
"page_sweep: start.\n");
3599 #if RGENGC_CHECK_MODE
3600 if (!
objspace->flags.immediate_sweep) {
3601 GC_ASSERT(sweep_page->flags.before_sweep == TRUE);
3604 sweep_page->flags.before_sweep = FALSE;
3605 sweep_page->free_slots = 0;
3607 p = (uintptr_t)sweep_page->start;
3608 bits = sweep_page->mark_bits;
3610 int page_rvalue_count = sweep_page->total_slots * (sweep_page->slot_size / BASE_SLOT_SIZE);
3611 int out_of_range_bits = (NUM_IN_PAGE(p) + page_rvalue_count) % BITS_BITLENGTH;
3612 if (out_of_range_bits != 0) {
3613 bits[BITMAP_INDEX(p) + page_rvalue_count / BITS_BITLENGTH] |= ~(((bits_t)1 << out_of_range_bits) - 1);
3619 int bitmap_plane_count = CEILDIV(NUM_IN_PAGE(p) + page_rvalue_count, BITS_BITLENGTH);
3620 GC_ASSERT(bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT - 1 ||
3621 bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT);
3625 bitset >>= NUM_IN_PAGE(p);
3627 gc_sweep_plane(
objspace, heap, p, bitset, ctx);
3629 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
3631 for (
int i = 1; i < bitmap_plane_count; i++) {
3634 gc_sweep_plane(
objspace, heap, p, bitset, ctx);
3636 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
3639 if (!heap->compact_cursor) {
3640 gc_setup_mark_bits(sweep_page);
3643 #if GC_PROFILE_MORE_DETAIL
3646 record->removing_objects += ctx->final_slots + ctx->freed_slots;
3647 record->empty_objects += ctx->empty_slots;
3650 if (0) fprintf(stderr,
"gc_sweep_page(%"PRIdSIZE
"): total_slots: %d, freed_slots: %d, empty_slots: %d, final_slots: %d\n",
3652 sweep_page->total_slots,
3653 ctx->freed_slots, ctx->empty_slots, ctx->final_slots);
3655 sweep_page->free_slots += ctx->freed_slots + ctx->empty_slots;
3656 sweep_page->heap->total_freed_objects += ctx->freed_slots;
3658 if (heap_pages_deferred_final && !finalizing) {
3659 gc_finalize_deferred_register(
objspace);
3662 #if RGENGC_CHECK_MODE
3663 short freelist_len = 0;
3664 asan_unlock_freelist(sweep_page);
3670 asan_lock_freelist(sweep_page);
3671 if (freelist_len != sweep_page->free_slots) {
3672 rb_bug(
"inconsistent freelist length: expected %d but was %d", sweep_page->free_slots, freelist_len);
3676 gc_report(2,
objspace,
"page_sweep: end.\n");
3680 gc_mode_name(
enum gc_mode mode)
3683 case gc_mode_none:
return "none";
3684 case gc_mode_marking:
return "marking";
3685 case gc_mode_sweeping:
return "sweeping";
3686 case gc_mode_compacting:
return "compacting";
3687 default:
rb_bug(
"gc_mode_name: unknown mode: %d", (
int)mode);
3694 #if RGENGC_CHECK_MODE
3695 enum gc_mode prev_mode = gc_mode(
objspace);
3696 switch (prev_mode) {
3697 case gc_mode_none: GC_ASSERT(mode == gc_mode_marking);
break;
3698 case gc_mode_marking: GC_ASSERT(mode == gc_mode_sweeping);
break;
3699 case gc_mode_sweeping: GC_ASSERT(mode == gc_mode_none || mode == gc_mode_compacting);
break;
3700 case gc_mode_compacting: GC_ASSERT(mode == gc_mode_none);
break;
3703 if (0) fprintf(stderr,
"gc_mode_transition: %s->%s\n", gc_mode_name(gc_mode(
objspace)), gc_mode_name(mode));
3711 asan_unlock_freelist(page);
3712 if (page->freelist) {
3714 asan_unpoison_object((
VALUE)p,
false);
3718 asan_poison_object((
VALUE)prev);
3719 asan_unpoison_object((
VALUE)p,
false);
3722 asan_poison_object((
VALUE)p);
3725 page->freelist = freelist;
3727 asan_lock_freelist(page);
3734 heap->sweeping_page = ccan_list_top(&heap->pages,
struct heap_page, page_node);
3735 heap->free_pages = NULL;
3736 heap->pooled_pages = NULL;
3737 if (!
objspace->flags.immediate_sweep) {
3740 ccan_list_for_each(&heap->pages, page, page_node) {
3741 page->flags.before_sweep = TRUE;
3746 #if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 4
3750 #if GC_CAN_COMPILE_COMPACTION
3751 static void gc_sort_heap_by_compare_func(
rb_objspace_t *
objspace, gc_compact_compare_func compare_func);
3752 static int compare_pinned_slots(
const void *left,
const void *right,
void *d);
3756 gc_ractor_newobj_cache_clear(
void *c,
void *data)
3760 newobj_cache->incremental_mark_step_allocated_slots = 0;
3762 for (
size_t heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) {
3765 struct heap_page *page = cache->using_page;
3766 struct free_slot *freelist = cache->freelist;
3767 RUBY_DEBUG_LOG(
"ractor using_page:%p freelist:%p", (
void *)page, (
void *)freelist);
3769 heap_page_freelist_append(page, freelist);
3771 cache->using_page = NULL;
3772 cache->freelist = NULL;
3779 gc_mode_transition(
objspace, gc_mode_sweeping);
3781 objspace->heap_pages.allocatable_slots = 0;
3783 #if GC_CAN_COMPILE_COMPACTION
3784 if (
objspace->flags.during_compacting) {
3785 gc_sort_heap_by_compare_func(
3787 objspace->rcompactor.compare_func ?
objspace->rcompactor.compare_func : compare_pinned_slots
3792 for (
int i = 0; i < HEAP_COUNT; i++) {
3794 gc_sweep_start_heap(
objspace, heap);
3797 if (heap->sweeping_page == NULL) {
3798 GC_ASSERT(heap->total_pages == 0);
3799 GC_ASSERT(heap->total_slots == 0);
3800 gc_sweep_finish_heap(
objspace, heap);
3804 rb_gc_ractor_newobj_cache_foreach(gc_ractor_newobj_cache_clear, NULL);
3810 size_t total_slots = heap->total_slots;
3811 size_t swept_slots = heap->freed_slots + heap->empty_slots;
3813 size_t init_slots = gc_params.heap_init_slots[heap - heaps];
3814 size_t min_free_slots = (size_t)(MAX(total_slots, init_slots) * gc_params.heap_free_slots_min_ratio);
3816 if (swept_slots < min_free_slots &&
3818 (heap->empty_slots == 0 || heap->freed_slots > heap->empty_slots)) {
3824 while (swept_slots < min_free_slots &&
3825 (resurrected_page = heap_page_resurrect(
objspace))) {
3826 heap_add_page(
objspace, heap, resurrected_page);
3827 heap_add_freepage(heap, resurrected_page);
3829 swept_slots += resurrected_page->free_slots;
3832 if (swept_slots < min_free_slots) {
3836 objspace->profile.count -
objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
3837 heap_allocatable_slots_expand(
objspace, heap, swept_slots, heap->total_slots);
3840 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
3841 heap->force_major_gc_count++;
3850 gc_report(1,
objspace,
"gc_sweep_finish\n");
3853 heap_pages_free_unused_pages(
objspace);
3855 for (
int i = 0; i < HEAP_COUNT; i++) {
3858 heap->freed_slots = 0;
3859 heap->empty_slots = 0;
3861 if (!will_be_incremental_marking(
objspace)) {
3862 struct heap_page *end_page = heap->free_pages;
3864 while (end_page->free_next) end_page = end_page->free_next;
3865 end_page->free_next = heap->pooled_pages;
3868 heap->free_pages = heap->pooled_pages;
3870 heap->pooled_pages = NULL;
3876 gc_mode_transition(
objspace, gc_mode_none);
3878 #if RGENGC_CHECK_MODE >= 2
3879 gc_verify_internal_consistency(
objspace);
3886 struct heap_page *sweep_page = heap->sweeping_page;
3887 int unlink_limit = GC_SWEEP_PAGES_FREEABLE_PER_STEP;
3888 int swept_slots = 0;
3889 int pooled_slots = 0;
3891 if (sweep_page == NULL)
return FALSE;
3893 #if GC_ENABLE_LAZY_SWEEP
3894 gc_prof_sweep_timer_start(
objspace);
3898 RUBY_DEBUG_LOG(
"sweep_page:%p", (
void *)sweep_page);
3906 gc_sweep_page(
objspace, heap, &ctx);
3907 int free_slots = ctx.freed_slots + ctx.empty_slots;
3909 heap->sweeping_page = ccan_list_next(&heap->pages, sweep_page, page_node);
3911 if (free_slots == sweep_page->total_slots &&
3912 heap_pages_freeable_pages > 0 &&
3914 heap_pages_freeable_pages--;
3917 heap_unlink_page(
objspace, heap, sweep_page);
3919 sweep_page->start = 0;
3920 sweep_page->total_slots = 0;
3921 sweep_page->slot_size = 0;
3922 sweep_page->heap = NULL;
3923 sweep_page->free_slots = 0;
3925 asan_unlock_freelist(sweep_page);
3926 sweep_page->freelist = NULL;
3927 asan_lock_freelist(sweep_page);
3929 asan_poison_memory_region(sweep_page->body, HEAP_PAGE_SIZE);
3932 sweep_page->free_next =
objspace->empty_pages;
3933 objspace->empty_pages = sweep_page;
3935 else if (free_slots > 0) {
3936 heap->freed_slots += ctx.freed_slots;
3937 heap->empty_slots += ctx.empty_slots;
3939 if (pooled_slots < GC_INCREMENTAL_SWEEP_POOL_SLOT_COUNT) {
3940 heap_add_poolpage(
objspace, heap, sweep_page);
3941 pooled_slots += free_slots;
3944 heap_add_freepage(heap, sweep_page);
3945 swept_slots += free_slots;
3946 if (swept_slots > GC_INCREMENTAL_SWEEP_SLOT_COUNT) {
3952 sweep_page->free_next = NULL;
3954 }
while ((sweep_page = heap->sweeping_page));
3956 if (!heap->sweeping_page) {
3957 gc_sweep_finish_heap(
objspace, heap);
3959 if (!has_sweeping_pages(
objspace)) {
3964 #if GC_ENABLE_LAZY_SWEEP
3965 gc_prof_sweep_timer_stop(
objspace);
3968 return heap->free_pages != NULL;
3974 for (
int i = 0; i < HEAP_COUNT; i++) {
3977 while (heap->sweeping_page) {
3986 GC_ASSERT(dont_gc_val() == FALSE ||
objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
3987 if (!GC_ENABLE_LAZY_SWEEP)
return;
3991 for (
int i = 0; i < HEAP_COUNT; i++) {
3993 if (!gc_sweep_step(
objspace, heap)) {
3996 if (heap == sweep_heap &&
objspace->heap_pages.allocatable_slots == 0) {
4008 rb_gc_impl_location(
void *objspace_ptr,
VALUE value)
4013 asan_unpoisoning_object(value) {
4015 destination = (
VALUE)RMOVED(value)->destination;
4019 destination = value;
4024 destination = value;
4030 #if GC_CAN_COMPILE_COMPACTION
4041 GC_ASSERT(RVALUE_PINNED(
objspace, forwarding_object));
4042 GC_ASSERT(!RVALUE_MARKED(
objspace, forwarding_object));
4044 CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object);
4046 object = rb_gc_impl_location(
objspace, forwarding_object);
4048 uint32_t original_shape_id = 0;
4050 original_shape_id = RMOVED(forwarding_object)->original_shape_id;
4053 gc_move(
objspace,
object, forwarding_object, GET_HEAP_PAGE(
object)->slot_size, page->slot_size);
4057 if (original_shape_id) {
4058 rb_gc_set_shape(forwarding_object, original_shape_id);
4061 struct heap_page *orig_page = GET_HEAP_PAGE(
object);
4062 orig_page->free_slots++;
4063 heap_page_add_freeobj(
objspace, orig_page,
object);
4065 GC_ASSERT(RVALUE_MARKED(
objspace, forwarding_object));
4070 p += BASE_SLOT_SIZE;
4080 bits_t *mark_bits, *pin_bits;
4083 mark_bits = page->mark_bits;
4084 pin_bits = page->pinned_bits;
4086 uintptr_t p = page->start;
4089 bitset = pin_bits[0] & ~mark_bits[0];
4090 bitset >>= NUM_IN_PAGE(p);
4091 invalidate_moved_plane(
objspace, page, p, bitset);
4092 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
4094 for (i=1; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
4097 bitset = pin_bits[i] & ~mark_bits[i];
4099 invalidate_moved_plane(
objspace, page, p, bitset);
4100 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
4109 gc_mode_transition(
objspace, gc_mode_compacting);
4111 for (
int i = 0; i < HEAP_COUNT; i++) {
4113 ccan_list_for_each(&heap->pages, page, page_node) {
4114 page->flags.before_sweep = TRUE;
4117 heap->compact_cursor = ccan_list_tail(&heap->pages,
struct heap_page, page_node);
4118 heap->compact_cursor_index = 0;
4123 record->moved_objects =
objspace->rcompactor.total_moved;
4126 memset(
objspace->rcompactor.considered_count_table, 0,
T_MASK *
sizeof(
size_t));
4127 memset(
objspace->rcompactor.moved_count_table, 0,
T_MASK *
sizeof(
size_t));
4128 memset(
objspace->rcompactor.moved_up_count_table, 0,
T_MASK *
sizeof(
size_t));
4129 memset(
objspace->rcompactor.moved_down_count_table, 0,
T_MASK *
sizeof(
size_t));
4142 const unsigned int immediate_sweep =
objspace->flags.immediate_sweep;
4144 gc_report(1,
objspace,
"gc_sweep: immediate: %d\n", immediate_sweep);
4147 if (
objspace->flags.during_compacting) {
4151 if (immediate_sweep) {
4152 #if !GC_ENABLE_LAZY_SWEEP
4153 gc_prof_sweep_timer_start(
objspace);
4156 #if !GC_ENABLE_LAZY_SWEEP
4157 gc_prof_sweep_timer_stop(
objspace);
4163 for (
int i = 0; i < HEAP_COUNT; i++) {
4175 stack_chunk_alloc(
void)
4189 return stack->chunk == NULL;
4195 size_t size = stack->index;
4196 stack_chunk_t *chunk = stack->chunk ? stack->chunk->next : NULL;
4199 size += stack->limit;
4200 chunk = chunk->next;
4208 chunk->next = stack->cache;
4209 stack->cache = chunk;
4210 stack->cache_size++;
4218 if (stack->unused_cache_size > (stack->cache_size/2)) {
4219 chunk = stack->cache;
4220 stack->cache = stack->cache->next;
4221 stack->cache_size--;
4224 stack->unused_cache_size = stack->cache_size;
4232 GC_ASSERT(stack->index == stack->limit);
4234 if (stack->cache_size > 0) {
4235 next = stack->cache;
4236 stack->cache = stack->cache->next;
4237 stack->cache_size--;
4238 if (stack->unused_cache_size > stack->cache_size)
4239 stack->unused_cache_size = stack->cache_size;
4242 next = stack_chunk_alloc();
4244 next->next = stack->chunk;
4245 stack->chunk = next;
4254 prev = stack->chunk->next;
4255 GC_ASSERT(stack->index == 0);
4256 add_stack_chunk_cache(stack, stack->chunk);
4257 stack->chunk = prev;
4258 stack->index = stack->limit;
4266 while (chunk != NULL) {
4276 mark_stack_chunk_list_free(stack->chunk);
4282 mark_stack_chunk_list_free(stack->cache);
4283 stack->cache_size = 0;
4284 stack->unused_cache_size = 0;
4311 if (stack->index == stack->limit) {
4312 push_mark_stack_chunk(stack);
4314 stack->chunk->data[stack->index++] = obj;
4324 rb_bug(
"push_mark_stack() called for broken object");
4328 rb_bug(
"push_mark_stack: unexpected T_NODE object");
4332 rb_bug(
"rb_gc_mark(): unknown data type 0x%x(%p) %s",
4334 is_pointer_to_heap((
rb_objspace_t *)rb_gc_get_objspace(), (
void *)obj) ?
"corrupted object" :
"non object");
4340 if (is_mark_stack_empty(stack)) {
4343 if (stack->index == 1) {
4344 *data = stack->chunk->data[--stack->index];
4345 pop_mark_stack_chunk(stack);
4348 *data = stack->chunk->data[--stack->index];
4359 stack->index = stack->limit = STACK_CHUNK_SIZE;
4361 for (i=0; i < 4; i++) {
4362 add_stack_chunk_cache(stack, stack_chunk_alloc());
4364 stack->unused_cache_size = stack->cache_size;
4375 if (RVALUE_WB_UNPROTECTED(
objspace, obj) || !RVALUE_OLD_P(
objspace, obj)) {
4376 rgengc_remember(
objspace, old_parent);
4380 GC_ASSERT(old_parent ==
objspace->rgengc.parent_object);
4386 if (RVALUE_MARKED(
objspace, obj))
return 0;
4387 MARK_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj);
4401 if(!gc_config_full_mark_val)
4404 struct heap_page *page = GET_HEAP_PAGE(obj);
4406 GC_ASSERT(RVALUE_MARKING(
objspace, obj) == FALSE);
4407 check_rvalue_consistency(
objspace, obj);
4409 if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) {
4410 if (!RVALUE_OLD_P(
objspace, obj)) {
4411 gc_report(3,
objspace,
"gc_aging: YOUNG: %s\n", rb_obj_info(obj));
4414 else if (is_full_marking(
objspace)) {
4415 GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE);
4416 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(
objspace, page, obj);
4419 check_rvalue_consistency(
objspace, obj);
4427 #if RGENGC_CHECK_MODE
4428 if (RVALUE_MARKED(
objspace, obj) == FALSE)
rb_bug(
"gc_grey: %s is not marked.", rb_obj_info(obj));
4429 if (RVALUE_MARKING(
objspace, obj) == TRUE)
rb_bug(
"gc_grey: %s is marking/remembered.", rb_obj_info(obj));
4432 if (is_incremental_marking(
objspace)) {
4433 MARK_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
4436 push_mark_stack(&
objspace->mark_stack, obj);
4442 GC_ASSERT(during_gc);
4444 rgengc_check_relation(
objspace, obj);
4445 if (!gc_mark_set(
objspace, obj))
return;
4448 if (
objspace->rgengc.parent_object) {
4449 RUBY_DEBUG_LOG(
"%p (%s) parent:%p (%s)",
4450 (
void *)obj, obj_type_name(obj),
4451 (
void *)
objspace->rgengc.parent_object, obj_type_name(
objspace->rgengc.parent_object));
4454 RUBY_DEBUG_LOG(
"%p (%s)", (
void *)obj, obj_type_name(obj));
4459 rb_obj_info_dump(obj);
4460 rb_bug(
"try to mark T_NONE object");
4473 if (!RVALUE_PINNED(
objspace, obj)) {
4474 GC_ASSERT(GET_HEAP_PAGE(obj)->pinned_slots <= GET_HEAP_PAGE(obj)->total_slots);
4475 GET_HEAP_PAGE(obj)->pinned_slots++;
4476 MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj);
4490 rb_gc_impl_mark_and_move(
void *objspace_ptr,
VALUE *
ptr)
4495 GC_ASSERT(
objspace->flags.during_compacting);
4496 GC_ASSERT(during_gc);
4506 rb_gc_impl_mark(
void *objspace_ptr,
VALUE obj)
4514 rb_gc_impl_mark_and_pin(
void *objspace_ptr,
VALUE obj)
4522 rb_gc_impl_mark_maybe(
void *objspace_ptr,
VALUE obj)
4526 (void)VALGRIND_MAKE_MEM_DEFINED(&obj,
sizeof(obj));
4528 if (is_pointer_to_heap(
objspace, (
void *)obj)) {
4529 asan_unpoisoning_object(obj) {
4544 rb_gc_impl_mark_weak(
void *objspace_ptr,
VALUE *
ptr)
4553 rb_obj_info_dump(obj);
4554 rb_bug(
"try to mark T_NONE object");
4561 GC_ASSERT(RVALUE_MARKED(
objspace, obj));
4562 GC_ASSERT(!
objspace->flags.during_compacting);
4567 rgengc_check_relation(
objspace, obj);
4569 DURING_GC_COULD_MALLOC_REGION_START();
4571 rb_darray_append(&
objspace->weak_references,
ptr);
4573 DURING_GC_COULD_MALLOC_REGION_END();
4575 objspace->profile.weak_references_count++;
4579 rb_gc_impl_remove_weak(
void *objspace_ptr,
VALUE parent_obj,
VALUE *
ptr)
4585 if (!is_incremental_marking(
objspace))
return;
4588 if (!RVALUE_MARKED(
objspace, parent_obj))
return;
4591 rb_darray_foreach(
objspace->weak_references, i, ptr_ptr) {
4592 if (*ptr_ptr ==
ptr) {
4600 pin_value(st_data_t key, st_data_t value, st_data_t data)
4602 rb_gc_impl_mark_and_pin((
void *)data, (
VALUE)value);
4610 #define MARK_CHECKPOINT(category) do { \
4611 if (categoryp) *categoryp = category; \
4614 MARK_CHECKPOINT(
"objspace");
4617 if (finalizer_table != NULL) {
4623 if (stress_to_class)
rb_gc_mark(stress_to_class);
4625 rb_gc_save_machine_context();
4626 rb_gc_mark_roots(
objspace, categoryp);
4633 objspace->rgengc.parent_object = obj;
4644 rb_gc_mark_children(
objspace, obj);
4656 size_t marked_slots_at_the_beginning =
objspace->marked_slots;
4657 size_t popped_count = 0;
4659 while (pop_mark_stack(mstack, &obj)) {
4660 if (obj ==
Qundef)
continue;
4662 if (RGENGC_CHECK_MODE && !RVALUE_MARKED(
objspace, obj)) {
4663 rb_bug(
"gc_mark_stacked_objects: %s is not marked.", rb_obj_info(obj));
4668 if (RGENGC_CHECK_MODE && !RVALUE_MARKING(
objspace, obj)) {
4669 rb_bug(
"gc_mark_stacked_objects: incremental, but marking bit is 0");
4671 CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
4674 if (popped_count + (
objspace->marked_slots - marked_slots_at_the_beginning) > count) {
4683 if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(
objspace);
4685 if (is_mark_stack_empty(mstack)) {
4686 shrink_stack_chunk_cache(mstack);
4697 return gc_mark_stacked_objects(
objspace, TRUE, count);
4703 return gc_mark_stacked_objects(
objspace, FALSE, 0);
4706 #if RGENGC_CHECK_MODE >= 4
4708 #define MAKE_ROOTSIG(obj) (((VALUE)(obj) << 1) | 0x01)
4709 #define IS_ROOTSIG(obj) ((VALUE)(obj) & 0x01)
4710 #define GET_ROOTSIG(obj) ((const char *)((VALUE)(obj) >> 1))
4718 static struct reflist *
4719 reflist_create(
VALUE obj)
4721 struct reflist *refs =
xmalloc(
sizeof(
struct reflist));
4724 refs->list[0] = obj;
4730 reflist_destruct(
struct reflist *refs)
4737 reflist_add(
struct reflist *refs,
VALUE obj)
4739 if (refs->pos == refs->size) {
4741 SIZED_REALLOC_N(refs->list,
VALUE, refs->size, refs->size/2);
4744 refs->list[refs->pos++] = obj;
4748 reflist_dump(
struct reflist *refs)
4751 for (i=0; i<refs->pos; i++) {
4752 VALUE obj = refs->list[i];
4753 if (IS_ROOTSIG(obj)) {
4754 fprintf(stderr,
"<root@%s>", GET_ROOTSIG(obj));
4757 fprintf(stderr,
"<%s>", rb_obj_info(obj));
4759 if (i+1 < refs->pos) fprintf(stderr,
", ");
4764 reflist_referred_from_machine_context(
struct reflist *refs)
4767 for (i=0; i<refs->pos; i++) {
4768 VALUE obj = refs->list[i];
4769 if (IS_ROOTSIG(obj) && strcmp(GET_ROOTSIG(obj),
"machine_context") == 0)
return 1;
4784 const char *category;
4790 allrefs_add(
struct allrefs *data,
VALUE obj)
4792 struct reflist *refs;
4795 if (st_lookup(data->references, obj, &r)) {
4796 refs = (
struct reflist *)r;
4797 reflist_add(refs, data->root_obj);
4801 refs = reflist_create(data->root_obj);
4802 st_insert(data->references, obj, (st_data_t)refs);
4810 struct allrefs *data = (
struct allrefs *)
ptr;
4812 if (allrefs_add(data, obj)) {
4813 push_mark_stack(&data->mark_stack, obj);
4818 allrefs_roots_i(
VALUE obj,
void *
ptr)
4820 struct allrefs *data = (
struct allrefs *)
ptr;
4821 if (strlen(data->category) == 0)
rb_bug(
"!!!");
4822 data->root_obj = MAKE_ROOTSIG(data->category);
4824 if (allrefs_add(data, obj)) {
4825 push_mark_stack(&data->mark_stack, obj);
4828 #define PUSH_MARK_FUNC_DATA(v) do { \
4829 struct gc_mark_func_data_struct *prev_mark_func_data = GET_VM()->gc.mark_func_data; \
4830 GET_VM()->gc.mark_func_data = (v);
4832 #define POP_MARK_FUNC_DATA() GET_VM()->gc.mark_func_data = prev_mark_func_data;} while (0)
4837 struct allrefs data;
4838 struct gc_mark_func_data_struct mfd;
4840 int prev_dont_gc = dont_gc_val();
4844 data.references = st_init_numtable();
4845 init_mark_stack(&data.mark_stack);
4847 mfd.mark_func = allrefs_roots_i;
4851 PUSH_MARK_FUNC_DATA(&mfd);
4852 GET_VM()->gc.mark_func_data = &mfd;
4853 mark_roots(
objspace, &data.category);
4854 POP_MARK_FUNC_DATA();
4857 while (pop_mark_stack(&data.mark_stack, &obj)) {
4858 rb_objspace_reachable_objects_from(data.root_obj = obj, allrefs_i, &data);
4860 free_stack_chunks(&data.mark_stack);
4862 dont_gc_set(prev_dont_gc);
4863 return data.references;
4867 objspace_allrefs_destruct_i(st_data_t key, st_data_t value, st_data_t
ptr)
4869 struct reflist *refs = (
struct reflist *)value;
4870 reflist_destruct(refs);
4875 objspace_allrefs_destruct(
struct st_table *refs)
4877 st_foreach(refs, objspace_allrefs_destruct_i, 0);
4878 st_free_table(refs);
4881 #if RGENGC_CHECK_MODE >= 5
4883 allrefs_dump_i(st_data_t k, st_data_t v, st_data_t
ptr)
4886 struct reflist *refs = (
struct reflist *)v;
4887 fprintf(stderr,
"[allrefs_dump_i] %s <- ", rb_obj_info(obj));
4889 fprintf(stderr,
"\n");
4897 fprintf(stderr,
"[all refs] (size: %"PRIuVALUE
")\n", size);
4903 gc_check_after_marks_i(st_data_t k, st_data_t v, st_data_t
ptr)
4906 struct reflist *refs = (
struct reflist *)v;
4910 if (!RVALUE_MARKED(
objspace, obj)) {
4911 fprintf(stderr,
"gc_check_after_marks_i: %s is not marked and not oldgen.\n", rb_obj_info(obj));
4912 fprintf(stderr,
"gc_check_after_marks_i: %p is referred from ", (
void *)obj);
4915 if (reflist_referred_from_machine_context(refs)) {
4916 fprintf(stderr,
" (marked from machine stack).\n");
4921 fprintf(stderr,
"\n");
4928 gc_marks_check(
rb_objspace_t *
objspace, st_foreach_callback_func *checker_func,
const char *checker_name)
4930 size_t saved_malloc_increase =
objspace->malloc_params.increase;
4931 #if RGENGC_ESTIMATE_OLDMALLOC
4932 size_t saved_oldmalloc_increase =
objspace->rgengc.oldmalloc_increase;
4942 if (
objspace->rgengc.error_count > 0) {
4943 #if RGENGC_CHECK_MODE >= 5
4946 if (checker_name)
rb_bug(
"%s: GC has problem.", checker_name);
4949 objspace_allrefs_destruct(
objspace->rgengc.allrefs_table);
4950 objspace->rgengc.allrefs_table = 0;
4953 objspace->malloc_params.increase = saved_malloc_increase;
4954 #if RGENGC_ESTIMATE_OLDMALLOC
4955 objspace->rgengc.oldmalloc_increase = saved_oldmalloc_increase;
4963 size_t live_object_count;
4964 size_t zombie_object_count;
4967 size_t old_object_count;
4968 size_t remembered_shady_count;
4972 check_generation_i(
const VALUE child,
void *
ptr)
4975 const VALUE parent = data->parent;
4977 if (RGENGC_CHECK_MODE) GC_ASSERT(RVALUE_OLD_P(data->objspace, parent));
4979 if (!RVALUE_OLD_P(data->objspace, child)) {
4980 if (!RVALUE_REMEMBERED(data->objspace, parent) &&
4981 !RVALUE_REMEMBERED(data->objspace, child) &&
4982 !RVALUE_UNCOLLECTIBLE(data->objspace, child)) {
4983 fprintf(stderr,
"verify_internal_consistency_reachable_i: WB miss (O->Y) %s -> %s\n", rb_obj_info(parent), rb_obj_info(child));
4990 check_color_i(
const VALUE child,
void *
ptr)
4993 const VALUE parent = data->parent;
4995 if (!RVALUE_WB_UNPROTECTED(data->objspace, parent) && RVALUE_WHITE_P(data->objspace, child)) {
4996 fprintf(stderr,
"verify_internal_consistency_reachable_i: WB miss (B->W) - %s -> %s\n",
4997 rb_obj_info(parent), rb_obj_info(child));
5003 check_children_i(
const VALUE child,
void *
ptr)
5006 if (check_rvalue_consistency_force(data->objspace, child, FALSE) != 0) {
5007 fprintf(stderr,
"check_children_i: %s has error (referenced from %s)",
5008 rb_obj_info(child), rb_obj_info(data->parent));
5015 verify_internal_consistency_i(
void *page_start,
void *page_end,
size_t stride,
5021 for (obj = (
VALUE)page_start; obj != (
VALUE)page_end; obj += stride) {
5022 asan_unpoisoning_object(obj) {
5023 if (!rb_gc_impl_garbage_object_p(
objspace, obj)) {
5025 data->live_object_count++;
5030 if (!gc_object_moved_p(
objspace, obj)) {
5032 rb_objspace_reachable_objects_from(obj, check_children_i, (
void *)data);
5036 if (RVALUE_OLD_P(
objspace, obj)) data->old_object_count++;
5037 if (RVALUE_WB_UNPROTECTED(
objspace, obj) && RVALUE_UNCOLLECTIBLE(
objspace, obj)) data->remembered_shady_count++;
5042 rb_objspace_reachable_objects_from(obj, check_generation_i, (
void *)data);
5045 if (is_incremental_marking(
objspace)) {
5046 if (RVALUE_BLACK_P(
objspace, obj)) {
5049 rb_objspace_reachable_objects_from(obj, check_color_i, (
void *)data);
5055 data->zombie_object_count++;
5058 fprintf(stderr,
"verify_internal_consistency_i: T_ZOMBIE has extra flags set: %s\n",
5064 fprintf(stderr,
"verify_internal_consistency_i: FL_FINALIZE %s but %s finalizer_table: %s\n",
5065 FL_TEST(obj,
FL_FINALIZE) ?
"set" :
"not set", st_is_member(finalizer_table, obj) ?
"in" :
"not in",
5080 unsigned int has_remembered_shady = FALSE;
5081 unsigned int has_remembered_old = FALSE;
5082 int remembered_old_objects = 0;
5083 int free_objects = 0;
5084 int zombie_objects = 0;
5086 short slot_size = page->slot_size;
5087 uintptr_t start = (uintptr_t)page->start;
5088 uintptr_t end = start + page->total_slots * slot_size;
5090 for (uintptr_t
ptr = start;
ptr < end;
ptr += slot_size) {
5092 asan_unpoisoning_object(val) {
5097 if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) {
5098 has_remembered_shady = TRUE;
5100 if (RVALUE_PAGE_MARKING(page, val)) {
5101 has_remembered_old = TRUE;
5102 remembered_old_objects++;
5107 if (!is_incremental_marking(
objspace) &&
5108 page->flags.has_remembered_objects == FALSE && has_remembered_old == TRUE) {
5110 for (uintptr_t
ptr = start;
ptr < end;
ptr += slot_size) {
5112 if (RVALUE_PAGE_MARKING(page, val)) {
5113 fprintf(stderr,
"marking -> %s\n", rb_obj_info(val));
5116 rb_bug(
"page %p's has_remembered_objects should be false, but there are remembered old objects (%d). %s",
5117 (
void *)page, remembered_old_objects, obj ? rb_obj_info(obj) :
"");
5120 if (page->flags.has_uncollectible_wb_unprotected_objects == FALSE && has_remembered_shady == TRUE) {
5121 rb_bug(
"page %p's has_remembered_shady should be false, but there are remembered shady objects. %s",
5122 (
void *)page, obj ? rb_obj_info(obj) :
"");
5127 if (page->free_slots != free_objects) {
5128 rb_bug(
"page %p's free_slots should be %d, but %d", (
void *)page, page->free_slots, free_objects);
5131 if (page->final_slots != zombie_objects) {
5132 rb_bug(
"page %p's final_slots should be %d, but %d", (
void *)page, page->final_slots, zombie_objects);
5135 return remembered_old_objects;
5141 int remembered_old_objects = 0;
5144 ccan_list_for_each(head, page, page_node) {
5145 asan_unlock_freelist(page);
5150 asan_unpoison_object(vp,
false);
5152 fprintf(stderr,
"freelist slot expected to be T_NONE but was: %s\n", rb_obj_info(vp));
5155 asan_poison_object(prev);
5157 asan_lock_freelist(page);
5159 if (page->flags.has_remembered_objects == FALSE) {
5160 remembered_old_objects += gc_verify_heap_page(
objspace, page,
Qfalse);
5164 return remembered_old_objects;
5170 int remembered_old_objects = 0;
5171 for (
int i = 0; i < HEAP_COUNT; i++) {
5172 remembered_old_objects += gc_verify_heap_pages_(
objspace, &((&heaps[i])->pages));
5174 return remembered_old_objects;
5183 gc_report(5,
objspace,
"gc_verify_internal_consistency: start\n");
5186 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
5188 short slot_size = page->slot_size;
5190 uintptr_t start = (uintptr_t)page->start;
5191 uintptr_t end = start + page->total_slots * slot_size;
5193 verify_internal_consistency_i((
void *)start, (
void *)end, slot_size, &data);
5196 if (data.err_count != 0) {
5197 #if RGENGC_CHECK_MODE >= 5
5198 objspace->rgengc.error_count = data.err_count;
5199 gc_marks_check(
objspace, NULL, NULL);
5202 rb_bug(
"gc_verify_internal_consistency: found internal inconsistency.");
5212 !rb_gc_multi_ractor_p()) {
5213 if (objspace_live_slots(
objspace) != data.live_object_count) {
5214 fprintf(stderr,
"heap_pages_final_slots: %"PRIdSIZE
", total_freed_objects: %"PRIdSIZE
"\n",
5216 rb_bug(
"inconsistent live slot number: expect %"PRIuSIZE
", but %"PRIuSIZE
".",
5217 objspace_live_slots(
objspace), data.live_object_count);
5222 if (
objspace->rgengc.old_objects != data.old_object_count) {
5223 rb_bug(
"inconsistent old slot number: expect %"PRIuSIZE
", but %"PRIuSIZE
".",
5224 objspace->rgengc.old_objects, data.old_object_count);
5226 if (
objspace->rgengc.uncollectible_wb_unprotected_objects != data.remembered_shady_count) {
5227 rb_bug(
"inconsistent number of wb unprotected objects: expect %"PRIuSIZE
", but %"PRIuSIZE
".",
5228 objspace->rgengc.uncollectible_wb_unprotected_objects, data.remembered_shady_count);
5233 size_t list_count = 0;
5236 VALUE z = heap_pages_deferred_final;
5239 z = RZOMBIE(z)->next;
5243 if (total_final_slots_count(
objspace) != data.zombie_object_count ||
5244 total_final_slots_count(
objspace) != list_count) {
5246 rb_bug(
"inconsistent finalizing object count:\n"
5247 " expect %"PRIuSIZE
"\n"
5248 " but %"PRIuSIZE
" zombies\n"
5249 " heap_pages_deferred_final list has %"PRIuSIZE
" items.",
5251 data.zombie_object_count,
5256 gc_report(5,
objspace,
"gc_verify_internal_consistency: OK\n");
5260 gc_verify_internal_consistency(
void *objspace_ptr)
5264 unsigned int lev = rb_gc_vm_lock();
5268 unsigned int prev_during_gc = during_gc;
5271 gc_verify_internal_consistency_(
objspace);
5273 during_gc = prev_during_gc;
5275 rb_gc_vm_unlock(lev);
5279 heap_move_pooled_pages_to_free_pages(
rb_heap_t *heap)
5281 if (heap->pooled_pages) {
5282 if (heap->free_pages) {
5283 struct heap_page *free_pages_tail = heap->free_pages;
5284 while (free_pages_tail->free_next) {
5285 free_pages_tail = free_pages_tail->free_next;
5287 free_pages_tail->free_next = heap->pooled_pages;
5290 heap->free_pages = heap->pooled_pages;
5293 heap->pooled_pages = NULL;
5300 struct heap_page *page = GET_HEAP_PAGE(obj);
5301 bits_t *uncollectible_bits = &page->uncollectible_bits[0];
5303 if (!MARKED_IN_BITMAP(uncollectible_bits, obj)) {
5304 page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
5305 MARK_IN_BITMAP(uncollectible_bits, obj);
5306 objspace->rgengc.uncollectible_wb_unprotected_objects++;
5308 #if RGENGC_PROFILE > 0
5309 objspace->profile.total_remembered_shady_object_count++;
5310 #if RGENGC_PROFILE >= 2
5327 gc_report(2,
objspace,
"gc_marks_wb_unprotected_objects: marked shady: %s\n", rb_obj_info((
VALUE)p));
5332 p += BASE_SLOT_SIZE;
5343 ccan_list_for_each(&heap->pages, page, page_node) {
5344 bits_t *mark_bits = page->mark_bits;
5345 bits_t *wbun_bits = page->wb_unprotected_bits;
5346 uintptr_t p = page->start;
5349 bits_t bits = mark_bits[0] & wbun_bits[0];
5350 bits >>= NUM_IN_PAGE(p);
5351 gc_marks_wb_unprotected_objects_plane(
objspace, p, bits);
5352 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
5354 for (j=1; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
5355 bits_t bits = mark_bits[j] & wbun_bits[j];
5357 gc_marks_wb_unprotected_objects_plane(
objspace, p, bits);
5358 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
5362 gc_mark_stacked_objects_all(
objspace);
5368 size_t retained_weak_references_count = 0;
5370 rb_darray_foreach(
objspace->weak_references, i, ptr_ptr) {
5371 if (!*ptr_ptr)
continue;
5373 VALUE obj = **ptr_ptr;
5377 if (!RVALUE_MARKED(
objspace, obj)) {
5381 retained_weak_references_count++;
5385 objspace->profile.retained_weak_references_count = retained_weak_references_count;
5387 rb_darray_clear(
objspace->weak_references);
5388 DURING_GC_COULD_MALLOC_REGION_START();
5390 rb_darray_resize_capa(&
objspace->weak_references, retained_weak_references_count);
5392 DURING_GC_COULD_MALLOC_REGION_END();
5399 if (is_incremental_marking(
objspace)) {
5400 if (RGENGC_CHECK_MODE && is_mark_stack_empty(&
objspace->mark_stack) == 0) {
5401 rb_bug(
"gc_marks_finish: mark stack is not empty (%"PRIdSIZE
").",
5402 mark_stack_size(&
objspace->mark_stack));
5406 while (gc_mark_stacked_objects_incremental(
objspace, INT_MAX) ==
false);
5408 #if RGENGC_CHECK_MODE >= 2
5409 if (gc_verify_heap_pages(
objspace) != 0) {
5410 rb_bug(
"gc_marks_finish (incremental): there are remembered old objects.");
5414 objspace->flags.during_incremental_marking = FALSE;
5416 for (
int i = 0; i < HEAP_COUNT; i++) {
5417 gc_marks_wb_unprotected_objects(
objspace, &heaps[i]);
5421 gc_update_weak_references(
objspace);
5423 #if RGENGC_CHECK_MODE >= 2
5424 gc_verify_internal_consistency(
objspace);
5427 #if RGENGC_CHECK_MODE >= 4
5429 gc_marks_check(
objspace, gc_check_after_marks_i,
"after_marks");
5434 const unsigned long r_mul =
objspace->live_ractor_cache_count > 8 ? 8 :
objspace->live_ractor_cache_count;
5436 size_t total_slots = objspace_available_slots(
objspace);
5437 size_t sweep_slots = total_slots -
objspace->marked_slots;
5438 size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio);
5439 size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio);
5440 if (min_free_slots < gc_params.heap_free_slots * r_mul) {
5441 min_free_slots = gc_params.heap_free_slots * r_mul;
5444 int full_marking = is_full_marking(
objspace);
5449 size_t total_init_slots = 0;
5450 for (
int i = 0; i < HEAP_COUNT; i++) {
5451 total_init_slots += gc_params.heap_init_slots[i] * r_mul;
5454 if (max_free_slots < total_init_slots) {
5455 max_free_slots = total_init_slots;
5458 if (sweep_slots > max_free_slots) {
5459 heap_pages_freeable_pages = (sweep_slots - max_free_slots) / HEAP_PAGE_OBJ_LIMIT;
5462 heap_pages_freeable_pages = 0;
5465 if (
objspace->heap_pages.allocatable_slots == 0 && sweep_slots < min_free_slots) {
5466 if (!full_marking) {
5467 if (
objspace->profile.count -
objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
5468 full_marking = TRUE;
5471 gc_report(1,
objspace,
"gc_marks_finish: next is full GC!!)\n");
5472 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
5479 const double r = gc_params.oldobject_limit_factor;
5480 objspace->rgengc.uncollectible_wb_unprotected_objects_limit = MAX(
5481 (
size_t)(
objspace->rgengc.uncollectible_wb_unprotected_objects * r),
5482 (
size_t)(
objspace->rgengc.old_objects * gc_params.uncollectible_wb_unprotected_objects_limit_ratio)
5484 objspace->rgengc.old_objects_limit = (size_t)(
objspace->rgengc.old_objects * r);
5487 if (
objspace->rgengc.uncollectible_wb_unprotected_objects >
objspace->rgengc.uncollectible_wb_unprotected_objects_limit) {
5488 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_SHADY;
5491 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDGEN;
5494 gc_report(1,
objspace,
"gc_marks_finish (marks %"PRIdSIZE
" objects, "
5495 "old %"PRIdSIZE
" objects, total %"PRIdSIZE
" slots, "
5496 "sweep %"PRIdSIZE
" slots, allocatable %"PRIdSIZE
" slots, next GC: %s)\n",
5498 gc_needs_major_flags ?
"major" :
"minor");
5502 rb_ractor_finish_marking();
5508 gc_compact_heap_cursors_met_p(
rb_heap_t *heap)
5510 return heap->sweeping_page == heap->compact_cursor;
5517 size_t obj_size = rb_gc_obj_optimal_size(obj);
5518 if (obj_size == 0) {
5523 if (rb_gc_impl_size_allocatable_p(obj_size)) {
5524 idx = heap_idx_for_size(obj_size);
5534 GC_ASSERT(gc_is_moveable_obj(
objspace, src));
5537 uint32_t orig_shape = 0;
5538 uint32_t new_shape = 0;
5540 if (gc_compact_heap_cursors_met_p(dest_pool)) {
5541 return dest_pool != heap;
5545 orig_shape = rb_gc_get_shape(src);
5547 if (dest_pool != heap) {
5548 new_shape = rb_gc_rebuild_shape(src, dest_pool - heaps);
5550 if (new_shape == 0) {
5556 while (!try_move(
objspace, dest_pool, dest_pool->free_pages, src)) {
5558 .page = dest_pool->sweeping_page,
5567 lock_page_body(
objspace, GET_PAGE_BODY(src));
5568 gc_sweep_page(
objspace, dest_pool, &ctx);
5569 unlock_page_body(
objspace, GET_PAGE_BODY(src));
5571 if (dest_pool->sweeping_page->free_slots > 0) {
5572 heap_add_freepage(dest_pool, dest_pool->sweeping_page);
5575 dest_pool->sweeping_page = ccan_list_next(&dest_pool->pages, dest_pool->sweeping_page, page_node);
5576 if (gc_compact_heap_cursors_met_p(dest_pool)) {
5577 return dest_pool != heap;
5581 if (orig_shape != 0) {
5582 if (new_shape != 0) {
5584 rb_gc_set_shape(dest, new_shape);
5586 RMOVED(src)->original_shape_id = orig_shape;
5595 short slot_size = page->slot_size;
5596 short slot_bits = slot_size / BASE_SLOT_SIZE;
5597 GC_ASSERT(slot_bits > 0);
5601 GC_ASSERT(vp % BASE_SLOT_SIZE == 0);
5606 if (gc_is_moveable_obj(
objspace, vp)) {
5607 if (!gc_compact_move(
objspace, heap, vp)) {
5614 bitset >>= slot_bits;
5624 GC_ASSERT(page == heap->compact_cursor);
5626 bits_t *mark_bits, *pin_bits;
5628 uintptr_t p = page->start;
5630 mark_bits = page->mark_bits;
5631 pin_bits = page->pinned_bits;
5634 bitset = (mark_bits[0] & ~pin_bits[0]);
5635 bitset >>= NUM_IN_PAGE(p);
5637 if (!gc_compact_plane(
objspace, heap, (uintptr_t)p, bitset, page))
5640 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
5642 for (
int j = 1; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
5643 bitset = (mark_bits[j] & ~pin_bits[j]);
5645 if (!gc_compact_plane(
objspace, heap, (uintptr_t)p, bitset, page))
5648 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
5657 for (
int i = 0; i < HEAP_COUNT; i++) {
5660 if (heap->total_pages > 0 &&
5661 !gc_compact_heap_cursors_met_p(heap)) {
5673 #if RGENGC_CHECK_MODE >= 2
5674 gc_verify_internal_consistency(
objspace);
5677 while (!gc_compact_all_compacted_p(
objspace)) {
5678 for (
int i = 0; i < HEAP_COUNT; i++) {
5681 if (gc_compact_heap_cursors_met_p(heap)) {
5685 struct heap_page *start_page = heap->compact_cursor;
5687 if (!gc_compact_page(
objspace, heap, start_page)) {
5688 lock_page_body(
objspace, start_page->body);
5695 lock_page_body(
objspace, start_page->body);
5696 heap->compact_cursor = ccan_list_prev(&heap->pages, heap->compact_cursor, page_node);
5702 #if RGENGC_CHECK_MODE >= 2
5703 gc_verify_internal_consistency(
objspace);
5710 gc_report(1,
objspace,
"gc_marks_rest\n");
5712 for (
int i = 0; i < HEAP_COUNT; i++) {
5713 (&heaps[i])->pooled_pages = NULL;
5716 if (is_incremental_marking(
objspace)) {
5717 while (gc_mark_stacked_objects_incremental(
objspace, INT_MAX) == FALSE);
5720 gc_mark_stacked_objects_all(
objspace);
5729 bool marking_finished =
false;
5732 if (gc_mark_stacked_objects_incremental(
objspace, slots)) {
5735 marking_finished =
true;
5738 return marking_finished;
5744 GC_ASSERT(dont_gc_val() == FALSE ||
objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
5745 bool marking_finished =
true;
5749 if (heap->free_pages) {
5750 gc_report(2,
objspace,
"gc_marks_continue: has pooled pages");
5755 gc_report(2,
objspace,
"gc_marks_continue: no more pooled pages (stack depth: %"PRIdSIZE
").\n",
5756 mark_stack_size(&
objspace->mark_stack));
5757 heap->force_incremental_marking_finish_count++;
5763 return marking_finished;
5770 gc_report(1,
objspace,
"gc_marks_start: (%s)\n", full_mark ?
"full" :
"minor");
5771 gc_mode_transition(
objspace, gc_mode_marking);
5774 size_t incremental_marking_steps = (
objspace->rincgc.pooled_slots / INCREMENTAL_MARK_STEP_ALLOCATIONS) + 1;
5775 objspace->rincgc.step_slots = (
objspace->marked_slots * 2) / incremental_marking_steps;
5777 if (0) fprintf(stderr,
"objspace->marked_slots: %"PRIdSIZE
", "
5778 "objspace->rincgc.pooled_page_num: %"PRIdSIZE
", "
5779 "objspace->rincgc.step_slots: %"PRIdSIZE
", \n",
5781 objspace->flags.during_minor_gc = FALSE;
5782 if (ruby_enable_autocompact) {
5783 objspace->flags.during_compacting |= TRUE;
5785 objspace->profile.major_gc_count++;
5786 objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
5791 for (
int i = 0; i < HEAP_COUNT; i++) {
5793 rgengc_mark_and_rememberset_clear(
objspace, heap);
5794 heap_move_pooled_pages_to_free_pages(heap);
5796 if (
objspace->flags.during_compacting) {
5799 ccan_list_for_each(&heap->pages, page, page_node) {
5800 page->pinned_slots = 0;
5806 objspace->flags.during_minor_gc = TRUE;
5808 objspace->rgengc.old_objects +
objspace->rgengc.uncollectible_wb_unprotected_objects;
5809 objspace->profile.minor_gc_count++;
5811 for (
int i = 0; i < HEAP_COUNT; i++) {
5812 rgengc_rememberset_mark(
objspace, &heaps[i]);
5818 gc_report(1,
objspace,
"gc_marks_start: (%s) end, stack in %"PRIdSIZE
"\n",
5819 full_mark ?
"full" :
"minor", mark_stack_size(&
objspace->mark_stack));
5825 gc_prof_mark_timer_start(
objspace);
5828 bool marking_finished =
false;
5832 gc_marks_start(
objspace, full_mark);
5833 if (!is_incremental_marking(
objspace)) {
5835 marking_finished =
true;
5838 #if RGENGC_PROFILE > 0
5841 record->old_objects =
objspace->rgengc.old_objects;
5848 return marking_finished;
5856 if (level <= RGENGC_DEBUG) {
5860 const char *status =
" ";
5863 status = is_full_marking(
objspace) ?
"+" :
"-";
5869 if (is_incremental_marking(
objspace)) {
5874 va_start(args, fmt);
5875 vsnprintf(buf, 1024, fmt, args);
5878 fprintf(out,
"%s|", status);
5888 struct heap_page *page = GET_HEAP_PAGE(obj);
5889 bits_t *bits = &page->remembered_bits[0];
5891 if (MARKED_IN_BITMAP(bits, obj)) {
5895 page->flags.has_remembered_objects = TRUE;
5896 MARK_IN_BITMAP(bits, obj);
5907 gc_report(6,
objspace,
"rgengc_remember: %s %s\n", rb_obj_info(obj),
5908 RVALUE_REMEMBERED(
objspace, obj) ?
"was already remembered" :
"is remembered now");
5910 check_rvalue_consistency(
objspace, obj);
5912 if (RGENGC_CHECK_MODE) {
5913 if (RVALUE_WB_UNPROTECTED(
objspace, obj))
rb_bug(
"rgengc_remember: %s is not wb protected.", rb_obj_info(obj));
5916 #if RGENGC_PROFILE > 0
5917 if (!RVALUE_REMEMBERED(
objspace, obj)) {
5918 if (RVALUE_WB_UNPROTECTED(
objspace, obj) == 0) {
5919 objspace->profile.total_remembered_normal_object_count++;
5920 #if RGENGC_PROFILE >= 2
5927 return rgengc_remembersetbits_set(
objspace, obj);
5930 #ifndef PROFILE_REMEMBERSET_MARK
5931 #define PROFILE_REMEMBERSET_MARK 0
5941 gc_report(2,
objspace,
"rgengc_rememberset_mark: mark %s\n", rb_obj_info(obj));
5942 GC_ASSERT(RVALUE_UNCOLLECTIBLE(
objspace, obj));
5943 GC_ASSERT(RVALUE_OLD_P(
objspace, obj) || RVALUE_WB_UNPROTECTED(
objspace, obj));
5947 p += BASE_SLOT_SIZE;
5958 #if PROFILE_REMEMBERSET_MARK
5959 int has_old = 0, has_shady = 0, has_both = 0, skip = 0;
5961 gc_report(1,
objspace,
"rgengc_rememberset_mark: start\n");
5963 ccan_list_for_each(&heap->pages, page, page_node) {
5964 if (page->flags.has_remembered_objects | page->flags.has_uncollectible_wb_unprotected_objects) {
5965 uintptr_t p = page->start;
5966 bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT];
5967 bits_t *remembered_bits = page->remembered_bits;
5968 bits_t *uncollectible_bits = page->uncollectible_bits;
5969 bits_t *wb_unprotected_bits = page->wb_unprotected_bits;
5970 #if PROFILE_REMEMBERSET_MARK
5971 if (page->flags.has_remembered_objects && page->flags.has_uncollectible_wb_unprotected_objects) has_both++;
5972 else if (page->flags.has_remembered_objects) has_old++;
5973 else if (page->flags.has_uncollectible_wb_unprotected_objects) has_shady++;
5975 for (j=0; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
5976 bits[j] = remembered_bits[j] | (uncollectible_bits[j] & wb_unprotected_bits[j]);
5977 remembered_bits[j] = 0;
5979 page->flags.has_remembered_objects = FALSE;
5982 bitset >>= NUM_IN_PAGE(p);
5983 rgengc_rememberset_mark_plane(
objspace, p, bitset);
5984 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
5986 for (j=1; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
5988 rgengc_rememberset_mark_plane(
objspace, p, bitset);
5989 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
5992 #if PROFILE_REMEMBERSET_MARK
5999 #if PROFILE_REMEMBERSET_MARK
6000 fprintf(stderr,
"%d\t%d\t%d\t%d\n", has_both, has_old, has_shady, skip);
6002 gc_report(1,
objspace,
"rgengc_rememberset_mark: finished\n");
6010 ccan_list_for_each(&heap->pages, page, page_node) {
6011 memset(&page->mark_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6012 memset(&page->uncollectible_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6013 memset(&page->marking_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6014 memset(&page->remembered_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6015 memset(&page->pinned_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6016 page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
6017 page->flags.has_remembered_objects = FALSE;
6028 if (RGENGC_CHECK_MODE) {
6029 if (!RVALUE_OLD_P(
objspace, a))
rb_bug(
"gc_writebarrier_generational: %s is not an old object.", rb_obj_info(a));
6030 if ( RVALUE_OLD_P(
objspace, b))
rb_bug(
"gc_writebarrier_generational: %s is an old object.", rb_obj_info(b));
6031 if (is_incremental_marking(
objspace))
rb_bug(
"gc_writebarrier_generational: called while incremental marking: %s -> %s", rb_obj_info(a), rb_obj_info(b));
6035 if (!RVALUE_REMEMBERED(
objspace, a)) {
6036 int lev = rb_gc_vm_lock_no_barrier();
6040 rb_gc_vm_unlock_no_barrier(lev);
6042 gc_report(1,
objspace,
"gc_writebarrier_generational: %s (remembered) -> %s\n", rb_obj_info(a), rb_obj_info(b));
6045 check_rvalue_consistency(
objspace, a);
6046 check_rvalue_consistency(
objspace, b);
6052 gc_mark_set_parent(
objspace, parent);
6053 rgengc_check_relation(
objspace, obj);
6054 if (gc_mark_set(
objspace, obj) == FALSE)
return;
6064 gc_report(2,
objspace,
"gc_writebarrier_incremental: [LG] %p -> %s\n", (
void *)a, rb_obj_info(b));
6068 if (!RVALUE_WB_UNPROTECTED(
objspace, a)) {
6069 gc_report(2,
objspace,
"gc_writebarrier_incremental: [IN] %p -> %s\n", (
void *)a, rb_obj_info(b));
6078 MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(b), b);
6084 rb_gc_impl_writebarrier(
void *objspace_ptr,
VALUE a,
VALUE b)
6088 if (RGENGC_CHECK_MODE) {
6101 if (!is_incremental_marking(
objspace)) {
6106 gc_writebarrier_generational(a, b,
objspace);
6112 int lev = rb_gc_vm_lock_no_barrier();
6114 if (is_incremental_marking(
objspace)) {
6115 gc_writebarrier_incremental(a, b,
objspace);
6121 rb_gc_vm_unlock_no_barrier(lev);
6123 if (retry)
goto retry;
6129 rb_gc_impl_writebarrier_unprotect(
void *objspace_ptr,
VALUE obj)
6133 if (RVALUE_WB_UNPROTECTED(
objspace, obj)) {
6137 gc_report(2,
objspace,
"rb_gc_writebarrier_unprotect: %s %s\n", rb_obj_info(obj),
6138 RVALUE_REMEMBERED(
objspace, obj) ?
" (already remembered)" :
"");
6140 unsigned int lev = rb_gc_vm_lock_no_barrier();
6143 gc_report(1,
objspace,
"rb_gc_writebarrier_unprotect: %s\n", rb_obj_info(obj));
6146 gc_remember_unprotected(
objspace, obj);
6149 objspace->profile.total_shade_operation_count++;
6150 #if RGENGC_PROFILE >= 2
6156 RVALUE_AGE_RESET(obj);
6159 RB_DEBUG_COUNTER_INC(obj_wb_unprotect);
6160 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
6162 rb_gc_vm_unlock_no_barrier(lev);
6167 rb_gc_impl_copy_attributes(
void *objspace_ptr,
VALUE dest,
VALUE obj)
6171 if (RVALUE_WB_UNPROTECTED(
objspace, obj)) {
6172 rb_gc_impl_writebarrier_unprotect(
objspace, dest);
6174 rb_gc_impl_copy_finalizer(
objspace, dest, obj);
6178 rb_gc_impl_active_gc_name(
void)
6184 rb_gc_impl_writebarrier_remember(
void *objspace_ptr,
VALUE obj)
6188 gc_report(1,
objspace,
"rb_gc_writebarrier_remember: %s\n", rb_obj_info(obj));
6190 if (is_incremental_marking(
objspace)) {
6191 if (RVALUE_BLACK_P(
objspace, obj)) {
6204 rb_gc_impl_obj_flags(
void *objspace_ptr,
VALUE obj,
ID* flags,
size_t max)
6208 static ID ID_marked;
6209 static ID ID_wb_protected, ID_old, ID_marking, ID_uncollectible, ID_pinned;
6212 #define I(s) ID_##s = rb_intern(#s);
6222 if (RVALUE_WB_UNPROTECTED(
objspace, obj) == 0 && n < max) flags[n++] = ID_wb_protected;
6223 if (RVALUE_OLD_P(
objspace, obj) && n < max) flags[n++] = ID_old;
6224 if (RVALUE_UNCOLLECTIBLE(
objspace, obj) && n < max) flags[n++] = ID_uncollectible;
6225 if (RVALUE_MARKING(
objspace, obj) && n < max) flags[n++] = ID_marking;
6226 if (RVALUE_MARKED(
objspace, obj) && n < max) flags[n++] = ID_marked;
6227 if (RVALUE_PINNED(
objspace, obj) && n < max) flags[n++] = ID_pinned;
6232 rb_gc_impl_ractor_cache_alloc(
void *objspace_ptr,
void *ractor)
6236 objspace->live_ractor_cache_count++;
6242 rb_gc_impl_ractor_cache_free(
void *objspace_ptr,
void *cache)
6246 objspace->live_ractor_cache_count--;
6248 gc_ractor_newobj_cache_clear(cache, NULL);
6255 if (!heap->free_pages) {
6256 if (!heap_page_allocate_and_initialize(
objspace, heap)) {
6257 objspace->heap_pages.allocatable_slots = 1;
6258 heap_page_allocate_and_initialize(
objspace, heap);
6266 if (dont_gc_val() || during_gc || ruby_disable_gc) {
6267 for (
int i = 0; i < HEAP_COUNT; i++) {
6284 size_t old_limit = malloc_limit;
6286 if (inc > malloc_limit) {
6287 malloc_limit = (size_t)(inc * gc_params.malloc_limit_growth_factor);
6288 if (malloc_limit > gc_params.malloc_limit_max) {
6289 malloc_limit = gc_params.malloc_limit_max;
6293 malloc_limit = (size_t)(malloc_limit * 0.98);
6294 if (malloc_limit < gc_params.malloc_limit_min) {
6295 malloc_limit = gc_params.malloc_limit_min;
6300 if (old_limit != malloc_limit) {
6301 fprintf(stderr,
"[%"PRIuSIZE
"] malloc_limit: %"PRIuSIZE
" -> %"PRIuSIZE
"\n",
6305 fprintf(stderr,
"[%"PRIuSIZE
"] malloc_limit: not changed (%"PRIuSIZE
")\n",
6312 #if RGENGC_ESTIMATE_OLDMALLOC
6314 if (
objspace->rgengc.oldmalloc_increase >
objspace->rgengc.oldmalloc_increase_limit) {
6315 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDMALLOC;
6316 objspace->rgengc.oldmalloc_increase_limit =
6317 (size_t)(
objspace->rgengc.oldmalloc_increase_limit * gc_params.oldmalloc_limit_growth_factor);
6319 if (
objspace->rgengc.oldmalloc_increase_limit > gc_params.oldmalloc_limit_max) {
6320 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_max;
6324 if (0) fprintf(stderr,
"%"PRIdSIZE
"\t%d\t%"PRIuSIZE
"\t%"PRIuSIZE
"\t%"PRIdSIZE
"\n",
6326 gc_needs_major_flags,
6327 objspace->rgengc.oldmalloc_increase,
6328 objspace->rgengc.oldmalloc_increase_limit,
6329 gc_params.oldmalloc_limit_max);
6333 objspace->rgengc.oldmalloc_increase = 0;
6335 if ((
objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) {
6336 objspace->rgengc.oldmalloc_increase_limit =
6337 (size_t)(
objspace->rgengc.oldmalloc_increase_limit / ((gc_params.oldmalloc_limit_growth_factor - 1)/10 + 1));
6338 if (
objspace->rgengc.oldmalloc_increase_limit < gc_params.oldmalloc_limit_min) {
6339 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
6351 int lev = rb_gc_vm_lock();
6353 #if GC_PROFILE_MORE_DETAIL
6354 objspace->profile.prepare_time = getrusage_time();
6359 #if GC_PROFILE_MORE_DETAIL
6360 objspace->profile.prepare_time = getrusage_time() -
objspace->profile.prepare_time;
6365 rb_gc_vm_unlock(lev);
6373 unsigned int do_full_mark = !!(reason & GPR_FLAG_FULL_MARK);
6376 objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP);
6378 if (!rb_darray_size(
objspace->heap_pages.sorted))
return TRUE;
6379 if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(
objspace))
return TRUE;
6381 GC_ASSERT(gc_mode(
objspace) == gc_mode_none);
6382 GC_ASSERT(!is_lazy_sweeping(
objspace));
6383 GC_ASSERT(!is_incremental_marking(
objspace));
6385 unsigned int lock_lev;
6386 gc_enter(
objspace, gc_enter_event_start, &lock_lev);
6388 #if RGENGC_CHECK_MODE >= 2
6389 gc_verify_internal_consistency(
objspace);
6392 if (ruby_gc_stressful) {
6393 int flag =
FIXNUM_P(ruby_gc_stress_mode) ?
FIX2INT(ruby_gc_stress_mode) : 0;
6395 if ((flag & (1 << gc_stress_no_major)) == 0) {
6396 do_full_mark = TRUE;
6399 objspace->flags.immediate_sweep = !(flag & (1<<gc_stress_no_immediate_sweep));
6402 if (gc_needs_major_flags) {
6403 reason |= gc_needs_major_flags;
6404 do_full_mark = TRUE;
6408 if (!gc_config_full_mark_val) {
6409 do_full_mark = FALSE;
6411 gc_needs_major_flags = GPR_FLAG_NONE;
6413 if (do_full_mark && (reason & GPR_FLAG_MAJOR_MASK) == 0) {
6414 reason |= GPR_FLAG_MAJOR_BY_FORCE;
6417 if (
objspace->flags.dont_incremental ||
6418 reason & GPR_FLAG_IMMEDIATE_MARK ||
6419 ruby_gc_stressful) {
6420 objspace->flags.during_incremental_marking = FALSE;
6423 objspace->flags.during_incremental_marking = do_full_mark;
6427 if (do_full_mark && ruby_enable_autocompact) {
6428 objspace->flags.during_compacting = TRUE;
6429 #if RGENGC_CHECK_MODE
6430 objspace->rcompactor.compare_func = ruby_autocompact_compare_func;
6434 objspace->flags.during_compacting = !!(reason & GPR_FLAG_COMPACT);
6437 if (!GC_ENABLE_LAZY_SWEEP ||
objspace->flags.dont_incremental) {
6438 objspace->flags.immediate_sweep = TRUE;
6441 if (
objspace->flags.immediate_sweep) reason |= GPR_FLAG_IMMEDIATE_SWEEP;
6443 gc_report(1,
objspace,
"gc_start(reason: %x) => %u, %d, %d\n",
6445 do_full_mark, !is_incremental_marking(
objspace),
objspace->flags.immediate_sweep);
6447 #if USE_DEBUG_COUNTER
6448 RB_DEBUG_COUNTER_INC(gc_count);
6450 if (reason & GPR_FLAG_MAJOR_MASK) {
6451 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_nofree, reason & GPR_FLAG_MAJOR_BY_NOFREE);
6452 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldgen, reason & GPR_FLAG_MAJOR_BY_OLDGEN);
6453 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_shady, reason & GPR_FLAG_MAJOR_BY_SHADY);
6454 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_force, reason & GPR_FLAG_MAJOR_BY_FORCE);
6455 #if RGENGC_ESTIMATE_OLDMALLOC
6456 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldmalloc, reason & GPR_FLAG_MAJOR_BY_OLDMALLOC);
6460 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_newobj, reason & GPR_FLAG_NEWOBJ);
6461 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_malloc, reason & GPR_FLAG_MALLOC);
6462 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_method, reason & GPR_FLAG_METHOD);
6463 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_capi, reason & GPR_FLAG_CAPI);
6464 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_stress, reason & GPR_FLAG_STRESS);
6469 objspace->profile.latest_gc_info = reason;
6470 objspace->profile.total_allocated_objects_at_gc_start = total_allocated_objects(
objspace);
6471 objspace->profile.heap_used_at_gc_start = rb_darray_size(
objspace->heap_pages.sorted);
6472 objspace->profile.weak_references_count = 0;
6473 objspace->profile.retained_weak_references_count = 0;
6474 gc_prof_setup_new_record(
objspace, reason);
6475 gc_reset_malloc_info(
objspace, do_full_mark);
6479 GC_ASSERT(during_gc);
6483 if (gc_marks(
objspace, do_full_mark)) {
6489 gc_exit(
objspace, gc_enter_event_start, &lock_lev);
6497 unsigned int lock_lev;
6498 gc_enter(
objspace, gc_enter_event_rest, &lock_lev);
6500 if (RGENGC_CHECK_MODE >= 2) gc_verify_internal_consistency(
objspace);
6502 if (is_incremental_marking(
objspace)) {
6516 gc_exit(
objspace, gc_enter_event_rest, &lock_lev);
6522 unsigned int reason;
6531 if (is_full_marking(
objspace)) buff[i++] =
'F';
6532 if (is_incremental_marking(
objspace)) buff[i++] =
'I';
6536 if (is_lazy_sweeping(
objspace)) buff[i++] =
'L';
6547 static char buff[0x10];
6548 gc_current_status_fill(
objspace, buff);
6552 #if PRINT_ENTER_EXIT_TICK
6554 static tick_t last_exit_tick;
6555 static tick_t enter_tick;
6556 static int enter_count = 0;
6557 static char last_gc_status[0x10];
6562 if (direction == 0) {
6564 enter_tick = tick();
6565 gc_current_status_fill(
objspace, last_gc_status);
6568 tick_t exit_tick = tick();
6569 char current_gc_status[0x10];
6570 gc_current_status_fill(
objspace, current_gc_status);
6573 fprintf(stderr,
"%"PRItick
"\t%"PRItick
"\t%s\t[%s->%s|%c]\n",
6574 enter_tick - last_exit_tick,
6575 exit_tick - enter_tick,
6577 last_gc_status, current_gc_status,
6578 (
objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ?
'+' :
'-');
6579 last_exit_tick = exit_tick;
6582 fprintf(stderr,
"%"PRItick
"\t%"PRItick
"\t%s\t[%s->%s|%c]\n",
6584 exit_tick - enter_tick,
6586 last_gc_status, current_gc_status,
6587 (
objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ?
'+' :
'-');
6600 gc_enter_event_cstr(
enum gc_enter_event event)
6603 case gc_enter_event_start:
return "start";
6604 case gc_enter_event_continue:
return "continue";
6605 case gc_enter_event_rest:
return "rest";
6606 case gc_enter_event_finalizer:
return "finalizer";
6612 gc_enter_count(
enum gc_enter_event event)
6615 case gc_enter_event_start: RB_DEBUG_COUNTER_INC(gc_enter_start);
break;
6616 case gc_enter_event_continue: RB_DEBUG_COUNTER_INC(gc_enter_continue);
break;
6617 case gc_enter_event_rest: RB_DEBUG_COUNTER_INC(gc_enter_rest);
break;
6618 case gc_enter_event_finalizer: RB_DEBUG_COUNTER_INC(gc_enter_finalizer);
break;
6622 static bool current_process_time(
struct timespec *ts);
6625 gc_clock_start(
struct timespec *ts)
6627 if (!current_process_time(ts)) {
6633 static unsigned long long
6638 if ((ts->tv_sec > 0 || ts->tv_nsec > 0) &&
6639 current_process_time(&end_time) &&
6640 end_time.tv_sec >= ts->tv_sec) {
6641 return (
unsigned long long)(end_time.tv_sec - ts->tv_sec) * (1000 * 1000 * 1000) +
6642 (end_time.tv_nsec - ts->tv_nsec);
6651 *lock_lev = rb_gc_vm_lock();
6654 case gc_enter_event_rest:
6657 case gc_enter_event_start:
6658 case gc_enter_event_continue:
6666 gc_enter_count(event);
6668 if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(
objspace);
6671 RUBY_DEBUG_LOG(
"%s (%s)",gc_enter_event_cstr(event), gc_current_status(
objspace));
6672 gc_report(1,
objspace,
"gc_enter: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(
objspace));
6673 gc_record(
objspace, 0, gc_enter_event_cstr(event));
6681 GC_ASSERT(during_gc != 0);
6685 gc_record(
objspace, 1, gc_enter_event_cstr(event));
6686 RUBY_DEBUG_LOG(
"%s (%s)", gc_enter_event_cstr(event), gc_current_status(
objspace));
6687 gc_report(1,
objspace,
"gc_exit: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(
objspace));
6690 rb_gc_vm_unlock(*lock_lev);
6694 #define MEASURE_GC (objspace->flags.measure_gc)
6700 GC_ASSERT(during_gc != 0);
6703 gc_clock_start(&
objspace->profile.marking_start_time);
6710 GC_ASSERT(during_gc != 0);
6713 objspace->profile.marking_time_ns += gc_clock_end(&
objspace->profile.marking_start_time);
6720 GC_ASSERT(during_gc != 0);
6723 gc_clock_start(&
objspace->profile.sweeping_start_time);
6730 GC_ASSERT(during_gc != 0);
6733 objspace->profile.sweeping_time_ns += gc_clock_end(&
objspace->profile.sweeping_start_time);
6738 gc_with_gvl(
void *
ptr)
6741 return (
void *)(
VALUE)garbage_collect(oar->objspace, oar->reason);
6744 int ruby_thread_has_gvl_p(
void);
6749 if (dont_gc_val())
return TRUE;
6750 if (ruby_thread_has_gvl_p()) {
6751 return garbage_collect(
objspace, reason);
6757 oar.reason = reason;
6762 fprintf(stderr,
"[FATAL] failed to allocate memory\n");
6769 gc_set_candidate_object_i(
void *vstart,
void *vend,
size_t stride,
void *data)
6774 for (; v != (
VALUE)vend; v += stride) {
6775 asan_unpoisoning_object(v) {
6781 rb_gc_prepare_heap_process_object(v);
6783 RVALUE_AGE_SET_CANDIDATE(
objspace, v);
6793 rb_gc_impl_start(
void *objspace_ptr,
bool full_mark,
bool immediate_mark,
bool immediate_sweep,
bool compact)
6796 unsigned int reason = (GPR_FLAG_FULL_MARK |
6797 GPR_FLAG_IMMEDIATE_MARK |
6798 GPR_FLAG_IMMEDIATE_SWEEP |
6801 int full_marking_p = gc_config_full_mark_val;
6802 gc_config_full_mark_set(TRUE);
6806 GC_ASSERT(GC_COMPACTION_SUPPORTED);
6808 reason |= GPR_FLAG_COMPACT;
6811 if (!full_mark) reason &= ~GPR_FLAG_FULL_MARK;
6812 if (!immediate_mark) reason &= ~GPR_FLAG_IMMEDIATE_MARK;
6813 if (!immediate_sweep) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
6819 gc_config_full_mark_set(full_marking_p);
6823 rb_gc_impl_prepare_heap(
void *objspace_ptr)
6827 size_t orig_total_slots = objspace_available_slots(
objspace);
6828 size_t orig_allocatable_slots =
objspace->heap_pages.allocatable_slots;
6830 rb_gc_impl_each_objects(
objspace, gc_set_candidate_object_i, objspace_ptr);
6832 double orig_max_free_slots = gc_params.heap_free_slots_max_ratio;
6834 gc_params.heap_free_slots_max_ratio = 0.0;
6835 rb_gc_impl_start(
objspace,
true,
true,
true,
true);
6836 gc_params.heap_free_slots_max_ratio = orig_max_free_slots;
6838 objspace->heap_pages.allocatable_slots = 0;
6839 heap_pages_free_unused_pages(objspace_ptr);
6840 GC_ASSERT(
objspace->empty_pages_count == 0);
6841 objspace->heap_pages.allocatable_slots = orig_allocatable_slots;
6843 size_t total_slots = objspace_available_slots(
objspace);
6844 if (orig_total_slots > total_slots) {
6845 objspace->heap_pages.allocatable_slots += orig_total_slots - total_slots;
6848 #if defined(HAVE_MALLOC_TRIM) && !defined(RUBY_ALTERNATIVE_MALLOC_HEADER)
6894 GC_ASSERT(st_is_member(finalizer_table, obj));
6898 GC_ASSERT(RVALUE_MARKED(
objspace, obj));
6899 GC_ASSERT(!RVALUE_PINNED(
objspace, obj));
6911 void rb_mv_generic_ivar(
VALUE src,
VALUE dst);
6921 gc_report(4,
objspace,
"Moving object: %p -> %p\n", (
void *)src, (
void *)dest);
6924 GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest));
6926 GC_ASSERT(!RVALUE_MARKING(
objspace, src));
6929 marked = RVALUE_MARKED(
objspace, src);
6930 wb_unprotected = RVALUE_WB_UNPROTECTED(
objspace, src);
6931 uncollectible = RVALUE_UNCOLLECTIBLE(
objspace, src);
6932 bool remembered = RVALUE_REMEMBERED(
objspace, src);
6933 age = RVALUE_AGE_GET(src);
6936 CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(src), src);
6937 CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(src), src);
6938 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(src), src);
6939 CLEAR_IN_BITMAP(GET_HEAP_PAGE(src)->remembered_bits, src);
6943 DURING_GC_COULD_MALLOC_REGION_START();
6945 rb_mv_generic_ivar(src, dest);
6947 DURING_GC_COULD_MALLOC_REGION_END();
6953 st_data_t srcid = (st_data_t)src,
id;
6955 gc_report(4,
objspace,
"Moving object with seen id: %p -> %p\n", (
void *)src, (
void *)dest);
6957 DURING_GC_COULD_MALLOC_REGION_START();
6959 if (!st_delete(
objspace->obj_to_id_tbl, &srcid, &
id)) {
6960 rb_bug(
"gc_move: object ID seen, but not in mapping table: %s", rb_obj_info((
VALUE)src));
6963 st_insert(
objspace->obj_to_id_tbl, (st_data_t)dest,
id);
6965 DURING_GC_COULD_MALLOC_REGION_END();
6968 GC_ASSERT(!st_lookup(
objspace->obj_to_id_tbl, (st_data_t)src, NULL));
6972 memcpy((
void *)dest, (
void *)src, MIN(src_slot_size, slot_size));
6974 if (RVALUE_OVERHEAD > 0) {
6975 void *dest_overhead = (
void *)(((uintptr_t)dest) + slot_size - RVALUE_OVERHEAD);
6976 void *src_overhead = (
void *)(((uintptr_t)src) + src_slot_size - RVALUE_OVERHEAD);
6978 memcpy(dest_overhead, src_overhead, RVALUE_OVERHEAD);
6981 memset((
void *)src, 0, src_slot_size);
6982 RVALUE_AGE_RESET(src);
6986 MARK_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
6989 CLEAR_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
6993 MARK_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
6996 CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
6999 if (wb_unprotected) {
7000 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
7003 CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
7006 if (uncollectible) {
7007 MARK_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
7010 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
7013 RVALUE_AGE_SET(dest, age);
7016 RMOVED(src)->dummy =
Qundef;
7017 RMOVED(src)->destination = dest;
7020 GET_HEAP_PAGE(src)->heap->total_freed_objects++;
7021 GET_HEAP_PAGE(dest)->heap->total_allocated_objects++;
7026 #if GC_CAN_COMPILE_COMPACTION
7028 compare_pinned_slots(
const void *left,
const void *right,
void *dummy)
7033 left_page = *(
struct heap_page *
const *)left;
7034 right_page = *(
struct heap_page *
const *)right;
7036 return left_page->pinned_slots - right_page->pinned_slots;
7040 compare_free_slots(
const void *left,
const void *right,
void *dummy)
7045 left_page = *(
struct heap_page *
const *)left;
7046 right_page = *(
struct heap_page *
const *)right;
7048 return left_page->free_slots - right_page->free_slots;
7054 for (
int j = 0; j < HEAP_COUNT; j++) {
7057 size_t total_pages = heap->total_pages;
7059 struct heap_page *page = 0, **page_list = malloc(size);
7062 heap->free_pages = NULL;
7063 ccan_list_for_each(&heap->pages, page, page_node) {
7064 page_list[i++] = page;
7068 GC_ASSERT((
size_t)i == total_pages);
7075 ccan_list_head_init(&heap->pages);
7077 for (i = 0; i < total_pages; i++) {
7078 ccan_list_add(&heap->pages, &page_list[i]->page_node);
7079 if (page_list[i]->free_slots != 0) {
7080 heap_add_freepage(heap, page_list[i]);
7090 rb_gc_impl_object_moved_p(
void *objspace_ptr,
VALUE obj)
7092 return gc_object_moved_p(objspace_ptr, obj);
7100 page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
7101 page->flags.has_remembered_objects = FALSE;
7104 for (; v != (
VALUE)vend; v += stride) {
7105 asan_unpoisoning_object(v) {
7112 if (RVALUE_WB_UNPROTECTED(
objspace, v)) {
7113 page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
7115 if (RVALUE_REMEMBERED(
objspace, v)) {
7116 page->flags.has_remembered_objects = TRUE;
7118 if (page->flags.before_sweep) {
7120 rb_gc_update_object_references(
objspace, v);
7124 rb_gc_update_object_references(
objspace, v);
7136 objspace->flags.during_reference_updating =
true;
7140 for (
int i = 0; i < HEAP_COUNT; i++) {
7141 bool should_set_mark_bits = TRUE;
7144 ccan_list_for_each(&heap->pages, page, page_node) {
7145 uintptr_t start = (uintptr_t)page->start;
7146 uintptr_t end = start + (page->total_slots * heap->slot_size);
7148 gc_ref_update((
void *)start, (
void *)end, heap->slot_size,
objspace, page);
7149 if (page == heap->sweeping_page) {
7150 should_set_mark_bits = FALSE;
7152 if (should_set_mark_bits) {
7153 gc_setup_mark_bits(page);
7157 gc_ref_update_table_values_only(
objspace->obj_to_id_tbl);
7158 gc_update_table_refs(
objspace->id_to_obj_tbl);
7159 gc_update_table_refs(finalizer_table);
7161 rb_gc_update_vm_references((
void *)
objspace);
7163 objspace->flags.during_reference_updating =
false;
7166 #if GC_CAN_COMPILE_COMPACTION
7168 root_obj_check_moved_i(
const char *category,
VALUE obj,
void *data)
7172 if (gc_object_moved_p(
objspace, obj)) {
7173 rb_bug(
"ROOT %s points to MOVED: %p -> %s", category, (
void *)obj, rb_obj_info(rb_gc_impl_location(
objspace, obj)));
7178 reachable_object_check_moved_i(
VALUE ref,
void *data)
7181 if (gc_object_moved_p(rb_gc_get_objspace(), ref)) {
7182 rb_bug(
"Object %s points to MOVED: %p -> %s", rb_obj_info(parent), (
void *)ref, rb_obj_info(rb_gc_impl_location(rb_gc_get_objspace(), ref)));
7187 heap_check_moved_i(
void *vstart,
void *vend,
size_t stride,
void *data)
7192 for (; v != (
VALUE)vend; v += stride) {
7193 if (gc_object_moved_p(
objspace, v)) {
7197 asan_unpoisoning_object(v) {
7203 if (!rb_gc_impl_garbage_object_p(
objspace, v)) {
7204 rb_objspace_reachable_objects_from(v, reachable_object_check_moved_i, (
void *)v);
7216 rb_gc_impl_during_gc_p(
void *objspace_ptr)
7223 #if RGENGC_PROFILE >= 2
7256 default:
return "unknown";
7261 gc_count_add_each_types(
VALUE hash,
const char *name,
const size_t *types)
7265 for (i=0; i<
T_MASK; i++) {
7266 const char *
type = type_name(i, 0);
7274 rb_gc_impl_gc_count(
void *objspace_ptr)
7284 static VALUE sym_major_by =
Qnil, sym_gc_by, sym_immediate_sweep, sym_have_finalizer, sym_state, sym_need_major_by;
7285 static VALUE sym_nofree, sym_oldgen, sym_shady, sym_force, sym_stress;
7286 #if RGENGC_ESTIMATE_OLDMALLOC
7287 static VALUE sym_oldmalloc;
7289 static VALUE sym_newobj, sym_malloc, sym_method, sym_capi;
7290 static VALUE sym_none, sym_marking, sym_sweeping;
7291 static VALUE sym_weak_references_count, sym_retained_weak_references_count;
7293 VALUE major_by, need_major_by;
7294 unsigned int flags = orig_flags ? orig_flags :
objspace->profile.latest_gc_info;
7303 rb_bug(
"gc_info_decode: non-hash or symbol given");
7306 if (
NIL_P(sym_major_by)) {
7307 #define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
7320 #if RGENGC_ESTIMATE_OLDMALLOC
7332 S(weak_references_count);
7333 S(retained_weak_references_count);
7337 #define SET(name, attr) \
7338 if (key == sym_##name) \
7340 else if (hash != Qnil) \
7341 rb_hash_aset(hash, sym_##name, (attr));
7344 (flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
7345 (flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
7346 (flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
7347 (flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
7348 #if RGENGC_ESTIMATE_OLDMALLOC
7349 (flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
7352 SET(major_by, major_by);
7354 if (orig_flags == 0) {
7355 unsigned int need_major_flags = gc_needs_major_flags;
7357 (need_major_flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
7358 (need_major_flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
7359 (need_major_flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
7360 (need_major_flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
7361 #if RGENGC_ESTIMATE_OLDMALLOC
7362 (need_major_flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
7365 SET(need_major_by, need_major_by);
7369 (flags & GPR_FLAG_NEWOBJ) ? sym_newobj :
7370 (flags & GPR_FLAG_MALLOC) ? sym_malloc :
7371 (flags & GPR_FLAG_METHOD) ? sym_method :
7372 (flags & GPR_FLAG_CAPI) ? sym_capi :
7373 (flags & GPR_FLAG_STRESS) ? sym_stress :
7377 SET(have_finalizer, (flags & GPR_FLAG_HAVE_FINALIZE) ?
Qtrue :
Qfalse);
7378 SET(immediate_sweep, (flags & GPR_FLAG_IMMEDIATE_SWEEP) ?
Qtrue :
Qfalse);
7380 if (orig_flags == 0) {
7381 SET(state, gc_mode(
objspace) == gc_mode_none ? sym_none :
7382 gc_mode(
objspace) == gc_mode_marking ? sym_marking : sym_sweeping);
7385 SET(weak_references_count,
LONG2FIX(
objspace->profile.weak_references_count));
7386 SET(retained_weak_references_count,
LONG2FIX(
objspace->profile.retained_weak_references_count));
7398 rb_gc_impl_latest_gc_info(
void *objspace_ptr,
VALUE key)
7402 return gc_info_decode(
objspace, key, 0);
7409 gc_stat_sym_marking_time,
7410 gc_stat_sym_sweeping_time,
7411 gc_stat_sym_heap_allocated_pages,
7412 gc_stat_sym_heap_empty_pages,
7413 gc_stat_sym_heap_allocatable_slots,
7414 gc_stat_sym_heap_available_slots,
7415 gc_stat_sym_heap_live_slots,
7416 gc_stat_sym_heap_free_slots,
7417 gc_stat_sym_heap_final_slots,
7418 gc_stat_sym_heap_marked_slots,
7419 gc_stat_sym_heap_eden_pages,
7420 gc_stat_sym_total_allocated_pages,
7421 gc_stat_sym_total_freed_pages,
7422 gc_stat_sym_total_allocated_objects,
7423 gc_stat_sym_total_freed_objects,
7424 gc_stat_sym_malloc_increase_bytes,
7425 gc_stat_sym_malloc_increase_bytes_limit,
7426 gc_stat_sym_minor_gc_count,
7427 gc_stat_sym_major_gc_count,
7428 gc_stat_sym_compact_count,
7429 gc_stat_sym_read_barrier_faults,
7430 gc_stat_sym_total_moved_objects,
7431 gc_stat_sym_remembered_wb_unprotected_objects,
7432 gc_stat_sym_remembered_wb_unprotected_objects_limit,
7433 gc_stat_sym_old_objects,
7434 gc_stat_sym_old_objects_limit,
7435 #if RGENGC_ESTIMATE_OLDMALLOC
7436 gc_stat_sym_oldmalloc_increase_bytes,
7437 gc_stat_sym_oldmalloc_increase_bytes_limit,
7439 gc_stat_sym_weak_references_count,
7441 gc_stat_sym_total_generated_normal_object_count,
7442 gc_stat_sym_total_generated_shady_object_count,
7443 gc_stat_sym_total_shade_operation_count,
7444 gc_stat_sym_total_promoted_count,
7445 gc_stat_sym_total_remembered_normal_object_count,
7446 gc_stat_sym_total_remembered_shady_object_count,
7451 static VALUE gc_stat_symbols[gc_stat_sym_last];
7454 setup_gc_stat_symbols(
void)
7456 if (gc_stat_symbols[0] == 0) {
7457 #define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s))
7462 S(heap_allocated_pages);
7463 S(heap_empty_pages);
7464 S(heap_allocatable_slots);
7465 S(heap_available_slots);
7468 S(heap_final_slots);
7469 S(heap_marked_slots);
7471 S(total_allocated_pages);
7472 S(total_freed_pages);
7473 S(total_allocated_objects);
7474 S(total_freed_objects);
7475 S(malloc_increase_bytes);
7476 S(malloc_increase_bytes_limit);
7480 S(read_barrier_faults);
7481 S(total_moved_objects);
7482 S(remembered_wb_unprotected_objects);
7483 S(remembered_wb_unprotected_objects_limit);
7485 S(old_objects_limit);
7486 #if RGENGC_ESTIMATE_OLDMALLOC
7487 S(oldmalloc_increase_bytes);
7488 S(oldmalloc_increase_bytes_limit);
7490 S(weak_references_count);
7492 S(total_generated_normal_object_count);
7493 S(total_generated_shady_object_count);
7494 S(total_shade_operation_count);
7495 S(total_promoted_count);
7496 S(total_remembered_normal_object_count);
7497 S(total_remembered_shady_object_count);
7504 ns_to_ms(uint64_t ns)
7506 return ns / (1000 * 1000);
7510 rb_gc_impl_stat(
void *objspace_ptr,
VALUE hash_or_sym)
7515 setup_gc_stat_symbols();
7524 rb_bug(
"non-hash or symbol given");
7527 #define SET(name, attr) \
7528 if (key == gc_stat_symbols[gc_stat_sym_##name]) \
7529 return SIZET2NUM(attr); \
7530 else if (hash != Qnil) \
7531 rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
7533 SET(count,
objspace->profile.count);
7534 SET(time, (
size_t)ns_to_ms(
objspace->profile.marking_time_ns +
objspace->profile.sweeping_time_ns));
7535 SET(marking_time, (
size_t)ns_to_ms(
objspace->profile.marking_time_ns));
7536 SET(sweeping_time, (
size_t)ns_to_ms(
objspace->profile.sweeping_time_ns));
7539 SET(heap_allocated_pages, rb_darray_size(
objspace->heap_pages.sorted));
7540 SET(heap_empty_pages,
objspace->empty_pages_count)
7541 SET(heap_allocatable_slots,
objspace->heap_pages.allocatable_slots);
7542 SET(heap_available_slots, objspace_available_slots(
objspace));
7543 SET(heap_live_slots, objspace_live_slots(
objspace));
7544 SET(heap_free_slots, objspace_free_slots(
objspace));
7545 SET(heap_final_slots, total_final_slots_count(
objspace));
7546 SET(heap_marked_slots,
objspace->marked_slots);
7547 SET(heap_eden_pages, heap_eden_total_pages(
objspace));
7548 SET(total_allocated_pages,
objspace->heap_pages.allocated_pages);
7549 SET(total_freed_pages,
objspace->heap_pages.freed_pages);
7550 SET(total_allocated_objects, total_allocated_objects(
objspace));
7551 SET(total_freed_objects, total_freed_objects(
objspace));
7552 SET(malloc_increase_bytes, malloc_increase);
7553 SET(malloc_increase_bytes_limit, malloc_limit);
7554 SET(minor_gc_count,
objspace->profile.minor_gc_count);
7555 SET(major_gc_count,
objspace->profile.major_gc_count);
7556 SET(compact_count,
objspace->profile.compact_count);
7557 SET(read_barrier_faults,
objspace->profile.read_barrier_faults);
7558 SET(total_moved_objects,
objspace->rcompactor.total_moved);
7559 SET(remembered_wb_unprotected_objects,
objspace->rgengc.uncollectible_wb_unprotected_objects);
7560 SET(remembered_wb_unprotected_objects_limit,
objspace->rgengc.uncollectible_wb_unprotected_objects_limit);
7561 SET(old_objects,
objspace->rgengc.old_objects);
7562 SET(old_objects_limit,
objspace->rgengc.old_objects_limit);
7563 #if RGENGC_ESTIMATE_OLDMALLOC
7564 SET(oldmalloc_increase_bytes,
objspace->rgengc.oldmalloc_increase);
7565 SET(oldmalloc_increase_bytes_limit,
objspace->rgengc.oldmalloc_increase_limit);
7569 SET(total_generated_normal_object_count,
objspace->profile.total_generated_normal_object_count);
7570 SET(total_generated_shady_object_count,
objspace->profile.total_generated_shady_object_count);
7571 SET(total_shade_operation_count,
objspace->profile.total_shade_operation_count);
7572 SET(total_promoted_count,
objspace->profile.total_promoted_count);
7573 SET(total_remembered_normal_object_count,
objspace->profile.total_remembered_normal_object_count);
7574 SET(total_remembered_shady_object_count,
objspace->profile.total_remembered_shady_object_count);
7583 #if defined(RGENGC_PROFILE) && RGENGC_PROFILE >= 2
7585 gc_count_add_each_types(hash,
"generated_normal_object_count_types",
objspace->profile.generated_normal_object_count_types);
7586 gc_count_add_each_types(hash,
"generated_shady_object_count_types",
objspace->profile.generated_shady_object_count_types);
7587 gc_count_add_each_types(hash,
"shade_operation_count_types",
objspace->profile.shade_operation_count_types);
7588 gc_count_add_each_types(hash,
"promoted_types",
objspace->profile.promoted_types);
7589 gc_count_add_each_types(hash,
"remembered_normal_object_count_types",
objspace->profile.remembered_normal_object_count_types);
7590 gc_count_add_each_types(hash,
"remembered_shady_object_count_types",
objspace->profile.remembered_shady_object_count_types);
7597 enum gc_stat_heap_sym {
7598 gc_stat_heap_sym_slot_size,
7599 gc_stat_heap_sym_heap_eden_pages,
7600 gc_stat_heap_sym_heap_eden_slots,
7601 gc_stat_heap_sym_total_allocated_pages,
7602 gc_stat_heap_sym_force_major_gc_count,
7603 gc_stat_heap_sym_force_incremental_marking_finish_count,
7604 gc_stat_heap_sym_total_allocated_objects,
7605 gc_stat_heap_sym_total_freed_objects,
7606 gc_stat_heap_sym_last
7609 static VALUE gc_stat_heap_symbols[gc_stat_heap_sym_last];
7612 setup_gc_stat_heap_symbols(
void)
7614 if (gc_stat_heap_symbols[0] == 0) {
7615 #define S(s) gc_stat_heap_symbols[gc_stat_heap_sym_##s] = ID2SYM(rb_intern_const(#s))
7619 S(total_allocated_pages);
7620 S(force_major_gc_count);
7621 S(force_incremental_marking_finish_count);
7622 S(total_allocated_objects);
7623 S(total_freed_objects);
7631 #define SET(name, attr) \
7632 if (key == gc_stat_heap_symbols[gc_stat_heap_sym_##name]) \
7633 return SIZET2NUM(attr); \
7634 else if (hash != Qnil) \
7635 rb_hash_aset(hash, gc_stat_heap_symbols[gc_stat_heap_sym_##name], SIZET2NUM(attr));
7637 SET(slot_size, heap->slot_size);
7638 SET(heap_eden_pages, heap->total_pages);
7639 SET(heap_eden_slots, heap->total_slots);
7640 SET(total_allocated_pages, heap->total_allocated_pages);
7641 SET(force_major_gc_count, heap->force_major_gc_count);
7642 SET(force_incremental_marking_finish_count, heap->force_incremental_marking_finish_count);
7643 SET(total_allocated_objects, heap->total_allocated_objects);
7644 SET(total_freed_objects, heap->total_freed_objects);
7656 rb_gc_impl_stat_heap(
void *objspace_ptr,
VALUE heap_name,
VALUE hash_or_sym)
7660 setup_gc_stat_heap_symbols();
7662 if (
NIL_P(heap_name)) {
7664 rb_bug(
"non-hash given");
7667 for (
int i = 0; i < HEAP_COUNT; i++) {
7674 stat_one_heap(&heaps[i], hash,
Qnil);
7678 int heap_idx =
FIX2INT(heap_name);
7680 if (heap_idx < 0 || heap_idx >= HEAP_COUNT) {
7685 return stat_one_heap(&heaps[heap_idx],
Qnil, hash_or_sym);
7688 return stat_one_heap(&heaps[heap_idx], hash_or_sym,
Qnil);
7691 rb_bug(
"non-hash or symbol given");
7695 rb_bug(
"heap_name must be nil or an Integer");
7707 #define RBOOL(v) (v ? Qtrue : Qfalse)
7711 rb_gc_impl_config_get(
void *objspace_ptr)
7713 #define sym(name) ID2SYM(rb_intern_const(name))
7717 rb_hash_aset(hash, sym(
"rgengc_allow_full_mark"), RBOOL(gc_config_full_mark_val));
7723 gc_config_set_key(st_data_t key, st_data_t value, st_data_t data)
7728 gc_config_full_mark_set(
RTEST(value));
7734 rb_gc_impl_config_set(
void *objspace_ptr,
VALUE hash)
7742 rb_hash_stlike_foreach(hash, gc_config_set_key, (st_data_t)
objspace);
7746 rb_gc_impl_stress_get(
void *objspace_ptr)
7749 return ruby_gc_stress_mode;
7753 rb_gc_impl_stress_set(
void *objspace_ptr,
VALUE flag)
7762 get_envparam_size(
const char *name,
size_t *default_value,
size_t lower_bound)
7764 const char *
ptr = getenv(name);
7767 if (
ptr != NULL && *
ptr) {
7770 #if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
7771 val = strtoll(
ptr, &end, 0);
7773 val = strtol(
ptr, &end, 0);
7785 unit = 1024*1024*1024;
7789 while (*end && isspace((
unsigned char)*end)) end++;
7795 if (val < -(ssize_t)(SIZE_MAX / 2 / unit) || (ssize_t)(SIZE_MAX / 2 / unit) < val) {
7796 if (
RTEST(
ruby_verbose)) fprintf(stderr,
"%s=%s is ignored because it overflows\n", name,
ptr);
7801 if (val > 0 && (
size_t)val > lower_bound) {
7803 fprintf(stderr,
"%s=%"PRIdSIZE
" (default value: %"PRIuSIZE
")\n", name, val, *default_value);
7805 *default_value = (size_t)val;
7810 fprintf(stderr,
"%s=%"PRIdSIZE
" (default value: %"PRIuSIZE
") is ignored because it must be greater than %"PRIuSIZE
".\n",
7811 name, val, *default_value, lower_bound);
7820 get_envparam_double(
const char *name,
double *default_value,
double lower_bound,
double upper_bound,
int accept_zero)
7822 const char *
ptr = getenv(name);
7825 if (
ptr != NULL && *
ptr) {
7828 if (!*
ptr || *end) {
7833 if (accept_zero && val == 0.0) {
7836 else if (val <= lower_bound) {
7838 fprintf(stderr,
"%s=%f (default value: %f) is ignored because it must be greater than %f.\n",
7839 name, val, *default_value, lower_bound);
7842 else if (upper_bound != 0.0 &&
7843 val > upper_bound) {
7845 fprintf(stderr,
"%s=%f (default value: %f) is ignored because it must be lower than %f.\n",
7846 name, val, *default_value, upper_bound);
7856 if (
RTEST(
ruby_verbose)) fprintf(stderr,
"%s=%f (default value: %f)\n", name, val, *default_value);
7857 *default_value = val;
7902 rb_gc_impl_set_params(
void *objspace_ptr)
7906 if (get_envparam_size(
"RUBY_GC_HEAP_FREE_SLOTS", &gc_params.heap_free_slots, 0)) {
7910 for (
int i = 0; i < HEAP_COUNT; i++) {
7911 char env_key[
sizeof(
"RUBY_GC_HEAP_" "_INIT_SLOTS") +
DECIMAL_SIZE_OF_BITS(
sizeof(
int) * CHAR_BIT)];
7912 snprintf(env_key,
sizeof(env_key),
"RUBY_GC_HEAP_%d_INIT_SLOTS", i);
7914 get_envparam_size(env_key, &gc_params.heap_init_slots[i], 0);
7917 get_envparam_double(
"RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE);
7918 get_envparam_size (
"RUBY_GC_HEAP_GROWTH_MAX_SLOTS", &gc_params.growth_max_slots, 0);
7919 get_envparam_double(
"RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio,
7921 get_envparam_double(
"RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", &gc_params.heap_free_slots_max_ratio,
7922 gc_params.heap_free_slots_min_ratio, 1.0, FALSE);
7923 get_envparam_double(
"RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", &gc_params.heap_free_slots_goal_ratio,
7924 gc_params.heap_free_slots_min_ratio, gc_params.heap_free_slots_max_ratio, TRUE);
7925 get_envparam_double(
"RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR", &gc_params.oldobject_limit_factor, 0.0, 0.0, TRUE);
7926 get_envparam_double(
"RUBY_GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO", &gc_params.uncollectible_wb_unprotected_objects_limit_ratio, 0.0, 0.0, TRUE);
7928 if (get_envparam_size(
"RUBY_GC_MALLOC_LIMIT", &gc_params.malloc_limit_min, 0)) {
7929 malloc_limit = gc_params.malloc_limit_min;
7931 get_envparam_size (
"RUBY_GC_MALLOC_LIMIT_MAX", &gc_params.malloc_limit_max, 0);
7932 if (!gc_params.malloc_limit_max) {
7933 gc_params.malloc_limit_max = SIZE_MAX;
7935 get_envparam_double(
"RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR", &gc_params.malloc_limit_growth_factor, 1.0, 0.0, FALSE);
7937 #if RGENGC_ESTIMATE_OLDMALLOC
7938 if (get_envparam_size(
"RUBY_GC_OLDMALLOC_LIMIT", &gc_params.oldmalloc_limit_min, 0)) {
7939 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
7941 get_envparam_size (
"RUBY_GC_OLDMALLOC_LIMIT_MAX", &gc_params.oldmalloc_limit_max, 0);
7942 get_envparam_double(
"RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR", &gc_params.oldmalloc_limit_growth_factor, 1.0, 0.0, FALSE);
7946 static inline size_t
7949 #ifdef HAVE_MALLOC_USABLE_SIZE
7950 return malloc_usable_size(
ptr);
7957 MEMOP_TYPE_MALLOC = 0,
7963 atomic_sub_nounderflow(
size_t *var,
size_t sub)
7965 if (sub == 0)
return;
7969 if (val < sub) sub = val;
7974 #define gc_stress_full_mark_after_malloc_p() \
7975 (FIXNUM_P(ruby_gc_stress_mode) && (FIX2LONG(ruby_gc_stress_mode) & (1<<gc_stress_full_mark_after_malloc)))
7981 unsigned int reason = (GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP |
7982 GPR_FLAG_STRESS | GPR_FLAG_MALLOC);
7984 if (gc_stress_full_mark_after_malloc_p()) {
7985 reason |= GPR_FLAG_FULL_MARK;
7987 garbage_collect_with_gvl(
objspace, reason);
7992 objspace_malloc_increase_report(
rb_objspace_t *
objspace,
void *mem,
size_t new_size,
size_t old_size,
enum memop_type
type)
7994 if (0) fprintf(stderr,
"increase - ptr: %p, type: %s, new_size: %"PRIdSIZE
", old_size: %"PRIdSIZE
"\n",
7996 type == MEMOP_TYPE_MALLOC ?
"malloc" :
7997 type == MEMOP_TYPE_FREE ?
"free " :
7998 type == MEMOP_TYPE_REALLOC ?
"realloc":
"error",
7999 new_size, old_size);
8004 objspace_malloc_increase_body(
rb_objspace_t *
objspace,
void *mem,
size_t new_size,
size_t old_size,
enum memop_type
type)
8006 if (new_size > old_size) {
8008 #if RGENGC_ESTIMATE_OLDMALLOC
8013 atomic_sub_nounderflow(&malloc_increase, old_size - new_size);
8014 #if RGENGC_ESTIMATE_OLDMALLOC
8015 atomic_sub_nounderflow(&
objspace->rgengc.oldmalloc_increase, old_size - new_size);
8019 if (
type == MEMOP_TYPE_MALLOC) {
8022 if (ruby_thread_has_gvl_p() && is_lazy_sweeping(
objspace)) {
8026 garbage_collect_with_gvl(
objspace, GPR_FLAG_MALLOC);
8030 #if MALLOC_ALLOCATED_SIZE
8031 if (new_size >= old_size) {
8035 size_t dec_size = old_size - new_size;
8036 size_t allocated_size =
objspace->malloc_params.allocated_size;
8038 #if MALLOC_ALLOCATED_SIZE_CHECK
8039 if (allocated_size < dec_size) {
8040 rb_bug(
"objspace_malloc_increase: underflow malloc_params.allocated_size.");
8043 atomic_sub_nounderflow(&
objspace->malloc_params.allocated_size, dec_size);
8047 case MEMOP_TYPE_MALLOC:
8050 case MEMOP_TYPE_FREE:
8052 size_t allocations =
objspace->malloc_params.allocations;
8053 if (allocations > 0) {
8054 atomic_sub_nounderflow(&
objspace->malloc_params.allocations, 1);
8056 #if MALLOC_ALLOCATED_SIZE_CHECK
8058 GC_ASSERT(
objspace->malloc_params.allocations > 0);
8063 case MEMOP_TYPE_REALLOC:
break;
8069 #define objspace_malloc_increase(...) \
8070 for (bool malloc_increase_done = objspace_malloc_increase_report(__VA_ARGS__); \
8071 !malloc_increase_done; \
8072 malloc_increase_done = objspace_malloc_increase_body(__VA_ARGS__))
8078 static inline size_t
8081 if (size == 0) size = 1;
8083 #if CALC_EXACT_MALLOC_SIZE
8097 return during_gc && !dont_gc_val() && !rb_gc_multi_ractor_p() && ruby_thread_has_gvl_p();
8100 static inline void *
8103 size = objspace_malloc_size(
objspace, mem, size);
8104 objspace_malloc_increase(
objspace, mem, size, 0, MEMOP_TYPE_MALLOC) {}
8106 #if CALC_EXACT_MALLOC_SIZE
8117 #if defined(__GNUC__) && RUBY_DEBUG
8118 #define RB_BUG_INSTEAD_OF_RB_MEMERROR 1
8121 #ifndef RB_BUG_INSTEAD_OF_RB_MEMERROR
8122 # define RB_BUG_INSTEAD_OF_RB_MEMERROR 0
8125 #define GC_MEMERROR(...) \
8126 ((RB_BUG_INSTEAD_OF_RB_MEMERROR+0) ? rb_bug("" __VA_ARGS__) : rb_memerror())
8128 #define TRY_WITH_GC(siz, expr) do { \
8129 const gc_profile_record_flag gpr = \
8130 GPR_FLAG_FULL_MARK | \
8131 GPR_FLAG_IMMEDIATE_MARK | \
8132 GPR_FLAG_IMMEDIATE_SWEEP | \
8134 objspace_malloc_gc_stress(objspace); \
8136 if (RB_LIKELY((expr))) { \
8139 else if (!garbage_collect_with_gvl(objspace, gpr)) { \
8141 GC_MEMERROR("TRY_WITH_GC: could not GC"); \
8143 else if ((expr)) { \
8147 GC_MEMERROR("TRY_WITH_GC: could not allocate:" \
8148 "%"PRIdSIZE" bytes for %s", \
8159 rb_bug(
"Cannot %s during GC", msg);
8164 rb_gc_impl_free(
void *objspace_ptr,
void *
ptr,
size_t old_size)
8175 #if CALC_EXACT_MALLOC_SIZE
8178 old_size = info->size;
8180 old_size = objspace_malloc_size(
objspace,
ptr, old_size);
8182 objspace_malloc_increase(
objspace,
ptr, 0, old_size, MEMOP_TYPE_FREE) {
8185 RB_DEBUG_COUNTER_INC(heap_xfree);
8190 rb_gc_impl_malloc(
void *objspace_ptr,
size_t size)
8193 check_malloc_not_in_gc(
objspace,
"malloc");
8197 size = objspace_malloc_prepare(
objspace, size);
8198 TRY_WITH_GC(size, mem = malloc(size));
8199 RB_DEBUG_COUNTER_INC(heap_xmalloc);
8200 return objspace_malloc_fixup(
objspace, mem, size);
8204 rb_gc_impl_calloc(
void *objspace_ptr,
size_t size)
8209 rb_warn(
"calloc during GC detected, this could cause crashes if it triggers another GC");
8210 #if RGENGC_CHECK_MODE || RUBY_DEBUG
8211 rb_bug(
"Cannot calloc during GC");
8217 size = objspace_malloc_prepare(
objspace, size);
8218 TRY_WITH_GC(size, mem = calloc1(size));
8219 return objspace_malloc_fixup(
objspace, mem, size);
8223 rb_gc_impl_realloc(
void *objspace_ptr,
void *
ptr,
size_t new_size,
size_t old_size)
8227 check_malloc_not_in_gc(
objspace,
"realloc");
8231 if (!
ptr)
return rb_gc_impl_malloc(
objspace, new_size);
8238 if (new_size == 0) {
8239 if ((mem = rb_gc_impl_malloc(
objspace, 0)) != NULL) {
8276 #if CALC_EXACT_MALLOC_SIZE
8281 old_size = info->size;
8285 old_size = objspace_malloc_size(
objspace,
ptr, old_size);
8287 new_size = objspace_malloc_size(
objspace, mem, new_size);
8289 #if CALC_EXACT_MALLOC_SIZE
8292 info->size = new_size;
8297 objspace_malloc_increase(
objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC);
8299 RB_DEBUG_COUNTER_INC(heap_xrealloc);
8304 rb_gc_impl_adjust_memory_usage(
void *objspace_ptr, ssize_t diff)
8309 objspace_malloc_increase(
objspace, 0, diff, 0, MEMOP_TYPE_REALLOC);
8311 else if (diff < 0) {
8312 objspace_malloc_increase(
objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC);
8321 #define GC_PROFILE_RECORD_DEFAULT_SIZE 100
8324 current_process_time(
struct timespec *ts)
8326 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
8328 static int try_clock_gettime = 1;
8329 if (try_clock_gettime && clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ts) == 0) {
8333 try_clock_gettime = 0;
8340 struct rusage usage;
8342 if (getrusage(RUSAGE_SELF, &usage) == 0) {
8343 time = usage.ru_utime;
8344 ts->tv_sec = time.tv_sec;
8345 ts->tv_nsec = (int32_t)time.tv_usec * 1000;
8353 FILETIME creation_time, exit_time, kernel_time, user_time;
8356 if (GetProcessTimes(GetCurrentProcess(),
8357 &creation_time, &exit_time, &kernel_time, &user_time) != 0) {
8358 memcpy(&ui, &user_time,
sizeof(FILETIME));
8359 #define PER100NSEC (uint64_t)(1000 * 1000 * 10)
8360 ts->tv_nsec = (long)(ui.QuadPart % PER100NSEC);
8361 ts->tv_sec = (time_t)(ui.QuadPart / PER100NSEC);
8371 getrusage_time(
void)
8374 if (current_process_time(&ts)) {
8375 return ts.tv_sec + ts.tv_nsec * 1e-9;
8387 size_t index =
objspace->profile.next_index;
8394 objspace->profile.size = GC_PROFILE_RECORD_DEFAULT_SIZE;
8397 if (index >=
objspace->profile.size) {
8405 rb_bug(
"gc_profile malloc or realloc miss");
8411 record->flags = reason | (ruby_gc_stressful ? GPR_FLAG_STRESS : 0);
8412 #if MALLOC_ALLOCATED_SIZE
8413 record->allocated_size = malloc_allocated_size;
8415 #if GC_PROFILE_MORE_DETAIL && GC_PROFILE_DETAIL_MEMORY
8418 struct rusage usage;
8419 if (getrusage(RUSAGE_SELF, &usage) == 0) {
8420 record->maxrss = usage.ru_maxrss;
8421 record->minflt = usage.ru_minflt;
8422 record->majflt = usage.ru_majflt;
8435 #if GC_PROFILE_MORE_DETAIL
8436 record->prepare_time =
objspace->profile.prepare_time;
8438 record->gc_time = 0;
8439 record->gc_invoke_time = getrusage_time();
8444 elapsed_time_from(
double time)
8446 double now = getrusage_time();
8460 record->gc_time = elapsed_time_from(record->gc_invoke_time);
8461 record->gc_invoke_time -=
objspace->profile.invoke_time;
8465 #ifdef BUILDING_SHARED_GC
8466 # define RUBY_DTRACE_GC_HOOK(name)
8468 # define RUBY_DTRACE_GC_HOOK(name) \
8469 do {if (RUBY_DTRACE_GC_##name##_ENABLED()) RUBY_DTRACE_GC_##name();} while (0)
8475 RUBY_DTRACE_GC_HOOK(MARK_BEGIN);
8476 #if GC_PROFILE_MORE_DETAIL
8478 gc_prof_record(
objspace)->gc_mark_time = getrusage_time();
8486 RUBY_DTRACE_GC_HOOK(MARK_END);
8487 #if GC_PROFILE_MORE_DETAIL
8490 record->gc_mark_time = elapsed_time_from(record->gc_mark_time);
8498 RUBY_DTRACE_GC_HOOK(SWEEP_BEGIN);
8502 if (record->gc_time > 0 || GC_PROFILE_MORE_DETAIL) {
8503 objspace->profile.gc_sweep_start_time = getrusage_time();
8511 RUBY_DTRACE_GC_HOOK(SWEEP_END);
8517 if (record->gc_time > 0) {
8518 sweep_time = elapsed_time_from(
objspace->profile.gc_sweep_start_time);
8520 record->gc_time += sweep_time;
8522 else if (GC_PROFILE_MORE_DETAIL) {
8523 sweep_time = elapsed_time_from(
objspace->profile.gc_sweep_start_time);
8526 #if GC_PROFILE_MORE_DETAIL
8527 record->gc_sweep_time += sweep_time;
8528 if (heap_pages_deferred_final) record->flags |= GPR_FLAG_HAVE_FINALIZE;
8530 if (heap_pages_deferred_final)
objspace->profile.latest_gc_info |= GPR_FLAG_HAVE_FINALIZE;
8537 #if GC_PROFILE_MORE_DETAIL
8540 record->allocate_increase = malloc_increase;
8541 record->allocate_limit = malloc_limit;
8551 size_t live =
objspace->profile.total_allocated_objects_at_gc_start - total_freed_objects(
objspace);
8552 size_t total =
objspace->profile.heap_used_at_gc_start * HEAP_PAGE_OBJ_LIMIT;
8554 #if GC_PROFILE_MORE_DETAIL
8555 record->heap_use_pages =
objspace->profile.heap_used_at_gc_start;
8556 record->heap_live_objects = live;
8557 record->heap_free_objects = total - live;
8560 record->heap_total_objects = total;
8561 record->heap_use_size = live * BASE_SLOT_SIZE;
8562 record->heap_total_size = total * BASE_SLOT_SIZE;
8575 gc_profile_clear(
VALUE _)
8578 void *p =
objspace->profile.records;
8582 objspace->profile.current_record = 0;
8638 gc_profile_record_get(
VALUE _)
8649 for (i =0; i <
objspace->profile.next_index; i++) {
8661 #if GC_PROFILE_MORE_DETAIL
8676 #if RGENGC_PROFILE > 0
8687 #if GC_PROFILE_MORE_DETAIL
8688 #define MAJOR_REASON_MAX 0x10
8691 gc_profile_dump_major_reason(
unsigned int flags,
char *buff)
8693 unsigned int reason = flags & GPR_FLAG_MAJOR_MASK;
8696 if (reason == GPR_FLAG_NONE) {
8702 if (reason & GPR_FLAG_MAJOR_BY_##x) { \
8703 buff[i++] = #x[0]; \
8704 if (i >= MAJOR_REASON_MAX) rb_bug("gc_profile_dump_major_reason: overflow"); \
8710 #if RGENGC_ESTIMATE_OLDMALLOC
8725 size_t count =
objspace->profile.next_index;
8726 #ifdef MAJOR_REASON_MAX
8727 char reason_str[MAJOR_REASON_MAX];
8730 if (
objspace->profile.run && count ) {
8735 append(out,
rb_str_new_cstr(
"Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC Time(ms)\n"));
8737 for (i = 0; i < count; i++) {
8738 record = &
objspace->profile.records[i];
8739 append(out,
rb_sprintf(
"%5"PRIuSIZE
" %19.3f %20"PRIuSIZE
" %20"PRIuSIZE
" %20"PRIuSIZE
" %30.20f\n",
8740 i+1, record->gc_invoke_time, record->heap_use_size,
8741 record->heap_total_size, record->heap_total_objects, record->gc_time*1000));
8744 #if GC_PROFILE_MORE_DETAIL
8745 const char *str =
"\n\n" \
8747 "Prepare Time = Previously GC's rest sweep time\n"
8748 "Index Flags Allocate Inc. Allocate Limit"
8749 #if CALC_EXACT_MALLOC_SIZE
8752 " Use Page Mark Time(ms) Sweep Time(ms) Prepare Time(ms) LivingObj FreeObj RemovedObj EmptyObj"
8754 " OldgenObj RemNormObj RemShadObj"
8756 #if GC_PROFILE_DETAIL_MEMORY
8757 " MaxRSS(KB) MinorFLT MajorFLT"
8762 for (i = 0; i < count; i++) {
8763 record = &
objspace->profile.records[i];
8764 append(out,
rb_sprintf(
"%5"PRIuSIZE
" %4s/%c/%6s%c %13"PRIuSIZE
" %15"PRIuSIZE
8765 #
if CALC_EXACT_MALLOC_SIZE
8768 " %9"PRIuSIZE
" %17.12f %17.12f %17.12f %10"PRIuSIZE
" %10"PRIuSIZE
" %10"PRIuSIZE
" %10"PRIuSIZE
8770 "%10"PRIuSIZE
" %10"PRIuSIZE
" %10"PRIuSIZE
8772 #
if GC_PROFILE_DETAIL_MEMORY
8778 gc_profile_dump_major_reason(record->flags, reason_str),
8779 (record->flags & GPR_FLAG_HAVE_FINALIZE) ?
'F' :
'.',
8780 (record->flags & GPR_FLAG_NEWOBJ) ?
"NEWOBJ" :
8781 (record->flags & GPR_FLAG_MALLOC) ?
"MALLOC" :
8782 (record->flags & GPR_FLAG_METHOD) ?
"METHOD" :
8783 (record->flags & GPR_FLAG_CAPI) ?
"CAPI__" :
"??????",
8784 (record->flags & GPR_FLAG_STRESS) ?
'!' :
' ',
8785 record->allocate_increase, record->allocate_limit,
8786 #
if CALC_EXACT_MALLOC_SIZE
8787 record->allocated_size,
8789 record->heap_use_pages,
8790 record->gc_mark_time*1000,
8791 record->gc_sweep_time*1000,
8792 record->prepare_time*1000,
8794 record->heap_live_objects,
8795 record->heap_free_objects,
8796 record->removing_objects,
8797 record->empty_objects
8800 record->old_objects,
8801 record->remembered_normal_objects,
8802 record->remembered_shady_objects
8804 #
if GC_PROFILE_DETAIL_MEMORY
8806 record->maxrss / 1024,
8829 gc_profile_result(
VALUE _)
8846 gc_profile_report(
int argc,
VALUE *argv,
VALUE self)
8864 gc_profile_total_time(
VALUE self)
8871 size_t count =
objspace->profile.next_index;
8873 for (i = 0; i < count; i++) {
8874 time +=
objspace->profile.records[i].gc_time;
8888 gc_profile_enable_get(
VALUE self)
8903 gc_profile_enable(
VALUE _)
8907 objspace->profile.current_record = 0;
8920 gc_profile_disable(
VALUE _)
8925 objspace->profile.current_record = 0;
8940 gc_verify_internal_consistency_m(
VALUE dummy)
8942 gc_verify_internal_consistency(rb_gc_get_objspace());
8946 #if GC_CAN_COMPILE_COMPACTION
8960 GC_ASSERT(GC_COMPACTION_SUPPORTED);
8962 ruby_enable_autocompact =
RTEST(v);
8964 #if RGENGC_CHECK_MODE
8965 ruby_autocompact_compare_func = NULL;
8970 ruby_autocompact_compare_func = compare_free_slots;
8978 # define gc_set_auto_compact rb_f_notimplement
8981 #if GC_CAN_COMPILE_COMPACTION
8989 gc_get_auto_compact(
VALUE _)
8994 # define gc_get_auto_compact rb_f_notimplement
8997 #if GC_CAN_COMPILE_COMPACTION
9023 gc_compact_stats(
VALUE self)
9032 for (
size_t i = 0; i <
T_MASK; i++) {
9033 if (
objspace->rcompactor.considered_count_table[i]) {
9037 if (
objspace->rcompactor.moved_count_table[i]) {
9041 if (
objspace->rcompactor.moved_up_count_table[i]) {
9045 if (
objspace->rcompactor.moved_down_count_table[i]) {
9058 # define gc_compact_stats rb_f_notimplement
9061 #if GC_CAN_COMPILE_COMPACTION
9080 gc_compact(
VALUE self)
9083 int full_marking_p = gc_config_full_mark_val;
9084 gc_config_full_mark_set(TRUE);
9087 rb_gc_impl_start(rb_gc_get_objspace(),
true,
true,
true,
true);
9088 gc_config_full_mark_set(full_marking_p);
9090 return gc_compact_stats(
self);
9093 # define gc_compact rb_f_notimplement
9096 #if GC_CAN_COMPILE_COMPACTION
9099 size_t required_slots[HEAP_COUNT];
9103 desired_compaction_pages_i(
struct heap_page *page,
void *data)
9108 VALUE vend = vstart + (
VALUE)(page->total_slots * page->heap->slot_size);
9111 for (
VALUE v = vstart; v != vend; v += page->heap->slot_size) {
9112 asan_unpoisoning_object(v) {
9116 size_t dest_pool_idx = dest_pool - heaps;
9117 tdata->required_slots[dest_pool_idx]++;
9141 gc_verify_compaction_references(
int argc,
VALUE* argv,
VALUE self)
9143 static ID keywords[3] = {0};
9154 int kwarg_count =
rb_get_kwargs(options, keywords, 0, 3, arguments);
9156 bool expand_heap = (kwarg_count > 1 &&
RTEST(arguments[1])) || (kwarg_count > 2 &&
RTEST(arguments[2]));
9161 rb_gc_impl_start(
objspace,
true,
true,
true,
false);
9163 unsigned int lev = rb_gc_vm_lock();
9171 .required_slots = {0},
9174 objspace_each_pages(
objspace, desired_compaction_pages_i, &desired_compaction, TRUE);
9177 size_t max_existing_pages = 0;
9178 for (
int i = 0; i < HEAP_COUNT; i++) {
9180 max_existing_pages = MAX(max_existing_pages, heap->total_pages);
9184 for (
int i = 0; i < HEAP_COUNT; i++) {
9187 size_t pages_to_add = 0;
9194 pages_to_add += max_existing_pages - heap->total_pages;
9199 objspace->heap_pages.allocatable_slots = desired_compaction.required_slots[i];
9200 while (
objspace->heap_pages.allocatable_slots > 0) {
9201 heap_page_allocate_and_initialize(
objspace, heap);
9209 for (; pages_to_add > 0; pages_to_add--) {
9210 heap_page_allocate_and_initialize_force(
objspace, heap);
9216 objspace->rcompactor.compare_func = compare_free_slots;
9219 rb_gc_vm_unlock(lev);
9221 rb_gc_impl_start(rb_gc_get_objspace(),
true,
true,
true,
true);
9223 rb_objspace_reachable_objects_from_root(root_obj_check_moved_i,
objspace);
9226 objspace->rcompactor.compare_func = NULL;
9228 return gc_compact_stats(
self);
9231 # define gc_verify_compaction_references rb_f_notimplement
9235 rb_gc_impl_objspace_free(
void *objspace_ptr)
9240 rb_bug(
"lazy sweeping underway when freeing object space");
9245 for (
size_t i = 0; i < rb_darray_size(
objspace->heap_pages.sorted); i++) {
9248 rb_darray_free(
objspace->heap_pages.sorted);
9249 heap_pages_lomem = 0;
9250 heap_pages_himem = 0;
9252 for (
int i = 0; i < HEAP_COUNT; i++) {
9254 heap->total_pages = 0;
9255 heap->total_slots = 0;
9258 st_free_table(
objspace->id_to_obj_tbl);
9259 st_free_table(
objspace->obj_to_id_tbl);
9261 free_stack_chunks(&
objspace->mark_stack);
9262 mark_stack_free_cache(&
objspace->mark_stack);
9264 rb_darray_free(
objspace->weak_references);
9269 #if MALLOC_ALLOCATED_SIZE
9280 gc_malloc_allocated_size(
VALUE self)
9295 gc_malloc_allocations(
VALUE self)
9301 void rb_gc_impl_before_fork(
void *objspace_ptr) { }
9302 void rb_gc_impl_after_fork(
void *objspace_ptr, rb_pid_t pid) { }
9305 rb_gc_impl_objspace_alloc(
void)
9313 rb_gc_impl_objspace_init(
void *objspace_ptr)
9317 gc_config_full_mark_set(TRUE);
9320 malloc_limit = gc_params.malloc_limit_min;
9322 if (
objspace->finalize_deferred_pjob == POSTPONED_JOB_HANDLE_INVALID) {
9323 rb_bug(
"Could not preregister postponed job for GC");
9326 for (
int i = 0; i < HEAP_COUNT; i++) {
9329 heap->slot_size = (1 << i) * BASE_SLOT_SIZE;
9331 ccan_list_head_init(&heap->pages);
9334 rb_darray_make(&
objspace->heap_pages.sorted, 0);
9335 rb_darray_make(&
objspace->weak_references, 0);
9342 #if defined(INIT_HEAP_PAGE_ALLOC_USE_MMAP)
9344 heap_page_alloc_use_mmap = INIT_HEAP_PAGE_ALLOC_USE_MMAP;
9346 objspace->next_object_id = OBJ_ID_INITIAL;
9347 objspace->id_to_obj_tbl = st_init_table(&object_id_hash_type);
9348 objspace->obj_to_id_tbl = st_init_numtable();
9349 #if RGENGC_ESTIMATE_OLDMALLOC
9350 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
9353 for (
int i = 0; i < HEAP_COUNT; i++) {
9355 gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS;
9358 init_mark_stack(&
objspace->mark_stack);
9360 objspace->profile.invoke_time = getrusage_time();
9361 finalizer_table = st_init_numtable();
9365 rb_gc_impl_init(
void)
9377 if (RB_BUG_INSTEAD_OF_RB_MEMERROR+0) {
9384 if (GC_COMPACTION_SUPPORTED) {
9402 #if MALLOC_ALLOCATED_SIZE
9421 #define OPT(o) if (o) rb_ary_push(opts, rb_interned_str(#o, sizeof(#o) - 1))
9425 OPT(RGENGC_CHECK_MODE);
9426 OPT(RGENGC_PROFILE);
9427 OPT(RGENGC_ESTIMATE_OLDMALLOC);
9428 OPT(GC_PROFILE_MORE_DETAIL);
9429 OPT(GC_ENABLE_LAZY_SWEEP);
9430 OPT(CALC_EXACT_MALLOC_SIZE);
9431 OPT(MALLOC_ALLOCATED_SIZE);
9432 OPT(MALLOC_ALLOCATED_SIZE_CHECK);
9433 OPT(GC_PROFILE_DETAIL_MEMORY);
9434 OPT(GC_COMPACTION_SUPPORTED);
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
#define RUBY_ATOMIC_VALUE_CAS(var, oldval, newval)
Identical to RUBY_ATOMIC_CAS, except it expects its arguments are VALUE.
#define RUBY_ATOMIC_SIZE_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are size_t.
#define RUBY_ATOMIC_SIZE_INC(var)
Identical to RUBY_ATOMIC_INC, except it expects its argument is size_t.
#define RUBY_ATOMIC_SIZE_CAS(var, oldval, newval)
Identical to RUBY_ATOMIC_CAS, except it expects its arguments are size_t.
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
#define RUBY_ATOMIC_SIZE_ADD(var, val)
Identical to RUBY_ATOMIC_ADD, except it expects its arguments are size_t.
#define RUBY_ATOMIC_VALUE_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are VALUE.
#define RUBY_ATOMIC_SET(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except for the return type.
#define RUBY_ATOMIC_EXCHANGE(var, val)
Atomically replaces the value pointed by var with val.
#define RB_LIKELY(x)
Asserts that the given Boolean expression likely holds.
#define RB_UNLIKELY(x)
Asserts that the given Boolean expression likely doesn't hold.
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
unsigned int rb_postponed_job_handle_t
The type of a handle returned from rb_postponed_job_preregister and passed to rb_postponed_job_trigge...
void rb_postponed_job_trigger(rb_postponed_job_handle_t h)
Triggers a pre-registered job registered with rb_postponed_job_preregister, scheduling it for executi...
rb_postponed_job_handle_t rb_postponed_job_preregister(unsigned int flags, rb_postponed_job_func_t func, void *data)
Pre-registers a func in Ruby's postponed job preregistration table, returning an opaque handle which ...
#define RUBY_INTERNAL_EVENT_GC_EXIT
gc_exit() is called.
#define RUBY_INTERNAL_EVENT_GC_ENTER
gc_enter() is called.
#define RUBY_INTERNAL_EVENT_GC_END_SWEEP
GC ended sweep phase.
#define RUBY_INTERNAL_EVENT_GC_END_MARK
GC ended mark phase.
#define RUBY_INTERNAL_EVENT_OBJSPACE_MASK
Bitmask of GC events.
#define RUBY_INTERNAL_EVENT_FREEOBJ
Object swept.
#define RUBY_INTERNAL_EVENT_GC_START
GC started.
uint32_t rb_event_flag_t
Represents event(s).
#define RUBY_INTERNAL_EVENT_NEWOBJ
Object allocated.
static VALUE RB_FL_TEST(VALUE obj, VALUE flags)
Tests if the given flag(s) are set or not.
static void RB_FL_SET_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_SET().
static void RB_FL_UNSET_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_UNSET().
@ RUBY_FL_PROMOTED
Ruby objects are "generational".
VALUE rb_define_module_under(VALUE outer, const char *name)
Defines a module under the namespace of outer.
int rb_scan_args_kw(int kw_flag, int argc, const VALUE *argv, const char *fmt,...)
Identical to rb_scan_args(), except it also accepts kw_splat.
int rb_keyword_given_p(void)
Determines if the current method is given a keyword argument.
int rb_get_kwargs(VALUE keyword_hash, const ID *table, int required, int optional, VALUE *values)
Keyword argument deconstructor.
#define T_COMPLEX
Old name of RUBY_T_COMPLEX.
#define T_FILE
Old name of RUBY_T_FILE.
#define FL_EXIVAR
Old name of RUBY_FL_EXIVAR.
#define T_STRING
Old name of RUBY_T_STRING.
#define xfree
Old name of ruby_xfree.
#define T_MASK
Old name of RUBY_T_MASK.
#define Qundef
Old name of RUBY_Qundef.
#define INT2FIX
Old name of RB_INT2FIX.
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
#define T_NIL
Old name of RUBY_T_NIL.
#define T_FLOAT
Old name of RUBY_T_FLOAT.
#define T_IMEMO
Old name of RUBY_T_IMEMO.
#define ID2SYM
Old name of RB_ID2SYM.
#define T_BIGNUM
Old name of RUBY_T_BIGNUM.
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
#define T_STRUCT
Old name of RUBY_T_STRUCT.
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
#define T_FIXNUM
Old name of RUBY_T_FIXNUM.
#define SYM2ID
Old name of RB_SYM2ID.
#define T_DATA
Old name of RUBY_T_DATA.
#define FL_SEEN_OBJ_ID
Old name of RUBY_FL_SEEN_OBJ_ID.
#define T_NONE
Old name of RUBY_T_NONE.
#define T_NODE
Old name of RUBY_T_NODE.
#define SIZET2NUM
Old name of RB_SIZE2NUM.
#define xmalloc
Old name of ruby_xmalloc.
#define LONG2FIX
Old name of RB_INT2FIX.
#define FIX2INT
Old name of RB_FIX2INT.
#define FL_FINALIZE
Old name of RUBY_FL_FINALIZE.
#define T_MODULE
Old name of RUBY_T_MODULE.
#define T_TRUE
Old name of RUBY_T_TRUE.
#define T_RATIONAL
Old name of RUBY_T_RATIONAL.
#define T_ICLASS
Old name of RUBY_T_ICLASS.
#define T_HASH
Old name of RUBY_T_HASH.
#define ALLOC_N
Old name of RB_ALLOC_N.
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
#define FL_SET
Old name of RB_FL_SET.
#define rb_ary_new3
Old name of rb_ary_new_from_args.
#define T_FALSE
Old name of RUBY_T_FALSE.
#define ULL2NUM
Old name of RB_ULL2NUM.
#define T_UNDEF
Old name of RUBY_T_UNDEF.
#define Qtrue
Old name of RUBY_Qtrue.
#define T_ZOMBIE
Old name of RUBY_T_ZOMBIE.
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define FIX2LONG
Old name of RB_FIX2LONG.
#define T_ARRAY
Old name of RUBY_T_ARRAY.
#define T_OBJECT
Old name of RUBY_T_OBJECT.
#define NIL_P
Old name of RB_NIL_P.
#define FL_WB_PROTECTED
Old name of RUBY_FL_WB_PROTECTED.
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
#define DBL2NUM
Old name of rb_float_new.
#define T_MATCH
Old name of RUBY_T_MATCH.
#define T_CLASS
Old name of RUBY_T_CLASS.
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
#define T_MOVED
Old name of RUBY_T_MOVED.
#define FL_TEST
Old name of RB_FL_TEST.
#define FL_UNSET
Old name of RB_FL_UNSET.
#define UINT2NUM
Old name of RB_UINT2NUM.
#define FIXNUM_P
Old name of RB_FIXNUM_P.
#define SYMBOL_P
Old name of RB_SYMBOL_P.
#define T_REGEXP
Old name of RUBY_T_REGEXP.
void rb_raise(VALUE exc_class, const char *fmt,...)
Exception entry point.
void rb_bug(const char *fmt,...)
Interpreter panic switch.
VALUE rb_eRangeError
RangeError exception.
#define ruby_verbose
This variable controls whether the interpreter is in debug mode.
VALUE rb_eRuntimeError
RuntimeError exception.
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
VALUE rb_eArgError
ArgumentError exception.
VALUE rb_ensure(VALUE(*b_proc)(VALUE), VALUE data1, VALUE(*e_proc)(VALUE), VALUE data2)
An equivalent to ensure clause.
VALUE rb_obj_hide(VALUE obj)
Make the object invisible from Ruby code.
VALUE rb_equal(VALUE lhs, VALUE rhs)
This function is an optimised version of calling #==.
VALUE rb_stdout
STDOUT constant.
#define RB_GNUC_EXTENSION_BLOCK(x)
This is expanded to the passed token for non-GCC compilers.
Routines to manipulate encodings of strings.
VALUE rb_funcall(VALUE recv, ID mid, int n,...)
Calls a method.
void rb_gc_mark(VALUE obj)
Marks an object.
void rb_memerror(void)
Triggers out-of-memory error.
static bool RB_OBJ_PROMOTED_RAW(VALUE obj)
This is the implementation of RB_OBJ_PROMOTED().
size_t rb_gc_count(void)
Identical to rb_gc_stat(), with "count" parameter.
VALUE rb_ary_new(void)
Allocates a new, empty array.
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
VALUE rb_big_eql(VALUE lhs, VALUE rhs)
Equality, in terms of eql?.
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
VALUE rb_hash_aref(VALUE hash, VALUE key)
Queries the given key in the given hash table.
VALUE rb_hash_aset(VALUE hash, VALUE key, VALUE val)
Inserts or replaces ("upsert"s) the objects into the given hash table.
VALUE rb_hash(VALUE obj)
Calculates a message authentication code of the passed object.
VALUE rb_hash_new(void)
Creates a new, empty hash object.
VALUE rb_io_write(VALUE io, VALUE str)
Writes the given string to the given IO.
VALUE rb_str_buf_append(VALUE dst, VALUE src)
Identical to rb_str_cat_cstr(), except it takes Ruby's string instead of C's.
VALUE rb_str_new_cstr(const char *ptr)
Identical to rb_str_new(), except it assumes the passed pointer is a pointer to a C string.
VALUE rb_str_buf_new(long capa)
Allocates a "string buffer".
const char * rb_sourcefile(void)
Resembles __FILE__.
VALUE rb_f_notimplement(int argc, const VALUE *argv, VALUE obj, VALUE marker)
Raises rb_eNotImpError.
int rb_sourceline(void)
Resembles __LINE__.
#define RB_SYM2ID
Just another name of rb_sym2id.
ID rb_intern(const char *name)
Finds or creates a symbol of the given name.
ID rb_sym2id(VALUE obj)
Converts an instance of rb_cSymbol into an ID.
void rb_define_const(VALUE klass, const char *name, VALUE val)
Defines a Ruby level constant under a namespace.
char * ptr
Pointer to the underlying memory region, of at least capa bytes.
int len
Length of the buffer.
void * rb_thread_call_with_gvl(void *(*func)(void *), void *data1)
(Re-)acquires the GVL.
#define strtod(s, e)
Just another name of ruby_strtod.
void ruby_qsort(void *, const size_t, const size_t, int(*)(const void *, const void *, void *), void *)
Reentrant implementation of quick sort.
#define DECIMAL_SIZE_OF_BITS(n)
an approximation of ceil(n * log10(2)), up to 1,048,576 (1<<20) without overflow within 32-bit calcul...
VALUE rb_sprintf(const char *fmt,...)
Ruby's extended sprintf(3).
#define MEMZERO(p, type, n)
Handy macro to erase a region of memory.
VALUE type(ANYARGS)
ANYARGS-ed function type.
int st_foreach(st_table *q, int_type *w, st_data_t e)
Iteration over the given table.
#define RARRAY_LEN
Just another name of rb_array_len.
#define RARRAY_AREF(a, i)
#define RBASIC(obj)
Convenient casting macro.
#define errno
Ractor-aware version of errno.
int ruby_native_thread_p(void)
Queries if the thread which calls this function is a ruby's thread.
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
#define RTEST
This is an old name of RB_TEST.
#define _(args)
This was a transition path from K&R to ANSI.
Ruby object's base components.
VALUE flags
Per-object flags.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
ruby_value_type
C-level type of an object.
@ RUBY_T_ICLASS
Hidden classes known as IClasses.
@ RUBY_T_FIXNUM
Integers formerly known as Fixnums.
@ RUBY_T_MASK
Bitmask of ruby_value_type.
@ RUBY_T_NONE
Non-object (swept etc.)