10#include "gc/gc_impl.h"
17#define WBCHECK_RCLASS_IS_ROOT FL_USER0
20static bool wbcheck_debug_enabled =
false;
23static bool wbcheck_verify_after_wb_enabled =
false;
26static bool wbcheck_warn_useless_wb_enabled =
false;
29wbcheck_debug(
const char *format, ...)
31 if (!wbcheck_debug_enabled)
return;
34 va_start(args, format);
35 vfprintf(stderr, format, args);
39#define WBCHECK_DEBUG(...) do { \
40 if (wbcheck_debug_enabled) { \
41 wbcheck_debug(__VA_ARGS__); \
46wbcheck_debug_obj_info_dump(
VALUE obj)
48 if (!wbcheck_debug_enabled)
return;
50 fprintf(stderr,
"%s\n", rb_raw_obj_info(buff,
sizeof(buff), obj));
54static void lock_and_maybe_gc(
void *objspace_ptr);
55static void force_gc(
void *objspace_ptr);
59wbcheck_configure_from_env(
void)
62 const char *debug_env = getenv(
"WBCHECK_DEBUG");
63 if (debug_env && (strcmp(debug_env,
"1") == 0 || strcmp(debug_env,
"true") == 0)) {
64 wbcheck_debug_enabled =
true;
68 const char *verify_after_wb_env = getenv(
"WBCHECK_VERIFY_AFTER_WB");
69 if (verify_after_wb_env && (strcmp(verify_after_wb_env,
"1") == 0 || strcmp(verify_after_wb_env,
"true") == 0)) {
70 wbcheck_verify_after_wb_enabled =
true;
74 const char *warn_useless_wb_env = getenv(
"WBCHECK_WARN_USELESS_WB");
75 if (warn_useless_wb_env && (strcmp(warn_useless_wb_env,
"1") == 0 || strcmp(warn_useless_wb_env,
"true") == 0)) {
76 wbcheck_warn_useless_wb_enabled =
true;
81static size_t heap_sizes[] = {
100#define HEAP_COUNT ((int)(sizeof(heap_sizes) / sizeof(heap_sizes[0])) - 1)
101#define MAX_HEAP_SIZE (heap_sizes[(HEAP_COUNT) - 1])
106 WBCHECK_STATE_MARKED,
108} wbcheck_object_state_t;
119 WBCHECK_PHASE_MUTATOR,
120 WBCHECK_PHASE_SNAPSHOT,
121 WBCHECK_PHASE_FULL_GC
133wbcheck_object_list_init_with_capacity(
size_t capacity)
136 if (!list) rb_bug(
"wbcheck: failed to allocate object list structure");
138 if (capacity < 4) capacity = 4;
139 list->items = malloc(capacity *
sizeof(
VALUE));
140 if (!list->items) rb_bug(
"wbcheck: failed to allocate object list array");
141 list->capacity = capacity;
147wbcheck_object_list_init(
void)
149 return wbcheck_object_list_init_with_capacity(4);
155 if (list->count >= list->capacity) {
156 size_t new_capacity = list->capacity == 0 ? 4 : list->capacity * 2;
157 VALUE *new_items = realloc(list->items, new_capacity *
sizeof(
VALUE));
158 if (!new_items) rb_bug(
"wbcheck: failed to reallocate object list array");
159 list->items = new_items;
160 list->capacity = new_capacity;
162 list->items[list->count++] = obj;
178 if (!wbcheck_debug_enabled)
return;
179 for (
size_t i = 0; i < list->count; i++) {
181 fprintf(stderr,
"-> %s\n", rb_raw_obj_info(buff,
sizeof(buff), list->items[i]));
188 for (
size_t i = 0; i < list->count; i++) {
189 if (list->items[i] == obj) {
204 wbcheck_object_state_t state;
205 wbcheck_color_t color;
212 WBCHECK_FINAL_JOB_DFREE,
213 WBCHECK_FINAL_JOB_FINALIZE,
217 void (*func)(
void *);
221 VALUE finalizer_array;
235 wbcheck_phase_t phase;
237 bool user_gc_disabled;
240 size_t missed_write_barrier_parents;
241 size_t missed_write_barrier_children;
242 size_t simulated_gc_count;
243 bool measure_total_time;
245 rb_nativethread_lock_t finalizer_lock;
258static void gc_run_finalizers(
void *data);
263wbcheck_get_object_info(
VALUE obj)
266 GC_ASSERT(wbcheck_global_objspace);
269 if (st_lookup(wbcheck_global_objspace->object_table, (st_data_t)obj, &value)) {
273 fprintf(stderr,
"wbcheck: object not found in tracking table\n");
275 fprintf(stderr,
"%s\n", rb_raw_obj_info(buff,
sizeof(buff), obj));
278 ((
volatile VALUE *)obj)[0];
281 rb_bug(
"wbcheck: object not found in tracking table");
291 size_t snapshot_count = gc_mark_snapshot ? gc_mark_snapshot->count : 0;
292 size_t wb_count = writebarrier_children ? writebarrier_children->count : 0;
294 fprintf(stderr,
"WBCHECK ERROR: Missed write barrier detected!\n");
295 fprintf(stderr,
" Parent object: %p (wb_protected: %s)\n",
296 (
void *)parent_obj, parent_info->wb_protected ?
"true" :
"false");
298 fprintf(stderr,
" %s\n", rb_raw_obj_info(buff,
sizeof(buff), parent_obj));
299 fprintf(stderr,
" Reference counts - snapshot: %zu, writebarrier: %zu, current: %zu, missed: %zu\n",
300 snapshot_count, wb_count, current_refs->count, missed_refs->count);
302 for (
size_t i = 0; i < missed_refs->count; i++) {
303 VALUE missed_ref = missed_refs->items[i];
305 fprintf(stderr,
" Missing reference to: %p\n %s\n", (
void *)missed_ref, rb_raw_obj_info(buff,
sizeof(buff), missed_ref));
308 fprintf(stderr,
"\n");
309 objspace->missed_write_barrier_parents++;
310 objspace->missed_write_barrier_children += missed_refs->count;
318 if (
objspace->missed_write_barrier_parents > 0) {
320 rb_bug(
"wbcheck: missed write barrier detected (%zu object(s), %zu reference(s))",
321 objspace->missed_write_barrier_parents,
322 objspace->missed_write_barrier_children);
332 size_t snapshot_count = gc_mark_snapshot ? gc_mark_snapshot->count : 0;
333 size_t wb_count = writebarrier_children ? writebarrier_children->count : 0;
335 WBCHECK_DEBUG(
"wbcheck: comparing references for object %p\n", (
void *)parent_obj);
336 WBCHECK_DEBUG(
"wbcheck: current refs: %zu, snapshot refs: %zu, wb refs: %zu\n",
337 current_refs->count, snapshot_count, wb_count);
343 size_t snapshot_idx = 0;
346 for (
size_t i = 0; i < current_refs->count; i++) {
347 VALUE current_ref = current_refs->items[i];
356 bool found_in_snapshot =
false;
357 if (gc_mark_snapshot && snapshot_count > 0) {
358 size_t start_idx = snapshot_idx;
360 if (gc_mark_snapshot->items[snapshot_idx] == current_ref) {
361 found_in_snapshot =
true;
363 if (snapshot_idx >= snapshot_count) snapshot_idx = 0;
367 if (snapshot_idx >= snapshot_count) snapshot_idx = 0;
368 }
while (snapshot_idx != start_idx);
371 if (found_in_snapshot) {
383 if (current_ref == parent_obj) {
389 if (writebarrier_children && wbcheck_object_list_contains(writebarrier_children, current_ref)) {
396 missed_refs = wbcheck_object_list_init();
398 wbcheck_object_list_append(missed_refs, current_ref);
403 wbcheck_report_error(objspace_ptr, parent_obj, current_refs, gc_mark_snapshot, writebarrier_children, missed_refs);
404 wbcheck_object_list_free(missed_refs);
409wbcheck_register_object(
void *objspace_ptr,
VALUE obj,
size_t alloc_size,
bool wb_protected)
416 if (!info) rb_bug(
"wbcheck_register_object: failed to allocate object info");
418 info->alloc_size = alloc_size;
419 info->wb_protected = wb_protected;
420 info->finalizers = 0;
421 info->gc_mark_snapshot = NULL;
422 info->mark_maybe_snapshot = NULL;
423 info->writebarrier_children = NULL;
424 info->state = WBCHECK_STATE_CLEAR;
425 info->color = WBCHECK_COLOR_BLACK;
428 st_insert(
objspace->object_table, (st_data_t)obj, (st_data_t)info);
432wbcheck_unregister_object(
void *objspace_ptr,
VALUE obj)
437 if (st_delete(
objspace->object_table, (st_data_t *)&obj, (st_data_t *)&info)) {
439 wbcheck_object_list_free(info->gc_mark_snapshot);
440 wbcheck_object_list_free(info->mark_maybe_snapshot);
441 wbcheck_object_list_free(info->writebarrier_children);
444 rb_bug(
"wbcheck_unregister_object: object not found in table");
450rb_gc_impl_objspace_alloc(
void)
452 wbcheck_configure_from_env();
455 if (!
objspace) rb_bug(
"wbcheck: failed to allocate objspace");
457 objspace->objects_to_capture = wbcheck_object_list_init();
458 objspace->objects_to_verify = wbcheck_object_list_init();
460 objspace->current_maybe_refs = NULL;
461 objspace->mark_queue = wbcheck_object_list_init();
462 objspace->weak_references = wbcheck_object_list_init();
463 objspace->phase = WBCHECK_PHASE_MUTATOR;
467 objspace->missed_write_barrier_parents = 0;
468 objspace->missed_write_barrier_children = 0;
470 objspace->measure_total_time =
true;
476rb_gc_impl_objspace_init(
void *objspace_ptr)
482 objspace->object_table = st_init_numtable();
484 rb_bug(
"wbcheck: failed to create object table");
496rb_gc_impl_ractor_cache_alloc(
void *objspace_ptr,
void *ractor)
503rb_gc_impl_zjit_new_obj_fastpath(
void *objspace_ptr,
size_t alloc_size,
VALUE flags,
VALUE klass,
510rb_gc_impl_set_params(
void *objspace_ptr)
516gc_verify_internal_consistency(
VALUE self)
524 VALUE gc_constants = rb_hash_new();
526 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVALUE_SIZE")),
SIZET2NUM(
sizeof(
struct RBasic) +
sizeof(
VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX])));
528 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVALUE_OVERHEAD")),
INT2NUM(0));
529 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVARGC_MAX_ALLOCATE_SIZE")),
SIZET2NUM(rb_gc_impl_max_allocation_size()));
530 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"HEAP_COUNT")),
LONG2FIX(HEAP_COUNT));
531 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"SIZE_POOL_COUNT")),
LONG2FIX(HEAP_COUNT));
532 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVALUE_OLD_AGE")),
INT2FIX(3));
534 rb_define_const(
rb_mGC,
"INTERNAL_CONSTANTS", gc_constants);
549rb_gc_impl_shutdown_free_objects(
void *objspace_ptr)
555rb_gc_impl_objspace_free(
void *objspace_ptr)
561rb_gc_impl_ractor_cache_free(
void *objspace_ptr,
void *cache)
568rb_gc_impl_start(
void *objspace_ptr,
bool full_mark,
bool immediate_mark,
bool immediate_sweep,
bool compact,
bool global)
577 unsigned int lev = RB_GC_VM_LOCK();
579 force_gc(objspace_ptr);
580 RB_GC_VM_UNLOCK(lev);
584rb_gc_impl_during_gc_p(
void *objspace_ptr)
587 return objspace->phase != WBCHECK_PHASE_MUTATOR;
591wbcheck_prepare_heap_i(
VALUE obj,
void *data)
593 rb_gc_prepare_heap_process_object(obj);
597rb_gc_impl_prepare_heap(
void *objspace_ptr)
599 rb_gc_impl_each_object(objspace_ptr, wbcheck_prepare_heap_i, NULL);
603rb_gc_impl_gc_enable(
void *objspace_ptr)
610rb_gc_impl_gc_disable(
void *objspace_ptr,
bool finish_current_gc)
617rb_gc_impl_gc_enabled_p(
void *objspace_ptr)
624rb_gc_impl_user_gc_disabled_set(
void *objspace_ptr,
bool disable)
627 const bool was =
objspace->user_gc_disabled;
628 objspace->user_gc_disabled = disable;
634rb_gc_impl_user_gc_disabled_p(
void *objspace_ptr)
641rb_gc_impl_gc_rest(
void *objspace_ptr)
647rb_gc_impl_stress_set(
void *objspace_ptr,
VALUE flag)
654rb_gc_impl_stress_get(
void *objspace_ptr)
661rb_gc_impl_config_get(
void *objspace_ptr)
663 return rb_hash_new();
667rb_gc_impl_config_set(
void *objspace_ptr,
VALUE hash)
672rb_gc_impl_get_vm_context(
void *objspace_ptr)
685 size_t snapshot_count = (info->gc_mark_snapshot) ? info->gc_mark_snapshot->count : 0;
686 size_t wb_children_count = (info->writebarrier_children) ? info->writebarrier_children->count : 0;
687 size_t capacity_hint = snapshot_count + wb_children_count;
693 objspace->current_maybe_refs = NULL;
694 objspace->phase = WBCHECK_PHASE_SNAPSHOT;
695 rb_gc_initialize_vm_context(&
objspace->vm_context);
701 objspace->phase = WBCHECK_PHASE_MUTATOR;
706 wbcheck_object_list_free(info->mark_maybe_snapshot);
707 info->mark_maybe_snapshot =
objspace->current_maybe_refs;
708 objspace->current_maybe_refs = NULL;
710 if (wbcheck_debug_enabled) {
711 WBCHECK_DEBUG(
"wbcheck: collected %zu references from %p\n", new_list->count, (
void *)obj);
713 fprintf(stderr,
"%s\n", rb_raw_obj_info(buff,
sizeof(buff), obj));
714 wbcheck_object_list_debug_print(new_list);
721wbcheck_collect_initial_references(
void *objspace_ptr,
VALUE obj)
723 WBCHECK_DEBUG(
"wbcheck: collecting initial references from %p:\n", obj);
724 wbcheck_debug_obj_info_dump(obj);
731 info->gc_mark_snapshot = new_list;
732 info->state = WBCHECK_STATE_MARKED;
736wbcheck_verify_object_references(
void *objspace_ptr,
VALUE obj)
741 if (!info->wb_protected) {
746 if (info->state == WBCHECK_STATE_CLEAR) {
751 WBCHECK_DEBUG(
"wbcheck: verifying references for object:\n");
752 wbcheck_debug_obj_info_dump(obj);
758 if (wbcheck_warn_useless_wb_enabled && info->writebarrier_children) {
759 for (
size_t i = 0; i < info->writebarrier_children->count; i++) {
760 VALUE wb_ref = info->writebarrier_children->items[i];
761 if (!wbcheck_object_list_contains(current_refs, wb_ref)) {
762 fprintf(stderr,
"WBCHECK WARNING: Potentially useless write barrier detected for object %p\n", (
void *)obj);
763 fprintf(stderr,
" Write barrier was recorded for reference to %p, but object no longer references it\n", (
void *)wb_ref);
765 fprintf(stderr,
" Parent: %s\n", rb_raw_obj_info(buff,
sizeof(buff), obj));
766 fprintf(stderr,
" Stale reference: %s\n", rb_raw_obj_info(buff,
sizeof(buff), wb_ref));
772 wbcheck_compare_references(objspace_ptr, obj, current_refs, info->gc_mark_snapshot, info->writebarrier_children);
775 wbcheck_object_list_free(info->gc_mark_snapshot);
776 wbcheck_object_list_free(info->writebarrier_children);
777 info->gc_mark_snapshot = current_refs;
778 info->writebarrier_children = NULL;
779 info->state = WBCHECK_STATE_MARKED;
789 if (!st_lookup(
objspace->object_table, (st_data_t)obj, &value)) {
790 rb_bug(
"wbcheck: asked to mark object %p not in our object table", (
void *)obj);
794 if (info->color != WBCHECK_COLOR_WHITE) {
798 info->color = WBCHECK_COLOR_GRAY;
799 wbcheck_object_list_append(
objspace->mark_queue, obj);
802 wbcheck_object_list_append(
objspace->weak_references, obj);
805 WBCHECK_DEBUG(
"wbcheck: marked gray: %p\n", (
void *)obj);
810st_foreach_reset_white(st_data_t key, st_data_t val, st_data_t arg)
813 info->color = WBCHECK_COLOR_WHITE;
819st_foreach_mark_finalizers(st_data_t key, st_data_t val, st_data_t arg)
824 if (info->finalizers) {
825 wbcheck_mark_gray(
objspace, info->finalizers);
835 WBCHECK_DEBUG(
"wbcheck: starting GC mark phase\n");
837 objspace->phase = WBCHECK_PHASE_FULL_GC;
838 rb_gc_initialize_vm_context(&
objspace->vm_context);
842 st_foreach(
objspace->object_table, st_foreach_reset_white, 0);
845 st_foreach(
objspace->object_table, st_foreach_mark_finalizers, (st_data_t)
objspace);
850 while (job != NULL) {
852 case WBCHECK_FINAL_JOB_DFREE:
854 case WBCHECK_FINAL_JOB_FINALIZE:
855 wbcheck_mark_gray(
objspace, job->as.finalize.finalizer_array);
858 rb_bug(
"wbcheck_mark_phase: unknown final job type %d", job->kind);
864 rb_gc_save_machine_context();
868 while (
objspace->mark_queue->count > 0) {
873 if (st_lookup(
objspace->object_table, (st_data_t)obj, &value)) {
875 if (info->color == WBCHECK_COLOR_GRAY) {
877 if (info->gc_mark_snapshot) {
878 for (
size_t i = 0; i < info->gc_mark_snapshot->count; i++) {
879 wbcheck_mark_gray(
objspace, info->gc_mark_snapshot->items[i]);
884 if (info->mark_maybe_snapshot) {
885 for (
size_t i = 0; i < info->mark_maybe_snapshot->count; i++) {
886 wbcheck_mark_gray(
objspace, info->mark_maybe_snapshot->items[i]);
891 info->color = WBCHECK_COLOR_BLACK;
892 WBCHECK_DEBUG(
"wbcheck: marked black: %p\n", (
void *)obj);
897 objspace->phase = WBCHECK_PHASE_MUTATOR;
899 WBCHECK_DEBUG(
"wbcheck: tri-color mark phase complete\n");
904wbcheck_sweep_callback(st_data_t key, st_data_t val, st_data_t arg,
int error)
910 if (info->color == WBCHECK_COLOR_WHITE) {
911 WBCHECK_DEBUG(
"wbcheck: sweeping unmarked object %p\n", (
void *)obj);
916 rb_gc_obj_free_vm_weak_references(obj);
919 if (info->finalizers) {
920 make_final_job(
objspace, obj, info->finalizers);
925 if (rb_gc_obj_free(
objspace, obj)) {
927 wbcheck_object_list_free(info->gc_mark_snapshot);
928 wbcheck_object_list_free(info->mark_maybe_snapshot);
929 wbcheck_object_list_free(info->writebarrier_children);
939 wbcheck_object_list_free(info->gc_mark_snapshot);
940 wbcheck_object_list_free(info->mark_maybe_snapshot);
941 wbcheck_object_list_free(info->writebarrier_children);
957 WBCHECK_DEBUG(
"wbcheck: starting sweep phase\n");
959 size_t objects_before = st_table_size(
objspace->object_table);
962 st_foreach_check(
objspace->object_table, wbcheck_sweep_callback, (st_data_t)
objspace, 0);
964 size_t objects_after = st_table_size(
objspace->object_table);
965 size_t freed_objects = objects_before - objects_after;
968 objspace->gc_threshold = objects_after * 2;
970 WBCHECK_DEBUG(
"wbcheck: sweep phase complete - freed %zu objects (%zu -> %zu), new threshold: %zu\n",
971 freed_objects, objects_before, objects_after,
objspace->gc_threshold);
980 WBCHECK_DEBUG(
"wbcheck: processing %zu weak reference objects\n",
objspace->weak_references->count);
982 for (
size_t i = 0; i <
objspace->weak_references->count; i++) {
984 rb_gc_handle_weak_references(obj);
987 objspace->weak_references->count = 0;
994 WBCHECK_DEBUG(
"wbcheck: starting full GC\n");
1000 WBCHECK_DEBUG(
"wbcheck: updating snapshots for all objects\n");
1001 wbcheck_foreach_object(
objspace, wbcheck_update_all_snapshots_callback,
objspace);
1009 wbcheck_process_weak_references(
objspace);
1017 WBCHECK_DEBUG(
"wbcheck: full GC complete\n");
1021gc_step(
void *objspace_ptr,
bool force)
1028 if (!
objspace->gc_enabled && !force)
return;
1031 if (wbcheck_verify_after_wb_enabled) {
1032 for (
size_t i = 0; i <
objspace->objects_to_verify->count; i++) {
1034 wbcheck_verify_object_references(objspace_ptr, obj);
1038 objspace->objects_to_verify->count = 0;
1044 for (
size_t i = 0; i <
objspace->objects_to_capture->count; i++) {
1046 wbcheck_collect_initial_references(objspace_ptr, obj);
1050 objspace->objects_to_capture->count = 0;
1064maybe_gc(
void *objspace_ptr)
1066 gc_step(objspace_ptr,
false);
1070force_gc(
void *objspace_ptr)
1072 gc_step(objspace_ptr,
true);
1075int ruby_thread_has_gvl_p(
void);
1078lock_and_maybe_gc_gvl(
void *objspace_ptr)
1080 unsigned int lev = RB_GC_VM_LOCK();
1083 maybe_gc(objspace_ptr);
1085 RB_GC_VM_UNLOCK(lev);
1090lock_and_maybe_gc(
void *objspace_ptr)
1094 if (!ruby_thread_has_gvl_p()) {
1098 lock_and_maybe_gc_gvl(objspace_ptr);
1103rb_gc_impl_new_obj(
void *objspace_ptr,
void *cache_ptr,
VALUE klass,
VALUE flags,
bool wb_protected,
size_t alloc_size,
size_t *actual_alloc_size)
1105 unsigned int lev = RB_GC_VM_LOCK();
1109 maybe_gc(objspace_ptr);
1112 alloc_size = rb_gc_impl_size_slot_size(objspace_ptr, alloc_size);
1115 VALUE *mem = malloc(alloc_size);
1116 if (!mem) rb_bug(
"FIXME: malloc failed");
1118 *actual_alloc_size = alloc_size;
1122 RBASIC(obj)->flags = flags;
1126 wbcheck_register_object(objspace_ptr, obj, alloc_size, wb_protected);
1130 wbcheck_object_list_append(
objspace->objects_to_capture, obj);
1132 RB_GC_VM_UNLOCK(lev);
1137rb_gc_impl_obj_slot_size(
VALUE obj)
1139 unsigned int lev = RB_GC_VM_LOCK();
1142 size_t result = info->alloc_size;
1144 RB_GC_VM_UNLOCK(lev);
1149rb_gc_impl_size_slot_size(
void *objspace_ptr,
size_t size)
1151 for (
int i = 0; i < HEAP_COUNT; i++) {
1152 if (size <= heap_sizes[i])
return heap_sizes[i];
1154 rb_bug(
"size too big");
1158rb_gc_impl_size_allocatable_p(
size_t size)
1161 return size <= rb_gc_impl_max_allocation_size();
1165rb_gc_impl_max_allocation_size(
void)
1167 return MAX_HEAP_SIZE;
1172rb_gc_impl_malloc(
void *objspace_ptr,
size_t size,
bool gc_allowed)
1175 lock_and_maybe_gc(objspace_ptr);
1177 return malloc(size);
1181rb_gc_impl_calloc(
void *objspace_ptr,
size_t size,
bool gc_allowed)
1184 lock_and_maybe_gc(objspace_ptr);
1186 return calloc(1, size);
1190rb_gc_impl_realloc(
void *objspace_ptr,
void *ptr,
size_t new_size,
size_t old_size,
bool gc_allowed)
1193 lock_and_maybe_gc(objspace_ptr);
1195 return realloc(ptr, new_size);
1199rb_gc_impl_free(
void *objspace_ptr,
void *ptr,
size_t old_size)
1205rb_gc_impl_adjust_memory_usage(
void *objspace_ptr, ssize_t diff)
1214 WBCHECK_DEBUG(
"wbcheck: gc_mark called\n");
1215 wbcheck_debug_obj_info_dump(obj);
1220 case WBCHECK_PHASE_SNAPSHOT:
1223 wbcheck_object_list_append(
objspace->current_refs, obj);
1225 case WBCHECK_PHASE_FULL_GC:
1229 case WBCHECK_PHASE_MUTATOR:
1231 rb_bug(
"wbcheck: gc_mark called during mutator phase");
1237rb_gc_impl_mark(
void *objspace_ptr,
VALUE obj)
1244rb_gc_impl_mark_and_move(
void *objspace_ptr,
VALUE *ptr)
1251rb_gc_impl_mark_and_pin(
void *objspace_ptr,
VALUE obj)
1258rb_gc_impl_mark_maybe(
void *objspace_ptr,
VALUE obj)
1262 if (!rb_gc_impl_live_object_p(objspace_ptr, (
void *)obj))
return;
1265 case WBCHECK_PHASE_SNAPSHOT:
1270 if (!
objspace->current_maybe_refs) {
1271 objspace->current_maybe_refs = wbcheck_object_list_init();
1273 wbcheck_object_list_append(
objspace->current_maybe_refs, obj);
1275 case WBCHECK_PHASE_FULL_GC:
1278 case WBCHECK_PHASE_MUTATOR:
1279 rb_bug(
"wbcheck: rb_gc_impl_mark_maybe called during mutator phase");
1286rb_gc_impl_declare_weak_references(
void *objspace_ptr,
VALUE obj)
1292rb_gc_impl_handle_weak_references_alive_p(
void *objspace_ptr,
VALUE obj)
1297 if (st_lookup(
objspace->object_table, (st_data_t)obj, &value)) {
1299 return info->color != WBCHECK_COLOR_WHITE;
1307rb_gc_impl_register_pinning_obj(
void *objspace_ptr,
VALUE obj)
1313rb_gc_impl_object_moved_p(
void *objspace_ptr,
VALUE obj)
1320rb_gc_impl_pinned_p(
void *objspace_ptr,
VALUE obj)
1327rb_gc_impl_location(
void *objspace_ptr,
VALUE value)
1335rb_gc_impl_writebarrier(
void *objspace_ptr,
VALUE a,
VALUE b)
1339 unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER();
1347 if (info->state != WBCHECK_STATE_CLEAR) {
1351 if (!info->writebarrier_children) {
1352 info->writebarrier_children = wbcheck_object_list_init();
1356 wbcheck_object_list_append(info->writebarrier_children, b);
1358 WBCHECK_DEBUG(
"wbcheck: write barrier recorded reference from %p to %p\n", (
void *)a, (
void *)b);
1361 if (wbcheck_verify_after_wb_enabled && info->state != WBCHECK_STATE_DIRTY) {
1362 WBCHECK_DEBUG(
"wbcheck: queueing object for verification after write barrier\n");
1363 info->state = WBCHECK_STATE_DIRTY;
1364 wbcheck_object_list_append(
objspace->objects_to_verify, a);
1367 WBCHECK_DEBUG(
"wbcheck: write barrier skipped (snapshot not initialized) from %p to %p\n", (
void *)a, (
void *)b);
1370 RB_GC_VM_UNLOCK_NO_BARRIER(lev);
1374rb_gc_impl_writebarrier_unprotect(
void *objspace_ptr,
VALUE obj)
1376 WBCHECK_DEBUG(
"wbcheck: writebarrier_unprotect called on object %p\n", (
void *)obj);
1378 unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER();
1381 info->wb_protected =
false;
1383 RB_GC_VM_UNLOCK_NO_BARRIER(lev);
1387rb_gc_impl_writebarrier_remember(
void *objspace_ptr,
VALUE obj)
1389 WBCHECK_DEBUG(
"wbcheck: writebarrier_remember called on object %p\n", (
void *)obj);
1391 unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER();
1397 if (info->state != WBCHECK_STATE_CLEAR) {
1399 wbcheck_object_list_free(info->gc_mark_snapshot);
1400 info->gc_mark_snapshot = NULL;
1402 wbcheck_object_list_free(info->mark_maybe_snapshot);
1403 info->mark_maybe_snapshot = NULL;
1407 wbcheck_object_list_append(
objspace->objects_to_capture, obj);
1410 if (info->writebarrier_children) {
1411 wbcheck_object_list_free(info->writebarrier_children);
1412 info->writebarrier_children = NULL;
1416 info->state = WBCHECK_STATE_CLEAR;
1422 RB_GC_VM_UNLOCK_NO_BARRIER(lev);
1432wbcheck_foreach_object_i(st_data_t key, st_data_t val, st_data_t arg)
1438 return foreach_data->callback(obj, info, foreach_data->data);
1445 .callback = callback,
1449 st_foreach(
objspace->object_table, wbcheck_foreach_object_i, (st_data_t)&foreach_data);
1454wbcheck_snapshot_collector(st_data_t key, st_data_t val, st_data_t arg)
1458 wbcheck_object_list_append(snapshot, obj);
1466 size_t object_count = st_table_size(
objspace->object_table);
1468 st_foreach(
objspace->object_table, wbcheck_snapshot_collector, (st_data_t)snapshot);
1474rb_gc_impl_each_objects(
void *objspace_ptr,
int (*callback)(
void *,
void *,
size_t,
void *),
void *data)
1481 for (
size_t i = 0; i < snapshot->count; i++) {
1482 VALUE obj = snapshot->items[i];
1484 if (st_lookup(
objspace->object_table, (st_data_t)obj, &value)) {
1486 int result = callback(
1488 (
void *)((
char *)obj + info->alloc_size),
1492 if (result != 0)
break;
1496 wbcheck_object_list_free(snapshot);
1500wbcheck_each_objects_shareable_i(
void *start,
void *end,
size_t stride,
void *d)
1502 struct { int (*func)(
void *,
void *, size_t,
void *);
void *data; } *args = d;
1504 return args->func(start, end, stride, args->data);
1510rb_gc_impl_each_objects_shareable(
void *objspace_ptr,
int (*callback)(
void *,
void *,
size_t,
void *),
void *data)
1512 struct { int (*func)(
void *,
void *, size_t,
void *);
void *data; } args = { callback, data };
1513 rb_gc_impl_each_objects(objspace_ptr, wbcheck_each_objects_shareable_i, &args);
1517rb_gc_impl_each_objects_foreign(
void *objspace_ptr,
int (*callback)(
void *,
void *,
size_t,
void *),
void *data)
1523rb_gc_impl_multi_objspace_p(
void)
1529rb_gc_impl_objspace_retire_gc(
void *objspace_ptr)
1534rb_gc_impl_during_global_gc_p(
void *objspace_ptr)
1539 return objspace->phase != WBCHECK_PHASE_MUTATOR;
1543rb_gc_impl_during_postmortem_p(
void *objspace_ptr)
1549rb_gc_impl_obj_foreign_p(
void *objspace_ptr,
VALUE obj)
1555rb_gc_impl_shref_marked_p(
void *objspace_ptr,
VALUE obj)
1561rb_gc_impl_heap_page_count(
void *objspace_ptr)
1567rb_gc_impl_objspace_absorb(
void *dst_ptr,
void *src_ptr)
1572rb_gc_impl_obj_became_shareable(
void *objspace_ptr,
VALUE obj)
1577rb_gc_impl_zjit_writebarrier_required_offset(
void)
1584rb_gc_impl_each_object(
void *objspace_ptr,
void (*func)(
VALUE obj,
void *data),
void *data)
1591 for (
size_t i = 0; i < snapshot->count; i++) {
1592 VALUE obj = snapshot->items[i];
1594 if (st_lookup(
objspace->object_table, (st_data_t)obj, &value)) {
1599 wbcheck_object_list_free(snapshot);
1606 job->next =
objspace->finalizer_jobs;
1617 objspace->finalizer_jobs = job->next;
1625rb_gc_impl_make_zombie(
void *objspace_ptr,
VALUE obj,
void (*dfree)(
void *),
void *data)
1627 if (dfree == NULL)
return;
1632 job->kind = WBCHECK_FINAL_JOB_DFREE;
1633 job->as.dfree.func = dfree;
1634 job->as.dfree.data = data;
1636 finalizer_jobs_push(
objspace, job);
1638 if (!ruby_free_at_exit_p()) {
1642 WBCHECK_DEBUG(
"wbcheck: made zombie for object %p with dfree function\n", (
void *)obj);
1646rb_gc_impl_define_finalizer(
void *objspace_ptr,
VALUE obj,
VALUE block)
1648 unsigned int lev = RB_GC_VM_LOCK();
1657 VALUE table = info->finalizers;
1658 VALUE result = block;
1664 info->finalizers = table;
1670 for (i = 1; i <
len; i++) {
1674 goto unlock_and_return;
1682 RB_GC_VM_UNLOCK(lev);
1687rb_gc_impl_undefine_finalizer(
void *objspace_ptr,
VALUE obj)
1689 unsigned int lev = RB_GC_VM_LOCK();
1696 info->finalizers = 0;
1699 RB_GC_VM_UNLOCK(lev);
1703rb_gc_impl_copy_finalizer(
void *objspace_ptr,
VALUE dest,
VALUE obj)
1709 unsigned int lev = RB_GC_VM_LOCK();
1714 if (src_info->finalizers) {
1718 dest_info->finalizers = table;
1722 RB_GC_VM_UNLOCK(lev);
1726wbcheck_get_final(
rb_len_t i,
void *data)
1742 job->kind = WBCHECK_FINAL_JOB_FINALIZE;
1743 job->as.finalize.finalizer_array = finalizer_array;
1745 finalizer_jobs_push(
objspace, job);
1749gc_run_finalizers(
void *data)
1753 rb_gc_set_pending_interrupt();
1756 while ((job = finalizer_jobs_pop(
objspace)) != NULL) {
1757 switch (job->kind) {
1758 case WBCHECK_FINAL_JOB_DFREE:
1759 job->as.dfree.func(job->as.dfree.data);
1761 case WBCHECK_FINAL_JOB_FINALIZE: {
1762 VALUE finalizer_array = job->as.finalize.finalizer_array;
1764 rb_gc_run_obj_finalizer(
1768 (
void *)finalizer_array
1779 rb_gc_unset_pending_interrupt();
1785 if (info->finalizers) {
1786 VALUE table = info->finalizers;
1788 rb_gc_run_obj_finalizer(
RARRAY_AREF(table, 0), count, wbcheck_get_final, (
void *)table);
1791 info->finalizers = 0;
1797 wbcheck_run_finalizers_for_object(obj, info);
1804 void *objspace_ptr = data;
1805 wbcheck_verify_object_references(objspace_ptr, obj);
1812 void *objspace_ptr = data;
1815 if (info->wb_protected && info->state != WBCHECK_STATE_CLEAR) {
1816 wbcheck_verify_object_references(objspace_ptr, obj);
1820 wbcheck_object_list_free(info->gc_mark_snapshot);
1821 info->gc_mark_snapshot = current_refs;
1822 info->state = WBCHECK_STATE_MARKED;
1831 void *objspace_ptr = data;
1833 if (rb_gc_shutdown_call_finalizer_p(obj)) {
1834 WBCHECK_DEBUG(
"wbcheck: finalizing object during shutdown: %p\n", (
void *)obj);
1835 rb_gc_obj_free_vm_weak_references(obj);
1836 if (rb_gc_obj_free(objspace_ptr, obj)) {
1846rb_gc_impl_shutdown_call_finalizer(
void *objspace_ptr)
1851 wbcheck_foreach_object(
objspace, wbcheck_shutdown_call_finalizer_callback, NULL);
1854 unsigned int verify_lev = RB_GC_VM_LOCK();
1855 WBCHECK_DEBUG(
"wbcheck: verifying references for all objects after finalizers\n");
1856 wbcheck_foreach_object(
objspace, wbcheck_verify_all_references_callback, objspace_ptr);
1857 WBCHECK_DEBUG(
"wbcheck: finished verifying all object references\n");
1858 RB_GC_VM_UNLOCK(verify_lev);
1861 WBCHECK_DEBUG(
"wbcheck: no write barrier violations detected\n");
1864 unsigned int lev = RB_GC_VM_LOCK();
1865 WBCHECK_DEBUG(
"wbcheck: calling rb_gc_obj_free on objects that need shutdown finalization\n");
1866 wbcheck_foreach_object(
objspace, wbcheck_shutdown_finalizer_callback, objspace_ptr);
1867 WBCHECK_DEBUG(
"wbcheck: finished calling rb_gc_obj_free\n");
1870 WBCHECK_DEBUG(
"wbcheck: running pending finalizer jobs\n");
1872 WBCHECK_DEBUG(
"wbcheck: finished running finalizer jobs\n");
1873 RB_GC_VM_UNLOCK(lev);
1878rb_gc_impl_before_fork(
void *objspace_ptr)
1882 unsigned int lev = RB_GC_VM_LOCK();
1884 force_gc(objspace_ptr);
1885 RB_GC_VM_UNLOCK(lev);
1889rb_gc_impl_after_fork(
void *objspace_ptr, rb_pid_t pid)
1896rb_gc_impl_set_measure_total_time(
void *objspace_ptr,
VALUE flag)
1903rb_gc_impl_get_measure_total_time(
void *objspace_ptr)
1906 return objspace->measure_total_time;
1910rb_gc_impl_get_total_time(
void *objspace_ptr)
1917rb_gc_impl_gc_count(
void *objspace_ptr)
1921 return objspace->simulated_gc_count;
1927rb_gc_impl_latest_gc_info(
void *objspace_ptr,
VALUE key)
1934rb_gc_impl_stat(
void *objspace_ptr,
VALUE hash_or_sym)
1936 if (objspace_ptr == NULL) {
1952 rb_bug(
"non-hash or symbol given");
1955#define SET(name, attr) \
1956 if (key == ID2SYM(rb_intern(#name))) \
1957 return SIZET2NUM(attr); \
1958 else if (hash != Qnil) \
1959 rb_hash_aset(hash, ID2SYM(rb_intern(#name)), SIZET2NUM(attr));
1962 SET(count,
objspace->simulated_gc_count);
1963 SET(time,
objspace->measure_total_time ?
objspace->simulated_gc_count : 0);
1964 SET(tracked_objects, st_table_size(
objspace->object_table));
1968 rb_raise(rb_eArgError,
"unknown key: %"PRIsVALUE,
rb_sym2str(key));
1977rb_gc_impl_stat_heap(
void *objspace_ptr,
VALUE heap_name,
VALUE hash_or_sym)
1980 int heap_idx =
FIX2INT(heap_name);
1981 if (heap_idx < 0 || heap_idx >= HEAP_COUNT) {
1982 rb_raise(rb_eArgError,
"size pool index out of range");
1985 if (hash_or_sym ==
ID2SYM(rb_intern(
"slot_size"))) {
2000rb_gc_impl_active_gc_name(
void)
2007#define WBCHECK_OBJECT_METADATA_ENTRY_COUNT 2
2011rb_gc_impl_object_metadata(
void *objspace_ptr,
VALUE obj)
2013 static ID ID_object_id, ID_shareable;
2015 if (!ID_object_id) {
2016 ID_object_id = rb_intern(
"object_id");
2017 ID_shareable = rb_intern(
"shareable");
2022#define SET_ENTRY(na, v) do { \
2023 GC_ASSERT(n < WBCHECK_OBJECT_METADATA_ENTRY_COUNT); \
2024 object_metadata_entries[n].name = ID_##na; \
2025 object_metadata_entries[n].val = v; \
2029 if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj));
2033 object_metadata_entries[n].name = 0;
2034 object_metadata_entries[n].val = 0;
2036 return object_metadata_entries;
2040rb_gc_impl_live_object_p(
void *objspace_ptr,
const void *ptr)
2042 GC_ASSERT(wbcheck_global_objspace);
2044 unsigned int lev = RB_GC_VM_LOCK();
2048 bool result = st_lookup(wbcheck_global_objspace->object_table, (st_data_t)ptr, &value);
2050 RB_GC_VM_UNLOCK(lev);
2055rb_gc_impl_garbage_object_p(
void *objspace_ptr,
VALUE obj)
2057 unsigned int lev = RB_GC_VM_LOCK();
2061 bool result = st_lookup(wbcheck_global_objspace->object_table, (st_data_t)obj, &value);
2063 RB_GC_VM_UNLOCK(lev);
2068rb_gc_impl_set_event_hook(
void *objspace_ptr,
const rb_event_flag_t event)
2074rb_gc_impl_copy_attributes(
void *objspace_ptr,
VALUE dest,
VALUE obj)
2078 if (!src_info->wb_protected) {
2079 rb_gc_impl_writebarrier_unprotect(objspace_ptr, dest);
2081 rb_gc_impl_copy_finalizer(objspace_ptr, dest, obj);
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
unsigned int rb_postponed_job_handle_t
The type of a handle returned from rb_postponed_job_preregister and passed to rb_postponed_job_trigge...
void rb_postponed_job_trigger(rb_postponed_job_handle_t h)
Triggers a pre-registered job registered with rb_postponed_job_preregister, scheduling it for executi...
rb_postponed_job_handle_t rb_postponed_job_preregister(unsigned int flags, rb_postponed_job_func_t func, void *data)
Pre-registers a func in Ruby's postponed job preregistration table, returning an opaque handle which ...
#define RUBY_INTERNAL_EVENT_GC_EXIT
gc_exit() is called.
#define RUBY_INTERNAL_EVENT_GC_ENTER
gc_enter() is called.
#define RUBY_INTERNAL_EVENT_GC_END_SWEEP
GC ended sweep phase.
#define RUBY_INTERNAL_EVENT_GC_END_MARK
GC ended mark phase.
#define RUBY_INTERNAL_EVENT_FREEOBJ
Object swept.
#define RUBY_INTERNAL_EVENT_GC_START
GC started.
uint32_t rb_event_flag_t
Represents event(s).
static VALUE RB_FL_TEST(VALUE obj, VALUE flags)
Tests if the given flag(s) are set or not.
static VALUE RB_FL_TEST_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_TEST().
static void RB_FL_UNSET(VALUE obj, VALUE flags)
Clears the given flag(s).
@ RUBY_FL_SHAREABLE
This flag has something to do with Ractor.
@ RUBY_FL_WEAK_REFERENCE
This object weakly refers to other objects.
#define Qundef
Old name of RUBY_Qundef.
#define INT2FIX
Old name of RB_INT2FIX.
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
#define ID2SYM
Old name of RB_ID2SYM.
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
#define FL_SHAREABLE
Old name of RUBY_FL_SHAREABLE.
#define SIZET2NUM
Old name of RB_SIZE2NUM.
#define LONG2FIX
Old name of RB_INT2FIX.
#define FIX2INT
Old name of RB_FIX2INT.
#define FL_FINALIZE
Old name of RUBY_FL_FINALIZE.
#define T_HASH
Old name of RUBY_T_HASH.
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
#define FL_SET
Old name of RB_FL_SET.
#define rb_ary_new3
Old name of rb_ary_new_from_args.
#define Qtrue
Old name of RUBY_Qtrue.
#define INT2NUM
Old name of RB_INT2NUM.
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define T_ARRAY
Old name of RUBY_T_ARRAY.
#define NIL_P
Old name of RB_NIL_P.
#define T_CLASS
Old name of RUBY_T_CLASS.
#define FL_TEST
Old name of RB_FL_TEST.
#define FL_UNSET
Old name of RB_FL_UNSET.
#define FIXNUM_P
Old name of RB_FIXNUM_P.
#define FL_SET_RAW
Old name of RB_FL_SET_RAW.
#define SYMBOL_P
Old name of RB_SYMBOL_P.
VALUE rb_obj_hide(VALUE obj)
Make the object invisible from Ruby code.
VALUE rb_equal(VALUE lhs, VALUE rhs)
This function is an optimised version of calling #==.
VALUE rb_ary_dup(VALUE ary)
Duplicates an array.
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
#define rb_str_new_cstr(str)
Identical to rb_str_new, except it assumes the passed pointer is a pointer to a C string.
VALUE rb_f_notimplement(int argc, const VALUE *argv, VALUE obj, VALUE marker)
Raises rb_eNotImpError.
VALUE rb_sym2str(VALUE symbol)
Obtain a frozen string representation of a symbol (not including the leading colon).
int len
Length of the buffer.
void * rb_thread_call_with_gvl(void *(*func)(void *), void *data1)
(Re-)acquires the GVL.
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
#define RARRAY_LEN
Just another name of rb_array_len.
#define RARRAY_AREF(a, i)
References an object in an array.
static void RARRAY_ASET(VALUE ary, rb_len_t i, VALUE v)
Assigns an object in an array.
#define RBASIC(obj)
Convenient casting macro.
int ruby_native_thread_p(void)
Queries if the thread which calls this function is a ruby's thread.
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
#define RTEST
This is an old name of RB_TEST.
Ruby object's base components.
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
long rb_len_t
A signed integer type for lengths and indices of objects such as String and Array.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.