10#include "gc/gc_impl.h"
17#define WBCHECK_RCLASS_IS_ROOT FL_USER0
20static bool wbcheck_debug_enabled =
false;
23static bool wbcheck_verify_after_wb_enabled =
false;
26static bool wbcheck_warn_useless_wb_enabled =
false;
29wbcheck_debug(
const char *format, ...)
31 if (!wbcheck_debug_enabled)
return;
34 va_start(args, format);
35 vfprintf(stderr, format, args);
39#define WBCHECK_DEBUG(...) do { \
40 if (wbcheck_debug_enabled) { \
41 wbcheck_debug(__VA_ARGS__); \
46wbcheck_debug_obj_info_dump(
VALUE obj)
48 if (!wbcheck_debug_enabled)
return;
50 fprintf(stderr,
"%s\n", rb_raw_obj_info(buff,
sizeof(buff), obj));
54static void lock_and_maybe_gc(
void *objspace_ptr);
55static void force_gc(
void *objspace_ptr);
59wbcheck_configure_from_env(
void)
62 const char *debug_env = getenv(
"WBCHECK_DEBUG");
63 if (debug_env && (strcmp(debug_env,
"1") == 0 || strcmp(debug_env,
"true") == 0)) {
64 wbcheck_debug_enabled =
true;
68 const char *verify_after_wb_env = getenv(
"WBCHECK_VERIFY_AFTER_WB");
69 if (verify_after_wb_env && (strcmp(verify_after_wb_env,
"1") == 0 || strcmp(verify_after_wb_env,
"true") == 0)) {
70 wbcheck_verify_after_wb_enabled =
true;
74 const char *warn_useless_wb_env = getenv(
"WBCHECK_WARN_USELESS_WB");
75 if (warn_useless_wb_env && (strcmp(warn_useless_wb_env,
"1") == 0 || strcmp(warn_useless_wb_env,
"true") == 0)) {
76 wbcheck_warn_useless_wb_enabled =
true;
81static size_t heap_sizes[] = {
100#define HEAP_COUNT ((int)(sizeof(heap_sizes) / sizeof(heap_sizes[0])) - 1)
101#define MAX_HEAP_SIZE (heap_sizes[(HEAP_COUNT) - 1])
106 WBCHECK_STATE_MARKED,
108} wbcheck_object_state_t;
119 WBCHECK_PHASE_MUTATOR,
120 WBCHECK_PHASE_SNAPSHOT,
121 WBCHECK_PHASE_FULL_GC
133wbcheck_object_list_init_with_capacity(
size_t capacity)
136 if (!list) rb_bug(
"wbcheck: failed to allocate object list structure");
138 if (capacity < 4) capacity = 4;
139 list->items = malloc(capacity *
sizeof(
VALUE));
140 if (!list->items) rb_bug(
"wbcheck: failed to allocate object list array");
141 list->capacity = capacity;
147wbcheck_object_list_init(
void)
149 return wbcheck_object_list_init_with_capacity(4);
155 if (list->count >= list->capacity) {
156 size_t new_capacity = list->capacity == 0 ? 4 : list->capacity * 2;
157 VALUE *new_items = realloc(list->items, new_capacity *
sizeof(
VALUE));
158 if (!new_items) rb_bug(
"wbcheck: failed to reallocate object list array");
159 list->items = new_items;
160 list->capacity = new_capacity;
162 list->items[list->count++] = obj;
178 if (!wbcheck_debug_enabled)
return;
179 for (
size_t i = 0; i < list->count; i++) {
181 fprintf(stderr,
"-> %s\n", rb_raw_obj_info(buff,
sizeof(buff), list->items[i]));
188 for (
size_t i = 0; i < list->count; i++) {
189 if (list->items[i] == obj) {
204 wbcheck_object_state_t state;
205 wbcheck_color_t color;
212 WBCHECK_FINAL_JOB_DFREE,
213 WBCHECK_FINAL_JOB_FINALIZE,
217 void (*func)(
void *);
221 VALUE finalizer_array;
235 wbcheck_phase_t phase;
237 bool user_gc_disabled;
240 size_t missed_write_barrier_parents;
241 size_t missed_write_barrier_children;
242 size_t simulated_gc_count;
243 size_t total_allocated_objects;
244 bool measure_total_time;
246 rb_nativethread_lock_t finalizer_lock;
259static void gc_run_finalizers(
void *data);
264wbcheck_get_object_info(
VALUE obj)
267 GC_ASSERT(wbcheck_global_objspace);
270 if (st_lookup(wbcheck_global_objspace->object_table, (st_data_t)obj, &value)) {
274 fprintf(stderr,
"wbcheck: object not found in tracking table\n");
276 fprintf(stderr,
"%s\n", rb_raw_obj_info(buff,
sizeof(buff), obj));
279 ((
volatile VALUE *)obj)[0];
282 rb_bug(
"wbcheck: object not found in tracking table");
292 size_t snapshot_count = gc_mark_snapshot ? gc_mark_snapshot->count : 0;
293 size_t wb_count = writebarrier_children ? writebarrier_children->count : 0;
295 fprintf(stderr,
"WBCHECK ERROR: Missed write barrier detected!\n");
296 fprintf(stderr,
" Parent object: %p (wb_protected: %s)\n",
297 (
void *)parent_obj, parent_info->wb_protected ?
"true" :
"false");
299 fprintf(stderr,
" %s\n", rb_raw_obj_info(buff,
sizeof(buff), parent_obj));
300 fprintf(stderr,
" Reference counts - snapshot: %zu, writebarrier: %zu, current: %zu, missed: %zu\n",
301 snapshot_count, wb_count, current_refs->count, missed_refs->count);
303 for (
size_t i = 0; i < missed_refs->count; i++) {
304 VALUE missed_ref = missed_refs->items[i];
306 fprintf(stderr,
" Missing reference to: %p\n %s\n", (
void *)missed_ref, rb_raw_obj_info(buff,
sizeof(buff), missed_ref));
309 fprintf(stderr,
"\n");
310 objspace->missed_write_barrier_parents++;
311 objspace->missed_write_barrier_children += missed_refs->count;
319 if (
objspace->missed_write_barrier_parents > 0) {
321 rb_bug(
"wbcheck: missed write barrier detected (%zu object(s), %zu reference(s))",
322 objspace->missed_write_barrier_parents,
323 objspace->missed_write_barrier_children);
333 size_t snapshot_count = gc_mark_snapshot ? gc_mark_snapshot->count : 0;
334 size_t wb_count = writebarrier_children ? writebarrier_children->count : 0;
336 WBCHECK_DEBUG(
"wbcheck: comparing references for object %p\n", (
void *)parent_obj);
337 WBCHECK_DEBUG(
"wbcheck: current refs: %zu, snapshot refs: %zu, wb refs: %zu\n",
338 current_refs->count, snapshot_count, wb_count);
344 size_t snapshot_idx = 0;
347 for (
size_t i = 0; i < current_refs->count; i++) {
348 VALUE current_ref = current_refs->items[i];
357 bool found_in_snapshot =
false;
358 if (gc_mark_snapshot && snapshot_count > 0) {
359 size_t start_idx = snapshot_idx;
361 if (gc_mark_snapshot->items[snapshot_idx] == current_ref) {
362 found_in_snapshot =
true;
364 if (snapshot_idx >= snapshot_count) snapshot_idx = 0;
368 if (snapshot_idx >= snapshot_count) snapshot_idx = 0;
369 }
while (snapshot_idx != start_idx);
372 if (found_in_snapshot) {
384 if (current_ref == parent_obj) {
390 if (writebarrier_children && wbcheck_object_list_contains(writebarrier_children, current_ref)) {
397 missed_refs = wbcheck_object_list_init();
399 wbcheck_object_list_append(missed_refs, current_ref);
404 wbcheck_report_error(objspace_ptr, parent_obj, current_refs, gc_mark_snapshot, writebarrier_children, missed_refs);
405 wbcheck_object_list_free(missed_refs);
410wbcheck_register_object(
void *objspace_ptr,
VALUE obj,
size_t alloc_size,
bool wb_protected)
417 if (!info) rb_bug(
"wbcheck_register_object: failed to allocate object info");
419 info->alloc_size = alloc_size;
420 info->wb_protected = wb_protected;
421 info->finalizers = 0;
422 info->gc_mark_snapshot = NULL;
423 info->mark_maybe_snapshot = NULL;
424 info->writebarrier_children = NULL;
425 info->state = WBCHECK_STATE_CLEAR;
426 info->color = WBCHECK_COLOR_BLACK;
429 st_insert(
objspace->object_table, (st_data_t)obj, (st_data_t)info);
430 objspace->total_allocated_objects++;
434wbcheck_unregister_object(
void *objspace_ptr,
VALUE obj)
439 if (st_delete(
objspace->object_table, (st_data_t *)&obj, (st_data_t *)&info)) {
441 wbcheck_object_list_free(info->gc_mark_snapshot);
442 wbcheck_object_list_free(info->mark_maybe_snapshot);
443 wbcheck_object_list_free(info->writebarrier_children);
446 rb_bug(
"wbcheck_unregister_object: object not found in table");
452rb_gc_impl_objspace_alloc(
void)
454 wbcheck_configure_from_env();
457 if (!
objspace) rb_bug(
"wbcheck: failed to allocate objspace");
459 objspace->objects_to_capture = wbcheck_object_list_init();
460 objspace->objects_to_verify = wbcheck_object_list_init();
462 objspace->current_maybe_refs = NULL;
463 objspace->mark_queue = wbcheck_object_list_init();
464 objspace->weak_references = wbcheck_object_list_init();
465 objspace->phase = WBCHECK_PHASE_MUTATOR;
469 objspace->missed_write_barrier_parents = 0;
470 objspace->missed_write_barrier_children = 0;
472 objspace->measure_total_time =
true;
478rb_gc_impl_objspace_init(
void *objspace_ptr)
484 objspace->object_table = st_init_numtable();
486 rb_bug(
"wbcheck: failed to create object table");
498rb_gc_impl_ractor_cache_alloc(
void *objspace_ptr,
void *ractor)
505rb_gc_impl_zjit_new_obj_fastpath(
void *objspace_ptr,
size_t alloc_size,
VALUE flags,
VALUE klass,
512rb_gc_impl_set_params(
void *objspace_ptr)
518gc_verify_internal_consistency(
VALUE self)
526 VALUE gc_constants = rb_hash_new();
528 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVALUE_SIZE")),
SIZET2NUM(
sizeof(
struct RBasic) +
sizeof(
VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX])));
530 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVALUE_OVERHEAD")),
INT2NUM(0));
531 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVARGC_MAX_ALLOCATE_SIZE")),
SIZET2NUM(rb_gc_impl_max_allocation_size()));
532 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"HEAP_COUNT")),
LONG2FIX(HEAP_COUNT));
533 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"SIZE_POOL_COUNT")),
LONG2FIX(HEAP_COUNT));
534 rb_hash_aset(gc_constants,
ID2SYM(rb_intern(
"RVALUE_OLD_AGE")),
INT2FIX(3));
536 rb_define_const(
rb_mGC,
"INTERNAL_CONSTANTS", gc_constants);
551rb_gc_impl_shutdown_free_objects(
void *objspace_ptr)
557rb_gc_impl_objspace_free(
void *objspace_ptr)
563rb_gc_impl_ractor_cache_free(
void *objspace_ptr,
void *cache)
570rb_gc_impl_start(
void *objspace_ptr,
bool full_mark,
bool immediate_mark,
bool immediate_sweep,
bool compact,
bool global)
574 unsigned int lev = RB_GC_VM_LOCK();
576 force_gc(objspace_ptr);
577 RB_GC_VM_UNLOCK(lev);
581rb_gc_impl_during_gc_p(
void *objspace_ptr)
584 return objspace->phase != WBCHECK_PHASE_MUTATOR;
588wbcheck_prepare_heap_i(
VALUE obj,
void *data)
590 rb_gc_prepare_heap_process_object(obj);
594rb_gc_impl_prepare_heap(
void *objspace_ptr)
596 rb_gc_impl_each_object(objspace_ptr, wbcheck_prepare_heap_i, NULL);
600rb_gc_impl_gc_enable(
void *objspace_ptr)
607rb_gc_impl_gc_disable(
void *objspace_ptr,
bool finish_current_gc)
614rb_gc_impl_gc_enabled_p(
void *objspace_ptr)
621rb_gc_impl_user_gc_disabled_set(
void *objspace_ptr,
bool disable)
624 const bool was =
objspace->user_gc_disabled;
625 objspace->user_gc_disabled = disable;
631rb_gc_impl_user_gc_disabled_p(
void *objspace_ptr)
638rb_gc_impl_gc_rest(
void *objspace_ptr)
644rb_gc_impl_stress_set(
void *objspace_ptr,
VALUE flag)
651rb_gc_impl_stress_get(
void *objspace_ptr)
658rb_gc_impl_config_get(
void *objspace_ptr)
660 return rb_hash_new();
664rb_gc_impl_config_set(
void *objspace_ptr,
VALUE hash)
669rb_gc_impl_get_vm_context(
void *objspace_ptr)
682 size_t snapshot_count = (info->gc_mark_snapshot) ? info->gc_mark_snapshot->count : 0;
683 size_t wb_children_count = (info->writebarrier_children) ? info->writebarrier_children->count : 0;
684 size_t capacity_hint = snapshot_count + wb_children_count;
690 objspace->current_maybe_refs = NULL;
691 objspace->phase = WBCHECK_PHASE_SNAPSHOT;
692 rb_gc_initialize_vm_context(&
objspace->vm_context);
698 objspace->phase = WBCHECK_PHASE_MUTATOR;
703 wbcheck_object_list_free(info->mark_maybe_snapshot);
704 info->mark_maybe_snapshot =
objspace->current_maybe_refs;
705 objspace->current_maybe_refs = NULL;
707 if (wbcheck_debug_enabled) {
708 WBCHECK_DEBUG(
"wbcheck: collected %zu references from %p\n", new_list->count, (
void *)obj);
710 fprintf(stderr,
"%s\n", rb_raw_obj_info(buff,
sizeof(buff), obj));
711 wbcheck_object_list_debug_print(new_list);
718wbcheck_collect_initial_references(
void *objspace_ptr,
VALUE obj)
720 WBCHECK_DEBUG(
"wbcheck: collecting initial references from %p:\n", obj);
721 wbcheck_debug_obj_info_dump(obj);
728 info->gc_mark_snapshot = new_list;
729 info->state = WBCHECK_STATE_MARKED;
733wbcheck_verify_object_references(
void *objspace_ptr,
VALUE obj)
738 if (!info->wb_protected) {
743 if (info->state == WBCHECK_STATE_CLEAR) {
748 WBCHECK_DEBUG(
"wbcheck: verifying references for object:\n");
749 wbcheck_debug_obj_info_dump(obj);
755 if (wbcheck_warn_useless_wb_enabled && info->writebarrier_children) {
756 for (
size_t i = 0; i < info->writebarrier_children->count; i++) {
757 VALUE wb_ref = info->writebarrier_children->items[i];
758 if (!wbcheck_object_list_contains(current_refs, wb_ref)) {
759 fprintf(stderr,
"WBCHECK WARNING: Potentially useless write barrier detected for object %p\n", (
void *)obj);
760 fprintf(stderr,
" Write barrier was recorded for reference to %p, but object no longer references it\n", (
void *)wb_ref);
762 fprintf(stderr,
" Parent: %s\n", rb_raw_obj_info(buff,
sizeof(buff), obj));
763 fprintf(stderr,
" Stale reference: %s\n", rb_raw_obj_info(buff,
sizeof(buff), wb_ref));
769 wbcheck_compare_references(objspace_ptr, obj, current_refs, info->gc_mark_snapshot, info->writebarrier_children);
772 wbcheck_object_list_free(info->gc_mark_snapshot);
773 wbcheck_object_list_free(info->writebarrier_children);
774 info->gc_mark_snapshot = current_refs;
775 info->writebarrier_children = NULL;
776 info->state = WBCHECK_STATE_MARKED;
786 if (!st_lookup(
objspace->object_table, (st_data_t)obj, &value)) {
787 rb_bug(
"wbcheck: asked to mark object %p not in our object table", (
void *)obj);
791 if (info->color != WBCHECK_COLOR_WHITE) {
795 info->color = WBCHECK_COLOR_GRAY;
796 wbcheck_object_list_append(
objspace->mark_queue, obj);
799 wbcheck_object_list_append(
objspace->weak_references, obj);
802 WBCHECK_DEBUG(
"wbcheck: marked gray: %p\n", (
void *)obj);
807st_foreach_reset_white(st_data_t key, st_data_t val, st_data_t arg)
810 info->color = WBCHECK_COLOR_WHITE;
816st_foreach_mark_finalizers(st_data_t key, st_data_t val, st_data_t arg)
821 if (info->finalizers) {
822 wbcheck_mark_gray(
objspace, info->finalizers);
832 WBCHECK_DEBUG(
"wbcheck: starting GC mark phase\n");
834 objspace->phase = WBCHECK_PHASE_FULL_GC;
835 rb_gc_initialize_vm_context(&
objspace->vm_context);
839 st_foreach(
objspace->object_table, st_foreach_reset_white, 0);
842 st_foreach(
objspace->object_table, st_foreach_mark_finalizers, (st_data_t)
objspace);
847 while (job != NULL) {
849 case WBCHECK_FINAL_JOB_DFREE:
851 case WBCHECK_FINAL_JOB_FINALIZE:
852 wbcheck_mark_gray(
objspace, job->as.finalize.finalizer_array);
855 rb_bug(
"wbcheck_mark_phase: unknown final job type %d", job->kind);
861 rb_gc_save_machine_context();
865 while (
objspace->mark_queue->count > 0) {
870 if (st_lookup(
objspace->object_table, (st_data_t)obj, &value)) {
872 if (info->color == WBCHECK_COLOR_GRAY) {
874 if (info->gc_mark_snapshot) {
875 for (
size_t i = 0; i < info->gc_mark_snapshot->count; i++) {
876 wbcheck_mark_gray(
objspace, info->gc_mark_snapshot->items[i]);
881 if (info->mark_maybe_snapshot) {
882 for (
size_t i = 0; i < info->mark_maybe_snapshot->count; i++) {
883 wbcheck_mark_gray(
objspace, info->mark_maybe_snapshot->items[i]);
888 info->color = WBCHECK_COLOR_BLACK;
889 WBCHECK_DEBUG(
"wbcheck: marked black: %p\n", (
void *)obj);
894 objspace->phase = WBCHECK_PHASE_MUTATOR;
896 WBCHECK_DEBUG(
"wbcheck: tri-color mark phase complete\n");
901wbcheck_sweep_callback(st_data_t key, st_data_t val, st_data_t arg,
int error)
907 if (info->color == WBCHECK_COLOR_WHITE) {
908 WBCHECK_DEBUG(
"wbcheck: sweeping unmarked object %p\n", (
void *)obj);
913 rb_gc_obj_free_vm_weak_references(obj);
916 if (info->finalizers) {
917 make_final_job(
objspace, obj, info->finalizers);
922 if (rb_gc_obj_free(
objspace, obj)) {
924 wbcheck_object_list_free(info->gc_mark_snapshot);
925 wbcheck_object_list_free(info->mark_maybe_snapshot);
926 wbcheck_object_list_free(info->writebarrier_children);
936 wbcheck_object_list_free(info->gc_mark_snapshot);
937 wbcheck_object_list_free(info->mark_maybe_snapshot);
938 wbcheck_object_list_free(info->writebarrier_children);
954 WBCHECK_DEBUG(
"wbcheck: starting sweep phase\n");
956 size_t objects_before = st_table_size(
objspace->object_table);
959 st_foreach_check(
objspace->object_table, wbcheck_sweep_callback, (st_data_t)
objspace, 0);
961 size_t objects_after = st_table_size(
objspace->object_table);
962 size_t freed_objects = objects_before - objects_after;
965 objspace->gc_threshold = objects_after * 2;
967 WBCHECK_DEBUG(
"wbcheck: sweep phase complete - freed %zu objects (%zu -> %zu), new threshold: %zu\n",
968 freed_objects, objects_before, objects_after,
objspace->gc_threshold);
977 WBCHECK_DEBUG(
"wbcheck: processing %zu weak reference objects\n",
objspace->weak_references->count);
979 for (
size_t i = 0; i <
objspace->weak_references->count; i++) {
981 rb_gc_handle_weak_references(
objspace, obj);
984 objspace->weak_references->count = 0;
991 WBCHECK_DEBUG(
"wbcheck: starting full GC\n");
999 WBCHECK_DEBUG(
"wbcheck: updating snapshots for all objects\n");
1000 wbcheck_foreach_object(
objspace, wbcheck_update_all_snapshots_callback,
objspace);
1008 wbcheck_process_weak_references(
objspace);
1016 WBCHECK_DEBUG(
"wbcheck: full GC complete\n");
1020gc_step(
void *objspace_ptr,
bool force)
1027 if (!
objspace->gc_enabled && !force)
return;
1030 if (wbcheck_verify_after_wb_enabled) {
1031 for (
size_t i = 0; i <
objspace->objects_to_verify->count; i++) {
1033 wbcheck_verify_object_references(objspace_ptr, obj);
1037 objspace->objects_to_verify->count = 0;
1043 for (
size_t i = 0; i <
objspace->objects_to_capture->count; i++) {
1045 wbcheck_collect_initial_references(objspace_ptr, obj);
1049 objspace->objects_to_capture->count = 0;
1064maybe_gc_with_barrier(
void *objspace_ptr)
1070 gc_step(objspace_ptr,
false);
1075force_gc(
void *objspace_ptr)
1077 gc_step(objspace_ptr,
true);
1080int ruby_thread_has_gvl_p(
void);
1083lock_and_maybe_gc_gvl(
void *objspace_ptr)
1085 unsigned int lev = RB_GC_VM_LOCK();
1086 maybe_gc_with_barrier(objspace_ptr);
1087 RB_GC_VM_UNLOCK(lev);
1092lock_and_maybe_gc(
void *objspace_ptr)
1096 if (!ruby_thread_has_gvl_p()) {
1100 lock_and_maybe_gc_gvl(objspace_ptr);
1105rb_gc_impl_new_obj(
void *objspace_ptr,
void *cache_ptr,
VALUE klass,
VALUE flags,
bool wb_protected,
size_t alloc_size,
size_t *actual_alloc_size)
1107 unsigned int lev = RB_GC_VM_LOCK();
1110 maybe_gc_with_barrier(objspace_ptr);
1113 alloc_size = rb_gc_impl_size_slot_size(objspace_ptr, alloc_size);
1116 VALUE *mem = malloc(alloc_size);
1117 if (!mem) rb_bug(
"FIXME: malloc failed");
1119 *actual_alloc_size = alloc_size;
1123 RBASIC(obj)->flags = flags;
1127 wbcheck_register_object(objspace_ptr, obj, alloc_size, wb_protected);
1131 wbcheck_object_list_append(
objspace->objects_to_capture, obj);
1133 RB_GC_VM_UNLOCK(lev);
1138rb_gc_impl_obj_slot_size(
VALUE obj)
1140 unsigned int lev = RB_GC_VM_LOCK();
1143 size_t result = info->alloc_size;
1145 RB_GC_VM_UNLOCK(lev);
1150rb_gc_impl_size_slot_size(
void *objspace_ptr,
size_t size)
1152 for (
int i = 0; i < HEAP_COUNT; i++) {
1153 if (size <= heap_sizes[i])
return heap_sizes[i];
1155 rb_bug(
"size too big");
1159rb_gc_impl_size_allocatable_p(
size_t size)
1162 return size <= rb_gc_impl_max_allocation_size();
1166rb_gc_impl_max_allocation_size(
void)
1168 return MAX_HEAP_SIZE;
1173rb_gc_impl_malloc(
void *objspace_ptr,
size_t size,
bool gc_allowed)
1176 lock_and_maybe_gc(objspace_ptr);
1178 return malloc(size);
1182rb_gc_impl_calloc(
void *objspace_ptr,
size_t size,
bool gc_allowed)
1185 lock_and_maybe_gc(objspace_ptr);
1187 return calloc(1, size);
1191rb_gc_impl_realloc(
void *objspace_ptr,
void *ptr,
size_t new_size,
size_t old_size,
bool gc_allowed)
1194 lock_and_maybe_gc(objspace_ptr);
1196 return realloc(ptr, new_size);
1200rb_gc_impl_free(
void *objspace_ptr,
void *ptr,
size_t old_size)
1206rb_gc_impl_adjust_memory_usage(
void *objspace_ptr, ssize_t diff)
1215 WBCHECK_DEBUG(
"wbcheck: gc_mark called\n");
1216 wbcheck_debug_obj_info_dump(obj);
1221 case WBCHECK_PHASE_SNAPSHOT:
1224 wbcheck_object_list_append(
objspace->current_refs, obj);
1226 case WBCHECK_PHASE_FULL_GC:
1230 case WBCHECK_PHASE_MUTATOR:
1232 rb_bug(
"wbcheck: gc_mark called during mutator phase");
1238rb_gc_impl_mark(
void *objspace_ptr,
VALUE obj)
1245rb_gc_impl_mark_and_move(
void *objspace_ptr,
VALUE *ptr)
1252rb_gc_impl_mark_and_pin(
void *objspace_ptr,
VALUE obj)
1259rb_gc_impl_mark_maybe(
void *objspace_ptr,
VALUE obj)
1263 if (!rb_gc_impl_live_object_p(objspace_ptr, (
void *)obj))
return;
1266 case WBCHECK_PHASE_SNAPSHOT:
1271 if (!
objspace->current_maybe_refs) {
1272 objspace->current_maybe_refs = wbcheck_object_list_init();
1274 wbcheck_object_list_append(
objspace->current_maybe_refs, obj);
1276 case WBCHECK_PHASE_FULL_GC:
1279 case WBCHECK_PHASE_MUTATOR:
1280 rb_bug(
"wbcheck: rb_gc_impl_mark_maybe called during mutator phase");
1287rb_gc_impl_declare_weak_references(
void *objspace_ptr,
VALUE obj)
1293rb_gc_impl_handle_weak_references_alive_p(
void *objspace_ptr,
VALUE obj)
1298 if (st_lookup(
objspace->object_table, (st_data_t)obj, &value)) {
1300 return info->color != WBCHECK_COLOR_WHITE;
1308rb_gc_impl_register_pinning_obj(
void *objspace_ptr,
VALUE obj)
1314rb_gc_impl_object_moved_p(
void *objspace_ptr,
VALUE obj)
1321rb_gc_impl_pinned_p(
void *objspace_ptr,
VALUE obj)
1328rb_gc_impl_location(
void *objspace_ptr,
VALUE value)
1336rb_gc_impl_writebarrier(
void *objspace_ptr,
VALUE a,
VALUE b)
1340 unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER();
1348 if (info->state != WBCHECK_STATE_CLEAR) {
1352 if (!info->writebarrier_children) {
1353 info->writebarrier_children = wbcheck_object_list_init();
1357 wbcheck_object_list_append(info->writebarrier_children, b);
1359 WBCHECK_DEBUG(
"wbcheck: write barrier recorded reference from %p to %p\n", (
void *)a, (
void *)b);
1362 if (wbcheck_verify_after_wb_enabled && info->state != WBCHECK_STATE_DIRTY) {
1363 WBCHECK_DEBUG(
"wbcheck: queueing object for verification after write barrier\n");
1364 info->state = WBCHECK_STATE_DIRTY;
1365 wbcheck_object_list_append(
objspace->objects_to_verify, a);
1368 WBCHECK_DEBUG(
"wbcheck: write barrier skipped (snapshot not initialized) from %p to %p\n", (
void *)a, (
void *)b);
1371 RB_GC_VM_UNLOCK_NO_BARRIER(lev);
1375rb_gc_impl_writebarrier_unprotect(
void *objspace_ptr,
VALUE obj)
1377 WBCHECK_DEBUG(
"wbcheck: writebarrier_unprotect called on object %p\n", (
void *)obj);
1379 unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER();
1382 info->wb_protected =
false;
1384 RB_GC_VM_UNLOCK_NO_BARRIER(lev);
1388rb_gc_impl_writebarrier_remember(
void *objspace_ptr,
VALUE obj)
1390 WBCHECK_DEBUG(
"wbcheck: writebarrier_remember called on object %p\n", (
void *)obj);
1392 unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER();
1398 if (info->state != WBCHECK_STATE_CLEAR) {
1400 wbcheck_object_list_free(info->gc_mark_snapshot);
1401 info->gc_mark_snapshot = NULL;
1403 wbcheck_object_list_free(info->mark_maybe_snapshot);
1404 info->mark_maybe_snapshot = NULL;
1408 wbcheck_object_list_append(
objspace->objects_to_capture, obj);
1411 if (info->writebarrier_children) {
1412 wbcheck_object_list_free(info->writebarrier_children);
1413 info->writebarrier_children = NULL;
1417 info->state = WBCHECK_STATE_CLEAR;
1423 RB_GC_VM_UNLOCK_NO_BARRIER(lev);
1433wbcheck_foreach_object_i(st_data_t key, st_data_t val, st_data_t arg)
1439 return foreach_data->callback(obj, info, foreach_data->data);
1446 .callback = callback,
1450 st_foreach(
objspace->object_table, wbcheck_foreach_object_i, (st_data_t)&foreach_data);
1455wbcheck_snapshot_collector(st_data_t key, st_data_t val, st_data_t arg)
1459 wbcheck_object_list_append(snapshot, obj);
1467 size_t object_count = st_table_size(
objspace->object_table);
1469 st_foreach(
objspace->object_table, wbcheck_snapshot_collector, (st_data_t)snapshot);
1475rb_gc_impl_each_objects(
void *objspace_ptr,
int (*callback)(
void *,
void *,
size_t,
void *),
void *data)
1482 for (
size_t i = 0; i < snapshot->count; i++) {
1483 VALUE obj = snapshot->items[i];
1485 if (st_lookup(
objspace->object_table, (st_data_t)obj, &value)) {
1487 int result = callback(
1489 (
void *)((
char *)obj + info->alloc_size),
1493 if (result != 0)
break;
1497 wbcheck_object_list_free(snapshot);
1501wbcheck_each_objects_shareable_i(
void *start,
void *end,
size_t stride,
void *d)
1503 struct { int (*func)(
void *,
void *, size_t,
void *);
void *data; } *args = d;
1505 return args->func(start, end, stride, args->data);
1511rb_gc_impl_each_objects_shareable(
void *objspace_ptr,
int (*callback)(
void *,
void *,
size_t,
void *),
void *data)
1513 struct { int (*func)(
void *,
void *, size_t,
void *);
void *data; } args = { callback, data };
1514 rb_gc_impl_each_objects(objspace_ptr, wbcheck_each_objects_shareable_i, &args);
1518rb_gc_impl_each_objects_foreign(
void *objspace_ptr,
int (*callback)(
void *,
void *,
size_t,
void *),
void *data)
1524rb_gc_impl_multi_objspace_p(
void)
1530rb_gc_impl_objspace_retire_gc(
void *objspace_ptr)
1535rb_gc_impl_during_global_gc_p(
void *objspace_ptr)
1540 return objspace->phase != WBCHECK_PHASE_MUTATOR;
1544rb_gc_impl_during_postmortem_p(
void *objspace_ptr)
1550rb_gc_impl_obj_foreign_p(
void *objspace_ptr,
VALUE obj)
1556rb_gc_impl_shref_marked_p(
void *objspace_ptr,
VALUE obj)
1562rb_gc_impl_objspace_zombify(
void *objspace_ptr)
1567rb_gc_impl_objspace_absorb(
void *dst_ptr,
void *src_ptr)
1572rb_gc_impl_obj_became_shareable(
void *objspace_ptr,
VALUE obj)
1577rb_gc_impl_zjit_writebarrier_required_offset(
void)
1584rb_gc_impl_each_object(
void *objspace_ptr,
void (*func)(
VALUE obj,
void *data),
void *data)
1591 for (
size_t i = 0; i < snapshot->count; i++) {
1592 VALUE obj = snapshot->items[i];
1594 if (st_lookup(
objspace->object_table, (st_data_t)obj, &value)) {
1599 wbcheck_object_list_free(snapshot);
1606 job->next =
objspace->finalizer_jobs;
1617 objspace->finalizer_jobs = job->next;
1625rb_gc_impl_make_zombie(
void *objspace_ptr,
VALUE obj,
void (*dfree)(
void *),
void *data)
1627 if (dfree == NULL)
return;
1632 job->kind = WBCHECK_FINAL_JOB_DFREE;
1633 job->as.dfree.func = dfree;
1634 job->as.dfree.data = data;
1636 finalizer_jobs_push(
objspace, job);
1638 if (!ruby_free_at_exit_p()) {
1642 WBCHECK_DEBUG(
"wbcheck: made zombie for object %p with dfree function\n", (
void *)obj);
1646rb_gc_impl_define_finalizer(
void *objspace_ptr,
VALUE obj,
VALUE block)
1648 unsigned int lev = RB_GC_VM_LOCK();
1657 VALUE table = info->finalizers;
1658 VALUE result = block;
1664 info->finalizers = table;
1670 for (i = 1; i <
len; i++) {
1672 if (rb_equal(recv, block)) {
1674 goto unlock_and_return;
1682 RB_GC_VM_UNLOCK(lev);
1687rb_gc_impl_undefine_finalizer(
void *objspace_ptr,
VALUE obj)
1689 unsigned int lev = RB_GC_VM_LOCK();
1696 info->finalizers = 0;
1699 RB_GC_VM_UNLOCK(lev);
1703rb_gc_impl_copy_finalizer(
void *objspace_ptr,
VALUE dest,
VALUE obj)
1709 unsigned int lev = RB_GC_VM_LOCK();
1714 if (src_info->finalizers) {
1718 dest_info->finalizers = table;
1722 RB_GC_VM_UNLOCK(lev);
1726wbcheck_get_final(
rb_len_t i,
void *data)
1742 job->kind = WBCHECK_FINAL_JOB_FINALIZE;
1743 job->as.finalize.finalizer_array = finalizer_array;
1745 finalizer_jobs_push(
objspace, job);
1749gc_run_finalizers(
void *data)
1753 rb_gc_set_pending_interrupt();
1756 while ((job = finalizer_jobs_pop(
objspace)) != NULL) {
1757 switch (job->kind) {
1758 case WBCHECK_FINAL_JOB_DFREE:
1759 job->as.dfree.func(job->as.dfree.data);
1761 case WBCHECK_FINAL_JOB_FINALIZE: {
1762 VALUE finalizer_array = job->as.finalize.finalizer_array;
1764 rb_gc_run_obj_finalizer(
1768 (
void *)finalizer_array
1779 rb_gc_unset_pending_interrupt();
1785 if (info->finalizers) {
1786 VALUE table = info->finalizers;
1788 rb_gc_run_obj_finalizer(
RARRAY_AREF(table, 0), count, wbcheck_get_final, (
void *)table);
1791 info->finalizers = 0;
1797 wbcheck_run_finalizers_for_object(obj, info);
1804 void *objspace_ptr = data;
1805 wbcheck_verify_object_references(objspace_ptr, obj);
1812 void *objspace_ptr = data;
1815 if (info->wb_protected && info->state != WBCHECK_STATE_CLEAR) {
1816 wbcheck_verify_object_references(objspace_ptr, obj);
1820 wbcheck_object_list_free(info->gc_mark_snapshot);
1821 info->gc_mark_snapshot = current_refs;
1822 info->state = WBCHECK_STATE_MARKED;
1831 void *objspace_ptr = data;
1833 if (rb_gc_shutdown_call_finalizer_p(obj)) {
1834 WBCHECK_DEBUG(
"wbcheck: finalizing object during shutdown: %p\n", (
void *)obj);
1835 rb_gc_obj_free_vm_weak_references(obj);
1836 if (rb_gc_obj_free(objspace_ptr, obj)) {
1846rb_gc_impl_shutdown_call_finalizer(
void *objspace_ptr)
1851 wbcheck_foreach_object(
objspace, wbcheck_shutdown_call_finalizer_callback, NULL);
1854 unsigned int verify_lev = RB_GC_VM_LOCK();
1855 WBCHECK_DEBUG(
"wbcheck: verifying references for all objects after finalizers\n");
1856 wbcheck_foreach_object(
objspace, wbcheck_verify_all_references_callback, objspace_ptr);
1857 WBCHECK_DEBUG(
"wbcheck: finished verifying all object references\n");
1858 RB_GC_VM_UNLOCK(verify_lev);
1861 WBCHECK_DEBUG(
"wbcheck: no write barrier violations detected\n");
1864 unsigned int lev = RB_GC_VM_LOCK();
1865 WBCHECK_DEBUG(
"wbcheck: calling rb_gc_obj_free on objects that need shutdown finalization\n");
1866 wbcheck_foreach_object(
objspace, wbcheck_shutdown_finalizer_callback, objspace_ptr);
1867 WBCHECK_DEBUG(
"wbcheck: finished calling rb_gc_obj_free\n");
1870 WBCHECK_DEBUG(
"wbcheck: running pending finalizer jobs\n");
1872 WBCHECK_DEBUG(
"wbcheck: finished running finalizer jobs\n");
1873 RB_GC_VM_UNLOCK(lev);
1878rb_gc_impl_before_fork(
void *objspace_ptr)
1882 unsigned int lev = RB_GC_VM_LOCK();
1884 force_gc(objspace_ptr);
1885 RB_GC_VM_UNLOCK(lev);
1889rb_gc_impl_after_fork(
void *objspace_ptr, rb_pid_t pid)
1896rb_gc_impl_set_measure_total_time(
void *objspace_ptr,
VALUE flag)
1903rb_gc_impl_get_measure_total_time(
void *objspace_ptr)
1906 return objspace->measure_total_time;
1910rb_gc_impl_get_total_time(
void *objspace_ptr)
1917rb_gc_impl_gc_count(
void *objspace_ptr)
1921 return objspace->simulated_gc_count;
1927rb_gc_impl_latest_gc_info(
void *objspace_ptr,
VALUE hash_or_key)
1938 rb_bug(
"rb_gc_impl_latest_gc_info: non-hash or symbol given");
1941#define SET(name, attr) \
1942 if (key == ID2SYM(rb_intern_const(#name))) \
1944 else if (hash != Qnil) \
1945 rb_hash_aset(hash, ID2SYM(rb_intern_const(#name)), (attr));
1959rb_gc_impl_stat(
void *objspace_ptr,
VALUE hash_or_sym)
1961 if (objspace_ptr == NULL) {
1977 rb_bug(
"non-hash or symbol given");
1980#define SET(name, attr) \
1981 if (key == ID2SYM(rb_intern(#name))) \
1982 return SIZET2NUM(attr); \
1983 else if (hash != Qnil) \
1984 rb_hash_aset(hash, ID2SYM(rb_intern(#name)), SIZET2NUM(attr));
1987 SET(count,
objspace->simulated_gc_count);
1988 SET(time,
objspace->measure_total_time ?
objspace->simulated_gc_count : 0);
1989 SET(total_allocated_objects,
objspace->total_allocated_objects);
1990 SET(tracked_objects, st_table_size(
objspace->object_table));
1994 rb_raise(rb_eArgError,
"unknown key: %"PRIsVALUE,
rb_sym2str(key));
2001rb_gc_impl_stat_heap(
void *objspace_ptr,
VALUE heap_name,
VALUE hash_or_sym)
2004 int heap_idx =
FIX2INT(heap_name);
2005 if (heap_idx < 0 || heap_idx >= HEAP_COUNT) {
2006 rb_raise(rb_eArgError,
"size pool index out of range");
2009 if (hash_or_sym ==
ID2SYM(rb_intern(
"slot_size"))) {
2024rb_gc_impl_active_gc_name(
void)
2031#define WBCHECK_OBJECT_METADATA_ENTRY_COUNT 3
2035rb_gc_impl_object_metadata(
void *objspace_ptr,
VALUE obj)
2037 static ID ID_wb_protected, ID_object_id, ID_shareable;
2039 if (!ID_object_id) {
2040 ID_wb_protected = rb_intern(
"wb_protected");
2041 ID_object_id = rb_intern(
"object_id");
2042 ID_shareable = rb_intern(
"shareable");
2045 unsigned int lev = RB_GC_VM_LOCK();
2046 bool wb_protected = wbcheck_get_object_info(obj)->wb_protected;
2047 RB_GC_VM_UNLOCK(lev);
2051#define SET_ENTRY(na, v) do { \
2052 GC_ASSERT(n < WBCHECK_OBJECT_METADATA_ENTRY_COUNT); \
2053 object_metadata_entries[n].name = ID_##na; \
2054 object_metadata_entries[n].val = v; \
2058 if (wb_protected) SET_ENTRY(wb_protected,
Qtrue);
2059 if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj));
2063 object_metadata_entries[n].name = 0;
2064 object_metadata_entries[n].val = 0;
2066 return object_metadata_entries;
2070rb_gc_impl_live_object_p(
void *objspace_ptr,
const void *ptr)
2072 GC_ASSERT(wbcheck_global_objspace);
2074 unsigned int lev = RB_GC_VM_LOCK();
2078 bool result = st_lookup(wbcheck_global_objspace->object_table, (st_data_t)ptr, &value);
2080 RB_GC_VM_UNLOCK(lev);
2085rb_gc_impl_garbage_object_p(
void *objspace_ptr,
VALUE obj)
2087 unsigned int lev = RB_GC_VM_LOCK();
2091 bool result = st_lookup(wbcheck_global_objspace->object_table, (st_data_t)obj, &value);
2093 RB_GC_VM_UNLOCK(lev);
2098rb_gc_impl_set_event_hook(
void *objspace_ptr,
const rb_event_flag_t event)
2104rb_gc_impl_copy_attributes(
void *objspace_ptr,
VALUE dest,
VALUE obj)
2108 if (!src_info->wb_protected) {
2109 rb_gc_impl_writebarrier_unprotect(objspace_ptr, dest);
2111 rb_gc_impl_copy_finalizer(objspace_ptr, dest, obj);
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
unsigned int rb_postponed_job_handle_t
The type of a handle returned from rb_postponed_job_preregister and passed to rb_postponed_job_trigge...
void rb_postponed_job_trigger(rb_postponed_job_handle_t h)
Triggers a pre-registered job registered with rb_postponed_job_preregister, scheduling it for executi...
rb_postponed_job_handle_t rb_postponed_job_preregister(unsigned int flags, rb_postponed_job_func_t func, void *data)
Pre-registers a func in Ruby's postponed job preregistration table, returning an opaque handle which ...
#define RUBY_INTERNAL_EVENT_GC_EXIT
gc_exit() is called.
#define RUBY_INTERNAL_EVENT_GC_ENTER
gc_enter() is called.
#define RUBY_INTERNAL_EVENT_GC_END_SWEEP
GC ended sweep phase.
#define RUBY_INTERNAL_EVENT_GC_END_MARK
GC ended mark phase.
#define RUBY_INTERNAL_EVENT_FREEOBJ
Object swept.
#define RUBY_INTERNAL_EVENT_GC_START
GC started.
uint32_t rb_event_flag_t
Represents event(s).
static VALUE RB_FL_TEST(VALUE obj, VALUE flags)
Tests if the given flag(s) are set or not.
static VALUE RB_FL_TEST_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_TEST().
static void RB_FL_UNSET(VALUE obj, VALUE flags)
Clears the given flag(s).
@ RUBY_FL_SHAREABLE
This flag has something to do with Ractor.
@ RUBY_FL_WEAK_REFERENCE
This object weakly refers to other objects.
#define Qundef
Old name of RUBY_Qundef.
#define INT2FIX
Old name of RB_INT2FIX.
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
#define ID2SYM
Old name of RB_ID2SYM.
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
#define FL_SHAREABLE
Old name of RUBY_FL_SHAREABLE.
#define SIZET2NUM
Old name of RB_SIZE2NUM.
#define LONG2FIX
Old name of RB_INT2FIX.
#define FIX2INT
Old name of RB_FIX2INT.
#define FL_FINALIZE
Old name of RUBY_FL_FINALIZE.
#define T_HASH
Old name of RUBY_T_HASH.
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
#define FL_SET
Old name of RB_FL_SET.
#define rb_ary_new3
Old name of rb_ary_new_from_args.
#define Qtrue
Old name of RUBY_Qtrue.
#define INT2NUM
Old name of RB_INT2NUM.
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define T_ARRAY
Old name of RUBY_T_ARRAY.
#define NIL_P
Old name of RB_NIL_P.
#define T_CLASS
Old name of RUBY_T_CLASS.
#define FL_TEST
Old name of RB_FL_TEST.
#define FL_UNSET
Old name of RB_FL_UNSET.
#define FIXNUM_P
Old name of RB_FIXNUM_P.
#define FL_SET_RAW
Old name of RB_FL_SET_RAW.
#define SYMBOL_P
Old name of RB_SYMBOL_P.
VALUE rb_obj_hide(VALUE obj)
Make the object invisible from Ruby code.
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
VALUE rb_ary_dup(VALUE ary)
Duplicates an array.
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
VALUE rb_f_notimplement(int argc, const VALUE *argv, VALUE obj, VALUE marker)
Raises rb_eNotImpError.
static ID rb_intern_const(const char *str)
This is a "tiny optimisation" over rb_intern().
VALUE rb_sym2str(VALUE symbol)
Obtain a frozen string representation of a symbol (not including the leading colon).
int len
Length of the buffer.
void * rb_thread_call_with_gvl(void *(*func)(void *), void *data1)
(Re-)acquires the GVL.
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
#define RARRAY_LEN
Just another name of rb_array_len.
#define RARRAY_AREF(a, i)
References an object in an array.
static void RARRAY_ASET(VALUE ary, rb_len_t i, VALUE v)
Assigns an object in an array.
#define RBASIC(obj)
Convenient casting macro.
int ruby_native_thread_p(void)
Queries if the thread which calls this function is a ruby's thread.
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
#define RTEST
This is an old name of RB_TEST.
Ruby object's base components.
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
long rb_len_t
A signed integer type for lengths and indices of objects such as String and Array.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.