Ruby 4.1.0dev (2026-11-20 revision 88e0be344d5fe2a189fb0d0e03ec62dd83406b4f)
vm_insnhelper.c (88e0be344d5fe2a189fb0d0e03ec62dd83406b4f)
1/**********************************************************************
2
3 vm_insnhelper.c - instruction helper functions. Included into vm.c.
4
5 $Author$
6
7 Copyright (C) 2007 Koichi Sasada
8
9**********************************************************************/
10
11#include "ruby/internal/config.h"
12
13#include <math.h>
14
15#ifdef HAVE_STDATOMIC_H
16 #include <stdatomic.h>
17#endif
18
19#include "constant.h"
20#include "debug_counter.h"
21#include "internal.h"
22#include "internal/class.h"
23#include "internal/compar.h"
24#include "internal/hash.h"
25#include "internal/numeric.h"
26#include "internal/proc.h"
27#include "internal/ractor.h"
28#include "internal/random.h"
29#include "internal/variable.h"
30#include "internal/set.h"
31#include "internal/set_table.h"
32#include "internal/struct.h"
33#include "ruby/thread.h"
34#include "variable.h"
35
36/* finish iseq array */
37#include "insns.inc"
38#include "insns_info.inc"
39
40extern rb_method_definition_t *rb_method_definition_create(rb_method_type_t type, ID mid);
41extern void rb_method_definition_set(const rb_method_entry_t *me, rb_method_definition_t *def, void *opts);
42extern int rb_method_definition_eq(const rb_method_definition_t *d1, const rb_method_definition_t *d2);
43extern VALUE rb_make_no_method_exception(VALUE exc, VALUE format, VALUE obj,
44 int argc, const VALUE *argv, int priv);
45
46/* control stack frame */
47
48static rb_control_frame_t *vm_get_ruby_level_caller_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp);
49
51ruby_vm_special_exception_copy(VALUE exc)
52{
54 rb_obj_copy_ivar(e, exc);
55 return e;
56}
57
58NORETURN(static void ec_stack_overflow(rb_execution_context_t *ec, int));
59static void
60ec_stack_overflow(rb_execution_context_t *ec, int setup)
61{
62 VALUE mesg = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_sysstack];
63 ec->raised_flag = RAISED_STACKOVERFLOW;
64 if (setup) {
65 VALUE at = rb_ec_backtrace_object(ec);
66 mesg = ruby_vm_special_exception_copy(mesg);
67 rb_ivar_set(mesg, idBt, at);
68 rb_ivar_set(mesg, idBt_locations, at);
69 }
70 ec->errinfo = mesg;
71 EC_JUMP_TAG(ec, TAG_RAISE);
72}
73
74NORETURN(static void vm_stackoverflow(void));
75
76static void
77vm_stackoverflow(void)
78{
79 ec_stack_overflow(GET_EC(), TRUE);
80}
81
82void
83rb_ec_stack_overflow(rb_execution_context_t *ec, ruby_stack_overflow_critical_level crit)
84{
85 if (rb_during_gc()) {
86 rb_bug("system stack overflow during GC. Faulty native extension?");
87 }
88 if (crit >= rb_stack_overflow_fatal) {
89 ec->raised_flag = RAISED_STACKOVERFLOW;
90 ec->errinfo = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_stackfatal];
91 EC_JUMP_TAG(ec, TAG_RAISE);
92 }
93 ec_stack_overflow(ec, crit < rb_stack_overflow_signal);
94}
95
96static inline void stack_check(rb_execution_context_t *ec);
97
98#if VM_CHECK_MODE > 0
99static int
100callable_class_p(VALUE klass)
101{
102#if VM_CHECK_MODE >= 2
103 if (!klass) return FALSE;
104 switch (RB_BUILTIN_TYPE(klass)) {
105 default:
106 break;
107 case T_ICLASS:
108 if (!RB_TYPE_P(RCLASS_SUPER(klass), T_MODULE)) break;
109 case T_MODULE:
110 return TRUE;
111 }
112 while (klass) {
113 if (klass == rb_cBasicObject) {
114 return TRUE;
115 }
116 klass = RCLASS_SUPER(klass);
117 }
118 return FALSE;
119#else
120 return klass != 0;
121#endif
122}
123
124static int
125callable_method_entry_p(const rb_callable_method_entry_t *cme)
126{
127 if (cme == NULL) {
128 return TRUE;
129 }
130 else {
131 VM_ASSERT(IMEMO_TYPE_P((VALUE)cme, imemo_ment), "imemo_type:%s", rb_imemo_name(imemo_type((VALUE)cme)));
132
133 if (callable_class_p(cme->defined_class)) {
134 return TRUE;
135 }
136 else {
137 return FALSE;
138 }
139 }
140}
141
142static void
143vm_check_frame_detail(VALUE type, int req_block, int req_me, int req_cref, VALUE specval, VALUE cref_or_me, int is_cframe, const rb_iseq_t *iseq)
144{
145 unsigned int magic = (unsigned int)(type & VM_FRAME_MAGIC_MASK);
146 enum imemo_type cref_or_me_type = imemo_env; /* impossible value */
147
148 if (RB_TYPE_P(cref_or_me, T_IMEMO)) {
149 cref_or_me_type = imemo_type(cref_or_me);
150 }
151 if (type & VM_FRAME_FLAG_BMETHOD) {
152 req_me = TRUE;
153 }
154
155 if (req_block && (type & VM_ENV_FLAG_LOCAL) == 0) {
156 rb_bug("vm_push_frame: specval (%p) should be a block_ptr on %x frame", (void *)specval, magic);
157 }
158 if (!req_block && (type & VM_ENV_FLAG_LOCAL) != 0) {
159 rb_bug("vm_push_frame: specval (%p) should not be a block_ptr on %x frame", (void *)specval, magic);
160 }
161
162 if (req_me) {
163 if (cref_or_me_type != imemo_ment) {
164 rb_bug("vm_push_frame: (%s) should be method entry on %x frame", rb_obj_info(cref_or_me), magic);
165 }
166 }
167 else {
168 if (req_cref && cref_or_me_type != imemo_cref) {
169 rb_bug("vm_push_frame: (%s) should be CREF on %x frame", rb_obj_info(cref_or_me), magic);
170 }
171 else { /* cref or Qfalse */
172 if (cref_or_me != Qfalse && cref_or_me_type != imemo_cref) {
173 if (((type & VM_FRAME_FLAG_LAMBDA) || magic == VM_FRAME_MAGIC_IFUNC || magic == VM_FRAME_MAGIC_DUMMY) && (cref_or_me_type == imemo_ment)) {
174 /* ignore */
175 }
176 else {
177 rb_bug("vm_push_frame: (%s) should be false or cref on %x frame", rb_obj_info(cref_or_me), magic);
178 }
179 }
180 }
181 }
182
183 if (cref_or_me_type == imemo_ment) {
184 const rb_callable_method_entry_t *me = (const rb_callable_method_entry_t *)cref_or_me;
185
186 if (!callable_method_entry_p(me)) {
187 rb_bug("vm_push_frame: ment (%s) should be callable on %x frame.", rb_obj_info(cref_or_me), magic);
188 }
189 }
190
191 if ((type & VM_FRAME_MAGIC_MASK) == VM_FRAME_MAGIC_DUMMY) {
192 VM_ASSERT(iseq == NULL ||
193 RBASIC_CLASS((VALUE)iseq) == 0 || // dummy frame for loading
194 RUBY_VM_NORMAL_ISEQ_P(iseq) //argument error
195 );
196 }
197 else {
198 VM_ASSERT(is_cframe == !RUBY_VM_NORMAL_ISEQ_P(iseq));
199 }
200}
201
202static void
203vm_check_frame(VALUE type,
204 VALUE specval,
205 VALUE cref_or_me,
206 const rb_iseq_t *iseq)
207{
208 VALUE given_magic = type & VM_FRAME_MAGIC_MASK;
209 VM_ASSERT(FIXNUM_P(type));
210
211#define CHECK(magic, req_block, req_me, req_cref, is_cframe) \
212 case magic: \
213 vm_check_frame_detail(type, req_block, req_me, req_cref, \
214 specval, cref_or_me, is_cframe, iseq); \
215 break
216 switch (given_magic) {
217 /* BLK ME CREF CFRAME */
218 CHECK(VM_FRAME_MAGIC_METHOD, TRUE, TRUE, FALSE, FALSE);
219 CHECK(VM_FRAME_MAGIC_CLASS, TRUE, FALSE, TRUE, FALSE);
220 CHECK(VM_FRAME_MAGIC_TOP, TRUE, FALSE, TRUE, FALSE);
221 CHECK(VM_FRAME_MAGIC_CFUNC, TRUE, TRUE, FALSE, TRUE);
222 CHECK(VM_FRAME_MAGIC_BLOCK, FALSE, FALSE, FALSE, FALSE);
223 CHECK(VM_FRAME_MAGIC_IFUNC, FALSE, FALSE, FALSE, TRUE);
224 CHECK(VM_FRAME_MAGIC_EVAL, FALSE, FALSE, FALSE, FALSE);
225 CHECK(VM_FRAME_MAGIC_RESCUE, FALSE, FALSE, FALSE, FALSE);
226 CHECK(VM_FRAME_MAGIC_DUMMY, TRUE, FALSE, FALSE, FALSE);
227 default:
228 rb_bug("vm_push_frame: unknown type (%x)", (unsigned int)given_magic);
229 }
230#undef CHECK
231}
232
233static VALUE vm_stack_canary; /* Initialized later */
234static bool vm_stack_canary_was_born = false;
235
236// Return the index of the instruction right before the given PC.
237// This is needed because insn_entry advances PC before the insn body.
238static unsigned int
239previous_insn_index(const rb_iseq_t *iseq, const VALUE *pc)
240{
241 unsigned int pos = 0;
242 while (pos < ISEQ_BODY(iseq)->iseq_size) {
243 int opcode = rb_vm_insn_addr2opcode((void *)ISEQ_BODY(iseq)->iseq_encoded[pos]);
244 unsigned int next_pos = pos + insn_len(opcode);
245 if (ISEQ_BODY(iseq)->iseq_encoded + next_pos == pc) {
246 return pos;
247 }
248 pos = next_pos;
249 }
250 rb_bug("failed to find the previous insn");
251}
252
253void
254rb_vm_check_canary(const rb_execution_context_t *ec, VALUE *sp)
255{
256 const struct rb_control_frame_struct *reg_cfp = ec->cfp;
257 const struct rb_iseq_struct *iseq;
258
259 if (! LIKELY(vm_stack_canary_was_born)) {
260 return; /* :FIXME: isn't it rather fatal to enter this branch? */
261 }
262 else if ((VALUE *)reg_cfp == ec->vm_stack + ec->vm_stack_size) {
263 /* This is at the very beginning of a thread. cfp does not exist. */
264 return;
265 }
266 else if (! VM_FRAME_RUBYFRAME_P(reg_cfp) || ! (iseq = GET_ISEQ())) {
267 /* The iseq field of an IFUNC frame holds the ifunc. */
268 return;
269 }
270 else if (LIKELY(sp[0] != vm_stack_canary)) {
271 return;
272 }
273 else {
274 /* we are going to call methods below; squash the canary to
275 * prevent infinite loop. */
276 sp[0] = Qundef;
277 }
278
279 const VALUE *orig = rb_iseq_original_iseq(iseq);
280 const VALUE iseqw = rb_iseqw_new(iseq);
281 const VALUE inspection = rb_inspect(iseqw);
282 const char *stri = rb_str_to_cstr(inspection);
283 const VALUE disasm = rb_iseq_disasm(iseq);
284 const char *strd = rb_str_to_cstr(disasm);
285 const ptrdiff_t pos = previous_insn_index(iseq, GET_PC());
286 const enum ruby_vminsn_type insn = (enum ruby_vminsn_type)orig[pos];
287 const char *name = insn_name(insn);
288
289 /* rb_bug() is not capable of outputting this large contents. It
290 is designed to run form a SIGSEGV handler, which tends to be
291 very restricted. */
292 ruby_debug_printf(
293 "We are killing the stack canary set by %s, "
294 "at %s@pc=%"PRIdPTR"\n"
295 "watch out the C stack trace.\n"
296 "%s",
297 name, stri, pos, strd);
298 rb_bug("see above.");
299}
300#define vm_check_canary(ec, sp) rb_vm_check_canary(ec, sp)
301
302#else
303#define vm_check_canary(ec, sp)
304#define vm_check_frame(a, b, c, d)
305#endif /* VM_CHECK_MODE > 0 */
306
307#if USE_DEBUG_COUNTER
308static void
309vm_push_frame_debug_counter_inc(
310 const struct rb_execution_context_struct *ec,
311 const struct rb_control_frame_struct *reg_cfp,
312 VALUE type)
313{
314 const struct rb_control_frame_struct *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(reg_cfp);
315
316 RB_DEBUG_COUNTER_INC(frame_push);
317
318 if (RUBY_VM_END_CONTROL_FRAME(ec) != prev_cfp) {
319 const bool curr = VM_FRAME_RUBYFRAME_P(reg_cfp);
320 const bool prev = VM_FRAME_RUBYFRAME_P(prev_cfp);
321 if (prev) {
322 if (curr) {
323 RB_DEBUG_COUNTER_INC(frame_R2R);
324 }
325 else {
326 RB_DEBUG_COUNTER_INC(frame_R2C);
327 }
328 }
329 else {
330 if (curr) {
331 RB_DEBUG_COUNTER_INC(frame_C2R);
332 }
333 else {
334 RB_DEBUG_COUNTER_INC(frame_C2C);
335 }
336 }
337 }
338
339 switch (type & VM_FRAME_MAGIC_MASK) {
340 case VM_FRAME_MAGIC_METHOD: RB_DEBUG_COUNTER_INC(frame_push_method); return;
341 case VM_FRAME_MAGIC_BLOCK: RB_DEBUG_COUNTER_INC(frame_push_block); return;
342 case VM_FRAME_MAGIC_CLASS: RB_DEBUG_COUNTER_INC(frame_push_class); return;
343 case VM_FRAME_MAGIC_TOP: RB_DEBUG_COUNTER_INC(frame_push_top); return;
344 case VM_FRAME_MAGIC_CFUNC: RB_DEBUG_COUNTER_INC(frame_push_cfunc); return;
345 case VM_FRAME_MAGIC_IFUNC: RB_DEBUG_COUNTER_INC(frame_push_ifunc); return;
346 case VM_FRAME_MAGIC_EVAL: RB_DEBUG_COUNTER_INC(frame_push_eval); return;
347 case VM_FRAME_MAGIC_RESCUE: RB_DEBUG_COUNTER_INC(frame_push_rescue); return;
348 case VM_FRAME_MAGIC_DUMMY: RB_DEBUG_COUNTER_INC(frame_push_dummy); return;
349 }
350
351 rb_bug("unreachable");
352}
353#else
354#define vm_push_frame_debug_counter_inc(ec, cfp, t) /* void */
355#endif
356
357// Return a poison value to be set above the stack top to verify leafness.
358VALUE
359rb_vm_stack_canary(void)
360{
361#if VM_CHECK_MODE > 0
362 return vm_stack_canary;
363#else
364 return 0;
365#endif
366}
367
368STATIC_ASSERT(VM_ENV_DATA_INDEX_ME_CREF, VM_ENV_DATA_INDEX_ME_CREF == -2);
369STATIC_ASSERT(VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL == -1);
370STATIC_ASSERT(VM_ENV_DATA_INDEX_FLAGS, VM_ENV_DATA_INDEX_FLAGS == -0);
371
372static void
373vm_push_frame(rb_execution_context_t *ec,
374 const rb_iseq_t *iseq,
375 VALUE type,
376 VALUE self,
377 VALUE specval,
378 VALUE cref_or_me,
379 const VALUE *pc,
380 VALUE *sp,
381 int local_size,
382 int stack_max)
383{
384 rb_control_frame_t *const cfp = RUBY_VM_NEXT_CONTROL_FRAME(ec->cfp);
385
386 vm_check_frame(type, specval, cref_or_me, iseq);
387 VM_ASSERT(local_size >= 0);
388
389 /* check stack overflow */
390 CHECK_VM_STACK_OVERFLOW0(cfp, sp, local_size + stack_max);
391 vm_check_canary(ec, sp);
392
393 /* setup vm value stack */
394
395 /* initialize local variables */
396 for (int i=0; i < local_size; i++) {
397 *sp++ = Qnil;
398 }
399
400 /* setup ep with managing data */
401 *sp++ = cref_or_me; /* ep[-2] / Qnil or T_IMEMO(cref) or T_IMEMO(ment) */
402 *sp++ = specval /* ep[-1] / block handler or prev env ptr */;
403 *sp++ = type; /* ep[-0] / ENV_FLAGS */
404
405 /* setup new frame */
406 *cfp = (const struct rb_control_frame_struct) {
407 .pc = pc,
408 .sp = sp,
409 ._iseq = iseq,
410 .self = self,
411 .ep = sp - 1,
412 .block_code = NULL,
413#if VM_DEBUG_BP_CHECK
414 .bp_check = sp,
415#endif
416 .jit_return = NULL,
417 };
418
419 /* Ensure the initialization of `*cfp` above never gets reordered with the update of `ec->cfp` below.
420 This is a no-op in all cases we've looked at (https://godbolt.org/z/3oxd1446K), but should guarantee it for all
421 future/untested compilers/platforms. */
422
423 #if defined HAVE_DECL_ATOMIC_SIGNAL_FENCE && HAVE_DECL_ATOMIC_SIGNAL_FENCE
424 atomic_signal_fence(memory_order_seq_cst);
425 #endif
426
427 ec->cfp = cfp;
428
429 if (VMDEBUG == 2) {
430 SDR();
431 }
432 vm_push_frame_debug_counter_inc(ec, cfp, type);
433}
434
435void
436rb_vm_pop_frame_no_int(rb_execution_context_t *ec)
437{
438 rb_control_frame_t *cfp = ec->cfp;
439
440 if (VMDEBUG == 2) SDR();
441
442 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
443}
444
445/* return TRUE if the frame is finished */
446static inline int
447vm_pop_frame(rb_execution_context_t *ec, rb_control_frame_t *cfp, const VALUE *ep)
448{
449 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
450
451 if (VMDEBUG == 2) SDR();
452
453 RUBY_VM_CHECK_INTS(ec);
454 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
455
456 return flags & VM_FRAME_FLAG_FINISH;
457}
458
459void
460rb_vm_pop_frame(rb_execution_context_t *ec)
461{
462 vm_pop_frame(ec, ec->cfp, ec->cfp->ep);
463}
464
465// it pushes pseudo-frame with fname filename.
466VALUE
467rb_vm_push_frame_fname(rb_execution_context_t *ec, VALUE fname)
468{
469 rb_iseq_t *rb_iseq_alloc_with_dummy_path(VALUE fname);
470 rb_iseq_t *dmy_iseq = rb_iseq_alloc_with_dummy_path(fname);
471
472 vm_push_frame(ec,
473 dmy_iseq, //const rb_iseq_t *iseq,
474 VM_FRAME_MAGIC_DUMMY | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, // VALUE type,
475 ec->cfp->self, // VALUE self,
476 VM_BLOCK_HANDLER_NONE, // VALUE specval,
477 Qfalse, // VALUE cref_or_me,
478 NULL, // const VALUE *pc,
479 ec->cfp->sp, // VALUE *sp,
480 0, // int local_size,
481 0); // int stack_max
482
483 return (VALUE)dmy_iseq;
484}
485
486/* method dispatch */
487static inline VALUE
488rb_arity_error_new(int argc, int min, int max)
489{
490 VALUE err_mess = rb_sprintf("wrong number of arguments (given %d, expected %d", argc, min);
491 if (min == max) {
492 /* max is not needed */
493 }
494 else if (max == UNLIMITED_ARGUMENTS) {
495 rb_str_cat_cstr(err_mess, "+");
496 }
497 else {
498 rb_str_catf(err_mess, "..%d", max);
499 }
500 rb_str_cat_cstr(err_mess, ")");
501 return rb_exc_new3(rb_eArgError, err_mess);
502}
503
504void
505rb_error_arity(int argc, int min, int max)
506{
507 rb_exc_raise(rb_arity_error_new(argc, min, max));
508}
509
510/* lvar */
511
512NOINLINE(static void vm_env_write_slowpath(const VALUE *ep, int index, VALUE v));
513
514static void
515vm_env_write_slowpath(const VALUE *ep, int index, VALUE v)
516{
517 const VALUE envval = VM_ENV_ENVVAL(ep);
518
519 if (RB_FL_TEST_RAW(envval, RUBY_FL_SHAREABLE)) {
520 /* Writing to a SHAREABLE env (an isolated proc) can create a shareable ->
521 * unshareable edge; only a full barrier sets the shref bit keeping v alive
522 * through its owner's local GC. WB_REQUIRED stays: later writes need it too. */
523 if (!SPECIAL_CONST_P(v)) {
524 rb_gc_writebarrier(envval, v);
525 }
526 VM_FORCE_WRITE(&ep[index], v);
527 }
528 else {
529 /* remember env value forcely */
530 rb_gc_writebarrier_remember(envval);
531 VM_FORCE_WRITE(&ep[index], v);
532 VM_ENV_FLAGS_UNSET(ep, VM_ENV_FLAG_WB_REQUIRED);
533 }
534 RB_DEBUG_COUNTER_INC(lvar_set_slowpath);
535}
536
537// YJIT assumes this function never runs GC
538static inline void
539vm_env_write(const VALUE *ep, int index, VALUE v)
540{
541 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
542 if (LIKELY((flags & VM_ENV_FLAG_WB_REQUIRED) == 0)) {
543 VM_STACK_ENV_WRITE(ep, index, v);
544 }
545 else {
546 vm_env_write_slowpath(ep, index, v);
547 }
548}
549
550void
551rb_vm_env_write(const VALUE *ep, int index, VALUE v)
552{
553 vm_env_write(ep, index, v);
554}
555
556VALUE
557rb_vm_bh_to_procval(const rb_execution_context_t *ec, VALUE block_handler)
558{
559 if (block_handler == VM_BLOCK_HANDLER_NONE) {
560 return Qnil;
561 }
562 else {
563 switch (vm_block_handler_type(block_handler)) {
564 case block_handler_type_iseq:
565 case block_handler_type_ifunc:
566 return rb_vm_make_proc(ec, VM_BH_TO_CAPT_BLOCK(block_handler), rb_cProc);
567 case block_handler_type_symbol:
568 return rb_sym_to_proc(VM_BH_TO_SYMBOL(block_handler));
569 case block_handler_type_proc:
570 return VM_BH_TO_PROC(block_handler);
571 default:
572 VM_UNREACHABLE(rb_vm_bh_to_procval);
573 }
574 }
575}
576
577/* svar */
578
579#if VM_CHECK_MODE > 0
580static int
581vm_svar_valid_p(VALUE svar)
582{
583 if (RB_TYPE_P((VALUE)svar, T_IMEMO)) {
584 switch (imemo_type(svar)) {
585 case imemo_svar:
586 case imemo_cref:
587 case imemo_ment:
588 return TRUE;
589 default:
590 break;
591 }
592 }
593 rb_bug("vm_svar_valid_p: unknown type: %s", rb_obj_info(svar));
594 return FALSE;
595}
596#endif
597
598/* Should this frame's special variables live in the env's svar slot? A SHAREABLE env
599 * (isolated proc) can run in several Ractors at once, making svar shared mutable state
600 * leaking $~/$_ across Ractors: keep them per-EC instead. */
601static inline bool
602lep_svar_in_env_p(const rb_execution_context_t *ec, const VALUE *lep)
603{
604 if (!lep) return false;
605 if (ec == NULL) return true;
606 if (ec->root_lep == lep) return false;
607 /* lep may be a stale on-stack ep of a frame whose env already escaped: lep[0]
608 * still holds the imemo_env, so flags is not a FIXNUM and VM_ENV_ESCAPED_P
609 * asserts. That is not a live shareable proc's env, so fall back to in-env. */
610 if (FIXNUM_P(lep[VM_ENV_DATA_INDEX_FLAGS]) &&
611 VM_ENV_ESCAPED_P(lep) &&
612 RB_FL_TEST_RAW(VM_ENV_ENVVAL(lep), RUBY_FL_SHAREABLE)) {
613 return false;
614 }
615 return true;
616}
617
618static inline struct vm_svar *
619lep_svar(const rb_execution_context_t *ec, const VALUE *lep)
620{
621 VALUE svar;
622
623 if (lep_svar_in_env_p(ec, lep)) {
624 svar = lep[VM_ENV_DATA_INDEX_ME_CREF];
625 }
626 else {
627 svar = ec->root_svar;
628 }
629
630 VM_ASSERT(svar == Qfalse || vm_svar_valid_p(svar));
631
632 return (struct vm_svar *)svar;
633}
634
635static inline void
636lep_svar_write(const rb_execution_context_t *ec, const VALUE *lep, const struct vm_svar *svar)
637{
638 VM_ASSERT(vm_svar_valid_p((VALUE)svar));
639
640 if (lep_svar_in_env_p(ec, lep)) {
641 vm_env_write(lep, VM_ENV_DATA_INDEX_ME_CREF, (VALUE)svar);
642 }
643 else {
644 RB_OBJ_WRITE(rb_ec_thread_ptr(ec)->self, &ec->root_svar, svar);
645 }
646}
647
648static VALUE
649lep_svar_get(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key)
650{
651 const struct vm_svar *svar = lep_svar(ec, lep);
652
653 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) return Qnil;
654
655 switch (key) {
656 case VM_SVAR_LASTLINE:
657 return svar->lastline;
658 case VM_SVAR_BACKREF:
659 return svar->backref;
660 default: {
661 const VALUE ary = svar->others;
662
663 if (NIL_P(ary)) {
664 return Qnil;
665 }
666 else {
667 return rb_ary_entry(ary, key - VM_SVAR_EXTRA_START);
668 }
669 }
670 }
671}
672
673static struct vm_svar *
674svar_new(VALUE obj)
675{
676 struct vm_svar *svar = IMEMO_NEW(struct vm_svar, imemo_svar, obj);
677 *((VALUE *)&svar->lastline) = Qnil;
678 *((VALUE *)&svar->backref) = Qnil;
679 *((VALUE *)&svar->others) = Qnil;
680
681 return svar;
682}
683
684static void
685lep_svar_set(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, VALUE val)
686{
687 struct vm_svar *svar = lep_svar(ec, lep);
688
689 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) {
690 lep_svar_write(ec, lep, svar = svar_new((VALUE)svar));
691 }
692
693 switch (key) {
694 case VM_SVAR_LASTLINE:
695 RB_OBJ_WRITE(svar, &svar->lastline, val);
696 return;
697 case VM_SVAR_BACKREF:
698 RB_OBJ_WRITE(svar, &svar->backref, val);
699 return;
700 default: {
701 VALUE ary = svar->others;
702
703 if (NIL_P(ary)) {
704 RB_OBJ_WRITE(svar, &svar->others, ary = rb_ary_new());
705 }
706 rb_ary_store(ary, key - VM_SVAR_EXTRA_START, val);
707 }
708 }
709}
710
711static inline VALUE
712vm_getspecial(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, rb_num_t type)
713{
714 VALUE val;
715
716 if (type == 0) {
717 val = lep_svar_get(ec, lep, key);
718 }
719 else {
720 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
721
722 if (type & 0x01) {
723 switch (type >> 1) {
724 case '&':
725 val = rb_reg_last_match(backref);
726 break;
727 case '`':
728 val = rb_reg_match_pre(backref);
729 break;
730 case '\'':
731 val = rb_reg_match_post(backref);
732 break;
733 case '+':
734 val = rb_reg_match_last(backref);
735 break;
736 default:
737 rb_bug("unexpected back-ref");
738 }
739 }
740 else {
741 val = rb_reg_nth_match((int)(type >> 1), backref);
742 }
743 }
744 return val;
745}
746
747static inline VALUE
748vm_backref_defined(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t type)
749{
750 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
751 int nth = 0;
752
753 if (type & 0x01) {
754 switch (type >> 1) {
755 case '&':
756 case '`':
757 case '\'':
758 break;
759 case '+':
760 return rb_reg_last_defined(backref);
761 default:
762 rb_bug("unexpected back-ref");
763 }
764 }
765 else {
766 nth = (int)(type >> 1);
767 }
768 return rb_reg_nth_defined(nth, backref);
769}
770
771PUREFUNC(static rb_callable_method_entry_t *check_method_entry(VALUE obj, int can_be_svar));
772static rb_callable_method_entry_t *
773check_method_entry(VALUE obj, int can_be_svar)
774{
775 if (obj == Qfalse) return NULL;
776
777#if VM_CHECK_MODE > 0
778 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_method_entry: unknown type: %s", rb_obj_info(obj));
779#endif
780
781 switch (imemo_type(obj)) {
782 case imemo_ment:
783 return (rb_callable_method_entry_t *)obj;
784 case imemo_cref:
785 return NULL;
786 case imemo_svar:
787 if (can_be_svar) {
788 return check_method_entry(((struct vm_svar *)obj)->cref_or_me, FALSE);
789 }
790 default:
791#if VM_CHECK_MODE > 0
792 rb_bug("check_method_entry: svar should not be there:");
793#endif
794 return NULL;
795 }
796}
797
798static rb_callable_method_entry_t *
799env_method_entry_unchecked(VALUE obj, int can_be_svar)
800{
801 if (obj == Qfalse) return NULL;
802
803 switch (imemo_type(obj)) {
804 case imemo_ment:
805 return (rb_callable_method_entry_t *)obj;
806 case imemo_cref:
807 return NULL;
808 case imemo_svar:
809 if (can_be_svar) {
810 return env_method_entry_unchecked(((struct vm_svar *)obj)->cref_or_me, FALSE);
811 }
812 default:
813 return NULL;
814 }
815}
816
817const rb_callable_method_entry_t *
818rb_vm_frame_method_entry(const rb_control_frame_t *cfp)
819{
820 const VALUE *ep = cfp->ep;
821 rb_callable_method_entry_t *me;
822
823 while (!VM_ENV_LOCAL_P(ep)) {
824 if ((me = check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
825 ep = VM_ENV_PREV_EP(ep);
826 }
827
828 return check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
829}
830
831const rb_callable_method_entry_t *
832rb_vm_frame_method_entry_unchecked(const rb_control_frame_t *cfp)
833{
834 const VALUE *ep = cfp->ep;
835 rb_callable_method_entry_t *me;
836
837 while (!VM_ENV_LOCAL_P_UNCHECKED(ep)) {
838 if ((me = env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
839 ep = VM_ENV_PREV_EP_UNCHECKED(ep);
840 }
841
842 return env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
843}
844
845static const rb_iseq_t *
846method_entry_iseqptr(const rb_callable_method_entry_t *me)
847{
848 switch (me->def->type) {
849 case VM_METHOD_TYPE_ISEQ:
850 return me->def->body.iseq.iseqptr;
851 default:
852 return NULL;
853 }
854}
855
856static rb_cref_t *
857method_entry_cref(const rb_callable_method_entry_t *me)
858{
859 switch (me->def->type) {
860 case VM_METHOD_TYPE_ISEQ:
861 return me->def->body.iseq.cref;
862 default:
863 return NULL;
864 }
865}
866
867#if VM_CHECK_MODE == 0
868PUREFUNC(static rb_cref_t *check_cref(VALUE, int));
869#endif
870static rb_cref_t *
871check_cref(VALUE obj, int can_be_svar)
872{
873 if (obj == Qfalse) return NULL;
874
875#if VM_CHECK_MODE > 0
876 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_cref: unknown type: %s", rb_obj_info(obj));
877#endif
878
879 switch (imemo_type(obj)) {
880 case imemo_ment:
881 return method_entry_cref((rb_callable_method_entry_t *)obj);
882 case imemo_cref:
883 return (rb_cref_t *)obj;
884 case imemo_svar:
885 if (can_be_svar) {
886 return check_cref(((struct vm_svar *)obj)->cref_or_me, FALSE);
887 }
888 default:
889#if VM_CHECK_MODE > 0
890 rb_bug("check_method_entry: svar should not be there:");
891#endif
892 return NULL;
893 }
894}
895
896static inline rb_cref_t *
897vm_env_cref(const VALUE *ep)
898{
899 rb_cref_t *cref;
900
901 while (!VM_ENV_LOCAL_P(ep)) {
902 if ((cref = check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return cref;
903 ep = VM_ENV_PREV_EP(ep);
904 }
905
906 return check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
907}
908
909static int
910is_cref(const VALUE v, int can_be_svar)
911{
912 if (RB_TYPE_P(v, T_IMEMO)) {
913 switch (imemo_type(v)) {
914 case imemo_cref:
915 return TRUE;
916 case imemo_svar:
917 if (can_be_svar) return is_cref(((struct vm_svar *)v)->cref_or_me, FALSE);
918 default:
919 break;
920 }
921 }
922 return FALSE;
923}
924
925static int
926vm_env_cref_by_cref(const VALUE *ep)
927{
928 while (!VM_ENV_LOCAL_P(ep)) {
929 if (is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) return TRUE;
930 ep = VM_ENV_PREV_EP(ep);
931 }
932 return is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
933}
934
935static rb_cref_t *
936cref_replace_with_duplicated_cref_each_frame(const VALUE *vptr, int can_be_svar, VALUE parent)
937{
938 const VALUE v = *vptr;
939 rb_cref_t *cref, *new_cref;
940
941 if (RB_TYPE_P(v, T_IMEMO)) {
942 switch (imemo_type(v)) {
943 case imemo_cref:
944 cref = (rb_cref_t *)v;
945 new_cref = rb_vm_cref_dup(cref);
946 if (parent) {
947 RB_OBJ_WRITE(parent, vptr, new_cref);
948 }
949 else {
950 VM_FORCE_WRITE(vptr, (VALUE)new_cref);
951 }
952 return (rb_cref_t *)new_cref;
953 case imemo_svar:
954 if (can_be_svar) {
955 return cref_replace_with_duplicated_cref_each_frame(&((struct vm_svar *)v)->cref_or_me, FALSE, v);
956 }
957 /* fall through */
958 case imemo_ment:
959 rb_bug("cref_replace_with_duplicated_cref_each_frame: unreachable");
960 default:
961 break;
962 }
963 }
964 return NULL;
965}
966
967static rb_cref_t *
968vm_cref_replace_with_duplicated_cref(const VALUE *ep)
969{
970 if (vm_env_cref_by_cref(ep)) {
971 rb_cref_t *cref;
972 VALUE envval;
973
974 while (!VM_ENV_LOCAL_P(ep)) {
975 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
976 if ((cref = cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE, envval)) != NULL) {
977 return cref;
978 }
979 ep = VM_ENV_PREV_EP(ep);
980 }
981 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
982 return cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE, envval);
983 }
984 else {
985 rb_bug("vm_cref_dup: unreachable");
986 }
987}
988
989static rb_cref_t *
990vm_get_cref(const VALUE *ep)
991{
992 rb_cref_t *cref = vm_env_cref(ep);
993
994 if (cref != NULL) {
995 return cref;
996 }
997 else {
998 rb_bug("vm_get_cref: unreachable");
999 }
1000}
1001
1002rb_cref_t *
1003rb_vm_get_cref(const VALUE *ep)
1004{
1005 return vm_get_cref(ep);
1006}
1007
1008static rb_cref_t *
1009vm_ec_cref(const rb_execution_context_t *ec)
1010{
1011 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1012
1013 if (cfp == NULL) {
1014 return NULL;
1015 }
1016 return vm_get_cref(cfp->ep);
1017}
1018
1019static const rb_cref_t *
1020vm_get_const_key_cref(const VALUE *ep)
1021{
1022 const rb_cref_t *cref = vm_get_cref(ep);
1023 const rb_cref_t *key_cref = cref;
1024
1025 while (cref) {
1026 if (CREF_DYNAMIC(cref)) {
1027 return key_cref;
1028 }
1029 cref = CREF_NEXT(cref);
1030 }
1031
1032 /* no dynamic singleton class or cloned class found */
1033 return NULL;
1034}
1035
1036static rb_cref_t *
1037vm_cref_push(const rb_execution_context_t *ec, VALUE klass, const VALUE *ep, int pushed_by_eval, int singleton)
1038{
1039 rb_cref_t *prev_cref = NULL;
1040
1041 if (ep) {
1042 prev_cref = vm_env_cref(ep);
1043 }
1044 else {
1045 rb_control_frame_t *cfp = vm_get_ruby_level_caller_cfp(ec, ec->cfp);
1046
1047 if (cfp) {
1048 prev_cref = vm_env_cref(cfp->ep);
1049 }
1050 }
1051
1052 return vm_cref_new(klass, METHOD_VISI_PUBLIC, FALSE, prev_cref, pushed_by_eval, singleton);
1053}
1054
1055static inline VALUE
1056vm_get_cbase(const VALUE *ep)
1057{
1058 const rb_cref_t *cref = vm_get_cref(ep);
1059
1060 return CREF_CLASS_FOR_DEFINITION(cref);
1061}
1062
1063static inline VALUE
1064vm_get_const_base(const VALUE *ep)
1065{
1066 const rb_cref_t *cref = vm_get_cref(ep);
1067
1068 while (cref) {
1069 if (!CREF_PUSHED_BY_EVAL(cref)) {
1070 return CREF_CLASS_FOR_DEFINITION(cref);
1071 }
1072 cref = CREF_NEXT(cref);
1073 }
1074
1075 return Qundef;
1076}
1077
1078static inline void
1079vm_check_if_namespace(VALUE klass)
1080{
1081 if (!RB_TYPE_P(klass, T_CLASS) && !RB_TYPE_P(klass, T_MODULE)) {
1082 rb_raise(rb_eTypeError, "%+"PRIsVALUE" is not a class/module", klass);
1083 }
1084}
1085
1086static inline void
1087vm_ensure_not_refinement_module(VALUE self)
1088{
1089 if (RB_TYPE_P(self, T_MODULE) && FL_TEST(self, RMODULE_IS_REFINEMENT)) {
1090 rb_warn("not defined at the refinement, but at the outer class/module");
1091 }
1092}
1093
1094static inline VALUE
1095vm_get_iclass(const rb_control_frame_t *cfp, VALUE klass)
1096{
1097 return klass;
1098}
1099
1100static inline VALUE
1101vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, bool allow_nil, int is_defined)
1102{
1103 void rb_const_warn_if_deprecated(const rb_const_entry_t *ce, VALUE klass, ID id);
1104 VALUE val;
1105
1106 if (NIL_P(orig_klass) && allow_nil) {
1107 /* in current lexical scope */
1108 const rb_cref_t *root_cref = vm_get_cref(ec->cfp->ep);
1109 const rb_cref_t *cref;
1110 VALUE klass = Qnil;
1111
1112 while (root_cref && CREF_PUSHED_BY_EVAL(root_cref)) {
1113 root_cref = CREF_NEXT(root_cref);
1114 }
1115 cref = root_cref;
1116 while (cref && CREF_NEXT(cref)) {
1117 if (CREF_PUSHED_BY_EVAL(cref)) {
1118 klass = Qnil;
1119 }
1120 else {
1121 klass = CREF_CLASS(cref);
1122 }
1123 cref = CREF_NEXT(cref);
1124
1125 if (!NIL_P(klass)) {
1126 VALUE av, am = 0;
1127 rb_const_entry_t entry, *ce = &entry;
1128 search_continue:
1129 if (rb_const_lookup_copy(klass, id, ce)) {
1130 val = ce->value;
1131 // before the warning, which may run Ruby: only this stack holds val until the owner has it
1132 if (!is_defined && !UNDEF_P(val) && UNLIKELY(!rb_class_owned_p(klass)) &&
1133 !rb_class_field_shareable_p(klass, val)) {
1134 rb_const_warn_if_deprecated(ce, klass, id);
1135 rb_const_unshareable_error(klass, id);
1136 }
1137 rb_const_warn_if_deprecated(ce, klass, id);
1138 if (UNDEF_P(val)) {
1139 if (am == klass) break;
1140 am = klass;
1141 if (is_defined) return 1;
1142 if (rb_autoloading_value(klass, id, &av, NULL)) {
1143 if (UNLIKELY(!rb_class_owned_p(klass)) && !rb_class_field_shareable_p(klass, av)) {
1144 rb_const_unshareable_error(klass, id);
1145 }
1146 return av;
1147 }
1148 rb_autoload_load(klass, id);
1149 goto search_continue;
1150 }
1151 else {
1152 if (is_defined) {
1153 return 1;
1154 }
1155 else {
1156 return val;
1157 }
1158 }
1159 }
1160 }
1161 }
1162
1163 /* search self */
1164 if (root_cref && !NIL_P(CREF_CLASS(root_cref))) {
1165 klass = vm_get_iclass(ec->cfp, CREF_CLASS(root_cref));
1166 }
1167 else {
1168 klass = CLASS_OF(ec->cfp->self);
1169 }
1170
1171 if (is_defined) {
1172 return rb_const_defined(klass, id);
1173 }
1174 else {
1175 return rb_const_get(klass, id);
1176 }
1177 }
1178 else {
1179 vm_check_if_namespace(orig_klass);
1180 if (is_defined) {
1181 return rb_public_const_defined_from(orig_klass, id);
1182 }
1183 else {
1184 return rb_public_const_get_from(orig_klass, id);
1185 }
1186 }
1187}
1188
1189VALUE
1190rb_vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, VALUE allow_nil)
1191{
1192 return vm_get_ev_const(ec, orig_klass, id, allow_nil == Qtrue, 0);
1193}
1194
1195static inline VALUE
1196vm_get_ev_const_chain(rb_execution_context_t *ec, const ID *segments)
1197{
1198 VALUE val = Qnil;
1199 int idx = 0;
1200 int allow_nil = TRUE;
1201 if (segments[0] == idNULL) {
1202 val = rb_cObject;
1203 idx++;
1204 allow_nil = FALSE;
1205 }
1206 while (segments[idx]) {
1207 ID id = segments[idx++];
1208 val = vm_get_ev_const(ec, val, id, allow_nil, 0);
1209 allow_nil = FALSE;
1210 }
1211 return val;
1212}
1213
1214
1215static inline VALUE
1216vm_get_cvar_base(const rb_cref_t *cref, const rb_control_frame_t *cfp, int top_level_raise)
1217{
1218 VALUE klass;
1219
1220 if (!cref) {
1221 rb_bug("vm_get_cvar_base: no cref");
1222 }
1223
1224 while (CREF_NEXT(cref) &&
1225 (NIL_P(CREF_CLASS(cref)) || RCLASS_SINGLETON_P(CREF_CLASS(cref)) ||
1226 CREF_PUSHED_BY_EVAL(cref) || CREF_SINGLETON(cref))) {
1227 cref = CREF_NEXT(cref);
1228 }
1229 if (top_level_raise && !CREF_NEXT(cref)) {
1230 rb_raise(rb_eRuntimeError, "class variable access from toplevel");
1231 }
1232
1233 klass = vm_get_iclass(cfp, CREF_CLASS(cref));
1234
1235 if (NIL_P(klass)) {
1236 rb_raise(rb_eTypeError, "no class variables available");
1237 }
1238 return klass;
1239}
1240
1241#define ractor_incidental_shareable_p(cond, val) \
1242 (!(cond) || rb_ractor_shareable_p(val))
1243#define ractor_object_incidental_shareable_p(obj, val) \
1244 ractor_incidental_shareable_p(rb_ractor_shareable_p(obj), val)
1245
1246ALWAYS_INLINE(static VALUE vm_getivar(VALUE, ID, const rb_iseq_t *, IVC, const struct rb_callcache *, int, VALUE));
1247static inline VALUE
1248vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, VALUE default_value)
1249{
1250 VALUE fields_obj;
1251#if OPT_IC_FOR_IVAR
1252 if (SPECIAL_CONST_P(obj)) {
1253 return default_value;
1254 }
1255
1256 switch (BUILTIN_TYPE(obj)) {
1257 case T_OBJECT:
1258 fields_obj = ROBJECT_FIELDS_OBJ(obj);
1259 break;
1260 case T_CLASS:
1261 case T_MODULE:
1262 {
1263 if (UNLIKELY(!rb_class_owned_p(obj))) {
1264 // For two reasons we can only use the fast path on the Ractor
1265 // that owns the class.
1266 // First, only the owner Ractor is allowed to set ivars on classes
1267 // and modules. So we can skip locking.
1268 // Second, other ractors need to check the shareability of the
1269 // values returned from the class ivars.
1270
1271 if (default_value == Qundef) { // defined?
1272 return rb_ivar_defined(obj, id) ? Qtrue : Qundef;
1273 }
1274 else {
1275 goto general_path;
1276 }
1277 }
1278
1279 fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1280 break;
1281 }
1282 default:
1283 fields_obj = rb_obj_fields(obj, id);
1284 }
1285
1286 if (!fields_obj) {
1287 return default_value;
1288 }
1289
1290 VALUE val = Qundef;
1291
1292 shape_id_t shape_id = RBASIC_SHAPE_ID_FOR_READ(fields_obj);
1293 VALUE *ivar_list = rb_imemo_fields_ptr(fields_obj);
1294
1295 rb_getivar_cache cache = rb_getivar_cache_unpack(vm_cache_attr_index_atomic_read(is_attr, ic, cc));
1296
1297 if (LIKELY(cache.shape_offset == shape_id)) {
1298 if (cache.index == ATTR_INDEX_NOT_SET) {
1299 return default_value;
1300 }
1301
1302 val = ivar_list[cache.index];
1303#if USE_DEBUG_COUNTER
1304 RB_DEBUG_COUNTER_INC(ivar_get_ic_hit);
1305
1306 if (RB_TYPE_P(obj, T_OBJECT)) {
1307 RB_DEBUG_COUNTER_INC(ivar_get_obj_hit);
1308 }
1309#endif
1310 RUBY_ASSERT(!UNDEF_P(val));
1311 }
1312 else { // cache miss case
1313#if USE_DEBUG_COUNTER
1314 if (is_attr) {
1315 if (cache.shape_offset != INVALID_SHAPE_ID) {
1316 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_set);
1317 }
1318 else {
1319 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_unset);
1320 }
1321 }
1322 else {
1323 if (cache.shape_offset != INVALID_SHAPE_ID) {
1324 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_set);
1325 }
1326 else {
1327 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_unset);
1328 }
1329 }
1330 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1331
1332 if (RB_TYPE_P(obj, T_OBJECT)) {
1333 RB_DEBUG_COUNTER_INC(ivar_get_obj_miss);
1334 }
1335#endif
1336
1337 if (UNLIKELY(rb_shape_complex_p(shape_id))) {
1338 st_table *table = (st_table *)ivar_list;
1339
1340 RUBY_ASSERT(table);
1341 RUBY_ASSERT(table == rb_imemo_fields_complex_tbl(fields_obj));
1342
1343 if (!st_lookup(table, id, &val)) {
1344 val = default_value;
1345 }
1346 }
1347 else {
1348 shape_id_t previous_cached_offset = cache.shape_offset;
1349 if (rb_shape_get_iv_index_with_hint(shape_id, id, &cache.index, &cache.shape_offset)) {
1350 if (cache.shape_offset != previous_cached_offset) {
1351 RUBY_ASSERT(!rb_shape_complex_p(cache.shape_offset));
1352 RUBY_ASSERT(cache.shape_offset != INVALID_SHAPE_ID);
1353
1354 uint64_t packed_cache = rb_getivar_cache_pack(cache.shape_offset, cache.index);
1355 vm_cache_attr_index_set(is_attr, ic, cc, packed_cache);
1356 }
1357
1358 if (cache.index == ATTR_INDEX_NOT_SET) {
1359 val = default_value;
1360 }
1361 else {
1362 // We fetched the ivar list above
1363 val = ivar_list[cache.index];
1364 RUBY_ASSERT(!UNDEF_P(val));
1365 }
1366 }
1367 else {
1368 vm_cache_attr_index_set(is_attr, ic, cc, rb_getivar_cache_pack(shape_id, ATTR_INDEX_NOT_SET));
1369 val = default_value;
1370 }
1371 }
1372 }
1373
1374 if (!UNDEF_P(default_value)) {
1375 RUBY_ASSERT(!UNDEF_P(val));
1376 }
1377
1378 return val;
1379
1380general_path:
1381#endif /* OPT_IC_FOR_IVAR */
1382 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1383
1384 if (is_attr) {
1385 return rb_attr_get(obj, id);
1386 }
1387 else {
1388 return rb_ivar_get(obj, id);
1389 }
1390}
1391
1392ALWAYS_INLINE(static VALUE vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr));
1393NOINLINE(static VALUE vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic));
1394NOINLINE(static VALUE vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc));
1395
1396static VALUE
1397vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr)
1398{
1399#if OPT_IC_FOR_IVAR
1400 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss);
1401
1402 rb_check_ivar_modifiable(obj);
1403
1404 shape_id_t previous_shape_id = RBASIC_SHAPE_ID(obj);
1405 attr_index_t index = rb_ivar_set_index(obj, id, val);
1406 shape_id_t next_shape_id = RBASIC_SHAPE_ID(obj);
1407
1408 if (!rb_shape_complex_p(next_shape_id)) {
1409 uint64_t packed_cache = rb_setivar_cache_pack(RSHAPE_OFFSET(previous_shape_id), RSHAPE_OFFSET(next_shape_id), index);
1410 vm_cache_attr_index_set(is_attr, ic, cc, packed_cache);
1411 }
1412
1413 RB_DEBUG_COUNTER_INC(ivar_set_obj_miss);
1414 return val;
1415#else
1416 return rb_ivar_set(obj, id, val);
1417#endif
1418}
1419
1420static VALUE
1421vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic)
1422{
1423 return vm_setivar_slowpath(obj, id, val, iseq, ic, NULL, false);
1424}
1425
1426static VALUE
1427vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc)
1428{
1429 return vm_setivar_slowpath(obj, id, val, NULL, NULL, cc, true);
1430}
1431
1432NOINLINE(static VALUE vm_setivar_class(VALUE obj, VALUE val, rb_setivar_cache cache));
1433static VALUE
1434vm_setivar_class(VALUE obj, VALUE val, rb_setivar_cache cache)
1435{
1436 if (UNLIKELY(!rb_class_owned_p(obj))) {
1437 return Qundef;
1438 }
1439
1440 VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1441 if (UNLIKELY(!fields_obj)) {
1442 return Qundef;
1443 }
1444
1445 shape_id_t shape_id = RBASIC_SHAPE_ID(fields_obj);
1446 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, RBASIC_SHAPE_ID(fields_obj), cache);
1447 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1448 return Qundef;
1449 }
1450
1451 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1452
1453 if (shape_id != dest_shape_id) {
1454 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1455 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1456 }
1457
1458 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1459
1460 return val;
1461}
1462
1463NOINLINE(static VALUE vm_setivar_default(VALUE obj, ID id, VALUE val, rb_setivar_cache cache));
1464static VALUE
1465vm_setivar_default(VALUE obj, ID id, VALUE val, rb_setivar_cache cache)
1466{
1467 VALUE fields_obj = rb_obj_fields(obj, id);
1468 if (UNLIKELY(!fields_obj)) {
1469 return Qundef;
1470 }
1471
1472 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1473 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, RBASIC_SHAPE_ID(fields_obj), cache);
1474 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1475 return Qundef;
1476 }
1477
1478 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1479
1480 if (shape_id != dest_shape_id) {
1481 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1482 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1483 }
1484
1485 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1486
1487 return val;
1488}
1489
1490static inline VALUE
1491vm_setivar(VALUE obj, VALUE val, rb_setivar_cache cache)
1492{
1493#if OPT_IC_FOR_IVAR
1494 switch (BUILTIN_TYPE(obj)) {
1495 case T_OBJECT:
1496 {
1497 VM_ASSERT(!rb_ractor_shareable_p(obj) || rb_obj_frozen_p(obj));
1498
1499 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1500 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, shape_id, cache);
1501 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1502 break;
1503 }
1504
1505 VALUE fields_obj = ROBJECT_FIELDS_OBJ(obj);
1506 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1507 if (shape_id != dest_shape_id) {
1508 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1509 if (fields_obj != obj) {
1510 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1511 }
1512 }
1513
1514 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1515 RB_DEBUG_COUNTER_INC(ivar_set_obj_hit);
1516 return val;
1517 }
1518 break;
1519 case T_CLASS:
1520 case T_MODULE:
1521 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss_noobject);
1522 default:
1523 break;
1524 }
1525
1526 return Qundef;
1527#endif /* OPT_IC_FOR_IVAR */
1528}
1529
1530static VALUE
1531update_classvariable_cache(const rb_iseq_t *iseq, VALUE klass, ID id, const rb_cref_t * cref, ICVARC ic)
1532{
1533 VALUE defined_class = 0;
1534 VALUE cvar_value = rb_cvar_find(klass, id, &defined_class);
1535
1536 if (RB_TYPE_P(defined_class, T_ICLASS)) {
1537 defined_class = RBASIC(defined_class)->klass;
1538 }
1539
1540 VALUE rb_cvc_tbl = RCLASS_CVC_TBL(defined_class);
1541 if (!rb_cvc_tbl) {
1542 rb_bug("the cvc table should be set");
1543 }
1544
1545 VALUE ent_data;
1546 if (!rb_marked_id_table_lookup(rb_cvc_tbl, id, &ent_data)) {
1547 rb_bug("should have cvar cache entry");
1548 }
1549
1550 struct rb_cvar_class_tbl_entry *ent = (void *)ent_data;
1551
1552 ent->global_cvar_state = GET_GLOBAL_CVAR_STATE();
1553 RB_OBJ_WRITE((VALUE)ent, &ent->cref, cref);
1554 RB_OBJ_WRITE(iseq, &ic->entry, ent);
1555
1556 RUBY_ASSERT(BUILTIN_TYPE((VALUE)cref) == T_IMEMO && IMEMO_TYPE_P(cref, imemo_cref));
1557
1558 return cvar_value;
1559}
1560
1561static inline VALUE
1562vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, ICVARC ic)
1563{
1564 const rb_cref_t *cref;
1565 cref = vm_get_cref(GET_EP());
1566
1567 // The fast path skips rb_cvar_find's checks, so it is for class_value's owner:
1568 // the sole writer, and the only one allowed to see an unshareable value.
1569 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref &&
1570 LIKELY(rb_class_owned_p(ic->entry->class_value))) {
1571 RB_DEBUG_COUNTER_INC(cvar_read_inline_hit);
1572
1573 VALUE v = rb_ivar_lookup(ic->entry->class_value, id, Qundef);
1574 RUBY_ASSERT(!UNDEF_P(v));
1575
1576 return v;
1577 }
1578
1579 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1580
1581 return update_classvariable_cache(iseq, klass, id, cref, ic);
1582}
1583
1584VALUE
1585rb_vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, ICVARC ic)
1586{
1587 return vm_getclassvariable(iseq, cfp, id, ic);
1588}
1589
1590static inline void
1591vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, VALUE val, ICVARC ic)
1592{
1593 const rb_cref_t *cref;
1594 cref = vm_get_cref(GET_EP());
1595
1596 // as on the read side: the fast path writes straight into class_value
1597 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref &&
1598 LIKELY(rb_class_owned_p(ic->entry->class_value))) {
1599 RB_DEBUG_COUNTER_INC(cvar_write_inline_hit);
1600
1601 rb_class_ivar_set(ic->entry->class_value, id, val);
1602 return;
1603 }
1604
1605 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1606
1607 rb_cvar_set(klass, id, val);
1608
1609 update_classvariable_cache(iseq, klass, id, cref, ic);
1610}
1611
1612void
1613rb_vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, VALUE val, ICVARC ic)
1614{
1615 vm_setclassvariable(iseq, cfp, id, val, ic);
1616}
1617
1618ALWAYS_INLINE(static VALUE vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic));
1619static inline VALUE
1620vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1621{
1622 return vm_getivar(obj, id, iseq, ic, NULL, FALSE, Qnil);
1623}
1624
1625static inline void
1626vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1627{
1628 if (RB_SPECIAL_CONST_P(obj)) {
1630 return;
1631 }
1632
1633 rb_setivar_cache cache = rb_setivar_cache_unpack(vm_ic_atomic_cache_read(ic));
1634 if (UNLIKELY(UNDEF_P(vm_setivar(obj, val, cache)))) {
1635 switch (BUILTIN_TYPE(obj)) {
1636 case T_OBJECT:
1637 break;
1638 case T_CLASS:
1639 case T_MODULE:
1640 if (!UNDEF_P(vm_setivar_class(obj, val, cache))) {
1641 return;
1642 }
1643 break;
1644 default:
1645 if (!UNDEF_P(vm_setivar_default(obj, id, val, cache))) {
1646 return;
1647 }
1648 }
1649 vm_setivar_slowpath_ivar(obj, id, val, iseq, ic);
1650 }
1651}
1652
1653void
1654rb_vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1655{
1656 vm_setinstancevariable(iseq, obj, id, val, ic);
1657}
1658
1659VALUE
1660rb_vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1661{
1662 return vm_getinstancevariable(iseq, obj, id, ic);
1663}
1664
1665static VALUE
1666vm_throw_continue(const rb_execution_context_t *ec, VALUE err)
1667{
1668 /* continue throw */
1669
1670 if (FIXNUM_P(err)) {
1671 ec->tag->state = RUBY_TAG_FATAL;
1672 }
1673 else if (SYMBOL_P(err)) {
1674 ec->tag->state = TAG_THROW;
1675 }
1676 else if (THROW_DATA_P(err)) {
1677 ec->tag->state = THROW_DATA_STATE((struct vm_throw_data *)err);
1678 }
1679 else {
1680 ec->tag->state = TAG_RAISE;
1681 }
1682 return err;
1683}
1684
1685static VALUE
1686vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, enum ruby_tag_type state,
1687 const int flag, const VALUE throwobj)
1688{
1689 const rb_control_frame_t *escape_cfp = NULL;
1690 const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */
1691
1692 if (flag != 0) {
1693 /* do nothing */
1694 }
1695 else if (state == TAG_BREAK) {
1696 int is_orphan = 1;
1697 const VALUE *ep = GET_EP();
1698 const rb_iseq_t *base_iseq = GET_ISEQ();
1699 escape_cfp = reg_cfp;
1700
1701 while (ISEQ_BODY(base_iseq)->type != ISEQ_TYPE_BLOCK) {
1702 if (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) {
1703 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1704 ep = escape_cfp->ep;
1705 base_iseq = CFP_ISEQ(escape_cfp);
1706 }
1707 else {
1708 ep = VM_ENV_PREV_EP(ep);
1709 base_iseq = ISEQ_BODY(base_iseq)->parent_iseq;
1710 escape_cfp = rb_vm_search_cf_from_ep(ec, escape_cfp, ep);
1711 VM_ASSERT(CFP_ISEQ(escape_cfp) == base_iseq);
1712 }
1713 }
1714
1715 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1716 /* lambda{... break ...} */
1717 is_orphan = 0;
1718 state = TAG_RETURN;
1719 }
1720 else {
1721 ep = VM_ENV_PREV_EP(ep);
1722
1723 while (escape_cfp < eocfp) {
1724 if (escape_cfp->ep == ep) {
1725 const rb_iseq_t *const iseq = CFP_ISEQ(escape_cfp);
1726 const VALUE epc = CFP_PC(escape_cfp) - ISEQ_BODY(iseq)->iseq_encoded;
1727 const struct iseq_catch_table *const ct = ISEQ_BODY(iseq)->catch_table;
1728 unsigned int i;
1729
1730 if (!ct) break;
1731 for (i=0; i < ct->size; i++) {
1732 const struct iseq_catch_table_entry *const entry =
1733 UNALIGNED_MEMBER_PTR(ct, entries[i]);
1734
1735 if (entry->type == CATCH_TYPE_BREAK &&
1736 entry->iseq == base_iseq &&
1737 entry->start < epc && entry->end >= epc) {
1738 if (entry->cont == epc) { /* found! */
1739 is_orphan = 0;
1740 }
1741 break;
1742 }
1743 }
1744 break;
1745 }
1746
1747 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1748 }
1749 }
1750
1751 if (is_orphan) {
1752 rb_vm_localjump_error("break from proc-closure", throwobj, TAG_BREAK);
1753 }
1754 }
1755 else if (state == TAG_RETRY) {
1756 const VALUE *ep = VM_ENV_PREV_EP(GET_EP());
1757
1758 escape_cfp = rb_vm_search_cf_from_ep(ec, reg_cfp, ep);
1759 }
1760 else if (state == TAG_RETURN) {
1761 const VALUE *current_ep = GET_EP();
1762 const VALUE *target_ep = NULL, *target_lep, *ep = current_ep;
1763 int in_class_frame = 0;
1764 int toplevel = 1;
1765 escape_cfp = reg_cfp;
1766
1767 // find target_lep, target_ep
1768 while (!VM_ENV_LOCAL_P(ep)) {
1769 if (VM_ENV_FLAGS(ep, VM_FRAME_FLAG_LAMBDA) && target_ep == NULL) {
1770 target_ep = ep;
1771 }
1772 ep = VM_ENV_PREV_EP(ep);
1773 }
1774 target_lep = ep;
1775
1776 while (escape_cfp < eocfp) {
1777 const VALUE *lep = VM_CF_LEP(escape_cfp);
1778
1779 if (!target_lep) {
1780 target_lep = lep;
1781 }
1782
1783 if (lep == target_lep &&
1784 VM_FRAME_RUBYFRAME_P(escape_cfp) &&
1785 ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) {
1786 in_class_frame = 1;
1787 target_lep = 0;
1788 }
1789
1790 if (lep == target_lep) {
1791 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1792 toplevel = 0;
1793 if (in_class_frame) {
1794 /* lambda {class A; ... return ...; end} */
1795 goto valid_return;
1796 }
1797 else {
1798 const VALUE *tep = current_ep;
1799
1800 while (target_lep != tep) {
1801 if (escape_cfp->ep == tep) {
1802 /* in lambda */
1803 if (tep == target_ep) {
1804 goto valid_return;
1805 }
1806 else {
1807 goto unexpected_return;
1808 }
1809 }
1810 tep = VM_ENV_PREV_EP(tep);
1811 }
1812 }
1813 }
1814 else if (VM_FRAME_RUBYFRAME_P(escape_cfp)) {
1815 switch (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type) {
1816 case ISEQ_TYPE_TOP:
1817 case ISEQ_TYPE_MAIN:
1818 if (toplevel) {
1819 if (in_class_frame) goto unexpected_return;
1820 if (target_ep == NULL) {
1821 goto valid_return;
1822 }
1823 else {
1824 goto unexpected_return;
1825 }
1826 }
1827 break;
1828 case ISEQ_TYPE_EVAL: {
1829 const rb_iseq_t *is = CFP_ISEQ(escape_cfp);
1830 enum rb_iseq_type t = ISEQ_BODY(is)->type;
1831 while (t == ISEQ_TYPE_RESCUE || t == ISEQ_TYPE_ENSURE || t == ISEQ_TYPE_EVAL) {
1832 if (!(is = ISEQ_BODY(is)->parent_iseq)) break;
1833 t = ISEQ_BODY(is)->type;
1834 }
1835 toplevel = t == ISEQ_TYPE_TOP || t == ISEQ_TYPE_MAIN;
1836 break;
1837 }
1838 case ISEQ_TYPE_CLASS:
1839 toplevel = 0;
1840 break;
1841 default:
1842 break;
1843 }
1844 }
1845 }
1846
1847 if (escape_cfp->ep == target_lep && ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_METHOD) {
1848 if (target_ep == NULL) {
1849 goto valid_return;
1850 }
1851 else {
1852 goto unexpected_return;
1853 }
1854 }
1855
1856 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1857 }
1858 unexpected_return:;
1859 rb_vm_localjump_error("unexpected return", throwobj, TAG_RETURN);
1860
1861 valid_return:;
1862 /* do nothing */
1863 }
1864 else {
1865 rb_bug("isns(throw): unsupported throw type");
1866 }
1867
1868 ec->tag->state = state;
1869 return (VALUE)THROW_DATA_NEW(throwobj, escape_cfp, state);
1870}
1871
1872static VALUE
1873vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
1874 rb_num_t throw_state, VALUE throwobj)
1875{
1876 const int state = (int)(throw_state & VM_THROW_STATE_MASK);
1877 const int flag = (int)(throw_state & VM_THROW_NO_ESCAPE_FLAG);
1878
1879 if (state != 0) {
1880 return vm_throw_start(ec, reg_cfp, state, flag, throwobj);
1881 }
1882 else {
1883 return vm_throw_continue(ec, throwobj);
1884 }
1885}
1886
1887// Fallback for YJIT. Prepare throw data and return it.
1888VALUE
1889rb_vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj)
1890{
1891 return vm_throw(ec, reg_cfp, throw_state, throwobj);
1892}
1893
1894NORETURN(VALUE rb_zjit_throw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj));
1895
1896// Fallback for ZJIT. Make a longjmp and unwind to the most recent vm_exec().
1897VALUE
1898rb_zjit_throw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj)
1899{
1900 VALUE val = vm_throw(ec, reg_cfp, throw_state, throwobj);
1901
1902 // vm_throw() has set ec->tag->state. On the longjmp path, vm_exec() reads the throw data from
1903 // ec->errinfo instead of vm_exec_core()'s return value like THROW_EXCEPTION()'s path does, so
1904 // we need to put it there instead.
1905 enum ruby_tag_type state = ec->tag->state;
1906 ec->errinfo = val;
1907 EC_JUMP_TAG(ec, state);
1908
1910}
1911
1912static inline void
1913vm_expandarray(struct rb_control_frame_struct *cfp, VALUE ary, rb_num_t num, int flag)
1914{
1915 int is_splat = flag & 0x01;
1916 const VALUE *ptr;
1917 rb_num_t len;
1918 const VALUE obj = ary;
1919
1920 if (!RB_TYPE_P(ary, T_ARRAY) && NIL_P(ary = rb_check_array_type(ary))) {
1921 ary = obj;
1922 ptr = &ary;
1923 len = 1;
1924 }
1925 else {
1926 ptr = RARRAY_CONST_PTR(ary);
1927 len = (rb_num_t)RARRAY_LEN(ary);
1928 }
1929
1930 if (num + is_splat == 0) {
1931 /* no space left on stack */
1932 }
1933 else if (flag & 0x02) {
1934 /* post: ..., nil ,ary[-1], ..., ary[0..-num] # top */
1935 rb_num_t i = 0, j;
1936
1937 if (len < num) {
1938 for (i = 0; i < num - len; i++) {
1939 *cfp->sp++ = Qnil;
1940 }
1941 }
1942
1943 for (j = 0; i < num; i++, j++) {
1944 VALUE v = ptr[len - j - 1];
1945 *cfp->sp++ = v;
1946 }
1947
1948 if (is_splat) {
1949 *cfp->sp++ = rb_ary_new4(len - j, ptr);
1950 }
1951 }
1952 else {
1953 /* normal: ary[num..-1], ary[num-2], ary[num-3], ..., ary[0] # top */
1954 if (is_splat) {
1955 if (num > len) {
1956 *cfp->sp++ = rb_ary_new();
1957 }
1958 else {
1959 *cfp->sp++ = rb_ary_new4(len - num, ptr + num);
1960 }
1961 }
1962
1963 if (num > len) {
1964 rb_num_t i = 0;
1965 for (; i < num - len; i++) {
1966 *cfp->sp++ = Qnil;
1967 }
1968
1969 for (rb_num_t j = 0; i < num; i++, j++) {
1970 *cfp->sp++ = ptr[len - j - 1];
1971 }
1972 }
1973 else {
1974 for (rb_num_t j = 0; j < num; j++) {
1975 *cfp->sp++ = ptr[num - j - 1];
1976 }
1977 }
1978 }
1979
1980 RB_GC_GUARD(ary);
1981}
1982
1983static VALUE vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
1984
1985static VALUE vm_mtbl_dump(VALUE klass, ID target_mid);
1986
1987static struct rb_class_cc_entries *
1988vm_ccs_create(VALUE klass, VALUE cc_tbl, ID mid, const rb_callable_method_entry_t *cme)
1989{
1990 int initial_capa = 2;
1991 struct rb_class_cc_entries *ccs = ruby_xmalloc(vm_ccs_alloc_size(initial_capa));
1992#if VM_CHECK_MODE > 0
1993 ccs->debug_sig = ~(VALUE)ccs;
1994#endif
1995 ccs->capa = initial_capa;
1996 ccs->len = 0;
1997 ccs->cme = cme;
1998 METHOD_ENTRY_CACHED_SET((rb_callable_method_entry_t *)cme);
1999
2000 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
2001 RB_OBJ_WRITTEN(cc_tbl, Qundef, cme);
2002 return ccs;
2003}
2004
2005static void
2006vm_ccs_push(VALUE cc_tbl, ID mid, struct rb_class_cc_entries *ccs, const struct rb_callinfo *ci, const struct rb_callcache *cc)
2007{
2008 if (! vm_cc_markable(cc)) {
2009 return;
2010 }
2011
2012 if (UNLIKELY(ccs->len == ccs->capa)) {
2013 RUBY_ASSERT(ccs->capa > 0);
2014 ccs->capa *= 2;
2015 ccs = ruby_xrealloc(ccs, vm_ccs_alloc_size(ccs->capa));
2016#if VM_CHECK_MODE > 0
2017 ccs->debug_sig = ~(VALUE)ccs;
2018#endif
2019 // GC?
2020 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
2021 }
2022 VM_ASSERT(ccs->len < ccs->capa);
2023
2024 const struct rb_callinfo_kwarg *kwarg = vm_ci_kwarg(ci);
2025 const int pos = ccs->len++;
2026 ccs->entries[pos].argc = vm_ci_argc(ci);
2027 ccs->entries[pos].flag = (unsigned short)vm_ci_flag(ci);
2028 ccs->entries[pos].kw_len = (unsigned short)(kwarg ? kwarg->keyword_len : 0);
2029 RB_OBJ_WRITE(cc_tbl, &ccs->entries[pos].cc, cc);
2030
2031 if (RB_DEBUG_COUNTER_SETMAX(ccs_maxlen, ccs->len)) {
2032 // for tuning
2033 // vm_mtbl_dump(klass, 0);
2034 }
2035}
2036
2037#if VM_CHECK_MODE > 0
2038void
2039rb_vm_ccs_dump(struct rb_class_cc_entries *ccs)
2040{
2041 ruby_debug_printf("ccs:%p (%d,%d)\n", (void *)ccs, ccs->len, ccs->capa);
2042 for (int i=0; i<ccs->len; i++) {
2043 ruby_debug_printf("CCS CI ID:flag:%x argc:%u kw_len:%u\n",
2044 ccs->entries[i].flag,
2045 ccs->entries[i].argc,
2046 ccs->entries[i].kw_len);
2047 rp(ccs->entries[i].cc);
2048 }
2049}
2050
2051static int
2052vm_ccs_verify(struct rb_class_cc_entries *ccs, ID mid, VALUE klass)
2053{
2054 VM_ASSERT(vm_ccs_p(ccs));
2055 VM_ASSERT(ccs->len <= ccs->capa);
2056
2057 for (int i=0; i<ccs->len; i++) {
2058 const struct rb_callcache *cc = ccs->entries[i].cc;
2059
2060 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
2061 VM_ASSERT(vm_cc_class_check(cc, klass));
2062 VM_ASSERT(vm_cc_check_cme(cc, ccs->cme));
2063 VM_ASSERT(!vm_cc_super_p(cc));
2064 VM_ASSERT(!vm_cc_refinement_p(cc));
2065 }
2066 return TRUE;
2067}
2068#endif
2069
2070const rb_callable_method_entry_t *rb_check_overloaded_cme(const rb_callable_method_entry_t *cme, const struct rb_callinfo * const ci);
2071
2072static void
2073vm_evict_cc(VALUE klass, VALUE cc_tbl, ID mid)
2074{
2075 ASSERT_vm_locking();
2076
2077 if (rb_multi_ractor_p()) {
2078 if (RCLASS_WRITABLE_CC_TBL(klass) != cc_tbl) {
2079 // Another ractor updated the CC table while we were waiting on the VM lock.
2080 // We have to retry.
2081 return;
2082 }
2083
2084 VALUE ccs_obj = 0;
2085 rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj);
2086 struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_obj;
2087
2088 if (!ccs || !METHOD_ENTRY_INVALIDATED(ccs->cme)) {
2089 // Another ractor replaced that entry while we were waiting on the VM lock.
2090 return;
2091 }
2092
2093 VALUE new_table = rb_vm_cc_table_dup(cc_tbl);
2094 rb_vm_cc_table_delete(new_table, mid);
2095 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), new_table);
2096 }
2097 else {
2098 rb_vm_cc_table_delete(cc_tbl, mid);
2099 }
2100}
2101
2102static const struct rb_callcache *
2103vm_populate_cc(VALUE klass, const struct rb_callinfo * const ci, ID mid)
2104{
2105 ASSERT_vm_locking();
2106
2107 RB_DEBUG_COUNTER_INC(cc_not_found_in_ccs);
2108
2109 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
2110
2111 VM_ASSERT(cme == NULL || IMEMO_TYPE_P(cme, imemo_ment));
2112
2113 if (cme == NULL) {
2114 // undef or not found: can't cache the information
2115 VM_ASSERT(vm_cc_cme(&rb_vm_empty_cc) == NULL);
2116 return &rb_vm_empty_cc;
2117 }
2118
2119 if (UNLIKELY(cme->def->uncached && !(vm_ci_flag(ci) & VM_CALL_SUPER))) {
2120 // Leaving a cc here would cost a ccs and a cc table entry on klass, which
2121 // is usually a metaclass, so the cost would scale with the number of
2122 // classes. invokesuper is excluded because vm_search_super_method needs a
2123 // cme in the cc it gets back.
2124 return &rb_vm_uncached_cc;
2125 }
2126
2127 VALUE cc_tbl = RCLASS_WRITABLE_CC_TBL(klass);
2128 const VALUE original_cc_table = cc_tbl;
2129 if (!cc_tbl) {
2130 // Is this possible after rb_callable_method_entry ?
2131 cc_tbl = rb_vm_cc_table_create(1);
2132 }
2133 else if (rb_multi_ractor_p()) {
2134 cc_tbl = rb_vm_cc_table_dup(cc_tbl);
2135 }
2136
2137 VM_ASSERT(cme == rb_callable_method_entry(klass, mid));
2138
2139 METHOD_ENTRY_CACHED_SET((struct rb_callable_method_entry_struct *)cme);
2140
2141 VM_ASSERT(cc_tbl);
2142
2143 struct rb_class_cc_entries *ccs = NULL;
2144 {
2145 VALUE ccs_obj;
2146 if (UNLIKELY(rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj))) {
2147 ccs = (struct rb_class_cc_entries *)ccs_obj;
2148 }
2149 else {
2150 // TODO: required?
2151 ccs = vm_ccs_create(klass, cc_tbl, mid, cme);
2152 }
2153 }
2154
2155 cme = rb_check_overloaded_cme(cme, ci);
2156
2157 const struct rb_callcache *cc = vm_cc_new(klass, cme, vm_call_general, cc_type_normal);
2158 vm_ccs_push(cc_tbl, mid, ccs, ci, cc);
2159
2160 VM_ASSERT(vm_cc_cme(cc) != NULL);
2161 VM_ASSERT(cme->called_id == mid);
2162 VM_ASSERT(vm_cc_cme(cc)->called_id == mid);
2163
2164 if (original_cc_table != cc_tbl) {
2165 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), cc_tbl);
2166 }
2167
2168 return cc;
2169}
2170
2171static const struct rb_callcache *
2172vm_lookup_cc(const VALUE klass, const struct rb_callinfo * const ci, ID mid)
2173{
2174 VALUE cc_tbl;
2175 struct rb_class_cc_entries *ccs;
2176retry:
2177 cc_tbl = RUBY_ATOMIC_VALUE_LOAD(RCLASS_WRITABLE_CC_TBL(klass));
2178 ccs = NULL;
2179
2180 if (cc_tbl) {
2181 // CCS data is keyed on method id, so we don't need the method id
2182 // for doing comparisons in the `for` loop below.
2183
2184 VALUE ccs_obj;
2185 if (rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj)) {
2186 ccs = (struct rb_class_cc_entries *)ccs_obj;
2187 const int ccs_len = ccs->len;
2188
2189 if (UNLIKELY(METHOD_ENTRY_INVALIDATED(ccs->cme))) {
2190 RB_VM_LOCKING() {
2191 vm_evict_cc(klass, cc_tbl, mid);
2192 }
2193 goto retry;
2194 }
2195 else if (UNDEFINED_METHOD_ENTRY_P(ccs->cme)) {
2196 // vm_populate_cc would return the empty cc for this too
2197 return &rb_vm_empty_cc;
2198 }
2199 else {
2200 VM_ASSERT(vm_ccs_verify(ccs, mid, klass));
2201
2202 // We already know the method id is correct because we had
2203 // to look up the ccs_data by method id. We compare argc and
2204 // flag, plus the keyword argument count: two call sites with
2205 // the same argc and flag can still differ in how many of the
2206 // arguments are keywords (e.g. `m(a: 1, b: 2)` vs `m(1, b: 2)`),
2207 // which selects a different argument setup fastpath.
2208 const struct rb_callinfo_kwarg *kwarg = vm_ci_kwarg(ci);
2209 unsigned int argc = vm_ci_argc(ci);
2210 unsigned int flag = vm_ci_flag(ci);
2211 unsigned int kw_len = kwarg ? kwarg->keyword_len : 0;
2212
2213 for (int i=0; i<ccs_len; i++) {
2214 unsigned int ccs_ci_argc = ccs->entries[i].argc;
2215 unsigned int ccs_ci_flag = ccs->entries[i].flag;
2216 unsigned int ccs_ci_kw_len = ccs->entries[i].kw_len;
2217 const struct rb_callcache *ccs_cc = ccs->entries[i].cc;
2218
2219 VM_ASSERT(IMEMO_TYPE_P(ccs_cc, imemo_callcache));
2220
2221 if (ccs_ci_argc == argc && ccs_ci_flag == flag && ccs_ci_kw_len == kw_len) {
2222 RB_DEBUG_COUNTER_INC(cc_found_in_ccs);
2223
2224 VM_ASSERT(vm_cc_cme(ccs_cc)->called_id == mid);
2225 VM_ASSERT(ccs_cc->klass == klass);
2226 VM_ASSERT(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(ccs_cc)));
2227
2228 return ccs_cc;
2229 }
2230 }
2231 }
2232 }
2233 }
2234
2235 RB_GC_GUARD(cc_tbl);
2236 return NULL;
2237}
2238
2239static const struct rb_callcache *
2240vm_search_cc(const VALUE klass, const struct rb_callinfo * const ci)
2241{
2242 const ID mid = vm_ci_mid(ci);
2243
2244 const struct rb_callcache *cc = vm_lookup_cc(klass, ci, mid);
2245 if (cc) {
2246 return cc;
2247 }
2248
2249 RB_VM_LOCKING() {
2250 if (rb_multi_ractor_p()) {
2251 // The CC may have been populated by another ractor while we were waiting on the lock,
2252 // so we must lookup a second time.
2253 cc = vm_lookup_cc(klass, ci, mid);
2254 }
2255
2256 if (!cc) {
2257 cc = vm_populate_cc(klass, ci, mid);
2258 }
2259 }
2260
2261 return cc;
2262}
2263
2264#if VM_CHECK_MODE > 0
2265static bool
2266vm_cc_resolved_p(const struct rb_callcache *cc)
2267{
2268 return cc != &rb_vm_empty_cc && cc != &rb_vm_uncached_cc;
2269}
2270#endif
2271
2272const struct rb_callcache *
2273rb_vm_search_method_slowpath(const struct rb_callinfo *ci, VALUE klass)
2274{
2275 const struct rb_callcache *cc;
2276
2277 VM_ASSERT(!SPECIAL_CONST_P(klass));
2278
2279 if (RB_BUILTIN_TYPE(klass) == T_NONE) {
2280 // If we find a T_NONE here, it's most likely we called CLASS_OF(obj) on a
2281 // garbage collected object (the freelist is stored in the class pointer),
2282 // but it's possible that just the class was GC'd.
2283 // This message intentionally tries to imply the former, but make an
2284 // accurate statement for either case.
2285 rb_bug("attempted to search method '%s' on a garbage collected object",
2286 rb_id2name(vm_ci_mid(ci)));
2287 }
2288
2289 VM_ASSERT_TYPE2(klass, T_CLASS, T_ICLASS);
2290
2291 cc = vm_search_cc(klass, ci);
2292
2293 VM_ASSERT(cc);
2294 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
2295 VM_ASSERT(!vm_cc_resolved_p(cc) || cc->klass == klass);
2296 VM_ASSERT(!vm_cc_resolved_p(cc) || callable_method_entry_p(vm_cc_cme(cc)));
2297 VM_ASSERT(!vm_cc_resolved_p(cc) || !METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)));
2298 VM_ASSERT(!vm_cc_resolved_p(cc) || vm_cc_cme(cc)->called_id == vm_ci_mid(ci));
2299
2300 return cc;
2301}
2302
2303#if VM_CHECK_MODE > 0
2304static bool
2305vm_cd_owned_by_iseq_p(const struct rb_call_data *cd, const rb_iseq_t *iseq)
2306{
2307 const struct rb_iseq_constant_body *body = ISEQ_BODY(iseq);
2308 return cd >= body->call_data && cd < body->call_data + body->ci_size;
2309}
2310#endif
2311
2312static const struct rb_callcache *
2313vm_search_method_slowpath0(VALUE cd_owner, struct rb_call_data *cd, VALUE klass)
2314{
2315#if USE_DEBUG_COUNTER
2316 const struct rb_callcache *old_cc = cd->cc;
2317 const rb_callable_method_entry_t *const old_cme = old_cc ? old_cc->cme_ : NULL;
2318#endif
2319
2320 const struct rb_callcache *cc = rb_vm_search_method_slowpath(cd->ci, klass);
2321 const rb_callable_method_entry_t *const new_cme = vm_cc_cme(cc);
2322
2323#if OPT_INLINE_METHOD_CACHE
2324 cd->cc = cc;
2325
2326 if (cd_owner && vm_cc_markable(cc)) {
2327 // invokesuper dispatches with a cd on the machine stack, which has no
2328 // owning iseq to remember and keeps its ci out of the GC.
2329 VM_ASSERT(!vm_ci_markable(cd->ci) ||
2330 vm_cd_owned_by_iseq_p(cd, (const rb_iseq_t *)cd_owner));
2331 RB_OBJ_WRITTEN(cd_owner, Qundef, cc);
2332 }
2333
2334#if USE_DEBUG_COUNTER
2335 const struct rb_callcache *empty_cc = &rb_vm_empty_cc;
2336 if (!old_cc || old_cc == empty_cc) {
2337 // empty
2338 RB_DEBUG_COUNTER_INC(mc_inline_miss_empty);
2339 }
2340 else if (old_cc == cc) {
2341 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cc);
2342 }
2343 else if (old_cme == new_cme) {
2344 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cme);
2345 }
2346 else if (old_cme && new_cme && old_cme->def == new_cme->def) {
2347 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_def);
2348 }
2349 else {
2350 RB_DEBUG_COUNTER_INC(mc_inline_miss_diff);
2351 }
2352#endif
2353#endif // OPT_INLINE_METHOD_CACHE
2354
2355 if (new_cme) VM_ASSERT(new_cme->called_id == vm_ci_mid(cd->ci));
2356
2357 return cc;
2358}
2359
2360ALWAYS_INLINE(static bool vm_cc_hit_p(const struct rb_callcache *cc, const struct rb_call_data *cd, VALUE klass));
2361static bool
2362vm_cc_hit_p(const struct rb_callcache *cc, const struct rb_call_data *cd, VALUE klass)
2363{
2364 VM_ASSERT_TYPE2(klass, T_CLASS, T_ICLASS);
2365
2366#if OPT_INLINE_METHOD_CACHE
2367 if (LIKELY(vm_cc_class_check(cc, klass))) {
2368 if (LIKELY(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)))) {
2369 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
2370 RB_DEBUG_COUNTER_INC(mc_inline_hit);
2371 VM_ASSERT(vm_cc_cme(cc) == NULL || // not found
2372 (vm_ci_flag(cd->ci) & VM_CALL_SUPER) || // search_super w/ define_method
2373 vm_cc_cme(cc)->called_id == vm_ci_mid(cd->ci)); // cme->called_id == ci->mid
2374
2375 return true;
2376 }
2377 RB_DEBUG_COUNTER_INC(mc_inline_miss_invalidated);
2378 }
2379 else {
2380 RB_DEBUG_COUNTER_INC(mc_inline_miss_klass);
2381 }
2382#endif
2383
2384 return false;
2385}
2386
2387ALWAYS_INLINE(static const struct rb_callcache *vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE klass));
2388static const struct rb_callcache *
2389vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE klass)
2390{
2391 const struct rb_callcache *cc = cd->cc;
2392 if (vm_cc_hit_p(cc, cd, klass)) {
2393 return cc;
2394 }
2395 if (UNLIKELY(cc == &rb_vm_uncached_cc)) {
2396 return cc;
2397 }
2398
2399 return vm_search_method_slowpath0((VALUE)CFP_ISEQ(reg_cfp), cd, klass);
2400}
2401
2402static const struct rb_callable_method_entry_struct *
2403vm_search_method(struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE recv)
2404{
2405 VALUE klass = CLASS_OF(recv);
2406 VM_ASSERT(klass != Qfalse);
2407 VM_ASSERT(RBASIC_CLASS(klass) == 0 || rb_obj_is_kind_of(klass, rb_cClass));
2408
2409 const struct rb_callcache *cc = vm_search_method_fastpath(reg_cfp, cd, klass);
2410 return vm_cc_cme(cc);
2411}
2412
2413static VALUE
2414vm_call_uncached(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2415{
2416 RB_DEBUG_COUNTER_INC(ccf_uncached);
2417
2418 const struct rb_call_data *cd = calling->cd;
2419 const VALUE klass = CLASS_OF(calling->recv);
2420 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, vm_ci_mid(cd->ci));
2421
2422 if (UNLIKELY(cme == NULL || !cme->def->uncached)) {
2423 // A site that saw an uncached method is now seeing a receiver whose
2424 // method is cacheable, so let it have a real cc after all.
2425 const struct rb_callcache *cc =
2426 vm_search_method_slowpath0((VALUE)CFP_ISEQ(cfp), (struct rb_call_data *)cd, klass);
2427 calling->cc = cc;
2428 return vm_cc_call(cc)(ec, cfp, calling);
2429 }
2430
2431 calling->cc = &VM_CC_ON_STACK(klass, vm_call_general, {{ 0 }},
2432 rb_check_overloaded_cme(cme, cd->ci));
2433 return vm_call_general(ec, cfp, calling);
2434}
2435
2437rb_zjit_vm_search_method(VALUE cd_owner, struct rb_call_data *cd, VALUE recv)
2438{
2439 // Called from ZJIT with the compile-time iseq, which may differ from
2440 // the iseq on the current CFP.
2441 VALUE klass = CLASS_OF(recv);
2442 const struct rb_callcache *cc = cd->cc;
2443 if (!vm_cc_hit_p(cc, cd, klass)) {
2444 if (UNLIKELY(cc == &rb_vm_uncached_cc)) {
2445 return rb_callable_method_entry(klass, vm_ci_mid(cd->ci));
2446 }
2447 cc = vm_search_method_slowpath0(cd_owner, cd, klass);
2448 }
2449 return vm_cc_cme(cc);
2450}
2451
2452#if __has_attribute(transparent_union)
2453typedef union {
2454 VALUE (*anyargs)(ANYARGS);
2455 VALUE (*f00)(VALUE);
2456 VALUE (*f01)(VALUE, VALUE);
2457 VALUE (*f02)(VALUE, VALUE, VALUE);
2458 VALUE (*f03)(VALUE, VALUE, VALUE, VALUE);
2459 VALUE (*f04)(VALUE, VALUE, VALUE, VALUE, VALUE);
2460 VALUE (*f05)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2461 VALUE (*f06)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2462 VALUE (*f07)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2471 VALUE (*fm1)(int, union { VALUE *x; const VALUE *y; } __attribute__((__transparent_union__)), VALUE);
2472} __attribute__((__transparent_union__)) cfunc_type;
2473# define make_cfunc_type(f) (cfunc_type){.anyargs = (VALUE (*)(ANYARGS))(f)}
2474#else
2475typedef VALUE (*cfunc_type)(ANYARGS);
2476# define make_cfunc_type(f) (cfunc_type)(f)
2477#endif
2478
2479static inline bool
2480cfunc_p(const rb_callable_method_entry_t *me)
2481{
2482 if (! me) {
2483 return false;
2484 }
2485
2486 VM_ASSERT(IMEMO_TYPE_P(me, imemo_ment));
2487 VM_ASSERT(callable_method_entry_p(me));
2488 VM_ASSERT(me->def);
2489
2490 return me->def->type == VM_METHOD_TYPE_CFUNC;
2491}
2492
2493static inline bool
2494cfunc_equal(const rb_callable_method_entry_t *me, cfunc_type func)
2495{
2496 VM_ASSERT(cfunc_p(me));
2497
2498#if __has_attribute(transparent_union)
2499 return me->def->body.cfunc.func == func.anyargs;
2500#else
2501 return me->def->body.cfunc.func == func;
2502#endif
2503}
2504
2505static inline bool
2506check_cfunc(const rb_callable_method_entry_t *me, cfunc_type func)
2507{
2508 return cfunc_p(me) && cfunc_equal(me, func);
2509}
2510
2511static inline int
2512check_method_basic_definition(const rb_callable_method_entry_t *me)
2513{
2514 return me && METHOD_ENTRY_BASIC(me);
2515}
2516
2517static inline int
2518vm_method_cfunc_is(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv, cfunc_type func)
2519{
2520 VM_ASSERT(reg_cfp != NULL);
2521 const struct rb_callable_method_entry_struct *cme = vm_search_method(reg_cfp, cd, recv);
2522 return check_cfunc(cme, func);
2523}
2524
2525bool
2526rb_zjit_cme_is_cfunc(const rb_callable_method_entry_t *me, const cfunc_type func)
2527{
2528 return check_cfunc(me, func);
2529}
2530
2531int
2532rb_vm_method_cfunc_is(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv, cfunc_type func)
2533{
2534 const struct rb_callable_method_entry_struct *cme = rb_zjit_vm_search_method((VALUE)iseq, cd, recv);
2535 return check_cfunc(cme, func);
2536}
2537
2538#define check_cfunc(me, func) check_cfunc(me, make_cfunc_type(func))
2539#define vm_method_cfunc_is(reg_cfp, cd, recv, func) vm_method_cfunc_is(reg_cfp, cd, recv, make_cfunc_type(func))
2540
2541#define OP_UNREDEFINED_P(op, t) BASIC_OP_UNREDEFINED_P(BOP_##op, t##_REDEFINED_OP_FLAG)
2542#define EQ_UNREDEFINED_P(t) OP_UNREDEFINED_P(EQ, t)
2543
2544static inline bool
2545FIXNUM_2_P(VALUE a, VALUE b)
2546{
2547 /* FIXNUM_P(a) && FIXNUM_P(b)
2548 * == ((a & 1) && (b & 1))
2549 * == a & b & 1 */
2550 SIGNED_VALUE x = a;
2551 SIGNED_VALUE y = b;
2552 SIGNED_VALUE z = x & y & 1;
2553 return z == 1;
2554}
2555
2556static inline bool
2557FLONUM_2_P(VALUE a, VALUE b)
2558{
2559#if USE_FLONUM
2560 /* FLONUM_P(a) && FLONUM_P(b)
2561 * == ((a & 3) == 2) && ((b & 3) == 2)
2562 * == ! ((a ^ 2) | (b ^ 2) & 3)
2563 */
2564 SIGNED_VALUE x = a;
2565 SIGNED_VALUE y = b;
2566 SIGNED_VALUE z = ((x ^ 2) | (y ^ 2)) & 3;
2567 return !z;
2568#else
2569 return false;
2570#endif
2571}
2572
2573static inline VALUE
2574opt_equality_specialized(VALUE recv, VALUE obj)
2575{
2576 if (recv == obj) {
2577 if (OP_UNREDEFINED_P(EQ, ANY)) {
2578 // Two identical immediate VALUE are always equal
2579 if (SPECIAL_CONST_P(recv)) {
2580 return Qtrue;
2581 }
2582 return Qundef;
2583 }
2584 return Qundef;
2585 }
2586
2587 return opt_equality_specialized_miss(recv, obj);
2588}
2589
2590static VALUE
2591opt_equality(struct rb_control_frame_struct *reg_cfp, VALUE recv, VALUE obj, CALL_DATA cd)
2592{
2593 VM_ASSERT(reg_cfp != NULL);
2594
2595 VALUE val = opt_equality_specialized(recv, obj);
2596 if (!UNDEF_P(val)) return val;
2597
2598 if (!vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_equal)) {
2599 return Qundef;
2600 }
2601 else {
2602 return RBOOL(recv == obj);
2603 }
2604}
2605
2606static inline const struct rb_callcache *gccct_method_search(rb_execution_context_t *ec, VALUE recv, ID mid, const struct rb_callinfo *ci); // vm_eval.c
2607
2608static inline VALUE
2609opt_equality_by_mid(VALUE recv, VALUE obj, ID mid)
2610{
2611 const struct rb_callcache *cc = gccct_method_search(GET_EC(), recv, mid, &VM_CI_ON_STACK(mid, 0, 1, NULL));
2612
2613 if (cc && check_cfunc(vm_cc_cme(cc), rb_obj_equal)) {
2614 return RBOOL(recv == obj);
2615 }
2616 else {
2617 return RBOOL(RTEST(rb_funcall(recv, mid, 1, obj)));
2618 }
2619}
2620
2621VALUE
2622rb_equal_opt(VALUE obj1, VALUE obj2)
2623{
2624 return opt_equality_by_mid(obj1, obj2, idEq);
2625}
2626
2627VALUE
2628rb_eql_opt(VALUE obj1, VALUE obj2)
2629{
2630 return opt_equality_by_mid(obj1, obj2, idEqlP);
2631}
2632
2633extern VALUE rb_vm_call0(rb_execution_context_t *ec, VALUE, ID, int, const VALUE*, const rb_callable_method_entry_t *, int kw_splat);
2634extern VALUE rb_vm_call_with_refinements(rb_execution_context_t *, VALUE, ID, int, const VALUE *, int);
2635
2636static VALUE
2637check_match(rb_execution_context_t *ec, VALUE pattern, VALUE target, enum vm_check_match_type type)
2638{
2639 switch (type) {
2640 case VM_CHECKMATCH_TYPE_WHEN:
2641 return pattern;
2642 case VM_CHECKMATCH_TYPE_RESCUE:
2643 if (!rb_obj_is_kind_of(pattern, rb_cModule)) {
2644 rb_raise(rb_eTypeError, "class or module required for rescue clause");
2645 }
2646 /* fall through */
2647 case VM_CHECKMATCH_TYPE_CASE: {
2648 return rb_vm_call_with_refinements(ec, pattern, idEqq, 1, &target, RB_NO_KEYWORDS);
2649 }
2650 default:
2651 rb_bug("check_match: unreachable");
2652 }
2653}
2654
2655
2656static inline VALUE
2657double_cmp_lt(double a, double b)
2658{
2659 return RBOOL(a < b);
2660}
2661
2662static inline VALUE
2663double_cmp_le(double a, double b)
2664{
2665 return RBOOL(a <= b);
2666}
2667
2668static inline VALUE
2669double_cmp_gt(double a, double b)
2670{
2671 return RBOOL(a > b);
2672}
2673
2674static inline VALUE
2675double_cmp_ge(double a, double b)
2676{
2677 return RBOOL(a >= b);
2678}
2679
2680// Copied by vm_dump.c
2681static inline VALUE *
2682vm_base_ptr(const rb_control_frame_t *cfp)
2683{
2684 const rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
2685
2686 if (CFP_ISEQ(cfp) && VM_FRAME_RUBYFRAME_P(cfp)) {
2687 VALUE *bp = prev_cfp->sp + ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size + VM_ENV_DATA_SIZE;
2688
2689 if (ISEQ_BODY(CFP_ISEQ(cfp))->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) {
2690 int lts = ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size;
2691 int params = ISEQ_BODY(CFP_ISEQ(cfp))->param.size;
2692
2693 CALL_INFO ci = (CALL_INFO)cfp->ep[-(VM_ENV_DATA_SIZE + (lts - params))]; // skip EP stuff, CI should be last local
2694 bp += vm_ci_argc(ci);
2695 }
2696
2697 if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) {
2698 /* adjust `self' */
2699 bp += 1;
2700 }
2701#if VM_DEBUG_BP_CHECK
2702 if (bp != cfp->bp_check) {
2703 ruby_debug_printf("bp_check: %ld, bp: %ld\n",
2704 (long)(cfp->bp_check - GET_EC()->vm_stack),
2705 (long)(bp - GET_EC()->vm_stack));
2706 rb_bug("vm_base_ptr: unreachable");
2707 }
2708#endif
2709 return bp;
2710 }
2711 else {
2712 return NULL;
2713 }
2714}
2715
2716VALUE *
2717rb_vm_base_ptr(const rb_control_frame_t *cfp)
2718{
2719 return vm_base_ptr(cfp);
2720}
2721
2722/* method call processes with call_info */
2723
2724#include "vm_args.c"
2725
2726static inline VALUE vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc, unsigned int param_size, unsigned int local_size);
2727ALWAYS_INLINE(static VALUE vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me, int opt_pc, int param_size, int local_size));
2728static inline VALUE vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc, unsigned int param_size, unsigned int local_size);
2729static VALUE vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
2730static VALUE vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2731static VALUE vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2732static inline VALUE vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2733
2734static vm_call_handler vm_call_iseq_setup_func(const struct rb_callinfo *ci, const int param_size, const int local_size);
2735
2736static VALUE
2737vm_call_iseq_setup_tailcall_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2738{
2739 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_tailcall_0start);
2740
2741 const struct rb_callcache *cc = calling->cc;
2742 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2743 unsigned int param = ISEQ_BODY(iseq)->param.size;
2744 unsigned int local = ISEQ_BODY(iseq)->local_table_size;
2745 return vm_call_iseq_setup_tailcall(ec, cfp, calling, 0, param, local);
2746}
2747
2748static VALUE
2749vm_call_iseq_setup_normal_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2750{
2751 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_0start);
2752
2753 const struct rb_callcache *cc = calling->cc;
2754 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2755 int param = ISEQ_BODY(iseq)->param.size;
2756 int local = ISEQ_BODY(iseq)->local_table_size;
2757 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
2758}
2759
2760bool
2761rb_simple_iseq_p(const rb_iseq_t *iseq)
2762{
2763 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2764 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2765 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2766 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2767 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2768 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2769 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2770 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2771 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2772}
2773
2774bool
2775rb_iseq_only_optparam_p(const rb_iseq_t *iseq)
2776{
2777 return ISEQ_BODY(iseq)->param.flags.has_opt == TRUE &&
2778 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2779 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2780 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2781 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2782 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2783 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2784 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2785 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2786}
2787
2788bool
2789rb_iseq_only_kwparam_p(const rb_iseq_t *iseq)
2790{
2791 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2792 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2793 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2794 ISEQ_BODY(iseq)->param.flags.has_kw == TRUE &&
2795 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2796 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2797 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2798 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2799}
2800
2801#define ALLOW_HEAP_ARGV (-2)
2802#define ALLOW_HEAP_ARGV_KEEP_KWSPLAT (-3)
2803
2804static inline bool
2805vm_caller_setup_arg_splat(rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE ary, int max_args)
2806{
2807 vm_check_canary(GET_EC(), cfp->sp);
2808 bool ret = false;
2809
2810 if (!NIL_P(ary)) {
2811 const VALUE *ptr = RARRAY_CONST_PTR(ary);
2812 rb_len_t len = RARRAY_LEN(ary);
2813 int argc = calling->argc;
2814
2815 if (UNLIKELY(max_args <= ALLOW_HEAP_ARGV && len + argc > VM_ARGC_STACK_MAX)) {
2816 /* Avoid SystemStackError when splatting large arrays by storing arguments in
2817 * a temporary array, instead of trying to keeping arguments on the VM stack.
2818 */
2819 VALUE *argv = cfp->sp - argc;
2820 VALUE argv_ary = rb_ary_hidden_new(len + argc + 1);
2821 rb_ary_cat(argv_ary, argv, argc);
2822 rb_ary_cat(argv_ary, ptr, len);
2823 cfp->sp -= argc - 1;
2824 cfp->sp[-1] = argv_ary;
2825 calling->argc = 1;
2826 calling->heap_argv = argv_ary;
2827 RB_GC_GUARD(ary);
2828 }
2829 else {
2830 long i;
2831
2832 if (max_args >= 0 && len + argc > max_args) {
2833 /* If only a given max_args is allowed, copy up to max args.
2834 * Used by vm_callee_setup_block_arg for non-lambda blocks,
2835 * where additional arguments are ignored.
2836 *
2837 * Also, copy up to one more argument than the maximum,
2838 * in case it is an empty keyword hash that will be removed.
2839 */
2840 calling->argc += len - (max_args - argc + 1);
2841 len = max_args - argc + 1;
2842 ret = true;
2843 }
2844 else {
2845 /* Unset heap_argv if set originally. Can happen when
2846 * forwarding modified arguments, where heap_argv was used
2847 * originally, but heap_argv not supported by the forwarded
2848 * method in all cases.
2849 */
2850 calling->heap_argv = 0;
2851 }
2852 CHECK_VM_STACK_OVERFLOW(cfp, len);
2853
2854 for (i = 0; i < len; i++) {
2855 *cfp->sp++ = ptr[i];
2856 }
2857 calling->argc += i;
2858 }
2859 }
2860
2861 return ret;
2862}
2863
2864static inline void
2865vm_caller_setup_arg_kw(rb_control_frame_t *cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci)
2866{
2867 const VALUE *const passed_keywords = vm_ci_kwarg(ci)->keywords;
2868 const int kw_len = vm_ci_kwarg(ci)->keyword_len;
2869 const VALUE h = rb_hash_new_capa(kw_len);
2870 VALUE *sp = cfp->sp;
2871 int i;
2872
2873 for (i=0; i<kw_len; i++) {
2874 rb_hash_aset(h, passed_keywords[i], (sp - kw_len)[i]);
2875 }
2876 (sp-kw_len)[0] = h;
2877
2878 cfp->sp -= kw_len - 1;
2879 calling->argc -= kw_len - 1;
2880 calling->kw_splat = 1;
2881}
2882
2883static inline VALUE
2884vm_caller_setup_keyword_hash(const struct rb_callinfo *ci, VALUE keyword_hash)
2885{
2886 if (UNLIKELY(!RB_TYPE_P(keyword_hash, T_HASH))) {
2887 if (keyword_hash != Qnil) {
2888 /* Convert a non-hash keyword splat to a new hash */
2889 keyword_hash = rb_hash_dup(rb_to_hash_type(keyword_hash));
2890 }
2891 }
2892 else if (!IS_ARGS_KW_SPLAT_MUT(ci) && !RHASH_EMPTY_P(keyword_hash)) {
2893 /* Convert a hash keyword splat to a new hash unless
2894 * a mutable keyword splat was passed.
2895 * Skip allocating new hash for empty keyword splat, as empty
2896 * keyword splat will be ignored by both callers.
2897 */
2898 keyword_hash = rb_hash_dup(keyword_hash);
2899 }
2900 return keyword_hash;
2901}
2902
2903static inline void
2904CALLER_SETUP_ARG(struct rb_control_frame_struct *restrict cfp,
2905 struct rb_calling_info *restrict calling,
2906 const struct rb_callinfo *restrict ci, int max_args)
2907{
2908 if (UNLIKELY(IS_ARGS_SPLAT(ci))) {
2909 if (IS_ARGS_KW_SPLAT(ci)) {
2910 // f(*a, **kw)
2911 VM_ASSERT(calling->kw_splat == 1);
2912
2913 cfp->sp -= 2;
2914 calling->argc -= 2;
2915 VALUE ary = cfp->sp[0];
2916 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[1]);
2917
2918 // splat a
2919 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) return;
2920
2921 // put kw
2922 if (kwh != Qnil && !RHASH_EMPTY_P(kwh)) {
2923 if (UNLIKELY(calling->heap_argv)) {
2924 rb_ary_push(calling->heap_argv, kwh);
2925 ((struct RHash *)kwh)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
2926 if (max_args != ALLOW_HEAP_ARGV_KEEP_KWSPLAT) {
2927 calling->kw_splat = 0;
2928 }
2929 }
2930 else {
2931 cfp->sp[0] = kwh;
2932 cfp->sp++;
2933 calling->argc++;
2934
2935 VM_ASSERT(calling->kw_splat == 1);
2936 }
2937 }
2938 else {
2939 calling->kw_splat = 0;
2940 }
2941 }
2942 else {
2943 // f(*a)
2944 VM_ASSERT(calling->kw_splat == 0);
2945
2946 cfp->sp -= 1;
2947 calling->argc -= 1;
2948 VALUE ary = cfp->sp[0];
2949
2950 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) {
2951 goto check_keyword;
2952 }
2953
2954 // check the last argument
2955 VALUE last_hash, argv_ary;
2956 if (UNLIKELY(argv_ary = calling->heap_argv)) {
2957 if (!IS_ARGS_KEYWORD(ci) &&
2958 RARRAY_LEN(argv_ary) > 0 &&
2959 RB_TYPE_P((last_hash = rb_ary_last(0, NULL, argv_ary)), T_HASH) &&
2960 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2961
2962 rb_ary_pop(argv_ary);
2963 if (!RHASH_EMPTY_P(last_hash)) {
2964 rb_ary_push(argv_ary, rb_hash_dup(last_hash));
2965 calling->kw_splat = 1;
2966 }
2967 }
2968 }
2969 else {
2970check_keyword:
2971 if (!IS_ARGS_KEYWORD(ci) &&
2972 calling->argc > 0 &&
2973 RB_TYPE_P((last_hash = cfp->sp[-1]), T_HASH) &&
2974 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2975
2976 if (RHASH_EMPTY_P(last_hash)) {
2977 calling->argc--;
2978 cfp->sp -= 1;
2979 }
2980 else {
2981 cfp->sp[-1] = rb_hash_dup(last_hash);
2982 calling->kw_splat = 1;
2983 }
2984 }
2985 }
2986 }
2987 }
2988 else if (UNLIKELY(IS_ARGS_KW_SPLAT(ci))) {
2989 // f(**kw)
2990 VM_ASSERT(calling->kw_splat == 1);
2991 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[-1]);
2992
2993 if (kwh == Qnil || RHASH_EMPTY_P(kwh)) {
2994 cfp->sp--;
2995 calling->argc--;
2996 calling->kw_splat = 0;
2997 }
2998 else {
2999 cfp->sp[-1] = kwh;
3000 }
3001 }
3002 else if (UNLIKELY(IS_ARGS_KEYWORD(ci))) {
3003 // f(k1:1, k2:2)
3004 VM_ASSERT(calling->kw_splat == 0);
3005
3006 /* This converts VM_CALL_KWARG style to VM_CALL_KW_SPLAT style
3007 * by creating a keyword hash.
3008 * So, vm_ci_flag(ci) & VM_CALL_KWARG is now inconsistent.
3009 */
3010 vm_caller_setup_arg_kw(cfp, calling, ci);
3011 }
3012}
3013
3014#define USE_OPT_HIST 0
3015
3016#if USE_OPT_HIST
3017#define OPT_HIST_MAX 64
3018static int opt_hist[OPT_HIST_MAX+1];
3019
3020__attribute__((destructor))
3021static void
3022opt_hist_show_results_at_exit(void)
3023{
3024 for (int i=0; i<OPT_HIST_MAX; i++) {
3025 ruby_debug_printf("opt_hist\t%d\t%d\n", i, opt_hist[i]);
3026 }
3027}
3028#endif
3029
3030static VALUE
3031vm_call_iseq_setup_normal_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3032 struct rb_calling_info *calling)
3033{
3034 const struct rb_callcache *cc = calling->cc;
3035 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3036 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3037 const int opt = calling->argc - lead_num;
3038 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
3039 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3040 const int param = ISEQ_BODY(iseq)->param.size;
3041 const int local = ISEQ_BODY(iseq)->local_table_size;
3042 const int delta = opt_num - opt;
3043
3044 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
3045
3046#if USE_OPT_HIST
3047 if (opt_pc < OPT_HIST_MAX) {
3048 opt_hist[opt]++;
3049 }
3050 else {
3051 opt_hist[OPT_HIST_MAX]++;
3052 }
3053#endif
3054
3055 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param - delta, local);
3056}
3057
3058static VALUE
3059vm_call_iseq_setup_tailcall_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3060 struct rb_calling_info *calling)
3061{
3062 const struct rb_callcache *cc = calling->cc;
3063 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3064 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3065 const int opt = calling->argc - lead_num;
3066 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3067 unsigned const int param = ISEQ_BODY(iseq)->param.size;
3068 unsigned const int local = ISEQ_BODY(iseq)->local_table_size;
3069
3070 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
3071
3072#if USE_OPT_HIST
3073 if (opt_pc < OPT_HIST_MAX) {
3074 opt_hist[opt]++;
3075 }
3076 else {
3077 opt_hist[OPT_HIST_MAX]++;
3078 }
3079#endif
3080
3081 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc, param, local);
3082}
3083
3084static void
3085args_setup_kw_parameters(rb_execution_context_t *const ec, const rb_iseq_t *const iseq, const rb_callable_method_entry_t *cme,
3086 VALUE *const passed_values, const int passed_keyword_len, const VALUE *const passed_keywords,
3087 VALUE *const locals);
3088
3089static VALUE
3090vm_call_iseq_forwardable(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3091 struct rb_calling_info *calling)
3092{
3093 const struct rb_callcache *cc = calling->cc;
3094 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3095 int param_size = ISEQ_BODY(iseq)->param.size;
3096 int local_size = ISEQ_BODY(iseq)->local_table_size;
3097
3098 // Setting up local size and param size
3099 VM_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
3100
3101 local_size = local_size + vm_ci_argc(calling->cd->ci);
3102 param_size = param_size + vm_ci_argc(calling->cd->ci);
3103
3104 cfp->sp[0] = (VALUE)calling->cd->ci;
3105
3106 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param_size, local_size);
3107}
3108
3109static VALUE
3110vm_call_iseq_setup_kwparm_kwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3111 struct rb_calling_info *calling)
3112{
3113 const struct rb_callinfo *ci = calling->cd->ci;
3114 const struct rb_callcache *cc = calling->cc;
3115
3116 VM_ASSERT(vm_ci_flag(ci) & VM_CALL_KWARG);
3117 RB_DEBUG_COUNTER_INC(ccf_iseq_kw1);
3118
3119 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3120 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3121 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
3122 const int ci_kw_len = kw_arg->keyword_len;
3123 const VALUE * const ci_keywords = kw_arg->keywords;
3124 VALUE *argv = cfp->sp - calling->argc;
3125 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3126 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3127 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
3128 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
3129 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
3130
3131 int param = ISEQ_BODY(iseq)->param.size;
3132 int local = ISEQ_BODY(iseq)->local_table_size;
3133 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
3134}
3135
3136static VALUE
3137vm_call_iseq_setup_kwparm_nokwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3138 struct rb_calling_info *calling)
3139{
3140 const struct rb_callinfo *MAYBE_UNUSED(ci) = calling->cd->ci;
3141 const struct rb_callcache *cc = calling->cc;
3142
3143 VM_ASSERT((vm_ci_flag(ci) & VM_CALL_KWARG) == 0);
3144 RB_DEBUG_COUNTER_INC(ccf_iseq_kw2);
3145
3146 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3147 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3148 VALUE * const argv = cfp->sp - calling->argc;
3149 VALUE * const klocals = argv + kw_param->bits_start - kw_param->num;
3150
3151 int i;
3152 for (i=0; i<kw_param->num; i++) {
3153 klocals[i] = kw_param->default_values[i];
3154 }
3155 klocals[i] = INT2FIX(0); // kw specify flag
3156 // NOTE:
3157 // nobody check this value, but it should be cleared because it can
3158 // points invalid VALUE (T_NONE objects, raw pointer and so on).
3159
3160 int param = ISEQ_BODY(iseq)->param.size;
3161 int local = ISEQ_BODY(iseq)->local_table_size;
3162 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
3163}
3164
3165static VALUE builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
3166
3167static VALUE
3168vm_call_single_noarg_leaf_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3169 struct rb_calling_info *calling)
3170{
3171 const struct rb_callcache *cc = calling->cc;
3172 // Pairs with the release store in vm_callee_setup_arg so aux_.bf is visible.
3173 vm_call_handler call = (vm_call_handler)(uintptr_t)rbimpl_atomic_ptr_load((void **)&cc->call_, RBIMPL_ATOMIC_ACQUIRE);
3174 if (UNLIKELY(call != vm_call_single_noarg_leaf_builtin)) {
3175 return call(ec, cfp, calling);
3176 }
3177 const struct rb_builtin_function *bf = cc->aux_.bf;
3178 cfp->sp -= (calling->argc + 1);
3179 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
3180 return builtin_invoker0(ec, calling->recv, NULL, func_ptr);
3181}
3182
3183VALUE rb_gen_method_name(VALUE owner, VALUE name); // in vm_backtrace.c
3184
3185static void
3186warn_unused_block(const rb_callable_method_entry_t *cme, const rb_iseq_t *iseq, void *pc)
3187{
3188 rb_vm_t *vm = GET_VM();
3189 set_table *dup_check_table = &vm->unused_block_warning_table;
3190 st_data_t key;
3191 bool strict_unused_block = rb_warning_category_enabled_p(RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK);
3192
3193 union {
3194 VALUE v;
3195 unsigned char b[SIZEOF_VALUE];
3196 } k1 = {
3197 .v = (VALUE)pc,
3198 }, k2 = {
3199 .v = (VALUE)cme->def,
3200 };
3201
3202 // relax check
3203 if (!strict_unused_block) {
3204 key = (st_data_t)cme->def->original_id;
3205
3206 if (set_table_lookup(dup_check_table, key)) {
3207 return;
3208 }
3209 }
3210
3211 // strict check
3212 // make unique key from pc and me->def pointer
3213 key = 0;
3214 for (int i=0; i<SIZEOF_VALUE; i++) {
3215 // fprintf(stderr, "k1:%3d k2:%3d\n", k1.b[i], k2.b[SIZEOF_VALUE-1-i]);
3216 key |= (st_data_t)(k1.b[i] ^ k2.b[SIZEOF_VALUE-1-i]) << (8 * i);
3217 }
3218
3219 if (0) {
3220 fprintf(stderr, "SIZEOF_VALUE:%d\n", SIZEOF_VALUE);
3221 fprintf(stderr, "pc:%p def:%p\n", pc, (void *)cme->def);
3222 fprintf(stderr, "key:%p\n", (void *)key);
3223 }
3224
3225 // duplication check
3226 if (set_insert(dup_check_table, key)) {
3227 // already shown
3228 }
3229 else if (RTEST(ruby_verbose) || strict_unused_block) {
3230 VALUE m_loc = rb_method_entry_location((const rb_method_entry_t *)cme);
3231 VALUE name = rb_gen_method_name(cme->defined_class, rb_iseq_base_label(iseq));
3232
3233 if (!NIL_P(m_loc)) {
3234 rb_warn("the block passed to '%"PRIsVALUE"' defined at %"PRIsVALUE":%"PRIsVALUE" may be ignored",
3235 name, RARRAY_AREF(m_loc, 0), RARRAY_AREF(m_loc, 1));
3236 }
3237 else {
3238 rb_warn("the block may be ignored because '%"PRIsVALUE"' does not use a block", name);
3239 }
3240 }
3241}
3242
3243static inline int
3244vm_callee_setup_arg(rb_execution_context_t *ec, struct rb_calling_info *calling,
3245 const rb_iseq_t *iseq, VALUE *argv, int param_size, int local_size)
3246{
3247 const struct rb_callinfo *ci = calling->cd->ci;
3248 const struct rb_callcache *cc = calling->cc;
3249
3250 VM_ASSERT((vm_ci_argc(ci), 1));
3251 VM_ASSERT(vm_cc_cme(cc) != NULL);
3252
3253 if (UNLIKELY(!ISEQ_BODY(iseq)->param.flags.use_block &&
3254 calling->block_handler != VM_BLOCK_HANDLER_NONE &&
3255 !(vm_ci_flag(calling->cd->ci) & (VM_CALL_OPT_SEND | VM_CALL_SUPER)))) {
3256 warn_unused_block(vm_cc_cme(cc), iseq, (void *)CFP_PC(ec->cfp));
3257 }
3258
3259 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_KW_SPLAT))) {
3260 if (LIKELY(rb_simple_iseq_p(iseq))) {
3261 rb_control_frame_t *cfp = ec->cfp;
3262 int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3263 CALLER_SETUP_ARG(cfp, calling, ci, lead_num);
3264
3265 if (calling->argc != lead_num) {
3266 argument_arity_error(ec, iseq, vm_cc_cme(cc), calling->argc, lead_num, lead_num);
3267 }
3268
3269 //VM_ASSERT(ci == calling->cd->ci);
3270 VM_ASSERT(cc == calling->cc);
3271
3272 if (vm_call_iseq_optimizable_p(ci, cc)) {
3273 if ((ISEQ_BODY(iseq)->builtin_attrs & BUILTIN_ATTR_SINGLE_NOARG_LEAF) && ruby_vm_c_events_enabled == 0) {
3274 VM_ASSERT(ISEQ_BODY(iseq)->builtin_attrs & BUILTIN_ATTR_LEAF);
3275 vm_cc_bf_set(cc, (void *)ISEQ_BODY(iseq)->iseq_encoded[1]);
3276 rbimpl_atomic_ptr_store((volatile void **)&cc->call_,
3277 (void *)(uintptr_t)vm_call_single_noarg_leaf_builtin,
3278 RBIMPL_ATOMIC_RELEASE);
3279 }
3280 else {
3281 CC_SET_FASTPATH(cc, vm_call_iseq_setup_func(ci, param_size, local_size), true);
3282 }
3283 }
3284 return 0;
3285 }
3286 else if (rb_iseq_only_optparam_p(iseq)) {
3287 rb_control_frame_t *cfp = ec->cfp;
3288
3289 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3290 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
3291
3292 CALLER_SETUP_ARG(cfp, calling, ci, lead_num + opt_num);
3293 const int argc = calling->argc;
3294 const int opt = argc - lead_num;
3295
3296 if (opt < 0 || opt > opt_num) {
3297 argument_arity_error(ec, iseq, vm_cc_cme(cc), argc, lead_num, lead_num + opt_num);
3298 }
3299
3300 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3301 CC_SET_FASTPATH(cc, vm_call_iseq_setup_normal_opt_start,
3302 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3303 vm_call_cacheable(ci, cc));
3304 }
3305 else {
3306 CC_SET_FASTPATH(cc, vm_call_iseq_setup_tailcall_opt_start,
3307 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3308 vm_call_cacheable(ci, cc));
3309 }
3310
3311 /* initialize opt vars for self-references */
3312 VM_ASSERT((int)ISEQ_BODY(iseq)->param.size == lead_num + opt_num);
3313 for (int i=argc; i<lead_num + opt_num; i++) {
3314 argv[i] = Qnil;
3315 }
3316 return (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3317 }
3318 else if (rb_iseq_only_kwparam_p(iseq) && !IS_ARGS_SPLAT(ci)) {
3319 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3320 const int argc = calling->argc;
3321 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3322
3323 if (vm_ci_flag(ci) & VM_CALL_KWARG) {
3324 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
3325
3326 if (argc - kw_arg->keyword_len == lead_num) {
3327 const int ci_kw_len = kw_arg->keyword_len;
3328 const VALUE * const ci_keywords = kw_arg->keywords;
3329 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
3330 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
3331
3332 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3333 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
3334
3335 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_kwarg,
3336 vm_call_cacheable(ci, cc));
3337
3338 return 0;
3339 }
3340 }
3341 else if (argc == lead_num) {
3342 /* no kwarg */
3343 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3344 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), NULL, 0, NULL, klocals);
3345
3346 if (klocals[kw_param->num] == INT2FIX(0)) {
3347 /* copy from default_values */
3348 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_nokwarg,
3349 vm_call_cacheable(ci, cc));
3350 }
3351
3352 return 0;
3353 }
3354 }
3355 }
3356
3357 // Called iseq is using ... param
3358 // def foo(...) # <- iseq for foo will have "forwardable"
3359 //
3360 // We want to set the `...` local to the caller's CI
3361 // foo(1, 2) # <- the ci for this should end up as `...`
3362 //
3363 // So hopefully the stack looks like:
3364 //
3365 // => 1
3366 // => 2
3367 // => *
3368 // => **
3369 // => &
3370 // => ... # <- points at `foo`s CI
3371 // => cref_or_me
3372 // => specval
3373 // => type
3374 //
3375 if (ISEQ_BODY(iseq)->param.flags.forwardable) {
3376 bool can_fastpath = true;
3377
3378 if ((vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
3379 struct rb_forwarding_call_data * forward_cd = (struct rb_forwarding_call_data *)calling->cd;
3380 if (vm_ci_argc(ci) != vm_ci_argc(forward_cd->caller_ci)) {
3381 ci = vm_ci_new_runtime(
3382 vm_ci_mid(ci),
3383 vm_ci_flag(ci),
3384 vm_ci_argc(ci),
3385 vm_ci_kwarg(ci));
3386 }
3387 else {
3388 ci = forward_cd->caller_ci;
3389 }
3390 can_fastpath = false;
3391 }
3392 // C functions calling iseqs will stack allocate a CI,
3393 // so we need to convert it to heap allocated
3394 if (!vm_ci_markable(ci)) {
3395 ci = vm_ci_new_runtime(
3396 vm_ci_mid(ci),
3397 vm_ci_flag(ci),
3398 vm_ci_argc(ci),
3399 vm_ci_kwarg(ci));
3400 can_fastpath = false;
3401 }
3402 argv[param_size - 1] = (VALUE)ci;
3403 CC_SET_FASTPATH(cc, vm_call_iseq_forwardable, can_fastpath);
3404 return 0;
3405 }
3406
3407 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_method);
3408}
3409
3410static void
3411vm_adjust_stack_forwarding(const struct rb_execution_context_struct *ec, struct rb_control_frame_struct *cfp, int argc, VALUE splat)
3412{
3413 // This case is when the caller is using a ... parameter.
3414 // For example `bar(...)`. The call info will have VM_CALL_FORWARDING
3415 // In this case the caller's caller's CI will be on the stack.
3416 //
3417 // For example:
3418 //
3419 // def bar(a, b); a + b; end
3420 // def foo(...); bar(...); end
3421 // foo(1, 2) # <- this CI will be on the stack when we call `bar(...)`
3422 //
3423 // Stack layout will be:
3424 //
3425 // > 1
3426 // > 2
3427 // > CI for foo(1, 2)
3428 // > cref_or_me
3429 // > specval
3430 // > type
3431 // > receiver
3432 // > CI for foo(1, 2), via `getlocal ...`
3433 // > ( SP points here )
3434 const VALUE * lep = VM_CF_LEP(cfp);
3435
3436 const rb_iseq_t *iseq;
3437
3438 // If we're in an escaped environment (lambda for example), get the iseq
3439 // from the captured env.
3440 if (VM_ENV_FLAGS(lep, VM_ENV_FLAG_ESCAPED)) {
3441 rb_env_t * env = (rb_env_t *)lep[VM_ENV_DATA_INDEX_ENV];
3442 iseq = env->iseq;
3443 }
3444 else { // Otherwise use the lep to find the caller
3445 iseq = CFP_ISEQ(rb_vm_search_cf_from_ep(ec, cfp, lep));
3446 }
3447
3448 // Our local storage is below the args we need to copy
3449 int local_size = ISEQ_BODY(iseq)->local_table_size + argc;
3450
3451 const VALUE * from = lep - (local_size + VM_ENV_DATA_SIZE - 1); // 2 for EP values
3452 VALUE * to = cfp->sp - 1; // clobber the CI
3453
3454 if (RTEST(splat)) {
3455 to -= 1; // clobber the splat array
3456 CHECK_VM_STACK_OVERFLOW0(cfp, to, RARRAY_LEN(splat));
3457 MEMCPY(to, RARRAY_CONST_PTR(splat), VALUE, RARRAY_LEN(splat));
3458 to += RARRAY_LEN(splat);
3459 }
3460
3461 CHECK_VM_STACK_OVERFLOW0(cfp, to, argc);
3462 MEMCPY(to, from, VALUE, argc);
3463 cfp->sp = to + argc;
3464
3465 // Stack layout should now be:
3466 //
3467 // > 1
3468 // > 2
3469 // > CI for foo(1, 2)
3470 // > cref_or_me
3471 // > specval
3472 // > type
3473 // > receiver
3474 // > 1
3475 // > 2
3476 // > ( SP points here )
3477}
3478
3479static VALUE
3480vm_call_iseq_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3481{
3482 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3483
3484 const struct rb_callcache *cc = calling->cc;
3485 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3486 unsigned int param_size = ISEQ_BODY(iseq)->param.size;
3487 unsigned int local_size = ISEQ_BODY(iseq)->local_table_size;
3488
3489 RUBY_ASSERT(!ISEQ_BODY(iseq)->param.flags.forwardable);
3490
3491 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3492 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3493}
3494
3495static VALUE
3496vm_call_iseq_fwd_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3497{
3498 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3499
3500 const struct rb_callcache *cc = calling->cc;
3501 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3502 unsigned int param_size = ISEQ_BODY(iseq)->param.size;
3503 unsigned int local_size = ISEQ_BODY(iseq)->local_table_size;
3504
3505 RUBY_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
3506
3507 // Setting up local size and param size
3508 local_size = local_size + vm_ci_argc(calling->cd->ci);
3509 param_size = param_size + vm_ci_argc(calling->cd->ci);
3510
3511 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3512 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3513}
3514
3515static inline VALUE
3516vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
3517 int opt_pc, unsigned int param_size, unsigned int local_size)
3518{
3519 const struct rb_callinfo *ci = calling->cd->ci;
3520 const struct rb_callcache *cc = calling->cc;
3521
3522 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3523 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param_size, local_size);
3524 }
3525 else {
3526 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc, param_size, local_size);
3527 }
3528}
3529
3530static inline VALUE
3531vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me,
3532 int opt_pc, int param_size, int local_size)
3533{
3534 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3535 VALUE *argv = cfp->sp - calling->argc;
3536 VALUE *sp = argv + param_size;
3537 cfp->sp = argv - 1 /* recv */;
3538
3539 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL, calling->recv,
3540 calling->block_handler, (VALUE)me,
3541 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3542 local_size - param_size,
3543 ISEQ_BODY(iseq)->stack_max);
3544 return Qundef;
3545}
3546
3547static inline VALUE
3548vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
3549 int opt_pc, unsigned int param_size, unsigned int local_size)
3550{
3551 const struct rb_callcache *cc = calling->cc;
3552 VALUE *argv = cfp->sp - calling->argc;
3553 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3554 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3555 VALUE *src_argv = argv;
3556 VALUE *sp_orig, *sp;
3557 VALUE finish_flag = VM_FRAME_FINISHED_P(cfp) ? VM_FRAME_FLAG_FINISH : 0;
3558
3559 if (VM_BH_FROM_CFP_P(calling->block_handler, cfp)) {
3560 struct rb_captured_block *dst_captured = VM_CFP_TO_CAPTURED_BLOCK(RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp));
3561 const struct rb_captured_block *src_captured = VM_BH_TO_CAPT_BLOCK(calling->block_handler);
3562 dst_captured->code.val = src_captured->code.val;
3563 if (VM_BH_ISEQ_BLOCK_P(calling->block_handler)) {
3564 calling->block_handler = VM_BH_FROM_ISEQ_BLOCK(dst_captured);
3565 }
3566 else {
3567 calling->block_handler = VM_BH_FROM_IFUNC_BLOCK(dst_captured);
3568 }
3569 }
3570
3571 vm_pop_frame(ec, cfp, cfp->ep);
3572 cfp = ec->cfp;
3573
3574 sp_orig = sp = cfp->sp;
3575
3576 /* push self */
3577 sp[0] = calling->recv;
3578 sp++;
3579
3580 /* copy arguments */
3581 for (unsigned int i = 0; i < param_size; i++) {
3582 *sp++ = src_argv[i];
3583 }
3584
3585 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL | finish_flag,
3586 calling->recv, calling->block_handler, (VALUE)me,
3587 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3588 local_size - param_size,
3589 ISEQ_BODY(iseq)->stack_max);
3590
3591 cfp->sp = sp_orig;
3592
3593 return Qundef;
3594}
3595
3596static void
3597ractor_unsafe_check(void)
3598{
3599 if (!rb_ractor_main_p()) {
3600 rb_raise(rb_eRactorUnsafeError, "ractor unsafe method called from not main ractor");
3601 }
3602}
3603
3604static VALUE
3605call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3606{
3607 ractor_unsafe_check();
3608 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3609 return (*f)(recv, rb_ary_new4(argc, argv));
3610}
3611
3612static VALUE
3613call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3614{
3615 ractor_unsafe_check();
3616 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3617 return (*f)(argc, argv, recv);
3618}
3619
3620static VALUE
3621call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3622{
3623 ractor_unsafe_check();
3624 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3625 return (*f)(recv);
3626}
3627
3628static VALUE
3629call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3630{
3631 ractor_unsafe_check();
3632 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3633 return (*f)(recv, argv[0]);
3634}
3635
3636static VALUE
3637call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3638{
3639 ractor_unsafe_check();
3640 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3641 return (*f)(recv, argv[0], argv[1]);
3642}
3643
3644static VALUE
3645call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3646{
3647 ractor_unsafe_check();
3648 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3649 return (*f)(recv, argv[0], argv[1], argv[2]);
3650}
3651
3652static VALUE
3653call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3654{
3655 ractor_unsafe_check();
3656 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3657 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3658}
3659
3660static VALUE
3661call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3662{
3663 ractor_unsafe_check();
3664 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3665 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3666}
3667
3668static VALUE
3669call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3670{
3671 ractor_unsafe_check();
3673 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3674}
3675
3676static VALUE
3677call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3678{
3679 ractor_unsafe_check();
3681 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3682}
3683
3684static VALUE
3685call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3686{
3687 ractor_unsafe_check();
3689 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3690}
3691
3692static VALUE
3693call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3694{
3695 ractor_unsafe_check();
3697 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3698}
3699
3700static VALUE
3701call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3702{
3703 ractor_unsafe_check();
3705 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3706}
3707
3708static VALUE
3709call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3710{
3711 ractor_unsafe_check();
3713 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3714}
3715
3716static VALUE
3717call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3718{
3719 ractor_unsafe_check();
3721 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3722}
3723
3724static VALUE
3725call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3726{
3727 ractor_unsafe_check();
3729 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3730}
3731
3732static VALUE
3733call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3734{
3735 ractor_unsafe_check();
3737 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3738}
3739
3740static VALUE
3741call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3742{
3743 ractor_unsafe_check();
3745 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3746}
3747
3748static VALUE
3749ractor_safe_call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3750{
3751 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3752 return (*f)(recv, rb_ary_new4(argc, argv));
3753}
3754
3755static VALUE
3756ractor_safe_call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3757{
3758 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3759 return (*f)(argc, argv, recv);
3760}
3761
3762static VALUE
3763ractor_safe_call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3764{
3765 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3766 return (*f)(recv);
3767}
3768
3769static VALUE
3770ractor_safe_call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3771{
3772 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3773 return (*f)(recv, argv[0]);
3774}
3775
3776static VALUE
3777ractor_safe_call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3778{
3779 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3780 return (*f)(recv, argv[0], argv[1]);
3781}
3782
3783static VALUE
3784ractor_safe_call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3785{
3786 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3787 return (*f)(recv, argv[0], argv[1], argv[2]);
3788}
3789
3790static VALUE
3791ractor_safe_call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3792{
3793 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3794 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3795}
3796
3797static VALUE
3798ractor_safe_call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3799{
3800 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3801 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3802}
3803
3804static VALUE
3805ractor_safe_call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3806{
3808 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3809}
3810
3811static VALUE
3812ractor_safe_call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3813{
3815 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3816}
3817
3818static VALUE
3819ractor_safe_call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3820{
3822 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3823}
3824
3825static VALUE
3826ractor_safe_call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3827{
3829 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3830}
3831
3832static VALUE
3833ractor_safe_call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3834{
3836 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3837}
3838
3839static VALUE
3840ractor_safe_call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3841{
3843 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3844}
3845
3846static VALUE
3847ractor_safe_call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3848{
3850 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3851}
3852
3853static VALUE
3854ractor_safe_call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3855{
3857 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3858}
3859
3860static VALUE
3861ractor_safe_call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3862{
3864 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3865}
3866
3867static VALUE
3868ractor_safe_call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3869{
3871 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3872}
3873
3874static inline int
3875vm_cfp_consistent_p(rb_execution_context_t *ec, const rb_control_frame_t *reg_cfp)
3876{
3877 const int ov_flags = RAISED_STACKOVERFLOW;
3878 if (LIKELY(reg_cfp == ec->cfp + 1)) return TRUE;
3879 if (rb_ec_raised_p(ec, ov_flags)) {
3880 rb_ec_raised_reset(ec, ov_flags);
3881 return TRUE;
3882 }
3883 return FALSE;
3884}
3885
3886#define CHECK_CFP_CONSISTENCY(func) \
3887 (LIKELY(vm_cfp_consistent_p(ec, reg_cfp)) ? (void)0 : \
3888 rb_bug(func ": cfp consistency error (%p, %p)", (void *)reg_cfp, (void *)(ec->cfp+1)))
3889
3890static inline
3891const rb_method_cfunc_t *
3892vm_method_cfunc_entry(const rb_callable_method_entry_t *me)
3893{
3894#if VM_DEBUG_VERIFY_METHOD_CACHE
3895 switch (me->def->type) {
3896 case VM_METHOD_TYPE_CFUNC:
3897 case VM_METHOD_TYPE_NOTIMPLEMENTED:
3898 break;
3899# define METHOD_BUG(t) case VM_METHOD_TYPE_##t: rb_bug("wrong method type: " #t)
3900 METHOD_BUG(ISEQ);
3901 METHOD_BUG(ATTRSET);
3902 METHOD_BUG(IVAR);
3903 METHOD_BUG(BMETHOD);
3904 METHOD_BUG(ZSUPER);
3905 METHOD_BUG(UNDEF);
3906 METHOD_BUG(OPTIMIZED);
3907 METHOD_BUG(MISSING);
3908 METHOD_BUG(REFINED);
3909 METHOD_BUG(ALIAS);
3910# undef METHOD_BUG
3911 default:
3912 rb_bug("wrong method type: %d", me->def->type);
3913 }
3914#endif
3915 return UNALIGNED_MEMBER_PTR(me->def, body.cfunc);
3916}
3917
3918static VALUE
3919vm_call_cfunc_with_frame_(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
3920 int argc, VALUE *argv, VALUE *stack_bottom)
3921{
3922 RB_DEBUG_COUNTER_INC(ccf_cfunc_with_frame);
3923 const struct rb_callinfo *ci = calling->cd->ci;
3924 const struct rb_callcache *cc = calling->cc;
3925 VALUE val;
3926 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3927 const rb_method_cfunc_t *cfunc = vm_method_cfunc_entry(me);
3928
3929 VALUE recv = calling->recv;
3930 VALUE block_handler = calling->block_handler;
3931 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3932
3933 if (UNLIKELY(calling->kw_splat)) {
3934 frame_type |= VM_FRAME_FLAG_CFRAME_KW;
3935 }
3936
3937 VM_ASSERT(reg_cfp == ec->cfp);
3938
3939 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, me->owner, me->def->original_id);
3940 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, recv, me->def->original_id, vm_ci_mid(ci), me->owner, Qundef);
3941
3942 vm_push_frame(ec, NULL, frame_type, recv,
3943 block_handler, (VALUE)me,
3944 0, ec->cfp->sp, 0, 0);
3945
3946 int len = cfunc->argc;
3947 if (len >= 0) rb_check_arity(argc, len, len);
3948
3949 reg_cfp->sp = stack_bottom;
3950 val = (*cfunc->invoker)(recv, argc, argv, cfunc->func);
3951
3952 CHECK_CFP_CONSISTENCY("vm_call_cfunc");
3953
3954 rb_vm_pop_frame(ec);
3955
3956 VM_ASSERT(ec->cfp->sp == stack_bottom);
3957
3958 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, recv, me->def->original_id, vm_ci_mid(ci), me->owner, val);
3959 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
3960
3961 return val;
3962}
3963
3964// Push a C method frame for a given cme. This is called when JIT code skipped
3965// pushing a frame but the C method reached a point where a frame is needed.
3966void
3967rb_vm_push_cfunc_frame(const rb_callable_method_entry_t *cme, int recv_idx)
3968{
3969 VM_ASSERT(cme->def->type == VM_METHOD_TYPE_CFUNC);
3970 rb_execution_context_t *ec = GET_EC();
3971 VALUE *sp = ec->cfp->sp;
3972 VALUE recv = *(sp - recv_idx - 1);
3973 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3974 VALUE block_handler = VM_BLOCK_HANDLER_NONE;
3975#if VM_CHECK_MODE > 0
3976 // Clean up the stack canary since we're about to satisfy the "leaf or lazy push" assumption
3977 *(GET_EC()->cfp->sp) = Qfalse;
3978#endif
3979 vm_push_frame(ec, NULL, frame_type, recv, block_handler, (VALUE)cme, 0, ec->cfp->sp, 0, 0);
3980}
3981
3982// If true, cc->call needs to include `CALLER_SETUP_ARG` (i.e. can't be skipped in fastpath)
3983bool
3984rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci)
3985{
3986 return IS_ARGS_SPLAT(ci) || IS_ARGS_KW_OR_KW_SPLAT(ci);
3987}
3988
3989static VALUE
3990vm_call_get_kwarg(rb_control_frame_t *reg_cfp, CALL_DATA cd, ID kw)
3991{
3992
3993 VALUE sym = ID2SYM(kw);
3994 if (vm_ci_flag(cd->ci) & VM_CALL_KW_SPLAT) {
3995 VALUE keywords = TOPN(0);
3996 if (UNLIKELY(!RB_TYPE_P(keywords, T_HASH))) {
3997 TOPN(0) = keywords = rb_to_hash_type(keywords);
3998 }
3999 return rb_hash_lookup2(keywords, sym, Qundef);
4000 }
4001 else {
4002 const struct rb_callinfo_kwarg *kwarg = vm_ci_kwarg(cd->ci);
4003 if (kwarg) {
4004 for (int index = 0; index < kwarg->keyword_len; index++) {
4005 if (kwarg->keywords[index] == sym) {
4006 return TOPN(kwarg->keyword_len - index - 1);
4007 }
4008 }
4009 }
4010 }
4011 return Qundef;
4012}
4013
4014static VALUE
4015vm_call_cfunc_with_frame(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4016{
4017 int argc = calling->argc;
4018 VALUE *stack_bottom = reg_cfp->sp - argc - 1;
4019 VALUE *argv = &stack_bottom[1];
4020
4021 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
4022}
4023
4024static VALUE
4025vm_call_cfunc_other(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4026{
4027 const struct rb_callinfo *ci = calling->cd->ci;
4028 RB_DEBUG_COUNTER_INC(ccf_cfunc_other);
4029
4030 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
4031 VALUE argv_ary;
4032 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4033 VM_ASSERT(!IS_ARGS_KEYWORD(ci));
4034 int argc = RARRAY_LENINT(argv_ary);
4035 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
4036 VALUE *stack_bottom = reg_cfp->sp - 2;
4037
4038 VM_ASSERT(calling->argc == 1);
4039 VM_ASSERT(RB_TYPE_P(argv_ary, T_ARRAY));
4040 VM_ASSERT(RBASIC_CLASS(argv_ary) == 0); // hidden ary
4041
4042 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
4043 }
4044 else {
4045 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_with_frame, !rb_splat_or_kwargs_p(ci) && !calling->kw_splat && !(vm_ci_flag(ci) & VM_CALL_FORWARDING));
4046
4047 return vm_call_cfunc_with_frame(ec, reg_cfp, calling);
4048 }
4049}
4050
4051static inline VALUE
4052vm_call_cfunc_array_argv(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int stack_offset, int argc_offset)
4053{
4054 VALUE argv_ary = reg_cfp->sp[-1 - stack_offset];
4055 int argc = RARRAY_LENINT(argv_ary) - argc_offset;
4056
4057 if (UNLIKELY(argc > VM_ARGC_STACK_MAX)) {
4058 return vm_call_cfunc_other(ec, reg_cfp, calling);
4059 }
4060
4061 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
4062 calling->kw_splat = 0;
4063 int i;
4064 VALUE *stack_bottom = reg_cfp->sp - 2 - stack_offset;
4065 VALUE *sp = stack_bottom;
4066 CHECK_VM_STACK_OVERFLOW(reg_cfp, argc);
4067 for(i = 0; i < argc; i++) {
4068 *++sp = argv[i];
4069 }
4070 reg_cfp->sp = sp+1;
4071
4072 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, stack_bottom+1, stack_bottom);
4073}
4074
4075static inline VALUE
4076vm_call_cfunc_only_splat(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4077{
4078 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat);
4079 VALUE argv_ary = reg_cfp->sp[-1];
4080 int argc = RARRAY_LENINT(argv_ary);
4081 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
4082 VALUE last_hash;
4083 int argc_offset = 0;
4084
4085 if (UNLIKELY(argc > 0 &&
4086 RB_TYPE_P((last_hash = argv[argc-1]), T_HASH) &&
4087 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS))) {
4088 if (!RHASH_EMPTY_P(last_hash)) {
4089 return vm_call_cfunc_other(ec, reg_cfp, calling);
4090 }
4091 argc_offset++;
4092 }
4093 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 0, argc_offset);
4094}
4095
4096static inline VALUE
4097vm_call_cfunc_only_splat_kw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4098{
4099 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat_kw);
4100 VALUE keyword_hash = reg_cfp->sp[-1];
4101
4102 if (keyword_hash == Qnil || (RB_TYPE_P(keyword_hash, T_HASH) && RHASH_EMPTY_P(keyword_hash))) {
4103 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 1, 0);
4104 }
4105
4106 return vm_call_cfunc_other(ec, reg_cfp, calling);
4107}
4108
4109static VALUE
4110vm_call_cfunc(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4111{
4112 const struct rb_callinfo *ci = calling->cd->ci;
4113 RB_DEBUG_COUNTER_INC(ccf_cfunc);
4114
4115 if (IS_ARGS_SPLAT(ci) && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
4116 if (!IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 1) {
4117 // f(*a)
4118 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat, TRUE);
4119 return vm_call_cfunc_only_splat(ec, reg_cfp, calling);
4120 }
4121 if (IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 2) {
4122 // f(*a, **kw)
4123 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat_kw, TRUE);
4124 return vm_call_cfunc_only_splat_kw(ec, reg_cfp, calling);
4125 }
4126 }
4127
4128 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_other, TRUE);
4129 return vm_call_cfunc_other(ec, reg_cfp, calling);
4130}
4131
4132static VALUE
4133vm_call_ivar(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4134{
4135 const struct rb_callcache *cc = calling->cc;
4136 RB_DEBUG_COUNTER_INC(ccf_ivar);
4137 cfp->sp -= 1;
4138 VALUE ivar = vm_getivar(calling->recv, vm_cc_cme(cc)->def->body.attr.id, NULL, NULL, cc, TRUE, Qnil);
4139 return ivar;
4140}
4141
4142static VALUE
4143vm_call_attrset_direct(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_callcache *cc, VALUE obj)
4144{
4145 RB_DEBUG_COUNTER_INC(ccf_attrset);
4146 VALUE val = *(cfp->sp - 1);
4147 cfp->sp -= 2;
4148 rb_setivar_cache cache = rb_setivar_cache_unpack(vm_cc_atomic_cache_read(cc));
4149 ID id = vm_cc_cme(cc)->def->body.attr.id;
4150 rb_check_frozen(obj);
4151 VALUE res = vm_setivar(obj, val, cache);
4152 if (UNDEF_P(res)) {
4153 switch (BUILTIN_TYPE(obj)) {
4154 case T_OBJECT:
4155 break;
4156 case T_CLASS:
4157 case T_MODULE:
4158 {
4159 res = vm_setivar_class(obj, val, cache);
4160 if (!UNDEF_P(res)) {
4161 return res;
4162 }
4163 }
4164 break;
4165 default:
4166 {
4167 res = vm_setivar_default(obj, id, val, cache);
4168 if (!UNDEF_P(res)) {
4169 return res;
4170 }
4171 }
4172 }
4173 res = vm_setivar_slowpath_attr(obj, id, val, cc);
4174 }
4175 return res;
4176}
4177
4178static VALUE
4179vm_call_attrset(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4180{
4181 return vm_call_attrset_direct(ec, cfp, calling->cc, calling->recv);
4182}
4183
4184static inline VALUE
4185vm_call_bmethod_body(rb_execution_context_t *ec, struct rb_calling_info *calling, const VALUE *argv)
4186{
4187 rb_proc_t *proc;
4188 VALUE val;
4189 const struct rb_callcache *cc = calling->cc;
4190 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4191 VALUE procv = cme->def->body.bmethod.proc;
4192
4193 if (!RB_OBJ_SHAREABLE_P(procv) &&
4194 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4195 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4196 }
4197
4198 /* control block frame */
4199 GetProcPtr(procv, proc);
4200 val = vm_invoke_bmethod(ec, proc, calling->recv, CALLING_ARGC(calling), argv, calling->kw_splat, calling->block_handler, vm_cc_cme(cc));
4201
4202 return val;
4203}
4204
4205static int vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type);
4206
4207static VALUE
4208vm_call_iseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4209{
4210 RB_DEBUG_COUNTER_INC(ccf_iseq_bmethod);
4211
4212 const struct rb_callcache *cc = calling->cc;
4213 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4214 VALUE procv = cme->def->body.bmethod.proc;
4215
4216 if (!RB_OBJ_SHAREABLE_P(procv) &&
4217 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4218 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4219 }
4220
4221 rb_proc_t *proc;
4222 GetProcPtr(procv, proc);
4223 const struct rb_block *block = &proc->block;
4224
4225 while (vm_block_type(block) == block_type_proc) {
4226 block = vm_proc_block(block->as.proc);
4227 }
4228 VM_ASSERT(vm_block_type(block) == block_type_iseq);
4229
4230 const struct rb_captured_block *captured = &block->as.captured;
4231 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
4232 VALUE * const argv = cfp->sp - calling->argc;
4233 const int arg_size = ISEQ_BODY(iseq)->param.size;
4234
4235 int opt_pc;
4236 if (vm_ci_flag(calling->cd->ci) & VM_CALL_ARGS_SIMPLE) {
4237 opt_pc = vm_callee_setup_block_arg(ec, calling, calling->cd->ci, iseq, argv, arg_setup_method);
4238 }
4239 else {
4240 opt_pc = setup_parameters_complex(ec, iseq, calling, calling->cd->ci, argv, arg_setup_method);
4241 }
4242
4243 cfp->sp = argv - 1; // -1 for the receiver
4244
4245 vm_push_frame(ec, iseq,
4246 VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_BMETHOD | VM_FRAME_FLAG_LAMBDA,
4247 calling->recv,
4248 VM_GUARDED_PREV_EP(captured->ep),
4249 (VALUE)cme,
4250 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
4251 argv + arg_size,
4252 ISEQ_BODY(iseq)->local_table_size - arg_size,
4253 ISEQ_BODY(iseq)->stack_max);
4254
4255 return Qundef;
4256}
4257
4258static VALUE
4259vm_call_noniseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4260{
4261 RB_DEBUG_COUNTER_INC(ccf_noniseq_bmethod);
4262
4263 VALUE *argv;
4264 int argc;
4265 CALLER_SETUP_ARG(cfp, calling, calling->cd->ci, ALLOW_HEAP_ARGV);
4266 if (UNLIKELY(calling->heap_argv)) {
4267 argv = RARRAY_PTR(calling->heap_argv);
4268 cfp->sp -= 2;
4269 }
4270 else {
4271 argc = calling->argc;
4272 argv = ALLOCA_N(VALUE, argc);
4273 MEMCPY(argv, cfp->sp - argc, VALUE, argc);
4274 cfp->sp += - argc - 1;
4275 }
4276
4277 return vm_call_bmethod_body(ec, calling, argv);
4278}
4279
4280static VALUE
4281vm_call_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4282{
4283 RB_DEBUG_COUNTER_INC(ccf_bmethod);
4284
4285 const struct rb_callcache *cc = calling->cc;
4286 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4287 VALUE procv = cme->def->body.bmethod.proc;
4288 rb_proc_t *proc;
4289 GetProcPtr(procv, proc);
4290 const struct rb_block *block = &proc->block;
4291
4292 while (vm_block_type(block) == block_type_proc) {
4293 block = vm_proc_block(block->as.proc);
4294 }
4295 if (vm_block_type(block) == block_type_iseq) {
4296 CC_SET_FASTPATH(cc, vm_call_iseq_bmethod, TRUE);
4297 return vm_call_iseq_bmethod(ec, cfp, calling);
4298 }
4299
4300 CC_SET_FASTPATH(cc, vm_call_noniseq_bmethod, TRUE);
4301 return vm_call_noniseq_bmethod(ec, cfp, calling);
4302}
4303
4304VALUE
4305rb_find_defined_class_by_owner(VALUE current_class, VALUE target_owner)
4306{
4307 VALUE klass = current_class;
4308
4309 /* for prepended Module, then start from cover class */
4310 if (RB_TYPE_P(klass, T_ICLASS) && RICLASS_IS_ORIGIN_P(klass) &&
4311 RB_TYPE_P(RBASIC_CLASS(klass), T_CLASS)) {
4312 klass = RBASIC_CLASS(klass);
4313 }
4314
4315 while (RTEST(klass)) {
4316 VALUE owner = RB_TYPE_P(klass, T_ICLASS) ? RBASIC_CLASS(klass) : klass;
4317 if (owner == target_owner) {
4318 return klass;
4319 }
4320 klass = RCLASS_SUPER(klass);
4321 }
4322
4323 return current_class; /* maybe module function */
4324}
4325
4326static const rb_callable_method_entry_t *
4327aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4328{
4329 const rb_method_entry_t *orig_me =
4330 rb_method_alias_original_me(rb_method_entry_def_acquire((const rb_method_entry_t *)me));
4331 const rb_callable_method_entry_t *cme = NULL;
4332
4333 if (orig_me->defined_class == 0) { // only true on first resolution
4334 // Resolving the alias mutates `me` (and may replace me->def, dropping a
4335 // reference to the shared definition), so it has to be serialized. Two
4336 // ractors calling the same aliased method would otherwise both release
4337 // the same definition which could result in a UAF.
4338 RB_VM_LOCKING() {
4339 orig_me = me->def->body.alias.original_me;
4340 // me is the ALIAS me. It has a def that's different from orig_me->def.
4341 if (orig_me->defined_class == 0) {
4342 // orig_me is the unresolved me clone that rb_add_alias stored.
4343 VALUE defined_class = rb_find_defined_class_by_owner(me->defined_class, orig_me->owner);
4344 VM_ASSERT_TYPE(orig_me->owner, T_MODULE);
4345 cme = rb_method_entry_complement_defined_class(orig_me, me->called_id, defined_class);
4346
4347 if (me->def->reference_count == 1) {
4348 rbimpl_atomic_ptr_store((volatile void **)&me->def->body.alias.original_me,
4349 (void *)cme, RBIMPL_ATOMIC_RELEASE);
4350 RB_OBJ_WRITTEN(me, Qundef, cme);
4351 }
4352 else {
4353 rb_method_definition_t *def =
4354 rb_method_definition_create(VM_METHOD_TYPE_ALIAS, me->def->original_id);
4355 def->body.alias.original_me = (rb_method_entry_t *)cme;
4356 def->original_module = cme->def->original_module;
4357 rb_method_definition_set((rb_method_entry_t *)me, def, NULL);
4358 RB_OBJ_WRITTEN(me, Qundef, cme);
4359 RB_OBJ_WRITTEN(me, Qundef, def->original_module);
4360 }
4361 }
4362 else {
4363 cme = (const rb_callable_method_entry_t *)orig_me;
4364 }
4365 }
4366 }
4367 else {
4368 cme = (const rb_callable_method_entry_t *)orig_me;
4369 }
4370
4371 VM_ASSERT(callable_method_entry_p(cme));
4372 return cme;
4373}
4374
4375const rb_callable_method_entry_t *
4376rb_aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4377{
4378 return aliased_callable_method_entry(me);
4379}
4380
4381static VALUE
4382vm_call_alias(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4383{
4384 calling->cc = &VM_CC_ON_STACK(Qundef,
4385 vm_call_general,
4386 {{0}},
4387 aliased_callable_method_entry(vm_cc_cme(calling->cc)));
4388
4389 return vm_call_method_each_type(ec, cfp, calling);
4390}
4391
4392static enum method_missing_reason
4393ci_missing_reason(const struct rb_callinfo *ci)
4394{
4395 enum method_missing_reason stat = MISSING_NOENTRY;
4396 if (vm_ci_flag(ci) & VM_CALL_VCALL && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) stat |= MISSING_VCALL;
4397 if (vm_ci_flag(ci) & VM_CALL_FCALL) stat |= MISSING_FCALL;
4398 if (vm_ci_flag(ci) & VM_CALL_SUPER) stat |= MISSING_SUPER;
4399 return stat;
4400}
4401
4402static VALUE vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
4403
4404static VALUE
4405vm_call_symbol(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4406 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE symbol, int flags)
4407{
4408 ASSUME(calling->argc >= 0);
4409
4410 enum method_missing_reason missing_reason = MISSING_NOENTRY;
4411 int argc = calling->argc;
4412 VALUE recv = calling->recv;
4413 VALUE klass = CLASS_OF(recv);
4414 ID mid = rb_check_id(&symbol);
4415 flags |= VM_CALL_OPT_SEND;
4416
4417 if (UNLIKELY(! mid)) {
4418 mid = idMethodMissing;
4419 missing_reason = ci_missing_reason(ci);
4420 ec->method_missing_reason = missing_reason;
4421
4422 VALUE argv_ary;
4423 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4424 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4425 rb_ary_unshift(argv_ary, symbol);
4426
4427 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4428 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4429 VALUE exc = rb_make_no_method_exception(
4430 rb_eNoMethodError, 0, recv, RARRAY_LENINT(argv_ary), RARRAY_CONST_PTR(argv_ary), priv);
4431
4432 rb_exc_raise(exc);
4433 }
4434 rb_ary_unshift(argv_ary, rb_str_intern(symbol));
4435 }
4436 else {
4437 /* E.g. when argc == 2
4438 *
4439 * | | | | TOPN
4440 * | | +------+
4441 * | | +---> | arg1 | 0
4442 * +------+ | +------+
4443 * | arg1 | -+ +-> | arg0 | 1
4444 * +------+ | +------+
4445 * | arg0 | ---+ | sym | 2
4446 * +------+ +------+
4447 * | recv | | recv | 3
4448 * --+------+--------+------+------
4449 */
4450 int i = argc;
4451 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4452 INC_SP(1);
4453 MEMMOVE(&TOPN(i - 1), &TOPN(i), VALUE, i);
4454 argc = ++calling->argc;
4455
4456 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4457 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4458 TOPN(i) = symbol;
4459 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4460 const VALUE *argv = STACK_ADDR_FROM_TOP(argc);
4461 VALUE exc = rb_make_no_method_exception(
4462 rb_eNoMethodError, 0, recv, argc, argv, priv);
4463
4464 rb_exc_raise(exc);
4465 }
4466 else {
4467 TOPN(i) = rb_str_intern(symbol);
4468 }
4469 }
4470 }
4471
4472 struct rb_forwarding_call_data new_fcd = {
4473 .cd = {
4474 .ci = &VM_CI_ON_STACK(mid, flags, argc, vm_ci_kwarg(ci)),
4475 .cc = NULL,
4476 },
4477 .caller_ci = NULL,
4478 };
4479
4480 if (!(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
4481 calling->cd = &new_fcd.cd;
4482 }
4483 else {
4484 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4485 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4486 new_fcd.caller_ci = caller_ci;
4487 calling->cd = (struct rb_call_data *)&new_fcd;
4488 }
4489 calling->cc = &VM_CC_ON_STACK(klass,
4490 vm_call_general,
4491 { .method_missing_reason = missing_reason },
4492 rb_callable_method_entry_with_refinements(klass, mid, NULL));
4493
4494 if (flags & VM_CALL_FCALL) {
4495 return vm_call_method(ec, reg_cfp, calling);
4496 }
4497
4498 const struct rb_callcache *cc = calling->cc;
4499 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
4500
4501 if (vm_cc_cme(cc) != NULL) {
4502 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
4503 case METHOD_VISI_PUBLIC: /* likely */
4504 return vm_call_method_each_type(ec, reg_cfp, calling);
4505 case METHOD_VISI_PRIVATE:
4506 vm_cc_method_missing_reason_set(cc, MISSING_PRIVATE);
4507 break;
4508 case METHOD_VISI_PROTECTED:
4509 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
4510 break;
4511 default:
4512 VM_UNREACHABLE(vm_call_method);
4513 }
4514 return vm_call_method_missing(ec, reg_cfp, calling);
4515 }
4516
4517 return vm_call_method_nome(ec, reg_cfp, calling);
4518}
4519
4520static VALUE
4521vm_call_opt_send0(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int flags)
4522{
4523 const struct rb_callinfo *ci = calling->cd->ci;
4524 int i;
4525 VALUE sym;
4526
4527 i = calling->argc - 1;
4528
4529 if (calling->argc == 0) {
4530 rb_raise(rb_eArgError, "no method name given");
4531 }
4532
4533 sym = TOPN(i);
4534 /* E.g. when i == 2
4535 *
4536 * | | | | TOPN
4537 * +------+ | |
4538 * | arg1 | ---+ | | 0
4539 * +------+ | +------+
4540 * | arg0 | -+ +-> | arg1 | 1
4541 * +------+ | +------+
4542 * | sym | +---> | arg0 | 2
4543 * +------+ +------+
4544 * | recv | | recv | 3
4545 * --+------+--------+------+------
4546 */
4547 /* shift arguments */
4548 if (i > 0) {
4549 MEMMOVE(&TOPN(i), &TOPN(i-1), VALUE, i);
4550 }
4551 calling->argc -= 1;
4552 DEC_SP(1);
4553
4554 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4555}
4556
4557static VALUE
4558vm_call_opt_send_complex(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4559{
4560 RB_DEBUG_COUNTER_INC(ccf_opt_send_complex);
4561 const struct rb_callinfo *ci = calling->cd->ci;
4562 int flags = VM_CALL_FCALL;
4563 VALUE sym;
4564
4565 VALUE argv_ary;
4566 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
4567 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4568 sym = rb_ary_shift(argv_ary);
4569 flags |= VM_CALL_ARGS_SPLAT;
4570 if (calling->kw_splat) {
4571 VALUE last_hash = rb_ary_last(0, NULL, argv_ary);
4572 ((struct RHash *)last_hash)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
4573 calling->kw_splat = 0;
4574 }
4575 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4576 }
4577
4578 if (calling->kw_splat) flags |= VM_CALL_KW_SPLAT;
4579 return vm_call_opt_send0(ec, reg_cfp, calling, flags);
4580}
4581
4582static VALUE
4583vm_call_opt_send_simple(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4584{
4585 RB_DEBUG_COUNTER_INC(ccf_opt_send_simple);
4586 return vm_call_opt_send0(ec, reg_cfp, calling, vm_ci_flag(calling->cd->ci) | VM_CALL_FCALL);
4587}
4588
4589static VALUE
4590vm_call_opt_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4591{
4592 RB_DEBUG_COUNTER_INC(ccf_opt_send);
4593
4594 const struct rb_callinfo *ci = calling->cd->ci;
4595 int flags = vm_ci_flag(ci);
4596
4597 if (UNLIKELY((flags & VM_CALL_FORWARDING) || (!(flags & VM_CALL_ARGS_SIMPLE) &&
4598 ((calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
4599 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
4600 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc)))))) {
4601 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_complex, TRUE);
4602 return vm_call_opt_send_complex(ec, reg_cfp, calling);
4603 }
4604
4605 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_simple, TRUE);
4606 return vm_call_opt_send_simple(ec, reg_cfp, calling);
4607}
4608
4609static VALUE
4610vm_call_method_missing_body(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
4611 const struct rb_callinfo *orig_ci, enum method_missing_reason reason)
4612{
4613 RB_DEBUG_COUNTER_INC(ccf_method_missing);
4614
4615 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
4616 unsigned int argc, flag;
4617
4618 flag = VM_CALL_FCALL | VM_CALL_OPT_SEND | vm_ci_flag(orig_ci);
4619 argc = ++calling->argc;
4620
4621 /* shift arguments: m(a, b, c) #=> method_missing(:m, a, b, c) */
4622 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4623 vm_check_canary(ec, reg_cfp->sp);
4624 if (argc > 1) {
4625 MEMMOVE(argv+1, argv, VALUE, argc-1);
4626 }
4627 argv[0] = ID2SYM(vm_ci_mid(orig_ci));
4628 INC_SP(1);
4629
4630 ec->method_missing_reason = reason;
4631
4632 struct rb_forwarding_call_data new_fcd = {
4633 .cd = {
4634 .ci = &VM_CI_ON_STACK(idMethodMissing, flag, argc, vm_ci_kwarg(orig_ci)),
4635 .cc = NULL,
4636 },
4637 .caller_ci = NULL,
4638 };
4639
4640 if (!(flag & VM_CALL_FORWARDING)) {
4641 calling->cd = &new_fcd.cd;
4642 }
4643 else {
4644 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4645 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4646 new_fcd.caller_ci = caller_ci;
4647 calling->cd = (struct rb_call_data *)&new_fcd;
4648 }
4649
4650 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }},
4651 rb_callable_method_entry_without_refinements(CLASS_OF(calling->recv), idMethodMissing, NULL));
4652 return vm_call_method(ec, reg_cfp, calling);
4653}
4654
4655static VALUE
4656vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4657{
4658 return vm_call_method_missing_body(ec, reg_cfp, calling, calling->cd->ci, vm_cc_cmethod_missing_reason(calling->cc));
4659}
4660
4661static const rb_callable_method_entry_t *refined_method_callable_without_refinement(const rb_callable_method_entry_t *me);
4662static VALUE
4663vm_call_zsuper(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE klass)
4664{
4665 klass = RCLASS_SUPER(klass);
4666
4667 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, vm_ci_mid(calling->cd->ci)) : NULL;
4668 if (cme == NULL) {
4669 return vm_call_method_nome(ec, cfp, calling);
4670 }
4671 if (cme->def->type == VM_METHOD_TYPE_REFINED &&
4672 cme->def->body.refined.orig_me) {
4673 cme = refined_method_callable_without_refinement(cme);
4674 }
4675
4676 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, cme);
4677
4678 return vm_call_method_each_type(ec, cfp, calling);
4679}
4680
4681static inline VALUE
4682find_refinement(VALUE refinements, VALUE klass)
4683{
4684 if (NIL_P(refinements)) {
4685 return Qnil;
4686 }
4687 return rb_hash_lookup(refinements, klass);
4688}
4689
4690PUREFUNC(static rb_control_frame_t * current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp));
4691static rb_control_frame_t *
4692current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp)
4693{
4694 rb_control_frame_t *top_cfp = cfp;
4695
4696 if (CFP_ISEQ(cfp) && !VM_FRAME_BMETHOD_P(cfp) && ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_BLOCK) {
4697 const rb_iseq_t *local_iseq = ISEQ_BODY(CFP_ISEQ(cfp))->local_iseq;
4698
4699 do {
4700 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
4701 if (RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
4702 /* TODO: orphan block */
4703 return top_cfp;
4704 }
4705 } while (CFP_ISEQ(cfp) != local_iseq && !VM_FRAME_BMETHOD_P(cfp));
4706 }
4707 return cfp;
4708}
4709
4710static const rb_callable_method_entry_t *
4711refined_method_callable_without_refinement(const rb_callable_method_entry_t *me)
4712{
4713 const rb_method_entry_t *orig_me = me->def->body.refined.orig_me;
4714 const rb_callable_method_entry_t *cme;
4715
4716 if (orig_me->defined_class == 0) {
4717 cme = NULL;
4719 }
4720 else {
4721 cme = (const rb_callable_method_entry_t *)orig_me;
4722 }
4723
4724 VM_ASSERT(callable_method_entry_p(cme));
4725
4726 if (UNDEFINED_METHOD_ENTRY_P(cme)) {
4727 cme = NULL;
4728 }
4729
4730 return cme;
4731}
4732
4733// Construct correct super chain for module refinements.
4734// Each class including/prepending a module has a weak map mapping refinement
4735// iclasses to refinement module iclasses with the correct super. Like
4736// singleton classes, these are lazily initialized (both the map and
4737// the refinement module iclasses).
4738static VALUE
4739module_refinement_iclass(VALUE refinement_iclass, VALUE defined_class)
4740{
4741 VALUE module_ref_iclass = Qundef;
4742
4743 RB_VM_LOCKING() {
4744 VALUE cache = RCLASS_MODULE_REFINEMENT_ICLASSES(defined_class);
4745
4746 if (LIKELY(cache)) {
4747 module_ref_iclass = rb_wmap_lookup(cache, refinement_iclass);
4748 }
4749 else {
4750 cache = rb_wmap_new_hidden();
4751 RCLASS_SET_MODULE_REFINEMENT_ICLASSES(defined_class, cache);
4752 }
4753
4754 if (UNDEF_P(module_ref_iclass)) {
4755 VALUE super;
4756 VALUE refinement_iclass_super = RCLASS_SUPER(refinement_iclass);
4757
4758 if (RICLASS_FOR_REFINEMENT_P(refinement_iclass_super)) {
4759 super = module_refinement_iclass(refinement_iclass_super, defined_class);
4760 }
4761 else {
4762 super = defined_class;
4763 }
4764
4765 module_ref_iclass = rb_include_class_new(RBASIC(refinement_iclass)->klass, super);
4766 RCLASS_SET_REFINED_CLASS(module_ref_iclass, RCLASS_REFINED_CLASS(refinement_iclass));
4767 rb_class_subclass_add(RCLASS_REFINED_CLASS(refinement_iclass), module_ref_iclass);
4768 rb_wmap_aset(cache, refinement_iclass, module_ref_iclass);
4769 }
4770 }
4771
4772 return module_ref_iclass;
4773}
4774
4775VALUE
4776rb_vm_module_refinement_iclass(VALUE refinement_iclass, VALUE defined_class)
4777{
4778 return module_refinement_iclass(refinement_iclass, defined_class);
4779}
4780
4781static VALUE
4782refinement_iclass_for_cme(VALUE refinement_iclass, const rb_callable_method_entry_t *cme)
4783{
4784 if (RB_TYPE_P(cme->owner, T_MODULE) && RB_TYPE_P(cme->defined_class, T_ICLASS)) {
4785 refinement_iclass = module_refinement_iclass(refinement_iclass, cme->defined_class);
4786 }
4787 return refinement_iclass;
4788}
4789
4790VALUE
4791rb_vm_refinement_iclass_for_cme(VALUE refinement_iclass, const rb_callable_method_entry_t *cme)
4792{
4793 return refinement_iclass_for_cme(refinement_iclass, cme);
4794}
4795
4796static const rb_callable_method_entry_t *
4797search_refined_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4798{
4799 const struct rb_callinfo *ci = calling->cd->ci;
4800 ID mid = vm_ci_mid(ci);
4801 // The initialize call of opt_new must behave like Class#new, which does not see the caller's refinements. [Bug #22501]
4802 const rb_cref_t *cref = (vm_ci_flag(ci) & VM_CALL_IGNORE_REFINEMENTS) ? NULL : vm_get_cref(cfp->ep);
4803 const struct rb_callcache * const cc = calling->cc;
4804 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4805
4806 for (; cref; cref = CREF_NEXT(cref)) {
4807 VALUE refinement = find_refinement(CREF_REFINEMENTS(cref), vm_cc_cme(cc)->owner);
4808 if (NIL_P(refinement)) continue;
4809
4810 refinement = refinement_iclass_for_cme(refinement, vm_cc_cme(cc));
4811
4812 const rb_callable_method_entry_t *const ref_me =
4813 rb_callable_method_entry(refinement, mid);
4814
4815 if (ref_me) {
4816 if (vm_cc_call(cc) == vm_call_super_method) {
4817 const rb_control_frame_t *top_cfp = current_method_entry(ec, cfp);
4818 const rb_callable_method_entry_t *top_me = rb_vm_frame_method_entry(top_cfp);
4819 if (top_me && rb_method_definition_eq(ref_me->def, top_me->def)) {
4820 continue;
4821 }
4822 }
4823
4824 if (cme->def->type != VM_METHOD_TYPE_REFINED ||
4825 cme->def != ref_me->def) {
4826 cme = ref_me;
4827 }
4828 if (ref_me->def->type != VM_METHOD_TYPE_REFINED) {
4829 return cme;
4830 }
4831 }
4832 else {
4833 return NULL;
4834 }
4835 }
4836
4837 if (vm_cc_cme(cc)->def->body.refined.orig_me) {
4838 return refined_method_callable_without_refinement(vm_cc_cme(cc));
4839 }
4840 else {
4841 VALUE klass = RCLASS_SUPER(vm_cc_cme(cc)->defined_class);
4842 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, mid) : NULL;
4843 return cme;
4844 }
4845}
4846
4847static VALUE
4848vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4849{
4850 const rb_callable_method_entry_t *ref_cme = search_refined_method(ec, cfp, calling);
4851
4852 if (ref_cme) {
4853 if (calling->cd->cc) {
4854 const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement);
4855 RB_OBJ_WRITE(CFP_ISEQ(cfp), &calling->cd->cc, cc);
4856 return vm_call_method(ec, cfp, calling);
4857 }
4858 else {
4859 struct rb_callcache *ref_cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, ref_cme);
4860 calling->cc= ref_cc;
4861 return vm_call_method(ec, cfp, calling);
4862 }
4863 }
4864 else {
4865 return vm_call_method_nome(ec, cfp, calling);
4866 }
4867}
4868
4869static inline VALUE vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci, bool is_lambda, VALUE block_handler);
4870
4871NOINLINE(static VALUE
4872 vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4873 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler));
4874
4875static VALUE
4876vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4877 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler)
4878{
4879 int argc = calling->argc;
4880
4881 /* remove self */
4882 if (argc > 0) MEMMOVE(&TOPN(argc), &TOPN(argc-1), VALUE, argc);
4883 DEC_SP(1);
4884
4885 return vm_invoke_block(ec, reg_cfp, calling, ci, false, block_handler);
4886}
4887
4888static VALUE
4889vm_call_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4890{
4891 RB_DEBUG_COUNTER_INC(ccf_opt_call);
4892
4893 const struct rb_callinfo *ci = calling->cd->ci;
4894 VALUE procval = calling->recv;
4895 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, VM_BH_FROM_PROC(procval));
4896}
4897
4898static VALUE
4899vm_call_opt_block_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4900{
4901 RB_DEBUG_COUNTER_INC(ccf_opt_block_call);
4902
4903 VALUE block_handler = VM_ENV_BLOCK_HANDLER(VM_CF_LEP(reg_cfp));
4904 const struct rb_callinfo *ci = calling->cd->ci;
4905
4906 enum ruby_basic_operators bop;
4907 switch (vm_ci_mid(ci)) {
4908 case idAREF: bop = BOP_AREF; break;
4909 case idYield: bop = BOP_YIELD; break;
4910 case idEqq: bop = BOP_EQQ; break;
4911 default: bop = BOP_CALL; break;
4912 }
4913
4914 if (BASIC_OP_UNREDEFINED_P(bop, PROC_REDEFINED_OP_FLAG)) {
4915 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, block_handler);
4916 }
4917 else {
4918 calling->recv = rb_vm_bh_to_procval(ec, block_handler);
4919 calling->cc = rb_vm_search_method_slowpath(ci, CLASS_OF(calling->recv));
4920 return vm_call_general(ec, reg_cfp, calling);
4921 }
4922}
4923
4924static VALUE
4925vm_call_opt_struct_aref0(rb_execution_context_t *ec, struct rb_calling_info *calling)
4926{
4927 VALUE recv = calling->recv;
4928
4929 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4930 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4931 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_AREF);
4932
4933 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4934 return RSTRUCT_GET_RAW(recv, off);
4935}
4936
4937static VALUE
4938vm_call_opt_struct_aref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4939{
4940 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aref);
4941
4942 VALUE ret = vm_call_opt_struct_aref0(ec, calling);
4943 reg_cfp->sp -= 1;
4944 return ret;
4945}
4946
4947static VALUE
4948vm_call_opt_struct_aset0(rb_execution_context_t *ec, struct rb_calling_info *calling, VALUE val)
4949{
4950 VALUE recv = calling->recv;
4951
4952 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4953 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4954 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_ASET);
4955
4956 rb_check_frozen(recv);
4957
4958 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4959 RSTRUCT_SET_RAW(recv, off, val);
4960
4961 return val;
4962}
4963
4964static VALUE
4965vm_call_opt_struct_aset(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4966{
4967 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aset);
4968
4969 VALUE ret = vm_call_opt_struct_aset0(ec, calling, *(reg_cfp->sp - 1));
4970 reg_cfp->sp -= 2;
4971 return ret;
4972}
4973
4974NOINLINE(static VALUE vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4975 const struct rb_callinfo *ci, const struct rb_callcache *cc));
4976
4977#define VM_CALL_METHOD_ATTR(var, func, nohook) \
4978 if (UNLIKELY(ruby_vm_c_events_enabled > 0)) { \
4979 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, calling->recv, vm_cc_cme(cc)->def->original_id, \
4980 vm_ci_mid(ci), vm_cc_cme(cc)->owner, Qundef); \
4981 var = func; \
4982 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, calling->recv, vm_cc_cme(cc)->def->original_id, \
4983 vm_ci_mid(ci), vm_cc_cme(cc)->owner, (var)); \
4984 } \
4985 else { \
4986 nohook; \
4987 var = func; \
4988 }
4989
4990static VALUE
4991vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4992 const struct rb_callinfo *ci, const struct rb_callcache *cc)
4993{
4994 switch (vm_cc_cme(cc)->def->body.optimized.type) {
4995 case OPTIMIZED_METHOD_TYPE_SEND:
4996 CC_SET_FASTPATH(cc, vm_call_opt_send, TRUE);
4997 return vm_call_opt_send(ec, cfp, calling);
4998 case OPTIMIZED_METHOD_TYPE_CALL:
4999 CC_SET_FASTPATH(cc, vm_call_opt_call, TRUE);
5000 return vm_call_opt_call(ec, cfp, calling);
5001 case OPTIMIZED_METHOD_TYPE_BLOCK_CALL:
5002 CC_SET_FASTPATH(cc, vm_call_opt_block_call, TRUE);
5003 return vm_call_opt_block_call(ec, cfp, calling);
5004 case OPTIMIZED_METHOD_TYPE_STRUCT_AREF: {
5005 CALLER_SETUP_ARG(cfp, calling, ci, 0);
5006 rb_check_arity(calling->argc, 0, 0);
5007
5008 VALUE v;
5009 VM_CALL_METHOD_ATTR(v,
5010 vm_call_opt_struct_aref(ec, cfp, calling),
5011 set_vm_cc_ivar(cc); \
5012 CC_SET_FASTPATH(cc, vm_call_opt_struct_aref, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
5013 return v;
5014 }
5015 case OPTIMIZED_METHOD_TYPE_STRUCT_ASET: {
5016 CALLER_SETUP_ARG(cfp, calling, ci, 1);
5017 rb_check_arity(calling->argc, 1, 1);
5018
5019 VALUE v;
5020 VM_CALL_METHOD_ATTR(v,
5021 vm_call_opt_struct_aset(ec, cfp, calling),
5022 set_vm_cc_ivar(cc); \
5023 CC_SET_FASTPATH(cc, vm_call_opt_struct_aset, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
5024 return v;
5025 }
5026 default:
5027 rb_bug("vm_call_method: unsupported optimized method type (%d)", vm_cc_cme(cc)->def->body.optimized.type);
5028 }
5029}
5030
5031static VALUE
5032vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
5033{
5034 const struct rb_callinfo *ci = calling->cd->ci;
5035 const struct rb_callcache *cc = calling->cc;
5036 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
5037 VALUE v;
5038
5039 VM_ASSERT(! METHOD_ENTRY_INVALIDATED(cme));
5040
5041 switch (cme->def->type) {
5042 case VM_METHOD_TYPE_ISEQ:
5043 if (ISEQ_BODY(def_iseq_ptr(cme->def))->param.flags.forwardable) {
5044 CC_SET_FASTPATH(cc, vm_call_iseq_fwd_setup, TRUE);
5045 return vm_call_iseq_fwd_setup(ec, cfp, calling);
5046 }
5047 else {
5048 CC_SET_FASTPATH(cc, vm_call_iseq_setup, TRUE);
5049 return vm_call_iseq_setup(ec, cfp, calling);
5050 }
5051
5052 case VM_METHOD_TYPE_NOTIMPLEMENTED:
5053 case VM_METHOD_TYPE_CFUNC:
5054 CC_SET_FASTPATH(cc, vm_call_cfunc, TRUE);
5055 return vm_call_cfunc(ec, cfp, calling);
5056
5057 case VM_METHOD_TYPE_ATTRSET:
5058 CALLER_SETUP_ARG(cfp, calling, ci, 1);
5059
5060 rb_check_arity(calling->argc, 1, 1);
5061
5062 const unsigned int aset_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_KWARG | VM_CALL_FORWARDING);
5063
5064 if (vm_cc_markable(cc)) {
5065 vm_cc_attr_index_set(cc, IVAR_CACHE_INIT);
5066 VM_CALL_METHOD_ATTR(v,
5067 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
5068 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
5069 }
5070 else {
5071 cc = &((struct rb_callcache) {
5072 .flags = T_IMEMO |
5073 (imemo_callcache << FL_USHIFT) |
5074 VM_CALLCACHE_UNMARKABLE |
5075 VM_CALLCACHE_ON_STACK,
5076 .klass = cc->klass,
5077 .cme_ = cc->cme_,
5078 .call_ = cc->call_,
5079 .aux_ = {
5080 .attr = {
5081 .value = IVAR_CACHE_INIT,
5082 }
5083 },
5084 });
5085
5086 VM_CALL_METHOD_ATTR(v,
5087 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
5088 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
5089 }
5090 return v;
5091
5092 case VM_METHOD_TYPE_IVAR:
5093 CALLER_SETUP_ARG(cfp, calling, ci, 0);
5094 rb_check_arity(calling->argc, 0, 0);
5095 vm_cc_attr_index_set(cc, rb_getivar_cache_pack(ROOT_SHAPE_ID, ATTR_INDEX_NOT_SET));
5096 const unsigned int ivar_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_FORWARDING);
5097 VM_CALL_METHOD_ATTR(v,
5098 vm_call_ivar(ec, cfp, calling),
5099 CC_SET_FASTPATH(cc, vm_call_ivar, !(vm_ci_flag(ci) & ivar_mask)));
5100 return v;
5101
5102 case VM_METHOD_TYPE_MISSING:
5103 vm_cc_method_missing_reason_set(cc, 0);
5104 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
5105 return vm_call_method_missing(ec, cfp, calling);
5106
5107 case VM_METHOD_TYPE_BMETHOD:
5108 CC_SET_FASTPATH(cc, vm_call_bmethod, TRUE);
5109 return vm_call_bmethod(ec, cfp, calling);
5110
5111 case VM_METHOD_TYPE_ALIAS:
5112 CC_SET_FASTPATH(cc, vm_call_alias, TRUE);
5113 return vm_call_alias(ec, cfp, calling);
5114
5115 case VM_METHOD_TYPE_OPTIMIZED:
5116 return vm_call_optimized(ec, cfp, calling, ci, cc);
5117
5118 case VM_METHOD_TYPE_UNDEF:
5119 break;
5120
5121 case VM_METHOD_TYPE_ZSUPER:
5122 return vm_call_zsuper(ec, cfp, calling, RCLASS_ORIGIN(vm_cc_cme(cc)->defined_class));
5123
5124 case VM_METHOD_TYPE_REFINED:
5125 // CC_SET_FASTPATH(cc, vm_call_refined, TRUE);
5126 // should not set FASTPATH since vm_call_refined assumes cc->call is vm_call_super_method on invokesuper.
5127 return vm_call_refined(ec, cfp, calling);
5128 }
5129
5130 rb_bug("vm_call_method: unsupported method type (%d)", vm_cc_cme(cc)->def->type);
5131}
5132
5133NORETURN(static void vm_raise_method_missing(rb_execution_context_t *ec, int argc, const VALUE *argv, VALUE obj, int call_status));
5134
5135static VALUE
5136vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
5137{
5138 /* method missing */
5139 const struct rb_callinfo *ci = calling->cd->ci;
5140 const int stat = ci_missing_reason(ci);
5141
5142 if (vm_ci_mid(ci) == idMethodMissing) {
5143 if (UNLIKELY(calling->heap_argv)) {
5144 vm_raise_method_missing(ec, RARRAY_LENINT(calling->heap_argv), RARRAY_CONST_PTR(calling->heap_argv), calling->recv, stat);
5145 }
5146 else {
5147 rb_control_frame_t *reg_cfp = cfp;
5148 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
5149 vm_raise_method_missing(ec, calling->argc, argv, calling->recv, stat);
5150 }
5151 }
5152 else {
5153 return vm_call_method_missing_body(ec, cfp, calling, ci, stat);
5154 }
5155}
5156
5157/* Protected method calls and super invocations need to check that the receiver
5158 * (self for super) inherits the module on which the method is defined.
5159 * In the case of refinements, it should consider the original class not the
5160 * refinement.
5161 */
5162static VALUE
5163vm_defined_class_for_protected_call(const rb_callable_method_entry_t *me)
5164{
5165 VALUE defined_class = me->defined_class;
5166 VALUE refined_class = RCLASS_REFINED_CLASS(defined_class);
5167 return NIL_P(refined_class) ? defined_class : refined_class;
5168}
5169
5170static inline VALUE
5171vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
5172{
5173 const struct rb_callinfo *ci = calling->cd->ci;
5174 const struct rb_callcache *cc = calling->cc;
5175
5176 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
5177
5178 if (vm_cc_cme(cc) != NULL) {
5179 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
5180 case METHOD_VISI_PUBLIC: /* likely */
5181 return vm_call_method_each_type(ec, cfp, calling);
5182
5183 case METHOD_VISI_PRIVATE:
5184 if (!(vm_ci_flag(ci) & VM_CALL_FCALL)) {
5185 enum method_missing_reason stat = MISSING_PRIVATE;
5186 if (vm_ci_flag(ci) & VM_CALL_VCALL) stat |= MISSING_VCALL;
5187
5188 vm_cc_method_missing_reason_set(cc, stat);
5189 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
5190 return vm_call_method_missing(ec, cfp, calling);
5191 }
5192 return vm_call_method_each_type(ec, cfp, calling);
5193
5194 case METHOD_VISI_PROTECTED:
5195 if (!(vm_ci_flag(ci) & (VM_CALL_OPT_SEND | VM_CALL_FCALL))) {
5196 VALUE defined_class = vm_defined_class_for_protected_call(vm_cc_cme(cc));
5197 if (!rb_obj_is_kind_of(cfp->self, defined_class)) {
5198 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
5199 return vm_call_method_missing(ec, cfp, calling);
5200 }
5201 else {
5202 /* caching method info to dummy cc */
5203 VM_ASSERT(vm_cc_cme(cc) != NULL);
5204 struct rb_callcache cc_on_stack = *cc;
5205 FL_SET_RAW((VALUE)&cc_on_stack, VM_CALLCACHE_UNMARKABLE);
5206 calling->cc = &cc_on_stack;
5207 return vm_call_method_each_type(ec, cfp, calling);
5208 }
5209 }
5210 return vm_call_method_each_type(ec, cfp, calling);
5211
5212 default:
5213 rb_bug("unreachable");
5214 }
5215 }
5216 else {
5217 return vm_call_method_nome(ec, cfp, calling);
5218 }
5219}
5220
5221static VALUE
5222vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
5223{
5224 RB_DEBUG_COUNTER_INC(ccf_general);
5225 return vm_call_method(ec, reg_cfp, calling);
5226}
5227
5228void
5229rb_vm_cc_general(const struct rb_callcache *cc)
5230{
5231 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
5232 VM_ASSERT(cc != vm_cc_empty());
5233
5234 *(vm_call_handler *)&cc->call_ = vm_call_general;
5235}
5236
5237static VALUE
5238vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
5239{
5240 RB_DEBUG_COUNTER_INC(ccf_super_method);
5241
5242 // This line is introduced to make different from `vm_call_general` because some compilers (VC we found)
5243 // can merge the function and the address of the function becomes same.
5244 // The address of `vm_call_super_method` is used in `search_refined_method`, so it should be different.
5245 if (ec == NULL) rb_bug("unreachable");
5246
5247 /* this check is required to distinguish with other functions. */
5248 VM_ASSERT(vm_cc_call(calling->cc) == vm_call_super_method);
5249 return vm_call_method(ec, reg_cfp, calling);
5250}
5251
5252/* super */
5253
5254static inline VALUE
5255vm_search_normal_superclass(VALUE klass)
5256{
5257 if (RICLASS_FOR_REFINEMENT_P(klass)) {
5258 if (RB_TYPE_P(RCLASS_REFINED_CLASS(klass), T_MODULE)) {
5259 do {
5260 klass = RCLASS_SUPER(klass);
5261 } while (RICLASS_FOR_REFINEMENT_P(klass));
5262
5263 return klass;
5264 }
5265
5266 klass = RBASIC(klass)->klass;
5267 }
5268 klass = RCLASS_ORIGIN(klass);
5269 return RCLASS_SUPER(klass);
5270}
5271
5272NORETURN(static void vm_super_outside(void));
5273
5274static void
5275vm_super_outside(void)
5276{
5277 rb_raise(rb_eNoMethodError, "super called outside of method");
5278}
5279
5280static const struct rb_callcache *
5281empty_cc_for_super(void)
5282{
5283 return &rb_vm_empty_cc_for_super;
5284}
5285
5286static const struct rb_callcache *
5287vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *cd, VALUE recv)
5288{
5289 VALUE current_defined_class;
5290 const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp);
5291 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
5292
5293 if (!me) {
5294 vm_super_outside();
5295 }
5296
5297 current_defined_class = vm_defined_class_for_protected_call(me);
5298
5299 if (BUILTIN_TYPE(current_defined_class) != T_MODULE &&
5300 iseq != method_entry_iseqptr(me) &&
5301 !rb_obj_is_kind_of(recv, current_defined_class)) {
5302 VALUE m = RB_TYPE_P(current_defined_class, T_ICLASS) ?
5303 RCLASS_INCLUDER(current_defined_class) : current_defined_class;
5304
5305 if (m) { /* not bound UnboundMethod */
5306 rb_raise(rb_eTypeError,
5307 "self has wrong type to call super in this context: "
5308 "%"PRIsVALUE" (expected %"PRIsVALUE")",
5309 rb_obj_class(recv), m);
5310 }
5311 }
5312
5313 if (me->def->type == VM_METHOD_TYPE_BMETHOD && (vm_ci_flag(cd->ci) & VM_CALL_ZSUPER)) {
5314 rb_raise(rb_eRuntimeError,
5315 "implicit argument passing of super from method defined"
5316 " by define_method() is not supported."
5317 " Specify all arguments explicitly.");
5318 }
5319
5320 ID mid = me->def->original_id;
5321
5322 if (!vm_ci_markable(cd->ci)) {
5323 VM_FORCE_WRITE((const VALUE *)&cd->ci->mid, (VALUE)mid);
5324 }
5325 else {
5326 // update iseq. really? (TODO)
5327 cd->ci = vm_ci_new_runtime(mid,
5328 vm_ci_flag(cd->ci),
5329 vm_ci_argc(cd->ci),
5330 vm_ci_kwarg(cd->ci));
5331
5332 RB_OBJ_WRITTEN(iseq, Qundef, cd->ci);
5333 }
5334
5335 const struct rb_callcache *cc;
5336
5337 VALUE klass = vm_search_normal_superclass(me->defined_class);
5338
5339 if (!klass) {
5340 /* bound instance method of module */
5341 cc = vm_cc_new(Qundef, NULL, vm_call_method_missing, cc_type_super);
5342 RB_OBJ_WRITE(iseq, &cd->cc, cc);
5343 }
5344 else {
5345 cc = vm_search_method_fastpath(reg_cfp, cd, klass);
5346 const rb_callable_method_entry_t *cached_cme = vm_cc_cme(cc);
5347
5348 // define_method can cache for different method id
5349 if (cached_cme == NULL) {
5350 // empty_cc_for_super is not markable object
5351 cd->cc = empty_cc_for_super();
5352 }
5353 else if (cached_cme->called_id != mid) {
5354 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
5355 if (cme) {
5356 cc = vm_cc_new(klass, cme, vm_call_super_method, cc_type_super);
5357 RB_OBJ_WRITE(iseq, &cd->cc, cc);
5358 }
5359 else {
5360 cd->cc = cc = empty_cc_for_super();
5361 }
5362 }
5363 else {
5364 switch (cached_cme->def->type) {
5365 // vm_call_refined (search_refined_method) assumes cc->call is vm_call_super_method on invokesuper
5366 case VM_METHOD_TYPE_REFINED:
5367 // cc->klass is superclass of receiver class. Checking cc->klass is not enough to invalidate IVC for the receiver class.
5368 case VM_METHOD_TYPE_ATTRSET:
5369 case VM_METHOD_TYPE_IVAR:
5370 vm_cc_call_set(cc, vm_call_super_method); // invalidate fastpath
5371 break;
5372 default:
5373 break; // use fastpath
5374 }
5375 }
5376 }
5377
5378 VM_ASSERT((vm_cc_cme(cc), true));
5379
5380 return cc;
5381}
5382
5383/* yield */
5384
5385static inline int
5386block_proc_is_lambda(const VALUE procval)
5387{
5388 rb_proc_t *proc;
5389
5390 if (procval) {
5391 GetProcPtr(procval, proc);
5392 return proc->header.is_lambda;
5393 }
5394 else {
5395 return 0;
5396 }
5397}
5398
5399static VALUE
5400vm_yield_with_cfunc(rb_execution_context_t *ec,
5401 const struct rb_captured_block *captured,
5402 VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE block_handler,
5403 const rb_callable_method_entry_t *me)
5404{
5405 int is_lambda = FALSE; /* TODO */
5406 VALUE val, arg, blockarg;
5407 int frame_flag;
5408 const struct vm_ifunc *ifunc = captured->code.ifunc;
5409
5410 if (is_lambda) {
5411 arg = rb_ary_new4(argc, argv);
5412 }
5413 else if (argc == 0) {
5414 arg = Qnil;
5415 }
5416 else {
5417 arg = argv[0];
5418 }
5419
5420 blockarg = rb_vm_bh_to_procval(ec, block_handler);
5421
5422 frame_flag = VM_FRAME_MAGIC_IFUNC | VM_FRAME_FLAG_CFRAME | (me ? VM_FRAME_FLAG_BMETHOD : 0);
5423 if (kw_splat) {
5424 frame_flag |= VM_FRAME_FLAG_CFRAME_KW;
5425 }
5426
5427 vm_push_frame(ec, (const rb_iseq_t *)captured->code.ifunc,
5428 frame_flag,
5429 self,
5430 VM_GUARDED_PREV_EP(captured->ep),
5431 (VALUE)me,
5432 0, ec->cfp->sp, 0, 0);
5433 val = (*ifunc->func)(arg, (VALUE)ifunc->data, argc, argv, blockarg);
5434 rb_vm_pop_frame(ec);
5435
5436 return val;
5437}
5438
5439VALUE
5440rb_vm_yield_with_cfunc(rb_execution_context_t *ec, const struct rb_captured_block *captured, int argc, const VALUE *argv)
5441{
5442 return vm_yield_with_cfunc(ec, captured, captured->self, argc, argv, 0, VM_BLOCK_HANDLER_NONE, NULL);
5443}
5444
5445static VALUE
5446vm_yield_with_symbol(rb_execution_context_t *ec, VALUE symbol, int argc, const VALUE *argv, int kw_splat, VALUE block_handler)
5447{
5448 VALUE passed_proc = rb_vm_bh_to_procval(ec, block_handler);
5449
5450 if (!rb_box_available()) {
5451 return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc);
5452 }
5453
5454 const rb_control_frame_t *reg_cfp = ec->cfp;
5455 const rb_control_frame_t *ruby_cfp = rb_vm_get_ruby_level_next_cfp(ec, reg_cfp);
5456 const rb_box_t *box = NULL;
5457
5458 /*
5459 * Traverse the frames until a user box (Main or Optional Box) is found.
5460 * Frames for built-in methods like <internal:xxx> run in the Root or Master Box,
5461 */
5462 while (ruby_cfp) {
5463 box = current_box_on_cfp(ec, ruby_cfp);
5464 if (BOX_USER_P(box)) {
5465 break;
5466 }
5467 ruby_cfp = rb_vm_get_ruby_level_next_cfp(ec, RUBY_VM_PREVIOUS_CONTROL_FRAME(ruby_cfp));
5468 }
5469
5470 // Fallback to the normal call if no user box is found
5471 if (!ruby_cfp || !box) {
5472 return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc);
5473 }
5474
5475 /*
5476 * Push a dummy block frame whose outer env is `ruby_cfp` so that the
5477 * method resolves in the caller's box (via the ep chain) and the
5478 * backtrace reads like a usual block invocation at the caller's site.
5479 */
5480 const rb_iseq_t *caller_iseq = CFP_ISEQ(ruby_cfp);
5481 VALUE name = rb_sprintf("block in %"PRIsVALUE, rb_iseq_label(caller_iseq));
5482 const rb_iseq_t *iseq = rb_iseq_new_with_opt(Qnil, name,
5483 rb_iseq_path(caller_iseq), rb_iseq_realpath(caller_iseq),
5484 rb_vm_get_sourceline(ruby_cfp), caller_iseq, 0,
5485 ISEQ_TYPE_BLOCK, NULL, Qnil);
5486 volatile VALUE val = Qnil;
5487 enum ruby_tag_type state;
5488
5489 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_FINISH,
5490 ruby_cfp->self, VM_GUARDED_PREV_EP(ruby_cfp->ep),
5491 Qfalse, /* cref or me */
5492 ISEQ_BODY(iseq)->iseq_encoded, reg_cfp->sp,
5493 ISEQ_BODY(iseq)->local_table_size, ISEQ_BODY(iseq)->stack_max);
5494
5495 /*
5496 * The pushed frame is finished (owned by no vm_exec loop), so catch the
5497 * non-local exit here, pop the frame, and let the caller's handlers see
5498 * the exception.
5499 */
5500 EC_PUSH_TAG(ec);
5501 if ((state = EC_EXEC_TAG()) == TAG_NONE) {
5502 val = rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc);
5503 }
5504 EC_POP_TAG();
5505
5506 if (state != TAG_NONE) {
5507 rb_vm_rewind_cfp(ec, (rb_control_frame_t *)reg_cfp);
5508 EC_JUMP_TAG(ec, state);
5509 }
5510
5511 rb_vm_pop_frame(ec);
5512
5513 return val;
5514}
5515
5516static inline int
5517vm_callee_setup_block_arg_arg0_splat(rb_control_frame_t *cfp, const rb_iseq_t *iseq, VALUE *argv, VALUE ary)
5518{
5519 int i;
5520 rb_len_t len = RARRAY_LEN(ary);
5521
5522 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5523
5524 for (i=0; i<len && i<ISEQ_BODY(iseq)->param.lead_num; i++) {
5525 argv[i] = RARRAY_AREF(ary, i);
5526 }
5527
5528 return i;
5529}
5530
5531static inline VALUE
5532vm_callee_setup_block_arg_arg0_check(VALUE *argv)
5533{
5534 VALUE ary, arg0 = argv[0];
5535 ary = rb_check_array_type(arg0);
5536#if 0
5537 argv[0] = arg0;
5538#else
5539 VM_ASSERT(argv[0] == arg0);
5540#endif
5541 return ary;
5542}
5543
5544static int
5545vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type)
5546{
5547 if (rb_simple_iseq_p(iseq)) {
5548 rb_control_frame_t *cfp = ec->cfp;
5549 VALUE arg0;
5550
5551 CALLER_SETUP_ARG(cfp, calling, ci, ISEQ_BODY(iseq)->param.lead_num);
5552
5553 if (arg_setup_type == arg_setup_block &&
5554 calling->argc == 1 &&
5555 ISEQ_BODY(iseq)->param.flags.has_lead &&
5556 !ISEQ_BODY(iseq)->param.flags.ambiguous_param0 &&
5557 !NIL_P(arg0 = vm_callee_setup_block_arg_arg0_check(argv))) {
5558 calling->argc = vm_callee_setup_block_arg_arg0_splat(cfp, iseq, argv, arg0);
5559 }
5560
5561 if (calling->argc != ISEQ_BODY(iseq)->param.lead_num) {
5562 if (arg_setup_type == arg_setup_block) {
5563 if (calling->argc < ISEQ_BODY(iseq)->param.lead_num) {
5564 int i;
5565 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5566 for (i=calling->argc; i<ISEQ_BODY(iseq)->param.lead_num; i++) argv[i] = Qnil;
5567 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* fill rest parameters */
5568 }
5569 else if (calling->argc > ISEQ_BODY(iseq)->param.lead_num) {
5570 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* simply truncate arguments */
5571 }
5572 }
5573 else {
5574 argument_arity_error(ec, iseq, NULL, calling->argc, ISEQ_BODY(iseq)->param.lead_num, ISEQ_BODY(iseq)->param.lead_num);
5575 }
5576 }
5577
5578 return 0;
5579 }
5580 else {
5581 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_type);
5582 }
5583}
5584
5585static int
5586vm_yield_setup_args(rb_execution_context_t *ec, const rb_iseq_t *iseq, const int argc, VALUE *argv, int flags, VALUE block_handler, enum arg_setup_type arg_setup_type)
5587{
5588 struct rb_calling_info calling_entry, *calling;
5589
5590 calling = &calling_entry;
5591 calling->argc = argc;
5592 calling->block_handler = block_handler;
5593 calling->kw_splat = (flags & VM_CALL_KW_SPLAT) ? 1 : 0;
5594 calling->recv = Qundef;
5595 calling->heap_argv = 0;
5596 calling->cc = NULL;
5597 struct rb_callinfo dummy_ci = VM_CI_ON_STACK(0, flags, 0, 0);
5598
5599 return vm_callee_setup_block_arg(ec, calling, &dummy_ci, iseq, argv, arg_setup_type);
5600}
5601
5602/* ruby iseq -> ruby block */
5603
5604static VALUE
5605vm_invoke_iseq_block_with_cref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5606 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5607 bool is_lambda, VALUE block_handler, const rb_cref_t *cref)
5608{
5609 const struct rb_captured_block *captured = VM_BH_TO_ISEQ_BLOCK(block_handler);
5610 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
5611 const int arg_size = ISEQ_BODY(iseq)->param.size;
5612 VALUE * const rsp = GET_SP() - calling->argc;
5613 VALUE * const argv = rsp;
5614 int opt_pc = vm_callee_setup_block_arg(ec, calling, ci, iseq, argv, is_lambda ? arg_setup_method : arg_setup_block);
5615 int frame_flag = VM_FRAME_MAGIC_BLOCK | (is_lambda ? VM_FRAME_FLAG_LAMBDA : 0);
5616
5617 SET_SP(rsp);
5618
5619 /* cref is normally 0; Proc#refined supplies a refinement cref */
5620 vm_push_frame(ec, iseq,
5621 frame_flag,
5622 captured->self,
5623 VM_GUARDED_PREV_EP(captured->ep), (VALUE)cref,
5624 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
5625 rsp + arg_size,
5626 ISEQ_BODY(iseq)->local_table_size - arg_size, ISEQ_BODY(iseq)->stack_max);
5627
5628 return Qundef;
5629}
5630
5631static VALUE
5632vm_invoke_iseq_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5633 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5634 bool is_lambda, VALUE block_handler)
5635{
5636 return vm_invoke_iseq_block_with_cref(ec, reg_cfp, calling, ci, is_lambda, block_handler, NULL);
5637}
5638
5639static VALUE
5640vm_invoke_symbol_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5641 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5642 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5643{
5644 VALUE symbol = VM_BH_TO_SYMBOL(block_handler);
5645 int flags = vm_ci_flag(ci);
5646
5647 if (UNLIKELY(!(flags & VM_CALL_ARGS_SIMPLE) &&
5648 ((calling->argc == 0) ||
5649 (calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
5650 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
5651 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc))))) {
5652 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
5653 flags = 0;
5654 if (UNLIKELY(calling->heap_argv)) {
5655#if VM_ARGC_STACK_MAX < 0
5656 if (RARRAY_LEN(calling->heap_argv) < 1) {
5657 rb_raise(rb_eArgError, "no receiver given");
5658 }
5659#endif
5660 calling->recv = rb_ary_shift(calling->heap_argv);
5661 // Modify stack to avoid cfp consistency error
5662 reg_cfp->sp++;
5663 reg_cfp->sp[-1] = reg_cfp->sp[-2];
5664 reg_cfp->sp[-2] = calling->recv;
5665 flags |= VM_CALL_ARGS_SPLAT;
5666 }
5667 else {
5668 if (calling->argc < 1) {
5669 rb_raise(rb_eArgError, "no receiver given");
5670 }
5671 calling->recv = TOPN(--calling->argc);
5672 }
5673 if (calling->kw_splat) {
5674 flags |= VM_CALL_KW_SPLAT;
5675 }
5676 }
5677 else {
5678 if (calling->argc < 1) {
5679 rb_raise(rb_eArgError, "no receiver given");
5680 }
5681 calling->recv = TOPN(--calling->argc);
5682 }
5683
5684 return vm_call_symbol(ec, reg_cfp, calling, ci, symbol, flags);
5685}
5686
5687static VALUE
5688vm_invoke_ifunc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5689 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5690 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5691{
5692 VALUE val;
5693 int argc;
5694 const struct rb_captured_block *captured = VM_BH_TO_IFUNC_BLOCK(block_handler);
5695 CALLER_SETUP_ARG(ec->cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
5696 argc = calling->argc;
5697 val = vm_yield_with_cfunc(ec, captured, captured->self, CALLING_ARGC(calling), calling->heap_argv ? RARRAY_CONST_PTR(calling->heap_argv) : STACK_ADDR_FROM_TOP(argc), calling->kw_splat, calling->block_handler, NULL);
5698 POPN(argc); /* TODO: should put before C/yield? */
5699 return val;
5700}
5701
5702static inline VALUE
5703vm_block_to_block_handler(const struct rb_block *block)
5704{
5705 switch (vm_block_type(block)) {
5706 case block_type_iseq:
5707 return VM_BH_FROM_ISEQ_BLOCK(&block->as.captured);
5708 case block_type_ifunc:
5709 return VM_BH_FROM_IFUNC_BLOCK(&block->as.captured);
5710 case block_type_symbol:
5711 return VM_BH_FROM_SYMBOL(block->as.symbol);
5712 case block_type_proc:
5713 return VM_BH_FROM_PROC(block->as.proc);
5714 }
5715 VM_UNREACHABLE(vm_yield_with_proc);
5716 return Qundef;
5717}
5718
5719static VALUE
5720vm_proc_to_block_handler(VALUE procval)
5721{
5722 return vm_block_to_block_handler(vm_proc_block(procval));
5723}
5724
5725NOINLINE(static VALUE
5726vm_invoke_proc_block_with_cref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5727 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5728 bool is_lambda, VALUE block_handler, VALUE refined_procval));
5729static VALUE
5730vm_invoke_proc_block_with_cref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5731 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5732 bool is_lambda, VALUE block_handler, VALUE refined_procval)
5733{
5734 const rb_cref_t *cref = rb_proc_refinements_cref_for_call(refined_procval);
5735 return vm_invoke_iseq_block_with_cref(ec, reg_cfp, calling, ci, is_lambda, block_handler, cref);
5736}
5737
5738static VALUE
5739vm_invoke_proc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5740 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5741 bool is_lambda, VALUE block_handler)
5742{
5743 VALUE refined_procval = 0;
5744
5745 while (vm_block_handler_type(block_handler) == block_handler_type_proc) {
5746 VALUE procval = VM_BH_TO_PROC(block_handler);
5747 rb_proc_t *po;
5748 GetProcPtr(procval, po);
5749 if (po->header.is_refined) refined_procval = procval;
5750 is_lambda = po->header.is_lambda;
5751 block_handler = vm_block_to_block_handler(&po->block);
5752 }
5753
5754 if (UNLIKELY(refined_procval) && vm_block_handler_type(block_handler) == block_handler_type_iseq) {
5755 /* should not be inlined so that vm_invoke_proc_block stays leaf/frameless. */
5756 return vm_invoke_proc_block_with_cref(ec, reg_cfp, calling, ci, is_lambda, block_handler,
5757 refined_procval);
5758 }
5759
5760 return vm_invoke_block(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5761}
5762
5763static inline VALUE
5764vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5765 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5766 bool is_lambda, VALUE block_handler)
5767{
5768 VALUE (*func)(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5769 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5770 bool is_lambda, VALUE block_handler);
5771
5772 switch (vm_block_handler_type(block_handler)) {
5773 case block_handler_type_iseq: func = vm_invoke_iseq_block; break;
5774 case block_handler_type_ifunc: func = vm_invoke_ifunc_block; break;
5775 case block_handler_type_proc: func = vm_invoke_proc_block; break;
5776 case block_handler_type_symbol: func = vm_invoke_symbol_block; break;
5777 default: rb_bug("vm_invoke_block: unreachable");
5778 }
5779
5780 return func(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5781}
5782
5783static VALUE
5784vm_make_proc_with_iseq(const rb_iseq_t *blockiseq)
5785{
5786 const rb_execution_context_t *ec = GET_EC();
5787 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
5788 struct rb_captured_block *captured;
5789
5790 if (cfp == 0) {
5791 rb_bug("vm_make_proc_with_iseq: unreachable");
5792 }
5793
5794 captured = VM_CFP_TO_CAPTURED_BLOCK(cfp);
5795 captured->code.iseq = blockiseq;
5796
5797 return rb_vm_make_proc(ec, captured, rb_cProc);
5798}
5799
5800static VALUE
5801vm_once_exec(VALUE iseq)
5802{
5803 VALUE proc = vm_make_proc_with_iseq((rb_iseq_t *)iseq);
5804 return rb_proc_call_with_block(proc, 0, 0, Qnil);
5805}
5806
5807#define RUNNING_THREAD_ONCE_DONE ((rb_thread_t *)0x1)
5808
5810 rb_vm_t *vm;
5811 union iseq_inline_storage_entry *is;
5812};
5813
5814static void
5815vm_once_broadcast(rb_vm_t *vm)
5816{
5817 rb_native_mutex_lock(&vm->once_lock);
5818 rb_native_cond_broadcast(&vm->once_cond);
5819 rb_native_mutex_unlock(&vm->once_lock);
5820}
5821
5822static void *
5823vm_once_wait_no_gvl(void *ptr)
5824{
5825 struct vm_once_wait_arg *arg = (struct vm_once_wait_arg *)ptr;
5826 rb_vm_t *vm = arg->vm;
5827 rb_thread_t *running_th;
5828
5829 rb_native_mutex_lock(&vm->once_lock);
5830 running_th = rbimpl_atomic_ptr_load((void **)&arg->is->once.running_thread, RBIMPL_ATOMIC_ACQUIRE);
5831 if (running_th != NULL && running_th != RUNNING_THREAD_ONCE_DONE) {
5832 rb_native_cond_wait(&vm->once_cond, &vm->once_lock);
5833 }
5834 rb_native_mutex_unlock(&vm->once_lock);
5835 return NULL;
5836}
5837
5838static void
5839vm_once_wait_ubf(void *ptr)
5840{
5841 vm_once_broadcast((rb_vm_t *)ptr);
5842}
5843
5844static VALUE
5845vm_once_clear(VALUE data)
5846{
5847 union iseq_inline_storage_entry *is = (union iseq_inline_storage_entry *)data;
5848 rbimpl_atomic_ptr_store((volatile void **)&is->once.running_thread, NULL, RBIMPL_ATOMIC_RELEASE);
5849 vm_once_broadcast(GET_VM());
5850 return Qnil;
5851}
5852
5853/* defined insn */
5854
5855static bool
5856check_respond_to_missing(VALUE obj, VALUE v)
5857{
5858 VALUE args[2];
5859 VALUE r;
5860
5861 args[0] = obj; args[1] = Qfalse;
5862 r = rb_check_funcall(v, idRespond_to_missing, 2, args);
5863 if (!UNDEF_P(r) && RTEST(r)) {
5864 return true;
5865 }
5866 else {
5867 return false;
5868 }
5869}
5870
5871static bool
5872vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5873{
5874 VALUE klass;
5875 enum defined_type type = (enum defined_type)op_type;
5876
5877 switch (type) {
5878 case DEFINED_IVAR:
5879 return rb_ivar_defined(GET_SELF(), SYM2ID(obj));
5880 break;
5881 case DEFINED_GVAR:
5882 return rb_gvar_defined(SYM2ID(obj));
5883 break;
5884 case DEFINED_CVAR: {
5885 const rb_cref_t *cref = vm_get_cref(GET_EP());
5886 klass = vm_get_cvar_base(cref, GET_CFP(), 0);
5887 return rb_cvar_defined(klass, SYM2ID(obj));
5888 break;
5889 }
5890 case DEFINED_CONST:
5891 case DEFINED_CONST_FROM: {
5892 bool allow_nil = type == DEFINED_CONST;
5893 klass = v;
5894 return vm_get_ev_const(ec, klass, SYM2ID(obj), allow_nil, true);
5895 break;
5896 }
5897 case DEFINED_FUNC:
5898 klass = CLASS_OF(v);
5899 return rb_ec_obj_respond_to(ec, v, SYM2ID(obj), TRUE);
5900 break;
5901 case DEFINED_METHOD:{
5902 VALUE klass = CLASS_OF(v);
5903 const rb_callable_method_entry_t *cme = rb_callable_method_entry_with_refinements(klass, SYM2ID(obj), NULL);
5904
5905 if (cme) {
5906 switch (METHOD_ENTRY_VISI(cme)) {
5907 case METHOD_VISI_PRIVATE:
5908 break;
5909 case METHOD_VISI_PROTECTED:
5910 if (!rb_obj_is_kind_of(GET_SELF(), vm_defined_class_for_protected_call(cme))) {
5911 break;
5912 }
5913 case METHOD_VISI_PUBLIC:
5914 return true;
5915 break;
5916 default:
5917 rb_bug("vm_defined: unreachable: %u", (unsigned int)METHOD_ENTRY_VISI(cme));
5918 }
5919 }
5920 else {
5921 return check_respond_to_missing(obj, v);
5922 }
5923 break;
5924 }
5925 case DEFINED_YIELD:
5926 if (GET_BLOCK_HANDLER() != VM_BLOCK_HANDLER_NONE) {
5927 return true;
5928 }
5929 break;
5930 case DEFINED_ZSUPER:
5931 {
5932 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(GET_CFP());
5933
5934 if (me) {
5935 VALUE klass = vm_search_normal_superclass(me->defined_class);
5936 if (!klass) return false;
5937
5938 ID id = me->def->original_id;
5939
5940 return rb_method_boundp(klass, id, 0);
5941 }
5942 }
5943 break;
5944 case DEFINED_REF:
5945 return RTEST(vm_backref_defined(ec, GET_LEP(), FIX2INT(obj)));
5946 default:
5947 rb_bug("unimplemented defined? type (VM)");
5948 break;
5949 }
5950
5951 return false;
5952}
5953
5954bool
5955rb_vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5956{
5957 return vm_defined(ec, reg_cfp, op_type, obj, v);
5958}
5959
5960static const VALUE *
5961vm_get_ep(const VALUE *const reg_ep, rb_num_t lv)
5962{
5963 rb_num_t i;
5964 const VALUE *ep = reg_ep;
5965 for (i = 0; i < lv; i++) {
5966 ep = GET_PREV_EP(ep);
5967 }
5968 return ep;
5969}
5970
5971static VALUE
5972vm_get_special_object(const VALUE *const reg_ep,
5973 enum vm_special_object_type type)
5974{
5975 switch (type) {
5976 case VM_SPECIAL_OBJECT_VMCORE:
5977 return rb_mRubyVMFrozenCore;
5978 case VM_SPECIAL_OBJECT_CBASE:
5979 return vm_get_cbase(reg_ep);
5980 case VM_SPECIAL_OBJECT_CONST_BASE:
5981 return vm_get_const_base(reg_ep);
5982 default:
5983 rb_bug("putspecialobject insn: unknown value_type %d", type);
5984 }
5985}
5986
5987// ZJIT implementation is using the C function
5988// and needs to call a non-static function
5989VALUE
5990rb_vm_get_special_object(const VALUE *reg_ep, enum vm_special_object_type type)
5991{
5992 return vm_get_special_object(reg_ep, type);
5993}
5994
5995static VALUE
5996vm_concat_array(VALUE ary1, VALUE ary2st)
5997{
5998 const VALUE ary2 = ary2st;
5999 VALUE tmp1 = rb_check_to_array(ary1);
6000 VALUE tmp2 = rb_check_to_array(ary2);
6001
6002 if (NIL_P(tmp1)) {
6003 tmp1 = rb_ary_new3(1, ary1);
6004 }
6005 if (tmp1 == ary1) {
6006 tmp1 = rb_ary_dup(ary1);
6007 }
6008
6009 if (NIL_P(tmp2)) {
6010 return rb_ary_push(tmp1, ary2);
6011 }
6012 else {
6013 return rb_ary_concat(tmp1, tmp2);
6014 }
6015}
6016
6017static VALUE
6018vm_concat_to_array(VALUE ary1, VALUE ary2st)
6019{
6020 /* ary1 must be a newly created array */
6021 const VALUE ary2 = ary2st;
6022
6023 if (NIL_P(ary2)) return ary1;
6024
6025 VALUE tmp2 = rb_check_to_array(ary2);
6026
6027 if (NIL_P(tmp2)) {
6028 return rb_ary_push(ary1, ary2);
6029 }
6030 else {
6031 return rb_ary_concat(ary1, tmp2);
6032 }
6033}
6034
6035// YJIT implementation is using the C function
6036// and needs to call a non-static function
6037VALUE
6038rb_vm_concat_array(VALUE ary1, VALUE ary2st)
6039{
6040 return vm_concat_array(ary1, ary2st);
6041}
6042
6043VALUE
6044rb_vm_concat_to_array(VALUE ary1, VALUE ary2st)
6045{
6046 return vm_concat_to_array(ary1, ary2st);
6047}
6048
6049static VALUE
6050vm_splat_array(VALUE flag, VALUE ary)
6051{
6052 if (NIL_P(ary)) {
6053 return RTEST(flag) ? rb_ary_new() : rb_cArray_empty_frozen;
6054 }
6055 VALUE tmp = rb_check_to_array(ary);
6056 if (NIL_P(tmp)) {
6057 return rb_ary_new3(1, ary);
6058 }
6059 else if (RTEST(flag)) {
6060 return rb_ary_dup(tmp);
6061 }
6062 else {
6063 return tmp;
6064 }
6065}
6066
6067// YJIT implementation is using the C function
6068// and needs to call a non-static function
6069VALUE
6070rb_vm_splat_array(VALUE flag, VALUE ary)
6071{
6072 return vm_splat_array(flag, ary);
6073}
6074
6075static VALUE
6076vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
6077{
6078 enum vm_check_match_type type = ((int)flag) & VM_CHECKMATCH_TYPE_MASK;
6079
6080 if (flag & VM_CHECKMATCH_ARRAY) {
6081 rb_len_t i;
6082 const rb_len_t n = RARRAY_LEN(pattern);
6083
6084 for (i = 0; i < n; i++) {
6085 VALUE v = RARRAY_AREF(pattern, i);
6086 VALUE c = check_match(ec, v, target, type);
6087
6088 if (RTEST(c)) {
6089 return c;
6090 }
6091 }
6092 return Qfalse;
6093 }
6094 else {
6095 return check_match(ec, pattern, target, type);
6096 }
6097}
6098
6099VALUE
6100rb_vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
6101{
6102 return vm_check_match(ec, target, pattern, flag);
6103}
6104
6105static VALUE
6106vm_check_keyword(lindex_t bits, lindex_t idx, const VALUE *ep)
6107{
6108 const VALUE kw_bits = *(ep - bits);
6109
6110 if (FIXNUM_P(kw_bits)) {
6111 unsigned int b = (unsigned int)FIX2ULONG(kw_bits);
6112 if ((idx < VM_KW_SPECIFIED_BITS_MAX) && (b & (0x01 << idx)))
6113 return Qfalse;
6114 }
6115 else {
6116 VM_ASSERT(rb_set_p(kw_bits), "%s", rb_obj_info(kw_bits));
6117 if (rb_set_lookup(kw_bits, INT2FIX(idx))) return Qfalse;
6118 }
6119 return Qtrue;
6120}
6121
6122static void
6123vm_dtrace(rb_event_flag_t flag, rb_execution_context_t *ec)
6124{
6125 if (RUBY_DTRACE_METHOD_ENTRY_ENABLED() ||
6126 RUBY_DTRACE_METHOD_RETURN_ENABLED() ||
6127 RUBY_DTRACE_CMETHOD_ENTRY_ENABLED() ||
6128 RUBY_DTRACE_CMETHOD_RETURN_ENABLED()) {
6129
6130 switch (flag) {
6131 case RUBY_EVENT_CALL:
6132 RUBY_DTRACE_METHOD_ENTRY_HOOK(ec, 0, 0);
6133 return;
6134 case RUBY_EVENT_C_CALL:
6135 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, 0, 0);
6136 return;
6137 case RUBY_EVENT_RETURN:
6138 RUBY_DTRACE_METHOD_RETURN_HOOK(ec, 0, 0);
6139 return;
6141 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, 0, 0);
6142 return;
6143 }
6144 }
6145}
6146
6147static VALUE
6148vm_const_get_under(ID id, rb_num_t flags, VALUE cbase)
6149{
6150 if (!rb_const_defined_at(cbase, id)) {
6151 return 0;
6152 }
6153 else if (VM_DEFINECLASS_SCOPED_P(flags)) {
6154 return rb_public_const_get_at(cbase, id);
6155 }
6156 else {
6157 return rb_const_get_at(cbase, id);
6158 }
6159}
6160
6161static VALUE
6162vm_check_if_class(ID id, rb_num_t flags, VALUE super, VALUE klass)
6163{
6164 if (!RB_TYPE_P(klass, T_CLASS)) {
6165 return 0;
6166 }
6167 else if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags)) {
6168 VALUE tmp = rb_class_real(RCLASS_SUPER(klass));
6169
6170 if (tmp != super) {
6171 rb_raise(rb_eTypeError,
6172 "superclass mismatch for class %"PRIsVALUE"",
6173 rb_id2str(id));
6174 }
6175 else {
6176 return klass;
6177 }
6178 }
6179 else {
6180 return klass;
6181 }
6182}
6183
6184static VALUE
6185vm_check_if_module(ID id, VALUE mod)
6186{
6187 if (!RB_TYPE_P(mod, T_MODULE)) {
6188 return 0;
6189 }
6190 else {
6191 return mod;
6192 }
6193}
6194
6195static VALUE
6196declare_under(ID id, VALUE cbase, VALUE c)
6197{
6198 rb_set_class_path_string(c, cbase, rb_id2str(id));
6199 rb_const_set(cbase, id, c);
6200 return c;
6201}
6202
6203static VALUE
6204vm_declare_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
6205{
6206 /* new class declaration */
6207 VALUE s = VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) ? super : rb_cObject;
6208 VALUE c = declare_under(id, cbase, rb_define_class_id(id, s));
6209 rb_class_inherited(s, c);
6210 return c;
6211}
6212
6213static VALUE
6214vm_declare_module(ID id, VALUE cbase)
6215{
6216 /* new module declaration */
6217 return declare_under(id, cbase, rb_module_new());
6218}
6219
6220NORETURN(static void unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old));
6221static void
6222unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old)
6223{
6224 VALUE name = rb_id2str(id);
6225 VALUE message = rb_sprintf("%"PRIsVALUE" is not a %s",
6226 name, type);
6227 VALUE location = rb_const_source_location_at(cbase, id);
6228 if (!NIL_P(location)) {
6229 rb_str_catf(message, "\n%"PRIsVALUE":%"PRIsVALUE":"
6230 " previous definition of %"PRIsVALUE" was here",
6231 rb_ary_entry(location, 0), rb_ary_entry(location, 1), name);
6232 }
6234}
6235
6236static VALUE
6237vm_define_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
6238{
6239 VALUE klass;
6240
6241 if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) && !RB_TYPE_P(super, T_CLASS)) {
6242 rb_raise(rb_eTypeError,
6243 "superclass must be an instance of Class (given an instance of %"PRIsVALUE")",
6244 rb_obj_class(super));
6245 }
6246
6247 vm_check_if_namespace(cbase);
6248
6249 /* find klass */
6250 rb_autoload_load(cbase, id);
6251
6252 if ((klass = vm_const_get_under(id, flags, cbase)) != 0) {
6253 if (!vm_check_if_class(id, flags, super, klass))
6254 unmatched_redefinition("class", cbase, id, klass);
6255 return klass;
6256 }
6257 else {
6258 return vm_declare_class(id, flags, cbase, super);
6259 }
6260}
6261
6262static VALUE
6263vm_define_module(ID id, rb_num_t flags, VALUE cbase)
6264{
6265 VALUE mod;
6266
6267 vm_check_if_namespace(cbase);
6268 if ((mod = vm_const_get_under(id, flags, cbase)) != 0) {
6269 if (!vm_check_if_module(id, mod))
6270 unmatched_redefinition("module", cbase, id, mod);
6271 return mod;
6272 }
6273 else {
6274 return vm_declare_module(id, cbase);
6275 }
6276}
6277
6278static VALUE
6279vm_find_or_create_class_by_id(ID id,
6280 rb_num_t flags,
6281 VALUE cbase,
6282 VALUE super)
6283{
6284 rb_vm_defineclass_type_t type = VM_DEFINECLASS_TYPE(flags);
6285
6286 switch (type) {
6287 case VM_DEFINECLASS_TYPE_CLASS:
6288 /* classdef returns class scope value */
6289 return vm_define_class(id, flags, cbase, super);
6290
6291 case VM_DEFINECLASS_TYPE_SINGLETON_CLASS:
6292 /* classdef returns class scope value */
6293 return rb_singleton_class(cbase);
6294
6295 case VM_DEFINECLASS_TYPE_MODULE:
6296 /* classdef returns class scope value */
6297 return vm_define_module(id, flags, cbase);
6298
6299 default:
6300 rb_bug("unknown defineclass type: %d", (int)type);
6301 }
6302}
6303
6304static rb_method_visibility_t
6305vm_scope_visibility_get(const rb_execution_context_t *ec)
6306{
6307 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
6308
6309 if (!vm_env_cref_by_cref(cfp->ep)) {
6310 return METHOD_VISI_PUBLIC;
6311 }
6312 else {
6313 return CREF_SCOPE_VISI(vm_ec_cref(ec))->method_visi;
6314 }
6315}
6316
6317static int
6318vm_scope_module_func_check(const rb_execution_context_t *ec)
6319{
6320 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
6321
6322 if (!vm_env_cref_by_cref(cfp->ep)) {
6323 return FALSE;
6324 }
6325 else {
6326 return CREF_SCOPE_VISI(vm_ec_cref(ec))->module_func;
6327 }
6328}
6329
6330static void
6331vm_define_method(const rb_execution_context_t *ec, VALUE obj, ID id, VALUE iseqval, int is_singleton)
6332{
6333 VALUE klass;
6334 rb_method_visibility_t visi;
6335 rb_cref_t *cref = vm_ec_cref(ec);
6336
6337 if (is_singleton) {
6338 klass = rb_singleton_class(obj); /* class and frozen checked in this API */
6339 visi = METHOD_VISI_PUBLIC;
6340 }
6341 else {
6342 klass = CREF_CLASS_FOR_DEFINITION(cref);
6343 visi = vm_scope_visibility_get(ec);
6344 }
6345
6346 if (NIL_P(klass)) {
6347 rb_raise(rb_eTypeError, "no class/module to add method");
6348 }
6349
6350 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, visi);
6351 // Set max_iv_count on klasses based on number of ivar sets that are in the initialize method
6352 if (id == idInitialize && RB_TYPE_P(klass, T_CLASS) &&
6353 !RCLASS_SINGLETON_P(klass) && !RCLASS_EXPECT_NO_IVAR(klass) &&
6354 (rb_get_alloc_func(klass) == rb_class_allocate_instance)) {
6355 RCLASS_SET_MAX_IV_COUNT(klass, rb_estimate_iv_count(klass, (const rb_iseq_t *)iseqval));
6356 }
6357
6358 if (!is_singleton && vm_scope_module_func_check(ec)) {
6359 klass = rb_singleton_class(klass);
6360 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, METHOD_VISI_PUBLIC);
6361 }
6362}
6363
6364// Return the untagged block handler:
6365// * If it's VM_BLOCK_HANDLER_NONE, return nil
6366// * If it's an ISEQ or an IFUNC, fetch it from its rb_captured_block
6367// * If it's a PROC or SYMBOL, return it as is
6368VALUE
6369rb_vm_untag_block_handler(VALUE block_handler)
6370{
6371 if (VM_BLOCK_HANDLER_NONE == block_handler) return Qnil;
6372
6373 switch (vm_block_handler_type(block_handler)) {
6374 case block_handler_type_iseq:
6375 case block_handler_type_ifunc: {
6376 struct rb_captured_block *captured = VM_TAGGED_PTR_REF(block_handler, 0x03);
6377 return captured->code.val;
6378 }
6379 case block_handler_type_proc:
6380 case block_handler_type_symbol:
6381 return block_handler;
6382 default:
6383 rb_bug("rb_vm_untag_block_handler: unreachable");
6384 }
6385}
6386
6387VALUE
6388rb_vm_get_untagged_block_handler(rb_control_frame_t *reg_cfp)
6389{
6390 return rb_vm_untag_block_handler(VM_CF_BLOCK_HANDLER(reg_cfp));
6391}
6392
6393static VALUE
6394vm_invokeblock_i(struct rb_execution_context_struct *ec,
6395 struct rb_control_frame_struct *reg_cfp,
6396 struct rb_calling_info *calling)
6397{
6398 const struct rb_callinfo *ci = calling->cd->ci;
6399 VALUE block_handler = VM_CF_BLOCK_HANDLER(GET_CFP());
6400
6401 if (block_handler == VM_BLOCK_HANDLER_NONE) {
6402 rb_vm_localjump_error("no block given (yield)", Qnil, 0);
6403 }
6404 else {
6405 return vm_invoke_block(ec, GET_CFP(), calling, ci, false, block_handler);
6406 }
6407}
6408
6409enum method_explorer_type {
6410 mexp_search_method,
6411 mexp_search_invokeblock,
6412 mexp_search_super,
6413};
6414
6415ALWAYS_INLINE(static VALUE vm_sendish(struct rb_execution_context_struct *ec,
6416 struct rb_control_frame_struct *reg_cfp,
6417 struct rb_call_data *cd, VALUE block_handler,
6418 enum method_explorer_type method_explorer));
6419static inline VALUE
6420vm_sendish(
6421 struct rb_execution_context_struct *ec,
6422 struct rb_control_frame_struct *reg_cfp,
6423 struct rb_call_data *cd,
6424 VALUE block_handler,
6425 enum method_explorer_type method_explorer
6426) {
6427 VALUE val = Qundef;
6428 const struct rb_callinfo *ci = cd->ci;
6429 const struct rb_callcache *cc;
6430 int argc = vm_ci_argc(ci);
6431 VALUE recv = TOPN(argc);
6432 struct rb_calling_info calling = {
6433 .block_handler = block_handler,
6434 .kw_splat = IS_ARGS_KW_SPLAT(ci) > 0,
6435 .recv = recv,
6436 .argc = argc,
6437 .cd = cd,
6438 };
6439
6440 switch (method_explorer) {
6441 case mexp_search_method:
6442 calling.cc = cc = vm_search_method_fastpath(reg_cfp, cd, CLASS_OF(recv));
6443 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6444 break;
6445 case mexp_search_super:
6446 calling.cc = cc = vm_search_super_method(reg_cfp, cd, recv);
6447 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6448 break;
6449 case mexp_search_invokeblock:
6450 val = vm_invokeblock_i(ec, GET_CFP(), &calling);
6451 break;
6452 }
6453 return val;
6454}
6455
6456VALUE
6457rb_vm_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6458{
6459 stack_check(ec);
6460 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), cd->ci, blockiseq, false);
6461 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6462 VM_EXEC(ec, val);
6463 return val;
6464}
6465
6466// Fallback for YJIT/ZJIT, not used by the interpreter
6467VALUE
6468rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6469{
6470 stack_check(ec);
6471
6472 struct rb_forwarding_call_data adjusted_cd;
6473 struct rb_callinfo adjusted_ci;
6474
6475 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, false, &adjusted_cd, &adjusted_ci);
6476
6477 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method);
6478
6479 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6480 RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc);
6481 }
6482
6483 VM_EXEC(ec, val);
6484 return val;
6485}
6486
6487VALUE
6488rb_vm_opt_send_without_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6489{
6490 stack_check(ec);
6491 VALUE bh = VM_BLOCK_HANDLER_NONE;
6492 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6493 VM_EXEC(ec, val);
6494 return val;
6495}
6496
6497VALUE
6498rb_vm_invokesuper(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6499{
6500 stack_check(ec);
6501
6502 struct rb_callinfo adjusted_ci = VM_CI_ON_STACK(vm_ci_mid(cd->ci),
6503 vm_ci_flag(cd->ci),
6504 vm_ci_argc(cd->ci),
6505 vm_ci_kwarg(cd->ci));
6506 const struct rb_callcache *original_cc = rbimpl_atomic_ptr_load((void **)&cd->cc, RBIMPL_ATOMIC_ACQUIRE);
6507 struct rb_call_data adjusted_cd = {
6508 .ci = &adjusted_ci,
6509 .cc = original_cc,
6510 };
6511
6512 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), adjusted_cd.ci, blockiseq, true);
6513 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd, bh, mexp_search_super);
6514
6515 if (original_cc != adjusted_cd.cc && vm_cc_markable(adjusted_cd.cc)) {
6516 rbimpl_atomic_ptr_store((volatile void **)&cd->cc, (void *)adjusted_cd.cc, RBIMPL_ATOMIC_RELEASE);
6517 RB_OBJ_WRITTEN(CFP_ISEQ(GET_CFP()), Qundef, adjusted_cd.cc);
6518 }
6519
6520 VM_EXEC(ec, val);
6521 return val;
6522}
6523
6524// Fallback for YJIT/ZJIT, not used by the interpreter
6525VALUE
6526rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6527{
6528 stack_check(ec);
6529 struct rb_forwarding_call_data adjusted_cd;
6530 struct rb_callinfo adjusted_ci;
6531
6532 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, true, &adjusted_cd, &adjusted_ci);
6533
6534 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super);
6535
6536 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6537 RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc);
6538 }
6539
6540 VM_EXEC(ec, val);
6541 return val;
6542}
6543
6544VALUE
6545rb_vm_invokeblock(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6546{
6547 stack_check(ec);
6548 VALUE bh = VM_BLOCK_HANDLER_NONE;
6549 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_invokeblock);
6550 VM_EXEC(ec, val);
6551 return val;
6552}
6553
6554/* object.c */
6555VALUE rb_nil_to_s(VALUE);
6556VALUE rb_true_to_s(VALUE);
6557VALUE rb_false_to_s(VALUE);
6558/* numeric.c */
6559VALUE rb_int_to_s(int argc, VALUE *argv, VALUE x);
6560VALUE rb_fix_to_s(VALUE);
6561/* variable.c */
6562VALUE rb_mod_to_s(VALUE);
6564
6565static VALUE
6566vm_objtostring(struct rb_control_frame_struct *reg_cfp, VALUE recv, CALL_DATA cd)
6567{
6568 int type = TYPE(recv);
6569 if (type == T_STRING) {
6570 return recv;
6571 }
6572
6573 const struct rb_callable_method_entry_struct *cme = vm_search_method(reg_cfp, cd, recv);
6574
6575 switch (type) {
6576 case T_SYMBOL:
6577 if (check_method_basic_definition(cme)) {
6578 return rb_sym2str(recv);
6579 }
6580 break;
6581 case T_MODULE:
6582 case T_CLASS:
6583 if (check_cfunc(cme, rb_mod_to_s)) {
6584 // rb_mod_to_s() allocates a mutable string, but since we are only
6585 // going to use this string for interpolation, it's fine to use the
6586 // frozen string.
6587 VALUE val = rb_mod_name(recv);
6588 if (NIL_P(val)) {
6589 val = rb_mod_to_s(recv);
6590 }
6591 return val;
6592 }
6593 break;
6594 case T_NIL:
6595 if (check_cfunc(cme, rb_nil_to_s)) {
6596 return rb_nil_to_s(recv);
6597 }
6598 break;
6599 case T_TRUE:
6600 if (check_cfunc(cme, rb_true_to_s)) {
6601 return rb_true_to_s(recv);
6602 }
6603 break;
6604 case T_FALSE:
6605 if (check_cfunc(cme, rb_false_to_s)) {
6606 return rb_false_to_s(recv);
6607 }
6608 break;
6609 case T_FIXNUM:
6610 if (check_cfunc(cme, rb_int_to_s)) {
6611 return rb_fix_to_s(recv);
6612 }
6613 break;
6614 }
6615 return Qundef;
6616}
6617
6618// ZJIT implementation is using the C function
6619// and needs to call a non-static function
6620VALUE
6621rb_vm_objtostring(struct rb_control_frame_struct *reg_cfp, VALUE recv, CALL_DATA cd)
6622{
6623 return vm_objtostring(reg_cfp, recv, cd);
6624}
6625
6626static VALUE
6627vm_opt_ary_freeze(VALUE ary, int bop, ID id)
6628{
6629 if (BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
6630 return ary;
6631 }
6632 else {
6633 return Qundef;
6634 }
6635}
6636
6637static VALUE
6638vm_opt_hash_freeze(VALUE hash, int bop, ID id)
6639{
6640 if (BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
6641 return hash;
6642 }
6643 else {
6644 return Qundef;
6645 }
6646}
6647
6648static VALUE
6649vm_opt_str_freeze(VALUE str, int bop, ID id)
6650{
6651 if (BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
6652 return str;
6653 }
6654 else {
6655 return Qundef;
6656 }
6657}
6658
6659/* this macro is mandatory to use OPTIMIZED_CMP. What a design! */
6660#define id_cmp idCmp
6661
6662static VALUE
6663vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6664{
6665 if (OP_UNREDEFINED_P(INCLUDE_P, ARRAY)) {
6666 return rb_ary_includes(ary, target);
6667 }
6668 else {
6669 VALUE args[1] = {target};
6670
6671 // duparray
6672 RUBY_DTRACE_CREATE_HOOK(ARRAY, RARRAY_LEN(ary));
6673 VALUE dupary = rb_ary_resurrect(ary);
6674
6675 return rb_vm_call_with_refinements(ec, dupary, idIncludeP, 1, args, RB_NO_KEYWORDS);
6676 }
6677}
6678
6679VALUE
6680rb_vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6681{
6682 return vm_opt_duparray_include_p(ec, ary, target);
6683}
6684
6685static VALUE
6686vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6687{
6688 if (OP_UNREDEFINED_P(MAX, ARRAY)) {
6689 if (array_len == 0) {
6690 return Qnil;
6691 }
6692 else {
6693 VALUE result = *ptr;
6694 rb_snum_t i = array_len - 1;
6695 while (i-- > 0) {
6696 const VALUE v = *++ptr;
6697 if (OPTIMIZED_CMP(v, result) > 0) {
6698 result = v;
6699 }
6700 }
6701 return result;
6702 }
6703 }
6704 else {
6705 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMax, 0, NULL, RB_NO_KEYWORDS);
6706 }
6707}
6708
6709VALUE
6710rb_vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6711{
6712 return vm_opt_newarray_max(ec, array_len, ptr);
6713}
6714
6715static VALUE
6716vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6717{
6718 if (OP_UNREDEFINED_P(MIN, ARRAY)) {
6719 if (array_len == 0) {
6720 return Qnil;
6721 }
6722 else {
6723 VALUE result = *ptr;
6724 rb_snum_t i = array_len - 1;
6725 while (i-- > 0) {
6726 const VALUE v = *++ptr;
6727 if (OPTIMIZED_CMP(v, result) < 0) {
6728 result = v;
6729 }
6730 }
6731 return result;
6732 }
6733 }
6734 else {
6735 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMin, 0, NULL, RB_NO_KEYWORDS);
6736 }
6737}
6738
6739VALUE
6740rb_vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6741{
6742 return vm_opt_newarray_min(ec, array_len, ptr);
6743}
6744
6745static VALUE
6746vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6747{
6748 // If Array#hash is _not_ monkeypatched, use the optimized call
6749 if (OP_UNREDEFINED_P(HASH, ARRAY)) {
6750 return rb_ary_hash_values(array_len, ptr);
6751 }
6752 else {
6753 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idHash, 0, NULL, RB_NO_KEYWORDS);
6754 }
6755}
6756
6757VALUE
6758rb_vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6759{
6760 return vm_opt_newarray_hash(ec, array_len, ptr);
6761}
6762
6763VALUE rb_setup_fake_ary(struct RArray *fake_ary, const VALUE *list, rb_len_t len);
6764VALUE rb_ec_pack_ary(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer);
6765
6766static VALUE
6767vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6768{
6769 if (OP_UNREDEFINED_P(INCLUDE_P, ARRAY)) {
6770 struct RArray fake_ary = {RBASIC_INIT};
6771 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6772 return rb_ary_includes(ary, target);
6773 }
6774 else {
6775 VALUE args[1] = {target};
6776 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idIncludeP, 1, args, RB_NO_KEYWORDS);
6777 }
6778}
6779
6780VALUE
6781rb_vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6782{
6783 return vm_opt_newarray_include_p(ec, array_len, ptr, target);
6784}
6785
6786static VALUE
6787vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6788{
6789 if (OP_UNREDEFINED_P(PACK, ARRAY)) {
6790 struct RArray fake_ary = {RBASIC_INIT};
6791 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6792 return rb_ec_pack_ary(ec, ary, fmt, (UNDEF_P(buffer) ? Qnil : buffer));
6793 }
6794 else {
6795 // The opt_newarray_send insn drops the keyword args so we need to rebuild them.
6796 // Setup an array with room for keyword hash.
6797 VALUE args[2];
6798 args[0] = fmt;
6799 int kw_splat = RB_NO_KEYWORDS;
6800 int argc = 1;
6801
6802 if (!UNDEF_P(buffer)) {
6803 args[1] = rb_hash_new_capa(1);
6804 rb_hash_aset(args[1], ID2SYM(idBuffer), buffer);
6805 kw_splat = RB_PASS_KEYWORDS;
6806 argc++;
6807 }
6808
6809 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idPack, argc, args, kw_splat);
6810 }
6811}
6812
6813VALUE
6814rb_vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6815{
6816 return vm_opt_newarray_pack_buffer(ec, array_len, ptr, fmt, buffer);
6817}
6818
6819
6820#undef id_cmp
6821
6822static void
6823vm_track_constant_cache(ID id, void *ic)
6824{
6825 rb_vm_t *vm = GET_VM();
6826 struct rb_id_table *const_cache = &vm->constant_cache;
6827 VALUE lookup_result;
6828 set_table *ics;
6829
6830 if (rb_id_table_lookup(const_cache, id, &lookup_result)) {
6831 ics = (set_table *)lookup_result;
6832 }
6833 else {
6834 ics = set_init_numtable();
6835 rb_id_table_insert(const_cache, id, (VALUE)ics);
6836 }
6837
6838 /* The call below to st_insert could allocate which could trigger a GC.
6839 * If it triggers a GC, it may free an iseq that also holds a cache to this
6840 * constant. If that iseq is the last iseq with a cache to this constant, then
6841 * it will free this ST table, which would cause an use-after-free during this
6842 * st_insert.
6843 *
6844 * So to fix this issue, we store the ID that is currently being inserted
6845 * and, in remove_from_constant_cache, we don't free the ST table for ID
6846 * equal to this one.
6847 *
6848 * See [Bug #20921].
6849 */
6850 vm->inserting_constant_cache_id = id;
6851
6852 set_insert(ics, (st_data_t)ic);
6853
6854 vm->inserting_constant_cache_id = (ID)0;
6855}
6856
6857static void
6858vm_ic_track_const_chain(rb_control_frame_t *cfp, IC ic, const ID *segments)
6859{
6860 RB_VM_LOCKING() {
6861 for (int i = 0; segments[i]; i++) {
6862 ID id = segments[i];
6863 if (id == idNULL) continue;
6864 vm_track_constant_cache(id, ic);
6865 }
6866 }
6867}
6868
6869// For JIT inlining
6870static inline bool
6871vm_inlined_ic_hit_p(VALUE flags, VALUE value, const rb_cref_t *ic_cref, rb_serial_t ractor_id, const VALUE *reg_ep)
6872{
6873 // Not rb_ractor_main_p(): an owner Ractor may now cache an unshareable constant
6874 // of its own class, and only it may be handed that value back.
6875 if ((flags & IMEMO_CONST_CACHE_SHAREABLE) || ractor_id == rb_ractor_id(GET_RACTOR())) {
6876 VM_ASSERT(ractor_incidental_shareable_p(flags & IMEMO_CONST_CACHE_SHAREABLE, value));
6877
6878 return (ic_cref == NULL || // no need to check CREF
6879 ic_cref == vm_get_cref(reg_ep));
6880 }
6881 return false;
6882}
6883
6884static bool
6885vm_ic_hit_p(const struct iseq_inline_constant_cache_entry *ice, const VALUE *reg_ep)
6886{
6887 VM_ASSERT(IMEMO_TYPE_P(ice, imemo_constcache));
6888 return vm_inlined_ic_hit_p(ice->flags, ice->value, ice->ic_cref, ice->ractor_id, reg_ep);
6889}
6890
6891// YJIT needs this function to never allocate and never raise
6892bool
6893rb_vm_ic_hit_p(IC ic, const VALUE *reg_ep)
6894{
6895 return ic->entry && vm_ic_hit_p(ic->entry, reg_ep);
6896}
6897
6898static void
6899vm_ic_update(const rb_execution_context_t *ec, const rb_iseq_t *iseq, IC ic, VALUE val, const VALUE *reg_ep, const VALUE *pc)
6900{
6901 if (ruby_vm_const_missing_count > 0) {
6902 ruby_vm_const_missing_count = 0;
6903 ic->entry = NULL;
6904 return;
6905 }
6906
6907 struct iseq_inline_constant_cache_entry *ice = SHAREABLE_IMEMO_NEW(struct iseq_inline_constant_cache_entry, imemo_constcache, 0);
6908 RB_OBJ_WRITE(ice, &ice->value, val);
6909 ice->ic_cref = vm_get_const_key_cref(reg_ep);
6910 ice->ractor_id = rb_ractor_id(rb_ec_ractor_ptr(ec));
6911
6912 if (rb_ractor_shareable_p(val)) {
6913 RUBY_ASSERT((rb_gc_verify_shareable(val), 1));
6914 ice->flags |= IMEMO_CONST_CACHE_SHAREABLE;
6915 }
6916 RB_OBJ_WRITE(iseq, &ic->entry, ice);
6917 RUBY_ASSERT(pc >= ISEQ_BODY(iseq)->iseq_encoded);
6918 unsigned pos = (unsigned)(pc - ISEQ_BODY(iseq)->iseq_encoded);
6919 rb_yjit_constant_ic_update(iseq, ic, pos);
6920}
6921
6922VALUE
6923rb_vm_opt_getconstant_path(rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, IC ic)
6924{
6925 VALUE val;
6926 const ID *segments = ic->segments;
6927 struct iseq_inline_constant_cache_entry *ice = ic->entry;
6928
6929 if (ice && vm_ic_hit_p(ice, GET_EP())) {
6930 val = ice->value;
6931
6932 // On a non-main ractor another ractor can concurrently reassign a
6933 // shareable constant (which invalidates ICs asynchronously), so the
6934 // cached-but-still-shareable value may legitimately differ from a
6935 // freshly recomputed chain. Only assert equality when single-ractor.
6936 VM_ASSERT(val == vm_get_ev_const_chain(ec, segments) || rb_multi_ractor_p());
6937 }
6938 else {
6939 ruby_vm_constant_cache_misses++;
6940 val = vm_get_ev_const_chain(ec, segments);
6941 vm_ic_track_const_chain(GET_CFP(), ic, segments);
6942 // Undo the PC increment to get the address to this instruction
6943 // INSN_ATTR(width) == 2
6944 vm_ic_update(ec, CFP_ISEQ(GET_CFP()), ic, val, GET_EP(), CFP_PC(GET_CFP()) - 2);
6945 }
6946 return val;
6947}
6948
6949// Return true if the once value is already computed and set *result to the value.
6950// Otherwise, return false.
6951// Used for ZJIT. Keep in sync with `vm_once_dispatch` below.
6952bool
6953rb_vm_once_done_value(ISE is, VALUE *result)
6954{
6955 rb_thread_t *running_th = rbimpl_atomic_ptr_load((void**)&is->once.running_thread, RBIMPL_ATOMIC_ACQUIRE);
6956 if (running_th == RUNNING_THREAD_ONCE_DONE) {
6957 *result = is->once.value;
6958 return true;
6959 }
6960 return false;
6961}
6962
6963static VALUE
6964vm_once_dispatch(rb_execution_context_t *ec, ISEQ iseq, ISE is)
6965{
6966 rb_thread_t *th = rb_ec_thread_ptr(ec);
6967 rb_thread_t *running_th;
6968
6969 again:
6970 running_th = rbimpl_atomic_ptr_load((void**)&is->once.running_thread, RBIMPL_ATOMIC_ACQUIRE);
6971 if (running_th == RUNNING_THREAD_ONCE_DONE) {
6972 return is->once.value;
6973 }
6974 else if (running_th == NULL) {
6975 VALUE val = Qundef;
6976 enum ruby_tag_type state;
6977 if (rbimpl_atomic_ptr_cas((void**)&is->once.running_thread, running_th, th, RBIMPL_ATOMIC_RELEASE, RBIMPL_ATOMIC_RELAXED) != running_th) {
6978 goto again;
6979 }
6980 EC_PUSH_TAG(ec);
6981 if ((state = EC_EXEC_TAG()) == TAG_NONE) {
6982 val = vm_once_exec((VALUE)iseq);
6983 }
6984 EC_POP_TAG();
6985 if (state != TAG_NONE) {
6986 vm_once_clear((VALUE)is);
6987 EC_JUMP_TAG(ec, state);
6988 }
6989
6990 // TODO: confirm that it is shareable
6991 if (RB_FL_ABLE(val)) {
6993 }
6994
6995 RB_OBJ_WRITE(CFP_ISEQ(ec->cfp), &is->once.value, val);
6996
6997 rbimpl_atomic_ptr_store((volatile void**)&is->once.running_thread, RUNNING_THREAD_ONCE_DONE, RBIMPL_ATOMIC_RELEASE); /* success */
6998 vm_once_broadcast(rb_ec_vm_ptr(ec));
6999 return val;
7000 }
7001 else if (running_th == th) {
7002 /* recursive once */
7003 return vm_once_exec((VALUE)iseq);
7004 }
7005 else {
7006 /* wait for the winner to finish */
7007 struct vm_once_wait_arg arg = { .vm = rb_ec_vm_ptr(ec), .is = is };
7008 rb_nogvl(vm_once_wait_no_gvl, &arg, vm_once_wait_ubf, arg.vm,
7010 RUBY_VM_CHECK_INTS(ec);
7011 goto again;
7012 }
7013}
7014
7015static OFFSET
7016vm_case_dispatch(CDHASH hash, OFFSET else_offset, VALUE key)
7017{
7018 switch (OBJ_BUILTIN_TYPE(key)) {
7019 case -1:
7020 case T_FLOAT:
7021 case T_SYMBOL:
7022 case T_BIGNUM:
7023 case T_STRING:
7024 if (BASIC_OP_UNREDEFINED_P(BOP_EQQ,
7025 SYMBOL_REDEFINED_OP_FLAG |
7026 INTEGER_REDEFINED_OP_FLAG |
7027 FLOAT_REDEFINED_OP_FLAG |
7028 NIL_REDEFINED_OP_FLAG |
7029 TRUE_REDEFINED_OP_FLAG |
7030 FALSE_REDEFINED_OP_FLAG |
7031 STRING_REDEFINED_OP_FLAG)) {
7032 st_data_t val;
7033 if (RB_FLOAT_TYPE_P(key)) {
7034 double kval = RFLOAT_VALUE(key);
7035 if (!isinf(kval) && modf(kval, &kval) == 0.0) {
7036 key = FIXABLE(kval) ? LONG2FIX((long)kval) : rb_dbl2big(kval);
7037 }
7038 }
7039 if (st_lookup(rb_imemo_cdhash_tbl(hash), key, &val)) {
7040 return (VALUE)val;
7041 }
7042 else {
7043 return else_offset;
7044 }
7045 }
7046 }
7047 return 0;
7048}
7049
7050NORETURN(static void
7051 vm_stack_consistency_error(const rb_execution_context_t *ec,
7052 const rb_control_frame_t *,
7053 const VALUE *));
7054static void
7055vm_stack_consistency_error(const rb_execution_context_t *ec,
7056 const rb_control_frame_t *cfp,
7057 const VALUE *bp)
7058{
7059 const ptrdiff_t nsp = VM_SP_CNT(ec, cfp->sp);
7060 const ptrdiff_t nbp = VM_SP_CNT(ec, bp);
7061 static const char stack_consistency_error[] =
7062 "Stack consistency error (sp: %"PRIdPTRDIFF", bp: %"PRIdPTRDIFF")";
7063#if defined RUBY_DEVEL
7064 VALUE mesg = rb_sprintf(stack_consistency_error, nsp, nbp);
7065 rb_str_cat_cstr(mesg, "\n");
7066 rb_str_append(mesg, rb_iseq_disasm(CFP_ISEQ(cfp)));
7068#else
7069 rb_bug(stack_consistency_error, nsp, nbp);
7070#endif
7071}
7072
7073static VALUE
7074vm_opt_plus(VALUE recv, VALUE obj)
7075{
7076 if (FIXNUM_2_P(recv, obj) &&
7077 OP_UNREDEFINED_P(PLUS, INTEGER)) {
7078 return rb_fix_plus_fix(recv, obj);
7079 }
7080 else if (FLONUM_2_P(recv, obj) &&
7081 OP_UNREDEFINED_P(PLUS, FLOAT)) {
7082 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
7083 }
7084 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7085 return Qundef;
7086 }
7087 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7088 RBASIC_CLASS(obj) == rb_cFloat &&
7089 OP_UNREDEFINED_P(PLUS, FLOAT)) {
7090 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
7091 }
7092 else if (RBASIC_CLASS(recv) == rb_cString &&
7093 RBASIC_CLASS(obj) == rb_cString &&
7094 OP_UNREDEFINED_P(PLUS, STRING)) {
7095 return rb_str_opt_plus(recv, obj);
7096 }
7097 else if (RBASIC_CLASS(recv) == rb_cArray &&
7098 RBASIC_CLASS(obj) == rb_cArray &&
7099 OP_UNREDEFINED_P(PLUS, ARRAY)) {
7100 return rb_ary_plus(recv, obj);
7101 }
7102 else {
7103 return Qundef;
7104 }
7105}
7106
7107static VALUE
7108vm_opt_minus(VALUE recv, VALUE obj)
7109{
7110 if (FIXNUM_2_P(recv, obj) &&
7111 OP_UNREDEFINED_P(MINUS, INTEGER)) {
7112 return rb_fix_minus_fix(recv, obj);
7113 }
7114 else if (FLONUM_2_P(recv, obj) &&
7115 OP_UNREDEFINED_P(MINUS, FLOAT)) {
7116 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
7117 }
7118 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7119 return Qundef;
7120 }
7121 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7122 RBASIC_CLASS(obj) == rb_cFloat &&
7123 OP_UNREDEFINED_P(MINUS, FLOAT)) {
7124 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
7125 }
7126 else {
7127 return Qundef;
7128 }
7129}
7130
7131static VALUE
7132vm_opt_mult(VALUE recv, VALUE obj)
7133{
7134 if (FIXNUM_2_P(recv, obj) &&
7135 OP_UNREDEFINED_P(MULT, INTEGER)) {
7136 return rb_fix_mul_fix(recv, obj);
7137 }
7138 else if (FLONUM_2_P(recv, obj) &&
7139 OP_UNREDEFINED_P(MULT, FLOAT)) {
7140 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
7141 }
7142 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7143 return Qundef;
7144 }
7145 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7146 RBASIC_CLASS(obj) == rb_cFloat &&
7147 OP_UNREDEFINED_P(MULT, FLOAT)) {
7148 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
7149 }
7150 else {
7151 return Qundef;
7152 }
7153}
7154
7155static VALUE
7156vm_opt_div(VALUE recv, VALUE obj)
7157{
7158 if (FIXNUM_2_P(recv, obj) &&
7159 OP_UNREDEFINED_P(DIV, INTEGER)) {
7160 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_div_fix(recv, obj);
7161 }
7162 else if (FLONUM_2_P(recv, obj) &&
7163 OP_UNREDEFINED_P(DIV, FLOAT)) {
7164 return rb_flo_div_flo(recv, obj);
7165 }
7166 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7167 return Qundef;
7168 }
7169 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7170 RBASIC_CLASS(obj) == rb_cFloat &&
7171 OP_UNREDEFINED_P(DIV, FLOAT)) {
7172 return rb_flo_div_flo(recv, obj);
7173 }
7174 else {
7175 return Qundef;
7176 }
7177}
7178
7179static VALUE
7180vm_opt_mod(VALUE recv, VALUE obj)
7181{
7182 if (FIXNUM_2_P(recv, obj) &&
7183 OP_UNREDEFINED_P(MOD, INTEGER)) {
7184 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_mod_fix(recv, obj);
7185 }
7186 else if (FLONUM_2_P(recv, obj) &&
7187 OP_UNREDEFINED_P(MOD, FLOAT)) {
7188 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
7189 }
7190 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7191 return Qundef;
7192 }
7193 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7194 RBASIC_CLASS(obj) == rb_cFloat &&
7195 OP_UNREDEFINED_P(MOD, FLOAT)) {
7196 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
7197 }
7198 else {
7199 return Qundef;
7200 }
7201}
7202
7203static VALUE
7204vm_opt_neq(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, CALL_DATA cd_eq, VALUE recv, VALUE obj)
7205{
7206 if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_not_equal)) {
7207 VALUE val = opt_equality(reg_cfp, recv, obj, cd_eq);
7208
7209 if (!UNDEF_P(val)) {
7210 return RBOOL(!RTEST(val));
7211 }
7212 }
7213
7214 return Qundef;
7215}
7216
7217static VALUE
7218vm_opt_lt(VALUE recv, VALUE obj)
7219{
7220 if (FIXNUM_2_P(recv, obj) &&
7221 OP_UNREDEFINED_P(LT, INTEGER)) {
7222 return RBOOL((SIGNED_VALUE)recv < (SIGNED_VALUE)obj);
7223 }
7224 else if (FLONUM_2_P(recv, obj) &&
7225 OP_UNREDEFINED_P(LT, FLOAT)) {
7226 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
7227 }
7228 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7229 return Qundef;
7230 }
7231 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7232 RBASIC_CLASS(obj) == rb_cFloat &&
7233 OP_UNREDEFINED_P(LT, FLOAT)) {
7234 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
7235 }
7236 else {
7237 return Qundef;
7238 }
7239}
7240
7241static VALUE
7242vm_opt_le(VALUE recv, VALUE obj)
7243{
7244 if (FIXNUM_2_P(recv, obj) &&
7245 OP_UNREDEFINED_P(LE, INTEGER)) {
7246 return RBOOL((SIGNED_VALUE)recv <= (SIGNED_VALUE)obj);
7247 }
7248 else if (FLONUM_2_P(recv, obj) &&
7249 OP_UNREDEFINED_P(LE, FLOAT)) {
7250 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
7251 }
7252 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7253 return Qundef;
7254 }
7255 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7256 RBASIC_CLASS(obj) == rb_cFloat &&
7257 OP_UNREDEFINED_P(LE, FLOAT)) {
7258 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
7259 }
7260 else {
7261 return Qundef;
7262 }
7263}
7264
7265static VALUE
7266vm_opt_gt(VALUE recv, VALUE obj)
7267{
7268 if (FIXNUM_2_P(recv, obj) &&
7269 OP_UNREDEFINED_P(GT, INTEGER)) {
7270 return RBOOL((SIGNED_VALUE)recv > (SIGNED_VALUE)obj);
7271 }
7272 else if (FLONUM_2_P(recv, obj) &&
7273 OP_UNREDEFINED_P(GT, FLOAT)) {
7274 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
7275 }
7276 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7277 return Qundef;
7278 }
7279 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7280 RBASIC_CLASS(obj) == rb_cFloat &&
7281 OP_UNREDEFINED_P(GT, FLOAT)) {
7282 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
7283 }
7284 else {
7285 return Qundef;
7286 }
7287}
7288
7289static VALUE
7290vm_opt_ge(VALUE recv, VALUE obj)
7291{
7292 if (FIXNUM_2_P(recv, obj) &&
7293 OP_UNREDEFINED_P(GE, INTEGER)) {
7294 return RBOOL((SIGNED_VALUE)recv >= (SIGNED_VALUE)obj);
7295 }
7296 else if (FLONUM_2_P(recv, obj) &&
7297 OP_UNREDEFINED_P(GE, FLOAT)) {
7298 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
7299 }
7300 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7301 return Qundef;
7302 }
7303 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7304 RBASIC_CLASS(obj) == rb_cFloat &&
7305 OP_UNREDEFINED_P(GE, FLOAT)) {
7306 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
7307 }
7308 else {
7309 return Qundef;
7310 }
7311}
7312
7313
7314static VALUE
7315vm_opt_ltlt(VALUE recv, VALUE obj)
7316{
7317 if (SPECIAL_CONST_P(recv)) {
7318 return Qundef;
7319 }
7320 else if (RBASIC_CLASS(recv) == rb_cString &&
7321 OP_UNREDEFINED_P(LTLT, STRING)) {
7322 if (LIKELY(RB_TYPE_P(obj, T_STRING))) {
7323 return rb_str_buf_append(recv, obj);
7324 }
7325 else {
7326 return rb_str_concat(recv, obj);
7327 }
7328 }
7329 else if (RBASIC_CLASS(recv) == rb_cArray &&
7330 OP_UNREDEFINED_P(LTLT, ARRAY)) {
7331 return rb_ary_push(recv, obj);
7332 }
7333 else {
7334 return Qundef;
7335 }
7336}
7337
7338static VALUE
7339vm_opt_and(VALUE recv, VALUE obj)
7340{
7341 // If recv and obj are both fixnums, then the bottom tag bit
7342 // will be 1 on both. 1 & 1 == 1, so the result value will also
7343 // be a fixnum. If either side is *not* a fixnum, then the tag bit
7344 // will be 0, and we return Qundef.
7345 VALUE ret = ((SIGNED_VALUE) recv) & ((SIGNED_VALUE) obj);
7346
7347 if (FIXNUM_P(ret) &&
7348 OP_UNREDEFINED_P(AND, INTEGER)) {
7349 return ret;
7350 }
7351 else {
7352 return Qundef;
7353 }
7354}
7355
7356static VALUE
7357vm_opt_or(VALUE recv, VALUE obj)
7358{
7359 if (FIXNUM_2_P(recv, obj) &&
7360 OP_UNREDEFINED_P(OR, INTEGER)) {
7361 return recv | obj;
7362 }
7363 else {
7364 return Qundef;
7365 }
7366}
7367
7368static VALUE
7369vm_opt_aref(VALUE recv, VALUE obj)
7370{
7371 if (SPECIAL_CONST_P(recv)) {
7372 if (FIXNUM_2_P(recv, obj) &&
7373 OP_UNREDEFINED_P(AREF, INTEGER)) {
7374 return rb_fix_aref(recv, obj);
7375 }
7376 return Qundef;
7377 }
7378 else if (RBASIC_CLASS(recv) == rb_cArray &&
7379 OP_UNREDEFINED_P(AREF, ARRAY)) {
7380 if (FIXNUM_P(obj)) {
7381 return rb_ary_entry_internal(recv, FIX2LONG(obj));
7382 }
7383 else {
7384 return rb_ary_aref1(recv, obj);
7385 }
7386 }
7387 else if (RBASIC_CLASS(recv) == rb_cHash &&
7388 OP_UNREDEFINED_P(AREF, HASH)) {
7389 return rb_hash_aref(recv, obj);
7390 }
7391 else {
7392 return Qundef;
7393 }
7394}
7395
7396static VALUE
7397vm_opt_aset(VALUE recv, VALUE obj, VALUE set)
7398{
7399 if (SPECIAL_CONST_P(recv)) {
7400 return Qundef;
7401 }
7402 else if (RBASIC_CLASS(recv) == rb_cArray &&
7403 OP_UNREDEFINED_P(ASET, ARRAY) &&
7404 FIXNUM_P(obj)) {
7405 rb_ary_store(recv, FIX2LONG(obj), set);
7406 return set;
7407 }
7408 else if (RBASIC_CLASS(recv) == rb_cHash &&
7409 OP_UNREDEFINED_P(ASET, HASH)) {
7410 rb_hash_aset(recv, obj, set);
7411 return set;
7412 }
7413 else {
7414 return Qundef;
7415 }
7416}
7417
7418static VALUE
7419vm_opt_length(VALUE recv, int bop)
7420{
7421 if (SPECIAL_CONST_P(recv)) {
7422 return Qundef;
7423 }
7424 else if (RBASIC_CLASS(recv) == rb_cString &&
7425 BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
7426 if (bop == BOP_EMPTY_P) {
7427 return LEN2NUM(RSTRING_LEN(recv));
7428 }
7429 else {
7430 return rb_str_length(recv);
7431 }
7432 }
7433 else if (RBASIC_CLASS(recv) == rb_cArray &&
7434 BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
7435 return LEN2NUM(RARRAY_LEN(recv));
7436 }
7437 else if (RBASIC_CLASS(recv) == rb_cHash &&
7438 BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
7439 return INT2FIX(RHASH_SIZE(recv));
7440 }
7441 else {
7442 return Qundef;
7443 }
7444}
7445
7446static VALUE
7447vm_opt_empty_p(VALUE recv)
7448{
7449 switch (vm_opt_length(recv, BOP_EMPTY_P)) {
7450 case Qundef: return Qundef;
7451 case INT2FIX(0): return Qtrue;
7452 default: return Qfalse;
7453 }
7454}
7455
7456VALUE rb_false(VALUE obj);
7457
7458static VALUE
7459vm_opt_nil_p(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv)
7460{
7461 if (NIL_P(recv) &&
7462 OP_UNREDEFINED_P(NIL_P, NIL)) {
7463 return Qtrue;
7464 }
7465 else if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_false)) {
7466 return Qfalse;
7467 }
7468 else {
7469 return Qundef;
7470 }
7471}
7472
7473static VALUE
7474fix_succ(VALUE x)
7475{
7476 switch (x) {
7477 case ~0UL:
7478 /* 0xFFFF_FFFF == INT2FIX(-1)
7479 * `-1.succ` is of course 0. */
7480 return INT2FIX(0);
7481 case RSHIFT(~0UL, 1):
7482 /* 0x7FFF_FFFF == LONG2FIX(0x3FFF_FFFF)
7483 * 0x3FFF_FFFF + 1 == 0x4000_0000, which is a Bignum. */
7484 return rb_uint2big(1UL << (SIZEOF_LONG * CHAR_BIT - 2));
7485 default:
7486 /* LONG2FIX(FIX2LONG(x)+FIX2LONG(y))
7487 * == ((lx*2+1)/2 + (ly*2+1)/2)*2+1
7488 * == lx*2 + ly*2 + 1
7489 * == (lx*2+1) + (ly*2+1) - 1
7490 * == x + y - 1
7491 *
7492 * Here, if we put y := INT2FIX(1):
7493 *
7494 * == x + INT2FIX(1) - 1
7495 * == x + 2 .
7496 */
7497 return x + 2;
7498 }
7499}
7500
7501static VALUE
7502vm_opt_succ(VALUE recv)
7503{
7504 if (FIXNUM_P(recv) &&
7505 OP_UNREDEFINED_P(SUCC, INTEGER)) {
7506 return fix_succ(recv);
7507 }
7508 else if (SPECIAL_CONST_P(recv)) {
7509 return Qundef;
7510 }
7511 else if (RBASIC_CLASS(recv) == rb_cString &&
7512 OP_UNREDEFINED_P(SUCC, STRING)) {
7513 return rb_str_succ(recv);
7514 }
7515 else {
7516 return Qundef;
7517 }
7518}
7519
7520static VALUE
7521vm_opt_not(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv)
7522{
7523 if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_not)) {
7524 return RBOOL(!RTEST(recv));
7525 }
7526 else {
7527 return Qundef;
7528 }
7529}
7530
7531static VALUE
7532vm_opt_regexpmatch2(VALUE recv, VALUE obj)
7533{
7534 if (SPECIAL_CONST_P(recv)) {
7535 return Qundef;
7536 }
7537 else if (RBASIC_CLASS(recv) == rb_cString &&
7538 CLASS_OF(obj) == rb_cRegexp &&
7539 OP_UNREDEFINED_P(MATCH, STRING)) {
7540 return rb_reg_match(obj, recv);
7541 }
7542 else if (RBASIC_CLASS(recv) == rb_cRegexp &&
7543 OP_UNREDEFINED_P(MATCH, REGEXP)) {
7544 return rb_reg_match(recv, obj);
7545 }
7546 else {
7547 return Qundef;
7548 }
7549}
7550
7551#undef EQ_UNREDEFINED_P
7552#undef OP_UNREDEFINED_P
7553
7554rb_event_flag_t rb_iseq_event_flags(const rb_iseq_t *iseq, size_t pos);
7555
7556NOINLINE(static void vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp));
7557
7558static inline void
7559vm_trace_hook(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE *pc,
7560 rb_event_flag_t pc_events, rb_event_flag_t target_event,
7561 rb_hook_list_t *global_hooks, rb_hook_list_t *local_hooks, VALUE val)
7562{
7563 rb_event_flag_t event = pc_events & target_event;
7564 VALUE self = GET_SELF();
7565
7566 VM_ASSERT(rb_popcount64((uint64_t)event) == 1);
7567
7568 if (local_hooks) local_hooks->running++; // make sure they don't get deleted while global hooks run
7569
7570 if (event & global_hooks->events) {
7571 /* increment PC because source line is calculated with PC-1 */
7572 reg_cfp->pc++;
7573 vm_dtrace(event, ec);
7574 rb_exec_event_hook_orig(ec, global_hooks, event, self, 0, 0, 0 , val, 0);
7575 reg_cfp->pc--;
7576 }
7577
7578 if (local_hooks) local_hooks->running--;
7579 if (local_hooks != NULL) {
7580 if (event & local_hooks->events) {
7581 /* increment PC because source line is calculated with PC-1 */
7582 reg_cfp->pc++;
7583 rb_exec_event_hook_orig(ec, local_hooks, event, self, 0, 0, 0 , val, 0);
7584 reg_cfp->pc--;
7585 }
7586 }
7587}
7588
7589/* Re-fetch the iseq-local hook list before each event: a hook fired by an
7590 * earlier event at this pc can disable the last targeted TracePoint on this
7591 * iseq, which frees local_hooks. Reload from its stable source (the ractor's
7592 * targeted-hooks table) so we never dereference a dangling pointer. */
7593#define VM_TRACE_HOOK(target_event, val) do { \
7594 if ((pc_events & (target_event)) & enabled_flags) { \
7595 if (local_hooks_cnt > 0) local_hooks = rb_iseq_local_hooks(iseq, r, false); \
7596 vm_trace_hook(ec, reg_cfp, pc, pc_events, (target_event), global_hooks, local_hooks, (val)); \
7597 } \
7598} while (0)
7599
7600static VALUE
7601rescue_errinfo(rb_execution_context_t *ec, rb_control_frame_t *cfp)
7602{
7603 VM_ASSERT(VM_FRAME_RUBYFRAME_P(cfp));
7604 VM_ASSERT(ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_RESCUE);
7605 return cfp->ep[VM_ENV_INDEX_LAST_LVAR];
7606}
7607
7608static void
7609vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp)
7610{
7611 const VALUE *pc = reg_cfp->pc;
7612 rb_ractor_t *r = rb_ec_ractor_ptr(ec);
7613 rb_event_flag_t enabled_flags = r->pub.hooks.events & ISEQ_TRACE_EVENTS;
7614 rb_event_flag_t ractor_events = enabled_flags;
7615
7616 if (enabled_flags == 0 && rb_ractor_targeted_hooks_cnt(r) == 0) {
7617 return;
7618 }
7619 else {
7620 const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp);
7621 size_t pos = pc - ISEQ_BODY(iseq)->iseq_encoded;
7622 rb_event_flag_t pc_events = rb_iseq_event_flags(iseq, pos);
7623 unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt;
7624 rb_hook_list_t *local_hooks = NULL;
7625 if (RB_UNLIKELY(local_hooks_cnt > 0)) {
7626 st_data_t val;
7627 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)iseq, &val)) {
7628 local_hooks = (rb_hook_list_t*)val;
7629 }
7630 }
7631 rb_event_flag_t iseq_local_events = local_hooks != NULL ? local_hooks->events : 0;
7632
7633 rb_hook_list_t *bmethod_local_hooks = NULL;
7634 rb_event_flag_t bmethod_local_events = 0;
7635 unsigned int bmethod_hooks_cnt = 0;
7636 rb_method_definition_t *bmethod_def = NULL;
7637 const bool bmethod_frame = VM_FRAME_BMETHOD_P(reg_cfp);
7638 enabled_flags |= iseq_local_events;
7639
7640 VM_ASSERT((iseq_local_events & ~ISEQ_TRACE_EVENTS) == 0);
7641
7642 if (bmethod_frame) {
7643 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
7644 VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD);
7645 bmethod_def = me->def;
7646 bmethod_hooks_cnt = me->def->body.bmethod.local_hooks_cnt;
7647 if (RB_UNLIKELY(bmethod_hooks_cnt > 0)) {
7648 st_data_t val;
7649 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)me->def, &val)) {
7650 bmethod_local_hooks = (rb_hook_list_t*)val;
7651 }
7652 if (bmethod_local_hooks) {
7653 bmethod_local_events = bmethod_local_hooks->events;
7654 }
7655 }
7656 }
7657
7658 if ((pc_events & enabled_flags) == 0 && !bmethod_frame) {
7659#if 0
7660 /* disable trace */
7661 /* TODO: incomplete */
7662 rb_iseq_trace_set(iseq, vm_event_flags & ISEQ_TRACE_EVENTS);
7663#else
7664 /* do not disable trace because of performance problem
7665 * (re-enable overhead)
7666 */
7667#endif
7668 return;
7669 }
7670 else if (ec->trace_arg != NULL) {
7671 /* already tracing */
7672 return;
7673 }
7674 else {
7675 rb_hook_list_t *global_hooks = rb_ec_ractor_hooks(ec);
7676 /* Note, not considering iseq local events here since the same
7677 * iseq could be used in multiple bmethods. */
7678 rb_event_flag_t bmethod_events = ractor_events | bmethod_local_events;
7679
7680 if (0) {
7681 VALUE path = rb_iseq_path(iseq);
7682 VALUE label = rb_iseq_label(iseq);
7683 ruby_debug_printf("vm_trace>>%4d (%4x) - %.*s:%d %.*s\n",
7684 (int)pos,
7685 (int)pc_events,
7686 RSTRING_LENINT(path), RSTRING_PTR(path),
7687 (int)rb_iseq_line_no(iseq, pos),
7688 RSTRING_LENINT(label), RSTRING_PTR(label));
7689 }
7690 VM_ASSERT(reg_cfp->pc == pc);
7691 VM_ASSERT(pc_events != 0);
7692
7693 /* check traces */
7694 if ((pc_events & RUBY_EVENT_B_CALL) && bmethod_frame && (bmethod_events & RUBY_EVENT_CALL)) {
7695 /* b_call instruction running as a method. Fire call event. */
7696 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_CALL, RUBY_EVENT_CALL, global_hooks, bmethod_local_hooks, Qundef);
7697 }
7699 VM_TRACE_HOOK(RUBY_EVENT_RESCUE, rescue_errinfo(ec, reg_cfp));
7700 VM_TRACE_HOOK(RUBY_EVENT_LINE, Qundef);
7701 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_LINE, Qundef);
7702 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_BRANCH, Qundef);
7703 VM_TRACE_HOOK(RUBY_EVENT_END | RUBY_EVENT_RETURN | RUBY_EVENT_B_RETURN, TOPN(0));
7704 if ((pc_events & RUBY_EVENT_B_RETURN) && bmethod_frame && (bmethod_events & RUBY_EVENT_RETURN)) {
7705 /* b_return instruction running as a method. Fire return event. */
7706 /* An earlier event's hook at this pc may have freed bmethod_local_hooks
7707 * (see VM_TRACE_HOOK); reload it from its stable source. */
7708 if (bmethod_hooks_cnt > 0) bmethod_local_hooks = rb_method_def_local_hooks(bmethod_def, r, false);
7709 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_RETURN, RUBY_EVENT_RETURN, global_hooks, bmethod_local_hooks, TOPN(0));
7710 }
7711 }
7712 }
7713}
7714#undef VM_TRACE_HOOK
7715
7716#if VM_CHECK_MODE > 0
7717NORETURN( NOINLINE( COLDFUNC
7718void rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)));
7719
7720void
7721Init_vm_stack_canary(void)
7722{
7723 /* This has to be called _after_ our PRNG is properly set up. */
7724 int n = ruby_fill_random_bytes(&vm_stack_canary, sizeof vm_stack_canary, false);
7725 /* Make it a large positive Fixnum. Arguments a block does not take
7726 * stay at the stack top, where vm_push_frame checks the canary. */
7727 vm_stack_canary >>= 2;
7728 vm_stack_canary |= (VALUE)1 << (SIZEOF_VALUE * CHAR_BIT - 2);
7729 vm_stack_canary |= 0x01; // valid VALUE (Fixnum)
7730
7731 vm_stack_canary_was_born = true;
7732 VM_ASSERT(n == 0);
7733}
7734
7735void
7736rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)
7737{
7738 /* Because a method has already been called, why not call
7739 * another one. */
7740 const char *insn = rb_insns_name(i);
7741 VALUE inspection = rb_inspect(c);
7742 const char *str = StringValueCStr(inspection);
7743
7744 rb_bug("dead canary found at %s: %s", insn, str);
7745}
7746
7747#else
7748void Init_vm_stack_canary(void) { /* nothing to do */ }
7749#endif
7750
7751
7752/* a part of the following code is generated by this ruby script:
7753
775416.times{|i|
7755 typedef_args = (0...i).map{|j| "VALUE v#{j+1}"}.join(", ")
7756 typedef_args.prepend(", ") if i != 0
7757 call_args = (0...i).map{|j| "argv[#{j}]"}.join(", ")
7758 call_args.prepend(", ") if i != 0
7759 puts %Q{
7760static VALUE
7761builtin_invoker#{i}(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7762{
7763 typedef VALUE (*rb_invoke_funcptr#{i}_t)(rb_execution_context_t *ec, VALUE self#{typedef_args});
7764 return (*(rb_invoke_funcptr#{i}_t)funcptr)(ec, self#{call_args});
7765}}
7766}
7767
7768puts
7769puts "static VALUE (* const cfunc_invokers[])(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr) = {"
777016.times{|i|
7771 puts " builtin_invoker#{i},"
7772}
7773puts "};"
7774*/
7775
7776static VALUE
7777builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7778{
7779 typedef VALUE (*rb_invoke_funcptr0_t)(rb_execution_context_t *ec, VALUE self);
7780 return (*(rb_invoke_funcptr0_t)funcptr)(ec, self);
7781}
7782
7783static VALUE
7784builtin_invoker1(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7785{
7786 typedef VALUE (*rb_invoke_funcptr1_t)(rb_execution_context_t *ec, VALUE self, VALUE v1);
7787 return (*(rb_invoke_funcptr1_t)funcptr)(ec, self, argv[0]);
7788}
7789
7790static VALUE
7791builtin_invoker2(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7792{
7793 typedef VALUE (*rb_invoke_funcptr2_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2);
7794 return (*(rb_invoke_funcptr2_t)funcptr)(ec, self, argv[0], argv[1]);
7795}
7796
7797static VALUE
7798builtin_invoker3(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7799{
7800 typedef VALUE (*rb_invoke_funcptr3_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3);
7801 return (*(rb_invoke_funcptr3_t)funcptr)(ec, self, argv[0], argv[1], argv[2]);
7802}
7803
7804static VALUE
7805builtin_invoker4(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7806{
7807 typedef VALUE (*rb_invoke_funcptr4_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4);
7808 return (*(rb_invoke_funcptr4_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3]);
7809}
7810
7811static VALUE
7812builtin_invoker5(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7813{
7814 typedef VALUE (*rb_invoke_funcptr5_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5);
7815 return (*(rb_invoke_funcptr5_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4]);
7816}
7817
7818static VALUE
7819builtin_invoker6(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7820{
7821 typedef VALUE (*rb_invoke_funcptr6_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6);
7822 return (*(rb_invoke_funcptr6_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
7823}
7824
7825static VALUE
7826builtin_invoker7(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7827{
7828 typedef VALUE (*rb_invoke_funcptr7_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7);
7829 return (*(rb_invoke_funcptr7_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
7830}
7831
7832static VALUE
7833builtin_invoker8(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7834{
7835 typedef VALUE (*rb_invoke_funcptr8_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8);
7836 return (*(rb_invoke_funcptr8_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
7837}
7838
7839static VALUE
7840builtin_invoker9(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7841{
7842 typedef VALUE (*rb_invoke_funcptr9_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9);
7843 return (*(rb_invoke_funcptr9_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
7844}
7845
7846static VALUE
7847builtin_invoker10(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7848{
7849 typedef VALUE (*rb_invoke_funcptr10_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10);
7850 return (*(rb_invoke_funcptr10_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
7851}
7852
7853static VALUE
7854builtin_invoker11(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7855{
7856 typedef VALUE (*rb_invoke_funcptr11_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11);
7857 return (*(rb_invoke_funcptr11_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
7858}
7859
7860static VALUE
7861builtin_invoker12(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7862{
7863 typedef VALUE (*rb_invoke_funcptr12_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12);
7864 return (*(rb_invoke_funcptr12_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
7865}
7866
7867static VALUE
7868builtin_invoker13(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7869{
7870 typedef VALUE (*rb_invoke_funcptr13_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13);
7871 return (*(rb_invoke_funcptr13_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
7872}
7873
7874static VALUE
7875builtin_invoker14(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7876{
7877 typedef VALUE (*rb_invoke_funcptr14_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14);
7878 return (*(rb_invoke_funcptr14_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
7879}
7880
7881static VALUE
7882builtin_invoker15(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7883{
7884 typedef VALUE (*rb_invoke_funcptr15_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14, VALUE v15);
7885 return (*(rb_invoke_funcptr15_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
7886}
7887
7888typedef VALUE (*builtin_invoker)(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
7889
7890static builtin_invoker
7891lookup_builtin_invoker(int argc)
7892{
7893 static const builtin_invoker invokers[] = {
7894 builtin_invoker0,
7895 builtin_invoker1,
7896 builtin_invoker2,
7897 builtin_invoker3,
7898 builtin_invoker4,
7899 builtin_invoker5,
7900 builtin_invoker6,
7901 builtin_invoker7,
7902 builtin_invoker8,
7903 builtin_invoker9,
7904 builtin_invoker10,
7905 builtin_invoker11,
7906 builtin_invoker12,
7907 builtin_invoker13,
7908 builtin_invoker14,
7909 builtin_invoker15,
7910 };
7911
7912 return invokers[argc];
7913}
7914
7915static inline VALUE
7916invoke_bf(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7917{
7918 const bool canary_p = ISEQ_BODY(CFP_ISEQ(reg_cfp))->builtin_attrs & BUILTIN_ATTR_LEAF; // Verify an assumption of `Primitive.attr! :leaf`
7919 SETUP_CANARY(canary_p);
7920 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
7921 VALUE ret = (*lookup_builtin_invoker(bf->argc))(ec, reg_cfp->self, argv, func_ptr);
7922 CHECK_CANARY(canary_p, BIN(invokebuiltin));
7923 return ret;
7924}
7925
7926static VALUE
7927vm_invoke_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7928{
7929 return invoke_bf(ec, cfp, bf, argv);
7930}
7931
7932static VALUE
7933vm_invoke_builtin_delegate(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function *bf, unsigned int start_index)
7934{
7935 if (0) { // debug print
7936 fputs("vm_invoke_builtin_delegate: passing -> ", stderr);
7937 for (int i=0; i<bf->argc; i++) {
7938 ruby_debug_printf(":%s ", rb_id2name(ISEQ_BODY(CFP_ISEQ(cfp))->local_table[i+start_index]));
7939 }
7940 ruby_debug_printf("\n" "%s %s(%d):%p\n", RUBY_FUNCTION_NAME_STRING, bf->name, bf->argc,
7941 (void *)(uintptr_t)bf->func_ptr);
7942 }
7943
7944 if (bf->argc == 0) {
7945 return invoke_bf(ec, cfp, bf, NULL);
7946 }
7947 else {
7948 const VALUE *argv = cfp->ep - ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size - VM_ENV_DATA_SIZE + 1 + start_index;
7949 return invoke_bf(ec, cfp, bf, argv);
7950 }
7951}
7952
7953// for __builtin_inline!()
7954
7955VALUE
7956rb_vm_lvar_exposed(rb_execution_context_t *ec, int index)
7957{
7958 const rb_control_frame_t *cfp = ec->cfp;
7959 return cfp->ep[index];
7960}
#define RUBY_EVENT_END
Encountered an end of a class clause.
Definition event.h:40
#define RUBY_EVENT_C_CALL
A method, written in C, is called.
Definition event.h:43
#define RUBY_EVENT_B_RETURN
Encountered a next statement.
Definition event.h:56
#define RUBY_EVENT_CLASS
Encountered a new class.
Definition event.h:39
#define RUBY_EVENT_LINE
Encountered a new line.
Definition event.h:38
#define RUBY_EVENT_RETURN
Encountered a return statement.
Definition event.h:42
#define RUBY_EVENT_C_RETURN
Return from a method, written in C.
Definition event.h:44
#define RUBY_EVENT_B_CALL
Encountered an yield statement.
Definition event.h:55
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
#define RUBY_EVENT_CALL
A method, written in Ruby, is called.
Definition event.h:41
#define RUBY_EVENT_RESCUE
Encountered a rescue statement.
Definition event.h:61
static VALUE RB_FL_TEST_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_TEST().
Definition fl_type.h:407
static bool RB_FL_ABLE(VALUE obj)
Checks if the object is flaggable.
Definition fl_type.h:384
@ RUBY_FL_SHAREABLE
This flag has something to do with Ractor.
Definition fl_type.h:253
VALUE rb_singleton_class(VALUE obj)
Finds or creates the singleton class of the passed object.
Definition class.c:3025
VALUE rb_module_new(void)
Creates a new, anonymous module.
Definition class.c:1629
VALUE rb_class_inherited(VALUE super, VALUE klass)
Calls Class::inherited.
Definition class.c:1529
VALUE rb_define_class_id(ID id, VALUE super)
This is a very badly designed API that creates an anonymous class.
Definition class.c:1508
#define TYPE(_)
Old name of rb_type.
Definition value_type.h:108
#define RFLOAT_VALUE
Old name of rb_float_value.
Definition double.h:28
#define T_STRING
Old name of RUBY_T_STRING.
Definition value_type.h:78
#define Qundef
Old name of RUBY_Qundef.
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
#define LEN2NUM
Old name of RB_LEN2NUM.
Definition len_t.h:33
#define T_NIL
Old name of RUBY_T_NIL.
Definition value_type.h:72
#define T_FLOAT
Old name of RUBY_T_FLOAT.
Definition value_type.h:64
#define T_IMEMO
Old name of RUBY_T_IMEMO.
Definition value_type.h:67
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
#define T_BIGNUM
Old name of RUBY_T_BIGNUM.
Definition value_type.h:57
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
#define T_STRUCT
Old name of RUBY_T_STRUCT.
Definition value_type.h:79
#define T_FIXNUM
Old name of RUBY_T_FIXNUM.
Definition value_type.h:63
#define UNREACHABLE_RETURN
Old name of RBIMPL_UNREACHABLE_RETURN.
Definition assume.h:29
#define SYM2ID
Old name of RB_SYM2ID.
Definition symbol.h:45
#define CLASS_OF
Old name of rb_class_of.
Definition globals.h:205
#define T_NONE
Old name of RUBY_T_NONE.
Definition value_type.h:74
#define rb_ary_new4
Old name of rb_ary_new_from_values.
Definition array.h:659
#define FIXABLE
Old name of RB_FIXABLE.
Definition fixnum.h:25
#define LONG2FIX
Old name of RB_INT2FIX.
Definition long.h:49
#define FIX2INT
Old name of RB_FIX2INT.
Definition int.h:41
#define T_MODULE
Old name of RUBY_T_MODULE.
Definition value_type.h:70
#define ASSUME
Old name of RBIMPL_ASSUME.
Definition assume.h:27
#define FIX2ULONG
Old name of RB_FIX2ULONG.
Definition long.h:47
#define T_TRUE
Old name of RUBY_T_TRUE.
Definition value_type.h:81
#define T_ICLASS
Old name of RUBY_T_ICLASS.
Definition value_type.h:66
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
#define rb_ary_new3
Old name of rb_ary_new_from_args.
Definition array.h:658
#define rb_exc_new3
Old name of rb_exc_new_str.
Definition error.h:38
#define T_FALSE
Old name of RUBY_T_FALSE.
Definition value_type.h:61
#define Qtrue
Old name of RUBY_Qtrue.
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define FIX2LONG
Old name of RB_FIX2LONG.
Definition long.h:46
#define T_ARRAY
Old name of RUBY_T_ARRAY.
Definition value_type.h:56
#define T_OBJECT
Old name of RUBY_T_OBJECT.
Definition value_type.h:75
#define NIL_P
Old name of RB_NIL_P.
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition value_type.h:80
#define DBL2NUM
Old name of rb_float_new.
Definition double.h:29
#define T_CLASS
Old name of RUBY_T_CLASS.
Definition value_type.h:58
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
Definition value_type.h:85
#define FL_TEST
Old name of RB_FL_TEST.
Definition fl_type.h:127
#define FIXNUM_P
Old name of RB_FIXNUM_P.
#define FL_USHIFT
Old name of RUBY_FL_USHIFT.
Definition fl_type.h:67
#define FL_SET_RAW
Old name of RB_FL_SET_RAW.
Definition fl_type.h:126
#define SYMBOL_P
Old name of RB_SYMBOL_P.
Definition value_type.h:88
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
Definition eval.c:678
#define rb_notimplement()
Definition flock.c:3
#define ruby_verbose
This variable controls whether the interpreter is in debug mode.
Definition error.h:476
VALUE rb_eTypeError
TypeError exception.
Definition error.c:1457
VALUE rb_eFatal
fatal exception.
Definition error.c:1453
VALUE rb_eNoMethodError
NoMethodError exception.
Definition error.c:1465
void rb_exc_fatal(VALUE mesg)
Raises a fatal error in the current thread.
Definition eval.c:691
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1455
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:464
void rb_error_frozen_object(VALUE frozen_obj)
Identical to rb_error_frozen(), except it takes arbitrary Ruby object instead of C's string.
Definition error.c:4305
VALUE rb_exc_new_str(VALUE etype, VALUE str)
Identical to rb_exc_new_cstr(), except it takes a Ruby's string instead of C's.
Definition error.c:1508
@ RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK
Warning is for checking unused block strictly.
Definition error.h:57
VALUE rb_cClass
Class class.
Definition object.c:62
VALUE rb_cArray
Array class.
VALUE rb_cObject
Object class.
Definition object.c:60
VALUE rb_obj_alloc(VALUE klass)
Allocates an instance of the given class.
Definition object.c:2264
VALUE rb_cRegexp
Regexp class.
Definition re.c:3413
VALUE rb_obj_frozen_p(VALUE obj)
Same as RB_OBJ_FROZEN(), but returns Qtrue/Qfalse instead of bool.
Definition object.c:1312
VALUE rb_cHash
Hash class.
Definition hash.c:123
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
Definition object.c:230
VALUE rb_inspect(VALUE obj)
Generates a human-readable textual representation of the given object.
Definition object.c:665
VALUE rb_cBasicObject
BasicObject class.
Definition object.c:58
VALUE rb_cModule
Module class.
Definition object.c:61
VALUE rb_class_real(VALUE klass)
Finds a "real" class.
Definition object.c:221
VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass)
Queries if the given object is an instance (of possibly descendants) of the given class.
Definition object.c:902
VALUE rb_cFloat
Float class.
Definition numeric.c:201
VALUE rb_cProc
Proc class.
Definition proc.c:46
VALUE rb_cString
String class.
Definition string.c:85
#define RB_OBJ_WRITTEN(old, oldv, young)
Identical to RB_OBJ_WRITE(), except it doesn't write any values, but only a WB declaration.
Definition gc.h:504
#define RB_OBJ_WRITE(old, slot, young)
Declaration of a "back" pointer.
Definition gc.h:492
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:219
VALUE rb_funcall(VALUE recv, ID mid, int n,...)
Calls a method.
Definition vm_eval.c:1149
int rb_during_gc(void)
Queries if the GC is busy.
Definition gc.c:5703
VALUE rb_ary_concat(VALUE lhs, VALUE rhs)
Destructively appends the contents of latter into the end of former.
VALUE rb_ary_shift(VALUE ary)
Destructively deletes an element from the beginning of the passed array and returns what was deleted.
VALUE rb_ary_resurrect(VALUE ary)
I guess there is no use case of this function in extension libraries, but this is a routine identical...
VALUE rb_ary_entry(VALUE ary, rb_len_t off)
Queries an element of an array.
VALUE rb_ary_dup(VALUE ary)
Duplicates an array.
VALUE rb_ary_includes(VALUE ary, VALUE elem)
Queries if the passed array has the passed entry.
void rb_ary_store(VALUE ary, rb_len_t key, VALUE val)
Destructively stores the passed value to the passed array's passed index.
VALUE rb_ary_plus(VALUE lhs, VALUE rhs)
Creates a new array, concatenating the former to the latter.
VALUE rb_check_array_type(VALUE obj)
Try converting an object to its array representation using its to_ary method, if any.
VALUE rb_ary_new(void)
Allocates a new, empty array.
VALUE rb_ary_pop(VALUE ary)
Destructively deletes an element from the end of the passed array and returns what was deleted.
VALUE rb_ary_hidden_new(rb_len_t capa)
Allocates a hidden (no class) empty array.
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
VALUE rb_ary_cat(VALUE ary, const VALUE *train, rb_len_t len)
Destructively appends multiple elements at the end of the array.
#define UNLIMITED_ARGUMENTS
This macro is used in conjunction with rb_check_arity().
Definition error.h:35
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
Definition error.h:284
VALUE rb_proc_call_with_block(VALUE recv, int argc, const VALUE *argv, VALUE proc)
Identical to rb_proc_call(), except you can additionally pass another proc object,...
Definition proc.c:1774
VALUE rb_reg_last_match(VALUE md)
This just returns the argument, stringified.
Definition re.c:2525
VALUE rb_reg_match(VALUE re, VALUE str)
This is the match operator.
Definition re.c:4596
VALUE rb_reg_nth_match(int n, VALUE md)
Queries the nth captured substring.
Definition re.c:2501
VALUE rb_reg_match_post(VALUE md)
The portion of the original string after the given match.
Definition re.c:2583
VALUE rb_reg_nth_defined(int n, VALUE md)
Identical to rb_reg_nth_match(), except it just returns Boolean.
Definition re.c:2485
VALUE rb_reg_match_pre(VALUE md)
The portion of the original string before the given match.
Definition re.c:2551
VALUE rb_reg_match_last(VALUE md)
The portion of the original string that captured at the very last.
Definition re.c:2612
bool rb_set_lookup(VALUE set, VALUE element)
Whether the set contains the given element.
Definition set.c:2381
VALUE rb_str_append(VALUE dst, VALUE src)
Identical to rb_str_buf_append(), except it converts the right hand side before concatenating.
Definition string.c:3940
VALUE rb_str_succ(VALUE orig)
Searches for the "successor" of a string.
Definition string.c:5681
VALUE rb_str_buf_append(VALUE dst, VALUE src)
Identical to rb_str_cat_cstr(), except it takes Ruby's string instead of C's.
Definition string.c:3904
VALUE rb_str_concat(VALUE dst, VALUE src)
Identical to rb_str_append(), except it also accepts an integer as a codepoint.
Definition string.c:4189
#define rb_str_cat_cstr(buf, str)
Identical to rb_str_cat(), except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1681
VALUE rb_str_length(VALUE)
Identical to rb_str_strlen(), except it returns the value in rb_cInteger.
Definition string.c:2517
VALUE rb_str_intern(VALUE str)
Identical to rb_to_symbol(), except it assumes the receiver being an instance of RString.
Definition symbol.c:1091
VALUE rb_const_get(VALUE space, ID name)
Identical to rb_const_defined(), except it returns the actual defined value.
Definition variable.c:3535
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
Definition variable.c:2168
void rb_cvar_set(VALUE klass, ID name, VALUE val)
Assigns a value to a class variable.
Definition variable.c:4329
VALUE rb_cvar_find(VALUE klass, ID name, VALUE *front)
Identical to rb_cvar_get(), except it takes additional "front" pointer.
Definition variable.c:4390
VALUE rb_ivar_get(VALUE obj, ID name)
Identical to rb_iv_get(), except it accepts the name as an ID instead of a C string.
Definition variable.c:1651
void rb_const_set(VALUE space, ID name, VALUE val)
Names a constant.
Definition variable.c:4018
VALUE rb_autoload_load(VALUE space, ID name)
Kicks the autoload procedure as if it was "touched".
Definition variable.c:3356
VALUE rb_mod_name(VALUE mod)
Queries the name of a module.
Definition variable.c:151
VALUE rb_const_get_at(VALUE space, ID name)
Identical to rb_const_defined_at(), except it returns the actual defined value.
Definition variable.c:3541
void rb_set_class_path_string(VALUE klass, VALUE space, VALUE name)
Identical to rb_set_class_path(), except it accepts the name as Ruby's string instead of C's.
Definition variable.c:440
VALUE rb_ivar_defined(VALUE obj, ID name)
Queries if the instance variable is defined at the object.
Definition variable.c:2228
int rb_const_defined_at(VALUE space, ID name)
Identical to rb_const_defined(), except it doesn't look for parent classes.
Definition variable.c:3876
VALUE rb_cvar_defined(VALUE klass, ID name)
Queries if the given class has the given class variable.
Definition variable.c:4415
int rb_const_defined(VALUE space, ID name)
Queries if the constant is defined at the namespace.
Definition variable.c:3870
int rb_method_basic_definition_p(VALUE klass, ID mid)
Well... Let us hesitate from describing what a "basic definition" is.
Definition vm_method.c:3630
VALUE rb_check_funcall(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it returns RUBY_Qundef instead of raising rb_eNoMethodError.
Definition vm_eval.c:716
rb_alloc_func_t rb_get_alloc_func(VALUE klass)
Queries the allocator function of a class.
Definition vm_method.c:1902
int rb_method_boundp(VALUE klass, ID id, int ex)
Queries if the klass has this method.
Definition vm_method.c:2524
ID rb_check_id(volatile VALUE *namep)
Detects if the given name is already interned or not.
Definition symbol.c:1296
VALUE rb_sym2str(VALUE symbol)
Obtain a frozen string representation of a symbol (not including the leading colon).
Definition symbol.c:1155
int off
Offset inside of ptr.
Definition io.h:5
int len
Length of the buffer.
Definition io.h:8
#define RB_OBJ_SET_SHAREABLE(obj)
Wrapper of rb_obj_set_shareable().
Definition ractor.h:290
static bool rb_ractor_shareable_p(VALUE obj)
Queries if multiple Ractors can share the passed object or not.
Definition ractor.h:269
#define RB_OBJ_SHAREABLE_P(obj)
Queries if the passed object has previously classified as shareable or not.
Definition ractor.h:255
#define RB_NOGVL_UBF_ASYNC_SAFE
Passing this flag to rb_nogvl() indicates that the passed UBF is async-signal-safe.
Definition thread.h:71
#define RB_NOGVL_INTR_FAIL
Passing this flag to rb_nogvl() prevents it from processing interrupts after the given function retur...
Definition thread.h:50
void * rb_nogvl(void *(*func)(void *), void *data1, rb_unblock_function_t *ubf, void *data2, int flags)
Identical to rb_thread_call_without_gvl(), except it additionally takes "flags" that change the behav...
Definition thread.c:1775
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:375
#define ALLOCA_N(type, n)
Definition memory.h:295
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:170
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
Definition memory.h:387
VALUE type(ANYARGS)
ANYARGS-ed function type.
#define RARRAY_LEN
Just another name of rb_array_len.
Definition rarray.h:50
static int RARRAY_LENINT(VALUE ary)
Identical to rb_array_len(), except it differs for the return type.
Definition rarray.h:280
static VALUE * RARRAY_PTR(VALUE ary)
Wild use of a C pointer.
Definition rarray.h:365
#define RARRAY_AREF(a, i)
References an object in an array.
Definition rarray.h:422
#define RARRAY_CONST_PTR
Just another name of rb_array_const_ptr.
Definition rarray.h:51
static VALUE RBASIC_CLASS(VALUE obj)
Queries the class of an object.
Definition rbasic.h:166
#define RBASIC(obj)
Convenient casting macro.
Definition rbasic.h:40
#define RCLASS_SUPER
Just another name of rb_class_get_superclass.
Definition rclass.h:44
#define RHASH_SIZE(h)
Queries the size of the hash.
Definition rhash.h:57
#define RHASH_EMPTY_P(h)
Checks if the hash is empty.
Definition rhash.h:67
static int RSTRING_LENINT(VALUE str)
Identical to RSTRING_LEN(), except it differs for the return type.
Definition rstring.h:438
#define StringValueCStr(v)
Identical to StringValuePtr, except it additionally checks for the contents for viability as a C stri...
Definition rstring.h:89
#define RB_PASS_KEYWORDS
Pass keywords, final argument must be a hash of keywords.
Definition scan_args.h:72
#define RB_NO_KEYWORDS
Do not pass keywords.
Definition scan_args.h:69
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
#define RTEST
This is an old name of RB_TEST.
#define ANYARGS
Functions declared using this macro take arbitrary arguments, including void.
Definition stdarg.h:64
Ruby's array.
Definition rarray.h:127
const VALUE ary[1]
Embedded elements.
Definition rarray.h:187
Definition hash.h:54
Definition iseq.h:360
Definition vm_core.h:262
const ID * segments
A null-terminated list of ids, used to represent a constant's path idNULL is used to represent the ::...
Definition vm_core.h:289
Definition method.h:63
Internal header for Class.
Definition class.h:31
rb_cref_t * cref
class reference, should be marked
Definition method.h:144
const rb_iseq_t * iseqptr
iseq pointer, should be separated from iseqval
Definition method.h:143
IFUNC (Internal FUNCtion).
Definition imemo.h:89
SVAR (Special VARiable).
Definition imemo.h:54
const VALUE cref_or_me
class reference or rb_method_entry_t
Definition imemo.h:56
THROW_DATA.
Definition imemo.h:63
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.
Definition vm_core.h:301
long rb_len_t
A signed integer type for lengths and indices of objects such as String and Array.
Definition value.h:142
intptr_t SIGNED_VALUE
A signed integer type that has the same width with VALUE.
Definition value.h:63
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
#define SIZEOF_VALUE
Identical to sizeof(VALUE), except it is a macro that can also be used inside of preprocessor directi...
Definition value.h:69
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
Definition value_type.h:182
static bool RB_FLOAT_TYPE_P(VALUE obj)
Queries if the object is an instance of rb_cFloat.
Definition value_type.h:264
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:376