Ruby 4.1.0dev (2026-10-10 revision 05917fd1e6d8539db013b087d4ea93cf068038e1)
vm_insnhelper.c (05917fd1e6d8539db013b087d4ea93cf068038e1)
1/**********************************************************************
2
3 vm_insnhelper.c - instruction helper functions. Included into vm.c.
4
5 $Author$
6
7 Copyright (C) 2007 Koichi Sasada
8
9**********************************************************************/
10
11#include "ruby/internal/config.h"
12
13#include <math.h>
14
15#ifdef HAVE_STDATOMIC_H
16 #include <stdatomic.h>
17#endif
18
19#include "constant.h"
20#include "debug_counter.h"
21#include "internal.h"
22#include "internal/class.h"
23#include "internal/compar.h"
24#include "internal/hash.h"
25#include "internal/numeric.h"
26#include "internal/proc.h"
27#include "internal/random.h"
28#include "internal/variable.h"
29#include "internal/set.h"
30#include "internal/set_table.h"
31#include "internal/struct.h"
32#include "ruby/thread.h"
33#include "variable.h"
34
35/* finish iseq array */
36#include "insns.inc"
37#include "insns_info.inc"
38
39extern rb_method_definition_t *rb_method_definition_create(rb_method_type_t type, ID mid);
40extern void rb_method_definition_set(const rb_method_entry_t *me, rb_method_definition_t *def, void *opts);
41extern int rb_method_definition_eq(const rb_method_definition_t *d1, const rb_method_definition_t *d2);
42extern VALUE rb_make_no_method_exception(VALUE exc, VALUE format, VALUE obj,
43 int argc, const VALUE *argv, int priv);
44
45static const struct rb_callcache vm_empty_cc;
46static const struct rb_callcache vm_empty_cc_for_super;
47
48/* control stack frame */
49
50static rb_control_frame_t *vm_get_ruby_level_caller_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp);
51
52VALUE
53ruby_vm_special_exception_copy(VALUE exc)
54{
55 VALUE e = rb_obj_alloc(rb_class_real(RBASIC_CLASS(exc)));
56 rb_obj_copy_ivar(e, exc);
57 return e;
58}
59
60NORETURN(static void ec_stack_overflow(rb_execution_context_t *ec, int));
61static void
62ec_stack_overflow(rb_execution_context_t *ec, int setup)
63{
64 VALUE mesg = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_sysstack];
65 ec->raised_flag = RAISED_STACKOVERFLOW;
66 if (setup) {
67 VALUE at = rb_ec_backtrace_object(ec);
68 mesg = ruby_vm_special_exception_copy(mesg);
69 rb_ivar_set(mesg, idBt, at);
70 rb_ivar_set(mesg, idBt_locations, at);
71 }
72 ec->errinfo = mesg;
73 EC_JUMP_TAG(ec, TAG_RAISE);
74}
75
76NORETURN(static void vm_stackoverflow(void));
77
78static void
79vm_stackoverflow(void)
80{
81 ec_stack_overflow(GET_EC(), TRUE);
82}
83
84void
85rb_ec_stack_overflow(rb_execution_context_t *ec, ruby_stack_overflow_critical_level crit)
86{
87 if (rb_during_gc()) {
88 rb_bug("system stack overflow during GC. Faulty native extension?");
89 }
90 if (crit >= rb_stack_overflow_fatal) {
91 ec->raised_flag = RAISED_STACKOVERFLOW;
92 ec->errinfo = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_stackfatal];
93 EC_JUMP_TAG(ec, TAG_RAISE);
94 }
95 ec_stack_overflow(ec, crit < rb_stack_overflow_signal);
96}
97
98static inline void stack_check(rb_execution_context_t *ec);
99
100#if VM_CHECK_MODE > 0
101static int
102callable_class_p(VALUE klass)
103{
104#if VM_CHECK_MODE >= 2
105 if (!klass) return FALSE;
106 switch (RB_BUILTIN_TYPE(klass)) {
107 default:
108 break;
109 case T_ICLASS:
110 if (!RB_TYPE_P(RCLASS_SUPER(klass), T_MODULE)) break;
111 case T_MODULE:
112 return TRUE;
113 }
114 while (klass) {
115 if (klass == rb_cBasicObject) {
116 return TRUE;
117 }
118 klass = RCLASS_SUPER(klass);
119 }
120 return FALSE;
121#else
122 return klass != 0;
123#endif
124}
125
126static int
127callable_method_entry_p(const rb_callable_method_entry_t *cme)
128{
129 if (cme == NULL) {
130 return TRUE;
131 }
132 else {
133 VM_ASSERT(IMEMO_TYPE_P((VALUE)cme, imemo_ment), "imemo_type:%s", rb_imemo_name(imemo_type((VALUE)cme)));
134
135 if (callable_class_p(cme->defined_class)) {
136 return TRUE;
137 }
138 else {
139 return FALSE;
140 }
141 }
142}
143
144static void
145vm_check_frame_detail(VALUE type, int req_block, int req_me, int req_cref, VALUE specval, VALUE cref_or_me, int is_cframe, const rb_iseq_t *iseq)
146{
147 unsigned int magic = (unsigned int)(type & VM_FRAME_MAGIC_MASK);
148 enum imemo_type cref_or_me_type = imemo_env; /* impossible value */
149
150 if (RB_TYPE_P(cref_or_me, T_IMEMO)) {
151 cref_or_me_type = imemo_type(cref_or_me);
152 }
153 if (type & VM_FRAME_FLAG_BMETHOD) {
154 req_me = TRUE;
155 }
156
157 if (req_block && (type & VM_ENV_FLAG_LOCAL) == 0) {
158 rb_bug("vm_push_frame: specval (%p) should be a block_ptr on %x frame", (void *)specval, magic);
159 }
160 if (!req_block && (type & VM_ENV_FLAG_LOCAL) != 0) {
161 rb_bug("vm_push_frame: specval (%p) should not be a block_ptr on %x frame", (void *)specval, magic);
162 }
163
164 if (req_me) {
165 if (cref_or_me_type != imemo_ment) {
166 rb_bug("vm_push_frame: (%s) should be method entry on %x frame", rb_obj_info(cref_or_me), magic);
167 }
168 }
169 else {
170 if (req_cref && cref_or_me_type != imemo_cref) {
171 rb_bug("vm_push_frame: (%s) should be CREF on %x frame", rb_obj_info(cref_or_me), magic);
172 }
173 else { /* cref or Qfalse */
174 if (cref_or_me != Qfalse && cref_or_me_type != imemo_cref) {
175 if (((type & VM_FRAME_FLAG_LAMBDA) || magic == VM_FRAME_MAGIC_IFUNC || magic == VM_FRAME_MAGIC_DUMMY) && (cref_or_me_type == imemo_ment)) {
176 /* ignore */
177 }
178 else {
179 rb_bug("vm_push_frame: (%s) should be false or cref on %x frame", rb_obj_info(cref_or_me), magic);
180 }
181 }
182 }
183 }
184
185 if (cref_or_me_type == imemo_ment) {
186 const rb_callable_method_entry_t *me = (const rb_callable_method_entry_t *)cref_or_me;
187
188 if (!callable_method_entry_p(me)) {
189 rb_bug("vm_push_frame: ment (%s) should be callable on %x frame.", rb_obj_info(cref_or_me), magic);
190 }
191 }
192
193 if ((type & VM_FRAME_MAGIC_MASK) == VM_FRAME_MAGIC_DUMMY) {
194 VM_ASSERT(iseq == NULL ||
195 RBASIC_CLASS((VALUE)iseq) == 0 || // dummy frame for loading
196 RUBY_VM_NORMAL_ISEQ_P(iseq) //argument error
197 );
198 }
199 else {
200 VM_ASSERT(is_cframe == !RUBY_VM_NORMAL_ISEQ_P(iseq));
201 }
202}
203
204static void
205vm_check_frame(VALUE type,
206 VALUE specval,
207 VALUE cref_or_me,
208 const rb_iseq_t *iseq)
209{
210 VALUE given_magic = type & VM_FRAME_MAGIC_MASK;
211 VM_ASSERT(FIXNUM_P(type));
212
213#define CHECK(magic, req_block, req_me, req_cref, is_cframe) \
214 case magic: \
215 vm_check_frame_detail(type, req_block, req_me, req_cref, \
216 specval, cref_or_me, is_cframe, iseq); \
217 break
218 switch (given_magic) {
219 /* BLK ME CREF CFRAME */
220 CHECK(VM_FRAME_MAGIC_METHOD, TRUE, TRUE, FALSE, FALSE);
221 CHECK(VM_FRAME_MAGIC_CLASS, TRUE, FALSE, TRUE, FALSE);
222 CHECK(VM_FRAME_MAGIC_TOP, TRUE, FALSE, TRUE, FALSE);
223 CHECK(VM_FRAME_MAGIC_CFUNC, TRUE, TRUE, FALSE, TRUE);
224 CHECK(VM_FRAME_MAGIC_BLOCK, FALSE, FALSE, FALSE, FALSE);
225 CHECK(VM_FRAME_MAGIC_IFUNC, FALSE, FALSE, FALSE, TRUE);
226 CHECK(VM_FRAME_MAGIC_EVAL, FALSE, FALSE, FALSE, FALSE);
227 CHECK(VM_FRAME_MAGIC_RESCUE, FALSE, FALSE, FALSE, FALSE);
228 CHECK(VM_FRAME_MAGIC_DUMMY, TRUE, FALSE, FALSE, FALSE);
229 default:
230 rb_bug("vm_push_frame: unknown type (%x)", (unsigned int)given_magic);
231 }
232#undef CHECK
233}
234
235static VALUE vm_stack_canary; /* Initialized later */
236static bool vm_stack_canary_was_born = false;
237
238// Return the index of the instruction right before the given PC.
239// This is needed because insn_entry advances PC before the insn body.
240static unsigned int
241previous_insn_index(const rb_iseq_t *iseq, const VALUE *pc)
242{
243 unsigned int pos = 0;
244 while (pos < ISEQ_BODY(iseq)->iseq_size) {
245 int opcode = rb_vm_insn_addr2opcode((void *)ISEQ_BODY(iseq)->iseq_encoded[pos]);
246 unsigned int next_pos = pos + insn_len(opcode);
247 if (ISEQ_BODY(iseq)->iseq_encoded + next_pos == pc) {
248 return pos;
249 }
250 pos = next_pos;
251 }
252 rb_bug("failed to find the previous insn");
253}
254
255void
256rb_vm_check_canary(const rb_execution_context_t *ec, VALUE *sp)
257{
258 const struct rb_control_frame_struct *reg_cfp = ec->cfp;
259 const struct rb_iseq_struct *iseq;
260
261 if (! LIKELY(vm_stack_canary_was_born)) {
262 return; /* :FIXME: isn't it rather fatal to enter this branch? */
263 }
264 else if ((VALUE *)reg_cfp == ec->vm_stack + ec->vm_stack_size) {
265 /* This is at the very beginning of a thread. cfp does not exist. */
266 return;
267 }
268 else if (! VM_FRAME_RUBYFRAME_P(reg_cfp) || ! (iseq = GET_ISEQ())) {
269 /* The iseq field of an IFUNC frame holds the ifunc. */
270 return;
271 }
272 else if (LIKELY(sp[0] != vm_stack_canary)) {
273 return;
274 }
275 else {
276 /* we are going to call methods below; squash the canary to
277 * prevent infinite loop. */
278 sp[0] = Qundef;
279 }
280
281 const VALUE *orig = rb_iseq_original_iseq(iseq);
282 const VALUE iseqw = rb_iseqw_new(iseq);
283 const VALUE inspection = rb_inspect(iseqw);
284 const char *stri = rb_str_to_cstr(inspection);
285 const VALUE disasm = rb_iseq_disasm(iseq);
286 const char *strd = rb_str_to_cstr(disasm);
287 const ptrdiff_t pos = previous_insn_index(iseq, GET_PC());
288 const enum ruby_vminsn_type insn = (enum ruby_vminsn_type)orig[pos];
289 const char *name = insn_name(insn);
290
291 /* rb_bug() is not capable of outputting this large contents. It
292 is designed to run form a SIGSEGV handler, which tends to be
293 very restricted. */
294 ruby_debug_printf(
295 "We are killing the stack canary set by %s, "
296 "at %s@pc=%"PRIdPTR"\n"
297 "watch out the C stack trace.\n"
298 "%s",
299 name, stri, pos, strd);
300 rb_bug("see above.");
301}
302#define vm_check_canary(ec, sp) rb_vm_check_canary(ec, sp)
303
304#else
305#define vm_check_canary(ec, sp)
306#define vm_check_frame(a, b, c, d)
307#endif /* VM_CHECK_MODE > 0 */
308
309#if USE_DEBUG_COUNTER
310static void
311vm_push_frame_debug_counter_inc(
312 const struct rb_execution_context_struct *ec,
313 const struct rb_control_frame_struct *reg_cfp,
314 VALUE type)
315{
316 const struct rb_control_frame_struct *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(reg_cfp);
317
318 RB_DEBUG_COUNTER_INC(frame_push);
319
320 if (RUBY_VM_END_CONTROL_FRAME(ec) != prev_cfp) {
321 const bool curr = VM_FRAME_RUBYFRAME_P(reg_cfp);
322 const bool prev = VM_FRAME_RUBYFRAME_P(prev_cfp);
323 if (prev) {
324 if (curr) {
325 RB_DEBUG_COUNTER_INC(frame_R2R);
326 }
327 else {
328 RB_DEBUG_COUNTER_INC(frame_R2C);
329 }
330 }
331 else {
332 if (curr) {
333 RB_DEBUG_COUNTER_INC(frame_C2R);
334 }
335 else {
336 RB_DEBUG_COUNTER_INC(frame_C2C);
337 }
338 }
339 }
340
341 switch (type & VM_FRAME_MAGIC_MASK) {
342 case VM_FRAME_MAGIC_METHOD: RB_DEBUG_COUNTER_INC(frame_push_method); return;
343 case VM_FRAME_MAGIC_BLOCK: RB_DEBUG_COUNTER_INC(frame_push_block); return;
344 case VM_FRAME_MAGIC_CLASS: RB_DEBUG_COUNTER_INC(frame_push_class); return;
345 case VM_FRAME_MAGIC_TOP: RB_DEBUG_COUNTER_INC(frame_push_top); return;
346 case VM_FRAME_MAGIC_CFUNC: RB_DEBUG_COUNTER_INC(frame_push_cfunc); return;
347 case VM_FRAME_MAGIC_IFUNC: RB_DEBUG_COUNTER_INC(frame_push_ifunc); return;
348 case VM_FRAME_MAGIC_EVAL: RB_DEBUG_COUNTER_INC(frame_push_eval); return;
349 case VM_FRAME_MAGIC_RESCUE: RB_DEBUG_COUNTER_INC(frame_push_rescue); return;
350 case VM_FRAME_MAGIC_DUMMY: RB_DEBUG_COUNTER_INC(frame_push_dummy); return;
351 }
352
353 rb_bug("unreachable");
354}
355#else
356#define vm_push_frame_debug_counter_inc(ec, cfp, t) /* void */
357#endif
358
359// Return a poison value to be set above the stack top to verify leafness.
360VALUE
361rb_vm_stack_canary(void)
362{
363#if VM_CHECK_MODE > 0
364 return vm_stack_canary;
365#else
366 return 0;
367#endif
368}
369
370STATIC_ASSERT(VM_ENV_DATA_INDEX_ME_CREF, VM_ENV_DATA_INDEX_ME_CREF == -2);
371STATIC_ASSERT(VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL == -1);
372STATIC_ASSERT(VM_ENV_DATA_INDEX_FLAGS, VM_ENV_DATA_INDEX_FLAGS == -0);
373
374static void
375vm_push_frame(rb_execution_context_t *ec,
376 const rb_iseq_t *iseq,
377 VALUE type,
378 VALUE self,
379 VALUE specval,
380 VALUE cref_or_me,
381 const VALUE *pc,
382 VALUE *sp,
383 int local_size,
384 int stack_max)
385{
386 rb_control_frame_t *const cfp = RUBY_VM_NEXT_CONTROL_FRAME(ec->cfp);
387
388 vm_check_frame(type, specval, cref_or_me, iseq);
389 VM_ASSERT(local_size >= 0);
390
391 /* check stack overflow */
392 CHECK_VM_STACK_OVERFLOW0(cfp, sp, local_size + stack_max);
393 vm_check_canary(ec, sp);
394
395 /* setup vm value stack */
396
397 /* initialize local variables */
398 for (int i=0; i < local_size; i++) {
399 *sp++ = Qnil;
400 }
401
402 /* setup ep with managing data */
403 *sp++ = cref_or_me; /* ep[-2] / Qnil or T_IMEMO(cref) or T_IMEMO(ment) */
404 *sp++ = specval /* ep[-1] / block handler or prev env ptr */;
405 *sp++ = type; /* ep[-0] / ENV_FLAGS */
406
407 /* setup new frame */
408 *cfp = (const struct rb_control_frame_struct) {
409 .pc = pc,
410 .sp = sp,
411 ._iseq = iseq,
412 .self = self,
413 .ep = sp - 1,
414 .block_code = NULL,
415#if VM_DEBUG_BP_CHECK
416 .bp_check = sp,
417#endif
418 .jit_return = NULL,
419 };
420
421 /* Ensure the initialization of `*cfp` above never gets reordered with the update of `ec->cfp` below.
422 This is a no-op in all cases we've looked at (https://godbolt.org/z/3oxd1446K), but should guarantee it for all
423 future/untested compilers/platforms. */
424
425 #if defined HAVE_DECL_ATOMIC_SIGNAL_FENCE && HAVE_DECL_ATOMIC_SIGNAL_FENCE
426 atomic_signal_fence(memory_order_seq_cst);
427 #endif
428
429 ec->cfp = cfp;
430
431 if (VMDEBUG == 2) {
432 SDR();
433 }
434 vm_push_frame_debug_counter_inc(ec, cfp, type);
435}
436
437void
438rb_vm_pop_frame_no_int(rb_execution_context_t *ec)
439{
440 rb_control_frame_t *cfp = ec->cfp;
441
442 if (VMDEBUG == 2) SDR();
443
444 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
445}
446
447/* return TRUE if the frame is finished */
448static inline int
449vm_pop_frame(rb_execution_context_t *ec, rb_control_frame_t *cfp, const VALUE *ep)
450{
451 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
452
453 if (VMDEBUG == 2) SDR();
454
455 RUBY_VM_CHECK_INTS(ec);
456 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
457
458 return flags & VM_FRAME_FLAG_FINISH;
459}
460
461void
462rb_vm_pop_frame(rb_execution_context_t *ec)
463{
464 vm_pop_frame(ec, ec->cfp, ec->cfp->ep);
465}
466
467// it pushes pseudo-frame with fname filename.
468VALUE
469rb_vm_push_frame_fname(rb_execution_context_t *ec, VALUE fname)
470{
471 rb_iseq_t *rb_iseq_alloc_with_dummy_path(VALUE fname);
472 rb_iseq_t *dmy_iseq = rb_iseq_alloc_with_dummy_path(fname);
473
474 vm_push_frame(ec,
475 dmy_iseq, //const rb_iseq_t *iseq,
476 VM_FRAME_MAGIC_DUMMY | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, // VALUE type,
477 ec->cfp->self, // VALUE self,
478 VM_BLOCK_HANDLER_NONE, // VALUE specval,
479 Qfalse, // VALUE cref_or_me,
480 NULL, // const VALUE *pc,
481 ec->cfp->sp, // VALUE *sp,
482 0, // int local_size,
483 0); // int stack_max
484
485 return (VALUE)dmy_iseq;
486}
487
488/* method dispatch */
489static inline VALUE
490rb_arity_error_new(int argc, int min, int max)
491{
492 VALUE err_mess = rb_sprintf("wrong number of arguments (given %d, expected %d", argc, min);
493 if (min == max) {
494 /* max is not needed */
495 }
496 else if (max == UNLIMITED_ARGUMENTS) {
497 rb_str_cat_cstr(err_mess, "+");
498 }
499 else {
500 rb_str_catf(err_mess, "..%d", max);
501 }
502 rb_str_cat_cstr(err_mess, ")");
503 return rb_exc_new3(rb_eArgError, err_mess);
504}
505
506void
507rb_error_arity(int argc, int min, int max)
508{
509 rb_exc_raise(rb_arity_error_new(argc, min, max));
510}
511
512/* lvar */
513
514NOINLINE(static void vm_env_write_slowpath(const VALUE *ep, int index, VALUE v));
515
516static void
517vm_env_write_slowpath(const VALUE *ep, int index, VALUE v)
518{
519 const VALUE envval = VM_ENV_ENVVAL(ep);
520
521 if (RB_FL_TEST_RAW(envval, RUBY_FL_SHAREABLE)) {
522 /* Writing to a SHAREABLE env (an isolated proc) can create a shareable ->
523 * unshareable edge; only a full barrier sets the shref bit keeping v alive
524 * through its owner's local GC. WB_REQUIRED stays: later writes need it too. */
525 if (!SPECIAL_CONST_P(v)) {
526 rb_gc_writebarrier(envval, v);
527 }
528 VM_FORCE_WRITE(&ep[index], v);
529 }
530 else {
531 /* remember env value forcely */
532 rb_gc_writebarrier_remember(envval);
533 VM_FORCE_WRITE(&ep[index], v);
534 VM_ENV_FLAGS_UNSET(ep, VM_ENV_FLAG_WB_REQUIRED);
535 }
536 RB_DEBUG_COUNTER_INC(lvar_set_slowpath);
537}
538
539// YJIT assumes this function never runs GC
540static inline void
541vm_env_write(const VALUE *ep, int index, VALUE v)
542{
543 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
544 if (LIKELY((flags & VM_ENV_FLAG_WB_REQUIRED) == 0)) {
545 VM_STACK_ENV_WRITE(ep, index, v);
546 }
547 else {
548 vm_env_write_slowpath(ep, index, v);
549 }
550}
551
552void
553rb_vm_env_write(const VALUE *ep, int index, VALUE v)
554{
555 vm_env_write(ep, index, v);
556}
557
558VALUE
559rb_vm_bh_to_procval(const rb_execution_context_t *ec, VALUE block_handler)
560{
561 if (block_handler == VM_BLOCK_HANDLER_NONE) {
562 return Qnil;
563 }
564 else {
565 switch (vm_block_handler_type(block_handler)) {
566 case block_handler_type_iseq:
567 case block_handler_type_ifunc:
568 return rb_vm_make_proc(ec, VM_BH_TO_CAPT_BLOCK(block_handler), rb_cProc);
569 case block_handler_type_symbol:
570 return rb_sym_to_proc(VM_BH_TO_SYMBOL(block_handler));
571 case block_handler_type_proc:
572 return VM_BH_TO_PROC(block_handler);
573 default:
574 VM_UNREACHABLE(rb_vm_bh_to_procval);
575 }
576 }
577}
578
579/* svar */
580
581#if VM_CHECK_MODE > 0
582static int
583vm_svar_valid_p(VALUE svar)
584{
585 if (RB_TYPE_P((VALUE)svar, T_IMEMO)) {
586 switch (imemo_type(svar)) {
587 case imemo_svar:
588 case imemo_cref:
589 case imemo_ment:
590 return TRUE;
591 default:
592 break;
593 }
594 }
595 rb_bug("vm_svar_valid_p: unknown type: %s", rb_obj_info(svar));
596 return FALSE;
597}
598#endif
599
600/* Should this frame's special variables live in the env's svar slot? A SHAREABLE env
601 * (isolated proc) can run in several Ractors at once, making svar shared mutable state
602 * leaking $~/$_ across Ractors: keep them per-EC instead. */
603static inline bool
604lep_svar_in_env_p(const rb_execution_context_t *ec, const VALUE *lep)
605{
606 if (!lep) return false;
607 if (ec == NULL) return true;
608 if (ec->root_lep == lep) return false;
609 /* lep may be a stale on-stack ep of a frame whose env already escaped: lep[0]
610 * still holds the imemo_env, so flags is not a FIXNUM and VM_ENV_ESCAPED_P
611 * asserts. That is not a live shareable proc's env, so fall back to in-env. */
612 if (FIXNUM_P(lep[VM_ENV_DATA_INDEX_FLAGS]) &&
613 VM_ENV_ESCAPED_P(lep) &&
614 RB_FL_TEST_RAW(VM_ENV_ENVVAL(lep), RUBY_FL_SHAREABLE)) {
615 return false;
616 }
617 return true;
618}
619
620static inline struct vm_svar *
621lep_svar(const rb_execution_context_t *ec, const VALUE *lep)
622{
623 VALUE svar;
624
625 if (lep_svar_in_env_p(ec, lep)) {
626 svar = lep[VM_ENV_DATA_INDEX_ME_CREF];
627 }
628 else {
629 svar = ec->root_svar;
630 }
631
632 VM_ASSERT(svar == Qfalse || vm_svar_valid_p(svar));
633
634 return (struct vm_svar *)svar;
635}
636
637static inline void
638lep_svar_write(const rb_execution_context_t *ec, const VALUE *lep, const struct vm_svar *svar)
639{
640 VM_ASSERT(vm_svar_valid_p((VALUE)svar));
641
642 if (lep_svar_in_env_p(ec, lep)) {
643 vm_env_write(lep, VM_ENV_DATA_INDEX_ME_CREF, (VALUE)svar);
644 }
645 else {
646 RB_OBJ_WRITE(rb_ec_thread_ptr(ec)->self, &ec->root_svar, svar);
647 }
648}
649
650static VALUE
651lep_svar_get(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key)
652{
653 const struct vm_svar *svar = lep_svar(ec, lep);
654
655 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) return Qnil;
656
657 switch (key) {
658 case VM_SVAR_LASTLINE:
659 return svar->lastline;
660 case VM_SVAR_BACKREF:
661 return svar->backref;
662 default: {
663 const VALUE ary = svar->others;
664
665 if (NIL_P(ary)) {
666 return Qnil;
667 }
668 else {
669 return rb_ary_entry(ary, key - VM_SVAR_EXTRA_START);
670 }
671 }
672 }
673}
674
675static struct vm_svar *
676svar_new(VALUE obj)
677{
678 struct vm_svar *svar = IMEMO_NEW(struct vm_svar, imemo_svar, obj);
679 *((VALUE *)&svar->lastline) = Qnil;
680 *((VALUE *)&svar->backref) = Qnil;
681 *((VALUE *)&svar->others) = Qnil;
682
683 return svar;
684}
685
686static void
687lep_svar_set(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, VALUE val)
688{
689 struct vm_svar *svar = lep_svar(ec, lep);
690
691 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) {
692 lep_svar_write(ec, lep, svar = svar_new((VALUE)svar));
693 }
694
695 switch (key) {
696 case VM_SVAR_LASTLINE:
697 RB_OBJ_WRITE(svar, &svar->lastline, val);
698 return;
699 case VM_SVAR_BACKREF:
700 RB_OBJ_WRITE(svar, &svar->backref, val);
701 return;
702 default: {
703 VALUE ary = svar->others;
704
705 if (NIL_P(ary)) {
706 RB_OBJ_WRITE(svar, &svar->others, ary = rb_ary_new());
707 }
708 rb_ary_store(ary, key - VM_SVAR_EXTRA_START, val);
709 }
710 }
711}
712
713static inline VALUE
714vm_getspecial(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, rb_num_t type)
715{
716 VALUE val;
717
718 if (type == 0) {
719 val = lep_svar_get(ec, lep, key);
720 }
721 else {
722 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
723
724 if (type & 0x01) {
725 switch (type >> 1) {
726 case '&':
727 val = rb_reg_last_match(backref);
728 break;
729 case '`':
730 val = rb_reg_match_pre(backref);
731 break;
732 case '\'':
733 val = rb_reg_match_post(backref);
734 break;
735 case '+':
736 val = rb_reg_match_last(backref);
737 break;
738 default:
739 rb_bug("unexpected back-ref");
740 }
741 }
742 else {
743 val = rb_reg_nth_match((int)(type >> 1), backref);
744 }
745 }
746 return val;
747}
748
749static inline VALUE
750vm_backref_defined(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t type)
751{
752 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
753 int nth = 0;
754
755 if (type & 0x01) {
756 switch (type >> 1) {
757 case '&':
758 case '`':
759 case '\'':
760 break;
761 case '+':
762 return rb_reg_last_defined(backref);
763 default:
764 rb_bug("unexpected back-ref");
765 }
766 }
767 else {
768 nth = (int)(type >> 1);
769 }
770 return rb_reg_nth_defined(nth, backref);
771}
772
773PUREFUNC(static rb_callable_method_entry_t *check_method_entry(VALUE obj, int can_be_svar));
774static rb_callable_method_entry_t *
775check_method_entry(VALUE obj, int can_be_svar)
776{
777 if (obj == Qfalse) return NULL;
778
779#if VM_CHECK_MODE > 0
780 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_method_entry: unknown type: %s", rb_obj_info(obj));
781#endif
782
783 switch (imemo_type(obj)) {
784 case imemo_ment:
785 return (rb_callable_method_entry_t *)obj;
786 case imemo_cref:
787 return NULL;
788 case imemo_svar:
789 if (can_be_svar) {
790 return check_method_entry(((struct vm_svar *)obj)->cref_or_me, FALSE);
791 }
792 default:
793#if VM_CHECK_MODE > 0
794 rb_bug("check_method_entry: svar should not be there:");
795#endif
796 return NULL;
797 }
798}
799
800static rb_callable_method_entry_t *
801env_method_entry_unchecked(VALUE obj, int can_be_svar)
802{
803 if (obj == Qfalse) return NULL;
804
805 switch (imemo_type(obj)) {
806 case imemo_ment:
807 return (rb_callable_method_entry_t *)obj;
808 case imemo_cref:
809 return NULL;
810 case imemo_svar:
811 if (can_be_svar) {
812 return env_method_entry_unchecked(((struct vm_svar *)obj)->cref_or_me, FALSE);
813 }
814 default:
815 return NULL;
816 }
817}
818
819const rb_callable_method_entry_t *
820rb_vm_frame_method_entry(const rb_control_frame_t *cfp)
821{
822 const VALUE *ep = cfp->ep;
823 rb_callable_method_entry_t *me;
824
825 while (!VM_ENV_LOCAL_P(ep)) {
826 if ((me = check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
827 ep = VM_ENV_PREV_EP(ep);
828 }
829
830 return check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
831}
832
833const rb_callable_method_entry_t *
834rb_vm_frame_method_entry_unchecked(const rb_control_frame_t *cfp)
835{
836 const VALUE *ep = cfp->ep;
837 rb_callable_method_entry_t *me;
838
839 while (!VM_ENV_LOCAL_P_UNCHECKED(ep)) {
840 if ((me = env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
841 ep = VM_ENV_PREV_EP_UNCHECKED(ep);
842 }
843
844 return env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
845}
846
847static const rb_iseq_t *
848method_entry_iseqptr(const rb_callable_method_entry_t *me)
849{
850 switch (me->def->type) {
851 case VM_METHOD_TYPE_ISEQ:
852 return me->def->body.iseq.iseqptr;
853 default:
854 return NULL;
855 }
856}
857
858static rb_cref_t *
859method_entry_cref(const rb_callable_method_entry_t *me)
860{
861 switch (me->def->type) {
862 case VM_METHOD_TYPE_ISEQ:
863 return me->def->body.iseq.cref;
864 default:
865 return NULL;
866 }
867}
868
869#if VM_CHECK_MODE == 0
870PUREFUNC(static rb_cref_t *check_cref(VALUE, int));
871#endif
872static rb_cref_t *
873check_cref(VALUE obj, int can_be_svar)
874{
875 if (obj == Qfalse) return NULL;
876
877#if VM_CHECK_MODE > 0
878 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_cref: unknown type: %s", rb_obj_info(obj));
879#endif
880
881 switch (imemo_type(obj)) {
882 case imemo_ment:
883 return method_entry_cref((rb_callable_method_entry_t *)obj);
884 case imemo_cref:
885 return (rb_cref_t *)obj;
886 case imemo_svar:
887 if (can_be_svar) {
888 return check_cref(((struct vm_svar *)obj)->cref_or_me, FALSE);
889 }
890 default:
891#if VM_CHECK_MODE > 0
892 rb_bug("check_method_entry: svar should not be there:");
893#endif
894 return NULL;
895 }
896}
897
898static inline rb_cref_t *
899vm_env_cref(const VALUE *ep)
900{
901 rb_cref_t *cref;
902
903 while (!VM_ENV_LOCAL_P(ep)) {
904 if ((cref = check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return cref;
905 ep = VM_ENV_PREV_EP(ep);
906 }
907
908 return check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
909}
910
911static int
912is_cref(const VALUE v, int can_be_svar)
913{
914 if (RB_TYPE_P(v, T_IMEMO)) {
915 switch (imemo_type(v)) {
916 case imemo_cref:
917 return TRUE;
918 case imemo_svar:
919 if (can_be_svar) return is_cref(((struct vm_svar *)v)->cref_or_me, FALSE);
920 default:
921 break;
922 }
923 }
924 return FALSE;
925}
926
927static int
928vm_env_cref_by_cref(const VALUE *ep)
929{
930 while (!VM_ENV_LOCAL_P(ep)) {
931 if (is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) return TRUE;
932 ep = VM_ENV_PREV_EP(ep);
933 }
934 return is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
935}
936
937static rb_cref_t *
938cref_replace_with_duplicated_cref_each_frame(const VALUE *vptr, int can_be_svar, VALUE parent)
939{
940 const VALUE v = *vptr;
941 rb_cref_t *cref, *new_cref;
942
943 if (RB_TYPE_P(v, T_IMEMO)) {
944 switch (imemo_type(v)) {
945 case imemo_cref:
946 cref = (rb_cref_t *)v;
947 new_cref = rb_vm_cref_dup(cref);
948 if (parent) {
949 RB_OBJ_WRITE(parent, vptr, new_cref);
950 }
951 else {
952 VM_FORCE_WRITE(vptr, (VALUE)new_cref);
953 }
954 return (rb_cref_t *)new_cref;
955 case imemo_svar:
956 if (can_be_svar) {
957 return cref_replace_with_duplicated_cref_each_frame(&((struct vm_svar *)v)->cref_or_me, FALSE, v);
958 }
959 /* fall through */
960 case imemo_ment:
961 rb_bug("cref_replace_with_duplicated_cref_each_frame: unreachable");
962 default:
963 break;
964 }
965 }
966 return NULL;
967}
968
969static rb_cref_t *
970vm_cref_replace_with_duplicated_cref(const VALUE *ep)
971{
972 if (vm_env_cref_by_cref(ep)) {
973 rb_cref_t *cref;
974 VALUE envval;
975
976 while (!VM_ENV_LOCAL_P(ep)) {
977 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
978 if ((cref = cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE, envval)) != NULL) {
979 return cref;
980 }
981 ep = VM_ENV_PREV_EP(ep);
982 }
983 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
984 return cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE, envval);
985 }
986 else {
987 rb_bug("vm_cref_dup: unreachable");
988 }
989}
990
991static rb_cref_t *
992vm_get_cref(const VALUE *ep)
993{
994 rb_cref_t *cref = vm_env_cref(ep);
995
996 if (cref != NULL) {
997 return cref;
998 }
999 else {
1000 rb_bug("vm_get_cref: unreachable");
1001 }
1002}
1003
1004rb_cref_t *
1005rb_vm_get_cref(const VALUE *ep)
1006{
1007 return vm_get_cref(ep);
1008}
1009
1010static rb_cref_t *
1011vm_ec_cref(const rb_execution_context_t *ec)
1012{
1013 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1014
1015 if (cfp == NULL) {
1016 return NULL;
1017 }
1018 return vm_get_cref(cfp->ep);
1019}
1020
1021static const rb_cref_t *
1022vm_get_const_key_cref(const VALUE *ep)
1023{
1024 const rb_cref_t *cref = vm_get_cref(ep);
1025 const rb_cref_t *key_cref = cref;
1026
1027 while (cref) {
1028 if (CREF_DYNAMIC(cref)) {
1029 return key_cref;
1030 }
1031 cref = CREF_NEXT(cref);
1032 }
1033
1034 /* no dynamic singleton class or cloned class found */
1035 return NULL;
1036}
1037
1038static rb_cref_t *
1039vm_cref_push(const rb_execution_context_t *ec, VALUE klass, const VALUE *ep, int pushed_by_eval, int singleton)
1040{
1041 rb_cref_t *prev_cref = NULL;
1042
1043 if (ep) {
1044 prev_cref = vm_env_cref(ep);
1045 }
1046 else {
1047 rb_control_frame_t *cfp = vm_get_ruby_level_caller_cfp(ec, ec->cfp);
1048
1049 if (cfp) {
1050 prev_cref = vm_env_cref(cfp->ep);
1051 }
1052 }
1053
1054 return vm_cref_new(klass, METHOD_VISI_PUBLIC, FALSE, prev_cref, pushed_by_eval, singleton);
1055}
1056
1057static inline VALUE
1058vm_get_cbase(const VALUE *ep)
1059{
1060 const rb_cref_t *cref = vm_get_cref(ep);
1061
1062 return CREF_CLASS_FOR_DEFINITION(cref);
1063}
1064
1065static inline VALUE
1066vm_get_const_base(const VALUE *ep)
1067{
1068 const rb_cref_t *cref = vm_get_cref(ep);
1069
1070 while (cref) {
1071 if (!CREF_PUSHED_BY_EVAL(cref)) {
1072 return CREF_CLASS_FOR_DEFINITION(cref);
1073 }
1074 cref = CREF_NEXT(cref);
1075 }
1076
1077 return Qundef;
1078}
1079
1080static inline void
1081vm_check_if_namespace(VALUE klass)
1082{
1083 if (!RB_TYPE_P(klass, T_CLASS) && !RB_TYPE_P(klass, T_MODULE)) {
1084 rb_raise(rb_eTypeError, "%+"PRIsVALUE" is not a class/module", klass);
1085 }
1086}
1087
1088static inline void
1089vm_ensure_not_refinement_module(VALUE self)
1090{
1091 if (RB_TYPE_P(self, T_MODULE) && FL_TEST(self, RMODULE_IS_REFINEMENT)) {
1092 rb_warn("not defined at the refinement, but at the outer class/module");
1093 }
1094}
1095
1096static inline VALUE
1097vm_get_iclass(const rb_control_frame_t *cfp, VALUE klass)
1098{
1099 return klass;
1100}
1101
1102static inline VALUE
1103vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, bool allow_nil, int is_defined)
1104{
1105 void rb_const_warn_if_deprecated(const rb_const_entry_t *ce, VALUE klass, ID id);
1106 VALUE val;
1107
1108 if (NIL_P(orig_klass) && allow_nil) {
1109 /* in current lexical scope */
1110 const rb_cref_t *root_cref = vm_get_cref(ec->cfp->ep);
1111 const rb_cref_t *cref;
1112 VALUE klass = Qnil;
1113
1114 while (root_cref && CREF_PUSHED_BY_EVAL(root_cref)) {
1115 root_cref = CREF_NEXT(root_cref);
1116 }
1117 cref = root_cref;
1118 while (cref && CREF_NEXT(cref)) {
1119 if (CREF_PUSHED_BY_EVAL(cref)) {
1120 klass = Qnil;
1121 }
1122 else {
1123 klass = CREF_CLASS(cref);
1124 }
1125 cref = CREF_NEXT(cref);
1126
1127 if (!NIL_P(klass)) {
1128 VALUE av, am = 0;
1129 rb_const_entry_t *ce;
1130 search_continue:
1131 if ((ce = rb_const_lookup(klass, id))) {
1132 rb_const_warn_if_deprecated(ce, klass, id);
1133 val = ce->value;
1134 if (UNDEF_P(val)) {
1135 if (am == klass) break;
1136 am = klass;
1137 if (is_defined) return 1;
1138 if (rb_autoloading_value(klass, id, &av, NULL)) return av;
1139 rb_autoload_load(klass, id);
1140 goto search_continue;
1141 }
1142 else {
1143 if (is_defined) {
1144 return 1;
1145 }
1146 else {
1147 if (UNLIKELY(!rb_class_owned_p(klass))) {
1148 if (!rb_ractor_shareable_p(val)) {
1149 rb_raise(rb_eRactorIsolationError,
1150 "can not access non-shareable objects in constant %"PRIsVALUE"::%"PRIsVALUE" of a class/module created by another Ractor.", rb_class_path(klass), rb_id2str(id));
1151 }
1152 }
1153 return val;
1154 }
1155 }
1156 }
1157 }
1158 }
1159
1160 /* search self */
1161 if (root_cref && !NIL_P(CREF_CLASS(root_cref))) {
1162 klass = vm_get_iclass(ec->cfp, CREF_CLASS(root_cref));
1163 }
1164 else {
1165 klass = CLASS_OF(ec->cfp->self);
1166 }
1167
1168 if (is_defined) {
1169 return rb_const_defined(klass, id);
1170 }
1171 else {
1172 return rb_const_get(klass, id);
1173 }
1174 }
1175 else {
1176 vm_check_if_namespace(orig_klass);
1177 if (is_defined) {
1178 return rb_public_const_defined_from(orig_klass, id);
1179 }
1180 else {
1181 return rb_public_const_get_from(orig_klass, id);
1182 }
1183 }
1184}
1185
1186VALUE
1187rb_vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, VALUE allow_nil)
1188{
1189 return vm_get_ev_const(ec, orig_klass, id, allow_nil == Qtrue, 0);
1190}
1191
1192static inline VALUE
1193vm_get_ev_const_chain(rb_execution_context_t *ec, const ID *segments)
1194{
1195 VALUE val = Qnil;
1196 int idx = 0;
1197 int allow_nil = TRUE;
1198 if (segments[0] == idNULL) {
1199 val = rb_cObject;
1200 idx++;
1201 allow_nil = FALSE;
1202 }
1203 while (segments[idx]) {
1204 ID id = segments[idx++];
1205 val = vm_get_ev_const(ec, val, id, allow_nil, 0);
1206 allow_nil = FALSE;
1207 }
1208 return val;
1209}
1210
1211
1212static inline VALUE
1213vm_get_cvar_base(const rb_cref_t *cref, const rb_control_frame_t *cfp, int top_level_raise)
1214{
1215 VALUE klass;
1216
1217 if (!cref) {
1218 rb_bug("vm_get_cvar_base: no cref");
1219 }
1220
1221 while (CREF_NEXT(cref) &&
1222 (NIL_P(CREF_CLASS(cref)) || RCLASS_SINGLETON_P(CREF_CLASS(cref)) ||
1223 CREF_PUSHED_BY_EVAL(cref) || CREF_SINGLETON(cref))) {
1224 cref = CREF_NEXT(cref);
1225 }
1226 if (top_level_raise && !CREF_NEXT(cref)) {
1227 rb_raise(rb_eRuntimeError, "class variable access from toplevel");
1228 }
1229
1230 klass = vm_get_iclass(cfp, CREF_CLASS(cref));
1231
1232 if (NIL_P(klass)) {
1233 rb_raise(rb_eTypeError, "no class variables available");
1234 }
1235 return klass;
1236}
1237
1238#define ractor_incidental_shareable_p(cond, val) \
1239 (!(cond) || rb_ractor_shareable_p(val))
1240#define ractor_object_incidental_shareable_p(obj, val) \
1241 ractor_incidental_shareable_p(rb_ractor_shareable_p(obj), val)
1242
1243ALWAYS_INLINE(static VALUE vm_getivar(VALUE, ID, const rb_iseq_t *, IVC, const struct rb_callcache *, int, VALUE));
1244static inline VALUE
1245vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, VALUE default_value)
1246{
1247 VALUE fields_obj;
1248#if OPT_IC_FOR_IVAR
1249 if (SPECIAL_CONST_P(obj)) {
1250 return default_value;
1251 }
1252
1253 switch (BUILTIN_TYPE(obj)) {
1254 case T_OBJECT:
1255 fields_obj = ROBJECT_FIELDS_OBJ(obj);
1256 break;
1257 case T_CLASS:
1258 case T_MODULE:
1259 {
1260 if (UNLIKELY(!rb_class_owned_p(obj))) {
1261 // For two reasons we can only use the fast path on the Ractor
1262 // that owns the class.
1263 // First, only the owner Ractor is allowed to set ivars on classes
1264 // and modules. So we can skip locking.
1265 // Second, other ractors need to check the shareability of the
1266 // values returned from the class ivars.
1267
1268 if (default_value == Qundef) { // defined?
1269 return rb_ivar_defined(obj, id) ? Qtrue : Qundef;
1270 }
1271 else {
1272 goto general_path;
1273 }
1274 }
1275
1276 fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1277 break;
1278 }
1279 default:
1280 fields_obj = rb_obj_fields(obj, id);
1281 }
1282
1283 if (!fields_obj) {
1284 return default_value;
1285 }
1286
1287 VALUE val = Qundef;
1288
1289 shape_id_t shape_id = RBASIC_SHAPE_ID_FOR_READ(fields_obj);
1290 VALUE *ivar_list = rb_imemo_fields_ptr(fields_obj);
1291
1292 rb_getivar_cache cache = rb_getivar_cache_unpack(vm_cache_attr_index_atomic_read(is_attr, ic, cc));
1293
1294 if (LIKELY(cache.shape_offset == shape_id)) {
1295 if (cache.index == ATTR_INDEX_NOT_SET) {
1296 return default_value;
1297 }
1298
1299 val = ivar_list[cache.index];
1300#if USE_DEBUG_COUNTER
1301 RB_DEBUG_COUNTER_INC(ivar_get_ic_hit);
1302
1303 if (RB_TYPE_P(obj, T_OBJECT)) {
1304 RB_DEBUG_COUNTER_INC(ivar_get_obj_hit);
1305 }
1306#endif
1307 RUBY_ASSERT(!UNDEF_P(val));
1308 }
1309 else { // cache miss case
1310#if USE_DEBUG_COUNTER
1311 if (is_attr) {
1312 if (cache.shape_offset != INVALID_SHAPE_ID) {
1313 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_set);
1314 }
1315 else {
1316 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_unset);
1317 }
1318 }
1319 else {
1320 if (cache.shape_offset != INVALID_SHAPE_ID) {
1321 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_set);
1322 }
1323 else {
1324 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_unset);
1325 }
1326 }
1327 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1328
1329 if (RB_TYPE_P(obj, T_OBJECT)) {
1330 RB_DEBUG_COUNTER_INC(ivar_get_obj_miss);
1331 }
1332#endif
1333
1334 if (UNLIKELY(rb_shape_complex_p(shape_id))) {
1335 st_table *table = (st_table *)ivar_list;
1336
1337 RUBY_ASSERT(table);
1338 RUBY_ASSERT(table == rb_imemo_fields_complex_tbl(fields_obj));
1339
1340 if (!st_lookup(table, id, &val)) {
1341 val = default_value;
1342 }
1343 }
1344 else {
1345 shape_id_t previous_cached_offset = cache.shape_offset;
1346 if (rb_shape_get_iv_index_with_hint(shape_id, id, &cache.index, &cache.shape_offset)) {
1347 if (cache.shape_offset != previous_cached_offset) {
1348 RUBY_ASSERT(!rb_shape_complex_p(cache.shape_offset));
1349 RUBY_ASSERT(cache.shape_offset != INVALID_SHAPE_ID);
1350
1351 uint64_t packed_cache = rb_getivar_cache_pack(cache.shape_offset, cache.index);
1352 vm_cache_attr_index_set(is_attr, ic, cc, packed_cache);
1353 }
1354
1355 if (cache.index == ATTR_INDEX_NOT_SET) {
1356 val = default_value;
1357 }
1358 else {
1359 // We fetched the ivar list above
1360 val = ivar_list[cache.index];
1361 RUBY_ASSERT(!UNDEF_P(val));
1362 }
1363 }
1364 else {
1365 vm_cache_attr_index_set(is_attr, ic, cc, rb_getivar_cache_pack(shape_id, ATTR_INDEX_NOT_SET));
1366 val = default_value;
1367 }
1368 }
1369 }
1370
1371 if (!UNDEF_P(default_value)) {
1372 RUBY_ASSERT(!UNDEF_P(val));
1373 }
1374
1375 return val;
1376
1377general_path:
1378#endif /* OPT_IC_FOR_IVAR */
1379 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1380
1381 if (is_attr) {
1382 return rb_attr_get(obj, id);
1383 }
1384 else {
1385 return rb_ivar_get(obj, id);
1386 }
1387}
1388
1389ALWAYS_INLINE(static VALUE vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr));
1390NOINLINE(static VALUE vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic));
1391NOINLINE(static VALUE vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc));
1392
1393static VALUE
1394vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr)
1395{
1396#if OPT_IC_FOR_IVAR
1397 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss);
1398
1399 rb_check_ivar_modifiable(obj);
1400
1401 shape_id_t previous_shape_id = RBASIC_SHAPE_ID(obj);
1402 attr_index_t index = rb_ivar_set_index(obj, id, val);
1403 shape_id_t next_shape_id = RBASIC_SHAPE_ID(obj);
1404
1405 if (!rb_shape_complex_p(next_shape_id)) {
1406 uint64_t packed_cache = rb_setivar_cache_pack(RSHAPE_OFFSET(previous_shape_id), RSHAPE_OFFSET(next_shape_id), index);
1407 vm_cache_attr_index_set(is_attr, ic, cc, packed_cache);
1408 }
1409
1410 RB_DEBUG_COUNTER_INC(ivar_set_obj_miss);
1411 return val;
1412#else
1413 return rb_ivar_set(obj, id, val);
1414#endif
1415}
1416
1417static VALUE
1418vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic)
1419{
1420 return vm_setivar_slowpath(obj, id, val, iseq, ic, NULL, false);
1421}
1422
1423static VALUE
1424vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc)
1425{
1426 return vm_setivar_slowpath(obj, id, val, NULL, NULL, cc, true);
1427}
1428
1429NOINLINE(static VALUE vm_setivar_class(VALUE obj, VALUE val, rb_setivar_cache cache));
1430static VALUE
1431vm_setivar_class(VALUE obj, VALUE val, rb_setivar_cache cache)
1432{
1433 if (UNLIKELY(!rb_class_owned_p(obj))) {
1434 return Qundef;
1435 }
1436
1437 VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1438 if (UNLIKELY(!fields_obj)) {
1439 return Qundef;
1440 }
1441
1442 shape_id_t shape_id = RBASIC_SHAPE_ID(fields_obj);
1443 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, RBASIC_SHAPE_ID(fields_obj), cache);
1444 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1445 return Qundef;
1446 }
1447
1448 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1449
1450 if (shape_id != dest_shape_id) {
1451 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1452 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1453 }
1454
1455 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1456
1457 return val;
1458}
1459
1460NOINLINE(static VALUE vm_setivar_default(VALUE obj, ID id, VALUE val, rb_setivar_cache cache));
1461static VALUE
1462vm_setivar_default(VALUE obj, ID id, VALUE val, rb_setivar_cache cache)
1463{
1464 VALUE fields_obj = rb_obj_fields(obj, id);
1465 if (UNLIKELY(!fields_obj)) {
1466 return Qundef;
1467 }
1468
1469 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1470 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, RBASIC_SHAPE_ID(fields_obj), cache);
1471 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1472 return Qundef;
1473 }
1474
1475 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1476
1477 if (shape_id != dest_shape_id) {
1478 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1479 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1480 }
1481
1482 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1483
1484 return val;
1485}
1486
1487static inline VALUE
1488vm_setivar(VALUE obj, VALUE val, rb_setivar_cache cache)
1489{
1490#if OPT_IC_FOR_IVAR
1491 switch (BUILTIN_TYPE(obj)) {
1492 case T_OBJECT:
1493 {
1494 VM_ASSERT(!rb_ractor_shareable_p(obj) || rb_obj_frozen_p(obj));
1495
1496 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1497 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, shape_id, cache);
1498 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1499 break;
1500 }
1501
1502 VALUE fields_obj = ROBJECT_FIELDS_OBJ(obj);
1503 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1504 if (shape_id != dest_shape_id) {
1505 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1506 if (fields_obj != obj) {
1507 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1508 }
1509 }
1510
1511 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1512 RB_DEBUG_COUNTER_INC(ivar_set_obj_hit);
1513 return val;
1514 }
1515 break;
1516 case T_CLASS:
1517 case T_MODULE:
1518 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss_noobject);
1519 default:
1520 break;
1521 }
1522
1523 return Qundef;
1524#endif /* OPT_IC_FOR_IVAR */
1525}
1526
1527static VALUE
1528update_classvariable_cache(const rb_iseq_t *iseq, VALUE klass, ID id, const rb_cref_t * cref, ICVARC ic)
1529{
1530 VALUE defined_class = 0;
1531 VALUE cvar_value = rb_cvar_find(klass, id, &defined_class);
1532
1533 if (RB_TYPE_P(defined_class, T_ICLASS)) {
1534 defined_class = RBASIC(defined_class)->klass;
1535 }
1536
1537 VALUE rb_cvc_tbl = RCLASS_CVC_TBL(defined_class);
1538 if (!rb_cvc_tbl) {
1539 rb_bug("the cvc table should be set");
1540 }
1541
1542 VALUE ent_data;
1543 if (!rb_marked_id_table_lookup(rb_cvc_tbl, id, &ent_data)) {
1544 rb_bug("should have cvar cache entry");
1545 }
1546
1547 struct rb_cvar_class_tbl_entry *ent = (void *)ent_data;
1548
1549 ent->global_cvar_state = GET_GLOBAL_CVAR_STATE();
1550 RB_OBJ_WRITE((VALUE)ent, &ent->cref, cref);
1551 RB_OBJ_WRITE(iseq, &ic->entry, ent);
1552
1553 RUBY_ASSERT(BUILTIN_TYPE((VALUE)cref) == T_IMEMO && IMEMO_TYPE_P(cref, imemo_cref));
1554
1555 return cvar_value;
1556}
1557
1558static inline VALUE
1559vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, ICVARC ic)
1560{
1561 const rb_cref_t *cref;
1562 cref = vm_get_cref(GET_EP());
1563
1564 // The fast path skips rb_cvar_find's checks, so it is for class_value's owner:
1565 // the sole writer, and the only one allowed to see an unshareable value.
1566 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref &&
1567 LIKELY(rb_class_owned_p(ic->entry->class_value))) {
1568 RB_DEBUG_COUNTER_INC(cvar_read_inline_hit);
1569
1570 VALUE v = rb_ivar_lookup(ic->entry->class_value, id, Qundef);
1571 RUBY_ASSERT(!UNDEF_P(v));
1572
1573 return v;
1574 }
1575
1576 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1577
1578 return update_classvariable_cache(iseq, klass, id, cref, ic);
1579}
1580
1581VALUE
1582rb_vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, ICVARC ic)
1583{
1584 return vm_getclassvariable(iseq, cfp, id, ic);
1585}
1586
1587static inline void
1588vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, VALUE val, ICVARC ic)
1589{
1590 const rb_cref_t *cref;
1591 cref = vm_get_cref(GET_EP());
1592
1593 // as on the read side: the fast path writes straight into class_value
1594 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref &&
1595 LIKELY(rb_class_owned_p(ic->entry->class_value))) {
1596 RB_DEBUG_COUNTER_INC(cvar_write_inline_hit);
1597
1598 rb_class_ivar_set(ic->entry->class_value, id, val);
1599 return;
1600 }
1601
1602 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1603
1604 rb_cvar_set(klass, id, val);
1605
1606 update_classvariable_cache(iseq, klass, id, cref, ic);
1607}
1608
1609void
1610rb_vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, VALUE val, ICVARC ic)
1611{
1612 vm_setclassvariable(iseq, cfp, id, val, ic);
1613}
1614
1615ALWAYS_INLINE(static VALUE vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic));
1616static inline VALUE
1617vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1618{
1619 return vm_getivar(obj, id, iseq, ic, NULL, FALSE, Qnil);
1620}
1621
1622static inline void
1623vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1624{
1625 if (RB_SPECIAL_CONST_P(obj)) {
1626 rb_error_frozen_object(obj);
1627 return;
1628 }
1629
1630 rb_setivar_cache cache = rb_setivar_cache_unpack(vm_ic_atomic_cache_read(ic));
1631 if (UNLIKELY(UNDEF_P(vm_setivar(obj, val, cache)))) {
1632 switch (BUILTIN_TYPE(obj)) {
1633 case T_OBJECT:
1634 break;
1635 case T_CLASS:
1636 case T_MODULE:
1637 if (!UNDEF_P(vm_setivar_class(obj, val, cache))) {
1638 return;
1639 }
1640 break;
1641 default:
1642 if (!UNDEF_P(vm_setivar_default(obj, id, val, cache))) {
1643 return;
1644 }
1645 }
1646 vm_setivar_slowpath_ivar(obj, id, val, iseq, ic);
1647 }
1648}
1649
1650void
1651rb_vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1652{
1653 vm_setinstancevariable(iseq, obj, id, val, ic);
1654}
1655
1656VALUE
1657rb_vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1658{
1659 return vm_getinstancevariable(iseq, obj, id, ic);
1660}
1661
1662static VALUE
1663vm_throw_continue(const rb_execution_context_t *ec, VALUE err)
1664{
1665 /* continue throw */
1666
1667 if (FIXNUM_P(err)) {
1668 ec->tag->state = RUBY_TAG_FATAL;
1669 }
1670 else if (SYMBOL_P(err)) {
1671 ec->tag->state = TAG_THROW;
1672 }
1673 else if (THROW_DATA_P(err)) {
1674 ec->tag->state = THROW_DATA_STATE((struct vm_throw_data *)err);
1675 }
1676 else {
1677 ec->tag->state = TAG_RAISE;
1678 }
1679 return err;
1680}
1681
1682static VALUE
1683vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, enum ruby_tag_type state,
1684 const int flag, const VALUE throwobj)
1685{
1686 const rb_control_frame_t *escape_cfp = NULL;
1687 const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */
1688
1689 if (flag != 0) {
1690 /* do nothing */
1691 }
1692 else if (state == TAG_BREAK) {
1693 int is_orphan = 1;
1694 const VALUE *ep = GET_EP();
1695 const rb_iseq_t *base_iseq = GET_ISEQ();
1696 escape_cfp = reg_cfp;
1697
1698 while (ISEQ_BODY(base_iseq)->type != ISEQ_TYPE_BLOCK) {
1699 if (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) {
1700 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1701 ep = escape_cfp->ep;
1702 base_iseq = CFP_ISEQ(escape_cfp);
1703 }
1704 else {
1705 ep = VM_ENV_PREV_EP(ep);
1706 base_iseq = ISEQ_BODY(base_iseq)->parent_iseq;
1707 escape_cfp = rb_vm_search_cf_from_ep(ec, escape_cfp, ep);
1708 VM_ASSERT(CFP_ISEQ(escape_cfp) == base_iseq);
1709 }
1710 }
1711
1712 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1713 /* lambda{... break ...} */
1714 is_orphan = 0;
1715 state = TAG_RETURN;
1716 }
1717 else {
1718 ep = VM_ENV_PREV_EP(ep);
1719
1720 while (escape_cfp < eocfp) {
1721 if (escape_cfp->ep == ep) {
1722 const rb_iseq_t *const iseq = CFP_ISEQ(escape_cfp);
1723 const VALUE epc = CFP_PC(escape_cfp) - ISEQ_BODY(iseq)->iseq_encoded;
1724 const struct iseq_catch_table *const ct = ISEQ_BODY(iseq)->catch_table;
1725 unsigned int i;
1726
1727 if (!ct) break;
1728 for (i=0; i < ct->size; i++) {
1729 const struct iseq_catch_table_entry *const entry =
1730 UNALIGNED_MEMBER_PTR(ct, entries[i]);
1731
1732 if (entry->type == CATCH_TYPE_BREAK &&
1733 entry->iseq == base_iseq &&
1734 entry->start < epc && entry->end >= epc) {
1735 if (entry->cont == epc) { /* found! */
1736 is_orphan = 0;
1737 }
1738 break;
1739 }
1740 }
1741 break;
1742 }
1743
1744 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1745 }
1746 }
1747
1748 if (is_orphan) {
1749 rb_vm_localjump_error("break from proc-closure", throwobj, TAG_BREAK);
1750 }
1751 }
1752 else if (state == TAG_RETRY) {
1753 const VALUE *ep = VM_ENV_PREV_EP(GET_EP());
1754
1755 escape_cfp = rb_vm_search_cf_from_ep(ec, reg_cfp, ep);
1756 }
1757 else if (state == TAG_RETURN) {
1758 const VALUE *current_ep = GET_EP();
1759 const VALUE *target_ep = NULL, *target_lep, *ep = current_ep;
1760 int in_class_frame = 0;
1761 int toplevel = 1;
1762 escape_cfp = reg_cfp;
1763
1764 // find target_lep, target_ep
1765 while (!VM_ENV_LOCAL_P(ep)) {
1766 if (VM_ENV_FLAGS(ep, VM_FRAME_FLAG_LAMBDA) && target_ep == NULL) {
1767 target_ep = ep;
1768 }
1769 ep = VM_ENV_PREV_EP(ep);
1770 }
1771 target_lep = ep;
1772
1773 while (escape_cfp < eocfp) {
1774 const VALUE *lep = VM_CF_LEP(escape_cfp);
1775
1776 if (!target_lep) {
1777 target_lep = lep;
1778 }
1779
1780 if (lep == target_lep &&
1781 VM_FRAME_RUBYFRAME_P(escape_cfp) &&
1782 ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) {
1783 in_class_frame = 1;
1784 target_lep = 0;
1785 }
1786
1787 if (lep == target_lep) {
1788 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1789 toplevel = 0;
1790 if (in_class_frame) {
1791 /* lambda {class A; ... return ...; end} */
1792 goto valid_return;
1793 }
1794 else {
1795 const VALUE *tep = current_ep;
1796
1797 while (target_lep != tep) {
1798 if (escape_cfp->ep == tep) {
1799 /* in lambda */
1800 if (tep == target_ep) {
1801 goto valid_return;
1802 }
1803 else {
1804 goto unexpected_return;
1805 }
1806 }
1807 tep = VM_ENV_PREV_EP(tep);
1808 }
1809 }
1810 }
1811 else if (VM_FRAME_RUBYFRAME_P(escape_cfp)) {
1812 switch (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type) {
1813 case ISEQ_TYPE_TOP:
1814 case ISEQ_TYPE_MAIN:
1815 if (toplevel) {
1816 if (in_class_frame) goto unexpected_return;
1817 if (target_ep == NULL) {
1818 goto valid_return;
1819 }
1820 else {
1821 goto unexpected_return;
1822 }
1823 }
1824 break;
1825 case ISEQ_TYPE_EVAL: {
1826 const rb_iseq_t *is = CFP_ISEQ(escape_cfp);
1827 enum rb_iseq_type t = ISEQ_BODY(is)->type;
1828 while (t == ISEQ_TYPE_RESCUE || t == ISEQ_TYPE_ENSURE || t == ISEQ_TYPE_EVAL) {
1829 if (!(is = ISEQ_BODY(is)->parent_iseq)) break;
1830 t = ISEQ_BODY(is)->type;
1831 }
1832 toplevel = t == ISEQ_TYPE_TOP || t == ISEQ_TYPE_MAIN;
1833 break;
1834 }
1835 case ISEQ_TYPE_CLASS:
1836 toplevel = 0;
1837 break;
1838 default:
1839 break;
1840 }
1841 }
1842 }
1843
1844 if (escape_cfp->ep == target_lep && ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_METHOD) {
1845 if (target_ep == NULL) {
1846 goto valid_return;
1847 }
1848 else {
1849 goto unexpected_return;
1850 }
1851 }
1852
1853 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1854 }
1855 unexpected_return:;
1856 rb_vm_localjump_error("unexpected return", throwobj, TAG_RETURN);
1857
1858 valid_return:;
1859 /* do nothing */
1860 }
1861 else {
1862 rb_bug("isns(throw): unsupported throw type");
1863 }
1864
1865 ec->tag->state = state;
1866 return (VALUE)THROW_DATA_NEW(throwobj, escape_cfp, state);
1867}
1868
1869static VALUE
1870vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
1871 rb_num_t throw_state, VALUE throwobj)
1872{
1873 const int state = (int)(throw_state & VM_THROW_STATE_MASK);
1874 const int flag = (int)(throw_state & VM_THROW_NO_ESCAPE_FLAG);
1875
1876 if (state != 0) {
1877 return vm_throw_start(ec, reg_cfp, state, flag, throwobj);
1878 }
1879 else {
1880 return vm_throw_continue(ec, throwobj);
1881 }
1882}
1883
1884// Fallback for YJIT. Prepare throw data and return it.
1885VALUE
1886rb_vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj)
1887{
1888 return vm_throw(ec, reg_cfp, throw_state, throwobj);
1889}
1890
1891NORETURN(VALUE rb_zjit_throw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj));
1892
1893// Fallback for ZJIT. Make a longjmp and unwind to the most recent vm_exec().
1894VALUE
1895rb_zjit_throw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj)
1896{
1897 VALUE val = vm_throw(ec, reg_cfp, throw_state, throwobj);
1898
1899 // vm_throw() has set ec->tag->state. On the longjmp path, vm_exec() reads the throw data from
1900 // ec->errinfo instead of vm_exec_core()'s return value like THROW_EXCEPTION()'s path does, so
1901 // we need to put it there instead.
1902 enum ruby_tag_type state = ec->tag->state;
1903 ec->errinfo = val;
1904 EC_JUMP_TAG(ec, state);
1905
1906 UNREACHABLE_RETURN(Qundef);
1907}
1908
1909static inline void
1910vm_expandarray(struct rb_control_frame_struct *cfp, VALUE ary, rb_num_t num, int flag)
1911{
1912 int is_splat = flag & 0x01;
1913 const VALUE *ptr;
1914 rb_num_t len;
1915 const VALUE obj = ary;
1916
1917 if (!RB_TYPE_P(ary, T_ARRAY) && NIL_P(ary = rb_check_array_type(ary))) {
1918 ary = obj;
1919 ptr = &ary;
1920 len = 1;
1921 }
1922 else {
1923 ptr = RARRAY_CONST_PTR(ary);
1924 len = (rb_num_t)RARRAY_LEN(ary);
1925 }
1926
1927 if (num + is_splat == 0) {
1928 /* no space left on stack */
1929 }
1930 else if (flag & 0x02) {
1931 /* post: ..., nil ,ary[-1], ..., ary[0..-num] # top */
1932 rb_num_t i = 0, j;
1933
1934 if (len < num) {
1935 for (i = 0; i < num - len; i++) {
1936 *cfp->sp++ = Qnil;
1937 }
1938 }
1939
1940 for (j = 0; i < num; i++, j++) {
1941 VALUE v = ptr[len - j - 1];
1942 *cfp->sp++ = v;
1943 }
1944
1945 if (is_splat) {
1946 *cfp->sp++ = rb_ary_new4(len - j, ptr);
1947 }
1948 }
1949 else {
1950 /* normal: ary[num..-1], ary[num-2], ary[num-3], ..., ary[0] # top */
1951 if (is_splat) {
1952 if (num > len) {
1953 *cfp->sp++ = rb_ary_new();
1954 }
1955 else {
1956 *cfp->sp++ = rb_ary_new4(len - num, ptr + num);
1957 }
1958 }
1959
1960 if (num > len) {
1961 rb_num_t i = 0;
1962 for (; i < num - len; i++) {
1963 *cfp->sp++ = Qnil;
1964 }
1965
1966 for (rb_num_t j = 0; i < num; i++, j++) {
1967 *cfp->sp++ = ptr[len - j - 1];
1968 }
1969 }
1970 else {
1971 for (rb_num_t j = 0; j < num; j++) {
1972 *cfp->sp++ = ptr[num - j - 1];
1973 }
1974 }
1975 }
1976
1977 RB_GC_GUARD(ary);
1978}
1979
1980static VALUE vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
1981
1982static VALUE vm_mtbl_dump(VALUE klass, ID target_mid);
1983
1984static struct rb_class_cc_entries *
1985vm_ccs_create(VALUE klass, VALUE cc_tbl, ID mid, const rb_callable_method_entry_t *cme)
1986{
1987 int initial_capa = 2;
1988 struct rb_class_cc_entries *ccs = ruby_xmalloc(vm_ccs_alloc_size(initial_capa));
1989#if VM_CHECK_MODE > 0
1990 ccs->debug_sig = ~(VALUE)ccs;
1991#endif
1992 ccs->capa = initial_capa;
1993 ccs->len = 0;
1994 ccs->cme = cme;
1995 METHOD_ENTRY_CACHED_SET((rb_callable_method_entry_t *)cme);
1996
1997 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
1998 RB_OBJ_WRITTEN(cc_tbl, Qundef, cme);
1999 return ccs;
2000}
2001
2002static void
2003vm_ccs_push(VALUE cc_tbl, ID mid, struct rb_class_cc_entries *ccs, const struct rb_callinfo *ci, const struct rb_callcache *cc)
2004{
2005 if (! vm_cc_markable(cc)) {
2006 return;
2007 }
2008
2009 if (UNLIKELY(ccs->len == ccs->capa)) {
2010 RUBY_ASSERT(ccs->capa > 0);
2011 ccs->capa *= 2;
2012 ccs = ruby_xrealloc(ccs, vm_ccs_alloc_size(ccs->capa));
2013#if VM_CHECK_MODE > 0
2014 ccs->debug_sig = ~(VALUE)ccs;
2015#endif
2016 // GC?
2017 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
2018 }
2019 VM_ASSERT(ccs->len < ccs->capa);
2020
2021 const struct rb_callinfo_kwarg *kwarg = vm_ci_kwarg(ci);
2022 const int pos = ccs->len++;
2023 ccs->entries[pos].argc = vm_ci_argc(ci);
2024 ccs->entries[pos].flag = (unsigned short)vm_ci_flag(ci);
2025 ccs->entries[pos].kw_len = (unsigned short)(kwarg ? kwarg->keyword_len : 0);
2026 RB_OBJ_WRITE(cc_tbl, &ccs->entries[pos].cc, cc);
2027
2028 if (RB_DEBUG_COUNTER_SETMAX(ccs_maxlen, ccs->len)) {
2029 // for tuning
2030 // vm_mtbl_dump(klass, 0);
2031 }
2032}
2033
2034#if VM_CHECK_MODE > 0
2035void
2036rb_vm_ccs_dump(struct rb_class_cc_entries *ccs)
2037{
2038 ruby_debug_printf("ccs:%p (%d,%d)\n", (void *)ccs, ccs->len, ccs->capa);
2039 for (int i=0; i<ccs->len; i++) {
2040 ruby_debug_printf("CCS CI ID:flag:%x argc:%u kw_len:%u\n",
2041 ccs->entries[i].flag,
2042 ccs->entries[i].argc,
2043 ccs->entries[i].kw_len);
2044 rp(ccs->entries[i].cc);
2045 }
2046}
2047
2048static int
2049vm_ccs_verify(struct rb_class_cc_entries *ccs, ID mid, VALUE klass)
2050{
2051 VM_ASSERT(vm_ccs_p(ccs));
2052 VM_ASSERT(ccs->len <= ccs->capa);
2053
2054 for (int i=0; i<ccs->len; i++) {
2055 const struct rb_callcache *cc = ccs->entries[i].cc;
2056
2057 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
2058 VM_ASSERT(vm_cc_class_check(cc, klass));
2059 VM_ASSERT(vm_cc_check_cme(cc, ccs->cme));
2060 VM_ASSERT(!vm_cc_super_p(cc));
2061 VM_ASSERT(!vm_cc_refinement_p(cc));
2062 }
2063 return TRUE;
2064}
2065#endif
2066
2067const rb_callable_method_entry_t *rb_check_overloaded_cme(const rb_callable_method_entry_t *cme, const struct rb_callinfo * const ci);
2068
2069static void
2070vm_evict_cc(VALUE klass, VALUE cc_tbl, ID mid)
2071{
2072 ASSERT_vm_locking();
2073
2074 if (rb_multi_ractor_p()) {
2075 if (RCLASS_WRITABLE_CC_TBL(klass) != cc_tbl) {
2076 // Another ractor updated the CC table while we were waiting on the VM lock.
2077 // We have to retry.
2078 return;
2079 }
2080
2081 VALUE ccs_obj = 0;
2082 rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj);
2083 struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_obj;
2084
2085 if (!ccs || !METHOD_ENTRY_INVALIDATED(ccs->cme)) {
2086 // Another ractor replaced that entry while we were waiting on the VM lock.
2087 return;
2088 }
2089
2090 VALUE new_table = rb_vm_cc_table_dup(cc_tbl);
2091 rb_vm_cc_table_delete(new_table, mid);
2092 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), new_table);
2093 }
2094 else {
2095 rb_vm_cc_table_delete(cc_tbl, mid);
2096 }
2097}
2098
2099static const struct rb_callcache *
2100vm_populate_cc(VALUE klass, const struct rb_callinfo * const ci, ID mid)
2101{
2102 ASSERT_vm_locking();
2103
2104 RB_DEBUG_COUNTER_INC(cc_not_found_in_ccs);
2105
2106 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
2107
2108 VM_ASSERT(cme == NULL || IMEMO_TYPE_P(cme, imemo_ment));
2109
2110 if (cme == NULL) {
2111 // undef or not found: can't cache the information
2112 VM_ASSERT(vm_cc_cme(&vm_empty_cc) == NULL);
2113 return &vm_empty_cc;
2114 }
2115
2116 VALUE cc_tbl = RCLASS_WRITABLE_CC_TBL(klass);
2117 const VALUE original_cc_table = cc_tbl;
2118 if (!cc_tbl) {
2119 // Is this possible after rb_callable_method_entry ?
2120 cc_tbl = rb_vm_cc_table_create(1);
2121 }
2122 else if (rb_multi_ractor_p()) {
2123 cc_tbl = rb_vm_cc_table_dup(cc_tbl);
2124 }
2125
2126 VM_ASSERT(cme == rb_callable_method_entry(klass, mid));
2127
2128 METHOD_ENTRY_CACHED_SET((struct rb_callable_method_entry_struct *)cme);
2129
2130 VM_ASSERT(cc_tbl);
2131
2132 struct rb_class_cc_entries *ccs = NULL;
2133 {
2134 VALUE ccs_obj;
2135 if (UNLIKELY(rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj))) {
2136 ccs = (struct rb_class_cc_entries *)ccs_obj;
2137 }
2138 else {
2139 // TODO: required?
2140 ccs = vm_ccs_create(klass, cc_tbl, mid, cme);
2141 }
2142 }
2143
2144 cme = rb_check_overloaded_cme(cme, ci);
2145
2146 const struct rb_callcache *cc = vm_cc_new(klass, cme, vm_call_general, cc_type_normal);
2147 vm_ccs_push(cc_tbl, mid, ccs, ci, cc);
2148
2149 VM_ASSERT(vm_cc_cme(cc) != NULL);
2150 VM_ASSERT(cme->called_id == mid);
2151 VM_ASSERT(vm_cc_cme(cc)->called_id == mid);
2152
2153 if (original_cc_table != cc_tbl) {
2154 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), cc_tbl);
2155 }
2156
2157 return cc;
2158}
2159
2160static const struct rb_callcache *
2161vm_lookup_cc(const VALUE klass, const struct rb_callinfo * const ci, ID mid)
2162{
2163 VALUE cc_tbl;
2164 struct rb_class_cc_entries *ccs;
2165retry:
2166 cc_tbl = RUBY_ATOMIC_VALUE_LOAD(RCLASS_WRITABLE_CC_TBL(klass));
2167 ccs = NULL;
2168
2169 if (cc_tbl) {
2170 // CCS data is keyed on method id, so we don't need the method id
2171 // for doing comparisons in the `for` loop below.
2172
2173 VALUE ccs_obj;
2174 if (rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj)) {
2175 ccs = (struct rb_class_cc_entries *)ccs_obj;
2176 const int ccs_len = ccs->len;
2177
2178 if (UNLIKELY(METHOD_ENTRY_INVALIDATED(ccs->cme))) {
2179 RB_VM_LOCKING() {
2180 vm_evict_cc(klass, cc_tbl, mid);
2181 }
2182 goto retry;
2183 }
2184 else {
2185 VM_ASSERT(vm_ccs_verify(ccs, mid, klass));
2186
2187 // We already know the method id is correct because we had
2188 // to look up the ccs_data by method id. We compare argc and
2189 // flag, plus the keyword argument count: two call sites with
2190 // the same argc and flag can still differ in how many of the
2191 // arguments are keywords (e.g. `m(a: 1, b: 2)` vs `m(1, b: 2)`),
2192 // which selects a different argument setup fastpath.
2193 const struct rb_callinfo_kwarg *kwarg = vm_ci_kwarg(ci);
2194 unsigned int argc = vm_ci_argc(ci);
2195 unsigned int flag = vm_ci_flag(ci);
2196 unsigned int kw_len = kwarg ? kwarg->keyword_len : 0;
2197
2198 for (int i=0; i<ccs_len; i++) {
2199 unsigned int ccs_ci_argc = ccs->entries[i].argc;
2200 unsigned int ccs_ci_flag = ccs->entries[i].flag;
2201 unsigned int ccs_ci_kw_len = ccs->entries[i].kw_len;
2202 const struct rb_callcache *ccs_cc = ccs->entries[i].cc;
2203
2204 VM_ASSERT(IMEMO_TYPE_P(ccs_cc, imemo_callcache));
2205
2206 if (ccs_ci_argc == argc && ccs_ci_flag == flag && ccs_ci_kw_len == kw_len) {
2207 RB_DEBUG_COUNTER_INC(cc_found_in_ccs);
2208
2209 VM_ASSERT(vm_cc_cme(ccs_cc)->called_id == mid);
2210 VM_ASSERT(ccs_cc->klass == klass);
2211 VM_ASSERT(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(ccs_cc)));
2212
2213 return ccs_cc;
2214 }
2215 }
2216 }
2217 }
2218 }
2219
2220 RB_GC_GUARD(cc_tbl);
2221 return NULL;
2222}
2223
2224static const struct rb_callcache *
2225vm_search_cc(const VALUE klass, const struct rb_callinfo * const ci)
2226{
2227 const ID mid = vm_ci_mid(ci);
2228
2229 const struct rb_callcache *cc = vm_lookup_cc(klass, ci, mid);
2230 if (cc) {
2231 return cc;
2232 }
2233
2234 RB_VM_LOCKING() {
2235 if (rb_multi_ractor_p()) {
2236 // The CC may have been populated by another ractor while we were waiting on the lock,
2237 // so we must lookup a second time.
2238 cc = vm_lookup_cc(klass, ci, mid);
2239 }
2240
2241 if (!cc) {
2242 cc = vm_populate_cc(klass, ci, mid);
2243 }
2244 }
2245
2246 return cc;
2247}
2248
2249const struct rb_callcache *
2250rb_vm_search_method_slowpath(const struct rb_callinfo *ci, VALUE klass)
2251{
2252 const struct rb_callcache *cc;
2253
2254 VM_ASSERT(!SPECIAL_CONST_P(klass));
2255
2256 if (RB_BUILTIN_TYPE(klass) == T_NONE) {
2257 // If we find a T_NONE here, it's most likely we called CLASS_OF(obj) on a
2258 // garbage collected object (the freelist is stored in the class pointer),
2259 // but it's possible that just the class was GC'd.
2260 // This message intentionally tries to imply the former, but make an
2261 // accurate statement for either case.
2262 rb_bug("attempted to search method '%s' on a garbage collected object",
2263 rb_id2name(vm_ci_mid(ci)));
2264 }
2265
2266 VM_ASSERT_TYPE2(klass, T_CLASS, T_ICLASS);
2267
2268 cc = vm_search_cc(klass, ci);
2269
2270 VM_ASSERT(cc);
2271 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
2272 VM_ASSERT(cc == vm_cc_empty() || cc->klass == klass);
2273 VM_ASSERT(cc == vm_cc_empty() || callable_method_entry_p(vm_cc_cme(cc)));
2274 VM_ASSERT(cc == vm_cc_empty() || !METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)));
2275 VM_ASSERT(cc == vm_cc_empty() || vm_cc_cme(cc)->called_id == vm_ci_mid(ci));
2276
2277 return cc;
2278}
2279
2280#if VM_CHECK_MODE > 0
2281static bool
2282vm_cd_owned_by_iseq_p(const struct rb_call_data *cd, const rb_iseq_t *iseq)
2283{
2284 const struct rb_iseq_constant_body *body = ISEQ_BODY(iseq);
2285 return cd >= body->call_data && cd < body->call_data + body->ci_size;
2286}
2287#endif
2288
2289static const struct rb_callcache *
2290vm_search_method_slowpath0(VALUE cd_owner, struct rb_call_data *cd, VALUE klass)
2291{
2292#if USE_DEBUG_COUNTER
2293 const struct rb_callcache *old_cc = cd->cc;
2294 const rb_callable_method_entry_t *const old_cme = old_cc ? old_cc->cme_ : NULL;
2295#endif
2296
2297 const struct rb_callcache *cc = rb_vm_search_method_slowpath(cd->ci, klass);
2298 const rb_callable_method_entry_t *const new_cme = vm_cc_cme(cc);
2299
2300#if OPT_INLINE_METHOD_CACHE
2301 cd->cc = cc;
2302
2303 const struct rb_callcache *empty_cc = &vm_empty_cc;
2304 if (cd_owner && cc != empty_cc) {
2305 // invokesuper dispatches with a cd on the machine stack, which has no
2306 // owning iseq to remember and keeps its ci out of the GC.
2307 VM_ASSERT(!vm_ci_markable(cd->ci) ||
2308 vm_cd_owned_by_iseq_p(cd, (const rb_iseq_t *)cd_owner));
2309 RB_OBJ_WRITTEN(cd_owner, Qundef, cc);
2310 }
2311
2312#if USE_DEBUG_COUNTER
2313 if (!old_cc || old_cc == empty_cc) {
2314 // empty
2315 RB_DEBUG_COUNTER_INC(mc_inline_miss_empty);
2316 }
2317 else if (old_cc == cc) {
2318 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cc);
2319 }
2320 else if (old_cme == new_cme) {
2321 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cme);
2322 }
2323 else if (old_cme && new_cme && old_cme->def == new_cme->def) {
2324 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_def);
2325 }
2326 else {
2327 RB_DEBUG_COUNTER_INC(mc_inline_miss_diff);
2328 }
2329#endif
2330#endif // OPT_INLINE_METHOD_CACHE
2331
2332 if (new_cme) VM_ASSERT(new_cme->called_id == vm_ci_mid(cd->ci));
2333
2334 return cc;
2335}
2336
2337ALWAYS_INLINE(static bool vm_cc_hit_p(const struct rb_callcache *cc, const struct rb_call_data *cd, VALUE klass));
2338static bool
2339vm_cc_hit_p(const struct rb_callcache *cc, const struct rb_call_data *cd, VALUE klass)
2340{
2341 VM_ASSERT_TYPE2(klass, T_CLASS, T_ICLASS);
2342
2343#if OPT_INLINE_METHOD_CACHE
2344 if (LIKELY(vm_cc_class_check(cc, klass))) {
2345 if (LIKELY(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)))) {
2346 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
2347 RB_DEBUG_COUNTER_INC(mc_inline_hit);
2348 VM_ASSERT(vm_cc_cme(cc) == NULL || // not found
2349 (vm_ci_flag(cd->ci) & VM_CALL_SUPER) || // search_super w/ define_method
2350 vm_cc_cme(cc)->called_id == vm_ci_mid(cd->ci)); // cme->called_id == ci->mid
2351
2352 return true;
2353 }
2354 RB_DEBUG_COUNTER_INC(mc_inline_miss_invalidated);
2355 }
2356 else {
2357 RB_DEBUG_COUNTER_INC(mc_inline_miss_klass);
2358 }
2359#endif
2360
2361 return false;
2362}
2363
2364ALWAYS_INLINE(static const struct rb_callcache *vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE klass));
2365static const struct rb_callcache *
2366vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE klass)
2367{
2368 const struct rb_callcache *cc = cd->cc;
2369 if (vm_cc_hit_p(cc, cd, klass)) {
2370 return cc;
2371 }
2372
2373 return vm_search_method_slowpath0((VALUE)CFP_ISEQ(reg_cfp), cd, klass);
2374}
2375
2376static const struct rb_callable_method_entry_struct *
2377vm_search_method(struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE recv)
2378{
2379 VALUE klass = CLASS_OF(recv);
2380 VM_ASSERT(klass != Qfalse);
2381 VM_ASSERT(RBASIC_CLASS(klass) == 0 || rb_obj_is_kind_of(klass, rb_cClass));
2382
2383 const struct rb_callcache *cc = vm_search_method_fastpath(reg_cfp, cd, klass);
2384 return vm_cc_cme(cc);
2385}
2386
2387const struct rb_callable_method_entry_struct *
2388rb_zjit_vm_search_method(VALUE cd_owner, struct rb_call_data *cd, VALUE recv)
2389{
2390 // Called from ZJIT with the compile-time iseq, which may differ from
2391 // the iseq on the current CFP.
2392 VALUE klass = CLASS_OF(recv);
2393 const struct rb_callcache *cc = cd->cc;
2394 if (!vm_cc_hit_p(cc, cd, klass)) {
2395 cc = vm_search_method_slowpath0(cd_owner, cd, klass);
2396 }
2397 return vm_cc_cme(cc);
2398}
2399
2400#if __has_attribute(transparent_union)
2401typedef union {
2402 VALUE (*anyargs)(ANYARGS);
2403 VALUE (*f00)(VALUE);
2404 VALUE (*f01)(VALUE, VALUE);
2405 VALUE (*f02)(VALUE, VALUE, VALUE);
2406 VALUE (*f03)(VALUE, VALUE, VALUE, VALUE);
2407 VALUE (*f04)(VALUE, VALUE, VALUE, VALUE, VALUE);
2408 VALUE (*f05)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2409 VALUE (*f06)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2410 VALUE (*f07)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2411 VALUE (*f08)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2412 VALUE (*f09)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2413 VALUE (*f10)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2414 VALUE (*f11)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2415 VALUE (*f12)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2416 VALUE (*f13)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2417 VALUE (*f14)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2418 VALUE (*f15)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2419 VALUE (*fm1)(int, union { VALUE *x; const VALUE *y; } __attribute__((__transparent_union__)), VALUE);
2420} __attribute__((__transparent_union__)) cfunc_type;
2421# define make_cfunc_type(f) (cfunc_type){.anyargs = (VALUE (*)(ANYARGS))(f)}
2422#else
2423typedef VALUE (*cfunc_type)(ANYARGS);
2424# define make_cfunc_type(f) (cfunc_type)(f)
2425#endif
2426
2427static inline int
2428check_cfunc(const rb_callable_method_entry_t *me, cfunc_type func)
2429{
2430 if (! me) {
2431 return false;
2432 }
2433 else {
2434 VM_ASSERT(IMEMO_TYPE_P(me, imemo_ment));
2435 VM_ASSERT(callable_method_entry_p(me));
2436 VM_ASSERT(me->def);
2437 if (me->def->type != VM_METHOD_TYPE_CFUNC) {
2438 return false;
2439 }
2440 else {
2441#if __has_attribute(transparent_union)
2442 return me->def->body.cfunc.func == func.anyargs;
2443#else
2444 return me->def->body.cfunc.func == func;
2445#endif
2446 }
2447 }
2448}
2449
2450static inline int
2451check_method_basic_definition(const rb_callable_method_entry_t *me)
2452{
2453 return me && METHOD_ENTRY_BASIC(me);
2454}
2455
2456static inline int
2457vm_method_cfunc_is(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv, cfunc_type func)
2458{
2459 VM_ASSERT(reg_cfp != NULL);
2460 const struct rb_callable_method_entry_struct *cme = vm_search_method(reg_cfp, cd, recv);
2461 return check_cfunc(cme, func);
2462}
2463
2464bool
2465rb_zjit_cme_is_cfunc(const rb_callable_method_entry_t *me, const cfunc_type func)
2466{
2467 return check_cfunc(me, func);
2468}
2469
2470int
2471rb_vm_method_cfunc_is(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv, cfunc_type func)
2472{
2473 const struct rb_callable_method_entry_struct *cme = rb_zjit_vm_search_method((VALUE)iseq, cd, recv);
2474 return check_cfunc(cme, func);
2475}
2476
2477#define check_cfunc(me, func) check_cfunc(me, make_cfunc_type(func))
2478#define vm_method_cfunc_is(reg_cfp, cd, recv, func) vm_method_cfunc_is(reg_cfp, cd, recv, make_cfunc_type(func))
2479
2480#define OP_UNREDEFINED_P(op, t) BASIC_OP_UNREDEFINED_P(BOP_##op, t##_REDEFINED_OP_FLAG)
2481#define EQ_UNREDEFINED_P(t) OP_UNREDEFINED_P(EQ, t)
2482
2483static inline bool
2484FIXNUM_2_P(VALUE a, VALUE b)
2485{
2486 /* FIXNUM_P(a) && FIXNUM_P(b)
2487 * == ((a & 1) && (b & 1))
2488 * == a & b & 1 */
2489 SIGNED_VALUE x = a;
2490 SIGNED_VALUE y = b;
2491 SIGNED_VALUE z = x & y & 1;
2492 return z == 1;
2493}
2494
2495static inline bool
2496FLONUM_2_P(VALUE a, VALUE b)
2497{
2498#if USE_FLONUM
2499 /* FLONUM_P(a) && FLONUM_P(b)
2500 * == ((a & 3) == 2) && ((b & 3) == 2)
2501 * == ! ((a ^ 2) | (b ^ 2) & 3)
2502 */
2503 SIGNED_VALUE x = a;
2504 SIGNED_VALUE y = b;
2505 SIGNED_VALUE z = ((x ^ 2) | (y ^ 2)) & 3;
2506 return !z;
2507#else
2508 return false;
2509#endif
2510}
2511
2512static VALUE
2513opt_equality_specialized(VALUE recv, VALUE obj)
2514{
2515 if (FIXNUM_2_P(recv, obj) && EQ_UNREDEFINED_P(INTEGER)) {
2516 goto compare_by_identity;
2517 }
2518 else if (FLONUM_2_P(recv, obj) && EQ_UNREDEFINED_P(FLOAT)) {
2519 goto compare_by_identity;
2520 }
2521 else if (STATIC_SYM_P(recv) && STATIC_SYM_P(obj) && EQ_UNREDEFINED_P(SYMBOL)) {
2522 goto compare_by_identity;
2523 }
2524 else if (SPECIAL_CONST_P(recv)) {
2525 //
2526 }
2527 else if (RBASIC_CLASS(recv) == rb_cFloat && RB_FLOAT_TYPE_P(obj) && EQ_UNREDEFINED_P(FLOAT)) {
2528 double a = RFLOAT_VALUE(recv);
2529 double b = RFLOAT_VALUE(obj);
2530
2531 return RBOOL(a == b);
2532 }
2533 else if (RBASIC_CLASS(recv) == rb_cString && EQ_UNREDEFINED_P(STRING)) {
2534 if (recv == obj) {
2535 return Qtrue;
2536 }
2537 else if (RB_TYPE_P(obj, T_STRING)) {
2538 return rb_str_eql_internal(obj, recv);
2539 }
2540 }
2541 return Qundef;
2542
2543 compare_by_identity:
2544 return RBOOL(recv == obj);
2545}
2546
2547static VALUE
2548opt_equality(struct rb_control_frame_struct *reg_cfp, VALUE recv, VALUE obj, CALL_DATA cd)
2549{
2550 VM_ASSERT(reg_cfp != NULL);
2551
2552 VALUE val = opt_equality_specialized(recv, obj);
2553 if (!UNDEF_P(val)) return val;
2554
2555 if (!vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_equal)) {
2556 return Qundef;
2557 }
2558 else {
2559 return RBOOL(recv == obj);
2560 }
2561}
2562
2563static inline const struct rb_callcache *gccct_method_search(rb_execution_context_t *ec, VALUE recv, ID mid, const struct rb_callinfo *ci); // vm_eval.c
2564NOINLINE(static VALUE opt_equality_by_mid_slowpath(VALUE recv, VALUE obj, ID mid));
2565
2566static VALUE
2567opt_equality_by_mid_slowpath(VALUE recv, VALUE obj, ID mid)
2568{
2569 const struct rb_callcache *cc = gccct_method_search(GET_EC(), recv, mid, &VM_CI_ON_STACK(mid, 0, 1, NULL));
2570
2571 if (cc && check_cfunc(vm_cc_cme(cc), rb_obj_equal)) {
2572 return RBOOL(recv == obj);
2573 }
2574 else {
2575 return Qundef;
2576 }
2577}
2578
2579static VALUE
2580opt_equality_by_mid(VALUE recv, VALUE obj, ID mid)
2581{
2582 VALUE val = opt_equality_specialized(recv, obj);
2583 if (!UNDEF_P(val)) {
2584 return val;
2585 }
2586 else {
2587 return opt_equality_by_mid_slowpath(recv, obj, mid);
2588 }
2589}
2590
2591VALUE
2592rb_equal_opt(VALUE obj1, VALUE obj2)
2593{
2594 return opt_equality_by_mid(obj1, obj2, idEq);
2595}
2596
2597VALUE
2598rb_eql_opt(VALUE obj1, VALUE obj2)
2599{
2600 return opt_equality_by_mid(obj1, obj2, idEqlP);
2601}
2602
2603extern VALUE rb_vm_call0(rb_execution_context_t *ec, VALUE, ID, int, const VALUE*, const rb_callable_method_entry_t *, int kw_splat);
2604extern VALUE rb_vm_call_with_refinements(rb_execution_context_t *, VALUE, ID, int, const VALUE *, int);
2605
2606static VALUE
2607check_match(rb_execution_context_t *ec, VALUE pattern, VALUE target, enum vm_check_match_type type)
2608{
2609 switch (type) {
2610 case VM_CHECKMATCH_TYPE_WHEN:
2611 return pattern;
2612 case VM_CHECKMATCH_TYPE_RESCUE:
2613 if (!rb_obj_is_kind_of(pattern, rb_cModule)) {
2614 rb_raise(rb_eTypeError, "class or module required for rescue clause");
2615 }
2616 /* fall through */
2617 case VM_CHECKMATCH_TYPE_CASE: {
2618 return rb_vm_call_with_refinements(ec, pattern, idEqq, 1, &target, RB_NO_KEYWORDS);
2619 }
2620 default:
2621 rb_bug("check_match: unreachable");
2622 }
2623}
2624
2625
2626static inline VALUE
2627double_cmp_lt(double a, double b)
2628{
2629 return RBOOL(a < b);
2630}
2631
2632static inline VALUE
2633double_cmp_le(double a, double b)
2634{
2635 return RBOOL(a <= b);
2636}
2637
2638static inline VALUE
2639double_cmp_gt(double a, double b)
2640{
2641 return RBOOL(a > b);
2642}
2643
2644static inline VALUE
2645double_cmp_ge(double a, double b)
2646{
2647 return RBOOL(a >= b);
2648}
2649
2650// Copied by vm_dump.c
2651static inline VALUE *
2652vm_base_ptr(const rb_control_frame_t *cfp)
2653{
2654 const rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
2655
2656 if (CFP_ISEQ(cfp) && VM_FRAME_RUBYFRAME_P(cfp)) {
2657 VALUE *bp = prev_cfp->sp + ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size + VM_ENV_DATA_SIZE;
2658
2659 if (ISEQ_BODY(CFP_ISEQ(cfp))->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) {
2660 int lts = ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size;
2661 int params = ISEQ_BODY(CFP_ISEQ(cfp))->param.size;
2662
2663 CALL_INFO ci = (CALL_INFO)cfp->ep[-(VM_ENV_DATA_SIZE + (lts - params))]; // skip EP stuff, CI should be last local
2664 bp += vm_ci_argc(ci);
2665 }
2666
2667 if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) {
2668 /* adjust `self' */
2669 bp += 1;
2670 }
2671#if VM_DEBUG_BP_CHECK
2672 if (bp != cfp->bp_check) {
2673 ruby_debug_printf("bp_check: %ld, bp: %ld\n",
2674 (long)(cfp->bp_check - GET_EC()->vm_stack),
2675 (long)(bp - GET_EC()->vm_stack));
2676 rb_bug("vm_base_ptr: unreachable");
2677 }
2678#endif
2679 return bp;
2680 }
2681 else {
2682 return NULL;
2683 }
2684}
2685
2686VALUE *
2687rb_vm_base_ptr(const rb_control_frame_t *cfp)
2688{
2689 return vm_base_ptr(cfp);
2690}
2691
2692/* method call processes with call_info */
2693
2694#include "vm_args.c"
2695
2696static inline VALUE vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc, int param_size, int local_size);
2697ALWAYS_INLINE(static VALUE vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me, int opt_pc, int param_size, int local_size));
2698static inline VALUE vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc);
2699static VALUE vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
2700static VALUE vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2701static VALUE vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2702static inline VALUE vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2703
2704static vm_call_handler vm_call_iseq_setup_func(const struct rb_callinfo *ci, const int param_size, const int local_size);
2705
2706static VALUE
2707vm_call_iseq_setup_tailcall_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2708{
2709 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_tailcall_0start);
2710
2711 return vm_call_iseq_setup_tailcall(ec, cfp, calling, 0);
2712}
2713
2714static VALUE
2715vm_call_iseq_setup_normal_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2716{
2717 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_0start);
2718
2719 const struct rb_callcache *cc = calling->cc;
2720 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2721 int param = ISEQ_BODY(iseq)->param.size;
2722 int local = ISEQ_BODY(iseq)->local_table_size;
2723 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
2724}
2725
2726bool
2727rb_simple_iseq_p(const rb_iseq_t *iseq)
2728{
2729 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2730 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2731 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2732 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2733 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2734 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2735 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2736 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2737 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2738}
2739
2740bool
2741rb_iseq_only_optparam_p(const rb_iseq_t *iseq)
2742{
2743 return ISEQ_BODY(iseq)->param.flags.has_opt == TRUE &&
2744 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2745 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2746 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2747 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2748 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2749 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2750 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2751 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2752}
2753
2754bool
2755rb_iseq_only_kwparam_p(const rb_iseq_t *iseq)
2756{
2757 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2758 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2759 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2760 ISEQ_BODY(iseq)->param.flags.has_kw == TRUE &&
2761 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2762 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2763 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2764 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2765}
2766
2767#define ALLOW_HEAP_ARGV (-2)
2768#define ALLOW_HEAP_ARGV_KEEP_KWSPLAT (-3)
2769
2770static inline bool
2771vm_caller_setup_arg_splat(rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE ary, int max_args)
2772{
2773 vm_check_canary(GET_EC(), cfp->sp);
2774 bool ret = false;
2775
2776 if (!NIL_P(ary)) {
2777 const VALUE *ptr = RARRAY_CONST_PTR(ary);
2778 rb_len_t len = RARRAY_LEN(ary);
2779 int argc = calling->argc;
2780
2781 if (UNLIKELY(max_args <= ALLOW_HEAP_ARGV && len + argc > VM_ARGC_STACK_MAX)) {
2782 /* Avoid SystemStackError when splatting large arrays by storing arguments in
2783 * a temporary array, instead of trying to keeping arguments on the VM stack.
2784 */
2785 VALUE *argv = cfp->sp - argc;
2786 VALUE argv_ary = rb_ary_hidden_new(len + argc + 1);
2787 rb_ary_cat(argv_ary, argv, argc);
2788 rb_ary_cat(argv_ary, ptr, len);
2789 cfp->sp -= argc - 1;
2790 cfp->sp[-1] = argv_ary;
2791 calling->argc = 1;
2792 calling->heap_argv = argv_ary;
2793 RB_GC_GUARD(ary);
2794 }
2795 else {
2796 long i;
2797
2798 if (max_args >= 0 && len + argc > max_args) {
2799 /* If only a given max_args is allowed, copy up to max args.
2800 * Used by vm_callee_setup_block_arg for non-lambda blocks,
2801 * where additional arguments are ignored.
2802 *
2803 * Also, copy up to one more argument than the maximum,
2804 * in case it is an empty keyword hash that will be removed.
2805 */
2806 calling->argc += len - (max_args - argc + 1);
2807 len = max_args - argc + 1;
2808 ret = true;
2809 }
2810 else {
2811 /* Unset heap_argv if set originally. Can happen when
2812 * forwarding modified arguments, where heap_argv was used
2813 * originally, but heap_argv not supported by the forwarded
2814 * method in all cases.
2815 */
2816 calling->heap_argv = 0;
2817 }
2818 CHECK_VM_STACK_OVERFLOW(cfp, len);
2819
2820 for (i = 0; i < len; i++) {
2821 *cfp->sp++ = ptr[i];
2822 }
2823 calling->argc += i;
2824 }
2825 }
2826
2827 return ret;
2828}
2829
2830static inline void
2831vm_caller_setup_arg_kw(rb_control_frame_t *cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci)
2832{
2833 const VALUE *const passed_keywords = vm_ci_kwarg(ci)->keywords;
2834 const int kw_len = vm_ci_kwarg(ci)->keyword_len;
2835 const VALUE h = rb_hash_new_capa(kw_len);
2836 VALUE *sp = cfp->sp;
2837 int i;
2838
2839 for (i=0; i<kw_len; i++) {
2840 rb_hash_aset(h, passed_keywords[i], (sp - kw_len)[i]);
2841 }
2842 (sp-kw_len)[0] = h;
2843
2844 cfp->sp -= kw_len - 1;
2845 calling->argc -= kw_len - 1;
2846 calling->kw_splat = 1;
2847}
2848
2849static inline VALUE
2850vm_caller_setup_keyword_hash(const struct rb_callinfo *ci, VALUE keyword_hash)
2851{
2852 if (UNLIKELY(!RB_TYPE_P(keyword_hash, T_HASH))) {
2853 if (keyword_hash != Qnil) {
2854 /* Convert a non-hash keyword splat to a new hash */
2855 keyword_hash = rb_hash_dup(rb_to_hash_type(keyword_hash));
2856 }
2857 }
2858 else if (!IS_ARGS_KW_SPLAT_MUT(ci) && !RHASH_EMPTY_P(keyword_hash)) {
2859 /* Convert a hash keyword splat to a new hash unless
2860 * a mutable keyword splat was passed.
2861 * Skip allocating new hash for empty keyword splat, as empty
2862 * keyword splat will be ignored by both callers.
2863 */
2864 keyword_hash = rb_hash_dup(keyword_hash);
2865 }
2866 return keyword_hash;
2867}
2868
2869static inline void
2870CALLER_SETUP_ARG(struct rb_control_frame_struct *restrict cfp,
2871 struct rb_calling_info *restrict calling,
2872 const struct rb_callinfo *restrict ci, int max_args)
2873{
2874 if (UNLIKELY(IS_ARGS_SPLAT(ci))) {
2875 if (IS_ARGS_KW_SPLAT(ci)) {
2876 // f(*a, **kw)
2877 VM_ASSERT(calling->kw_splat == 1);
2878
2879 cfp->sp -= 2;
2880 calling->argc -= 2;
2881 VALUE ary = cfp->sp[0];
2882 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[1]);
2883
2884 // splat a
2885 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) return;
2886
2887 // put kw
2888 if (kwh != Qnil && !RHASH_EMPTY_P(kwh)) {
2889 if (UNLIKELY(calling->heap_argv)) {
2890 rb_ary_push(calling->heap_argv, kwh);
2891 ((struct RHash *)kwh)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
2892 if (max_args != ALLOW_HEAP_ARGV_KEEP_KWSPLAT) {
2893 calling->kw_splat = 0;
2894 }
2895 }
2896 else {
2897 cfp->sp[0] = kwh;
2898 cfp->sp++;
2899 calling->argc++;
2900
2901 VM_ASSERT(calling->kw_splat == 1);
2902 }
2903 }
2904 else {
2905 calling->kw_splat = 0;
2906 }
2907 }
2908 else {
2909 // f(*a)
2910 VM_ASSERT(calling->kw_splat == 0);
2911
2912 cfp->sp -= 1;
2913 calling->argc -= 1;
2914 VALUE ary = cfp->sp[0];
2915
2916 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) {
2917 goto check_keyword;
2918 }
2919
2920 // check the last argument
2921 VALUE last_hash, argv_ary;
2922 if (UNLIKELY(argv_ary = calling->heap_argv)) {
2923 if (!IS_ARGS_KEYWORD(ci) &&
2924 RARRAY_LEN(argv_ary) > 0 &&
2925 RB_TYPE_P((last_hash = rb_ary_last(0, NULL, argv_ary)), T_HASH) &&
2926 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2927
2928 rb_ary_pop(argv_ary);
2929 if (!RHASH_EMPTY_P(last_hash)) {
2930 rb_ary_push(argv_ary, rb_hash_dup(last_hash));
2931 calling->kw_splat = 1;
2932 }
2933 }
2934 }
2935 else {
2936check_keyword:
2937 if (!IS_ARGS_KEYWORD(ci) &&
2938 calling->argc > 0 &&
2939 RB_TYPE_P((last_hash = cfp->sp[-1]), T_HASH) &&
2940 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2941
2942 if (RHASH_EMPTY_P(last_hash)) {
2943 calling->argc--;
2944 cfp->sp -= 1;
2945 }
2946 else {
2947 cfp->sp[-1] = rb_hash_dup(last_hash);
2948 calling->kw_splat = 1;
2949 }
2950 }
2951 }
2952 }
2953 }
2954 else if (UNLIKELY(IS_ARGS_KW_SPLAT(ci))) {
2955 // f(**kw)
2956 VM_ASSERT(calling->kw_splat == 1);
2957 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[-1]);
2958
2959 if (kwh == Qnil || RHASH_EMPTY_P(kwh)) {
2960 cfp->sp--;
2961 calling->argc--;
2962 calling->kw_splat = 0;
2963 }
2964 else {
2965 cfp->sp[-1] = kwh;
2966 }
2967 }
2968 else if (UNLIKELY(IS_ARGS_KEYWORD(ci))) {
2969 // f(k1:1, k2:2)
2970 VM_ASSERT(calling->kw_splat == 0);
2971
2972 /* This converts VM_CALL_KWARG style to VM_CALL_KW_SPLAT style
2973 * by creating a keyword hash.
2974 * So, vm_ci_flag(ci) & VM_CALL_KWARG is now inconsistent.
2975 */
2976 vm_caller_setup_arg_kw(cfp, calling, ci);
2977 }
2978}
2979
2980#define USE_OPT_HIST 0
2981
2982#if USE_OPT_HIST
2983#define OPT_HIST_MAX 64
2984static int opt_hist[OPT_HIST_MAX+1];
2985
2986__attribute__((destructor))
2987static void
2988opt_hist_show_results_at_exit(void)
2989{
2990 for (int i=0; i<OPT_HIST_MAX; i++) {
2991 ruby_debug_printf("opt_hist\t%d\t%d\n", i, opt_hist[i]);
2992 }
2993}
2994#endif
2995
2996static VALUE
2997vm_call_iseq_setup_normal_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
2998 struct rb_calling_info *calling)
2999{
3000 const struct rb_callcache *cc = calling->cc;
3001 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3002 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3003 const int opt = calling->argc - lead_num;
3004 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
3005 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3006 const int param = ISEQ_BODY(iseq)->param.size;
3007 const int local = ISEQ_BODY(iseq)->local_table_size;
3008 const int delta = opt_num - opt;
3009
3010 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
3011
3012#if USE_OPT_HIST
3013 if (opt_pc < OPT_HIST_MAX) {
3014 opt_hist[opt]++;
3015 }
3016 else {
3017 opt_hist[OPT_HIST_MAX]++;
3018 }
3019#endif
3020
3021 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param - delta, local);
3022}
3023
3024static VALUE
3025vm_call_iseq_setup_tailcall_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3026 struct rb_calling_info *calling)
3027{
3028 const struct rb_callcache *cc = calling->cc;
3029 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3030 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3031 const int opt = calling->argc - lead_num;
3032 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3033
3034 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
3035
3036#if USE_OPT_HIST
3037 if (opt_pc < OPT_HIST_MAX) {
3038 opt_hist[opt]++;
3039 }
3040 else {
3041 opt_hist[OPT_HIST_MAX]++;
3042 }
3043#endif
3044
3045 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc);
3046}
3047
3048static void
3049args_setup_kw_parameters(rb_execution_context_t *const ec, const rb_iseq_t *const iseq, const rb_callable_method_entry_t *cme,
3050 VALUE *const passed_values, const int passed_keyword_len, const VALUE *const passed_keywords,
3051 VALUE *const locals);
3052
3053static VALUE
3054vm_call_iseq_forwardable(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3055 struct rb_calling_info *calling)
3056{
3057 const struct rb_callcache *cc = calling->cc;
3058 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3059 int param_size = ISEQ_BODY(iseq)->param.size;
3060 int local_size = ISEQ_BODY(iseq)->local_table_size;
3061
3062 // Setting up local size and param size
3063 VM_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
3064
3065 local_size = local_size + vm_ci_argc(calling->cd->ci);
3066 param_size = param_size + vm_ci_argc(calling->cd->ci);
3067
3068 cfp->sp[0] = (VALUE)calling->cd->ci;
3069
3070 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param_size, local_size);
3071}
3072
3073static VALUE
3074vm_call_iseq_setup_kwparm_kwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3075 struct rb_calling_info *calling)
3076{
3077 const struct rb_callinfo *ci = calling->cd->ci;
3078 const struct rb_callcache *cc = calling->cc;
3079
3080 VM_ASSERT(vm_ci_flag(ci) & VM_CALL_KWARG);
3081 RB_DEBUG_COUNTER_INC(ccf_iseq_kw1);
3082
3083 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3084 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3085 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
3086 const int ci_kw_len = kw_arg->keyword_len;
3087 const VALUE * const ci_keywords = kw_arg->keywords;
3088 VALUE *argv = cfp->sp - calling->argc;
3089 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3090 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3091 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
3092 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
3093 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
3094
3095 int param = ISEQ_BODY(iseq)->param.size;
3096 int local = ISEQ_BODY(iseq)->local_table_size;
3097 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
3098}
3099
3100static VALUE
3101vm_call_iseq_setup_kwparm_nokwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3102 struct rb_calling_info *calling)
3103{
3104 const struct rb_callinfo *MAYBE_UNUSED(ci) = calling->cd->ci;
3105 const struct rb_callcache *cc = calling->cc;
3106
3107 VM_ASSERT((vm_ci_flag(ci) & VM_CALL_KWARG) == 0);
3108 RB_DEBUG_COUNTER_INC(ccf_iseq_kw2);
3109
3110 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3111 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3112 VALUE * const argv = cfp->sp - calling->argc;
3113 VALUE * const klocals = argv + kw_param->bits_start - kw_param->num;
3114
3115 int i;
3116 for (i=0; i<kw_param->num; i++) {
3117 klocals[i] = kw_param->default_values[i];
3118 }
3119 klocals[i] = INT2FIX(0); // kw specify flag
3120 // NOTE:
3121 // nobody check this value, but it should be cleared because it can
3122 // points invalid VALUE (T_NONE objects, raw pointer and so on).
3123
3124 int param = ISEQ_BODY(iseq)->param.size;
3125 int local = ISEQ_BODY(iseq)->local_table_size;
3126 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
3127}
3128
3129static VALUE builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
3130
3131static VALUE
3132vm_call_single_noarg_leaf_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3133 struct rb_calling_info *calling)
3134{
3135 const struct rb_builtin_function *bf = calling->cc->aux_.bf;
3136 cfp->sp -= (calling->argc + 1);
3137 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
3138 return builtin_invoker0(ec, calling->recv, NULL, func_ptr);
3139}
3140
3141VALUE rb_gen_method_name(VALUE owner, VALUE name); // in vm_backtrace.c
3142
3143static void
3144warn_unused_block(const rb_callable_method_entry_t *cme, const rb_iseq_t *iseq, void *pc)
3145{
3146 rb_vm_t *vm = GET_VM();
3147 set_table *dup_check_table = &vm->unused_block_warning_table;
3148 st_data_t key;
3149 bool strict_unused_block = rb_warning_category_enabled_p(RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK);
3150
3151 union {
3152 VALUE v;
3153 unsigned char b[SIZEOF_VALUE];
3154 } k1 = {
3155 .v = (VALUE)pc,
3156 }, k2 = {
3157 .v = (VALUE)cme->def,
3158 };
3159
3160 // relax check
3161 if (!strict_unused_block) {
3162 key = (st_data_t)cme->def->original_id;
3163
3164 if (set_table_lookup(dup_check_table, key)) {
3165 return;
3166 }
3167 }
3168
3169 // strict check
3170 // make unique key from pc and me->def pointer
3171 key = 0;
3172 for (int i=0; i<SIZEOF_VALUE; i++) {
3173 // fprintf(stderr, "k1:%3d k2:%3d\n", k1.b[i], k2.b[SIZEOF_VALUE-1-i]);
3174 key |= (st_data_t)(k1.b[i] ^ k2.b[SIZEOF_VALUE-1-i]) << (8 * i);
3175 }
3176
3177 if (0) {
3178 fprintf(stderr, "SIZEOF_VALUE:%d\n", SIZEOF_VALUE);
3179 fprintf(stderr, "pc:%p def:%p\n", pc, (void *)cme->def);
3180 fprintf(stderr, "key:%p\n", (void *)key);
3181 }
3182
3183 // duplication check
3184 if (set_insert(dup_check_table, key)) {
3185 // already shown
3186 }
3187 else if (RTEST(ruby_verbose) || strict_unused_block) {
3188 VALUE m_loc = rb_method_entry_location((const rb_method_entry_t *)cme);
3189 VALUE name = rb_gen_method_name(cme->defined_class, rb_iseq_base_label(iseq));
3190
3191 if (!NIL_P(m_loc)) {
3192 rb_warn("the block passed to '%"PRIsVALUE"' defined at %"PRIsVALUE":%"PRIsVALUE" may be ignored",
3193 name, RARRAY_AREF(m_loc, 0), RARRAY_AREF(m_loc, 1));
3194 }
3195 else {
3196 rb_warn("the block may be ignored because '%"PRIsVALUE"' does not use a block", name);
3197 }
3198 }
3199}
3200
3201static inline int
3202vm_callee_setup_arg(rb_execution_context_t *ec, struct rb_calling_info *calling,
3203 const rb_iseq_t *iseq, VALUE *argv, int param_size, int local_size)
3204{
3205 const struct rb_callinfo *ci = calling->cd->ci;
3206 const struct rb_callcache *cc = calling->cc;
3207
3208 VM_ASSERT((vm_ci_argc(ci), 1));
3209 VM_ASSERT(vm_cc_cme(cc) != NULL);
3210
3211 if (UNLIKELY(!ISEQ_BODY(iseq)->param.flags.use_block &&
3212 calling->block_handler != VM_BLOCK_HANDLER_NONE &&
3213 !(vm_ci_flag(calling->cd->ci) & (VM_CALL_OPT_SEND | VM_CALL_SUPER)))) {
3214 warn_unused_block(vm_cc_cme(cc), iseq, (void *)CFP_PC(ec->cfp));
3215 }
3216
3217 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_KW_SPLAT))) {
3218 if (LIKELY(rb_simple_iseq_p(iseq))) {
3219 rb_control_frame_t *cfp = ec->cfp;
3220 int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3221 CALLER_SETUP_ARG(cfp, calling, ci, lead_num);
3222
3223 if (calling->argc != lead_num) {
3224 argument_arity_error(ec, iseq, vm_cc_cme(cc), calling->argc, lead_num, lead_num);
3225 }
3226
3227 //VM_ASSERT(ci == calling->cd->ci);
3228 VM_ASSERT(cc == calling->cc);
3229
3230 if (vm_call_iseq_optimizable_p(ci, cc)) {
3231 if ((ISEQ_BODY(iseq)->builtin_attrs & BUILTIN_ATTR_SINGLE_NOARG_LEAF) && ruby_vm_c_events_enabled == 0) {
3232 VM_ASSERT(ISEQ_BODY(iseq)->builtin_attrs & BUILTIN_ATTR_LEAF);
3233 vm_cc_bf_set(cc, (void *)ISEQ_BODY(iseq)->iseq_encoded[1]);
3234 CC_SET_FASTPATH(cc, vm_call_single_noarg_leaf_builtin, true);
3235 }
3236 else {
3237 CC_SET_FASTPATH(cc, vm_call_iseq_setup_func(ci, param_size, local_size), true);
3238 }
3239 }
3240 return 0;
3241 }
3242 else if (rb_iseq_only_optparam_p(iseq)) {
3243 rb_control_frame_t *cfp = ec->cfp;
3244
3245 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3246 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
3247
3248 CALLER_SETUP_ARG(cfp, calling, ci, lead_num + opt_num);
3249 const int argc = calling->argc;
3250 const int opt = argc - lead_num;
3251
3252 if (opt < 0 || opt > opt_num) {
3253 argument_arity_error(ec, iseq, vm_cc_cme(cc), argc, lead_num, lead_num + opt_num);
3254 }
3255
3256 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3257 CC_SET_FASTPATH(cc, vm_call_iseq_setup_normal_opt_start,
3258 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3259 vm_call_cacheable(ci, cc));
3260 }
3261 else {
3262 CC_SET_FASTPATH(cc, vm_call_iseq_setup_tailcall_opt_start,
3263 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3264 vm_call_cacheable(ci, cc));
3265 }
3266
3267 /* initialize opt vars for self-references */
3268 VM_ASSERT((int)ISEQ_BODY(iseq)->param.size == lead_num + opt_num);
3269 for (int i=argc; i<lead_num + opt_num; i++) {
3270 argv[i] = Qnil;
3271 }
3272 return (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3273 }
3274 else if (rb_iseq_only_kwparam_p(iseq) && !IS_ARGS_SPLAT(ci)) {
3275 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3276 const int argc = calling->argc;
3277 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3278
3279 if (vm_ci_flag(ci) & VM_CALL_KWARG) {
3280 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
3281
3282 if (argc - kw_arg->keyword_len == lead_num) {
3283 const int ci_kw_len = kw_arg->keyword_len;
3284 const VALUE * const ci_keywords = kw_arg->keywords;
3285 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
3286 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
3287
3288 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3289 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
3290
3291 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_kwarg,
3292 vm_call_cacheable(ci, cc));
3293
3294 return 0;
3295 }
3296 }
3297 else if (argc == lead_num) {
3298 /* no kwarg */
3299 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3300 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), NULL, 0, NULL, klocals);
3301
3302 if (klocals[kw_param->num] == INT2FIX(0)) {
3303 /* copy from default_values */
3304 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_nokwarg,
3305 vm_call_cacheable(ci, cc));
3306 }
3307
3308 return 0;
3309 }
3310 }
3311 }
3312
3313 // Called iseq is using ... param
3314 // def foo(...) # <- iseq for foo will have "forwardable"
3315 //
3316 // We want to set the `...` local to the caller's CI
3317 // foo(1, 2) # <- the ci for this should end up as `...`
3318 //
3319 // So hopefully the stack looks like:
3320 //
3321 // => 1
3322 // => 2
3323 // => *
3324 // => **
3325 // => &
3326 // => ... # <- points at `foo`s CI
3327 // => cref_or_me
3328 // => specval
3329 // => type
3330 //
3331 if (ISEQ_BODY(iseq)->param.flags.forwardable) {
3332 bool can_fastpath = true;
3333
3334 if ((vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
3335 struct rb_forwarding_call_data * forward_cd = (struct rb_forwarding_call_data *)calling->cd;
3336 if (vm_ci_argc(ci) != vm_ci_argc(forward_cd->caller_ci)) {
3337 ci = vm_ci_new_runtime(
3338 vm_ci_mid(ci),
3339 vm_ci_flag(ci),
3340 vm_ci_argc(ci),
3341 vm_ci_kwarg(ci));
3342 }
3343 else {
3344 ci = forward_cd->caller_ci;
3345 }
3346 can_fastpath = false;
3347 }
3348 // C functions calling iseqs will stack allocate a CI,
3349 // so we need to convert it to heap allocated
3350 if (!vm_ci_markable(ci)) {
3351 ci = vm_ci_new_runtime(
3352 vm_ci_mid(ci),
3353 vm_ci_flag(ci),
3354 vm_ci_argc(ci),
3355 vm_ci_kwarg(ci));
3356 can_fastpath = false;
3357 }
3358 argv[param_size - 1] = (VALUE)ci;
3359 CC_SET_FASTPATH(cc, vm_call_iseq_forwardable, can_fastpath);
3360 return 0;
3361 }
3362
3363 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_method);
3364}
3365
3366static void
3367vm_adjust_stack_forwarding(const struct rb_execution_context_struct *ec, struct rb_control_frame_struct *cfp, int argc, VALUE splat)
3368{
3369 // This case is when the caller is using a ... parameter.
3370 // For example `bar(...)`. The call info will have VM_CALL_FORWARDING
3371 // In this case the caller's caller's CI will be on the stack.
3372 //
3373 // For example:
3374 //
3375 // def bar(a, b); a + b; end
3376 // def foo(...); bar(...); end
3377 // foo(1, 2) # <- this CI will be on the stack when we call `bar(...)`
3378 //
3379 // Stack layout will be:
3380 //
3381 // > 1
3382 // > 2
3383 // > CI for foo(1, 2)
3384 // > cref_or_me
3385 // > specval
3386 // > type
3387 // > receiver
3388 // > CI for foo(1, 2), via `getlocal ...`
3389 // > ( SP points here )
3390 const VALUE * lep = VM_CF_LEP(cfp);
3391
3392 const rb_iseq_t *iseq;
3393
3394 // If we're in an escaped environment (lambda for example), get the iseq
3395 // from the captured env.
3396 if (VM_ENV_FLAGS(lep, VM_ENV_FLAG_ESCAPED)) {
3397 rb_env_t * env = (rb_env_t *)lep[VM_ENV_DATA_INDEX_ENV];
3398 iseq = env->iseq;
3399 }
3400 else { // Otherwise use the lep to find the caller
3401 iseq = CFP_ISEQ(rb_vm_search_cf_from_ep(ec, cfp, lep));
3402 }
3403
3404 // Our local storage is below the args we need to copy
3405 int local_size = ISEQ_BODY(iseq)->local_table_size + argc;
3406
3407 const VALUE * from = lep - (local_size + VM_ENV_DATA_SIZE - 1); // 2 for EP values
3408 VALUE * to = cfp->sp - 1; // clobber the CI
3409
3410 if (RTEST(splat)) {
3411 to -= 1; // clobber the splat array
3412 CHECK_VM_STACK_OVERFLOW0(cfp, to, RARRAY_LEN(splat));
3413 MEMCPY(to, RARRAY_CONST_PTR(splat), VALUE, RARRAY_LEN(splat));
3414 to += RARRAY_LEN(splat);
3415 }
3416
3417 CHECK_VM_STACK_OVERFLOW0(cfp, to, argc);
3418 MEMCPY(to, from, VALUE, argc);
3419 cfp->sp = to + argc;
3420
3421 // Stack layout should now be:
3422 //
3423 // > 1
3424 // > 2
3425 // > CI for foo(1, 2)
3426 // > cref_or_me
3427 // > specval
3428 // > type
3429 // > receiver
3430 // > 1
3431 // > 2
3432 // > ( SP points here )
3433}
3434
3435static VALUE
3436vm_call_iseq_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3437{
3438 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3439
3440 const struct rb_callcache *cc = calling->cc;
3441 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3442 int param_size = ISEQ_BODY(iseq)->param.size;
3443 int local_size = ISEQ_BODY(iseq)->local_table_size;
3444
3445 RUBY_ASSERT(!ISEQ_BODY(iseq)->param.flags.forwardable);
3446
3447 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3448 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3449}
3450
3451static VALUE
3452vm_call_iseq_fwd_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3453{
3454 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3455
3456 const struct rb_callcache *cc = calling->cc;
3457 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3458 int param_size = ISEQ_BODY(iseq)->param.size;
3459 int local_size = ISEQ_BODY(iseq)->local_table_size;
3460
3461 RUBY_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
3462
3463 // Setting up local size and param size
3464 local_size = local_size + vm_ci_argc(calling->cd->ci);
3465 param_size = param_size + vm_ci_argc(calling->cd->ci);
3466
3467 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3468 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3469}
3470
3471static inline VALUE
3472vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
3473 int opt_pc, int param_size, int local_size)
3474{
3475 const struct rb_callinfo *ci = calling->cd->ci;
3476 const struct rb_callcache *cc = calling->cc;
3477
3478 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3479 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param_size, local_size);
3480 }
3481 else {
3482 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc);
3483 }
3484}
3485
3486static inline VALUE
3487vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me,
3488 int opt_pc, int param_size, int local_size)
3489{
3490 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3491 VALUE *argv = cfp->sp - calling->argc;
3492 VALUE *sp = argv + param_size;
3493 cfp->sp = argv - 1 /* recv */;
3494
3495 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL, calling->recv,
3496 calling->block_handler, (VALUE)me,
3497 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3498 local_size - param_size,
3499 ISEQ_BODY(iseq)->stack_max);
3500 return Qundef;
3501}
3502
3503static inline VALUE
3504vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc)
3505{
3506 const struct rb_callcache *cc = calling->cc;
3507 unsigned int i;
3508 VALUE *argv = cfp->sp - calling->argc;
3509 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3510 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3511 VALUE *src_argv = argv;
3512 VALUE *sp_orig, *sp;
3513 VALUE finish_flag = VM_FRAME_FINISHED_P(cfp) ? VM_FRAME_FLAG_FINISH : 0;
3514
3515 if (VM_BH_FROM_CFP_P(calling->block_handler, cfp)) {
3516 struct rb_captured_block *dst_captured = VM_CFP_TO_CAPTURED_BLOCK(RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp));
3517 const struct rb_captured_block *src_captured = VM_BH_TO_CAPT_BLOCK(calling->block_handler);
3518 dst_captured->code.val = src_captured->code.val;
3519 if (VM_BH_ISEQ_BLOCK_P(calling->block_handler)) {
3520 calling->block_handler = VM_BH_FROM_ISEQ_BLOCK(dst_captured);
3521 }
3522 else {
3523 calling->block_handler = VM_BH_FROM_IFUNC_BLOCK(dst_captured);
3524 }
3525 }
3526
3527 vm_pop_frame(ec, cfp, cfp->ep);
3528 cfp = ec->cfp;
3529
3530 sp_orig = sp = cfp->sp;
3531
3532 /* push self */
3533 sp[0] = calling->recv;
3534 sp++;
3535
3536 /* copy arguments */
3537 for (i=0; i < ISEQ_BODY(iseq)->param.size; i++) {
3538 *sp++ = src_argv[i];
3539 }
3540
3541 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL | finish_flag,
3542 calling->recv, calling->block_handler, (VALUE)me,
3543 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3544 ISEQ_BODY(iseq)->local_table_size - ISEQ_BODY(iseq)->param.size,
3545 ISEQ_BODY(iseq)->stack_max);
3546
3547 cfp->sp = sp_orig;
3548
3549 return Qundef;
3550}
3551
3552static void
3553ractor_unsafe_check(void)
3554{
3555 if (!rb_ractor_main_p()) {
3556 rb_raise(rb_eRactorUnsafeError, "ractor unsafe method called from not main ractor");
3557 }
3558}
3559
3560static VALUE
3561call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3562{
3563 ractor_unsafe_check();
3564 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3565 return (*f)(recv, rb_ary_new4(argc, argv));
3566}
3567
3568static VALUE
3569call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3570{
3571 ractor_unsafe_check();
3572 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3573 return (*f)(argc, argv, recv);
3574}
3575
3576static VALUE
3577call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3578{
3579 ractor_unsafe_check();
3580 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3581 return (*f)(recv);
3582}
3583
3584static VALUE
3585call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3586{
3587 ractor_unsafe_check();
3588 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3589 return (*f)(recv, argv[0]);
3590}
3591
3592static VALUE
3593call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3594{
3595 ractor_unsafe_check();
3596 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3597 return (*f)(recv, argv[0], argv[1]);
3598}
3599
3600static VALUE
3601call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3602{
3603 ractor_unsafe_check();
3604 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3605 return (*f)(recv, argv[0], argv[1], argv[2]);
3606}
3607
3608static VALUE
3609call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3610{
3611 ractor_unsafe_check();
3612 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3613 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3614}
3615
3616static VALUE
3617call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3618{
3619 ractor_unsafe_check();
3620 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3621 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3622}
3623
3624static VALUE
3625call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3626{
3627 ractor_unsafe_check();
3628 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3629 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3630}
3631
3632static VALUE
3633call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3634{
3635 ractor_unsafe_check();
3636 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3637 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3638}
3639
3640static VALUE
3641call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3642{
3643 ractor_unsafe_check();
3644 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3645 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3646}
3647
3648static VALUE
3649call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3650{
3651 ractor_unsafe_check();
3652 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3653 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3654}
3655
3656static VALUE
3657call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3658{
3659 ractor_unsafe_check();
3660 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3661 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3662}
3663
3664static VALUE
3665call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3666{
3667 ractor_unsafe_check();
3668 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3669 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3670}
3671
3672static VALUE
3673call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3674{
3675 ractor_unsafe_check();
3676 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3677 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3678}
3679
3680static VALUE
3681call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3682{
3683 ractor_unsafe_check();
3684 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3685 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3686}
3687
3688static VALUE
3689call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3690{
3691 ractor_unsafe_check();
3692 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3693 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3694}
3695
3696static VALUE
3697call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3698{
3699 ractor_unsafe_check();
3700 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3701 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3702}
3703
3704static VALUE
3705ractor_safe_call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3706{
3707 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3708 return (*f)(recv, rb_ary_new4(argc, argv));
3709}
3710
3711static VALUE
3712ractor_safe_call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3713{
3714 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3715 return (*f)(argc, argv, recv);
3716}
3717
3718static VALUE
3719ractor_safe_call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3720{
3721 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3722 return (*f)(recv);
3723}
3724
3725static VALUE
3726ractor_safe_call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3727{
3728 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3729 return (*f)(recv, argv[0]);
3730}
3731
3732static VALUE
3733ractor_safe_call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3734{
3735 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3736 return (*f)(recv, argv[0], argv[1]);
3737}
3738
3739static VALUE
3740ractor_safe_call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3741{
3742 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3743 return (*f)(recv, argv[0], argv[1], argv[2]);
3744}
3745
3746static VALUE
3747ractor_safe_call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3748{
3749 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3750 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3751}
3752
3753static VALUE
3754ractor_safe_call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3755{
3756 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3757 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3758}
3759
3760static VALUE
3761ractor_safe_call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3762{
3763 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3764 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3765}
3766
3767static VALUE
3768ractor_safe_call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3769{
3770 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3771 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3772}
3773
3774static VALUE
3775ractor_safe_call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3776{
3777 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3778 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3779}
3780
3781static VALUE
3782ractor_safe_call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3783{
3784 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3785 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3786}
3787
3788static VALUE
3789ractor_safe_call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3790{
3791 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3792 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3793}
3794
3795static VALUE
3796ractor_safe_call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3797{
3798 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3799 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3800}
3801
3802static VALUE
3803ractor_safe_call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3804{
3805 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3806 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3807}
3808
3809static VALUE
3810ractor_safe_call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3811{
3812 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3813 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3814}
3815
3816static VALUE
3817ractor_safe_call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3818{
3819 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3820 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3821}
3822
3823static VALUE
3824ractor_safe_call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3825{
3826 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3827 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3828}
3829
3830static inline int
3831vm_cfp_consistent_p(rb_execution_context_t *ec, const rb_control_frame_t *reg_cfp)
3832{
3833 const int ov_flags = RAISED_STACKOVERFLOW;
3834 if (LIKELY(reg_cfp == ec->cfp + 1)) return TRUE;
3835 if (rb_ec_raised_p(ec, ov_flags)) {
3836 rb_ec_raised_reset(ec, ov_flags);
3837 return TRUE;
3838 }
3839 return FALSE;
3840}
3841
3842#define CHECK_CFP_CONSISTENCY(func) \
3843 (LIKELY(vm_cfp_consistent_p(ec, reg_cfp)) ? (void)0 : \
3844 rb_bug(func ": cfp consistency error (%p, %p)", (void *)reg_cfp, (void *)(ec->cfp+1)))
3845
3846static inline
3847const rb_method_cfunc_t *
3848vm_method_cfunc_entry(const rb_callable_method_entry_t *me)
3849{
3850#if VM_DEBUG_VERIFY_METHOD_CACHE
3851 switch (me->def->type) {
3852 case VM_METHOD_TYPE_CFUNC:
3853 case VM_METHOD_TYPE_NOTIMPLEMENTED:
3854 break;
3855# define METHOD_BUG(t) case VM_METHOD_TYPE_##t: rb_bug("wrong method type: " #t)
3856 METHOD_BUG(ISEQ);
3857 METHOD_BUG(ATTRSET);
3858 METHOD_BUG(IVAR);
3859 METHOD_BUG(BMETHOD);
3860 METHOD_BUG(ZSUPER);
3861 METHOD_BUG(UNDEF);
3862 METHOD_BUG(OPTIMIZED);
3863 METHOD_BUG(MISSING);
3864 METHOD_BUG(REFINED);
3865 METHOD_BUG(ALIAS);
3866# undef METHOD_BUG
3867 default:
3868 rb_bug("wrong method type: %d", me->def->type);
3869 }
3870#endif
3871 return UNALIGNED_MEMBER_PTR(me->def, body.cfunc);
3872}
3873
3874static VALUE
3875vm_call_cfunc_with_frame_(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
3876 int argc, VALUE *argv, VALUE *stack_bottom)
3877{
3878 RB_DEBUG_COUNTER_INC(ccf_cfunc_with_frame);
3879 const struct rb_callinfo *ci = calling->cd->ci;
3880 const struct rb_callcache *cc = calling->cc;
3881 VALUE val;
3882 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3883 const rb_method_cfunc_t *cfunc = vm_method_cfunc_entry(me);
3884
3885 VALUE recv = calling->recv;
3886 VALUE block_handler = calling->block_handler;
3887 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3888
3889 if (UNLIKELY(calling->kw_splat)) {
3890 frame_type |= VM_FRAME_FLAG_CFRAME_KW;
3891 }
3892
3893 VM_ASSERT(reg_cfp == ec->cfp);
3894
3895 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, me->owner, me->def->original_id);
3896 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, recv, me->def->original_id, vm_ci_mid(ci), me->owner, Qundef);
3897
3898 vm_push_frame(ec, NULL, frame_type, recv,
3899 block_handler, (VALUE)me,
3900 0, ec->cfp->sp, 0, 0);
3901
3902 int len = cfunc->argc;
3903 if (len >= 0) rb_check_arity(argc, len, len);
3904
3905 reg_cfp->sp = stack_bottom;
3906 val = (*cfunc->invoker)(recv, argc, argv, cfunc->func);
3907
3908 CHECK_CFP_CONSISTENCY("vm_call_cfunc");
3909
3910 rb_vm_pop_frame(ec);
3911
3912 VM_ASSERT(ec->cfp->sp == stack_bottom);
3913
3914 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, recv, me->def->original_id, vm_ci_mid(ci), me->owner, val);
3915 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
3916
3917 return val;
3918}
3919
3920// Push a C method frame for a given cme. This is called when JIT code skipped
3921// pushing a frame but the C method reached a point where a frame is needed.
3922void
3923rb_vm_push_cfunc_frame(const rb_callable_method_entry_t *cme, int recv_idx)
3924{
3925 VM_ASSERT(cme->def->type == VM_METHOD_TYPE_CFUNC);
3926 rb_execution_context_t *ec = GET_EC();
3927 VALUE *sp = ec->cfp->sp;
3928 VALUE recv = *(sp - recv_idx - 1);
3929 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3930 VALUE block_handler = VM_BLOCK_HANDLER_NONE;
3931#if VM_CHECK_MODE > 0
3932 // Clean up the stack canary since we're about to satisfy the "leaf or lazy push" assumption
3933 *(GET_EC()->cfp->sp) = Qfalse;
3934#endif
3935 vm_push_frame(ec, NULL, frame_type, recv, block_handler, (VALUE)cme, 0, ec->cfp->sp, 0, 0);
3936}
3937
3938// If true, cc->call needs to include `CALLER_SETUP_ARG` (i.e. can't be skipped in fastpath)
3939bool
3940rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci)
3941{
3942 return IS_ARGS_SPLAT(ci) || IS_ARGS_KW_OR_KW_SPLAT(ci);
3943}
3944
3945static VALUE
3946vm_call_cfunc_with_frame(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3947{
3948 int argc = calling->argc;
3949 VALUE *stack_bottom = reg_cfp->sp - argc - 1;
3950 VALUE *argv = &stack_bottom[1];
3951
3952 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
3953}
3954
3955static VALUE
3956vm_call_cfunc_other(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3957{
3958 const struct rb_callinfo *ci = calling->cd->ci;
3959 RB_DEBUG_COUNTER_INC(ccf_cfunc_other);
3960
3961 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
3962 VALUE argv_ary;
3963 if (UNLIKELY(argv_ary = calling->heap_argv)) {
3964 VM_ASSERT(!IS_ARGS_KEYWORD(ci));
3965 int argc = RARRAY_LENINT(argv_ary);
3966 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
3967 VALUE *stack_bottom = reg_cfp->sp - 2;
3968
3969 VM_ASSERT(calling->argc == 1);
3970 VM_ASSERT(RB_TYPE_P(argv_ary, T_ARRAY));
3971 VM_ASSERT(RBASIC_CLASS(argv_ary) == 0); // hidden ary
3972
3973 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
3974 }
3975 else {
3976 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_with_frame, !rb_splat_or_kwargs_p(ci) && !calling->kw_splat && !(vm_ci_flag(ci) & VM_CALL_FORWARDING));
3977
3978 return vm_call_cfunc_with_frame(ec, reg_cfp, calling);
3979 }
3980}
3981
3982static inline VALUE
3983vm_call_cfunc_array_argv(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int stack_offset, int argc_offset)
3984{
3985 VALUE argv_ary = reg_cfp->sp[-1 - stack_offset];
3986 int argc = RARRAY_LENINT(argv_ary) - argc_offset;
3987
3988 if (UNLIKELY(argc > VM_ARGC_STACK_MAX)) {
3989 return vm_call_cfunc_other(ec, reg_cfp, calling);
3990 }
3991
3992 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
3993 calling->kw_splat = 0;
3994 int i;
3995 VALUE *stack_bottom = reg_cfp->sp - 2 - stack_offset;
3996 VALUE *sp = stack_bottom;
3997 CHECK_VM_STACK_OVERFLOW(reg_cfp, argc);
3998 for(i = 0; i < argc; i++) {
3999 *++sp = argv[i];
4000 }
4001 reg_cfp->sp = sp+1;
4002
4003 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, stack_bottom+1, stack_bottom);
4004}
4005
4006static inline VALUE
4007vm_call_cfunc_only_splat(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4008{
4009 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat);
4010 VALUE argv_ary = reg_cfp->sp[-1];
4011 int argc = RARRAY_LENINT(argv_ary);
4012 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
4013 VALUE last_hash;
4014 int argc_offset = 0;
4015
4016 if (UNLIKELY(argc > 0 &&
4017 RB_TYPE_P((last_hash = argv[argc-1]), T_HASH) &&
4018 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS))) {
4019 if (!RHASH_EMPTY_P(last_hash)) {
4020 return vm_call_cfunc_other(ec, reg_cfp, calling);
4021 }
4022 argc_offset++;
4023 }
4024 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 0, argc_offset);
4025}
4026
4027static inline VALUE
4028vm_call_cfunc_only_splat_kw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4029{
4030 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat_kw);
4031 VALUE keyword_hash = reg_cfp->sp[-1];
4032
4033 if (keyword_hash == Qnil || (RB_TYPE_P(keyword_hash, T_HASH) && RHASH_EMPTY_P(keyword_hash))) {
4034 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 1, 0);
4035 }
4036
4037 return vm_call_cfunc_other(ec, reg_cfp, calling);
4038}
4039
4040static VALUE
4041vm_call_cfunc(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4042{
4043 const struct rb_callinfo *ci = calling->cd->ci;
4044 RB_DEBUG_COUNTER_INC(ccf_cfunc);
4045
4046 if (IS_ARGS_SPLAT(ci) && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
4047 if (!IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 1) {
4048 // f(*a)
4049 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat, TRUE);
4050 return vm_call_cfunc_only_splat(ec, reg_cfp, calling);
4051 }
4052 if (IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 2) {
4053 // f(*a, **kw)
4054 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat_kw, TRUE);
4055 return vm_call_cfunc_only_splat_kw(ec, reg_cfp, calling);
4056 }
4057 }
4058
4059 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_other, TRUE);
4060 return vm_call_cfunc_other(ec, reg_cfp, calling);
4061}
4062
4063static VALUE
4064vm_call_ivar(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4065{
4066 const struct rb_callcache *cc = calling->cc;
4067 RB_DEBUG_COUNTER_INC(ccf_ivar);
4068 cfp->sp -= 1;
4069 VALUE ivar = vm_getivar(calling->recv, vm_cc_cme(cc)->def->body.attr.id, NULL, NULL, cc, TRUE, Qnil);
4070 return ivar;
4071}
4072
4073static VALUE
4074vm_call_attrset_direct(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_callcache *cc, VALUE obj)
4075{
4076 RB_DEBUG_COUNTER_INC(ccf_attrset);
4077 VALUE val = *(cfp->sp - 1);
4078 cfp->sp -= 2;
4079 rb_setivar_cache cache = rb_setivar_cache_unpack(vm_cc_atomic_cache_read(cc));
4080 ID id = vm_cc_cme(cc)->def->body.attr.id;
4081 rb_check_frozen(obj);
4082 VALUE res = vm_setivar(obj, val, cache);
4083 if (UNDEF_P(res)) {
4084 switch (BUILTIN_TYPE(obj)) {
4085 case T_OBJECT:
4086 break;
4087 case T_CLASS:
4088 case T_MODULE:
4089 {
4090 res = vm_setivar_class(obj, val, cache);
4091 if (!UNDEF_P(res)) {
4092 return res;
4093 }
4094 }
4095 break;
4096 default:
4097 {
4098 res = vm_setivar_default(obj, id, val, cache);
4099 if (!UNDEF_P(res)) {
4100 return res;
4101 }
4102 }
4103 }
4104 res = vm_setivar_slowpath_attr(obj, id, val, cc);
4105 }
4106 return res;
4107}
4108
4109static VALUE
4110vm_call_attrset(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4111{
4112 return vm_call_attrset_direct(ec, cfp, calling->cc, calling->recv);
4113}
4114
4115static inline VALUE
4116vm_call_bmethod_body(rb_execution_context_t *ec, struct rb_calling_info *calling, const VALUE *argv)
4117{
4118 rb_proc_t *proc;
4119 VALUE val;
4120 const struct rb_callcache *cc = calling->cc;
4121 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4122 VALUE procv = cme->def->body.bmethod.proc;
4123
4124 if (!RB_OBJ_SHAREABLE_P(procv) &&
4125 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4126 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4127 }
4128
4129 /* control block frame */
4130 GetProcPtr(procv, proc);
4131 val = vm_invoke_bmethod(ec, proc, calling->recv, CALLING_ARGC(calling), argv, calling->kw_splat, calling->block_handler, vm_cc_cme(cc));
4132
4133 return val;
4134}
4135
4136static int vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type);
4137
4138static VALUE
4139vm_call_iseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4140{
4141 RB_DEBUG_COUNTER_INC(ccf_iseq_bmethod);
4142
4143 const struct rb_callcache *cc = calling->cc;
4144 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4145 VALUE procv = cme->def->body.bmethod.proc;
4146
4147 if (!RB_OBJ_SHAREABLE_P(procv) &&
4148 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4149 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4150 }
4151
4152 rb_proc_t *proc;
4153 GetProcPtr(procv, proc);
4154 const struct rb_block *block = &proc->block;
4155
4156 while (vm_block_type(block) == block_type_proc) {
4157 block = vm_proc_block(block->as.proc);
4158 }
4159 VM_ASSERT(vm_block_type(block) == block_type_iseq);
4160
4161 const struct rb_captured_block *captured = &block->as.captured;
4162 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
4163 VALUE * const argv = cfp->sp - calling->argc;
4164 const int arg_size = ISEQ_BODY(iseq)->param.size;
4165
4166 int opt_pc;
4167 if (vm_ci_flag(calling->cd->ci) & VM_CALL_ARGS_SIMPLE) {
4168 opt_pc = vm_callee_setup_block_arg(ec, calling, calling->cd->ci, iseq, argv, arg_setup_method);
4169 }
4170 else {
4171 opt_pc = setup_parameters_complex(ec, iseq, calling, calling->cd->ci, argv, arg_setup_method);
4172 }
4173
4174 cfp->sp = argv - 1; // -1 for the receiver
4175
4176 vm_push_frame(ec, iseq,
4177 VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_BMETHOD | VM_FRAME_FLAG_LAMBDA,
4178 calling->recv,
4179 VM_GUARDED_PREV_EP(captured->ep),
4180 (VALUE)cme,
4181 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
4182 argv + arg_size,
4183 ISEQ_BODY(iseq)->local_table_size - arg_size,
4184 ISEQ_BODY(iseq)->stack_max);
4185
4186 return Qundef;
4187}
4188
4189static VALUE
4190vm_call_noniseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4191{
4192 RB_DEBUG_COUNTER_INC(ccf_noniseq_bmethod);
4193
4194 VALUE *argv;
4195 int argc;
4196 CALLER_SETUP_ARG(cfp, calling, calling->cd->ci, ALLOW_HEAP_ARGV);
4197 if (UNLIKELY(calling->heap_argv)) {
4198 argv = RARRAY_PTR(calling->heap_argv);
4199 cfp->sp -= 2;
4200 }
4201 else {
4202 argc = calling->argc;
4203 argv = ALLOCA_N(VALUE, argc);
4204 MEMCPY(argv, cfp->sp - argc, VALUE, argc);
4205 cfp->sp += - argc - 1;
4206 }
4207
4208 return vm_call_bmethod_body(ec, calling, argv);
4209}
4210
4211static VALUE
4212vm_call_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4213{
4214 RB_DEBUG_COUNTER_INC(ccf_bmethod);
4215
4216 const struct rb_callcache *cc = calling->cc;
4217 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4218 VALUE procv = cme->def->body.bmethod.proc;
4219 rb_proc_t *proc;
4220 GetProcPtr(procv, proc);
4221 const struct rb_block *block = &proc->block;
4222
4223 while (vm_block_type(block) == block_type_proc) {
4224 block = vm_proc_block(block->as.proc);
4225 }
4226 if (vm_block_type(block) == block_type_iseq) {
4227 CC_SET_FASTPATH(cc, vm_call_iseq_bmethod, TRUE);
4228 return vm_call_iseq_bmethod(ec, cfp, calling);
4229 }
4230
4231 CC_SET_FASTPATH(cc, vm_call_noniseq_bmethod, TRUE);
4232 return vm_call_noniseq_bmethod(ec, cfp, calling);
4233}
4234
4235VALUE
4236rb_find_defined_class_by_owner(VALUE current_class, VALUE target_owner)
4237{
4238 VALUE klass = current_class;
4239
4240 /* for prepended Module, then start from cover class */
4241 if (RB_TYPE_P(klass, T_ICLASS) && RICLASS_IS_ORIGIN_P(klass) &&
4242 RB_TYPE_P(RBASIC_CLASS(klass), T_CLASS)) {
4243 klass = RBASIC_CLASS(klass);
4244 }
4245
4246 while (RTEST(klass)) {
4247 VALUE owner = RB_TYPE_P(klass, T_ICLASS) ? RBASIC_CLASS(klass) : klass;
4248 if (owner == target_owner) {
4249 return klass;
4250 }
4251 klass = RCLASS_SUPER(klass);
4252 }
4253
4254 return current_class; /* maybe module function */
4255}
4256
4257static const rb_callable_method_entry_t *
4258aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4259{
4260 const rb_method_entry_t *orig_me = me->def->body.alias.original_me;
4261 const rb_callable_method_entry_t *cme;
4262
4263 if (orig_me->defined_class == 0) {
4264 VALUE defined_class = rb_find_defined_class_by_owner(me->defined_class, orig_me->owner);
4265 VM_ASSERT_TYPE(orig_me->owner, T_MODULE);
4266 cme = rb_method_entry_complement_defined_class(orig_me, me->called_id, defined_class);
4267
4268 if (me->def->reference_count == 1) {
4269 RB_OBJ_WRITE(me, &me->def->body.alias.original_me, cme);
4270 }
4271 else {
4272 rb_method_definition_t *def =
4273 rb_method_definition_create(VM_METHOD_TYPE_ALIAS, me->def->original_id);
4274 rb_method_definition_set((rb_method_entry_t *)me, def, (void *)cme);
4275 }
4276 }
4277 else {
4278 cme = (const rb_callable_method_entry_t *)orig_me;
4279 }
4280
4281 VM_ASSERT(callable_method_entry_p(cme));
4282 return cme;
4283}
4284
4285const rb_callable_method_entry_t *
4286rb_aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4287{
4288 return aliased_callable_method_entry(me);
4289}
4290
4291static VALUE
4292vm_call_alias(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4293{
4294 calling->cc = &VM_CC_ON_STACK(Qundef,
4295 vm_call_general,
4296 {{0}},
4297 aliased_callable_method_entry(vm_cc_cme(calling->cc)));
4298
4299 return vm_call_method_each_type(ec, cfp, calling);
4300}
4301
4302static enum method_missing_reason
4303ci_missing_reason(const struct rb_callinfo *ci)
4304{
4305 enum method_missing_reason stat = MISSING_NOENTRY;
4306 if (vm_ci_flag(ci) & VM_CALL_VCALL && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) stat |= MISSING_VCALL;
4307 if (vm_ci_flag(ci) & VM_CALL_FCALL) stat |= MISSING_FCALL;
4308 if (vm_ci_flag(ci) & VM_CALL_SUPER) stat |= MISSING_SUPER;
4309 return stat;
4310}
4311
4312static VALUE vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
4313
4314static VALUE
4315vm_call_symbol(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4316 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE symbol, int flags)
4317{
4318 ASSUME(calling->argc >= 0);
4319
4320 enum method_missing_reason missing_reason = MISSING_NOENTRY;
4321 int argc = calling->argc;
4322 VALUE recv = calling->recv;
4323 VALUE klass = CLASS_OF(recv);
4324 ID mid = rb_check_id(&symbol);
4325 flags |= VM_CALL_OPT_SEND;
4326
4327 if (UNLIKELY(! mid)) {
4328 mid = idMethodMissing;
4329 missing_reason = ci_missing_reason(ci);
4330 ec->method_missing_reason = missing_reason;
4331
4332 VALUE argv_ary;
4333 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4334 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4335 rb_ary_unshift(argv_ary, symbol);
4336
4337 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4338 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4339 VALUE exc = rb_make_no_method_exception(
4340 rb_eNoMethodError, 0, recv, RARRAY_LENINT(argv_ary), RARRAY_CONST_PTR(argv_ary), priv);
4341
4342 rb_exc_raise(exc);
4343 }
4344 rb_ary_unshift(argv_ary, rb_str_intern(symbol));
4345 }
4346 else {
4347 /* E.g. when argc == 2
4348 *
4349 * | | | | TOPN
4350 * | | +------+
4351 * | | +---> | arg1 | 0
4352 * +------+ | +------+
4353 * | arg1 | -+ +-> | arg0 | 1
4354 * +------+ | +------+
4355 * | arg0 | ---+ | sym | 2
4356 * +------+ +------+
4357 * | recv | | recv | 3
4358 * --+------+--------+------+------
4359 */
4360 int i = argc;
4361 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4362 INC_SP(1);
4363 MEMMOVE(&TOPN(i - 1), &TOPN(i), VALUE, i);
4364 argc = ++calling->argc;
4365
4366 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4367 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4368 TOPN(i) = symbol;
4369 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4370 const VALUE *argv = STACK_ADDR_FROM_TOP(argc);
4371 VALUE exc = rb_make_no_method_exception(
4372 rb_eNoMethodError, 0, recv, argc, argv, priv);
4373
4374 rb_exc_raise(exc);
4375 }
4376 else {
4377 TOPN(i) = rb_str_intern(symbol);
4378 }
4379 }
4380 }
4381
4382 struct rb_forwarding_call_data new_fcd = {
4383 .cd = {
4384 .ci = &VM_CI_ON_STACK(mid, flags, argc, vm_ci_kwarg(ci)),
4385 .cc = NULL,
4386 },
4387 .caller_ci = NULL,
4388 };
4389
4390 if (!(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
4391 calling->cd = &new_fcd.cd;
4392 }
4393 else {
4394 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4395 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4396 new_fcd.caller_ci = caller_ci;
4397 calling->cd = (struct rb_call_data *)&new_fcd;
4398 }
4399 calling->cc = &VM_CC_ON_STACK(klass,
4400 vm_call_general,
4401 { .method_missing_reason = missing_reason },
4402 rb_callable_method_entry_with_refinements(klass, mid, NULL));
4403
4404 if (flags & VM_CALL_FCALL) {
4405 return vm_call_method(ec, reg_cfp, calling);
4406 }
4407
4408 const struct rb_callcache *cc = calling->cc;
4409 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
4410
4411 if (vm_cc_cme(cc) != NULL) {
4412 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
4413 case METHOD_VISI_PUBLIC: /* likely */
4414 return vm_call_method_each_type(ec, reg_cfp, calling);
4415 case METHOD_VISI_PRIVATE:
4416 vm_cc_method_missing_reason_set(cc, MISSING_PRIVATE);
4417 break;
4418 case METHOD_VISI_PROTECTED:
4419 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
4420 break;
4421 default:
4422 VM_UNREACHABLE(vm_call_method);
4423 }
4424 return vm_call_method_missing(ec, reg_cfp, calling);
4425 }
4426
4427 return vm_call_method_nome(ec, reg_cfp, calling);
4428}
4429
4430static VALUE
4431vm_call_opt_send0(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int flags)
4432{
4433 const struct rb_callinfo *ci = calling->cd->ci;
4434 int i;
4435 VALUE sym;
4436
4437 i = calling->argc - 1;
4438
4439 if (calling->argc == 0) {
4440 rb_raise(rb_eArgError, "no method name given");
4441 }
4442
4443 sym = TOPN(i);
4444 /* E.g. when i == 2
4445 *
4446 * | | | | TOPN
4447 * +------+ | |
4448 * | arg1 | ---+ | | 0
4449 * +------+ | +------+
4450 * | arg0 | -+ +-> | arg1 | 1
4451 * +------+ | +------+
4452 * | sym | +---> | arg0 | 2
4453 * +------+ +------+
4454 * | recv | | recv | 3
4455 * --+------+--------+------+------
4456 */
4457 /* shift arguments */
4458 if (i > 0) {
4459 MEMMOVE(&TOPN(i), &TOPN(i-1), VALUE, i);
4460 }
4461 calling->argc -= 1;
4462 DEC_SP(1);
4463
4464 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4465}
4466
4467static VALUE
4468vm_call_opt_send_complex(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4469{
4470 RB_DEBUG_COUNTER_INC(ccf_opt_send_complex);
4471 const struct rb_callinfo *ci = calling->cd->ci;
4472 int flags = VM_CALL_FCALL;
4473 VALUE sym;
4474
4475 VALUE argv_ary;
4476 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
4477 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4478 sym = rb_ary_shift(argv_ary);
4479 flags |= VM_CALL_ARGS_SPLAT;
4480 if (calling->kw_splat) {
4481 VALUE last_hash = rb_ary_last(0, NULL, argv_ary);
4482 ((struct RHash *)last_hash)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
4483 calling->kw_splat = 0;
4484 }
4485 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4486 }
4487
4488 if (calling->kw_splat) flags |= VM_CALL_KW_SPLAT;
4489 return vm_call_opt_send0(ec, reg_cfp, calling, flags);
4490}
4491
4492static VALUE
4493vm_call_opt_send_simple(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4494{
4495 RB_DEBUG_COUNTER_INC(ccf_opt_send_simple);
4496 return vm_call_opt_send0(ec, reg_cfp, calling, vm_ci_flag(calling->cd->ci) | VM_CALL_FCALL);
4497}
4498
4499static VALUE
4500vm_call_opt_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4501{
4502 RB_DEBUG_COUNTER_INC(ccf_opt_send);
4503
4504 const struct rb_callinfo *ci = calling->cd->ci;
4505 int flags = vm_ci_flag(ci);
4506
4507 if (UNLIKELY((flags & VM_CALL_FORWARDING) || (!(flags & VM_CALL_ARGS_SIMPLE) &&
4508 ((calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
4509 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
4510 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc)))))) {
4511 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_complex, TRUE);
4512 return vm_call_opt_send_complex(ec, reg_cfp, calling);
4513 }
4514
4515 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_simple, TRUE);
4516 return vm_call_opt_send_simple(ec, reg_cfp, calling);
4517}
4518
4519static VALUE
4520vm_call_method_missing_body(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
4521 const struct rb_callinfo *orig_ci, enum method_missing_reason reason)
4522{
4523 RB_DEBUG_COUNTER_INC(ccf_method_missing);
4524
4525 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
4526 unsigned int argc, flag;
4527
4528 flag = VM_CALL_FCALL | VM_CALL_OPT_SEND | vm_ci_flag(orig_ci);
4529 argc = ++calling->argc;
4530
4531 /* shift arguments: m(a, b, c) #=> method_missing(:m, a, b, c) */
4532 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4533 vm_check_canary(ec, reg_cfp->sp);
4534 if (argc > 1) {
4535 MEMMOVE(argv+1, argv, VALUE, argc-1);
4536 }
4537 argv[0] = ID2SYM(vm_ci_mid(orig_ci));
4538 INC_SP(1);
4539
4540 ec->method_missing_reason = reason;
4541
4542 struct rb_forwarding_call_data new_fcd = {
4543 .cd = {
4544 .ci = &VM_CI_ON_STACK(idMethodMissing, flag, argc, vm_ci_kwarg(orig_ci)),
4545 .cc = NULL,
4546 },
4547 .caller_ci = NULL,
4548 };
4549
4550 if (!(flag & VM_CALL_FORWARDING)) {
4551 calling->cd = &new_fcd.cd;
4552 }
4553 else {
4554 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4555 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4556 new_fcd.caller_ci = caller_ci;
4557 calling->cd = (struct rb_call_data *)&new_fcd;
4558 }
4559
4560 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }},
4561 rb_callable_method_entry_without_refinements(CLASS_OF(calling->recv), idMethodMissing, NULL));
4562 return vm_call_method(ec, reg_cfp, calling);
4563}
4564
4565static VALUE
4566vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4567{
4568 return vm_call_method_missing_body(ec, reg_cfp, calling, calling->cd->ci, vm_cc_cmethod_missing_reason(calling->cc));
4569}
4570
4571static const rb_callable_method_entry_t *refined_method_callable_without_refinement(const rb_callable_method_entry_t *me);
4572static VALUE
4573vm_call_zsuper(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE klass)
4574{
4575 klass = RCLASS_SUPER(klass);
4576
4577 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, vm_ci_mid(calling->cd->ci)) : NULL;
4578 if (cme == NULL) {
4579 return vm_call_method_nome(ec, cfp, calling);
4580 }
4581 if (cme->def->type == VM_METHOD_TYPE_REFINED &&
4582 cme->def->body.refined.orig_me) {
4583 cme = refined_method_callable_without_refinement(cme);
4584 }
4585
4586 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, cme);
4587
4588 return vm_call_method_each_type(ec, cfp, calling);
4589}
4590
4591static inline VALUE
4592find_refinement(VALUE refinements, VALUE klass)
4593{
4594 if (NIL_P(refinements)) {
4595 return Qnil;
4596 }
4597 return rb_hash_lookup(refinements, klass);
4598}
4599
4600PUREFUNC(static rb_control_frame_t * current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp));
4601static rb_control_frame_t *
4602current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp)
4603{
4604 rb_control_frame_t *top_cfp = cfp;
4605
4606 if (CFP_ISEQ(cfp) && !VM_FRAME_BMETHOD_P(cfp) && ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_BLOCK) {
4607 const rb_iseq_t *local_iseq = ISEQ_BODY(CFP_ISEQ(cfp))->local_iseq;
4608
4609 do {
4610 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
4611 if (RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
4612 /* TODO: orphan block */
4613 return top_cfp;
4614 }
4615 } while (CFP_ISEQ(cfp) != local_iseq && !VM_FRAME_BMETHOD_P(cfp));
4616 }
4617 return cfp;
4618}
4619
4620static const rb_callable_method_entry_t *
4621refined_method_callable_without_refinement(const rb_callable_method_entry_t *me)
4622{
4623 const rb_method_entry_t *orig_me = me->def->body.refined.orig_me;
4624 const rb_callable_method_entry_t *cme;
4625
4626 if (orig_me->defined_class == 0) {
4627 cme = NULL;
4628 rb_notimplement();
4629 }
4630 else {
4631 cme = (const rb_callable_method_entry_t *)orig_me;
4632 }
4633
4634 VM_ASSERT(callable_method_entry_p(cme));
4635
4636 if (UNDEFINED_METHOD_ENTRY_P(cme)) {
4637 cme = NULL;
4638 }
4639
4640 return cme;
4641}
4642
4643// Construct correct super chain for module refinements.
4644// Each class including/prepending a module has a weak map mapping refinement
4645// iclasses to refinement module iclasses with the correct super. Like
4646// singleton classes, these are lazily initialized (both the map and
4647// the refinement module iclasses).
4648static VALUE
4649module_refinement_iclass(VALUE refinement_iclass, VALUE defined_class)
4650{
4651 VALUE module_ref_iclass = Qundef;
4652
4653 RB_VM_LOCKING() {
4654 VALUE cache = RCLASS_MODULE_REFINEMENT_ICLASSES(defined_class);
4655
4656 if (LIKELY(cache)) {
4657 module_ref_iclass = rb_wmap_lookup(cache, refinement_iclass);
4658 }
4659 else {
4660 cache = rb_wmap_new_hidden();
4661 RCLASS_SET_MODULE_REFINEMENT_ICLASSES(defined_class, cache);
4662 }
4663
4664 if (UNDEF_P(module_ref_iclass)) {
4665 VALUE super;
4666 VALUE refinement_iclass_super = RCLASS_SUPER(refinement_iclass);
4667
4668 if (RICLASS_FOR_REFINEMENT_P(refinement_iclass_super)) {
4669 super = module_refinement_iclass(refinement_iclass_super, defined_class);
4670 }
4671 else {
4672 super = defined_class;
4673 }
4674
4675 module_ref_iclass = rb_include_class_new(RBASIC(refinement_iclass)->klass, super);
4676 RCLASS_SET_REFINED_CLASS(module_ref_iclass, RCLASS_REFINED_CLASS(refinement_iclass));
4677 rb_class_subclass_add(RCLASS_REFINED_CLASS(refinement_iclass), module_ref_iclass);
4678 rb_wmap_aset(cache, refinement_iclass, module_ref_iclass);
4679 }
4680 }
4681
4682 return module_ref_iclass;
4683}
4684
4685VALUE
4686rb_vm_module_refinement_iclass(VALUE refinement_iclass, VALUE defined_class)
4687{
4688 return module_refinement_iclass(refinement_iclass, defined_class);
4689}
4690
4691static VALUE
4692refinement_iclass_for_cme(VALUE refinement_iclass, const rb_callable_method_entry_t *cme)
4693{
4694 if (RB_TYPE_P(cme->owner, T_MODULE) && RB_TYPE_P(cme->defined_class, T_ICLASS)) {
4695 refinement_iclass = module_refinement_iclass(refinement_iclass, cme->defined_class);
4696 }
4697 return refinement_iclass;
4698}
4699
4700VALUE
4701rb_vm_refinement_iclass_for_cme(VALUE refinement_iclass, const rb_callable_method_entry_t *cme)
4702{
4703 return refinement_iclass_for_cme(refinement_iclass, cme);
4704}
4705
4706static const rb_callable_method_entry_t *
4707search_refined_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4708{
4709 ID mid = vm_ci_mid(calling->cd->ci);
4710 const rb_cref_t *cref = vm_get_cref(cfp->ep);
4711 const struct rb_callcache * const cc = calling->cc;
4712 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4713
4714 for (; cref; cref = CREF_NEXT(cref)) {
4715 VALUE refinement = find_refinement(CREF_REFINEMENTS(cref), vm_cc_cme(cc)->owner);
4716 if (NIL_P(refinement)) continue;
4717
4718 refinement = refinement_iclass_for_cme(refinement, vm_cc_cme(cc));
4719
4720 const rb_callable_method_entry_t *const ref_me =
4721 rb_callable_method_entry(refinement, mid);
4722
4723 if (ref_me) {
4724 if (vm_cc_call(cc) == vm_call_super_method) {
4725 const rb_control_frame_t *top_cfp = current_method_entry(ec, cfp);
4726 const rb_callable_method_entry_t *top_me = rb_vm_frame_method_entry(top_cfp);
4727 if (top_me && rb_method_definition_eq(ref_me->def, top_me->def)) {
4728 continue;
4729 }
4730 }
4731
4732 if (cme->def->type != VM_METHOD_TYPE_REFINED ||
4733 cme->def != ref_me->def) {
4734 cme = ref_me;
4735 }
4736 if (ref_me->def->type != VM_METHOD_TYPE_REFINED) {
4737 return cme;
4738 }
4739 }
4740 else {
4741 return NULL;
4742 }
4743 }
4744
4745 if (vm_cc_cme(cc)->def->body.refined.orig_me) {
4746 return refined_method_callable_without_refinement(vm_cc_cme(cc));
4747 }
4748 else {
4749 VALUE klass = RCLASS_SUPER(vm_cc_cme(cc)->defined_class);
4750 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, mid) : NULL;
4751 return cme;
4752 }
4753}
4754
4755static VALUE
4756vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4757{
4758 const rb_callable_method_entry_t *ref_cme = search_refined_method(ec, cfp, calling);
4759
4760 if (ref_cme) {
4761 if (calling->cd->cc) {
4762 const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement);
4763 RB_OBJ_WRITE(CFP_ISEQ(cfp), &calling->cd->cc, cc);
4764 return vm_call_method(ec, cfp, calling);
4765 }
4766 else {
4767 struct rb_callcache *ref_cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, ref_cme);
4768 calling->cc= ref_cc;
4769 return vm_call_method(ec, cfp, calling);
4770 }
4771 }
4772 else {
4773 return vm_call_method_nome(ec, cfp, calling);
4774 }
4775}
4776
4777static inline VALUE vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci, bool is_lambda, VALUE block_handler);
4778
4779NOINLINE(static VALUE
4780 vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4781 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler));
4782
4783static VALUE
4784vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4785 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler)
4786{
4787 int argc = calling->argc;
4788
4789 /* remove self */
4790 if (argc > 0) MEMMOVE(&TOPN(argc), &TOPN(argc-1), VALUE, argc);
4791 DEC_SP(1);
4792
4793 return vm_invoke_block(ec, reg_cfp, calling, ci, false, block_handler);
4794}
4795
4796static VALUE
4797vm_call_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4798{
4799 RB_DEBUG_COUNTER_INC(ccf_opt_call);
4800
4801 const struct rb_callinfo *ci = calling->cd->ci;
4802 VALUE procval = calling->recv;
4803 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, VM_BH_FROM_PROC(procval));
4804}
4805
4806static VALUE
4807vm_call_opt_block_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4808{
4809 RB_DEBUG_COUNTER_INC(ccf_opt_block_call);
4810
4811 VALUE block_handler = VM_ENV_BLOCK_HANDLER(VM_CF_LEP(reg_cfp));
4812 const struct rb_callinfo *ci = calling->cd->ci;
4813
4814 enum ruby_basic_operators bop;
4815 switch (vm_ci_mid(ci)) {
4816 case idAREF: bop = BOP_AREF; break;
4817 case idYield: bop = BOP_YIELD; break;
4818 case idEqq: bop = BOP_EQQ; break;
4819 default: bop = BOP_CALL; break;
4820 }
4821
4822 if (BASIC_OP_UNREDEFINED_P(bop, PROC_REDEFINED_OP_FLAG)) {
4823 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, block_handler);
4824 }
4825 else {
4826 calling->recv = rb_vm_bh_to_procval(ec, block_handler);
4827 calling->cc = rb_vm_search_method_slowpath(ci, CLASS_OF(calling->recv));
4828 return vm_call_general(ec, reg_cfp, calling);
4829 }
4830}
4831
4832static VALUE
4833vm_call_opt_struct_aref0(rb_execution_context_t *ec, struct rb_calling_info *calling)
4834{
4835 VALUE recv = calling->recv;
4836
4837 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4838 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4839 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_AREF);
4840
4841 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4842 return RSTRUCT_GET_RAW(recv, off);
4843}
4844
4845static VALUE
4846vm_call_opt_struct_aref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4847{
4848 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aref);
4849
4850 VALUE ret = vm_call_opt_struct_aref0(ec, calling);
4851 reg_cfp->sp -= 1;
4852 return ret;
4853}
4854
4855static VALUE
4856vm_call_opt_struct_aset0(rb_execution_context_t *ec, struct rb_calling_info *calling, VALUE val)
4857{
4858 VALUE recv = calling->recv;
4859
4860 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4861 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4862 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_ASET);
4863
4864 rb_check_frozen(recv);
4865
4866 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4867 RSTRUCT_SET_RAW(recv, off, val);
4868
4869 return val;
4870}
4871
4872static VALUE
4873vm_call_opt_struct_aset(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4874{
4875 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aset);
4876
4877 VALUE ret = vm_call_opt_struct_aset0(ec, calling, *(reg_cfp->sp - 1));
4878 reg_cfp->sp -= 2;
4879 return ret;
4880}
4881
4882NOINLINE(static VALUE vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4883 const struct rb_callinfo *ci, const struct rb_callcache *cc));
4884
4885#define VM_CALL_METHOD_ATTR(var, func, nohook) \
4886 if (UNLIKELY(ruby_vm_c_events_enabled > 0)) { \
4887 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, calling->recv, vm_cc_cme(cc)->def->original_id, \
4888 vm_ci_mid(ci), vm_cc_cme(cc)->owner, Qundef); \
4889 var = func; \
4890 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, calling->recv, vm_cc_cme(cc)->def->original_id, \
4891 vm_ci_mid(ci), vm_cc_cme(cc)->owner, (var)); \
4892 } \
4893 else { \
4894 nohook; \
4895 var = func; \
4896 }
4897
4898static VALUE
4899vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4900 const struct rb_callinfo *ci, const struct rb_callcache *cc)
4901{
4902 switch (vm_cc_cme(cc)->def->body.optimized.type) {
4903 case OPTIMIZED_METHOD_TYPE_SEND:
4904 CC_SET_FASTPATH(cc, vm_call_opt_send, TRUE);
4905 return vm_call_opt_send(ec, cfp, calling);
4906 case OPTIMIZED_METHOD_TYPE_CALL:
4907 CC_SET_FASTPATH(cc, vm_call_opt_call, TRUE);
4908 return vm_call_opt_call(ec, cfp, calling);
4909 case OPTIMIZED_METHOD_TYPE_BLOCK_CALL:
4910 CC_SET_FASTPATH(cc, vm_call_opt_block_call, TRUE);
4911 return vm_call_opt_block_call(ec, cfp, calling);
4912 case OPTIMIZED_METHOD_TYPE_STRUCT_AREF: {
4913 CALLER_SETUP_ARG(cfp, calling, ci, 0);
4914 rb_check_arity(calling->argc, 0, 0);
4915
4916 VALUE v;
4917 VM_CALL_METHOD_ATTR(v,
4918 vm_call_opt_struct_aref(ec, cfp, calling),
4919 set_vm_cc_ivar(cc); \
4920 CC_SET_FASTPATH(cc, vm_call_opt_struct_aref, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
4921 return v;
4922 }
4923 case OPTIMIZED_METHOD_TYPE_STRUCT_ASET: {
4924 CALLER_SETUP_ARG(cfp, calling, ci, 1);
4925 rb_check_arity(calling->argc, 1, 1);
4926
4927 VALUE v;
4928 VM_CALL_METHOD_ATTR(v,
4929 vm_call_opt_struct_aset(ec, cfp, calling),
4930 set_vm_cc_ivar(cc); \
4931 CC_SET_FASTPATH(cc, vm_call_opt_struct_aset, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
4932 return v;
4933 }
4934 default:
4935 rb_bug("vm_call_method: unsupported optimized method type (%d)", vm_cc_cme(cc)->def->body.optimized.type);
4936 }
4937}
4938
4939static VALUE
4940vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4941{
4942 const struct rb_callinfo *ci = calling->cd->ci;
4943 const struct rb_callcache *cc = calling->cc;
4944 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4945 VALUE v;
4946
4947 VM_ASSERT(! METHOD_ENTRY_INVALIDATED(cme));
4948
4949 switch (cme->def->type) {
4950 case VM_METHOD_TYPE_ISEQ:
4951 if (ISEQ_BODY(def_iseq_ptr(cme->def))->param.flags.forwardable) {
4952 CC_SET_FASTPATH(cc, vm_call_iseq_fwd_setup, TRUE);
4953 return vm_call_iseq_fwd_setup(ec, cfp, calling);
4954 }
4955 else {
4956 CC_SET_FASTPATH(cc, vm_call_iseq_setup, TRUE);
4957 return vm_call_iseq_setup(ec, cfp, calling);
4958 }
4959
4960 case VM_METHOD_TYPE_NOTIMPLEMENTED:
4961 case VM_METHOD_TYPE_CFUNC:
4962 CC_SET_FASTPATH(cc, vm_call_cfunc, TRUE);
4963 return vm_call_cfunc(ec, cfp, calling);
4964
4965 case VM_METHOD_TYPE_ATTRSET:
4966 CALLER_SETUP_ARG(cfp, calling, ci, 1);
4967
4968 rb_check_arity(calling->argc, 1, 1);
4969
4970 const unsigned int aset_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_KWARG | VM_CALL_FORWARDING);
4971
4972 if (vm_cc_markable(cc)) {
4973 vm_cc_attr_index_set(cc, IVAR_CACHE_INIT);
4974 VM_CALL_METHOD_ATTR(v,
4975 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
4976 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
4977 }
4978 else {
4979 cc = &((struct rb_callcache) {
4980 .flags = T_IMEMO |
4981 (imemo_callcache << FL_USHIFT) |
4982 VM_CALLCACHE_UNMARKABLE |
4983 VM_CALLCACHE_ON_STACK,
4984 .klass = cc->klass,
4985 .cme_ = cc->cme_,
4986 .call_ = cc->call_,
4987 .aux_ = {
4988 .attr = {
4989 .value = IVAR_CACHE_INIT,
4990 }
4991 },
4992 });
4993
4994 VM_CALL_METHOD_ATTR(v,
4995 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
4996 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
4997 }
4998 return v;
4999
5000 case VM_METHOD_TYPE_IVAR:
5001 CALLER_SETUP_ARG(cfp, calling, ci, 0);
5002 rb_check_arity(calling->argc, 0, 0);
5003 vm_cc_attr_index_set(cc, rb_getivar_cache_pack(ROOT_SHAPE_ID, ATTR_INDEX_NOT_SET));
5004 const unsigned int ivar_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_FORWARDING);
5005 VM_CALL_METHOD_ATTR(v,
5006 vm_call_ivar(ec, cfp, calling),
5007 CC_SET_FASTPATH(cc, vm_call_ivar, !(vm_ci_flag(ci) & ivar_mask)));
5008 return v;
5009
5010 case VM_METHOD_TYPE_MISSING:
5011 vm_cc_method_missing_reason_set(cc, 0);
5012 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
5013 return vm_call_method_missing(ec, cfp, calling);
5014
5015 case VM_METHOD_TYPE_BMETHOD:
5016 CC_SET_FASTPATH(cc, vm_call_bmethod, TRUE);
5017 return vm_call_bmethod(ec, cfp, calling);
5018
5019 case VM_METHOD_TYPE_ALIAS:
5020 CC_SET_FASTPATH(cc, vm_call_alias, TRUE);
5021 return vm_call_alias(ec, cfp, calling);
5022
5023 case VM_METHOD_TYPE_OPTIMIZED:
5024 return vm_call_optimized(ec, cfp, calling, ci, cc);
5025
5026 case VM_METHOD_TYPE_UNDEF:
5027 break;
5028
5029 case VM_METHOD_TYPE_ZSUPER:
5030 return vm_call_zsuper(ec, cfp, calling, RCLASS_ORIGIN(vm_cc_cme(cc)->defined_class));
5031
5032 case VM_METHOD_TYPE_REFINED:
5033 // CC_SET_FASTPATH(cc, vm_call_refined, TRUE);
5034 // should not set FASTPATH since vm_call_refined assumes cc->call is vm_call_super_method on invokesuper.
5035 return vm_call_refined(ec, cfp, calling);
5036 }
5037
5038 rb_bug("vm_call_method: unsupported method type (%d)", vm_cc_cme(cc)->def->type);
5039}
5040
5041NORETURN(static void vm_raise_method_missing(rb_execution_context_t *ec, int argc, const VALUE *argv, VALUE obj, int call_status));
5042
5043static VALUE
5044vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
5045{
5046 /* method missing */
5047 const struct rb_callinfo *ci = calling->cd->ci;
5048 const int stat = ci_missing_reason(ci);
5049
5050 if (vm_ci_mid(ci) == idMethodMissing) {
5051 if (UNLIKELY(calling->heap_argv)) {
5052 vm_raise_method_missing(ec, RARRAY_LENINT(calling->heap_argv), RARRAY_CONST_PTR(calling->heap_argv), calling->recv, stat);
5053 }
5054 else {
5055 rb_control_frame_t *reg_cfp = cfp;
5056 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
5057 vm_raise_method_missing(ec, calling->argc, argv, calling->recv, stat);
5058 }
5059 }
5060 else {
5061 return vm_call_method_missing_body(ec, cfp, calling, ci, stat);
5062 }
5063}
5064
5065/* Protected method calls and super invocations need to check that the receiver
5066 * (self for super) inherits the module on which the method is defined.
5067 * In the case of refinements, it should consider the original class not the
5068 * refinement.
5069 */
5070static VALUE
5071vm_defined_class_for_protected_call(const rb_callable_method_entry_t *me)
5072{
5073 VALUE defined_class = me->defined_class;
5074 VALUE refined_class = RCLASS_REFINED_CLASS(defined_class);
5075 return NIL_P(refined_class) ? defined_class : refined_class;
5076}
5077
5078static inline VALUE
5079vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
5080{
5081 const struct rb_callinfo *ci = calling->cd->ci;
5082 const struct rb_callcache *cc = calling->cc;
5083
5084 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
5085
5086 if (vm_cc_cme(cc) != NULL) {
5087 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
5088 case METHOD_VISI_PUBLIC: /* likely */
5089 return vm_call_method_each_type(ec, cfp, calling);
5090
5091 case METHOD_VISI_PRIVATE:
5092 if (!(vm_ci_flag(ci) & VM_CALL_FCALL)) {
5093 enum method_missing_reason stat = MISSING_PRIVATE;
5094 if (vm_ci_flag(ci) & VM_CALL_VCALL) stat |= MISSING_VCALL;
5095
5096 vm_cc_method_missing_reason_set(cc, stat);
5097 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
5098 return vm_call_method_missing(ec, cfp, calling);
5099 }
5100 return vm_call_method_each_type(ec, cfp, calling);
5101
5102 case METHOD_VISI_PROTECTED:
5103 if (!(vm_ci_flag(ci) & (VM_CALL_OPT_SEND | VM_CALL_FCALL))) {
5104 VALUE defined_class = vm_defined_class_for_protected_call(vm_cc_cme(cc));
5105 if (!rb_obj_is_kind_of(cfp->self, defined_class)) {
5106 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
5107 return vm_call_method_missing(ec, cfp, calling);
5108 }
5109 else {
5110 /* caching method info to dummy cc */
5111 VM_ASSERT(vm_cc_cme(cc) != NULL);
5112 struct rb_callcache cc_on_stack = *cc;
5113 FL_SET_RAW((VALUE)&cc_on_stack, VM_CALLCACHE_UNMARKABLE);
5114 calling->cc = &cc_on_stack;
5115 return vm_call_method_each_type(ec, cfp, calling);
5116 }
5117 }
5118 return vm_call_method_each_type(ec, cfp, calling);
5119
5120 default:
5121 rb_bug("unreachable");
5122 }
5123 }
5124 else {
5125 return vm_call_method_nome(ec, cfp, calling);
5126 }
5127}
5128
5129static VALUE
5130vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
5131{
5132 RB_DEBUG_COUNTER_INC(ccf_general);
5133 return vm_call_method(ec, reg_cfp, calling);
5134}
5135
5136void
5137rb_vm_cc_general(const struct rb_callcache *cc)
5138{
5139 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
5140 VM_ASSERT(cc != vm_cc_empty());
5141
5142 *(vm_call_handler *)&cc->call_ = vm_call_general;
5143}
5144
5145static VALUE
5146vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
5147{
5148 RB_DEBUG_COUNTER_INC(ccf_super_method);
5149
5150 // This line is introduced to make different from `vm_call_general` because some compilers (VC we found)
5151 // can merge the function and the address of the function becomes same.
5152 // The address of `vm_call_super_method` is used in `search_refined_method`, so it should be different.
5153 if (ec == NULL) rb_bug("unreachable");
5154
5155 /* this check is required to distinguish with other functions. */
5156 VM_ASSERT(vm_cc_call(calling->cc) == vm_call_super_method);
5157 return vm_call_method(ec, reg_cfp, calling);
5158}
5159
5160/* super */
5161
5162static inline VALUE
5163vm_search_normal_superclass(VALUE klass)
5164{
5165 if (RICLASS_FOR_REFINEMENT_P(klass)) {
5166 if (RB_TYPE_P(RCLASS_REFINED_CLASS(klass), T_MODULE)) {
5167 do {
5168 klass = RCLASS_SUPER(klass);
5169 } while (RICLASS_FOR_REFINEMENT_P(klass));
5170
5171 return klass;
5172 }
5173
5174 klass = RBASIC(klass)->klass;
5175 }
5176 klass = RCLASS_ORIGIN(klass);
5177 return RCLASS_SUPER(klass);
5178}
5179
5180NORETURN(static void vm_super_outside(void));
5181
5182static void
5183vm_super_outside(void)
5184{
5185 rb_raise(rb_eNoMethodError, "super called outside of method");
5186}
5187
5188static const struct rb_callcache *
5189empty_cc_for_super(void)
5190{
5191 return &vm_empty_cc_for_super;
5192}
5193
5194static const struct rb_callcache *
5195vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *cd, VALUE recv)
5196{
5197 VALUE current_defined_class;
5198 const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp);
5199 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
5200
5201 if (!me) {
5202 vm_super_outside();
5203 }
5204
5205 current_defined_class = vm_defined_class_for_protected_call(me);
5206
5207 if (BUILTIN_TYPE(current_defined_class) != T_MODULE &&
5208 iseq != method_entry_iseqptr(me) &&
5209 !rb_obj_is_kind_of(recv, current_defined_class)) {
5210 VALUE m = RB_TYPE_P(current_defined_class, T_ICLASS) ?
5211 RCLASS_INCLUDER(current_defined_class) : current_defined_class;
5212
5213 if (m) { /* not bound UnboundMethod */
5214 rb_raise(rb_eTypeError,
5215 "self has wrong type to call super in this context: "
5216 "%"PRIsVALUE" (expected %"PRIsVALUE")",
5217 rb_obj_class(recv), m);
5218 }
5219 }
5220
5221 if (me->def->type == VM_METHOD_TYPE_BMETHOD && (vm_ci_flag(cd->ci) & VM_CALL_ZSUPER)) {
5222 rb_raise(rb_eRuntimeError,
5223 "implicit argument passing of super from method defined"
5224 " by define_method() is not supported."
5225 " Specify all arguments explicitly.");
5226 }
5227
5228 ID mid = me->def->original_id;
5229
5230 if (!vm_ci_markable(cd->ci)) {
5231 VM_FORCE_WRITE((const VALUE *)&cd->ci->mid, (VALUE)mid);
5232 }
5233 else {
5234 // update iseq. really? (TODO)
5235 cd->ci = vm_ci_new_runtime(mid,
5236 vm_ci_flag(cd->ci),
5237 vm_ci_argc(cd->ci),
5238 vm_ci_kwarg(cd->ci));
5239
5240 RB_OBJ_WRITTEN(iseq, Qundef, cd->ci);
5241 }
5242
5243 const struct rb_callcache *cc;
5244
5245 VALUE klass = vm_search_normal_superclass(me->defined_class);
5246
5247 if (!klass) {
5248 /* bound instance method of module */
5249 cc = vm_cc_new(Qundef, NULL, vm_call_method_missing, cc_type_super);
5250 RB_OBJ_WRITE(iseq, &cd->cc, cc);
5251 }
5252 else {
5253 cc = vm_search_method_fastpath(reg_cfp, cd, klass);
5254 const rb_callable_method_entry_t *cached_cme = vm_cc_cme(cc);
5255
5256 // define_method can cache for different method id
5257 if (cached_cme == NULL) {
5258 // empty_cc_for_super is not markable object
5259 cd->cc = empty_cc_for_super();
5260 }
5261 else if (cached_cme->called_id != mid) {
5262 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
5263 if (cme) {
5264 cc = vm_cc_new(klass, cme, vm_call_super_method, cc_type_super);
5265 RB_OBJ_WRITE(iseq, &cd->cc, cc);
5266 }
5267 else {
5268 cd->cc = cc = empty_cc_for_super();
5269 }
5270 }
5271 else {
5272 switch (cached_cme->def->type) {
5273 // vm_call_refined (search_refined_method) assumes cc->call is vm_call_super_method on invokesuper
5274 case VM_METHOD_TYPE_REFINED:
5275 // cc->klass is superclass of receiver class. Checking cc->klass is not enough to invalidate IVC for the receiver class.
5276 case VM_METHOD_TYPE_ATTRSET:
5277 case VM_METHOD_TYPE_IVAR:
5278 vm_cc_call_set(cc, vm_call_super_method); // invalidate fastpath
5279 break;
5280 default:
5281 break; // use fastpath
5282 }
5283 }
5284 }
5285
5286 VM_ASSERT((vm_cc_cme(cc), true));
5287
5288 return cc;
5289}
5290
5291/* yield */
5292
5293static inline int
5294block_proc_is_lambda(const VALUE procval)
5295{
5296 rb_proc_t *proc;
5297
5298 if (procval) {
5299 GetProcPtr(procval, proc);
5300 return proc->header.is_lambda;
5301 }
5302 else {
5303 return 0;
5304 }
5305}
5306
5307static VALUE
5308vm_yield_with_cfunc(rb_execution_context_t *ec,
5309 const struct rb_captured_block *captured,
5310 VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE block_handler,
5311 const rb_callable_method_entry_t *me)
5312{
5313 int is_lambda = FALSE; /* TODO */
5314 VALUE val, arg, blockarg;
5315 int frame_flag;
5316 const struct vm_ifunc *ifunc = captured->code.ifunc;
5317
5318 if (is_lambda) {
5319 arg = rb_ary_new4(argc, argv);
5320 }
5321 else if (argc == 0) {
5322 arg = Qnil;
5323 }
5324 else {
5325 arg = argv[0];
5326 }
5327
5328 blockarg = rb_vm_bh_to_procval(ec, block_handler);
5329
5330 frame_flag = VM_FRAME_MAGIC_IFUNC | VM_FRAME_FLAG_CFRAME | (me ? VM_FRAME_FLAG_BMETHOD : 0);
5331 if (kw_splat) {
5332 frame_flag |= VM_FRAME_FLAG_CFRAME_KW;
5333 }
5334
5335 vm_push_frame(ec, (const rb_iseq_t *)captured->code.ifunc,
5336 frame_flag,
5337 self,
5338 VM_GUARDED_PREV_EP(captured->ep),
5339 (VALUE)me,
5340 0, ec->cfp->sp, 0, 0);
5341 val = (*ifunc->func)(arg, (VALUE)ifunc->data, argc, argv, blockarg);
5342 rb_vm_pop_frame(ec);
5343
5344 return val;
5345}
5346
5347VALUE
5348rb_vm_yield_with_cfunc(rb_execution_context_t *ec, const struct rb_captured_block *captured, int argc, const VALUE *argv)
5349{
5350 return vm_yield_with_cfunc(ec, captured, captured->self, argc, argv, 0, VM_BLOCK_HANDLER_NONE, NULL);
5351}
5352
5353static VALUE
5354vm_yield_with_symbol(rb_execution_context_t *ec, VALUE symbol, int argc, const VALUE *argv, int kw_splat, VALUE block_handler)
5355{
5356 VALUE passed_proc = rb_vm_bh_to_procval(ec, block_handler);
5357
5358 if (!rb_box_available()) {
5359 return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc);
5360 }
5361
5362 const rb_control_frame_t *reg_cfp = ec->cfp;
5363 const rb_control_frame_t *ruby_cfp = rb_vm_get_ruby_level_next_cfp(ec, reg_cfp);
5364 const rb_box_t *box = NULL;
5365
5366 /*
5367 * Traverse the frames until a user box (Main or Optional Box) is found.
5368 * Frames for built-in methods like <internal:xxx> run in the Root or Master Box,
5369 */
5370 while (ruby_cfp) {
5371 box = current_box_on_cfp(ec, ruby_cfp);
5372 if (BOX_USER_P(box)) {
5373 break;
5374 }
5375 ruby_cfp = rb_vm_get_ruby_level_next_cfp(ec, RUBY_VM_PREVIOUS_CONTROL_FRAME(ruby_cfp));
5376 }
5377
5378 // Fallback to the normal call if no user box is found
5379 if (!ruby_cfp || !box) {
5380 return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc);
5381 }
5382
5383 /*
5384 * Push a dummy block frame whose outer env is `ruby_cfp` so that the
5385 * method resolves in the caller's box (via the ep chain) and the
5386 * backtrace reads like a usual block invocation at the caller's site.
5387 */
5388 const rb_iseq_t *caller_iseq = CFP_ISEQ(ruby_cfp);
5389 VALUE name = rb_sprintf("block in %"PRIsVALUE, rb_iseq_label(caller_iseq));
5390 const rb_iseq_t *iseq = rb_iseq_new_with_opt(Qnil, name,
5391 rb_iseq_path(caller_iseq), rb_iseq_realpath(caller_iseq),
5392 rb_vm_get_sourceline(ruby_cfp), caller_iseq, 0,
5393 ISEQ_TYPE_BLOCK, NULL, Qnil);
5394 volatile VALUE val = Qnil;
5395 enum ruby_tag_type state;
5396
5397 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_FINISH,
5398 ruby_cfp->self, VM_GUARDED_PREV_EP(ruby_cfp->ep),
5399 Qfalse, /* cref or me */
5400 ISEQ_BODY(iseq)->iseq_encoded, reg_cfp->sp,
5401 ISEQ_BODY(iseq)->local_table_size, ISEQ_BODY(iseq)->stack_max);
5402
5403 /*
5404 * The pushed frame is finished (owned by no vm_exec loop), so catch the
5405 * non-local exit here, pop the frame, and let the caller's handlers see
5406 * the exception.
5407 */
5408 EC_PUSH_TAG(ec);
5409 if ((state = EC_EXEC_TAG()) == TAG_NONE) {
5410 val = rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc);
5411 }
5412 EC_POP_TAG();
5413
5414 if (state != TAG_NONE) {
5415 rb_vm_rewind_cfp(ec, (rb_control_frame_t *)reg_cfp);
5416 EC_JUMP_TAG(ec, state);
5417 }
5418
5419 rb_vm_pop_frame(ec);
5420
5421 return val;
5422}
5423
5424static inline int
5425vm_callee_setup_block_arg_arg0_splat(rb_control_frame_t *cfp, const rb_iseq_t *iseq, VALUE *argv, VALUE ary)
5426{
5427 int i;
5428 rb_len_t len = RARRAY_LEN(ary);
5429
5430 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5431
5432 for (i=0; i<len && i<ISEQ_BODY(iseq)->param.lead_num; i++) {
5433 argv[i] = RARRAY_AREF(ary, i);
5434 }
5435
5436 return i;
5437}
5438
5439static inline VALUE
5440vm_callee_setup_block_arg_arg0_check(VALUE *argv)
5441{
5442 VALUE ary, arg0 = argv[0];
5443 ary = rb_check_array_type(arg0);
5444#if 0
5445 argv[0] = arg0;
5446#else
5447 VM_ASSERT(argv[0] == arg0);
5448#endif
5449 return ary;
5450}
5451
5452static int
5453vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type)
5454{
5455 if (rb_simple_iseq_p(iseq)) {
5456 rb_control_frame_t *cfp = ec->cfp;
5457 VALUE arg0;
5458
5459 CALLER_SETUP_ARG(cfp, calling, ci, ISEQ_BODY(iseq)->param.lead_num);
5460
5461 if (arg_setup_type == arg_setup_block &&
5462 calling->argc == 1 &&
5463 ISEQ_BODY(iseq)->param.flags.has_lead &&
5464 !ISEQ_BODY(iseq)->param.flags.ambiguous_param0 &&
5465 !NIL_P(arg0 = vm_callee_setup_block_arg_arg0_check(argv))) {
5466 calling->argc = vm_callee_setup_block_arg_arg0_splat(cfp, iseq, argv, arg0);
5467 }
5468
5469 if (calling->argc != ISEQ_BODY(iseq)->param.lead_num) {
5470 if (arg_setup_type == arg_setup_block) {
5471 if (calling->argc < ISEQ_BODY(iseq)->param.lead_num) {
5472 int i;
5473 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5474 for (i=calling->argc; i<ISEQ_BODY(iseq)->param.lead_num; i++) argv[i] = Qnil;
5475 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* fill rest parameters */
5476 }
5477 else if (calling->argc > ISEQ_BODY(iseq)->param.lead_num) {
5478 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* simply truncate arguments */
5479 }
5480 }
5481 else {
5482 argument_arity_error(ec, iseq, NULL, calling->argc, ISEQ_BODY(iseq)->param.lead_num, ISEQ_BODY(iseq)->param.lead_num);
5483 }
5484 }
5485
5486 return 0;
5487 }
5488 else {
5489 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_type);
5490 }
5491}
5492
5493static int
5494vm_yield_setup_args(rb_execution_context_t *ec, const rb_iseq_t *iseq, const int argc, VALUE *argv, int flags, VALUE block_handler, enum arg_setup_type arg_setup_type)
5495{
5496 struct rb_calling_info calling_entry, *calling;
5497
5498 calling = &calling_entry;
5499 calling->argc = argc;
5500 calling->block_handler = block_handler;
5501 calling->kw_splat = (flags & VM_CALL_KW_SPLAT) ? 1 : 0;
5502 calling->recv = Qundef;
5503 calling->heap_argv = 0;
5504 calling->cc = NULL;
5505 struct rb_callinfo dummy_ci = VM_CI_ON_STACK(0, flags, 0, 0);
5506
5507 return vm_callee_setup_block_arg(ec, calling, &dummy_ci, iseq, argv, arg_setup_type);
5508}
5509
5510/* ruby iseq -> ruby block */
5511
5512static VALUE
5513vm_invoke_iseq_block_with_cref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5514 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5515 bool is_lambda, VALUE block_handler, const rb_cref_t *cref)
5516{
5517 const struct rb_captured_block *captured = VM_BH_TO_ISEQ_BLOCK(block_handler);
5518 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
5519 const int arg_size = ISEQ_BODY(iseq)->param.size;
5520 VALUE * const rsp = GET_SP() - calling->argc;
5521 VALUE * const argv = rsp;
5522 int opt_pc = vm_callee_setup_block_arg(ec, calling, ci, iseq, argv, is_lambda ? arg_setup_method : arg_setup_block);
5523 int frame_flag = VM_FRAME_MAGIC_BLOCK | (is_lambda ? VM_FRAME_FLAG_LAMBDA : 0);
5524
5525 SET_SP(rsp);
5526
5527 /* cref is normally 0; Proc#refined supplies a refinement cref */
5528 vm_push_frame(ec, iseq,
5529 frame_flag,
5530 captured->self,
5531 VM_GUARDED_PREV_EP(captured->ep), (VALUE)cref,
5532 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
5533 rsp + arg_size,
5534 ISEQ_BODY(iseq)->local_table_size - arg_size, ISEQ_BODY(iseq)->stack_max);
5535
5536 return Qundef;
5537}
5538
5539static VALUE
5540vm_invoke_iseq_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5541 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5542 bool is_lambda, VALUE block_handler)
5543{
5544 return vm_invoke_iseq_block_with_cref(ec, reg_cfp, calling, ci, is_lambda, block_handler, NULL);
5545}
5546
5547static VALUE
5548vm_invoke_symbol_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5549 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5550 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5551{
5552 VALUE symbol = VM_BH_TO_SYMBOL(block_handler);
5553 int flags = vm_ci_flag(ci);
5554
5555 if (UNLIKELY(!(flags & VM_CALL_ARGS_SIMPLE) &&
5556 ((calling->argc == 0) ||
5557 (calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
5558 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
5559 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc))))) {
5560 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
5561 flags = 0;
5562 if (UNLIKELY(calling->heap_argv)) {
5563#if VM_ARGC_STACK_MAX < 0
5564 if (RARRAY_LEN(calling->heap_argv) < 1) {
5565 rb_raise(rb_eArgError, "no receiver given");
5566 }
5567#endif
5568 calling->recv = rb_ary_shift(calling->heap_argv);
5569 // Modify stack to avoid cfp consistency error
5570 reg_cfp->sp++;
5571 reg_cfp->sp[-1] = reg_cfp->sp[-2];
5572 reg_cfp->sp[-2] = calling->recv;
5573 flags |= VM_CALL_ARGS_SPLAT;
5574 }
5575 else {
5576 if (calling->argc < 1) {
5577 rb_raise(rb_eArgError, "no receiver given");
5578 }
5579 calling->recv = TOPN(--calling->argc);
5580 }
5581 if (calling->kw_splat) {
5582 flags |= VM_CALL_KW_SPLAT;
5583 }
5584 }
5585 else {
5586 if (calling->argc < 1) {
5587 rb_raise(rb_eArgError, "no receiver given");
5588 }
5589 calling->recv = TOPN(--calling->argc);
5590 }
5591
5592 return vm_call_symbol(ec, reg_cfp, calling, ci, symbol, flags);
5593}
5594
5595static VALUE
5596vm_invoke_ifunc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5597 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5598 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5599{
5600 VALUE val;
5601 int argc;
5602 const struct rb_captured_block *captured = VM_BH_TO_IFUNC_BLOCK(block_handler);
5603 CALLER_SETUP_ARG(ec->cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
5604 argc = calling->argc;
5605 val = vm_yield_with_cfunc(ec, captured, captured->self, CALLING_ARGC(calling), calling->heap_argv ? RARRAY_CONST_PTR(calling->heap_argv) : STACK_ADDR_FROM_TOP(argc), calling->kw_splat, calling->block_handler, NULL);
5606 POPN(argc); /* TODO: should put before C/yield? */
5607 return val;
5608}
5609
5610static inline VALUE
5611vm_block_to_block_handler(const struct rb_block *block)
5612{
5613 switch (vm_block_type(block)) {
5614 case block_type_iseq:
5615 return VM_BH_FROM_ISEQ_BLOCK(&block->as.captured);
5616 case block_type_ifunc:
5617 return VM_BH_FROM_IFUNC_BLOCK(&block->as.captured);
5618 case block_type_symbol:
5619 return VM_BH_FROM_SYMBOL(block->as.symbol);
5620 case block_type_proc:
5621 return VM_BH_FROM_PROC(block->as.proc);
5622 }
5623 VM_UNREACHABLE(vm_yield_with_proc);
5624 return Qundef;
5625}
5626
5627static VALUE
5628vm_proc_to_block_handler(VALUE procval)
5629{
5630 return vm_block_to_block_handler(vm_proc_block(procval));
5631}
5632
5633NOINLINE(static VALUE
5634vm_invoke_proc_block_with_cref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5635 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5636 bool is_lambda, VALUE block_handler, VALUE refined_procval));
5637static VALUE
5638vm_invoke_proc_block_with_cref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5639 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5640 bool is_lambda, VALUE block_handler, VALUE refined_procval)
5641{
5642 const rb_cref_t *cref = rb_proc_refinements_cref_for_call(refined_procval);
5643 return vm_invoke_iseq_block_with_cref(ec, reg_cfp, calling, ci, is_lambda, block_handler, cref);
5644}
5645
5646static VALUE
5647vm_invoke_proc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5648 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5649 bool is_lambda, VALUE block_handler)
5650{
5651 VALUE refined_procval = 0;
5652
5653 while (vm_block_handler_type(block_handler) == block_handler_type_proc) {
5654 VALUE procval = VM_BH_TO_PROC(block_handler);
5655 rb_proc_t *po;
5656 GetProcPtr(procval, po);
5657 if (po->header.is_refined) refined_procval = procval;
5658 is_lambda = po->header.is_lambda;
5659 block_handler = vm_block_to_block_handler(&po->block);
5660 }
5661
5662 if (UNLIKELY(refined_procval) && vm_block_handler_type(block_handler) == block_handler_type_iseq) {
5663 /* should not be inlined so that vm_invoke_proc_block stays leaf/frameless. */
5664 return vm_invoke_proc_block_with_cref(ec, reg_cfp, calling, ci, is_lambda, block_handler,
5665 refined_procval);
5666 }
5667
5668 return vm_invoke_block(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5669}
5670
5671static inline VALUE
5672vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5673 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5674 bool is_lambda, VALUE block_handler)
5675{
5676 VALUE (*func)(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5677 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5678 bool is_lambda, VALUE block_handler);
5679
5680 switch (vm_block_handler_type(block_handler)) {
5681 case block_handler_type_iseq: func = vm_invoke_iseq_block; break;
5682 case block_handler_type_ifunc: func = vm_invoke_ifunc_block; break;
5683 case block_handler_type_proc: func = vm_invoke_proc_block; break;
5684 case block_handler_type_symbol: func = vm_invoke_symbol_block; break;
5685 default: rb_bug("vm_invoke_block: unreachable");
5686 }
5687
5688 return func(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5689}
5690
5691static VALUE
5692vm_make_proc_with_iseq(const rb_iseq_t *blockiseq)
5693{
5694 const rb_execution_context_t *ec = GET_EC();
5695 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
5696 struct rb_captured_block *captured;
5697
5698 if (cfp == 0) {
5699 rb_bug("vm_make_proc_with_iseq: unreachable");
5700 }
5701
5702 captured = VM_CFP_TO_CAPTURED_BLOCK(cfp);
5703 captured->code.iseq = blockiseq;
5704
5705 return rb_vm_make_proc(ec, captured, rb_cProc);
5706}
5707
5708static VALUE
5709vm_once_exec(VALUE iseq)
5710{
5711 VALUE proc = vm_make_proc_with_iseq((rb_iseq_t *)iseq);
5712 return rb_proc_call_with_block(proc, 0, 0, Qnil);
5713}
5714
5715#define RUNNING_THREAD_ONCE_DONE ((rb_thread_t *)0x1)
5716
5717struct vm_once_wait_arg {
5718 rb_vm_t *vm;
5719 union iseq_inline_storage_entry *is;
5720};
5721
5722static void
5723vm_once_broadcast(rb_vm_t *vm)
5724{
5725 rb_native_mutex_lock(&vm->once_lock);
5726 rb_native_cond_broadcast(&vm->once_cond);
5727 rb_native_mutex_unlock(&vm->once_lock);
5728}
5729
5730static void *
5731vm_once_wait_no_gvl(void *ptr)
5732{
5733 struct vm_once_wait_arg *arg = (struct vm_once_wait_arg *)ptr;
5734 rb_vm_t *vm = arg->vm;
5735 rb_thread_t *running_th;
5736
5737 rb_native_mutex_lock(&vm->once_lock);
5738 running_th = rbimpl_atomic_ptr_load((void **)&arg->is->once.running_thread, RBIMPL_ATOMIC_ACQUIRE);
5739 if (running_th != NULL && running_th != RUNNING_THREAD_ONCE_DONE) {
5740 rb_native_cond_wait(&vm->once_cond, &vm->once_lock);
5741 }
5742 rb_native_mutex_unlock(&vm->once_lock);
5743 return NULL;
5744}
5745
5746static void
5747vm_once_wait_ubf(void *ptr)
5748{
5749 vm_once_broadcast((rb_vm_t *)ptr);
5750}
5751
5752static VALUE
5753vm_once_clear(VALUE data)
5754{
5755 union iseq_inline_storage_entry *is = (union iseq_inline_storage_entry *)data;
5756 rbimpl_atomic_ptr_store((volatile void **)&is->once.running_thread, NULL, RBIMPL_ATOMIC_RELEASE);
5757 vm_once_broadcast(GET_VM());
5758 return Qnil;
5759}
5760
5761/* defined insn */
5762
5763static bool
5764check_respond_to_missing(VALUE obj, VALUE v)
5765{
5766 VALUE args[2];
5767 VALUE r;
5768
5769 args[0] = obj; args[1] = Qfalse;
5770 r = rb_check_funcall(v, idRespond_to_missing, 2, args);
5771 if (!UNDEF_P(r) && RTEST(r)) {
5772 return true;
5773 }
5774 else {
5775 return false;
5776 }
5777}
5778
5779static bool
5780vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5781{
5782 VALUE klass;
5783 enum defined_type type = (enum defined_type)op_type;
5784
5785 switch (type) {
5786 case DEFINED_IVAR:
5787 return rb_ivar_defined(GET_SELF(), SYM2ID(obj));
5788 break;
5789 case DEFINED_GVAR:
5790 return rb_gvar_defined(SYM2ID(obj));
5791 break;
5792 case DEFINED_CVAR: {
5793 const rb_cref_t *cref = vm_get_cref(GET_EP());
5794 klass = vm_get_cvar_base(cref, GET_CFP(), 0);
5795 return rb_cvar_defined(klass, SYM2ID(obj));
5796 break;
5797 }
5798 case DEFINED_CONST:
5799 case DEFINED_CONST_FROM: {
5800 bool allow_nil = type == DEFINED_CONST;
5801 klass = v;
5802 return vm_get_ev_const(ec, klass, SYM2ID(obj), allow_nil, true);
5803 break;
5804 }
5805 case DEFINED_FUNC:
5806 klass = CLASS_OF(v);
5807 return rb_ec_obj_respond_to(ec, v, SYM2ID(obj), TRUE);
5808 break;
5809 case DEFINED_METHOD:{
5810 VALUE klass = CLASS_OF(v);
5811 const rb_callable_method_entry_t *cme = rb_callable_method_entry_with_refinements(klass, SYM2ID(obj), NULL);
5812
5813 if (cme) {
5814 switch (METHOD_ENTRY_VISI(cme)) {
5815 case METHOD_VISI_PRIVATE:
5816 break;
5817 case METHOD_VISI_PROTECTED:
5818 if (!rb_obj_is_kind_of(GET_SELF(), vm_defined_class_for_protected_call(cme))) {
5819 break;
5820 }
5821 case METHOD_VISI_PUBLIC:
5822 return true;
5823 break;
5824 default:
5825 rb_bug("vm_defined: unreachable: %u", (unsigned int)METHOD_ENTRY_VISI(cme));
5826 }
5827 }
5828 else {
5829 return check_respond_to_missing(obj, v);
5830 }
5831 break;
5832 }
5833 case DEFINED_YIELD:
5834 if (GET_BLOCK_HANDLER() != VM_BLOCK_HANDLER_NONE) {
5835 return true;
5836 }
5837 break;
5838 case DEFINED_ZSUPER:
5839 {
5840 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(GET_CFP());
5841
5842 if (me) {
5843 VALUE klass = vm_search_normal_superclass(me->defined_class);
5844 if (!klass) return false;
5845
5846 ID id = me->def->original_id;
5847
5848 return rb_method_boundp(klass, id, 0);
5849 }
5850 }
5851 break;
5852 case DEFINED_REF:
5853 return RTEST(vm_backref_defined(ec, GET_LEP(), FIX2INT(obj)));
5854 default:
5855 rb_bug("unimplemented defined? type (VM)");
5856 break;
5857 }
5858
5859 return false;
5860}
5861
5862bool
5863rb_vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5864{
5865 return vm_defined(ec, reg_cfp, op_type, obj, v);
5866}
5867
5868static const VALUE *
5869vm_get_ep(const VALUE *const reg_ep, rb_num_t lv)
5870{
5871 rb_num_t i;
5872 const VALUE *ep = reg_ep;
5873 for (i = 0; i < lv; i++) {
5874 ep = GET_PREV_EP(ep);
5875 }
5876 return ep;
5877}
5878
5879static VALUE
5880vm_get_special_object(const VALUE *const reg_ep,
5881 enum vm_special_object_type type)
5882{
5883 switch (type) {
5884 case VM_SPECIAL_OBJECT_VMCORE:
5885 return rb_mRubyVMFrozenCore;
5886 case VM_SPECIAL_OBJECT_CBASE:
5887 return vm_get_cbase(reg_ep);
5888 case VM_SPECIAL_OBJECT_CONST_BASE:
5889 return vm_get_const_base(reg_ep);
5890 default:
5891 rb_bug("putspecialobject insn: unknown value_type %d", type);
5892 }
5893}
5894
5895// ZJIT implementation is using the C function
5896// and needs to call a non-static function
5897VALUE
5898rb_vm_get_special_object(const VALUE *reg_ep, enum vm_special_object_type type)
5899{
5900 return vm_get_special_object(reg_ep, type);
5901}
5902
5903static VALUE
5904vm_concat_array(VALUE ary1, VALUE ary2st)
5905{
5906 const VALUE ary2 = ary2st;
5907 VALUE tmp1 = rb_check_to_array(ary1);
5908 VALUE tmp2 = rb_check_to_array(ary2);
5909
5910 if (NIL_P(tmp1)) {
5911 tmp1 = rb_ary_new3(1, ary1);
5912 }
5913 if (tmp1 == ary1) {
5914 tmp1 = rb_ary_dup(ary1);
5915 }
5916
5917 if (NIL_P(tmp2)) {
5918 return rb_ary_push(tmp1, ary2);
5919 }
5920 else {
5921 return rb_ary_concat(tmp1, tmp2);
5922 }
5923}
5924
5925static VALUE
5926vm_concat_to_array(VALUE ary1, VALUE ary2st)
5927{
5928 /* ary1 must be a newly created array */
5929 const VALUE ary2 = ary2st;
5930
5931 if (NIL_P(ary2)) return ary1;
5932
5933 VALUE tmp2 = rb_check_to_array(ary2);
5934
5935 if (NIL_P(tmp2)) {
5936 return rb_ary_push(ary1, ary2);
5937 }
5938 else {
5939 return rb_ary_concat(ary1, tmp2);
5940 }
5941}
5942
5943// YJIT implementation is using the C function
5944// and needs to call a non-static function
5945VALUE
5946rb_vm_concat_array(VALUE ary1, VALUE ary2st)
5947{
5948 return vm_concat_array(ary1, ary2st);
5949}
5950
5951VALUE
5952rb_vm_concat_to_array(VALUE ary1, VALUE ary2st)
5953{
5954 return vm_concat_to_array(ary1, ary2st);
5955}
5956
5957static VALUE
5958vm_splat_array(VALUE flag, VALUE ary)
5959{
5960 if (NIL_P(ary)) {
5961 return RTEST(flag) ? rb_ary_new() : rb_cArray_empty_frozen;
5962 }
5963 VALUE tmp = rb_check_to_array(ary);
5964 if (NIL_P(tmp)) {
5965 return rb_ary_new3(1, ary);
5966 }
5967 else if (RTEST(flag)) {
5968 return rb_ary_dup(tmp);
5969 }
5970 else {
5971 return tmp;
5972 }
5973}
5974
5975// YJIT implementation is using the C function
5976// and needs to call a non-static function
5977VALUE
5978rb_vm_splat_array(VALUE flag, VALUE ary)
5979{
5980 return vm_splat_array(flag, ary);
5981}
5982
5983static VALUE
5984vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
5985{
5986 enum vm_check_match_type type = ((int)flag) & VM_CHECKMATCH_TYPE_MASK;
5987
5988 if (flag & VM_CHECKMATCH_ARRAY) {
5989 rb_len_t i;
5990 const rb_len_t n = RARRAY_LEN(pattern);
5991
5992 for (i = 0; i < n; i++) {
5993 VALUE v = RARRAY_AREF(pattern, i);
5994 VALUE c = check_match(ec, v, target, type);
5995
5996 if (RTEST(c)) {
5997 return c;
5998 }
5999 }
6000 return Qfalse;
6001 }
6002 else {
6003 return check_match(ec, pattern, target, type);
6004 }
6005}
6006
6007VALUE
6008rb_vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
6009{
6010 return vm_check_match(ec, target, pattern, flag);
6011}
6012
6013static VALUE
6014vm_check_keyword(lindex_t bits, lindex_t idx, const VALUE *ep)
6015{
6016 const VALUE kw_bits = *(ep - bits);
6017
6018 if (FIXNUM_P(kw_bits)) {
6019 unsigned int b = (unsigned int)FIX2ULONG(kw_bits);
6020 if ((idx < VM_KW_SPECIFIED_BITS_MAX) && (b & (0x01 << idx)))
6021 return Qfalse;
6022 }
6023 else {
6024 VM_ASSERT(rb_set_p(kw_bits), "%s", rb_obj_info(kw_bits));
6025 if (rb_set_lookup(kw_bits, INT2FIX(idx))) return Qfalse;
6026 }
6027 return Qtrue;
6028}
6029
6030static void
6031vm_dtrace(rb_event_flag_t flag, rb_execution_context_t *ec)
6032{
6033 if (RUBY_DTRACE_METHOD_ENTRY_ENABLED() ||
6034 RUBY_DTRACE_METHOD_RETURN_ENABLED() ||
6035 RUBY_DTRACE_CMETHOD_ENTRY_ENABLED() ||
6036 RUBY_DTRACE_CMETHOD_RETURN_ENABLED()) {
6037
6038 switch (flag) {
6039 case RUBY_EVENT_CALL:
6040 RUBY_DTRACE_METHOD_ENTRY_HOOK(ec, 0, 0);
6041 return;
6042 case RUBY_EVENT_C_CALL:
6043 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, 0, 0);
6044 return;
6045 case RUBY_EVENT_RETURN:
6046 RUBY_DTRACE_METHOD_RETURN_HOOK(ec, 0, 0);
6047 return;
6048 case RUBY_EVENT_C_RETURN:
6049 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, 0, 0);
6050 return;
6051 }
6052 }
6053}
6054
6055static VALUE
6056vm_const_get_under(ID id, rb_num_t flags, VALUE cbase)
6057{
6058 if (!rb_const_defined_at(cbase, id)) {
6059 return 0;
6060 }
6061 else if (VM_DEFINECLASS_SCOPED_P(flags)) {
6062 return rb_public_const_get_at(cbase, id);
6063 }
6064 else {
6065 return rb_const_get_at(cbase, id);
6066 }
6067}
6068
6069static VALUE
6070vm_check_if_class(ID id, rb_num_t flags, VALUE super, VALUE klass)
6071{
6072 if (!RB_TYPE_P(klass, T_CLASS)) {
6073 return 0;
6074 }
6075 else if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags)) {
6076 VALUE tmp = rb_class_real(RCLASS_SUPER(klass));
6077
6078 if (tmp != super) {
6079 rb_raise(rb_eTypeError,
6080 "superclass mismatch for class %"PRIsVALUE"",
6081 rb_id2str(id));
6082 }
6083 else {
6084 return klass;
6085 }
6086 }
6087 else {
6088 return klass;
6089 }
6090}
6091
6092static VALUE
6093vm_check_if_module(ID id, VALUE mod)
6094{
6095 if (!RB_TYPE_P(mod, T_MODULE)) {
6096 return 0;
6097 }
6098 else {
6099 return mod;
6100 }
6101}
6102
6103static VALUE
6104declare_under(ID id, VALUE cbase, VALUE c)
6105{
6106 rb_set_class_path_string(c, cbase, rb_id2str(id));
6107 rb_const_set(cbase, id, c);
6108 return c;
6109}
6110
6111static VALUE
6112vm_declare_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
6113{
6114 /* new class declaration */
6115 VALUE s = VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) ? super : rb_cObject;
6116 VALUE c = declare_under(id, cbase, rb_define_class_id(id, s));
6117 rb_class_inherited(s, c);
6118 return c;
6119}
6120
6121static VALUE
6122vm_declare_module(ID id, VALUE cbase)
6123{
6124 /* new module declaration */
6125 return declare_under(id, cbase, rb_module_new());
6126}
6127
6128NORETURN(static void unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old));
6129static void
6130unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old)
6131{
6132 VALUE name = rb_id2str(id);
6133 VALUE message = rb_sprintf("%"PRIsVALUE" is not a %s",
6134 name, type);
6135 VALUE location = rb_const_source_location_at(cbase, id);
6136 if (!NIL_P(location)) {
6137 rb_str_catf(message, "\n%"PRIsVALUE":%"PRIsVALUE":"
6138 " previous definition of %"PRIsVALUE" was here",
6139 rb_ary_entry(location, 0), rb_ary_entry(location, 1), name);
6140 }
6141 rb_exc_raise(rb_exc_new_str(rb_eTypeError, message));
6142}
6143
6144static VALUE
6145vm_define_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
6146{
6147 VALUE klass;
6148
6149 if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) && !RB_TYPE_P(super, T_CLASS)) {
6150 rb_raise(rb_eTypeError,
6151 "superclass must be an instance of Class (given an instance of %"PRIsVALUE")",
6152 rb_obj_class(super));
6153 }
6154
6155 vm_check_if_namespace(cbase);
6156
6157 /* find klass */
6158 rb_autoload_load(cbase, id);
6159
6160 if ((klass = vm_const_get_under(id, flags, cbase)) != 0) {
6161 if (!vm_check_if_class(id, flags, super, klass))
6162 unmatched_redefinition("class", cbase, id, klass);
6163 return klass;
6164 }
6165 else {
6166 return vm_declare_class(id, flags, cbase, super);
6167 }
6168}
6169
6170static VALUE
6171vm_define_module(ID id, rb_num_t flags, VALUE cbase)
6172{
6173 VALUE mod;
6174
6175 vm_check_if_namespace(cbase);
6176 if ((mod = vm_const_get_under(id, flags, cbase)) != 0) {
6177 if (!vm_check_if_module(id, mod))
6178 unmatched_redefinition("module", cbase, id, mod);
6179 return mod;
6180 }
6181 else {
6182 return vm_declare_module(id, cbase);
6183 }
6184}
6185
6186static VALUE
6187vm_find_or_create_class_by_id(ID id,
6188 rb_num_t flags,
6189 VALUE cbase,
6190 VALUE super)
6191{
6192 rb_vm_defineclass_type_t type = VM_DEFINECLASS_TYPE(flags);
6193
6194 switch (type) {
6195 case VM_DEFINECLASS_TYPE_CLASS:
6196 /* classdef returns class scope value */
6197 return vm_define_class(id, flags, cbase, super);
6198
6199 case VM_DEFINECLASS_TYPE_SINGLETON_CLASS:
6200 /* classdef returns class scope value */
6201 return rb_singleton_class(cbase);
6202
6203 case VM_DEFINECLASS_TYPE_MODULE:
6204 /* classdef returns class scope value */
6205 return vm_define_module(id, flags, cbase);
6206
6207 default:
6208 rb_bug("unknown defineclass type: %d", (int)type);
6209 }
6210}
6211
6212static rb_method_visibility_t
6213vm_scope_visibility_get(const rb_execution_context_t *ec)
6214{
6215 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
6216
6217 if (!vm_env_cref_by_cref(cfp->ep)) {
6218 return METHOD_VISI_PUBLIC;
6219 }
6220 else {
6221 return CREF_SCOPE_VISI(vm_ec_cref(ec))->method_visi;
6222 }
6223}
6224
6225static int
6226vm_scope_module_func_check(const rb_execution_context_t *ec)
6227{
6228 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
6229
6230 if (!vm_env_cref_by_cref(cfp->ep)) {
6231 return FALSE;
6232 }
6233 else {
6234 return CREF_SCOPE_VISI(vm_ec_cref(ec))->module_func;
6235 }
6236}
6237
6238static void
6239vm_define_method(const rb_execution_context_t *ec, VALUE obj, ID id, VALUE iseqval, int is_singleton)
6240{
6241 VALUE klass;
6242 rb_method_visibility_t visi;
6243 rb_cref_t *cref = vm_ec_cref(ec);
6244
6245 if (is_singleton) {
6246 klass = rb_singleton_class(obj); /* class and frozen checked in this API */
6247 visi = METHOD_VISI_PUBLIC;
6248 }
6249 else {
6250 klass = CREF_CLASS_FOR_DEFINITION(cref);
6251 visi = vm_scope_visibility_get(ec);
6252 }
6253
6254 if (NIL_P(klass)) {
6255 rb_raise(rb_eTypeError, "no class/module to add method");
6256 }
6257
6258 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, visi);
6259 // Set max_iv_count on klasses based on number of ivar sets that are in the initialize method
6260 if (id == idInitialize && RB_TYPE_P(klass, T_CLASS) &&
6261 !RCLASS_SINGLETON_P(klass) && !RCLASS_EXPECT_NO_IVAR(klass) &&
6262 (rb_get_alloc_func(klass) == rb_class_allocate_instance)) {
6263 RCLASS_SET_MAX_IV_COUNT(klass, rb_estimate_iv_count(klass, (const rb_iseq_t *)iseqval));
6264 }
6265
6266 if (!is_singleton && vm_scope_module_func_check(ec)) {
6267 klass = rb_singleton_class(klass);
6268 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, METHOD_VISI_PUBLIC);
6269 }
6270}
6271
6272// Return the untagged block handler:
6273// * If it's VM_BLOCK_HANDLER_NONE, return nil
6274// * If it's an ISEQ or an IFUNC, fetch it from its rb_captured_block
6275// * If it's a PROC or SYMBOL, return it as is
6276VALUE
6277rb_vm_untag_block_handler(VALUE block_handler)
6278{
6279 if (VM_BLOCK_HANDLER_NONE == block_handler) return Qnil;
6280
6281 switch (vm_block_handler_type(block_handler)) {
6282 case block_handler_type_iseq:
6283 case block_handler_type_ifunc: {
6284 struct rb_captured_block *captured = VM_TAGGED_PTR_REF(block_handler, 0x03);
6285 return captured->code.val;
6286 }
6287 case block_handler_type_proc:
6288 case block_handler_type_symbol:
6289 return block_handler;
6290 default:
6291 rb_bug("rb_vm_untag_block_handler: unreachable");
6292 }
6293}
6294
6295VALUE
6296rb_vm_get_untagged_block_handler(rb_control_frame_t *reg_cfp)
6297{
6298 return rb_vm_untag_block_handler(VM_CF_BLOCK_HANDLER(reg_cfp));
6299}
6300
6301static VALUE
6302vm_invokeblock_i(struct rb_execution_context_struct *ec,
6303 struct rb_control_frame_struct *reg_cfp,
6304 struct rb_calling_info *calling)
6305{
6306 const struct rb_callinfo *ci = calling->cd->ci;
6307 VALUE block_handler = VM_CF_BLOCK_HANDLER(GET_CFP());
6308
6309 if (block_handler == VM_BLOCK_HANDLER_NONE) {
6310 rb_vm_localjump_error("no block given (yield)", Qnil, 0);
6311 }
6312 else {
6313 return vm_invoke_block(ec, GET_CFP(), calling, ci, false, block_handler);
6314 }
6315}
6316
6317enum method_explorer_type {
6318 mexp_search_method,
6319 mexp_search_invokeblock,
6320 mexp_search_super,
6321};
6322
6323ALWAYS_INLINE(static VALUE vm_sendish(struct rb_execution_context_struct *ec,
6324 struct rb_control_frame_struct *reg_cfp,
6325 struct rb_call_data *cd, VALUE block_handler,
6326 enum method_explorer_type method_explorer));
6327static inline VALUE
6328vm_sendish(
6329 struct rb_execution_context_struct *ec,
6330 struct rb_control_frame_struct *reg_cfp,
6331 struct rb_call_data *cd,
6332 VALUE block_handler,
6333 enum method_explorer_type method_explorer
6334) {
6335 VALUE val = Qundef;
6336 const struct rb_callinfo *ci = cd->ci;
6337 const struct rb_callcache *cc;
6338 int argc = vm_ci_argc(ci);
6339 VALUE recv = TOPN(argc);
6340 struct rb_calling_info calling = {
6341 .block_handler = block_handler,
6342 .kw_splat = IS_ARGS_KW_SPLAT(ci) > 0,
6343 .recv = recv,
6344 .argc = argc,
6345 .cd = cd,
6346 };
6347
6348 switch (method_explorer) {
6349 case mexp_search_method:
6350 calling.cc = cc = vm_search_method_fastpath(reg_cfp, cd, CLASS_OF(recv));
6351 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6352 break;
6353 case mexp_search_super:
6354 calling.cc = cc = vm_search_super_method(reg_cfp, cd, recv);
6355 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6356 break;
6357 case mexp_search_invokeblock:
6358 val = vm_invokeblock_i(ec, GET_CFP(), &calling);
6359 break;
6360 }
6361 return val;
6362}
6363
6364VALUE
6365rb_vm_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6366{
6367 stack_check(ec);
6368 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), cd->ci, blockiseq, false);
6369 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6370 VM_EXEC(ec, val);
6371 return val;
6372}
6373
6374// Fallback for YJIT/ZJIT, not used by the interpreter
6375VALUE
6376rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6377{
6378 stack_check(ec);
6379
6380 struct rb_forwarding_call_data adjusted_cd;
6381 struct rb_callinfo adjusted_ci;
6382
6383 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, false, &adjusted_cd, &adjusted_ci);
6384
6385 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method);
6386
6387 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6388 RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc);
6389 }
6390
6391 VM_EXEC(ec, val);
6392 return val;
6393}
6394
6395VALUE
6396rb_vm_opt_send_without_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6397{
6398 stack_check(ec);
6399 VALUE bh = VM_BLOCK_HANDLER_NONE;
6400 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6401 VM_EXEC(ec, val);
6402 return val;
6403}
6404
6405VALUE
6406rb_vm_invokesuper(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6407{
6408 stack_check(ec);
6409
6410 struct rb_callinfo adjusted_ci = VM_CI_ON_STACK(vm_ci_mid(cd->ci),
6411 vm_ci_flag(cd->ci),
6412 vm_ci_argc(cd->ci),
6413 vm_ci_kwarg(cd->ci));
6414 const struct rb_callcache *original_cc = rbimpl_atomic_ptr_load((void **)&cd->cc, RBIMPL_ATOMIC_ACQUIRE);
6415 struct rb_call_data adjusted_cd = {
6416 .ci = &adjusted_ci,
6417 .cc = original_cc,
6418 };
6419
6420 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), adjusted_cd.ci, blockiseq, true);
6421 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd, bh, mexp_search_super);
6422
6423 if (original_cc != adjusted_cd.cc && vm_cc_markable(adjusted_cd.cc)) {
6424 rbimpl_atomic_ptr_store((volatile void **)&cd->cc, (void *)adjusted_cd.cc, RBIMPL_ATOMIC_RELEASE);
6425 RB_OBJ_WRITTEN(CFP_ISEQ(GET_CFP()), Qundef, adjusted_cd.cc);
6426 }
6427
6428 VM_EXEC(ec, val);
6429 return val;
6430}
6431
6432// Fallback for YJIT/ZJIT, not used by the interpreter
6433VALUE
6434rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6435{
6436 stack_check(ec);
6437 struct rb_forwarding_call_data adjusted_cd;
6438 struct rb_callinfo adjusted_ci;
6439
6440 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, true, &adjusted_cd, &adjusted_ci);
6441
6442 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super);
6443
6444 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6445 RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc);
6446 }
6447
6448 VM_EXEC(ec, val);
6449 return val;
6450}
6451
6452VALUE
6453rb_vm_invokeblock(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6454{
6455 stack_check(ec);
6456 VALUE bh = VM_BLOCK_HANDLER_NONE;
6457 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_invokeblock);
6458 VM_EXEC(ec, val);
6459 return val;
6460}
6461
6462/* object.c */
6463VALUE rb_nil_to_s(VALUE);
6464VALUE rb_true_to_s(VALUE);
6465VALUE rb_false_to_s(VALUE);
6466/* numeric.c */
6467VALUE rb_int_to_s(int argc, VALUE *argv, VALUE x);
6468VALUE rb_fix_to_s(VALUE);
6469/* variable.c */
6470VALUE rb_mod_to_s(VALUE);
6471VALUE rb_mod_name(VALUE);
6472
6473static VALUE
6474vm_objtostring(struct rb_control_frame_struct *reg_cfp, VALUE recv, CALL_DATA cd)
6475{
6476 int type = TYPE(recv);
6477 if (type == T_STRING) {
6478 return recv;
6479 }
6480
6481 const struct rb_callable_method_entry_struct *cme = vm_search_method(reg_cfp, cd, recv);
6482
6483 switch (type) {
6484 case T_SYMBOL:
6485 if (check_method_basic_definition(cme)) {
6486 return rb_sym2str(recv);
6487 }
6488 break;
6489 case T_MODULE:
6490 case T_CLASS:
6491 if (check_cfunc(cme, rb_mod_to_s)) {
6492 // rb_mod_to_s() allocates a mutable string, but since we are only
6493 // going to use this string for interpolation, it's fine to use the
6494 // frozen string.
6495 VALUE val = rb_mod_name(recv);
6496 if (NIL_P(val)) {
6497 val = rb_mod_to_s(recv);
6498 }
6499 return val;
6500 }
6501 break;
6502 case T_NIL:
6503 if (check_cfunc(cme, rb_nil_to_s)) {
6504 return rb_nil_to_s(recv);
6505 }
6506 break;
6507 case T_TRUE:
6508 if (check_cfunc(cme, rb_true_to_s)) {
6509 return rb_true_to_s(recv);
6510 }
6511 break;
6512 case T_FALSE:
6513 if (check_cfunc(cme, rb_false_to_s)) {
6514 return rb_false_to_s(recv);
6515 }
6516 break;
6517 case T_FIXNUM:
6518 if (check_cfunc(cme, rb_int_to_s)) {
6519 return rb_fix_to_s(recv);
6520 }
6521 break;
6522 }
6523 return Qundef;
6524}
6525
6526// ZJIT implementation is using the C function
6527// and needs to call a non-static function
6528VALUE
6529rb_vm_objtostring(struct rb_control_frame_struct *reg_cfp, VALUE recv, CALL_DATA cd)
6530{
6531 return vm_objtostring(reg_cfp, recv, cd);
6532}
6533
6534static VALUE
6535vm_opt_ary_freeze(VALUE ary, int bop, ID id)
6536{
6537 if (BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
6538 return ary;
6539 }
6540 else {
6541 return Qundef;
6542 }
6543}
6544
6545static VALUE
6546vm_opt_hash_freeze(VALUE hash, int bop, ID id)
6547{
6548 if (BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
6549 return hash;
6550 }
6551 else {
6552 return Qundef;
6553 }
6554}
6555
6556static VALUE
6557vm_opt_str_freeze(VALUE str, int bop, ID id)
6558{
6559 if (BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
6560 return str;
6561 }
6562 else {
6563 return Qundef;
6564 }
6565}
6566
6567/* this macro is mandatory to use OPTIMIZED_CMP. What a design! */
6568#define id_cmp idCmp
6569
6570static VALUE
6571vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6572{
6573 if (OP_UNREDEFINED_P(INCLUDE_P, ARRAY)) {
6574 return rb_ary_includes(ary, target);
6575 }
6576 else {
6577 VALUE args[1] = {target};
6578
6579 // duparray
6580 RUBY_DTRACE_CREATE_HOOK(ARRAY, RARRAY_LEN(ary));
6581 VALUE dupary = rb_ary_resurrect(ary);
6582
6583 return rb_vm_call_with_refinements(ec, dupary, idIncludeP, 1, args, RB_NO_KEYWORDS);
6584 }
6585}
6586
6587VALUE
6588rb_vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6589{
6590 return vm_opt_duparray_include_p(ec, ary, target);
6591}
6592
6593static VALUE
6594vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6595{
6596 if (OP_UNREDEFINED_P(MAX, ARRAY)) {
6597 if (array_len == 0) {
6598 return Qnil;
6599 }
6600 else {
6601 VALUE result = *ptr;
6602 rb_snum_t i = array_len - 1;
6603 while (i-- > 0) {
6604 const VALUE v = *++ptr;
6605 if (OPTIMIZED_CMP(v, result) > 0) {
6606 result = v;
6607 }
6608 }
6609 return result;
6610 }
6611 }
6612 else {
6613 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMax, 0, NULL, RB_NO_KEYWORDS);
6614 }
6615}
6616
6617VALUE
6618rb_vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6619{
6620 return vm_opt_newarray_max(ec, array_len, ptr);
6621}
6622
6623static VALUE
6624vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6625{
6626 if (OP_UNREDEFINED_P(MIN, ARRAY)) {
6627 if (array_len == 0) {
6628 return Qnil;
6629 }
6630 else {
6631 VALUE result = *ptr;
6632 rb_snum_t i = array_len - 1;
6633 while (i-- > 0) {
6634 const VALUE v = *++ptr;
6635 if (OPTIMIZED_CMP(v, result) < 0) {
6636 result = v;
6637 }
6638 }
6639 return result;
6640 }
6641 }
6642 else {
6643 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMin, 0, NULL, RB_NO_KEYWORDS);
6644 }
6645}
6646
6647VALUE
6648rb_vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6649{
6650 return vm_opt_newarray_min(ec, array_len, ptr);
6651}
6652
6653static VALUE
6654vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6655{
6656 // If Array#hash is _not_ monkeypatched, use the optimized call
6657 if (OP_UNREDEFINED_P(HASH, ARRAY)) {
6658 return rb_ary_hash_values(array_len, ptr);
6659 }
6660 else {
6661 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idHash, 0, NULL, RB_NO_KEYWORDS);
6662 }
6663}
6664
6665VALUE
6666rb_vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6667{
6668 return vm_opt_newarray_hash(ec, array_len, ptr);
6669}
6670
6671VALUE rb_setup_fake_ary(struct RArray *fake_ary, const VALUE *list, rb_len_t len);
6672VALUE rb_ec_pack_ary(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer);
6673
6674static VALUE
6675vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6676{
6677 if (OP_UNREDEFINED_P(INCLUDE_P, ARRAY)) {
6678 struct RArray fake_ary = {RBASIC_INIT};
6679 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6680 return rb_ary_includes(ary, target);
6681 }
6682 else {
6683 VALUE args[1] = {target};
6684 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idIncludeP, 1, args, RB_NO_KEYWORDS);
6685 }
6686}
6687
6688VALUE
6689rb_vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6690{
6691 return vm_opt_newarray_include_p(ec, array_len, ptr, target);
6692}
6693
6694static VALUE
6695vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6696{
6697 if (OP_UNREDEFINED_P(PACK, ARRAY)) {
6698 struct RArray fake_ary = {RBASIC_INIT};
6699 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6700 return rb_ec_pack_ary(ec, ary, fmt, (UNDEF_P(buffer) ? Qnil : buffer));
6701 }
6702 else {
6703 // The opt_newarray_send insn drops the keyword args so we need to rebuild them.
6704 // Setup an array with room for keyword hash.
6705 VALUE args[2];
6706 args[0] = fmt;
6707 int kw_splat = RB_NO_KEYWORDS;
6708 int argc = 1;
6709
6710 if (!UNDEF_P(buffer)) {
6711 args[1] = rb_hash_new_capa(1);
6712 rb_hash_aset(args[1], ID2SYM(idBuffer), buffer);
6713 kw_splat = RB_PASS_KEYWORDS;
6714 argc++;
6715 }
6716
6717 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idPack, argc, args, kw_splat);
6718 }
6719}
6720
6721VALUE
6722rb_vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6723{
6724 return vm_opt_newarray_pack_buffer(ec, array_len, ptr, fmt, buffer);
6725}
6726
6727
6728#undef id_cmp
6729
6730static void
6731vm_track_constant_cache(ID id, void *ic)
6732{
6733 rb_vm_t *vm = GET_VM();
6734 struct rb_id_table *const_cache = &vm->constant_cache;
6735 VALUE lookup_result;
6736 set_table *ics;
6737
6738 if (rb_id_table_lookup(const_cache, id, &lookup_result)) {
6739 ics = (set_table *)lookup_result;
6740 }
6741 else {
6742 ics = set_init_numtable();
6743 rb_id_table_insert(const_cache, id, (VALUE)ics);
6744 }
6745
6746 /* The call below to st_insert could allocate which could trigger a GC.
6747 * If it triggers a GC, it may free an iseq that also holds a cache to this
6748 * constant. If that iseq is the last iseq with a cache to this constant, then
6749 * it will free this ST table, which would cause an use-after-free during this
6750 * st_insert.
6751 *
6752 * So to fix this issue, we store the ID that is currently being inserted
6753 * and, in remove_from_constant_cache, we don't free the ST table for ID
6754 * equal to this one.
6755 *
6756 * See [Bug #20921].
6757 */
6758 vm->inserting_constant_cache_id = id;
6759
6760 set_insert(ics, (st_data_t)ic);
6761
6762 vm->inserting_constant_cache_id = (ID)0;
6763}
6764
6765static void
6766vm_ic_track_const_chain(rb_control_frame_t *cfp, IC ic, const ID *segments)
6767{
6768 RB_VM_LOCKING() {
6769 for (int i = 0; segments[i]; i++) {
6770 ID id = segments[i];
6771 if (id == idNULL) continue;
6772 vm_track_constant_cache(id, ic);
6773 }
6774 }
6775}
6776
6777// For JIT inlining
6778static inline bool
6779vm_inlined_ic_hit_p(VALUE flags, VALUE value, const rb_cref_t *ic_cref, rb_serial_t ractor_id, const VALUE *reg_ep)
6780{
6781 // Not rb_ractor_main_p(): an owner Ractor may now cache an unshareable constant
6782 // of its own class, and only it may be handed that value back.
6783 if ((flags & IMEMO_CONST_CACHE_SHAREABLE) || ractor_id == rb_ractor_id(GET_RACTOR())) {
6784 VM_ASSERT(ractor_incidental_shareable_p(flags & IMEMO_CONST_CACHE_SHAREABLE, value));
6785
6786 return (ic_cref == NULL || // no need to check CREF
6787 ic_cref == vm_get_cref(reg_ep));
6788 }
6789 return false;
6790}
6791
6792static bool
6793vm_ic_hit_p(const struct iseq_inline_constant_cache_entry *ice, const VALUE *reg_ep)
6794{
6795 VM_ASSERT(IMEMO_TYPE_P(ice, imemo_constcache));
6796 return vm_inlined_ic_hit_p(ice->flags, ice->value, ice->ic_cref, ice->ractor_id, reg_ep);
6797}
6798
6799// YJIT needs this function to never allocate and never raise
6800bool
6801rb_vm_ic_hit_p(IC ic, const VALUE *reg_ep)
6802{
6803 return ic->entry && vm_ic_hit_p(ic->entry, reg_ep);
6804}
6805
6806static void
6807vm_ic_update(const rb_iseq_t *iseq, IC ic, VALUE val, const VALUE *reg_ep, const VALUE *pc)
6808{
6809 if (ruby_vm_const_missing_count > 0) {
6810 ruby_vm_const_missing_count = 0;
6811 ic->entry = NULL;
6812 return;
6813 }
6814
6815 struct iseq_inline_constant_cache_entry *ice = SHAREABLE_IMEMO_NEW(struct iseq_inline_constant_cache_entry, imemo_constcache, 0);
6816 RB_OBJ_WRITE(ice, &ice->value, val);
6817 ice->ic_cref = vm_get_const_key_cref(reg_ep);
6818 ice->ractor_id = rb_ractor_id(GET_RACTOR());
6819
6820 if (rb_ractor_shareable_p(val)) {
6821 RUBY_ASSERT((rb_gc_verify_shareable(val), 1));
6822 ice->flags |= IMEMO_CONST_CACHE_SHAREABLE;
6823 }
6824 RB_OBJ_WRITE(iseq, &ic->entry, ice);
6825 RUBY_ASSERT(pc >= ISEQ_BODY(iseq)->iseq_encoded);
6826 unsigned pos = (unsigned)(pc - ISEQ_BODY(iseq)->iseq_encoded);
6827 rb_yjit_constant_ic_update(iseq, ic, pos);
6828}
6829
6830VALUE
6831rb_vm_opt_getconstant_path(rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, IC ic)
6832{
6833 VALUE val;
6834 const ID *segments = ic->segments;
6835 struct iseq_inline_constant_cache_entry *ice = ic->entry;
6836
6837 if (ice && vm_ic_hit_p(ice, GET_EP())) {
6838 val = ice->value;
6839
6840 // On a non-main ractor another ractor can concurrently reassign a
6841 // shareable constant (which invalidates ICs asynchronously), so the
6842 // cached-but-still-shareable value may legitimately differ from a
6843 // freshly recomputed chain. Only assert equality when single-ractor.
6844 VM_ASSERT(val == vm_get_ev_const_chain(ec, segments) || rb_multi_ractor_p());
6845 }
6846 else {
6847 ruby_vm_constant_cache_misses++;
6848 val = vm_get_ev_const_chain(ec, segments);
6849 vm_ic_track_const_chain(GET_CFP(), ic, segments);
6850 // Undo the PC increment to get the address to this instruction
6851 // INSN_ATTR(width) == 2
6852 vm_ic_update(CFP_ISEQ(GET_CFP()), ic, val, GET_EP(), CFP_PC(GET_CFP()) - 2);
6853 }
6854 return val;
6855}
6856
6857// Return true if the once value is already computed and set *result to the value.
6858// Otherwise, return false.
6859// Used for ZJIT. Keep in sync with `vm_once_dispatch` below.
6860bool
6861rb_vm_once_done_value(ISE is, VALUE *result)
6862{
6863 rb_thread_t *running_th = rbimpl_atomic_ptr_load((void**)&is->once.running_thread, RBIMPL_ATOMIC_ACQUIRE);
6864 if (running_th == RUNNING_THREAD_ONCE_DONE) {
6865 *result = is->once.value;
6866 return true;
6867 }
6868 return false;
6869}
6870
6871static VALUE
6872vm_once_dispatch(rb_execution_context_t *ec, ISEQ iseq, ISE is)
6873{
6874 rb_thread_t *th = rb_ec_thread_ptr(ec);
6875 rb_thread_t *running_th;
6876
6877 again:
6878 running_th = rbimpl_atomic_ptr_load((void**)&is->once.running_thread, RBIMPL_ATOMIC_ACQUIRE);
6879 if (running_th == RUNNING_THREAD_ONCE_DONE) {
6880 return is->once.value;
6881 }
6882 else if (running_th == NULL) {
6883 VALUE val = Qundef;
6884 enum ruby_tag_type state;
6885 if (rbimpl_atomic_ptr_cas((void**)&is->once.running_thread, running_th, th, RBIMPL_ATOMIC_RELEASE, RBIMPL_ATOMIC_RELAXED) != running_th) {
6886 goto again;
6887 }
6888 EC_PUSH_TAG(ec);
6889 if ((state = EC_EXEC_TAG()) == TAG_NONE) {
6890 val = vm_once_exec((VALUE)iseq);
6891 }
6892 EC_POP_TAG();
6893 if (state != TAG_NONE) {
6894 vm_once_clear((VALUE)is);
6895 EC_JUMP_TAG(ec, state);
6896 }
6897
6898 // TODO: confirm that it is shareable
6899 if (RB_FL_ABLE(val)) {
6900 RB_OBJ_SET_SHAREABLE(val);
6901 }
6902
6903 RB_OBJ_WRITE(CFP_ISEQ(ec->cfp), &is->once.value, val);
6904
6905 rbimpl_atomic_ptr_store((volatile void**)&is->once.running_thread, RUNNING_THREAD_ONCE_DONE, RBIMPL_ATOMIC_RELEASE); /* success */
6906 vm_once_broadcast(rb_ec_vm_ptr(ec));
6907 return val;
6908 }
6909 else if (running_th == th) {
6910 /* recursive once */
6911 return vm_once_exec((VALUE)iseq);
6912 }
6913 else {
6914 /* wait for the winner to finish */
6915 struct vm_once_wait_arg arg = { .vm = rb_ec_vm_ptr(ec), .is = is };
6916 rb_nogvl(vm_once_wait_no_gvl, &arg, vm_once_wait_ubf, arg.vm,
6917 RB_NOGVL_UBF_ASYNC_SAFE | RB_NOGVL_INTR_FAIL);
6918 RUBY_VM_CHECK_INTS(ec);
6919 goto again;
6920 }
6921}
6922
6923static OFFSET
6924vm_case_dispatch(CDHASH hash, OFFSET else_offset, VALUE key)
6925{
6926 switch (OBJ_BUILTIN_TYPE(key)) {
6927 case -1:
6928 case T_FLOAT:
6929 case T_SYMBOL:
6930 case T_BIGNUM:
6931 case T_STRING:
6932 if (BASIC_OP_UNREDEFINED_P(BOP_EQQ,
6933 SYMBOL_REDEFINED_OP_FLAG |
6934 INTEGER_REDEFINED_OP_FLAG |
6935 FLOAT_REDEFINED_OP_FLAG |
6936 NIL_REDEFINED_OP_FLAG |
6937 TRUE_REDEFINED_OP_FLAG |
6938 FALSE_REDEFINED_OP_FLAG |
6939 STRING_REDEFINED_OP_FLAG)) {
6940 st_data_t val;
6941 if (RB_FLOAT_TYPE_P(key)) {
6942 double kval = RFLOAT_VALUE(key);
6943 if (!isinf(kval) && modf(kval, &kval) == 0.0) {
6944 key = FIXABLE(kval) ? LONG2FIX((long)kval) : rb_dbl2big(kval);
6945 }
6946 }
6947 if (st_lookup(rb_imemo_cdhash_tbl(hash), key, &val)) {
6948 return (VALUE)val;
6949 }
6950 else {
6951 return else_offset;
6952 }
6953 }
6954 }
6955 return 0;
6956}
6957
6958NORETURN(static void
6959 vm_stack_consistency_error(const rb_execution_context_t *ec,
6960 const rb_control_frame_t *,
6961 const VALUE *));
6962static void
6963vm_stack_consistency_error(const rb_execution_context_t *ec,
6964 const rb_control_frame_t *cfp,
6965 const VALUE *bp)
6966{
6967 const ptrdiff_t nsp = VM_SP_CNT(ec, cfp->sp);
6968 const ptrdiff_t nbp = VM_SP_CNT(ec, bp);
6969 static const char stack_consistency_error[] =
6970 "Stack consistency error (sp: %"PRIdPTRDIFF", bp: %"PRIdPTRDIFF")";
6971#if defined RUBY_DEVEL
6972 VALUE mesg = rb_sprintf(stack_consistency_error, nsp, nbp);
6973 rb_str_cat_cstr(mesg, "\n");
6974 rb_str_append(mesg, rb_iseq_disasm(CFP_ISEQ(cfp)));
6975 rb_exc_fatal(rb_exc_new3(rb_eFatal, mesg));
6976#else
6977 rb_bug(stack_consistency_error, nsp, nbp);
6978#endif
6979}
6980
6981static VALUE
6982vm_opt_plus(VALUE recv, VALUE obj)
6983{
6984 if (FIXNUM_2_P(recv, obj) &&
6985 OP_UNREDEFINED_P(PLUS, INTEGER)) {
6986 return rb_fix_plus_fix(recv, obj);
6987 }
6988 else if (FLONUM_2_P(recv, obj) &&
6989 OP_UNREDEFINED_P(PLUS, FLOAT)) {
6990 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
6991 }
6992 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6993 return Qundef;
6994 }
6995 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6996 RBASIC_CLASS(obj) == rb_cFloat &&
6997 OP_UNREDEFINED_P(PLUS, FLOAT)) {
6998 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
6999 }
7000 else if (RBASIC_CLASS(recv) == rb_cString &&
7001 RBASIC_CLASS(obj) == rb_cString &&
7002 OP_UNREDEFINED_P(PLUS, STRING)) {
7003 return rb_str_opt_plus(recv, obj);
7004 }
7005 else if (RBASIC_CLASS(recv) == rb_cArray &&
7006 RBASIC_CLASS(obj) == rb_cArray &&
7007 OP_UNREDEFINED_P(PLUS, ARRAY)) {
7008 return rb_ary_plus(recv, obj);
7009 }
7010 else {
7011 return Qundef;
7012 }
7013}
7014
7015static VALUE
7016vm_opt_minus(VALUE recv, VALUE obj)
7017{
7018 if (FIXNUM_2_P(recv, obj) &&
7019 OP_UNREDEFINED_P(MINUS, INTEGER)) {
7020 return rb_fix_minus_fix(recv, obj);
7021 }
7022 else if (FLONUM_2_P(recv, obj) &&
7023 OP_UNREDEFINED_P(MINUS, FLOAT)) {
7024 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
7025 }
7026 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7027 return Qundef;
7028 }
7029 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7030 RBASIC_CLASS(obj) == rb_cFloat &&
7031 OP_UNREDEFINED_P(MINUS, FLOAT)) {
7032 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
7033 }
7034 else {
7035 return Qundef;
7036 }
7037}
7038
7039static VALUE
7040vm_opt_mult(VALUE recv, VALUE obj)
7041{
7042 if (FIXNUM_2_P(recv, obj) &&
7043 OP_UNREDEFINED_P(MULT, INTEGER)) {
7044 return rb_fix_mul_fix(recv, obj);
7045 }
7046 else if (FLONUM_2_P(recv, obj) &&
7047 OP_UNREDEFINED_P(MULT, FLOAT)) {
7048 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
7049 }
7050 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7051 return Qundef;
7052 }
7053 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7054 RBASIC_CLASS(obj) == rb_cFloat &&
7055 OP_UNREDEFINED_P(MULT, FLOAT)) {
7056 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
7057 }
7058 else {
7059 return Qundef;
7060 }
7061}
7062
7063static VALUE
7064vm_opt_div(VALUE recv, VALUE obj)
7065{
7066 if (FIXNUM_2_P(recv, obj) &&
7067 OP_UNREDEFINED_P(DIV, INTEGER)) {
7068 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_div_fix(recv, obj);
7069 }
7070 else if (FLONUM_2_P(recv, obj) &&
7071 OP_UNREDEFINED_P(DIV, FLOAT)) {
7072 return rb_flo_div_flo(recv, obj);
7073 }
7074 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7075 return Qundef;
7076 }
7077 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7078 RBASIC_CLASS(obj) == rb_cFloat &&
7079 OP_UNREDEFINED_P(DIV, FLOAT)) {
7080 return rb_flo_div_flo(recv, obj);
7081 }
7082 else {
7083 return Qundef;
7084 }
7085}
7086
7087static VALUE
7088vm_opt_mod(VALUE recv, VALUE obj)
7089{
7090 if (FIXNUM_2_P(recv, obj) &&
7091 OP_UNREDEFINED_P(MOD, INTEGER)) {
7092 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_mod_fix(recv, obj);
7093 }
7094 else if (FLONUM_2_P(recv, obj) &&
7095 OP_UNREDEFINED_P(MOD, FLOAT)) {
7096 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
7097 }
7098 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7099 return Qundef;
7100 }
7101 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7102 RBASIC_CLASS(obj) == rb_cFloat &&
7103 OP_UNREDEFINED_P(MOD, FLOAT)) {
7104 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
7105 }
7106 else {
7107 return Qundef;
7108 }
7109}
7110
7111static VALUE
7112vm_opt_neq(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, CALL_DATA cd_eq, VALUE recv, VALUE obj)
7113{
7114 if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_not_equal)) {
7115 VALUE val = opt_equality(reg_cfp, recv, obj, cd_eq);
7116
7117 if (!UNDEF_P(val)) {
7118 return RBOOL(!RTEST(val));
7119 }
7120 }
7121
7122 return Qundef;
7123}
7124
7125static VALUE
7126vm_opt_lt(VALUE recv, VALUE obj)
7127{
7128 if (FIXNUM_2_P(recv, obj) &&
7129 OP_UNREDEFINED_P(LT, INTEGER)) {
7130 return RBOOL((SIGNED_VALUE)recv < (SIGNED_VALUE)obj);
7131 }
7132 else if (FLONUM_2_P(recv, obj) &&
7133 OP_UNREDEFINED_P(LT, FLOAT)) {
7134 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
7135 }
7136 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7137 return Qundef;
7138 }
7139 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7140 RBASIC_CLASS(obj) == rb_cFloat &&
7141 OP_UNREDEFINED_P(LT, FLOAT)) {
7142 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
7143 }
7144 else {
7145 return Qundef;
7146 }
7147}
7148
7149static VALUE
7150vm_opt_le(VALUE recv, VALUE obj)
7151{
7152 if (FIXNUM_2_P(recv, obj) &&
7153 OP_UNREDEFINED_P(LE, INTEGER)) {
7154 return RBOOL((SIGNED_VALUE)recv <= (SIGNED_VALUE)obj);
7155 }
7156 else if (FLONUM_2_P(recv, obj) &&
7157 OP_UNREDEFINED_P(LE, FLOAT)) {
7158 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
7159 }
7160 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7161 return Qundef;
7162 }
7163 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7164 RBASIC_CLASS(obj) == rb_cFloat &&
7165 OP_UNREDEFINED_P(LE, FLOAT)) {
7166 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
7167 }
7168 else {
7169 return Qundef;
7170 }
7171}
7172
7173static VALUE
7174vm_opt_gt(VALUE recv, VALUE obj)
7175{
7176 if (FIXNUM_2_P(recv, obj) &&
7177 OP_UNREDEFINED_P(GT, INTEGER)) {
7178 return RBOOL((SIGNED_VALUE)recv > (SIGNED_VALUE)obj);
7179 }
7180 else if (FLONUM_2_P(recv, obj) &&
7181 OP_UNREDEFINED_P(GT, FLOAT)) {
7182 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
7183 }
7184 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7185 return Qundef;
7186 }
7187 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7188 RBASIC_CLASS(obj) == rb_cFloat &&
7189 OP_UNREDEFINED_P(GT, FLOAT)) {
7190 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
7191 }
7192 else {
7193 return Qundef;
7194 }
7195}
7196
7197static VALUE
7198vm_opt_ge(VALUE recv, VALUE obj)
7199{
7200 if (FIXNUM_2_P(recv, obj) &&
7201 OP_UNREDEFINED_P(GE, INTEGER)) {
7202 return RBOOL((SIGNED_VALUE)recv >= (SIGNED_VALUE)obj);
7203 }
7204 else if (FLONUM_2_P(recv, obj) &&
7205 OP_UNREDEFINED_P(GE, FLOAT)) {
7206 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
7207 }
7208 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7209 return Qundef;
7210 }
7211 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7212 RBASIC_CLASS(obj) == rb_cFloat &&
7213 OP_UNREDEFINED_P(GE, FLOAT)) {
7214 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
7215 }
7216 else {
7217 return Qundef;
7218 }
7219}
7220
7221
7222static VALUE
7223vm_opt_ltlt(VALUE recv, VALUE obj)
7224{
7225 if (SPECIAL_CONST_P(recv)) {
7226 return Qundef;
7227 }
7228 else if (RBASIC_CLASS(recv) == rb_cString &&
7229 OP_UNREDEFINED_P(LTLT, STRING)) {
7230 if (LIKELY(RB_TYPE_P(obj, T_STRING))) {
7231 return rb_str_buf_append(recv, obj);
7232 }
7233 else {
7234 return rb_str_concat(recv, obj);
7235 }
7236 }
7237 else if (RBASIC_CLASS(recv) == rb_cArray &&
7238 OP_UNREDEFINED_P(LTLT, ARRAY)) {
7239 return rb_ary_push(recv, obj);
7240 }
7241 else {
7242 return Qundef;
7243 }
7244}
7245
7246static VALUE
7247vm_opt_and(VALUE recv, VALUE obj)
7248{
7249 // If recv and obj are both fixnums, then the bottom tag bit
7250 // will be 1 on both. 1 & 1 == 1, so the result value will also
7251 // be a fixnum. If either side is *not* a fixnum, then the tag bit
7252 // will be 0, and we return Qundef.
7253 VALUE ret = ((SIGNED_VALUE) recv) & ((SIGNED_VALUE) obj);
7254
7255 if (FIXNUM_P(ret) &&
7256 OP_UNREDEFINED_P(AND, INTEGER)) {
7257 return ret;
7258 }
7259 else {
7260 return Qundef;
7261 }
7262}
7263
7264static VALUE
7265vm_opt_or(VALUE recv, VALUE obj)
7266{
7267 if (FIXNUM_2_P(recv, obj) &&
7268 OP_UNREDEFINED_P(OR, INTEGER)) {
7269 return recv | obj;
7270 }
7271 else {
7272 return Qundef;
7273 }
7274}
7275
7276static VALUE
7277vm_opt_aref(VALUE recv, VALUE obj)
7278{
7279 if (SPECIAL_CONST_P(recv)) {
7280 if (FIXNUM_2_P(recv, obj) &&
7281 OP_UNREDEFINED_P(AREF, INTEGER)) {
7282 return rb_fix_aref(recv, obj);
7283 }
7284 return Qundef;
7285 }
7286 else if (RBASIC_CLASS(recv) == rb_cArray &&
7287 OP_UNREDEFINED_P(AREF, ARRAY)) {
7288 if (FIXNUM_P(obj)) {
7289 return rb_ary_entry_internal(recv, FIX2LONG(obj));
7290 }
7291 else {
7292 return rb_ary_aref1(recv, obj);
7293 }
7294 }
7295 else if (RBASIC_CLASS(recv) == rb_cHash &&
7296 OP_UNREDEFINED_P(AREF, HASH)) {
7297 return rb_hash_aref(recv, obj);
7298 }
7299 else {
7300 return Qundef;
7301 }
7302}
7303
7304static VALUE
7305vm_opt_aset(VALUE recv, VALUE obj, VALUE set)
7306{
7307 if (SPECIAL_CONST_P(recv)) {
7308 return Qundef;
7309 }
7310 else if (RBASIC_CLASS(recv) == rb_cArray &&
7311 OP_UNREDEFINED_P(ASET, ARRAY) &&
7312 FIXNUM_P(obj)) {
7313 rb_ary_store(recv, FIX2LONG(obj), set);
7314 return set;
7315 }
7316 else if (RBASIC_CLASS(recv) == rb_cHash &&
7317 OP_UNREDEFINED_P(ASET, HASH)) {
7318 rb_hash_aset(recv, obj, set);
7319 return set;
7320 }
7321 else {
7322 return Qundef;
7323 }
7324}
7325
7326static VALUE
7327vm_opt_length(VALUE recv, int bop)
7328{
7329 if (SPECIAL_CONST_P(recv)) {
7330 return Qundef;
7331 }
7332 else if (RBASIC_CLASS(recv) == rb_cString &&
7333 BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
7334 if (bop == BOP_EMPTY_P) {
7335 return LEN2NUM(RSTRING_LEN(recv));
7336 }
7337 else {
7338 return rb_str_length(recv);
7339 }
7340 }
7341 else if (RBASIC_CLASS(recv) == rb_cArray &&
7342 BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
7343 return LEN2NUM(RARRAY_LEN(recv));
7344 }
7345 else if (RBASIC_CLASS(recv) == rb_cHash &&
7346 BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
7347 return INT2FIX(RHASH_SIZE(recv));
7348 }
7349 else {
7350 return Qundef;
7351 }
7352}
7353
7354static VALUE
7355vm_opt_empty_p(VALUE recv)
7356{
7357 switch (vm_opt_length(recv, BOP_EMPTY_P)) {
7358 case Qundef: return Qundef;
7359 case INT2FIX(0): return Qtrue;
7360 default: return Qfalse;
7361 }
7362}
7363
7364VALUE rb_false(VALUE obj);
7365
7366static VALUE
7367vm_opt_nil_p(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv)
7368{
7369 if (NIL_P(recv) &&
7370 OP_UNREDEFINED_P(NIL_P, NIL)) {
7371 return Qtrue;
7372 }
7373 else if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_false)) {
7374 return Qfalse;
7375 }
7376 else {
7377 return Qundef;
7378 }
7379}
7380
7381static VALUE
7382fix_succ(VALUE x)
7383{
7384 switch (x) {
7385 case ~0UL:
7386 /* 0xFFFF_FFFF == INT2FIX(-1)
7387 * `-1.succ` is of course 0. */
7388 return INT2FIX(0);
7389 case RSHIFT(~0UL, 1):
7390 /* 0x7FFF_FFFF == LONG2FIX(0x3FFF_FFFF)
7391 * 0x3FFF_FFFF + 1 == 0x4000_0000, which is a Bignum. */
7392 return rb_uint2big(1UL << (SIZEOF_LONG * CHAR_BIT - 2));
7393 default:
7394 /* LONG2FIX(FIX2LONG(x)+FIX2LONG(y))
7395 * == ((lx*2+1)/2 + (ly*2+1)/2)*2+1
7396 * == lx*2 + ly*2 + 1
7397 * == (lx*2+1) + (ly*2+1) - 1
7398 * == x + y - 1
7399 *
7400 * Here, if we put y := INT2FIX(1):
7401 *
7402 * == x + INT2FIX(1) - 1
7403 * == x + 2 .
7404 */
7405 return x + 2;
7406 }
7407}
7408
7409static VALUE
7410vm_opt_succ(VALUE recv)
7411{
7412 if (FIXNUM_P(recv) &&
7413 OP_UNREDEFINED_P(SUCC, INTEGER)) {
7414 return fix_succ(recv);
7415 }
7416 else if (SPECIAL_CONST_P(recv)) {
7417 return Qundef;
7418 }
7419 else if (RBASIC_CLASS(recv) == rb_cString &&
7420 OP_UNREDEFINED_P(SUCC, STRING)) {
7421 return rb_str_succ(recv);
7422 }
7423 else {
7424 return Qundef;
7425 }
7426}
7427
7428static VALUE
7429vm_opt_not(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv)
7430{
7431 if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_not)) {
7432 return RBOOL(!RTEST(recv));
7433 }
7434 else {
7435 return Qundef;
7436 }
7437}
7438
7439static VALUE
7440vm_opt_regexpmatch2(VALUE recv, VALUE obj)
7441{
7442 if (SPECIAL_CONST_P(recv)) {
7443 return Qundef;
7444 }
7445 else if (RBASIC_CLASS(recv) == rb_cString &&
7446 CLASS_OF(obj) == rb_cRegexp &&
7447 OP_UNREDEFINED_P(MATCH, STRING)) {
7448 return rb_reg_match(obj, recv);
7449 }
7450 else if (RBASIC_CLASS(recv) == rb_cRegexp &&
7451 OP_UNREDEFINED_P(MATCH, REGEXP)) {
7452 return rb_reg_match(recv, obj);
7453 }
7454 else {
7455 return Qundef;
7456 }
7457}
7458
7459#undef EQ_UNREDEFINED_P
7460#undef OP_UNREDEFINED_P
7461
7462rb_event_flag_t rb_iseq_event_flags(const rb_iseq_t *iseq, size_t pos);
7463
7464NOINLINE(static void vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp));
7465
7466static inline void
7467vm_trace_hook(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE *pc,
7468 rb_event_flag_t pc_events, rb_event_flag_t target_event,
7469 rb_hook_list_t *global_hooks, rb_hook_list_t *local_hooks, VALUE val)
7470{
7471 rb_event_flag_t event = pc_events & target_event;
7472 VALUE self = GET_SELF();
7473
7474 VM_ASSERT(rb_popcount64((uint64_t)event) == 1);
7475
7476 if (local_hooks) local_hooks->running++; // make sure they don't get deleted while global hooks run
7477
7478 if (event & global_hooks->events) {
7479 /* increment PC because source line is calculated with PC-1 */
7480 reg_cfp->pc++;
7481 vm_dtrace(event, ec);
7482 rb_exec_event_hook_orig(ec, global_hooks, event, self, 0, 0, 0 , val, 0);
7483 reg_cfp->pc--;
7484 }
7485
7486 if (local_hooks) local_hooks->running--;
7487 if (local_hooks != NULL) {
7488 if (event & local_hooks->events) {
7489 /* increment PC because source line is calculated with PC-1 */
7490 reg_cfp->pc++;
7491 rb_exec_event_hook_orig(ec, local_hooks, event, self, 0, 0, 0 , val, 0);
7492 reg_cfp->pc--;
7493 }
7494 }
7495}
7496
7497/* Re-fetch the iseq-local hook list before each event: a hook fired by an
7498 * earlier event at this pc can disable the last targeted TracePoint on this
7499 * iseq, which frees local_hooks. Reload from its stable source (the ractor's
7500 * targeted-hooks table) so we never dereference a dangling pointer. */
7501#define VM_TRACE_HOOK(target_event, val) do { \
7502 if ((pc_events & (target_event)) & enabled_flags) { \
7503 if (local_hooks_cnt > 0) local_hooks = rb_iseq_local_hooks(iseq, r, false); \
7504 vm_trace_hook(ec, reg_cfp, pc, pc_events, (target_event), global_hooks, local_hooks, (val)); \
7505 } \
7506} while (0)
7507
7508static VALUE
7509rescue_errinfo(rb_execution_context_t *ec, rb_control_frame_t *cfp)
7510{
7511 VM_ASSERT(VM_FRAME_RUBYFRAME_P(cfp));
7512 VM_ASSERT(ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_RESCUE);
7513 return cfp->ep[VM_ENV_INDEX_LAST_LVAR];
7514}
7515
7516static void
7517vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp)
7518{
7519 const VALUE *pc = reg_cfp->pc;
7520 rb_ractor_t *r = rb_ec_ractor_ptr(ec);
7521 rb_event_flag_t enabled_flags = r->pub.hooks.events & ISEQ_TRACE_EVENTS;
7522 rb_event_flag_t ractor_events = enabled_flags;
7523
7524 if (enabled_flags == 0 && rb_ractor_targeted_hooks_cnt(r) == 0) {
7525 return;
7526 }
7527 else {
7528 const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp);
7529 size_t pos = pc - ISEQ_BODY(iseq)->iseq_encoded;
7530 rb_event_flag_t pc_events = rb_iseq_event_flags(iseq, pos);
7531 unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt;
7532 rb_hook_list_t *local_hooks = NULL;
7533 if (RB_UNLIKELY(local_hooks_cnt > 0)) {
7534 st_data_t val;
7535 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)iseq, &val)) {
7536 local_hooks = (rb_hook_list_t*)val;
7537 }
7538 }
7539 rb_event_flag_t iseq_local_events = local_hooks != NULL ? local_hooks->events : 0;
7540
7541 rb_hook_list_t *bmethod_local_hooks = NULL;
7542 rb_event_flag_t bmethod_local_events = 0;
7543 unsigned int bmethod_hooks_cnt = 0;
7544 rb_method_definition_t *bmethod_def = NULL;
7545 const bool bmethod_frame = VM_FRAME_BMETHOD_P(reg_cfp);
7546 enabled_flags |= iseq_local_events;
7547
7548 VM_ASSERT((iseq_local_events & ~ISEQ_TRACE_EVENTS) == 0);
7549
7550 if (bmethod_frame) {
7551 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
7552 VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD);
7553 bmethod_def = me->def;
7554 bmethod_hooks_cnt = me->def->body.bmethod.local_hooks_cnt;
7555 if (RB_UNLIKELY(bmethod_hooks_cnt > 0)) {
7556 st_data_t val;
7557 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)me->def, &val)) {
7558 bmethod_local_hooks = (rb_hook_list_t*)val;
7559 }
7560 if (bmethod_local_hooks) {
7561 bmethod_local_events = bmethod_local_hooks->events;
7562 }
7563 }
7564 }
7565
7566 if ((pc_events & enabled_flags) == 0 && !bmethod_frame) {
7567#if 0
7568 /* disable trace */
7569 /* TODO: incomplete */
7570 rb_iseq_trace_set(iseq, vm_event_flags & ISEQ_TRACE_EVENTS);
7571#else
7572 /* do not disable trace because of performance problem
7573 * (re-enable overhead)
7574 */
7575#endif
7576 return;
7577 }
7578 else if (ec->trace_arg != NULL) {
7579 /* already tracing */
7580 return;
7581 }
7582 else {
7583 rb_hook_list_t *global_hooks = rb_ec_ractor_hooks(ec);
7584 /* Note, not considering iseq local events here since the same
7585 * iseq could be used in multiple bmethods. */
7586 rb_event_flag_t bmethod_events = ractor_events | bmethod_local_events;
7587
7588 if (0) {
7589 VALUE path = rb_iseq_path(iseq);
7590 VALUE label = rb_iseq_label(iseq);
7591 ruby_debug_printf("vm_trace>>%4d (%4x) - %.*s:%d %.*s\n",
7592 (int)pos,
7593 (int)pc_events,
7594 RSTRING_LENINT(path), RSTRING_PTR(path),
7595 (int)rb_iseq_line_no(iseq, pos),
7596 RSTRING_LENINT(label), RSTRING_PTR(label));
7597 }
7598 VM_ASSERT(reg_cfp->pc == pc);
7599 VM_ASSERT(pc_events != 0);
7600
7601 /* check traces */
7602 if ((pc_events & RUBY_EVENT_B_CALL) && bmethod_frame && (bmethod_events & RUBY_EVENT_CALL)) {
7603 /* b_call instruction running as a method. Fire call event. */
7604 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_CALL, RUBY_EVENT_CALL, global_hooks, bmethod_local_hooks, Qundef);
7605 }
7606 VM_TRACE_HOOK(RUBY_EVENT_CLASS | RUBY_EVENT_CALL | RUBY_EVENT_B_CALL, Qundef);
7607 VM_TRACE_HOOK(RUBY_EVENT_RESCUE, rescue_errinfo(ec, reg_cfp));
7608 VM_TRACE_HOOK(RUBY_EVENT_LINE, Qundef);
7609 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_LINE, Qundef);
7610 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_BRANCH, Qundef);
7611 VM_TRACE_HOOK(RUBY_EVENT_END | RUBY_EVENT_RETURN | RUBY_EVENT_B_RETURN, TOPN(0));
7612 if ((pc_events & RUBY_EVENT_B_RETURN) && bmethod_frame && (bmethod_events & RUBY_EVENT_RETURN)) {
7613 /* b_return instruction running as a method. Fire return event. */
7614 /* An earlier event's hook at this pc may have freed bmethod_local_hooks
7615 * (see VM_TRACE_HOOK); reload it from its stable source. */
7616 if (bmethod_hooks_cnt > 0) bmethod_local_hooks = rb_method_def_local_hooks(bmethod_def, r, false);
7617 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_RETURN, RUBY_EVENT_RETURN, global_hooks, bmethod_local_hooks, TOPN(0));
7618 }
7619 }
7620 }
7621}
7622#undef VM_TRACE_HOOK
7623
7624#if VM_CHECK_MODE > 0
7625NORETURN( NOINLINE( COLDFUNC
7626void rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)));
7627
7628void
7629Init_vm_stack_canary(void)
7630{
7631 /* This has to be called _after_ our PRNG is properly set up. */
7632 int n = ruby_fill_random_bytes(&vm_stack_canary, sizeof vm_stack_canary, false);
7633 /* Make it a large positive Fixnum. Arguments a block does not take
7634 * stay at the stack top, where vm_push_frame checks the canary. */
7635 vm_stack_canary >>= 2;
7636 vm_stack_canary |= (VALUE)1 << (SIZEOF_VALUE * CHAR_BIT - 2);
7637 vm_stack_canary |= 0x01; // valid VALUE (Fixnum)
7638
7639 vm_stack_canary_was_born = true;
7640 VM_ASSERT(n == 0);
7641}
7642
7643void
7644rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)
7645{
7646 /* Because a method has already been called, why not call
7647 * another one. */
7648 const char *insn = rb_insns_name(i);
7649 VALUE inspection = rb_inspect(c);
7650 const char *str = StringValueCStr(inspection);
7651
7652 rb_bug("dead canary found at %s: %s", insn, str);
7653}
7654
7655#else
7656void Init_vm_stack_canary(void) { /* nothing to do */ }
7657#endif
7658
7659
7660/* a part of the following code is generated by this ruby script:
7661
766216.times{|i|
7663 typedef_args = (0...i).map{|j| "VALUE v#{j+1}"}.join(", ")
7664 typedef_args.prepend(", ") if i != 0
7665 call_args = (0...i).map{|j| "argv[#{j}]"}.join(", ")
7666 call_args.prepend(", ") if i != 0
7667 puts %Q{
7668static VALUE
7669builtin_invoker#{i}(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7670{
7671 typedef VALUE (*rb_invoke_funcptr#{i}_t)(rb_execution_context_t *ec, VALUE self#{typedef_args});
7672 return (*(rb_invoke_funcptr#{i}_t)funcptr)(ec, self#{call_args});
7673}}
7674}
7675
7676puts
7677puts "static VALUE (* const cfunc_invokers[])(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr) = {"
767816.times{|i|
7679 puts " builtin_invoker#{i},"
7680}
7681puts "};"
7682*/
7683
7684static VALUE
7685builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7686{
7687 typedef VALUE (*rb_invoke_funcptr0_t)(rb_execution_context_t *ec, VALUE self);
7688 return (*(rb_invoke_funcptr0_t)funcptr)(ec, self);
7689}
7690
7691static VALUE
7692builtin_invoker1(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7693{
7694 typedef VALUE (*rb_invoke_funcptr1_t)(rb_execution_context_t *ec, VALUE self, VALUE v1);
7695 return (*(rb_invoke_funcptr1_t)funcptr)(ec, self, argv[0]);
7696}
7697
7698static VALUE
7699builtin_invoker2(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7700{
7701 typedef VALUE (*rb_invoke_funcptr2_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2);
7702 return (*(rb_invoke_funcptr2_t)funcptr)(ec, self, argv[0], argv[1]);
7703}
7704
7705static VALUE
7706builtin_invoker3(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7707{
7708 typedef VALUE (*rb_invoke_funcptr3_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3);
7709 return (*(rb_invoke_funcptr3_t)funcptr)(ec, self, argv[0], argv[1], argv[2]);
7710}
7711
7712static VALUE
7713builtin_invoker4(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7714{
7715 typedef VALUE (*rb_invoke_funcptr4_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4);
7716 return (*(rb_invoke_funcptr4_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3]);
7717}
7718
7719static VALUE
7720builtin_invoker5(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7721{
7722 typedef VALUE (*rb_invoke_funcptr5_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5);
7723 return (*(rb_invoke_funcptr5_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4]);
7724}
7725
7726static VALUE
7727builtin_invoker6(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7728{
7729 typedef VALUE (*rb_invoke_funcptr6_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6);
7730 return (*(rb_invoke_funcptr6_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
7731}
7732
7733static VALUE
7734builtin_invoker7(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7735{
7736 typedef VALUE (*rb_invoke_funcptr7_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7);
7737 return (*(rb_invoke_funcptr7_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
7738}
7739
7740static VALUE
7741builtin_invoker8(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7742{
7743 typedef VALUE (*rb_invoke_funcptr8_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8);
7744 return (*(rb_invoke_funcptr8_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
7745}
7746
7747static VALUE
7748builtin_invoker9(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7749{
7750 typedef VALUE (*rb_invoke_funcptr9_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9);
7751 return (*(rb_invoke_funcptr9_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
7752}
7753
7754static VALUE
7755builtin_invoker10(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7756{
7757 typedef VALUE (*rb_invoke_funcptr10_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10);
7758 return (*(rb_invoke_funcptr10_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
7759}
7760
7761static VALUE
7762builtin_invoker11(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7763{
7764 typedef VALUE (*rb_invoke_funcptr11_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11);
7765 return (*(rb_invoke_funcptr11_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
7766}
7767
7768static VALUE
7769builtin_invoker12(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7770{
7771 typedef VALUE (*rb_invoke_funcptr12_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12);
7772 return (*(rb_invoke_funcptr12_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
7773}
7774
7775static VALUE
7776builtin_invoker13(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7777{
7778 typedef VALUE (*rb_invoke_funcptr13_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13);
7779 return (*(rb_invoke_funcptr13_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
7780}
7781
7782static VALUE
7783builtin_invoker14(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7784{
7785 typedef VALUE (*rb_invoke_funcptr14_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14);
7786 return (*(rb_invoke_funcptr14_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
7787}
7788
7789static VALUE
7790builtin_invoker15(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7791{
7792 typedef VALUE (*rb_invoke_funcptr15_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14, VALUE v15);
7793 return (*(rb_invoke_funcptr15_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
7794}
7795
7796typedef VALUE (*builtin_invoker)(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
7797
7798static builtin_invoker
7799lookup_builtin_invoker(int argc)
7800{
7801 static const builtin_invoker invokers[] = {
7802 builtin_invoker0,
7803 builtin_invoker1,
7804 builtin_invoker2,
7805 builtin_invoker3,
7806 builtin_invoker4,
7807 builtin_invoker5,
7808 builtin_invoker6,
7809 builtin_invoker7,
7810 builtin_invoker8,
7811 builtin_invoker9,
7812 builtin_invoker10,
7813 builtin_invoker11,
7814 builtin_invoker12,
7815 builtin_invoker13,
7816 builtin_invoker14,
7817 builtin_invoker15,
7818 };
7819
7820 return invokers[argc];
7821}
7822
7823static inline VALUE
7824invoke_bf(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7825{
7826 const bool canary_p = ISEQ_BODY(CFP_ISEQ(reg_cfp))->builtin_attrs & BUILTIN_ATTR_LEAF; // Verify an assumption of `Primitive.attr! :leaf`
7827 SETUP_CANARY(canary_p);
7828 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
7829 VALUE ret = (*lookup_builtin_invoker(bf->argc))(ec, reg_cfp->self, argv, func_ptr);
7830 CHECK_CANARY(canary_p, BIN(invokebuiltin));
7831 return ret;
7832}
7833
7834static VALUE
7835vm_invoke_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7836{
7837 return invoke_bf(ec, cfp, bf, argv);
7838}
7839
7840static VALUE
7841vm_invoke_builtin_delegate(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function *bf, unsigned int start_index)
7842{
7843 if (0) { // debug print
7844 fputs("vm_invoke_builtin_delegate: passing -> ", stderr);
7845 for (int i=0; i<bf->argc; i++) {
7846 ruby_debug_printf(":%s ", rb_id2name(ISEQ_BODY(CFP_ISEQ(cfp))->local_table[i+start_index]));
7847 }
7848 ruby_debug_printf("\n" "%s %s(%d):%p\n", RUBY_FUNCTION_NAME_STRING, bf->name, bf->argc,
7849 (void *)(uintptr_t)bf->func_ptr);
7850 }
7851
7852 if (bf->argc == 0) {
7853 return invoke_bf(ec, cfp, bf, NULL);
7854 }
7855 else {
7856 const VALUE *argv = cfp->ep - ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size - VM_ENV_DATA_SIZE + 1 + start_index;
7857 return invoke_bf(ec, cfp, bf, argv);
7858 }
7859}
7860
7861// for __builtin_inline!()
7862
7863VALUE
7864rb_vm_lvar_exposed(rb_execution_context_t *ec, int index)
7865{
7866 const rb_control_frame_t *cfp = ec->cfp;
7867 return cfp->ep[index];
7868}
RUBY_ASSERT
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:219
RUBY_EVENT_END
#define RUBY_EVENT_END
Encountered an end of a class clause.
Definition event.h:40
RUBY_EVENT_C_CALL
#define RUBY_EVENT_C_CALL
A method, written in C, is called.
Definition event.h:43
RUBY_EVENT_B_RETURN
#define RUBY_EVENT_B_RETURN
Encountered a next statement.
Definition event.h:56
RUBY_EVENT_CLASS
#define RUBY_EVENT_CLASS
Encountered a new class.
Definition event.h:39
RUBY_EVENT_LINE
#define RUBY_EVENT_LINE
Encountered a new line.
Definition event.h:38
RUBY_EVENT_RETURN
#define RUBY_EVENT_RETURN
Encountered a return statement.
Definition event.h:42
RUBY_EVENT_C_RETURN
#define RUBY_EVENT_C_RETURN
Return from a method, written in C.
Definition event.h:44
RUBY_EVENT_B_CALL
#define RUBY_EVENT_B_CALL
Encountered an yield statement.
Definition event.h:55
rb_event_flag_t
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
RUBY_EVENT_CALL
#define RUBY_EVENT_CALL
A method, written in Ruby, is called.
Definition event.h:41
RUBY_EVENT_RESCUE
#define RUBY_EVENT_RESCUE
Encountered a rescue statement.
Definition event.h:61
RB_FL_TEST_RAW
static VALUE RB_FL_TEST_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_TEST().
Definition fl_type.h:407
RB_FL_ABLE
static bool RB_FL_ABLE(VALUE obj)
Checks if the object is flaggable.
Definition fl_type.h:384
RUBY_FL_SHAREABLE
@ RUBY_FL_SHAREABLE
This flag has something to do with Ractor.
Definition fl_type.h:253
rb_singleton_class
VALUE rb_singleton_class(VALUE obj)
Finds or creates the singleton class of the passed object.
Definition class.c:3053
rb_module_new
VALUE rb_module_new(void)
Creates a new, anonymous module.
Definition class.c:1663
rb_class_inherited
VALUE rb_class_inherited(VALUE super, VALUE klass)
Calls Class::inherited.
Definition class.c:1563
rb_define_class_id
VALUE rb_define_class_id(ID id, VALUE super)
This is a very badly designed API that creates an anonymous class.
Definition class.c:1542
TYPE
#define TYPE(_)
Old name of rb_type.
Definition value_type.h:108
RFLOAT_VALUE
#define RFLOAT_VALUE
Old name of rb_float_value.
Definition double.h:28
T_STRING
#define T_STRING
Old name of RUBY_T_STRING.
Definition value_type.h:78
Qundef
#define Qundef
Old name of RUBY_Qundef.
Definition special_consts.h:62
INT2FIX
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
LEN2NUM
#define LEN2NUM
Old name of RB_LEN2NUM.
Definition len_t.h:29
T_NIL
#define T_NIL
Old name of RUBY_T_NIL.
Definition value_type.h:72
T_FLOAT
#define T_FLOAT
Old name of RUBY_T_FLOAT.
Definition value_type.h:64
T_IMEMO
#define T_IMEMO
Old name of RUBY_T_IMEMO.
Definition value_type.h:67
ID2SYM
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
T_BIGNUM
#define T_BIGNUM
Old name of RUBY_T_BIGNUM.
Definition value_type.h:57
SPECIAL_CONST_P
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
Definition special_consts.h:56
T_STRUCT
#define T_STRUCT
Old name of RUBY_T_STRUCT.
Definition value_type.h:79
T_FIXNUM
#define T_FIXNUM
Old name of RUBY_T_FIXNUM.
Definition value_type.h:63
UNREACHABLE_RETURN
#define UNREACHABLE_RETURN
Old name of RBIMPL_UNREACHABLE_RETURN.
Definition assume.h:29
SYM2ID
#define SYM2ID
Old name of RB_SYM2ID.
Definition symbol.h:45
CLASS_OF
#define CLASS_OF
Old name of rb_class_of.
Definition globals.h:205
T_NONE
#define T_NONE
Old name of RUBY_T_NONE.
Definition value_type.h:74
rb_ary_new4
#define rb_ary_new4
Old name of rb_ary_new_from_values.
Definition array.h:659
FIXABLE
#define FIXABLE
Old name of RB_FIXABLE.
Definition fixnum.h:25
LONG2FIX
#define LONG2FIX
Old name of RB_INT2FIX.
Definition long.h:49
FIX2INT
#define FIX2INT
Old name of RB_FIX2INT.
Definition int.h:41
T_MODULE
#define T_MODULE
Old name of RUBY_T_MODULE.
Definition value_type.h:70
STATIC_SYM_P
#define STATIC_SYM_P
Old name of RB_STATIC_SYM_P.
Definition special_consts.h:57
ASSUME
#define ASSUME
Old name of RBIMPL_ASSUME.
Definition assume.h:27
FIX2ULONG
#define FIX2ULONG
Old name of RB_FIX2ULONG.
Definition long.h:47
T_TRUE
#define T_TRUE
Old name of RUBY_T_TRUE.
Definition value_type.h:81
T_ICLASS
#define T_ICLASS
Old name of RUBY_T_ICLASS.
Definition value_type.h:66
T_HASH
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
rb_ary_new3
#define rb_ary_new3
Old name of rb_ary_new_from_args.
Definition array.h:658
rb_exc_new3
#define rb_exc_new3
Old name of rb_exc_new_str.
Definition error.h:38
T_FALSE
#define T_FALSE
Old name of RUBY_T_FALSE.
Definition value_type.h:61
Qtrue
#define Qtrue
Old name of RUBY_Qtrue.
Definition special_consts.h:61
Qnil
#define Qnil
Old name of RUBY_Qnil.
Definition special_consts.h:60
Qfalse
#define Qfalse
Old name of RUBY_Qfalse.
Definition special_consts.h:59
FIX2LONG
#define FIX2LONG
Old name of RB_FIX2LONG.
Definition long.h:46
T_ARRAY
#define T_ARRAY
Old name of RUBY_T_ARRAY.
Definition value_type.h:56
T_OBJECT
#define T_OBJECT
Old name of RUBY_T_OBJECT.
Definition value_type.h:75
NIL_P
#define NIL_P
Old name of RB_NIL_P.
Definition special_consts.h:55
T_SYMBOL
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition value_type.h:80
DBL2NUM
#define DBL2NUM
Old name of rb_float_new.
Definition double.h:29
T_CLASS
#define T_CLASS
Old name of RUBY_T_CLASS.
Definition value_type.h:58
BUILTIN_TYPE
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
Definition value_type.h:85
FL_TEST
#define FL_TEST
Old name of RB_FL_TEST.
Definition fl_type.h:127
FIXNUM_P
#define FIXNUM_P
Old name of RB_FIXNUM_P.
Definition special_consts.h:53
FL_USHIFT
#define FL_USHIFT
Old name of RUBY_FL_USHIFT.
Definition fl_type.h:67
FL_SET_RAW
#define FL_SET_RAW
Old name of RB_FL_SET_RAW.
Definition fl_type.h:126
SYMBOL_P
#define SYMBOL_P
Old name of RB_SYMBOL_P.
Definition value_type.h:88
rb_exc_raise
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
Definition eval.c:678
ruby_verbose
#define ruby_verbose
This variable controls whether the interpreter is in debug mode.
Definition error.h:476
rb_eTypeError
VALUE rb_eTypeError
TypeError exception.
Definition error.c:1457
rb_eFatal
VALUE rb_eFatal
fatal exception.
Definition error.c:1453
rb_eNoMethodError
VALUE rb_eNoMethodError
NoMethodError exception.
Definition error.c:1465
rb_exc_fatal
void rb_exc_fatal(VALUE mesg)
Raises a fatal error in the current thread.
Definition eval.c:691
rb_eRuntimeError
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1455
rb_warn
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:464
rb_error_frozen_object
void rb_error_frozen_object(VALUE frozen_obj)
Identical to rb_error_frozen(), except it takes arbitrary Ruby object instead of C's string.
Definition error.c:4320
rb_exc_new_str
VALUE rb_exc_new_str(VALUE etype, VALUE str)
Identical to rb_exc_new_cstr(), except it takes a Ruby's string instead of C's.
Definition error.c:1508
RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK
@ RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK
Warning is for checking unused block strictly.
Definition error.h:57
rb_cClass
VALUE rb_cClass
Class class.
Definition object.c:62
rb_cArray
VALUE rb_cArray
Array class.
rb_cObject
VALUE rb_cObject
Object class.
Definition object.c:60
rb_obj_alloc
VALUE rb_obj_alloc(VALUE klass)
Allocates an instance of the given class.
Definition object.c:2253
rb_cRegexp
VALUE rb_cRegexp
Regexp class.
Definition re.c:2828
rb_obj_frozen_p
VALUE rb_obj_frozen_p(VALUE obj)
Same as RB_OBJ_FROZEN(), but returns Qtrue/Qfalse instead of #bool.
Definition object.c:1316
rb_cHash
VALUE rb_cHash
Hash class.
Definition hash.c:123
rb_obj_class
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
Definition object.c:234
rb_inspect
VALUE rb_inspect(VALUE obj)
Generates a human-readable textual representation of the given object.
Definition object.c:669
rb_cBasicObject
VALUE rb_cBasicObject
BasicObject class.
Definition object.c:58
rb_cModule
VALUE rb_cModule
Module class.
Definition object.c:61
rb_class_real
VALUE rb_class_real(VALUE klass)
Finds a "real" class.
Definition object.c:225
rb_obj_is_kind_of
VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass)
Queries if the given object is an instance (of possibly descendants) of the given class.
Definition object.c:906
rb_cFloat
VALUE rb_cFloat
Float class.
Definition numeric.c:201
rb_cProc
VALUE rb_cProc
Proc class.
Definition proc.c:46
rb_cString
VALUE rb_cString
String class.
Definition string.c:85
RB_OBJ_WRITTEN
#define RB_OBJ_WRITTEN(old, oldv, young)
Identical to RB_OBJ_WRITE(), except it doesn't write any values, but only a WB declaration.
Definition gc.h:504
RB_OBJ_WRITE
#define RB_OBJ_WRITE(old, slot, young)
Declaration of a "back" pointer.
Definition gc.h:492
rb_ary_concat
VALUE rb_ary_concat(VALUE lhs, VALUE rhs)
Destructively appends the contents of latter into the end of former.
rb_ary_shift
VALUE rb_ary_shift(VALUE ary)
Destructively deletes an element from the beginning of the passed array and returns what was deleted.
rb_ary_resurrect
VALUE rb_ary_resurrect(VALUE ary)
I guess there is no use case of this function in extension libraries, but this is a routine identical...
rb_ary_entry
VALUE rb_ary_entry(VALUE ary, rb_len_t off)
Queries an element of an array.
rb_ary_dup
VALUE rb_ary_dup(VALUE ary)
Duplicates an array.
rb_ary_includes
VALUE rb_ary_includes(VALUE ary, VALUE elem)
Queries if the passed array has the passed entry.
rb_ary_store
void rb_ary_store(VALUE ary, rb_len_t key, VALUE val)
Destructively stores the passed value to the passed array's passed index.
rb_ary_plus
VALUE rb_ary_plus(VALUE lhs, VALUE rhs)
Creates a new array, concatenating the former to the latter.
rb_check_array_type
VALUE rb_check_array_type(VALUE obj)
Try converting an object to its array representation using its to_ary method, if any.
rb_ary_new
VALUE rb_ary_new(void)
Allocates a new, empty array.
rb_ary_pop
VALUE rb_ary_pop(VALUE ary)
Destructively deletes an element from the end of the passed array and returns what was deleted.
rb_ary_hidden_new
VALUE rb_ary_hidden_new(rb_len_t capa)
Allocates a hidden (no class) empty array.
rb_ary_push
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
rb_ary_cat
VALUE rb_ary_cat(VALUE ary, const VALUE *train, rb_len_t len)
Destructively appends multiple elements at the end of the array.
UNLIMITED_ARGUMENTS
#define UNLIMITED_ARGUMENTS
This macro is used in conjunction with rb_check_arity().
Definition error.h:35
rb_check_arity
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
Definition error.h:284
rb_proc_call_with_block
VALUE rb_proc_call_with_block(VALUE recv, int argc, const VALUE *argv, VALUE proc)
Identical to rb_proc_call(), except you can additionally pass another proc object,...
Definition proc.c:1768
rb_reg_last_match
VALUE rb_reg_last_match(VALUE md)
This just returns the argument, stringified.
Definition re.c:2095
rb_reg_match
VALUE rb_reg_match(VALUE re, VALUE str)
This is the match operator.
Definition re.c:3970
rb_reg_nth_match
VALUE rb_reg_nth_match(int n, VALUE md)
Queries the nth captured substring.
Definition re.c:2071
rb_reg_match_post
VALUE rb_reg_match_post(VALUE md)
The portion of the original string after the given match.
Definition re.c:2150
rb_reg_nth_defined
VALUE rb_reg_nth_defined(int n, VALUE md)
Identical to rb_reg_nth_match(), except it just returns Boolean.
Definition re.c:2055
rb_reg_match_pre
VALUE rb_reg_match_pre(VALUE md)
The portion of the original string before the given match.
Definition re.c:2119
rb_reg_match_last
VALUE rb_reg_match_last(VALUE md)
The portion of the original string that captured at the very last.
Definition re.c:2179
rb_set_lookup
bool rb_set_lookup(VALUE set, VALUE element)
Whether the set contains the given element.
Definition set.c:2364
rb_str_append
VALUE rb_str_append(VALUE dst, VALUE src)
Identical to rb_str_buf_append(), except it converts the right hand side before concatenating.
Definition string.c:3913
rb_str_succ
VALUE rb_str_succ(VALUE orig)
Searches for the "successor" of a string.
Definition string.c:5456
rb_str_buf_append
VALUE rb_str_buf_append(VALUE dst, VALUE src)
Identical to rb_str_cat_cstr(), except it takes Ruby's string instead of C's.
Definition string.c:3879
rb_str_concat
VALUE rb_str_concat(VALUE dst, VALUE src)
Identical to rb_str_append(), except it also accepts an integer as a codepoint.
Definition string.c:4149
rb_str_cat_cstr
#define rb_str_cat_cstr(buf, str)
Identical to rb_str_cat(), except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1681
rb_str_length
VALUE rb_str_length(VALUE)
Identical to rb_str_strlen(), except it returns the value in rb_cInteger.
Definition string.c:2506
rb_str_intern
VALUE rb_str_intern(VALUE str)
Identical to rb_to_symbol(), except it assumes the receiver being an instance of RString.
Definition symbol.c:1085
rb_const_get
VALUE rb_const_get(VALUE space, ID name)
Identical to rb_const_defined(), except it returns the actual defined value.
Definition variable.c:3505
rb_ivar_set
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
Definition variable.c:2141
rb_cvar_set
void rb_cvar_set(VALUE klass, ID name, VALUE val)
Assigns a value to a class variable.
Definition variable.c:4294
rb_cvar_find
VALUE rb_cvar_find(VALUE klass, ID name, VALUE *front)
Identical to rb_cvar_get(), except it takes additional "front" pointer.
Definition variable.c:4354
rb_ivar_get
VALUE rb_ivar_get(VALUE obj, ID name)
Identical to rb_iv_get(), except it accepts the name as an ID instead of a C string.
Definition variable.c:1641
rb_const_set
void rb_const_set(VALUE space, ID name, VALUE val)
Names a constant.
Definition variable.c:3984
rb_autoload_load
VALUE rb_autoload_load(VALUE space, ID name)
Kicks the autoload procedure as if it was "touched".
Definition variable.c:3335
rb_mod_name
VALUE rb_mod_name(VALUE mod)
Queries the name of a module.
Definition variable.c:152
rb_const_get_at
VALUE rb_const_get_at(VALUE space, ID name)
Identical to rb_const_defined_at(), except it returns the actual defined value.
Definition variable.c:3511
rb_set_class_path_string
void rb_set_class_path_string(VALUE klass, VALUE space, VALUE name)
Identical to rb_set_class_path(), except it accepts the name as Ruby's string instead of C's.
Definition variable.c:441
rb_ivar_defined
VALUE rb_ivar_defined(VALUE obj, ID name)
Queries if the instance variable is defined at the object.
Definition variable.c:2201
rb_const_defined_at
int rb_const_defined_at(VALUE space, ID name)
Identical to rb_const_defined(), except it doesn't look for parent classes.
Definition variable.c:3844
rb_cvar_defined
VALUE rb_cvar_defined(VALUE klass, ID name)
Queries if the given class has the given class variable.
Definition variable.c:4380
rb_class_path
VALUE rb_class_path(VALUE mod)
Identical to rb_mod_name(), except it returns #<Class: ...> style inspection for anonymous modules.
Definition variable.c:398
rb_const_defined
int rb_const_defined(VALUE space, ID name)
Queries if the constant is defined at the namespace.
Definition variable.c:3838
rb_method_basic_definition_p
int rb_method_basic_definition_p(VALUE klass, ID mid)
Well... Let us hesitate from describing what a "basic definition" is.
Definition vm_method.c:3574
rb_check_funcall
VALUE rb_check_funcall(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it returns RUBY_Qundef instead of raising rb_eNoMethodError.
Definition vm_eval.c:691
rb_get_alloc_func
rb_alloc_func_t rb_get_alloc_func(VALUE klass)
Queries the allocator function of a class.
Definition vm_method.c:1855
rb_method_boundp
int rb_method_boundp(VALUE klass, ID id, int ex)
Queries if the klass has this method.
Definition vm_method.c:2468
rb_check_id
ID rb_check_id(volatile VALUE *namep)
Detects if the given name is already interned or not.
Definition symbol.c:1289
rb_sym2str
VALUE rb_sym2str(VALUE symbol)
Obtain a frozen string representation of a symbol (not including the leading colon).
Definition symbol.c:1148
off
int off
Offset inside of ptr.
Definition io.h:5
len
int len
Length of the buffer.
Definition io.h:8
RB_OBJ_SET_SHAREABLE
#define RB_OBJ_SET_SHAREABLE(obj)
Wrapper of rb_obj_set_shareable().
Definition ractor.h:290
rb_ractor_shareable_p
static bool rb_ractor_shareable_p(VALUE obj)
Queries if multiple Ractors can share the passed object or not.
Definition ractor.h:269
RB_OBJ_SHAREABLE_P
#define RB_OBJ_SHAREABLE_P(obj)
Queries if the passed object has previously classified as shareable or not.
Definition ractor.h:255
RB_NOGVL_UBF_ASYNC_SAFE
#define RB_NOGVL_UBF_ASYNC_SAFE
Passing this flag to rb_nogvl() indicates that the passed UBF is async-signal-safe.
Definition thread.h:71
RB_NOGVL_INTR_FAIL
#define RB_NOGVL_INTR_FAIL
Passing this flag to rb_nogvl() prevents it from processing interrupts after the given function retur...
Definition thread.h:50
rb_nogvl
void * rb_nogvl(void *(*func)(void *), void *data1, rb_unblock_function_t *ubf, void *data2, int flags)
Identical to rb_thread_call_without_gvl(), except it additionally takes "flags" that change the behav...
Definition thread.c:1773
MEMCPY
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:372
ALLOCA_N
#define ALLOCA_N(type, n)
Definition memory.h:292
RB_GC_GUARD
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:167
MEMMOVE
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
Definition memory.h:384
ruby::backward::cxxanyargs::type
VALUE type(ANYARGS)
ANYARGS-ed function type.
Definition cxxanyargs.hpp:56
RARRAY_LEN
#define RARRAY_LEN
Just another name of rb_array_len.
Definition rarray.h:50
RARRAY_LENINT
static int RARRAY_LENINT(VALUE ary)
Identical to rb_array_len(), except it differs for the return type.
Definition rarray.h:280
RARRAY_PTR
static VALUE * RARRAY_PTR(VALUE ary)
Wild use of a C pointer.
Definition rarray.h:365
RARRAY_AREF
#define RARRAY_AREF(a, i)
Definition rarray.h:402
RARRAY_CONST_PTR
#define RARRAY_CONST_PTR
Just another name of rb_array_const_ptr.
Definition rarray.h:51
RBASIC_CLASS
static VALUE RBASIC_CLASS(VALUE obj)
Queries the class of an object.
Definition rbasic.h:166
RBASIC
#define RBASIC(obj)
Convenient casting macro.
Definition rbasic.h:40
RCLASS_SUPER
#define RCLASS_SUPER
Just another name of rb_class_get_superclass.
Definition rclass.h:44
RHASH_SIZE
#define RHASH_SIZE(h)
Queries the size of the hash.
Definition rhash.h:57
RHASH_EMPTY_P
#define RHASH_EMPTY_P(h)
Checks if the hash is empty.
Definition rhash.h:67
RSTRING_LENINT
static int RSTRING_LENINT(VALUE str)
Identical to RSTRING_LEN(), except it differs for the return type.
Definition rstring.h:438
StringValueCStr
#define StringValueCStr(v)
Identical to StringValuePtr, except it additionally checks for the contents for viability as a C stri...
Definition rstring.h:89
RB_PASS_KEYWORDS
#define RB_PASS_KEYWORDS
Pass keywords, final argument must be a hash of keywords.
Definition scan_args.h:72
RB_NO_KEYWORDS
#define RB_NO_KEYWORDS
Do not pass keywords.
Definition scan_args.h:69
RB_SPECIAL_CONST_P
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
Definition special_consts.h:327
RTEST
#define RTEST
This is an old name of RB_TEST.
Definition special_consts.h:51
ANYARGS
#define ANYARGS
Functions declared using this macro take arbitrary arguments, including void.
Definition stdarg.h:64
RArray
Ruby's array.
Definition rarray.h:127
RArray::ary
const VALUE ary[1]
Embedded elements.
Definition rarray.h:187
RArray::ptr
const VALUE * ptr
Pointer to the C array that holds the elements of the array.
Definition rarray.h:174
RHash
Definition hash.h:54
iseq_catch_table_entry
Definition iseq.h:368
iseq_catch_table
Definition iseq.h:379
iseq_inline_constant_cache_entry
Definition vm_core.h:258
iseq_inline_constant_cache
Definition vm_core.h:271
iseq_inline_constant_cache::segments
const ID * segments
A null-terminated list of ids, used to represent a constant's path idNULL is used to represent the ::...
Definition vm_core.h:285
iseq_inline_cvar_cache_entry
Definition vm_core.h:293
iseq_inline_iv_cache_entry
Definition vm_core.h:288
rb_block
Definition vm_core.h:939
rb_box_struct
Internal header for Ruby Box.
Definition box.h:14
rb_builtin_function
Definition builtin.h:6
rb_call_data
Definition vm_callinfo.h:606
rb_callable_method_entry_struct
Definition method.h:63
rb_callcache
Definition vm_callinfo.h:292
rb_callinfo_kwarg
Definition vm_callinfo.h:48
rb_callinfo
Definition vm_callinfo.h:79
rb_calling_info
Definition vm_core.h:306
rb_captured_block
Definition vm_core.h:915
rb_class_cc_entries
Definition vm_callinfo.h:611
rb_const_entry_struct
Definition constant.h:33
rb_control_frame_struct
Definition vm_core.h:948
rb_cref_struct
CREF (Class REFerence)
Definition method.h:45
rb_cvar_class_tbl_entry
Internal header for Class.
Definition class.h:31
rb_env_t
Definition vm_core.h:1400
rb_execution_context_struct
Definition vm_core.h:1082
rb_forwarding_call_data
Definition vm_insnhelper.h:69
rb_getivar_cache
Definition shape.h:595
rb_hook_list_struct
Definition vm_core.h:684
rb_id_table
Definition id_table.h:7
rb_iseq_constant_body
Definition vm_core.h:430
rb_iseq_struct
Definition vm_core.h:595
rb_method_cfunc_struct
Definition method.h:148
rb_method_definition_struct
Definition method.h:187
rb_method_entry_struct
Definition method.h:55
rb_method_iseq_struct::cref
rb_cref_t * cref
class reference, should be marked
Definition method.h:144
rb_method_iseq_struct::iseqptr
const rb_iseq_t * iseqptr
iseq pointer, should be separated from iseqval
Definition method.h:143
rb_ractor_struct
Definition ractor_core.h:98
rb_setivar_cache
Definition shape.h:637
rb_thread_struct
Definition vm_core.h:1183
rb_vm_struct
Definition vm_core.h:709
set_table
Definition set_table.h:10
st_table
Definition st.h:79
vm_ifunc
IFUNC (Internal FUNCtion)
Definition imemo.h:89
vm_once_wait_arg
Definition vm_insnhelper.c:5717
vm_svar
SVAR (Special VARiable)
Definition imemo.h:54
vm_svar::cref_or_me
const VALUE cref_or_me
class reference or rb_method_entry_t
Definition imemo.h:56
vm_throw_data
THROW_DATA.
Definition imemo.h:63
rb_native_mutex_lock
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
rb_native_cond_broadcast
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
rb_native_mutex_unlock
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
rb_native_cond_wait
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.
__attribute__
Definition vm_insnhelper.c:2401
iseq_inline_storage_entry
Definition vm_core.h:297
rb_proc_t
Definition vm_core.h:1370
rb_len_t
long rb_len_t
A signed integer type for lengths and indices of objects such as String and Array.
Definition value.h:131
SIGNED_VALUE
intptr_t SIGNED_VALUE
A signed integer type that has the same width with VALUE.
Definition value.h:63
ID
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
SIZEOF_VALUE
#define SIZEOF_VALUE
Identical to sizeof(VALUE), except it is a macro that can also be used inside of preprocessor directi...
Definition value.h:69
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
RB_BUILTIN_TYPE
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
Definition value_type.h:182
RB_FLOAT_TYPE_P
static bool RB_FLOAT_TYPE_P(VALUE obj)
Queries if the object is an instance of rb_cFloat.
Definition value_type.h:264
RB_TYPE_P
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:376
Generated by doxygen 1.9.8