Ruby 4.1.0dev (2026-10-01 revision 1c03c515fa595e0f5783b81c30e3bfcaf23a8935)
vm_insnhelper.c (1c03c515fa595e0f5783b81c30e3bfcaf23a8935)
1/**********************************************************************
2
3 vm_insnhelper.c - instruction helper functions. Included into vm.c.
4
5 $Author$
6
7 Copyright (C) 2007 Koichi Sasada
8
9**********************************************************************/
10
11#include "ruby/internal/config.h"
12
13#include <math.h>
14
15#ifdef HAVE_STDATOMIC_H
16 #include <stdatomic.h>
17#endif
18
19#include "constant.h"
20#include "debug_counter.h"
21#include "internal.h"
22#include "internal/class.h"
23#include "internal/compar.h"
24#include "internal/hash.h"
25#include "internal/numeric.h"
26#include "internal/proc.h"
27#include "internal/random.h"
28#include "internal/variable.h"
29#include "internal/set.h"
30#include "internal/set_table.h"
31#include "internal/struct.h"
32#include "ruby/thread.h"
33#include "variable.h"
34
35/* finish iseq array */
36#include "insns.inc"
37#include "insns_info.inc"
38
39extern rb_method_definition_t *rb_method_definition_create(rb_method_type_t type, ID mid);
40extern void rb_method_definition_set(const rb_method_entry_t *me, rb_method_definition_t *def, void *opts);
41extern int rb_method_definition_eq(const rb_method_definition_t *d1, const rb_method_definition_t *d2);
42extern VALUE rb_make_no_method_exception(VALUE exc, VALUE format, VALUE obj,
43 int argc, const VALUE *argv, int priv);
44
45static const struct rb_callcache vm_empty_cc;
46static const struct rb_callcache vm_empty_cc_for_super;
47
48/* control stack frame */
49
50static rb_control_frame_t *vm_get_ruby_level_caller_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp);
51
52VALUE
53ruby_vm_special_exception_copy(VALUE exc)
54{
55 VALUE e = rb_obj_alloc(rb_class_real(RBASIC_CLASS(exc)));
56 rb_obj_copy_ivar(e, exc);
57 return e;
58}
59
60NORETURN(static void ec_stack_overflow(rb_execution_context_t *ec, int));
61static void
62ec_stack_overflow(rb_execution_context_t *ec, int setup)
63{
64 VALUE mesg = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_sysstack];
65 ec->raised_flag = RAISED_STACKOVERFLOW;
66 if (setup) {
67 VALUE at = rb_ec_backtrace_object(ec);
68 mesg = ruby_vm_special_exception_copy(mesg);
69 rb_ivar_set(mesg, idBt, at);
70 rb_ivar_set(mesg, idBt_locations, at);
71 }
72 ec->errinfo = mesg;
73 EC_JUMP_TAG(ec, TAG_RAISE);
74}
75
76NORETURN(static void vm_stackoverflow(void));
77
78static void
79vm_stackoverflow(void)
80{
81 ec_stack_overflow(GET_EC(), TRUE);
82}
83
84void
85rb_ec_stack_overflow(rb_execution_context_t *ec, ruby_stack_overflow_critical_level crit)
86{
87 if (rb_during_gc()) {
88 rb_bug("system stack overflow during GC. Faulty native extension?");
89 }
90 if (crit >= rb_stack_overflow_fatal) {
91 ec->raised_flag = RAISED_STACKOVERFLOW;
92 ec->errinfo = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_stackfatal];
93 EC_JUMP_TAG(ec, TAG_RAISE);
94 }
95 ec_stack_overflow(ec, crit < rb_stack_overflow_signal);
96}
97
98static inline void stack_check(rb_execution_context_t *ec);
99
100#if VM_CHECK_MODE > 0
101static int
102callable_class_p(VALUE klass)
103{
104#if VM_CHECK_MODE >= 2
105 if (!klass) return FALSE;
106 switch (RB_BUILTIN_TYPE(klass)) {
107 default:
108 break;
109 case T_ICLASS:
110 if (!RB_TYPE_P(RCLASS_SUPER(klass), T_MODULE)) break;
111 case T_MODULE:
112 return TRUE;
113 }
114 while (klass) {
115 if (klass == rb_cBasicObject) {
116 return TRUE;
117 }
118 klass = RCLASS_SUPER(klass);
119 }
120 return FALSE;
121#else
122 return klass != 0;
123#endif
124}
125
126static int
127callable_method_entry_p(const rb_callable_method_entry_t *cme)
128{
129 if (cme == NULL) {
130 return TRUE;
131 }
132 else {
133 VM_ASSERT(IMEMO_TYPE_P((VALUE)cme, imemo_ment), "imemo_type:%s", rb_imemo_name(imemo_type((VALUE)cme)));
134
135 if (callable_class_p(cme->defined_class)) {
136 return TRUE;
137 }
138 else {
139 return FALSE;
140 }
141 }
142}
143
144static void
145vm_check_frame_detail(VALUE type, int req_block, int req_me, int req_cref, VALUE specval, VALUE cref_or_me, int is_cframe, const rb_iseq_t *iseq)
146{
147 unsigned int magic = (unsigned int)(type & VM_FRAME_MAGIC_MASK);
148 enum imemo_type cref_or_me_type = imemo_env; /* impossible value */
149
150 if (RB_TYPE_P(cref_or_me, T_IMEMO)) {
151 cref_or_me_type = imemo_type(cref_or_me);
152 }
153 if (type & VM_FRAME_FLAG_BMETHOD) {
154 req_me = TRUE;
155 }
156
157 if (req_block && (type & VM_ENV_FLAG_LOCAL) == 0) {
158 rb_bug("vm_push_frame: specval (%p) should be a block_ptr on %x frame", (void *)specval, magic);
159 }
160 if (!req_block && (type & VM_ENV_FLAG_LOCAL) != 0) {
161 rb_bug("vm_push_frame: specval (%p) should not be a block_ptr on %x frame", (void *)specval, magic);
162 }
163
164 if (req_me) {
165 if (cref_or_me_type != imemo_ment) {
166 rb_bug("vm_push_frame: (%s) should be method entry on %x frame", rb_obj_info(cref_or_me), magic);
167 }
168 }
169 else {
170 if (req_cref && cref_or_me_type != imemo_cref) {
171 rb_bug("vm_push_frame: (%s) should be CREF on %x frame", rb_obj_info(cref_or_me), magic);
172 }
173 else { /* cref or Qfalse */
174 if (cref_or_me != Qfalse && cref_or_me_type != imemo_cref) {
175 if (((type & VM_FRAME_FLAG_LAMBDA) || magic == VM_FRAME_MAGIC_IFUNC || magic == VM_FRAME_MAGIC_DUMMY) && (cref_or_me_type == imemo_ment)) {
176 /* ignore */
177 }
178 else {
179 rb_bug("vm_push_frame: (%s) should be false or cref on %x frame", rb_obj_info(cref_or_me), magic);
180 }
181 }
182 }
183 }
184
185 if (cref_or_me_type == imemo_ment) {
186 const rb_callable_method_entry_t *me = (const rb_callable_method_entry_t *)cref_or_me;
187
188 if (!callable_method_entry_p(me)) {
189 rb_bug("vm_push_frame: ment (%s) should be callable on %x frame.", rb_obj_info(cref_or_me), magic);
190 }
191 }
192
193 if ((type & VM_FRAME_MAGIC_MASK) == VM_FRAME_MAGIC_DUMMY) {
194 VM_ASSERT(iseq == NULL ||
195 RBASIC_CLASS((VALUE)iseq) == 0 || // dummy frame for loading
196 RUBY_VM_NORMAL_ISEQ_P(iseq) //argument error
197 );
198 }
199 else {
200 VM_ASSERT(is_cframe == !RUBY_VM_NORMAL_ISEQ_P(iseq));
201 }
202}
203
204static void
205vm_check_frame(VALUE type,
206 VALUE specval,
207 VALUE cref_or_me,
208 const rb_iseq_t *iseq)
209{
210 VALUE given_magic = type & VM_FRAME_MAGIC_MASK;
211 VM_ASSERT(FIXNUM_P(type));
212
213#define CHECK(magic, req_block, req_me, req_cref, is_cframe) \
214 case magic: \
215 vm_check_frame_detail(type, req_block, req_me, req_cref, \
216 specval, cref_or_me, is_cframe, iseq); \
217 break
218 switch (given_magic) {
219 /* BLK ME CREF CFRAME */
220 CHECK(VM_FRAME_MAGIC_METHOD, TRUE, TRUE, FALSE, FALSE);
221 CHECK(VM_FRAME_MAGIC_CLASS, TRUE, FALSE, TRUE, FALSE);
222 CHECK(VM_FRAME_MAGIC_TOP, TRUE, FALSE, TRUE, FALSE);
223 CHECK(VM_FRAME_MAGIC_CFUNC, TRUE, TRUE, FALSE, TRUE);
224 CHECK(VM_FRAME_MAGIC_BLOCK, FALSE, FALSE, FALSE, FALSE);
225 CHECK(VM_FRAME_MAGIC_IFUNC, FALSE, FALSE, FALSE, TRUE);
226 CHECK(VM_FRAME_MAGIC_EVAL, FALSE, FALSE, FALSE, FALSE);
227 CHECK(VM_FRAME_MAGIC_RESCUE, FALSE, FALSE, FALSE, FALSE);
228 CHECK(VM_FRAME_MAGIC_DUMMY, TRUE, FALSE, FALSE, FALSE);
229 default:
230 rb_bug("vm_push_frame: unknown type (%x)", (unsigned int)given_magic);
231 }
232#undef CHECK
233}
234
235static VALUE vm_stack_canary; /* Initialized later */
236static bool vm_stack_canary_was_born = false;
237
238// Return the index of the instruction right before the given PC.
239// This is needed because insn_entry advances PC before the insn body.
240static unsigned int
241previous_insn_index(const rb_iseq_t *iseq, const VALUE *pc)
242{
243 unsigned int pos = 0;
244 while (pos < ISEQ_BODY(iseq)->iseq_size) {
245 int opcode = rb_vm_insn_addr2opcode((void *)ISEQ_BODY(iseq)->iseq_encoded[pos]);
246 unsigned int next_pos = pos + insn_len(opcode);
247 if (ISEQ_BODY(iseq)->iseq_encoded + next_pos == pc) {
248 return pos;
249 }
250 pos = next_pos;
251 }
252 rb_bug("failed to find the previous insn");
253}
254
255void
256rb_vm_check_canary(const rb_execution_context_t *ec, VALUE *sp)
257{
258 const struct rb_control_frame_struct *reg_cfp = ec->cfp;
259 const struct rb_iseq_struct *iseq;
260
261 if (! LIKELY(vm_stack_canary_was_born)) {
262 return; /* :FIXME: isn't it rather fatal to enter this branch? */
263 }
264 else if ((VALUE *)reg_cfp == ec->vm_stack + ec->vm_stack_size) {
265 /* This is at the very beginning of a thread. cfp does not exist. */
266 return;
267 }
268 else if (! VM_FRAME_RUBYFRAME_P(reg_cfp) || ! (iseq = GET_ISEQ())) {
269 /* The iseq field of an IFUNC frame holds the ifunc. */
270 return;
271 }
272 else if (LIKELY(sp[0] != vm_stack_canary)) {
273 return;
274 }
275 else {
276 /* we are going to call methods below; squash the canary to
277 * prevent infinite loop. */
278 sp[0] = Qundef;
279 }
280
281 const VALUE *orig = rb_iseq_original_iseq(iseq);
282 const VALUE iseqw = rb_iseqw_new(iseq);
283 const VALUE inspection = rb_inspect(iseqw);
284 const char *stri = rb_str_to_cstr(inspection);
285 const VALUE disasm = rb_iseq_disasm(iseq);
286 const char *strd = rb_str_to_cstr(disasm);
287 const ptrdiff_t pos = previous_insn_index(iseq, GET_PC());
288 const enum ruby_vminsn_type insn = (enum ruby_vminsn_type)orig[pos];
289 const char *name = insn_name(insn);
290
291 /* rb_bug() is not capable of outputting this large contents. It
292 is designed to run form a SIGSEGV handler, which tends to be
293 very restricted. */
294 ruby_debug_printf(
295 "We are killing the stack canary set by %s, "
296 "at %s@pc=%"PRIdPTR"\n"
297 "watch out the C stack trace.\n"
298 "%s",
299 name, stri, pos, strd);
300 rb_bug("see above.");
301}
302#define vm_check_canary(ec, sp) rb_vm_check_canary(ec, sp)
303
304#else
305#define vm_check_canary(ec, sp)
306#define vm_check_frame(a, b, c, d)
307#endif /* VM_CHECK_MODE > 0 */
308
309#if USE_DEBUG_COUNTER
310static void
311vm_push_frame_debug_counter_inc(
312 const struct rb_execution_context_struct *ec,
313 const struct rb_control_frame_struct *reg_cfp,
314 VALUE type)
315{
316 const struct rb_control_frame_struct *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(reg_cfp);
317
318 RB_DEBUG_COUNTER_INC(frame_push);
319
320 if (RUBY_VM_END_CONTROL_FRAME(ec) != prev_cfp) {
321 const bool curr = VM_FRAME_RUBYFRAME_P(reg_cfp);
322 const bool prev = VM_FRAME_RUBYFRAME_P(prev_cfp);
323 if (prev) {
324 if (curr) {
325 RB_DEBUG_COUNTER_INC(frame_R2R);
326 }
327 else {
328 RB_DEBUG_COUNTER_INC(frame_R2C);
329 }
330 }
331 else {
332 if (curr) {
333 RB_DEBUG_COUNTER_INC(frame_C2R);
334 }
335 else {
336 RB_DEBUG_COUNTER_INC(frame_C2C);
337 }
338 }
339 }
340
341 switch (type & VM_FRAME_MAGIC_MASK) {
342 case VM_FRAME_MAGIC_METHOD: RB_DEBUG_COUNTER_INC(frame_push_method); return;
343 case VM_FRAME_MAGIC_BLOCK: RB_DEBUG_COUNTER_INC(frame_push_block); return;
344 case VM_FRAME_MAGIC_CLASS: RB_DEBUG_COUNTER_INC(frame_push_class); return;
345 case VM_FRAME_MAGIC_TOP: RB_DEBUG_COUNTER_INC(frame_push_top); return;
346 case VM_FRAME_MAGIC_CFUNC: RB_DEBUG_COUNTER_INC(frame_push_cfunc); return;
347 case VM_FRAME_MAGIC_IFUNC: RB_DEBUG_COUNTER_INC(frame_push_ifunc); return;
348 case VM_FRAME_MAGIC_EVAL: RB_DEBUG_COUNTER_INC(frame_push_eval); return;
349 case VM_FRAME_MAGIC_RESCUE: RB_DEBUG_COUNTER_INC(frame_push_rescue); return;
350 case VM_FRAME_MAGIC_DUMMY: RB_DEBUG_COUNTER_INC(frame_push_dummy); return;
351 }
352
353 rb_bug("unreachable");
354}
355#else
356#define vm_push_frame_debug_counter_inc(ec, cfp, t) /* void */
357#endif
358
359// Return a poison value to be set above the stack top to verify leafness.
360VALUE
361rb_vm_stack_canary(void)
362{
363#if VM_CHECK_MODE > 0
364 return vm_stack_canary;
365#else
366 return 0;
367#endif
368}
369
370STATIC_ASSERT(VM_ENV_DATA_INDEX_ME_CREF, VM_ENV_DATA_INDEX_ME_CREF == -2);
371STATIC_ASSERT(VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL == -1);
372STATIC_ASSERT(VM_ENV_DATA_INDEX_FLAGS, VM_ENV_DATA_INDEX_FLAGS == -0);
373
374static void
375vm_push_frame(rb_execution_context_t *ec,
376 const rb_iseq_t *iseq,
377 VALUE type,
378 VALUE self,
379 VALUE specval,
380 VALUE cref_or_me,
381 const VALUE *pc,
382 VALUE *sp,
383 int local_size,
384 int stack_max)
385{
386 rb_control_frame_t *const cfp = RUBY_VM_NEXT_CONTROL_FRAME(ec->cfp);
387
388 vm_check_frame(type, specval, cref_or_me, iseq);
389 VM_ASSERT(local_size >= 0);
390
391 /* check stack overflow */
392 CHECK_VM_STACK_OVERFLOW0(cfp, sp, local_size + stack_max);
393 vm_check_canary(ec, sp);
394
395 /* setup vm value stack */
396
397 /* initialize local variables */
398 for (int i=0; i < local_size; i++) {
399 *sp++ = Qnil;
400 }
401
402 /* setup ep with managing data */
403 *sp++ = cref_or_me; /* ep[-2] / Qnil or T_IMEMO(cref) or T_IMEMO(ment) */
404 *sp++ = specval /* ep[-1] / block handler or prev env ptr */;
405 *sp++ = type; /* ep[-0] / ENV_FLAGS */
406
407 /* setup new frame */
408 *cfp = (const struct rb_control_frame_struct) {
409 .pc = pc,
410 .sp = sp,
411 ._iseq = iseq,
412 .self = self,
413 .ep = sp - 1,
414 .block_code = NULL,
415#if VM_DEBUG_BP_CHECK
416 .bp_check = sp,
417#endif
418 .jit_return = NULL,
419 };
420
421 /* Ensure the initialization of `*cfp` above never gets reordered with the update of `ec->cfp` below.
422 This is a no-op in all cases we've looked at (https://godbolt.org/z/3oxd1446K), but should guarantee it for all
423 future/untested compilers/platforms. */
424
425 #if defined HAVE_DECL_ATOMIC_SIGNAL_FENCE && HAVE_DECL_ATOMIC_SIGNAL_FENCE
426 atomic_signal_fence(memory_order_seq_cst);
427 #endif
428
429 ec->cfp = cfp;
430
431 if (VMDEBUG == 2) {
432 SDR();
433 }
434 vm_push_frame_debug_counter_inc(ec, cfp, type);
435}
436
437void
438rb_vm_pop_frame_no_int(rb_execution_context_t *ec)
439{
440 rb_control_frame_t *cfp = ec->cfp;
441
442 if (VMDEBUG == 2) SDR();
443
444 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
445}
446
447/* return TRUE if the frame is finished */
448static inline int
449vm_pop_frame(rb_execution_context_t *ec, rb_control_frame_t *cfp, const VALUE *ep)
450{
451 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
452
453 if (VMDEBUG == 2) SDR();
454
455 RUBY_VM_CHECK_INTS(ec);
456 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
457
458 return flags & VM_FRAME_FLAG_FINISH;
459}
460
461void
462rb_vm_pop_frame(rb_execution_context_t *ec)
463{
464 vm_pop_frame(ec, ec->cfp, ec->cfp->ep);
465}
466
467// it pushes pseudo-frame with fname filename.
468VALUE
469rb_vm_push_frame_fname(rb_execution_context_t *ec, VALUE fname)
470{
471 rb_iseq_t *rb_iseq_alloc_with_dummy_path(VALUE fname);
472 rb_iseq_t *dmy_iseq = rb_iseq_alloc_with_dummy_path(fname);
473
474 vm_push_frame(ec,
475 dmy_iseq, //const rb_iseq_t *iseq,
476 VM_FRAME_MAGIC_DUMMY | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, // VALUE type,
477 ec->cfp->self, // VALUE self,
478 VM_BLOCK_HANDLER_NONE, // VALUE specval,
479 Qfalse, // VALUE cref_or_me,
480 NULL, // const VALUE *pc,
481 ec->cfp->sp, // VALUE *sp,
482 0, // int local_size,
483 0); // int stack_max
484
485 return (VALUE)dmy_iseq;
486}
487
488/* method dispatch */
489static inline VALUE
490rb_arity_error_new(int argc, int min, int max)
491{
492 VALUE err_mess = rb_sprintf("wrong number of arguments (given %d, expected %d", argc, min);
493 if (min == max) {
494 /* max is not needed */
495 }
496 else if (max == UNLIMITED_ARGUMENTS) {
497 rb_str_cat_cstr(err_mess, "+");
498 }
499 else {
500 rb_str_catf(err_mess, "..%d", max);
501 }
502 rb_str_cat_cstr(err_mess, ")");
503 return rb_exc_new3(rb_eArgError, err_mess);
504}
505
506void
507rb_error_arity(int argc, int min, int max)
508{
509 rb_exc_raise(rb_arity_error_new(argc, min, max));
510}
511
512/* lvar */
513
514NOINLINE(static void vm_env_write_slowpath(const VALUE *ep, int index, VALUE v));
515
516static void
517vm_env_write_slowpath(const VALUE *ep, int index, VALUE v)
518{
519 const VALUE envval = VM_ENV_ENVVAL(ep);
520
521 if (RB_FL_TEST_RAW(envval, RUBY_FL_SHAREABLE)) {
522 /* Writing to a SHAREABLE env (an isolated proc) can create a shareable ->
523 * unshareable edge; only a full barrier sets the shref bit keeping v alive
524 * through its owner's local GC. WB_REQUIRED stays: later writes need it too. */
525 if (!SPECIAL_CONST_P(v)) {
526 rb_gc_writebarrier(envval, v);
527 }
528 VM_FORCE_WRITE(&ep[index], v);
529 }
530 else {
531 /* remember env value forcely */
532 rb_gc_writebarrier_remember(envval);
533 VM_FORCE_WRITE(&ep[index], v);
534 VM_ENV_FLAGS_UNSET(ep, VM_ENV_FLAG_WB_REQUIRED);
535 }
536 RB_DEBUG_COUNTER_INC(lvar_set_slowpath);
537}
538
539// YJIT assumes this function never runs GC
540static inline void
541vm_env_write(const VALUE *ep, int index, VALUE v)
542{
543 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
544 if (LIKELY((flags & VM_ENV_FLAG_WB_REQUIRED) == 0)) {
545 VM_STACK_ENV_WRITE(ep, index, v);
546 }
547 else {
548 vm_env_write_slowpath(ep, index, v);
549 }
550}
551
552void
553rb_vm_env_write(const VALUE *ep, int index, VALUE v)
554{
555 vm_env_write(ep, index, v);
556}
557
558VALUE
559rb_vm_bh_to_procval(const rb_execution_context_t *ec, VALUE block_handler)
560{
561 if (block_handler == VM_BLOCK_HANDLER_NONE) {
562 return Qnil;
563 }
564 else {
565 switch (vm_block_handler_type(block_handler)) {
566 case block_handler_type_iseq:
567 case block_handler_type_ifunc:
568 return rb_vm_make_proc(ec, VM_BH_TO_CAPT_BLOCK(block_handler), rb_cProc);
569 case block_handler_type_symbol:
570 return rb_sym_to_proc(VM_BH_TO_SYMBOL(block_handler));
571 case block_handler_type_proc:
572 return VM_BH_TO_PROC(block_handler);
573 default:
574 VM_UNREACHABLE(rb_vm_bh_to_procval);
575 }
576 }
577}
578
579/* svar */
580
581#if VM_CHECK_MODE > 0
582static int
583vm_svar_valid_p(VALUE svar)
584{
585 if (RB_TYPE_P((VALUE)svar, T_IMEMO)) {
586 switch (imemo_type(svar)) {
587 case imemo_svar:
588 case imemo_cref:
589 case imemo_ment:
590 return TRUE;
591 default:
592 break;
593 }
594 }
595 rb_bug("vm_svar_valid_p: unknown type: %s", rb_obj_info(svar));
596 return FALSE;
597}
598#endif
599
600/* Should this frame's special variables live in the env's svar slot? A SHAREABLE env
601 * (isolated proc) can run in several Ractors at once, making svar shared mutable state
602 * leaking $~/$_ across Ractors: keep them per-EC instead. */
603static inline bool
604lep_svar_in_env_p(const rb_execution_context_t *ec, const VALUE *lep)
605{
606 if (!lep) return false;
607 if (ec == NULL) return true;
608 if (ec->root_lep == lep) return false;
609 /* lep may be a stale on-stack ep of a frame whose env already escaped: lep[0]
610 * still holds the imemo_env, so flags is not a FIXNUM and VM_ENV_ESCAPED_P
611 * asserts. That is not a live shareable proc's env, so fall back to in-env. */
612 if (FIXNUM_P(lep[VM_ENV_DATA_INDEX_FLAGS]) &&
613 VM_ENV_ESCAPED_P(lep) &&
614 RB_FL_TEST_RAW(VM_ENV_ENVVAL(lep), RUBY_FL_SHAREABLE)) {
615 return false;
616 }
617 return true;
618}
619
620static inline struct vm_svar *
621lep_svar(const rb_execution_context_t *ec, const VALUE *lep)
622{
623 VALUE svar;
624
625 if (lep_svar_in_env_p(ec, lep)) {
626 svar = lep[VM_ENV_DATA_INDEX_ME_CREF];
627 }
628 else {
629 svar = ec->root_svar;
630 }
631
632 VM_ASSERT(svar == Qfalse || vm_svar_valid_p(svar));
633
634 return (struct vm_svar *)svar;
635}
636
637static inline void
638lep_svar_write(const rb_execution_context_t *ec, const VALUE *lep, const struct vm_svar *svar)
639{
640 VM_ASSERT(vm_svar_valid_p((VALUE)svar));
641
642 if (lep_svar_in_env_p(ec, lep)) {
643 vm_env_write(lep, VM_ENV_DATA_INDEX_ME_CREF, (VALUE)svar);
644 }
645 else {
646 RB_OBJ_WRITE(rb_ec_thread_ptr(ec)->self, &ec->root_svar, svar);
647 }
648}
649
650static VALUE
651lep_svar_get(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key)
652{
653 const struct vm_svar *svar = lep_svar(ec, lep);
654
655 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) return Qnil;
656
657 switch (key) {
658 case VM_SVAR_LASTLINE:
659 return svar->lastline;
660 case VM_SVAR_BACKREF:
661 return svar->backref;
662 default: {
663 const VALUE ary = svar->others;
664
665 if (NIL_P(ary)) {
666 return Qnil;
667 }
668 else {
669 return rb_ary_entry(ary, key - VM_SVAR_EXTRA_START);
670 }
671 }
672 }
673}
674
675static struct vm_svar *
676svar_new(VALUE obj)
677{
678 struct vm_svar *svar = IMEMO_NEW(struct vm_svar, imemo_svar, obj);
679 *((VALUE *)&svar->lastline) = Qnil;
680 *((VALUE *)&svar->backref) = Qnil;
681 *((VALUE *)&svar->others) = Qnil;
682
683 return svar;
684}
685
686static void
687lep_svar_set(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, VALUE val)
688{
689 struct vm_svar *svar = lep_svar(ec, lep);
690
691 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) {
692 lep_svar_write(ec, lep, svar = svar_new((VALUE)svar));
693 }
694
695 switch (key) {
696 case VM_SVAR_LASTLINE:
697 RB_OBJ_WRITE(svar, &svar->lastline, val);
698 return;
699 case VM_SVAR_BACKREF:
700 RB_OBJ_WRITE(svar, &svar->backref, val);
701 return;
702 default: {
703 VALUE ary = svar->others;
704
705 if (NIL_P(ary)) {
706 RB_OBJ_WRITE(svar, &svar->others, ary = rb_ary_new());
707 }
708 rb_ary_store(ary, key - VM_SVAR_EXTRA_START, val);
709 }
710 }
711}
712
713static inline VALUE
714vm_getspecial(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, rb_num_t type)
715{
716 VALUE val;
717
718 if (type == 0) {
719 val = lep_svar_get(ec, lep, key);
720 }
721 else {
722 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
723
724 if (type & 0x01) {
725 switch (type >> 1) {
726 case '&':
727 val = rb_reg_last_match(backref);
728 break;
729 case '`':
730 val = rb_reg_match_pre(backref);
731 break;
732 case '\'':
733 val = rb_reg_match_post(backref);
734 break;
735 case '+':
736 val = rb_reg_match_last(backref);
737 break;
738 default:
739 rb_bug("unexpected back-ref");
740 }
741 }
742 else {
743 val = rb_reg_nth_match((int)(type >> 1), backref);
744 }
745 }
746 return val;
747}
748
749static inline VALUE
750vm_backref_defined(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t type)
751{
752 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
753 int nth = 0;
754
755 if (type & 0x01) {
756 switch (type >> 1) {
757 case '&':
758 case '`':
759 case '\'':
760 break;
761 case '+':
762 return rb_reg_last_defined(backref);
763 default:
764 rb_bug("unexpected back-ref");
765 }
766 }
767 else {
768 nth = (int)(type >> 1);
769 }
770 return rb_reg_nth_defined(nth, backref);
771}
772
773PUREFUNC(static rb_callable_method_entry_t *check_method_entry(VALUE obj, int can_be_svar));
774static rb_callable_method_entry_t *
775check_method_entry(VALUE obj, int can_be_svar)
776{
777 if (obj == Qfalse) return NULL;
778
779#if VM_CHECK_MODE > 0
780 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_method_entry: unknown type: %s", rb_obj_info(obj));
781#endif
782
783 switch (imemo_type(obj)) {
784 case imemo_ment:
785 return (rb_callable_method_entry_t *)obj;
786 case imemo_cref:
787 return NULL;
788 case imemo_svar:
789 if (can_be_svar) {
790 return check_method_entry(((struct vm_svar *)obj)->cref_or_me, FALSE);
791 }
792 default:
793#if VM_CHECK_MODE > 0
794 rb_bug("check_method_entry: svar should not be there:");
795#endif
796 return NULL;
797 }
798}
799
800static rb_callable_method_entry_t *
801env_method_entry_unchecked(VALUE obj, int can_be_svar)
802{
803 if (obj == Qfalse) return NULL;
804
805 switch (imemo_type(obj)) {
806 case imemo_ment:
807 return (rb_callable_method_entry_t *)obj;
808 case imemo_cref:
809 return NULL;
810 case imemo_svar:
811 if (can_be_svar) {
812 return env_method_entry_unchecked(((struct vm_svar *)obj)->cref_or_me, FALSE);
813 }
814 default:
815 return NULL;
816 }
817}
818
819const rb_callable_method_entry_t *
820rb_vm_frame_method_entry(const rb_control_frame_t *cfp)
821{
822 const VALUE *ep = cfp->ep;
823 rb_callable_method_entry_t *me;
824
825 while (!VM_ENV_LOCAL_P(ep)) {
826 if ((me = check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
827 ep = VM_ENV_PREV_EP(ep);
828 }
829
830 return check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
831}
832
833const rb_callable_method_entry_t *
834rb_vm_frame_method_entry_unchecked(const rb_control_frame_t *cfp)
835{
836 const VALUE *ep = cfp->ep;
837 rb_callable_method_entry_t *me;
838
839 while (!VM_ENV_LOCAL_P_UNCHECKED(ep)) {
840 if ((me = env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
841 ep = VM_ENV_PREV_EP_UNCHECKED(ep);
842 }
843
844 return env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
845}
846
847static const rb_iseq_t *
848method_entry_iseqptr(const rb_callable_method_entry_t *me)
849{
850 switch (me->def->type) {
851 case VM_METHOD_TYPE_ISEQ:
852 return me->def->body.iseq.iseqptr;
853 default:
854 return NULL;
855 }
856}
857
858static rb_cref_t *
859method_entry_cref(const rb_callable_method_entry_t *me)
860{
861 switch (me->def->type) {
862 case VM_METHOD_TYPE_ISEQ:
863 return me->def->body.iseq.cref;
864 default:
865 return NULL;
866 }
867}
868
869#if VM_CHECK_MODE == 0
870PUREFUNC(static rb_cref_t *check_cref(VALUE, int));
871#endif
872static rb_cref_t *
873check_cref(VALUE obj, int can_be_svar)
874{
875 if (obj == Qfalse) return NULL;
876
877#if VM_CHECK_MODE > 0
878 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_cref: unknown type: %s", rb_obj_info(obj));
879#endif
880
881 switch (imemo_type(obj)) {
882 case imemo_ment:
883 return method_entry_cref((rb_callable_method_entry_t *)obj);
884 case imemo_cref:
885 return (rb_cref_t *)obj;
886 case imemo_svar:
887 if (can_be_svar) {
888 return check_cref(((struct vm_svar *)obj)->cref_or_me, FALSE);
889 }
890 default:
891#if VM_CHECK_MODE > 0
892 rb_bug("check_method_entry: svar should not be there:");
893#endif
894 return NULL;
895 }
896}
897
898static inline rb_cref_t *
899vm_env_cref(const VALUE *ep)
900{
901 rb_cref_t *cref;
902
903 while (!VM_ENV_LOCAL_P(ep)) {
904 if ((cref = check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return cref;
905 ep = VM_ENV_PREV_EP(ep);
906 }
907
908 return check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
909}
910
911static int
912is_cref(const VALUE v, int can_be_svar)
913{
914 if (RB_TYPE_P(v, T_IMEMO)) {
915 switch (imemo_type(v)) {
916 case imemo_cref:
917 return TRUE;
918 case imemo_svar:
919 if (can_be_svar) return is_cref(((struct vm_svar *)v)->cref_or_me, FALSE);
920 default:
921 break;
922 }
923 }
924 return FALSE;
925}
926
927static int
928vm_env_cref_by_cref(const VALUE *ep)
929{
930 while (!VM_ENV_LOCAL_P(ep)) {
931 if (is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) return TRUE;
932 ep = VM_ENV_PREV_EP(ep);
933 }
934 return is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
935}
936
937static rb_cref_t *
938cref_replace_with_duplicated_cref_each_frame(const VALUE *vptr, int can_be_svar, VALUE parent)
939{
940 const VALUE v = *vptr;
941 rb_cref_t *cref, *new_cref;
942
943 if (RB_TYPE_P(v, T_IMEMO)) {
944 switch (imemo_type(v)) {
945 case imemo_cref:
946 cref = (rb_cref_t *)v;
947 new_cref = rb_vm_cref_dup(cref);
948 if (parent) {
949 RB_OBJ_WRITE(parent, vptr, new_cref);
950 }
951 else {
952 VM_FORCE_WRITE(vptr, (VALUE)new_cref);
953 }
954 return (rb_cref_t *)new_cref;
955 case imemo_svar:
956 if (can_be_svar) {
957 return cref_replace_with_duplicated_cref_each_frame(&((struct vm_svar *)v)->cref_or_me, FALSE, v);
958 }
959 /* fall through */
960 case imemo_ment:
961 rb_bug("cref_replace_with_duplicated_cref_each_frame: unreachable");
962 default:
963 break;
964 }
965 }
966 return NULL;
967}
968
969static rb_cref_t *
970vm_cref_replace_with_duplicated_cref(const VALUE *ep)
971{
972 if (vm_env_cref_by_cref(ep)) {
973 rb_cref_t *cref;
974 VALUE envval;
975
976 while (!VM_ENV_LOCAL_P(ep)) {
977 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
978 if ((cref = cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE, envval)) != NULL) {
979 return cref;
980 }
981 ep = VM_ENV_PREV_EP(ep);
982 }
983 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
984 return cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE, envval);
985 }
986 else {
987 rb_bug("vm_cref_dup: unreachable");
988 }
989}
990
991static rb_cref_t *
992vm_get_cref(const VALUE *ep)
993{
994 rb_cref_t *cref = vm_env_cref(ep);
995
996 if (cref != NULL) {
997 return cref;
998 }
999 else {
1000 rb_bug("vm_get_cref: unreachable");
1001 }
1002}
1003
1004rb_cref_t *
1005rb_vm_get_cref(const VALUE *ep)
1006{
1007 return vm_get_cref(ep);
1008}
1009
1010static rb_cref_t *
1011vm_ec_cref(const rb_execution_context_t *ec)
1012{
1013 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1014
1015 if (cfp == NULL) {
1016 return NULL;
1017 }
1018 return vm_get_cref(cfp->ep);
1019}
1020
1021static const rb_cref_t *
1022vm_get_const_key_cref(const VALUE *ep)
1023{
1024 const rb_cref_t *cref = vm_get_cref(ep);
1025 const rb_cref_t *key_cref = cref;
1026
1027 while (cref) {
1028 if (CREF_DYNAMIC(cref)) {
1029 return key_cref;
1030 }
1031 cref = CREF_NEXT(cref);
1032 }
1033
1034 /* no dynamic singleton class or cloned class found */
1035 return NULL;
1036}
1037
1038static rb_cref_t *
1039vm_cref_push(const rb_execution_context_t *ec, VALUE klass, const VALUE *ep, int pushed_by_eval, int singleton)
1040{
1041 rb_cref_t *prev_cref = NULL;
1042
1043 if (ep) {
1044 prev_cref = vm_env_cref(ep);
1045 }
1046 else {
1047 rb_control_frame_t *cfp = vm_get_ruby_level_caller_cfp(ec, ec->cfp);
1048
1049 if (cfp) {
1050 prev_cref = vm_env_cref(cfp->ep);
1051 }
1052 }
1053
1054 return vm_cref_new(klass, METHOD_VISI_PUBLIC, FALSE, prev_cref, pushed_by_eval, singleton);
1055}
1056
1057static inline VALUE
1058vm_get_cbase(const VALUE *ep)
1059{
1060 const rb_cref_t *cref = vm_get_cref(ep);
1061
1062 return CREF_CLASS_FOR_DEFINITION(cref);
1063}
1064
1065static inline VALUE
1066vm_get_const_base(const VALUE *ep)
1067{
1068 const rb_cref_t *cref = vm_get_cref(ep);
1069
1070 while (cref) {
1071 if (!CREF_PUSHED_BY_EVAL(cref)) {
1072 return CREF_CLASS_FOR_DEFINITION(cref);
1073 }
1074 cref = CREF_NEXT(cref);
1075 }
1076
1077 return Qundef;
1078}
1079
1080static inline void
1081vm_check_if_namespace(VALUE klass)
1082{
1083 if (!RB_TYPE_P(klass, T_CLASS) && !RB_TYPE_P(klass, T_MODULE)) {
1084 rb_raise(rb_eTypeError, "%+"PRIsVALUE" is not a class/module", klass);
1085 }
1086}
1087
1088static inline void
1089vm_ensure_not_refinement_module(VALUE self)
1090{
1091 if (RB_TYPE_P(self, T_MODULE) && FL_TEST(self, RMODULE_IS_REFINEMENT)) {
1092 rb_warn("not defined at the refinement, but at the outer class/module");
1093 }
1094}
1095
1096static inline VALUE
1097vm_get_iclass(const rb_control_frame_t *cfp, VALUE klass)
1098{
1099 return klass;
1100}
1101
1102static inline VALUE
1103vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, bool allow_nil, int is_defined)
1104{
1105 void rb_const_warn_if_deprecated(const rb_const_entry_t *ce, VALUE klass, ID id);
1106 VALUE val;
1107
1108 if (NIL_P(orig_klass) && allow_nil) {
1109 /* in current lexical scope */
1110 const rb_cref_t *root_cref = vm_get_cref(ec->cfp->ep);
1111 const rb_cref_t *cref;
1112 VALUE klass = Qnil;
1113
1114 while (root_cref && CREF_PUSHED_BY_EVAL(root_cref)) {
1115 root_cref = CREF_NEXT(root_cref);
1116 }
1117 cref = root_cref;
1118 while (cref && CREF_NEXT(cref)) {
1119 if (CREF_PUSHED_BY_EVAL(cref)) {
1120 klass = Qnil;
1121 }
1122 else {
1123 klass = CREF_CLASS(cref);
1124 }
1125 cref = CREF_NEXT(cref);
1126
1127 if (!NIL_P(klass)) {
1128 VALUE av, am = 0;
1129 rb_const_entry_t *ce;
1130 search_continue:
1131 if ((ce = rb_const_lookup(klass, id))) {
1132 rb_const_warn_if_deprecated(ce, klass, id);
1133 val = ce->value;
1134 if (UNDEF_P(val)) {
1135 if (am == klass) break;
1136 am = klass;
1137 if (is_defined) return 1;
1138 if (rb_autoloading_value(klass, id, &av, NULL)) return av;
1139 rb_autoload_load(klass, id);
1140 goto search_continue;
1141 }
1142 else {
1143 if (is_defined) {
1144 return 1;
1145 }
1146 else {
1147 if (UNLIKELY(!rb_class_owned_p(klass))) {
1148 if (!rb_ractor_shareable_p(val)) {
1149 rb_raise(rb_eRactorIsolationError,
1150 "can not access non-shareable objects in constant %"PRIsVALUE"::%"PRIsVALUE" of a class/module created by another Ractor.", rb_class_path(klass), rb_id2str(id));
1151 }
1152 }
1153 return val;
1154 }
1155 }
1156 }
1157 }
1158 }
1159
1160 /* search self */
1161 if (root_cref && !NIL_P(CREF_CLASS(root_cref))) {
1162 klass = vm_get_iclass(ec->cfp, CREF_CLASS(root_cref));
1163 }
1164 else {
1165 klass = CLASS_OF(ec->cfp->self);
1166 }
1167
1168 if (is_defined) {
1169 return rb_const_defined(klass, id);
1170 }
1171 else {
1172 return rb_const_get(klass, id);
1173 }
1174 }
1175 else {
1176 vm_check_if_namespace(orig_klass);
1177 if (is_defined) {
1178 return rb_public_const_defined_from(orig_klass, id);
1179 }
1180 else {
1181 return rb_public_const_get_from(orig_klass, id);
1182 }
1183 }
1184}
1185
1186VALUE
1187rb_vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, VALUE allow_nil)
1188{
1189 return vm_get_ev_const(ec, orig_klass, id, allow_nil == Qtrue, 0);
1190}
1191
1192static inline VALUE
1193vm_get_ev_const_chain(rb_execution_context_t *ec, const ID *segments)
1194{
1195 VALUE val = Qnil;
1196 int idx = 0;
1197 int allow_nil = TRUE;
1198 if (segments[0] == idNULL) {
1199 val = rb_cObject;
1200 idx++;
1201 allow_nil = FALSE;
1202 }
1203 while (segments[idx]) {
1204 ID id = segments[idx++];
1205 val = vm_get_ev_const(ec, val, id, allow_nil, 0);
1206 allow_nil = FALSE;
1207 }
1208 return val;
1209}
1210
1211
1212static inline VALUE
1213vm_get_cvar_base(const rb_cref_t *cref, const rb_control_frame_t *cfp, int top_level_raise)
1214{
1215 VALUE klass;
1216
1217 if (!cref) {
1218 rb_bug("vm_get_cvar_base: no cref");
1219 }
1220
1221 while (CREF_NEXT(cref) &&
1222 (NIL_P(CREF_CLASS(cref)) || RCLASS_SINGLETON_P(CREF_CLASS(cref)) ||
1223 CREF_PUSHED_BY_EVAL(cref) || CREF_SINGLETON(cref))) {
1224 cref = CREF_NEXT(cref);
1225 }
1226 if (top_level_raise && !CREF_NEXT(cref)) {
1227 rb_raise(rb_eRuntimeError, "class variable access from toplevel");
1228 }
1229
1230 klass = vm_get_iclass(cfp, CREF_CLASS(cref));
1231
1232 if (NIL_P(klass)) {
1233 rb_raise(rb_eTypeError, "no class variables available");
1234 }
1235 return klass;
1236}
1237
1238#define ractor_incidental_shareable_p(cond, val) \
1239 (!(cond) || rb_ractor_shareable_p(val))
1240#define ractor_object_incidental_shareable_p(obj, val) \
1241 ractor_incidental_shareable_p(rb_ractor_shareable_p(obj), val)
1242
1243ALWAYS_INLINE(static VALUE vm_getivar(VALUE, ID, const rb_iseq_t *, IVC, const struct rb_callcache *, int, VALUE));
1244static inline VALUE
1245vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, VALUE default_value)
1246{
1247 VALUE fields_obj;
1248#if OPT_IC_FOR_IVAR
1249 if (SPECIAL_CONST_P(obj)) {
1250 return default_value;
1251 }
1252
1253 switch (BUILTIN_TYPE(obj)) {
1254 case T_OBJECT:
1255 fields_obj = ROBJECT_FIELDS_OBJ(obj);
1256 break;
1257 case T_CLASS:
1258 case T_MODULE:
1259 {
1260 if (UNLIKELY(!rb_class_owned_p(obj))) {
1261 // For two reasons we can only use the fast path on the Ractor
1262 // that owns the class.
1263 // First, only the owner Ractor is allowed to set ivars on classes
1264 // and modules. So we can skip locking.
1265 // Second, other ractors need to check the shareability of the
1266 // values returned from the class ivars.
1267
1268 if (default_value == Qundef) { // defined?
1269 return rb_ivar_defined(obj, id) ? Qtrue : Qundef;
1270 }
1271 else {
1272 goto general_path;
1273 }
1274 }
1275
1276 fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1277 break;
1278 }
1279 default:
1280 fields_obj = rb_obj_fields(obj, id);
1281 }
1282
1283 if (!fields_obj) {
1284 return default_value;
1285 }
1286
1287 VALUE val = Qundef;
1288
1289 shape_id_t shape_id = RBASIC_SHAPE_ID_FOR_READ(fields_obj);
1290 VALUE *ivar_list = rb_imemo_fields_ptr(fields_obj);
1291
1292 rb_getivar_cache cache = rb_getivar_cache_unpack(vm_cache_attr_index_atomic_read(is_attr, ic, cc));
1293
1294 if (LIKELY(cache.shape_offset == shape_id)) {
1295 if (cache.index == ATTR_INDEX_NOT_SET) {
1296 return default_value;
1297 }
1298
1299 val = ivar_list[cache.index];
1300#if USE_DEBUG_COUNTER
1301 RB_DEBUG_COUNTER_INC(ivar_get_ic_hit);
1302
1303 if (RB_TYPE_P(obj, T_OBJECT)) {
1304 RB_DEBUG_COUNTER_INC(ivar_get_obj_hit);
1305 }
1306#endif
1307 RUBY_ASSERT(!UNDEF_P(val));
1308 }
1309 else { // cache miss case
1310#if USE_DEBUG_COUNTER
1311 if (is_attr) {
1312 if (cache.shape_offset != INVALID_SHAPE_ID) {
1313 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_set);
1314 }
1315 else {
1316 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_unset);
1317 }
1318 }
1319 else {
1320 if (cache.shape_offset != INVALID_SHAPE_ID) {
1321 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_set);
1322 }
1323 else {
1324 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_unset);
1325 }
1326 }
1327 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1328
1329 if (RB_TYPE_P(obj, T_OBJECT)) {
1330 RB_DEBUG_COUNTER_INC(ivar_get_obj_miss);
1331 }
1332#endif
1333
1334 if (UNLIKELY(rb_shape_complex_p(shape_id))) {
1335 st_table *table = (st_table *)ivar_list;
1336
1337 RUBY_ASSERT(table);
1338 RUBY_ASSERT(table == rb_imemo_fields_complex_tbl(fields_obj));
1339
1340 if (!st_lookup(table, id, &val)) {
1341 val = default_value;
1342 }
1343 }
1344 else {
1345 shape_id_t previous_cached_offset = cache.shape_offset;
1346 if (rb_shape_get_iv_index_with_hint(shape_id, id, &cache.index, &cache.shape_offset)) {
1347 if (cache.shape_offset != previous_cached_offset) {
1348 RUBY_ASSERT(!rb_shape_complex_p(cache.shape_offset));
1349 RUBY_ASSERT(cache.shape_offset != INVALID_SHAPE_ID);
1350
1351 uint64_t packed_cache = rb_getivar_cache_pack(cache.shape_offset, cache.index);
1352 vm_cache_attr_index_set(is_attr, ic, cc, packed_cache);
1353 }
1354
1355 if (cache.index == ATTR_INDEX_NOT_SET) {
1356 val = default_value;
1357 }
1358 else {
1359 // We fetched the ivar list above
1360 val = ivar_list[cache.index];
1361 RUBY_ASSERT(!UNDEF_P(val));
1362 }
1363 }
1364 else {
1365 vm_cache_attr_index_set(is_attr, ic, cc, rb_getivar_cache_pack(shape_id, ATTR_INDEX_NOT_SET));
1366 val = default_value;
1367 }
1368 }
1369 }
1370
1371 if (!UNDEF_P(default_value)) {
1372 RUBY_ASSERT(!UNDEF_P(val));
1373 }
1374
1375 return val;
1376
1377general_path:
1378#endif /* OPT_IC_FOR_IVAR */
1379 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1380
1381 if (is_attr) {
1382 return rb_attr_get(obj, id);
1383 }
1384 else {
1385 return rb_ivar_get(obj, id);
1386 }
1387}
1388
1389ALWAYS_INLINE(static VALUE vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr));
1390NOINLINE(static VALUE vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic));
1391NOINLINE(static VALUE vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc));
1392
1393static VALUE
1394vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr)
1395{
1396#if OPT_IC_FOR_IVAR
1397 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss);
1398
1399 rb_check_ivar_modifiable(obj);
1400
1401 shape_id_t previous_shape_id = RBASIC_SHAPE_ID(obj);
1402 attr_index_t index = rb_ivar_set_index(obj, id, val);
1403 shape_id_t next_shape_id = RBASIC_SHAPE_ID(obj);
1404
1405 if (!rb_shape_complex_p(next_shape_id)) {
1406 uint64_t packed_cache = rb_setivar_cache_pack(RSHAPE_OFFSET(previous_shape_id), RSHAPE_OFFSET(next_shape_id), index);
1407 vm_cache_attr_index_set(is_attr, ic, cc, packed_cache);
1408 }
1409
1410 RB_DEBUG_COUNTER_INC(ivar_set_obj_miss);
1411 return val;
1412#else
1413 return rb_ivar_set(obj, id, val);
1414#endif
1415}
1416
1417static VALUE
1418vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic)
1419{
1420 return vm_setivar_slowpath(obj, id, val, iseq, ic, NULL, false);
1421}
1422
1423static VALUE
1424vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc)
1425{
1426 return vm_setivar_slowpath(obj, id, val, NULL, NULL, cc, true);
1427}
1428
1429NOINLINE(static VALUE vm_setivar_class(VALUE obj, VALUE val, rb_setivar_cache cache));
1430static VALUE
1431vm_setivar_class(VALUE obj, VALUE val, rb_setivar_cache cache)
1432{
1433 if (UNLIKELY(!rb_class_owned_p(obj))) {
1434 return Qundef;
1435 }
1436
1437 VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1438 if (UNLIKELY(!fields_obj)) {
1439 return Qundef;
1440 }
1441
1442 shape_id_t shape_id = RBASIC_SHAPE_ID(fields_obj);
1443 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, RBASIC_SHAPE_ID(fields_obj), cache);
1444 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1445 return Qundef;
1446 }
1447
1448 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1449
1450 if (shape_id != dest_shape_id) {
1451 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1452 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1453 }
1454
1455 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1456
1457 return val;
1458}
1459
1460NOINLINE(static VALUE vm_setivar_default(VALUE obj, ID id, VALUE val, rb_setivar_cache cache));
1461static VALUE
1462vm_setivar_default(VALUE obj, ID id, VALUE val, rb_setivar_cache cache)
1463{
1464 VALUE fields_obj = rb_obj_fields(obj, id);
1465 if (UNLIKELY(!fields_obj)) {
1466 return Qundef;
1467 }
1468
1469 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1470 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, RBASIC_SHAPE_ID(fields_obj), cache);
1471 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1472 return Qundef;
1473 }
1474
1475 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1476
1477 if (shape_id != dest_shape_id) {
1478 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1479 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1480 }
1481
1482 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1483
1484 return val;
1485}
1486
1487static inline VALUE
1488vm_setivar(VALUE obj, VALUE val, rb_setivar_cache cache)
1489{
1490#if OPT_IC_FOR_IVAR
1491 switch (BUILTIN_TYPE(obj)) {
1492 case T_OBJECT:
1493 {
1494 VM_ASSERT(!rb_ractor_shareable_p(obj) || rb_obj_frozen_p(obj));
1495
1496 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1497 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, shape_id, cache);
1498 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1499 break;
1500 }
1501
1502 VALUE fields_obj = ROBJECT_FIELDS_OBJ(obj);
1503 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1504 if (shape_id != dest_shape_id) {
1505 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1506 if (fields_obj != obj) {
1507 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1508 }
1509 }
1510
1511 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1512 RB_DEBUG_COUNTER_INC(ivar_set_obj_hit);
1513 return val;
1514 }
1515 break;
1516 case T_CLASS:
1517 case T_MODULE:
1518 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss_noobject);
1519 default:
1520 break;
1521 }
1522
1523 return Qundef;
1524#endif /* OPT_IC_FOR_IVAR */
1525}
1526
1527static VALUE
1528update_classvariable_cache(const rb_iseq_t *iseq, VALUE klass, ID id, const rb_cref_t * cref, ICVARC ic)
1529{
1530 VALUE defined_class = 0;
1531 VALUE cvar_value = rb_cvar_find(klass, id, &defined_class);
1532
1533 if (RB_TYPE_P(defined_class, T_ICLASS)) {
1534 defined_class = RBASIC(defined_class)->klass;
1535 }
1536
1537 VALUE rb_cvc_tbl = RCLASS_CVC_TBL(defined_class);
1538 if (!rb_cvc_tbl) {
1539 rb_bug("the cvc table should be set");
1540 }
1541
1542 VALUE ent_data;
1543 if (!rb_marked_id_table_lookup(rb_cvc_tbl, id, &ent_data)) {
1544 rb_bug("should have cvar cache entry");
1545 }
1546
1547 struct rb_cvar_class_tbl_entry *ent = (void *)ent_data;
1548
1549 ent->global_cvar_state = GET_GLOBAL_CVAR_STATE();
1550 RB_OBJ_WRITE((VALUE)ent, &ent->cref, cref);
1551 RB_OBJ_WRITE(iseq, &ic->entry, ent);
1552
1553 RUBY_ASSERT(BUILTIN_TYPE((VALUE)cref) == T_IMEMO && IMEMO_TYPE_P(cref, imemo_cref));
1554
1555 return cvar_value;
1556}
1557
1558static inline VALUE
1559vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, ICVARC ic)
1560{
1561 const rb_cref_t *cref;
1562 cref = vm_get_cref(GET_EP());
1563
1564 // The fast path skips rb_cvar_find's checks, so it is for class_value's owner:
1565 // the sole writer, and the only one allowed to see an unshareable value.
1566 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref &&
1567 LIKELY(rb_class_owned_p(ic->entry->class_value))) {
1568 RB_DEBUG_COUNTER_INC(cvar_read_inline_hit);
1569
1570 VALUE v = rb_ivar_lookup(ic->entry->class_value, id, Qundef);
1571 RUBY_ASSERT(!UNDEF_P(v));
1572
1573 return v;
1574 }
1575
1576 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1577
1578 return update_classvariable_cache(iseq, klass, id, cref, ic);
1579}
1580
1581VALUE
1582rb_vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, ICVARC ic)
1583{
1584 return vm_getclassvariable(iseq, cfp, id, ic);
1585}
1586
1587static inline void
1588vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, VALUE val, ICVARC ic)
1589{
1590 const rb_cref_t *cref;
1591 cref = vm_get_cref(GET_EP());
1592
1593 // as on the read side: the fast path writes straight into class_value
1594 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref &&
1595 LIKELY(rb_class_owned_p(ic->entry->class_value))) {
1596 RB_DEBUG_COUNTER_INC(cvar_write_inline_hit);
1597
1598 rb_class_ivar_set(ic->entry->class_value, id, val);
1599 return;
1600 }
1601
1602 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1603
1604 rb_cvar_set(klass, id, val);
1605
1606 update_classvariable_cache(iseq, klass, id, cref, ic);
1607}
1608
1609void
1610rb_vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, VALUE val, ICVARC ic)
1611{
1612 vm_setclassvariable(iseq, cfp, id, val, ic);
1613}
1614
1615ALWAYS_INLINE(static VALUE vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic));
1616static inline VALUE
1617vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1618{
1619 return vm_getivar(obj, id, iseq, ic, NULL, FALSE, Qnil);
1620}
1621
1622static inline void
1623vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1624{
1625 if (RB_SPECIAL_CONST_P(obj)) {
1626 rb_error_frozen_object(obj);
1627 return;
1628 }
1629
1630 rb_setivar_cache cache = rb_setivar_cache_unpack(vm_ic_atomic_cache_read(ic));
1631 if (UNLIKELY(UNDEF_P(vm_setivar(obj, val, cache)))) {
1632 switch (BUILTIN_TYPE(obj)) {
1633 case T_OBJECT:
1634 break;
1635 case T_CLASS:
1636 case T_MODULE:
1637 if (!UNDEF_P(vm_setivar_class(obj, val, cache))) {
1638 return;
1639 }
1640 break;
1641 default:
1642 if (!UNDEF_P(vm_setivar_default(obj, id, val, cache))) {
1643 return;
1644 }
1645 }
1646 vm_setivar_slowpath_ivar(obj, id, val, iseq, ic);
1647 }
1648}
1649
1650void
1651rb_vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1652{
1653 vm_setinstancevariable(iseq, obj, id, val, ic);
1654}
1655
1656VALUE
1657rb_vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1658{
1659 return vm_getinstancevariable(iseq, obj, id, ic);
1660}
1661
1662static VALUE
1663vm_throw_continue(const rb_execution_context_t *ec, VALUE err)
1664{
1665 /* continue throw */
1666
1667 if (FIXNUM_P(err)) {
1668 ec->tag->state = RUBY_TAG_FATAL;
1669 }
1670 else if (SYMBOL_P(err)) {
1671 ec->tag->state = TAG_THROW;
1672 }
1673 else if (THROW_DATA_P(err)) {
1674 ec->tag->state = THROW_DATA_STATE((struct vm_throw_data *)err);
1675 }
1676 else {
1677 ec->tag->state = TAG_RAISE;
1678 }
1679 return err;
1680}
1681
1682static VALUE
1683vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, enum ruby_tag_type state,
1684 const int flag, const VALUE throwobj)
1685{
1686 const rb_control_frame_t *escape_cfp = NULL;
1687 const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */
1688
1689 if (flag != 0) {
1690 /* do nothing */
1691 }
1692 else if (state == TAG_BREAK) {
1693 int is_orphan = 1;
1694 const VALUE *ep = GET_EP();
1695 const rb_iseq_t *base_iseq = GET_ISEQ();
1696 escape_cfp = reg_cfp;
1697
1698 while (ISEQ_BODY(base_iseq)->type != ISEQ_TYPE_BLOCK) {
1699 if (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) {
1700 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1701 ep = escape_cfp->ep;
1702 base_iseq = CFP_ISEQ(escape_cfp);
1703 }
1704 else {
1705 ep = VM_ENV_PREV_EP(ep);
1706 base_iseq = ISEQ_BODY(base_iseq)->parent_iseq;
1707 escape_cfp = rb_vm_search_cf_from_ep(ec, escape_cfp, ep);
1708 VM_ASSERT(CFP_ISEQ(escape_cfp) == base_iseq);
1709 }
1710 }
1711
1712 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1713 /* lambda{... break ...} */
1714 is_orphan = 0;
1715 state = TAG_RETURN;
1716 }
1717 else {
1718 ep = VM_ENV_PREV_EP(ep);
1719
1720 while (escape_cfp < eocfp) {
1721 if (escape_cfp->ep == ep) {
1722 const rb_iseq_t *const iseq = CFP_ISEQ(escape_cfp);
1723 const VALUE epc = CFP_PC(escape_cfp) - ISEQ_BODY(iseq)->iseq_encoded;
1724 const struct iseq_catch_table *const ct = ISEQ_BODY(iseq)->catch_table;
1725 unsigned int i;
1726
1727 if (!ct) break;
1728 for (i=0; i < ct->size; i++) {
1729 const struct iseq_catch_table_entry *const entry =
1730 UNALIGNED_MEMBER_PTR(ct, entries[i]);
1731
1732 if (entry->type == CATCH_TYPE_BREAK &&
1733 entry->iseq == base_iseq &&
1734 entry->start < epc && entry->end >= epc) {
1735 if (entry->cont == epc) { /* found! */
1736 is_orphan = 0;
1737 }
1738 break;
1739 }
1740 }
1741 break;
1742 }
1743
1744 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1745 }
1746 }
1747
1748 if (is_orphan) {
1749 rb_vm_localjump_error("break from proc-closure", throwobj, TAG_BREAK);
1750 }
1751 }
1752 else if (state == TAG_RETRY) {
1753 const VALUE *ep = VM_ENV_PREV_EP(GET_EP());
1754
1755 escape_cfp = rb_vm_search_cf_from_ep(ec, reg_cfp, ep);
1756 }
1757 else if (state == TAG_RETURN) {
1758 const VALUE *current_ep = GET_EP();
1759 const VALUE *target_ep = NULL, *target_lep, *ep = current_ep;
1760 int in_class_frame = 0;
1761 int toplevel = 1;
1762 escape_cfp = reg_cfp;
1763
1764 // find target_lep, target_ep
1765 while (!VM_ENV_LOCAL_P(ep)) {
1766 if (VM_ENV_FLAGS(ep, VM_FRAME_FLAG_LAMBDA) && target_ep == NULL) {
1767 target_ep = ep;
1768 }
1769 ep = VM_ENV_PREV_EP(ep);
1770 }
1771 target_lep = ep;
1772
1773 while (escape_cfp < eocfp) {
1774 const VALUE *lep = VM_CF_LEP(escape_cfp);
1775
1776 if (!target_lep) {
1777 target_lep = lep;
1778 }
1779
1780 if (lep == target_lep &&
1781 VM_FRAME_RUBYFRAME_P(escape_cfp) &&
1782 ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) {
1783 in_class_frame = 1;
1784 target_lep = 0;
1785 }
1786
1787 if (lep == target_lep) {
1788 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1789 toplevel = 0;
1790 if (in_class_frame) {
1791 /* lambda {class A; ... return ...; end} */
1792 goto valid_return;
1793 }
1794 else {
1795 const VALUE *tep = current_ep;
1796
1797 while (target_lep != tep) {
1798 if (escape_cfp->ep == tep) {
1799 /* in lambda */
1800 if (tep == target_ep) {
1801 goto valid_return;
1802 }
1803 else {
1804 goto unexpected_return;
1805 }
1806 }
1807 tep = VM_ENV_PREV_EP(tep);
1808 }
1809 }
1810 }
1811 else if (VM_FRAME_RUBYFRAME_P(escape_cfp)) {
1812 switch (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type) {
1813 case ISEQ_TYPE_TOP:
1814 case ISEQ_TYPE_MAIN:
1815 if (toplevel) {
1816 if (in_class_frame) goto unexpected_return;
1817 if (target_ep == NULL) {
1818 goto valid_return;
1819 }
1820 else {
1821 goto unexpected_return;
1822 }
1823 }
1824 break;
1825 case ISEQ_TYPE_EVAL: {
1826 const rb_iseq_t *is = CFP_ISEQ(escape_cfp);
1827 enum rb_iseq_type t = ISEQ_BODY(is)->type;
1828 while (t == ISEQ_TYPE_RESCUE || t == ISEQ_TYPE_ENSURE || t == ISEQ_TYPE_EVAL) {
1829 if (!(is = ISEQ_BODY(is)->parent_iseq)) break;
1830 t = ISEQ_BODY(is)->type;
1831 }
1832 toplevel = t == ISEQ_TYPE_TOP || t == ISEQ_TYPE_MAIN;
1833 break;
1834 }
1835 case ISEQ_TYPE_CLASS:
1836 toplevel = 0;
1837 break;
1838 default:
1839 break;
1840 }
1841 }
1842 }
1843
1844 if (escape_cfp->ep == target_lep && ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_METHOD) {
1845 if (target_ep == NULL) {
1846 goto valid_return;
1847 }
1848 else {
1849 goto unexpected_return;
1850 }
1851 }
1852
1853 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1854 }
1855 unexpected_return:;
1856 rb_vm_localjump_error("unexpected return", throwobj, TAG_RETURN);
1857
1858 valid_return:;
1859 /* do nothing */
1860 }
1861 else {
1862 rb_bug("isns(throw): unsupported throw type");
1863 }
1864
1865 ec->tag->state = state;
1866 return (VALUE)THROW_DATA_NEW(throwobj, escape_cfp, state);
1867}
1868
1869static VALUE
1870vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
1871 rb_num_t throw_state, VALUE throwobj)
1872{
1873 const int state = (int)(throw_state & VM_THROW_STATE_MASK);
1874 const int flag = (int)(throw_state & VM_THROW_NO_ESCAPE_FLAG);
1875
1876 if (state != 0) {
1877 return vm_throw_start(ec, reg_cfp, state, flag, throwobj);
1878 }
1879 else {
1880 return vm_throw_continue(ec, throwobj);
1881 }
1882}
1883
1884// Fallback for YJIT. Prepare throw data and return it.
1885VALUE
1886rb_vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj)
1887{
1888 return vm_throw(ec, reg_cfp, throw_state, throwobj);
1889}
1890
1891NORETURN(VALUE rb_zjit_throw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj));
1892
1893// Fallback for ZJIT. Make a longjmp and unwind to the most recent vm_exec().
1894VALUE
1895rb_zjit_throw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj)
1896{
1897 VALUE val = vm_throw(ec, reg_cfp, throw_state, throwobj);
1898
1899 // vm_throw() has set ec->tag->state. On the longjmp path, vm_exec() reads the throw data from
1900 // ec->errinfo instead of vm_exec_core()'s return value like THROW_EXCEPTION()'s path does, so
1901 // we need to put it there instead.
1902 enum ruby_tag_type state = ec->tag->state;
1903 ec->errinfo = val;
1904 EC_JUMP_TAG(ec, state);
1905
1906 UNREACHABLE_RETURN(Qundef);
1907}
1908
1909static inline void
1910vm_expandarray(struct rb_control_frame_struct *cfp, VALUE ary, rb_num_t num, int flag)
1911{
1912 int is_splat = flag & 0x01;
1913 const VALUE *ptr;
1914 rb_num_t len;
1915 const VALUE obj = ary;
1916
1917 if (!RB_TYPE_P(ary, T_ARRAY) && NIL_P(ary = rb_check_array_type(ary))) {
1918 ary = obj;
1919 ptr = &ary;
1920 len = 1;
1921 }
1922 else {
1923 ptr = RARRAY_CONST_PTR(ary);
1924 len = (rb_num_t)RARRAY_LEN(ary);
1925 }
1926
1927 if (num + is_splat == 0) {
1928 /* no space left on stack */
1929 }
1930 else if (flag & 0x02) {
1931 /* post: ..., nil ,ary[-1], ..., ary[0..-num] # top */
1932 rb_num_t i = 0, j;
1933
1934 if (len < num) {
1935 for (i = 0; i < num - len; i++) {
1936 *cfp->sp++ = Qnil;
1937 }
1938 }
1939
1940 for (j = 0; i < num; i++, j++) {
1941 VALUE v = ptr[len - j - 1];
1942 *cfp->sp++ = v;
1943 }
1944
1945 if (is_splat) {
1946 *cfp->sp++ = rb_ary_new4(len - j, ptr);
1947 }
1948 }
1949 else {
1950 /* normal: ary[num..-1], ary[num-2], ary[num-3], ..., ary[0] # top */
1951 if (is_splat) {
1952 if (num > len) {
1953 *cfp->sp++ = rb_ary_new();
1954 }
1955 else {
1956 *cfp->sp++ = rb_ary_new4(len - num, ptr + num);
1957 }
1958 }
1959
1960 if (num > len) {
1961 rb_num_t i = 0;
1962 for (; i < num - len; i++) {
1963 *cfp->sp++ = Qnil;
1964 }
1965
1966 for (rb_num_t j = 0; i < num; i++, j++) {
1967 *cfp->sp++ = ptr[len - j - 1];
1968 }
1969 }
1970 else {
1971 for (rb_num_t j = 0; j < num; j++) {
1972 *cfp->sp++ = ptr[num - j - 1];
1973 }
1974 }
1975 }
1976
1977 RB_GC_GUARD(ary);
1978}
1979
1980static VALUE vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
1981
1982static VALUE vm_mtbl_dump(VALUE klass, ID target_mid);
1983
1984static struct rb_class_cc_entries *
1985vm_ccs_create(VALUE klass, VALUE cc_tbl, ID mid, const rb_callable_method_entry_t *cme)
1986{
1987 int initial_capa = 2;
1988 struct rb_class_cc_entries *ccs = ruby_xmalloc(vm_ccs_alloc_size(initial_capa));
1989#if VM_CHECK_MODE > 0
1990 ccs->debug_sig = ~(VALUE)ccs;
1991#endif
1992 ccs->capa = initial_capa;
1993 ccs->len = 0;
1994 ccs->cme = cme;
1995 METHOD_ENTRY_CACHED_SET((rb_callable_method_entry_t *)cme);
1996
1997 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
1998 RB_OBJ_WRITTEN(cc_tbl, Qundef, cme);
1999 return ccs;
2000}
2001
2002static void
2003vm_ccs_push(VALUE cc_tbl, ID mid, struct rb_class_cc_entries *ccs, const struct rb_callinfo *ci, const struct rb_callcache *cc)
2004{
2005 if (! vm_cc_markable(cc)) {
2006 return;
2007 }
2008
2009 if (UNLIKELY(ccs->len == ccs->capa)) {
2010 RUBY_ASSERT(ccs->capa > 0);
2011 ccs->capa *= 2;
2012 ccs = ruby_xrealloc(ccs, vm_ccs_alloc_size(ccs->capa));
2013#if VM_CHECK_MODE > 0
2014 ccs->debug_sig = ~(VALUE)ccs;
2015#endif
2016 // GC?
2017 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
2018 }
2019 VM_ASSERT(ccs->len < ccs->capa);
2020
2021 const struct rb_callinfo_kwarg *kwarg = vm_ci_kwarg(ci);
2022 const int pos = ccs->len++;
2023 ccs->entries[pos].argc = vm_ci_argc(ci);
2024 ccs->entries[pos].flag = (unsigned short)vm_ci_flag(ci);
2025 ccs->entries[pos].kw_len = (unsigned short)(kwarg ? kwarg->keyword_len : 0);
2026 RB_OBJ_WRITE(cc_tbl, &ccs->entries[pos].cc, cc);
2027
2028 if (RB_DEBUG_COUNTER_SETMAX(ccs_maxlen, ccs->len)) {
2029 // for tuning
2030 // vm_mtbl_dump(klass, 0);
2031 }
2032}
2033
2034#if VM_CHECK_MODE > 0
2035void
2036rb_vm_ccs_dump(struct rb_class_cc_entries *ccs)
2037{
2038 ruby_debug_printf("ccs:%p (%d,%d)\n", (void *)ccs, ccs->len, ccs->capa);
2039 for (int i=0; i<ccs->len; i++) {
2040 ruby_debug_printf("CCS CI ID:flag:%x argc:%u kw_len:%u\n",
2041 ccs->entries[i].flag,
2042 ccs->entries[i].argc,
2043 ccs->entries[i].kw_len);
2044 rp(ccs->entries[i].cc);
2045 }
2046}
2047
2048static int
2049vm_ccs_verify(struct rb_class_cc_entries *ccs, ID mid, VALUE klass)
2050{
2051 VM_ASSERT(vm_ccs_p(ccs));
2052 VM_ASSERT(ccs->len <= ccs->capa);
2053
2054 for (int i=0; i<ccs->len; i++) {
2055 const struct rb_callcache *cc = ccs->entries[i].cc;
2056
2057 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
2058 VM_ASSERT(vm_cc_class_check(cc, klass));
2059 VM_ASSERT(vm_cc_check_cme(cc, ccs->cme));
2060 VM_ASSERT(!vm_cc_super_p(cc));
2061 VM_ASSERT(!vm_cc_refinement_p(cc));
2062 }
2063 return TRUE;
2064}
2065#endif
2066
2067const rb_callable_method_entry_t *rb_check_overloaded_cme(const rb_callable_method_entry_t *cme, const struct rb_callinfo * const ci);
2068
2069static void
2070vm_evict_cc(VALUE klass, VALUE cc_tbl, ID mid)
2071{
2072 ASSERT_vm_locking();
2073
2074 if (rb_multi_ractor_p()) {
2075 if (RCLASS_WRITABLE_CC_TBL(klass) != cc_tbl) {
2076 // Another ractor updated the CC table while we were waiting on the VM lock.
2077 // We have to retry.
2078 return;
2079 }
2080
2081 VALUE ccs_obj = 0;
2082 rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj);
2083 struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_obj;
2084
2085 if (!ccs || !METHOD_ENTRY_INVALIDATED(ccs->cme)) {
2086 // Another ractor replaced that entry while we were waiting on the VM lock.
2087 return;
2088 }
2089
2090 VALUE new_table = rb_vm_cc_table_dup(cc_tbl);
2091 rb_vm_cc_table_delete(new_table, mid);
2092 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), new_table);
2093 }
2094 else {
2095 rb_vm_cc_table_delete(cc_tbl, mid);
2096 }
2097}
2098
2099static const struct rb_callcache *
2100vm_populate_cc(VALUE klass, const struct rb_callinfo * const ci, ID mid)
2101{
2102 ASSERT_vm_locking();
2103
2104 RB_DEBUG_COUNTER_INC(cc_not_found_in_ccs);
2105
2106 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
2107
2108 VM_ASSERT(cme == NULL || IMEMO_TYPE_P(cme, imemo_ment));
2109
2110 if (cme == NULL) {
2111 // undef or not found: can't cache the information
2112 VM_ASSERT(vm_cc_cme(&vm_empty_cc) == NULL);
2113 return &vm_empty_cc;
2114 }
2115
2116 VALUE cc_tbl = RCLASS_WRITABLE_CC_TBL(klass);
2117 const VALUE original_cc_table = cc_tbl;
2118 if (!cc_tbl) {
2119 // Is this possible after rb_callable_method_entry ?
2120 cc_tbl = rb_vm_cc_table_create(1);
2121 }
2122 else if (rb_multi_ractor_p()) {
2123 cc_tbl = rb_vm_cc_table_dup(cc_tbl);
2124 }
2125
2126 VM_ASSERT(cme == rb_callable_method_entry(klass, mid));
2127
2128 METHOD_ENTRY_CACHED_SET((struct rb_callable_method_entry_struct *)cme);
2129
2130 VM_ASSERT(cc_tbl);
2131
2132 struct rb_class_cc_entries *ccs = NULL;
2133 {
2134 VALUE ccs_obj;
2135 if (UNLIKELY(rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj))) {
2136 ccs = (struct rb_class_cc_entries *)ccs_obj;
2137 }
2138 else {
2139 // TODO: required?
2140 ccs = vm_ccs_create(klass, cc_tbl, mid, cme);
2141 }
2142 }
2143
2144 cme = rb_check_overloaded_cme(cme, ci);
2145
2146 const struct rb_callcache *cc = vm_cc_new(klass, cme, vm_call_general, cc_type_normal);
2147 vm_ccs_push(cc_tbl, mid, ccs, ci, cc);
2148
2149 VM_ASSERT(vm_cc_cme(cc) != NULL);
2150 VM_ASSERT(cme->called_id == mid);
2151 VM_ASSERT(vm_cc_cme(cc)->called_id == mid);
2152
2153 if (original_cc_table != cc_tbl) {
2154 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), cc_tbl);
2155 }
2156
2157 return cc;
2158}
2159
2160static const struct rb_callcache *
2161vm_lookup_cc(const VALUE klass, const struct rb_callinfo * const ci, ID mid)
2162{
2163 VALUE cc_tbl;
2164 struct rb_class_cc_entries *ccs;
2165retry:
2166 cc_tbl = RUBY_ATOMIC_VALUE_LOAD(RCLASS_WRITABLE_CC_TBL(klass));
2167 ccs = NULL;
2168
2169 if (cc_tbl) {
2170 // CCS data is keyed on method id, so we don't need the method id
2171 // for doing comparisons in the `for` loop below.
2172
2173 VALUE ccs_obj;
2174 if (rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj)) {
2175 ccs = (struct rb_class_cc_entries *)ccs_obj;
2176 const int ccs_len = ccs->len;
2177
2178 if (UNLIKELY(METHOD_ENTRY_INVALIDATED(ccs->cme))) {
2179 RB_VM_LOCKING() {
2180 vm_evict_cc(klass, cc_tbl, mid);
2181 }
2182 goto retry;
2183 }
2184 else {
2185 VM_ASSERT(vm_ccs_verify(ccs, mid, klass));
2186
2187 // We already know the method id is correct because we had
2188 // to look up the ccs_data by method id. We compare argc and
2189 // flag, plus the keyword argument count: two call sites with
2190 // the same argc and flag can still differ in how many of the
2191 // arguments are keywords (e.g. `m(a: 1, b: 2)` vs `m(1, b: 2)`),
2192 // which selects a different argument setup fastpath.
2193 const struct rb_callinfo_kwarg *kwarg = vm_ci_kwarg(ci);
2194 unsigned int argc = vm_ci_argc(ci);
2195 unsigned int flag = vm_ci_flag(ci);
2196 unsigned int kw_len = kwarg ? kwarg->keyword_len : 0;
2197
2198 for (int i=0; i<ccs_len; i++) {
2199 unsigned int ccs_ci_argc = ccs->entries[i].argc;
2200 unsigned int ccs_ci_flag = ccs->entries[i].flag;
2201 unsigned int ccs_ci_kw_len = ccs->entries[i].kw_len;
2202 const struct rb_callcache *ccs_cc = ccs->entries[i].cc;
2203
2204 VM_ASSERT(IMEMO_TYPE_P(ccs_cc, imemo_callcache));
2205
2206 if (ccs_ci_argc == argc && ccs_ci_flag == flag && ccs_ci_kw_len == kw_len) {
2207 RB_DEBUG_COUNTER_INC(cc_found_in_ccs);
2208
2209 VM_ASSERT(vm_cc_cme(ccs_cc)->called_id == mid);
2210 VM_ASSERT(ccs_cc->klass == klass);
2211 VM_ASSERT(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(ccs_cc)));
2212
2213 return ccs_cc;
2214 }
2215 }
2216 }
2217 }
2218 }
2219
2220 RB_GC_GUARD(cc_tbl);
2221 return NULL;
2222}
2223
2224static const struct rb_callcache *
2225vm_search_cc(const VALUE klass, const struct rb_callinfo * const ci)
2226{
2227 const ID mid = vm_ci_mid(ci);
2228
2229 const struct rb_callcache *cc = vm_lookup_cc(klass, ci, mid);
2230 if (cc) {
2231 return cc;
2232 }
2233
2234 RB_VM_LOCKING() {
2235 if (rb_multi_ractor_p()) {
2236 // The CC may have been populated by another ractor while we were waiting on the lock,
2237 // so we must lookup a second time.
2238 cc = vm_lookup_cc(klass, ci, mid);
2239 }
2240
2241 if (!cc) {
2242 cc = vm_populate_cc(klass, ci, mid);
2243 }
2244 }
2245
2246 return cc;
2247}
2248
2249const struct rb_callcache *
2250rb_vm_search_method_slowpath(const struct rb_callinfo *ci, VALUE klass)
2251{
2252 const struct rb_callcache *cc;
2253
2254 VM_ASSERT(!SPECIAL_CONST_P(klass));
2255
2256 if (RB_BUILTIN_TYPE(klass) == T_NONE) {
2257 // If we find a T_NONE here, it's most likely we called CLASS_OF(obj) on a
2258 // garbage collected object (the freelist is stored in the class pointer),
2259 // but it's possible that just the class was GC'd.
2260 // This message intentionally tries to imply the former, but make an
2261 // accurate statement for either case.
2262 rb_bug("attempted to search method '%s' on a garbage collected object",
2263 rb_id2name(vm_ci_mid(ci)));
2264 }
2265
2266 VM_ASSERT_TYPE2(klass, T_CLASS, T_ICLASS);
2267
2268 cc = vm_search_cc(klass, ci);
2269
2270 VM_ASSERT(cc);
2271 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
2272 VM_ASSERT(cc == vm_cc_empty() || cc->klass == klass);
2273 VM_ASSERT(cc == vm_cc_empty() || callable_method_entry_p(vm_cc_cme(cc)));
2274 VM_ASSERT(cc == vm_cc_empty() || !METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)));
2275 VM_ASSERT(cc == vm_cc_empty() || vm_cc_cme(cc)->called_id == vm_ci_mid(ci));
2276
2277 return cc;
2278}
2279
2280#if VM_CHECK_MODE > 0
2281static bool
2282vm_cd_owned_by_iseq_p(const struct rb_call_data *cd, const rb_iseq_t *iseq)
2283{
2284 const struct rb_iseq_constant_body *body = ISEQ_BODY(iseq);
2285 return cd >= body->call_data && cd < body->call_data + body->ci_size;
2286}
2287#endif
2288
2289static const struct rb_callcache *
2290vm_search_method_slowpath0(VALUE cd_owner, struct rb_call_data *cd, VALUE klass)
2291{
2292#if USE_DEBUG_COUNTER
2293 const struct rb_callcache *old_cc = cd->cc;
2294 const rb_callable_method_entry_t *const old_cme = old_cc ? old_cc->cme_ : NULL;
2295#endif
2296
2297 const struct rb_callcache *cc = rb_vm_search_method_slowpath(cd->ci, klass);
2298 const rb_callable_method_entry_t *const new_cme = vm_cc_cme(cc);
2299
2300#if OPT_INLINE_METHOD_CACHE
2301 cd->cc = cc;
2302
2303 const struct rb_callcache *empty_cc = &vm_empty_cc;
2304 if (cd_owner && cc != empty_cc) {
2305 // invokesuper dispatches with a cd on the machine stack, which has no
2306 // owning iseq to remember and keeps its ci out of the GC.
2307 VM_ASSERT(!vm_ci_markable(cd->ci) ||
2308 vm_cd_owned_by_iseq_p(cd, (const rb_iseq_t *)cd_owner));
2309 RB_OBJ_WRITTEN(cd_owner, Qundef, cc);
2310 }
2311
2312#if USE_DEBUG_COUNTER
2313 if (!old_cc || old_cc == empty_cc) {
2314 // empty
2315 RB_DEBUG_COUNTER_INC(mc_inline_miss_empty);
2316 }
2317 else if (old_cc == cc) {
2318 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cc);
2319 }
2320 else if (old_cme == new_cme) {
2321 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cme);
2322 }
2323 else if (old_cme && new_cme && old_cme->def == new_cme->def) {
2324 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_def);
2325 }
2326 else {
2327 RB_DEBUG_COUNTER_INC(mc_inline_miss_diff);
2328 }
2329#endif
2330#endif // OPT_INLINE_METHOD_CACHE
2331
2332 if (new_cme) VM_ASSERT(new_cme->called_id == vm_ci_mid(cd->ci));
2333
2334 return cc;
2335}
2336
2337ALWAYS_INLINE(static bool vm_cc_hit_p(const struct rb_callcache *cc, const struct rb_call_data *cd, VALUE klass));
2338static bool
2339vm_cc_hit_p(const struct rb_callcache *cc, const struct rb_call_data *cd, VALUE klass)
2340{
2341 VM_ASSERT_TYPE2(klass, T_CLASS, T_ICLASS);
2342
2343#if OPT_INLINE_METHOD_CACHE
2344 if (LIKELY(vm_cc_class_check(cc, klass))) {
2345 if (LIKELY(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)))) {
2346 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
2347 RB_DEBUG_COUNTER_INC(mc_inline_hit);
2348 VM_ASSERT(vm_cc_cme(cc) == NULL || // not found
2349 (vm_ci_flag(cd->ci) & VM_CALL_SUPER) || // search_super w/ define_method
2350 vm_cc_cme(cc)->called_id == vm_ci_mid(cd->ci)); // cme->called_id == ci->mid
2351
2352 return true;
2353 }
2354 RB_DEBUG_COUNTER_INC(mc_inline_miss_invalidated);
2355 }
2356 else {
2357 RB_DEBUG_COUNTER_INC(mc_inline_miss_klass);
2358 }
2359#endif
2360
2361 return false;
2362}
2363
2364ALWAYS_INLINE(static const struct rb_callcache *vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE klass));
2365static const struct rb_callcache *
2366vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE klass)
2367{
2368 const struct rb_callcache *cc = cd->cc;
2369 if (vm_cc_hit_p(cc, cd, klass)) {
2370 return cc;
2371 }
2372
2373 return vm_search_method_slowpath0((VALUE)CFP_ISEQ(reg_cfp), cd, klass);
2374}
2375
2376static const struct rb_callable_method_entry_struct *
2377vm_search_method(struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE recv)
2378{
2379 VALUE klass = CLASS_OF(recv);
2380 VM_ASSERT(klass != Qfalse);
2381 VM_ASSERT(RBASIC_CLASS(klass) == 0 || rb_obj_is_kind_of(klass, rb_cClass));
2382
2383 const struct rb_callcache *cc = vm_search_method_fastpath(reg_cfp, cd, klass);
2384 return vm_cc_cme(cc);
2385}
2386
2387const struct rb_callable_method_entry_struct *
2388rb_zjit_vm_search_method(VALUE cd_owner, struct rb_call_data *cd, VALUE recv)
2389{
2390 // Called from ZJIT with the compile-time iseq, which may differ from
2391 // the iseq on the current CFP.
2392 VALUE klass = CLASS_OF(recv);
2393 const struct rb_callcache *cc = cd->cc;
2394 if (!vm_cc_hit_p(cc, cd, klass)) {
2395 cc = vm_search_method_slowpath0(cd_owner, cd, klass);
2396 }
2397 return vm_cc_cme(cc);
2398}
2399
2400#if __has_attribute(transparent_union)
2401typedef union {
2402 VALUE (*anyargs)(ANYARGS);
2403 VALUE (*f00)(VALUE);
2404 VALUE (*f01)(VALUE, VALUE);
2405 VALUE (*f02)(VALUE, VALUE, VALUE);
2406 VALUE (*f03)(VALUE, VALUE, VALUE, VALUE);
2407 VALUE (*f04)(VALUE, VALUE, VALUE, VALUE, VALUE);
2408 VALUE (*f05)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2409 VALUE (*f06)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2410 VALUE (*f07)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2411 VALUE (*f08)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2412 VALUE (*f09)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2413 VALUE (*f10)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2414 VALUE (*f11)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2415 VALUE (*f12)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2416 VALUE (*f13)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2417 VALUE (*f14)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2418 VALUE (*f15)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2419 VALUE (*fm1)(int, union { VALUE *x; const VALUE *y; } __attribute__((__transparent_union__)), VALUE);
2420} __attribute__((__transparent_union__)) cfunc_type;
2421# define make_cfunc_type(f) (cfunc_type){.anyargs = (VALUE (*)(ANYARGS))(f)}
2422#else
2423typedef VALUE (*cfunc_type)(ANYARGS);
2424# define make_cfunc_type(f) (cfunc_type)(f)
2425#endif
2426
2427static inline int
2428check_cfunc(const rb_callable_method_entry_t *me, cfunc_type func)
2429{
2430 if (! me) {
2431 return false;
2432 }
2433 else {
2434 VM_ASSERT(IMEMO_TYPE_P(me, imemo_ment));
2435 VM_ASSERT(callable_method_entry_p(me));
2436 VM_ASSERT(me->def);
2437 if (me->def->type != VM_METHOD_TYPE_CFUNC) {
2438 return false;
2439 }
2440 else {
2441#if __has_attribute(transparent_union)
2442 return me->def->body.cfunc.func == func.anyargs;
2443#else
2444 return me->def->body.cfunc.func == func;
2445#endif
2446 }
2447 }
2448}
2449
2450static inline int
2451check_method_basic_definition(const rb_callable_method_entry_t *me)
2452{
2453 return me && METHOD_ENTRY_BASIC(me);
2454}
2455
2456static inline int
2457vm_method_cfunc_is(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv, cfunc_type func)
2458{
2459 VM_ASSERT(reg_cfp != NULL);
2460 const struct rb_callable_method_entry_struct *cme = vm_search_method(reg_cfp, cd, recv);
2461 return check_cfunc(cme, func);
2462}
2463
2464bool
2465rb_zjit_cme_is_cfunc(const rb_callable_method_entry_t *me, const cfunc_type func)
2466{
2467 return check_cfunc(me, func);
2468}
2469
2470int
2471rb_vm_method_cfunc_is(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv, cfunc_type func)
2472{
2473 const struct rb_callable_method_entry_struct *cme = rb_zjit_vm_search_method((VALUE)iseq, cd, recv);
2474 return check_cfunc(cme, func);
2475}
2476
2477#define check_cfunc(me, func) check_cfunc(me, make_cfunc_type(func))
2478#define vm_method_cfunc_is(reg_cfp, cd, recv, func) vm_method_cfunc_is(reg_cfp, cd, recv, make_cfunc_type(func))
2479
2480#define OP_UNREDEFINED_P(op, t) BASIC_OP_UNREDEFINED_P(BOP_##op, t##_REDEFINED_OP_FLAG)
2481#define EQ_UNREDEFINED_P(t) OP_UNREDEFINED_P(EQ, t)
2482
2483static inline bool
2484FIXNUM_2_P(VALUE a, VALUE b)
2485{
2486 /* FIXNUM_P(a) && FIXNUM_P(b)
2487 * == ((a & 1) && (b & 1))
2488 * == a & b & 1 */
2489 SIGNED_VALUE x = a;
2490 SIGNED_VALUE y = b;
2491 SIGNED_VALUE z = x & y & 1;
2492 return z == 1;
2493}
2494
2495static inline bool
2496FLONUM_2_P(VALUE a, VALUE b)
2497{
2498#if USE_FLONUM
2499 /* FLONUM_P(a) && FLONUM_P(b)
2500 * == ((a & 3) == 2) && ((b & 3) == 2)
2501 * == ! ((a ^ 2) | (b ^ 2) & 3)
2502 */
2503 SIGNED_VALUE x = a;
2504 SIGNED_VALUE y = b;
2505 SIGNED_VALUE z = ((x ^ 2) | (y ^ 2)) & 3;
2506 return !z;
2507#else
2508 return false;
2509#endif
2510}
2511
2512static VALUE
2513opt_equality_specialized(VALUE recv, VALUE obj)
2514{
2515 if (FIXNUM_2_P(recv, obj) && EQ_UNREDEFINED_P(INTEGER)) {
2516 goto compare_by_identity;
2517 }
2518 else if (FLONUM_2_P(recv, obj) && EQ_UNREDEFINED_P(FLOAT)) {
2519 goto compare_by_identity;
2520 }
2521 else if (STATIC_SYM_P(recv) && STATIC_SYM_P(obj) && EQ_UNREDEFINED_P(SYMBOL)) {
2522 goto compare_by_identity;
2523 }
2524 else if (SPECIAL_CONST_P(recv)) {
2525 //
2526 }
2527 else if (RBASIC_CLASS(recv) == rb_cFloat && RB_FLOAT_TYPE_P(obj) && EQ_UNREDEFINED_P(FLOAT)) {
2528 double a = RFLOAT_VALUE(recv);
2529 double b = RFLOAT_VALUE(obj);
2530
2531 return RBOOL(a == b);
2532 }
2533 else if (RBASIC_CLASS(recv) == rb_cString && EQ_UNREDEFINED_P(STRING)) {
2534 if (recv == obj) {
2535 return Qtrue;
2536 }
2537 else if (RB_TYPE_P(obj, T_STRING)) {
2538 return rb_str_eql_internal(obj, recv);
2539 }
2540 }
2541 return Qundef;
2542
2543 compare_by_identity:
2544 return RBOOL(recv == obj);
2545}
2546
2547static VALUE
2548opt_equality(struct rb_control_frame_struct *reg_cfp, VALUE recv, VALUE obj, CALL_DATA cd)
2549{
2550 VM_ASSERT(reg_cfp != NULL);
2551
2552 VALUE val = opt_equality_specialized(recv, obj);
2553 if (!UNDEF_P(val)) return val;
2554
2555 if (!vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_equal)) {
2556 return Qundef;
2557 }
2558 else {
2559 return RBOOL(recv == obj);
2560 }
2561}
2562
2563static inline const struct rb_callcache *gccct_method_search(rb_execution_context_t *ec, VALUE recv, ID mid, const struct rb_callinfo *ci); // vm_eval.c
2564NOINLINE(static VALUE opt_equality_by_mid_slowpath(VALUE recv, VALUE obj, ID mid));
2565
2566static VALUE
2567opt_equality_by_mid_slowpath(VALUE recv, VALUE obj, ID mid)
2568{
2569 const struct rb_callcache *cc = gccct_method_search(GET_EC(), recv, mid, &VM_CI_ON_STACK(mid, 0, 1, NULL));
2570
2571 if (cc && check_cfunc(vm_cc_cme(cc), rb_obj_equal)) {
2572 return RBOOL(recv == obj);
2573 }
2574 else {
2575 return Qundef;
2576 }
2577}
2578
2579static VALUE
2580opt_equality_by_mid(VALUE recv, VALUE obj, ID mid)
2581{
2582 VALUE val = opt_equality_specialized(recv, obj);
2583 if (!UNDEF_P(val)) {
2584 return val;
2585 }
2586 else {
2587 return opt_equality_by_mid_slowpath(recv, obj, mid);
2588 }
2589}
2590
2591VALUE
2592rb_equal_opt(VALUE obj1, VALUE obj2)
2593{
2594 return opt_equality_by_mid(obj1, obj2, idEq);
2595}
2596
2597VALUE
2598rb_eql_opt(VALUE obj1, VALUE obj2)
2599{
2600 return opt_equality_by_mid(obj1, obj2, idEqlP);
2601}
2602
2603extern VALUE rb_vm_call0(rb_execution_context_t *ec, VALUE, ID, int, const VALUE*, const rb_callable_method_entry_t *, int kw_splat);
2604extern VALUE rb_vm_call_with_refinements(rb_execution_context_t *, VALUE, ID, int, const VALUE *, int);
2605
2606static VALUE
2607check_match(rb_execution_context_t *ec, VALUE pattern, VALUE target, enum vm_check_match_type type)
2608{
2609 switch (type) {
2610 case VM_CHECKMATCH_TYPE_WHEN:
2611 return pattern;
2612 case VM_CHECKMATCH_TYPE_RESCUE:
2613 if (!rb_obj_is_kind_of(pattern, rb_cModule)) {
2614 rb_raise(rb_eTypeError, "class or module required for rescue clause");
2615 }
2616 /* fall through */
2617 case VM_CHECKMATCH_TYPE_CASE: {
2618 return rb_vm_call_with_refinements(ec, pattern, idEqq, 1, &target, RB_NO_KEYWORDS);
2619 }
2620 default:
2621 rb_bug("check_match: unreachable");
2622 }
2623}
2624
2625
2626static inline VALUE
2627double_cmp_lt(double a, double b)
2628{
2629 return RBOOL(a < b);
2630}
2631
2632static inline VALUE
2633double_cmp_le(double a, double b)
2634{
2635 return RBOOL(a <= b);
2636}
2637
2638static inline VALUE
2639double_cmp_gt(double a, double b)
2640{
2641 return RBOOL(a > b);
2642}
2643
2644static inline VALUE
2645double_cmp_ge(double a, double b)
2646{
2647 return RBOOL(a >= b);
2648}
2649
2650// Copied by vm_dump.c
2651static inline VALUE *
2652vm_base_ptr(const rb_control_frame_t *cfp)
2653{
2654 const rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
2655
2656 if (CFP_ISEQ(cfp) && VM_FRAME_RUBYFRAME_P(cfp)) {
2657 VALUE *bp = prev_cfp->sp + ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size + VM_ENV_DATA_SIZE;
2658
2659 if (ISEQ_BODY(CFP_ISEQ(cfp))->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) {
2660 int lts = ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size;
2661 int params = ISEQ_BODY(CFP_ISEQ(cfp))->param.size;
2662
2663 CALL_INFO ci = (CALL_INFO)cfp->ep[-(VM_ENV_DATA_SIZE + (lts - params))]; // skip EP stuff, CI should be last local
2664 bp += vm_ci_argc(ci);
2665 }
2666
2667 if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) {
2668 /* adjust `self' */
2669 bp += 1;
2670 }
2671#if VM_DEBUG_BP_CHECK
2672 if (bp != cfp->bp_check) {
2673 ruby_debug_printf("bp_check: %ld, bp: %ld\n",
2674 (long)(cfp->bp_check - GET_EC()->vm_stack),
2675 (long)(bp - GET_EC()->vm_stack));
2676 rb_bug("vm_base_ptr: unreachable");
2677 }
2678#endif
2679 return bp;
2680 }
2681 else {
2682 return NULL;
2683 }
2684}
2685
2686VALUE *
2687rb_vm_base_ptr(const rb_control_frame_t *cfp)
2688{
2689 return vm_base_ptr(cfp);
2690}
2691
2692/* method call processes with call_info */
2693
2694#include "vm_args.c"
2695
2696static inline VALUE vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc, int param_size, int local_size);
2697ALWAYS_INLINE(static VALUE vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me, int opt_pc, int param_size, int local_size));
2698static inline VALUE vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc);
2699static VALUE vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
2700static VALUE vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2701static VALUE vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2702static inline VALUE vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2703
2704static vm_call_handler vm_call_iseq_setup_func(const struct rb_callinfo *ci, const int param_size, const int local_size);
2705
2706static VALUE
2707vm_call_iseq_setup_tailcall_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2708{
2709 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_tailcall_0start);
2710
2711 return vm_call_iseq_setup_tailcall(ec, cfp, calling, 0);
2712}
2713
2714static VALUE
2715vm_call_iseq_setup_normal_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2716{
2717 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_0start);
2718
2719 const struct rb_callcache *cc = calling->cc;
2720 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2721 int param = ISEQ_BODY(iseq)->param.size;
2722 int local = ISEQ_BODY(iseq)->local_table_size;
2723 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
2724}
2725
2726bool
2727rb_simple_iseq_p(const rb_iseq_t *iseq)
2728{
2729 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2730 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2731 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2732 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2733 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2734 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2735 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2736 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2737 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2738}
2739
2740bool
2741rb_iseq_only_optparam_p(const rb_iseq_t *iseq)
2742{
2743 return ISEQ_BODY(iseq)->param.flags.has_opt == TRUE &&
2744 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2745 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2746 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2747 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2748 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2749 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2750 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2751 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2752}
2753
2754bool
2755rb_iseq_only_kwparam_p(const rb_iseq_t *iseq)
2756{
2757 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2758 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2759 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2760 ISEQ_BODY(iseq)->param.flags.has_kw == TRUE &&
2761 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2762 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2763 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2764 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2765}
2766
2767#define ALLOW_HEAP_ARGV (-2)
2768#define ALLOW_HEAP_ARGV_KEEP_KWSPLAT (-3)
2769
2770static inline bool
2771vm_caller_setup_arg_splat(rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE ary, int max_args)
2772{
2773 vm_check_canary(GET_EC(), cfp->sp);
2774 bool ret = false;
2775
2776 if (!NIL_P(ary)) {
2777 const VALUE *ptr = RARRAY_CONST_PTR(ary);
2778 long len = RARRAY_LEN(ary);
2779 int argc = calling->argc;
2780
2781 if (UNLIKELY(max_args <= ALLOW_HEAP_ARGV && len + argc > VM_ARGC_STACK_MAX)) {
2782 /* Avoid SystemStackError when splatting large arrays by storing arguments in
2783 * a temporary array, instead of trying to keeping arguments on the VM stack.
2784 */
2785 VALUE *argv = cfp->sp - argc;
2786 VALUE argv_ary = rb_ary_hidden_new(len + argc + 1);
2787 rb_ary_cat(argv_ary, argv, argc);
2788 rb_ary_cat(argv_ary, ptr, len);
2789 cfp->sp -= argc - 1;
2790 cfp->sp[-1] = argv_ary;
2791 calling->argc = 1;
2792 calling->heap_argv = argv_ary;
2793 RB_GC_GUARD(ary);
2794 }
2795 else {
2796 long i;
2797
2798 if (max_args >= 0 && len + argc > max_args) {
2799 /* If only a given max_args is allowed, copy up to max args.
2800 * Used by vm_callee_setup_block_arg for non-lambda blocks,
2801 * where additional arguments are ignored.
2802 *
2803 * Also, copy up to one more argument than the maximum,
2804 * in case it is an empty keyword hash that will be removed.
2805 */
2806 calling->argc += len - (max_args - argc + 1);
2807 len = max_args - argc + 1;
2808 ret = true;
2809 }
2810 else {
2811 /* Unset heap_argv if set originally. Can happen when
2812 * forwarding modified arguments, where heap_argv was used
2813 * originally, but heap_argv not supported by the forwarded
2814 * method in all cases.
2815 */
2816 calling->heap_argv = 0;
2817 }
2818 CHECK_VM_STACK_OVERFLOW(cfp, len);
2819
2820 for (i = 0; i < len; i++) {
2821 *cfp->sp++ = ptr[i];
2822 }
2823 calling->argc += i;
2824 }
2825 }
2826
2827 return ret;
2828}
2829
2830static inline void
2831vm_caller_setup_arg_kw(rb_control_frame_t *cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci)
2832{
2833 const VALUE *const passed_keywords = vm_ci_kwarg(ci)->keywords;
2834 const int kw_len = vm_ci_kwarg(ci)->keyword_len;
2835 const VALUE h = rb_hash_new_capa(kw_len);
2836 VALUE *sp = cfp->sp;
2837 int i;
2838
2839 for (i=0; i<kw_len; i++) {
2840 rb_hash_aset(h, passed_keywords[i], (sp - kw_len)[i]);
2841 }
2842 (sp-kw_len)[0] = h;
2843
2844 cfp->sp -= kw_len - 1;
2845 calling->argc -= kw_len - 1;
2846 calling->kw_splat = 1;
2847}
2848
2849static inline VALUE
2850vm_caller_setup_keyword_hash(const struct rb_callinfo *ci, VALUE keyword_hash)
2851{
2852 if (UNLIKELY(!RB_TYPE_P(keyword_hash, T_HASH))) {
2853 if (keyword_hash != Qnil) {
2854 /* Convert a non-hash keyword splat to a new hash */
2855 keyword_hash = rb_hash_dup(rb_to_hash_type(keyword_hash));
2856 }
2857 }
2858 else if (!IS_ARGS_KW_SPLAT_MUT(ci) && !RHASH_EMPTY_P(keyword_hash)) {
2859 /* Convert a hash keyword splat to a new hash unless
2860 * a mutable keyword splat was passed.
2861 * Skip allocating new hash for empty keyword splat, as empty
2862 * keyword splat will be ignored by both callers.
2863 */
2864 keyword_hash = rb_hash_dup(keyword_hash);
2865 }
2866 return keyword_hash;
2867}
2868
2869static inline void
2870CALLER_SETUP_ARG(struct rb_control_frame_struct *restrict cfp,
2871 struct rb_calling_info *restrict calling,
2872 const struct rb_callinfo *restrict ci, int max_args)
2873{
2874 if (UNLIKELY(IS_ARGS_SPLAT(ci))) {
2875 if (IS_ARGS_KW_SPLAT(ci)) {
2876 // f(*a, **kw)
2877 VM_ASSERT(calling->kw_splat == 1);
2878
2879 cfp->sp -= 2;
2880 calling->argc -= 2;
2881 VALUE ary = cfp->sp[0];
2882 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[1]);
2883
2884 // splat a
2885 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) return;
2886
2887 // put kw
2888 if (kwh != Qnil && !RHASH_EMPTY_P(kwh)) {
2889 if (UNLIKELY(calling->heap_argv)) {
2890 rb_ary_push(calling->heap_argv, kwh);
2891 ((struct RHash *)kwh)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
2892 if (max_args != ALLOW_HEAP_ARGV_KEEP_KWSPLAT) {
2893 calling->kw_splat = 0;
2894 }
2895 }
2896 else {
2897 cfp->sp[0] = kwh;
2898 cfp->sp++;
2899 calling->argc++;
2900
2901 VM_ASSERT(calling->kw_splat == 1);
2902 }
2903 }
2904 else {
2905 calling->kw_splat = 0;
2906 }
2907 }
2908 else {
2909 // f(*a)
2910 VM_ASSERT(calling->kw_splat == 0);
2911
2912 cfp->sp -= 1;
2913 calling->argc -= 1;
2914 VALUE ary = cfp->sp[0];
2915
2916 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) {
2917 goto check_keyword;
2918 }
2919
2920 // check the last argument
2921 VALUE last_hash, argv_ary;
2922 if (UNLIKELY(argv_ary = calling->heap_argv)) {
2923 if (!IS_ARGS_KEYWORD(ci) &&
2924 RARRAY_LEN(argv_ary) > 0 &&
2925 RB_TYPE_P((last_hash = rb_ary_last(0, NULL, argv_ary)), T_HASH) &&
2926 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2927
2928 rb_ary_pop(argv_ary);
2929 if (!RHASH_EMPTY_P(last_hash)) {
2930 rb_ary_push(argv_ary, rb_hash_dup(last_hash));
2931 calling->kw_splat = 1;
2932 }
2933 }
2934 }
2935 else {
2936check_keyword:
2937 if (!IS_ARGS_KEYWORD(ci) &&
2938 calling->argc > 0 &&
2939 RB_TYPE_P((last_hash = cfp->sp[-1]), T_HASH) &&
2940 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2941
2942 if (RHASH_EMPTY_P(last_hash)) {
2943 calling->argc--;
2944 cfp->sp -= 1;
2945 }
2946 else {
2947 cfp->sp[-1] = rb_hash_dup(last_hash);
2948 calling->kw_splat = 1;
2949 }
2950 }
2951 }
2952 }
2953 }
2954 else if (UNLIKELY(IS_ARGS_KW_SPLAT(ci))) {
2955 // f(**kw)
2956 VM_ASSERT(calling->kw_splat == 1);
2957 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[-1]);
2958
2959 if (kwh == Qnil || RHASH_EMPTY_P(kwh)) {
2960 cfp->sp--;
2961 calling->argc--;
2962 calling->kw_splat = 0;
2963 }
2964 else {
2965 cfp->sp[-1] = kwh;
2966 }
2967 }
2968 else if (UNLIKELY(IS_ARGS_KEYWORD(ci))) {
2969 // f(k1:1, k2:2)
2970 VM_ASSERT(calling->kw_splat == 0);
2971
2972 /* This converts VM_CALL_KWARG style to VM_CALL_KW_SPLAT style
2973 * by creating a keyword hash.
2974 * So, vm_ci_flag(ci) & VM_CALL_KWARG is now inconsistent.
2975 */
2976 vm_caller_setup_arg_kw(cfp, calling, ci);
2977 }
2978}
2979
2980#define USE_OPT_HIST 0
2981
2982#if USE_OPT_HIST
2983#define OPT_HIST_MAX 64
2984static int opt_hist[OPT_HIST_MAX+1];
2985
2986__attribute__((destructor))
2987static void
2988opt_hist_show_results_at_exit(void)
2989{
2990 for (int i=0; i<OPT_HIST_MAX; i++) {
2991 ruby_debug_printf("opt_hist\t%d\t%d\n", i, opt_hist[i]);
2992 }
2993}
2994#endif
2995
2996static VALUE
2997vm_call_iseq_setup_normal_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
2998 struct rb_calling_info *calling)
2999{
3000 const struct rb_callcache *cc = calling->cc;
3001 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3002 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3003 const int opt = calling->argc - lead_num;
3004 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
3005 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3006 const int param = ISEQ_BODY(iseq)->param.size;
3007 const int local = ISEQ_BODY(iseq)->local_table_size;
3008 const int delta = opt_num - opt;
3009
3010 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
3011
3012#if USE_OPT_HIST
3013 if (opt_pc < OPT_HIST_MAX) {
3014 opt_hist[opt]++;
3015 }
3016 else {
3017 opt_hist[OPT_HIST_MAX]++;
3018 }
3019#endif
3020
3021 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param - delta, local);
3022}
3023
3024static VALUE
3025vm_call_iseq_setup_tailcall_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3026 struct rb_calling_info *calling)
3027{
3028 const struct rb_callcache *cc = calling->cc;
3029 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3030 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3031 const int opt = calling->argc - lead_num;
3032 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3033
3034 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
3035
3036#if USE_OPT_HIST
3037 if (opt_pc < OPT_HIST_MAX) {
3038 opt_hist[opt]++;
3039 }
3040 else {
3041 opt_hist[OPT_HIST_MAX]++;
3042 }
3043#endif
3044
3045 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc);
3046}
3047
3048static void
3049args_setup_kw_parameters(rb_execution_context_t *const ec, const rb_iseq_t *const iseq, const rb_callable_method_entry_t *cme,
3050 VALUE *const passed_values, const int passed_keyword_len, const VALUE *const passed_keywords,
3051 VALUE *const locals);
3052
3053static VALUE
3054vm_call_iseq_forwardable(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3055 struct rb_calling_info *calling)
3056{
3057 const struct rb_callcache *cc = calling->cc;
3058 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3059 int param_size = ISEQ_BODY(iseq)->param.size;
3060 int local_size = ISEQ_BODY(iseq)->local_table_size;
3061
3062 // Setting up local size and param size
3063 VM_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
3064
3065 local_size = local_size + vm_ci_argc(calling->cd->ci);
3066 param_size = param_size + vm_ci_argc(calling->cd->ci);
3067
3068 cfp->sp[0] = (VALUE)calling->cd->ci;
3069
3070 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param_size, local_size);
3071}
3072
3073static VALUE
3074vm_call_iseq_setup_kwparm_kwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3075 struct rb_calling_info *calling)
3076{
3077 const struct rb_callinfo *ci = calling->cd->ci;
3078 const struct rb_callcache *cc = calling->cc;
3079
3080 VM_ASSERT(vm_ci_flag(ci) & VM_CALL_KWARG);
3081 RB_DEBUG_COUNTER_INC(ccf_iseq_kw1);
3082
3083 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3084 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3085 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
3086 const int ci_kw_len = kw_arg->keyword_len;
3087 const VALUE * const ci_keywords = kw_arg->keywords;
3088 VALUE *argv = cfp->sp - calling->argc;
3089 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3090 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3091 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
3092 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
3093 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
3094
3095 int param = ISEQ_BODY(iseq)->param.size;
3096 int local = ISEQ_BODY(iseq)->local_table_size;
3097 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
3098}
3099
3100static VALUE
3101vm_call_iseq_setup_kwparm_nokwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3102 struct rb_calling_info *calling)
3103{
3104 const struct rb_callinfo *MAYBE_UNUSED(ci) = calling->cd->ci;
3105 const struct rb_callcache *cc = calling->cc;
3106
3107 VM_ASSERT((vm_ci_flag(ci) & VM_CALL_KWARG) == 0);
3108 RB_DEBUG_COUNTER_INC(ccf_iseq_kw2);
3109
3110 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3111 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3112 VALUE * const argv = cfp->sp - calling->argc;
3113 VALUE * const klocals = argv + kw_param->bits_start - kw_param->num;
3114
3115 int i;
3116 for (i=0; i<kw_param->num; i++) {
3117 klocals[i] = kw_param->default_values[i];
3118 }
3119 klocals[i] = INT2FIX(0); // kw specify flag
3120 // NOTE:
3121 // nobody check this value, but it should be cleared because it can
3122 // points invalid VALUE (T_NONE objects, raw pointer and so on).
3123
3124 int param = ISEQ_BODY(iseq)->param.size;
3125 int local = ISEQ_BODY(iseq)->local_table_size;
3126 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
3127}
3128
3129static VALUE builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
3130
3131static VALUE
3132vm_call_single_noarg_leaf_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3133 struct rb_calling_info *calling)
3134{
3135 const struct rb_builtin_function *bf = calling->cc->aux_.bf;
3136 cfp->sp -= (calling->argc + 1);
3137 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
3138 return builtin_invoker0(ec, calling->recv, NULL, func_ptr);
3139}
3140
3141VALUE rb_gen_method_name(VALUE owner, VALUE name); // in vm_backtrace.c
3142
3143static void
3144warn_unused_block(const rb_callable_method_entry_t *cme, const rb_iseq_t *iseq, void *pc)
3145{
3146 rb_vm_t *vm = GET_VM();
3147 set_table *dup_check_table = &vm->unused_block_warning_table;
3148 st_data_t key;
3149 bool strict_unused_block = rb_warning_category_enabled_p(RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK);
3150
3151 union {
3152 VALUE v;
3153 unsigned char b[SIZEOF_VALUE];
3154 } k1 = {
3155 .v = (VALUE)pc,
3156 }, k2 = {
3157 .v = (VALUE)cme->def,
3158 };
3159
3160 // relax check
3161 if (!strict_unused_block) {
3162 key = (st_data_t)cme->def->original_id;
3163
3164 if (set_table_lookup(dup_check_table, key)) {
3165 return;
3166 }
3167 }
3168
3169 // strict check
3170 // make unique key from pc and me->def pointer
3171 key = 0;
3172 for (int i=0; i<SIZEOF_VALUE; i++) {
3173 // fprintf(stderr, "k1:%3d k2:%3d\n", k1.b[i], k2.b[SIZEOF_VALUE-1-i]);
3174 key |= (st_data_t)(k1.b[i] ^ k2.b[SIZEOF_VALUE-1-i]) << (8 * i);
3175 }
3176
3177 if (0) {
3178 fprintf(stderr, "SIZEOF_VALUE:%d\n", SIZEOF_VALUE);
3179 fprintf(stderr, "pc:%p def:%p\n", pc, (void *)cme->def);
3180 fprintf(stderr, "key:%p\n", (void *)key);
3181 }
3182
3183 // duplication check
3184 if (set_insert(dup_check_table, key)) {
3185 // already shown
3186 }
3187 else if (RTEST(ruby_verbose) || strict_unused_block) {
3188 VALUE m_loc = rb_method_entry_location((const rb_method_entry_t *)cme);
3189 VALUE name = rb_gen_method_name(cme->defined_class, rb_iseq_base_label(iseq));
3190
3191 if (!NIL_P(m_loc)) {
3192 rb_warn("the block passed to '%"PRIsVALUE"' defined at %"PRIsVALUE":%"PRIsVALUE" may be ignored",
3193 name, RARRAY_AREF(m_loc, 0), RARRAY_AREF(m_loc, 1));
3194 }
3195 else {
3196 rb_warn("the block may be ignored because '%"PRIsVALUE"' does not use a block", name);
3197 }
3198 }
3199}
3200
3201static inline int
3202vm_callee_setup_arg(rb_execution_context_t *ec, struct rb_calling_info *calling,
3203 const rb_iseq_t *iseq, VALUE *argv, int param_size, int local_size)
3204{
3205 const struct rb_callinfo *ci = calling->cd->ci;
3206 const struct rb_callcache *cc = calling->cc;
3207
3208 VM_ASSERT((vm_ci_argc(ci), 1));
3209 VM_ASSERT(vm_cc_cme(cc) != NULL);
3210
3211 if (UNLIKELY(!ISEQ_BODY(iseq)->param.flags.use_block &&
3212 calling->block_handler != VM_BLOCK_HANDLER_NONE &&
3213 !(vm_ci_flag(calling->cd->ci) & (VM_CALL_OPT_SEND | VM_CALL_SUPER)))) {
3214 warn_unused_block(vm_cc_cme(cc), iseq, (void *)CFP_PC(ec->cfp));
3215 }
3216
3217 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_KW_SPLAT))) {
3218 if (LIKELY(rb_simple_iseq_p(iseq))) {
3219 rb_control_frame_t *cfp = ec->cfp;
3220 int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3221 CALLER_SETUP_ARG(cfp, calling, ci, lead_num);
3222
3223 if (calling->argc != lead_num) {
3224 argument_arity_error(ec, iseq, vm_cc_cme(cc), calling->argc, lead_num, lead_num);
3225 }
3226
3227 //VM_ASSERT(ci == calling->cd->ci);
3228 VM_ASSERT(cc == calling->cc);
3229
3230 if (vm_call_iseq_optimizable_p(ci, cc)) {
3231 if ((ISEQ_BODY(iseq)->builtin_attrs & BUILTIN_ATTR_SINGLE_NOARG_LEAF) && ruby_vm_c_events_enabled == 0) {
3232 VM_ASSERT(ISEQ_BODY(iseq)->builtin_attrs & BUILTIN_ATTR_LEAF);
3233 vm_cc_bf_set(cc, (void *)ISEQ_BODY(iseq)->iseq_encoded[1]);
3234 CC_SET_FASTPATH(cc, vm_call_single_noarg_leaf_builtin, true);
3235 }
3236 else {
3237 CC_SET_FASTPATH(cc, vm_call_iseq_setup_func(ci, param_size, local_size), true);
3238 }
3239 }
3240 return 0;
3241 }
3242 else if (rb_iseq_only_optparam_p(iseq)) {
3243 rb_control_frame_t *cfp = ec->cfp;
3244
3245 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3246 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
3247
3248 CALLER_SETUP_ARG(cfp, calling, ci, lead_num + opt_num);
3249 const int argc = calling->argc;
3250 const int opt = argc - lead_num;
3251
3252 if (opt < 0 || opt > opt_num) {
3253 argument_arity_error(ec, iseq, vm_cc_cme(cc), argc, lead_num, lead_num + opt_num);
3254 }
3255
3256 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3257 CC_SET_FASTPATH(cc, vm_call_iseq_setup_normal_opt_start,
3258 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3259 vm_call_cacheable(ci, cc));
3260 }
3261 else {
3262 CC_SET_FASTPATH(cc, vm_call_iseq_setup_tailcall_opt_start,
3263 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3264 vm_call_cacheable(ci, cc));
3265 }
3266
3267 /* initialize opt vars for self-references */
3268 VM_ASSERT((int)ISEQ_BODY(iseq)->param.size == lead_num + opt_num);
3269 for (int i=argc; i<lead_num + opt_num; i++) {
3270 argv[i] = Qnil;
3271 }
3272 return (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3273 }
3274 else if (rb_iseq_only_kwparam_p(iseq) && !IS_ARGS_SPLAT(ci)) {
3275 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3276 const int argc = calling->argc;
3277 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3278
3279 if (vm_ci_flag(ci) & VM_CALL_KWARG) {
3280 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
3281
3282 if (argc - kw_arg->keyword_len == lead_num) {
3283 const int ci_kw_len = kw_arg->keyword_len;
3284 const VALUE * const ci_keywords = kw_arg->keywords;
3285 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
3286 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
3287
3288 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3289 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
3290
3291 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_kwarg,
3292 vm_call_cacheable(ci, cc));
3293
3294 return 0;
3295 }
3296 }
3297 else if (argc == lead_num) {
3298 /* no kwarg */
3299 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3300 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), NULL, 0, NULL, klocals);
3301
3302 if (klocals[kw_param->num] == INT2FIX(0)) {
3303 /* copy from default_values */
3304 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_nokwarg,
3305 vm_call_cacheable(ci, cc));
3306 }
3307
3308 return 0;
3309 }
3310 }
3311 }
3312
3313 // Called iseq is using ... param
3314 // def foo(...) # <- iseq for foo will have "forwardable"
3315 //
3316 // We want to set the `...` local to the caller's CI
3317 // foo(1, 2) # <- the ci for this should end up as `...`
3318 //
3319 // So hopefully the stack looks like:
3320 //
3321 // => 1
3322 // => 2
3323 // => *
3324 // => **
3325 // => &
3326 // => ... # <- points at `foo`s CI
3327 // => cref_or_me
3328 // => specval
3329 // => type
3330 //
3331 if (ISEQ_BODY(iseq)->param.flags.forwardable) {
3332 bool can_fastpath = true;
3333
3334 if ((vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
3335 struct rb_forwarding_call_data * forward_cd = (struct rb_forwarding_call_data *)calling->cd;
3336 if (vm_ci_argc(ci) != vm_ci_argc(forward_cd->caller_ci)) {
3337 ci = vm_ci_new_runtime(
3338 vm_ci_mid(ci),
3339 vm_ci_flag(ci),
3340 vm_ci_argc(ci),
3341 vm_ci_kwarg(ci));
3342 }
3343 else {
3344 ci = forward_cd->caller_ci;
3345 }
3346 can_fastpath = false;
3347 }
3348 // C functions calling iseqs will stack allocate a CI,
3349 // so we need to convert it to heap allocated
3350 if (!vm_ci_markable(ci)) {
3351 ci = vm_ci_new_runtime(
3352 vm_ci_mid(ci),
3353 vm_ci_flag(ci),
3354 vm_ci_argc(ci),
3355 vm_ci_kwarg(ci));
3356 can_fastpath = false;
3357 }
3358 argv[param_size - 1] = (VALUE)ci;
3359 CC_SET_FASTPATH(cc, vm_call_iseq_forwardable, can_fastpath);
3360 return 0;
3361 }
3362
3363 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_method);
3364}
3365
3366static void
3367vm_adjust_stack_forwarding(const struct rb_execution_context_struct *ec, struct rb_control_frame_struct *cfp, int argc, VALUE splat)
3368{
3369 // This case is when the caller is using a ... parameter.
3370 // For example `bar(...)`. The call info will have VM_CALL_FORWARDING
3371 // In this case the caller's caller's CI will be on the stack.
3372 //
3373 // For example:
3374 //
3375 // def bar(a, b); a + b; end
3376 // def foo(...); bar(...); end
3377 // foo(1, 2) # <- this CI will be on the stack when we call `bar(...)`
3378 //
3379 // Stack layout will be:
3380 //
3381 // > 1
3382 // > 2
3383 // > CI for foo(1, 2)
3384 // > cref_or_me
3385 // > specval
3386 // > type
3387 // > receiver
3388 // > CI for foo(1, 2), via `getlocal ...`
3389 // > ( SP points here )
3390 const VALUE * lep = VM_CF_LEP(cfp);
3391
3392 const rb_iseq_t *iseq;
3393
3394 // If we're in an escaped environment (lambda for example), get the iseq
3395 // from the captured env.
3396 if (VM_ENV_FLAGS(lep, VM_ENV_FLAG_ESCAPED)) {
3397 rb_env_t * env = (rb_env_t *)lep[VM_ENV_DATA_INDEX_ENV];
3398 iseq = env->iseq;
3399 }
3400 else { // Otherwise use the lep to find the caller
3401 iseq = CFP_ISEQ(rb_vm_search_cf_from_ep(ec, cfp, lep));
3402 }
3403
3404 // Our local storage is below the args we need to copy
3405 int local_size = ISEQ_BODY(iseq)->local_table_size + argc;
3406
3407 const VALUE * from = lep - (local_size + VM_ENV_DATA_SIZE - 1); // 2 for EP values
3408 VALUE * to = cfp->sp - 1; // clobber the CI
3409
3410 if (RTEST(splat)) {
3411 to -= 1; // clobber the splat array
3412 CHECK_VM_STACK_OVERFLOW0(cfp, to, RARRAY_LEN(splat));
3413 MEMCPY(to, RARRAY_CONST_PTR(splat), VALUE, RARRAY_LEN(splat));
3414 to += RARRAY_LEN(splat);
3415 }
3416
3417 CHECK_VM_STACK_OVERFLOW0(cfp, to, argc);
3418 MEMCPY(to, from, VALUE, argc);
3419 cfp->sp = to + argc;
3420
3421 // Stack layout should now be:
3422 //
3423 // > 1
3424 // > 2
3425 // > CI for foo(1, 2)
3426 // > cref_or_me
3427 // > specval
3428 // > type
3429 // > receiver
3430 // > 1
3431 // > 2
3432 // > ( SP points here )
3433}
3434
3435static VALUE
3436vm_call_iseq_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3437{
3438 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3439
3440 const struct rb_callcache *cc = calling->cc;
3441 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3442 int param_size = ISEQ_BODY(iseq)->param.size;
3443 int local_size = ISEQ_BODY(iseq)->local_table_size;
3444
3445 RUBY_ASSERT(!ISEQ_BODY(iseq)->param.flags.forwardable);
3446
3447 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3448 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3449}
3450
3451static VALUE
3452vm_call_iseq_fwd_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3453{
3454 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3455
3456 const struct rb_callcache *cc = calling->cc;
3457 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3458 int param_size = ISEQ_BODY(iseq)->param.size;
3459 int local_size = ISEQ_BODY(iseq)->local_table_size;
3460
3461 RUBY_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
3462
3463 // Setting up local size and param size
3464 local_size = local_size + vm_ci_argc(calling->cd->ci);
3465 param_size = param_size + vm_ci_argc(calling->cd->ci);
3466
3467 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3468 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3469}
3470
3471static inline VALUE
3472vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
3473 int opt_pc, int param_size, int local_size)
3474{
3475 const struct rb_callinfo *ci = calling->cd->ci;
3476 const struct rb_callcache *cc = calling->cc;
3477
3478 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3479 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param_size, local_size);
3480 }
3481 else {
3482 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc);
3483 }
3484}
3485
3486static inline VALUE
3487vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me,
3488 int opt_pc, int param_size, int local_size)
3489{
3490 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3491 VALUE *argv = cfp->sp - calling->argc;
3492 VALUE *sp = argv + param_size;
3493 cfp->sp = argv - 1 /* recv */;
3494
3495 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL, calling->recv,
3496 calling->block_handler, (VALUE)me,
3497 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3498 local_size - param_size,
3499 ISEQ_BODY(iseq)->stack_max);
3500 return Qundef;
3501}
3502
3503static inline VALUE
3504vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc)
3505{
3506 const struct rb_callcache *cc = calling->cc;
3507 unsigned int i;
3508 VALUE *argv = cfp->sp - calling->argc;
3509 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3510 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3511 VALUE *src_argv = argv;
3512 VALUE *sp_orig, *sp;
3513 VALUE finish_flag = VM_FRAME_FINISHED_P(cfp) ? VM_FRAME_FLAG_FINISH : 0;
3514
3515 if (VM_BH_FROM_CFP_P(calling->block_handler, cfp)) {
3516 struct rb_captured_block *dst_captured = VM_CFP_TO_CAPTURED_BLOCK(RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp));
3517 const struct rb_captured_block *src_captured = VM_BH_TO_CAPT_BLOCK(calling->block_handler);
3518 dst_captured->code.val = src_captured->code.val;
3519 if (VM_BH_ISEQ_BLOCK_P(calling->block_handler)) {
3520 calling->block_handler = VM_BH_FROM_ISEQ_BLOCK(dst_captured);
3521 }
3522 else {
3523 calling->block_handler = VM_BH_FROM_IFUNC_BLOCK(dst_captured);
3524 }
3525 }
3526
3527 vm_pop_frame(ec, cfp, cfp->ep);
3528 cfp = ec->cfp;
3529
3530 sp_orig = sp = cfp->sp;
3531
3532 /* push self */
3533 sp[0] = calling->recv;
3534 sp++;
3535
3536 /* copy arguments */
3537 for (i=0; i < ISEQ_BODY(iseq)->param.size; i++) {
3538 *sp++ = src_argv[i];
3539 }
3540
3541 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL | finish_flag,
3542 calling->recv, calling->block_handler, (VALUE)me,
3543 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3544 ISEQ_BODY(iseq)->local_table_size - ISEQ_BODY(iseq)->param.size,
3545 ISEQ_BODY(iseq)->stack_max);
3546
3547 cfp->sp = sp_orig;
3548
3549 return Qundef;
3550}
3551
3552static void
3553ractor_unsafe_check(void)
3554{
3555 if (!rb_ractor_main_p()) {
3556 rb_raise(rb_eRactorUnsafeError, "ractor unsafe method called from not main ractor");
3557 }
3558}
3559
3560static VALUE
3561call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3562{
3563 ractor_unsafe_check();
3564 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3565 return (*f)(recv, rb_ary_new4(argc, argv));
3566}
3567
3568static VALUE
3569call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3570{
3571 ractor_unsafe_check();
3572 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3573 return (*f)(argc, argv, recv);
3574}
3575
3576static VALUE
3577call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3578{
3579 ractor_unsafe_check();
3580 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3581 return (*f)(recv);
3582}
3583
3584static VALUE
3585call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3586{
3587 ractor_unsafe_check();
3588 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3589 return (*f)(recv, argv[0]);
3590}
3591
3592static VALUE
3593call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3594{
3595 ractor_unsafe_check();
3596 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3597 return (*f)(recv, argv[0], argv[1]);
3598}
3599
3600static VALUE
3601call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3602{
3603 ractor_unsafe_check();
3604 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3605 return (*f)(recv, argv[0], argv[1], argv[2]);
3606}
3607
3608static VALUE
3609call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3610{
3611 ractor_unsafe_check();
3612 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3613 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3614}
3615
3616static VALUE
3617call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3618{
3619 ractor_unsafe_check();
3620 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3621 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3622}
3623
3624static VALUE
3625call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3626{
3627 ractor_unsafe_check();
3628 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3629 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3630}
3631
3632static VALUE
3633call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3634{
3635 ractor_unsafe_check();
3636 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3637 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3638}
3639
3640static VALUE
3641call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3642{
3643 ractor_unsafe_check();
3644 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3645 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3646}
3647
3648static VALUE
3649call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3650{
3651 ractor_unsafe_check();
3652 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3653 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3654}
3655
3656static VALUE
3657call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3658{
3659 ractor_unsafe_check();
3660 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3661 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3662}
3663
3664static VALUE
3665call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3666{
3667 ractor_unsafe_check();
3668 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3669 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3670}
3671
3672static VALUE
3673call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3674{
3675 ractor_unsafe_check();
3676 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3677 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3678}
3679
3680static VALUE
3681call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3682{
3683 ractor_unsafe_check();
3684 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3685 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3686}
3687
3688static VALUE
3689call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3690{
3691 ractor_unsafe_check();
3692 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3693 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3694}
3695
3696static VALUE
3697call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3698{
3699 ractor_unsafe_check();
3700 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3701 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3702}
3703
3704static VALUE
3705ractor_safe_call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3706{
3707 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3708 return (*f)(recv, rb_ary_new4(argc, argv));
3709}
3710
3711static VALUE
3712ractor_safe_call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3713{
3714 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3715 return (*f)(argc, argv, recv);
3716}
3717
3718static VALUE
3719ractor_safe_call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3720{
3721 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3722 return (*f)(recv);
3723}
3724
3725static VALUE
3726ractor_safe_call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3727{
3728 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3729 return (*f)(recv, argv[0]);
3730}
3731
3732static VALUE
3733ractor_safe_call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3734{
3735 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3736 return (*f)(recv, argv[0], argv[1]);
3737}
3738
3739static VALUE
3740ractor_safe_call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3741{
3742 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3743 return (*f)(recv, argv[0], argv[1], argv[2]);
3744}
3745
3746static VALUE
3747ractor_safe_call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3748{
3749 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3750 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3751}
3752
3753static VALUE
3754ractor_safe_call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3755{
3756 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3757 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3758}
3759
3760static VALUE
3761ractor_safe_call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3762{
3763 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3764 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3765}
3766
3767static VALUE
3768ractor_safe_call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3769{
3770 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3771 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3772}
3773
3774static VALUE
3775ractor_safe_call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3776{
3777 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3778 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3779}
3780
3781static VALUE
3782ractor_safe_call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3783{
3784 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3785 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3786}
3787
3788static VALUE
3789ractor_safe_call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3790{
3791 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3792 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3793}
3794
3795static VALUE
3796ractor_safe_call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3797{
3798 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3799 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3800}
3801
3802static VALUE
3803ractor_safe_call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3804{
3805 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3806 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3807}
3808
3809static VALUE
3810ractor_safe_call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3811{
3812 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3813 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3814}
3815
3816static VALUE
3817ractor_safe_call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3818{
3819 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3820 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3821}
3822
3823static VALUE
3824ractor_safe_call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3825{
3826 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3827 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3828}
3829
3830static inline int
3831vm_cfp_consistent_p(rb_execution_context_t *ec, const rb_control_frame_t *reg_cfp)
3832{
3833 const int ov_flags = RAISED_STACKOVERFLOW;
3834 if (LIKELY(reg_cfp == ec->cfp + 1)) return TRUE;
3835 if (rb_ec_raised_p(ec, ov_flags)) {
3836 rb_ec_raised_reset(ec, ov_flags);
3837 return TRUE;
3838 }
3839 return FALSE;
3840}
3841
3842#define CHECK_CFP_CONSISTENCY(func) \
3843 (LIKELY(vm_cfp_consistent_p(ec, reg_cfp)) ? (void)0 : \
3844 rb_bug(func ": cfp consistency error (%p, %p)", (void *)reg_cfp, (void *)(ec->cfp+1)))
3845
3846static inline
3847const rb_method_cfunc_t *
3848vm_method_cfunc_entry(const rb_callable_method_entry_t *me)
3849{
3850#if VM_DEBUG_VERIFY_METHOD_CACHE
3851 switch (me->def->type) {
3852 case VM_METHOD_TYPE_CFUNC:
3853 case VM_METHOD_TYPE_NOTIMPLEMENTED:
3854 break;
3855# define METHOD_BUG(t) case VM_METHOD_TYPE_##t: rb_bug("wrong method type: " #t)
3856 METHOD_BUG(ISEQ);
3857 METHOD_BUG(ATTRSET);
3858 METHOD_BUG(IVAR);
3859 METHOD_BUG(BMETHOD);
3860 METHOD_BUG(ZSUPER);
3861 METHOD_BUG(UNDEF);
3862 METHOD_BUG(OPTIMIZED);
3863 METHOD_BUG(MISSING);
3864 METHOD_BUG(REFINED);
3865 METHOD_BUG(ALIAS);
3866# undef METHOD_BUG
3867 default:
3868 rb_bug("wrong method type: %d", me->def->type);
3869 }
3870#endif
3871 return UNALIGNED_MEMBER_PTR(me->def, body.cfunc);
3872}
3873
3874static VALUE
3875vm_call_cfunc_with_frame_(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
3876 int argc, VALUE *argv, VALUE *stack_bottom)
3877{
3878 RB_DEBUG_COUNTER_INC(ccf_cfunc_with_frame);
3879 const struct rb_callinfo *ci = calling->cd->ci;
3880 const struct rb_callcache *cc = calling->cc;
3881 VALUE val;
3882 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3883 const rb_method_cfunc_t *cfunc = vm_method_cfunc_entry(me);
3884
3885 VALUE recv = calling->recv;
3886 VALUE block_handler = calling->block_handler;
3887 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3888
3889 if (UNLIKELY(calling->kw_splat)) {
3890 frame_type |= VM_FRAME_FLAG_CFRAME_KW;
3891 }
3892
3893 VM_ASSERT(reg_cfp == ec->cfp);
3894
3895 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, me->owner, me->def->original_id);
3896 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, recv, me->def->original_id, vm_ci_mid(ci), me->owner, Qundef);
3897
3898 vm_push_frame(ec, NULL, frame_type, recv,
3899 block_handler, (VALUE)me,
3900 0, ec->cfp->sp, 0, 0);
3901
3902 int len = cfunc->argc;
3903 if (len >= 0) rb_check_arity(argc, len, len);
3904
3905 reg_cfp->sp = stack_bottom;
3906 val = (*cfunc->invoker)(recv, argc, argv, cfunc->func);
3907
3908 CHECK_CFP_CONSISTENCY("vm_call_cfunc");
3909
3910 rb_vm_pop_frame(ec);
3911
3912 VM_ASSERT(ec->cfp->sp == stack_bottom);
3913
3914 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, recv, me->def->original_id, vm_ci_mid(ci), me->owner, val);
3915 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
3916
3917 return val;
3918}
3919
3920// Push a C method frame for a given cme. This is called when JIT code skipped
3921// pushing a frame but the C method reached a point where a frame is needed.
3922void
3923rb_vm_push_cfunc_frame(const rb_callable_method_entry_t *cme, int recv_idx)
3924{
3925 VM_ASSERT(cme->def->type == VM_METHOD_TYPE_CFUNC);
3926 rb_execution_context_t *ec = GET_EC();
3927 VALUE *sp = ec->cfp->sp;
3928 VALUE recv = *(sp - recv_idx - 1);
3929 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3930 VALUE block_handler = VM_BLOCK_HANDLER_NONE;
3931#if VM_CHECK_MODE > 0
3932 // Clean up the stack canary since we're about to satisfy the "leaf or lazy push" assumption
3933 *(GET_EC()->cfp->sp) = Qfalse;
3934#endif
3935 vm_push_frame(ec, NULL, frame_type, recv, block_handler, (VALUE)cme, 0, ec->cfp->sp, 0, 0);
3936}
3937
3938// If true, cc->call needs to include `CALLER_SETUP_ARG` (i.e. can't be skipped in fastpath)
3939bool
3940rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci)
3941{
3942 return IS_ARGS_SPLAT(ci) || IS_ARGS_KW_OR_KW_SPLAT(ci);
3943}
3944
3945static VALUE
3946vm_call_cfunc_with_frame(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3947{
3948 int argc = calling->argc;
3949 VALUE *stack_bottom = reg_cfp->sp - argc - 1;
3950 VALUE *argv = &stack_bottom[1];
3951
3952 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
3953}
3954
3955static VALUE
3956vm_call_cfunc_other(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3957{
3958 const struct rb_callinfo *ci = calling->cd->ci;
3959 RB_DEBUG_COUNTER_INC(ccf_cfunc_other);
3960
3961 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
3962 VALUE argv_ary;
3963 if (UNLIKELY(argv_ary = calling->heap_argv)) {
3964 VM_ASSERT(!IS_ARGS_KEYWORD(ci));
3965 int argc = RARRAY_LENINT(argv_ary);
3966 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
3967 VALUE *stack_bottom = reg_cfp->sp - 2;
3968
3969 VM_ASSERT(calling->argc == 1);
3970 VM_ASSERT(RB_TYPE_P(argv_ary, T_ARRAY));
3971 VM_ASSERT(RBASIC_CLASS(argv_ary) == 0); // hidden ary
3972
3973 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
3974 }
3975 else {
3976 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_with_frame, !rb_splat_or_kwargs_p(ci) && !calling->kw_splat && !(vm_ci_flag(ci) & VM_CALL_FORWARDING));
3977
3978 return vm_call_cfunc_with_frame(ec, reg_cfp, calling);
3979 }
3980}
3981
3982static inline VALUE
3983vm_call_cfunc_array_argv(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int stack_offset, int argc_offset)
3984{
3985 VALUE argv_ary = reg_cfp->sp[-1 - stack_offset];
3986 int argc = RARRAY_LENINT(argv_ary) - argc_offset;
3987
3988 if (UNLIKELY(argc > VM_ARGC_STACK_MAX)) {
3989 return vm_call_cfunc_other(ec, reg_cfp, calling);
3990 }
3991
3992 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
3993 calling->kw_splat = 0;
3994 int i;
3995 VALUE *stack_bottom = reg_cfp->sp - 2 - stack_offset;
3996 VALUE *sp = stack_bottom;
3997 CHECK_VM_STACK_OVERFLOW(reg_cfp, argc);
3998 for(i = 0; i < argc; i++) {
3999 *++sp = argv[i];
4000 }
4001 reg_cfp->sp = sp+1;
4002
4003 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, stack_bottom+1, stack_bottom);
4004}
4005
4006static inline VALUE
4007vm_call_cfunc_only_splat(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4008{
4009 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat);
4010 VALUE argv_ary = reg_cfp->sp[-1];
4011 int argc = RARRAY_LENINT(argv_ary);
4012 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
4013 VALUE last_hash;
4014 int argc_offset = 0;
4015
4016 if (UNLIKELY(argc > 0 &&
4017 RB_TYPE_P((last_hash = argv[argc-1]), T_HASH) &&
4018 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS))) {
4019 if (!RHASH_EMPTY_P(last_hash)) {
4020 return vm_call_cfunc_other(ec, reg_cfp, calling);
4021 }
4022 argc_offset++;
4023 }
4024 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 0, argc_offset);
4025}
4026
4027static inline VALUE
4028vm_call_cfunc_only_splat_kw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4029{
4030 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat_kw);
4031 VALUE keyword_hash = reg_cfp->sp[-1];
4032
4033 if (keyword_hash == Qnil || (RB_TYPE_P(keyword_hash, T_HASH) && RHASH_EMPTY_P(keyword_hash))) {
4034 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 1, 0);
4035 }
4036
4037 return vm_call_cfunc_other(ec, reg_cfp, calling);
4038}
4039
4040static VALUE
4041vm_call_cfunc(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4042{
4043 const struct rb_callinfo *ci = calling->cd->ci;
4044 RB_DEBUG_COUNTER_INC(ccf_cfunc);
4045
4046 if (IS_ARGS_SPLAT(ci) && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
4047 if (!IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 1) {
4048 // f(*a)
4049 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat, TRUE);
4050 return vm_call_cfunc_only_splat(ec, reg_cfp, calling);
4051 }
4052 if (IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 2) {
4053 // f(*a, **kw)
4054 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat_kw, TRUE);
4055 return vm_call_cfunc_only_splat_kw(ec, reg_cfp, calling);
4056 }
4057 }
4058
4059 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_other, TRUE);
4060 return vm_call_cfunc_other(ec, reg_cfp, calling);
4061}
4062
4063static VALUE
4064vm_call_ivar(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4065{
4066 const struct rb_callcache *cc = calling->cc;
4067 RB_DEBUG_COUNTER_INC(ccf_ivar);
4068 cfp->sp -= 1;
4069 VALUE ivar = vm_getivar(calling->recv, vm_cc_cme(cc)->def->body.attr.id, NULL, NULL, cc, TRUE, Qnil);
4070 return ivar;
4071}
4072
4073static VALUE
4074vm_call_attrset_direct(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_callcache *cc, VALUE obj)
4075{
4076 RB_DEBUG_COUNTER_INC(ccf_attrset);
4077 VALUE val = *(cfp->sp - 1);
4078 cfp->sp -= 2;
4079 rb_setivar_cache cache = rb_setivar_cache_unpack(vm_cc_atomic_cache_read(cc));
4080 ID id = vm_cc_cme(cc)->def->body.attr.id;
4081 rb_check_frozen(obj);
4082 VALUE res = vm_setivar(obj, val, cache);
4083 if (UNDEF_P(res)) {
4084 switch (BUILTIN_TYPE(obj)) {
4085 case T_OBJECT:
4086 break;
4087 case T_CLASS:
4088 case T_MODULE:
4089 {
4090 res = vm_setivar_class(obj, val, cache);
4091 if (!UNDEF_P(res)) {
4092 return res;
4093 }
4094 }
4095 break;
4096 default:
4097 {
4098 res = vm_setivar_default(obj, id, val, cache);
4099 if (!UNDEF_P(res)) {
4100 return res;
4101 }
4102 }
4103 }
4104 res = vm_setivar_slowpath_attr(obj, id, val, cc);
4105 }
4106 return res;
4107}
4108
4109static VALUE
4110vm_call_attrset(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4111{
4112 return vm_call_attrset_direct(ec, cfp, calling->cc, calling->recv);
4113}
4114
4115static inline VALUE
4116vm_call_bmethod_body(rb_execution_context_t *ec, struct rb_calling_info *calling, const VALUE *argv)
4117{
4118 rb_proc_t *proc;
4119 VALUE val;
4120 const struct rb_callcache *cc = calling->cc;
4121 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4122 VALUE procv = cme->def->body.bmethod.proc;
4123
4124 if (!RB_OBJ_SHAREABLE_P(procv) &&
4125 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4126 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4127 }
4128
4129 /* control block frame */
4130 GetProcPtr(procv, proc);
4131 val = vm_invoke_bmethod(ec, proc, calling->recv, CALLING_ARGC(calling), argv, calling->kw_splat, calling->block_handler, vm_cc_cme(cc));
4132
4133 return val;
4134}
4135
4136static int vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type);
4137
4138static VALUE
4139vm_call_iseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4140{
4141 RB_DEBUG_COUNTER_INC(ccf_iseq_bmethod);
4142
4143 const struct rb_callcache *cc = calling->cc;
4144 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4145 VALUE procv = cme->def->body.bmethod.proc;
4146
4147 if (!RB_OBJ_SHAREABLE_P(procv) &&
4148 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4149 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4150 }
4151
4152 rb_proc_t *proc;
4153 GetProcPtr(procv, proc);
4154 const struct rb_block *block = &proc->block;
4155
4156 while (vm_block_type(block) == block_type_proc) {
4157 block = vm_proc_block(block->as.proc);
4158 }
4159 VM_ASSERT(vm_block_type(block) == block_type_iseq);
4160
4161 const struct rb_captured_block *captured = &block->as.captured;
4162 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
4163 VALUE * const argv = cfp->sp - calling->argc;
4164 const int arg_size = ISEQ_BODY(iseq)->param.size;
4165
4166 int opt_pc;
4167 if (vm_ci_flag(calling->cd->ci) & VM_CALL_ARGS_SIMPLE) {
4168 opt_pc = vm_callee_setup_block_arg(ec, calling, calling->cd->ci, iseq, argv, arg_setup_method);
4169 }
4170 else {
4171 opt_pc = setup_parameters_complex(ec, iseq, calling, calling->cd->ci, argv, arg_setup_method);
4172 }
4173
4174 cfp->sp = argv - 1; // -1 for the receiver
4175
4176 vm_push_frame(ec, iseq,
4177 VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_BMETHOD | VM_FRAME_FLAG_LAMBDA,
4178 calling->recv,
4179 VM_GUARDED_PREV_EP(captured->ep),
4180 (VALUE)cme,
4181 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
4182 argv + arg_size,
4183 ISEQ_BODY(iseq)->local_table_size - arg_size,
4184 ISEQ_BODY(iseq)->stack_max);
4185
4186 return Qundef;
4187}
4188
4189static VALUE
4190vm_call_noniseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4191{
4192 RB_DEBUG_COUNTER_INC(ccf_noniseq_bmethod);
4193
4194 VALUE *argv;
4195 int argc;
4196 CALLER_SETUP_ARG(cfp, calling, calling->cd->ci, ALLOW_HEAP_ARGV);
4197 if (UNLIKELY(calling->heap_argv)) {
4198 argv = RARRAY_PTR(calling->heap_argv);
4199 cfp->sp -= 2;
4200 }
4201 else {
4202 argc = calling->argc;
4203 argv = ALLOCA_N(VALUE, argc);
4204 MEMCPY(argv, cfp->sp - argc, VALUE, argc);
4205 cfp->sp += - argc - 1;
4206 }
4207
4208 return vm_call_bmethod_body(ec, calling, argv);
4209}
4210
4211static VALUE
4212vm_call_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4213{
4214 RB_DEBUG_COUNTER_INC(ccf_bmethod);
4215
4216 const struct rb_callcache *cc = calling->cc;
4217 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4218 VALUE procv = cme->def->body.bmethod.proc;
4219 rb_proc_t *proc;
4220 GetProcPtr(procv, proc);
4221 const struct rb_block *block = &proc->block;
4222
4223 while (vm_block_type(block) == block_type_proc) {
4224 block = vm_proc_block(block->as.proc);
4225 }
4226 if (vm_block_type(block) == block_type_iseq) {
4227 CC_SET_FASTPATH(cc, vm_call_iseq_bmethod, TRUE);
4228 return vm_call_iseq_bmethod(ec, cfp, calling);
4229 }
4230
4231 CC_SET_FASTPATH(cc, vm_call_noniseq_bmethod, TRUE);
4232 return vm_call_noniseq_bmethod(ec, cfp, calling);
4233}
4234
4235VALUE
4236rb_find_defined_class_by_owner(VALUE current_class, VALUE target_owner)
4237{
4238 VALUE klass = current_class;
4239
4240 /* for prepended Module, then start from cover class */
4241 if (RB_TYPE_P(klass, T_ICLASS) && RICLASS_IS_ORIGIN_P(klass) &&
4242 RB_TYPE_P(RBASIC_CLASS(klass), T_CLASS)) {
4243 klass = RBASIC_CLASS(klass);
4244 }
4245
4246 while (RTEST(klass)) {
4247 VALUE owner = RB_TYPE_P(klass, T_ICLASS) ? RBASIC_CLASS(klass) : klass;
4248 if (owner == target_owner) {
4249 return klass;
4250 }
4251 klass = RCLASS_SUPER(klass);
4252 }
4253
4254 return current_class; /* maybe module function */
4255}
4256
4257static const rb_callable_method_entry_t *
4258aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4259{
4260 const rb_method_entry_t *orig_me = me->def->body.alias.original_me;
4261 const rb_callable_method_entry_t *cme;
4262
4263 if (orig_me->defined_class == 0) {
4264 VALUE defined_class = rb_find_defined_class_by_owner(me->defined_class, orig_me->owner);
4265 VM_ASSERT_TYPE(orig_me->owner, T_MODULE);
4266 cme = rb_method_entry_complement_defined_class(orig_me, me->called_id, defined_class);
4267
4268 if (me->def->reference_count == 1) {
4269 RB_OBJ_WRITE(me, &me->def->body.alias.original_me, cme);
4270 }
4271 else {
4272 rb_method_definition_t *def =
4273 rb_method_definition_create(VM_METHOD_TYPE_ALIAS, me->def->original_id);
4274 rb_method_definition_set((rb_method_entry_t *)me, def, (void *)cme);
4275 }
4276 }
4277 else {
4278 cme = (const rb_callable_method_entry_t *)orig_me;
4279 }
4280
4281 VM_ASSERT(callable_method_entry_p(cme));
4282 return cme;
4283}
4284
4285const rb_callable_method_entry_t *
4286rb_aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4287{
4288 return aliased_callable_method_entry(me);
4289}
4290
4291static VALUE
4292vm_call_alias(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4293{
4294 calling->cc = &VM_CC_ON_STACK(Qundef,
4295 vm_call_general,
4296 {{0}},
4297 aliased_callable_method_entry(vm_cc_cme(calling->cc)));
4298
4299 return vm_call_method_each_type(ec, cfp, calling);
4300}
4301
4302static enum method_missing_reason
4303ci_missing_reason(const struct rb_callinfo *ci)
4304{
4305 enum method_missing_reason stat = MISSING_NOENTRY;
4306 if (vm_ci_flag(ci) & VM_CALL_VCALL && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) stat |= MISSING_VCALL;
4307 if (vm_ci_flag(ci) & VM_CALL_FCALL) stat |= MISSING_FCALL;
4308 if (vm_ci_flag(ci) & VM_CALL_SUPER) stat |= MISSING_SUPER;
4309 return stat;
4310}
4311
4312static VALUE vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
4313
4314static VALUE
4315vm_call_symbol(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4316 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE symbol, int flags)
4317{
4318 ASSUME(calling->argc >= 0);
4319
4320 enum method_missing_reason missing_reason = MISSING_NOENTRY;
4321 int argc = calling->argc;
4322 VALUE recv = calling->recv;
4323 VALUE klass = CLASS_OF(recv);
4324 ID mid = rb_check_id(&symbol);
4325 flags |= VM_CALL_OPT_SEND;
4326
4327 if (UNLIKELY(! mid)) {
4328 mid = idMethodMissing;
4329 missing_reason = ci_missing_reason(ci);
4330 ec->method_missing_reason = missing_reason;
4331
4332 VALUE argv_ary;
4333 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4334 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4335 rb_ary_unshift(argv_ary, symbol);
4336
4337 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4338 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4339 VALUE exc = rb_make_no_method_exception(
4340 rb_eNoMethodError, 0, recv, RARRAY_LENINT(argv_ary), RARRAY_CONST_PTR(argv_ary), priv);
4341
4342 rb_exc_raise(exc);
4343 }
4344 rb_ary_unshift(argv_ary, rb_str_intern(symbol));
4345 }
4346 else {
4347 /* E.g. when argc == 2
4348 *
4349 * | | | | TOPN
4350 * | | +------+
4351 * | | +---> | arg1 | 0
4352 * +------+ | +------+
4353 * | arg1 | -+ +-> | arg0 | 1
4354 * +------+ | +------+
4355 * | arg0 | ---+ | sym | 2
4356 * +------+ +------+
4357 * | recv | | recv | 3
4358 * --+------+--------+------+------
4359 */
4360 int i = argc;
4361 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4362 INC_SP(1);
4363 MEMMOVE(&TOPN(i - 1), &TOPN(i), VALUE, i);
4364 argc = ++calling->argc;
4365
4366 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4367 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4368 TOPN(i) = symbol;
4369 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4370 const VALUE *argv = STACK_ADDR_FROM_TOP(argc);
4371 VALUE exc = rb_make_no_method_exception(
4372 rb_eNoMethodError, 0, recv, argc, argv, priv);
4373
4374 rb_exc_raise(exc);
4375 }
4376 else {
4377 TOPN(i) = rb_str_intern(symbol);
4378 }
4379 }
4380 }
4381
4382 struct rb_forwarding_call_data new_fcd = {
4383 .cd = {
4384 .ci = &VM_CI_ON_STACK(mid, flags, argc, vm_ci_kwarg(ci)),
4385 .cc = NULL,
4386 },
4387 .caller_ci = NULL,
4388 };
4389
4390 if (!(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
4391 calling->cd = &new_fcd.cd;
4392 }
4393 else {
4394 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4395 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4396 new_fcd.caller_ci = caller_ci;
4397 calling->cd = (struct rb_call_data *)&new_fcd;
4398 }
4399 calling->cc = &VM_CC_ON_STACK(klass,
4400 vm_call_general,
4401 { .method_missing_reason = missing_reason },
4402 rb_callable_method_entry_with_refinements(klass, mid, NULL));
4403
4404 if (flags & VM_CALL_FCALL) {
4405 return vm_call_method(ec, reg_cfp, calling);
4406 }
4407
4408 const struct rb_callcache *cc = calling->cc;
4409 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
4410
4411 if (vm_cc_cme(cc) != NULL) {
4412 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
4413 case METHOD_VISI_PUBLIC: /* likely */
4414 return vm_call_method_each_type(ec, reg_cfp, calling);
4415 case METHOD_VISI_PRIVATE:
4416 vm_cc_method_missing_reason_set(cc, MISSING_PRIVATE);
4417 break;
4418 case METHOD_VISI_PROTECTED:
4419 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
4420 break;
4421 default:
4422 VM_UNREACHABLE(vm_call_method);
4423 }
4424 return vm_call_method_missing(ec, reg_cfp, calling);
4425 }
4426
4427 return vm_call_method_nome(ec, reg_cfp, calling);
4428}
4429
4430static VALUE
4431vm_call_opt_send0(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int flags)
4432{
4433 const struct rb_callinfo *ci = calling->cd->ci;
4434 int i;
4435 VALUE sym;
4436
4437 i = calling->argc - 1;
4438
4439 if (calling->argc == 0) {
4440 rb_raise(rb_eArgError, "no method name given");
4441 }
4442
4443 sym = TOPN(i);
4444 /* E.g. when i == 2
4445 *
4446 * | | | | TOPN
4447 * +------+ | |
4448 * | arg1 | ---+ | | 0
4449 * +------+ | +------+
4450 * | arg0 | -+ +-> | arg1 | 1
4451 * +------+ | +------+
4452 * | sym | +---> | arg0 | 2
4453 * +------+ +------+
4454 * | recv | | recv | 3
4455 * --+------+--------+------+------
4456 */
4457 /* shift arguments */
4458 if (i > 0) {
4459 MEMMOVE(&TOPN(i), &TOPN(i-1), VALUE, i);
4460 }
4461 calling->argc -= 1;
4462 DEC_SP(1);
4463
4464 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4465}
4466
4467static VALUE
4468vm_call_opt_send_complex(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4469{
4470 RB_DEBUG_COUNTER_INC(ccf_opt_send_complex);
4471 const struct rb_callinfo *ci = calling->cd->ci;
4472 int flags = VM_CALL_FCALL;
4473 VALUE sym;
4474
4475 VALUE argv_ary;
4476 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
4477 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4478 sym = rb_ary_shift(argv_ary);
4479 flags |= VM_CALL_ARGS_SPLAT;
4480 if (calling->kw_splat) {
4481 VALUE last_hash = rb_ary_last(0, NULL, argv_ary);
4482 ((struct RHash *)last_hash)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
4483 calling->kw_splat = 0;
4484 }
4485 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4486 }
4487
4488 if (calling->kw_splat) flags |= VM_CALL_KW_SPLAT;
4489 return vm_call_opt_send0(ec, reg_cfp, calling, flags);
4490}
4491
4492static VALUE
4493vm_call_opt_send_simple(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4494{
4495 RB_DEBUG_COUNTER_INC(ccf_opt_send_simple);
4496 return vm_call_opt_send0(ec, reg_cfp, calling, vm_ci_flag(calling->cd->ci) | VM_CALL_FCALL);
4497}
4498
4499static VALUE
4500vm_call_opt_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4501{
4502 RB_DEBUG_COUNTER_INC(ccf_opt_send);
4503
4504 const struct rb_callinfo *ci = calling->cd->ci;
4505 int flags = vm_ci_flag(ci);
4506
4507 if (UNLIKELY((flags & VM_CALL_FORWARDING) || (!(flags & VM_CALL_ARGS_SIMPLE) &&
4508 ((calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
4509 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
4510 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc)))))) {
4511 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_complex, TRUE);
4512 return vm_call_opt_send_complex(ec, reg_cfp, calling);
4513 }
4514
4515 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_simple, TRUE);
4516 return vm_call_opt_send_simple(ec, reg_cfp, calling);
4517}
4518
4519static VALUE
4520vm_call_method_missing_body(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
4521 const struct rb_callinfo *orig_ci, enum method_missing_reason reason)
4522{
4523 RB_DEBUG_COUNTER_INC(ccf_method_missing);
4524
4525 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
4526 unsigned int argc, flag;
4527
4528 flag = VM_CALL_FCALL | VM_CALL_OPT_SEND | vm_ci_flag(orig_ci);
4529 argc = ++calling->argc;
4530
4531 /* shift arguments: m(a, b, c) #=> method_missing(:m, a, b, c) */
4532 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4533 vm_check_canary(ec, reg_cfp->sp);
4534 if (argc > 1) {
4535 MEMMOVE(argv+1, argv, VALUE, argc-1);
4536 }
4537 argv[0] = ID2SYM(vm_ci_mid(orig_ci));
4538 INC_SP(1);
4539
4540 ec->method_missing_reason = reason;
4541
4542 struct rb_forwarding_call_data new_fcd = {
4543 .cd = {
4544 .ci = &VM_CI_ON_STACK(idMethodMissing, flag, argc, vm_ci_kwarg(orig_ci)),
4545 .cc = NULL,
4546 },
4547 .caller_ci = NULL,
4548 };
4549
4550 if (!(flag & VM_CALL_FORWARDING)) {
4551 calling->cd = &new_fcd.cd;
4552 }
4553 else {
4554 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4555 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4556 new_fcd.caller_ci = caller_ci;
4557 calling->cd = (struct rb_call_data *)&new_fcd;
4558 }
4559
4560 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }},
4561 rb_callable_method_entry_without_refinements(CLASS_OF(calling->recv), idMethodMissing, NULL));
4562 return vm_call_method(ec, reg_cfp, calling);
4563}
4564
4565static VALUE
4566vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4567{
4568 return vm_call_method_missing_body(ec, reg_cfp, calling, calling->cd->ci, vm_cc_cmethod_missing_reason(calling->cc));
4569}
4570
4571static const rb_callable_method_entry_t *refined_method_callable_without_refinement(const rb_callable_method_entry_t *me);
4572static VALUE
4573vm_call_zsuper(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE klass)
4574{
4575 klass = RCLASS_SUPER(klass);
4576
4577 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, vm_ci_mid(calling->cd->ci)) : NULL;
4578 if (cme == NULL) {
4579 return vm_call_method_nome(ec, cfp, calling);
4580 }
4581 if (cme->def->type == VM_METHOD_TYPE_REFINED &&
4582 cme->def->body.refined.orig_me) {
4583 cme = refined_method_callable_without_refinement(cme);
4584 }
4585
4586 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, cme);
4587
4588 return vm_call_method_each_type(ec, cfp, calling);
4589}
4590
4591static inline VALUE
4592find_refinement(VALUE refinements, VALUE klass)
4593{
4594 if (NIL_P(refinements)) {
4595 return Qnil;
4596 }
4597 return rb_hash_lookup(refinements, klass);
4598}
4599
4600PUREFUNC(static rb_control_frame_t * current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp));
4601static rb_control_frame_t *
4602current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp)
4603{
4604 rb_control_frame_t *top_cfp = cfp;
4605
4606 if (CFP_ISEQ(cfp) && ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_BLOCK) {
4607 const rb_iseq_t *local_iseq = ISEQ_BODY(CFP_ISEQ(cfp))->local_iseq;
4608
4609 do {
4610 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
4611 if (RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
4612 /* TODO: orphan block */
4613 return top_cfp;
4614 }
4615 } while (CFP_ISEQ(cfp) != local_iseq);
4616 }
4617 return cfp;
4618}
4619
4620static const rb_callable_method_entry_t *
4621refined_method_callable_without_refinement(const rb_callable_method_entry_t *me)
4622{
4623 const rb_method_entry_t *orig_me = me->def->body.refined.orig_me;
4624 const rb_callable_method_entry_t *cme;
4625
4626 if (orig_me->defined_class == 0) {
4627 cme = NULL;
4628 rb_notimplement();
4629 }
4630 else {
4631 cme = (const rb_callable_method_entry_t *)orig_me;
4632 }
4633
4634 VM_ASSERT(callable_method_entry_p(cme));
4635
4636 if (UNDEFINED_METHOD_ENTRY_P(cme)) {
4637 cme = NULL;
4638 }
4639
4640 return cme;
4641}
4642
4643static const rb_callable_method_entry_t *
4644search_refined_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4645{
4646 ID mid = vm_ci_mid(calling->cd->ci);
4647 const rb_cref_t *cref = vm_get_cref(cfp->ep);
4648 const struct rb_callcache * const cc = calling->cc;
4649 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4650
4651 for (; cref; cref = CREF_NEXT(cref)) {
4652 const VALUE refinement = find_refinement(CREF_REFINEMENTS(cref), vm_cc_cme(cc)->owner);
4653 if (NIL_P(refinement)) continue;
4654
4655 const rb_callable_method_entry_t *const ref_me =
4656 rb_callable_method_entry(refinement, mid);
4657
4658 if (ref_me) {
4659 if (vm_cc_call(cc) == vm_call_super_method) {
4660 const rb_control_frame_t *top_cfp = current_method_entry(ec, cfp);
4661 const rb_callable_method_entry_t *top_me = rb_vm_frame_method_entry(top_cfp);
4662 if (top_me && rb_method_definition_eq(ref_me->def, top_me->def)) {
4663 continue;
4664 }
4665 }
4666
4667 if (cme->def->type != VM_METHOD_TYPE_REFINED ||
4668 cme->def != ref_me->def) {
4669 cme = ref_me;
4670 }
4671 if (ref_me->def->type != VM_METHOD_TYPE_REFINED) {
4672 return cme;
4673 }
4674 }
4675 else {
4676 return NULL;
4677 }
4678 }
4679
4680 if (vm_cc_cme(cc)->def->body.refined.orig_me) {
4681 return refined_method_callable_without_refinement(vm_cc_cme(cc));
4682 }
4683 else {
4684 VALUE klass = RCLASS_SUPER(vm_cc_cme(cc)->defined_class);
4685 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, mid) : NULL;
4686 return cme;
4687 }
4688}
4689
4690static VALUE
4691vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4692{
4693 const rb_callable_method_entry_t *ref_cme = search_refined_method(ec, cfp, calling);
4694
4695 if (ref_cme) {
4696 if (calling->cd->cc) {
4697 const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement);
4698 RB_OBJ_WRITE(CFP_ISEQ(cfp), &calling->cd->cc, cc);
4699 return vm_call_method(ec, cfp, calling);
4700 }
4701 else {
4702 struct rb_callcache *ref_cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, ref_cme);
4703 calling->cc= ref_cc;
4704 return vm_call_method(ec, cfp, calling);
4705 }
4706 }
4707 else {
4708 return vm_call_method_nome(ec, cfp, calling);
4709 }
4710}
4711
4712static inline VALUE vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci, bool is_lambda, VALUE block_handler);
4713
4714NOINLINE(static VALUE
4715 vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4716 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler));
4717
4718static VALUE
4719vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4720 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler)
4721{
4722 int argc = calling->argc;
4723
4724 /* remove self */
4725 if (argc > 0) MEMMOVE(&TOPN(argc), &TOPN(argc-1), VALUE, argc);
4726 DEC_SP(1);
4727
4728 return vm_invoke_block(ec, reg_cfp, calling, ci, false, block_handler);
4729}
4730
4731static VALUE
4732vm_call_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4733{
4734 RB_DEBUG_COUNTER_INC(ccf_opt_call);
4735
4736 const struct rb_callinfo *ci = calling->cd->ci;
4737 VALUE procval = calling->recv;
4738 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, VM_BH_FROM_PROC(procval));
4739}
4740
4741static VALUE
4742vm_call_opt_block_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4743{
4744 RB_DEBUG_COUNTER_INC(ccf_opt_block_call);
4745
4746 VALUE block_handler = VM_ENV_BLOCK_HANDLER(VM_CF_LEP(reg_cfp));
4747 const struct rb_callinfo *ci = calling->cd->ci;
4748
4749 enum ruby_basic_operators bop;
4750 switch (vm_ci_mid(ci)) {
4751 case idAREF: bop = BOP_AREF; break;
4752 case idYield: bop = BOP_YIELD; break;
4753 case idEqq: bop = BOP_EQQ; break;
4754 default: bop = BOP_CALL; break;
4755 }
4756
4757 if (BASIC_OP_UNREDEFINED_P(bop, PROC_REDEFINED_OP_FLAG)) {
4758 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, block_handler);
4759 }
4760 else {
4761 calling->recv = rb_vm_bh_to_procval(ec, block_handler);
4762 calling->cc = rb_vm_search_method_slowpath(ci, CLASS_OF(calling->recv));
4763 return vm_call_general(ec, reg_cfp, calling);
4764 }
4765}
4766
4767static VALUE
4768vm_call_opt_struct_aref0(rb_execution_context_t *ec, struct rb_calling_info *calling)
4769{
4770 VALUE recv = calling->recv;
4771
4772 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4773 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4774 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_AREF);
4775
4776 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4777 return RSTRUCT_GET_RAW(recv, off);
4778}
4779
4780static VALUE
4781vm_call_opt_struct_aref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4782{
4783 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aref);
4784
4785 VALUE ret = vm_call_opt_struct_aref0(ec, calling);
4786 reg_cfp->sp -= 1;
4787 return ret;
4788}
4789
4790static VALUE
4791vm_call_opt_struct_aset0(rb_execution_context_t *ec, struct rb_calling_info *calling, VALUE val)
4792{
4793 VALUE recv = calling->recv;
4794
4795 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4796 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4797 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_ASET);
4798
4799 rb_check_frozen(recv);
4800
4801 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4802 RSTRUCT_SET_RAW(recv, off, val);
4803
4804 return val;
4805}
4806
4807static VALUE
4808vm_call_opt_struct_aset(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4809{
4810 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aset);
4811
4812 VALUE ret = vm_call_opt_struct_aset0(ec, calling, *(reg_cfp->sp - 1));
4813 reg_cfp->sp -= 2;
4814 return ret;
4815}
4816
4817NOINLINE(static VALUE vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4818 const struct rb_callinfo *ci, const struct rb_callcache *cc));
4819
4820#define VM_CALL_METHOD_ATTR(var, func, nohook) \
4821 if (UNLIKELY(ruby_vm_c_events_enabled > 0)) { \
4822 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, calling->recv, vm_cc_cme(cc)->def->original_id, \
4823 vm_ci_mid(ci), vm_cc_cme(cc)->owner, Qundef); \
4824 var = func; \
4825 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, calling->recv, vm_cc_cme(cc)->def->original_id, \
4826 vm_ci_mid(ci), vm_cc_cme(cc)->owner, (var)); \
4827 } \
4828 else { \
4829 nohook; \
4830 var = func; \
4831 }
4832
4833static VALUE
4834vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4835 const struct rb_callinfo *ci, const struct rb_callcache *cc)
4836{
4837 switch (vm_cc_cme(cc)->def->body.optimized.type) {
4838 case OPTIMIZED_METHOD_TYPE_SEND:
4839 CC_SET_FASTPATH(cc, vm_call_opt_send, TRUE);
4840 return vm_call_opt_send(ec, cfp, calling);
4841 case OPTIMIZED_METHOD_TYPE_CALL:
4842 CC_SET_FASTPATH(cc, vm_call_opt_call, TRUE);
4843 return vm_call_opt_call(ec, cfp, calling);
4844 case OPTIMIZED_METHOD_TYPE_BLOCK_CALL:
4845 CC_SET_FASTPATH(cc, vm_call_opt_block_call, TRUE);
4846 return vm_call_opt_block_call(ec, cfp, calling);
4847 case OPTIMIZED_METHOD_TYPE_STRUCT_AREF: {
4848 CALLER_SETUP_ARG(cfp, calling, ci, 0);
4849 rb_check_arity(calling->argc, 0, 0);
4850
4851 VALUE v;
4852 VM_CALL_METHOD_ATTR(v,
4853 vm_call_opt_struct_aref(ec, cfp, calling),
4854 set_vm_cc_ivar(cc); \
4855 CC_SET_FASTPATH(cc, vm_call_opt_struct_aref, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
4856 return v;
4857 }
4858 case OPTIMIZED_METHOD_TYPE_STRUCT_ASET: {
4859 CALLER_SETUP_ARG(cfp, calling, ci, 1);
4860 rb_check_arity(calling->argc, 1, 1);
4861
4862 VALUE v;
4863 VM_CALL_METHOD_ATTR(v,
4864 vm_call_opt_struct_aset(ec, cfp, calling),
4865 set_vm_cc_ivar(cc); \
4866 CC_SET_FASTPATH(cc, vm_call_opt_struct_aset, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
4867 return v;
4868 }
4869 default:
4870 rb_bug("vm_call_method: unsupported optimized method type (%d)", vm_cc_cme(cc)->def->body.optimized.type);
4871 }
4872}
4873
4874static VALUE
4875vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4876{
4877 const struct rb_callinfo *ci = calling->cd->ci;
4878 const struct rb_callcache *cc = calling->cc;
4879 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4880 VALUE v;
4881
4882 VM_ASSERT(! METHOD_ENTRY_INVALIDATED(cme));
4883
4884 switch (cme->def->type) {
4885 case VM_METHOD_TYPE_ISEQ:
4886 if (ISEQ_BODY(def_iseq_ptr(cme->def))->param.flags.forwardable) {
4887 CC_SET_FASTPATH(cc, vm_call_iseq_fwd_setup, TRUE);
4888 return vm_call_iseq_fwd_setup(ec, cfp, calling);
4889 }
4890 else {
4891 CC_SET_FASTPATH(cc, vm_call_iseq_setup, TRUE);
4892 return vm_call_iseq_setup(ec, cfp, calling);
4893 }
4894
4895 case VM_METHOD_TYPE_NOTIMPLEMENTED:
4896 case VM_METHOD_TYPE_CFUNC:
4897 CC_SET_FASTPATH(cc, vm_call_cfunc, TRUE);
4898 return vm_call_cfunc(ec, cfp, calling);
4899
4900 case VM_METHOD_TYPE_ATTRSET:
4901 CALLER_SETUP_ARG(cfp, calling, ci, 1);
4902
4903 rb_check_arity(calling->argc, 1, 1);
4904
4905 const unsigned int aset_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_KWARG | VM_CALL_FORWARDING);
4906
4907 if (vm_cc_markable(cc)) {
4908 vm_cc_attr_index_set(cc, IVAR_CACHE_INIT);
4909 VM_CALL_METHOD_ATTR(v,
4910 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
4911 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
4912 }
4913 else {
4914 cc = &((struct rb_callcache) {
4915 .flags = T_IMEMO |
4916 (imemo_callcache << FL_USHIFT) |
4917 VM_CALLCACHE_UNMARKABLE |
4918 VM_CALLCACHE_ON_STACK,
4919 .klass = cc->klass,
4920 .cme_ = cc->cme_,
4921 .call_ = cc->call_,
4922 .aux_ = {
4923 .attr = {
4924 .value = IVAR_CACHE_INIT,
4925 }
4926 },
4927 });
4928
4929 VM_CALL_METHOD_ATTR(v,
4930 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
4931 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
4932 }
4933 return v;
4934
4935 case VM_METHOD_TYPE_IVAR:
4936 CALLER_SETUP_ARG(cfp, calling, ci, 0);
4937 rb_check_arity(calling->argc, 0, 0);
4938 vm_cc_attr_index_set(cc, rb_getivar_cache_pack(ROOT_SHAPE_ID, ATTR_INDEX_NOT_SET));
4939 const unsigned int ivar_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_FORWARDING);
4940 VM_CALL_METHOD_ATTR(v,
4941 vm_call_ivar(ec, cfp, calling),
4942 CC_SET_FASTPATH(cc, vm_call_ivar, !(vm_ci_flag(ci) & ivar_mask)));
4943 return v;
4944
4945 case VM_METHOD_TYPE_MISSING:
4946 vm_cc_method_missing_reason_set(cc, 0);
4947 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
4948 return vm_call_method_missing(ec, cfp, calling);
4949
4950 case VM_METHOD_TYPE_BMETHOD:
4951 CC_SET_FASTPATH(cc, vm_call_bmethod, TRUE);
4952 return vm_call_bmethod(ec, cfp, calling);
4953
4954 case VM_METHOD_TYPE_ALIAS:
4955 CC_SET_FASTPATH(cc, vm_call_alias, TRUE);
4956 return vm_call_alias(ec, cfp, calling);
4957
4958 case VM_METHOD_TYPE_OPTIMIZED:
4959 return vm_call_optimized(ec, cfp, calling, ci, cc);
4960
4961 case VM_METHOD_TYPE_UNDEF:
4962 break;
4963
4964 case VM_METHOD_TYPE_ZSUPER:
4965 return vm_call_zsuper(ec, cfp, calling, RCLASS_ORIGIN(vm_cc_cme(cc)->defined_class));
4966
4967 case VM_METHOD_TYPE_REFINED:
4968 // CC_SET_FASTPATH(cc, vm_call_refined, TRUE);
4969 // should not set FASTPATH since vm_call_refined assumes cc->call is vm_call_super_method on invokesuper.
4970 return vm_call_refined(ec, cfp, calling);
4971 }
4972
4973 rb_bug("vm_call_method: unsupported method type (%d)", vm_cc_cme(cc)->def->type);
4974}
4975
4976NORETURN(static void vm_raise_method_missing(rb_execution_context_t *ec, int argc, const VALUE *argv, VALUE obj, int call_status));
4977
4978static VALUE
4979vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4980{
4981 /* method missing */
4982 const struct rb_callinfo *ci = calling->cd->ci;
4983 const int stat = ci_missing_reason(ci);
4984
4985 if (vm_ci_mid(ci) == idMethodMissing) {
4986 if (UNLIKELY(calling->heap_argv)) {
4987 vm_raise_method_missing(ec, RARRAY_LENINT(calling->heap_argv), RARRAY_CONST_PTR(calling->heap_argv), calling->recv, stat);
4988 }
4989 else {
4990 rb_control_frame_t *reg_cfp = cfp;
4991 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
4992 vm_raise_method_missing(ec, calling->argc, argv, calling->recv, stat);
4993 }
4994 }
4995 else {
4996 return vm_call_method_missing_body(ec, cfp, calling, ci, stat);
4997 }
4998}
4999
5000/* Protected method calls and super invocations need to check that the receiver
5001 * (self for super) inherits the module on which the method is defined.
5002 * In the case of refinements, it should consider the original class not the
5003 * refinement.
5004 */
5005static VALUE
5006vm_defined_class_for_protected_call(const rb_callable_method_entry_t *me)
5007{
5008 VALUE defined_class = me->defined_class;
5009 VALUE refined_class = RCLASS_REFINED_CLASS(defined_class);
5010 return NIL_P(refined_class) ? defined_class : refined_class;
5011}
5012
5013static inline VALUE
5014vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
5015{
5016 const struct rb_callinfo *ci = calling->cd->ci;
5017 const struct rb_callcache *cc = calling->cc;
5018
5019 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
5020
5021 if (vm_cc_cme(cc) != NULL) {
5022 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
5023 case METHOD_VISI_PUBLIC: /* likely */
5024 return vm_call_method_each_type(ec, cfp, calling);
5025
5026 case METHOD_VISI_PRIVATE:
5027 if (!(vm_ci_flag(ci) & VM_CALL_FCALL)) {
5028 enum method_missing_reason stat = MISSING_PRIVATE;
5029 if (vm_ci_flag(ci) & VM_CALL_VCALL) stat |= MISSING_VCALL;
5030
5031 vm_cc_method_missing_reason_set(cc, stat);
5032 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
5033 return vm_call_method_missing(ec, cfp, calling);
5034 }
5035 return vm_call_method_each_type(ec, cfp, calling);
5036
5037 case METHOD_VISI_PROTECTED:
5038 if (!(vm_ci_flag(ci) & (VM_CALL_OPT_SEND | VM_CALL_FCALL))) {
5039 VALUE defined_class = vm_defined_class_for_protected_call(vm_cc_cme(cc));
5040 if (!rb_obj_is_kind_of(cfp->self, defined_class)) {
5041 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
5042 return vm_call_method_missing(ec, cfp, calling);
5043 }
5044 else {
5045 /* caching method info to dummy cc */
5046 VM_ASSERT(vm_cc_cme(cc) != NULL);
5047 struct rb_callcache cc_on_stack = *cc;
5048 FL_SET_RAW((VALUE)&cc_on_stack, VM_CALLCACHE_UNMARKABLE);
5049 calling->cc = &cc_on_stack;
5050 return vm_call_method_each_type(ec, cfp, calling);
5051 }
5052 }
5053 return vm_call_method_each_type(ec, cfp, calling);
5054
5055 default:
5056 rb_bug("unreachable");
5057 }
5058 }
5059 else {
5060 return vm_call_method_nome(ec, cfp, calling);
5061 }
5062}
5063
5064static VALUE
5065vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
5066{
5067 RB_DEBUG_COUNTER_INC(ccf_general);
5068 return vm_call_method(ec, reg_cfp, calling);
5069}
5070
5071void
5072rb_vm_cc_general(const struct rb_callcache *cc)
5073{
5074 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
5075 VM_ASSERT(cc != vm_cc_empty());
5076
5077 *(vm_call_handler *)&cc->call_ = vm_call_general;
5078}
5079
5080static VALUE
5081vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
5082{
5083 RB_DEBUG_COUNTER_INC(ccf_super_method);
5084
5085 // This line is introduced to make different from `vm_call_general` because some compilers (VC we found)
5086 // can merge the function and the address of the function becomes same.
5087 // The address of `vm_call_super_method` is used in `search_refined_method`, so it should be different.
5088 if (ec == NULL) rb_bug("unreachable");
5089
5090 /* this check is required to distinguish with other functions. */
5091 VM_ASSERT(vm_cc_call(calling->cc) == vm_call_super_method);
5092 return vm_call_method(ec, reg_cfp, calling);
5093}
5094
5095/* super */
5096
5097static inline VALUE
5098vm_search_normal_superclass(VALUE klass)
5099{
5100 if (RICLASS_FOR_REFINEMENT_P(klass)) {
5101 klass = RBASIC(klass)->klass;
5102 }
5103 klass = RCLASS_ORIGIN(klass);
5104 return RCLASS_SUPER(klass);
5105}
5106
5107NORETURN(static void vm_super_outside(void));
5108
5109static void
5110vm_super_outside(void)
5111{
5112 rb_raise(rb_eNoMethodError, "super called outside of method");
5113}
5114
5115static const struct rb_callcache *
5116empty_cc_for_super(void)
5117{
5118 return &vm_empty_cc_for_super;
5119}
5120
5121static const struct rb_callcache *
5122vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *cd, VALUE recv)
5123{
5124 VALUE current_defined_class;
5125 const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp);
5126 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
5127
5128 if (!me) {
5129 vm_super_outside();
5130 }
5131
5132 current_defined_class = vm_defined_class_for_protected_call(me);
5133
5134 if (BUILTIN_TYPE(current_defined_class) != T_MODULE &&
5135 iseq != method_entry_iseqptr(me) &&
5136 !rb_obj_is_kind_of(recv, current_defined_class)) {
5137 VALUE m = RB_TYPE_P(current_defined_class, T_ICLASS) ?
5138 RCLASS_INCLUDER(current_defined_class) : current_defined_class;
5139
5140 if (m) { /* not bound UnboundMethod */
5141 rb_raise(rb_eTypeError,
5142 "self has wrong type to call super in this context: "
5143 "%"PRIsVALUE" (expected %"PRIsVALUE")",
5144 rb_obj_class(recv), m);
5145 }
5146 }
5147
5148 if (me->def->type == VM_METHOD_TYPE_BMETHOD && (vm_ci_flag(cd->ci) & VM_CALL_ZSUPER)) {
5149 rb_raise(rb_eRuntimeError,
5150 "implicit argument passing of super from method defined"
5151 " by define_method() is not supported."
5152 " Specify all arguments explicitly.");
5153 }
5154
5155 ID mid = me->def->original_id;
5156
5157 if (!vm_ci_markable(cd->ci)) {
5158 VM_FORCE_WRITE((const VALUE *)&cd->ci->mid, (VALUE)mid);
5159 }
5160 else {
5161 // update iseq. really? (TODO)
5162 cd->ci = vm_ci_new_runtime(mid,
5163 vm_ci_flag(cd->ci),
5164 vm_ci_argc(cd->ci),
5165 vm_ci_kwarg(cd->ci));
5166
5167 RB_OBJ_WRITTEN(iseq, Qundef, cd->ci);
5168 }
5169
5170 const struct rb_callcache *cc;
5171
5172 VALUE klass = vm_search_normal_superclass(me->defined_class);
5173
5174 if (!klass) {
5175 /* bound instance method of module */
5176 cc = vm_cc_new(Qundef, NULL, vm_call_method_missing, cc_type_super);
5177 RB_OBJ_WRITE(iseq, &cd->cc, cc);
5178 }
5179 else if (klass == rb_cBasicObject &&
5180 RB_TYPE_P(me->defined_class, T_ICLASS) &&
5181 RCLASS_INCLUDER(me->defined_class) == 0) {
5182 rb_raise(rb_eNoMethodError,
5183 "super in a method in a module that has been refined and that is called via super"
5184 " from a refinement method is not supported.");
5185 }
5186 else {
5187 cc = vm_search_method_fastpath(reg_cfp, cd, klass);
5188 const rb_callable_method_entry_t *cached_cme = vm_cc_cme(cc);
5189
5190 // define_method can cache for different method id
5191 if (cached_cme == NULL) {
5192 // empty_cc_for_super is not markable object
5193 cd->cc = empty_cc_for_super();
5194 }
5195 else if (cached_cme->called_id != mid) {
5196 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
5197 if (cme) {
5198 cc = vm_cc_new(klass, cme, vm_call_super_method, cc_type_super);
5199 RB_OBJ_WRITE(iseq, &cd->cc, cc);
5200 }
5201 else {
5202 cd->cc = cc = empty_cc_for_super();
5203 }
5204 }
5205 else {
5206 switch (cached_cme->def->type) {
5207 // vm_call_refined (search_refined_method) assumes cc->call is vm_call_super_method on invokesuper
5208 case VM_METHOD_TYPE_REFINED:
5209 // cc->klass is superclass of receiver class. Checking cc->klass is not enough to invalidate IVC for the receiver class.
5210 case VM_METHOD_TYPE_ATTRSET:
5211 case VM_METHOD_TYPE_IVAR:
5212 vm_cc_call_set(cc, vm_call_super_method); // invalidate fastpath
5213 break;
5214 default:
5215 break; // use fastpath
5216 }
5217 }
5218 }
5219
5220 VM_ASSERT((vm_cc_cme(cc), true));
5221
5222 return cc;
5223}
5224
5225/* yield */
5226
5227static inline int
5228block_proc_is_lambda(const VALUE procval)
5229{
5230 rb_proc_t *proc;
5231
5232 if (procval) {
5233 GetProcPtr(procval, proc);
5234 return proc->header.is_lambda;
5235 }
5236 else {
5237 return 0;
5238 }
5239}
5240
5241static VALUE
5242vm_yield_with_cfunc(rb_execution_context_t *ec,
5243 const struct rb_captured_block *captured,
5244 VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE block_handler,
5245 const rb_callable_method_entry_t *me)
5246{
5247 int is_lambda = FALSE; /* TODO */
5248 VALUE val, arg, blockarg;
5249 int frame_flag;
5250 const struct vm_ifunc *ifunc = captured->code.ifunc;
5251
5252 if (is_lambda) {
5253 arg = rb_ary_new4(argc, argv);
5254 }
5255 else if (argc == 0) {
5256 arg = Qnil;
5257 }
5258 else {
5259 arg = argv[0];
5260 }
5261
5262 blockarg = rb_vm_bh_to_procval(ec, block_handler);
5263
5264 frame_flag = VM_FRAME_MAGIC_IFUNC | VM_FRAME_FLAG_CFRAME | (me ? VM_FRAME_FLAG_BMETHOD : 0);
5265 if (kw_splat) {
5266 frame_flag |= VM_FRAME_FLAG_CFRAME_KW;
5267 }
5268
5269 vm_push_frame(ec, (const rb_iseq_t *)captured->code.ifunc,
5270 frame_flag,
5271 self,
5272 VM_GUARDED_PREV_EP(captured->ep),
5273 (VALUE)me,
5274 0, ec->cfp->sp, 0, 0);
5275 val = (*ifunc->func)(arg, (VALUE)ifunc->data, argc, argv, blockarg);
5276 rb_vm_pop_frame(ec);
5277
5278 return val;
5279}
5280
5281VALUE
5282rb_vm_yield_with_cfunc(rb_execution_context_t *ec, const struct rb_captured_block *captured, int argc, const VALUE *argv)
5283{
5284 return vm_yield_with_cfunc(ec, captured, captured->self, argc, argv, 0, VM_BLOCK_HANDLER_NONE, NULL);
5285}
5286
5287static VALUE
5288vm_yield_with_symbol(rb_execution_context_t *ec, VALUE symbol, int argc, const VALUE *argv, int kw_splat, VALUE block_handler)
5289{
5290 VALUE passed_proc = rb_vm_bh_to_procval(ec, block_handler);
5291
5292 if (!rb_box_available()) {
5293 return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc);
5294 }
5295
5296 const rb_control_frame_t *reg_cfp = ec->cfp;
5297 const rb_control_frame_t *ruby_cfp = rb_vm_get_ruby_level_next_cfp(ec, reg_cfp);
5298 const rb_box_t *box = NULL;
5299
5300 /*
5301 * Traverse the frames until a user box (Main or Optional Box) is found.
5302 * Frames for built-in methods like <internal:xxx> run in the Root or Master Box,
5303 */
5304 while (ruby_cfp) {
5305 box = current_box_on_cfp(ec, ruby_cfp);
5306 if (BOX_USER_P(box)) {
5307 break;
5308 }
5309 ruby_cfp = rb_vm_get_ruby_level_next_cfp(ec, RUBY_VM_PREVIOUS_CONTROL_FRAME(ruby_cfp));
5310 }
5311
5312 // Fallback to the normal call if no user box is found
5313 if (!ruby_cfp || !box) {
5314 return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc);
5315 }
5316
5317 /*
5318 * Push a dummy block frame whose outer env is `ruby_cfp` so that the
5319 * method resolves in the caller's box (via the ep chain) and the
5320 * backtrace reads like a usual block invocation at the caller's site.
5321 */
5322 const rb_iseq_t *caller_iseq = CFP_ISEQ(ruby_cfp);
5323 VALUE name = rb_sprintf("block in %"PRIsVALUE, rb_iseq_label(caller_iseq));
5324 const rb_iseq_t *iseq = rb_iseq_new_with_opt(Qnil, name,
5325 rb_iseq_path(caller_iseq), rb_iseq_realpath(caller_iseq),
5326 rb_vm_get_sourceline(ruby_cfp), caller_iseq, 0,
5327 ISEQ_TYPE_BLOCK, NULL, Qnil);
5328 volatile VALUE val = Qnil;
5329 enum ruby_tag_type state;
5330
5331 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_FINISH,
5332 ruby_cfp->self, VM_GUARDED_PREV_EP(ruby_cfp->ep),
5333 Qfalse, /* cref or me */
5334 ISEQ_BODY(iseq)->iseq_encoded, reg_cfp->sp,
5335 ISEQ_BODY(iseq)->local_table_size, ISEQ_BODY(iseq)->stack_max);
5336
5337 /*
5338 * The pushed frame is finished (owned by no vm_exec loop), so catch the
5339 * non-local exit here, pop the frame, and let the caller's handlers see
5340 * the exception.
5341 */
5342 EC_PUSH_TAG(ec);
5343 if ((state = EC_EXEC_TAG()) == TAG_NONE) {
5344 val = rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc);
5345 }
5346 EC_POP_TAG();
5347
5348 if (state != TAG_NONE) {
5349 rb_vm_rewind_cfp(ec, (rb_control_frame_t *)reg_cfp);
5350 EC_JUMP_TAG(ec, state);
5351 }
5352
5353 rb_vm_pop_frame(ec);
5354
5355 return val;
5356}
5357
5358static inline int
5359vm_callee_setup_block_arg_arg0_splat(rb_control_frame_t *cfp, const rb_iseq_t *iseq, VALUE *argv, VALUE ary)
5360{
5361 int i;
5362 long len = RARRAY_LEN(ary);
5363
5364 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5365
5366 for (i=0; i<len && i<ISEQ_BODY(iseq)->param.lead_num; i++) {
5367 argv[i] = RARRAY_AREF(ary, i);
5368 }
5369
5370 return i;
5371}
5372
5373static inline VALUE
5374vm_callee_setup_block_arg_arg0_check(VALUE *argv)
5375{
5376 VALUE ary, arg0 = argv[0];
5377 ary = rb_check_array_type(arg0);
5378#if 0
5379 argv[0] = arg0;
5380#else
5381 VM_ASSERT(argv[0] == arg0);
5382#endif
5383 return ary;
5384}
5385
5386static int
5387vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type)
5388{
5389 if (rb_simple_iseq_p(iseq)) {
5390 rb_control_frame_t *cfp = ec->cfp;
5391 VALUE arg0;
5392
5393 CALLER_SETUP_ARG(cfp, calling, ci, ISEQ_BODY(iseq)->param.lead_num);
5394
5395 if (arg_setup_type == arg_setup_block &&
5396 calling->argc == 1 &&
5397 ISEQ_BODY(iseq)->param.flags.has_lead &&
5398 !ISEQ_BODY(iseq)->param.flags.ambiguous_param0 &&
5399 !NIL_P(arg0 = vm_callee_setup_block_arg_arg0_check(argv))) {
5400 calling->argc = vm_callee_setup_block_arg_arg0_splat(cfp, iseq, argv, arg0);
5401 }
5402
5403 if (calling->argc != ISEQ_BODY(iseq)->param.lead_num) {
5404 if (arg_setup_type == arg_setup_block) {
5405 if (calling->argc < ISEQ_BODY(iseq)->param.lead_num) {
5406 int i;
5407 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5408 for (i=calling->argc; i<ISEQ_BODY(iseq)->param.lead_num; i++) argv[i] = Qnil;
5409 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* fill rest parameters */
5410 }
5411 else if (calling->argc > ISEQ_BODY(iseq)->param.lead_num) {
5412 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* simply truncate arguments */
5413 }
5414 }
5415 else {
5416 argument_arity_error(ec, iseq, NULL, calling->argc, ISEQ_BODY(iseq)->param.lead_num, ISEQ_BODY(iseq)->param.lead_num);
5417 }
5418 }
5419
5420 return 0;
5421 }
5422 else {
5423 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_type);
5424 }
5425}
5426
5427static int
5428vm_yield_setup_args(rb_execution_context_t *ec, const rb_iseq_t *iseq, const int argc, VALUE *argv, int flags, VALUE block_handler, enum arg_setup_type arg_setup_type)
5429{
5430 struct rb_calling_info calling_entry, *calling;
5431
5432 calling = &calling_entry;
5433 calling->argc = argc;
5434 calling->block_handler = block_handler;
5435 calling->kw_splat = (flags & VM_CALL_KW_SPLAT) ? 1 : 0;
5436 calling->recv = Qundef;
5437 calling->heap_argv = 0;
5438 calling->cc = NULL;
5439 struct rb_callinfo dummy_ci = VM_CI_ON_STACK(0, flags, 0, 0);
5440
5441 return vm_callee_setup_block_arg(ec, calling, &dummy_ci, iseq, argv, arg_setup_type);
5442}
5443
5444/* ruby iseq -> ruby block */
5445
5446static VALUE
5447vm_invoke_iseq_block_with_cref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5448 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5449 bool is_lambda, VALUE block_handler, const rb_cref_t *cref)
5450{
5451 const struct rb_captured_block *captured = VM_BH_TO_ISEQ_BLOCK(block_handler);
5452 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
5453 const int arg_size = ISEQ_BODY(iseq)->param.size;
5454 VALUE * const rsp = GET_SP() - calling->argc;
5455 VALUE * const argv = rsp;
5456 int opt_pc = vm_callee_setup_block_arg(ec, calling, ci, iseq, argv, is_lambda ? arg_setup_method : arg_setup_block);
5457 int frame_flag = VM_FRAME_MAGIC_BLOCK | (is_lambda ? VM_FRAME_FLAG_LAMBDA : 0);
5458
5459 SET_SP(rsp);
5460
5461 /* cref is normally 0; Proc#refined supplies a refinement cref */
5462 vm_push_frame(ec, iseq,
5463 frame_flag,
5464 captured->self,
5465 VM_GUARDED_PREV_EP(captured->ep), (VALUE)cref,
5466 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
5467 rsp + arg_size,
5468 ISEQ_BODY(iseq)->local_table_size - arg_size, ISEQ_BODY(iseq)->stack_max);
5469
5470 return Qundef;
5471}
5472
5473static VALUE
5474vm_invoke_iseq_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5475 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5476 bool is_lambda, VALUE block_handler)
5477{
5478 return vm_invoke_iseq_block_with_cref(ec, reg_cfp, calling, ci, is_lambda, block_handler, NULL);
5479}
5480
5481static VALUE
5482vm_invoke_symbol_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5483 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5484 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5485{
5486 VALUE symbol = VM_BH_TO_SYMBOL(block_handler);
5487 int flags = vm_ci_flag(ci);
5488
5489 if (UNLIKELY(!(flags & VM_CALL_ARGS_SIMPLE) &&
5490 ((calling->argc == 0) ||
5491 (calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
5492 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
5493 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc))))) {
5494 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
5495 flags = 0;
5496 if (UNLIKELY(calling->heap_argv)) {
5497#if VM_ARGC_STACK_MAX < 0
5498 if (RARRAY_LEN(calling->heap_argv) < 1) {
5499 rb_raise(rb_eArgError, "no receiver given");
5500 }
5501#endif
5502 calling->recv = rb_ary_shift(calling->heap_argv);
5503 // Modify stack to avoid cfp consistency error
5504 reg_cfp->sp++;
5505 reg_cfp->sp[-1] = reg_cfp->sp[-2];
5506 reg_cfp->sp[-2] = calling->recv;
5507 flags |= VM_CALL_ARGS_SPLAT;
5508 }
5509 else {
5510 if (calling->argc < 1) {
5511 rb_raise(rb_eArgError, "no receiver given");
5512 }
5513 calling->recv = TOPN(--calling->argc);
5514 }
5515 if (calling->kw_splat) {
5516 flags |= VM_CALL_KW_SPLAT;
5517 }
5518 }
5519 else {
5520 if (calling->argc < 1) {
5521 rb_raise(rb_eArgError, "no receiver given");
5522 }
5523 calling->recv = TOPN(--calling->argc);
5524 }
5525
5526 return vm_call_symbol(ec, reg_cfp, calling, ci, symbol, flags);
5527}
5528
5529static VALUE
5530vm_invoke_ifunc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5531 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5532 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5533{
5534 VALUE val;
5535 int argc;
5536 const struct rb_captured_block *captured = VM_BH_TO_IFUNC_BLOCK(block_handler);
5537 CALLER_SETUP_ARG(ec->cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
5538 argc = calling->argc;
5539 val = vm_yield_with_cfunc(ec, captured, captured->self, CALLING_ARGC(calling), calling->heap_argv ? RARRAY_CONST_PTR(calling->heap_argv) : STACK_ADDR_FROM_TOP(argc), calling->kw_splat, calling->block_handler, NULL);
5540 POPN(argc); /* TODO: should put before C/yield? */
5541 return val;
5542}
5543
5544static inline VALUE
5545vm_block_to_block_handler(const struct rb_block *block)
5546{
5547 switch (vm_block_type(block)) {
5548 case block_type_iseq:
5549 return VM_BH_FROM_ISEQ_BLOCK(&block->as.captured);
5550 case block_type_ifunc:
5551 return VM_BH_FROM_IFUNC_BLOCK(&block->as.captured);
5552 case block_type_symbol:
5553 return VM_BH_FROM_SYMBOL(block->as.symbol);
5554 case block_type_proc:
5555 return VM_BH_FROM_PROC(block->as.proc);
5556 }
5557 VM_UNREACHABLE(vm_yield_with_proc);
5558 return Qundef;
5559}
5560
5561static VALUE
5562vm_proc_to_block_handler(VALUE procval)
5563{
5564 return vm_block_to_block_handler(vm_proc_block(procval));
5565}
5566
5567NOINLINE(static VALUE
5568vm_invoke_proc_block_with_cref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5569 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5570 bool is_lambda, VALUE block_handler, VALUE refined_procval));
5571static VALUE
5572vm_invoke_proc_block_with_cref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5573 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5574 bool is_lambda, VALUE block_handler, VALUE refined_procval)
5575{
5576 const rb_cref_t *cref = rb_proc_refinements_cref_for_call(refined_procval);
5577 return vm_invoke_iseq_block_with_cref(ec, reg_cfp, calling, ci, is_lambda, block_handler, cref);
5578}
5579
5580static VALUE
5581vm_invoke_proc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5582 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5583 bool is_lambda, VALUE block_handler)
5584{
5585 VALUE refined_procval = 0;
5586
5587 while (vm_block_handler_type(block_handler) == block_handler_type_proc) {
5588 VALUE procval = VM_BH_TO_PROC(block_handler);
5589 rb_proc_t *po;
5590 GetProcPtr(procval, po);
5591 if (po->header.is_refined) refined_procval = procval;
5592 is_lambda = po->header.is_lambda;
5593 block_handler = vm_block_to_block_handler(&po->block);
5594 }
5595
5596 if (UNLIKELY(refined_procval) && vm_block_handler_type(block_handler) == block_handler_type_iseq) {
5597 /* should not be inlined so that vm_invoke_proc_block stays leaf/frameless. */
5598 return vm_invoke_proc_block_with_cref(ec, reg_cfp, calling, ci, is_lambda, block_handler,
5599 refined_procval);
5600 }
5601
5602 return vm_invoke_block(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5603}
5604
5605static inline VALUE
5606vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5607 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5608 bool is_lambda, VALUE block_handler)
5609{
5610 VALUE (*func)(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5611 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5612 bool is_lambda, VALUE block_handler);
5613
5614 switch (vm_block_handler_type(block_handler)) {
5615 case block_handler_type_iseq: func = vm_invoke_iseq_block; break;
5616 case block_handler_type_ifunc: func = vm_invoke_ifunc_block; break;
5617 case block_handler_type_proc: func = vm_invoke_proc_block; break;
5618 case block_handler_type_symbol: func = vm_invoke_symbol_block; break;
5619 default: rb_bug("vm_invoke_block: unreachable");
5620 }
5621
5622 return func(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5623}
5624
5625static VALUE
5626vm_make_proc_with_iseq(const rb_iseq_t *blockiseq)
5627{
5628 const rb_execution_context_t *ec = GET_EC();
5629 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
5630 struct rb_captured_block *captured;
5631
5632 if (cfp == 0) {
5633 rb_bug("vm_make_proc_with_iseq: unreachable");
5634 }
5635
5636 captured = VM_CFP_TO_CAPTURED_BLOCK(cfp);
5637 captured->code.iseq = blockiseq;
5638
5639 return rb_vm_make_proc(ec, captured, rb_cProc);
5640}
5641
5642static VALUE
5643vm_once_exec(VALUE iseq)
5644{
5645 VALUE proc = vm_make_proc_with_iseq((rb_iseq_t *)iseq);
5646 return rb_proc_call_with_block(proc, 0, 0, Qnil);
5647}
5648
5649#define RUNNING_THREAD_ONCE_DONE ((rb_thread_t *)0x1)
5650
5651struct vm_once_wait_arg {
5652 rb_vm_t *vm;
5653 union iseq_inline_storage_entry *is;
5654};
5655
5656static void
5657vm_once_broadcast(rb_vm_t *vm)
5658{
5659 rb_native_mutex_lock(&vm->once_lock);
5660 rb_native_cond_broadcast(&vm->once_cond);
5661 rb_native_mutex_unlock(&vm->once_lock);
5662}
5663
5664static void *
5665vm_once_wait_no_gvl(void *ptr)
5666{
5667 struct vm_once_wait_arg *arg = (struct vm_once_wait_arg *)ptr;
5668 rb_vm_t *vm = arg->vm;
5669 rb_thread_t *running_th;
5670
5671 rb_native_mutex_lock(&vm->once_lock);
5672 running_th = rbimpl_atomic_ptr_load((void **)&arg->is->once.running_thread, RBIMPL_ATOMIC_ACQUIRE);
5673 if (running_th != NULL && running_th != RUNNING_THREAD_ONCE_DONE) {
5674 rb_native_cond_wait(&vm->once_cond, &vm->once_lock);
5675 }
5676 rb_native_mutex_unlock(&vm->once_lock);
5677 return NULL;
5678}
5679
5680static void
5681vm_once_wait_ubf(void *ptr)
5682{
5683 vm_once_broadcast((rb_vm_t *)ptr);
5684}
5685
5686static VALUE
5687vm_once_clear(VALUE data)
5688{
5689 union iseq_inline_storage_entry *is = (union iseq_inline_storage_entry *)data;
5690 rbimpl_atomic_ptr_store((volatile void **)&is->once.running_thread, NULL, RBIMPL_ATOMIC_RELEASE);
5691 vm_once_broadcast(GET_VM());
5692 return Qnil;
5693}
5694
5695/* defined insn */
5696
5697static bool
5698check_respond_to_missing(VALUE obj, VALUE v)
5699{
5700 VALUE args[2];
5701 VALUE r;
5702
5703 args[0] = obj; args[1] = Qfalse;
5704 r = rb_check_funcall(v, idRespond_to_missing, 2, args);
5705 if (!UNDEF_P(r) && RTEST(r)) {
5706 return true;
5707 }
5708 else {
5709 return false;
5710 }
5711}
5712
5713static bool
5714vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5715{
5716 VALUE klass;
5717 enum defined_type type = (enum defined_type)op_type;
5718
5719 switch (type) {
5720 case DEFINED_IVAR:
5721 return rb_ivar_defined(GET_SELF(), SYM2ID(obj));
5722 break;
5723 case DEFINED_GVAR:
5724 return rb_gvar_defined(SYM2ID(obj));
5725 break;
5726 case DEFINED_CVAR: {
5727 const rb_cref_t *cref = vm_get_cref(GET_EP());
5728 klass = vm_get_cvar_base(cref, GET_CFP(), 0);
5729 return rb_cvar_defined(klass, SYM2ID(obj));
5730 break;
5731 }
5732 case DEFINED_CONST:
5733 case DEFINED_CONST_FROM: {
5734 bool allow_nil = type == DEFINED_CONST;
5735 klass = v;
5736 return vm_get_ev_const(ec, klass, SYM2ID(obj), allow_nil, true);
5737 break;
5738 }
5739 case DEFINED_FUNC:
5740 klass = CLASS_OF(v);
5741 return rb_ec_obj_respond_to(ec, v, SYM2ID(obj), TRUE);
5742 break;
5743 case DEFINED_METHOD:{
5744 VALUE klass = CLASS_OF(v);
5745 const rb_callable_method_entry_t *cme = rb_callable_method_entry_with_refinements(klass, SYM2ID(obj), NULL);
5746
5747 if (cme) {
5748 switch (METHOD_ENTRY_VISI(cme)) {
5749 case METHOD_VISI_PRIVATE:
5750 break;
5751 case METHOD_VISI_PROTECTED:
5752 if (!rb_obj_is_kind_of(GET_SELF(), vm_defined_class_for_protected_call(cme))) {
5753 break;
5754 }
5755 case METHOD_VISI_PUBLIC:
5756 return true;
5757 break;
5758 default:
5759 rb_bug("vm_defined: unreachable: %u", (unsigned int)METHOD_ENTRY_VISI(cme));
5760 }
5761 }
5762 else {
5763 return check_respond_to_missing(obj, v);
5764 }
5765 break;
5766 }
5767 case DEFINED_YIELD:
5768 if (GET_BLOCK_HANDLER() != VM_BLOCK_HANDLER_NONE) {
5769 return true;
5770 }
5771 break;
5772 case DEFINED_ZSUPER:
5773 {
5774 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(GET_CFP());
5775
5776 if (me) {
5777 VALUE klass = vm_search_normal_superclass(me->defined_class);
5778 if (!klass) return false;
5779
5780 ID id = me->def->original_id;
5781
5782 return rb_method_boundp(klass, id, 0);
5783 }
5784 }
5785 break;
5786 case DEFINED_REF:
5787 return RTEST(vm_backref_defined(ec, GET_LEP(), FIX2INT(obj)));
5788 default:
5789 rb_bug("unimplemented defined? type (VM)");
5790 break;
5791 }
5792
5793 return false;
5794}
5795
5796bool
5797rb_vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5798{
5799 return vm_defined(ec, reg_cfp, op_type, obj, v);
5800}
5801
5802static const VALUE *
5803vm_get_ep(const VALUE *const reg_ep, rb_num_t lv)
5804{
5805 rb_num_t i;
5806 const VALUE *ep = reg_ep;
5807 for (i = 0; i < lv; i++) {
5808 ep = GET_PREV_EP(ep);
5809 }
5810 return ep;
5811}
5812
5813static VALUE
5814vm_get_special_object(const VALUE *const reg_ep,
5815 enum vm_special_object_type type)
5816{
5817 switch (type) {
5818 case VM_SPECIAL_OBJECT_VMCORE:
5819 return rb_mRubyVMFrozenCore;
5820 case VM_SPECIAL_OBJECT_CBASE:
5821 return vm_get_cbase(reg_ep);
5822 case VM_SPECIAL_OBJECT_CONST_BASE:
5823 return vm_get_const_base(reg_ep);
5824 default:
5825 rb_bug("putspecialobject insn: unknown value_type %d", type);
5826 }
5827}
5828
5829// ZJIT implementation is using the C function
5830// and needs to call a non-static function
5831VALUE
5832rb_vm_get_special_object(const VALUE *reg_ep, enum vm_special_object_type type)
5833{
5834 return vm_get_special_object(reg_ep, type);
5835}
5836
5837static VALUE
5838vm_concat_array(VALUE ary1, VALUE ary2st)
5839{
5840 const VALUE ary2 = ary2st;
5841 VALUE tmp1 = rb_check_to_array(ary1);
5842 VALUE tmp2 = rb_check_to_array(ary2);
5843
5844 if (NIL_P(tmp1)) {
5845 tmp1 = rb_ary_new3(1, ary1);
5846 }
5847 if (tmp1 == ary1) {
5848 tmp1 = rb_ary_dup(ary1);
5849 }
5850
5851 if (NIL_P(tmp2)) {
5852 return rb_ary_push(tmp1, ary2);
5853 }
5854 else {
5855 return rb_ary_concat(tmp1, tmp2);
5856 }
5857}
5858
5859static VALUE
5860vm_concat_to_array(VALUE ary1, VALUE ary2st)
5861{
5862 /* ary1 must be a newly created array */
5863 const VALUE ary2 = ary2st;
5864
5865 if (NIL_P(ary2)) return ary1;
5866
5867 VALUE tmp2 = rb_check_to_array(ary2);
5868
5869 if (NIL_P(tmp2)) {
5870 return rb_ary_push(ary1, ary2);
5871 }
5872 else {
5873 return rb_ary_concat(ary1, tmp2);
5874 }
5875}
5876
5877// YJIT implementation is using the C function
5878// and needs to call a non-static function
5879VALUE
5880rb_vm_concat_array(VALUE ary1, VALUE ary2st)
5881{
5882 return vm_concat_array(ary1, ary2st);
5883}
5884
5885VALUE
5886rb_vm_concat_to_array(VALUE ary1, VALUE ary2st)
5887{
5888 return vm_concat_to_array(ary1, ary2st);
5889}
5890
5891static VALUE
5892vm_splat_array(VALUE flag, VALUE ary)
5893{
5894 if (NIL_P(ary)) {
5895 return RTEST(flag) ? rb_ary_new() : rb_cArray_empty_frozen;
5896 }
5897 VALUE tmp = rb_check_to_array(ary);
5898 if (NIL_P(tmp)) {
5899 return rb_ary_new3(1, ary);
5900 }
5901 else if (RTEST(flag)) {
5902 return rb_ary_dup(tmp);
5903 }
5904 else {
5905 return tmp;
5906 }
5907}
5908
5909// YJIT implementation is using the C function
5910// and needs to call a non-static function
5911VALUE
5912rb_vm_splat_array(VALUE flag, VALUE ary)
5913{
5914 return vm_splat_array(flag, ary);
5915}
5916
5917static VALUE
5918vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
5919{
5920 enum vm_check_match_type type = ((int)flag) & VM_CHECKMATCH_TYPE_MASK;
5921
5922 if (flag & VM_CHECKMATCH_ARRAY) {
5923 long i;
5924 const long n = RARRAY_LEN(pattern);
5925
5926 for (i = 0; i < n; i++) {
5927 VALUE v = RARRAY_AREF(pattern, i);
5928 VALUE c = check_match(ec, v, target, type);
5929
5930 if (RTEST(c)) {
5931 return c;
5932 }
5933 }
5934 return Qfalse;
5935 }
5936 else {
5937 return check_match(ec, pattern, target, type);
5938 }
5939}
5940
5941VALUE
5942rb_vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
5943{
5944 return vm_check_match(ec, target, pattern, flag);
5945}
5946
5947static VALUE
5948vm_check_keyword(lindex_t bits, lindex_t idx, const VALUE *ep)
5949{
5950 const VALUE kw_bits = *(ep - bits);
5951
5952 if (FIXNUM_P(kw_bits)) {
5953 unsigned int b = (unsigned int)FIX2ULONG(kw_bits);
5954 if ((idx < VM_KW_SPECIFIED_BITS_MAX) && (b & (0x01 << idx)))
5955 return Qfalse;
5956 }
5957 else {
5958 VM_ASSERT(rb_set_p(kw_bits), "%s", rb_obj_info(kw_bits));
5959 if (rb_set_lookup(kw_bits, INT2FIX(idx))) return Qfalse;
5960 }
5961 return Qtrue;
5962}
5963
5964static void
5965vm_dtrace(rb_event_flag_t flag, rb_execution_context_t *ec)
5966{
5967 if (RUBY_DTRACE_METHOD_ENTRY_ENABLED() ||
5968 RUBY_DTRACE_METHOD_RETURN_ENABLED() ||
5969 RUBY_DTRACE_CMETHOD_ENTRY_ENABLED() ||
5970 RUBY_DTRACE_CMETHOD_RETURN_ENABLED()) {
5971
5972 switch (flag) {
5973 case RUBY_EVENT_CALL:
5974 RUBY_DTRACE_METHOD_ENTRY_HOOK(ec, 0, 0);
5975 return;
5976 case RUBY_EVENT_C_CALL:
5977 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, 0, 0);
5978 return;
5979 case RUBY_EVENT_RETURN:
5980 RUBY_DTRACE_METHOD_RETURN_HOOK(ec, 0, 0);
5981 return;
5982 case RUBY_EVENT_C_RETURN:
5983 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, 0, 0);
5984 return;
5985 }
5986 }
5987}
5988
5989static VALUE
5990vm_const_get_under(ID id, rb_num_t flags, VALUE cbase)
5991{
5992 if (!rb_const_defined_at(cbase, id)) {
5993 return 0;
5994 }
5995 else if (VM_DEFINECLASS_SCOPED_P(flags)) {
5996 return rb_public_const_get_at(cbase, id);
5997 }
5998 else {
5999 return rb_const_get_at(cbase, id);
6000 }
6001}
6002
6003static VALUE
6004vm_check_if_class(ID id, rb_num_t flags, VALUE super, VALUE klass)
6005{
6006 if (!RB_TYPE_P(klass, T_CLASS)) {
6007 return 0;
6008 }
6009 else if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags)) {
6010 VALUE tmp = rb_class_real(RCLASS_SUPER(klass));
6011
6012 if (tmp != super) {
6013 rb_raise(rb_eTypeError,
6014 "superclass mismatch for class %"PRIsVALUE"",
6015 rb_id2str(id));
6016 }
6017 else {
6018 return klass;
6019 }
6020 }
6021 else {
6022 return klass;
6023 }
6024}
6025
6026static VALUE
6027vm_check_if_module(ID id, VALUE mod)
6028{
6029 if (!RB_TYPE_P(mod, T_MODULE)) {
6030 return 0;
6031 }
6032 else {
6033 return mod;
6034 }
6035}
6036
6037static VALUE
6038declare_under(ID id, VALUE cbase, VALUE c)
6039{
6040 rb_set_class_path_string(c, cbase, rb_id2str(id));
6041 rb_const_set(cbase, id, c);
6042 return c;
6043}
6044
6045static VALUE
6046vm_declare_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
6047{
6048 /* new class declaration */
6049 VALUE s = VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) ? super : rb_cObject;
6050 VALUE c = declare_under(id, cbase, rb_define_class_id(id, s));
6051 rb_class_inherited(s, c);
6052 return c;
6053}
6054
6055static VALUE
6056vm_declare_module(ID id, VALUE cbase)
6057{
6058 /* new module declaration */
6059 return declare_under(id, cbase, rb_module_new());
6060}
6061
6062NORETURN(static void unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old));
6063static void
6064unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old)
6065{
6066 VALUE name = rb_id2str(id);
6067 VALUE message = rb_sprintf("%"PRIsVALUE" is not a %s",
6068 name, type);
6069 VALUE location = rb_const_source_location_at(cbase, id);
6070 if (!NIL_P(location)) {
6071 rb_str_catf(message, "\n%"PRIsVALUE":%"PRIsVALUE":"
6072 " previous definition of %"PRIsVALUE" was here",
6073 rb_ary_entry(location, 0), rb_ary_entry(location, 1), name);
6074 }
6075 rb_exc_raise(rb_exc_new_str(rb_eTypeError, message));
6076}
6077
6078static VALUE
6079vm_define_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
6080{
6081 VALUE klass;
6082
6083 if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) && !RB_TYPE_P(super, T_CLASS)) {
6084 rb_raise(rb_eTypeError,
6085 "superclass must be an instance of Class (given an instance of %"PRIsVALUE")",
6086 rb_obj_class(super));
6087 }
6088
6089 vm_check_if_namespace(cbase);
6090
6091 /* find klass */
6092 rb_autoload_load(cbase, id);
6093
6094 if ((klass = vm_const_get_under(id, flags, cbase)) != 0) {
6095 if (!vm_check_if_class(id, flags, super, klass))
6096 unmatched_redefinition("class", cbase, id, klass);
6097 return klass;
6098 }
6099 else {
6100 return vm_declare_class(id, flags, cbase, super);
6101 }
6102}
6103
6104static VALUE
6105vm_define_module(ID id, rb_num_t flags, VALUE cbase)
6106{
6107 VALUE mod;
6108
6109 vm_check_if_namespace(cbase);
6110 if ((mod = vm_const_get_under(id, flags, cbase)) != 0) {
6111 if (!vm_check_if_module(id, mod))
6112 unmatched_redefinition("module", cbase, id, mod);
6113 return mod;
6114 }
6115 else {
6116 return vm_declare_module(id, cbase);
6117 }
6118}
6119
6120static VALUE
6121vm_find_or_create_class_by_id(ID id,
6122 rb_num_t flags,
6123 VALUE cbase,
6124 VALUE super)
6125{
6126 rb_vm_defineclass_type_t type = VM_DEFINECLASS_TYPE(flags);
6127
6128 switch (type) {
6129 case VM_DEFINECLASS_TYPE_CLASS:
6130 /* classdef returns class scope value */
6131 return vm_define_class(id, flags, cbase, super);
6132
6133 case VM_DEFINECLASS_TYPE_SINGLETON_CLASS:
6134 /* classdef returns class scope value */
6135 return rb_singleton_class(cbase);
6136
6137 case VM_DEFINECLASS_TYPE_MODULE:
6138 /* classdef returns class scope value */
6139 return vm_define_module(id, flags, cbase);
6140
6141 default:
6142 rb_bug("unknown defineclass type: %d", (int)type);
6143 }
6144}
6145
6146static rb_method_visibility_t
6147vm_scope_visibility_get(const rb_execution_context_t *ec)
6148{
6149 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
6150
6151 if (!vm_env_cref_by_cref(cfp->ep)) {
6152 return METHOD_VISI_PUBLIC;
6153 }
6154 else {
6155 return CREF_SCOPE_VISI(vm_ec_cref(ec))->method_visi;
6156 }
6157}
6158
6159static int
6160vm_scope_module_func_check(const rb_execution_context_t *ec)
6161{
6162 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
6163
6164 if (!vm_env_cref_by_cref(cfp->ep)) {
6165 return FALSE;
6166 }
6167 else {
6168 return CREF_SCOPE_VISI(vm_ec_cref(ec))->module_func;
6169 }
6170}
6171
6172static void
6173vm_define_method(const rb_execution_context_t *ec, VALUE obj, ID id, VALUE iseqval, int is_singleton)
6174{
6175 VALUE klass;
6176 rb_method_visibility_t visi;
6177 rb_cref_t *cref = vm_ec_cref(ec);
6178
6179 if (is_singleton) {
6180 klass = rb_singleton_class(obj); /* class and frozen checked in this API */
6181 visi = METHOD_VISI_PUBLIC;
6182 }
6183 else {
6184 klass = CREF_CLASS_FOR_DEFINITION(cref);
6185 visi = vm_scope_visibility_get(ec);
6186 }
6187
6188 if (NIL_P(klass)) {
6189 rb_raise(rb_eTypeError, "no class/module to add method");
6190 }
6191
6192 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, visi);
6193 // Set max_iv_count on klasses based on number of ivar sets that are in the initialize method
6194 if (id == idInitialize && RB_TYPE_P(klass, T_CLASS) &&
6195 !RCLASS_SINGLETON_P(klass) && !RCLASS_EXPECT_NO_IVAR(klass) &&
6196 (rb_get_alloc_func(klass) == rb_class_allocate_instance)) {
6197 RCLASS_SET_MAX_IV_COUNT(klass, rb_estimate_iv_count(klass, (const rb_iseq_t *)iseqval));
6198 }
6199
6200 if (!is_singleton && vm_scope_module_func_check(ec)) {
6201 klass = rb_singleton_class(klass);
6202 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, METHOD_VISI_PUBLIC);
6203 }
6204}
6205
6206// Return the untagged block handler:
6207// * If it's VM_BLOCK_HANDLER_NONE, return nil
6208// * If it's an ISEQ or an IFUNC, fetch it from its rb_captured_block
6209// * If it's a PROC or SYMBOL, return it as is
6210VALUE
6211rb_vm_untag_block_handler(VALUE block_handler)
6212{
6213 if (VM_BLOCK_HANDLER_NONE == block_handler) return Qnil;
6214
6215 switch (vm_block_handler_type(block_handler)) {
6216 case block_handler_type_iseq:
6217 case block_handler_type_ifunc: {
6218 struct rb_captured_block *captured = VM_TAGGED_PTR_REF(block_handler, 0x03);
6219 return captured->code.val;
6220 }
6221 case block_handler_type_proc:
6222 case block_handler_type_symbol:
6223 return block_handler;
6224 default:
6225 rb_bug("rb_vm_untag_block_handler: unreachable");
6226 }
6227}
6228
6229VALUE
6230rb_vm_get_untagged_block_handler(rb_control_frame_t *reg_cfp)
6231{
6232 return rb_vm_untag_block_handler(VM_CF_BLOCK_HANDLER(reg_cfp));
6233}
6234
6235static VALUE
6236vm_invokeblock_i(struct rb_execution_context_struct *ec,
6237 struct rb_control_frame_struct *reg_cfp,
6238 struct rb_calling_info *calling)
6239{
6240 const struct rb_callinfo *ci = calling->cd->ci;
6241 VALUE block_handler = VM_CF_BLOCK_HANDLER(GET_CFP());
6242
6243 if (block_handler == VM_BLOCK_HANDLER_NONE) {
6244 rb_vm_localjump_error("no block given (yield)", Qnil, 0);
6245 }
6246 else {
6247 return vm_invoke_block(ec, GET_CFP(), calling, ci, false, block_handler);
6248 }
6249}
6250
6251enum method_explorer_type {
6252 mexp_search_method,
6253 mexp_search_invokeblock,
6254 mexp_search_super,
6255};
6256
6257ALWAYS_INLINE(static VALUE vm_sendish(struct rb_execution_context_struct *ec,
6258 struct rb_control_frame_struct *reg_cfp,
6259 struct rb_call_data *cd, VALUE block_handler,
6260 enum method_explorer_type method_explorer));
6261static inline VALUE
6262vm_sendish(
6263 struct rb_execution_context_struct *ec,
6264 struct rb_control_frame_struct *reg_cfp,
6265 struct rb_call_data *cd,
6266 VALUE block_handler,
6267 enum method_explorer_type method_explorer
6268) {
6269 VALUE val = Qundef;
6270 const struct rb_callinfo *ci = cd->ci;
6271 const struct rb_callcache *cc;
6272 int argc = vm_ci_argc(ci);
6273 VALUE recv = TOPN(argc);
6274 struct rb_calling_info calling = {
6275 .block_handler = block_handler,
6276 .kw_splat = IS_ARGS_KW_SPLAT(ci) > 0,
6277 .recv = recv,
6278 .argc = argc,
6279 .cd = cd,
6280 };
6281
6282 switch (method_explorer) {
6283 case mexp_search_method:
6284 calling.cc = cc = vm_search_method_fastpath(reg_cfp, cd, CLASS_OF(recv));
6285 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6286 break;
6287 case mexp_search_super:
6288 calling.cc = cc = vm_search_super_method(reg_cfp, cd, recv);
6289 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6290 break;
6291 case mexp_search_invokeblock:
6292 val = vm_invokeblock_i(ec, GET_CFP(), &calling);
6293 break;
6294 }
6295 return val;
6296}
6297
6298VALUE
6299rb_vm_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6300{
6301 stack_check(ec);
6302 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), cd->ci, blockiseq, false);
6303 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6304 VM_EXEC(ec, val);
6305 return val;
6306}
6307
6308// Fallback for YJIT/ZJIT, not used by the interpreter
6309VALUE
6310rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6311{
6312 stack_check(ec);
6313
6314 struct rb_forwarding_call_data adjusted_cd;
6315 struct rb_callinfo adjusted_ci;
6316
6317 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, false, &adjusted_cd, &adjusted_ci);
6318
6319 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method);
6320
6321 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6322 RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc);
6323 }
6324
6325 VM_EXEC(ec, val);
6326 return val;
6327}
6328
6329VALUE
6330rb_vm_opt_send_without_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6331{
6332 stack_check(ec);
6333 VALUE bh = VM_BLOCK_HANDLER_NONE;
6334 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6335 VM_EXEC(ec, val);
6336 return val;
6337}
6338
6339VALUE
6340rb_vm_invokesuper(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6341{
6342 stack_check(ec);
6343
6344 struct rb_callinfo adjusted_ci = VM_CI_ON_STACK(vm_ci_mid(cd->ci),
6345 vm_ci_flag(cd->ci),
6346 vm_ci_argc(cd->ci),
6347 vm_ci_kwarg(cd->ci));
6348 const struct rb_callcache *original_cc = rbimpl_atomic_ptr_load((void **)&cd->cc, RBIMPL_ATOMIC_ACQUIRE);
6349 struct rb_call_data adjusted_cd = {
6350 .ci = &adjusted_ci,
6351 .cc = original_cc,
6352 };
6353
6354 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), adjusted_cd.ci, blockiseq, true);
6355 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd, bh, mexp_search_super);
6356
6357 if (original_cc != adjusted_cd.cc && vm_cc_markable(adjusted_cd.cc)) {
6358 rbimpl_atomic_ptr_store((volatile void **)&cd->cc, (void *)adjusted_cd.cc, RBIMPL_ATOMIC_RELEASE);
6359 RB_OBJ_WRITTEN(CFP_ISEQ(GET_CFP()), Qundef, adjusted_cd.cc);
6360 }
6361
6362 VM_EXEC(ec, val);
6363 return val;
6364}
6365
6366// Fallback for YJIT/ZJIT, not used by the interpreter
6367VALUE
6368rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6369{
6370 stack_check(ec);
6371 struct rb_forwarding_call_data adjusted_cd;
6372 struct rb_callinfo adjusted_ci;
6373
6374 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, true, &adjusted_cd, &adjusted_ci);
6375
6376 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super);
6377
6378 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6379 RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc);
6380 }
6381
6382 VM_EXEC(ec, val);
6383 return val;
6384}
6385
6386VALUE
6387rb_vm_invokeblock(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6388{
6389 stack_check(ec);
6390 VALUE bh = VM_BLOCK_HANDLER_NONE;
6391 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_invokeblock);
6392 VM_EXEC(ec, val);
6393 return val;
6394}
6395
6396/* object.c */
6397VALUE rb_nil_to_s(VALUE);
6398VALUE rb_true_to_s(VALUE);
6399VALUE rb_false_to_s(VALUE);
6400/* numeric.c */
6401VALUE rb_int_to_s(int argc, VALUE *argv, VALUE x);
6402VALUE rb_fix_to_s(VALUE);
6403/* variable.c */
6404VALUE rb_mod_to_s(VALUE);
6405VALUE rb_mod_name(VALUE);
6406
6407static VALUE
6408vm_objtostring(struct rb_control_frame_struct *reg_cfp, VALUE recv, CALL_DATA cd)
6409{
6410 int type = TYPE(recv);
6411 if (type == T_STRING) {
6412 return recv;
6413 }
6414
6415 const struct rb_callable_method_entry_struct *cme = vm_search_method(reg_cfp, cd, recv);
6416
6417 switch (type) {
6418 case T_SYMBOL:
6419 if (check_method_basic_definition(cme)) {
6420 return rb_sym2str(recv);
6421 }
6422 break;
6423 case T_MODULE:
6424 case T_CLASS:
6425 if (check_cfunc(cme, rb_mod_to_s)) {
6426 // rb_mod_to_s() allocates a mutable string, but since we are only
6427 // going to use this string for interpolation, it's fine to use the
6428 // frozen string.
6429 VALUE val = rb_mod_name(recv);
6430 if (NIL_P(val)) {
6431 val = rb_mod_to_s(recv);
6432 }
6433 return val;
6434 }
6435 break;
6436 case T_NIL:
6437 if (check_cfunc(cme, rb_nil_to_s)) {
6438 return rb_nil_to_s(recv);
6439 }
6440 break;
6441 case T_TRUE:
6442 if (check_cfunc(cme, rb_true_to_s)) {
6443 return rb_true_to_s(recv);
6444 }
6445 break;
6446 case T_FALSE:
6447 if (check_cfunc(cme, rb_false_to_s)) {
6448 return rb_false_to_s(recv);
6449 }
6450 break;
6451 case T_FIXNUM:
6452 if (check_cfunc(cme, rb_int_to_s)) {
6453 return rb_fix_to_s(recv);
6454 }
6455 break;
6456 }
6457 return Qundef;
6458}
6459
6460// ZJIT implementation is using the C function
6461// and needs to call a non-static function
6462VALUE
6463rb_vm_objtostring(struct rb_control_frame_struct *reg_cfp, VALUE recv, CALL_DATA cd)
6464{
6465 return vm_objtostring(reg_cfp, recv, cd);
6466}
6467
6468static VALUE
6469vm_opt_ary_freeze(VALUE ary, int bop, ID id)
6470{
6471 if (BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
6472 return ary;
6473 }
6474 else {
6475 return Qundef;
6476 }
6477}
6478
6479static VALUE
6480vm_opt_hash_freeze(VALUE hash, int bop, ID id)
6481{
6482 if (BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
6483 return hash;
6484 }
6485 else {
6486 return Qundef;
6487 }
6488}
6489
6490static VALUE
6491vm_opt_str_freeze(VALUE str, int bop, ID id)
6492{
6493 if (BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
6494 return str;
6495 }
6496 else {
6497 return Qundef;
6498 }
6499}
6500
6501/* this macro is mandatory to use OPTIMIZED_CMP. What a design! */
6502#define id_cmp idCmp
6503
6504static VALUE
6505vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6506{
6507 if (OP_UNREDEFINED_P(INCLUDE_P, ARRAY)) {
6508 return rb_ary_includes(ary, target);
6509 }
6510 else {
6511 VALUE args[1] = {target};
6512
6513 // duparray
6514 RUBY_DTRACE_CREATE_HOOK(ARRAY, RARRAY_LEN(ary));
6515 VALUE dupary = rb_ary_resurrect(ary);
6516
6517 return rb_vm_call_with_refinements(ec, dupary, idIncludeP, 1, args, RB_NO_KEYWORDS);
6518 }
6519}
6520
6521VALUE
6522rb_vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6523{
6524 return vm_opt_duparray_include_p(ec, ary, target);
6525}
6526
6527static VALUE
6528vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6529{
6530 if (OP_UNREDEFINED_P(MAX, ARRAY)) {
6531 if (array_len == 0) {
6532 return Qnil;
6533 }
6534 else {
6535 VALUE result = *ptr;
6536 rb_snum_t i = array_len - 1;
6537 while (i-- > 0) {
6538 const VALUE v = *++ptr;
6539 if (OPTIMIZED_CMP(v, result) > 0) {
6540 result = v;
6541 }
6542 }
6543 return result;
6544 }
6545 }
6546 else {
6547 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMax, 0, NULL, RB_NO_KEYWORDS);
6548 }
6549}
6550
6551VALUE
6552rb_vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6553{
6554 return vm_opt_newarray_max(ec, array_len, ptr);
6555}
6556
6557static VALUE
6558vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6559{
6560 if (OP_UNREDEFINED_P(MIN, ARRAY)) {
6561 if (array_len == 0) {
6562 return Qnil;
6563 }
6564 else {
6565 VALUE result = *ptr;
6566 rb_snum_t i = array_len - 1;
6567 while (i-- > 0) {
6568 const VALUE v = *++ptr;
6569 if (OPTIMIZED_CMP(v, result) < 0) {
6570 result = v;
6571 }
6572 }
6573 return result;
6574 }
6575 }
6576 else {
6577 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMin, 0, NULL, RB_NO_KEYWORDS);
6578 }
6579}
6580
6581VALUE
6582rb_vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6583{
6584 return vm_opt_newarray_min(ec, array_len, ptr);
6585}
6586
6587static VALUE
6588vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6589{
6590 // If Array#hash is _not_ monkeypatched, use the optimized call
6591 if (OP_UNREDEFINED_P(HASH, ARRAY)) {
6592 return rb_ary_hash_values(array_len, ptr);
6593 }
6594 else {
6595 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idHash, 0, NULL, RB_NO_KEYWORDS);
6596 }
6597}
6598
6599VALUE
6600rb_vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6601{
6602 return vm_opt_newarray_hash(ec, array_len, ptr);
6603}
6604
6605VALUE rb_setup_fake_ary(struct RArray *fake_ary, const VALUE *list, long len);
6606VALUE rb_ec_pack_ary(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer);
6607
6608static VALUE
6609vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6610{
6611 if (OP_UNREDEFINED_P(INCLUDE_P, ARRAY)) {
6612 struct RArray fake_ary = {RBASIC_INIT};
6613 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6614 return rb_ary_includes(ary, target);
6615 }
6616 else {
6617 VALUE args[1] = {target};
6618 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idIncludeP, 1, args, RB_NO_KEYWORDS);
6619 }
6620}
6621
6622VALUE
6623rb_vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6624{
6625 return vm_opt_newarray_include_p(ec, array_len, ptr, target);
6626}
6627
6628static VALUE
6629vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6630{
6631 if (OP_UNREDEFINED_P(PACK, ARRAY)) {
6632 struct RArray fake_ary = {RBASIC_INIT};
6633 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6634 return rb_ec_pack_ary(ec, ary, fmt, (UNDEF_P(buffer) ? Qnil : buffer));
6635 }
6636 else {
6637 // The opt_newarray_send insn drops the keyword args so we need to rebuild them.
6638 // Setup an array with room for keyword hash.
6639 VALUE args[2];
6640 args[0] = fmt;
6641 int kw_splat = RB_NO_KEYWORDS;
6642 int argc = 1;
6643
6644 if (!UNDEF_P(buffer)) {
6645 args[1] = rb_hash_new_capa(1);
6646 rb_hash_aset(args[1], ID2SYM(idBuffer), buffer);
6647 kw_splat = RB_PASS_KEYWORDS;
6648 argc++;
6649 }
6650
6651 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idPack, argc, args, kw_splat);
6652 }
6653}
6654
6655VALUE
6656rb_vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6657{
6658 return vm_opt_newarray_pack_buffer(ec, array_len, ptr, fmt, buffer);
6659}
6660
6661
6662#undef id_cmp
6663
6664static void
6665vm_track_constant_cache(ID id, void *ic)
6666{
6667 rb_vm_t *vm = GET_VM();
6668 struct rb_id_table *const_cache = &vm->constant_cache;
6669 VALUE lookup_result;
6670 set_table *ics;
6671
6672 if (rb_id_table_lookup(const_cache, id, &lookup_result)) {
6673 ics = (set_table *)lookup_result;
6674 }
6675 else {
6676 ics = set_init_numtable();
6677 rb_id_table_insert(const_cache, id, (VALUE)ics);
6678 }
6679
6680 /* The call below to st_insert could allocate which could trigger a GC.
6681 * If it triggers a GC, it may free an iseq that also holds a cache to this
6682 * constant. If that iseq is the last iseq with a cache to this constant, then
6683 * it will free this ST table, which would cause an use-after-free during this
6684 * st_insert.
6685 *
6686 * So to fix this issue, we store the ID that is currently being inserted
6687 * and, in remove_from_constant_cache, we don't free the ST table for ID
6688 * equal to this one.
6689 *
6690 * See [Bug #20921].
6691 */
6692 vm->inserting_constant_cache_id = id;
6693
6694 set_insert(ics, (st_data_t)ic);
6695
6696 vm->inserting_constant_cache_id = (ID)0;
6697}
6698
6699static void
6700vm_ic_track_const_chain(rb_control_frame_t *cfp, IC ic, const ID *segments)
6701{
6702 RB_VM_LOCKING() {
6703 for (int i = 0; segments[i]; i++) {
6704 ID id = segments[i];
6705 if (id == idNULL) continue;
6706 vm_track_constant_cache(id, ic);
6707 }
6708 }
6709}
6710
6711// For JIT inlining
6712static inline bool
6713vm_inlined_ic_hit_p(VALUE flags, VALUE value, const rb_cref_t *ic_cref, rb_serial_t ractor_id, const VALUE *reg_ep)
6714{
6715 // Not rb_ractor_main_p(): an owner Ractor may now cache an unshareable constant
6716 // of its own class, and only it may be handed that value back.
6717 if ((flags & IMEMO_CONST_CACHE_SHAREABLE) || ractor_id == rb_ractor_id(GET_RACTOR())) {
6718 VM_ASSERT(ractor_incidental_shareable_p(flags & IMEMO_CONST_CACHE_SHAREABLE, value));
6719
6720 return (ic_cref == NULL || // no need to check CREF
6721 ic_cref == vm_get_cref(reg_ep));
6722 }
6723 return false;
6724}
6725
6726static bool
6727vm_ic_hit_p(const struct iseq_inline_constant_cache_entry *ice, const VALUE *reg_ep)
6728{
6729 VM_ASSERT(IMEMO_TYPE_P(ice, imemo_constcache));
6730 return vm_inlined_ic_hit_p(ice->flags, ice->value, ice->ic_cref, ice->ractor_id, reg_ep);
6731}
6732
6733// YJIT needs this function to never allocate and never raise
6734bool
6735rb_vm_ic_hit_p(IC ic, const VALUE *reg_ep)
6736{
6737 return ic->entry && vm_ic_hit_p(ic->entry, reg_ep);
6738}
6739
6740static void
6741vm_ic_update(const rb_iseq_t *iseq, IC ic, VALUE val, const VALUE *reg_ep, const VALUE *pc)
6742{
6743 if (ruby_vm_const_missing_count > 0) {
6744 ruby_vm_const_missing_count = 0;
6745 ic->entry = NULL;
6746 return;
6747 }
6748
6749 struct iseq_inline_constant_cache_entry *ice = SHAREABLE_IMEMO_NEW(struct iseq_inline_constant_cache_entry, imemo_constcache, 0);
6750 RB_OBJ_WRITE(ice, &ice->value, val);
6751 ice->ic_cref = vm_get_const_key_cref(reg_ep);
6752 ice->ractor_id = rb_ractor_id(GET_RACTOR());
6753
6754 if (rb_ractor_shareable_p(val)) {
6755 RUBY_ASSERT((rb_gc_verify_shareable(val), 1));
6756 ice->flags |= IMEMO_CONST_CACHE_SHAREABLE;
6757 }
6758 RB_OBJ_WRITE(iseq, &ic->entry, ice);
6759 RUBY_ASSERT(pc >= ISEQ_BODY(iseq)->iseq_encoded);
6760 unsigned pos = (unsigned)(pc - ISEQ_BODY(iseq)->iseq_encoded);
6761 rb_yjit_constant_ic_update(iseq, ic, pos);
6762}
6763
6764VALUE
6765rb_vm_opt_getconstant_path(rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, IC ic)
6766{
6767 VALUE val;
6768 const ID *segments = ic->segments;
6769 struct iseq_inline_constant_cache_entry *ice = ic->entry;
6770
6771 if (ice && vm_ic_hit_p(ice, GET_EP())) {
6772 val = ice->value;
6773
6774 // On a non-main ractor another ractor can concurrently reassign a
6775 // shareable constant (which invalidates ICs asynchronously), so the
6776 // cached-but-still-shareable value may legitimately differ from a
6777 // freshly recomputed chain. Only assert equality when single-ractor.
6778 VM_ASSERT(val == vm_get_ev_const_chain(ec, segments) || rb_multi_ractor_p());
6779 }
6780 else {
6781 ruby_vm_constant_cache_misses++;
6782 val = vm_get_ev_const_chain(ec, segments);
6783 vm_ic_track_const_chain(GET_CFP(), ic, segments);
6784 // Undo the PC increment to get the address to this instruction
6785 // INSN_ATTR(width) == 2
6786 vm_ic_update(CFP_ISEQ(GET_CFP()), ic, val, GET_EP(), CFP_PC(GET_CFP()) - 2);
6787 }
6788 return val;
6789}
6790
6791// Return true if the once value is already computed and set *result to the value.
6792// Otherwise, return false.
6793// Used for ZJIT. Keep in sync with `vm_once_dispatch` below.
6794bool
6795rb_vm_once_done_value(ISE is, VALUE *result)
6796{
6797 rb_thread_t *running_th = rbimpl_atomic_ptr_load((void**)&is->once.running_thread, RBIMPL_ATOMIC_ACQUIRE);
6798 if (running_th == RUNNING_THREAD_ONCE_DONE) {
6799 *result = is->once.value;
6800 return true;
6801 }
6802 return false;
6803}
6804
6805static VALUE
6806vm_once_dispatch(rb_execution_context_t *ec, ISEQ iseq, ISE is)
6807{
6808 rb_thread_t *th = rb_ec_thread_ptr(ec);
6809 rb_thread_t *running_th;
6810
6811 again:
6812 running_th = rbimpl_atomic_ptr_load((void**)&is->once.running_thread, RBIMPL_ATOMIC_ACQUIRE);
6813 if (running_th == RUNNING_THREAD_ONCE_DONE) {
6814 return is->once.value;
6815 }
6816 else if (running_th == NULL) {
6817 VALUE val = Qundef;
6818 enum ruby_tag_type state;
6819 if (rbimpl_atomic_ptr_cas((void**)&is->once.running_thread, running_th, th, RBIMPL_ATOMIC_RELEASE, RBIMPL_ATOMIC_RELAXED) != running_th) {
6820 goto again;
6821 }
6822 EC_PUSH_TAG(ec);
6823 if ((state = EC_EXEC_TAG()) == TAG_NONE) {
6824 val = vm_once_exec((VALUE)iseq);
6825 }
6826 EC_POP_TAG();
6827 if (state != TAG_NONE) {
6828 vm_once_clear((VALUE)is);
6829 EC_JUMP_TAG(ec, state);
6830 }
6831
6832 // TODO: confirm that it is shareable
6833 if (RB_FL_ABLE(val)) {
6834 RB_OBJ_SET_SHAREABLE(val);
6835 }
6836
6837 RB_OBJ_WRITE(CFP_ISEQ(ec->cfp), &is->once.value, val);
6838
6839 rbimpl_atomic_ptr_store((volatile void**)&is->once.running_thread, RUNNING_THREAD_ONCE_DONE, RBIMPL_ATOMIC_RELEASE); /* success */
6840 vm_once_broadcast(rb_ec_vm_ptr(ec));
6841 return val;
6842 }
6843 else if (running_th == th) {
6844 /* recursive once */
6845 return vm_once_exec((VALUE)iseq);
6846 }
6847 else {
6848 /* wait for the winner to finish */
6849 struct vm_once_wait_arg arg = { .vm = rb_ec_vm_ptr(ec), .is = is };
6850 rb_nogvl(vm_once_wait_no_gvl, &arg, vm_once_wait_ubf, arg.vm,
6851 RB_NOGVL_UBF_ASYNC_SAFE | RB_NOGVL_INTR_FAIL);
6852 RUBY_VM_CHECK_INTS(ec);
6853 goto again;
6854 }
6855}
6856
6857static OFFSET
6858vm_case_dispatch(CDHASH hash, OFFSET else_offset, VALUE key)
6859{
6860 switch (OBJ_BUILTIN_TYPE(key)) {
6861 case -1:
6862 case T_FLOAT:
6863 case T_SYMBOL:
6864 case T_BIGNUM:
6865 case T_STRING:
6866 if (BASIC_OP_UNREDEFINED_P(BOP_EQQ,
6867 SYMBOL_REDEFINED_OP_FLAG |
6868 INTEGER_REDEFINED_OP_FLAG |
6869 FLOAT_REDEFINED_OP_FLAG |
6870 NIL_REDEFINED_OP_FLAG |
6871 TRUE_REDEFINED_OP_FLAG |
6872 FALSE_REDEFINED_OP_FLAG |
6873 STRING_REDEFINED_OP_FLAG)) {
6874 st_data_t val;
6875 if (RB_FLOAT_TYPE_P(key)) {
6876 double kval = RFLOAT_VALUE(key);
6877 if (!isinf(kval) && modf(kval, &kval) == 0.0) {
6878 key = FIXABLE(kval) ? LONG2FIX((long)kval) : rb_dbl2big(kval);
6879 }
6880 }
6881 if (st_lookup(rb_imemo_cdhash_tbl(hash), key, &val)) {
6882 return (VALUE)val;
6883 }
6884 else {
6885 return else_offset;
6886 }
6887 }
6888 }
6889 return 0;
6890}
6891
6892NORETURN(static void
6893 vm_stack_consistency_error(const rb_execution_context_t *ec,
6894 const rb_control_frame_t *,
6895 const VALUE *));
6896static void
6897vm_stack_consistency_error(const rb_execution_context_t *ec,
6898 const rb_control_frame_t *cfp,
6899 const VALUE *bp)
6900{
6901 const ptrdiff_t nsp = VM_SP_CNT(ec, cfp->sp);
6902 const ptrdiff_t nbp = VM_SP_CNT(ec, bp);
6903 static const char stack_consistency_error[] =
6904 "Stack consistency error (sp: %"PRIdPTRDIFF", bp: %"PRIdPTRDIFF")";
6905#if defined RUBY_DEVEL
6906 VALUE mesg = rb_sprintf(stack_consistency_error, nsp, nbp);
6907 rb_str_cat_cstr(mesg, "\n");
6908 rb_str_append(mesg, rb_iseq_disasm(CFP_ISEQ(cfp)));
6909 rb_exc_fatal(rb_exc_new3(rb_eFatal, mesg));
6910#else
6911 rb_bug(stack_consistency_error, nsp, nbp);
6912#endif
6913}
6914
6915static VALUE
6916vm_opt_plus(VALUE recv, VALUE obj)
6917{
6918 if (FIXNUM_2_P(recv, obj) &&
6919 OP_UNREDEFINED_P(PLUS, INTEGER)) {
6920 return rb_fix_plus_fix(recv, obj);
6921 }
6922 else if (FLONUM_2_P(recv, obj) &&
6923 OP_UNREDEFINED_P(PLUS, FLOAT)) {
6924 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
6925 }
6926 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6927 return Qundef;
6928 }
6929 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6930 RBASIC_CLASS(obj) == rb_cFloat &&
6931 OP_UNREDEFINED_P(PLUS, FLOAT)) {
6932 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
6933 }
6934 else if (RBASIC_CLASS(recv) == rb_cString &&
6935 RBASIC_CLASS(obj) == rb_cString &&
6936 OP_UNREDEFINED_P(PLUS, STRING)) {
6937 return rb_str_opt_plus(recv, obj);
6938 }
6939 else if (RBASIC_CLASS(recv) == rb_cArray &&
6940 RBASIC_CLASS(obj) == rb_cArray &&
6941 OP_UNREDEFINED_P(PLUS, ARRAY)) {
6942 return rb_ary_plus(recv, obj);
6943 }
6944 else {
6945 return Qundef;
6946 }
6947}
6948
6949static VALUE
6950vm_opt_minus(VALUE recv, VALUE obj)
6951{
6952 if (FIXNUM_2_P(recv, obj) &&
6953 OP_UNREDEFINED_P(MINUS, INTEGER)) {
6954 return rb_fix_minus_fix(recv, obj);
6955 }
6956 else if (FLONUM_2_P(recv, obj) &&
6957 OP_UNREDEFINED_P(MINUS, FLOAT)) {
6958 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
6959 }
6960 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6961 return Qundef;
6962 }
6963 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6964 RBASIC_CLASS(obj) == rb_cFloat &&
6965 OP_UNREDEFINED_P(MINUS, FLOAT)) {
6966 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
6967 }
6968 else {
6969 return Qundef;
6970 }
6971}
6972
6973static VALUE
6974vm_opt_mult(VALUE recv, VALUE obj)
6975{
6976 if (FIXNUM_2_P(recv, obj) &&
6977 OP_UNREDEFINED_P(MULT, INTEGER)) {
6978 return rb_fix_mul_fix(recv, obj);
6979 }
6980 else if (FLONUM_2_P(recv, obj) &&
6981 OP_UNREDEFINED_P(MULT, FLOAT)) {
6982 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
6983 }
6984 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6985 return Qundef;
6986 }
6987 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6988 RBASIC_CLASS(obj) == rb_cFloat &&
6989 OP_UNREDEFINED_P(MULT, FLOAT)) {
6990 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
6991 }
6992 else {
6993 return Qundef;
6994 }
6995}
6996
6997static VALUE
6998vm_opt_div(VALUE recv, VALUE obj)
6999{
7000 if (FIXNUM_2_P(recv, obj) &&
7001 OP_UNREDEFINED_P(DIV, INTEGER)) {
7002 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_div_fix(recv, obj);
7003 }
7004 else if (FLONUM_2_P(recv, obj) &&
7005 OP_UNREDEFINED_P(DIV, FLOAT)) {
7006 return rb_flo_div_flo(recv, obj);
7007 }
7008 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7009 return Qundef;
7010 }
7011 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7012 RBASIC_CLASS(obj) == rb_cFloat &&
7013 OP_UNREDEFINED_P(DIV, FLOAT)) {
7014 return rb_flo_div_flo(recv, obj);
7015 }
7016 else {
7017 return Qundef;
7018 }
7019}
7020
7021static VALUE
7022vm_opt_mod(VALUE recv, VALUE obj)
7023{
7024 if (FIXNUM_2_P(recv, obj) &&
7025 OP_UNREDEFINED_P(MOD, INTEGER)) {
7026 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_mod_fix(recv, obj);
7027 }
7028 else if (FLONUM_2_P(recv, obj) &&
7029 OP_UNREDEFINED_P(MOD, FLOAT)) {
7030 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
7031 }
7032 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7033 return Qundef;
7034 }
7035 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7036 RBASIC_CLASS(obj) == rb_cFloat &&
7037 OP_UNREDEFINED_P(MOD, FLOAT)) {
7038 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
7039 }
7040 else {
7041 return Qundef;
7042 }
7043}
7044
7045static VALUE
7046vm_opt_neq(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, CALL_DATA cd_eq, VALUE recv, VALUE obj)
7047{
7048 if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_not_equal)) {
7049 VALUE val = opt_equality(reg_cfp, recv, obj, cd_eq);
7050
7051 if (!UNDEF_P(val)) {
7052 return RBOOL(!RTEST(val));
7053 }
7054 }
7055
7056 return Qundef;
7057}
7058
7059static VALUE
7060vm_opt_lt(VALUE recv, VALUE obj)
7061{
7062 if (FIXNUM_2_P(recv, obj) &&
7063 OP_UNREDEFINED_P(LT, INTEGER)) {
7064 return RBOOL((SIGNED_VALUE)recv < (SIGNED_VALUE)obj);
7065 }
7066 else if (FLONUM_2_P(recv, obj) &&
7067 OP_UNREDEFINED_P(LT, FLOAT)) {
7068 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
7069 }
7070 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7071 return Qundef;
7072 }
7073 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7074 RBASIC_CLASS(obj) == rb_cFloat &&
7075 OP_UNREDEFINED_P(LT, FLOAT)) {
7076 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
7077 }
7078 else {
7079 return Qundef;
7080 }
7081}
7082
7083static VALUE
7084vm_opt_le(VALUE recv, VALUE obj)
7085{
7086 if (FIXNUM_2_P(recv, obj) &&
7087 OP_UNREDEFINED_P(LE, INTEGER)) {
7088 return RBOOL((SIGNED_VALUE)recv <= (SIGNED_VALUE)obj);
7089 }
7090 else if (FLONUM_2_P(recv, obj) &&
7091 OP_UNREDEFINED_P(LE, FLOAT)) {
7092 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
7093 }
7094 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7095 return Qundef;
7096 }
7097 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7098 RBASIC_CLASS(obj) == rb_cFloat &&
7099 OP_UNREDEFINED_P(LE, FLOAT)) {
7100 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
7101 }
7102 else {
7103 return Qundef;
7104 }
7105}
7106
7107static VALUE
7108vm_opt_gt(VALUE recv, VALUE obj)
7109{
7110 if (FIXNUM_2_P(recv, obj) &&
7111 OP_UNREDEFINED_P(GT, INTEGER)) {
7112 return RBOOL((SIGNED_VALUE)recv > (SIGNED_VALUE)obj);
7113 }
7114 else if (FLONUM_2_P(recv, obj) &&
7115 OP_UNREDEFINED_P(GT, FLOAT)) {
7116 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
7117 }
7118 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7119 return Qundef;
7120 }
7121 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7122 RBASIC_CLASS(obj) == rb_cFloat &&
7123 OP_UNREDEFINED_P(GT, FLOAT)) {
7124 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
7125 }
7126 else {
7127 return Qundef;
7128 }
7129}
7130
7131static VALUE
7132vm_opt_ge(VALUE recv, VALUE obj)
7133{
7134 if (FIXNUM_2_P(recv, obj) &&
7135 OP_UNREDEFINED_P(GE, INTEGER)) {
7136 return RBOOL((SIGNED_VALUE)recv >= (SIGNED_VALUE)obj);
7137 }
7138 else if (FLONUM_2_P(recv, obj) &&
7139 OP_UNREDEFINED_P(GE, FLOAT)) {
7140 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
7141 }
7142 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7143 return Qundef;
7144 }
7145 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7146 RBASIC_CLASS(obj) == rb_cFloat &&
7147 OP_UNREDEFINED_P(GE, FLOAT)) {
7148 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
7149 }
7150 else {
7151 return Qundef;
7152 }
7153}
7154
7155
7156static VALUE
7157vm_opt_ltlt(VALUE recv, VALUE obj)
7158{
7159 if (SPECIAL_CONST_P(recv)) {
7160 return Qundef;
7161 }
7162 else if (RBASIC_CLASS(recv) == rb_cString &&
7163 OP_UNREDEFINED_P(LTLT, STRING)) {
7164 if (LIKELY(RB_TYPE_P(obj, T_STRING))) {
7165 return rb_str_buf_append(recv, obj);
7166 }
7167 else {
7168 return rb_str_concat(recv, obj);
7169 }
7170 }
7171 else if (RBASIC_CLASS(recv) == rb_cArray &&
7172 OP_UNREDEFINED_P(LTLT, ARRAY)) {
7173 return rb_ary_push(recv, obj);
7174 }
7175 else {
7176 return Qundef;
7177 }
7178}
7179
7180static VALUE
7181vm_opt_and(VALUE recv, VALUE obj)
7182{
7183 // If recv and obj are both fixnums, then the bottom tag bit
7184 // will be 1 on both. 1 & 1 == 1, so the result value will also
7185 // be a fixnum. If either side is *not* a fixnum, then the tag bit
7186 // will be 0, and we return Qundef.
7187 VALUE ret = ((SIGNED_VALUE) recv) & ((SIGNED_VALUE) obj);
7188
7189 if (FIXNUM_P(ret) &&
7190 OP_UNREDEFINED_P(AND, INTEGER)) {
7191 return ret;
7192 }
7193 else {
7194 return Qundef;
7195 }
7196}
7197
7198static VALUE
7199vm_opt_or(VALUE recv, VALUE obj)
7200{
7201 if (FIXNUM_2_P(recv, obj) &&
7202 OP_UNREDEFINED_P(OR, INTEGER)) {
7203 return recv | obj;
7204 }
7205 else {
7206 return Qundef;
7207 }
7208}
7209
7210static VALUE
7211vm_opt_aref(VALUE recv, VALUE obj)
7212{
7213 if (SPECIAL_CONST_P(recv)) {
7214 if (FIXNUM_2_P(recv, obj) &&
7215 OP_UNREDEFINED_P(AREF, INTEGER)) {
7216 return rb_fix_aref(recv, obj);
7217 }
7218 return Qundef;
7219 }
7220 else if (RBASIC_CLASS(recv) == rb_cArray &&
7221 OP_UNREDEFINED_P(AREF, ARRAY)) {
7222 if (FIXNUM_P(obj)) {
7223 return rb_ary_entry_internal(recv, FIX2LONG(obj));
7224 }
7225 else {
7226 return rb_ary_aref1(recv, obj);
7227 }
7228 }
7229 else if (RBASIC_CLASS(recv) == rb_cHash &&
7230 OP_UNREDEFINED_P(AREF, HASH)) {
7231 return rb_hash_aref(recv, obj);
7232 }
7233 else {
7234 return Qundef;
7235 }
7236}
7237
7238static VALUE
7239vm_opt_aset(VALUE recv, VALUE obj, VALUE set)
7240{
7241 if (SPECIAL_CONST_P(recv)) {
7242 return Qundef;
7243 }
7244 else if (RBASIC_CLASS(recv) == rb_cArray &&
7245 OP_UNREDEFINED_P(ASET, ARRAY) &&
7246 FIXNUM_P(obj)) {
7247 rb_ary_store(recv, FIX2LONG(obj), set);
7248 return set;
7249 }
7250 else if (RBASIC_CLASS(recv) == rb_cHash &&
7251 OP_UNREDEFINED_P(ASET, HASH)) {
7252 rb_hash_aset(recv, obj, set);
7253 return set;
7254 }
7255 else {
7256 return Qundef;
7257 }
7258}
7259
7260static VALUE
7261vm_opt_length(VALUE recv, int bop)
7262{
7263 if (SPECIAL_CONST_P(recv)) {
7264 return Qundef;
7265 }
7266 else if (RBASIC_CLASS(recv) == rb_cString &&
7267 BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
7268 if (bop == BOP_EMPTY_P) {
7269 return LONG2NUM(RSTRING_LEN(recv));
7270 }
7271 else {
7272 return rb_str_length(recv);
7273 }
7274 }
7275 else if (RBASIC_CLASS(recv) == rb_cArray &&
7276 BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
7277 return LONG2NUM(RARRAY_LEN(recv));
7278 }
7279 else if (RBASIC_CLASS(recv) == rb_cHash &&
7280 BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
7281 return INT2FIX(RHASH_SIZE(recv));
7282 }
7283 else {
7284 return Qundef;
7285 }
7286}
7287
7288static VALUE
7289vm_opt_empty_p(VALUE recv)
7290{
7291 switch (vm_opt_length(recv, BOP_EMPTY_P)) {
7292 case Qundef: return Qundef;
7293 case INT2FIX(0): return Qtrue;
7294 default: return Qfalse;
7295 }
7296}
7297
7298VALUE rb_false(VALUE obj);
7299
7300static VALUE
7301vm_opt_nil_p(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv)
7302{
7303 if (NIL_P(recv) &&
7304 OP_UNREDEFINED_P(NIL_P, NIL)) {
7305 return Qtrue;
7306 }
7307 else if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_false)) {
7308 return Qfalse;
7309 }
7310 else {
7311 return Qundef;
7312 }
7313}
7314
7315static VALUE
7316fix_succ(VALUE x)
7317{
7318 switch (x) {
7319 case ~0UL:
7320 /* 0xFFFF_FFFF == INT2FIX(-1)
7321 * `-1.succ` is of course 0. */
7322 return INT2FIX(0);
7323 case RSHIFT(~0UL, 1):
7324 /* 0x7FFF_FFFF == LONG2FIX(0x3FFF_FFFF)
7325 * 0x3FFF_FFFF + 1 == 0x4000_0000, which is a Bignum. */
7326 return rb_uint2big(1UL << (SIZEOF_LONG * CHAR_BIT - 2));
7327 default:
7328 /* LONG2FIX(FIX2LONG(x)+FIX2LONG(y))
7329 * == ((lx*2+1)/2 + (ly*2+1)/2)*2+1
7330 * == lx*2 + ly*2 + 1
7331 * == (lx*2+1) + (ly*2+1) - 1
7332 * == x + y - 1
7333 *
7334 * Here, if we put y := INT2FIX(1):
7335 *
7336 * == x + INT2FIX(1) - 1
7337 * == x + 2 .
7338 */
7339 return x + 2;
7340 }
7341}
7342
7343static VALUE
7344vm_opt_succ(VALUE recv)
7345{
7346 if (FIXNUM_P(recv) &&
7347 OP_UNREDEFINED_P(SUCC, INTEGER)) {
7348 return fix_succ(recv);
7349 }
7350 else if (SPECIAL_CONST_P(recv)) {
7351 return Qundef;
7352 }
7353 else if (RBASIC_CLASS(recv) == rb_cString &&
7354 OP_UNREDEFINED_P(SUCC, STRING)) {
7355 return rb_str_succ(recv);
7356 }
7357 else {
7358 return Qundef;
7359 }
7360}
7361
7362static VALUE
7363vm_opt_not(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv)
7364{
7365 if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_not)) {
7366 return RBOOL(!RTEST(recv));
7367 }
7368 else {
7369 return Qundef;
7370 }
7371}
7372
7373static VALUE
7374vm_opt_regexpmatch2(VALUE recv, VALUE obj)
7375{
7376 if (SPECIAL_CONST_P(recv)) {
7377 return Qundef;
7378 }
7379 else if (RBASIC_CLASS(recv) == rb_cString &&
7380 CLASS_OF(obj) == rb_cRegexp &&
7381 OP_UNREDEFINED_P(MATCH, STRING)) {
7382 return rb_reg_match(obj, recv);
7383 }
7384 else if (RBASIC_CLASS(recv) == rb_cRegexp &&
7385 OP_UNREDEFINED_P(MATCH, REGEXP)) {
7386 return rb_reg_match(recv, obj);
7387 }
7388 else {
7389 return Qundef;
7390 }
7391}
7392
7393#undef EQ_UNREDEFINED_P
7394#undef OP_UNREDEFINED_P
7395
7396rb_event_flag_t rb_iseq_event_flags(const rb_iseq_t *iseq, size_t pos);
7397
7398NOINLINE(static void vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp));
7399
7400static inline void
7401vm_trace_hook(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE *pc,
7402 rb_event_flag_t pc_events, rb_event_flag_t target_event,
7403 rb_hook_list_t *global_hooks, rb_hook_list_t *local_hooks, VALUE val)
7404{
7405 rb_event_flag_t event = pc_events & target_event;
7406 VALUE self = GET_SELF();
7407
7408 VM_ASSERT(rb_popcount64((uint64_t)event) == 1);
7409
7410 if (local_hooks) local_hooks->running++; // make sure they don't get deleted while global hooks run
7411
7412 if (event & global_hooks->events) {
7413 /* increment PC because source line is calculated with PC-1 */
7414 reg_cfp->pc++;
7415 vm_dtrace(event, ec);
7416 rb_exec_event_hook_orig(ec, global_hooks, event, self, 0, 0, 0 , val, 0);
7417 reg_cfp->pc--;
7418 }
7419
7420 if (local_hooks) local_hooks->running--;
7421 if (local_hooks != NULL) {
7422 if (event & local_hooks->events) {
7423 /* increment PC because source line is calculated with PC-1 */
7424 reg_cfp->pc++;
7425 rb_exec_event_hook_orig(ec, local_hooks, event, self, 0, 0, 0 , val, 0);
7426 reg_cfp->pc--;
7427 }
7428 }
7429}
7430
7431/* Re-fetch the iseq-local hook list before each event: a hook fired by an
7432 * earlier event at this pc can disable the last targeted TracePoint on this
7433 * iseq, which frees local_hooks. Reload from its stable source (the ractor's
7434 * targeted-hooks table) so we never dereference a dangling pointer. */
7435#define VM_TRACE_HOOK(target_event, val) do { \
7436 if ((pc_events & (target_event)) & enabled_flags) { \
7437 if (local_hooks_cnt > 0) local_hooks = rb_iseq_local_hooks(iseq, r, false); \
7438 vm_trace_hook(ec, reg_cfp, pc, pc_events, (target_event), global_hooks, local_hooks, (val)); \
7439 } \
7440} while (0)
7441
7442static VALUE
7443rescue_errinfo(rb_execution_context_t *ec, rb_control_frame_t *cfp)
7444{
7445 VM_ASSERT(VM_FRAME_RUBYFRAME_P(cfp));
7446 VM_ASSERT(ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_RESCUE);
7447 return cfp->ep[VM_ENV_INDEX_LAST_LVAR];
7448}
7449
7450static void
7451vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp)
7452{
7453 const VALUE *pc = reg_cfp->pc;
7454 rb_ractor_t *r = rb_ec_ractor_ptr(ec);
7455 rb_event_flag_t enabled_flags = r->pub.hooks.events & ISEQ_TRACE_EVENTS;
7456 rb_event_flag_t ractor_events = enabled_flags;
7457
7458 if (enabled_flags == 0 && rb_ractor_targeted_hooks_cnt(r) == 0) {
7459 return;
7460 }
7461 else {
7462 const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp);
7463 size_t pos = pc - ISEQ_BODY(iseq)->iseq_encoded;
7464 rb_event_flag_t pc_events = rb_iseq_event_flags(iseq, pos);
7465 unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt;
7466 rb_hook_list_t *local_hooks = NULL;
7467 if (RB_UNLIKELY(local_hooks_cnt > 0)) {
7468 st_data_t val;
7469 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)iseq, &val)) {
7470 local_hooks = (rb_hook_list_t*)val;
7471 }
7472 }
7473 rb_event_flag_t iseq_local_events = local_hooks != NULL ? local_hooks->events : 0;
7474
7475 rb_hook_list_t *bmethod_local_hooks = NULL;
7476 rb_event_flag_t bmethod_local_events = 0;
7477 unsigned int bmethod_hooks_cnt = 0;
7478 rb_method_definition_t *bmethod_def = NULL;
7479 const bool bmethod_frame = VM_FRAME_BMETHOD_P(reg_cfp);
7480 enabled_flags |= iseq_local_events;
7481
7482 VM_ASSERT((iseq_local_events & ~ISEQ_TRACE_EVENTS) == 0);
7483
7484 if (bmethod_frame) {
7485 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
7486 VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD);
7487 bmethod_def = me->def;
7488 bmethod_hooks_cnt = me->def->body.bmethod.local_hooks_cnt;
7489 if (RB_UNLIKELY(bmethod_hooks_cnt > 0)) {
7490 st_data_t val;
7491 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)me->def, &val)) {
7492 bmethod_local_hooks = (rb_hook_list_t*)val;
7493 }
7494 if (bmethod_local_hooks) {
7495 bmethod_local_events = bmethod_local_hooks->events;
7496 }
7497 }
7498 }
7499
7500 if ((pc_events & enabled_flags) == 0 && !bmethod_frame) {
7501#if 0
7502 /* disable trace */
7503 /* TODO: incomplete */
7504 rb_iseq_trace_set(iseq, vm_event_flags & ISEQ_TRACE_EVENTS);
7505#else
7506 /* do not disable trace because of performance problem
7507 * (re-enable overhead)
7508 */
7509#endif
7510 return;
7511 }
7512 else if (ec->trace_arg != NULL) {
7513 /* already tracing */
7514 return;
7515 }
7516 else {
7517 rb_hook_list_t *global_hooks = rb_ec_ractor_hooks(ec);
7518 /* Note, not considering iseq local events here since the same
7519 * iseq could be used in multiple bmethods. */
7520 rb_event_flag_t bmethod_events = ractor_events | bmethod_local_events;
7521
7522 if (0) {
7523 VALUE path = rb_iseq_path(iseq);
7524 VALUE label = rb_iseq_label(iseq);
7525 ruby_debug_printf("vm_trace>>%4d (%4x) - %.*s:%d %.*s\n",
7526 (int)pos,
7527 (int)pc_events,
7528 RSTRING_LENINT(path), RSTRING_PTR(path),
7529 (int)rb_iseq_line_no(iseq, pos),
7530 RSTRING_LENINT(label), RSTRING_PTR(label));
7531 }
7532 VM_ASSERT(reg_cfp->pc == pc);
7533 VM_ASSERT(pc_events != 0);
7534
7535 /* check traces */
7536 if ((pc_events & RUBY_EVENT_B_CALL) && bmethod_frame && (bmethod_events & RUBY_EVENT_CALL)) {
7537 /* b_call instruction running as a method. Fire call event. */
7538 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_CALL, RUBY_EVENT_CALL, global_hooks, bmethod_local_hooks, Qundef);
7539 }
7540 VM_TRACE_HOOK(RUBY_EVENT_CLASS | RUBY_EVENT_CALL | RUBY_EVENT_B_CALL, Qundef);
7541 VM_TRACE_HOOK(RUBY_EVENT_RESCUE, rescue_errinfo(ec, reg_cfp));
7542 VM_TRACE_HOOK(RUBY_EVENT_LINE, Qundef);
7543 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_LINE, Qundef);
7544 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_BRANCH, Qundef);
7545 VM_TRACE_HOOK(RUBY_EVENT_END | RUBY_EVENT_RETURN | RUBY_EVENT_B_RETURN, TOPN(0));
7546 if ((pc_events & RUBY_EVENT_B_RETURN) && bmethod_frame && (bmethod_events & RUBY_EVENT_RETURN)) {
7547 /* b_return instruction running as a method. Fire return event. */
7548 /* An earlier event's hook at this pc may have freed bmethod_local_hooks
7549 * (see VM_TRACE_HOOK); reload it from its stable source. */
7550 if (bmethod_hooks_cnt > 0) bmethod_local_hooks = rb_method_def_local_hooks(bmethod_def, r, false);
7551 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_RETURN, RUBY_EVENT_RETURN, global_hooks, bmethod_local_hooks, TOPN(0));
7552 }
7553 }
7554 }
7555}
7556#undef VM_TRACE_HOOK
7557
7558#if VM_CHECK_MODE > 0
7559NORETURN( NOINLINE( COLDFUNC
7560void rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)));
7561
7562void
7563Init_vm_stack_canary(void)
7564{
7565 /* This has to be called _after_ our PRNG is properly set up. */
7566 int n = ruby_fill_random_bytes(&vm_stack_canary, sizeof vm_stack_canary, false);
7567 /* Make it a large positive Fixnum. Arguments a block does not take
7568 * stay at the stack top, where vm_push_frame checks the canary. */
7569 vm_stack_canary >>= 2;
7570 vm_stack_canary |= (VALUE)1 << (SIZEOF_VALUE * CHAR_BIT - 2);
7571 vm_stack_canary |= 0x01; // valid VALUE (Fixnum)
7572
7573 vm_stack_canary_was_born = true;
7574 VM_ASSERT(n == 0);
7575}
7576
7577void
7578rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)
7579{
7580 /* Because a method has already been called, why not call
7581 * another one. */
7582 const char *insn = rb_insns_name(i);
7583 VALUE inspection = rb_inspect(c);
7584 const char *str = StringValueCStr(inspection);
7585
7586 rb_bug("dead canary found at %s: %s", insn, str);
7587}
7588
7589#else
7590void Init_vm_stack_canary(void) { /* nothing to do */ }
7591#endif
7592
7593
7594/* a part of the following code is generated by this ruby script:
7595
759616.times{|i|
7597 typedef_args = (0...i).map{|j| "VALUE v#{j+1}"}.join(", ")
7598 typedef_args.prepend(", ") if i != 0
7599 call_args = (0...i).map{|j| "argv[#{j}]"}.join(", ")
7600 call_args.prepend(", ") if i != 0
7601 puts %Q{
7602static VALUE
7603builtin_invoker#{i}(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7604{
7605 typedef VALUE (*rb_invoke_funcptr#{i}_t)(rb_execution_context_t *ec, VALUE self#{typedef_args});
7606 return (*(rb_invoke_funcptr#{i}_t)funcptr)(ec, self#{call_args});
7607}}
7608}
7609
7610puts
7611puts "static VALUE (* const cfunc_invokers[])(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr) = {"
761216.times{|i|
7613 puts " builtin_invoker#{i},"
7614}
7615puts "};"
7616*/
7617
7618static VALUE
7619builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7620{
7621 typedef VALUE (*rb_invoke_funcptr0_t)(rb_execution_context_t *ec, VALUE self);
7622 return (*(rb_invoke_funcptr0_t)funcptr)(ec, self);
7623}
7624
7625static VALUE
7626builtin_invoker1(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7627{
7628 typedef VALUE (*rb_invoke_funcptr1_t)(rb_execution_context_t *ec, VALUE self, VALUE v1);
7629 return (*(rb_invoke_funcptr1_t)funcptr)(ec, self, argv[0]);
7630}
7631
7632static VALUE
7633builtin_invoker2(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7634{
7635 typedef VALUE (*rb_invoke_funcptr2_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2);
7636 return (*(rb_invoke_funcptr2_t)funcptr)(ec, self, argv[0], argv[1]);
7637}
7638
7639static VALUE
7640builtin_invoker3(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7641{
7642 typedef VALUE (*rb_invoke_funcptr3_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3);
7643 return (*(rb_invoke_funcptr3_t)funcptr)(ec, self, argv[0], argv[1], argv[2]);
7644}
7645
7646static VALUE
7647builtin_invoker4(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7648{
7649 typedef VALUE (*rb_invoke_funcptr4_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4);
7650 return (*(rb_invoke_funcptr4_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3]);
7651}
7652
7653static VALUE
7654builtin_invoker5(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7655{
7656 typedef VALUE (*rb_invoke_funcptr5_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5);
7657 return (*(rb_invoke_funcptr5_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4]);
7658}
7659
7660static VALUE
7661builtin_invoker6(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7662{
7663 typedef VALUE (*rb_invoke_funcptr6_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6);
7664 return (*(rb_invoke_funcptr6_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
7665}
7666
7667static VALUE
7668builtin_invoker7(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7669{
7670 typedef VALUE (*rb_invoke_funcptr7_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7);
7671 return (*(rb_invoke_funcptr7_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
7672}
7673
7674static VALUE
7675builtin_invoker8(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7676{
7677 typedef VALUE (*rb_invoke_funcptr8_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8);
7678 return (*(rb_invoke_funcptr8_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
7679}
7680
7681static VALUE
7682builtin_invoker9(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7683{
7684 typedef VALUE (*rb_invoke_funcptr9_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9);
7685 return (*(rb_invoke_funcptr9_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
7686}
7687
7688static VALUE
7689builtin_invoker10(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7690{
7691 typedef VALUE (*rb_invoke_funcptr10_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10);
7692 return (*(rb_invoke_funcptr10_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
7693}
7694
7695static VALUE
7696builtin_invoker11(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7697{
7698 typedef VALUE (*rb_invoke_funcptr11_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11);
7699 return (*(rb_invoke_funcptr11_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
7700}
7701
7702static VALUE
7703builtin_invoker12(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7704{
7705 typedef VALUE (*rb_invoke_funcptr12_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12);
7706 return (*(rb_invoke_funcptr12_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
7707}
7708
7709static VALUE
7710builtin_invoker13(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7711{
7712 typedef VALUE (*rb_invoke_funcptr13_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13);
7713 return (*(rb_invoke_funcptr13_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
7714}
7715
7716static VALUE
7717builtin_invoker14(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7718{
7719 typedef VALUE (*rb_invoke_funcptr14_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14);
7720 return (*(rb_invoke_funcptr14_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
7721}
7722
7723static VALUE
7724builtin_invoker15(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7725{
7726 typedef VALUE (*rb_invoke_funcptr15_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14, VALUE v15);
7727 return (*(rb_invoke_funcptr15_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
7728}
7729
7730typedef VALUE (*builtin_invoker)(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
7731
7732static builtin_invoker
7733lookup_builtin_invoker(int argc)
7734{
7735 static const builtin_invoker invokers[] = {
7736 builtin_invoker0,
7737 builtin_invoker1,
7738 builtin_invoker2,
7739 builtin_invoker3,
7740 builtin_invoker4,
7741 builtin_invoker5,
7742 builtin_invoker6,
7743 builtin_invoker7,
7744 builtin_invoker8,
7745 builtin_invoker9,
7746 builtin_invoker10,
7747 builtin_invoker11,
7748 builtin_invoker12,
7749 builtin_invoker13,
7750 builtin_invoker14,
7751 builtin_invoker15,
7752 };
7753
7754 return invokers[argc];
7755}
7756
7757static inline VALUE
7758invoke_bf(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7759{
7760 const bool canary_p = ISEQ_BODY(CFP_ISEQ(reg_cfp))->builtin_attrs & BUILTIN_ATTR_LEAF; // Verify an assumption of `Primitive.attr! :leaf`
7761 SETUP_CANARY(canary_p);
7762 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
7763 VALUE ret = (*lookup_builtin_invoker(bf->argc))(ec, reg_cfp->self, argv, func_ptr);
7764 CHECK_CANARY(canary_p, BIN(invokebuiltin));
7765 return ret;
7766}
7767
7768static VALUE
7769vm_invoke_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7770{
7771 return invoke_bf(ec, cfp, bf, argv);
7772}
7773
7774static VALUE
7775vm_invoke_builtin_delegate(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function *bf, unsigned int start_index)
7776{
7777 if (0) { // debug print
7778 fputs("vm_invoke_builtin_delegate: passing -> ", stderr);
7779 for (int i=0; i<bf->argc; i++) {
7780 ruby_debug_printf(":%s ", rb_id2name(ISEQ_BODY(CFP_ISEQ(cfp))->local_table[i+start_index]));
7781 }
7782 ruby_debug_printf("\n" "%s %s(%d):%p\n", RUBY_FUNCTION_NAME_STRING, bf->name, bf->argc,
7783 (void *)(uintptr_t)bf->func_ptr);
7784 }
7785
7786 if (bf->argc == 0) {
7787 return invoke_bf(ec, cfp, bf, NULL);
7788 }
7789 else {
7790 const VALUE *argv = cfp->ep - ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size - VM_ENV_DATA_SIZE + 1 + start_index;
7791 return invoke_bf(ec, cfp, bf, argv);
7792 }
7793}
7794
7795// for __builtin_inline!()
7796
7797VALUE
7798rb_vm_lvar_exposed(rb_execution_context_t *ec, int index)
7799{
7800 const rb_control_frame_t *cfp = ec->cfp;
7801 return cfp->ep[index];
7802}
RUBY_ASSERT
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:219
RUBY_EVENT_END
#define RUBY_EVENT_END
Encountered an end of a class clause.
Definition event.h:40
RUBY_EVENT_C_CALL
#define RUBY_EVENT_C_CALL
A method, written in C, is called.
Definition event.h:43
RUBY_EVENT_B_RETURN
#define RUBY_EVENT_B_RETURN
Encountered a next statement.
Definition event.h:56
RUBY_EVENT_CLASS
#define RUBY_EVENT_CLASS
Encountered a new class.
Definition event.h:39
RUBY_EVENT_LINE
#define RUBY_EVENT_LINE
Encountered a new line.
Definition event.h:38
RUBY_EVENT_RETURN
#define RUBY_EVENT_RETURN
Encountered a return statement.
Definition event.h:42
RUBY_EVENT_C_RETURN
#define RUBY_EVENT_C_RETURN
Return from a method, written in C.
Definition event.h:44
RUBY_EVENT_B_CALL
#define RUBY_EVENT_B_CALL
Encountered an yield statement.
Definition event.h:55
rb_event_flag_t
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
RUBY_EVENT_CALL
#define RUBY_EVENT_CALL
A method, written in Ruby, is called.
Definition event.h:41
RUBY_EVENT_RESCUE
#define RUBY_EVENT_RESCUE
Encountered a rescue statement.
Definition event.h:61
RB_FL_TEST_RAW
static VALUE RB_FL_TEST_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_TEST().
Definition fl_type.h:407
RB_FL_ABLE
static bool RB_FL_ABLE(VALUE obj)
Checks if the object is flaggable.
Definition fl_type.h:384
RUBY_FL_SHAREABLE
@ RUBY_FL_SHAREABLE
This flag has something to do with Ractor.
Definition fl_type.h:253
rb_singleton_class
VALUE rb_singleton_class(VALUE obj)
Finds or creates the singleton class of the passed object.
Definition class.c:3051
rb_module_new
VALUE rb_module_new(void)
Creates a new, anonymous module.
Definition class.c:1661
rb_class_inherited
VALUE rb_class_inherited(VALUE super, VALUE klass)
Calls Class::inherited.
Definition class.c:1561
rb_define_class_id
VALUE rb_define_class_id(ID id, VALUE super)
This is a very badly designed API that creates an anonymous class.
Definition class.c:1540
TYPE
#define TYPE(_)
Old name of rb_type.
Definition value_type.h:108
RFLOAT_VALUE
#define RFLOAT_VALUE
Old name of rb_float_value.
Definition double.h:28
T_STRING
#define T_STRING
Old name of RUBY_T_STRING.
Definition value_type.h:78
Qundef
#define Qundef
Old name of RUBY_Qundef.
Definition special_consts.h:62
INT2FIX
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
T_NIL
#define T_NIL
Old name of RUBY_T_NIL.
Definition value_type.h:72
T_FLOAT
#define T_FLOAT
Old name of RUBY_T_FLOAT.
Definition value_type.h:64
T_IMEMO
#define T_IMEMO
Old name of RUBY_T_IMEMO.
Definition value_type.h:67
ID2SYM
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
T_BIGNUM
#define T_BIGNUM
Old name of RUBY_T_BIGNUM.
Definition value_type.h:57
SPECIAL_CONST_P
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
Definition special_consts.h:56
T_STRUCT
#define T_STRUCT
Old name of RUBY_T_STRUCT.
Definition value_type.h:79
T_FIXNUM
#define T_FIXNUM
Old name of RUBY_T_FIXNUM.
Definition value_type.h:63
UNREACHABLE_RETURN
#define UNREACHABLE_RETURN
Old name of RBIMPL_UNREACHABLE_RETURN.
Definition assume.h:29
SYM2ID
#define SYM2ID
Old name of RB_SYM2ID.
Definition symbol.h:45
CLASS_OF
#define CLASS_OF
Old name of rb_class_of.
Definition globals.h:205
T_NONE
#define T_NONE
Old name of RUBY_T_NONE.
Definition value_type.h:74
rb_ary_new4
#define rb_ary_new4
Old name of rb_ary_new_from_values.
Definition array.h:659
FIXABLE
#define FIXABLE
Old name of RB_FIXABLE.
Definition fixnum.h:25
LONG2FIX
#define LONG2FIX
Old name of RB_INT2FIX.
Definition long.h:49
FIX2INT
#define FIX2INT
Old name of RB_FIX2INT.
Definition int.h:41
T_MODULE
#define T_MODULE
Old name of RUBY_T_MODULE.
Definition value_type.h:70
STATIC_SYM_P
#define STATIC_SYM_P
Old name of RB_STATIC_SYM_P.
Definition special_consts.h:57
ASSUME
#define ASSUME
Old name of RBIMPL_ASSUME.
Definition assume.h:27
FIX2ULONG
#define FIX2ULONG
Old name of RB_FIX2ULONG.
Definition long.h:47
T_TRUE
#define T_TRUE
Old name of RUBY_T_TRUE.
Definition value_type.h:81
T_ICLASS
#define T_ICLASS
Old name of RUBY_T_ICLASS.
Definition value_type.h:66
T_HASH
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
rb_ary_new3
#define rb_ary_new3
Old name of rb_ary_new_from_args.
Definition array.h:658
LONG2NUM
#define LONG2NUM
Old name of RB_LONG2NUM.
Definition long.h:50
rb_exc_new3
#define rb_exc_new3
Old name of rb_exc_new_str.
Definition error.h:38
T_FALSE
#define T_FALSE
Old name of RUBY_T_FALSE.
Definition value_type.h:61
Qtrue
#define Qtrue
Old name of RUBY_Qtrue.
Definition special_consts.h:61
Qnil
#define Qnil
Old name of RUBY_Qnil.
Definition special_consts.h:60
Qfalse
#define Qfalse
Old name of RUBY_Qfalse.
Definition special_consts.h:59
FIX2LONG
#define FIX2LONG
Old name of RB_FIX2LONG.
Definition long.h:46
T_ARRAY
#define T_ARRAY
Old name of RUBY_T_ARRAY.
Definition value_type.h:56
T_OBJECT
#define T_OBJECT
Old name of RUBY_T_OBJECT.
Definition value_type.h:75
NIL_P
#define NIL_P
Old name of RB_NIL_P.
Definition special_consts.h:55
T_SYMBOL
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition value_type.h:80
DBL2NUM
#define DBL2NUM
Old name of rb_float_new.
Definition double.h:29
T_CLASS
#define T_CLASS
Old name of RUBY_T_CLASS.
Definition value_type.h:58
BUILTIN_TYPE
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
Definition value_type.h:85
FL_TEST
#define FL_TEST
Old name of RB_FL_TEST.
Definition fl_type.h:127
FIXNUM_P
#define FIXNUM_P
Old name of RB_FIXNUM_P.
Definition special_consts.h:53
FL_USHIFT
#define FL_USHIFT
Old name of RUBY_FL_USHIFT.
Definition fl_type.h:67
FL_SET_RAW
#define FL_SET_RAW
Old name of RB_FL_SET_RAW.
Definition fl_type.h:126
SYMBOL_P
#define SYMBOL_P
Old name of RB_SYMBOL_P.
Definition value_type.h:88
rb_exc_raise
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
Definition eval.c:678
ruby_verbose
#define ruby_verbose
This variable controls whether the interpreter is in debug mode.
Definition error.h:476
rb_eTypeError
VALUE rb_eTypeError
TypeError exception.
Definition error.c:1473
rb_eFatal
VALUE rb_eFatal
fatal exception.
Definition error.c:1469
rb_eNoMethodError
VALUE rb_eNoMethodError
NoMethodError exception.
Definition error.c:1481
rb_exc_fatal
void rb_exc_fatal(VALUE mesg)
Raises a fatal error in the current thread.
Definition eval.c:691
rb_eRuntimeError
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1471
rb_warn
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:468
rb_error_frozen_object
void rb_error_frozen_object(VALUE frozen_obj)
Identical to rb_error_frozen(), except it takes arbitrary Ruby object instead of C's string.
Definition error.c:4336
rb_exc_new_str
VALUE rb_exc_new_str(VALUE etype, VALUE str)
Identical to rb_exc_new_cstr(), except it takes a Ruby's string instead of C's.
Definition error.c:1524
RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK
@ RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK
Warning is for checking unused block strictly.
Definition error.h:57
rb_cClass
VALUE rb_cClass
Class class.
Definition object.c:62
rb_cArray
VALUE rb_cArray
Array class.
rb_cObject
VALUE rb_cObject
Object class.
Definition object.c:60
rb_obj_alloc
VALUE rb_obj_alloc(VALUE klass)
Allocates an instance of the given class.
Definition object.c:2252
rb_cRegexp
VALUE rb_cRegexp
Regexp class.
Definition re.c:2828
rb_obj_frozen_p
VALUE rb_obj_frozen_p(VALUE obj)
Same as RB_OBJ_FROZEN(), but returns Qtrue/Qfalse instead of #bool.
Definition object.c:1316
rb_cHash
VALUE rb_cHash
Hash class.
Definition hash.c:123
rb_obj_class
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
Definition object.c:234
rb_inspect
VALUE rb_inspect(VALUE obj)
Generates a human-readable textual representation of the given object.
Definition object.c:669
rb_cBasicObject
VALUE rb_cBasicObject
BasicObject class.
Definition object.c:58
rb_cModule
VALUE rb_cModule
Module class.
Definition object.c:61
rb_class_real
VALUE rb_class_real(VALUE klass)
Finds a "real" class.
Definition object.c:225
rb_obj_is_kind_of
VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass)
Queries if the given object is an instance (of possibly descendants) of the given class.
Definition object.c:906
rb_cFloat
VALUE rb_cFloat
Float class.
Definition numeric.c:201
rb_cProc
VALUE rb_cProc
Proc class.
Definition proc.c:46
rb_cString
VALUE rb_cString
String class.
Definition string.c:85
RB_OBJ_WRITTEN
#define RB_OBJ_WRITTEN(old, oldv, young)
Identical to RB_OBJ_WRITE(), except it doesn't write any values, but only a WB declaration.
Definition gc.h:504
RB_OBJ_WRITE
#define RB_OBJ_WRITE(old, slot, young)
Declaration of a "back" pointer.
Definition gc.h:492
rb_ary_concat
VALUE rb_ary_concat(VALUE lhs, VALUE rhs)
Destructively appends the contents of latter into the end of former.
rb_ary_shift
VALUE rb_ary_shift(VALUE ary)
Destructively deletes an element from the beginning of the passed array and returns what was deleted.
rb_ary_resurrect
VALUE rb_ary_resurrect(VALUE ary)
I guess there is no use case of this function in extension libraries, but this is a routine identical...
rb_ary_dup
VALUE rb_ary_dup(VALUE ary)
Duplicates an array.
rb_ary_includes
VALUE rb_ary_includes(VALUE ary, VALUE elem)
Queries if the passed array has the passed entry.
rb_ary_plus
VALUE rb_ary_plus(VALUE lhs, VALUE rhs)
Creates a new array, concatenating the former to the latter.
rb_ary_cat
VALUE rb_ary_cat(VALUE ary, const VALUE *train, long len)
Destructively appends multiple elements at the end of the array.
rb_check_array_type
VALUE rb_check_array_type(VALUE obj)
Try converting an object to its array representation using its to_ary method, if any.
rb_ary_new
VALUE rb_ary_new(void)
Allocates a new, empty array.
rb_ary_pop
VALUE rb_ary_pop(VALUE ary)
Destructively deletes an element from the end of the passed array and returns what was deleted.
rb_ary_hidden_new
VALUE rb_ary_hidden_new(long capa)
Allocates a hidden (no class) empty array.
rb_ary_push
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
rb_ary_entry
VALUE rb_ary_entry(VALUE ary, long off)
Queries an element of an array.
rb_ary_store
void rb_ary_store(VALUE ary, long key, VALUE val)
Destructively stores the passed value to the passed array's passed index.
UNLIMITED_ARGUMENTS
#define UNLIMITED_ARGUMENTS
This macro is used in conjunction with rb_check_arity().
Definition error.h:35
rb_check_arity
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
Definition error.h:284
rb_proc_call_with_block
VALUE rb_proc_call_with_block(VALUE recv, int argc, const VALUE *argv, VALUE proc)
Identical to rb_proc_call(), except you can additionally pass another proc object,...
Definition proc.c:1763
rb_reg_last_match
VALUE rb_reg_last_match(VALUE md)
This just returns the argument, stringified.
Definition re.c:2095
rb_reg_match
VALUE rb_reg_match(VALUE re, VALUE str)
This is the match operator.
Definition re.c:3970
rb_reg_nth_match
VALUE rb_reg_nth_match(int n, VALUE md)
Queries the nth captured substring.
Definition re.c:2071
rb_reg_match_post
VALUE rb_reg_match_post(VALUE md)
The portion of the original string after the given match.
Definition re.c:2150
rb_reg_nth_defined
VALUE rb_reg_nth_defined(int n, VALUE md)
Identical to rb_reg_nth_match(), except it just returns Boolean.
Definition re.c:2055
rb_reg_match_pre
VALUE rb_reg_match_pre(VALUE md)
The portion of the original string before the given match.
Definition re.c:2119
rb_reg_match_last
VALUE rb_reg_match_last(VALUE md)
The portion of the original string that captured at the very last.
Definition re.c:2179
rb_set_lookup
bool rb_set_lookup(VALUE set, VALUE element)
Whether the set contains the given element.
Definition set.c:2364
rb_str_append
VALUE rb_str_append(VALUE dst, VALUE src)
Identical to rb_str_buf_append(), except it converts the right hand side before concatenating.
Definition string.c:3913
rb_str_succ
VALUE rb_str_succ(VALUE orig)
Searches for the "successor" of a string.
Definition string.c:5457
rb_str_buf_append
VALUE rb_str_buf_append(VALUE dst, VALUE src)
Identical to rb_str_cat_cstr(), except it takes Ruby's string instead of C's.
Definition string.c:3879
rb_str_concat
VALUE rb_str_concat(VALUE dst, VALUE src)
Identical to rb_str_append(), except it also accepts an integer as a codepoint.
Definition string.c:4150
rb_str_cat_cstr
#define rb_str_cat_cstr(buf, str)
Identical to rb_str_cat(), except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1681
rb_str_length
VALUE rb_str_length(VALUE)
Identical to rb_str_strlen(), except it returns the value in rb_cInteger.
Definition string.c:2506
rb_str_intern
VALUE rb_str_intern(VALUE str)
Identical to rb_to_symbol(), except it assumes the receiver being an instance of RString.
Definition symbol.c:1085
rb_const_get
VALUE rb_const_get(VALUE space, ID name)
Identical to rb_const_defined(), except it returns the actual defined value.
Definition variable.c:3505
rb_ivar_set
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
Definition variable.c:2141
rb_cvar_set
void rb_cvar_set(VALUE klass, ID name, VALUE val)
Assigns a value to a class variable.
Definition variable.c:4294
rb_cvar_find
VALUE rb_cvar_find(VALUE klass, ID name, VALUE *front)
Identical to rb_cvar_get(), except it takes additional "front" pointer.
Definition variable.c:4354
rb_ivar_get
VALUE rb_ivar_get(VALUE obj, ID name)
Identical to rb_iv_get(), except it accepts the name as an ID instead of a C string.
Definition variable.c:1641
rb_const_set
void rb_const_set(VALUE space, ID name, VALUE val)
Names a constant.
Definition variable.c:3984
rb_autoload_load
VALUE rb_autoload_load(VALUE space, ID name)
Kicks the autoload procedure as if it was "touched".
Definition variable.c:3335
rb_mod_name
VALUE rb_mod_name(VALUE mod)
Queries the name of a module.
Definition variable.c:152
rb_const_get_at
VALUE rb_const_get_at(VALUE space, ID name)
Identical to rb_const_defined_at(), except it returns the actual defined value.
Definition variable.c:3511
rb_set_class_path_string
void rb_set_class_path_string(VALUE klass, VALUE space, VALUE name)
Identical to rb_set_class_path(), except it accepts the name as Ruby's string instead of C's.
Definition variable.c:441
rb_ivar_defined
VALUE rb_ivar_defined(VALUE obj, ID name)
Queries if the instance variable is defined at the object.
Definition variable.c:2201
rb_const_defined_at
int rb_const_defined_at(VALUE space, ID name)
Identical to rb_const_defined(), except it doesn't look for parent classes.
Definition variable.c:3844
rb_cvar_defined
VALUE rb_cvar_defined(VALUE klass, ID name)
Queries if the given class has the given class variable.
Definition variable.c:4380
rb_class_path
VALUE rb_class_path(VALUE mod)
Identical to rb_mod_name(), except it returns #<Class: ...> style inspection for anonymous modules.
Definition variable.c:398
rb_const_defined
int rb_const_defined(VALUE space, ID name)
Queries if the constant is defined at the namespace.
Definition variable.c:3838
rb_method_basic_definition_p
int rb_method_basic_definition_p(VALUE klass, ID mid)
Well... Let us hesitate from describing what a "basic definition" is.
Definition vm_method.c:3571
rb_check_funcall
VALUE rb_check_funcall(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it returns RUBY_Qundef instead of raising rb_eNoMethodError.
Definition vm_eval.c:691
rb_get_alloc_func
rb_alloc_func_t rb_get_alloc_func(VALUE klass)
Queries the allocator function of a class.
Definition vm_method.c:1855
rb_method_boundp
int rb_method_boundp(VALUE klass, ID id, int ex)
Queries if the klass has this method.
Definition vm_method.c:2465
rb_check_id
ID rb_check_id(volatile VALUE *namep)
Detects if the given name is already interned or not.
Definition symbol.c:1289
rb_sym2str
VALUE rb_sym2str(VALUE symbol)
Obtain a frozen string representation of a symbol (not including the leading colon).
Definition symbol.c:1148
off
int off
Offset inside of ptr.
Definition io.h:5
len
int len
Length of the buffer.
Definition io.h:8
RB_OBJ_SET_SHAREABLE
#define RB_OBJ_SET_SHAREABLE(obj)
Wrapper of rb_obj_set_shareable().
Definition ractor.h:290
rb_ractor_shareable_p
static bool rb_ractor_shareable_p(VALUE obj)
Queries if multiple Ractors can share the passed object or not.
Definition ractor.h:269
RB_OBJ_SHAREABLE_P
#define RB_OBJ_SHAREABLE_P(obj)
Queries if the passed object has previously classified as shareable or not.
Definition ractor.h:255
RB_NOGVL_UBF_ASYNC_SAFE
#define RB_NOGVL_UBF_ASYNC_SAFE
Passing this flag to rb_nogvl() indicates that the passed UBF is async-signal-safe.
Definition thread.h:71
RB_NOGVL_INTR_FAIL
#define RB_NOGVL_INTR_FAIL
Passing this flag to rb_nogvl() prevents it from processing interrupts after the given function retur...
Definition thread.h:50
rb_nogvl
void * rb_nogvl(void *(*func)(void *), void *data1, rb_unblock_function_t *ubf, void *data2, int flags)
Identical to rb_thread_call_without_gvl(), except it additionally takes "flags" that change the behav...
Definition thread.c:1773
MEMCPY
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:372
ALLOCA_N
#define ALLOCA_N(type, n)
Definition memory.h:292
RB_GC_GUARD
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:167
MEMMOVE
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
Definition memory.h:384
ruby::backward::cxxanyargs::type
VALUE type(ANYARGS)
ANYARGS-ed function type.
Definition cxxanyargs.hpp:56
RARRAY_LEN
#define RARRAY_LEN
Just another name of rb_array_len.
Definition rarray.h:50
RARRAY_LENINT
static int RARRAY_LENINT(VALUE ary)
Identical to rb_array_len(), except it differs for the return type.
Definition rarray.h:280
RARRAY_PTR
static VALUE * RARRAY_PTR(VALUE ary)
Wild use of a C pointer.
Definition rarray.h:365
RARRAY_AREF
#define RARRAY_AREF(a, i)
Definition rarray.h:402
RARRAY_CONST_PTR
#define RARRAY_CONST_PTR
Just another name of rb_array_const_ptr.
Definition rarray.h:51
RBASIC_CLASS
static VALUE RBASIC_CLASS(VALUE obj)
Queries the class of an object.
Definition rbasic.h:166
RBASIC
#define RBASIC(obj)
Convenient casting macro.
Definition rbasic.h:40
RCLASS_SUPER
#define RCLASS_SUPER
Just another name of rb_class_get_superclass.
Definition rclass.h:44
RHASH_SIZE
#define RHASH_SIZE(h)
Queries the size of the hash.
Definition rhash.h:57
RHASH_EMPTY_P
#define RHASH_EMPTY_P(h)
Checks if the hash is empty.
Definition rhash.h:67
RSTRING_LENINT
static int RSTRING_LENINT(VALUE str)
Identical to RSTRING_LEN(), except it differs for the return type.
Definition rstring.h:438
StringValueCStr
#define StringValueCStr(v)
Identical to StringValuePtr, except it additionally checks for the contents for viability as a C stri...
Definition rstring.h:89
RB_PASS_KEYWORDS
#define RB_PASS_KEYWORDS
Pass keywords, final argument must be a hash of keywords.
Definition scan_args.h:72
RB_NO_KEYWORDS
#define RB_NO_KEYWORDS
Do not pass keywords.
Definition scan_args.h:69
RB_SPECIAL_CONST_P
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
Definition special_consts.h:327
RTEST
#define RTEST
This is an old name of RB_TEST.
Definition special_consts.h:51
ANYARGS
#define ANYARGS
Functions declared using this macro take arbitrary arguments, including void.
Definition stdarg.h:64
RArray
Ruby's array.
Definition rarray.h:127
RArray::ary
const VALUE ary[1]
Embedded elements.
Definition rarray.h:187
RArray::ptr
const VALUE * ptr
Pointer to the C array that holds the elements of the array.
Definition rarray.h:174
RHash
Definition hash.h:54
iseq_catch_table_entry
Definition iseq.h:367
iseq_catch_table
Definition iseq.h:378
iseq_inline_constant_cache_entry
Definition vm_core.h:258
iseq_inline_constant_cache
Definition vm_core.h:271
iseq_inline_constant_cache::segments
const ID * segments
A null-terminated list of ids, used to represent a constant's path idNULL is used to represent the ::...
Definition vm_core.h:285
iseq_inline_cvar_cache_entry
Definition vm_core.h:293
iseq_inline_iv_cache_entry
Definition vm_core.h:288
rb_block
Definition vm_core.h:939
rb_box_struct
Internal header for Ruby Box.
Definition box.h:14
rb_builtin_function
Definition builtin.h:6
rb_call_data
Definition vm_callinfo.h:606
rb_callable_method_entry_struct
Definition method.h:63
rb_callcache
Definition vm_callinfo.h:292
rb_callinfo_kwarg
Definition vm_callinfo.h:48
rb_callinfo
Definition vm_callinfo.h:79
rb_calling_info
Definition vm_core.h:306
rb_captured_block
Definition vm_core.h:915
rb_class_cc_entries
Definition vm_callinfo.h:611
rb_const_entry_struct
Definition constant.h:33
rb_control_frame_struct
Definition vm_core.h:948
rb_cref_struct
CREF (Class REFerence)
Definition method.h:45
rb_cvar_class_tbl_entry
Internal header for Class.
Definition class.h:31
rb_env_t
Definition vm_core.h:1400
rb_execution_context_struct
Definition vm_core.h:1082
rb_forwarding_call_data
Definition vm_insnhelper.h:69
rb_getivar_cache
Definition shape.h:595
rb_hook_list_struct
Definition vm_core.h:684
rb_id_table
Definition id_table.h:7
rb_iseq_constant_body
Definition vm_core.h:430
rb_iseq_struct
Definition vm_core.h:595
rb_method_cfunc_struct
Definition method.h:148
rb_method_definition_struct
Definition method.h:187
rb_method_entry_struct
Definition method.h:55
rb_method_iseq_struct::cref
rb_cref_t * cref
class reference, should be marked
Definition method.h:144
rb_method_iseq_struct::iseqptr
const rb_iseq_t * iseqptr
iseq pointer, should be separated from iseqval
Definition method.h:143
rb_ractor_struct
Definition ractor_core.h:98
rb_setivar_cache
Definition shape.h:637
rb_thread_struct
Definition vm_core.h:1183
rb_vm_struct
Definition vm_core.h:709
set_table
Definition set_table.h:10
st_table
Definition st.h:79
vm_ifunc
IFUNC (Internal FUNCtion)
Definition imemo.h:87
vm_once_wait_arg
Definition vm_insnhelper.c:5651
vm_svar
SVAR (Special VARiable)
Definition imemo.h:52
vm_svar::cref_or_me
const VALUE cref_or_me
class reference or rb_method_entry_t
Definition imemo.h:54
vm_throw_data
THROW_DATA.
Definition imemo.h:61
rb_native_mutex_lock
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
rb_native_cond_broadcast
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
rb_native_mutex_unlock
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
rb_native_cond_wait
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.
__attribute__
Definition vm_insnhelper.c:2401
iseq_inline_storage_entry
Definition vm_core.h:297
rb_proc_t
Definition vm_core.h:1370
SIGNED_VALUE
intptr_t SIGNED_VALUE
A signed integer type that has the same width with VALUE.
Definition value.h:63
ID
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
SIZEOF_VALUE
#define SIZEOF_VALUE
Identical to sizeof(VALUE), except it is a macro that can also be used inside of preprocessor directi...
Definition value.h:69
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
RB_BUILTIN_TYPE
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
Definition value_type.h:182
RB_FLOAT_TYPE_P
static bool RB_FLOAT_TYPE_P(VALUE obj)
Queries if the object is an instance of rb_cFloat.
Definition value_type.h:264
RB_TYPE_P
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:376
Generated by doxygen 1.9.8