Ruby 4.1.0dev (2026-05-27 revision dcfba5725794815fa36e129457f9d748f24371f9)
vm_insnhelper.c (dcfba5725794815fa36e129457f9d748f24371f9)
1/**********************************************************************
2
3 vm_insnhelper.c - instruction helper functions. Included into vm.c.
4
5 $Author$
6
7 Copyright (C) 2007 Koichi Sasada
8
9**********************************************************************/
10
11#include "ruby/internal/config.h"
12
13#include <math.h>
14
15#ifdef HAVE_STDATOMIC_H
16 #include <stdatomic.h>
17#endif
18
19#include "constant.h"
20#include "debug_counter.h"
21#include "internal.h"
22#include "internal/class.h"
23#include "internal/compar.h"
24#include "internal/hash.h"
25#include "internal/numeric.h"
26#include "internal/proc.h"
27#include "internal/random.h"
28#include "internal/variable.h"
29#include "internal/set_table.h"
30#include "internal/struct.h"
31#include "variable.h"
32
33/* finish iseq array */
34#include "insns.inc"
35#include "insns_info.inc"
36
37extern rb_method_definition_t *rb_method_definition_create(rb_method_type_t type, ID mid);
38extern void rb_method_definition_set(const rb_method_entry_t *me, rb_method_definition_t *def, void *opts);
39extern int rb_method_definition_eq(const rb_method_definition_t *d1, const rb_method_definition_t *d2);
40extern VALUE rb_make_no_method_exception(VALUE exc, VALUE format, VALUE obj,
41 int argc, const VALUE *argv, int priv);
42
43static const struct rb_callcache vm_empty_cc;
44static const struct rb_callcache vm_empty_cc_for_super;
45
46/* control stack frame */
47
48static rb_control_frame_t *vm_get_ruby_level_caller_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp);
49
50VALUE
51ruby_vm_special_exception_copy(VALUE exc)
52{
53 VALUE e = rb_obj_alloc(rb_class_real(RBASIC_CLASS(exc)));
54 rb_obj_copy_ivar(e, exc);
55 return e;
56}
57
58NORETURN(static void ec_stack_overflow(rb_execution_context_t *ec, int));
59static void
60ec_stack_overflow(rb_execution_context_t *ec, int setup)
61{
62 VALUE mesg = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_sysstack];
63 ec->raised_flag = RAISED_STACKOVERFLOW;
64 if (setup) {
65 VALUE at = rb_ec_backtrace_object(ec);
66 mesg = ruby_vm_special_exception_copy(mesg);
67 rb_ivar_set(mesg, idBt, at);
68 rb_ivar_set(mesg, idBt_locations, at);
69 }
70 ec->errinfo = mesg;
71 EC_JUMP_TAG(ec, TAG_RAISE);
72}
73
74NORETURN(static void vm_stackoverflow(void));
75
76static void
77vm_stackoverflow(void)
78{
79 ec_stack_overflow(GET_EC(), TRUE);
80}
81
82void
83rb_ec_stack_overflow(rb_execution_context_t *ec, ruby_stack_overflow_critical_level crit)
84{
85 if (rb_during_gc()) {
86 rb_bug("system stack overflow during GC. Faulty native extension?");
87 }
88 if (crit >= rb_stack_overflow_fatal) {
89 ec->raised_flag = RAISED_STACKOVERFLOW;
90 ec->errinfo = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_stackfatal];
91 EC_JUMP_TAG(ec, TAG_RAISE);
92 }
93 ec_stack_overflow(ec, crit < rb_stack_overflow_signal);
94}
95
96static inline void stack_check(rb_execution_context_t *ec);
97
98#if VM_CHECK_MODE > 0
99static int
100callable_class_p(VALUE klass)
101{
102#if VM_CHECK_MODE >= 2
103 if (!klass) return FALSE;
104 switch (RB_BUILTIN_TYPE(klass)) {
105 default:
106 break;
107 case T_ICLASS:
108 if (!RB_TYPE_P(RCLASS_SUPER(klass), T_MODULE)) break;
109 case T_MODULE:
110 return TRUE;
111 }
112 while (klass) {
113 if (klass == rb_cBasicObject) {
114 return TRUE;
115 }
116 klass = RCLASS_SUPER(klass);
117 }
118 return FALSE;
119#else
120 return klass != 0;
121#endif
122}
123
124static int
125callable_method_entry_p(const rb_callable_method_entry_t *cme)
126{
127 if (cme == NULL) {
128 return TRUE;
129 }
130 else {
131 VM_ASSERT(IMEMO_TYPE_P((VALUE)cme, imemo_ment), "imemo_type:%s", rb_imemo_name(imemo_type((VALUE)cme)));
132
133 if (callable_class_p(cme->defined_class)) {
134 return TRUE;
135 }
136 else {
137 return FALSE;
138 }
139 }
140}
141
142static void
143vm_check_frame_detail(VALUE type, int req_block, int req_me, int req_cref, VALUE specval, VALUE cref_or_me, int is_cframe, const rb_iseq_t *iseq)
144{
145 unsigned int magic = (unsigned int)(type & VM_FRAME_MAGIC_MASK);
146 enum imemo_type cref_or_me_type = imemo_env; /* impossible value */
147
148 if (RB_TYPE_P(cref_or_me, T_IMEMO)) {
149 cref_or_me_type = imemo_type(cref_or_me);
150 }
151 if (type & VM_FRAME_FLAG_BMETHOD) {
152 req_me = TRUE;
153 }
154
155 if (req_block && (type & VM_ENV_FLAG_LOCAL) == 0) {
156 rb_bug("vm_push_frame: specval (%p) should be a block_ptr on %x frame", (void *)specval, magic);
157 }
158 if (!req_block && (type & VM_ENV_FLAG_LOCAL) != 0) {
159 rb_bug("vm_push_frame: specval (%p) should not be a block_ptr on %x frame", (void *)specval, magic);
160 }
161
162 if (req_me) {
163 if (cref_or_me_type != imemo_ment) {
164 rb_bug("vm_push_frame: (%s) should be method entry on %x frame", rb_obj_info(cref_or_me), magic);
165 }
166 }
167 else {
168 if (req_cref && cref_or_me_type != imemo_cref) {
169 rb_bug("vm_push_frame: (%s) should be CREF on %x frame", rb_obj_info(cref_or_me), magic);
170 }
171 else { /* cref or Qfalse */
172 if (cref_or_me != Qfalse && cref_or_me_type != imemo_cref) {
173 if (((type & VM_FRAME_FLAG_LAMBDA) || magic == VM_FRAME_MAGIC_IFUNC || magic == VM_FRAME_MAGIC_DUMMY) && (cref_or_me_type == imemo_ment)) {
174 /* ignore */
175 }
176 else {
177 rb_bug("vm_push_frame: (%s) should be false or cref on %x frame", rb_obj_info(cref_or_me), magic);
178 }
179 }
180 }
181 }
182
183 if (cref_or_me_type == imemo_ment) {
184 const rb_callable_method_entry_t *me = (const rb_callable_method_entry_t *)cref_or_me;
185
186 if (!callable_method_entry_p(me)) {
187 rb_bug("vm_push_frame: ment (%s) should be callable on %x frame.", rb_obj_info(cref_or_me), magic);
188 }
189 }
190
191 if ((type & VM_FRAME_MAGIC_MASK) == VM_FRAME_MAGIC_DUMMY) {
192 VM_ASSERT(iseq == NULL ||
193 RBASIC_CLASS((VALUE)iseq) == 0 || // dummy frame for loading
194 RUBY_VM_NORMAL_ISEQ_P(iseq) //argument error
195 );
196 }
197 else {
198 VM_ASSERT(is_cframe == !RUBY_VM_NORMAL_ISEQ_P(iseq));
199 }
200}
201
202static void
203vm_check_frame(VALUE type,
204 VALUE specval,
205 VALUE cref_or_me,
206 const rb_iseq_t *iseq)
207{
208 VALUE given_magic = type & VM_FRAME_MAGIC_MASK;
209 VM_ASSERT(FIXNUM_P(type));
210
211#define CHECK(magic, req_block, req_me, req_cref, is_cframe) \
212 case magic: \
213 vm_check_frame_detail(type, req_block, req_me, req_cref, \
214 specval, cref_or_me, is_cframe, iseq); \
215 break
216 switch (given_magic) {
217 /* BLK ME CREF CFRAME */
218 CHECK(VM_FRAME_MAGIC_METHOD, TRUE, TRUE, FALSE, FALSE);
219 CHECK(VM_FRAME_MAGIC_CLASS, TRUE, FALSE, TRUE, FALSE);
220 CHECK(VM_FRAME_MAGIC_TOP, TRUE, FALSE, TRUE, FALSE);
221 CHECK(VM_FRAME_MAGIC_CFUNC, TRUE, TRUE, FALSE, TRUE);
222 CHECK(VM_FRAME_MAGIC_BLOCK, FALSE, FALSE, FALSE, FALSE);
223 CHECK(VM_FRAME_MAGIC_IFUNC, FALSE, FALSE, FALSE, TRUE);
224 CHECK(VM_FRAME_MAGIC_EVAL, FALSE, FALSE, FALSE, FALSE);
225 CHECK(VM_FRAME_MAGIC_RESCUE, FALSE, FALSE, FALSE, FALSE);
226 CHECK(VM_FRAME_MAGIC_DUMMY, TRUE, FALSE, FALSE, FALSE);
227 default:
228 rb_bug("vm_push_frame: unknown type (%x)", (unsigned int)given_magic);
229 }
230#undef CHECK
231}
232
233static VALUE vm_stack_canary; /* Initialized later */
234static bool vm_stack_canary_was_born = false;
235
236// Return the index of the instruction right before the given PC.
237// This is needed because insn_entry advances PC before the insn body.
238static unsigned int
239previous_insn_index(const rb_iseq_t *iseq, const VALUE *pc)
240{
241 unsigned int pos = 0;
242 while (pos < ISEQ_BODY(iseq)->iseq_size) {
243 int opcode = rb_vm_insn_addr2opcode((void *)ISEQ_BODY(iseq)->iseq_encoded[pos]);
244 unsigned int next_pos = pos + insn_len(opcode);
245 if (ISEQ_BODY(iseq)->iseq_encoded + next_pos == pc) {
246 return pos;
247 }
248 pos = next_pos;
249 }
250 rb_bug("failed to find the previous insn");
251}
252
253void
254rb_vm_check_canary(const rb_execution_context_t *ec, VALUE *sp)
255{
256 const struct rb_control_frame_struct *reg_cfp = ec->cfp;
257 const struct rb_iseq_struct *iseq;
258
259 if (! LIKELY(vm_stack_canary_was_born)) {
260 return; /* :FIXME: isn't it rather fatal to enter this branch? */
261 }
262 else if ((VALUE *)reg_cfp == ec->vm_stack + ec->vm_stack_size) {
263 /* This is at the very beginning of a thread. cfp does not exist. */
264 return;
265 }
266 else if (! (iseq = GET_ISEQ())) {
267 return;
268 }
269 else if (LIKELY(sp[0] != vm_stack_canary)) {
270 return;
271 }
272 else {
273 /* we are going to call methods below; squash the canary to
274 * prevent infinite loop. */
275 sp[0] = Qundef;
276 }
277
278 const VALUE *orig = rb_iseq_original_iseq(iseq);
279 const VALUE iseqw = rb_iseqw_new(iseq);
280 const VALUE inspection = rb_inspect(iseqw);
281 const char *stri = rb_str_to_cstr(inspection);
282 const VALUE disasm = rb_iseq_disasm(iseq);
283 const char *strd = rb_str_to_cstr(disasm);
284 const ptrdiff_t pos = previous_insn_index(iseq, GET_PC());
285 const enum ruby_vminsn_type insn = (enum ruby_vminsn_type)orig[pos];
286 const char *name = insn_name(insn);
287
288 /* rb_bug() is not capable of outputting this large contents. It
289 is designed to run form a SIGSEGV handler, which tends to be
290 very restricted. */
291 ruby_debug_printf(
292 "We are killing the stack canary set by %s, "
293 "at %s@pc=%"PRIdPTR"\n"
294 "watch out the C stack trace.\n"
295 "%s",
296 name, stri, pos, strd);
297 rb_bug("see above.");
298}
299#define vm_check_canary(ec, sp) rb_vm_check_canary(ec, sp)
300
301#else
302#define vm_check_canary(ec, sp)
303#define vm_check_frame(a, b, c, d)
304#endif /* VM_CHECK_MODE > 0 */
305
306#if USE_DEBUG_COUNTER
307static void
308vm_push_frame_debug_counter_inc(
309 const struct rb_execution_context_struct *ec,
310 const struct rb_control_frame_struct *reg_cfp,
311 VALUE type)
312{
313 const struct rb_control_frame_struct *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(reg_cfp);
314
315 RB_DEBUG_COUNTER_INC(frame_push);
316
317 if (RUBY_VM_END_CONTROL_FRAME(ec) != prev_cfp) {
318 const bool curr = VM_FRAME_RUBYFRAME_P(reg_cfp);
319 const bool prev = VM_FRAME_RUBYFRAME_P(prev_cfp);
320 if (prev) {
321 if (curr) {
322 RB_DEBUG_COUNTER_INC(frame_R2R);
323 }
324 else {
325 RB_DEBUG_COUNTER_INC(frame_R2C);
326 }
327 }
328 else {
329 if (curr) {
330 RB_DEBUG_COUNTER_INC(frame_C2R);
331 }
332 else {
333 RB_DEBUG_COUNTER_INC(frame_C2C);
334 }
335 }
336 }
337
338 switch (type & VM_FRAME_MAGIC_MASK) {
339 case VM_FRAME_MAGIC_METHOD: RB_DEBUG_COUNTER_INC(frame_push_method); return;
340 case VM_FRAME_MAGIC_BLOCK: RB_DEBUG_COUNTER_INC(frame_push_block); return;
341 case VM_FRAME_MAGIC_CLASS: RB_DEBUG_COUNTER_INC(frame_push_class); return;
342 case VM_FRAME_MAGIC_TOP: RB_DEBUG_COUNTER_INC(frame_push_top); return;
343 case VM_FRAME_MAGIC_CFUNC: RB_DEBUG_COUNTER_INC(frame_push_cfunc); return;
344 case VM_FRAME_MAGIC_IFUNC: RB_DEBUG_COUNTER_INC(frame_push_ifunc); return;
345 case VM_FRAME_MAGIC_EVAL: RB_DEBUG_COUNTER_INC(frame_push_eval); return;
346 case VM_FRAME_MAGIC_RESCUE: RB_DEBUG_COUNTER_INC(frame_push_rescue); return;
347 case VM_FRAME_MAGIC_DUMMY: RB_DEBUG_COUNTER_INC(frame_push_dummy); return;
348 }
349
350 rb_bug("unreachable");
351}
352#else
353#define vm_push_frame_debug_counter_inc(ec, cfp, t) /* void */
354#endif
355
356// Return a poison value to be set above the stack top to verify leafness.
357VALUE
358rb_vm_stack_canary(void)
359{
360#if VM_CHECK_MODE > 0
361 return vm_stack_canary;
362#else
363 return 0;
364#endif
365}
366
367STATIC_ASSERT(VM_ENV_DATA_INDEX_ME_CREF, VM_ENV_DATA_INDEX_ME_CREF == -2);
368STATIC_ASSERT(VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL == -1);
369STATIC_ASSERT(VM_ENV_DATA_INDEX_FLAGS, VM_ENV_DATA_INDEX_FLAGS == -0);
370
371static void
372vm_push_frame(rb_execution_context_t *ec,
373 const rb_iseq_t *iseq,
374 VALUE type,
375 VALUE self,
376 VALUE specval,
377 VALUE cref_or_me,
378 const VALUE *pc,
379 VALUE *sp,
380 int local_size,
381 int stack_max)
382{
383 rb_control_frame_t *const cfp = RUBY_VM_NEXT_CONTROL_FRAME(ec->cfp);
384
385 vm_check_frame(type, specval, cref_or_me, iseq);
386 VM_ASSERT(local_size >= 0);
387
388 /* check stack overflow */
389 CHECK_VM_STACK_OVERFLOW0(cfp, sp, local_size + stack_max);
390 vm_check_canary(ec, sp);
391
392 /* setup vm value stack */
393
394 /* initialize local variables */
395 for (int i=0; i < local_size; i++) {
396 *sp++ = Qnil;
397 }
398
399 /* setup ep with managing data */
400 *sp++ = cref_or_me; /* ep[-2] / Qnil or T_IMEMO(cref) or T_IMEMO(ment) */
401 *sp++ = specval /* ep[-1] / block handler or prev env ptr */;
402 *sp++ = type; /* ep[-0] / ENV_FLAGS */
403
404 /* setup new frame */
405 *cfp = (const struct rb_control_frame_struct) {
406 .pc = pc,
407 .sp = sp,
408 ._iseq = iseq,
409 .self = self,
410 .ep = sp - 1,
411 .block_code = NULL,
412#if VM_DEBUG_BP_CHECK
413 .bp_check = sp,
414#endif
415 .jit_return = NULL,
416 };
417
418 /* Ensure the initialization of `*cfp` above never gets reordered with the update of `ec->cfp` below.
419 This is a no-op in all cases we've looked at (https://godbolt.org/z/3oxd1446K), but should guarantee it for all
420 future/untested compilers/platforms. */
421
422 #if defined HAVE_DECL_ATOMIC_SIGNAL_FENCE && HAVE_DECL_ATOMIC_SIGNAL_FENCE
423 atomic_signal_fence(memory_order_seq_cst);
424 #endif
425
426 ec->cfp = cfp;
427
428 if (VMDEBUG == 2) {
429 SDR();
430 }
431 vm_push_frame_debug_counter_inc(ec, cfp, type);
432}
433
434void
435rb_vm_pop_frame_no_int(rb_execution_context_t *ec)
436{
437 rb_control_frame_t *cfp = ec->cfp;
438
439 if (VMDEBUG == 2) SDR();
440
441 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
442}
443
444/* return TRUE if the frame is finished */
445static inline int
446vm_pop_frame(rb_execution_context_t *ec, rb_control_frame_t *cfp, const VALUE *ep)
447{
448 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
449
450 if (VMDEBUG == 2) SDR();
451
452 RUBY_VM_CHECK_INTS(ec);
453 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
454
455 return flags & VM_FRAME_FLAG_FINISH;
456}
457
458void
459rb_vm_pop_frame(rb_execution_context_t *ec)
460{
461 vm_pop_frame(ec, ec->cfp, ec->cfp->ep);
462}
463
464// it pushes pseudo-frame with fname filename.
465VALUE
466rb_vm_push_frame_fname(rb_execution_context_t *ec, VALUE fname)
467{
468 rb_iseq_t *rb_iseq_alloc_with_dummy_path(VALUE fname);
469 rb_iseq_t *dmy_iseq = rb_iseq_alloc_with_dummy_path(fname);
470
471 vm_push_frame(ec,
472 dmy_iseq, //const rb_iseq_t *iseq,
473 VM_FRAME_MAGIC_DUMMY | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, // VALUE type,
474 ec->cfp->self, // VALUE self,
475 VM_BLOCK_HANDLER_NONE, // VALUE specval,
476 Qfalse, // VALUE cref_or_me,
477 NULL, // const VALUE *pc,
478 ec->cfp->sp, // VALUE *sp,
479 0, // int local_size,
480 0); // int stack_max
481
482 return (VALUE)dmy_iseq;
483}
484
485/* method dispatch */
486static inline VALUE
487rb_arity_error_new(int argc, int min, int max)
488{
489 VALUE err_mess = rb_sprintf("wrong number of arguments (given %d, expected %d", argc, min);
490 if (min == max) {
491 /* max is not needed */
492 }
493 else if (max == UNLIMITED_ARGUMENTS) {
494 rb_str_cat_cstr(err_mess, "+");
495 }
496 else {
497 rb_str_catf(err_mess, "..%d", max);
498 }
499 rb_str_cat_cstr(err_mess, ")");
500 return rb_exc_new3(rb_eArgError, err_mess);
501}
502
503void
504rb_error_arity(int argc, int min, int max)
505{
506 rb_exc_raise(rb_arity_error_new(argc, min, max));
507}
508
509/* lvar */
510
511NOINLINE(static void vm_env_write_slowpath(const VALUE *ep, int index, VALUE v));
512
513static void
514vm_env_write_slowpath(const VALUE *ep, int index, VALUE v)
515{
516 /* remember env value forcely */
517 rb_gc_writebarrier_remember(VM_ENV_ENVVAL(ep));
518 VM_FORCE_WRITE(&ep[index], v);
519 VM_ENV_FLAGS_UNSET(ep, VM_ENV_FLAG_WB_REQUIRED);
520 RB_DEBUG_COUNTER_INC(lvar_set_slowpath);
521}
522
523// YJIT assumes this function never runs GC
524static inline void
525vm_env_write(const VALUE *ep, int index, VALUE v)
526{
527 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
528 if (LIKELY((flags & VM_ENV_FLAG_WB_REQUIRED) == 0)) {
529 VM_STACK_ENV_WRITE(ep, index, v);
530 }
531 else {
532 vm_env_write_slowpath(ep, index, v);
533 }
534}
535
536void
537rb_vm_env_write(const VALUE *ep, int index, VALUE v)
538{
539 vm_env_write(ep, index, v);
540}
541
542VALUE
543rb_vm_bh_to_procval(const rb_execution_context_t *ec, VALUE block_handler)
544{
545 if (block_handler == VM_BLOCK_HANDLER_NONE) {
546 return Qnil;
547 }
548 else {
549 switch (vm_block_handler_type(block_handler)) {
550 case block_handler_type_iseq:
551 case block_handler_type_ifunc:
552 return rb_vm_make_proc(ec, VM_BH_TO_CAPT_BLOCK(block_handler), rb_cProc);
553 case block_handler_type_symbol:
554 return rb_sym_to_proc(VM_BH_TO_SYMBOL(block_handler));
555 case block_handler_type_proc:
556 return VM_BH_TO_PROC(block_handler);
557 default:
558 VM_UNREACHABLE(rb_vm_bh_to_procval);
559 }
560 }
561}
562
563/* svar */
564
565#if VM_CHECK_MODE > 0
566static int
567vm_svar_valid_p(VALUE svar)
568{
569 if (RB_TYPE_P((VALUE)svar, T_IMEMO)) {
570 switch (imemo_type(svar)) {
571 case imemo_svar:
572 case imemo_cref:
573 case imemo_ment:
574 return TRUE;
575 default:
576 break;
577 }
578 }
579 rb_bug("vm_svar_valid_p: unknown type: %s", rb_obj_info(svar));
580 return FALSE;
581}
582#endif
583
584static inline struct vm_svar *
585lep_svar(const rb_execution_context_t *ec, const VALUE *lep)
586{
587 VALUE svar;
588
589 if (lep && (ec == NULL || ec->root_lep != lep)) {
590 svar = lep[VM_ENV_DATA_INDEX_ME_CREF];
591 }
592 else {
593 svar = ec->root_svar;
594 }
595
596 VM_ASSERT(svar == Qfalse || vm_svar_valid_p(svar));
597
598 return (struct vm_svar *)svar;
599}
600
601static inline void
602lep_svar_write(const rb_execution_context_t *ec, const VALUE *lep, const struct vm_svar *svar)
603{
604 VM_ASSERT(vm_svar_valid_p((VALUE)svar));
605
606 if (lep && (ec == NULL || ec->root_lep != lep)) {
607 vm_env_write(lep, VM_ENV_DATA_INDEX_ME_CREF, (VALUE)svar);
608 }
609 else {
610 RB_OBJ_WRITE(rb_ec_thread_ptr(ec)->self, &ec->root_svar, svar);
611 }
612}
613
614static VALUE
615lep_svar_get(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key)
616{
617 const struct vm_svar *svar = lep_svar(ec, lep);
618
619 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) return Qnil;
620
621 switch (key) {
622 case VM_SVAR_LASTLINE:
623 return svar->lastline;
624 case VM_SVAR_BACKREF:
625 return svar->backref;
626 default: {
627 const VALUE ary = svar->others;
628
629 if (NIL_P(ary)) {
630 return Qnil;
631 }
632 else {
633 return rb_ary_entry(ary, key - VM_SVAR_EXTRA_START);
634 }
635 }
636 }
637}
638
639static struct vm_svar *
640svar_new(VALUE obj)
641{
642 struct vm_svar *svar = IMEMO_NEW(struct vm_svar, imemo_svar, obj);
643 *((VALUE *)&svar->lastline) = Qnil;
644 *((VALUE *)&svar->backref) = Qnil;
645 *((VALUE *)&svar->others) = Qnil;
646
647 return svar;
648}
649
650static void
651lep_svar_set(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, VALUE val)
652{
653 struct vm_svar *svar = lep_svar(ec, lep);
654
655 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) {
656 lep_svar_write(ec, lep, svar = svar_new((VALUE)svar));
657 }
658
659 switch (key) {
660 case VM_SVAR_LASTLINE:
661 RB_OBJ_WRITE(svar, &svar->lastline, val);
662 return;
663 case VM_SVAR_BACKREF:
664 RB_OBJ_WRITE(svar, &svar->backref, val);
665 return;
666 default: {
667 VALUE ary = svar->others;
668
669 if (NIL_P(ary)) {
670 RB_OBJ_WRITE(svar, &svar->others, ary = rb_ary_new());
671 }
672 rb_ary_store(ary, key - VM_SVAR_EXTRA_START, val);
673 }
674 }
675}
676
677static inline VALUE
678vm_getspecial(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, rb_num_t type)
679{
680 VALUE val;
681
682 if (type == 0) {
683 val = lep_svar_get(ec, lep, key);
684 }
685 else {
686 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
687
688 if (type & 0x01) {
689 switch (type >> 1) {
690 case '&':
691 val = rb_reg_last_match(backref);
692 break;
693 case '`':
694 val = rb_reg_match_pre(backref);
695 break;
696 case '\'':
697 val = rb_reg_match_post(backref);
698 break;
699 case '+':
700 val = rb_reg_match_last(backref);
701 break;
702 default:
703 rb_bug("unexpected back-ref");
704 }
705 }
706 else {
707 val = rb_reg_nth_match((int)(type >> 1), backref);
708 }
709 }
710 return val;
711}
712
713static inline VALUE
714vm_backref_defined(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t type)
715{
716 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
717 int nth = 0;
718
719 if (type & 0x01) {
720 switch (type >> 1) {
721 case '&':
722 case '`':
723 case '\'':
724 break;
725 case '+':
726 return rb_reg_last_defined(backref);
727 default:
728 rb_bug("unexpected back-ref");
729 }
730 }
731 else {
732 nth = (int)(type >> 1);
733 }
734 return rb_reg_nth_defined(nth, backref);
735}
736
737PUREFUNC(static rb_callable_method_entry_t *check_method_entry(VALUE obj, int can_be_svar));
738static rb_callable_method_entry_t *
739check_method_entry(VALUE obj, int can_be_svar)
740{
741 if (obj == Qfalse) return NULL;
742
743#if VM_CHECK_MODE > 0
744 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_method_entry: unknown type: %s", rb_obj_info(obj));
745#endif
746
747 switch (imemo_type(obj)) {
748 case imemo_ment:
749 return (rb_callable_method_entry_t *)obj;
750 case imemo_cref:
751 return NULL;
752 case imemo_svar:
753 if (can_be_svar) {
754 return check_method_entry(((struct vm_svar *)obj)->cref_or_me, FALSE);
755 }
756 default:
757#if VM_CHECK_MODE > 0
758 rb_bug("check_method_entry: svar should not be there:");
759#endif
760 return NULL;
761 }
762}
763
764static rb_callable_method_entry_t *
765env_method_entry_unchecked(VALUE obj, int can_be_svar)
766{
767 if (obj == Qfalse) return NULL;
768
769 switch (imemo_type(obj)) {
770 case imemo_ment:
771 return (rb_callable_method_entry_t *)obj;
772 case imemo_cref:
773 return NULL;
774 case imemo_svar:
775 if (can_be_svar) {
776 return env_method_entry_unchecked(((struct vm_svar *)obj)->cref_or_me, FALSE);
777 }
778 default:
779 return NULL;
780 }
781}
782
783const rb_callable_method_entry_t *
784rb_vm_frame_method_entry(const rb_control_frame_t *cfp)
785{
786 const VALUE *ep = cfp->ep;
787 rb_callable_method_entry_t *me;
788
789 while (!VM_ENV_LOCAL_P(ep)) {
790 if ((me = check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
791 ep = VM_ENV_PREV_EP(ep);
792 }
793
794 return check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
795}
796
797const rb_callable_method_entry_t *
798rb_vm_frame_method_entry_unchecked(const rb_control_frame_t *cfp)
799{
800 const VALUE *ep = cfp->ep;
801 rb_callable_method_entry_t *me;
802
803 while (!VM_ENV_LOCAL_P_UNCHECKED(ep)) {
804 if ((me = env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
805 ep = VM_ENV_PREV_EP_UNCHECKED(ep);
806 }
807
808 return env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
809}
810
811static const rb_iseq_t *
812method_entry_iseqptr(const rb_callable_method_entry_t *me)
813{
814 switch (me->def->type) {
815 case VM_METHOD_TYPE_ISEQ:
816 return me->def->body.iseq.iseqptr;
817 default:
818 return NULL;
819 }
820}
821
822static rb_cref_t *
823method_entry_cref(const rb_callable_method_entry_t *me)
824{
825 switch (me->def->type) {
826 case VM_METHOD_TYPE_ISEQ:
827 return me->def->body.iseq.cref;
828 default:
829 return NULL;
830 }
831}
832
833#if VM_CHECK_MODE == 0
834PUREFUNC(static rb_cref_t *check_cref(VALUE, int));
835#endif
836static rb_cref_t *
837check_cref(VALUE obj, int can_be_svar)
838{
839 if (obj == Qfalse) return NULL;
840
841#if VM_CHECK_MODE > 0
842 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_cref: unknown type: %s", rb_obj_info(obj));
843#endif
844
845 switch (imemo_type(obj)) {
846 case imemo_ment:
847 return method_entry_cref((rb_callable_method_entry_t *)obj);
848 case imemo_cref:
849 return (rb_cref_t *)obj;
850 case imemo_svar:
851 if (can_be_svar) {
852 return check_cref(((struct vm_svar *)obj)->cref_or_me, FALSE);
853 }
854 default:
855#if VM_CHECK_MODE > 0
856 rb_bug("check_method_entry: svar should not be there:");
857#endif
858 return NULL;
859 }
860}
861
862static inline rb_cref_t *
863vm_env_cref(const VALUE *ep)
864{
865 rb_cref_t *cref;
866
867 while (!VM_ENV_LOCAL_P(ep)) {
868 if ((cref = check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return cref;
869 ep = VM_ENV_PREV_EP(ep);
870 }
871
872 return check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
873}
874
875static int
876is_cref(const VALUE v, int can_be_svar)
877{
878 if (RB_TYPE_P(v, T_IMEMO)) {
879 switch (imemo_type(v)) {
880 case imemo_cref:
881 return TRUE;
882 case imemo_svar:
883 if (can_be_svar) return is_cref(((struct vm_svar *)v)->cref_or_me, FALSE);
884 default:
885 break;
886 }
887 }
888 return FALSE;
889}
890
891static int
892vm_env_cref_by_cref(const VALUE *ep)
893{
894 while (!VM_ENV_LOCAL_P(ep)) {
895 if (is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) return TRUE;
896 ep = VM_ENV_PREV_EP(ep);
897 }
898 return is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
899}
900
901static rb_cref_t *
902cref_replace_with_duplicated_cref_each_frame(const VALUE *vptr, int can_be_svar, VALUE parent)
903{
904 const VALUE v = *vptr;
905 rb_cref_t *cref, *new_cref;
906
907 if (RB_TYPE_P(v, T_IMEMO)) {
908 switch (imemo_type(v)) {
909 case imemo_cref:
910 cref = (rb_cref_t *)v;
911 new_cref = vm_cref_dup(cref);
912 if (parent) {
913 RB_OBJ_WRITE(parent, vptr, new_cref);
914 }
915 else {
916 VM_FORCE_WRITE(vptr, (VALUE)new_cref);
917 }
918 return (rb_cref_t *)new_cref;
919 case imemo_svar:
920 if (can_be_svar) {
921 return cref_replace_with_duplicated_cref_each_frame(&((struct vm_svar *)v)->cref_or_me, FALSE, v);
922 }
923 /* fall through */
924 case imemo_ment:
925 rb_bug("cref_replace_with_duplicated_cref_each_frame: unreachable");
926 default:
927 break;
928 }
929 }
930 return NULL;
931}
932
933static rb_cref_t *
934vm_cref_replace_with_duplicated_cref(const VALUE *ep)
935{
936 if (vm_env_cref_by_cref(ep)) {
937 rb_cref_t *cref;
938 VALUE envval;
939
940 while (!VM_ENV_LOCAL_P(ep)) {
941 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
942 if ((cref = cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE, envval)) != NULL) {
943 return cref;
944 }
945 ep = VM_ENV_PREV_EP(ep);
946 }
947 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
948 return cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE, envval);
949 }
950 else {
951 rb_bug("vm_cref_dup: unreachable");
952 }
953}
954
955static rb_cref_t *
956vm_get_cref(const VALUE *ep)
957{
958 rb_cref_t *cref = vm_env_cref(ep);
959
960 if (cref != NULL) {
961 return cref;
962 }
963 else {
964 rb_bug("vm_get_cref: unreachable");
965 }
966}
967
968rb_cref_t *
969rb_vm_get_cref(const VALUE *ep)
970{
971 return vm_get_cref(ep);
972}
973
974static rb_cref_t *
975vm_ec_cref(const rb_execution_context_t *ec)
976{
977 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
978
979 if (cfp == NULL) {
980 return NULL;
981 }
982 return vm_get_cref(cfp->ep);
983}
984
985static const rb_cref_t *
986vm_get_const_key_cref(const VALUE *ep)
987{
988 const rb_cref_t *cref = vm_get_cref(ep);
989 const rb_cref_t *key_cref = cref;
990
991 while (cref) {
992 if (CREF_DYNAMIC(cref)) {
993 return key_cref;
994 }
995 cref = CREF_NEXT(cref);
996 }
997
998 /* no dynamic singleton class or cloned class found */
999 return NULL;
1000}
1001
1002static rb_cref_t *
1003vm_cref_push(const rb_execution_context_t *ec, VALUE klass, const VALUE *ep, int pushed_by_eval, int singleton)
1004{
1005 rb_cref_t *prev_cref = NULL;
1006
1007 if (ep) {
1008 prev_cref = vm_env_cref(ep);
1009 }
1010 else {
1011 rb_control_frame_t *cfp = vm_get_ruby_level_caller_cfp(ec, ec->cfp);
1012
1013 if (cfp) {
1014 prev_cref = vm_env_cref(cfp->ep);
1015 }
1016 }
1017
1018 return vm_cref_new(klass, METHOD_VISI_PUBLIC, FALSE, prev_cref, pushed_by_eval, singleton);
1019}
1020
1021static inline VALUE
1022vm_get_cbase(const VALUE *ep)
1023{
1024 const rb_cref_t *cref = vm_get_cref(ep);
1025
1026 return CREF_CLASS_FOR_DEFINITION(cref);
1027}
1028
1029static inline VALUE
1030vm_get_const_base(const VALUE *ep)
1031{
1032 const rb_cref_t *cref = vm_get_cref(ep);
1033
1034 while (cref) {
1035 if (!CREF_PUSHED_BY_EVAL(cref)) {
1036 return CREF_CLASS_FOR_DEFINITION(cref);
1037 }
1038 cref = CREF_NEXT(cref);
1039 }
1040
1041 return Qundef;
1042}
1043
1044static inline void
1045vm_check_if_namespace(VALUE klass)
1046{
1047 if (!RB_TYPE_P(klass, T_CLASS) && !RB_TYPE_P(klass, T_MODULE)) {
1048 rb_raise(rb_eTypeError, "%+"PRIsVALUE" is not a class/module", klass);
1049 }
1050}
1051
1052static inline void
1053vm_ensure_not_refinement_module(VALUE self)
1054{
1055 if (RB_TYPE_P(self, T_MODULE) && FL_TEST(self, RMODULE_IS_REFINEMENT)) {
1056 rb_warn("not defined at the refinement, but at the outer class/module");
1057 }
1058}
1059
1060static inline VALUE
1061vm_get_iclass(const rb_control_frame_t *cfp, VALUE klass)
1062{
1063 return klass;
1064}
1065
1066static inline VALUE
1067vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, bool allow_nil, int is_defined)
1068{
1069 void rb_const_warn_if_deprecated(const rb_const_entry_t *ce, VALUE klass, ID id);
1070 VALUE val;
1071
1072 if (NIL_P(orig_klass) && allow_nil) {
1073 /* in current lexical scope */
1074 const rb_cref_t *root_cref = vm_get_cref(ec->cfp->ep);
1075 const rb_cref_t *cref;
1076 VALUE klass = Qnil;
1077
1078 while (root_cref && CREF_PUSHED_BY_EVAL(root_cref)) {
1079 root_cref = CREF_NEXT(root_cref);
1080 }
1081 cref = root_cref;
1082 while (cref && CREF_NEXT(cref)) {
1083 if (CREF_PUSHED_BY_EVAL(cref)) {
1084 klass = Qnil;
1085 }
1086 else {
1087 klass = CREF_CLASS(cref);
1088 }
1089 cref = CREF_NEXT(cref);
1090
1091 if (!NIL_P(klass)) {
1092 VALUE av, am = 0;
1093 rb_const_entry_t *ce;
1094 search_continue:
1095 if ((ce = rb_const_lookup(klass, id))) {
1096 rb_const_warn_if_deprecated(ce, klass, id);
1097 val = ce->value;
1098 if (UNDEF_P(val)) {
1099 if (am == klass) break;
1100 am = klass;
1101 if (is_defined) return 1;
1102 if (rb_autoloading_value(klass, id, &av, NULL)) return av;
1103 rb_autoload_load(klass, id);
1104 goto search_continue;
1105 }
1106 else {
1107 if (is_defined) {
1108 return 1;
1109 }
1110 else {
1111 if (UNLIKELY(!rb_ractor_main_p())) {
1112 if (!rb_ractor_shareable_p(val)) {
1113 rb_raise(rb_eRactorIsolationError,
1114 "can not access non-shareable objects in constant %"PRIsVALUE"::%"PRIsVALUE" by non-main ractor.", rb_class_path(klass), rb_id2str(id));
1115 }
1116 }
1117 return val;
1118 }
1119 }
1120 }
1121 }
1122 }
1123
1124 /* search self */
1125 if (root_cref && !NIL_P(CREF_CLASS(root_cref))) {
1126 klass = vm_get_iclass(ec->cfp, CREF_CLASS(root_cref));
1127 }
1128 else {
1129 klass = CLASS_OF(ec->cfp->self);
1130 }
1131
1132 if (is_defined) {
1133 return rb_const_defined(klass, id);
1134 }
1135 else {
1136 return rb_const_get(klass, id);
1137 }
1138 }
1139 else {
1140 vm_check_if_namespace(orig_klass);
1141 if (is_defined) {
1142 return rb_public_const_defined_from(orig_klass, id);
1143 }
1144 else {
1145 return rb_public_const_get_from(orig_klass, id);
1146 }
1147 }
1148}
1149
1150VALUE
1151rb_vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, VALUE allow_nil)
1152{
1153 return vm_get_ev_const(ec, orig_klass, id, allow_nil == Qtrue, 0);
1154}
1155
1156static inline VALUE
1157vm_get_ev_const_chain(rb_execution_context_t *ec, const ID *segments)
1158{
1159 VALUE val = Qnil;
1160 int idx = 0;
1161 int allow_nil = TRUE;
1162 if (segments[0] == idNULL) {
1163 val = rb_cObject;
1164 idx++;
1165 allow_nil = FALSE;
1166 }
1167 while (segments[idx]) {
1168 ID id = segments[idx++];
1169 val = vm_get_ev_const(ec, val, id, allow_nil, 0);
1170 allow_nil = FALSE;
1171 }
1172 return val;
1173}
1174
1175
1176static inline VALUE
1177vm_get_cvar_base(const rb_cref_t *cref, const rb_control_frame_t *cfp, int top_level_raise)
1178{
1179 VALUE klass;
1180
1181 if (!cref) {
1182 rb_bug("vm_get_cvar_base: no cref");
1183 }
1184
1185 while (CREF_NEXT(cref) &&
1186 (NIL_P(CREF_CLASS(cref)) || RCLASS_SINGLETON_P(CREF_CLASS(cref)) ||
1187 CREF_PUSHED_BY_EVAL(cref) || CREF_SINGLETON(cref))) {
1188 cref = CREF_NEXT(cref);
1189 }
1190 if (top_level_raise && !CREF_NEXT(cref)) {
1191 rb_raise(rb_eRuntimeError, "class variable access from toplevel");
1192 }
1193
1194 klass = vm_get_iclass(cfp, CREF_CLASS(cref));
1195
1196 if (NIL_P(klass)) {
1197 rb_raise(rb_eTypeError, "no class variables available");
1198 }
1199 return klass;
1200}
1201
1202#define ractor_incidental_shareable_p(cond, val) \
1203 (!(cond) || rb_ractor_shareable_p(val))
1204#define ractor_object_incidental_shareable_p(obj, val) \
1205 ractor_incidental_shareable_p(rb_ractor_shareable_p(obj), val)
1206
1207ALWAYS_INLINE(static VALUE vm_getivar(VALUE, ID, const rb_iseq_t *, IVC, const struct rb_callcache *, int, VALUE));
1208static inline VALUE
1209vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, VALUE default_value)
1210{
1211 VALUE fields_obj;
1212#if OPT_IC_FOR_IVAR
1213 if (SPECIAL_CONST_P(obj)) {
1214 return default_value;
1215 }
1216
1217 switch (BUILTIN_TYPE(obj)) {
1218 case T_OBJECT:
1219 fields_obj = obj;
1220 break;
1221 case T_CLASS:
1222 case T_MODULE:
1223 {
1224 if (UNLIKELY(!rb_ractor_main_p())) {
1225 // For two reasons we can only use the fast path on the main
1226 // ractor.
1227 // First, only the main ractor is allowed to set ivars on classes
1228 // and modules. So we can skip locking.
1229 // Second, other ractors need to check the shareability of the
1230 // values returned from the class ivars.
1231
1232 if (default_value == Qundef) { // defined?
1233 return rb_ivar_defined(obj, id) ? Qtrue : Qundef;
1234 }
1235 else {
1236 goto general_path;
1237 }
1238 }
1239
1240 fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1241 break;
1242 }
1243 default:
1244 fields_obj = rb_obj_fields(obj, id);
1245 }
1246
1247 if (!fields_obj) {
1248 return default_value;
1249 }
1250
1251 VALUE val = Qundef;
1252
1253 shape_id_t shape_id = RBASIC_SHAPE_ID_FOR_READ(fields_obj);
1254 VALUE *ivar_list = rb_imemo_fields_ptr(fields_obj);
1255
1256 rb_getivar_cache cache = rb_getivar_cache_unpack(vm_cache_attr_index_atomic_read(is_attr, ic, cc));
1257
1258 if (LIKELY(cache.shape_offset == shape_id)) {
1259 if (cache.index == ATTR_INDEX_NOT_SET) {
1260 return default_value;
1261 }
1262
1263 val = ivar_list[cache.index];
1264#if USE_DEBUG_COUNTER
1265 RB_DEBUG_COUNTER_INC(ivar_get_ic_hit);
1266
1267 if (RB_TYPE_P(obj, T_OBJECT)) {
1268 RB_DEBUG_COUNTER_INC(ivar_get_obj_hit);
1269 }
1270#endif
1271 RUBY_ASSERT(!UNDEF_P(val));
1272 }
1273 else { // cache miss case
1274#if USE_DEBUG_COUNTER
1275 if (is_attr) {
1276 if (cache.shape_offset != INVALID_SHAPE_ID) {
1277 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_set);
1278 }
1279 else {
1280 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_unset);
1281 }
1282 }
1283 else {
1284 if (cache.shape_offset != INVALID_SHAPE_ID) {
1285 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_set);
1286 }
1287 else {
1288 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_unset);
1289 }
1290 }
1291 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1292
1293 if (RB_TYPE_P(obj, T_OBJECT)) {
1294 RB_DEBUG_COUNTER_INC(ivar_get_obj_miss);
1295 }
1296#endif
1297
1298 if (UNLIKELY(rb_shape_complex_p(shape_id))) {
1299 st_table *table = (st_table *)ivar_list;
1300
1301 RUBY_ASSERT(table);
1302 RUBY_ASSERT(table == rb_imemo_fields_complex_tbl(fields_obj));
1303
1304 if (!st_lookup(table, id, &val)) {
1305 val = default_value;
1306 }
1307 }
1308 else {
1309 shape_id_t previous_cached_offset = cache.shape_offset;
1310 if (rb_shape_get_iv_index_with_hint(shape_id, id, &cache.index, &cache.shape_offset)) {
1311 if (cache.shape_offset != previous_cached_offset) {
1312 RUBY_ASSERT(!rb_shape_complex_p(cache.shape_offset));
1313 RUBY_ASSERT(cache.shape_offset != INVALID_SHAPE_ID);
1314
1315 uint64_t packed_cache = rb_getivar_cache_pack(cache.shape_offset, cache.index);
1316 vm_cache_attr_index_set(is_attr, ic, cc, packed_cache);
1317 }
1318
1319 if (cache.index == ATTR_INDEX_NOT_SET) {
1320 val = default_value;
1321 }
1322 else {
1323 // We fetched the ivar list above
1324 val = ivar_list[cache.index];
1325 RUBY_ASSERT(!UNDEF_P(val));
1326 }
1327 }
1328 else {
1329 vm_cache_attr_index_set(is_attr, ic, cc, rb_getivar_cache_pack(shape_id, ATTR_INDEX_NOT_SET));
1330 val = default_value;
1331 }
1332 }
1333 }
1334
1335 if (!UNDEF_P(default_value)) {
1336 RUBY_ASSERT(!UNDEF_P(val));
1337 }
1338
1339 return val;
1340
1341general_path:
1342#endif /* OPT_IC_FOR_IVAR */
1343 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1344
1345 if (is_attr) {
1346 return rb_attr_get(obj, id);
1347 }
1348 else {
1349 return rb_ivar_get(obj, id);
1350 }
1351}
1352
1353ALWAYS_INLINE(static VALUE vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr));
1354NOINLINE(static VALUE vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic));
1355NOINLINE(static VALUE vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc));
1356
1357static VALUE
1358vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr)
1359{
1360#if OPT_IC_FOR_IVAR
1361 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss);
1362
1363 rb_check_frozen(obj);
1364
1365 shape_id_t previous_shape_id = RBASIC_SHAPE_ID(obj);
1366 attr_index_t index = rb_ivar_set_index(obj, id, val);
1367 shape_id_t next_shape_id = RBASIC_SHAPE_ID(obj);
1368
1369 if (!rb_shape_complex_p(next_shape_id)) {
1370 uint64_t packed_cache = rb_setivar_cache_pack(RSHAPE_OFFSET(previous_shape_id), RSHAPE_OFFSET(next_shape_id), index);
1371 vm_cache_attr_index_set(is_attr, ic, cc, packed_cache);
1372 }
1373
1374 RB_DEBUG_COUNTER_INC(ivar_set_obj_miss);
1375 return val;
1376#else
1377 return rb_ivar_set(obj, id, val);
1378#endif
1379}
1380
1381static VALUE
1382vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic)
1383{
1384 return vm_setivar_slowpath(obj, id, val, iseq, ic, NULL, false);
1385}
1386
1387static VALUE
1388vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc)
1389{
1390 return vm_setivar_slowpath(obj, id, val, NULL, NULL, cc, true);
1391}
1392
1393NOINLINE(static VALUE vm_setivar_class(VALUE obj, VALUE val, rb_setivar_cache cache));
1394static VALUE
1395vm_setivar_class(VALUE obj, VALUE val, rb_setivar_cache cache)
1396{
1397 if (UNLIKELY(!rb_ractor_main_p())) {
1398 return Qundef;
1399 }
1400
1401 VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1402 if (UNLIKELY(!fields_obj)) {
1403 return Qundef;
1404 }
1405
1406 shape_id_t shape_id = RBASIC_SHAPE_ID(fields_obj);
1407 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, cache);
1408 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1409 return Qundef;
1410 }
1411
1412 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1413
1414 if (shape_id != dest_shape_id) {
1415 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1416 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1417 }
1418
1419 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1420
1421 return val;
1422}
1423
1424NOINLINE(static VALUE vm_setivar_default(VALUE obj, ID id, VALUE val, rb_setivar_cache cache));
1425static VALUE
1426vm_setivar_default(VALUE obj, ID id, VALUE val, rb_setivar_cache cache)
1427{
1428 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1429 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, cache);
1430 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1431 return Qundef;
1432 }
1433
1434 VALUE fields_obj = rb_obj_fields(obj, id);
1435 RUBY_ASSERT(fields_obj);
1436 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1437
1438 if (shape_id != dest_shape_id) {
1439 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1440 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1441 }
1442
1443 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1444
1445 return val;
1446}
1447
1448static inline VALUE
1449vm_setivar(VALUE obj, VALUE val, rb_setivar_cache cache)
1450{
1451#if OPT_IC_FOR_IVAR
1452 switch (BUILTIN_TYPE(obj)) {
1453 case T_OBJECT:
1454 {
1455 VM_ASSERT(!rb_ractor_shareable_p(obj) || rb_obj_frozen_p(obj));
1456
1457 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1458 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, cache);
1459 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1460 break;
1461 }
1462
1463 RB_OBJ_WRITE(obj, &ROBJECT_FIELDS(obj)[cache.index], val);
1464 if (shape_id != dest_shape_id) {
1465 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1466 }
1467
1468 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1469 RB_DEBUG_COUNTER_INC(ivar_set_obj_hit);
1470 return val;
1471 }
1472 break;
1473 case T_CLASS:
1474 case T_MODULE:
1475 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss_noobject);
1476 default:
1477 break;
1478 }
1479
1480 return Qundef;
1481#endif /* OPT_IC_FOR_IVAR */
1482}
1483
1484static VALUE
1485update_classvariable_cache(const rb_iseq_t *iseq, VALUE klass, ID id, const rb_cref_t * cref, ICVARC ic)
1486{
1487 VALUE defined_class = 0;
1488 VALUE cvar_value = rb_cvar_find(klass, id, &defined_class);
1489
1490 if (RB_TYPE_P(defined_class, T_ICLASS)) {
1491 defined_class = RBASIC(defined_class)->klass;
1492 }
1493
1494 VALUE rb_cvc_tbl = RCLASS_CVC_TBL(defined_class);
1495 if (!rb_cvc_tbl) {
1496 rb_bug("the cvc table should be set");
1497 }
1498
1499 VALUE ent_data;
1500 if (!rb_marked_id_table_lookup(rb_cvc_tbl, id, &ent_data)) {
1501 rb_bug("should have cvar cache entry");
1502 }
1503
1504 struct rb_cvar_class_tbl_entry *ent = (void *)ent_data;
1505
1506 ent->global_cvar_state = GET_GLOBAL_CVAR_STATE();
1507 RB_OBJ_WRITE((VALUE)ent, &ent->cref, cref);
1508 RB_OBJ_WRITE(iseq, &ic->entry, ent);
1509
1510 RUBY_ASSERT(BUILTIN_TYPE((VALUE)cref) == T_IMEMO && IMEMO_TYPE_P(cref, imemo_cref));
1511
1512 return cvar_value;
1513}
1514
1515static inline VALUE
1516vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, ICVARC ic)
1517{
1518 const rb_cref_t *cref;
1519 cref = vm_get_cref(GET_EP());
1520
1521 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref && LIKELY(rb_ractor_main_p())) {
1522 RB_DEBUG_COUNTER_INC(cvar_read_inline_hit);
1523
1524 VALUE v = rb_ivar_lookup(ic->entry->class_value, id, Qundef);
1525 RUBY_ASSERT(!UNDEF_P(v));
1526
1527 return v;
1528 }
1529
1530 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1531
1532 return update_classvariable_cache(iseq, klass, id, cref, ic);
1533}
1534
1535VALUE
1536rb_vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, ICVARC ic)
1537{
1538 return vm_getclassvariable(iseq, cfp, id, ic);
1539}
1540
1541static inline void
1542vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, VALUE val, ICVARC ic)
1543{
1544 const rb_cref_t *cref;
1545 cref = vm_get_cref(GET_EP());
1546
1547 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref && LIKELY(rb_ractor_main_p())) {
1548 RB_DEBUG_COUNTER_INC(cvar_write_inline_hit);
1549
1550 rb_class_ivar_set(ic->entry->class_value, id, val);
1551 return;
1552 }
1553
1554 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1555
1556 rb_cvar_set(klass, id, val);
1557
1558 update_classvariable_cache(iseq, klass, id, cref, ic);
1559}
1560
1561void
1562rb_vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, VALUE val, ICVARC ic)
1563{
1564 vm_setclassvariable(iseq, cfp, id, val, ic);
1565}
1566
1567ALWAYS_INLINE(static VALUE vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic));
1568static inline VALUE
1569vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1570{
1571 return vm_getivar(obj, id, iseq, ic, NULL, FALSE, Qnil);
1572}
1573
1574static inline void
1575vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1576{
1577 if (RB_SPECIAL_CONST_P(obj)) {
1578 rb_error_frozen_object(obj);
1579 return;
1580 }
1581
1582 rb_setivar_cache cache = rb_setivar_cache_unpack(vm_ic_atomic_cache_read(ic));
1583 if (UNLIKELY(UNDEF_P(vm_setivar(obj, val, cache)))) {
1584 switch (BUILTIN_TYPE(obj)) {
1585 case T_OBJECT:
1586 break;
1587 case T_CLASS:
1588 case T_MODULE:
1589 if (!UNDEF_P(vm_setivar_class(obj, val, cache))) {
1590 return;
1591 }
1592 break;
1593 default:
1594 if (!UNDEF_P(vm_setivar_default(obj, id, val, cache))) {
1595 return;
1596 }
1597 }
1598 vm_setivar_slowpath_ivar(obj, id, val, iseq, ic);
1599 }
1600}
1601
1602void
1603rb_vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1604{
1605 vm_setinstancevariable(iseq, obj, id, val, ic);
1606}
1607
1608VALUE
1609rb_vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1610{
1611 return vm_getinstancevariable(iseq, obj, id, ic);
1612}
1613
1614static VALUE
1615vm_throw_continue(const rb_execution_context_t *ec, VALUE err)
1616{
1617 /* continue throw */
1618
1619 if (FIXNUM_P(err)) {
1620 ec->tag->state = RUBY_TAG_FATAL;
1621 }
1622 else if (SYMBOL_P(err)) {
1623 ec->tag->state = TAG_THROW;
1624 }
1625 else if (THROW_DATA_P(err)) {
1626 ec->tag->state = THROW_DATA_STATE((struct vm_throw_data *)err);
1627 }
1628 else {
1629 ec->tag->state = TAG_RAISE;
1630 }
1631 return err;
1632}
1633
1634static VALUE
1635vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, enum ruby_tag_type state,
1636 const int flag, const VALUE throwobj)
1637{
1638 const rb_control_frame_t *escape_cfp = NULL;
1639 const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */
1640
1641 if (flag != 0) {
1642 /* do nothing */
1643 }
1644 else if (state == TAG_BREAK) {
1645 int is_orphan = 1;
1646 const VALUE *ep = GET_EP();
1647 const rb_iseq_t *base_iseq = GET_ISEQ();
1648 escape_cfp = reg_cfp;
1649
1650 while (ISEQ_BODY(base_iseq)->type != ISEQ_TYPE_BLOCK) {
1651 if (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) {
1652 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1653 ep = escape_cfp->ep;
1654 base_iseq = CFP_ISEQ(escape_cfp);
1655 }
1656 else {
1657 ep = VM_ENV_PREV_EP(ep);
1658 base_iseq = ISEQ_BODY(base_iseq)->parent_iseq;
1659 escape_cfp = rb_vm_search_cf_from_ep(ec, escape_cfp, ep);
1660 VM_ASSERT(CFP_ISEQ(escape_cfp) == base_iseq);
1661 }
1662 }
1663
1664 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1665 /* lambda{... break ...} */
1666 is_orphan = 0;
1667 state = TAG_RETURN;
1668 }
1669 else {
1670 ep = VM_ENV_PREV_EP(ep);
1671
1672 while (escape_cfp < eocfp) {
1673 if (escape_cfp->ep == ep) {
1674 const rb_iseq_t *const iseq = CFP_ISEQ(escape_cfp);
1675 const VALUE epc = CFP_PC(escape_cfp) - ISEQ_BODY(iseq)->iseq_encoded;
1676 const struct iseq_catch_table *const ct = ISEQ_BODY(iseq)->catch_table;
1677 unsigned int i;
1678
1679 if (!ct) break;
1680 for (i=0; i < ct->size; i++) {
1681 const struct iseq_catch_table_entry *const entry =
1682 UNALIGNED_MEMBER_PTR(ct, entries[i]);
1683
1684 if (entry->type == CATCH_TYPE_BREAK &&
1685 entry->iseq == base_iseq &&
1686 entry->start < epc && entry->end >= epc) {
1687 if (entry->cont == epc) { /* found! */
1688 is_orphan = 0;
1689 }
1690 break;
1691 }
1692 }
1693 break;
1694 }
1695
1696 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1697 }
1698 }
1699
1700 if (is_orphan) {
1701 rb_vm_localjump_error("break from proc-closure", throwobj, TAG_BREAK);
1702 }
1703 }
1704 else if (state == TAG_RETRY) {
1705 const VALUE *ep = VM_ENV_PREV_EP(GET_EP());
1706
1707 escape_cfp = rb_vm_search_cf_from_ep(ec, reg_cfp, ep);
1708 }
1709 else if (state == TAG_RETURN) {
1710 const VALUE *current_ep = GET_EP();
1711 const VALUE *target_ep = NULL, *target_lep, *ep = current_ep;
1712 int in_class_frame = 0;
1713 int toplevel = 1;
1714 escape_cfp = reg_cfp;
1715
1716 // find target_lep, target_ep
1717 while (!VM_ENV_LOCAL_P(ep)) {
1718 if (VM_ENV_FLAGS(ep, VM_FRAME_FLAG_LAMBDA) && target_ep == NULL) {
1719 target_ep = ep;
1720 }
1721 ep = VM_ENV_PREV_EP(ep);
1722 }
1723 target_lep = ep;
1724
1725 while (escape_cfp < eocfp) {
1726 const VALUE *lep = VM_CF_LEP(escape_cfp);
1727
1728 if (!target_lep) {
1729 target_lep = lep;
1730 }
1731
1732 if (lep == target_lep &&
1733 VM_FRAME_RUBYFRAME_P(escape_cfp) &&
1734 ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) {
1735 in_class_frame = 1;
1736 target_lep = 0;
1737 }
1738
1739 if (lep == target_lep) {
1740 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1741 toplevel = 0;
1742 if (in_class_frame) {
1743 /* lambda {class A; ... return ...; end} */
1744 goto valid_return;
1745 }
1746 else {
1747 const VALUE *tep = current_ep;
1748
1749 while (target_lep != tep) {
1750 if (escape_cfp->ep == tep) {
1751 /* in lambda */
1752 if (tep == target_ep) {
1753 goto valid_return;
1754 }
1755 else {
1756 goto unexpected_return;
1757 }
1758 }
1759 tep = VM_ENV_PREV_EP(tep);
1760 }
1761 }
1762 }
1763 else if (VM_FRAME_RUBYFRAME_P(escape_cfp)) {
1764 switch (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type) {
1765 case ISEQ_TYPE_TOP:
1766 case ISEQ_TYPE_MAIN:
1767 if (toplevel) {
1768 if (in_class_frame) goto unexpected_return;
1769 if (target_ep == NULL) {
1770 goto valid_return;
1771 }
1772 else {
1773 goto unexpected_return;
1774 }
1775 }
1776 break;
1777 case ISEQ_TYPE_EVAL: {
1778 const rb_iseq_t *is = CFP_ISEQ(escape_cfp);
1779 enum rb_iseq_type t = ISEQ_BODY(is)->type;
1780 while (t == ISEQ_TYPE_RESCUE || t == ISEQ_TYPE_ENSURE || t == ISEQ_TYPE_EVAL) {
1781 if (!(is = ISEQ_BODY(is)->parent_iseq)) break;
1782 t = ISEQ_BODY(is)->type;
1783 }
1784 toplevel = t == ISEQ_TYPE_TOP || t == ISEQ_TYPE_MAIN;
1785 break;
1786 }
1787 case ISEQ_TYPE_CLASS:
1788 toplevel = 0;
1789 break;
1790 default:
1791 break;
1792 }
1793 }
1794 }
1795
1796 if (escape_cfp->ep == target_lep && ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_METHOD) {
1797 if (target_ep == NULL) {
1798 goto valid_return;
1799 }
1800 else {
1801 goto unexpected_return;
1802 }
1803 }
1804
1805 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1806 }
1807 unexpected_return:;
1808 rb_vm_localjump_error("unexpected return", throwobj, TAG_RETURN);
1809
1810 valid_return:;
1811 /* do nothing */
1812 }
1813 else {
1814 rb_bug("isns(throw): unsupported throw type");
1815 }
1816
1817 ec->tag->state = state;
1818 return (VALUE)THROW_DATA_NEW(throwobj, escape_cfp, state);
1819}
1820
1821static VALUE
1822vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
1823 rb_num_t throw_state, VALUE throwobj)
1824{
1825 const int state = (int)(throw_state & VM_THROW_STATE_MASK);
1826 const int flag = (int)(throw_state & VM_THROW_NO_ESCAPE_FLAG);
1827
1828 if (state != 0) {
1829 return vm_throw_start(ec, reg_cfp, state, flag, throwobj);
1830 }
1831 else {
1832 return vm_throw_continue(ec, throwobj);
1833 }
1834}
1835
1836VALUE
1837rb_vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj)
1838{
1839 return vm_throw(ec, reg_cfp, throw_state, throwobj);
1840}
1841
1842static inline void
1843vm_expandarray(struct rb_control_frame_struct *cfp, VALUE ary, rb_num_t num, int flag)
1844{
1845 int is_splat = flag & 0x01;
1846 const VALUE *ptr;
1847 rb_num_t len;
1848 const VALUE obj = ary;
1849
1850 if (!RB_TYPE_P(ary, T_ARRAY) && NIL_P(ary = rb_check_array_type(ary))) {
1851 ary = obj;
1852 ptr = &ary;
1853 len = 1;
1854 }
1855 else {
1856 ptr = RARRAY_CONST_PTR(ary);
1857 len = (rb_num_t)RARRAY_LEN(ary);
1858 }
1859
1860 if (num + is_splat == 0) {
1861 /* no space left on stack */
1862 }
1863 else if (flag & 0x02) {
1864 /* post: ..., nil ,ary[-1], ..., ary[0..-num] # top */
1865 rb_num_t i = 0, j;
1866
1867 if (len < num) {
1868 for (i = 0; i < num - len; i++) {
1869 *cfp->sp++ = Qnil;
1870 }
1871 }
1872
1873 for (j = 0; i < num; i++, j++) {
1874 VALUE v = ptr[len - j - 1];
1875 *cfp->sp++ = v;
1876 }
1877
1878 if (is_splat) {
1879 *cfp->sp++ = rb_ary_new4(len - j, ptr);
1880 }
1881 }
1882 else {
1883 /* normal: ary[num..-1], ary[num-2], ary[num-3], ..., ary[0] # top */
1884 if (is_splat) {
1885 if (num > len) {
1886 *cfp->sp++ = rb_ary_new();
1887 }
1888 else {
1889 *cfp->sp++ = rb_ary_new4(len - num, ptr + num);
1890 }
1891 }
1892
1893 if (num > len) {
1894 rb_num_t i = 0;
1895 for (; i < num - len; i++) {
1896 *cfp->sp++ = Qnil;
1897 }
1898
1899 for (rb_num_t j = 0; i < num; i++, j++) {
1900 *cfp->sp++ = ptr[len - j - 1];
1901 }
1902 }
1903 else {
1904 for (rb_num_t j = 0; j < num; j++) {
1905 *cfp->sp++ = ptr[num - j - 1];
1906 }
1907 }
1908 }
1909
1910 RB_GC_GUARD(ary);
1911}
1912
1913static VALUE vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
1914
1915static VALUE vm_mtbl_dump(VALUE klass, ID target_mid);
1916
1917static struct rb_class_cc_entries *
1918vm_ccs_create(VALUE klass, VALUE cc_tbl, ID mid, const rb_callable_method_entry_t *cme)
1919{
1920 int initial_capa = 2;
1921 struct rb_class_cc_entries *ccs = ruby_xmalloc(vm_ccs_alloc_size(initial_capa));
1922#if VM_CHECK_MODE > 0
1923 ccs->debug_sig = ~(VALUE)ccs;
1924#endif
1925 ccs->capa = initial_capa;
1926 ccs->len = 0;
1927 ccs->cme = cme;
1928 METHOD_ENTRY_CACHED_SET((rb_callable_method_entry_t *)cme);
1929
1930 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
1931 RB_OBJ_WRITTEN(cc_tbl, Qundef, cme);
1932 return ccs;
1933}
1934
1935static void
1936vm_ccs_push(VALUE cc_tbl, ID mid, struct rb_class_cc_entries *ccs, const struct rb_callinfo *ci, const struct rb_callcache *cc)
1937{
1938 if (! vm_cc_markable(cc)) {
1939 return;
1940 }
1941
1942 if (UNLIKELY(ccs->len == ccs->capa)) {
1943 RUBY_ASSERT(ccs->capa > 0);
1944 ccs->capa *= 2;
1945 ccs = ruby_xrealloc(ccs, vm_ccs_alloc_size(ccs->capa));
1946#if VM_CHECK_MODE > 0
1947 ccs->debug_sig = ~(VALUE)ccs;
1948#endif
1949 // GC?
1950 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
1951 }
1952 VM_ASSERT(ccs->len < ccs->capa);
1953
1954 const int pos = ccs->len++;
1955 ccs->entries[pos].argc = vm_ci_argc(ci);
1956 ccs->entries[pos].flag = vm_ci_flag(ci);
1957 RB_OBJ_WRITE(cc_tbl, &ccs->entries[pos].cc, cc);
1958
1959 if (RB_DEBUG_COUNTER_SETMAX(ccs_maxlen, ccs->len)) {
1960 // for tuning
1961 // vm_mtbl_dump(klass, 0);
1962 }
1963}
1964
1965#if VM_CHECK_MODE > 0
1966void
1967rb_vm_ccs_dump(struct rb_class_cc_entries *ccs)
1968{
1969 ruby_debug_printf("ccs:%p (%d,%d)\n", (void *)ccs, ccs->len, ccs->capa);
1970 for (int i=0; i<ccs->len; i++) {
1971 ruby_debug_printf("CCS CI ID:flag:%x argc:%u\n",
1972 ccs->entries[i].flag,
1973 ccs->entries[i].argc);
1974 rp(ccs->entries[i].cc);
1975 }
1976}
1977
1978static int
1979vm_ccs_verify(struct rb_class_cc_entries *ccs, ID mid, VALUE klass)
1980{
1981 VM_ASSERT(vm_ccs_p(ccs));
1982 VM_ASSERT(ccs->len <= ccs->capa);
1983
1984 for (int i=0; i<ccs->len; i++) {
1985 const struct rb_callcache *cc = ccs->entries[i].cc;
1986
1987 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
1988 VM_ASSERT(vm_cc_class_check(cc, klass));
1989 VM_ASSERT(vm_cc_check_cme(cc, ccs->cme));
1990 VM_ASSERT(!vm_cc_super_p(cc));
1991 VM_ASSERT(!vm_cc_refinement_p(cc));
1992 }
1993 return TRUE;
1994}
1995#endif
1996
1997const rb_callable_method_entry_t *rb_check_overloaded_cme(const rb_callable_method_entry_t *cme, const struct rb_callinfo * const ci);
1998
1999static void
2000vm_evict_cc(VALUE klass, VALUE cc_tbl, ID mid)
2001{
2002 ASSERT_vm_locking();
2003
2004 if (rb_multi_ractor_p()) {
2005 if (RCLASS_WRITABLE_CC_TBL(klass) != cc_tbl) {
2006 // Another ractor updated the CC table while we were waiting on the VM lock.
2007 // We have to retry.
2008 return;
2009 }
2010
2011 VALUE ccs_obj = 0;
2012 rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj);
2013 struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_obj;
2014
2015 if (!ccs || !METHOD_ENTRY_INVALIDATED(ccs->cme)) {
2016 // Another ractor replaced that entry while we were waiting on the VM lock.
2017 return;
2018 }
2019
2020 VALUE new_table = rb_vm_cc_table_dup(cc_tbl);
2021 rb_vm_cc_table_delete(new_table, mid);
2022 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), new_table);
2023 }
2024 else {
2025 rb_vm_cc_table_delete(cc_tbl, mid);
2026 }
2027}
2028
2029static const struct rb_callcache *
2030vm_populate_cc(VALUE klass, const struct rb_callinfo * const ci, ID mid)
2031{
2032 ASSERT_vm_locking();
2033
2034 RB_DEBUG_COUNTER_INC(cc_not_found_in_ccs);
2035
2036 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
2037
2038 VM_ASSERT(cme == NULL || IMEMO_TYPE_P(cme, imemo_ment));
2039
2040 if (cme == NULL) {
2041 // undef or not found: can't cache the information
2042 VM_ASSERT(vm_cc_cme(&vm_empty_cc) == NULL);
2043 return &vm_empty_cc;
2044 }
2045
2046 VALUE cc_tbl = RCLASS_WRITABLE_CC_TBL(klass);
2047 const VALUE original_cc_table = cc_tbl;
2048 if (!cc_tbl) {
2049 // Is this possible after rb_callable_method_entry ?
2050 cc_tbl = rb_vm_cc_table_create(1);
2051 }
2052 else if (rb_multi_ractor_p()) {
2053 cc_tbl = rb_vm_cc_table_dup(cc_tbl);
2054 }
2055
2056 VM_ASSERT(cme == rb_callable_method_entry(klass, mid));
2057
2058 METHOD_ENTRY_CACHED_SET((struct rb_callable_method_entry_struct *)cme);
2059
2060 VM_ASSERT(cc_tbl);
2061
2062 struct rb_class_cc_entries *ccs = NULL;
2063 {
2064 VALUE ccs_obj;
2065 if (UNLIKELY(rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj))) {
2066 ccs = (struct rb_class_cc_entries *)ccs_obj;
2067 }
2068 else {
2069 // TODO: required?
2070 ccs = vm_ccs_create(klass, cc_tbl, mid, cme);
2071 }
2072 }
2073
2074 cme = rb_check_overloaded_cme(cme, ci);
2075
2076 const struct rb_callcache *cc = vm_cc_new(klass, cme, vm_call_general, cc_type_normal);
2077 vm_ccs_push(cc_tbl, mid, ccs, ci, cc);
2078
2079 VM_ASSERT(vm_cc_cme(cc) != NULL);
2080 VM_ASSERT(cme->called_id == mid);
2081 VM_ASSERT(vm_cc_cme(cc)->called_id == mid);
2082
2083 if (original_cc_table != cc_tbl) {
2084 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), cc_tbl);
2085 }
2086
2087 return cc;
2088}
2089
2090static const struct rb_callcache *
2091vm_lookup_cc(const VALUE klass, const struct rb_callinfo * const ci, ID mid)
2092{
2093 VALUE cc_tbl;
2094 struct rb_class_cc_entries *ccs;
2095retry:
2096 cc_tbl = RUBY_ATOMIC_VALUE_LOAD(RCLASS_WRITABLE_CC_TBL(klass));
2097 ccs = NULL;
2098
2099 if (cc_tbl) {
2100 // CCS data is keyed on method id, so we don't need the method id
2101 // for doing comparisons in the `for` loop below.
2102
2103 VALUE ccs_obj;
2104 if (rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj)) {
2105 ccs = (struct rb_class_cc_entries *)ccs_obj;
2106 const int ccs_len = ccs->len;
2107
2108 if (UNLIKELY(METHOD_ENTRY_INVALIDATED(ccs->cme))) {
2109 RB_VM_LOCKING() {
2110 vm_evict_cc(klass, cc_tbl, mid);
2111 }
2112 goto retry;
2113 }
2114 else {
2115 VM_ASSERT(vm_ccs_verify(ccs, mid, klass));
2116
2117 // We already know the method id is correct because we had
2118 // to look up the ccs_data by method id. All we need to
2119 // compare is argc and flag
2120 unsigned int argc = vm_ci_argc(ci);
2121 unsigned int flag = vm_ci_flag(ci);
2122
2123 for (int i=0; i<ccs_len; i++) {
2124 unsigned int ccs_ci_argc = ccs->entries[i].argc;
2125 unsigned int ccs_ci_flag = ccs->entries[i].flag;
2126 const struct rb_callcache *ccs_cc = ccs->entries[i].cc;
2127
2128 VM_ASSERT(IMEMO_TYPE_P(ccs_cc, imemo_callcache));
2129
2130 if (ccs_ci_argc == argc && ccs_ci_flag == flag) {
2131 RB_DEBUG_COUNTER_INC(cc_found_in_ccs);
2132
2133 VM_ASSERT(vm_cc_cme(ccs_cc)->called_id == mid);
2134 VM_ASSERT(ccs_cc->klass == klass);
2135 VM_ASSERT(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(ccs_cc)));
2136
2137 return ccs_cc;
2138 }
2139 }
2140 }
2141 }
2142 }
2143
2144 RB_GC_GUARD(cc_tbl);
2145 return NULL;
2146}
2147
2148static const struct rb_callcache *
2149vm_search_cc(const VALUE klass, const struct rb_callinfo * const ci)
2150{
2151 const ID mid = vm_ci_mid(ci);
2152
2153 const struct rb_callcache *cc = vm_lookup_cc(klass, ci, mid);
2154 if (cc) {
2155 return cc;
2156 }
2157
2158 RB_VM_LOCKING() {
2159 if (rb_multi_ractor_p()) {
2160 // The CC may have been populated by another ractor while we were waiting on the lock,
2161 // so we must lookup a second time.
2162 cc = vm_lookup_cc(klass, ci, mid);
2163 }
2164
2165 if (!cc) {
2166 cc = vm_populate_cc(klass, ci, mid);
2167 }
2168 }
2169
2170 return cc;
2171}
2172
2173const struct rb_callcache *
2174rb_vm_search_method_slowpath(const struct rb_callinfo *ci, VALUE klass)
2175{
2176 const struct rb_callcache *cc;
2177
2178 VM_ASSERT_TYPE2(klass, T_CLASS, T_ICLASS);
2179
2180 cc = vm_search_cc(klass, ci);
2181
2182 VM_ASSERT(cc);
2183 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
2184 VM_ASSERT(cc == vm_cc_empty() || cc->klass == klass);
2185 VM_ASSERT(cc == vm_cc_empty() || callable_method_entry_p(vm_cc_cme(cc)));
2186 VM_ASSERT(cc == vm_cc_empty() || !METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)));
2187 VM_ASSERT(cc == vm_cc_empty() || vm_cc_cme(cc)->called_id == vm_ci_mid(ci));
2188
2189 return cc;
2190}
2191
2192static const struct rb_callcache *
2193vm_search_method_slowpath0(VALUE cd_owner, struct rb_call_data *cd, VALUE klass)
2194{
2195#if USE_DEBUG_COUNTER
2196 const struct rb_callcache *old_cc = cd->cc;
2197#endif
2198
2199 const struct rb_callcache *cc = rb_vm_search_method_slowpath(cd->ci, klass);
2200
2201#if OPT_INLINE_METHOD_CACHE
2202 cd->cc = cc;
2203
2204 const struct rb_callcache *empty_cc = &vm_empty_cc;
2205 if (cd_owner && cc != empty_cc) {
2206 RB_OBJ_WRITTEN(cd_owner, Qundef, cc);
2207 }
2208
2209#if USE_DEBUG_COUNTER
2210 if (!old_cc || old_cc == empty_cc) {
2211 // empty
2212 RB_DEBUG_COUNTER_INC(mc_inline_miss_empty);
2213 }
2214 else if (old_cc == cc) {
2215 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cc);
2216 }
2217 else if (vm_cc_cme(old_cc) == vm_cc_cme(cc)) {
2218 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cme);
2219 }
2220 else if (vm_cc_cme(old_cc) && vm_cc_cme(cc) &&
2221 vm_cc_cme(old_cc)->def == vm_cc_cme(cc)->def) {
2222 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_def);
2223 }
2224 else {
2225 RB_DEBUG_COUNTER_INC(mc_inline_miss_diff);
2226 }
2227#endif
2228#endif // OPT_INLINE_METHOD_CACHE
2229
2230 VM_ASSERT(vm_cc_cme(cc) == NULL ||
2231 vm_cc_cme(cc)->called_id == vm_ci_mid(cd->ci));
2232
2233 return cc;
2234}
2235
2236ALWAYS_INLINE(static const struct rb_callcache *vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE klass));
2237static const struct rb_callcache *
2238vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE klass)
2239{
2240 const struct rb_callcache *cc = cd->cc;
2241
2242 VM_ASSERT_TYPE2(klass, T_CLASS, T_ICLASS);
2243
2244#if OPT_INLINE_METHOD_CACHE
2245 if (LIKELY(vm_cc_class_check(cc, klass))) {
2246 if (LIKELY(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)))) {
2247 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
2248 RB_DEBUG_COUNTER_INC(mc_inline_hit);
2249 VM_ASSERT(vm_cc_cme(cc) == NULL || // not found
2250 (vm_ci_flag(cd->ci) & VM_CALL_SUPER) || // search_super w/ define_method
2251 vm_cc_cme(cc)->called_id == vm_ci_mid(cd->ci)); // cme->called_id == ci->mid
2252
2253 return cc;
2254 }
2255 RB_DEBUG_COUNTER_INC(mc_inline_miss_invalidated);
2256 }
2257 else {
2258 RB_DEBUG_COUNTER_INC(mc_inline_miss_klass);
2259 }
2260#endif
2261
2262 return vm_search_method_slowpath0((VALUE)CFP_ISEQ(reg_cfp), cd, klass);
2263}
2264
2265static const struct rb_callable_method_entry_struct *
2266vm_search_method(struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE recv)
2267{
2268 VALUE klass = CLASS_OF(recv);
2269 VM_ASSERT(klass != Qfalse);
2270 VM_ASSERT(RBASIC_CLASS(klass) == 0 || rb_obj_is_kind_of(klass, rb_cClass));
2271
2272 const struct rb_callcache *cc = vm_search_method_fastpath(reg_cfp, cd, klass);
2273 return vm_cc_cme(cc);
2274}
2275
2276const struct rb_callable_method_entry_struct *
2277rb_zjit_vm_search_method(VALUE cd_owner, struct rb_call_data *cd, VALUE recv)
2278{
2279 // Called from ZJIT with the compile-time iseq, which may differ from
2280 // the iseq on the current CFP. Use the slowpath to avoid stale caches.
2281 VALUE klass = CLASS_OF(recv);
2282 const struct rb_callcache *cc = vm_search_method_slowpath0(cd_owner, cd, klass);
2283 return vm_cc_cme(cc);
2284}
2285
2286#if __has_attribute(transparent_union)
2287typedef union {
2288 VALUE (*anyargs)(ANYARGS);
2289 VALUE (*f00)(VALUE);
2290 VALUE (*f01)(VALUE, VALUE);
2291 VALUE (*f02)(VALUE, VALUE, VALUE);
2292 VALUE (*f03)(VALUE, VALUE, VALUE, VALUE);
2293 VALUE (*f04)(VALUE, VALUE, VALUE, VALUE, VALUE);
2294 VALUE (*f05)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2295 VALUE (*f06)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2296 VALUE (*f07)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2297 VALUE (*f08)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2298 VALUE (*f09)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2299 VALUE (*f10)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2300 VALUE (*f11)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2301 VALUE (*f12)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2302 VALUE (*f13)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2303 VALUE (*f14)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2304 VALUE (*f15)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2305 VALUE (*fm1)(int, union { VALUE *x; const VALUE *y; } __attribute__((__transparent_union__)), VALUE);
2306} __attribute__((__transparent_union__)) cfunc_type;
2307# define make_cfunc_type(f) (cfunc_type){.anyargs = (VALUE (*)(ANYARGS))(f)}
2308#else
2309typedef VALUE (*cfunc_type)(ANYARGS);
2310# define make_cfunc_type(f) (cfunc_type)(f)
2311#endif
2312
2313static inline int
2314check_cfunc(const rb_callable_method_entry_t *me, cfunc_type func)
2315{
2316 if (! me) {
2317 return false;
2318 }
2319 else {
2320 VM_ASSERT(IMEMO_TYPE_P(me, imemo_ment));
2321 VM_ASSERT(callable_method_entry_p(me));
2322 VM_ASSERT(me->def);
2323 if (me->def->type != VM_METHOD_TYPE_CFUNC) {
2324 return false;
2325 }
2326 else {
2327#if __has_attribute(transparent_union)
2328 return me->def->body.cfunc.func == func.anyargs;
2329#else
2330 return me->def->body.cfunc.func == func;
2331#endif
2332 }
2333 }
2334}
2335
2336static inline int
2337check_method_basic_definition(const rb_callable_method_entry_t *me)
2338{
2339 return me && METHOD_ENTRY_BASIC(me);
2340}
2341
2342static inline int
2343vm_method_cfunc_is(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv, cfunc_type func)
2344{
2345 VM_ASSERT(reg_cfp != NULL);
2346 const struct rb_callable_method_entry_struct *cme = vm_search_method(reg_cfp, cd, recv);
2347 return check_cfunc(cme, func);
2348}
2349
2350bool
2351rb_zjit_cme_is_cfunc(const rb_callable_method_entry_t *me, const cfunc_type func)
2352{
2353 return check_cfunc(me, func);
2354}
2355
2356int
2357rb_vm_method_cfunc_is(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv, cfunc_type func)
2358{
2359 // Called from ZJIT with the compile-time iseq, which may differ from
2360 // the iseq on the current CFP. Use the slowpath to avoid stale caches.
2361 VALUE klass = CLASS_OF(recv);
2362 const struct rb_callcache *cc = vm_search_method_slowpath0((VALUE)iseq, cd, klass);
2363 const struct rb_callable_method_entry_struct *cme = vm_cc_cme(cc);
2364 return check_cfunc(cme, func);
2365}
2366
2367#define check_cfunc(me, func) check_cfunc(me, make_cfunc_type(func))
2368#define vm_method_cfunc_is(reg_cfp, cd, recv, func) vm_method_cfunc_is(reg_cfp, cd, recv, make_cfunc_type(func))
2369
2370#define EQ_UNREDEFINED_P(t) BASIC_OP_UNREDEFINED_P(BOP_EQ, t##_REDEFINED_OP_FLAG)
2371
2372static inline bool
2373FIXNUM_2_P(VALUE a, VALUE b)
2374{
2375 /* FIXNUM_P(a) && FIXNUM_P(b)
2376 * == ((a & 1) && (b & 1))
2377 * == a & b & 1 */
2378 SIGNED_VALUE x = a;
2379 SIGNED_VALUE y = b;
2380 SIGNED_VALUE z = x & y & 1;
2381 return z == 1;
2382}
2383
2384static inline bool
2385FLONUM_2_P(VALUE a, VALUE b)
2386{
2387#if USE_FLONUM
2388 /* FLONUM_P(a) && FLONUM_P(b)
2389 * == ((a & 3) == 2) && ((b & 3) == 2)
2390 * == ! ((a ^ 2) | (b ^ 2) & 3)
2391 */
2392 SIGNED_VALUE x = a;
2393 SIGNED_VALUE y = b;
2394 SIGNED_VALUE z = ((x ^ 2) | (y ^ 2)) & 3;
2395 return !z;
2396#else
2397 return false;
2398#endif
2399}
2400
2401static VALUE
2402opt_equality_specialized(VALUE recv, VALUE obj)
2403{
2404 if (FIXNUM_2_P(recv, obj) && EQ_UNREDEFINED_P(INTEGER)) {
2405 goto compare_by_identity;
2406 }
2407 else if (FLONUM_2_P(recv, obj) && EQ_UNREDEFINED_P(FLOAT)) {
2408 goto compare_by_identity;
2409 }
2410 else if (STATIC_SYM_P(recv) && STATIC_SYM_P(obj) && EQ_UNREDEFINED_P(SYMBOL)) {
2411 goto compare_by_identity;
2412 }
2413 else if (SPECIAL_CONST_P(recv)) {
2414 //
2415 }
2416 else if (RBASIC_CLASS(recv) == rb_cFloat && RB_FLOAT_TYPE_P(obj) && EQ_UNREDEFINED_P(FLOAT)) {
2417 double a = RFLOAT_VALUE(recv);
2418 double b = RFLOAT_VALUE(obj);
2419
2420 return RBOOL(a == b);
2421 }
2422 else if (RBASIC_CLASS(recv) == rb_cString && EQ_UNREDEFINED_P(STRING)) {
2423 if (recv == obj) {
2424 return Qtrue;
2425 }
2426 else if (RB_TYPE_P(obj, T_STRING)) {
2427 return rb_str_eql_internal(obj, recv);
2428 }
2429 }
2430 return Qundef;
2431
2432 compare_by_identity:
2433 return RBOOL(recv == obj);
2434}
2435
2436static VALUE
2437opt_equality(struct rb_control_frame_struct *reg_cfp, VALUE recv, VALUE obj, CALL_DATA cd)
2438{
2439 VM_ASSERT(reg_cfp != NULL);
2440
2441 VALUE val = opt_equality_specialized(recv, obj);
2442 if (!UNDEF_P(val)) return val;
2443
2444 if (!vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_equal)) {
2445 return Qundef;
2446 }
2447 else {
2448 return RBOOL(recv == obj);
2449 }
2450}
2451
2452#undef EQ_UNREDEFINED_P
2453
2454static inline const struct rb_callcache *gccct_method_search(rb_execution_context_t *ec, VALUE recv, ID mid, const struct rb_callinfo *ci); // vm_eval.c
2455NOINLINE(static VALUE opt_equality_by_mid_slowpath(VALUE recv, VALUE obj, ID mid));
2456
2457static VALUE
2458opt_equality_by_mid_slowpath(VALUE recv, VALUE obj, ID mid)
2459{
2460 const struct rb_callcache *cc = gccct_method_search(GET_EC(), recv, mid, &VM_CI_ON_STACK(mid, 0, 1, NULL));
2461
2462 if (cc && check_cfunc(vm_cc_cme(cc), rb_obj_equal)) {
2463 return RBOOL(recv == obj);
2464 }
2465 else {
2466 return Qundef;
2467 }
2468}
2469
2470static VALUE
2471opt_equality_by_mid(VALUE recv, VALUE obj, ID mid)
2472{
2473 VALUE val = opt_equality_specialized(recv, obj);
2474 if (!UNDEF_P(val)) {
2475 return val;
2476 }
2477 else {
2478 return opt_equality_by_mid_slowpath(recv, obj, mid);
2479 }
2480}
2481
2482VALUE
2483rb_equal_opt(VALUE obj1, VALUE obj2)
2484{
2485 return opt_equality_by_mid(obj1, obj2, idEq);
2486}
2487
2488VALUE
2489rb_eql_opt(VALUE obj1, VALUE obj2)
2490{
2491 return opt_equality_by_mid(obj1, obj2, idEqlP);
2492}
2493
2494extern VALUE rb_vm_call0(rb_execution_context_t *ec, VALUE, ID, int, const VALUE*, const rb_callable_method_entry_t *, int kw_splat);
2495extern VALUE rb_vm_call_with_refinements(rb_execution_context_t *, VALUE, ID, int, const VALUE *, int);
2496
2497static VALUE
2498check_match(rb_execution_context_t *ec, VALUE pattern, VALUE target, enum vm_check_match_type type)
2499{
2500 switch (type) {
2501 case VM_CHECKMATCH_TYPE_WHEN:
2502 return pattern;
2503 case VM_CHECKMATCH_TYPE_RESCUE:
2504 if (!rb_obj_is_kind_of(pattern, rb_cModule)) {
2505 rb_raise(rb_eTypeError, "class or module required for rescue clause");
2506 }
2507 /* fall through */
2508 case VM_CHECKMATCH_TYPE_CASE: {
2509 return rb_vm_call_with_refinements(ec, pattern, idEqq, 1, &target, RB_NO_KEYWORDS);
2510 }
2511 default:
2512 rb_bug("check_match: unreachable");
2513 }
2514}
2515
2516
2517static inline VALUE
2518double_cmp_lt(double a, double b)
2519{
2520 return RBOOL(a < b);
2521}
2522
2523static inline VALUE
2524double_cmp_le(double a, double b)
2525{
2526 return RBOOL(a <= b);
2527}
2528
2529static inline VALUE
2530double_cmp_gt(double a, double b)
2531{
2532 return RBOOL(a > b);
2533}
2534
2535static inline VALUE
2536double_cmp_ge(double a, double b)
2537{
2538 return RBOOL(a >= b);
2539}
2540
2541// Copied by vm_dump.c
2542static inline VALUE *
2543vm_base_ptr(const rb_control_frame_t *cfp)
2544{
2545 const rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
2546
2547 if (CFP_ISEQ(cfp) && VM_FRAME_RUBYFRAME_P(cfp)) {
2548 VALUE *bp = prev_cfp->sp + ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size + VM_ENV_DATA_SIZE;
2549
2550 if (ISEQ_BODY(CFP_ISEQ(cfp))->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) {
2551 int lts = ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size;
2552 int params = ISEQ_BODY(CFP_ISEQ(cfp))->param.size;
2553
2554 CALL_INFO ci = (CALL_INFO)cfp->ep[-(VM_ENV_DATA_SIZE + (lts - params))]; // skip EP stuff, CI should be last local
2555 bp += vm_ci_argc(ci);
2556 }
2557
2558 if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) {
2559 /* adjust `self' */
2560 bp += 1;
2561 }
2562#if VM_DEBUG_BP_CHECK
2563 if (bp != cfp->bp_check) {
2564 ruby_debug_printf("bp_check: %ld, bp: %ld\n",
2565 (long)(cfp->bp_check - GET_EC()->vm_stack),
2566 (long)(bp - GET_EC()->vm_stack));
2567 rb_bug("vm_base_ptr: unreachable");
2568 }
2569#endif
2570 return bp;
2571 }
2572 else {
2573 return NULL;
2574 }
2575}
2576
2577VALUE *
2578rb_vm_base_ptr(const rb_control_frame_t *cfp)
2579{
2580 return vm_base_ptr(cfp);
2581}
2582
2583/* method call processes with call_info */
2584
2585#include "vm_args.c"
2586
2587static inline VALUE vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc, int param_size, int local_size);
2588ALWAYS_INLINE(static VALUE vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me, int opt_pc, int param_size, int local_size));
2589static inline VALUE vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc);
2590static VALUE vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
2591static VALUE vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2592static VALUE vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2593static inline VALUE vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2594
2595static vm_call_handler vm_call_iseq_setup_func(const struct rb_callinfo *ci, const int param_size, const int local_size);
2596
2597static VALUE
2598vm_call_iseq_setup_tailcall_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2599{
2600 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_tailcall_0start);
2601
2602 return vm_call_iseq_setup_tailcall(ec, cfp, calling, 0);
2603}
2604
2605static VALUE
2606vm_call_iseq_setup_normal_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2607{
2608 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_0start);
2609
2610 const struct rb_callcache *cc = calling->cc;
2611 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2612 int param = ISEQ_BODY(iseq)->param.size;
2613 int local = ISEQ_BODY(iseq)->local_table_size;
2614 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
2615}
2616
2617bool
2618rb_simple_iseq_p(const rb_iseq_t *iseq)
2619{
2620 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2621 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2622 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2623 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2624 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2625 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2626 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2627 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2628 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2629}
2630
2631bool
2632rb_iseq_only_optparam_p(const rb_iseq_t *iseq)
2633{
2634 return ISEQ_BODY(iseq)->param.flags.has_opt == TRUE &&
2635 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2636 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2637 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2638 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2639 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2640 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2641 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2642 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2643}
2644
2645bool
2646rb_iseq_only_kwparam_p(const rb_iseq_t *iseq)
2647{
2648 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2649 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2650 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2651 ISEQ_BODY(iseq)->param.flags.has_kw == TRUE &&
2652 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2653 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2654 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2655 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2656}
2657
2658#define ALLOW_HEAP_ARGV (-2)
2659#define ALLOW_HEAP_ARGV_KEEP_KWSPLAT (-3)
2660
2661static inline bool
2662vm_caller_setup_arg_splat(rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE ary, int max_args)
2663{
2664 vm_check_canary(GET_EC(), cfp->sp);
2665 bool ret = false;
2666
2667 if (!NIL_P(ary)) {
2668 const VALUE *ptr = RARRAY_CONST_PTR(ary);
2669 long len = RARRAY_LEN(ary);
2670 int argc = calling->argc;
2671
2672 if (UNLIKELY(max_args <= ALLOW_HEAP_ARGV && len + argc > VM_ARGC_STACK_MAX)) {
2673 /* Avoid SystemStackError when splatting large arrays by storing arguments in
2674 * a temporary array, instead of trying to keeping arguments on the VM stack.
2675 */
2676 VALUE *argv = cfp->sp - argc;
2677 VALUE argv_ary = rb_ary_hidden_new(len + argc + 1);
2678 rb_ary_cat(argv_ary, argv, argc);
2679 rb_ary_cat(argv_ary, ptr, len);
2680 cfp->sp -= argc - 1;
2681 cfp->sp[-1] = argv_ary;
2682 calling->argc = 1;
2683 calling->heap_argv = argv_ary;
2684 RB_GC_GUARD(ary);
2685 }
2686 else {
2687 long i;
2688
2689 if (max_args >= 0 && len + argc > max_args) {
2690 /* If only a given max_args is allowed, copy up to max args.
2691 * Used by vm_callee_setup_block_arg for non-lambda blocks,
2692 * where additional arguments are ignored.
2693 *
2694 * Also, copy up to one more argument than the maximum,
2695 * in case it is an empty keyword hash that will be removed.
2696 */
2697 calling->argc += len - (max_args - argc + 1);
2698 len = max_args - argc + 1;
2699 ret = true;
2700 }
2701 else {
2702 /* Unset heap_argv if set originally. Can happen when
2703 * forwarding modified arguments, where heap_argv was used
2704 * originally, but heap_argv not supported by the forwarded
2705 * method in all cases.
2706 */
2707 calling->heap_argv = 0;
2708 }
2709 CHECK_VM_STACK_OVERFLOW(cfp, len);
2710
2711 for (i = 0; i < len; i++) {
2712 *cfp->sp++ = ptr[i];
2713 }
2714 calling->argc += i;
2715 }
2716 }
2717
2718 return ret;
2719}
2720
2721static inline void
2722vm_caller_setup_arg_kw(rb_control_frame_t *cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci)
2723{
2724 const VALUE *const passed_keywords = vm_ci_kwarg(ci)->keywords;
2725 const int kw_len = vm_ci_kwarg(ci)->keyword_len;
2726 const VALUE h = rb_hash_new_with_size(kw_len);
2727 VALUE *sp = cfp->sp;
2728 int i;
2729
2730 for (i=0; i<kw_len; i++) {
2731 rb_hash_aset(h, passed_keywords[i], (sp - kw_len)[i]);
2732 }
2733 (sp-kw_len)[0] = h;
2734
2735 cfp->sp -= kw_len - 1;
2736 calling->argc -= kw_len - 1;
2737 calling->kw_splat = 1;
2738}
2739
2740static inline VALUE
2741vm_caller_setup_keyword_hash(const struct rb_callinfo *ci, VALUE keyword_hash)
2742{
2743 if (UNLIKELY(!RB_TYPE_P(keyword_hash, T_HASH))) {
2744 if (keyword_hash != Qnil) {
2745 /* Convert a non-hash keyword splat to a new hash */
2746 keyword_hash = rb_hash_dup(rb_to_hash_type(keyword_hash));
2747 }
2748 }
2749 else if (!IS_ARGS_KW_SPLAT_MUT(ci) && !RHASH_EMPTY_P(keyword_hash)) {
2750 /* Convert a hash keyword splat to a new hash unless
2751 * a mutable keyword splat was passed.
2752 * Skip allocating new hash for empty keyword splat, as empty
2753 * keyword splat will be ignored by both callers.
2754 */
2755 keyword_hash = rb_hash_dup(keyword_hash);
2756 }
2757 return keyword_hash;
2758}
2759
2760static inline void
2761CALLER_SETUP_ARG(struct rb_control_frame_struct *restrict cfp,
2762 struct rb_calling_info *restrict calling,
2763 const struct rb_callinfo *restrict ci, int max_args)
2764{
2765 if (UNLIKELY(IS_ARGS_SPLAT(ci))) {
2766 if (IS_ARGS_KW_SPLAT(ci)) {
2767 // f(*a, **kw)
2768 VM_ASSERT(calling->kw_splat == 1);
2769
2770 cfp->sp -= 2;
2771 calling->argc -= 2;
2772 VALUE ary = cfp->sp[0];
2773 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[1]);
2774
2775 // splat a
2776 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) return;
2777
2778 // put kw
2779 if (kwh != Qnil && !RHASH_EMPTY_P(kwh)) {
2780 if (UNLIKELY(calling->heap_argv)) {
2781 rb_ary_push(calling->heap_argv, kwh);
2782 ((struct RHash *)kwh)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
2783 if (max_args != ALLOW_HEAP_ARGV_KEEP_KWSPLAT) {
2784 calling->kw_splat = 0;
2785 }
2786 }
2787 else {
2788 cfp->sp[0] = kwh;
2789 cfp->sp++;
2790 calling->argc++;
2791
2792 VM_ASSERT(calling->kw_splat == 1);
2793 }
2794 }
2795 else {
2796 calling->kw_splat = 0;
2797 }
2798 }
2799 else {
2800 // f(*a)
2801 VM_ASSERT(calling->kw_splat == 0);
2802
2803 cfp->sp -= 1;
2804 calling->argc -= 1;
2805 VALUE ary = cfp->sp[0];
2806
2807 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) {
2808 goto check_keyword;
2809 }
2810
2811 // check the last argument
2812 VALUE last_hash, argv_ary;
2813 if (UNLIKELY(argv_ary = calling->heap_argv)) {
2814 if (!IS_ARGS_KEYWORD(ci) &&
2815 RARRAY_LEN(argv_ary) > 0 &&
2816 RB_TYPE_P((last_hash = rb_ary_last(0, NULL, argv_ary)), T_HASH) &&
2817 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2818
2819 rb_ary_pop(argv_ary);
2820 if (!RHASH_EMPTY_P(last_hash)) {
2821 rb_ary_push(argv_ary, rb_hash_dup(last_hash));
2822 calling->kw_splat = 1;
2823 }
2824 }
2825 }
2826 else {
2827check_keyword:
2828 if (!IS_ARGS_KEYWORD(ci) &&
2829 calling->argc > 0 &&
2830 RB_TYPE_P((last_hash = cfp->sp[-1]), T_HASH) &&
2831 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2832
2833 if (RHASH_EMPTY_P(last_hash)) {
2834 calling->argc--;
2835 cfp->sp -= 1;
2836 }
2837 else {
2838 cfp->sp[-1] = rb_hash_dup(last_hash);
2839 calling->kw_splat = 1;
2840 }
2841 }
2842 }
2843 }
2844 }
2845 else if (UNLIKELY(IS_ARGS_KW_SPLAT(ci))) {
2846 // f(**kw)
2847 VM_ASSERT(calling->kw_splat == 1);
2848 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[-1]);
2849
2850 if (kwh == Qnil || RHASH_EMPTY_P(kwh)) {
2851 cfp->sp--;
2852 calling->argc--;
2853 calling->kw_splat = 0;
2854 }
2855 else {
2856 cfp->sp[-1] = kwh;
2857 }
2858 }
2859 else if (UNLIKELY(IS_ARGS_KEYWORD(ci))) {
2860 // f(k1:1, k2:2)
2861 VM_ASSERT(calling->kw_splat == 0);
2862
2863 /* This converts VM_CALL_KWARG style to VM_CALL_KW_SPLAT style
2864 * by creating a keyword hash.
2865 * So, vm_ci_flag(ci) & VM_CALL_KWARG is now inconsistent.
2866 */
2867 vm_caller_setup_arg_kw(cfp, calling, ci);
2868 }
2869}
2870
2871#define USE_OPT_HIST 0
2872
2873#if USE_OPT_HIST
2874#define OPT_HIST_MAX 64
2875static int opt_hist[OPT_HIST_MAX+1];
2876
2877__attribute__((destructor))
2878static void
2879opt_hist_show_results_at_exit(void)
2880{
2881 for (int i=0; i<OPT_HIST_MAX; i++) {
2882 ruby_debug_printf("opt_hist\t%d\t%d\n", i, opt_hist[i]);
2883 }
2884}
2885#endif
2886
2887static VALUE
2888vm_call_iseq_setup_normal_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
2889 struct rb_calling_info *calling)
2890{
2891 const struct rb_callcache *cc = calling->cc;
2892 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2893 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
2894 const int opt = calling->argc - lead_num;
2895 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
2896 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
2897 const int param = ISEQ_BODY(iseq)->param.size;
2898 const int local = ISEQ_BODY(iseq)->local_table_size;
2899 const int delta = opt_num - opt;
2900
2901 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
2902
2903#if USE_OPT_HIST
2904 if (opt_pc < OPT_HIST_MAX) {
2905 opt_hist[opt]++;
2906 }
2907 else {
2908 opt_hist[OPT_HIST_MAX]++;
2909 }
2910#endif
2911
2912 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param - delta, local);
2913}
2914
2915static VALUE
2916vm_call_iseq_setup_tailcall_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
2917 struct rb_calling_info *calling)
2918{
2919 const struct rb_callcache *cc = calling->cc;
2920 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2921 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
2922 const int opt = calling->argc - lead_num;
2923 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
2924
2925 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
2926
2927#if USE_OPT_HIST
2928 if (opt_pc < OPT_HIST_MAX) {
2929 opt_hist[opt]++;
2930 }
2931 else {
2932 opt_hist[OPT_HIST_MAX]++;
2933 }
2934#endif
2935
2936 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc);
2937}
2938
2939static void
2940args_setup_kw_parameters(rb_execution_context_t *const ec, const rb_iseq_t *const iseq, const rb_callable_method_entry_t *cme,
2941 VALUE *const passed_values, const int passed_keyword_len, const VALUE *const passed_keywords,
2942 VALUE *const locals);
2943
2944static VALUE
2945vm_call_iseq_forwardable(rb_execution_context_t *ec, rb_control_frame_t *cfp,
2946 struct rb_calling_info *calling)
2947{
2948 const struct rb_callcache *cc = calling->cc;
2949 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2950 int param_size = ISEQ_BODY(iseq)->param.size;
2951 int local_size = ISEQ_BODY(iseq)->local_table_size;
2952
2953 // Setting up local size and param size
2954 VM_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
2955
2956 local_size = local_size + vm_ci_argc(calling->cd->ci);
2957 param_size = param_size + vm_ci_argc(calling->cd->ci);
2958
2959 cfp->sp[0] = (VALUE)calling->cd->ci;
2960
2961 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param_size, local_size);
2962}
2963
2964static VALUE
2965vm_call_iseq_setup_kwparm_kwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
2966 struct rb_calling_info *calling)
2967{
2968 const struct rb_callinfo *ci = calling->cd->ci;
2969 const struct rb_callcache *cc = calling->cc;
2970
2971 VM_ASSERT(vm_ci_flag(ci) & VM_CALL_KWARG);
2972 RB_DEBUG_COUNTER_INC(ccf_iseq_kw1);
2973
2974 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2975 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
2976 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
2977 const int ci_kw_len = kw_arg->keyword_len;
2978 const VALUE * const ci_keywords = kw_arg->keywords;
2979 VALUE *argv = cfp->sp - calling->argc;
2980 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
2981 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
2982 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
2983 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
2984 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
2985
2986 int param = ISEQ_BODY(iseq)->param.size;
2987 int local = ISEQ_BODY(iseq)->local_table_size;
2988 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
2989}
2990
2991static VALUE
2992vm_call_iseq_setup_kwparm_nokwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
2993 struct rb_calling_info *calling)
2994{
2995 const struct rb_callinfo *MAYBE_UNUSED(ci) = calling->cd->ci;
2996 const struct rb_callcache *cc = calling->cc;
2997
2998 VM_ASSERT((vm_ci_flag(ci) & VM_CALL_KWARG) == 0);
2999 RB_DEBUG_COUNTER_INC(ccf_iseq_kw2);
3000
3001 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3002 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3003 VALUE * const argv = cfp->sp - calling->argc;
3004 VALUE * const klocals = argv + kw_param->bits_start - kw_param->num;
3005
3006 int i;
3007 for (i=0; i<kw_param->num; i++) {
3008 klocals[i] = kw_param->default_values[i];
3009 }
3010 klocals[i] = INT2FIX(0); // kw specify flag
3011 // NOTE:
3012 // nobody check this value, but it should be cleared because it can
3013 // points invalid VALUE (T_NONE objects, raw pointer and so on).
3014
3015 int param = ISEQ_BODY(iseq)->param.size;
3016 int local = ISEQ_BODY(iseq)->local_table_size;
3017 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
3018}
3019
3020static VALUE builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
3021
3022static VALUE
3023vm_call_single_noarg_leaf_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3024 struct rb_calling_info *calling)
3025{
3026 const struct rb_builtin_function *bf = calling->cc->aux_.bf;
3027 cfp->sp -= (calling->argc + 1);
3028 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
3029 return builtin_invoker0(ec, calling->recv, NULL, func_ptr);
3030}
3031
3032VALUE rb_gen_method_name(VALUE owner, VALUE name); // in vm_backtrace.c
3033
3034static void
3035warn_unused_block(const rb_callable_method_entry_t *cme, const rb_iseq_t *iseq, void *pc)
3036{
3037 rb_vm_t *vm = GET_VM();
3038 set_table *dup_check_table = &vm->unused_block_warning_table;
3039 st_data_t key;
3040 bool strict_unused_block = rb_warning_category_enabled_p(RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK);
3041
3042 union {
3043 VALUE v;
3044 unsigned char b[SIZEOF_VALUE];
3045 } k1 = {
3046 .v = (VALUE)pc,
3047 }, k2 = {
3048 .v = (VALUE)cme->def,
3049 };
3050
3051 // relax check
3052 if (!strict_unused_block) {
3053 key = (st_data_t)cme->def->original_id;
3054
3055 if (set_table_lookup(dup_check_table, key)) {
3056 return;
3057 }
3058 }
3059
3060 // strict check
3061 // make unique key from pc and me->def pointer
3062 key = 0;
3063 for (int i=0; i<SIZEOF_VALUE; i++) {
3064 // fprintf(stderr, "k1:%3d k2:%3d\n", k1.b[i], k2.b[SIZEOF_VALUE-1-i]);
3065 key |= (st_data_t)(k1.b[i] ^ k2.b[SIZEOF_VALUE-1-i]) << (8 * i);
3066 }
3067
3068 if (0) {
3069 fprintf(stderr, "SIZEOF_VALUE:%d\n", SIZEOF_VALUE);
3070 fprintf(stderr, "pc:%p def:%p\n", pc, (void *)cme->def);
3071 fprintf(stderr, "key:%p\n", (void *)key);
3072 }
3073
3074 // duplication check
3075 if (set_insert(dup_check_table, key)) {
3076 // already shown
3077 }
3078 else if (RTEST(ruby_verbose) || strict_unused_block) {
3079 VALUE m_loc = rb_method_entry_location((const rb_method_entry_t *)cme);
3080 VALUE name = rb_gen_method_name(cme->defined_class, ISEQ_BODY(iseq)->location.base_label);
3081
3082 if (!NIL_P(m_loc)) {
3083 rb_warn("the block passed to '%"PRIsVALUE"' defined at %"PRIsVALUE":%"PRIsVALUE" may be ignored",
3084 name, RARRAY_AREF(m_loc, 0), RARRAY_AREF(m_loc, 1));
3085 }
3086 else {
3087 rb_warn("the block may be ignored because '%"PRIsVALUE"' does not use a block", name);
3088 }
3089 }
3090}
3091
3092static inline int
3093vm_callee_setup_arg(rb_execution_context_t *ec, struct rb_calling_info *calling,
3094 const rb_iseq_t *iseq, VALUE *argv, int param_size, int local_size)
3095{
3096 const struct rb_callinfo *ci = calling->cd->ci;
3097 const struct rb_callcache *cc = calling->cc;
3098
3099 VM_ASSERT((vm_ci_argc(ci), 1));
3100 VM_ASSERT(vm_cc_cme(cc) != NULL);
3101
3102 if (UNLIKELY(!ISEQ_BODY(iseq)->param.flags.use_block &&
3103 calling->block_handler != VM_BLOCK_HANDLER_NONE &&
3104 !(vm_ci_flag(calling->cd->ci) & (VM_CALL_OPT_SEND | VM_CALL_SUPER)))) {
3105 warn_unused_block(vm_cc_cme(cc), iseq, (void *)CFP_PC(ec->cfp));
3106 }
3107
3108 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_KW_SPLAT))) {
3109 if (LIKELY(rb_simple_iseq_p(iseq))) {
3110 rb_control_frame_t *cfp = ec->cfp;
3111 int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3112 CALLER_SETUP_ARG(cfp, calling, ci, lead_num);
3113
3114 if (calling->argc != lead_num) {
3115 argument_arity_error(ec, iseq, vm_cc_cme(cc), calling->argc, lead_num, lead_num);
3116 }
3117
3118 //VM_ASSERT(ci == calling->cd->ci);
3119 VM_ASSERT(cc == calling->cc);
3120
3121 if (vm_call_iseq_optimizable_p(ci, cc)) {
3122 if ((iseq->body->builtin_attrs & BUILTIN_ATTR_SINGLE_NOARG_LEAF) && ruby_vm_c_events_enabled == 0) {
3123 VM_ASSERT(iseq->body->builtin_attrs & BUILTIN_ATTR_LEAF);
3124 vm_cc_bf_set(cc, (void *)iseq->body->iseq_encoded[1]);
3125 CC_SET_FASTPATH(cc, vm_call_single_noarg_leaf_builtin, true);
3126 }
3127 else {
3128 CC_SET_FASTPATH(cc, vm_call_iseq_setup_func(ci, param_size, local_size), true);
3129 }
3130 }
3131 return 0;
3132 }
3133 else if (rb_iseq_only_optparam_p(iseq)) {
3134 rb_control_frame_t *cfp = ec->cfp;
3135
3136 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3137 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
3138
3139 CALLER_SETUP_ARG(cfp, calling, ci, lead_num + opt_num);
3140 const int argc = calling->argc;
3141 const int opt = argc - lead_num;
3142
3143 if (opt < 0 || opt > opt_num) {
3144 argument_arity_error(ec, iseq, vm_cc_cme(cc), argc, lead_num, lead_num + opt_num);
3145 }
3146
3147 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3148 CC_SET_FASTPATH(cc, vm_call_iseq_setup_normal_opt_start,
3149 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3150 vm_call_cacheable(ci, cc));
3151 }
3152 else {
3153 CC_SET_FASTPATH(cc, vm_call_iseq_setup_tailcall_opt_start,
3154 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3155 vm_call_cacheable(ci, cc));
3156 }
3157
3158 /* initialize opt vars for self-references */
3159 VM_ASSERT((int)ISEQ_BODY(iseq)->param.size == lead_num + opt_num);
3160 for (int i=argc; i<lead_num + opt_num; i++) {
3161 argv[i] = Qnil;
3162 }
3163 return (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3164 }
3165 else if (rb_iseq_only_kwparam_p(iseq) && !IS_ARGS_SPLAT(ci)) {
3166 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3167 const int argc = calling->argc;
3168 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3169
3170 if (vm_ci_flag(ci) & VM_CALL_KWARG) {
3171 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
3172
3173 if (argc - kw_arg->keyword_len == lead_num) {
3174 const int ci_kw_len = kw_arg->keyword_len;
3175 const VALUE * const ci_keywords = kw_arg->keywords;
3176 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
3177 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
3178
3179 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3180 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
3181
3182 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_kwarg,
3183 vm_call_cacheable(ci, cc));
3184
3185 return 0;
3186 }
3187 }
3188 else if (argc == lead_num) {
3189 /* no kwarg */
3190 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3191 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), NULL, 0, NULL, klocals);
3192
3193 if (klocals[kw_param->num] == INT2FIX(0)) {
3194 /* copy from default_values */
3195 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_nokwarg,
3196 vm_call_cacheable(ci, cc));
3197 }
3198
3199 return 0;
3200 }
3201 }
3202 }
3203
3204 // Called iseq is using ... param
3205 // def foo(...) # <- iseq for foo will have "forwardable"
3206 //
3207 // We want to set the `...` local to the caller's CI
3208 // foo(1, 2) # <- the ci for this should end up as `...`
3209 //
3210 // So hopefully the stack looks like:
3211 //
3212 // => 1
3213 // => 2
3214 // => *
3215 // => **
3216 // => &
3217 // => ... # <- points at `foo`s CI
3218 // => cref_or_me
3219 // => specval
3220 // => type
3221 //
3222 if (ISEQ_BODY(iseq)->param.flags.forwardable) {
3223 bool can_fastpath = true;
3224
3225 if ((vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
3226 struct rb_forwarding_call_data * forward_cd = (struct rb_forwarding_call_data *)calling->cd;
3227 if (vm_ci_argc(ci) != vm_ci_argc(forward_cd->caller_ci)) {
3228 ci = vm_ci_new_runtime(
3229 vm_ci_mid(ci),
3230 vm_ci_flag(ci),
3231 vm_ci_argc(ci),
3232 vm_ci_kwarg(ci));
3233 }
3234 else {
3235 ci = forward_cd->caller_ci;
3236 }
3237 can_fastpath = false;
3238 }
3239 // C functions calling iseqs will stack allocate a CI,
3240 // so we need to convert it to heap allocated
3241 if (!vm_ci_markable(ci)) {
3242 ci = vm_ci_new_runtime(
3243 vm_ci_mid(ci),
3244 vm_ci_flag(ci),
3245 vm_ci_argc(ci),
3246 vm_ci_kwarg(ci));
3247 can_fastpath = false;
3248 }
3249 argv[param_size - 1] = (VALUE)ci;
3250 CC_SET_FASTPATH(cc, vm_call_iseq_forwardable, can_fastpath);
3251 return 0;
3252 }
3253
3254 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_method);
3255}
3256
3257static void
3258vm_adjust_stack_forwarding(const struct rb_execution_context_struct *ec, struct rb_control_frame_struct *cfp, int argc, VALUE splat)
3259{
3260 // This case is when the caller is using a ... parameter.
3261 // For example `bar(...)`. The call info will have VM_CALL_FORWARDING
3262 // In this case the caller's caller's CI will be on the stack.
3263 //
3264 // For example:
3265 //
3266 // def bar(a, b); a + b; end
3267 // def foo(...); bar(...); end
3268 // foo(1, 2) # <- this CI will be on the stack when we call `bar(...)`
3269 //
3270 // Stack layout will be:
3271 //
3272 // > 1
3273 // > 2
3274 // > CI for foo(1, 2)
3275 // > cref_or_me
3276 // > specval
3277 // > type
3278 // > receiver
3279 // > CI for foo(1, 2), via `getlocal ...`
3280 // > ( SP points here )
3281 const VALUE * lep = VM_CF_LEP(cfp);
3282
3283 const rb_iseq_t *iseq;
3284
3285 // If we're in an escaped environment (lambda for example), get the iseq
3286 // from the captured env.
3287 if (VM_ENV_FLAGS(lep, VM_ENV_FLAG_ESCAPED)) {
3288 rb_env_t * env = (rb_env_t *)lep[VM_ENV_DATA_INDEX_ENV];
3289 iseq = env->iseq;
3290 }
3291 else { // Otherwise use the lep to find the caller
3292 iseq = CFP_ISEQ(rb_vm_search_cf_from_ep(ec, cfp, lep));
3293 }
3294
3295 // Our local storage is below the args we need to copy
3296 int local_size = ISEQ_BODY(iseq)->local_table_size + argc;
3297
3298 const VALUE * from = lep - (local_size + VM_ENV_DATA_SIZE - 1); // 2 for EP values
3299 VALUE * to = cfp->sp - 1; // clobber the CI
3300
3301 if (RTEST(splat)) {
3302 to -= 1; // clobber the splat array
3303 CHECK_VM_STACK_OVERFLOW0(cfp, to, RARRAY_LEN(splat));
3304 MEMCPY(to, RARRAY_CONST_PTR(splat), VALUE, RARRAY_LEN(splat));
3305 to += RARRAY_LEN(splat);
3306 }
3307
3308 CHECK_VM_STACK_OVERFLOW0(cfp, to, argc);
3309 MEMCPY(to, from, VALUE, argc);
3310 cfp->sp = to + argc;
3311
3312 // Stack layout should now be:
3313 //
3314 // > 1
3315 // > 2
3316 // > CI for foo(1, 2)
3317 // > cref_or_me
3318 // > specval
3319 // > type
3320 // > receiver
3321 // > 1
3322 // > 2
3323 // > ( SP points here )
3324}
3325
3326static VALUE
3327vm_call_iseq_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3328{
3329 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3330
3331 const struct rb_callcache *cc = calling->cc;
3332 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3333 int param_size = ISEQ_BODY(iseq)->param.size;
3334 int local_size = ISEQ_BODY(iseq)->local_table_size;
3335
3336 RUBY_ASSERT(!ISEQ_BODY(iseq)->param.flags.forwardable);
3337
3338 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3339 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3340}
3341
3342static VALUE
3343vm_call_iseq_fwd_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3344{
3345 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3346
3347 const struct rb_callcache *cc = calling->cc;
3348 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3349 int param_size = ISEQ_BODY(iseq)->param.size;
3350 int local_size = ISEQ_BODY(iseq)->local_table_size;
3351
3352 RUBY_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
3353
3354 // Setting up local size and param size
3355 local_size = local_size + vm_ci_argc(calling->cd->ci);
3356 param_size = param_size + vm_ci_argc(calling->cd->ci);
3357
3358 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3359 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3360}
3361
3362static inline VALUE
3363vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
3364 int opt_pc, int param_size, int local_size)
3365{
3366 const struct rb_callinfo *ci = calling->cd->ci;
3367 const struct rb_callcache *cc = calling->cc;
3368
3369 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3370 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param_size, local_size);
3371 }
3372 else {
3373 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc);
3374 }
3375}
3376
3377static inline VALUE
3378vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me,
3379 int opt_pc, int param_size, int local_size)
3380{
3381 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3382 VALUE *argv = cfp->sp - calling->argc;
3383 VALUE *sp = argv + param_size;
3384 cfp->sp = argv - 1 /* recv */;
3385
3386 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL, calling->recv,
3387 calling->block_handler, (VALUE)me,
3388 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3389 local_size - param_size,
3390 ISEQ_BODY(iseq)->stack_max);
3391 return Qundef;
3392}
3393
3394static inline VALUE
3395vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc)
3396{
3397 const struct rb_callcache *cc = calling->cc;
3398 unsigned int i;
3399 VALUE *argv = cfp->sp - calling->argc;
3400 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3401 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3402 VALUE *src_argv = argv;
3403 VALUE *sp_orig, *sp;
3404 VALUE finish_flag = VM_FRAME_FINISHED_P(cfp) ? VM_FRAME_FLAG_FINISH : 0;
3405
3406 if (VM_BH_FROM_CFP_P(calling->block_handler, cfp)) {
3407 struct rb_captured_block *dst_captured = VM_CFP_TO_CAPTURED_BLOCK(RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp));
3408 const struct rb_captured_block *src_captured = VM_BH_TO_CAPT_BLOCK(calling->block_handler);
3409 dst_captured->code.val = src_captured->code.val;
3410 if (VM_BH_ISEQ_BLOCK_P(calling->block_handler)) {
3411 calling->block_handler = VM_BH_FROM_ISEQ_BLOCK(dst_captured);
3412 }
3413 else {
3414 calling->block_handler = VM_BH_FROM_IFUNC_BLOCK(dst_captured);
3415 }
3416 }
3417
3418 vm_pop_frame(ec, cfp, cfp->ep);
3419 cfp = ec->cfp;
3420
3421 sp_orig = sp = cfp->sp;
3422
3423 /* push self */
3424 sp[0] = calling->recv;
3425 sp++;
3426
3427 /* copy arguments */
3428 for (i=0; i < ISEQ_BODY(iseq)->param.size; i++) {
3429 *sp++ = src_argv[i];
3430 }
3431
3432 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL | finish_flag,
3433 calling->recv, calling->block_handler, (VALUE)me,
3434 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3435 ISEQ_BODY(iseq)->local_table_size - ISEQ_BODY(iseq)->param.size,
3436 ISEQ_BODY(iseq)->stack_max);
3437
3438 cfp->sp = sp_orig;
3439
3440 return Qundef;
3441}
3442
3443static void
3444ractor_unsafe_check(void)
3445{
3446 if (!rb_ractor_main_p()) {
3447 rb_raise(rb_eRactorUnsafeError, "ractor unsafe method called from not main ractor");
3448 }
3449}
3450
3451static VALUE
3452call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3453{
3454 ractor_unsafe_check();
3455 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3456 return (*f)(recv, rb_ary_new4(argc, argv));
3457}
3458
3459static VALUE
3460call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3461{
3462 ractor_unsafe_check();
3463 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3464 return (*f)(argc, argv, recv);
3465}
3466
3467static VALUE
3468call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3469{
3470 ractor_unsafe_check();
3471 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3472 return (*f)(recv);
3473}
3474
3475static VALUE
3476call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3477{
3478 ractor_unsafe_check();
3479 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3480 return (*f)(recv, argv[0]);
3481}
3482
3483static VALUE
3484call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3485{
3486 ractor_unsafe_check();
3487 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3488 return (*f)(recv, argv[0], argv[1]);
3489}
3490
3491static VALUE
3492call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3493{
3494 ractor_unsafe_check();
3495 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3496 return (*f)(recv, argv[0], argv[1], argv[2]);
3497}
3498
3499static VALUE
3500call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3501{
3502 ractor_unsafe_check();
3503 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3504 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3505}
3506
3507static VALUE
3508call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3509{
3510 ractor_unsafe_check();
3511 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3512 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3513}
3514
3515static VALUE
3516call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3517{
3518 ractor_unsafe_check();
3519 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3520 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3521}
3522
3523static VALUE
3524call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3525{
3526 ractor_unsafe_check();
3527 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3528 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3529}
3530
3531static VALUE
3532call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3533{
3534 ractor_unsafe_check();
3535 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3536 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3537}
3538
3539static VALUE
3540call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3541{
3542 ractor_unsafe_check();
3543 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3544 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3545}
3546
3547static VALUE
3548call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3549{
3550 ractor_unsafe_check();
3551 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3552 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3553}
3554
3555static VALUE
3556call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3557{
3558 ractor_unsafe_check();
3559 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3560 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3561}
3562
3563static VALUE
3564call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3565{
3566 ractor_unsafe_check();
3567 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3568 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3569}
3570
3571static VALUE
3572call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3573{
3574 ractor_unsafe_check();
3575 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3576 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3577}
3578
3579static VALUE
3580call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3581{
3582 ractor_unsafe_check();
3583 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3584 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3585}
3586
3587static VALUE
3588call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3589{
3590 ractor_unsafe_check();
3591 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3592 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3593}
3594
3595static VALUE
3596ractor_safe_call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3597{
3598 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3599 return (*f)(recv, rb_ary_new4(argc, argv));
3600}
3601
3602static VALUE
3603ractor_safe_call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3604{
3605 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3606 return (*f)(argc, argv, recv);
3607}
3608
3609static VALUE
3610ractor_safe_call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3611{
3612 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3613 return (*f)(recv);
3614}
3615
3616static VALUE
3617ractor_safe_call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3618{
3619 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3620 return (*f)(recv, argv[0]);
3621}
3622
3623static VALUE
3624ractor_safe_call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3625{
3626 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3627 return (*f)(recv, argv[0], argv[1]);
3628}
3629
3630static VALUE
3631ractor_safe_call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3632{
3633 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3634 return (*f)(recv, argv[0], argv[1], argv[2]);
3635}
3636
3637static VALUE
3638ractor_safe_call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3639{
3640 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3641 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3642}
3643
3644static VALUE
3645ractor_safe_call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3646{
3647 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3648 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3649}
3650
3651static VALUE
3652ractor_safe_call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3653{
3654 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3655 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3656}
3657
3658static VALUE
3659ractor_safe_call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3660{
3661 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3662 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3663}
3664
3665static VALUE
3666ractor_safe_call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3667{
3668 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3669 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3670}
3671
3672static VALUE
3673ractor_safe_call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3674{
3675 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3676 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3677}
3678
3679static VALUE
3680ractor_safe_call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3681{
3682 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3683 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3684}
3685
3686static VALUE
3687ractor_safe_call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3688{
3689 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3690 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3691}
3692
3693static VALUE
3694ractor_safe_call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3695{
3696 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3697 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3698}
3699
3700static VALUE
3701ractor_safe_call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3702{
3703 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3704 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3705}
3706
3707static VALUE
3708ractor_safe_call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3709{
3710 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3711 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3712}
3713
3714static VALUE
3715ractor_safe_call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3716{
3717 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3718 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3719}
3720
3721static inline int
3722vm_cfp_consistent_p(rb_execution_context_t *ec, const rb_control_frame_t *reg_cfp)
3723{
3724 const int ov_flags = RAISED_STACKOVERFLOW;
3725 if (LIKELY(reg_cfp == ec->cfp + 1)) return TRUE;
3726 if (rb_ec_raised_p(ec, ov_flags)) {
3727 rb_ec_raised_reset(ec, ov_flags);
3728 return TRUE;
3729 }
3730 return FALSE;
3731}
3732
3733#define CHECK_CFP_CONSISTENCY(func) \
3734 (LIKELY(vm_cfp_consistent_p(ec, reg_cfp)) ? (void)0 : \
3735 rb_bug(func ": cfp consistency error (%p, %p)", (void *)reg_cfp, (void *)(ec->cfp+1)))
3736
3737static inline
3738const rb_method_cfunc_t *
3739vm_method_cfunc_entry(const rb_callable_method_entry_t *me)
3740{
3741#if VM_DEBUG_VERIFY_METHOD_CACHE
3742 switch (me->def->type) {
3743 case VM_METHOD_TYPE_CFUNC:
3744 case VM_METHOD_TYPE_NOTIMPLEMENTED:
3745 break;
3746# define METHOD_BUG(t) case VM_METHOD_TYPE_##t: rb_bug("wrong method type: " #t)
3747 METHOD_BUG(ISEQ);
3748 METHOD_BUG(ATTRSET);
3749 METHOD_BUG(IVAR);
3750 METHOD_BUG(BMETHOD);
3751 METHOD_BUG(ZSUPER);
3752 METHOD_BUG(UNDEF);
3753 METHOD_BUG(OPTIMIZED);
3754 METHOD_BUG(MISSING);
3755 METHOD_BUG(REFINED);
3756 METHOD_BUG(ALIAS);
3757# undef METHOD_BUG
3758 default:
3759 rb_bug("wrong method type: %d", me->def->type);
3760 }
3761#endif
3762 return UNALIGNED_MEMBER_PTR(me->def, body.cfunc);
3763}
3764
3765static VALUE
3766vm_call_cfunc_with_frame_(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
3767 int argc, VALUE *argv, VALUE *stack_bottom)
3768{
3769 RB_DEBUG_COUNTER_INC(ccf_cfunc_with_frame);
3770 const struct rb_callinfo *ci = calling->cd->ci;
3771 const struct rb_callcache *cc = calling->cc;
3772 VALUE val;
3773 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3774 const rb_method_cfunc_t *cfunc = vm_method_cfunc_entry(me);
3775
3776 VALUE recv = calling->recv;
3777 VALUE block_handler = calling->block_handler;
3778 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3779
3780 if (UNLIKELY(calling->kw_splat)) {
3781 frame_type |= VM_FRAME_FLAG_CFRAME_KW;
3782 }
3783
3784 VM_ASSERT(reg_cfp == ec->cfp);
3785
3786 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, me->owner, me->def->original_id);
3787 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, recv, me->def->original_id, vm_ci_mid(ci), me->owner, Qundef);
3788
3789 vm_push_frame(ec, NULL, frame_type, recv,
3790 block_handler, (VALUE)me,
3791 0, ec->cfp->sp, 0, 0);
3792
3793 int len = cfunc->argc;
3794 if (len >= 0) rb_check_arity(argc, len, len);
3795
3796 reg_cfp->sp = stack_bottom;
3797 val = (*cfunc->invoker)(recv, argc, argv, cfunc->func);
3798
3799 CHECK_CFP_CONSISTENCY("vm_call_cfunc");
3800
3801 rb_vm_pop_frame(ec);
3802
3803 VM_ASSERT(ec->cfp->sp == stack_bottom);
3804
3805 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, recv, me->def->original_id, vm_ci_mid(ci), me->owner, val);
3806 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
3807
3808 return val;
3809}
3810
3811// Push a C method frame for a given cme. This is called when JIT code skipped
3812// pushing a frame but the C method reached a point where a frame is needed.
3813void
3814rb_vm_push_cfunc_frame(const rb_callable_method_entry_t *cme, int recv_idx)
3815{
3816 VM_ASSERT(cme->def->type == VM_METHOD_TYPE_CFUNC);
3817 rb_execution_context_t *ec = GET_EC();
3818 VALUE *sp = ec->cfp->sp;
3819 VALUE recv = *(sp - recv_idx - 1);
3820 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3821 VALUE block_handler = VM_BLOCK_HANDLER_NONE;
3822#if VM_CHECK_MODE > 0
3823 // Clean up the stack canary since we're about to satisfy the "leaf or lazy push" assumption
3824 *(GET_EC()->cfp->sp) = Qfalse;
3825#endif
3826 vm_push_frame(ec, NULL, frame_type, recv, block_handler, (VALUE)cme, 0, ec->cfp->sp, 0, 0);
3827}
3828
3829// If true, cc->call needs to include `CALLER_SETUP_ARG` (i.e. can't be skipped in fastpath)
3830bool
3831rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci)
3832{
3833 return IS_ARGS_SPLAT(ci) || IS_ARGS_KW_OR_KW_SPLAT(ci);
3834}
3835
3836static VALUE
3837vm_call_cfunc_with_frame(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3838{
3839 int argc = calling->argc;
3840 VALUE *stack_bottom = reg_cfp->sp - argc - 1;
3841 VALUE *argv = &stack_bottom[1];
3842
3843 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
3844}
3845
3846static VALUE
3847vm_call_cfunc_other(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3848{
3849 const struct rb_callinfo *ci = calling->cd->ci;
3850 RB_DEBUG_COUNTER_INC(ccf_cfunc_other);
3851
3852 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
3853 VALUE argv_ary;
3854 if (UNLIKELY(argv_ary = calling->heap_argv)) {
3855 VM_ASSERT(!IS_ARGS_KEYWORD(ci));
3856 int argc = RARRAY_LENINT(argv_ary);
3857 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
3858 VALUE *stack_bottom = reg_cfp->sp - 2;
3859
3860 VM_ASSERT(calling->argc == 1);
3861 VM_ASSERT(RB_TYPE_P(argv_ary, T_ARRAY));
3862 VM_ASSERT(RBASIC_CLASS(argv_ary) == 0); // hidden ary
3863
3864 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
3865 }
3866 else {
3867 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_with_frame, !rb_splat_or_kwargs_p(ci) && !calling->kw_splat && !(vm_ci_flag(ci) & VM_CALL_FORWARDING));
3868
3869 return vm_call_cfunc_with_frame(ec, reg_cfp, calling);
3870 }
3871}
3872
3873static inline VALUE
3874vm_call_cfunc_array_argv(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int stack_offset, int argc_offset)
3875{
3876 VALUE argv_ary = reg_cfp->sp[-1 - stack_offset];
3877 int argc = RARRAY_LENINT(argv_ary) - argc_offset;
3878
3879 if (UNLIKELY(argc > VM_ARGC_STACK_MAX)) {
3880 return vm_call_cfunc_other(ec, reg_cfp, calling);
3881 }
3882
3883 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
3884 calling->kw_splat = 0;
3885 int i;
3886 VALUE *stack_bottom = reg_cfp->sp - 2 - stack_offset;
3887 VALUE *sp = stack_bottom;
3888 CHECK_VM_STACK_OVERFLOW(reg_cfp, argc);
3889 for(i = 0; i < argc; i++) {
3890 *++sp = argv[i];
3891 }
3892 reg_cfp->sp = sp+1;
3893
3894 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, stack_bottom+1, stack_bottom);
3895}
3896
3897static inline VALUE
3898vm_call_cfunc_only_splat(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3899{
3900 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat);
3901 VALUE argv_ary = reg_cfp->sp[-1];
3902 int argc = RARRAY_LENINT(argv_ary);
3903 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
3904 VALUE last_hash;
3905 int argc_offset = 0;
3906
3907 if (UNLIKELY(argc > 0 &&
3908 RB_TYPE_P((last_hash = argv[argc-1]), T_HASH) &&
3909 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS))) {
3910 if (!RHASH_EMPTY_P(last_hash)) {
3911 return vm_call_cfunc_other(ec, reg_cfp, calling);
3912 }
3913 argc_offset++;
3914 }
3915 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 0, argc_offset);
3916}
3917
3918static inline VALUE
3919vm_call_cfunc_only_splat_kw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3920{
3921 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat_kw);
3922 VALUE keyword_hash = reg_cfp->sp[-1];
3923
3924 if (keyword_hash == Qnil || (RB_TYPE_P(keyword_hash, T_HASH) && RHASH_EMPTY_P(keyword_hash))) {
3925 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 1, 0);
3926 }
3927
3928 return vm_call_cfunc_other(ec, reg_cfp, calling);
3929}
3930
3931static VALUE
3932vm_call_cfunc(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3933{
3934 const struct rb_callinfo *ci = calling->cd->ci;
3935 RB_DEBUG_COUNTER_INC(ccf_cfunc);
3936
3937 if (IS_ARGS_SPLAT(ci) && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
3938 if (!IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 1) {
3939 // f(*a)
3940 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat, TRUE);
3941 return vm_call_cfunc_only_splat(ec, reg_cfp, calling);
3942 }
3943 if (IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 2) {
3944 // f(*a, **kw)
3945 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat_kw, TRUE);
3946 return vm_call_cfunc_only_splat_kw(ec, reg_cfp, calling);
3947 }
3948 }
3949
3950 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_other, TRUE);
3951 return vm_call_cfunc_other(ec, reg_cfp, calling);
3952}
3953
3954static VALUE
3955vm_call_ivar(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3956{
3957 const struct rb_callcache *cc = calling->cc;
3958 RB_DEBUG_COUNTER_INC(ccf_ivar);
3959 cfp->sp -= 1;
3960 VALUE ivar = vm_getivar(calling->recv, vm_cc_cme(cc)->def->body.attr.id, NULL, NULL, cc, TRUE, Qnil);
3961 return ivar;
3962}
3963
3964static VALUE
3965vm_call_attrset_direct(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_callcache *cc, VALUE obj)
3966{
3967 RB_DEBUG_COUNTER_INC(ccf_attrset);
3968 VALUE val = *(cfp->sp - 1);
3969 cfp->sp -= 2;
3970 rb_setivar_cache cache = rb_setivar_cache_unpack(vm_cc_atomic_cache_read(cc));
3971 ID id = vm_cc_cme(cc)->def->body.attr.id;
3972 rb_check_frozen(obj);
3973 VALUE res = vm_setivar(obj, val, cache);
3974 if (UNDEF_P(res)) {
3975 switch (BUILTIN_TYPE(obj)) {
3976 case T_OBJECT:
3977 break;
3978 case T_CLASS:
3979 case T_MODULE:
3980 {
3981 res = vm_setivar_class(obj, val, cache);
3982 if (!UNDEF_P(res)) {
3983 return res;
3984 }
3985 }
3986 break;
3987 default:
3988 {
3989 res = vm_setivar_default(obj, id, val, cache);
3990 if (!UNDEF_P(res)) {
3991 return res;
3992 }
3993 }
3994 }
3995 res = vm_setivar_slowpath_attr(obj, id, val, cc);
3996 }
3997 return res;
3998}
3999
4000static VALUE
4001vm_call_attrset(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4002{
4003 return vm_call_attrset_direct(ec, cfp, calling->cc, calling->recv);
4004}
4005
4006static inline VALUE
4007vm_call_bmethod_body(rb_execution_context_t *ec, struct rb_calling_info *calling, const VALUE *argv)
4008{
4009 rb_proc_t *proc;
4010 VALUE val;
4011 const struct rb_callcache *cc = calling->cc;
4012 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4013 VALUE procv = cme->def->body.bmethod.proc;
4014
4015 if (!RB_OBJ_SHAREABLE_P(procv) &&
4016 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4017 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4018 }
4019
4020 /* control block frame */
4021 GetProcPtr(procv, proc);
4022 val = vm_invoke_bmethod(ec, proc, calling->recv, CALLING_ARGC(calling), argv, calling->kw_splat, calling->block_handler, vm_cc_cme(cc));
4023
4024 return val;
4025}
4026
4027static int vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type);
4028
4029static VALUE
4030vm_call_iseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4031{
4032 RB_DEBUG_COUNTER_INC(ccf_iseq_bmethod);
4033
4034 const struct rb_callcache *cc = calling->cc;
4035 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4036 VALUE procv = cme->def->body.bmethod.proc;
4037
4038 if (!RB_OBJ_SHAREABLE_P(procv) &&
4039 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4040 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4041 }
4042
4043 rb_proc_t *proc;
4044 GetProcPtr(procv, proc);
4045 const struct rb_block *block = &proc->block;
4046
4047 while (vm_block_type(block) == block_type_proc) {
4048 block = vm_proc_block(block->as.proc);
4049 }
4050 VM_ASSERT(vm_block_type(block) == block_type_iseq);
4051
4052 const struct rb_captured_block *captured = &block->as.captured;
4053 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
4054 VALUE * const argv = cfp->sp - calling->argc;
4055 const int arg_size = ISEQ_BODY(iseq)->param.size;
4056
4057 int opt_pc;
4058 if (vm_ci_flag(calling->cd->ci) & VM_CALL_ARGS_SIMPLE) {
4059 opt_pc = vm_callee_setup_block_arg(ec, calling, calling->cd->ci, iseq, argv, arg_setup_method);
4060 }
4061 else {
4062 opt_pc = setup_parameters_complex(ec, iseq, calling, calling->cd->ci, argv, arg_setup_method);
4063 }
4064
4065 cfp->sp = argv - 1; // -1 for the receiver
4066
4067 vm_push_frame(ec, iseq,
4068 VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_BMETHOD | VM_FRAME_FLAG_LAMBDA,
4069 calling->recv,
4070 VM_GUARDED_PREV_EP(captured->ep),
4071 (VALUE)cme,
4072 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
4073 argv + arg_size,
4074 ISEQ_BODY(iseq)->local_table_size - arg_size,
4075 ISEQ_BODY(iseq)->stack_max);
4076
4077 return Qundef;
4078}
4079
4080static VALUE
4081vm_call_noniseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4082{
4083 RB_DEBUG_COUNTER_INC(ccf_noniseq_bmethod);
4084
4085 VALUE *argv;
4086 int argc;
4087 CALLER_SETUP_ARG(cfp, calling, calling->cd->ci, ALLOW_HEAP_ARGV);
4088 if (UNLIKELY(calling->heap_argv)) {
4089 argv = RARRAY_PTR(calling->heap_argv);
4090 cfp->sp -= 2;
4091 }
4092 else {
4093 argc = calling->argc;
4094 argv = ALLOCA_N(VALUE, argc);
4095 MEMCPY(argv, cfp->sp - argc, VALUE, argc);
4096 cfp->sp += - argc - 1;
4097 }
4098
4099 return vm_call_bmethod_body(ec, calling, argv);
4100}
4101
4102static VALUE
4103vm_call_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4104{
4105 RB_DEBUG_COUNTER_INC(ccf_bmethod);
4106
4107 const struct rb_callcache *cc = calling->cc;
4108 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4109 VALUE procv = cme->def->body.bmethod.proc;
4110 rb_proc_t *proc;
4111 GetProcPtr(procv, proc);
4112 const struct rb_block *block = &proc->block;
4113
4114 while (vm_block_type(block) == block_type_proc) {
4115 block = vm_proc_block(block->as.proc);
4116 }
4117 if (vm_block_type(block) == block_type_iseq) {
4118 CC_SET_FASTPATH(cc, vm_call_iseq_bmethod, TRUE);
4119 return vm_call_iseq_bmethod(ec, cfp, calling);
4120 }
4121
4122 CC_SET_FASTPATH(cc, vm_call_noniseq_bmethod, TRUE);
4123 return vm_call_noniseq_bmethod(ec, cfp, calling);
4124}
4125
4126VALUE
4127rb_find_defined_class_by_owner(VALUE current_class, VALUE target_owner)
4128{
4129 VALUE klass = current_class;
4130
4131 /* for prepended Module, then start from cover class */
4132 if (RB_TYPE_P(klass, T_ICLASS) && RICLASS_IS_ORIGIN_P(klass) &&
4133 RB_TYPE_P(RBASIC_CLASS(klass), T_CLASS)) {
4134 klass = RBASIC_CLASS(klass);
4135 }
4136
4137 while (RTEST(klass)) {
4138 VALUE owner = RB_TYPE_P(klass, T_ICLASS) ? RBASIC_CLASS(klass) : klass;
4139 if (owner == target_owner) {
4140 return klass;
4141 }
4142 klass = RCLASS_SUPER(klass);
4143 }
4144
4145 return current_class; /* maybe module function */
4146}
4147
4148static const rb_callable_method_entry_t *
4149aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4150{
4151 const rb_method_entry_t *orig_me = me->def->body.alias.original_me;
4152 const rb_callable_method_entry_t *cme;
4153
4154 if (orig_me->defined_class == 0) {
4155 VALUE defined_class = rb_find_defined_class_by_owner(me->defined_class, orig_me->owner);
4156 VM_ASSERT_TYPE(orig_me->owner, T_MODULE);
4157 cme = rb_method_entry_complement_defined_class(orig_me, me->called_id, defined_class);
4158
4159 if (me->def->reference_count == 1) {
4160 RB_OBJ_WRITE(me, &me->def->body.alias.original_me, cme);
4161 }
4162 else {
4163 rb_method_definition_t *def =
4164 rb_method_definition_create(VM_METHOD_TYPE_ALIAS, me->def->original_id);
4165 rb_method_definition_set((rb_method_entry_t *)me, def, (void *)cme);
4166 }
4167 }
4168 else {
4169 cme = (const rb_callable_method_entry_t *)orig_me;
4170 }
4171
4172 VM_ASSERT(callable_method_entry_p(cme));
4173 return cme;
4174}
4175
4176const rb_callable_method_entry_t *
4177rb_aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4178{
4179 return aliased_callable_method_entry(me);
4180}
4181
4182static VALUE
4183vm_call_alias(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4184{
4185 calling->cc = &VM_CC_ON_STACK(Qundef,
4186 vm_call_general,
4187 {{0}},
4188 aliased_callable_method_entry(vm_cc_cme(calling->cc)));
4189
4190 return vm_call_method_each_type(ec, cfp, calling);
4191}
4192
4193static enum method_missing_reason
4194ci_missing_reason(const struct rb_callinfo *ci)
4195{
4196 enum method_missing_reason stat = MISSING_NOENTRY;
4197 if (vm_ci_flag(ci) & VM_CALL_VCALL && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) stat |= MISSING_VCALL;
4198 if (vm_ci_flag(ci) & VM_CALL_FCALL) stat |= MISSING_FCALL;
4199 if (vm_ci_flag(ci) & VM_CALL_SUPER) stat |= MISSING_SUPER;
4200 return stat;
4201}
4202
4203static VALUE vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
4204
4205static VALUE
4206vm_call_symbol(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4207 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE symbol, int flags)
4208{
4209 ASSUME(calling->argc >= 0);
4210
4211 enum method_missing_reason missing_reason = MISSING_NOENTRY;
4212 int argc = calling->argc;
4213 VALUE recv = calling->recv;
4214 VALUE klass = CLASS_OF(recv);
4215 ID mid = rb_check_id(&symbol);
4216 flags |= VM_CALL_OPT_SEND;
4217
4218 if (UNLIKELY(! mid)) {
4219 mid = idMethodMissing;
4220 missing_reason = ci_missing_reason(ci);
4221 ec->method_missing_reason = missing_reason;
4222
4223 VALUE argv_ary;
4224 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4225 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4226 rb_ary_unshift(argv_ary, symbol);
4227
4228 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4229 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4230 VALUE exc = rb_make_no_method_exception(
4231 rb_eNoMethodError, 0, recv, RARRAY_LENINT(argv_ary), RARRAY_CONST_PTR(argv_ary), priv);
4232
4233 rb_exc_raise(exc);
4234 }
4235 rb_ary_unshift(argv_ary, rb_str_intern(symbol));
4236 }
4237 else {
4238 /* E.g. when argc == 2
4239 *
4240 * | | | | TOPN
4241 * | | +------+
4242 * | | +---> | arg1 | 0
4243 * +------+ | +------+
4244 * | arg1 | -+ +-> | arg0 | 1
4245 * +------+ | +------+
4246 * | arg0 | ---+ | sym | 2
4247 * +------+ +------+
4248 * | recv | | recv | 3
4249 * --+------+--------+------+------
4250 */
4251 int i = argc;
4252 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4253 INC_SP(1);
4254 MEMMOVE(&TOPN(i - 1), &TOPN(i), VALUE, i);
4255 argc = ++calling->argc;
4256
4257 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4258 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4259 TOPN(i) = symbol;
4260 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4261 const VALUE *argv = STACK_ADDR_FROM_TOP(argc);
4262 VALUE exc = rb_make_no_method_exception(
4263 rb_eNoMethodError, 0, recv, argc, argv, priv);
4264
4265 rb_exc_raise(exc);
4266 }
4267 else {
4268 TOPN(i) = rb_str_intern(symbol);
4269 }
4270 }
4271 }
4272
4273 struct rb_forwarding_call_data new_fcd = {
4274 .cd = {
4275 .ci = &VM_CI_ON_STACK(mid, flags, argc, vm_ci_kwarg(ci)),
4276 .cc = NULL,
4277 },
4278 .caller_ci = NULL,
4279 };
4280
4281 if (!(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
4282 calling->cd = &new_fcd.cd;
4283 }
4284 else {
4285 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4286 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4287 new_fcd.caller_ci = caller_ci;
4288 calling->cd = (struct rb_call_data *)&new_fcd;
4289 }
4290 calling->cc = &VM_CC_ON_STACK(klass,
4291 vm_call_general,
4292 { .method_missing_reason = missing_reason },
4293 rb_callable_method_entry_with_refinements(klass, mid, NULL));
4294
4295 if (flags & VM_CALL_FCALL) {
4296 return vm_call_method(ec, reg_cfp, calling);
4297 }
4298
4299 const struct rb_callcache *cc = calling->cc;
4300 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
4301
4302 if (vm_cc_cme(cc) != NULL) {
4303 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
4304 case METHOD_VISI_PUBLIC: /* likely */
4305 return vm_call_method_each_type(ec, reg_cfp, calling);
4306 case METHOD_VISI_PRIVATE:
4307 vm_cc_method_missing_reason_set(cc, MISSING_PRIVATE);
4308 break;
4309 case METHOD_VISI_PROTECTED:
4310 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
4311 break;
4312 default:
4313 VM_UNREACHABLE(vm_call_method);
4314 }
4315 return vm_call_method_missing(ec, reg_cfp, calling);
4316 }
4317
4318 return vm_call_method_nome(ec, reg_cfp, calling);
4319}
4320
4321static VALUE
4322vm_call_opt_send0(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int flags)
4323{
4324 const struct rb_callinfo *ci = calling->cd->ci;
4325 int i;
4326 VALUE sym;
4327
4328 i = calling->argc - 1;
4329
4330 if (calling->argc == 0) {
4331 rb_raise(rb_eArgError, "no method name given");
4332 }
4333
4334 sym = TOPN(i);
4335 /* E.g. when i == 2
4336 *
4337 * | | | | TOPN
4338 * +------+ | |
4339 * | arg1 | ---+ | | 0
4340 * +------+ | +------+
4341 * | arg0 | -+ +-> | arg1 | 1
4342 * +------+ | +------+
4343 * | sym | +---> | arg0 | 2
4344 * +------+ +------+
4345 * | recv | | recv | 3
4346 * --+------+--------+------+------
4347 */
4348 /* shift arguments */
4349 if (i > 0) {
4350 MEMMOVE(&TOPN(i), &TOPN(i-1), VALUE, i);
4351 }
4352 calling->argc -= 1;
4353 DEC_SP(1);
4354
4355 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4356}
4357
4358static VALUE
4359vm_call_opt_send_complex(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4360{
4361 RB_DEBUG_COUNTER_INC(ccf_opt_send_complex);
4362 const struct rb_callinfo *ci = calling->cd->ci;
4363 int flags = VM_CALL_FCALL;
4364 VALUE sym;
4365
4366 VALUE argv_ary;
4367 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
4368 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4369 sym = rb_ary_shift(argv_ary);
4370 flags |= VM_CALL_ARGS_SPLAT;
4371 if (calling->kw_splat) {
4372 VALUE last_hash = rb_ary_last(0, NULL, argv_ary);
4373 ((struct RHash *)last_hash)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
4374 calling->kw_splat = 0;
4375 }
4376 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4377 }
4378
4379 if (calling->kw_splat) flags |= VM_CALL_KW_SPLAT;
4380 return vm_call_opt_send0(ec, reg_cfp, calling, flags);
4381}
4382
4383static VALUE
4384vm_call_opt_send_simple(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4385{
4386 RB_DEBUG_COUNTER_INC(ccf_opt_send_simple);
4387 return vm_call_opt_send0(ec, reg_cfp, calling, vm_ci_flag(calling->cd->ci) | VM_CALL_FCALL);
4388}
4389
4390static VALUE
4391vm_call_opt_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4392{
4393 RB_DEBUG_COUNTER_INC(ccf_opt_send);
4394
4395 const struct rb_callinfo *ci = calling->cd->ci;
4396 int flags = vm_ci_flag(ci);
4397
4398 if (UNLIKELY((flags & VM_CALL_FORWARDING) || (!(flags & VM_CALL_ARGS_SIMPLE) &&
4399 ((calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
4400 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
4401 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc)))))) {
4402 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_complex, TRUE);
4403 return vm_call_opt_send_complex(ec, reg_cfp, calling);
4404 }
4405
4406 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_simple, TRUE);
4407 return vm_call_opt_send_simple(ec, reg_cfp, calling);
4408}
4409
4410static VALUE
4411vm_call_method_missing_body(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
4412 const struct rb_callinfo *orig_ci, enum method_missing_reason reason)
4413{
4414 RB_DEBUG_COUNTER_INC(ccf_method_missing);
4415
4416 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
4417 unsigned int argc, flag;
4418
4419 flag = VM_CALL_FCALL | VM_CALL_OPT_SEND | vm_ci_flag(orig_ci);
4420 argc = ++calling->argc;
4421
4422 /* shift arguments: m(a, b, c) #=> method_missing(:m, a, b, c) */
4423 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4424 vm_check_canary(ec, reg_cfp->sp);
4425 if (argc > 1) {
4426 MEMMOVE(argv+1, argv, VALUE, argc-1);
4427 }
4428 argv[0] = ID2SYM(vm_ci_mid(orig_ci));
4429 INC_SP(1);
4430
4431 ec->method_missing_reason = reason;
4432
4433 struct rb_forwarding_call_data new_fcd = {
4434 .cd = {
4435 .ci = &VM_CI_ON_STACK(idMethodMissing, flag, argc, vm_ci_kwarg(orig_ci)),
4436 .cc = NULL,
4437 },
4438 .caller_ci = NULL,
4439 };
4440
4441 if (!(flag & VM_CALL_FORWARDING)) {
4442 calling->cd = &new_fcd.cd;
4443 }
4444 else {
4445 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4446 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4447 new_fcd.caller_ci = caller_ci;
4448 calling->cd = (struct rb_call_data *)&new_fcd;
4449 }
4450
4451 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }},
4452 rb_callable_method_entry_without_refinements(CLASS_OF(calling->recv), idMethodMissing, NULL));
4453 return vm_call_method(ec, reg_cfp, calling);
4454}
4455
4456static VALUE
4457vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4458{
4459 return vm_call_method_missing_body(ec, reg_cfp, calling, calling->cd->ci, vm_cc_cmethod_missing_reason(calling->cc));
4460}
4461
4462static const rb_callable_method_entry_t *refined_method_callable_without_refinement(const rb_callable_method_entry_t *me);
4463static VALUE
4464vm_call_zsuper(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE klass)
4465{
4466 klass = RCLASS_SUPER(klass);
4467
4468 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, vm_ci_mid(calling->cd->ci)) : NULL;
4469 if (cme == NULL) {
4470 return vm_call_method_nome(ec, cfp, calling);
4471 }
4472 if (cme->def->type == VM_METHOD_TYPE_REFINED &&
4473 cme->def->body.refined.orig_me) {
4474 cme = refined_method_callable_without_refinement(cme);
4475 }
4476
4477 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, cme);
4478
4479 return vm_call_method_each_type(ec, cfp, calling);
4480}
4481
4482static inline VALUE
4483find_refinement(VALUE refinements, VALUE klass)
4484{
4485 if (NIL_P(refinements)) {
4486 return Qnil;
4487 }
4488 return rb_hash_lookup(refinements, klass);
4489}
4490
4491PUREFUNC(static rb_control_frame_t * current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp));
4492static rb_control_frame_t *
4493current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp)
4494{
4495 rb_control_frame_t *top_cfp = cfp;
4496
4497 if (CFP_ISEQ(cfp) && ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_BLOCK) {
4498 const rb_iseq_t *local_iseq = ISEQ_BODY(CFP_ISEQ(cfp))->local_iseq;
4499
4500 do {
4501 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
4502 if (RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
4503 /* TODO: orphan block */
4504 return top_cfp;
4505 }
4506 } while (CFP_ISEQ(cfp) != local_iseq);
4507 }
4508 return cfp;
4509}
4510
4511static const rb_callable_method_entry_t *
4512refined_method_callable_without_refinement(const rb_callable_method_entry_t *me)
4513{
4514 const rb_method_entry_t *orig_me = me->def->body.refined.orig_me;
4515 const rb_callable_method_entry_t *cme;
4516
4517 if (orig_me->defined_class == 0) {
4518 cme = NULL;
4519 rb_notimplement();
4520 }
4521 else {
4522 cme = (const rb_callable_method_entry_t *)orig_me;
4523 }
4524
4525 VM_ASSERT(callable_method_entry_p(cme));
4526
4527 if (UNDEFINED_METHOD_ENTRY_P(cme)) {
4528 cme = NULL;
4529 }
4530
4531 return cme;
4532}
4533
4534static const rb_callable_method_entry_t *
4535search_refined_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4536{
4537 ID mid = vm_ci_mid(calling->cd->ci);
4538 const rb_cref_t *cref = vm_get_cref(cfp->ep);
4539 const struct rb_callcache * const cc = calling->cc;
4540 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4541
4542 for (; cref; cref = CREF_NEXT(cref)) {
4543 const VALUE refinement = find_refinement(CREF_REFINEMENTS(cref), vm_cc_cme(cc)->owner);
4544 if (NIL_P(refinement)) continue;
4545
4546 const rb_callable_method_entry_t *const ref_me =
4547 rb_callable_method_entry(refinement, mid);
4548
4549 if (ref_me) {
4550 if (vm_cc_call(cc) == vm_call_super_method) {
4551 const rb_control_frame_t *top_cfp = current_method_entry(ec, cfp);
4552 const rb_callable_method_entry_t *top_me = rb_vm_frame_method_entry(top_cfp);
4553 if (top_me && rb_method_definition_eq(ref_me->def, top_me->def)) {
4554 continue;
4555 }
4556 }
4557
4558 if (cme->def->type != VM_METHOD_TYPE_REFINED ||
4559 cme->def != ref_me->def) {
4560 cme = ref_me;
4561 }
4562 if (ref_me->def->type != VM_METHOD_TYPE_REFINED) {
4563 return cme;
4564 }
4565 }
4566 else {
4567 return NULL;
4568 }
4569 }
4570
4571 if (vm_cc_cme(cc)->def->body.refined.orig_me) {
4572 return refined_method_callable_without_refinement(vm_cc_cme(cc));
4573 }
4574 else {
4575 VALUE klass = RCLASS_SUPER(vm_cc_cme(cc)->defined_class);
4576 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, mid) : NULL;
4577 return cme;
4578 }
4579}
4580
4581static VALUE
4582vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4583{
4584 const rb_callable_method_entry_t *ref_cme = search_refined_method(ec, cfp, calling);
4585
4586 if (ref_cme) {
4587 if (calling->cd->cc) {
4588 const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement);
4589 RB_OBJ_WRITE(CFP_ISEQ(cfp), &calling->cd->cc, cc);
4590 return vm_call_method(ec, cfp, calling);
4591 }
4592 else {
4593 struct rb_callcache *ref_cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, ref_cme);
4594 calling->cc= ref_cc;
4595 return vm_call_method(ec, cfp, calling);
4596 }
4597 }
4598 else {
4599 return vm_call_method_nome(ec, cfp, calling);
4600 }
4601}
4602
4603static inline VALUE vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci, bool is_lambda, VALUE block_handler);
4604
4605NOINLINE(static VALUE
4606 vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4607 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler));
4608
4609static VALUE
4610vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4611 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler)
4612{
4613 int argc = calling->argc;
4614
4615 /* remove self */
4616 if (argc > 0) MEMMOVE(&TOPN(argc), &TOPN(argc-1), VALUE, argc);
4617 DEC_SP(1);
4618
4619 return vm_invoke_block(ec, reg_cfp, calling, ci, false, block_handler);
4620}
4621
4622static VALUE
4623vm_call_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4624{
4625 RB_DEBUG_COUNTER_INC(ccf_opt_call);
4626
4627 const struct rb_callinfo *ci = calling->cd->ci;
4628 VALUE procval = calling->recv;
4629 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, VM_BH_FROM_PROC(procval));
4630}
4631
4632static VALUE
4633vm_call_opt_block_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4634{
4635 RB_DEBUG_COUNTER_INC(ccf_opt_block_call);
4636
4637 VALUE block_handler = VM_ENV_BLOCK_HANDLER(VM_CF_LEP(reg_cfp));
4638 const struct rb_callinfo *ci = calling->cd->ci;
4639
4640 if (BASIC_OP_UNREDEFINED_P(BOP_CALL, PROC_REDEFINED_OP_FLAG)) {
4641 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, block_handler);
4642 }
4643 else {
4644 calling->recv = rb_vm_bh_to_procval(ec, block_handler);
4645 calling->cc = rb_vm_search_method_slowpath(ci, CLASS_OF(calling->recv));
4646 return vm_call_general(ec, reg_cfp, calling);
4647 }
4648}
4649
4650static VALUE
4651vm_call_opt_struct_aref0(rb_execution_context_t *ec, struct rb_calling_info *calling)
4652{
4653 VALUE recv = calling->recv;
4654
4655 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4656 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4657 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_AREF);
4658
4659 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4660 return RSTRUCT_GET_RAW(recv, off);
4661}
4662
4663static VALUE
4664vm_call_opt_struct_aref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4665{
4666 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aref);
4667
4668 VALUE ret = vm_call_opt_struct_aref0(ec, calling);
4669 reg_cfp->sp -= 1;
4670 return ret;
4671}
4672
4673static VALUE
4674vm_call_opt_struct_aset0(rb_execution_context_t *ec, struct rb_calling_info *calling, VALUE val)
4675{
4676 VALUE recv = calling->recv;
4677
4678 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4679 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4680 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_ASET);
4681
4682 rb_check_frozen(recv);
4683
4684 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4685 RSTRUCT_SET_RAW(recv, off, val);
4686
4687 return val;
4688}
4689
4690static VALUE
4691vm_call_opt_struct_aset(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4692{
4693 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aset);
4694
4695 VALUE ret = vm_call_opt_struct_aset0(ec, calling, *(reg_cfp->sp - 1));
4696 reg_cfp->sp -= 2;
4697 return ret;
4698}
4699
4700NOINLINE(static VALUE vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4701 const struct rb_callinfo *ci, const struct rb_callcache *cc));
4702
4703#define VM_CALL_METHOD_ATTR(var, func, nohook) \
4704 if (UNLIKELY(ruby_vm_c_events_enabled > 0)) { \
4705 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, calling->recv, vm_cc_cme(cc)->def->original_id, \
4706 vm_ci_mid(ci), vm_cc_cme(cc)->owner, Qundef); \
4707 var = func; \
4708 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, calling->recv, vm_cc_cme(cc)->def->original_id, \
4709 vm_ci_mid(ci), vm_cc_cme(cc)->owner, (var)); \
4710 } \
4711 else { \
4712 nohook; \
4713 var = func; \
4714 }
4715
4716static VALUE
4717vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4718 const struct rb_callinfo *ci, const struct rb_callcache *cc)
4719{
4720 switch (vm_cc_cme(cc)->def->body.optimized.type) {
4721 case OPTIMIZED_METHOD_TYPE_SEND:
4722 CC_SET_FASTPATH(cc, vm_call_opt_send, TRUE);
4723 return vm_call_opt_send(ec, cfp, calling);
4724 case OPTIMIZED_METHOD_TYPE_CALL:
4725 CC_SET_FASTPATH(cc, vm_call_opt_call, TRUE);
4726 return vm_call_opt_call(ec, cfp, calling);
4727 case OPTIMIZED_METHOD_TYPE_BLOCK_CALL:
4728 CC_SET_FASTPATH(cc, vm_call_opt_block_call, TRUE);
4729 return vm_call_opt_block_call(ec, cfp, calling);
4730 case OPTIMIZED_METHOD_TYPE_STRUCT_AREF: {
4731 CALLER_SETUP_ARG(cfp, calling, ci, 0);
4732 rb_check_arity(calling->argc, 0, 0);
4733
4734 VALUE v;
4735 VM_CALL_METHOD_ATTR(v,
4736 vm_call_opt_struct_aref(ec, cfp, calling),
4737 set_vm_cc_ivar(cc); \
4738 CC_SET_FASTPATH(cc, vm_call_opt_struct_aref, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
4739 return v;
4740 }
4741 case OPTIMIZED_METHOD_TYPE_STRUCT_ASET: {
4742 CALLER_SETUP_ARG(cfp, calling, ci, 1);
4743 rb_check_arity(calling->argc, 1, 1);
4744
4745 VALUE v;
4746 VM_CALL_METHOD_ATTR(v,
4747 vm_call_opt_struct_aset(ec, cfp, calling),
4748 set_vm_cc_ivar(cc); \
4749 CC_SET_FASTPATH(cc, vm_call_opt_struct_aset, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
4750 return v;
4751 }
4752 default:
4753 rb_bug("vm_call_method: unsupported optimized method type (%d)", vm_cc_cme(cc)->def->body.optimized.type);
4754 }
4755}
4756
4757static VALUE
4758vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4759{
4760 const struct rb_callinfo *ci = calling->cd->ci;
4761 const struct rb_callcache *cc = calling->cc;
4762 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4763 VALUE v;
4764
4765 VM_ASSERT(! METHOD_ENTRY_INVALIDATED(cme));
4766
4767 switch (cme->def->type) {
4768 case VM_METHOD_TYPE_ISEQ:
4769 if (ISEQ_BODY(def_iseq_ptr(cme->def))->param.flags.forwardable) {
4770 CC_SET_FASTPATH(cc, vm_call_iseq_fwd_setup, TRUE);
4771 return vm_call_iseq_fwd_setup(ec, cfp, calling);
4772 }
4773 else {
4774 CC_SET_FASTPATH(cc, vm_call_iseq_setup, TRUE);
4775 return vm_call_iseq_setup(ec, cfp, calling);
4776 }
4777
4778 case VM_METHOD_TYPE_NOTIMPLEMENTED:
4779 case VM_METHOD_TYPE_CFUNC:
4780 CC_SET_FASTPATH(cc, vm_call_cfunc, TRUE);
4781 return vm_call_cfunc(ec, cfp, calling);
4782
4783 case VM_METHOD_TYPE_ATTRSET:
4784 CALLER_SETUP_ARG(cfp, calling, ci, 1);
4785
4786 rb_check_arity(calling->argc, 1, 1);
4787
4788 const unsigned int aset_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_KWARG | VM_CALL_FORWARDING);
4789
4790 if (vm_cc_markable(cc)) {
4791 vm_cc_attr_index_set(cc, IVAR_CACHE_INIT);
4792 VM_CALL_METHOD_ATTR(v,
4793 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
4794 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
4795 }
4796 else {
4797 cc = &((struct rb_callcache) {
4798 .flags = T_IMEMO |
4799 (imemo_callcache << FL_USHIFT) |
4800 VM_CALLCACHE_UNMARKABLE |
4801 VM_CALLCACHE_ON_STACK,
4802 .klass = cc->klass,
4803 .cme_ = cc->cme_,
4804 .call_ = cc->call_,
4805 .aux_ = {
4806 .attr = {
4807 .value = IVAR_CACHE_INIT,
4808 }
4809 },
4810 });
4811
4812 VM_CALL_METHOD_ATTR(v,
4813 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
4814 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
4815 }
4816 return v;
4817
4818 case VM_METHOD_TYPE_IVAR:
4819 CALLER_SETUP_ARG(cfp, calling, ci, 0);
4820 rb_check_arity(calling->argc, 0, 0);
4821 vm_cc_attr_index_set(cc, rb_getivar_cache_pack(ROOT_SHAPE_ID, ATTR_INDEX_NOT_SET));
4822 const unsigned int ivar_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_FORWARDING);
4823 VM_CALL_METHOD_ATTR(v,
4824 vm_call_ivar(ec, cfp, calling),
4825 CC_SET_FASTPATH(cc, vm_call_ivar, !(vm_ci_flag(ci) & ivar_mask)));
4826 return v;
4827
4828 case VM_METHOD_TYPE_MISSING:
4829 vm_cc_method_missing_reason_set(cc, 0);
4830 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
4831 return vm_call_method_missing(ec, cfp, calling);
4832
4833 case VM_METHOD_TYPE_BMETHOD:
4834 CC_SET_FASTPATH(cc, vm_call_bmethod, TRUE);
4835 return vm_call_bmethod(ec, cfp, calling);
4836
4837 case VM_METHOD_TYPE_ALIAS:
4838 CC_SET_FASTPATH(cc, vm_call_alias, TRUE);
4839 return vm_call_alias(ec, cfp, calling);
4840
4841 case VM_METHOD_TYPE_OPTIMIZED:
4842 return vm_call_optimized(ec, cfp, calling, ci, cc);
4843
4844 case VM_METHOD_TYPE_UNDEF:
4845 break;
4846
4847 case VM_METHOD_TYPE_ZSUPER:
4848 return vm_call_zsuper(ec, cfp, calling, RCLASS_ORIGIN(vm_cc_cme(cc)->defined_class));
4849
4850 case VM_METHOD_TYPE_REFINED:
4851 // CC_SET_FASTPATH(cc, vm_call_refined, TRUE);
4852 // should not set FASTPATH since vm_call_refined assumes cc->call is vm_call_super_method on invokesuper.
4853 return vm_call_refined(ec, cfp, calling);
4854 }
4855
4856 rb_bug("vm_call_method: unsupported method type (%d)", vm_cc_cme(cc)->def->type);
4857}
4858
4859NORETURN(static void vm_raise_method_missing(rb_execution_context_t *ec, int argc, const VALUE *argv, VALUE obj, int call_status));
4860
4861static VALUE
4862vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4863{
4864 /* method missing */
4865 const struct rb_callinfo *ci = calling->cd->ci;
4866 const int stat = ci_missing_reason(ci);
4867
4868 if (vm_ci_mid(ci) == idMethodMissing) {
4869 if (UNLIKELY(calling->heap_argv)) {
4870 vm_raise_method_missing(ec, RARRAY_LENINT(calling->heap_argv), RARRAY_CONST_PTR(calling->heap_argv), calling->recv, stat);
4871 }
4872 else {
4873 rb_control_frame_t *reg_cfp = cfp;
4874 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
4875 vm_raise_method_missing(ec, calling->argc, argv, calling->recv, stat);
4876 }
4877 }
4878 else {
4879 return vm_call_method_missing_body(ec, cfp, calling, ci, stat);
4880 }
4881}
4882
4883/* Protected method calls and super invocations need to check that the receiver
4884 * (self for super) inherits the module on which the method is defined.
4885 * In the case of refinements, it should consider the original class not the
4886 * refinement.
4887 */
4888static VALUE
4889vm_defined_class_for_protected_call(const rb_callable_method_entry_t *me)
4890{
4891 VALUE defined_class = me->defined_class;
4892 VALUE refined_class = RCLASS_REFINED_CLASS(defined_class);
4893 return NIL_P(refined_class) ? defined_class : refined_class;
4894}
4895
4896static inline VALUE
4897vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4898{
4899 const struct rb_callinfo *ci = calling->cd->ci;
4900 const struct rb_callcache *cc = calling->cc;
4901
4902 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
4903
4904 if (vm_cc_cme(cc) != NULL) {
4905 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
4906 case METHOD_VISI_PUBLIC: /* likely */
4907 return vm_call_method_each_type(ec, cfp, calling);
4908
4909 case METHOD_VISI_PRIVATE:
4910 if (!(vm_ci_flag(ci) & VM_CALL_FCALL)) {
4911 enum method_missing_reason stat = MISSING_PRIVATE;
4912 if (vm_ci_flag(ci) & VM_CALL_VCALL) stat |= MISSING_VCALL;
4913
4914 vm_cc_method_missing_reason_set(cc, stat);
4915 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
4916 return vm_call_method_missing(ec, cfp, calling);
4917 }
4918 return vm_call_method_each_type(ec, cfp, calling);
4919
4920 case METHOD_VISI_PROTECTED:
4921 if (!(vm_ci_flag(ci) & (VM_CALL_OPT_SEND | VM_CALL_FCALL))) {
4922 VALUE defined_class = vm_defined_class_for_protected_call(vm_cc_cme(cc));
4923 if (!rb_obj_is_kind_of(cfp->self, defined_class)) {
4924 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
4925 return vm_call_method_missing(ec, cfp, calling);
4926 }
4927 else {
4928 /* caching method info to dummy cc */
4929 VM_ASSERT(vm_cc_cme(cc) != NULL);
4930 struct rb_callcache cc_on_stack = *cc;
4931 FL_SET_RAW((VALUE)&cc_on_stack, VM_CALLCACHE_UNMARKABLE);
4932 calling->cc = &cc_on_stack;
4933 return vm_call_method_each_type(ec, cfp, calling);
4934 }
4935 }
4936 return vm_call_method_each_type(ec, cfp, calling);
4937
4938 default:
4939 rb_bug("unreachable");
4940 }
4941 }
4942 else {
4943 return vm_call_method_nome(ec, cfp, calling);
4944 }
4945}
4946
4947static VALUE
4948vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4949{
4950 RB_DEBUG_COUNTER_INC(ccf_general);
4951 return vm_call_method(ec, reg_cfp, calling);
4952}
4953
4954void
4955rb_vm_cc_general(const struct rb_callcache *cc)
4956{
4957 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
4958 VM_ASSERT(cc != vm_cc_empty());
4959
4960 *(vm_call_handler *)&cc->call_ = vm_call_general;
4961}
4962
4963static VALUE
4964vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4965{
4966 RB_DEBUG_COUNTER_INC(ccf_super_method);
4967
4968 // This line is introduced to make different from `vm_call_general` because some compilers (VC we found)
4969 // can merge the function and the address of the function becomes same.
4970 // The address of `vm_call_super_method` is used in `search_refined_method`, so it should be different.
4971 if (ec == NULL) rb_bug("unreachable");
4972
4973 /* this check is required to distinguish with other functions. */
4974 VM_ASSERT(vm_cc_call(calling->cc) == vm_call_super_method);
4975 return vm_call_method(ec, reg_cfp, calling);
4976}
4977
4978/* super */
4979
4980static inline VALUE
4981vm_search_normal_superclass(VALUE klass)
4982{
4983 if (RICLASS_FOR_REFINEMENT_P(klass)) {
4984 klass = RBASIC(klass)->klass;
4985 }
4986 klass = RCLASS_ORIGIN(klass);
4987 return RCLASS_SUPER(klass);
4988}
4989
4990NORETURN(static void vm_super_outside(void));
4991
4992static void
4993vm_super_outside(void)
4994{
4995 rb_raise(rb_eNoMethodError, "super called outside of method");
4996}
4997
4998static const struct rb_callcache *
4999empty_cc_for_super(void)
5000{
5001 return &vm_empty_cc_for_super;
5002}
5003
5004static const struct rb_callcache *
5005vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *cd, VALUE recv)
5006{
5007 VALUE current_defined_class;
5008 const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp);
5009 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
5010
5011 if (!me) {
5012 vm_super_outside();
5013 }
5014
5015 current_defined_class = vm_defined_class_for_protected_call(me);
5016
5017 if (BUILTIN_TYPE(current_defined_class) != T_MODULE &&
5018 iseq != method_entry_iseqptr(me) &&
5019 !rb_obj_is_kind_of(recv, current_defined_class)) {
5020 VALUE m = RB_TYPE_P(current_defined_class, T_ICLASS) ?
5021 RCLASS_INCLUDER(current_defined_class) : current_defined_class;
5022
5023 if (m) { /* not bound UnboundMethod */
5024 rb_raise(rb_eTypeError,
5025 "self has wrong type to call super in this context: "
5026 "%"PRIsVALUE" (expected %"PRIsVALUE")",
5027 rb_obj_class(recv), m);
5028 }
5029 }
5030
5031 if (me->def->type == VM_METHOD_TYPE_BMETHOD && (vm_ci_flag(cd->ci) & VM_CALL_ZSUPER)) {
5032 rb_raise(rb_eRuntimeError,
5033 "implicit argument passing of super from method defined"
5034 " by define_method() is not supported."
5035 " Specify all arguments explicitly.");
5036 }
5037
5038 ID mid = me->def->original_id;
5039
5040 if (!vm_ci_markable(cd->ci)) {
5041 VM_FORCE_WRITE((const VALUE *)&cd->ci->mid, (VALUE)mid);
5042 }
5043 else {
5044 // update iseq. really? (TODO)
5045 cd->ci = vm_ci_new_runtime(mid,
5046 vm_ci_flag(cd->ci),
5047 vm_ci_argc(cd->ci),
5048 vm_ci_kwarg(cd->ci));
5049
5050 RB_OBJ_WRITTEN(iseq, Qundef, cd->ci);
5051 }
5052
5053 const struct rb_callcache *cc;
5054
5055 VALUE klass = vm_search_normal_superclass(me->defined_class);
5056
5057 if (!klass) {
5058 /* bound instance method of module */
5059 cc = vm_cc_new(Qundef, NULL, vm_call_method_missing, cc_type_super);
5060 RB_OBJ_WRITE(iseq, &cd->cc, cc);
5061 }
5062 else if (klass == rb_cBasicObject &&
5063 RB_TYPE_P(me->defined_class, T_ICLASS) &&
5064 RCLASS_INCLUDER(me->defined_class) == 0) {
5065 rb_raise(rb_eNoMethodError,
5066 "super in a method in a module that has been refined and that is called via super"
5067 " from a refinement method is not supported.");
5068 }
5069 else {
5070 cc = vm_search_method_fastpath(reg_cfp, cd, klass);
5071 const rb_callable_method_entry_t *cached_cme = vm_cc_cme(cc);
5072
5073 // define_method can cache for different method id
5074 if (cached_cme == NULL) {
5075 // empty_cc_for_super is not markable object
5076 cd->cc = empty_cc_for_super();
5077 }
5078 else if (cached_cme->called_id != mid) {
5079 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
5080 if (cme) {
5081 cc = vm_cc_new(klass, cme, vm_call_super_method, cc_type_super);
5082 RB_OBJ_WRITE(iseq, &cd->cc, cc);
5083 }
5084 else {
5085 cd->cc = cc = empty_cc_for_super();
5086 }
5087 }
5088 else {
5089 switch (cached_cme->def->type) {
5090 // vm_call_refined (search_refined_method) assumes cc->call is vm_call_super_method on invokesuper
5091 case VM_METHOD_TYPE_REFINED:
5092 // cc->klass is superclass of receiver class. Checking cc->klass is not enough to invalidate IVC for the receiver class.
5093 case VM_METHOD_TYPE_ATTRSET:
5094 case VM_METHOD_TYPE_IVAR:
5095 vm_cc_call_set(cc, vm_call_super_method); // invalidate fastpath
5096 break;
5097 default:
5098 break; // use fastpath
5099 }
5100 }
5101 }
5102
5103 VM_ASSERT((vm_cc_cme(cc), true));
5104
5105 return cc;
5106}
5107
5108/* yield */
5109
5110static inline int
5111block_proc_is_lambda(const VALUE procval)
5112{
5113 rb_proc_t *proc;
5114
5115 if (procval) {
5116 GetProcPtr(procval, proc);
5117 return proc->is_lambda;
5118 }
5119 else {
5120 return 0;
5121 }
5122}
5123
5124static VALUE
5125vm_yield_with_cfunc(rb_execution_context_t *ec,
5126 const struct rb_captured_block *captured,
5127 VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE block_handler,
5128 const rb_callable_method_entry_t *me)
5129{
5130 int is_lambda = FALSE; /* TODO */
5131 VALUE val, arg, blockarg;
5132 int frame_flag;
5133 const struct vm_ifunc *ifunc = captured->code.ifunc;
5134
5135 if (is_lambda) {
5136 arg = rb_ary_new4(argc, argv);
5137 }
5138 else if (argc == 0) {
5139 arg = Qnil;
5140 }
5141 else {
5142 arg = argv[0];
5143 }
5144
5145 blockarg = rb_vm_bh_to_procval(ec, block_handler);
5146
5147 frame_flag = VM_FRAME_MAGIC_IFUNC | VM_FRAME_FLAG_CFRAME | (me ? VM_FRAME_FLAG_BMETHOD : 0);
5148 if (kw_splat) {
5149 frame_flag |= VM_FRAME_FLAG_CFRAME_KW;
5150 }
5151
5152 vm_push_frame(ec, (const rb_iseq_t *)captured->code.ifunc,
5153 frame_flag,
5154 self,
5155 VM_GUARDED_PREV_EP(captured->ep),
5156 (VALUE)me,
5157 0, ec->cfp->sp, 0, 0);
5158 val = (*ifunc->func)(arg, (VALUE)ifunc->data, argc, argv, blockarg);
5159 rb_vm_pop_frame(ec);
5160
5161 return val;
5162}
5163
5164VALUE
5165rb_vm_yield_with_cfunc(rb_execution_context_t *ec, const struct rb_captured_block *captured, int argc, const VALUE *argv)
5166{
5167 return vm_yield_with_cfunc(ec, captured, captured->self, argc, argv, 0, VM_BLOCK_HANDLER_NONE, NULL);
5168}
5169
5170static VALUE
5171vm_yield_with_symbol(rb_execution_context_t *ec, VALUE symbol, int argc, const VALUE *argv, int kw_splat, VALUE block_handler)
5172{
5173 return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, rb_vm_bh_to_procval(ec, block_handler));
5174}
5175
5176static inline int
5177vm_callee_setup_block_arg_arg0_splat(rb_control_frame_t *cfp, const rb_iseq_t *iseq, VALUE *argv, VALUE ary)
5178{
5179 int i;
5180 long len = RARRAY_LEN(ary);
5181
5182 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5183
5184 for (i=0; i<len && i<ISEQ_BODY(iseq)->param.lead_num; i++) {
5185 argv[i] = RARRAY_AREF(ary, i);
5186 }
5187
5188 return i;
5189}
5190
5191static inline VALUE
5192vm_callee_setup_block_arg_arg0_check(VALUE *argv)
5193{
5194 VALUE ary, arg0 = argv[0];
5195 ary = rb_check_array_type(arg0);
5196#if 0
5197 argv[0] = arg0;
5198#else
5199 VM_ASSERT(argv[0] == arg0);
5200#endif
5201 return ary;
5202}
5203
5204static int
5205vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type)
5206{
5207 if (rb_simple_iseq_p(iseq)) {
5208 rb_control_frame_t *cfp = ec->cfp;
5209 VALUE arg0;
5210
5211 CALLER_SETUP_ARG(cfp, calling, ci, ISEQ_BODY(iseq)->param.lead_num);
5212
5213 if (arg_setup_type == arg_setup_block &&
5214 calling->argc == 1 &&
5215 ISEQ_BODY(iseq)->param.flags.has_lead &&
5216 !ISEQ_BODY(iseq)->param.flags.ambiguous_param0 &&
5217 !NIL_P(arg0 = vm_callee_setup_block_arg_arg0_check(argv))) {
5218 calling->argc = vm_callee_setup_block_arg_arg0_splat(cfp, iseq, argv, arg0);
5219 }
5220
5221 if (calling->argc != ISEQ_BODY(iseq)->param.lead_num) {
5222 if (arg_setup_type == arg_setup_block) {
5223 if (calling->argc < ISEQ_BODY(iseq)->param.lead_num) {
5224 int i;
5225 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5226 for (i=calling->argc; i<ISEQ_BODY(iseq)->param.lead_num; i++) argv[i] = Qnil;
5227 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* fill rest parameters */
5228 }
5229 else if (calling->argc > ISEQ_BODY(iseq)->param.lead_num) {
5230 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* simply truncate arguments */
5231 }
5232 }
5233 else {
5234 argument_arity_error(ec, iseq, NULL, calling->argc, ISEQ_BODY(iseq)->param.lead_num, ISEQ_BODY(iseq)->param.lead_num);
5235 }
5236 }
5237
5238 return 0;
5239 }
5240 else {
5241 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_type);
5242 }
5243}
5244
5245static int
5246vm_yield_setup_args(rb_execution_context_t *ec, const rb_iseq_t *iseq, const int argc, VALUE *argv, int flags, VALUE block_handler, enum arg_setup_type arg_setup_type)
5247{
5248 struct rb_calling_info calling_entry, *calling;
5249
5250 calling = &calling_entry;
5251 calling->argc = argc;
5252 calling->block_handler = block_handler;
5253 calling->kw_splat = (flags & VM_CALL_KW_SPLAT) ? 1 : 0;
5254 calling->recv = Qundef;
5255 calling->heap_argv = 0;
5256 calling->cc = NULL;
5257 struct rb_callinfo dummy_ci = VM_CI_ON_STACK(0, flags, 0, 0);
5258
5259 return vm_callee_setup_block_arg(ec, calling, &dummy_ci, iseq, argv, arg_setup_type);
5260}
5261
5262/* ruby iseq -> ruby block */
5263
5264static VALUE
5265vm_invoke_iseq_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5266 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5267 bool is_lambda, VALUE block_handler)
5268{
5269 const struct rb_captured_block *captured = VM_BH_TO_ISEQ_BLOCK(block_handler);
5270 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
5271 const int arg_size = ISEQ_BODY(iseq)->param.size;
5272 VALUE * const rsp = GET_SP() - calling->argc;
5273 VALUE * const argv = rsp;
5274 int opt_pc = vm_callee_setup_block_arg(ec, calling, ci, iseq, argv, is_lambda ? arg_setup_method : arg_setup_block);
5275 int frame_flag = VM_FRAME_MAGIC_BLOCK | (is_lambda ? VM_FRAME_FLAG_LAMBDA : 0);
5276
5277 SET_SP(rsp);
5278
5279 vm_push_frame(ec, iseq,
5280 frame_flag,
5281 captured->self,
5282 VM_GUARDED_PREV_EP(captured->ep), 0,
5283 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
5284 rsp + arg_size,
5285 ISEQ_BODY(iseq)->local_table_size - arg_size, ISEQ_BODY(iseq)->stack_max);
5286
5287 return Qundef;
5288}
5289
5290static VALUE
5291vm_invoke_symbol_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5292 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5293 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5294{
5295 VALUE symbol = VM_BH_TO_SYMBOL(block_handler);
5296 int flags = vm_ci_flag(ci);
5297
5298 if (UNLIKELY(!(flags & VM_CALL_ARGS_SIMPLE) &&
5299 ((calling->argc == 0) ||
5300 (calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
5301 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
5302 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc))))) {
5303 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
5304 flags = 0;
5305 if (UNLIKELY(calling->heap_argv)) {
5306#if VM_ARGC_STACK_MAX < 0
5307 if (RARRAY_LEN(calling->heap_argv) < 1) {
5308 rb_raise(rb_eArgError, "no receiver given");
5309 }
5310#endif
5311 calling->recv = rb_ary_shift(calling->heap_argv);
5312 // Modify stack to avoid cfp consistency error
5313 reg_cfp->sp++;
5314 reg_cfp->sp[-1] = reg_cfp->sp[-2];
5315 reg_cfp->sp[-2] = calling->recv;
5316 flags |= VM_CALL_ARGS_SPLAT;
5317 }
5318 else {
5319 if (calling->argc < 1) {
5320 rb_raise(rb_eArgError, "no receiver given");
5321 }
5322 calling->recv = TOPN(--calling->argc);
5323 }
5324 if (calling->kw_splat) {
5325 flags |= VM_CALL_KW_SPLAT;
5326 }
5327 }
5328 else {
5329 if (calling->argc < 1) {
5330 rb_raise(rb_eArgError, "no receiver given");
5331 }
5332 calling->recv = TOPN(--calling->argc);
5333 }
5334
5335 return vm_call_symbol(ec, reg_cfp, calling, ci, symbol, flags);
5336}
5337
5338static VALUE
5339vm_invoke_ifunc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5340 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5341 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5342{
5343 VALUE val;
5344 int argc;
5345 const struct rb_captured_block *captured = VM_BH_TO_IFUNC_BLOCK(block_handler);
5346 CALLER_SETUP_ARG(ec->cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
5347 argc = calling->argc;
5348 val = vm_yield_with_cfunc(ec, captured, captured->self, CALLING_ARGC(calling), calling->heap_argv ? RARRAY_CONST_PTR(calling->heap_argv) : STACK_ADDR_FROM_TOP(argc), calling->kw_splat, calling->block_handler, NULL);
5349 POPN(argc); /* TODO: should put before C/yield? */
5350 return val;
5351}
5352
5353static VALUE
5354vm_proc_to_block_handler(VALUE procval)
5355{
5356 const struct rb_block *block = vm_proc_block(procval);
5357
5358 switch (vm_block_type(block)) {
5359 case block_type_iseq:
5360 return VM_BH_FROM_ISEQ_BLOCK(&block->as.captured);
5361 case block_type_ifunc:
5362 return VM_BH_FROM_IFUNC_BLOCK(&block->as.captured);
5363 case block_type_symbol:
5364 return VM_BH_FROM_SYMBOL(block->as.symbol);
5365 case block_type_proc:
5366 return VM_BH_FROM_PROC(block->as.proc);
5367 }
5368 VM_UNREACHABLE(vm_yield_with_proc);
5369 return Qundef;
5370}
5371
5372static VALUE
5373vm_invoke_proc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5374 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5375 bool is_lambda, VALUE block_handler)
5376{
5377 while (vm_block_handler_type(block_handler) == block_handler_type_proc) {
5378 VALUE proc = VM_BH_TO_PROC(block_handler);
5379 is_lambda = block_proc_is_lambda(proc);
5380 block_handler = vm_proc_to_block_handler(proc);
5381 }
5382
5383 return vm_invoke_block(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5384}
5385
5386static inline VALUE
5387vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5388 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5389 bool is_lambda, VALUE block_handler)
5390{
5391 VALUE (*func)(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5392 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5393 bool is_lambda, VALUE block_handler);
5394
5395 switch (vm_block_handler_type(block_handler)) {
5396 case block_handler_type_iseq: func = vm_invoke_iseq_block; break;
5397 case block_handler_type_ifunc: func = vm_invoke_ifunc_block; break;
5398 case block_handler_type_proc: func = vm_invoke_proc_block; break;
5399 case block_handler_type_symbol: func = vm_invoke_symbol_block; break;
5400 default: rb_bug("vm_invoke_block: unreachable");
5401 }
5402
5403 return func(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5404}
5405
5406static VALUE
5407vm_make_proc_with_iseq(const rb_iseq_t *blockiseq)
5408{
5409 const rb_execution_context_t *ec = GET_EC();
5410 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
5411 struct rb_captured_block *captured;
5412
5413 if (cfp == 0) {
5414 rb_bug("vm_make_proc_with_iseq: unreachable");
5415 }
5416
5417 captured = VM_CFP_TO_CAPTURED_BLOCK(cfp);
5418 captured->code.iseq = blockiseq;
5419
5420 return rb_vm_make_proc(ec, captured, rb_cProc);
5421}
5422
5423static VALUE
5424vm_once_exec(VALUE iseq)
5425{
5426 VALUE proc = vm_make_proc_with_iseq((rb_iseq_t *)iseq);
5427 return rb_proc_call_with_block(proc, 0, 0, Qnil);
5428}
5429
5430static VALUE
5431vm_once_clear(VALUE data)
5432{
5433 union iseq_inline_storage_entry *is = (union iseq_inline_storage_entry *)data;
5434 is->once.running_thread = NULL;
5435 return Qnil;
5436}
5437
5438/* defined insn */
5439
5440static bool
5441check_respond_to_missing(VALUE obj, VALUE v)
5442{
5443 VALUE args[2];
5444 VALUE r;
5445
5446 args[0] = obj; args[1] = Qfalse;
5447 r = rb_check_funcall(v, idRespond_to_missing, 2, args);
5448 if (!UNDEF_P(r) && RTEST(r)) {
5449 return true;
5450 }
5451 else {
5452 return false;
5453 }
5454}
5455
5456static bool
5457vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5458{
5459 VALUE klass;
5460 enum defined_type type = (enum defined_type)op_type;
5461
5462 switch (type) {
5463 case DEFINED_IVAR:
5464 return rb_ivar_defined(GET_SELF(), SYM2ID(obj));
5465 break;
5466 case DEFINED_GVAR:
5467 return rb_gvar_defined(SYM2ID(obj));
5468 break;
5469 case DEFINED_CVAR: {
5470 const rb_cref_t *cref = vm_get_cref(GET_EP());
5471 klass = vm_get_cvar_base(cref, GET_CFP(), 0);
5472 return rb_cvar_defined(klass, SYM2ID(obj));
5473 break;
5474 }
5475 case DEFINED_CONST:
5476 case DEFINED_CONST_FROM: {
5477 bool allow_nil = type == DEFINED_CONST;
5478 klass = v;
5479 return vm_get_ev_const(ec, klass, SYM2ID(obj), allow_nil, true);
5480 break;
5481 }
5482 case DEFINED_FUNC:
5483 klass = CLASS_OF(v);
5484 return rb_ec_obj_respond_to(ec, v, SYM2ID(obj), TRUE);
5485 break;
5486 case DEFINED_METHOD:{
5487 VALUE klass = CLASS_OF(v);
5488 const rb_callable_method_entry_t *cme = rb_callable_method_entry_with_refinements(klass, SYM2ID(obj), NULL);
5489
5490 if (cme) {
5491 switch (METHOD_ENTRY_VISI(cme)) {
5492 case METHOD_VISI_PRIVATE:
5493 break;
5494 case METHOD_VISI_PROTECTED:
5495 if (!rb_obj_is_kind_of(GET_SELF(), vm_defined_class_for_protected_call(cme))) {
5496 break;
5497 }
5498 case METHOD_VISI_PUBLIC:
5499 return true;
5500 break;
5501 default:
5502 rb_bug("vm_defined: unreachable: %u", (unsigned int)METHOD_ENTRY_VISI(cme));
5503 }
5504 }
5505 else {
5506 return check_respond_to_missing(obj, v);
5507 }
5508 break;
5509 }
5510 case DEFINED_YIELD:
5511 if (GET_BLOCK_HANDLER() != VM_BLOCK_HANDLER_NONE) {
5512 return true;
5513 }
5514 break;
5515 case DEFINED_ZSUPER:
5516 {
5517 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(GET_CFP());
5518
5519 if (me) {
5520 VALUE klass = vm_search_normal_superclass(me->defined_class);
5521 if (!klass) return false;
5522
5523 ID id = me->def->original_id;
5524
5525 return rb_method_boundp(klass, id, 0);
5526 }
5527 }
5528 break;
5529 case DEFINED_REF:
5530 return RTEST(vm_backref_defined(ec, GET_LEP(), FIX2INT(obj)));
5531 default:
5532 rb_bug("unimplemented defined? type (VM)");
5533 break;
5534 }
5535
5536 return false;
5537}
5538
5539bool
5540rb_vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5541{
5542 return vm_defined(ec, reg_cfp, op_type, obj, v);
5543}
5544
5545static const VALUE *
5546vm_get_ep(const VALUE *const reg_ep, rb_num_t lv)
5547{
5548 rb_num_t i;
5549 const VALUE *ep = reg_ep;
5550 for (i = 0; i < lv; i++) {
5551 ep = GET_PREV_EP(ep);
5552 }
5553 return ep;
5554}
5555
5556static VALUE
5557vm_get_special_object(const VALUE *const reg_ep,
5558 enum vm_special_object_type type)
5559{
5560 switch (type) {
5561 case VM_SPECIAL_OBJECT_VMCORE:
5562 return rb_mRubyVMFrozenCore;
5563 case VM_SPECIAL_OBJECT_CBASE:
5564 return vm_get_cbase(reg_ep);
5565 case VM_SPECIAL_OBJECT_CONST_BASE:
5566 return vm_get_const_base(reg_ep);
5567 default:
5568 rb_bug("putspecialobject insn: unknown value_type %d", type);
5569 }
5570}
5571
5572// ZJIT implementation is using the C function
5573// and needs to call a non-static function
5574VALUE
5575rb_vm_get_special_object(const VALUE *reg_ep, enum vm_special_object_type type)
5576{
5577 return vm_get_special_object(reg_ep, type);
5578}
5579
5580static VALUE
5581vm_concat_array(VALUE ary1, VALUE ary2st)
5582{
5583 const VALUE ary2 = ary2st;
5584 VALUE tmp1 = rb_check_to_array(ary1);
5585 VALUE tmp2 = rb_check_to_array(ary2);
5586
5587 if (NIL_P(tmp1)) {
5588 tmp1 = rb_ary_new3(1, ary1);
5589 }
5590 if (tmp1 == ary1) {
5591 tmp1 = rb_ary_dup(ary1);
5592 }
5593
5594 if (NIL_P(tmp2)) {
5595 return rb_ary_push(tmp1, ary2);
5596 }
5597 else {
5598 return rb_ary_concat(tmp1, tmp2);
5599 }
5600}
5601
5602static VALUE
5603vm_concat_to_array(VALUE ary1, VALUE ary2st)
5604{
5605 /* ary1 must be a newly created array */
5606 const VALUE ary2 = ary2st;
5607
5608 if (NIL_P(ary2)) return ary1;
5609
5610 VALUE tmp2 = rb_check_to_array(ary2);
5611
5612 if (NIL_P(tmp2)) {
5613 return rb_ary_push(ary1, ary2);
5614 }
5615 else {
5616 return rb_ary_concat(ary1, tmp2);
5617 }
5618}
5619
5620// YJIT implementation is using the C function
5621// and needs to call a non-static function
5622VALUE
5623rb_vm_concat_array(VALUE ary1, VALUE ary2st)
5624{
5625 return vm_concat_array(ary1, ary2st);
5626}
5627
5628VALUE
5629rb_vm_concat_to_array(VALUE ary1, VALUE ary2st)
5630{
5631 return vm_concat_to_array(ary1, ary2st);
5632}
5633
5634static VALUE
5635vm_splat_array(VALUE flag, VALUE ary)
5636{
5637 if (NIL_P(ary)) {
5638 return RTEST(flag) ? rb_ary_new() : rb_cArray_empty_frozen;
5639 }
5640 VALUE tmp = rb_check_to_array(ary);
5641 if (NIL_P(tmp)) {
5642 return rb_ary_new3(1, ary);
5643 }
5644 else if (RTEST(flag)) {
5645 return rb_ary_dup(tmp);
5646 }
5647 else {
5648 return tmp;
5649 }
5650}
5651
5652// YJIT implementation is using the C function
5653// and needs to call a non-static function
5654VALUE
5655rb_vm_splat_array(VALUE flag, VALUE ary)
5656{
5657 return vm_splat_array(flag, ary);
5658}
5659
5660static VALUE
5661vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
5662{
5663 enum vm_check_match_type type = ((int)flag) & VM_CHECKMATCH_TYPE_MASK;
5664
5665 if (flag & VM_CHECKMATCH_ARRAY) {
5666 long i;
5667 const long n = RARRAY_LEN(pattern);
5668
5669 for (i = 0; i < n; i++) {
5670 VALUE v = RARRAY_AREF(pattern, i);
5671 VALUE c = check_match(ec, v, target, type);
5672
5673 if (RTEST(c)) {
5674 return c;
5675 }
5676 }
5677 return Qfalse;
5678 }
5679 else {
5680 return check_match(ec, pattern, target, type);
5681 }
5682}
5683
5684VALUE
5685rb_vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
5686{
5687 return vm_check_match(ec, target, pattern, flag);
5688}
5689
5690static VALUE
5691vm_check_keyword(lindex_t bits, lindex_t idx, const VALUE *ep)
5692{
5693 const VALUE kw_bits = *(ep - bits);
5694
5695 if (FIXNUM_P(kw_bits)) {
5696 unsigned int b = (unsigned int)FIX2ULONG(kw_bits);
5697 if ((idx < VM_KW_SPECIFIED_BITS_MAX) && (b & (0x01 << idx)))
5698 return Qfalse;
5699 }
5700 else {
5701 VM_ASSERT(RB_TYPE_P(kw_bits, T_HASH));
5702 if (rb_hash_has_key(kw_bits, INT2FIX(idx))) return Qfalse;
5703 }
5704 return Qtrue;
5705}
5706
5707static void
5708vm_dtrace(rb_event_flag_t flag, rb_execution_context_t *ec)
5709{
5710 if (RUBY_DTRACE_METHOD_ENTRY_ENABLED() ||
5711 RUBY_DTRACE_METHOD_RETURN_ENABLED() ||
5712 RUBY_DTRACE_CMETHOD_ENTRY_ENABLED() ||
5713 RUBY_DTRACE_CMETHOD_RETURN_ENABLED()) {
5714
5715 switch (flag) {
5716 case RUBY_EVENT_CALL:
5717 RUBY_DTRACE_METHOD_ENTRY_HOOK(ec, 0, 0);
5718 return;
5719 case RUBY_EVENT_C_CALL:
5720 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, 0, 0);
5721 return;
5722 case RUBY_EVENT_RETURN:
5723 RUBY_DTRACE_METHOD_RETURN_HOOK(ec, 0, 0);
5724 return;
5725 case RUBY_EVENT_C_RETURN:
5726 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, 0, 0);
5727 return;
5728 }
5729 }
5730}
5731
5732static VALUE
5733vm_const_get_under(ID id, rb_num_t flags, VALUE cbase)
5734{
5735 if (!rb_const_defined_at(cbase, id)) {
5736 return 0;
5737 }
5738 else if (VM_DEFINECLASS_SCOPED_P(flags)) {
5739 return rb_public_const_get_at(cbase, id);
5740 }
5741 else {
5742 return rb_const_get_at(cbase, id);
5743 }
5744}
5745
5746static VALUE
5747vm_check_if_class(ID id, rb_num_t flags, VALUE super, VALUE klass)
5748{
5749 if (!RB_TYPE_P(klass, T_CLASS)) {
5750 return 0;
5751 }
5752 else if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags)) {
5753 VALUE tmp = rb_class_real(RCLASS_SUPER(klass));
5754
5755 if (tmp != super) {
5756 rb_raise(rb_eTypeError,
5757 "superclass mismatch for class %"PRIsVALUE"",
5758 rb_id2str(id));
5759 }
5760 else {
5761 return klass;
5762 }
5763 }
5764 else {
5765 return klass;
5766 }
5767}
5768
5769static VALUE
5770vm_check_if_module(ID id, VALUE mod)
5771{
5772 if (!RB_TYPE_P(mod, T_MODULE)) {
5773 return 0;
5774 }
5775 else {
5776 return mod;
5777 }
5778}
5779
5780static VALUE
5781declare_under(ID id, VALUE cbase, VALUE c)
5782{
5783 rb_set_class_path_string(c, cbase, rb_id2str(id));
5784 rb_const_set(cbase, id, c);
5785 return c;
5786}
5787
5788static VALUE
5789vm_declare_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
5790{
5791 /* new class declaration */
5792 VALUE s = VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) ? super : rb_cObject;
5793 VALUE c = declare_under(id, cbase, rb_define_class_id(id, s));
5794 rb_class_inherited(s, c);
5795 return c;
5796}
5797
5798static VALUE
5799vm_declare_module(ID id, VALUE cbase)
5800{
5801 /* new module declaration */
5802 return declare_under(id, cbase, rb_module_new());
5803}
5804
5805NORETURN(static void unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old));
5806static void
5807unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old)
5808{
5809 VALUE name = rb_id2str(id);
5810 VALUE message = rb_sprintf("%"PRIsVALUE" is not a %s",
5811 name, type);
5812 VALUE location = rb_const_source_location_at(cbase, id);
5813 if (!NIL_P(location)) {
5814 rb_str_catf(message, "\n%"PRIsVALUE":%"PRIsVALUE":"
5815 " previous definition of %"PRIsVALUE" was here",
5816 rb_ary_entry(location, 0), rb_ary_entry(location, 1), name);
5817 }
5818 rb_exc_raise(rb_exc_new_str(rb_eTypeError, message));
5819}
5820
5821static VALUE
5822vm_define_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
5823{
5824 VALUE klass;
5825
5826 if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) && !RB_TYPE_P(super, T_CLASS)) {
5827 rb_raise(rb_eTypeError,
5828 "superclass must be an instance of Class (given an instance of %"PRIsVALUE")",
5829 rb_obj_class(super));
5830 }
5831
5832 vm_check_if_namespace(cbase);
5833
5834 /* find klass */
5835 rb_autoload_load(cbase, id);
5836
5837 if ((klass = vm_const_get_under(id, flags, cbase)) != 0) {
5838 if (!vm_check_if_class(id, flags, super, klass))
5839 unmatched_redefinition("class", cbase, id, klass);
5840 return klass;
5841 }
5842 else {
5843 return vm_declare_class(id, flags, cbase, super);
5844 }
5845}
5846
5847static VALUE
5848vm_define_module(ID id, rb_num_t flags, VALUE cbase)
5849{
5850 VALUE mod;
5851
5852 vm_check_if_namespace(cbase);
5853 if ((mod = vm_const_get_under(id, flags, cbase)) != 0) {
5854 if (!vm_check_if_module(id, mod))
5855 unmatched_redefinition("module", cbase, id, mod);
5856 return mod;
5857 }
5858 else {
5859 return vm_declare_module(id, cbase);
5860 }
5861}
5862
5863static VALUE
5864vm_find_or_create_class_by_id(ID id,
5865 rb_num_t flags,
5866 VALUE cbase,
5867 VALUE super)
5868{
5869 rb_vm_defineclass_type_t type = VM_DEFINECLASS_TYPE(flags);
5870
5871 switch (type) {
5872 case VM_DEFINECLASS_TYPE_CLASS:
5873 /* classdef returns class scope value */
5874 return vm_define_class(id, flags, cbase, super);
5875
5876 case VM_DEFINECLASS_TYPE_SINGLETON_CLASS:
5877 /* classdef returns class scope value */
5878 return rb_singleton_class(cbase);
5879
5880 case VM_DEFINECLASS_TYPE_MODULE:
5881 /* classdef returns class scope value */
5882 return vm_define_module(id, flags, cbase);
5883
5884 default:
5885 rb_bug("unknown defineclass type: %d", (int)type);
5886 }
5887}
5888
5889static rb_method_visibility_t
5890vm_scope_visibility_get(const rb_execution_context_t *ec)
5891{
5892 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
5893
5894 if (!vm_env_cref_by_cref(cfp->ep)) {
5895 return METHOD_VISI_PUBLIC;
5896 }
5897 else {
5898 return CREF_SCOPE_VISI(vm_ec_cref(ec))->method_visi;
5899 }
5900}
5901
5902static int
5903vm_scope_module_func_check(const rb_execution_context_t *ec)
5904{
5905 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
5906
5907 if (!vm_env_cref_by_cref(cfp->ep)) {
5908 return FALSE;
5909 }
5910 else {
5911 return CREF_SCOPE_VISI(vm_ec_cref(ec))->module_func;
5912 }
5913}
5914
5915static void
5916vm_define_method(const rb_execution_context_t *ec, VALUE obj, ID id, VALUE iseqval, int is_singleton)
5917{
5918 VALUE klass;
5919 rb_method_visibility_t visi;
5920 rb_cref_t *cref = vm_ec_cref(ec);
5921
5922 if (is_singleton) {
5923 klass = rb_singleton_class(obj); /* class and frozen checked in this API */
5924 visi = METHOD_VISI_PUBLIC;
5925 }
5926 else {
5927 klass = CREF_CLASS_FOR_DEFINITION(cref);
5928 visi = vm_scope_visibility_get(ec);
5929 }
5930
5931 if (NIL_P(klass)) {
5932 rb_raise(rb_eTypeError, "no class/module to add method");
5933 }
5934
5935 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, visi);
5936 // Set max_iv_count on klasses based on number of ivar sets that are in the initialize method
5937 if (id == idInitialize && RB_TYPE_P(klass, T_CLASS) &&
5938 !RCLASS_SINGLETON_P(klass) && !RCLASS_EXPECT_NO_IVAR(klass) &&
5939 (rb_get_alloc_func(klass) == rb_class_allocate_instance)) {
5940 RCLASS_SET_MAX_IV_COUNT(klass, rb_estimate_iv_count(klass, (const rb_iseq_t *)iseqval));
5941 }
5942
5943 if (!is_singleton && vm_scope_module_func_check(ec)) {
5944 klass = rb_singleton_class(klass);
5945 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, METHOD_VISI_PUBLIC);
5946 }
5947}
5948
5949// Return the untagged block handler:
5950// * If it's VM_BLOCK_HANDLER_NONE, return nil
5951// * If it's an ISEQ or an IFUNC, fetch it from its rb_captured_block
5952// * If it's a PROC or SYMBOL, return it as is
5953VALUE
5954rb_vm_untag_block_handler(VALUE block_handler)
5955{
5956 if (VM_BLOCK_HANDLER_NONE == block_handler) return Qnil;
5957
5958 switch (vm_block_handler_type(block_handler)) {
5959 case block_handler_type_iseq:
5960 case block_handler_type_ifunc: {
5961 struct rb_captured_block *captured = VM_TAGGED_PTR_REF(block_handler, 0x03);
5962 return captured->code.val;
5963 }
5964 case block_handler_type_proc:
5965 case block_handler_type_symbol:
5966 return block_handler;
5967 default:
5968 rb_bug("rb_vm_untag_block_handler: unreachable");
5969 }
5970}
5971
5972VALUE
5973rb_vm_get_untagged_block_handler(rb_control_frame_t *reg_cfp)
5974{
5975 return rb_vm_untag_block_handler(VM_CF_BLOCK_HANDLER(reg_cfp));
5976}
5977
5978static VALUE
5979vm_invokeblock_i(struct rb_execution_context_struct *ec,
5980 struct rb_control_frame_struct *reg_cfp,
5981 struct rb_calling_info *calling)
5982{
5983 const struct rb_callinfo *ci = calling->cd->ci;
5984 VALUE block_handler = VM_CF_BLOCK_HANDLER(GET_CFP());
5985
5986 if (block_handler == VM_BLOCK_HANDLER_NONE) {
5987 rb_vm_localjump_error("no block given (yield)", Qnil, 0);
5988 }
5989 else {
5990 return vm_invoke_block(ec, GET_CFP(), calling, ci, false, block_handler);
5991 }
5992}
5993
5994enum method_explorer_type {
5995 mexp_search_method,
5996 mexp_search_invokeblock,
5997 mexp_search_super,
5998};
5999
6000static inline VALUE
6001vm_sendish(
6002 struct rb_execution_context_struct *ec,
6003 struct rb_control_frame_struct *reg_cfp,
6004 struct rb_call_data *cd,
6005 VALUE block_handler,
6006 enum method_explorer_type method_explorer
6007) {
6008 VALUE val = Qundef;
6009 const struct rb_callinfo *ci = cd->ci;
6010 const struct rb_callcache *cc;
6011 int argc = vm_ci_argc(ci);
6012 VALUE recv = TOPN(argc);
6013 struct rb_calling_info calling = {
6014 .block_handler = block_handler,
6015 .kw_splat = IS_ARGS_KW_SPLAT(ci) > 0,
6016 .recv = recv,
6017 .argc = argc,
6018 .cd = cd,
6019 };
6020
6021 switch (method_explorer) {
6022 case mexp_search_method:
6023 calling.cc = cc = vm_search_method_fastpath(reg_cfp, cd, CLASS_OF(recv));
6024 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6025 break;
6026 case mexp_search_super:
6027 calling.cc = cc = vm_search_super_method(reg_cfp, cd, recv);
6028 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6029 break;
6030 case mexp_search_invokeblock:
6031 val = vm_invokeblock_i(ec, GET_CFP(), &calling);
6032 break;
6033 }
6034 return val;
6035}
6036
6037VALUE
6038rb_vm_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6039{
6040 stack_check(ec);
6041 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), cd->ci, blockiseq, false);
6042 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6043 VM_EXEC(ec, val);
6044 return val;
6045}
6046
6047// Fallback for YJIT/ZJIT, not used by the interpreter
6048VALUE
6049rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6050{
6051 stack_check(ec);
6052
6053 struct rb_forwarding_call_data adjusted_cd;
6054 struct rb_callinfo adjusted_ci;
6055
6056 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, false, &adjusted_cd, &adjusted_ci);
6057
6058 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method);
6059
6060 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6061 RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc);
6062 }
6063
6064 VM_EXEC(ec, val);
6065 return val;
6066}
6067
6068VALUE
6069rb_vm_opt_send_without_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6070{
6071 stack_check(ec);
6072 VALUE bh = VM_BLOCK_HANDLER_NONE;
6073 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6074 VM_EXEC(ec, val);
6075 return val;
6076}
6077
6078VALUE
6079rb_vm_invokesuper(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6080{
6081 stack_check(ec);
6082
6083 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), cd->ci, blockiseq, true);
6084 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_super);
6085
6086 VM_EXEC(ec, val);
6087 return val;
6088}
6089
6090// Fallback for YJIT/ZJIT, not used by the interpreter
6091VALUE
6092rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6093{
6094 stack_check(ec);
6095 struct rb_forwarding_call_data adjusted_cd;
6096 struct rb_callinfo adjusted_ci;
6097
6098 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, true, &adjusted_cd, &adjusted_ci);
6099
6100 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super);
6101
6102 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6103 RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc);
6104 }
6105
6106 VM_EXEC(ec, val);
6107 return val;
6108}
6109
6110VALUE
6111rb_vm_invokeblock(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6112{
6113 stack_check(ec);
6114 VALUE bh = VM_BLOCK_HANDLER_NONE;
6115 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_invokeblock);
6116 VM_EXEC(ec, val);
6117 return val;
6118}
6119
6120/* object.c */
6121VALUE rb_nil_to_s(VALUE);
6122VALUE rb_true_to_s(VALUE);
6123VALUE rb_false_to_s(VALUE);
6124/* numeric.c */
6125VALUE rb_int_to_s(int argc, VALUE *argv, VALUE x);
6126VALUE rb_fix_to_s(VALUE);
6127/* variable.c */
6128VALUE rb_mod_to_s(VALUE);
6129VALUE rb_mod_name(VALUE);
6130
6131static VALUE
6132vm_objtostring(struct rb_control_frame_struct *reg_cfp, VALUE recv, CALL_DATA cd)
6133{
6134 int type = TYPE(recv);
6135 if (type == T_STRING) {
6136 return recv;
6137 }
6138
6139 const struct rb_callable_method_entry_struct *cme = vm_search_method(reg_cfp, cd, recv);
6140
6141 switch (type) {
6142 case T_SYMBOL:
6143 if (check_method_basic_definition(cme)) {
6144 // rb_sym_to_s() allocates a mutable string, but since we are only
6145 // going to use this string for interpolation, it's fine to use the
6146 // frozen string.
6147 return rb_sym2str(recv);
6148 }
6149 break;
6150 case T_MODULE:
6151 case T_CLASS:
6152 if (check_cfunc(cme, rb_mod_to_s)) {
6153 // rb_mod_to_s() allocates a mutable string, but since we are only
6154 // going to use this string for interpolation, it's fine to use the
6155 // frozen string.
6156 VALUE val = rb_mod_name(recv);
6157 if (NIL_P(val)) {
6158 val = rb_mod_to_s(recv);
6159 }
6160 return val;
6161 }
6162 break;
6163 case T_NIL:
6164 if (check_cfunc(cme, rb_nil_to_s)) {
6165 return rb_nil_to_s(recv);
6166 }
6167 break;
6168 case T_TRUE:
6169 if (check_cfunc(cme, rb_true_to_s)) {
6170 return rb_true_to_s(recv);
6171 }
6172 break;
6173 case T_FALSE:
6174 if (check_cfunc(cme, rb_false_to_s)) {
6175 return rb_false_to_s(recv);
6176 }
6177 break;
6178 case T_FIXNUM:
6179 if (check_cfunc(cme, rb_int_to_s)) {
6180 return rb_fix_to_s(recv);
6181 }
6182 break;
6183 }
6184 return Qundef;
6185}
6186
6187// ZJIT implementation is using the C function
6188// and needs to call a non-static function
6189VALUE
6190rb_vm_objtostring(struct rb_control_frame_struct *reg_cfp, VALUE recv, CALL_DATA cd)
6191{
6192 return vm_objtostring(reg_cfp, recv, cd);
6193}
6194
6195static VALUE
6196vm_opt_ary_freeze(VALUE ary, int bop, ID id)
6197{
6198 if (BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
6199 return ary;
6200 }
6201 else {
6202 return Qundef;
6203 }
6204}
6205
6206static VALUE
6207vm_opt_hash_freeze(VALUE hash, int bop, ID id)
6208{
6209 if (BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
6210 return hash;
6211 }
6212 else {
6213 return Qundef;
6214 }
6215}
6216
6217static VALUE
6218vm_opt_str_freeze(VALUE str, int bop, ID id)
6219{
6220 if (BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
6221 return str;
6222 }
6223 else {
6224 return Qundef;
6225 }
6226}
6227
6228/* this macro is mandatory to use OPTIMIZED_CMP. What a design! */
6229#define id_cmp idCmp
6230
6231static VALUE
6232vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6233{
6234 if (BASIC_OP_UNREDEFINED_P(BOP_INCLUDE_P, ARRAY_REDEFINED_OP_FLAG)) {
6235 return rb_ary_includes(ary, target);
6236 }
6237 else {
6238 VALUE args[1] = {target};
6239
6240 // duparray
6241 RUBY_DTRACE_CREATE_HOOK(ARRAY, RARRAY_LEN(ary));
6242 VALUE dupary = rb_ary_resurrect(ary);
6243
6244 return rb_vm_call_with_refinements(ec, dupary, idIncludeP, 1, args, RB_NO_KEYWORDS);
6245 }
6246}
6247
6248VALUE
6249rb_vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6250{
6251 return vm_opt_duparray_include_p(ec, ary, target);
6252}
6253
6254static VALUE
6255vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6256{
6257 if (BASIC_OP_UNREDEFINED_P(BOP_MAX, ARRAY_REDEFINED_OP_FLAG)) {
6258 if (array_len == 0) {
6259 return Qnil;
6260 }
6261 else {
6262 VALUE result = *ptr;
6263 rb_snum_t i = array_len - 1;
6264 while (i-- > 0) {
6265 const VALUE v = *++ptr;
6266 if (OPTIMIZED_CMP(v, result) > 0) {
6267 result = v;
6268 }
6269 }
6270 return result;
6271 }
6272 }
6273 else {
6274 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMax, 0, NULL, RB_NO_KEYWORDS);
6275 }
6276}
6277
6278VALUE
6279rb_vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6280{
6281 return vm_opt_newarray_max(ec, array_len, ptr);
6282}
6283
6284static VALUE
6285vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6286{
6287 if (BASIC_OP_UNREDEFINED_P(BOP_MIN, ARRAY_REDEFINED_OP_FLAG)) {
6288 if (array_len == 0) {
6289 return Qnil;
6290 }
6291 else {
6292 VALUE result = *ptr;
6293 rb_snum_t i = array_len - 1;
6294 while (i-- > 0) {
6295 const VALUE v = *++ptr;
6296 if (OPTIMIZED_CMP(v, result) < 0) {
6297 result = v;
6298 }
6299 }
6300 return result;
6301 }
6302 }
6303 else {
6304 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMin, 0, NULL, RB_NO_KEYWORDS);
6305 }
6306}
6307
6308VALUE
6309rb_vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6310{
6311 return vm_opt_newarray_min(ec, array_len, ptr);
6312}
6313
6314static VALUE
6315vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6316{
6317 // If Array#hash is _not_ monkeypatched, use the optimized call
6318 if (BASIC_OP_UNREDEFINED_P(BOP_HASH, ARRAY_REDEFINED_OP_FLAG)) {
6319 return rb_ary_hash_values(array_len, ptr);
6320 }
6321 else {
6322 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idHash, 0, NULL, RB_NO_KEYWORDS);
6323 }
6324}
6325
6326VALUE
6327rb_vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6328{
6329 return vm_opt_newarray_hash(ec, array_len, ptr);
6330}
6331
6332VALUE rb_setup_fake_ary(struct RArray *fake_ary, const VALUE *list, long len);
6333VALUE rb_ec_pack_ary(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer);
6334
6335static VALUE
6336vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6337{
6338 if (BASIC_OP_UNREDEFINED_P(BOP_INCLUDE_P, ARRAY_REDEFINED_OP_FLAG)) {
6339 struct RArray fake_ary = {RBASIC_INIT};
6340 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6341 return rb_ary_includes(ary, target);
6342 }
6343 else {
6344 VALUE args[1] = {target};
6345 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idIncludeP, 1, args, RB_NO_KEYWORDS);
6346 }
6347}
6348
6349VALUE
6350rb_vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6351{
6352 return vm_opt_newarray_include_p(ec, array_len, ptr, target);
6353}
6354
6355static VALUE
6356vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6357{
6358 if (BASIC_OP_UNREDEFINED_P(BOP_PACK, ARRAY_REDEFINED_OP_FLAG)) {
6359 struct RArray fake_ary = {RBASIC_INIT};
6360 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6361 return rb_ec_pack_ary(ec, ary, fmt, (UNDEF_P(buffer) ? Qnil : buffer));
6362 }
6363 else {
6364 // The opt_newarray_send insn drops the keyword args so we need to rebuild them.
6365 // Setup an array with room for keyword hash.
6366 VALUE args[2];
6367 args[0] = fmt;
6368 int kw_splat = RB_NO_KEYWORDS;
6369 int argc = 1;
6370
6371 if (!UNDEF_P(buffer)) {
6372 args[1] = rb_hash_new_with_size(1);
6373 rb_hash_aset(args[1], ID2SYM(idBuffer), buffer);
6374 kw_splat = RB_PASS_KEYWORDS;
6375 argc++;
6376 }
6377
6378 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idPack, argc, args, kw_splat);
6379 }
6380}
6381
6382VALUE
6383rb_vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6384{
6385 return vm_opt_newarray_pack_buffer(ec, array_len, ptr, fmt, buffer);
6386}
6387
6388VALUE
6389rb_vm_opt_newarray_pack(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt)
6390{
6391 return vm_opt_newarray_pack_buffer(ec, array_len, ptr, fmt, Qundef);
6392}
6393
6394#undef id_cmp
6395
6396static void
6397vm_track_constant_cache(ID id, void *ic)
6398{
6399 rb_vm_t *vm = GET_VM();
6400 struct rb_id_table *const_cache = &vm->constant_cache;
6401 VALUE lookup_result;
6402 set_table *ics;
6403
6404 if (rb_id_table_lookup(const_cache, id, &lookup_result)) {
6405 ics = (set_table *)lookup_result;
6406 }
6407 else {
6408 ics = set_init_numtable();
6409 rb_id_table_insert(const_cache, id, (VALUE)ics);
6410 }
6411
6412 /* The call below to st_insert could allocate which could trigger a GC.
6413 * If it triggers a GC, it may free an iseq that also holds a cache to this
6414 * constant. If that iseq is the last iseq with a cache to this constant, then
6415 * it will free this ST table, which would cause an use-after-free during this
6416 * st_insert.
6417 *
6418 * So to fix this issue, we store the ID that is currently being inserted
6419 * and, in remove_from_constant_cache, we don't free the ST table for ID
6420 * equal to this one.
6421 *
6422 * See [Bug #20921].
6423 */
6424 vm->inserting_constant_cache_id = id;
6425
6426 set_insert(ics, (st_data_t)ic);
6427
6428 vm->inserting_constant_cache_id = (ID)0;
6429}
6430
6431static void
6432vm_ic_track_const_chain(rb_control_frame_t *cfp, IC ic, const ID *segments)
6433{
6434 RB_VM_LOCKING() {
6435 for (int i = 0; segments[i]; i++) {
6436 ID id = segments[i];
6437 if (id == idNULL) continue;
6438 vm_track_constant_cache(id, ic);
6439 }
6440 }
6441}
6442
6443// For JIT inlining
6444static inline bool
6445vm_inlined_ic_hit_p(VALUE flags, VALUE value, const rb_cref_t *ic_cref, const VALUE *reg_ep)
6446{
6447 if ((flags & IMEMO_CONST_CACHE_SHAREABLE) || rb_ractor_main_p()) {
6448 VM_ASSERT(ractor_incidental_shareable_p(flags & IMEMO_CONST_CACHE_SHAREABLE, value));
6449
6450 return (ic_cref == NULL || // no need to check CREF
6451 ic_cref == vm_get_cref(reg_ep));
6452 }
6453 return false;
6454}
6455
6456static bool
6457vm_ic_hit_p(const struct iseq_inline_constant_cache_entry *ice, const VALUE *reg_ep)
6458{
6459 VM_ASSERT(IMEMO_TYPE_P(ice, imemo_constcache));
6460 return vm_inlined_ic_hit_p(ice->flags, ice->value, ice->ic_cref, reg_ep);
6461}
6462
6463// YJIT needs this function to never allocate and never raise
6464bool
6465rb_vm_ic_hit_p(IC ic, const VALUE *reg_ep)
6466{
6467 return ic->entry && vm_ic_hit_p(ic->entry, reg_ep);
6468}
6469
6470static void
6471vm_ic_update(const rb_iseq_t *iseq, IC ic, VALUE val, const VALUE *reg_ep, const VALUE *pc)
6472{
6473 if (ruby_vm_const_missing_count > 0) {
6474 ruby_vm_const_missing_count = 0;
6475 ic->entry = NULL;
6476 return;
6477 }
6478
6479 struct iseq_inline_constant_cache_entry *ice = SHAREABLE_IMEMO_NEW(struct iseq_inline_constant_cache_entry, imemo_constcache, 0);
6480 RB_OBJ_WRITE(ice, &ice->value, val);
6481 ice->ic_cref = vm_get_const_key_cref(reg_ep);
6482
6483 if (rb_ractor_shareable_p(val)) {
6484 RUBY_ASSERT((rb_gc_verify_shareable(val), 1));
6485 ice->flags |= IMEMO_CONST_CACHE_SHAREABLE;
6486 }
6487 RB_OBJ_WRITE(iseq, &ic->entry, ice);
6488 RUBY_ASSERT(pc >= ISEQ_BODY(iseq)->iseq_encoded);
6489 unsigned pos = (unsigned)(pc - ISEQ_BODY(iseq)->iseq_encoded);
6490 rb_yjit_constant_ic_update(iseq, ic, pos);
6491}
6492
6493VALUE
6494rb_vm_opt_getconstant_path(rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, IC ic)
6495{
6496 VALUE val;
6497 const ID *segments = ic->segments;
6498 struct iseq_inline_constant_cache_entry *ice = ic->entry;
6499
6500 if (ice && vm_ic_hit_p(ice, GET_EP())) {
6501 val = ice->value;
6502
6503 VM_ASSERT(val == vm_get_ev_const_chain(ec, segments));
6504 }
6505 else {
6506 ruby_vm_constant_cache_misses++;
6507 val = vm_get_ev_const_chain(ec, segments);
6508 vm_ic_track_const_chain(GET_CFP(), ic, segments);
6509 // Undo the PC increment to get the address to this instruction
6510 // INSN_ATTR(width) == 2
6511 vm_ic_update(CFP_ISEQ(GET_CFP()), ic, val, GET_EP(), CFP_PC(GET_CFP()) - 2);
6512 }
6513 return val;
6514}
6515
6516static VALUE
6517vm_once_dispatch(rb_execution_context_t *ec, ISEQ iseq, ISE is)
6518{
6519 rb_thread_t *th = rb_ec_thread_ptr(ec);
6520 rb_thread_t *const RUNNING_THREAD_ONCE_DONE = (rb_thread_t *)(0x1);
6521
6522 again:
6523 if (is->once.running_thread == RUNNING_THREAD_ONCE_DONE) {
6524 return is->once.value;
6525 }
6526 else if (is->once.running_thread == NULL) {
6527 VALUE val;
6528 is->once.running_thread = th;
6529 val = rb_ensure(vm_once_exec, (VALUE)iseq, vm_once_clear, (VALUE)is);
6530 // TODO: confirm that it is shareable
6531
6532 if (RB_FL_ABLE(val)) {
6533 RB_OBJ_SET_SHAREABLE(val);
6534 }
6535
6536 RB_OBJ_WRITE(CFP_ISEQ(ec->cfp), &is->once.value, val);
6537
6538 /* is->once.running_thread is cleared by vm_once_clear() */
6539 is->once.running_thread = RUNNING_THREAD_ONCE_DONE; /* success */
6540 return val;
6541 }
6542 else if (is->once.running_thread == th) {
6543 /* recursive once */
6544 return vm_once_exec((VALUE)iseq);
6545 }
6546 else {
6547 /* waiting for finish */
6548 RUBY_VM_CHECK_INTS(ec);
6549 rb_thread_schedule();
6550 goto again;
6551 }
6552}
6553
6554static OFFSET
6555vm_case_dispatch(CDHASH hash, OFFSET else_offset, VALUE key)
6556{
6557 switch (OBJ_BUILTIN_TYPE(key)) {
6558 case -1:
6559 case T_FLOAT:
6560 case T_SYMBOL:
6561 case T_BIGNUM:
6562 case T_STRING:
6563 if (BASIC_OP_UNREDEFINED_P(BOP_EQQ,
6564 SYMBOL_REDEFINED_OP_FLAG |
6565 INTEGER_REDEFINED_OP_FLAG |
6566 FLOAT_REDEFINED_OP_FLAG |
6567 NIL_REDEFINED_OP_FLAG |
6568 TRUE_REDEFINED_OP_FLAG |
6569 FALSE_REDEFINED_OP_FLAG |
6570 STRING_REDEFINED_OP_FLAG)) {
6571 st_data_t val;
6572 if (RB_FLOAT_TYPE_P(key)) {
6573 double kval = RFLOAT_VALUE(key);
6574 if (!isinf(kval) && modf(kval, &kval) == 0.0) {
6575 key = FIXABLE(kval) ? LONG2FIX((long)kval) : rb_dbl2big(kval);
6576 }
6577 }
6578 if (st_lookup(rb_imemo_cdhash_tbl(hash), key, &val)) {
6579 return (VALUE)val;
6580 }
6581 else {
6582 return else_offset;
6583 }
6584 }
6585 }
6586 return 0;
6587}
6588
6589NORETURN(static void
6590 vm_stack_consistency_error(const rb_execution_context_t *ec,
6591 const rb_control_frame_t *,
6592 const VALUE *));
6593static void
6594vm_stack_consistency_error(const rb_execution_context_t *ec,
6595 const rb_control_frame_t *cfp,
6596 const VALUE *bp)
6597{
6598 const ptrdiff_t nsp = VM_SP_CNT(ec, cfp->sp);
6599 const ptrdiff_t nbp = VM_SP_CNT(ec, bp);
6600 static const char stack_consistency_error[] =
6601 "Stack consistency error (sp: %"PRIdPTRDIFF", bp: %"PRIdPTRDIFF")";
6602#if defined RUBY_DEVEL
6603 VALUE mesg = rb_sprintf(stack_consistency_error, nsp, nbp);
6604 rb_str_cat_cstr(mesg, "\n");
6605 rb_str_append(mesg, rb_iseq_disasm(CFP_ISEQ(cfp)));
6606 rb_exc_fatal(rb_exc_new3(rb_eFatal, mesg));
6607#else
6608 rb_bug(stack_consistency_error, nsp, nbp);
6609#endif
6610}
6611
6612static VALUE
6613vm_opt_plus(VALUE recv, VALUE obj)
6614{
6615 if (FIXNUM_2_P(recv, obj) &&
6616 BASIC_OP_UNREDEFINED_P(BOP_PLUS, INTEGER_REDEFINED_OP_FLAG)) {
6617 return rb_fix_plus_fix(recv, obj);
6618 }
6619 else if (FLONUM_2_P(recv, obj) &&
6620 BASIC_OP_UNREDEFINED_P(BOP_PLUS, FLOAT_REDEFINED_OP_FLAG)) {
6621 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
6622 }
6623 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6624 return Qundef;
6625 }
6626 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6627 RBASIC_CLASS(obj) == rb_cFloat &&
6628 BASIC_OP_UNREDEFINED_P(BOP_PLUS, FLOAT_REDEFINED_OP_FLAG)) {
6629 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
6630 }
6631 else if (RBASIC_CLASS(recv) == rb_cString &&
6632 RBASIC_CLASS(obj) == rb_cString &&
6633 BASIC_OP_UNREDEFINED_P(BOP_PLUS, STRING_REDEFINED_OP_FLAG)) {
6634 return rb_str_opt_plus(recv, obj);
6635 }
6636 else if (RBASIC_CLASS(recv) == rb_cArray &&
6637 RBASIC_CLASS(obj) == rb_cArray &&
6638 BASIC_OP_UNREDEFINED_P(BOP_PLUS, ARRAY_REDEFINED_OP_FLAG)) {
6639 return rb_ary_plus(recv, obj);
6640 }
6641 else {
6642 return Qundef;
6643 }
6644}
6645
6646static VALUE
6647vm_opt_minus(VALUE recv, VALUE obj)
6648{
6649 if (FIXNUM_2_P(recv, obj) &&
6650 BASIC_OP_UNREDEFINED_P(BOP_MINUS, INTEGER_REDEFINED_OP_FLAG)) {
6651 return rb_fix_minus_fix(recv, obj);
6652 }
6653 else if (FLONUM_2_P(recv, obj) &&
6654 BASIC_OP_UNREDEFINED_P(BOP_MINUS, FLOAT_REDEFINED_OP_FLAG)) {
6655 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
6656 }
6657 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6658 return Qundef;
6659 }
6660 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6661 RBASIC_CLASS(obj) == rb_cFloat &&
6662 BASIC_OP_UNREDEFINED_P(BOP_MINUS, FLOAT_REDEFINED_OP_FLAG)) {
6663 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
6664 }
6665 else {
6666 return Qundef;
6667 }
6668}
6669
6670static VALUE
6671vm_opt_mult(VALUE recv, VALUE obj)
6672{
6673 if (FIXNUM_2_P(recv, obj) &&
6674 BASIC_OP_UNREDEFINED_P(BOP_MULT, INTEGER_REDEFINED_OP_FLAG)) {
6675 return rb_fix_mul_fix(recv, obj);
6676 }
6677 else if (FLONUM_2_P(recv, obj) &&
6678 BASIC_OP_UNREDEFINED_P(BOP_MULT, FLOAT_REDEFINED_OP_FLAG)) {
6679 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
6680 }
6681 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6682 return Qundef;
6683 }
6684 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6685 RBASIC_CLASS(obj) == rb_cFloat &&
6686 BASIC_OP_UNREDEFINED_P(BOP_MULT, FLOAT_REDEFINED_OP_FLAG)) {
6687 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
6688 }
6689 else {
6690 return Qundef;
6691 }
6692}
6693
6694static VALUE
6695vm_opt_div(VALUE recv, VALUE obj)
6696{
6697 if (FIXNUM_2_P(recv, obj) &&
6698 BASIC_OP_UNREDEFINED_P(BOP_DIV, INTEGER_REDEFINED_OP_FLAG)) {
6699 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_div_fix(recv, obj);
6700 }
6701 else if (FLONUM_2_P(recv, obj) &&
6702 BASIC_OP_UNREDEFINED_P(BOP_DIV, FLOAT_REDEFINED_OP_FLAG)) {
6703 return rb_flo_div_flo(recv, obj);
6704 }
6705 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6706 return Qundef;
6707 }
6708 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6709 RBASIC_CLASS(obj) == rb_cFloat &&
6710 BASIC_OP_UNREDEFINED_P(BOP_DIV, FLOAT_REDEFINED_OP_FLAG)) {
6711 return rb_flo_div_flo(recv, obj);
6712 }
6713 else {
6714 return Qundef;
6715 }
6716}
6717
6718static VALUE
6719vm_opt_mod(VALUE recv, VALUE obj)
6720{
6721 if (FIXNUM_2_P(recv, obj) &&
6722 BASIC_OP_UNREDEFINED_P(BOP_MOD, INTEGER_REDEFINED_OP_FLAG)) {
6723 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_mod_fix(recv, obj);
6724 }
6725 else if (FLONUM_2_P(recv, obj) &&
6726 BASIC_OP_UNREDEFINED_P(BOP_MOD, FLOAT_REDEFINED_OP_FLAG)) {
6727 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
6728 }
6729 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6730 return Qundef;
6731 }
6732 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6733 RBASIC_CLASS(obj) == rb_cFloat &&
6734 BASIC_OP_UNREDEFINED_P(BOP_MOD, FLOAT_REDEFINED_OP_FLAG)) {
6735 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
6736 }
6737 else {
6738 return Qundef;
6739 }
6740}
6741
6742static VALUE
6743vm_opt_neq(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, CALL_DATA cd_eq, VALUE recv, VALUE obj)
6744{
6745 if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_not_equal)) {
6746 VALUE val = opt_equality(reg_cfp, recv, obj, cd_eq);
6747
6748 if (!UNDEF_P(val)) {
6749 return RBOOL(!RTEST(val));
6750 }
6751 }
6752
6753 return Qundef;
6754}
6755
6756static VALUE
6757vm_opt_lt(VALUE recv, VALUE obj)
6758{
6759 if (FIXNUM_2_P(recv, obj) &&
6760 BASIC_OP_UNREDEFINED_P(BOP_LT, INTEGER_REDEFINED_OP_FLAG)) {
6761 return RBOOL((SIGNED_VALUE)recv < (SIGNED_VALUE)obj);
6762 }
6763 else if (FLONUM_2_P(recv, obj) &&
6764 BASIC_OP_UNREDEFINED_P(BOP_LT, FLOAT_REDEFINED_OP_FLAG)) {
6765 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
6766 }
6767 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6768 return Qundef;
6769 }
6770 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6771 RBASIC_CLASS(obj) == rb_cFloat &&
6772 BASIC_OP_UNREDEFINED_P(BOP_LT, FLOAT_REDEFINED_OP_FLAG)) {
6773 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
6774 }
6775 else {
6776 return Qundef;
6777 }
6778}
6779
6780static VALUE
6781vm_opt_le(VALUE recv, VALUE obj)
6782{
6783 if (FIXNUM_2_P(recv, obj) &&
6784 BASIC_OP_UNREDEFINED_P(BOP_LE, INTEGER_REDEFINED_OP_FLAG)) {
6785 return RBOOL((SIGNED_VALUE)recv <= (SIGNED_VALUE)obj);
6786 }
6787 else if (FLONUM_2_P(recv, obj) &&
6788 BASIC_OP_UNREDEFINED_P(BOP_LE, FLOAT_REDEFINED_OP_FLAG)) {
6789 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
6790 }
6791 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6792 return Qundef;
6793 }
6794 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6795 RBASIC_CLASS(obj) == rb_cFloat &&
6796 BASIC_OP_UNREDEFINED_P(BOP_LE, FLOAT_REDEFINED_OP_FLAG)) {
6797 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
6798 }
6799 else {
6800 return Qundef;
6801 }
6802}
6803
6804static VALUE
6805vm_opt_gt(VALUE recv, VALUE obj)
6806{
6807 if (FIXNUM_2_P(recv, obj) &&
6808 BASIC_OP_UNREDEFINED_P(BOP_GT, INTEGER_REDEFINED_OP_FLAG)) {
6809 return RBOOL((SIGNED_VALUE)recv > (SIGNED_VALUE)obj);
6810 }
6811 else if (FLONUM_2_P(recv, obj) &&
6812 BASIC_OP_UNREDEFINED_P(BOP_GT, FLOAT_REDEFINED_OP_FLAG)) {
6813 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
6814 }
6815 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6816 return Qundef;
6817 }
6818 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6819 RBASIC_CLASS(obj) == rb_cFloat &&
6820 BASIC_OP_UNREDEFINED_P(BOP_GT, FLOAT_REDEFINED_OP_FLAG)) {
6821 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
6822 }
6823 else {
6824 return Qundef;
6825 }
6826}
6827
6828static VALUE
6829vm_opt_ge(VALUE recv, VALUE obj)
6830{
6831 if (FIXNUM_2_P(recv, obj) &&
6832 BASIC_OP_UNREDEFINED_P(BOP_GE, INTEGER_REDEFINED_OP_FLAG)) {
6833 return RBOOL((SIGNED_VALUE)recv >= (SIGNED_VALUE)obj);
6834 }
6835 else if (FLONUM_2_P(recv, obj) &&
6836 BASIC_OP_UNREDEFINED_P(BOP_GE, FLOAT_REDEFINED_OP_FLAG)) {
6837 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
6838 }
6839 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6840 return Qundef;
6841 }
6842 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6843 RBASIC_CLASS(obj) == rb_cFloat &&
6844 BASIC_OP_UNREDEFINED_P(BOP_GE, FLOAT_REDEFINED_OP_FLAG)) {
6845 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
6846 }
6847 else {
6848 return Qundef;
6849 }
6850}
6851
6852
6853static VALUE
6854vm_opt_ltlt(VALUE recv, VALUE obj)
6855{
6856 if (SPECIAL_CONST_P(recv)) {
6857 return Qundef;
6858 }
6859 else if (RBASIC_CLASS(recv) == rb_cString &&
6860 BASIC_OP_UNREDEFINED_P(BOP_LTLT, STRING_REDEFINED_OP_FLAG)) {
6861 if (LIKELY(RB_TYPE_P(obj, T_STRING))) {
6862 return rb_str_buf_append(recv, obj);
6863 }
6864 else {
6865 return rb_str_concat(recv, obj);
6866 }
6867 }
6868 else if (RBASIC_CLASS(recv) == rb_cArray &&
6869 BASIC_OP_UNREDEFINED_P(BOP_LTLT, ARRAY_REDEFINED_OP_FLAG)) {
6870 return rb_ary_push(recv, obj);
6871 }
6872 else {
6873 return Qundef;
6874 }
6875}
6876
6877static VALUE
6878vm_opt_and(VALUE recv, VALUE obj)
6879{
6880 // If recv and obj are both fixnums, then the bottom tag bit
6881 // will be 1 on both. 1 & 1 == 1, so the result value will also
6882 // be a fixnum. If either side is *not* a fixnum, then the tag bit
6883 // will be 0, and we return Qundef.
6884 VALUE ret = ((SIGNED_VALUE) recv) & ((SIGNED_VALUE) obj);
6885
6886 if (FIXNUM_P(ret) &&
6887 BASIC_OP_UNREDEFINED_P(BOP_AND, INTEGER_REDEFINED_OP_FLAG)) {
6888 return ret;
6889 }
6890 else {
6891 return Qundef;
6892 }
6893}
6894
6895static VALUE
6896vm_opt_or(VALUE recv, VALUE obj)
6897{
6898 if (FIXNUM_2_P(recv, obj) &&
6899 BASIC_OP_UNREDEFINED_P(BOP_OR, INTEGER_REDEFINED_OP_FLAG)) {
6900 return recv | obj;
6901 }
6902 else {
6903 return Qundef;
6904 }
6905}
6906
6907static VALUE
6908vm_opt_aref(VALUE recv, VALUE obj)
6909{
6910 if (SPECIAL_CONST_P(recv)) {
6911 if (FIXNUM_2_P(recv, obj) &&
6912 BASIC_OP_UNREDEFINED_P(BOP_AREF, INTEGER_REDEFINED_OP_FLAG)) {
6913 return rb_fix_aref(recv, obj);
6914 }
6915 return Qundef;
6916 }
6917 else if (RBASIC_CLASS(recv) == rb_cArray &&
6918 BASIC_OP_UNREDEFINED_P(BOP_AREF, ARRAY_REDEFINED_OP_FLAG)) {
6919 if (FIXNUM_P(obj)) {
6920 return rb_ary_entry_internal(recv, FIX2LONG(obj));
6921 }
6922 else {
6923 return rb_ary_aref1(recv, obj);
6924 }
6925 }
6926 else if (RBASIC_CLASS(recv) == rb_cHash &&
6927 BASIC_OP_UNREDEFINED_P(BOP_AREF, HASH_REDEFINED_OP_FLAG)) {
6928 return rb_hash_aref(recv, obj);
6929 }
6930 else {
6931 return Qundef;
6932 }
6933}
6934
6935static VALUE
6936vm_opt_aset(VALUE recv, VALUE obj, VALUE set)
6937{
6938 if (SPECIAL_CONST_P(recv)) {
6939 return Qundef;
6940 }
6941 else if (RBASIC_CLASS(recv) == rb_cArray &&
6942 BASIC_OP_UNREDEFINED_P(BOP_ASET, ARRAY_REDEFINED_OP_FLAG) &&
6943 FIXNUM_P(obj)) {
6944 rb_ary_store(recv, FIX2LONG(obj), set);
6945 return set;
6946 }
6947 else if (RBASIC_CLASS(recv) == rb_cHash &&
6948 BASIC_OP_UNREDEFINED_P(BOP_ASET, HASH_REDEFINED_OP_FLAG)) {
6949 rb_hash_aset(recv, obj, set);
6950 return set;
6951 }
6952 else {
6953 return Qundef;
6954 }
6955}
6956
6957static VALUE
6958vm_opt_length(VALUE recv, int bop)
6959{
6960 if (SPECIAL_CONST_P(recv)) {
6961 return Qundef;
6962 }
6963 else if (RBASIC_CLASS(recv) == rb_cString &&
6964 BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
6965 if (bop == BOP_EMPTY_P) {
6966 return LONG2NUM(RSTRING_LEN(recv));
6967 }
6968 else {
6969 return rb_str_length(recv);
6970 }
6971 }
6972 else if (RBASIC_CLASS(recv) == rb_cArray &&
6973 BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
6974 return LONG2NUM(RARRAY_LEN(recv));
6975 }
6976 else if (RBASIC_CLASS(recv) == rb_cHash &&
6977 BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
6978 return INT2FIX(RHASH_SIZE(recv));
6979 }
6980 else {
6981 return Qundef;
6982 }
6983}
6984
6985static VALUE
6986vm_opt_empty_p(VALUE recv)
6987{
6988 switch (vm_opt_length(recv, BOP_EMPTY_P)) {
6989 case Qundef: return Qundef;
6990 case INT2FIX(0): return Qtrue;
6991 default: return Qfalse;
6992 }
6993}
6994
6995VALUE rb_false(VALUE obj);
6996
6997static VALUE
6998vm_opt_nil_p(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv)
6999{
7000 if (NIL_P(recv) &&
7001 BASIC_OP_UNREDEFINED_P(BOP_NIL_P, NIL_REDEFINED_OP_FLAG)) {
7002 return Qtrue;
7003 }
7004 else if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_false)) {
7005 return Qfalse;
7006 }
7007 else {
7008 return Qundef;
7009 }
7010}
7011
7012static VALUE
7013fix_succ(VALUE x)
7014{
7015 switch (x) {
7016 case ~0UL:
7017 /* 0xFFFF_FFFF == INT2FIX(-1)
7018 * `-1.succ` is of course 0. */
7019 return INT2FIX(0);
7020 case RSHIFT(~0UL, 1):
7021 /* 0x7FFF_FFFF == LONG2FIX(0x3FFF_FFFF)
7022 * 0x3FFF_FFFF + 1 == 0x4000_0000, which is a Bignum. */
7023 return rb_uint2big(1UL << (SIZEOF_LONG * CHAR_BIT - 2));
7024 default:
7025 /* LONG2FIX(FIX2LONG(x)+FIX2LONG(y))
7026 * == ((lx*2+1)/2 + (ly*2+1)/2)*2+1
7027 * == lx*2 + ly*2 + 1
7028 * == (lx*2+1) + (ly*2+1) - 1
7029 * == x + y - 1
7030 *
7031 * Here, if we put y := INT2FIX(1):
7032 *
7033 * == x + INT2FIX(1) - 1
7034 * == x + 2 .
7035 */
7036 return x + 2;
7037 }
7038}
7039
7040static VALUE
7041vm_opt_succ(VALUE recv)
7042{
7043 if (FIXNUM_P(recv) &&
7044 BASIC_OP_UNREDEFINED_P(BOP_SUCC, INTEGER_REDEFINED_OP_FLAG)) {
7045 return fix_succ(recv);
7046 }
7047 else if (SPECIAL_CONST_P(recv)) {
7048 return Qundef;
7049 }
7050 else if (RBASIC_CLASS(recv) == rb_cString &&
7051 BASIC_OP_UNREDEFINED_P(BOP_SUCC, STRING_REDEFINED_OP_FLAG)) {
7052 return rb_str_succ(recv);
7053 }
7054 else {
7055 return Qundef;
7056 }
7057}
7058
7059static VALUE
7060vm_opt_not(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv)
7061{
7062 if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_not)) {
7063 return RBOOL(!RTEST(recv));
7064 }
7065 else {
7066 return Qundef;
7067 }
7068}
7069
7070static VALUE
7071vm_opt_regexpmatch2(VALUE recv, VALUE obj)
7072{
7073 if (SPECIAL_CONST_P(recv)) {
7074 return Qundef;
7075 }
7076 else if (RBASIC_CLASS(recv) == rb_cString &&
7077 CLASS_OF(obj) == rb_cRegexp &&
7078 BASIC_OP_UNREDEFINED_P(BOP_MATCH, STRING_REDEFINED_OP_FLAG)) {
7079 return rb_reg_match(obj, recv);
7080 }
7081 else if (RBASIC_CLASS(recv) == rb_cRegexp &&
7082 BASIC_OP_UNREDEFINED_P(BOP_MATCH, REGEXP_REDEFINED_OP_FLAG)) {
7083 return rb_reg_match(recv, obj);
7084 }
7085 else {
7086 return Qundef;
7087 }
7088}
7089
7090rb_event_flag_t rb_iseq_event_flags(const rb_iseq_t *iseq, size_t pos);
7091
7092NOINLINE(static void vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp));
7093
7094static inline void
7095vm_trace_hook(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE *pc,
7096 rb_event_flag_t pc_events, rb_event_flag_t target_event,
7097 rb_hook_list_t *global_hooks, rb_hook_list_t *local_hooks, VALUE val)
7098{
7099 rb_event_flag_t event = pc_events & target_event;
7100 VALUE self = GET_SELF();
7101
7102 VM_ASSERT(rb_popcount64((uint64_t)event) == 1);
7103
7104 if (local_hooks) local_hooks->running++; // make sure they don't get deleted while global hooks run
7105
7106 if (event & global_hooks->events) {
7107 /* increment PC because source line is calculated with PC-1 */
7108 reg_cfp->pc++;
7109 vm_dtrace(event, ec);
7110 rb_exec_event_hook_orig(ec, global_hooks, event, self, 0, 0, 0 , val, 0);
7111 reg_cfp->pc--;
7112 }
7113
7114 if (local_hooks) local_hooks->running--;
7115 if (local_hooks != NULL) {
7116 if (event & local_hooks->events) {
7117 /* increment PC because source line is calculated with PC-1 */
7118 reg_cfp->pc++;
7119 rb_exec_event_hook_orig(ec, local_hooks, event, self, 0, 0, 0 , val, 0);
7120 reg_cfp->pc--;
7121 }
7122 }
7123}
7124
7125#define VM_TRACE_HOOK(target_event, val) do { \
7126 if ((pc_events & (target_event)) & enabled_flags) { \
7127 vm_trace_hook(ec, reg_cfp, pc, pc_events, (target_event), global_hooks, local_hooks, (val)); \
7128 } \
7129} while (0)
7130
7131static VALUE
7132rescue_errinfo(rb_execution_context_t *ec, rb_control_frame_t *cfp)
7133{
7134 VM_ASSERT(VM_FRAME_RUBYFRAME_P(cfp));
7135 VM_ASSERT(ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_RESCUE);
7136 return cfp->ep[VM_ENV_INDEX_LAST_LVAR];
7137}
7138
7139static void
7140vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp)
7141{
7142 const VALUE *pc = reg_cfp->pc;
7143 rb_ractor_t *r = rb_ec_ractor_ptr(ec);
7144 rb_event_flag_t enabled_flags = r->pub.hooks.events & ISEQ_TRACE_EVENTS;
7145 rb_event_flag_t ractor_events = enabled_flags;
7146
7147 if (enabled_flags == 0 && rb_ractor_targeted_hooks_cnt(r) == 0) {
7148 return;
7149 }
7150 else {
7151 const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp);
7152 size_t pos = pc - ISEQ_BODY(iseq)->iseq_encoded;
7153 rb_event_flag_t pc_events = rb_iseq_event_flags(iseq, pos);
7154 unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt;
7155 rb_hook_list_t *local_hooks = NULL;
7156 if (RB_UNLIKELY(local_hooks_cnt > 0)) {
7157 st_data_t val;
7158 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)iseq, &val)) {
7159 local_hooks = (rb_hook_list_t*)val;
7160 }
7161 }
7162 rb_event_flag_t iseq_local_events = local_hooks != NULL ? local_hooks->events : 0;
7163
7164 rb_hook_list_t *bmethod_local_hooks = NULL;
7165 rb_event_flag_t bmethod_local_events = 0;
7166 const bool bmethod_frame = VM_FRAME_BMETHOD_P(reg_cfp);
7167 enabled_flags |= iseq_local_events;
7168
7169 VM_ASSERT((iseq_local_events & ~ISEQ_TRACE_EVENTS) == 0);
7170
7171 if (bmethod_frame) {
7172 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
7173 VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD);
7174 unsigned int bmethod_hooks_cnt = me->def->body.bmethod.local_hooks_cnt;
7175 if (RB_UNLIKELY(bmethod_hooks_cnt > 0)) {
7176 st_data_t val;
7177 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)me->def, &val)) {
7178 bmethod_local_hooks = (rb_hook_list_t*)val;
7179 }
7180 if (bmethod_local_hooks) {
7181 bmethod_local_events = bmethod_local_hooks->events;
7182 }
7183 }
7184 }
7185
7186 if ((pc_events & enabled_flags) == 0 && !bmethod_frame) {
7187#if 0
7188 /* disable trace */
7189 /* TODO: incomplete */
7190 rb_iseq_trace_set(iseq, vm_event_flags & ISEQ_TRACE_EVENTS);
7191#else
7192 /* do not disable trace because of performance problem
7193 * (re-enable overhead)
7194 */
7195#endif
7196 return;
7197 }
7198 else if (ec->trace_arg != NULL) {
7199 /* already tracing */
7200 return;
7201 }
7202 else {
7203 rb_hook_list_t *global_hooks = rb_ec_ractor_hooks(ec);
7204 /* Note, not considering iseq local events here since the same
7205 * iseq could be used in multiple bmethods. */
7206 rb_event_flag_t bmethod_events = ractor_events | bmethod_local_events;
7207
7208 if (0) {
7209 ruby_debug_printf("vm_trace>>%4d (%4x) - %s:%d %s\n",
7210 (int)pos,
7211 (int)pc_events,
7212 RSTRING_PTR(rb_iseq_path(iseq)),
7213 (int)rb_iseq_line_no(iseq, pos),
7214 RSTRING_PTR(rb_iseq_label(iseq)));
7215 }
7216 VM_ASSERT(reg_cfp->pc == pc);
7217 VM_ASSERT(pc_events != 0);
7218
7219 /* check traces */
7220 if ((pc_events & RUBY_EVENT_B_CALL) && bmethod_frame && (bmethod_events & RUBY_EVENT_CALL)) {
7221 /* b_call instruction running as a method. Fire call event. */
7222 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_CALL, RUBY_EVENT_CALL, global_hooks, bmethod_local_hooks, Qundef);
7223 }
7224 VM_TRACE_HOOK(RUBY_EVENT_CLASS | RUBY_EVENT_CALL | RUBY_EVENT_B_CALL, Qundef);
7225 VM_TRACE_HOOK(RUBY_EVENT_RESCUE, rescue_errinfo(ec, reg_cfp));
7226 VM_TRACE_HOOK(RUBY_EVENT_LINE, Qundef);
7227 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_LINE, Qundef);
7228 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_BRANCH, Qundef);
7229 VM_TRACE_HOOK(RUBY_EVENT_END | RUBY_EVENT_RETURN | RUBY_EVENT_B_RETURN, TOPN(0));
7230 if ((pc_events & RUBY_EVENT_B_RETURN) && bmethod_frame && (bmethod_events & RUBY_EVENT_RETURN)) {
7231 /* b_return instruction running as a method. Fire return event. */
7232 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_RETURN, RUBY_EVENT_RETURN, global_hooks, bmethod_local_hooks, TOPN(0));
7233 }
7234 }
7235 }
7236}
7237#undef VM_TRACE_HOOK
7238
7239#if VM_CHECK_MODE > 0
7240NORETURN( NOINLINE( COLDFUNC
7241void rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)));
7242
7243void
7244Init_vm_stack_canary(void)
7245{
7246 /* This has to be called _after_ our PRNG is properly set up. */
7247 int n = ruby_fill_random_bytes(&vm_stack_canary, sizeof vm_stack_canary, false);
7248 vm_stack_canary |= 0x01; // valid VALUE (Fixnum)
7249
7250 vm_stack_canary_was_born = true;
7251 VM_ASSERT(n == 0);
7252}
7253
7254void
7255rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)
7256{
7257 /* Because a method has already been called, why not call
7258 * another one. */
7259 const char *insn = rb_insns_name(i);
7260 VALUE inspection = rb_inspect(c);
7261 const char *str = StringValueCStr(inspection);
7262
7263 rb_bug("dead canary found at %s: %s", insn, str);
7264}
7265
7266#else
7267void Init_vm_stack_canary(void) { /* nothing to do */ }
7268#endif
7269
7270
7271/* a part of the following code is generated by this ruby script:
7272
727316.times{|i|
7274 typedef_args = (0...i).map{|j| "VALUE v#{j+1}"}.join(", ")
7275 typedef_args.prepend(", ") if i != 0
7276 call_args = (0...i).map{|j| "argv[#{j}]"}.join(", ")
7277 call_args.prepend(", ") if i != 0
7278 puts %Q{
7279static VALUE
7280builtin_invoker#{i}(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7281{
7282 typedef VALUE (*rb_invoke_funcptr#{i}_t)(rb_execution_context_t *ec, VALUE self#{typedef_args});
7283 return (*(rb_invoke_funcptr#{i}_t)funcptr)(ec, self#{call_args});
7284}}
7285}
7286
7287puts
7288puts "static VALUE (* const cfunc_invokers[])(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr) = {"
728916.times{|i|
7290 puts " builtin_invoker#{i},"
7291}
7292puts "};"
7293*/
7294
7295static VALUE
7296builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7297{
7298 typedef VALUE (*rb_invoke_funcptr0_t)(rb_execution_context_t *ec, VALUE self);
7299 return (*(rb_invoke_funcptr0_t)funcptr)(ec, self);
7300}
7301
7302static VALUE
7303builtin_invoker1(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7304{
7305 typedef VALUE (*rb_invoke_funcptr1_t)(rb_execution_context_t *ec, VALUE self, VALUE v1);
7306 return (*(rb_invoke_funcptr1_t)funcptr)(ec, self, argv[0]);
7307}
7308
7309static VALUE
7310builtin_invoker2(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7311{
7312 typedef VALUE (*rb_invoke_funcptr2_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2);
7313 return (*(rb_invoke_funcptr2_t)funcptr)(ec, self, argv[0], argv[1]);
7314}
7315
7316static VALUE
7317builtin_invoker3(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7318{
7319 typedef VALUE (*rb_invoke_funcptr3_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3);
7320 return (*(rb_invoke_funcptr3_t)funcptr)(ec, self, argv[0], argv[1], argv[2]);
7321}
7322
7323static VALUE
7324builtin_invoker4(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7325{
7326 typedef VALUE (*rb_invoke_funcptr4_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4);
7327 return (*(rb_invoke_funcptr4_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3]);
7328}
7329
7330static VALUE
7331builtin_invoker5(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7332{
7333 typedef VALUE (*rb_invoke_funcptr5_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5);
7334 return (*(rb_invoke_funcptr5_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4]);
7335}
7336
7337static VALUE
7338builtin_invoker6(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7339{
7340 typedef VALUE (*rb_invoke_funcptr6_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6);
7341 return (*(rb_invoke_funcptr6_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
7342}
7343
7344static VALUE
7345builtin_invoker7(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7346{
7347 typedef VALUE (*rb_invoke_funcptr7_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7);
7348 return (*(rb_invoke_funcptr7_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
7349}
7350
7351static VALUE
7352builtin_invoker8(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7353{
7354 typedef VALUE (*rb_invoke_funcptr8_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8);
7355 return (*(rb_invoke_funcptr8_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
7356}
7357
7358static VALUE
7359builtin_invoker9(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7360{
7361 typedef VALUE (*rb_invoke_funcptr9_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9);
7362 return (*(rb_invoke_funcptr9_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
7363}
7364
7365static VALUE
7366builtin_invoker10(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7367{
7368 typedef VALUE (*rb_invoke_funcptr10_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10);
7369 return (*(rb_invoke_funcptr10_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
7370}
7371
7372static VALUE
7373builtin_invoker11(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7374{
7375 typedef VALUE (*rb_invoke_funcptr11_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11);
7376 return (*(rb_invoke_funcptr11_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
7377}
7378
7379static VALUE
7380builtin_invoker12(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7381{
7382 typedef VALUE (*rb_invoke_funcptr12_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12);
7383 return (*(rb_invoke_funcptr12_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
7384}
7385
7386static VALUE
7387builtin_invoker13(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7388{
7389 typedef VALUE (*rb_invoke_funcptr13_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13);
7390 return (*(rb_invoke_funcptr13_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
7391}
7392
7393static VALUE
7394builtin_invoker14(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7395{
7396 typedef VALUE (*rb_invoke_funcptr14_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14);
7397 return (*(rb_invoke_funcptr14_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
7398}
7399
7400static VALUE
7401builtin_invoker15(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7402{
7403 typedef VALUE (*rb_invoke_funcptr15_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14, VALUE v15);
7404 return (*(rb_invoke_funcptr15_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
7405}
7406
7407typedef VALUE (*builtin_invoker)(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
7408
7409static builtin_invoker
7410lookup_builtin_invoker(int argc)
7411{
7412 static const builtin_invoker invokers[] = {
7413 builtin_invoker0,
7414 builtin_invoker1,
7415 builtin_invoker2,
7416 builtin_invoker3,
7417 builtin_invoker4,
7418 builtin_invoker5,
7419 builtin_invoker6,
7420 builtin_invoker7,
7421 builtin_invoker8,
7422 builtin_invoker9,
7423 builtin_invoker10,
7424 builtin_invoker11,
7425 builtin_invoker12,
7426 builtin_invoker13,
7427 builtin_invoker14,
7428 builtin_invoker15,
7429 };
7430
7431 return invokers[argc];
7432}
7433
7434static inline VALUE
7435invoke_bf(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7436{
7437 const bool canary_p = ISEQ_BODY(CFP_ISEQ(reg_cfp))->builtin_attrs & BUILTIN_ATTR_LEAF; // Verify an assumption of `Primitive.attr! :leaf`
7438 SETUP_CANARY(canary_p);
7439 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
7440 VALUE ret = (*lookup_builtin_invoker(bf->argc))(ec, reg_cfp->self, argv, func_ptr);
7441 CHECK_CANARY(canary_p, BIN(invokebuiltin));
7442 return ret;
7443}
7444
7445static VALUE
7446vm_invoke_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7447{
7448 return invoke_bf(ec, cfp, bf, argv);
7449}
7450
7451static VALUE
7452vm_invoke_builtin_delegate(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function *bf, unsigned int start_index)
7453{
7454 if (0) { // debug print
7455 fputs("vm_invoke_builtin_delegate: passing -> ", stderr);
7456 for (int i=0; i<bf->argc; i++) {
7457 ruby_debug_printf(":%s ", rb_id2name(ISEQ_BODY(CFP_ISEQ(cfp))->local_table[i+start_index]));
7458 }
7459 ruby_debug_printf("\n" "%s %s(%d):%p\n", RUBY_FUNCTION_NAME_STRING, bf->name, bf->argc,
7460 (void *)(uintptr_t)bf->func_ptr);
7461 }
7462
7463 if (bf->argc == 0) {
7464 return invoke_bf(ec, cfp, bf, NULL);
7465 }
7466 else {
7467 const VALUE *argv = cfp->ep - ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size - VM_ENV_DATA_SIZE + 1 + start_index;
7468 return invoke_bf(ec, cfp, bf, argv);
7469 }
7470}
7471
7472// for __builtin_inline!()
7473
7474VALUE
7475rb_vm_lvar_exposed(rb_execution_context_t *ec, int index)
7476{
7477 const rb_control_frame_t *cfp = ec->cfp;
7478 return cfp->ep[index];
7479}
RUBY_ASSERT
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:219
RUBY_EVENT_END
#define RUBY_EVENT_END
Encountered an end of a class clause.
Definition event.h:40
RUBY_EVENT_C_CALL
#define RUBY_EVENT_C_CALL
A method, written in C, is called.
Definition event.h:43
RUBY_EVENT_B_RETURN
#define RUBY_EVENT_B_RETURN
Encountered a next statement.
Definition event.h:56
RUBY_EVENT_CLASS
#define RUBY_EVENT_CLASS
Encountered a new class.
Definition event.h:39
RUBY_EVENT_LINE
#define RUBY_EVENT_LINE
Encountered a new line.
Definition event.h:38
RUBY_EVENT_RETURN
#define RUBY_EVENT_RETURN
Encountered a return statement.
Definition event.h:42
RUBY_EVENT_C_RETURN
#define RUBY_EVENT_C_RETURN
Return from a method, written in C.
Definition event.h:44
RUBY_EVENT_B_CALL
#define RUBY_EVENT_B_CALL
Encountered an yield statement.
Definition event.h:55
rb_event_flag_t
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
RUBY_EVENT_CALL
#define RUBY_EVENT_CALL
A method, written in Ruby, is called.
Definition event.h:41
RUBY_EVENT_RESCUE
#define RUBY_EVENT_RESCUE
Encountered a rescue statement.
Definition event.h:61
RB_FL_ABLE
static bool RB_FL_ABLE(VALUE obj)
Checks if the object is flaggable.
Definition fl_type.h:381
rb_singleton_class
VALUE rb_singleton_class(VALUE obj)
Finds or creates the singleton class of the passed object.
Definition class.c:2728
rb_module_new
VALUE rb_module_new(void)
Creates a new, anonymous module.
Definition class.c:1490
rb_class_inherited
VALUE rb_class_inherited(VALUE super, VALUE klass)
Calls Class::inherited.
Definition class.c:1387
rb_define_class_id
VALUE rb_define_class_id(ID id, VALUE super)
This is a very badly designed API that creates an anonymous class.
Definition class.c:1366
TYPE
#define TYPE(_)
Old name of rb_type.
Definition value_type.h:108
RFLOAT_VALUE
#define RFLOAT_VALUE
Old name of rb_float_value.
Definition double.h:28
T_STRING
#define T_STRING
Old name of RUBY_T_STRING.
Definition value_type.h:78
Qundef
#define Qundef
Old name of RUBY_Qundef.
Definition special_consts.h:62
INT2FIX
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
T_NIL
#define T_NIL
Old name of RUBY_T_NIL.
Definition value_type.h:72
T_FLOAT
#define T_FLOAT
Old name of RUBY_T_FLOAT.
Definition value_type.h:64
T_IMEMO
#define T_IMEMO
Old name of RUBY_T_IMEMO.
Definition value_type.h:67
ID2SYM
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
T_BIGNUM
#define T_BIGNUM
Old name of RUBY_T_BIGNUM.
Definition value_type.h:57
SPECIAL_CONST_P
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
Definition special_consts.h:56
T_STRUCT
#define T_STRUCT
Old name of RUBY_T_STRUCT.
Definition value_type.h:79
T_FIXNUM
#define T_FIXNUM
Old name of RUBY_T_FIXNUM.
Definition value_type.h:63
SYM2ID
#define SYM2ID
Old name of RB_SYM2ID.
Definition symbol.h:45
CLASS_OF
#define CLASS_OF
Old name of rb_class_of.
Definition globals.h:205
rb_ary_new4
#define rb_ary_new4
Old name of rb_ary_new_from_values.
Definition array.h:659
FIXABLE
#define FIXABLE
Old name of RB_FIXABLE.
Definition fixnum.h:25
LONG2FIX
#define LONG2FIX
Old name of RB_INT2FIX.
Definition long.h:49
FIX2INT
#define FIX2INT
Old name of RB_FIX2INT.
Definition int.h:41
T_MODULE
#define T_MODULE
Old name of RUBY_T_MODULE.
Definition value_type.h:70
STATIC_SYM_P
#define STATIC_SYM_P
Old name of RB_STATIC_SYM_P.
Definition special_consts.h:57
ASSUME
#define ASSUME
Old name of RBIMPL_ASSUME.
Definition assume.h:27
FIX2ULONG
#define FIX2ULONG
Old name of RB_FIX2ULONG.
Definition long.h:47
T_TRUE
#define T_TRUE
Old name of RUBY_T_TRUE.
Definition value_type.h:81
T_ICLASS
#define T_ICLASS
Old name of RUBY_T_ICLASS.
Definition value_type.h:66
T_HASH
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
rb_ary_new3
#define rb_ary_new3
Old name of rb_ary_new_from_args.
Definition array.h:658
LONG2NUM
#define LONG2NUM
Old name of RB_LONG2NUM.
Definition long.h:50
rb_exc_new3
#define rb_exc_new3
Old name of rb_exc_new_str.
Definition error.h:38
T_FALSE
#define T_FALSE
Old name of RUBY_T_FALSE.
Definition value_type.h:61
Qtrue
#define Qtrue
Old name of RUBY_Qtrue.
Definition special_consts.h:61
Qnil
#define Qnil
Old name of RUBY_Qnil.
Definition special_consts.h:60
Qfalse
#define Qfalse
Old name of RUBY_Qfalse.
Definition special_consts.h:59
FIX2LONG
#define FIX2LONG
Old name of RB_FIX2LONG.
Definition long.h:46
T_ARRAY
#define T_ARRAY
Old name of RUBY_T_ARRAY.
Definition value_type.h:56
T_OBJECT
#define T_OBJECT
Old name of RUBY_T_OBJECT.
Definition value_type.h:75
NIL_P
#define NIL_P
Old name of RB_NIL_P.
Definition special_consts.h:55
T_SYMBOL
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition value_type.h:80
DBL2NUM
#define DBL2NUM
Old name of rb_float_new.
Definition double.h:29
T_CLASS
#define T_CLASS
Old name of RUBY_T_CLASS.
Definition value_type.h:58
BUILTIN_TYPE
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
Definition value_type.h:85
FL_TEST
#define FL_TEST
Old name of RB_FL_TEST.
Definition fl_type.h:127
FIXNUM_P
#define FIXNUM_P
Old name of RB_FIXNUM_P.
Definition special_consts.h:53
FL_USHIFT
#define FL_USHIFT
Old name of RUBY_FL_USHIFT.
Definition fl_type.h:67
FL_SET_RAW
#define FL_SET_RAW
Old name of RB_FL_SET_RAW.
Definition fl_type.h:126
SYMBOL_P
#define SYMBOL_P
Old name of RB_SYMBOL_P.
Definition value_type.h:88
rb_notimplement
void rb_notimplement(void)
Definition error.c:3898
rb_exc_raise
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
Definition eval.c:661
ruby_verbose
#define ruby_verbose
This variable controls whether the interpreter is in debug mode.
Definition error.h:476
rb_eTypeError
VALUE rb_eTypeError
TypeError exception.
Definition error.c:1427
rb_eFatal
VALUE rb_eFatal
fatal exception.
Definition error.c:1423
rb_eNoMethodError
VALUE rb_eNoMethodError
NoMethodError exception.
Definition error.c:1435
rb_exc_fatal
void rb_exc_fatal(VALUE mesg)
Raises a fatal error in the current thread.
Definition eval.c:674
rb_eRuntimeError
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1425
rb_warn
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:467
rb_error_frozen_object
void rb_error_frozen_object(VALUE frozen_obj)
Identical to rb_error_frozen(), except it takes arbitrary Ruby object instead of C's string.
Definition error.c:4219
rb_exc_new_str
VALUE rb_exc_new_str(VALUE etype, VALUE str)
Identical to rb_exc_new_cstr(), except it takes a Ruby's string instead of C's.
Definition error.c:1478
RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK
@ RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK
Warning is for checking unused block strictly.
Definition error.h:57
rb_cClass
VALUE rb_cClass
Class class.
Definition object.c:63
rb_cArray
VALUE rb_cArray
Array class.
rb_cObject
VALUE rb_cObject
Object class.
Definition object.c:61
rb_obj_alloc
VALUE rb_obj_alloc(VALUE klass)
Allocates an instance of the given class.
Definition object.c:2255
rb_cRegexp
VALUE rb_cRegexp
Regexp class.
Definition re.c:2726
rb_obj_frozen_p
VALUE rb_obj_frozen_p(VALUE obj)
Just calls RB_OBJ_FROZEN() inside.
Definition object.c:1325
rb_cHash
VALUE rb_cHash
Hash class.
Definition hash.c:109
rb_obj_class
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
Definition object.c:235
rb_inspect
VALUE rb_inspect(VALUE obj)
Generates a human-readable textual representation of the given object.
Definition object.c:657
rb_cBasicObject
VALUE rb_cBasicObject
BasicObject class.
Definition object.c:59
rb_cModule
VALUE rb_cModule
Module class.
Definition object.c:62
rb_class_real
VALUE rb_class_real(VALUE klass)
Finds a "real" class.
Definition object.c:226
rb_obj_is_kind_of
VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass)
Queries if the given object is an instance (of possibly descendants) of the given class.
Definition object.c:894
rb_cFloat
VALUE rb_cFloat
Float class.
Definition numeric.c:200
rb_cProc
VALUE rb_cProc
Proc class.
Definition proc.c:45
rb_cString
VALUE rb_cString
String class.
Definition string.c:81
RB_OBJ_WRITTEN
#define RB_OBJ_WRITTEN(old, oldv, young)
Identical to RB_OBJ_WRITE(), except it doesn't write any values, but only a WB declaration.
Definition gc.h:468
RB_OBJ_WRITE
#define RB_OBJ_WRITE(old, slot, young)
Declaration of a "back" pointer.
Definition gc.h:456
rb_ary_concat
VALUE rb_ary_concat(VALUE lhs, VALUE rhs)
Destructively appends the contents of latter into the end of former.
rb_ary_shift
VALUE rb_ary_shift(VALUE ary)
Destructively deletes an element from the beginning of the passed array and returns what was deleted.
rb_ary_resurrect
VALUE rb_ary_resurrect(VALUE ary)
I guess there is no use case of this function in extension libraries, but this is a routine identical...
rb_ary_dup
VALUE rb_ary_dup(VALUE ary)
Duplicates an array.
rb_ary_includes
VALUE rb_ary_includes(VALUE ary, VALUE elem)
Queries if the passed array has the passed entry.
rb_ary_plus
VALUE rb_ary_plus(VALUE lhs, VALUE rhs)
Creates a new array, concatenating the former to the latter.
rb_ary_cat
VALUE rb_ary_cat(VALUE ary, const VALUE *train, long len)
Destructively appends multiple elements at the end of the array.
rb_check_array_type
VALUE rb_check_array_type(VALUE obj)
Try converting an object to its array representation using its to_ary method, if any.
rb_ary_new
VALUE rb_ary_new(void)
Allocates a new, empty array.
rb_ary_pop
VALUE rb_ary_pop(VALUE ary)
Destructively deletes an element from the end of the passed array and returns what was deleted.
rb_ary_hidden_new
VALUE rb_ary_hidden_new(long capa)
Allocates a hidden (no class) empty array.
rb_ary_push
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
rb_ary_entry
VALUE rb_ary_entry(VALUE ary, long off)
Queries an element of an array.
rb_ary_store
void rb_ary_store(VALUE ary, long key, VALUE val)
Destructively stores the passed value to the passed array's passed index.
UNLIMITED_ARGUMENTS
#define UNLIMITED_ARGUMENTS
This macro is used in conjunction with rb_check_arity().
Definition error.h:35
rb_check_arity
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
Definition error.h:284
rb_proc_call_with_block
VALUE rb_proc_call_with_block(VALUE recv, int argc, const VALUE *argv, VALUE proc)
Identical to rb_proc_call(), except you can additionally pass another proc object,...
Definition proc.c:1186
rb_reg_last_match
VALUE rb_reg_last_match(VALUE md)
This just returns the argument, stringified.
Definition re.c:1993
rb_reg_match
VALUE rb_reg_match(VALUE re, VALUE str)
This is the match operator.
Definition re.c:3857
rb_reg_nth_match
VALUE rb_reg_nth_match(int n, VALUE md)
Queries the nth captured substring.
Definition re.c:1969
rb_reg_match_post
VALUE rb_reg_match_post(VALUE md)
The portion of the original string after the given match.
Definition re.c:2048
rb_reg_nth_defined
VALUE rb_reg_nth_defined(int n, VALUE md)
Identical to rb_reg_nth_match(), except it just returns Boolean.
Definition re.c:1953
rb_reg_match_pre
VALUE rb_reg_match_pre(VALUE md)
The portion of the original string before the given match.
Definition re.c:2017
rb_reg_match_last
VALUE rb_reg_match_last(VALUE md)
The portion of the original string that captured at the very last.
Definition re.c:2077
rb_str_append
VALUE rb_str_append(VALUE dst, VALUE src)
Identical to rb_str_buf_append(), except it converts the right hand side before concatenating.
Definition string.c:3836
rb_str_succ
VALUE rb_str_succ(VALUE orig)
Searches for the "successor" of a string.
Definition string.c:5377
rb_str_buf_append
VALUE rb_str_buf_append(VALUE dst, VALUE src)
Identical to rb_str_cat_cstr(), except it takes Ruby's string instead of C's.
Definition string.c:3802
rb_str_concat
VALUE rb_str_concat(VALUE dst, VALUE src)
Identical to rb_str_append(), except it also accepts an integer as a codepoint.
Definition string.c:4073
rb_str_cat_cstr
#define rb_str_cat_cstr(buf, str)
Identical to rb_str_cat(), except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1657
rb_str_length
VALUE rb_str_length(VALUE)
Identical to rb_str_strlen(), except it returns the value in rb_cInteger.
Definition string.c:2437
rb_str_intern
VALUE rb_str_intern(VALUE str)
Identical to rb_to_symbol(), except it assumes the receiver being an instance of RString.
Definition symbol.c:968
rb_thread_schedule
void rb_thread_schedule(void)
Tries to switch to another thread.
Definition thread.c:1514
rb_const_get
VALUE rb_const_get(VALUE space, ID name)
Identical to rb_const_defined(), except it returns the actual defined value.
Definition variable.c:3536
rb_ivar_set
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
Definition variable.c:2064
rb_cvar_set
void rb_cvar_set(VALUE klass, ID name, VALUE val)
Assigns a value to a class variable.
Definition variable.c:4323
rb_cvar_find
VALUE rb_cvar_find(VALUE klass, ID name, VALUE *front)
Identical to rb_cvar_get(), except it takes additional "front" pointer.
Definition variable.c:4384
rb_ivar_get
VALUE rb_ivar_get(VALUE obj, ID name)
Identical to rb_iv_get(), except it accepts the name as an ID instead of a C string.
Definition variable.c:1515
rb_const_set
void rb_const_set(VALUE space, ID name, VALUE val)
Names a constant.
Definition variable.c:4014
rb_autoload_load
VALUE rb_autoload_load(VALUE space, ID name)
Kicks the autoload procedure as if it was "touched".
Definition variable.c:3366
rb_mod_name
VALUE rb_mod_name(VALUE mod)
Queries the name of a module.
Definition variable.c:136
rb_const_get_at
VALUE rb_const_get_at(VALUE space, ID name)
Identical to rb_const_defined_at(), except it returns the actual defined value.
Definition variable.c:3542
rb_set_class_path_string
void rb_set_class_path_string(VALUE klass, VALUE space, VALUE name)
Identical to rb_set_class_path(), except it accepts the name as Ruby's string instead of C's.
Definition variable.c:423
rb_ivar_defined
VALUE rb_ivar_defined(VALUE obj, ID name)
Queries if the instance variable is defined at the object.
Definition variable.c:2143
rb_const_defined_at
int rb_const_defined_at(VALUE space, ID name)
Identical to rb_const_defined(), except it doesn't look for parent classes.
Definition variable.c:3874
rb_cvar_defined
VALUE rb_cvar_defined(VALUE klass, ID name)
Queries if the given class has the given class variable.
Definition variable.c:4407
rb_class_path
VALUE rb_class_path(VALUE mod)
Identical to rb_mod_name(), except it returns #<Class: ...> style inspection for anonymous modules.
Definition variable.c:380
rb_const_defined
int rb_const_defined(VALUE space, ID name)
Queries if the constant is defined at the namespace.
Definition variable.c:3868
rb_check_funcall
VALUE rb_check_funcall(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it returns RUBY_Qundef instead of raising rb_eNoMethodError.
Definition vm_eval.c:690
rb_get_alloc_func
rb_alloc_func_t rb_get_alloc_func(VALUE klass)
Queries the allocator function of a class.
Definition vm_method.c:1797
rb_method_boundp
int rb_method_boundp(VALUE klass, ID id, int ex)
Queries if the klass has this method.
Definition vm_method.c:2367
rb_check_id
ID rb_check_id(volatile VALUE *namep)
Detects if the given name is already interned or not.
Definition symbol.c:1164
rb_sym2str
VALUE rb_sym2str(VALUE symbol)
Obtain a frozen string representation of a symbol (not including the leading colon).
Definition symbol.c:1024
off
int off
Offset inside of ptr.
Definition io.h:5
len
int len
Length of the buffer.
Definition io.h:8
rb_ractor_shareable_p
static bool rb_ractor_shareable_p(VALUE obj)
Queries if multiple Ractors can share the passed object or not.
Definition ractor.h:249
RB_OBJ_SHAREABLE_P
#define RB_OBJ_SHAREABLE_P(obj)
Queries if the passed object has previously classified as shareable or not.
Definition ractor.h:235
MEMCPY
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:372
ALLOCA_N
#define ALLOCA_N(type, n)
Definition memory.h:292
RB_GC_GUARD
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:167
MEMMOVE
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
Definition memory.h:384
ruby::backward::cxxanyargs::type
VALUE type(ANYARGS)
ANYARGS-ed function type.
Definition cxxanyargs.hpp:56
ruby::backward::cxxanyargs::rb_ensure
VALUE rb_ensure(type *q, VALUE w, type *e, VALUE r)
An equivalent of ensure clause.
Definition cxxanyargs.hpp:283
RARRAY_LEN
#define RARRAY_LEN
Just another name of rb_array_len.
Definition rarray.h:51
RARRAY_LENINT
static int RARRAY_LENINT(VALUE ary)
Identical to rb_array_len(), except it differs for the return type.
Definition rarray.h:281
RARRAY_PTR
static VALUE * RARRAY_PTR(VALUE ary)
Wild use of a C pointer.
Definition rarray.h:366
RARRAY_AREF
#define RARRAY_AREF(a, i)
Definition rarray.h:403
RARRAY_CONST_PTR
#define RARRAY_CONST_PTR
Just another name of rb_array_const_ptr.
Definition rarray.h:52
RBASIC_CLASS
static VALUE RBASIC_CLASS(VALUE obj)
Queries the class of an object.
Definition rbasic.h:166
RBASIC
#define RBASIC(obj)
Convenient casting macro.
Definition rbasic.h:40
RCLASS_SUPER
#define RCLASS_SUPER
Just another name of rb_class_get_superclass.
Definition rclass.h:44
RHASH_SIZE
#define RHASH_SIZE(h)
Queries the size of the hash.
Definition rhash.h:69
RHASH_EMPTY_P
#define RHASH_EMPTY_P(h)
Checks if the hash is empty.
Definition rhash.h:79
ROBJECT_FIELDS
static VALUE * ROBJECT_FIELDS(VALUE obj)
Queries the instance variables.
Definition robject.h:133
StringValueCStr
#define StringValueCStr(v)
Identical to StringValuePtr, except it additionally checks for the contents for viability as a C stri...
Definition rstring.h:89
RB_PASS_KEYWORDS
#define RB_PASS_KEYWORDS
Pass keywords, final argument must be a hash of keywords.
Definition scan_args.h:72
RB_NO_KEYWORDS
#define RB_NO_KEYWORDS
Do not pass keywords.
Definition scan_args.h:69
RB_SPECIAL_CONST_P
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
Definition special_consts.h:327
RTEST
#define RTEST
This is an old name of RB_TEST.
Definition special_consts.h:51
ANYARGS
#define ANYARGS
Functions declared using this macro take arbitrary arguments, including void.
Definition stdarg.h:64
RArray
Ruby's array.
Definition rarray.h:128
RArray::ary
const VALUE ary[1]
Embedded elements.
Definition rarray.h:188
RArray::ptr
const VALUE * ptr
Pointer to the C array that holds the elements of the array.
Definition rarray.h:175
RHash
Definition hash.h:53
iseq_catch_table_entry
Definition iseq.h:289
iseq_catch_table
Definition iseq.h:300
iseq_inline_constant_cache_entry
Definition vm_core.h:261
iseq_inline_constant_cache
Definition vm_core.h:271
iseq_inline_constant_cache::segments
const ID * segments
A null-terminated list of ids, used to represent a constant's path idNULL is used to represent the ::...
Definition vm_core.h:285
iseq_inline_cvar_cache_entry
Definition vm_core.h:293
iseq_inline_iv_cache_entry
Definition vm_core.h:288
rb_block
Definition vm_core.h:909
rb_builtin_function
Definition builtin.h:6
rb_call_data
Definition vm_callinfo.h:577
rb_callable_method_entry_struct
Definition method.h:63
rb_callcache
Definition vm_callinfo.h:278
rb_callinfo_kwarg
Definition vm_callinfo.h:48
rb_callinfo
Definition vm_callinfo.h:65
rb_calling_info
Definition vm_core.h:306
rb_captured_block
Definition vm_core.h:885
rb_class_cc_entries
Definition vm_callinfo.h:582
rb_const_entry_struct
Definition constant.h:33
rb_control_frame_struct
Definition vm_core.h:918
rb_cref_struct
CREF (Class REFerence)
Definition method.h:45
rb_cvar_class_tbl_entry
Internal header for Class.
Definition class.h:30
rb_env_t
Definition vm_core.h:1318
rb_execution_context_struct
Definition vm_core.h:1045
rb_forwarding_call_data
Definition vm_insnhelper.h:69
rb_getivar_cache
Definition shape.h:525
rb_hook_list_struct
Definition vm_core.h:671
rb_id_table
Definition id_table.h:9
rb_iseq_struct
Definition vm_core.h:581
rb_method_cfunc_struct
Definition method.h:148
rb_method_definition_struct
Definition method.h:187
rb_method_entry_struct
Definition method.h:55
rb_method_iseq_struct::cref
rb_cref_t * cref
class reference, should be marked
Definition method.h:144
rb_method_iseq_struct::iseqptr
const rb_iseq_t * iseqptr
iseq pointer, should be separated from iseqval
Definition method.h:143
rb_proc_t
Definition vm_core.h:1305
rb_ractor_struct
Definition ractor_core.h:68
rb_setivar_cache
Definition shape.h:567
rb_thread_struct
Definition vm_core.h:1142
rb_vm_struct
Definition vm_core.h:682
set_table
Definition set_table.h:10
st_table
Definition st.h:79
vm_ifunc
IFUNC (Internal FUNCtion)
Definition imemo.h:87
vm_svar
SVAR (Special VARiable)
Definition imemo.h:52
vm_svar::cref_or_me
const VALUE cref_or_me
class reference or rb_method_entry_t
Definition imemo.h:54
vm_throw_data
THROW_DATA.
Definition imemo.h:61
__attribute__
Definition vm_insnhelper.c:2287
iseq_inline_storage_entry
Definition vm_core.h:297
SIGNED_VALUE
intptr_t SIGNED_VALUE
A signed integer type that has the same width with VALUE.
Definition value.h:63
ID
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
SIZEOF_VALUE
#define SIZEOF_VALUE
Identical to sizeof(VALUE), except it is a macro that can also be used inside of preprocessor directi...
Definition value.h:69
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
RB_BUILTIN_TYPE
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
Definition value_type.h:182
RB_FLOAT_TYPE_P
static bool RB_FLOAT_TYPE_P(VALUE obj)
Queries if the object is an instance of rb_cFloat.
Definition value_type.h:264
RB_TYPE_P
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:376
Generated by doxygen 1.9.8